source: trunk/essentials/dev-lang/python/Lib/email/charset.py@ 3393

Last change on this file since 3393 was 3225, checked in by bird, 19 years ago

Python 2.5
File size: 15.3 KB
Line 
1# Copyright (C) 2001-2006 Python Software Foundation
2# Author: Ben Gertzfield, Barry Warsaw
3# Contact: [email protected]
4
5__all__ = [
6 'Charset',
7 'add_alias',
8 'add_charset',
9 'add_codec',
10 ]
11
12import email.base64mime
13import email.quoprimime
14
15from email import errors
16from email.encoders import encode_7or8bit
17
18
19
20
21# Flags for types of header encodings
22QP = 1 # Quoted-Printable
23BASE64 = 2 # Base64
24SHORTEST = 3 # the shorter of QP and base64, but only for headers
25
26# In "=?charset?q?hello_world?=", the =?, ?q?, and ?= add up to 7
27MISC_LEN = 7
28
29DEFAULT_CHARSET = 'us-ascii'
30
31
32
33
34# Defaults
35CHARSETS = {
36 # input header enc body enc output conv
37 'iso-8859-1': (QP, QP, None),
38 'iso-8859-2': (QP, QP, None),
39 'iso-8859-3': (QP, QP, None),
40 'iso-8859-4': (QP, QP, None),
41 # iso-8859-5 is Cyrillic, and not especially used
42 # iso-8859-6 is Arabic, also not particularly used
43 # iso-8859-7 is Greek, QP will not make it readable
44 # iso-8859-8 is Hebrew, QP will not make it readable
45 'iso-8859-9': (QP, QP, None),
46 'iso-8859-10': (QP, QP, None),
47 # iso-8859-11 is Thai, QP will not make it readable
48 'iso-8859-13': (QP, QP, None),
49 'iso-8859-14': (QP, QP, None),
50 'iso-8859-15': (QP, QP, None),
51 'windows-1252':(QP, QP, None),
52 'viscii': (QP, QP, None),
53 'us-ascii': (None, None, None),
54 'big5': (BASE64, BASE64, None),
55 'gb2312': (BASE64, BASE64, None),
56 'euc-jp': (BASE64, None, 'iso-2022-jp'),
57 'shift_jis': (BASE64, None, 'iso-2022-jp'),
58 'iso-2022-jp': (BASE64, None, None),
59 'koi8-r': (BASE64, BASE64, None),
60 'utf-8': (SHORTEST, BASE64, 'utf-8'),
61 # We're making this one up to represent raw unencoded 8-bit
62 '8bit': (None, BASE64, 'utf-8'),
63 }
64
65# Aliases for other commonly-used names for character sets. Map
66# them to the real ones used in email.
67ALIASES = {
68 'latin_1': 'iso-8859-1',
69 'latin-1': 'iso-8859-1',
70 'latin_2': 'iso-8859-2',
71 'latin-2': 'iso-8859-2',
72 'latin_3': 'iso-8859-3',
73 'latin-3': 'iso-8859-3',
74 'latin_4': 'iso-8859-4',
75 'latin-4': 'iso-8859-4',
76 'latin_5': 'iso-8859-9',
77 'latin-5': 'iso-8859-9',
78 'latin_6': 'iso-8859-10',
79 'latin-6': 'iso-8859-10',
80 'latin_7': 'iso-8859-13',
81 'latin-7': 'iso-8859-13',
82 'latin_8': 'iso-8859-14',
83 'latin-8': 'iso-8859-14',
84 'latin_9': 'iso-8859-15',
85 'latin-9': 'iso-8859-15',
86 'cp949': 'ks_c_5601-1987',
87 'euc_jp': 'euc-jp',
88 'euc_kr': 'euc-kr',
89 'ascii': 'us-ascii',
90 }
91
92
93# Map charsets to their Unicode codec strings.
94CODEC_MAP = {
95 'gb2312': 'eucgb2312_cn',
96 'big5': 'big5_tw',
97 # Hack: We don't want *any* conversion for stuff marked us-ascii, as all
98 # sorts of garbage might be sent to us in the guise of 7-bit us-ascii.
99 # Let that stuff pass through without conversion to/from Unicode.
100 'us-ascii': None,
101 }
102
103
104
105
106# Convenience functions for extending the above mappings
107def add_charset(charset, header_enc=None, body_enc=None, output_charset=None):
108 """Add character set properties to the global registry.
109
110 charset is the input character set, and must be the canonical name of a
111 character set.
112
113 Optional header_enc and body_enc is either Charset.QP for
114 quoted-printable, Charset.BASE64 for base64 encoding, Charset.SHORTEST for
115 the shortest of qp or base64 encoding, or None for no encoding. SHORTEST
116 is only valid for header_enc. It describes how message headers and
117 message bodies in the input charset are to be encoded. Default is no
118 encoding.
119
120 Optional output_charset is the character set that the output should be
121 in. Conversions will proceed from input charset, to Unicode, to the
122 output charset when the method Charset.convert() is called. The default
123 is to output in the same character set as the input.
124
125 Both input_charset and output_charset must have Unicode codec entries in
126 the module's charset-to-codec mapping; use add_codec(charset, codecname)
127 to add codecs the module does not know about. See the codecs module's
128 documentation for more information.
129 """
130 if body_enc == SHORTEST:
131 raise ValueError('SHORTEST not allowed for body_enc')
132 CHARSETS[charset] = (header_enc, body_enc, output_charset)
133
134
135def add_alias(alias, canonical):
136 """Add a character set alias.
137
138 alias is the alias name, e.g. latin-1
139 canonical is the character set's canonical name, e.g. iso-8859-1
140 """
141 ALIASES[alias] = canonical
142
143
144def add_codec(charset, codecname):
145 """Add a codec that map characters in the given charset to/from Unicode.
146
147 charset is the canonical name of a character set. codecname is the name
148 of a Python codec, as appropriate for the second argument to the unicode()
149 built-in, or to the encode() method of a Unicode string.
150 """
151 CODEC_MAP[charset] = codecname
152
153
154
155
156class Charset:
157 """Map character sets to their email properties.
158
159 This class provides information about the requirements imposed on email
160 for a specific character set. It also provides convenience routines for
161 converting between character sets, given the availability of the
162 applicable codecs. Given a character set, it will do its best to provide
163 information on how to use that character set in an email in an
164 RFC-compliant way.
165
166 Certain character sets must be encoded with quoted-printable or base64
167 when used in email headers or bodies. Certain character sets must be
168 converted outright, and are not allowed in email. Instances of this
169 module expose the following information about a character set:
170
171 input_charset: The initial character set specified. Common aliases
172 are converted to their `official' email names (e.g. latin_1
173 is converted to iso-8859-1). Defaults to 7-bit us-ascii.
174
175 header_encoding: If the character set must be encoded before it can be
176 used in an email header, this attribute will be set to
177 Charset.QP (for quoted-printable), Charset.BASE64 (for
178 base64 encoding), or Charset.SHORTEST for the shortest of
179 QP or BASE64 encoding. Otherwise, it will be None.
180
181 body_encoding: Same as header_encoding, but describes the encoding for the
182 mail message's body, which indeed may be different than the
183 header encoding. Charset.SHORTEST is not allowed for
184 body_encoding.
185
186 output_charset: Some character sets must be converted before the can be
187 used in email headers or bodies. If the input_charset is
188 one of them, this attribute will contain the name of the
189 charset output will be converted to. Otherwise, it will
190 be None.
191
192 input_codec: The name of the Python codec used to convert the
193 input_charset to Unicode. If no conversion codec is
194 necessary, this attribute will be None.
195
196 output_codec: The name of the Python codec used to convert Unicode
197 to the output_charset. If no conversion codec is necessary,
198 this attribute will have the same value as the input_codec.
199 """
200 def __init__(self, input_charset=DEFAULT_CHARSET):
201 # RFC 2046, $4.1.2 says charsets are not case sensitive. We coerce to
202 # unicode because its .lower() is locale insensitive. If the argument
203 # is already a unicode, we leave it at that, but ensure that the
204 # charset is ASCII, as the standard (RFC XXX) requires.
205 try:
206 if isinstance(input_charset, unicode):
207 input_charset.encode('ascii')
208 else:
209 input_charset = unicode(input_charset, 'ascii')
210 except UnicodeError:
211 raise errors.CharsetError(input_charset)
212 input_charset = input_charset.lower()
213 # Set the input charset after filtering through the aliases
214 self.input_charset = ALIASES.get(input_charset, input_charset)
215 # We can try to guess which encoding and conversion to use by the
216 # charset_map dictionary. Try that first, but let the user override
217 # it.
218 henc, benc, conv = CHARSETS.get(self.input_charset,
219 (SHORTEST, BASE64, None))
220 if not conv:
221 conv = self.input_charset
222 # Set the attributes, allowing the arguments to override the default.
223 self.header_encoding = henc
224 self.body_encoding = benc
225 self.output_charset = ALIASES.get(conv, conv)
226 # Now set the codecs. If one isn't defined for input_charset,
227 # guess and try a Unicode codec with the same name as input_codec.
228 self.input_codec = CODEC_MAP.get(self.input_charset,
229 self.input_charset)
230 self.output_codec = CODEC_MAP.get(self.output_charset,
231 self.output_charset)
232
233 def __str__(self):
234 return self.input_charset.lower()
235
236 __repr__ = __str__
237
238 def __eq__(self, other):
239 return str(self) == str(other).lower()
240
241 def __ne__(self, other):
242 return not self.__eq__(other)
243
244 def get_body_encoding(self):
245 """Return the content-transfer-encoding used for body encoding.
246
247 This is either the string `quoted-printable' or `base64' depending on
248 the encoding used, or it is a function in which case you should call
249 the function with a single argument, the Message object being
250 encoded. The function should then set the Content-Transfer-Encoding
251 header itself to whatever is appropriate.
252
253 Returns "quoted-printable" if self.body_encoding is QP.
254 Returns "base64" if self.body_encoding is BASE64.
255 Returns "7bit" otherwise.
256 """
257 assert self.body_encoding <> SHORTEST
258 if self.body_encoding == QP:
259 return 'quoted-printable'
260 elif self.body_encoding == BASE64:
261 return 'base64'
262 else:
263 return encode_7or8bit
264
265 def convert(self, s):
266 """Convert a string from the input_codec to the output_codec."""
267 if self.input_codec <> self.output_codec:
268 return unicode(s, self.input_codec).encode(self.output_codec)
269 else:
270 return s
271
272 def to_splittable(self, s):
273 """Convert a possibly multibyte string to a safely splittable format.
274
275 Uses the input_codec to try and convert the string to Unicode, so it
276 can be safely split on character boundaries (even for multibyte
277 characters).
278
279 Returns the string as-is if it isn't known how to convert it to
280 Unicode with the input_charset.
281
282 Characters that could not be converted to Unicode will be replaced
283 with the Unicode replacement character U+FFFD.
284 """
285 if isinstance(s, unicode) or self.input_codec is None:
286 return s
287 try:
288 return unicode(s, self.input_codec, 'replace')
289 except LookupError:
290 # Input codec not installed on system, so return the original
291 # string unchanged.
292 return s
293
294 def from_splittable(self, ustr, to_output=True):
295 """Convert a splittable string back into an encoded string.
296
297 Uses the proper codec to try and convert the string from Unicode back
298 into an encoded format. Return the string as-is if it is not Unicode,
299 or if it could not be converted from Unicode.
300
301 Characters that could not be converted from Unicode will be replaced
302 with an appropriate character (usually '?').
303
304 If to_output is True (the default), uses output_codec to convert to an
305 encoded format. If to_output is False, uses input_codec.
306 """
307 if to_output:
308 codec = self.output_codec
309 else:
310 codec = self.input_codec
311 if not isinstance(ustr, unicode) or codec is None:
312 return ustr
313 try:
314 return ustr.encode(codec, 'replace')
315 except LookupError:
316 # Output codec not installed
317 return ustr
318
319 def get_output_charset(self):
320 """Return the output character set.
321
322 This is self.output_charset if that is not None, otherwise it is
323 self.input_charset.
324 """
325 return self.output_charset or self.input_charset
326
327 def encoded_header_len(self, s):
328 """Return the length of the encoded header string."""
329 cset = self.get_output_charset()
330 # The len(s) of a 7bit encoding is len(s)
331 if self.header_encoding == BASE64:
332 return email.base64mime.base64_len(s) + len(cset) + MISC_LEN
333 elif self.header_encoding == QP:
334 return email.quoprimime.header_quopri_len(s) + len(cset) + MISC_LEN
335 elif self.header_encoding == SHORTEST:
336 lenb64 = email.base64mime.base64_len(s)
337 lenqp = email.quoprimime.header_quopri_len(s)
338 return min(lenb64, lenqp) + len(cset) + MISC_LEN
339 else:
340 return len(s)
341
342 def header_encode(self, s, convert=False):
343 """Header-encode a string, optionally converting it to output_charset.
344
345 If convert is True, the string will be converted from the input
346 charset to the output charset automatically. This is not useful for
347 multibyte character sets, which have line length issues (multibyte
348 characters must be split on a character, not a byte boundary); use the
349 high-level Header class to deal with these issues. convert defaults
350 to False.
351
352 The type of encoding (base64 or quoted-printable) will be based on
353 self.header_encoding.
354 """
355 cset = self.get_output_charset()
356 if convert:
357 s = self.convert(s)
358 # 7bit/8bit encodings return the string unchanged (modulo conversions)
359 if self.header_encoding == BASE64:
360 return email.base64mime.header_encode(s, cset)
361 elif self.header_encoding == QP:
362 return email.quoprimime.header_encode(s, cset, maxlinelen=None)
363 elif self.header_encoding == SHORTEST:
364 lenb64 = email.base64mime.base64_len(s)
365 lenqp = email.quoprimime.header_quopri_len(s)
366 if lenb64 < lenqp:
367 return email.base64mime.header_encode(s, cset)
368 else:
369 return email.quoprimime.header_encode(s, cset, maxlinelen=None)
370 else:
371 return s
372
373 def body_encode(self, s, convert=True):
374 """Body-encode a string and convert it to output_charset.
375
376 If convert is True (the default), the string will be converted from
377 the input charset to output charset automatically. Unlike
378 header_encode(), there are no issues with byte boundaries and
379 multibyte charsets in email bodies, so this is usually pretty safe.
380
381 The type of encoding (base64 or quoted-printable) will be based on
382 self.body_encoding.
383 """
384 if convert:
385 s = self.convert(s)
386 # 7bit/8bit encodings return the string unchanged (module conversions)
387 if self.body_encoding is BASE64:
388 return email.base64mime.body_encode(s)
389 elif self.body_encoding is QP:
390 return email.quoprimime.body_encode(s)
391 else:
392 return s
Note: See TracBrowser for help on using the repository browser.