Debian Codegraph @

src/tinycss/decoding.py

raw
1 # coding: utf8
2 """
3 tinycss.decoding
4 ----------------
5
6 Decoding stylesheets from bytes to Unicode.
7 http://www.w3.org/TR/CSS21/syndata.html#charset
8
9 :copyright: (c) 2012 by Simon Sapin.
10 :license: BSD, see LICENSE for more details.
11 """
12
13
14 import operator
15 import re
16
17 from polyglot.binary import from_hex_bytes
18
19 __all__ = ['decode'] # Everything else is implementation detail
20
21
22 def decode(css_bytes, protocol_encoding=None,
23 linking_encoding=None, document_encoding=None):
24 """
25 Determine the character encoding from the passed metadata and the
26 ``@charset`` rule in the stylesheet (if any); and decode accordingly.
27 If no encoding information is available or decoding fails,
28 decoding defaults to UTF-8 and then fall back on ISO-8859-1.
29
30 :param css_bytes:
31 a CSS stylesheet as a byte string
32 :param protocol_encoding:
33 The "charset" parameter of a "Content-Type" HTTP header (if any),
34 or similar metadata for other protocols.
35 :param linking_encoding:
36 ``<link charset="">`` or other metadata from the linking mechanism
37 (if any)
38 :param document_encoding:
39 Encoding of the referring style sheet or document (if any)
40 :return:
41 A tuple of an Unicode string, with any BOM removed, and the
42 encoding that was used.
43
44 """
45 if protocol_encoding:
46 css_unicode = try_encoding(css_bytes, protocol_encoding)
47 if css_unicode is not None:
48 return css_unicode, protocol_encoding
49 for encoding, pattern in ENCODING_MAGIC_NUMBERS:
50 match = pattern(css_bytes)
51 if match:
52 has_at_charset = isinstance(encoding, tuple)
53 if has_at_charset:
54 extract, endianness = encoding
55 encoding = extract(match.group(1))
56 # Get an ASCII-only unicode value.
57 # This is the only thing that works on both Python 2 and 3
58 # for bytes.decode()
59 # Non-ASCII encoding names are invalid anyway,
60 # but make sure they stay invalid.
61 encoding = encoding.decode('ascii', 'replace')
62 encoding = encoding.replace('\ufffd', '?')
63 if encoding.replace('-', '').replace('_', '').lower() in [
64 'utf16', 'utf32']:
65 encoding += endianness
66 encoding = encoding.encode('ascii', 'replace').decode('ascii')
67 css_unicode = try_encoding(css_bytes, encoding)
68 if css_unicode and not (has_at_charset and not
69 css_unicode.startswith('@charset "')):
70 return css_unicode, encoding
71 break
72 for encoding in [linking_encoding, document_encoding]:
73 if encoding:
74 css_unicode = try_encoding(css_bytes, encoding)
75 if css_unicode is not None:
76 return css_unicode, encoding
77 css_unicode = try_encoding(css_bytes, 'UTF-8')
78 if css_unicode is not None:
79 return css_unicode, 'UTF-8'
80 return try_encoding(css_bytes, 'ISO-8859-1', fallback=False), 'ISO-8859-1'
81
82
83 def try_encoding(css_bytes, encoding, fallback=True):
84 if fallback:
85 try:
86 css_unicode = css_bytes.decode(encoding)
87 # LookupError means unknown encoding
88 except (UnicodeDecodeError, LookupError):
89 return None
90 else:
91 css_unicode = css_bytes.decode(encoding)
92 if css_unicode and css_unicode[0] == '\ufeff':
93 # Remove any Byte Order Mark
94 css_unicode = css_unicode[1:]
95 return css_unicode
96
97
98 def hex2re(hex_data):
99 return re.escape(from_hex_bytes(hex_data.replace(' ', '').encode('ascii')))
100
101
102 class Slicer:
103 """Slice()[start:stop:end] == slice(start, stop, end)"""
104 def __getitem__(self, slice_):
105 return operator.itemgetter(slice_)
106
107
108 Slice = Slicer()
109
110
111 # List of (bom_size, encoding, pattern)
112 # bom_size is in bytes and can be zero
113 # encoding is a string or (slice_, endianness) for "as specified"
114 # slice_ is a slice object.How to extract the specified
115
116 ENCODING_MAGIC_NUMBERS = [
117 ((Slice[:], ''), re.compile(
118 hex2re('EF BB BF 40 63 68 61 72 73 65 74 20 22')
119 + b'([^\x22]*?)'
120 + hex2re('22 3B')).match),
121
122 ('UTF-8', re.compile(
123 hex2re('EF BB BF')).match),
124
125 ((Slice[:], ''), re.compile(
126 hex2re('40 63 68 61 72 73 65 74 20 22')
127 + b'([^\x22]*?)'
128 + hex2re('22 3B')).match),
129
130 ((Slice[1::2], '-BE'), re.compile(
131 hex2re('FE FF 00 40 00 63 00 68 00 61 00 72 00 73 00 65 00'
132 '74 00 20 00 22')
133 + b'((\x00[^\x22])*?)'
134 + hex2re('00 22 00 3B')).match),
135
136 ((Slice[1::2], '-BE'), re.compile(
137 hex2re('00 40 00 63 00 68 00 61 00 72 00 73 00 65 00 74 00'
138 '20 00 22')
139 + b'((\x00[^\x22])*?)'
140 + hex2re('00 22 00 3B')).match),
141
142 ((Slice[::2], '-LE'), re.compile(
143 hex2re('FF FE 40 00 63 00 68 00 61 00 72 00 73 00 65 00 74'
144 '00 20 00 22 00')
145 + b'(([^\x22]\x00)*?)'
146 + hex2re('22 00 3B 00')).match),
147
148 ((Slice[::2], '-LE'), re.compile(
149 hex2re('40 00 63 00 68 00 61 00 72 00 73 00 65 00 74 00 20'
150 '00 22 00')
151 + b'(([^\x22]\x00)*?)'
152 + hex2re('22 00 3B 00')).match),
153
154 ((Slice[3::4], '-BE'), re.compile(
155 hex2re('00 00 FE FF 00 00 00 40 00 00 00 63 00 00 00 68 00'
156 '00 00 61 00 00 00 72 00 00 00 73 00 00 00 65 00 00'
157 '00 74 00 00 00 20 00 00 00 22')
158 + b'((\x00\x00\x00[^\x22])*?)'
159 + hex2re('00 00 00 22 00 00 00 3B')).match),
160
161 ((Slice[3::4], '-BE'), re.compile(
162 hex2re('00 00 00 40 00 00 00 63 00 00 00 68 00 00 00 61 00'
163 '00 00 72 00 00 00 73 00 00 00 65 00 00 00 74 00 00'
164 '00 20 00 00 00 22')
165 + b'((\x00\x00\x00[^\x22])*?)'
166 + hex2re('00 00 00 22 00 00 00 3B')).match),
167
168
169 # Python does not support 2143 or 3412 endianness, AFAIK.
170 # I guess we could fix it up ourselves but meh. Patches welcome.
171
172 # ((Slice[2::4], '-2143'), re.compile(
173 # hex2re('00 00 FF FE 00 00 40 00 00 00 63 00 00 00 68 00 00'
174 # '00 61 00 00 00 72 00 00 00 73 00 00 00 65 00 00 00'
175 # '74 00 00 00 20 00 00 00 22 00')
176 # + b'((\x00\x00[^\x22]\x00)*?)'
177 # + hex2re('00 00 22 00 00 00 3B 00')).match),
178
179 # ((Slice[2::4], '-2143'), re.compile(
180 # hex2re('00 00 40 00 00 00 63 00 00 00 68 00 00 00 61 00 00'
181 # '00 72 00 00 00 73 00 00 00 65 00 00 00 74 00 00 00'
182 # '20 00 00 00 22 00')
183 # + b'((\x00\x00[^\x22]\x00)*?)'
184 # + hex2re('00 00 22 00 00 00 3B 00')).match),
185
186 # ((Slice[1::4], '-3412'), re.compile(
187 # hex2re('FE FF 00 00 00 40 00 00 00 63 00 00 00 68 00 00 00'
188 # '61 00 00 00 72 00 00 00 73 00 00 00 65 00 00 00 74'
189 # '00 00 00 20 00 00 00 22 00 00')
190 # + b'((\x00[^\x22]\x00\x00)*?)'
191 # + hex2re('00 22 00 00 00 3B 00 00')).match),
192
193 # ((Slice[1::4], '-3412'), re.compile(
194 # hex2re('00 40 00 00 00 63 00 00 00 68 00 00 00 61 00 00 00'
195 # '72 00 00 00 73 00 00 00 65 00 00 00 74 00 00 00 20'
196 # '00 00 00 22 00 00')
197 # + b'((\x00[^\x22]\x00\x00)*?)'
198 # + hex2re('00 22 00 00 00 3B 00 00')).match),
199
200 ((Slice[::4], '-LE'), re.compile(
201 hex2re('FF FE 00 00 40 00 00 00 63 00 00 00 68 00 00 00 61'
202 '00 00 00 72 00 00 00 73 00 00 00 65 00 00 00 74 00'
203 '00 00 20 00 00 00 22 00 00 00')
204 + b'(([^\x22]\x00\x00\x00)*?)'
205 + hex2re('22 00 00 00 3B 00 00 00')).match),
206
207 ((Slice[::4], '-LE'), re.compile(
208 hex2re('40 00 00 00 63 00 00 00 68 00 00 00 61 00 00 00 72'
209 '00 00 00 73 00 00 00 65 00 00 00 74 00 00 00 20 00'
210 '00 00 22 00 00 00')
211 + b'(([^\x22]\x00\x00\x00)*?)'
212 + hex2re('22 00 00 00 3B 00 00 00')).match),
213
214 ('UTF-32-BE', re.compile(
215 hex2re('00 00 FE FF')).match),
216
217 ('UTF-32-LE', re.compile(
218 hex2re('FF FE 00 00')).match),
219
220 # ('UTF-32-2143', re.compile(
221 # hex2re('00 00 FF FE')).match),
222
223 # ('UTF-32-3412', re.compile(
224 # hex2re('FE FF 00 00')).match),
225
226 ('UTF-16-BE', re.compile(
227 hex2re('FE FF')).match),
228
229 ('UTF-16-LE', re.compile(
230 hex2re('FF FE')).match),
231
232
233 # Some of there are supported by Python, but I didn’t bother.
234 # You know the story with patches ...
235
236 # # as specified, transcoded from EBCDIC to ASCII
237 # ('as_specified-EBCDIC', re.compile(
238 # hex2re('7C 83 88 81 99 A2 85 A3 40 7F')
239 # + b'([^\x7F]*?)'
240 # + hex2re('7F 5E')).match),
241
242 # # as specified, transcoded from IBM1026 to ASCII
243 # ('as_specified-IBM1026', re.compile(
244 # hex2re('AE 83 88 81 99 A2 85 A3 40 FC')
245 # + b'([^\xFC]*?)'
246 # + hex2re('FC 5E')).match),
247
248 # # as specified, transcoded from GSM 03.38 to ASCII
249 # ('as_specified-GSM_03.38', re.compile(
250 # hex2re('00 63 68 61 72 73 65 74 20 22')
251 # + b'([^\x22]*?)'
252 # + hex2re('22 3B')).match),
253 ]