Debian Codegraph @

Lib/test/test_capi/test_codecs.py

raw
1 import codecs
2 import contextlib
3 import io
4 import re
5 import sys
6 import unittest
7 import unittest.mock as mock
8 import _testcapi
9 from test.support import import_helper
10
11 _testlimitedcapi = import_helper.import_module('_testlimitedcapi')
12
13 NULL = None
14 BAD_ARGUMENT = re.escape('bad argument type for built-in operation')
15
16
17 class CAPIUnicodeTest(unittest.TestCase):
18 # TODO: Test the following functions:
19 #
20 # PyUnicode_BuildEncodingMap
21 # PyUnicode_FSConverter
22 # PyUnicode_FSDecoder
23 # PyUnicode_DecodeMBCS
24 # PyUnicode_DecodeMBCSStateful
25 # PyUnicode_DecodeCodePageStateful
26 # PyUnicode_AsMBCSString
27 # PyUnicode_EncodeCodePage
28 # PyUnicode_DecodeLocaleAndSize
29 # PyUnicode_DecodeLocale
30 # PyUnicode_EncodeLocale
31 # PyUnicode_DecodeFSDefault
32 # PyUnicode_DecodeFSDefaultAndSize
33 # PyUnicode_EncodeFSDefault
34
35 def test_fromencodedobject(self):
36 """Test PyUnicode_FromEncodedObject()"""
37 fromencodedobject = _testlimitedcapi.unicode_fromencodedobject
38
39 self.assertEqual(fromencodedobject(b'abc', NULL), 'abc')
40 self.assertEqual(fromencodedobject(b'abc', 'ascii'), 'abc')
41 b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80'
42 s = 'a\xa1\u4f60\U0001f600'
43 self.assertEqual(fromencodedobject(b, NULL), s)
44 self.assertEqual(fromencodedobject(b, 'utf-8'), s)
45 self.assertEqual(fromencodedobject(b, 'latin1'), b.decode('latin1'))
46 self.assertRaises(UnicodeDecodeError, fromencodedobject, b, 'ascii')
47 self.assertEqual(fromencodedobject(b, 'ascii', 'replace'),
48 'a' + '\ufffd'*9)
49 self.assertEqual(fromencodedobject(bytearray(b), NULL), s)
50 self.assertEqual(fromencodedobject(bytearray(b), 'utf-8'), s)
51 self.assertRaises(LookupError, fromencodedobject, b'abc', 'foo')
52 self.assertRaises(LookupError, fromencodedobject, b, 'ascii', 'foo')
53 self.assertRaises(TypeError, fromencodedobject, 'abc', NULL)
54 self.assertRaises(TypeError, fromencodedobject, 'abc', 'ascii')
55 self.assertRaises(TypeError, fromencodedobject, [], NULL)
56 self.assertRaises(TypeError, fromencodedobject, [], 'ascii')
57 self.assertRaises(SystemError, fromencodedobject, NULL, NULL)
58 self.assertRaises(SystemError, fromencodedobject, NULL, 'ascii')
59
60 def test_decode(self):
61 """Test PyUnicode_Decode()"""
62 decode = _testlimitedcapi.unicode_decode
63
64 self.assertEqual(decode(b'[\xe2\x82\xac]', 'utf-8'), '[\u20ac]')
65 self.assertEqual(decode(b'[\xa4]', 'iso8859-15'), '[\u20ac]')
66 self.assertEqual(decode(b'[\xa4]', 'iso8859-15', 'strict'), '[\u20ac]')
67 self.assertRaises(UnicodeDecodeError, decode, b'[\xa4]', 'utf-8')
68 self.assertEqual(decode(b'[\xa4]', 'utf-8', 'replace'), '[\ufffd]')
69
70 self.assertEqual(decode(b'[\xe2\x82\xac]', NULL), '[\u20ac]')
71 self.assertEqual(decode(b'[\xa4]', NULL, 'replace'), '[\ufffd]')
72
73 self.assertRaises(LookupError, decode, b'\xa4', 'foo')
74 self.assertRaises(LookupError, decode, b'\xa4', 'utf-8', 'foo')
75 # TODO: Test PyUnicode_Decode() with NULL as data and
76 # negative size.
77
78 def test_asencodedstring(self):
79 """Test PyUnicode_AsEncodedString()"""
80 asencodedstring = _testlimitedcapi.unicode_asencodedstring
81
82 self.assertEqual(asencodedstring('abc', NULL), b'abc')
83 self.assertEqual(asencodedstring('abc', 'ascii'), b'abc')
84 s = 'a\xa1\u4f60\U0001f600'
85 b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80'
86 self.assertEqual(asencodedstring(s, NULL), b)
87 self.assertEqual(asencodedstring(s, 'utf-8'), b)
88 self.assertEqual(asencodedstring('\xa1\xa2', 'latin1'), b'\xa1\xa2')
89 self.assertRaises(UnicodeEncodeError, asencodedstring, '\xa1\xa2', 'ascii')
90 self.assertEqual(asencodedstring(s, 'ascii', 'replace'), b'a???')
91
92 self.assertRaises(LookupError, asencodedstring, 'abc', 'foo')
93 self.assertRaises(LookupError, asencodedstring, s, 'ascii', 'foo')
94 self.assertRaises(TypeError, asencodedstring, b'abc', NULL)
95 self.assertRaises(TypeError, asencodedstring, b'abc', 'ascii')
96 self.assertRaises(TypeError, asencodedstring, [], NULL)
97 self.assertRaises(TypeError, asencodedstring, [], 'ascii')
98 # CRASHES asencodedstring(NULL, NULL)
99 # CRASHES asencodedstring(NULL, 'ascii')
100
101 def test_decodeutf8(self):
102 """Test PyUnicode_DecodeUTF8()"""
103 decodeutf8 = _testlimitedcapi.unicode_decodeutf8
104
105 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
106 b = s.encode('utf-8')
107 self.assertEqual(decodeutf8(b), s)
108 self.assertEqual(decodeutf8(b, 'strict'), s)
109
110 self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80')
111 self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0')
112 self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff')
113 self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f')
114 self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd')
115 self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb')
116
117 self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo')
118 # TODO: Test PyUnicode_DecodeUTF8() with NULL as data and
119 # negative size.
120
121 def test_decodeutf8stateful(self):
122 """Test PyUnicode_DecodeUTF8Stateful()"""
123 decodeutf8stateful = _testlimitedcapi.unicode_decodeutf8stateful
124
125 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
126 b = s.encode('utf-8')
127 self.assertEqual(decodeutf8stateful(b), (s, len(b)))
128 self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b)))
129
130 self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80')
131 self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0')
132 self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff')
133 self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1))
134 self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1))
135 self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb')
136 self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4))
137
138 self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo')
139 # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and
140 # negative size.
141 # TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of
142 # "consumed".
143
144 def test_asutf8string(self):
145 """Test PyUnicode_AsUTF8String()"""
146 asutf8string = _testlimitedcapi.unicode_asutf8string
147
148 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
149 self.assertEqual(asutf8string(s), s.encode('utf-8'))
150
151 self.assertRaises(UnicodeEncodeError, asutf8string, '\ud8ff')
152 self.assertRaises(TypeError, asutf8string, b'abc')
153 self.assertRaises(TypeError, asutf8string, [])
154 # CRASHES asutf8string(NULL)
155
156 def test_decodeutf16(self):
157 """Test PyUnicode_DecodeUTF16()"""
158 decodeutf16 = _testlimitedcapi.unicode_decodeutf16
159
160 naturalbyteorder = -1 if sys.byteorder == 'little' else 1
161 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
162 b = s.encode('utf-16')
163 self.assertEqual(decodeutf16(0, b), (naturalbyteorder, s))
164 b = s.encode('utf-16le')
165 self.assertEqual(decodeutf16(-1, b), (-1, s))
166 self.assertEqual(decodeutf16(0, b'\xff\xfe'+b), (-1, s))
167 b = s.encode('utf-16be')
168 self.assertEqual(decodeutf16(1, b), (1, s))
169 self.assertEqual(decodeutf16(0, b'\xfe\xff'+b), (1, s))
170
171 self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'a')
172 self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'a')
173 self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xff\xfea')
174 self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xfe\xffa')
175
176 self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x00\xde')
177 self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xde\x00')
178 self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xde\xde')
179 self.assertEqual(decodeutf16(-1, b'\x00\xde', 'replace'), (-1, '\ufffd'))
180 self.assertEqual(decodeutf16(1, b'\xde\x00', 'replace'), (1, '\ufffd'))
181 self.assertEqual(decodeutf16(0, b'\xde\xde', 'replace'), (0, '\ufffd'))
182 self.assertEqual(decodeutf16(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd'))
183 self.assertEqual(decodeutf16(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd'))
184
185 self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x3d\xd8')
186 self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xd8\x3d')
187 self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xd8\xd8')
188 self.assertEqual(decodeutf16(-1, b'\x3d\xd8', 'replace'), (-1, '\ufffd'))
189 self.assertEqual(decodeutf16(1, b'\xd8\x3d', 'replace'), (1, '\ufffd'))
190 self.assertEqual(decodeutf16(0, b'\xd8\xd8', 'replace'), (0, '\ufffd'))
191 self.assertEqual(decodeutf16(0, b'\xff\xfe\x3d\xd8', 'replace'), (-1, '\ufffd'))
192 self.assertEqual(decodeutf16(0, b'\xfe\xff\xd8\x3d', 'replace'), (1, '\ufffd'))
193
194 self.assertRaises(LookupError, decodeutf16, -1, b'\x00\xde', 'foo')
195 self.assertRaises(LookupError, decodeutf16, 1, b'\xde\x00', 'foo')
196 self.assertRaises(LookupError, decodeutf16, 0, b'\xde\xde', 'foo')
197 # TODO: Test PyUnicode_DecodeUTF16() with NULL as data and
198 # negative size.
199
200 def test_decodeutf16stateful(self):
201 """Test PyUnicode_DecodeUTF16Stateful()"""
202 decodeutf16stateful = _testlimitedcapi.unicode_decodeutf16stateful
203
204 naturalbyteorder = -1 if sys.byteorder == 'little' else 1
205 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
206 b = s.encode('utf-16')
207 self.assertEqual(decodeutf16stateful(0, b), (naturalbyteorder, s, len(b)))
208 b = s.encode('utf-16le')
209 self.assertEqual(decodeutf16stateful(-1, b), (-1, s, len(b)))
210 self.assertEqual(decodeutf16stateful(0, b'\xff\xfe'+b), (-1, s, len(b)+2))
211 b = s.encode('utf-16be')
212 self.assertEqual(decodeutf16stateful(1, b), (1, s, len(b)))
213 self.assertEqual(decodeutf16stateful(0, b'\xfe\xff'+b), (1, s, len(b)+2))
214
215 self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d'), (-1, 'a', 2))
216 self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8'), (-1, 'a', 2))
217 self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8\x00'), (-1, 'a', 2))
218 self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8'), (1, 'a', 2))
219 self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d'), (1, 'a', 2))
220 self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d\xde'), (1, 'a', 2))
221 self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x61\x00\x3d\xd8\x00'), (-1, 'a', 4))
222 self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\x00\x61\xd8\x3d\xde'), (1, 'a', 4))
223
224 self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x00\xde')
225 self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xde\x00')
226 self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 0, b'\xde\xde')
227 self.assertEqual(decodeutf16stateful(-1, b'\x00\xde', 'replace'), (-1, '\ufffd', 2))
228 self.assertEqual(decodeutf16stateful(1, b'\xde\x00', 'replace'), (1, '\ufffd', 2))
229 self.assertEqual(decodeutf16stateful(0, b'\xde\xde', 'replace'), (0, '\ufffd', 2))
230 self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd', 4))
231 self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd', 4))
232
233 self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x3d\xd8\x61\x00')
234 self.assertEqual(decodeutf16stateful(-1, b'\x3d\xd8\x61\x00', 'replace'), (-1, '\ufffda', 4))
235 self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xd8\x3d\x00\x61')
236 self.assertEqual(decodeutf16stateful(1, b'\xd8\x3d\x00\x61', 'replace'), (1, '\ufffda', 4))
237
238 self.assertRaises(LookupError, decodeutf16stateful, -1, b'\x00\xde', 'foo')
239 self.assertRaises(LookupError, decodeutf16stateful, 1, b'\xde\x00', 'foo')
240 self.assertRaises(LookupError, decodeutf16stateful, 0, b'\xde\xde', 'foo')
241 # TODO: Test PyUnicode_DecodeUTF16Stateful() with NULL as data and
242 # negative size.
243 # TODO: Test PyUnicode_DecodeUTF16Stateful() with NULL as the address of
244 # "consumed".
245
246 def test_asutf16string(self):
247 """Test PyUnicode_AsUTF16String()"""
248 asutf16string = _testlimitedcapi.unicode_asutf16string
249
250 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
251 self.assertEqual(asutf16string(s), s.encode('utf-16'))
252
253 self.assertRaises(UnicodeEncodeError, asutf16string, '\ud8ff')
254 self.assertRaises(TypeError, asutf16string, b'abc')
255 self.assertRaises(TypeError, asutf16string, [])
256 # CRASHES asutf16string(NULL)
257
258 def test_decodeutf32(self):
259 """Test PyUnicode_DecodeUTF8()"""
260 decodeutf32 = _testlimitedcapi.unicode_decodeutf32
261
262 naturalbyteorder = -1 if sys.byteorder == 'little' else 1
263 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
264 b = s.encode('utf-32')
265 self.assertEqual(decodeutf32(0, b), (naturalbyteorder, s))
266 b = s.encode('utf-32le')
267 self.assertEqual(decodeutf32(-1, b), (-1, s))
268 self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00'+b), (-1, s))
269 b = s.encode('utf-32be')
270 self.assertEqual(decodeutf32(1, b), (1, s))
271 self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff'+b), (1, s))
272
273 self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x61\x00\x00\x00\x00')
274 self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\x00\x61\x00')
275 self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00')
276 self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00')
277
278 self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\xff\xff\xff\xff')
279 self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\xff\xff\xff\xff')
280 self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xff\xff\xff')
281 self.assertEqual(decodeutf32(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd'))
282 self.assertEqual(decodeutf32(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd'))
283 self.assertEqual(decodeutf32(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd'))
284 self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd'))
285 self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd'))
286
287 self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x3d\xd8\x00\x00')
288 self.assertEqual(decodeutf32(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd'))
289 self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\xd8\x3d')
290 self.assertEqual(decodeutf32(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd'))
291
292 self.assertRaises(LookupError, decodeutf32, -1, b'\xff\xff\xff\xff', 'foo')
293 self.assertRaises(LookupError, decodeutf32, 1, b'\xff\xff\xff\xff', 'foo')
294 self.assertRaises(LookupError, decodeutf32, 0, b'\xff\xff\xff\xff', 'foo')
295 # TODO: Test PyUnicode_DecodeUTF32() with NULL as data and
296 # negative size.
297
298 def test_decodeutf32stateful(self):
299 """Test PyUnicode_DecodeUTF32Stateful()"""
300 decodeutf32stateful = _testlimitedcapi.unicode_decodeutf32stateful
301
302 naturalbyteorder = -1 if sys.byteorder == 'little' else 1
303 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
304 b = s.encode('utf-32')
305 self.assertEqual(decodeutf32stateful(0, b), (naturalbyteorder, s, len(b)))
306 b = s.encode('utf-32le')
307 self.assertEqual(decodeutf32stateful(-1, b), (-1, s, len(b)))
308 self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, s, len(b)+4))
309 b = s.encode('utf-32be')
310 self.assertEqual(decodeutf32stateful(1, b), (1, s, len(b)))
311 self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, s, len(b)+4))
312
313 self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00'), (-1, 'a', 4))
314 self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6'), (-1, 'a', 4))
315 self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 4))
316 self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00'), (1, 'a', 4))
317 self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01'), (1, 'a', 4))
318 self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 4))
319 self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 8))
320 self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 8))
321
322 for b in b'\xff', b'\xff\xff', b'\xff\xff\xff':
323 self.assertEqual(decodeutf32stateful(-1, b), (-1, '', 0))
324 self.assertEqual(decodeutf32stateful(1, b), (1, '', 0))
325 self.assertEqual(decodeutf32stateful(0, b), (0, '', 0))
326 self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, '', 4))
327 self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, '', 4))
328 self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\xff\xff\xff\xff')
329 self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\xff\xff\xff\xff')
330 self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 0, b'\xff\xff\xff\xff')
331 self.assertEqual(decodeutf32stateful(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 4))
332 self.assertEqual(decodeutf32stateful(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 4))
333 self.assertEqual(decodeutf32stateful(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd', 4))
334 self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 8))
335 self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 8))
336
337 self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\x3d\xd8\x00\x00')
338 self.assertEqual(decodeutf32stateful(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd', 4))
339 self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\x00\x00\xd8\x3d')
340 self.assertEqual(decodeutf32stateful(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd', 4))
341
342 self.assertRaises(LookupError, decodeutf32stateful, -1, b'\xff\xff\xff\xff', 'foo')
343 self.assertRaises(LookupError, decodeutf32stateful, 1, b'\xff\xff\xff\xff', 'foo')
344 self.assertRaises(LookupError, decodeutf32stateful, 0, b'\xff\xff\xff\xff', 'foo')
345 # TODO: Test PyUnicode_DecodeUTF32Stateful() with NULL as data and
346 # negative size.
347 # TODO: Test PyUnicode_DecodeUTF32Stateful() with NULL as the address of
348 # "consumed".
349
350 def test_asutf32string(self):
351 """Test PyUnicode_AsUTF32String()"""
352 asutf32string = _testlimitedcapi.unicode_asutf32string
353
354 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']:
355 self.assertEqual(asutf32string(s), s.encode('utf-32'))
356
357 self.assertRaises(UnicodeEncodeError, asutf32string, '\ud8ff')
358 self.assertRaises(TypeError, asutf32string, b'abc')
359 self.assertRaises(TypeError, asutf32string, [])
360 # CRASHES asutf32string(NULL)
361
362 def test_decodelatin1(self):
363 """Test PyUnicode_DecodeLatin1()"""
364 decodelatin1 = _testlimitedcapi.unicode_decodelatin1
365
366 self.assertEqual(decodelatin1(b'abc'), 'abc')
367 self.assertEqual(decodelatin1(b'abc', 'strict'), 'abc')
368 self.assertEqual(decodelatin1(b'\xa1\xa2'), '\xa1\xa2')
369 self.assertEqual(decodelatin1(b'\xa1\xa2', 'strict'), '\xa1\xa2')
370 # TODO: Test PyUnicode_DecodeLatin1() with NULL as data and
371 # negative size.
372
373 def test_aslatin1string(self):
374 """Test PyUnicode_AsLatin1String()"""
375 aslatin1string = _testlimitedcapi.unicode_aslatin1string
376
377 self.assertEqual(aslatin1string('abc'), b'abc')
378 self.assertEqual(aslatin1string('\xa1\xa2'), b'\xa1\xa2')
379
380 self.assertRaises(UnicodeEncodeError, aslatin1string, '\u4f60')
381 self.assertRaises(TypeError, aslatin1string, b'abc')
382 self.assertRaises(TypeError, aslatin1string, [])
383 # CRASHES aslatin1string(NULL)
384
385 def test_decodeascii(self):
386 """Test PyUnicode_DecodeASCII()"""
387 decodeascii = _testlimitedcapi.unicode_decodeascii
388
389 self.assertEqual(decodeascii(b'abc'), 'abc')
390 self.assertEqual(decodeascii(b'abc', 'strict'), 'abc')
391
392 self.assertRaises(UnicodeDecodeError, decodeascii, b'\xff')
393 self.assertEqual(decodeascii(b'a\xff', 'replace'), 'a\ufffd')
394 self.assertEqual(decodeascii(b'a\xffb', 'replace'), 'a\ufffdb')
395
396 self.assertRaises(LookupError, decodeascii, b'a\xff', 'foo')
397 # TODO: Test PyUnicode_DecodeASCII() with NULL as data and
398 # negative size.
399
400 def test_asasciistring(self):
401 """Test PyUnicode_AsASCIIString()"""
402 asasciistring = _testlimitedcapi.unicode_asasciistring
403
404 self.assertEqual(asasciistring('abc'), b'abc')
405
406 self.assertRaises(UnicodeEncodeError, asasciistring, '\x80')
407 self.assertRaises(TypeError, asasciistring, b'abc')
408 self.assertRaises(TypeError, asasciistring, [])
409 # CRASHES asasciistring(NULL)
410
411 def test_decodecharmap(self):
412 """Test PyUnicode_DecodeCharmap()"""
413 decodecharmap = _testlimitedcapi.unicode_decodecharmap
414
415 self.assertEqual(decodecharmap(b'\3\0\7', {0: 'a', 3: 'b', 7: 'c'}), 'bac')
416 self.assertEqual(decodecharmap(b'\1\0\2', ['a', 'b', 'c']), 'bac')
417 self.assertEqual(decodecharmap(b'\1\0\2', 'abc'), 'bac')
418 self.assertEqual(decodecharmap(b'\1\0\2', ['\xa1', '\xa2', '\xa3']), '\xa2\xa1\xa3')
419 self.assertEqual(decodecharmap(b'\1\0\2', ['\u4f60', '\u597d', '\u4e16']), '\u597d\u4f60\u4e16')
420 self.assertEqual(decodecharmap(b'\1\0\2', ['\U0001f600', '\U0001f601', '\U0001f602']), '\U0001f601\U0001f600\U0001f602')
421
422 self.assertEqual(decodecharmap(b'\1\0\2', [97, 98, 99]), 'bac')
423 self.assertEqual(decodecharmap(b'\1\0\2', ['', 'b', 'cd']), 'bcd')
424
425 self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {})
426 self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {0: None})
427 self.assertEqual(decodecharmap(b'\1\0\2', [None, 'b', 'c'], 'replace'), 'b\ufffdc')
428 self.assertEqual(decodecharmap(b'\1\0\2\xff', NULL), '\1\0\2\xff')
429 self.assertRaises(TypeError, decodecharmap, b'\0', 42)
430
431 # TODO: Test PyUnicode_DecodeCharmap() with NULL as data and
432 # negative size.
433
434 def test_ascharmapstring(self):
435 """Test PyUnicode_AsCharmapString()"""
436 ascharmapstring = _testlimitedcapi.unicode_ascharmapstring
437
438 self.assertEqual(ascharmapstring('abc', {97: 3, 98: 0, 99: 7}), b'\3\0\7')
439 self.assertEqual(ascharmapstring('\xa1\xa2\xa3', {0xa1: 3, 0xa2: 0, 0xa3: 7}), b'\3\0\7')
440 self.assertEqual(ascharmapstring('\u4f60\u597d\u4e16', {0x4f60: 3, 0x597d: 0, 0x4e16: 7}), b'\3\0\7')
441 self.assertEqual(ascharmapstring('\U0001f600\U0001f601\U0001f602', {0x1f600: 3, 0x1f601: 0, 0x1f602: 7}), b'\3\0\7')
442 self.assertEqual(ascharmapstring('abc', {97: 3, 98: b'', 99: b'spam'}), b'\3spam')
443
444 self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {})
445 self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {97: None})
446 self.assertRaises(TypeError, ascharmapstring, b'a', {})
447 self.assertRaises(TypeError, ascharmapstring, [], {})
448 self.assertRaises(TypeError, ascharmapstring, 'a', NULL)
449 # CRASHES ascharmapstring(NULL, {})
450
451 def test_decodeunicodeescape(self):
452 """Test PyUnicode_DecodeUnicodeEscape()"""
453 decodeunicodeescape = _testlimitedcapi.unicode_decodeunicodeescape
454
455 self.assertEqual(decodeunicodeescape(b'abc'), 'abc')
456 self.assertEqual(decodeunicodeescape(br'\t\n\r\x0b\x0c\x00\\'), '\t\n\r\v\f\0\\')
457 self.assertEqual(decodeunicodeescape(b'\t\n\r\x0b\x0c\x00'), '\t\n\r\v\f\0')
458 self.assertEqual(decodeunicodeescape(br'\xa1\xa2'), '\xa1\xa2')
459 self.assertEqual(decodeunicodeescape(b'\xa1\xa2'), '\xa1\xa2')
460 self.assertEqual(decodeunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d')
461 self.assertEqual(decodeunicodeescape(br'\U0001f600'), '\U0001f600')
462 with self.assertWarns(DeprecationWarning):
463 self.assertEqual(decodeunicodeescape(br'\z'), r'\z')
464
465 for b in b'\\', br'\xa', br'\u4f6', br'\U0001f60':
466 self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b)
467 self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b, 'strict')
468 self.assertEqual(decodeunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd')
469 self.assertEqual(decodeunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy')
470
471 self.assertRaises(LookupError, decodeunicodeescape, b'\\', 'foo')
472 # TODO: Test PyUnicode_DecodeUnicodeEscape() with NULL as data and
473 # negative size.
474
475 def test_asunicodeescapestring(self):
476 """Test PyUnicode_AsUnicodeEscapeString()"""
477 asunicodeescapestring = _testlimitedcapi.unicode_asunicodeescapestring
478
479 self.assertEqual(asunicodeescapestring('abc'), b'abc')
480 self.assertEqual(asunicodeescapestring('\t\n\r\v\f\0\\'), br'\t\n\r\x0b\x0c\x00\\')
481 self.assertEqual(asunicodeescapestring('\xa1\xa2'), br'\xa1\xa2')
482 self.assertEqual(asunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d')
483 self.assertEqual(asunicodeescapestring('\U0001f600'), br'\U0001f600')
484
485 self.assertRaises(TypeError, asunicodeescapestring, b'abc')
486 self.assertRaises(TypeError, asunicodeescapestring, [])
487 # CRASHES asunicodeescapestring(NULL)
488
489 def test_decoderawunicodeescape(self):
490 """Test PyUnicode_DecodeRawUnicodeEscape()"""
491 decoderawunicodeescape = _testlimitedcapi.unicode_decoderawunicodeescape
492
493 self.assertEqual(decoderawunicodeescape(b'abc'), 'abc')
494 self.assertEqual(decoderawunicodeescape(b'\t\n\r\v\f\0\\'), '\t\n\r\v\f\0\\')
495 self.assertEqual(decoderawunicodeescape(b'\xa1\xa2'), '\xa1\xa2')
496 self.assertEqual(decoderawunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d')
497 self.assertEqual(decoderawunicodeescape(br'\U0001f600'), '\U0001f600')
498 self.assertEqual(decoderawunicodeescape(br'\xa1\xa2'), r'\xa1\xa2')
499 self.assertEqual(decoderawunicodeescape(br'\z'), r'\z')
500
501 for b in br'\u4f6', br'\U0001f60':
502 self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b)
503 self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b, 'strict')
504 self.assertEqual(decoderawunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd')
505 self.assertEqual(decoderawunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy')
506
507 self.assertRaises(LookupError, decoderawunicodeescape, br'\U0001f60', 'foo')
508 # TODO: Test PyUnicode_DecodeRawUnicodeEscape() with NULL as data and
509 # negative size.
510
511 def test_asrawunicodeescapestring(self):
512 """Test PyUnicode_AsRawUnicodeEscapeString()"""
513 asrawunicodeescapestring = _testlimitedcapi.unicode_asrawunicodeescapestring
514
515 self.assertEqual(asrawunicodeescapestring('abc'), b'abc')
516 self.assertEqual(asrawunicodeescapestring('\t\n\r\v\f\0\\'), b'\t\n\r\v\f\0\\')
517 self.assertEqual(asrawunicodeescapestring('\xa1\xa2'), b'\xa1\xa2')
518 self.assertEqual(asrawunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d')
519 self.assertEqual(asrawunicodeescapestring('\U0001f600'), br'\U0001f600')
520
521 self.assertRaises(TypeError, asrawunicodeescapestring, b'abc')
522 self.assertRaises(TypeError, asrawunicodeescapestring, [])
523 # CRASHES asrawunicodeescapestring(NULL)
524
525
526 class CAPICodecs(unittest.TestCase):
527
528 def setUp(self):
529 # Encoding names are normalized internally by converting them
530 # to lowercase and their hyphens are replaced by underscores.
531 self.encoding_name = 'test.test_capi.test_codecs.codec_reversed'
532 # Make sure that our custom codec is not already registered (that
533 # way we know whether we correctly unregistered the custom codec
534 # after a test or not).
535 self.assertRaises(LookupError, codecs.lookup, self.encoding_name)
536 # create the search function without registering yet
537 self._create_custom_codec()
538
539 def _create_custom_codec(self):
540 def codec_encoder(m, errors='strict'):
541 return (type(m)().join(reversed(m)), len(m))
542
543 def codec_decoder(c, errors='strict'):
544 return (type(c)().join(reversed(c)), len(c))
545
546 class IncrementalEncoder(codecs.IncrementalEncoder):
547 def encode(self, input, final=False):
548 return codec_encoder(input)
549
550 class IncrementalDecoder(codecs.IncrementalDecoder):
551 def decode(self, input, final=False):
552 return codec_decoder(input)
553
554 class StreamReader(codecs.StreamReader):
555 def encode(self, input, errors='strict'):
556 return codec_encoder(input, errors=errors)
557
558 def decode(self, input, errors='strict'):
559 return codec_decoder(input, errors=errors)
560
561 class StreamWriter(codecs.StreamWriter):
562 def encode(self, input, errors='strict'):
563 return codec_encoder(input, errors=errors)
564
565 def decode(self, input, errors='strict'):
566 return codec_decoder(input, errors=errors)
567
568 info = codecs.CodecInfo(
569 encode=codec_encoder,
570 decode=codec_decoder,
571 streamreader=StreamReader,
572 streamwriter=StreamWriter,
573 incrementalencoder=IncrementalEncoder,
574 incrementaldecoder=IncrementalDecoder,
575 name=self.encoding_name
576 )
577
578 def search_function(encoding):
579 if encoding == self.encoding_name:
580 return info
581 return None
582
583 self.codec_info = info
584 self.search_function = search_function
585
586 @contextlib.contextmanager
587 def use_custom_encoder(self):
588 self.assertRaises(LookupError, codecs.lookup, self.encoding_name)
589 codecs.register(self.search_function)
590 yield
591 codecs.unregister(self.search_function)
592 self.assertRaises(LookupError, codecs.lookup, self.encoding_name)
593
594 def test_codec_register(self):
595 search_function, encoding = self.search_function, self.encoding_name
596 # register the search function using the C API
597 self.assertIsNone(_testcapi.codec_register(search_function))
598 # in case the test failed before cleaning up
599 self.addCleanup(codecs.unregister, self.search_function)
600 self.assertIs(codecs.lookup(encoding), search_function(encoding))
601 self.assertEqual(codecs.encode('123', encoding=encoding), '321')
602 # unregister the search function using the regular API
603 codecs.unregister(search_function)
604 self.assertRaises(LookupError, codecs.lookup, encoding)
605
606 def test_codec_unregister(self):
607 search_function, encoding = self.search_function, self.encoding_name
608 self.assertRaises(LookupError, codecs.lookup, encoding)
609 # register the search function using the regular API
610 codecs.register(search_function)
611 # in case the test failed before cleaning up
612 self.addCleanup(codecs.unregister, self.search_function)
613 self.assertIsNotNone(codecs.lookup(encoding))
614 # unregister the search function using the C API
615 self.assertIsNone(_testcapi.codec_unregister(search_function))
616 self.assertRaises(LookupError, codecs.lookup, encoding)
617
618 def test_codec_known_encoding(self):
619 self.assertRaises(LookupError, codecs.lookup, 'unknown-codec')
620 self.assertFalse(_testcapi.codec_known_encoding('unknown-codec'))
621 self.assertFalse(_testcapi.codec_known_encoding('unknown_codec'))
622 self.assertFalse(_testcapi.codec_known_encoding('UNKNOWN-codec'))
623
624 encoding_name = self.encoding_name
625 self.assertRaises(LookupError, codecs.lookup, encoding_name)
626
627 codecs.register(self.search_function)
628 self.addCleanup(codecs.unregister, self.search_function)
629
630 for name in [
631 encoding_name,
632 encoding_name.upper(),
633 encoding_name.replace('_', '-'),
634 ]:
635 with self.subTest(name):
636 self.assertTrue(_testcapi.codec_known_encoding(name))
637
638 def test_codec_encode(self):
639 encode = _testcapi.codec_encode
640 self.assertEqual(encode('a', 'utf-8', NULL), b'a')
641 self.assertEqual(encode('a', 'utf-8', 'strict'), b'a')
642 self.assertEqual(encode('[é]', 'ascii', 'ignore'), b'[]')
643
644 self.assertRaises(TypeError, encode, NULL, 'ascii', 'strict')
645 with self.assertRaisesRegex(TypeError, BAD_ARGUMENT):
646 encode('a', NULL, 'strict')
647
648 def test_codec_decode(self):
649 decode = _testcapi.codec_decode
650
651 s = 'a\xa1\u4f60\U0001f600'
652 b = s.encode()
653
654 self.assertEqual(decode(b, 'utf-8', 'strict'), s)
655 self.assertEqual(decode(b, 'utf-8', NULL), s)
656 self.assertEqual(decode(b, 'latin1', 'strict'), b.decode('latin1'))
657 self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', 'strict')
658 self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', NULL)
659 self.assertEqual(decode(b, 'ascii', 'replace'), 'a' + '\ufffd'*9)
660
661 # _codecs.decode() only reports an unknown error handling name when
662 # the corresponding error handling function is used; this difers
663 # from PyUnicode_Decode() which checks that both the encoding and
664 # the error handling name are recognized before even attempting to
665 # call the decoder.
666 self.assertEqual(decode(b'', 'utf-8', 'unknown-error-handler'), '')
667 self.assertEqual(decode(b'a', 'utf-8', 'unknown-error-handler'), 'a')
668
669 self.assertRaises(TypeError, decode, NULL, 'ascii', 'strict')
670 with self.assertRaisesRegex(TypeError, BAD_ARGUMENT):
671 decode(b, NULL, 'strict')
672
673 def test_codec_encoder(self):
674 codec_encoder = _testcapi.codec_encoder
675
676 with self.use_custom_encoder():
677 encoder = codec_encoder(self.encoding_name)
678 self.assertIs(encoder, self.codec_info.encode)
679
680 with self.assertRaisesRegex(TypeError, BAD_ARGUMENT):
681 codec_encoder(NULL)
682
683 def test_codec_decoder(self):
684 codec_decoder = _testcapi.codec_decoder
685
686 with self.use_custom_encoder():
687 decoder = codec_decoder(self.encoding_name)
688 self.assertIs(decoder, self.codec_info.decode)
689
690 with self.assertRaisesRegex(TypeError, BAD_ARGUMENT):
691 codec_decoder(NULL)
692
693 def test_codec_incremental_encoder(self):
694 codec_incremental_encoder = _testcapi.codec_incremental_encoder
695
696 with self.use_custom_encoder():
697 encoding = self.encoding_name
698
699 for errors in ['strict', NULL]:
700 with self.subTest(errors):
701 encoder = codec_incremental_encoder(encoding, errors)
702 self.assertIsInstance(encoder, self.codec_info.incrementalencoder)
703
704 with self.assertRaisesRegex(TypeError, BAD_ARGUMENT):
705 codec_incremental_encoder(NULL, 'strict')
706
707 def test_codec_incremental_decoder(self):
708 codec_incremental_decoder = _testcapi.codec_incremental_decoder
709
710 with self.use_custom_encoder():
711 encoding = self.encoding_name
712
713 for errors in ['strict', NULL]:
714 with self.subTest(errors):
715 decoder = codec_incremental_decoder(encoding, errors)
716 self.assertIsInstance(decoder, self.codec_info.incrementaldecoder)
717
718 with self.assertRaisesRegex(TypeError, BAD_ARGUMENT):
719 codec_incremental_decoder(NULL, 'strict')
720
721 def test_codec_stream_reader(self):
722 codec_stream_reader = _testcapi.codec_stream_reader
723
724 with self.use_custom_encoder():
725 encoding, stream = self.encoding_name, io.StringIO()
726 for errors in ['strict', NULL]:
727 with self.subTest(errors):
728 writer = codec_stream_reader(encoding, stream, errors)
729 self.assertIsInstance(writer, self.codec_info.streamreader)
730
731 with self.assertRaisesRegex(TypeError, BAD_ARGUMENT):
732 codec_stream_reader(NULL, stream, 'strict')
733
734 def test_codec_stream_writer(self):
735 codec_stream_writer = _testcapi.codec_stream_writer
736
737 with self.use_custom_encoder():
738 encoding, stream = self.encoding_name, io.StringIO()
739 for errors in ['strict', NULL]:
740 with self.subTest(errors):
741 writer = codec_stream_writer(encoding, stream, errors)
742 self.assertIsInstance(writer, self.codec_info.streamwriter)
743
744 with self.assertRaisesRegex(TypeError, BAD_ARGUMENT):
745 codec_stream_writer(NULL, stream, 'strict')
746
747
748 class CAPICodecErrors(unittest.TestCase):
749
750 @classmethod
751 def _generate_exception_args(cls):
752 for objlen in range(5):
753 maxind = 2 * max(2, objlen)
754 for start in range(-maxind, maxind + 1):
755 for end in range(-maxind, maxind + 1):
756 yield objlen, start, end
757
758 @classmethod
759 def generate_encode_errors(cls):
760 return tuple(
761 UnicodeEncodeError('utf-8', '0' * objlen, start, end, 'why')
762 for objlen, start, end in cls._generate_exception_args()
763 )
764
765 @classmethod
766 def generate_decode_errors(cls):
767 return tuple(
768 UnicodeDecodeError('utf-8', b'0' * objlen, start, end, 'why')
769 for objlen, start, end in cls._generate_exception_args()
770 )
771
772 @classmethod
773 def generate_translate_errors(cls):
774 return tuple(
775 UnicodeTranslateError('0' * objlen, start, end, 'why')
776 for objlen, start, end in cls._generate_exception_args()
777 )
778
779 @classmethod
780 def setUpClass(cls):
781 cls.unicode_encode_errors = cls.generate_encode_errors()
782 cls.unicode_decode_errors = cls.generate_decode_errors()
783 cls.unicode_translate_errors = cls.generate_translate_errors()
784 cls.all_unicode_errors = (
785 cls.unicode_encode_errors
786 + cls.unicode_decode_errors
787 + cls.unicode_translate_errors
788 )
789 cls.bad_unicode_errors = (
790 ValueError(),
791 )
792
793 def test_codec_register_error(self):
794 # for cleaning up between tests
795 from _codecs import _unregister_error as _codecs_unregister_error
796
797 self.assertRaises(LookupError, _testcapi.codec_lookup_error, 'custom')
798
799 def custom_error_handler(exc):
800 raise exc
801
802 error_handler = mock.Mock(wraps=custom_error_handler)
803 _testcapi.codec_register_error('custom', error_handler)
804 self.addCleanup(_codecs_unregister_error, 'custom')
805
806 self.assertRaises(UnicodeEncodeError, codecs.encode,
807 '\xff', 'ascii', errors='custom')
808 error_handler.assert_called_once()
809 error_handler.reset_mock()
810
811 self.assertRaises(UnicodeDecodeError, codecs.decode,
812 b'\xff', 'ascii', errors='custom')
813 error_handler.assert_called_once()
814
815 # _codecs._unregister_error directly delegates to the internal C
816 # function so a Python-level function test is sufficient (it is
817 # tested in test_codeccallbacks).
818
819 def test_codec_lookup_error(self):
820 codec_lookup_error = _testcapi.codec_lookup_error
821 self.assertIs(codec_lookup_error(NULL), codecs.strict_errors)
822 self.assertIs(codec_lookup_error('strict'), codecs.strict_errors)
823 self.assertIs(codec_lookup_error('ignore'), codecs.ignore_errors)
824 self.assertIs(codec_lookup_error('replace'), codecs.replace_errors)
825 self.assertIs(codec_lookup_error('xmlcharrefreplace'), codecs.xmlcharrefreplace_errors)
826 self.assertIs(codec_lookup_error('backslashreplace'), codecs.backslashreplace_errors)
827 self.assertIs(codec_lookup_error('namereplace'), codecs.namereplace_errors)
828 self.assertRaises(LookupError, codec_lookup_error, 'unknown')
829
830 def test_codec_strict_errors_handler(self):
831 handler = _testcapi.codec_strict_errors
832 for exc in self.all_unicode_errors + self.bad_unicode_errors:
833 with self.subTest(handler=handler, exc=exc):
834 self.assertRaises(type(exc), handler, exc)
835
836 def test_codec_ignore_errors_handler(self):
837 handler = _testcapi.codec_ignore_errors
838 self.do_test_codec_errors_handler(handler, self.all_unicode_errors)
839
840 def test_codec_replace_errors_handler(self):
841 handler = _testcapi.codec_replace_errors
842 self.do_test_codec_errors_handler(handler, self.all_unicode_errors)
843
844 def test_codec_xmlcharrefreplace_errors_handler(self):
845 handler = _testcapi.codec_xmlcharrefreplace_errors
846 self.do_test_codec_errors_handler(handler, self.unicode_encode_errors)
847
848 def test_codec_backslashreplace_errors_handler(self):
849 handler = _testcapi.codec_backslashreplace_errors
850 self.do_test_codec_errors_handler(handler, self.all_unicode_errors)
851
852 def test_codec_namereplace_errors_handler(self):
853 handler = _testlimitedcapi.codec_namereplace_errors
854 self.do_test_codec_errors_handler(handler, self.unicode_encode_errors)
855
856 def do_test_codec_errors_handler(self, handler, exceptions):
857 self.assertNotEqual(len(exceptions), 0)
858 for exc in exceptions:
859 with self.subTest(handler=handler, exc=exc):
860 # test that the handler does not crash
861 res = handler(exc)
862 self.assertIsInstance(res, tuple)
863 self.assertEqual(len(res), 2)
864 replacement, continue_from = res
865 self.assertIsInstance(replacement, str)
866 self.assertIsInstance(continue_from, int)
867 self.assertGreaterEqual(continue_from, 0)
868 self.assertLessEqual(continue_from, len(exc.object))
869
870 for bad_exc in (
871 self.bad_unicode_errors
872 + tuple(e for e in self.all_unicode_errors if e not in exceptions)
873 ):
874 with self.subTest('bad type', handler=handler, exc=bad_exc):
875 self.assertRaises(TypeError, handler, bad_exc)
876
877
878 if __name__ == "__main__":
879 unittest.main()