Debian Codegraph @

Lib/test/multibytecodec_support.py

raw
1 #
2 # multibytecodec_support.py
3 # Common Unittest Routines for CJK codecs
4 #
5
6 import codecs
7 import os
8 import re
9 import sys
10 import unittest
11 from http.client import HTTPException
12 from test import support
13 from io import BytesIO
14
15 class TestBase:
16 encoding = '' # codec name
17 codec = None # codec tuple (with 4 elements)
18 tstring = None # must set. 2 strings to test StreamReader
19
20 codectests = None # must set. codec test tuple
21 roundtriptest = 1 # set if roundtrip is possible with unicode
22 has_iso10646 = 0 # set if this encoding contains whole iso10646 map
23 xmlcharnametest = None # string to test xmlcharrefreplace
24 unmappedunicode = '\udeee' # a unicode code point that is not mapped.
25
26 def setUp(self):
27 if self.codec is None:
28 self.codec = codecs.lookup(self.encoding)
29 self.encode = self.codec.encode
30 self.decode = self.codec.decode
31 self.reader = self.codec.streamreader
32 self.writer = self.codec.streamwriter
33 self.incrementalencoder = self.codec.incrementalencoder
34 self.incrementaldecoder = self.codec.incrementaldecoder
35
36 def test_chunkcoding(self):
37 tstring_lines = []
38 for b in self.tstring:
39 lines = b.split(b"\n")
40 last = lines.pop()
41 assert last == b""
42 lines = [line + b"\n" for line in lines]
43 tstring_lines.append(lines)
44 for native, utf8 in zip(*tstring_lines):
45 u = self.decode(native)[0]
46 self.assertEqual(u, utf8.decode('utf-8'))
47 if self.roundtriptest:
48 self.assertEqual(native, self.encode(u)[0])
49
50 def test_errorhandle(self):
51 for source, scheme, expected in self.codectests:
52 if isinstance(source, bytes):
53 func = self.decode
54 else:
55 func = self.encode
56 if expected:
57 result = func(source, scheme)[0]
58 if func is self.decode:
59 self.assertTrue(type(result) is str, type(result))
60 self.assertEqual(result, expected,
61 '%a.decode(%r, %r)=%a != %a'
62 % (source, self.encoding, scheme, result,
63 expected))
64 else:
65 self.assertTrue(type(result) is bytes, type(result))
66 self.assertEqual(result, expected,
67 '%a.encode(%r, %r)=%a != %a'
68 % (source, self.encoding, scheme, result,
69 expected))
70 else:
71 self.assertRaises(UnicodeError, func, source, scheme)
72
73 def test_xmlcharrefreplace(self):
74 if self.has_iso10646:
75 self.skipTest('encoding contains full ISO 10646 map')
76
77 s = "\u0b13\u0b23\u0b60 nd eggs"
78 self.assertEqual(
79 self.encode(s, "xmlcharrefreplace")[0],
80 b"ଓଣୠ nd eggs"
81 )
82
83 def test_customreplace_encode(self):
84 if self.has_iso10646:
85 self.skipTest('encoding contains full ISO 10646 map')
86
87 from html.entities import codepoint2name
88
89 def xmlcharnamereplace(exc):
90 if not isinstance(exc, UnicodeEncodeError):
91 raise TypeError("don't know how to handle %r" % exc)
92 l = []
93 for c in exc.object[exc.start:exc.end]:
94 if ord(c) in codepoint2name:
95 l.append("&%s;" % codepoint2name[ord(c)])
96 else:
97 l.append("&#%d;" % ord(c))
98 return ("".join(l), exc.end)
99
100 codecs.register_error("test.xmlcharnamereplace", xmlcharnamereplace)
101
102 if self.xmlcharnametest:
103 sin, sout = self.xmlcharnametest
104 else:
105 sin = "\xab\u211c\xbb = \u2329\u1234\u232a"
106 sout = b"«ℜ» = ⟨ሴ⟩"
107 self.assertEqual(self.encode(sin,
108 "test.xmlcharnamereplace")[0], sout)
109
110 def test_callback_returns_bytes(self):
111 def myreplace(exc):
112 return (b"1234", exc.end)
113 codecs.register_error("test.cjktest", myreplace)
114 enc = self.encode("abc" + self.unmappedunicode + "def", "test.cjktest")[0]
115 self.assertEqual(enc, b"abc1234def")
116
117 def test_callback_wrong_objects(self):
118 def myreplace(exc):
119 return (ret, exc.end)
120 codecs.register_error("test.cjktest", myreplace)
121
122 for ret in ([1, 2, 3], [], None, object()):
123 self.assertRaises(TypeError, self.encode, self.unmappedunicode,
124 'test.cjktest')
125
126 def test_callback_long_index(self):
127 def myreplace(exc):
128 return ('x', int(exc.end))
129 codecs.register_error("test.cjktest", myreplace)
130 self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh',
131 'test.cjktest'), (b'abcdxefgh', 9))
132
133 def myreplace(exc):
134 return ('x', sys.maxsize + 1)
135 codecs.register_error("test.cjktest", myreplace)
136 self.assertRaises(IndexError, self.encode, self.unmappedunicode,
137 'test.cjktest')
138
139 def test_callback_None_index(self):
140 def myreplace(exc):
141 return ('x', None)
142 codecs.register_error("test.cjktest", myreplace)
143 self.assertRaises(TypeError, self.encode, self.unmappedunicode,
144 'test.cjktest')
145
146 def test_callback_backward_index(self):
147 def myreplace(exc):
148 if myreplace.limit > 0:
149 myreplace.limit -= 1
150 return ('REPLACED', 0)
151 else:
152 return ('TERMINAL', exc.end)
153 myreplace.limit = 3
154 codecs.register_error("test.cjktest", myreplace)
155 self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh',
156 'test.cjktest'),
157 (b'abcdREPLACEDabcdREPLACEDabcdREPLACEDabcdTERMINALefgh', 9))
158
159 def test_callback_forward_index(self):
160 def myreplace(exc):
161 return ('REPLACED', exc.end + 2)
162 codecs.register_error("test.cjktest", myreplace)
163 self.assertEqual(self.encode('abcd' + self.unmappedunicode + 'efgh',
164 'test.cjktest'), (b'abcdREPLACEDgh', 9))
165
166 def test_callback_index_outofbound(self):
167 def myreplace(exc):
168 return ('TERM', 100)
169 codecs.register_error("test.cjktest", myreplace)
170 self.assertRaises(IndexError, self.encode, self.unmappedunicode,
171 'test.cjktest')
172
173 def test_incrementalencoder(self):
174 UTF8Reader = codecs.getreader('utf-8')
175 for sizehint in [None] + list(range(1, 33)) + \
176 [64, 128, 256, 512, 1024]:
177 istream = UTF8Reader(BytesIO(self.tstring[1]))
178 ostream = BytesIO()
179 encoder = self.incrementalencoder()
180 while 1:
181 if sizehint is not None:
182 data = istream.read(sizehint)
183 else:
184 data = istream.read()
185
186 if not data:
187 break
188 e = encoder.encode(data)
189 ostream.write(e)
190
191 self.assertEqual(ostream.getvalue(), self.tstring[0])
192
193 def test_incrementaldecoder(self):
194 UTF8Writer = codecs.getwriter('utf-8')
195 for sizehint in [None, -1] + list(range(1, 33)) + \
196 [64, 128, 256, 512, 1024]:
197 istream = BytesIO(self.tstring[0])
198 ostream = UTF8Writer(BytesIO())
199 decoder = self.incrementaldecoder()
200 while 1:
201 data = istream.read(sizehint)
202 if not data:
203 break
204 else:
205 u = decoder.decode(data)
206 ostream.write(u)
207
208 self.assertEqual(ostream.getvalue(), self.tstring[1])
209
210 def test_incrementalencoder_error_callback(self):
211 inv = self.unmappedunicode
212
213 e = self.incrementalencoder()
214 self.assertRaises(UnicodeEncodeError, e.encode, inv, True)
215
216 e.errors = 'ignore'
217 self.assertEqual(e.encode(inv, True), b'')
218
219 e.reset()
220 def tempreplace(exc):
221 return ('called', exc.end)
222 codecs.register_error('test.incremental_error_callback', tempreplace)
223 e.errors = 'test.incremental_error_callback'
224 self.assertEqual(e.encode(inv, True), b'called')
225
226 # again
227 e.errors = 'ignore'
228 self.assertEqual(e.encode(inv, True), b'')
229
230 def test_streamreader(self):
231 UTF8Writer = codecs.getwriter('utf-8')
232 for name in ["read", "readline", "readlines"]:
233 for sizehint in [None, -1] + list(range(1, 33)) + \
234 [64, 128, 256, 512, 1024]:
235 istream = self.reader(BytesIO(self.tstring[0]))
236 ostream = UTF8Writer(BytesIO())
237 func = getattr(istream, name)
238 while 1:
239 data = func(sizehint)
240 if not data:
241 break
242 if name == "readlines":
243 ostream.writelines(data)
244 else:
245 ostream.write(data)
246
247 self.assertEqual(ostream.getvalue(), self.tstring[1])
248
249 def test_streamwriter(self):
250 readfuncs = ('read', 'readline', 'readlines')
251 UTF8Reader = codecs.getreader('utf-8')
252 for name in readfuncs:
253 for sizehint in [None] + list(range(1, 33)) + \
254 [64, 128, 256, 512, 1024]:
255 istream = UTF8Reader(BytesIO(self.tstring[1]))
256 ostream = self.writer(BytesIO())
257 func = getattr(istream, name)
258 while 1:
259 if sizehint is not None:
260 data = func(sizehint)
261 else:
262 data = func()
263
264 if not data:
265 break
266 if name == "readlines":
267 ostream.writelines(data)
268 else:
269 ostream.write(data)
270
271 self.assertEqual(ostream.getvalue(), self.tstring[0])
272
273 def test_streamwriter_reset_no_pending(self):
274 # Issue #23247: Calling reset() on a fresh StreamWriter instance
275 # (without pending data) must not crash
276 stream = BytesIO()
277 writer = self.writer(stream)
278 writer.reset()
279
280 def test_incrementalencoder_del_segfault(self):
281 e = self.incrementalencoder()
282 with self.assertRaises(AttributeError):
283 del e.errors
284
285 def test_null_terminator(self):
286 # see gh-101828
287 text = "フルーツ"
288 try:
289 text.encode(self.encoding)
290 except UnicodeEncodeError:
291 text = "Python is cool"
292 encode_w_null = (text + "\0").encode(self.encoding)
293 encode_plus_null = text.encode(self.encoding) + "\0".encode(self.encoding)
294 self.assertTrue(encode_w_null.endswith(b'\x00'))
295 self.assertEqual(encode_w_null, encode_plus_null)
296
297 encode_w_null_2 = (text + "\0" + text + "\0").encode(self.encoding)
298 encode_plus_null_2 = encode_plus_null + encode_plus_null
299 self.assertEqual(encode_w_null_2.count(b'\x00'), 2)
300 self.assertEqual(encode_w_null_2, encode_plus_null_2)
301
302
303 class TestBase_Mapping(unittest.TestCase):
304 pass_enctest = []
305 pass_dectest = []
306 supmaps = []
307 codectests = []
308
309 def setUp(self):
310 try:
311 self.open_mapping_file().close() # test it to report the error early
312 except (OSError, HTTPException):
313 self.skipTest("Could not retrieve "+self.mapfileurl)
314
315 def open_mapping_file(self):
316 return support.open_urlresource(self.mapfileurl, encoding="utf-8")
317
318 def test_mapping_file(self):
319 if self.mapfileurl.endswith('.xml'):
320 self._test_mapping_file_ucm()
321 else:
322 self._test_mapping_file_plain()
323
324 def _test_mapping_file_plain(self):
325 def unichrs(s):
326 return ''.join(chr(int(x, 16)) for x in s.split('+'))
327
328 urt_wa = {}
329
330 with self.open_mapping_file() as f:
331 for line in f:
332 if not line:
333 break
334 data = line.split('#')[0].split()
335 if len(data) != 2:
336 continue
337
338 if data[0][:2] != '0x':
339 self.fail(f"Invalid line: {line!r}")
340 csetch = bytes.fromhex(data[0][2:])
341 if len(csetch) == 1 and 0x80 <= csetch[0]:
342 continue
343
344 unich = unichrs(data[1])
345 if ord(unich) == 0xfffd or unich in urt_wa:
346 continue
347 urt_wa[unich] = csetch
348
349 self._testpoint(csetch, unich)
350
351 def _test_mapping_file_ucm(self):
352 with self.open_mapping_file() as f:
353 ucmdata = f.read()
354 uc = re.findall('<a u="([A-F0-9]{4})" b="([0-9A-F ]+)"/>', ucmdata)
355 for uni, coded in uc:
356 unich = chr(int(uni, 16))
357 codech = bytes.fromhex(coded)
358 self._testpoint(codech, unich)
359
360 def test_mapping_supplemental(self):
361 for mapping in self.supmaps:
362 self._testpoint(*mapping)
363
364 def _testpoint(self, csetch, unich):
365 if (csetch, unich) not in self.pass_enctest:
366 self.assertEqual(unich.encode(self.encoding), csetch)
367 if (csetch, unich) not in self.pass_dectest:
368 self.assertEqual(str(csetch, self.encoding), unich)
369
370 def test_errorhandle(self):
371 for source, scheme, expected in self.codectests:
372 if isinstance(source, bytes):
373 func = source.decode
374 else:
375 func = source.encode
376 if expected:
377 if isinstance(source, bytes):
378 result = func(self.encoding, scheme)
379 self.assertTrue(type(result) is str, type(result))
380 self.assertEqual(result, expected,
381 '%a.decode(%r, %r)=%a != %a'
382 % (source, self.encoding, scheme, result,
383 expected))
384 else:
385 result = func(self.encoding, scheme)
386 self.assertTrue(type(result) is bytes, type(result))
387 self.assertEqual(result, expected,
388 '%a.encode(%r, %r)=%a != %a'
389 % (source, self.encoding, scheme, result,
390 expected))
391 else:
392 self.assertRaises(UnicodeError, func, self.encoding, scheme)
393
394 def load_teststring(name):
395 dir = os.path.join(os.path.dirname(__file__), 'cjkencodings')
396 with open(os.path.join(dir, name + '.txt'), 'rb') as f:
397 encoded = f.read()
398 with open(os.path.join(dir, name + '-utf8.txt'), 'rb') as f:
399 utf8 = f.read()
400 return encoded, utf8