Debian Codegraph @

Lib/test/test_multibytecodec.py

raw
1 #
2 # test_multibytecodec.py
3 # Unit test for multibytecodec itself
4 #
5
6 import _multibytecodec
7 import codecs
8 import io
9 import sys
10 import textwrap
11 import unittest
12 from test import support
13 from test.support import os_helper
14 from test.support.os_helper import TESTFN
15 from test.support.import_helper import import_module
16
17 ALL_CJKENCODINGS = [
18 # _codecs_cn
19 'gb2312', 'gbk', 'gb18030', 'hz',
20 # _codecs_hk
21 'big5hkscs',
22 # _codecs_jp
23 'cp932', 'shift_jis', 'euc_jp', 'euc_jisx0213', 'shift_jisx0213',
24 'euc_jis_2004', 'shift_jis_2004',
25 # _codecs_kr
26 'cp949', 'euc_kr', 'johab',
27 # _codecs_tw
28 'big5', 'cp950',
29 # _codecs_iso2022
30 'iso2022_jp', 'iso2022_jp_1', 'iso2022_jp_2', 'iso2022_jp_2004',
31 'iso2022_jp_3', 'iso2022_jp_ext', 'iso2022_kr',
32 ]
33
34 class Test_MultibyteCodec(unittest.TestCase):
35
36 def test_nullcoding(self):
37 for enc in ALL_CJKENCODINGS:
38 self.assertEqual(b''.decode(enc), '')
39 self.assertEqual(str(b'', enc), '')
40 self.assertEqual(''.encode(enc), b'')
41
42 def test_str_decode(self):
43 for enc in ALL_CJKENCODINGS:
44 self.assertEqual('abcd'.encode(enc), b'abcd')
45
46 def test_errorcallback_longindex(self):
47 dec = codecs.getdecoder('euc-kr')
48 myreplace = lambda exc: ('', sys.maxsize+1)
49 codecs.register_error('test.cjktest', myreplace)
50 self.assertRaises(IndexError, dec,
51 b'apple\x92ham\x93spam', 'test.cjktest')
52
53 def test_errorcallback_custom_ignore(self):
54 # Issue #23215: MemoryError with custom error handlers and multibyte codecs
55 data = 100 * "\udc00"
56 codecs.register_error("test.ignore", codecs.ignore_errors)
57 for enc in ALL_CJKENCODINGS:
58 self.assertEqual(data.encode(enc, "test.ignore"), b'')
59
60 def test_codingspec(self):
61 try:
62 for enc in ALL_CJKENCODINGS:
63 code = '# coding: {}\n'.format(enc)
64 exec(code)
65 finally:
66 os_helper.unlink(TESTFN)
67
68 def test_init_segfault(self):
69 # bug #3305: this used to segfault
70 self.assertRaises(AttributeError,
71 _multibytecodec.MultibyteStreamReader, None)
72 self.assertRaises(AttributeError,
73 _multibytecodec.MultibyteStreamWriter, None)
74
75 def test_decode_unicode(self):
76 # Trying to decode a unicode string should raise a TypeError
77 for enc in ALL_CJKENCODINGS:
78 self.assertRaises(TypeError, codecs.getdecoder(enc), "")
79
80 class Test_IncrementalEncoder(unittest.TestCase):
81
82 def test_stateless(self):
83 # cp949 encoder isn't stateful at all.
84 encoder = codecs.getincrementalencoder('cp949')()
85 self.assertEqual(encoder.encode('\ud30c\uc774\uc36c \ub9c8\uc744'),
86 b'\xc6\xc4\xc0\xcc\xbd\xe3 \xb8\xb6\xc0\xbb')
87 self.assertEqual(encoder.reset(), None)
88 self.assertEqual(encoder.encode('\u2606\u223c\u2606', True),
89 b'\xa1\xd9\xa1\xad\xa1\xd9')
90 self.assertEqual(encoder.reset(), None)
91 self.assertEqual(encoder.encode('', True), b'')
92 self.assertEqual(encoder.encode('', False), b'')
93 self.assertEqual(encoder.reset(), None)
94
95 def test_stateful(self):
96 # jisx0213 encoder is stateful for a few code points. eg)
97 # U+00E6 => A9DC
98 # U+00E6 U+0300 => ABC4
99 # U+0300 => ABDC
100
101 encoder = codecs.getincrementalencoder('jisx0213')()
102 self.assertEqual(encoder.encode('\u00e6\u0300'), b'\xab\xc4')
103 self.assertEqual(encoder.encode('\u00e6'), b'')
104 self.assertEqual(encoder.encode('\u0300'), b'\xab\xc4')
105 self.assertEqual(encoder.encode('\u00e6', True), b'\xa9\xdc')
106
107 self.assertEqual(encoder.reset(), None)
108 self.assertEqual(encoder.encode('\u0300'), b'\xab\xdc')
109
110 self.assertEqual(encoder.encode('\u00e6'), b'')
111 self.assertEqual(encoder.encode('', True), b'\xa9\xdc')
112 self.assertEqual(encoder.encode('', True), b'')
113
114 def test_stateful_keep_buffer(self):
115 encoder = codecs.getincrementalencoder('jisx0213')()
116 self.assertEqual(encoder.encode('\u00e6'), b'')
117 self.assertRaises(UnicodeEncodeError, encoder.encode, '\u0123')
118 self.assertEqual(encoder.encode('\u0300\u00e6'), b'\xab\xc4')
119 self.assertRaises(UnicodeEncodeError, encoder.encode, '\u0123')
120 self.assertEqual(encoder.reset(), None)
121 self.assertEqual(encoder.encode('\u0300'), b'\xab\xdc')
122 self.assertEqual(encoder.encode('\u00e6'), b'')
123 self.assertRaises(UnicodeEncodeError, encoder.encode, '\u0123')
124 self.assertEqual(encoder.encode('', True), b'\xa9\xdc')
125
126 def test_state_methods_with_buffer_state(self):
127 # euc_jis_2004 stores state as a buffer of pending bytes
128 encoder = codecs.getincrementalencoder('euc_jis_2004')()
129
130 initial_state = encoder.getstate()
131 self.assertEqual(encoder.encode('\u00e6\u0300'), b'\xab\xc4')
132 encoder.setstate(initial_state)
133 self.assertEqual(encoder.encode('\u00e6\u0300'), b'\xab\xc4')
134
135 self.assertEqual(encoder.encode('\u00e6'), b'')
136 partial_state = encoder.getstate()
137 self.assertEqual(encoder.encode('\u0300'), b'\xab\xc4')
138 encoder.setstate(partial_state)
139 self.assertEqual(encoder.encode('\u0300'), b'\xab\xc4')
140
141 def test_state_methods_with_non_buffer_state(self):
142 # iso2022_jp stores state without using a buffer
143 encoder = codecs.getincrementalencoder('iso2022_jp')()
144
145 self.assertEqual(encoder.encode('z'), b'z')
146 en_state = encoder.getstate()
147
148 self.assertEqual(encoder.encode('\u3042'), b'\x1b\x24\x42\x24\x22')
149 jp_state = encoder.getstate()
150 self.assertEqual(encoder.encode('z'), b'\x1b\x28\x42z')
151
152 encoder.setstate(jp_state)
153 self.assertEqual(encoder.encode('\u3042'), b'\x24\x22')
154
155 encoder.setstate(en_state)
156 self.assertEqual(encoder.encode('z'), b'z')
157
158 def test_getstate_returns_expected_value(self):
159 # Note: getstate is implemented such that these state values
160 # are expected to be the same across all builds of Python,
161 # regardless of x32/64 bit, endianness and compiler.
162
163 # euc_jis_2004 stores state as a buffer of pending bytes
164 buffer_state_encoder = codecs.getincrementalencoder('euc_jis_2004')()
165 self.assertEqual(buffer_state_encoder.getstate(), 0)
166 buffer_state_encoder.encode('\u00e6')
167 self.assertEqual(buffer_state_encoder.getstate(),
168 int.from_bytes(
169 b"\x02"
170 b"\xc3\xa6"
171 b"\x00\x00\x00\x00\x00\x00\x00\x00",
172 'little'))
173 buffer_state_encoder.encode('\u0300')
174 self.assertEqual(buffer_state_encoder.getstate(), 0)
175
176 # iso2022_jp stores state without using a buffer
177 non_buffer_state_encoder = codecs.getincrementalencoder('iso2022_jp')()
178 self.assertEqual(non_buffer_state_encoder.getstate(),
179 int.from_bytes(
180 b"\x00"
181 b"\x42\x42\x00\x00\x00\x00\x00\x00",
182 'little'))
183 non_buffer_state_encoder.encode('\u3042')
184 self.assertEqual(non_buffer_state_encoder.getstate(),
185 int.from_bytes(
186 b"\x00"
187 b"\xc2\x42\x00\x00\x00\x00\x00\x00",
188 'little'))
189
190 def test_setstate_validates_input_size(self):
191 encoder = codecs.getincrementalencoder('euc_jp')()
192 pending_size_nine = int.from_bytes(
193 b"\x09"
194 b"\x00\x00\x00\x00\x00\x00\x00\x00"
195 b"\x00\x00\x00\x00\x00\x00\x00\x00",
196 'little')
197 self.assertRaises(UnicodeError, encoder.setstate, pending_size_nine)
198
199 def test_setstate_validates_input_bytes(self):
200 encoder = codecs.getincrementalencoder('euc_jp')()
201 invalid_utf8 = int.from_bytes(
202 b"\x01"
203 b"\xff"
204 b"\x00\x00\x00\x00\x00\x00\x00\x00",
205 'little')
206 self.assertRaises(UnicodeDecodeError, encoder.setstate, invalid_utf8)
207
208 def test_issue5640(self):
209 encoder = codecs.getincrementalencoder('shift-jis')('backslashreplace')
210 self.assertEqual(encoder.encode('\xff'), b'\\xff')
211 self.assertEqual(encoder.encode('\n'), b'\n')
212
213 @support.cpython_only
214 def test_subinterp(self):
215 # bpo-42846: Test a CJK codec in a subinterpreter
216 _testcapi = import_module("_testcapi")
217 encoding = 'cp932'
218 text = "Python の開発は、1990 年ごろから開始されています。"
219 code = textwrap.dedent("""
220 import codecs
221 encoding = %r
222 text = %r
223 encoder = codecs.getincrementalencoder(encoding)()
224 text2 = encoder.encode(text).decode(encoding)
225 if text2 != text:
226 raise ValueError(f"encoding issue: {text2!a} != {text!a}")
227 """) % (encoding, text)
228 res = _testcapi.run_in_subinterp(code)
229 self.assertEqual(res, 0)
230
231 class Test_IncrementalDecoder(unittest.TestCase):
232
233 def test_dbcs(self):
234 # cp949 decoder is simple with only 1 or 2 bytes sequences.
235 decoder = codecs.getincrementaldecoder('cp949')()
236 self.assertEqual(decoder.decode(b'\xc6\xc4\xc0\xcc\xbd'),
237 '\ud30c\uc774')
238 self.assertEqual(decoder.decode(b'\xe3 \xb8\xb6\xc0\xbb'),
239 '\uc36c \ub9c8\uc744')
240 self.assertEqual(decoder.decode(b''), '')
241
242 def test_dbcs_keep_buffer(self):
243 decoder = codecs.getincrementaldecoder('cp949')()
244 self.assertEqual(decoder.decode(b'\xc6\xc4\xc0'), '\ud30c')
245 self.assertRaises(UnicodeDecodeError, decoder.decode, b'', True)
246 self.assertEqual(decoder.decode(b'\xcc'), '\uc774')
247
248 self.assertEqual(decoder.decode(b'\xc6\xc4\xc0'), '\ud30c')
249 self.assertRaises(UnicodeDecodeError, decoder.decode,
250 b'\xcc\xbd', True)
251 self.assertEqual(decoder.decode(b'\xcc'), '\uc774')
252
253 def test_iso2022(self):
254 decoder = codecs.getincrementaldecoder('iso2022-jp')()
255 ESC = b'\x1b'
256 self.assertEqual(decoder.decode(ESC + b'('), '')
257 self.assertEqual(decoder.decode(b'B', True), '')
258 self.assertEqual(decoder.decode(ESC + b'$'), '')
259 self.assertEqual(decoder.decode(b'B@$'), '\u4e16')
260 self.assertEqual(decoder.decode(b'@$@'), '\u4e16')
261 self.assertEqual(decoder.decode(b'$', True), '\u4e16')
262 self.assertEqual(decoder.reset(), None)
263 self.assertEqual(decoder.decode(b'@$'), '@$')
264 self.assertEqual(decoder.decode(ESC + b'$'), '')
265 self.assertRaises(UnicodeDecodeError, decoder.decode, b'', True)
266 self.assertEqual(decoder.decode(b'B@$'), '\u4e16')
267
268 def test_decode_unicode(self):
269 # Trying to decode a unicode string should raise a TypeError
270 for enc in ALL_CJKENCODINGS:
271 decoder = codecs.getincrementaldecoder(enc)()
272 self.assertRaises(TypeError, decoder.decode, "")
273
274 def test_state_methods(self):
275 decoder = codecs.getincrementaldecoder('euc_jp')()
276
277 # Decode a complete input sequence
278 self.assertEqual(decoder.decode(b'\xa4\xa6'), '\u3046')
279 pending1, _ = decoder.getstate()
280 self.assertEqual(pending1, b'')
281
282 # Decode first half of a partial input sequence
283 self.assertEqual(decoder.decode(b'\xa4'), '')
284 pending2, flags2 = decoder.getstate()
285 self.assertEqual(pending2, b'\xa4')
286
287 # Decode second half of a partial input sequence
288 self.assertEqual(decoder.decode(b'\xa6'), '\u3046')
289 pending3, _ = decoder.getstate()
290 self.assertEqual(pending3, b'')
291
292 # Jump back and decode second half of partial input sequence again
293 decoder.setstate((pending2, flags2))
294 self.assertEqual(decoder.decode(b'\xa6'), '\u3046')
295 pending4, _ = decoder.getstate()
296 self.assertEqual(pending4, b'')
297
298 # Ensure state values are preserved correctly
299 decoder.setstate((b'abc', 123456789))
300 self.assertEqual(decoder.getstate(), (b'abc', 123456789))
301
302 def test_setstate_validates_input(self):
303 decoder = codecs.getincrementaldecoder('euc_jp')()
304 self.assertRaises(TypeError, decoder.setstate, 123)
305 self.assertRaises(TypeError, decoder.setstate, ("invalid", 0))
306 self.assertRaises(TypeError, decoder.setstate, (b"1234", "invalid"))
307 self.assertRaises(UnicodeDecodeError, decoder.setstate, (b"123456789", 0))
308
309 class Test_StreamReader(unittest.TestCase):
310 def test_bug1728403(self):
311 try:
312 f = open(TESTFN, 'wb')
313 try:
314 f.write(b'\xa1')
315 finally:
316 f.close()
317 with self.assertWarns(DeprecationWarning):
318 f = codecs.open(TESTFN, encoding='cp949')
319 try:
320 self.assertRaises(UnicodeDecodeError, f.read, 2)
321 finally:
322 f.close()
323 finally:
324 os_helper.unlink(TESTFN)
325
326 class Test_StreamWriter(unittest.TestCase):
327 def test_gb18030(self):
328 s= io.BytesIO()
329 c = codecs.getwriter('gb18030')(s)
330 c.write('123')
331 self.assertEqual(s.getvalue(), b'123')
332 c.write('\U00012345')
333 self.assertEqual(s.getvalue(), b'123\x907\x959')
334 c.write('\uac00\u00ac')
335 self.assertEqual(s.getvalue(),
336 b'123\x907\x959\x827\xcf5\x810\x851')
337
338 def test_utf_8(self):
339 s= io.BytesIO()
340 c = codecs.getwriter('utf-8')(s)
341 c.write('123')
342 self.assertEqual(s.getvalue(), b'123')
343 c.write('\U00012345')
344 self.assertEqual(s.getvalue(), b'123\xf0\x92\x8d\x85')
345 c.write('\uac00\u00ac')
346 self.assertEqual(s.getvalue(),
347 b'123\xf0\x92\x8d\x85'
348 b'\xea\xb0\x80\xc2\xac')
349
350 def test_streamwriter_strwrite(self):
351 s = io.BytesIO()
352 wr = codecs.getwriter('gb18030')(s)
353 wr.write('abcd')
354 self.assertEqual(s.getvalue(), b'abcd')
355
356 class Test_ISO2022(unittest.TestCase):
357 def test_g2(self):
358 iso2022jp2 = b'\x1b(B:hu4:unit\x1b.A\x1bNi de famille'
359 uni = ':hu4:unit\xe9 de famille'
360 self.assertEqual(iso2022jp2.decode('iso2022-jp-2'), uni)
361
362 def test_iso2022_jp_g0(self):
363 self.assertNotIn(b'\x0e', '\N{SOFT HYPHEN}'.encode('iso-2022-jp-2'))
364 for encoding in ('iso-2022-jp-2004', 'iso-2022-jp-3'):
365 e = '\u3406'.encode(encoding)
366 self.assertFalse(any(x > 0x80 for x in e))
367
368 @support.requires_resource('cpu')
369 def test_bug1572832(self):
370 for x in range(0x10000, 0x110000):
371 # Any ISO 2022 codec will cause the segfault
372 chr(x).encode('iso_2022_jp', 'ignore')
373
374 class TestStateful(unittest.TestCase):
375 text = '\u4E16\u4E16'
376 encoding = 'iso-2022-jp'
377 expected = b'\x1b$B@$@$'
378 reset = b'\x1b(B'
379 expected_reset = expected + reset
380
381 def test_encode(self):
382 self.assertEqual(self.text.encode(self.encoding), self.expected_reset)
383
384 def test_incrementalencoder(self):
385 encoder = codecs.getincrementalencoder(self.encoding)()
386 output = b''.join(
387 encoder.encode(char)
388 for char in self.text)
389 self.assertEqual(output, self.expected)
390 self.assertEqual(encoder.encode('', final=True), self.reset)
391 self.assertEqual(encoder.encode('', final=True), b'')
392
393 def test_incrementalencoder_final(self):
394 encoder = codecs.getincrementalencoder(self.encoding)()
395 last_index = len(self.text) - 1
396 output = b''.join(
397 encoder.encode(char, index == last_index)
398 for index, char in enumerate(self.text))
399 self.assertEqual(output, self.expected_reset)
400 self.assertEqual(encoder.encode('', final=True), b'')
401
402 class TestHZStateful(TestStateful):
403 text = '\u804a\u804a'
404 encoding = 'hz'
405 expected = b'~{ADAD'
406 reset = b'~}'
407 expected_reset = expected + reset
408
409
410 if __name__ == "__main__":
411 unittest.main()