Debian Codegraph @

Lib/test/test_codecs.py

raw
1 import codecs
2 import contextlib
3 import copy
4 import importlib
5 import io
6 import pickle
7 import os
8 import sys
9 import unittest
10 import encodings
11 from unittest import mock
12 import warnings
13
14 from test import support
15 from test.support import os_helper
16
17 try:
18 import _testlimitedcapi
19 except ImportError:
20 _testlimitedcapi = None
21 try:
22 import _testinternalcapi
23 except ImportError:
24 _testinternalcapi = None
25
26
27 def codecs_open_no_warn(*args, **kwargs):
28 """Call codecs.open(*args, **kwargs) ignoring DeprecationWarning."""
29 with warnings.catch_warnings():
30 warnings.simplefilter("ignore")
31 return codecs.open(*args, **kwargs)
32
33 def coding_checker(self, coder):
34 def check(input, expect):
35 self.assertEqual(coder(input), (expect, len(input)))
36 return check
37
38 # On small versions of Windows like Windows IoT or Windows Nano Server not all codepages are present
39 def is_code_page_present(cp):
40 from ctypes import POINTER, WINFUNCTYPE, WinDLL, Structure
41 from ctypes.wintypes import BOOL, BYTE, WCHAR, UINT, DWORD
42
43 MAX_LEADBYTES = 12 # 5 ranges, 2 bytes ea., 0 term.
44 MAX_DEFAULTCHAR = 2 # single or double byte
45 MAX_PATH = 260
46 class CPINFOEXW(Structure):
47 _fields_ = [("MaxCharSize", UINT),
48 ("DefaultChar", BYTE*MAX_DEFAULTCHAR),
49 ("LeadByte", BYTE*MAX_LEADBYTES),
50 ("UnicodeDefaultChar", WCHAR),
51 ("CodePage", UINT),
52 ("CodePageName", WCHAR*MAX_PATH)]
53
54 prototype = WINFUNCTYPE(BOOL, UINT, DWORD, POINTER(CPINFOEXW))
55 GetCPInfoEx = prototype(("GetCPInfoExW", WinDLL("kernel32")))
56 info = CPINFOEXW()
57 return GetCPInfoEx(cp, 0, info)
58
59 class Queue(object):
60 """
61 queue: write bytes at one end, read bytes from the other end
62 """
63 def __init__(self, buffer):
64 self._buffer = buffer
65
66 def write(self, chars):
67 self._buffer += chars
68
69 def read(self, size=-1):
70 if size<0:
71 s = self._buffer
72 self._buffer = self._buffer[:0] # make empty
73 return s
74 else:
75 s = self._buffer[:size]
76 self._buffer = self._buffer[size:]
77 return s
78
79
80 class MixInCheckStateHandling:
81 def check_state_handling_decode(self, encoding, u, s):
82 for i in range(len(s)+1):
83 d = codecs.getincrementaldecoder(encoding)()
84 part1 = d.decode(s[:i])
85 state = d.getstate()
86 self.assertIsInstance(state[1], int)
87 # Check that the condition stated in the documentation for
88 # IncrementalDecoder.getstate() holds
89 if not state[1]:
90 # reset decoder to the default state without anything buffered
91 d.setstate((state[0][:0], 0))
92 # Feeding the previous input may not produce any output
93 self.assertTrue(not d.decode(state[0]))
94 # The decoder must return to the same state
95 self.assertEqual(state, d.getstate())
96 # Create a new decoder and set it to the state
97 # we extracted from the old one
98 d = codecs.getincrementaldecoder(encoding)()
99 d.setstate(state)
100 part2 = d.decode(s[i:], True)
101 self.assertEqual(u, part1+part2)
102
103 def check_state_handling_encode(self, encoding, u, s):
104 for i in range(len(u)+1):
105 d = codecs.getincrementalencoder(encoding)()
106 part1 = d.encode(u[:i])
107 state = d.getstate()
108 d = codecs.getincrementalencoder(encoding)()
109 d.setstate(state)
110 part2 = d.encode(u[i:], True)
111 self.assertEqual(s, part1+part2)
112
113
114 class ReadTest(MixInCheckStateHandling):
115 def check_partial(self, input, partialresults):
116 # get a StreamReader for the encoding and feed the bytestring version
117 # of input to the reader byte by byte. Read everything available from
118 # the StreamReader and check that the results equal the appropriate
119 # entries from partialresults.
120 q = Queue(b"")
121 r = codecs.getreader(self.encoding)(q)
122 result = ""
123 for (c, partialresult) in zip(input.encode(self.encoding), partialresults, strict=True):
124 q.write(bytes([c]))
125 result += r.read()
126 self.assertEqual(result, partialresult)
127 # check that there's nothing left in the buffers
128 self.assertEqual(r.read(), "")
129 self.assertEqual(r.bytebuffer, b"")
130
131 # do the check again, this time using an incremental decoder
132 d = codecs.getincrementaldecoder(self.encoding)()
133 result = ""
134 for (c, partialresult) in zip(input.encode(self.encoding), partialresults, strict=True):
135 result += d.decode(bytes([c]))
136 self.assertEqual(result, partialresult)
137 # check that there's nothing left in the buffers
138 self.assertEqual(d.decode(b"", True), "")
139 self.assertEqual(d.buffer, b"")
140
141 # Check whether the reset method works properly
142 d.reset()
143 result = ""
144 for (c, partialresult) in zip(input.encode(self.encoding), partialresults, strict=True):
145 result += d.decode(bytes([c]))
146 self.assertEqual(result, partialresult)
147 # check that there's nothing left in the buffers
148 self.assertEqual(d.decode(b"", True), "")
149 self.assertEqual(d.buffer, b"")
150
151 # check iterdecode()
152 encoded = input.encode(self.encoding)
153 self.assertEqual(
154 input,
155 "".join(codecs.iterdecode([bytes([c]) for c in encoded], self.encoding))
156 )
157
158 def test_readline(self):
159 def getreader(input):
160 stream = io.BytesIO(input.encode(self.encoding))
161 return codecs.getreader(self.encoding)(stream)
162
163 def readalllines(input, keepends=True, size=None):
164 reader = getreader(input)
165 lines = []
166 while True:
167 line = reader.readline(size=size, keepends=keepends)
168 if not line:
169 break
170 lines.append(line)
171 return "|".join(lines)
172
173 s = "foo\nbar\r\nbaz\rspam\u2028eggs"
174 sexpected = "foo\n|bar\r\n|baz\r|spam\u2028|eggs"
175 sexpectednoends = "foo|bar|baz|spam|eggs"
176 self.assertEqual(readalllines(s, True), sexpected)
177 self.assertEqual(readalllines(s, False), sexpectednoends)
178 self.assertEqual(readalllines(s, True, 10), sexpected)
179 self.assertEqual(readalllines(s, False, 10), sexpectednoends)
180
181 lineends = ("\n", "\r\n", "\r", "\u2028")
182 # Test long lines (multiple calls to read() in readline())
183 vw = []
184 vwo = []
185 for (i, lineend) in enumerate(lineends):
186 vw.append((i*200+200)*"\u3042" + lineend)
187 vwo.append((i*200+200)*"\u3042")
188 self.assertEqual(readalllines("".join(vw), True), "|".join(vw))
189 self.assertEqual(readalllines("".join(vw), False), "|".join(vwo))
190
191 # Test lines where the first read might end with \r, so the
192 # reader has to look ahead whether this is a lone \r or a \r\n
193 for size in range(80):
194 for lineend in lineends:
195 s = 10*(size*"a" + lineend + "xxx\n")
196 reader = getreader(s)
197 for i in range(10):
198 self.assertEqual(
199 reader.readline(keepends=True),
200 size*"a" + lineend,
201 )
202 self.assertEqual(
203 reader.readline(keepends=True),
204 "xxx\n",
205 )
206 reader = getreader(s)
207 for i in range(10):
208 self.assertEqual(
209 reader.readline(keepends=False),
210 size*"a",
211 )
212 self.assertEqual(
213 reader.readline(keepends=False),
214 "xxx",
215 )
216
217 def test_mixed_readline_and_read(self):
218 lines = ["Humpty Dumpty sat on a wall,\n",
219 "Humpty Dumpty had a great fall.\r\n",
220 "All the king's horses and all the king's men\r",
221 "Couldn't put Humpty together again."]
222 data = ''.join(lines)
223 def getreader():
224 stream = io.BytesIO(data.encode(self.encoding))
225 return codecs.getreader(self.encoding)(stream)
226
227 # Issue #8260: Test readline() followed by read()
228 f = getreader()
229 self.assertEqual(f.readline(), lines[0])
230 self.assertEqual(f.read(), ''.join(lines[1:]))
231 self.assertEqual(f.read(), '')
232
233 # Issue #32110: Test readline() followed by read(n)
234 f = getreader()
235 self.assertEqual(f.readline(), lines[0])
236 self.assertEqual(f.read(1), lines[1][0])
237 self.assertEqual(f.read(0), '')
238 self.assertEqual(f.read(100), data[len(lines[0]) + 1:][:100])
239
240 # Issue #16636: Test readline() followed by readlines()
241 f = getreader()
242 self.assertEqual(f.readline(), lines[0])
243 self.assertEqual(f.readlines(), lines[1:])
244 self.assertEqual(f.read(), '')
245
246 # Test read(n) followed by read()
247 f = getreader()
248 self.assertEqual(f.read(size=40, chars=5), data[:5])
249 self.assertEqual(f.read(), data[5:])
250 self.assertEqual(f.read(), '')
251
252 # Issue #32110: Test read(n) followed by read(n)
253 f = getreader()
254 self.assertEqual(f.read(size=40, chars=5), data[:5])
255 self.assertEqual(f.read(1), data[5])
256 self.assertEqual(f.read(0), '')
257 self.assertEqual(f.read(100), data[6:106])
258
259 # Issue #12446: Test read(n) followed by readlines()
260 f = getreader()
261 self.assertEqual(f.read(size=40, chars=5), data[:5])
262 self.assertEqual(f.readlines(), [lines[0][5:]] + lines[1:])
263 self.assertEqual(f.read(), '')
264
265 def test_bug1175396(self):
266 s = [
267 '<%!--===================================================\r\n',
268 ' BLOG index page: show recent articles,\r\n',
269 ' today\'s articles, or articles of a specific date.\r\n',
270 '========================================================--%>\r\n',
271 '<%@inputencoding="ISO-8859-1"%>\r\n',
272 '<%@pagetemplate=TEMPLATE.y%>\r\n',
273 '<%@import=import frog.util, frog%>\r\n',
274 '<%@import=import frog.objects%>\r\n',
275 '<%@import=from frog.storageerrors import StorageError%>\r\n',
276 '<%\r\n',
277 '\r\n',
278 'import logging\r\n',
279 'log=logging.getLogger("Snakelets.logger")\r\n',
280 '\r\n',
281 '\r\n',
282 'user=self.SessionCtx.user\r\n',
283 'storageEngine=self.SessionCtx.storageEngine\r\n',
284 '\r\n',
285 '\r\n',
286 'def readArticlesFromDate(date, count=None):\r\n',
287 ' entryids=storageEngine.listBlogEntries(date)\r\n',
288 ' entryids.reverse() # descending\r\n',
289 ' if count:\r\n',
290 ' entryids=entryids[:count]\r\n',
291 ' try:\r\n',
292 ' return [ frog.objects.BlogEntry.load(storageEngine, date, Id) for Id in entryids ]\r\n',
293 ' except StorageError,x:\r\n',
294 ' log.error("Error loading articles: "+str(x))\r\n',
295 ' self.abort("cannot load articles")\r\n',
296 '\r\n',
297 'showdate=None\r\n',
298 '\r\n',
299 'arg=self.Request.getArg()\r\n',
300 'if arg=="today":\r\n',
301 ' #-------------------- TODAY\'S ARTICLES\r\n',
302 ' self.write("<h2>Today\'s articles</h2>")\r\n',
303 ' showdate = frog.util.isodatestr() \r\n',
304 ' entries = readArticlesFromDate(showdate)\r\n',
305 'elif arg=="active":\r\n',
306 ' #-------------------- ACTIVE ARTICLES redirect\r\n',
307 ' self.Yredirect("active.y")\r\n',
308 'elif arg=="login":\r\n',
309 ' #-------------------- LOGIN PAGE redirect\r\n',
310 ' self.Yredirect("login.y")\r\n',
311 'elif arg=="date":\r\n',
312 ' #-------------------- ARTICLES OF A SPECIFIC DATE\r\n',
313 ' showdate = self.Request.getParameter("date")\r\n',
314 ' self.write("<h2>Articles written on %s</h2>"% frog.util.mediumdatestr(showdate))\r\n',
315 ' entries = readArticlesFromDate(showdate)\r\n',
316 'else:\r\n',
317 ' #-------------------- RECENT ARTICLES\r\n',
318 ' self.write("<h2>Recent articles</h2>")\r\n',
319 ' dates=storageEngine.listBlogEntryDates()\r\n',
320 ' if dates:\r\n',
321 ' entries=[]\r\n',
322 ' SHOWAMOUNT=10\r\n',
323 ' for showdate in dates:\r\n',
324 ' entries.extend( readArticlesFromDate(showdate, SHOWAMOUNT-len(entries)) )\r\n',
325 ' if len(entries)>=SHOWAMOUNT:\r\n',
326 ' break\r\n',
327 ' \r\n',
328 ]
329 stream = io.BytesIO("".join(s).encode(self.encoding))
330 reader = codecs.getreader(self.encoding)(stream)
331 for (i, line) in enumerate(reader):
332 self.assertEqual(line, s[i])
333
334 def test_readlinequeue(self):
335 q = Queue(b"")
336 writer = codecs.getwriter(self.encoding)(q)
337 reader = codecs.getreader(self.encoding)(q)
338
339 # No lineends
340 writer.write("foo\r")
341 self.assertEqual(reader.readline(keepends=False), "foo")
342 writer.write("\nbar\r")
343 self.assertEqual(reader.readline(keepends=False), "")
344 self.assertEqual(reader.readline(keepends=False), "bar")
345 writer.write("baz")
346 self.assertEqual(reader.readline(keepends=False), "baz")
347 self.assertEqual(reader.readline(keepends=False), "")
348
349 # Lineends
350 writer.write("foo\r")
351 self.assertEqual(reader.readline(keepends=True), "foo\r")
352 writer.write("\nbar\r")
353 self.assertEqual(reader.readline(keepends=True), "\n")
354 self.assertEqual(reader.readline(keepends=True), "bar\r")
355 writer.write("baz")
356 self.assertEqual(reader.readline(keepends=True), "baz")
357 self.assertEqual(reader.readline(keepends=True), "")
358 writer.write("foo\r\n")
359 self.assertEqual(reader.readline(keepends=True), "foo\r\n")
360
361 def test_bug1098990_a(self):
362 s1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n"
363 s2 = "offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n"
364 s3 = "next line.\r\n"
365
366 s = (s1+s2+s3).encode(self.encoding)
367 stream = io.BytesIO(s)
368 reader = codecs.getreader(self.encoding)(stream)
369 self.assertEqual(reader.readline(), s1)
370 self.assertEqual(reader.readline(), s2)
371 self.assertEqual(reader.readline(), s3)
372 self.assertEqual(reader.readline(), "")
373
374 def test_bug1098990_b(self):
375 s1 = "aaaaaaaaaaaaaaaaaaaaaaaa\r\n"
376 s2 = "bbbbbbbbbbbbbbbbbbbbbbbb\r\n"
377 s3 = "stillokay:bbbbxx\r\n"
378 s4 = "broken!!!!badbad\r\n"
379 s5 = "againokay.\r\n"
380
381 s = (s1+s2+s3+s4+s5).encode(self.encoding)
382 stream = io.BytesIO(s)
383 reader = codecs.getreader(self.encoding)(stream)
384 self.assertEqual(reader.readline(), s1)
385 self.assertEqual(reader.readline(), s2)
386 self.assertEqual(reader.readline(), s3)
387 self.assertEqual(reader.readline(), s4)
388 self.assertEqual(reader.readline(), s5)
389 self.assertEqual(reader.readline(), "")
390
391 ill_formed_sequence_replace = "\ufffd"
392
393 def test_lone_surrogates(self):
394 self.assertRaises(UnicodeEncodeError, "\ud800".encode, self.encoding)
395 self.assertEqual("[\uDC80]".encode(self.encoding, "backslashreplace"),
396 "[\\udc80]".encode(self.encoding))
397 self.assertEqual("[\uDC80]".encode(self.encoding, "namereplace"),
398 "[\\udc80]".encode(self.encoding))
399 self.assertEqual("[\uDC80]".encode(self.encoding, "xmlcharrefreplace"),
400 "[&#56448;]".encode(self.encoding))
401 self.assertEqual("[\uDC80]".encode(self.encoding, "ignore"),
402 "[]".encode(self.encoding))
403 self.assertEqual("[\uDC80]".encode(self.encoding, "replace"),
404 "[?]".encode(self.encoding))
405
406 # sequential surrogate characters
407 self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "ignore"),
408 "[]".encode(self.encoding))
409 self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "replace"),
410 "[??]".encode(self.encoding))
411
412 bom = "".encode(self.encoding)
413 for before, after in [("\U00010fff", "A"), ("[", "]"),
414 ("A", "\U00010fff")]:
415 before_sequence = before.encode(self.encoding)[len(bom):]
416 after_sequence = after.encode(self.encoding)[len(bom):]
417 test_string = before + "\uDC80" + after
418 test_sequence = (bom + before_sequence +
419 self.ill_formed_sequence + after_sequence)
420 self.assertRaises(UnicodeDecodeError, test_sequence.decode,
421 self.encoding)
422 self.assertEqual(test_string.encode(self.encoding,
423 "surrogatepass"),
424 test_sequence)
425 self.assertEqual(test_sequence.decode(self.encoding,
426 "surrogatepass"),
427 test_string)
428 self.assertEqual(test_sequence.decode(self.encoding, "ignore"),
429 before + after)
430 self.assertEqual(test_sequence.decode(self.encoding, "replace"),
431 before + self.ill_formed_sequence_replace + after)
432 backslashreplace = ''.join('\\x%02x' % b
433 for b in self.ill_formed_sequence)
434 self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"),
435 before + backslashreplace + after)
436
437 def test_incremental_surrogatepass(self):
438 # Test incremental decoder for surrogatepass handler:
439 # see issue #24214
440 # High surrogate
441 data = '\uD901'.encode(self.encoding, 'surrogatepass')
442 for i in range(1, len(data)):
443 dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass')
444 self.assertEqual(dec.decode(data[:i]), '')
445 self.assertEqual(dec.decode(data[i:], True), '\uD901')
446 # Low surrogate
447 data = '\uDC02'.encode(self.encoding, 'surrogatepass')
448 for i in range(1, len(data)):
449 dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass')
450 self.assertEqual(dec.decode(data[:i]), '')
451 self.assertEqual(dec.decode(data[i:]), '\uDC02')
452
453
454 class UTF32Test(ReadTest, unittest.TestCase):
455 encoding = "utf-32"
456 if sys.byteorder == 'little':
457 ill_formed_sequence = b"\x80\xdc\x00\x00"
458 else:
459 ill_formed_sequence = b"\x00\x00\xdc\x80"
460
461 spamle = (b'\xff\xfe\x00\x00'
462 b's\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m\x00\x00\x00'
463 b's\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m\x00\x00\x00')
464 spambe = (b'\x00\x00\xfe\xff'
465 b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m'
466 b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m')
467
468 def test_only_one_bom(self):
469 _,_,reader,writer = codecs.lookup(self.encoding)
470 # encode some stream
471 s = io.BytesIO()
472 f = writer(s)
473 f.write("spam")
474 f.write("spam")
475 d = s.getvalue()
476 # check whether there is exactly one BOM in it
477 self.assertTrue(d == self.spamle or d == self.spambe)
478 # try to read it back
479 s = io.BytesIO(d)
480 f = reader(s)
481 self.assertEqual(f.read(), "spamspam")
482
483 def test_badbom(self):
484 s = io.BytesIO(4*b"\xff")
485 f = codecs.getreader(self.encoding)(s)
486 self.assertRaises(UnicodeDecodeError, f.read)
487
488 s = io.BytesIO(8*b"\xff")
489 f = codecs.getreader(self.encoding)(s)
490 self.assertRaises(UnicodeDecodeError, f.read)
491
492 def test_partial(self):
493 self.check_partial(
494 "\x00\xff\u0100\uffff\U00010000",
495 [
496 "", # first byte of BOM read
497 "", # second byte of BOM read
498 "", # third byte of BOM read
499 "", # fourth byte of BOM read => byteorder known
500 "",
501 "",
502 "",
503 "\x00",
504 "\x00",
505 "\x00",
506 "\x00",
507 "\x00\xff",
508 "\x00\xff",
509 "\x00\xff",
510 "\x00\xff",
511 "\x00\xff\u0100",
512 "\x00\xff\u0100",
513 "\x00\xff\u0100",
514 "\x00\xff\u0100",
515 "\x00\xff\u0100\uffff",
516 "\x00\xff\u0100\uffff",
517 "\x00\xff\u0100\uffff",
518 "\x00\xff\u0100\uffff",
519 "\x00\xff\u0100\uffff\U00010000",
520 ]
521 )
522
523 def test_handlers(self):
524 self.assertEqual(('\ufffd', 1),
525 codecs.utf_32_decode(b'\x01', 'replace', True))
526 self.assertEqual(('', 1),
527 codecs.utf_32_decode(b'\x01', 'ignore', True))
528
529 def test_errors(self):
530 self.assertRaises(UnicodeDecodeError, codecs.utf_32_decode,
531 b"\xff", "strict", True)
532
533 def test_decoder_state(self):
534 self.check_state_handling_decode(self.encoding,
535 "spamspam", self.spamle)
536 self.check_state_handling_decode(self.encoding,
537 "spamspam", self.spambe)
538
539 def test_issue8941(self):
540 # Issue #8941: insufficient result allocation when decoding into
541 # surrogate pairs on UCS-2 builds.
542 encoded_le = b'\xff\xfe\x00\x00' + b'\x00\x00\x01\x00' * 1024
543 self.assertEqual('\U00010000' * 1024,
544 codecs.utf_32_decode(encoded_le)[0])
545 encoded_be = b'\x00\x00\xfe\xff' + b'\x00\x01\x00\x00' * 1024
546 self.assertEqual('\U00010000' * 1024,
547 codecs.utf_32_decode(encoded_be)[0])
548
549
550 class UTF32LETest(ReadTest, unittest.TestCase):
551 encoding = "utf-32-le"
552 ill_formed_sequence = b"\x80\xdc\x00\x00"
553
554 def test_partial(self):
555 self.check_partial(
556 "\x00\xff\u0100\uffff\U00010000",
557 [
558 "",
559 "",
560 "",
561 "\x00",
562 "\x00",
563 "\x00",
564 "\x00",
565 "\x00\xff",
566 "\x00\xff",
567 "\x00\xff",
568 "\x00\xff",
569 "\x00\xff\u0100",
570 "\x00\xff\u0100",
571 "\x00\xff\u0100",
572 "\x00\xff\u0100",
573 "\x00\xff\u0100\uffff",
574 "\x00\xff\u0100\uffff",
575 "\x00\xff\u0100\uffff",
576 "\x00\xff\u0100\uffff",
577 "\x00\xff\u0100\uffff\U00010000",
578 ]
579 )
580
581 def test_simple(self):
582 self.assertEqual("\U00010203".encode(self.encoding), b"\x03\x02\x01\x00")
583
584 def test_errors(self):
585 self.assertRaises(UnicodeDecodeError, codecs.utf_32_le_decode,
586 b"\xff", "strict", True)
587
588 def test_issue8941(self):
589 # Issue #8941: insufficient result allocation when decoding into
590 # surrogate pairs on UCS-2 builds.
591 encoded = b'\x00\x00\x01\x00' * 1024
592 self.assertEqual('\U00010000' * 1024,
593 codecs.utf_32_le_decode(encoded)[0])
594
595
596 class UTF32BETest(ReadTest, unittest.TestCase):
597 encoding = "utf-32-be"
598 ill_formed_sequence = b"\x00\x00\xdc\x80"
599
600 def test_partial(self):
601 self.check_partial(
602 "\x00\xff\u0100\uffff\U00010000",
603 [
604 "",
605 "",
606 "",
607 "\x00",
608 "\x00",
609 "\x00",
610 "\x00",
611 "\x00\xff",
612 "\x00\xff",
613 "\x00\xff",
614 "\x00\xff",
615 "\x00\xff\u0100",
616 "\x00\xff\u0100",
617 "\x00\xff\u0100",
618 "\x00\xff\u0100",
619 "\x00\xff\u0100\uffff",
620 "\x00\xff\u0100\uffff",
621 "\x00\xff\u0100\uffff",
622 "\x00\xff\u0100\uffff",
623 "\x00\xff\u0100\uffff\U00010000",
624 ]
625 )
626
627 def test_simple(self):
628 self.assertEqual("\U00010203".encode(self.encoding), b"\x00\x01\x02\x03")
629
630 def test_errors(self):
631 self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode,
632 b"\xff", "strict", True)
633
634 def test_issue8941(self):
635 # Issue #8941: insufficient result allocation when decoding into
636 # surrogate pairs on UCS-2 builds.
637 encoded = b'\x00\x01\x00\x00' * 1024
638 self.assertEqual('\U00010000' * 1024,
639 codecs.utf_32_be_decode(encoded)[0])
640
641
642 class UTF16Test(ReadTest, unittest.TestCase):
643 encoding = "utf-16"
644 if sys.byteorder == 'little':
645 ill_formed_sequence = b"\x80\xdc"
646 else:
647 ill_formed_sequence = b"\xdc\x80"
648
649 spamle = b'\xff\xfes\x00p\x00a\x00m\x00s\x00p\x00a\x00m\x00'
650 spambe = b'\xfe\xff\x00s\x00p\x00a\x00m\x00s\x00p\x00a\x00m'
651
652 def test_only_one_bom(self):
653 _,_,reader,writer = codecs.lookup(self.encoding)
654 # encode some stream
655 s = io.BytesIO()
656 f = writer(s)
657 f.write("spam")
658 f.write("spam")
659 d = s.getvalue()
660 # check whether there is exactly one BOM in it
661 self.assertTrue(d == self.spamle or d == self.spambe)
662 # try to read it back
663 s = io.BytesIO(d)
664 f = reader(s)
665 self.assertEqual(f.read(), "spamspam")
666
667 def test_badbom(self):
668 s = io.BytesIO(b"\xff\xff")
669 f = codecs.getreader(self.encoding)(s)
670 self.assertRaises(UnicodeDecodeError, f.read)
671
672 s = io.BytesIO(b"\xff\xff\xff\xff")
673 f = codecs.getreader(self.encoding)(s)
674 self.assertRaises(UnicodeDecodeError, f.read)
675
676 def test_partial(self):
677 self.check_partial(
678 "\x00\xff\u0100\uffff\U00010000",
679 [
680 "", # first byte of BOM read
681 "", # second byte of BOM read => byteorder known
682 "",
683 "\x00",
684 "\x00",
685 "\x00\xff",
686 "\x00\xff",
687 "\x00\xff\u0100",
688 "\x00\xff\u0100",
689 "\x00\xff\u0100\uffff",
690 "\x00\xff\u0100\uffff",
691 "\x00\xff\u0100\uffff",
692 "\x00\xff\u0100\uffff",
693 "\x00\xff\u0100\uffff\U00010000",
694 ]
695 )
696
697 def test_handlers(self):
698 self.assertEqual(('\ufffd', 1),
699 codecs.utf_16_decode(b'\x01', 'replace', True))
700 self.assertEqual(('', 1),
701 codecs.utf_16_decode(b'\x01', 'ignore', True))
702
703 def test_errors(self):
704 self.assertRaises(UnicodeDecodeError, codecs.utf_16_decode,
705 b"\xff", "strict", True)
706
707 def test_decoder_state(self):
708 self.check_state_handling_decode(self.encoding,
709 "spamspam", self.spamle)
710 self.check_state_handling_decode(self.encoding,
711 "spamspam", self.spambe)
712
713 def test_bug691291(self):
714 # If encoding is not None, then
715 # files are always opened in binary mode, even if no binary mode was
716 # specified. This means that no automatic conversion of '\n' is done
717 # on reading and writing.
718 s1 = 'Hello\r\nworld\r\n'
719
720 s = s1.encode(self.encoding)
721 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
722 with open(os_helper.TESTFN, 'wb') as fp:
723 fp.write(s)
724 with codecs_open_no_warn(os_helper.TESTFN, 'r',
725 encoding=self.encoding) as reader:
726 self.assertEqual(reader.read(), s1)
727
728 def test_invalid_modes(self):
729 for mode in ('U', 'rU', 'r+U'):
730 with self.assertRaises(ValueError) as cm:
731 codecs_open_no_warn(os_helper.TESTFN, mode, encoding=self.encoding)
732 self.assertIn('invalid mode', str(cm.exception))
733
734 for mode in ('rt', 'wt', 'at', 'r+t'):
735 with self.assertRaises(ValueError) as cm:
736 codecs_open_no_warn(os_helper.TESTFN, mode, encoding=self.encoding)
737 self.assertIn("can't have text and binary mode at once",
738 str(cm.exception))
739
740
741 class UTF16LETest(ReadTest, unittest.TestCase):
742 encoding = "utf-16-le"
743 ill_formed_sequence = b"\x80\xdc"
744
745 def test_partial(self):
746 self.check_partial(
747 "\x00\xff\u0100\uffff\U00010000",
748 [
749 "",
750 "\x00",
751 "\x00",
752 "\x00\xff",
753 "\x00\xff",
754 "\x00\xff\u0100",
755 "\x00\xff\u0100",
756 "\x00\xff\u0100\uffff",
757 "\x00\xff\u0100\uffff",
758 "\x00\xff\u0100\uffff",
759 "\x00\xff\u0100\uffff",
760 "\x00\xff\u0100\uffff\U00010000",
761 ]
762 )
763
764 def test_errors(self):
765 tests = [
766 (b'\xff', '\ufffd'),
767 (b'A\x00Z', 'A\ufffd'),
768 (b'A\x00B\x00C\x00D\x00Z', 'ABCD\ufffd'),
769 (b'\x00\xd8', '\ufffd'),
770 (b'\x00\xd8A', '\ufffd'),
771 (b'\x00\xd8A\x00', '\ufffdA'),
772 (b'\x00\xdcA\x00', '\ufffdA'),
773 ]
774 for raw, expected in tests:
775 self.assertRaises(UnicodeDecodeError, codecs.utf_16_le_decode,
776 raw, 'strict', True)
777 self.assertEqual(raw.decode('utf-16le', 'replace'), expected)
778
779 def test_nonbmp(self):
780 self.assertEqual("\U00010203".encode(self.encoding),
781 b'\x00\xd8\x03\xde')
782 self.assertEqual(b'\x00\xd8\x03\xde'.decode(self.encoding),
783 "\U00010203")
784
785 class UTF16BETest(ReadTest, unittest.TestCase):
786 encoding = "utf-16-be"
787 ill_formed_sequence = b"\xdc\x80"
788
789 def test_partial(self):
790 self.check_partial(
791 "\x00\xff\u0100\uffff\U00010000",
792 [
793 "",
794 "\x00",
795 "\x00",
796 "\x00\xff",
797 "\x00\xff",
798 "\x00\xff\u0100",
799 "\x00\xff\u0100",
800 "\x00\xff\u0100\uffff",
801 "\x00\xff\u0100\uffff",
802 "\x00\xff\u0100\uffff",
803 "\x00\xff\u0100\uffff",
804 "\x00\xff\u0100\uffff\U00010000",
805 ]
806 )
807
808 def test_errors(self):
809 tests = [
810 (b'\xff', '\ufffd'),
811 (b'\x00A\xff', 'A\ufffd'),
812 (b'\x00A\x00B\x00C\x00DZ', 'ABCD\ufffd'),
813 (b'\xd8\x00', '\ufffd'),
814 (b'\xd8\x00\xdc', '\ufffd'),
815 (b'\xd8\x00\x00A', '\ufffdA'),
816 (b'\xdc\x00\x00A', '\ufffdA'),
817 ]
818 for raw, expected in tests:
819 self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode,
820 raw, 'strict', True)
821 self.assertEqual(raw.decode('utf-16be', 'replace'), expected)
822
823 def test_nonbmp(self):
824 self.assertEqual("\U00010203".encode(self.encoding),
825 b'\xd8\x00\xde\x03')
826 self.assertEqual(b'\xd8\x00\xde\x03'.decode(self.encoding),
827 "\U00010203")
828
829 class UTF8Test(ReadTest, unittest.TestCase):
830 encoding = "utf-8"
831 ill_formed_sequence = b"\xed\xb2\x80"
832 ill_formed_sequence_replace = "\ufffd" * 3
833 BOM = b''
834
835 def test_partial(self):
836 self.check_partial(
837 "\x00\xff\u07ff\u0800\uffff\U00010000",
838 [
839 "\x00",
840 "\x00",
841 "\x00\xff",
842 "\x00\xff",
843 "\x00\xff\u07ff",
844 "\x00\xff\u07ff",
845 "\x00\xff\u07ff",
846 "\x00\xff\u07ff\u0800",
847 "\x00\xff\u07ff\u0800",
848 "\x00\xff\u07ff\u0800",
849 "\x00\xff\u07ff\u0800\uffff",
850 "\x00\xff\u07ff\u0800\uffff",
851 "\x00\xff\u07ff\u0800\uffff",
852 "\x00\xff\u07ff\u0800\uffff",
853 "\x00\xff\u07ff\u0800\uffff\U00010000",
854 ]
855 )
856
857 def test_decoder_state(self):
858 u = "\x00\x7f\x80\xff\u0100\u07ff\u0800\uffff\U0010ffff"
859 self.check_state_handling_decode(self.encoding,
860 u, u.encode(self.encoding))
861
862 def test_decode_error(self):
863 for data, error_handler, expected in (
864 (b'[\x80\xff]', 'ignore', '[]'),
865 (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'),
866 (b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'),
867 (b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'),
868 ):
869 with self.subTest(data=data, error_handler=error_handler,
870 expected=expected):
871 self.assertEqual(data.decode(self.encoding, error_handler),
872 expected)
873
874 def test_lone_surrogates(self):
875 super().test_lone_surrogates()
876 # not sure if this is making sense for
877 # UTF-16 and UTF-32
878 self.assertEqual("[\uDC80]".encode(self.encoding, "surrogateescape"),
879 self.BOM + b'[\x80]')
880
881 with self.assertRaises(UnicodeEncodeError) as cm:
882 "[\uDC80\uD800\uDFFF]".encode(self.encoding, "surrogateescape")
883 exc = cm.exception
884 self.assertEqual(exc.object[exc.start:exc.end], '\uD800\uDFFF')
885
886 def test_surrogatepass_handler(self):
887 self.assertEqual("abc\ud800def".encode(self.encoding, "surrogatepass"),
888 self.BOM + b"abc\xed\xa0\x80def")
889 self.assertEqual("\U00010fff\uD800".encode(self.encoding, "surrogatepass"),
890 self.BOM + b"\xf0\x90\xbf\xbf\xed\xa0\x80")
891 self.assertEqual("[\uD800\uDC80]".encode(self.encoding, "surrogatepass"),
892 self.BOM + b'[\xed\xa0\x80\xed\xb2\x80]')
893
894 self.assertEqual(b"abc\xed\xa0\x80def".decode(self.encoding, "surrogatepass"),
895 "abc\ud800def")
896 self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode(self.encoding, "surrogatepass"),
897 "\U00010fff\uD800")
898
899 self.assertTrue(codecs.lookup_error("surrogatepass"))
900 with self.assertRaises(UnicodeDecodeError):
901 b"abc\xed\xa0".decode(self.encoding, "surrogatepass")
902 with self.assertRaises(UnicodeDecodeError):
903 b"abc\xed\xa0z".decode(self.encoding, "surrogatepass")
904
905 def test_incremental_errors(self):
906 # Test that the incremental decoder can fail with final=False.
907 # See issue #24214
908 cases = [b'\x80', b'\xBF', b'\xC0', b'\xC1', b'\xF5', b'\xF6', b'\xFF']
909 for prefix in (b'\xC2', b'\xDF', b'\xE0', b'\xE0\xA0', b'\xEF',
910 b'\xEF\xBF', b'\xF0', b'\xF0\x90', b'\xF0\x90\x80',
911 b'\xF4', b'\xF4\x8F', b'\xF4\x8F\xBF'):
912 for suffix in b'\x7F', b'\xC0':
913 cases.append(prefix + suffix)
914 cases.extend((b'\xE0\x80', b'\xE0\x9F', b'\xED\xA0\x80',
915 b'\xED\xBF\xBF', b'\xF0\x80', b'\xF0\x8F', b'\xF4\x90'))
916
917 for data in cases:
918 with self.subTest(data=data):
919 dec = codecs.getincrementaldecoder(self.encoding)()
920 self.assertRaises(UnicodeDecodeError, dec.decode, data)
921
922
923 class UTF7Test(ReadTest, unittest.TestCase):
924 encoding = "utf-7"
925
926 def test_ascii(self):
927 # Set D (directly encoded characters)
928 set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
929 'abcdefghijklmnopqrstuvwxyz'
930 '0123456789'
931 '\'(),-./:?')
932 self.assertEqual(set_d.encode(self.encoding), set_d.encode('ascii'))
933 self.assertEqual(set_d.encode('ascii').decode(self.encoding), set_d)
934 # Set O (optional direct characters)
935 set_o = ' !"#$%&*;<=>@[]^_`{|}'
936 self.assertEqual(set_o.encode(self.encoding), set_o.encode('ascii'))
937 self.assertEqual(set_o.encode('ascii').decode(self.encoding), set_o)
938 # +
939 self.assertEqual('a+b'.encode(self.encoding), b'a+-b')
940 self.assertEqual(b'a+-b'.decode(self.encoding), 'a+b')
941 # White spaces
942 ws = ' \t\n\r'
943 self.assertEqual(ws.encode(self.encoding), ws.encode('ascii'))
944 self.assertEqual(ws.encode('ascii').decode(self.encoding), ws)
945 # Other ASCII characters
946 other_ascii = ''.join(sorted(set(bytes(range(0x80)).decode()) -
947 set(set_d + set_o + '+' + ws)))
948 self.assertEqual(other_ascii.encode(self.encoding),
949 b'+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU'
950 b'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-')
951
952 def test_partial(self):
953 self.check_partial(
954 'a+-b\x00c\x80d\u0100e\U00010000f',
955 [
956 'a',
957 'a',
958 'a+',
959 'a+-',
960 'a+-b',
961 'a+-b',
962 'a+-b',
963 'a+-b',
964 'a+-b',
965 'a+-b\x00',
966 'a+-b\x00c',
967 'a+-b\x00c',
968 'a+-b\x00c',
969 'a+-b\x00c',
970 'a+-b\x00c',
971 'a+-b\x00c\x80',
972 'a+-b\x00c\x80d',
973 'a+-b\x00c\x80d',
974 'a+-b\x00c\x80d',
975 'a+-b\x00c\x80d',
976 'a+-b\x00c\x80d',
977 'a+-b\x00c\x80d\u0100',
978 'a+-b\x00c\x80d\u0100e',
979 'a+-b\x00c\x80d\u0100e',
980 'a+-b\x00c\x80d\u0100e',
981 'a+-b\x00c\x80d\u0100e',
982 'a+-b\x00c\x80d\u0100e',
983 'a+-b\x00c\x80d\u0100e',
984 'a+-b\x00c\x80d\u0100e',
985 'a+-b\x00c\x80d\u0100e',
986 'a+-b\x00c\x80d\u0100e\U00010000',
987 'a+-b\x00c\x80d\u0100e\U00010000f',
988 ]
989 )
990
991 def test_errors(self):
992 tests = [
993 (b'\xffb', '\ufffdb'),
994 (b'a\xffb', 'a\ufffdb'),
995 (b'a\xff\xffb', 'a\ufffd\ufffdb'),
996 (b'a+IK', 'a\ufffd'),
997 (b'a+IK-b', 'a\ufffdb'),
998 (b'a+IK,b', 'a\ufffdb'),
999 (b'a+IKx', 'a\u20ac\ufffd'),
1000 (b'a+IKx-b', 'a\u20ac\ufffdb'),
1001 (b'a+IKwgr', 'a\u20ac\ufffd'),
1002 (b'a+IKwgr-b', 'a\u20ac\ufffdb'),
1003 (b'a+IKwgr,', 'a\u20ac\ufffd'),
1004 (b'a+IKwgr,-b', 'a\u20ac\ufffd-b'),
1005 (b'a+IKwgrB', 'a\u20ac\u20ac\ufffd'),
1006 (b'a+IKwgrB-b', 'a\u20ac\u20ac\ufffdb'),
1007 (b'a+/,+IKw-b', 'a\ufffd\u20acb'),
1008 (b'a+//,+IKw-b', 'a\ufffd\u20acb'),
1009 (b'a+///,+IKw-b', 'a\uffff\ufffd\u20acb'),
1010 (b'a+////,+IKw-b', 'a\uffff\ufffd\u20acb'),
1011 (b'a+IKw-b\xff', 'a\u20acb\ufffd'),
1012 (b'a+IKw\xffb', 'a\u20ac\ufffdb'),
1013 (b'a+@b', 'a\ufffdb'),
1014 ]
1015 for raw, expected in tests:
1016 with self.subTest(raw=raw):
1017 self.assertRaises(UnicodeDecodeError, codecs.utf_7_decode,
1018 raw, 'strict', True)
1019 self.assertEqual(raw.decode('utf-7', 'replace'), expected)
1020
1021 def test_nonbmp(self):
1022 self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-')
1023 self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-')
1024 self.assertEqual(b'+2AHcoA-'.decode(self.encoding), '\U000104A0')
1025 self.assertEqual(b'+2AHcoA'.decode(self.encoding), '\U000104A0')
1026 self.assertEqual('\u20ac\U000104A0'.encode(self.encoding), b'+IKzYAdyg-')
1027 self.assertEqual(b'+IKzYAdyg-'.decode(self.encoding), '\u20ac\U000104A0')
1028 self.assertEqual(b'+IKzYAdyg'.decode(self.encoding), '\u20ac\U000104A0')
1029 self.assertEqual('\u20ac\u20ac\U000104A0'.encode(self.encoding),
1030 b'+IKwgrNgB3KA-')
1031 self.assertEqual(b'+IKwgrNgB3KA-'.decode(self.encoding),
1032 '\u20ac\u20ac\U000104A0')
1033 self.assertEqual(b'+IKwgrNgB3KA'.decode(self.encoding),
1034 '\u20ac\u20ac\U000104A0')
1035
1036 def test_lone_surrogates(self):
1037 tests = [
1038 (b'a+2AE-b', 'a\ud801b'),
1039 (b'a+2AE\xffb', 'a\ufffdb'),
1040 (b'a+2AE', 'a\ufffd'),
1041 (b'a+2AEA-b', 'a\ufffdb'),
1042 (b'a+2AH-b', 'a\ufffdb'),
1043 (b'a+IKzYAQ-b', 'a\u20ac\ud801b'),
1044 (b'a+IKzYAQ\xffb', 'a\u20ac\ufffdb'),
1045 (b'a+IKzYAQA-b', 'a\u20ac\ufffdb'),
1046 (b'a+IKzYAd-b', 'a\u20ac\ufffdb'),
1047 (b'a+IKwgrNgB-b', 'a\u20ac\u20ac\ud801b'),
1048 (b'a+IKwgrNgB\xffb', 'a\u20ac\u20ac\ufffdb'),
1049 (b'a+IKwgrNgB', 'a\u20ac\u20ac\ufffd'),
1050 (b'a+IKwgrNgBA-b', 'a\u20ac\u20ac\ufffdb'),
1051 ]
1052 for raw, expected in tests:
1053 with self.subTest(raw=raw):
1054 self.assertEqual(raw.decode('utf-7', 'replace'), expected)
1055
1056
1057 class UTF16ExTest(unittest.TestCase):
1058
1059 def test_errors(self):
1060 self.assertRaises(UnicodeDecodeError, codecs.utf_16_ex_decode, b"\xff", "strict", 0, True)
1061
1062 def test_bad_args(self):
1063 self.assertRaises(TypeError, codecs.utf_16_ex_decode)
1064
1065 class ReadBufferTest(unittest.TestCase):
1066
1067 def test_array(self):
1068 import array
1069 self.assertEqual(
1070 codecs.readbuffer_encode(array.array("b", b"spam")),
1071 (b"spam", 4)
1072 )
1073
1074 def test_empty(self):
1075 self.assertEqual(codecs.readbuffer_encode(""), (b"", 0))
1076
1077 def test_bad_args(self):
1078 self.assertRaises(TypeError, codecs.readbuffer_encode)
1079 self.assertRaises(TypeError, codecs.readbuffer_encode, 42)
1080
1081 class UTF8SigTest(UTF8Test, unittest.TestCase):
1082 encoding = "utf-8-sig"
1083 BOM = codecs.BOM_UTF8
1084
1085 def test_partial(self):
1086 self.check_partial(
1087 "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
1088 [
1089 "",
1090 "",
1091 "", # First BOM has been read and skipped
1092 "",
1093 "",
1094 "\ufeff", # Second BOM has been read and emitted
1095 "\ufeff\x00", # "\x00" read and emitted
1096 "\ufeff\x00", # First byte of encoded "\xff" read
1097 "\ufeff\x00\xff", # Second byte of encoded "\xff" read
1098 "\ufeff\x00\xff", # First byte of encoded "\u07ff" read
1099 "\ufeff\x00\xff\u07ff", # Second byte of encoded "\u07ff" read
1100 "\ufeff\x00\xff\u07ff",
1101 "\ufeff\x00\xff\u07ff",
1102 "\ufeff\x00\xff\u07ff\u0800",
1103 "\ufeff\x00\xff\u07ff\u0800",
1104 "\ufeff\x00\xff\u07ff\u0800",
1105 "\ufeff\x00\xff\u07ff\u0800\uffff",
1106 "\ufeff\x00\xff\u07ff\u0800\uffff",
1107 "\ufeff\x00\xff\u07ff\u0800\uffff",
1108 "\ufeff\x00\xff\u07ff\u0800\uffff",
1109 "\ufeff\x00\xff\u07ff\u0800\uffff\U00010000",
1110 ]
1111 )
1112
1113 def test_bug1601501(self):
1114 # SF bug #1601501: check that the codec works with a buffer
1115 self.assertEqual(str(b"\xef\xbb\xbf", "utf-8-sig"), "")
1116
1117 def test_bom(self):
1118 d = codecs.getincrementaldecoder("utf-8-sig")()
1119 s = "spam"
1120 self.assertEqual(d.decode(s.encode("utf-8-sig")), s)
1121
1122 def test_stream_bom(self):
1123 unistring = "ABC\u00A1\u2200XYZ"
1124 bytestring = codecs.BOM_UTF8 + b"ABC\xC2\xA1\xE2\x88\x80XYZ"
1125
1126 reader = codecs.getreader("utf-8-sig")
1127 for sizehint in [None] + list(range(1, 11)) + \
1128 [64, 128, 256, 512, 1024]:
1129 istream = reader(io.BytesIO(bytestring))
1130 ostream = io.StringIO()
1131 while 1:
1132 if sizehint is not None:
1133 data = istream.read(sizehint)
1134 else:
1135 data = istream.read()
1136
1137 if not data:
1138 break
1139 ostream.write(data)
1140
1141 got = ostream.getvalue()
1142 self.assertEqual(got, unistring)
1143
1144 def test_stream_bare(self):
1145 unistring = "ABC\u00A1\u2200XYZ"
1146 bytestring = b"ABC\xC2\xA1\xE2\x88\x80XYZ"
1147
1148 reader = codecs.getreader("utf-8-sig")
1149 for sizehint in [None] + list(range(1, 11)) + \
1150 [64, 128, 256, 512, 1024]:
1151 istream = reader(io.BytesIO(bytestring))
1152 ostream = io.StringIO()
1153 while 1:
1154 if sizehint is not None:
1155 data = istream.read(sizehint)
1156 else:
1157 data = istream.read()
1158
1159 if not data:
1160 break
1161 ostream.write(data)
1162
1163 got = ostream.getvalue()
1164 self.assertEqual(got, unistring)
1165
1166
1167 class EscapeDecodeTest(unittest.TestCase):
1168 def test_empty(self):
1169 self.assertEqual(codecs.escape_decode(b""), (b"", 0))
1170 self.assertEqual(codecs.escape_decode(bytearray()), (b"", 0))
1171
1172 def test_raw(self):
1173 decode = codecs.escape_decode
1174 for b in range(256):
1175 b = bytes([b])
1176 if b != b'\\':
1177 self.assertEqual(decode(b + b'0'), (b + b'0', 2))
1178
1179 def test_escape(self):
1180 decode = codecs.escape_decode
1181 check = coding_checker(self, decode)
1182 check(b"[\\\n]", b"[]")
1183 check(br'[\"]', b'["]')
1184 check(br"[\']", b"[']")
1185 check(br"[\\]", b"[\\]")
1186 check(br"[\a]", b"[\x07]")
1187 check(br"[\b]", b"[\x08]")
1188 check(br"[\t]", b"[\x09]")
1189 check(br"[\n]", b"[\x0a]")
1190 check(br"[\v]", b"[\x0b]")
1191 check(br"[\f]", b"[\x0c]")
1192 check(br"[\r]", b"[\x0d]")
1193 check(br"[\7]", b"[\x07]")
1194 check(br"[\78]", b"[\x078]")
1195 check(br"[\41]", b"[!]")
1196 check(br"[\418]", b"[!8]")
1197 check(br"[\101]", b"[A]")
1198 check(br"[\1010]", b"[A0]")
1199 check(br"[\x41]", b"[A]")
1200 check(br"[\x410]", b"[A0]")
1201
1202 def test_warnings(self):
1203 decode = codecs.escape_decode
1204 check = coding_checker(self, decode)
1205 for i in range(97, 123):
1206 b = bytes([i])
1207 if b not in b'abfnrtvx':
1208 with self.assertWarnsRegex(DeprecationWarning,
1209 r'"\\%c" is an invalid escape sequence' % i):
1210 check(b"\\" + b, b"\\" + b)
1211 with self.assertWarnsRegex(DeprecationWarning,
1212 r'"\\%c" is an invalid escape sequence' % (i-32)):
1213 check(b"\\" + b.upper(), b"\\" + b.upper())
1214 with self.assertWarnsRegex(DeprecationWarning,
1215 r'"\\8" is an invalid escape sequence'):
1216 check(br"\8", b"\\8")
1217 with self.assertWarns(DeprecationWarning):
1218 check(br"\9", b"\\9")
1219 with self.assertWarnsRegex(DeprecationWarning,
1220 r'"\\\xfa" is an invalid escape sequence') as cm:
1221 check(b"\\\xfa", b"\\\xfa")
1222 for i in range(0o400, 0o1000):
1223 with self.assertWarnsRegex(DeprecationWarning,
1224 r'"\\%o" is an invalid octal escape sequence' % i):
1225 check(rb'\%o' % i, bytes([i & 0o377]))
1226
1227 with self.assertWarnsRegex(DeprecationWarning,
1228 r'"\\z" is an invalid escape sequence'):
1229 self.assertEqual(decode(br'\x\z', 'ignore'), (b'\\z', 4))
1230 with self.assertWarnsRegex(DeprecationWarning,
1231 r'"\\501" is an invalid octal escape sequence'):
1232 self.assertEqual(decode(br'\x\501', 'ignore'), (b'A', 6))
1233
1234 def test_errors(self):
1235 decode = codecs.escape_decode
1236 self.assertRaises(ValueError, decode, br"\x")
1237 self.assertRaises(ValueError, decode, br"[\x]")
1238 self.assertEqual(decode(br"[\x]\x", "ignore"), (b"[]", 6))
1239 self.assertEqual(decode(br"[\x]\x", "replace"), (b"[?]?", 6))
1240 self.assertRaises(ValueError, decode, br"\x0")
1241 self.assertRaises(ValueError, decode, br"[\x0]")
1242 self.assertEqual(decode(br"[\x0]\x0", "ignore"), (b"[]", 8))
1243 self.assertEqual(decode(br"[\x0]\x0", "replace"), (b"[?]?", 8))
1244
1245
1246 # From RFC 3492
1247 punycode_testcases = [
1248 # A Arabic (Egyptian):
1249 ("\u0644\u064A\u0647\u0645\u0627\u0628\u062A\u0643\u0644"
1250 "\u0645\u0648\u0634\u0639\u0631\u0628\u064A\u061F",
1251 b"egbpdaj6bu4bxfgehfvwxn"),
1252 # B Chinese (simplified):
1253 ("\u4ED6\u4EEC\u4E3A\u4EC0\u4E48\u4E0D\u8BF4\u4E2D\u6587",
1254 b"ihqwcrb4cv8a8dqg056pqjye"),
1255 # C Chinese (traditional):
1256 ("\u4ED6\u5011\u7232\u4EC0\u9EBD\u4E0D\u8AAA\u4E2D\u6587",
1257 b"ihqwctvzc91f659drss3x8bo0yb"),
1258 # D Czech: Pro<ccaron>prost<ecaron>nemluv<iacute><ccaron>esky
1259 ("\u0050\u0072\u006F\u010D\u0070\u0072\u006F\u0073\u0074"
1260 "\u011B\u006E\u0065\u006D\u006C\u0075\u0076\u00ED\u010D"
1261 "\u0065\u0073\u006B\u0079",
1262 b"Proprostnemluvesky-uyb24dma41a"),
1263 # E Hebrew:
1264 ("\u05DC\u05DE\u05D4\u05D4\u05DD\u05E4\u05E9\u05D5\u05D8"
1265 "\u05DC\u05D0\u05DE\u05D3\u05D1\u05E8\u05D9\u05DD\u05E2"
1266 "\u05D1\u05E8\u05D9\u05EA",
1267 b"4dbcagdahymbxekheh6e0a7fei0b"),
1268 # F Hindi (Devanagari):
1269 ("\u092F\u0939\u0932\u094B\u0917\u0939\u093F\u0928\u094D"
1270 "\u0926\u0940\u0915\u094D\u092F\u094B\u0902\u0928\u0939"
1271 "\u0940\u0902\u092C\u094B\u0932\u0938\u0915\u0924\u0947"
1272 "\u0939\u0948\u0902",
1273 b"i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd"),
1274
1275 #(G) Japanese (kanji and hiragana):
1276 ("\u306A\u305C\u307F\u3093\u306A\u65E5\u672C\u8A9E\u3092"
1277 "\u8A71\u3057\u3066\u304F\u308C\u306A\u3044\u306E\u304B",
1278 b"n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa"),
1279
1280 # (H) Korean (Hangul syllables):
1281 ("\uC138\uACC4\uC758\uBAA8\uB4E0\uC0AC\uB78C\uB4E4\uC774"
1282 "\uD55C\uAD6D\uC5B4\uB97C\uC774\uD574\uD55C\uB2E4\uBA74"
1283 "\uC5BC\uB9C8\uB098\uC88B\uC744\uAE4C",
1284 b"989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5j"
1285 b"psd879ccm6fea98c"),
1286
1287 # (I) Russian (Cyrillic):
1288 ("\u043F\u043E\u0447\u0435\u043C\u0443\u0436\u0435\u043E"
1289 "\u043D\u0438\u043D\u0435\u0433\u043E\u0432\u043E\u0440"
1290 "\u044F\u0442\u043F\u043E\u0440\u0443\u0441\u0441\u043A"
1291 "\u0438",
1292 b"b1abfaaepdrnnbgefbaDotcwatmq2g4l"),
1293
1294 # (J) Spanish: Porqu<eacute>nopuedensimplementehablarenEspa<ntilde>ol
1295 ("\u0050\u006F\u0072\u0071\u0075\u00E9\u006E\u006F\u0070"
1296 "\u0075\u0065\u0064\u0065\u006E\u0073\u0069\u006D\u0070"
1297 "\u006C\u0065\u006D\u0065\u006E\u0074\u0065\u0068\u0061"
1298 "\u0062\u006C\u0061\u0072\u0065\u006E\u0045\u0073\u0070"
1299 "\u0061\u00F1\u006F\u006C",
1300 b"PorqunopuedensimplementehablarenEspaol-fmd56a"),
1301
1302 # (K) Vietnamese:
1303 # T<adotbelow>isaoh<odotbelow>kh<ocirc>ngth<ecirchookabove>ch\
1304 # <ihookabove>n<oacute>iti<ecircacute>ngVi<ecircdotbelow>t
1305 ("\u0054\u1EA1\u0069\u0073\u0061\u006F\u0068\u1ECD\u006B"
1306 "\u0068\u00F4\u006E\u0067\u0074\u0068\u1EC3\u0063\u0068"
1307 "\u1EC9\u006E\u00F3\u0069\u0074\u0069\u1EBF\u006E\u0067"
1308 "\u0056\u0069\u1EC7\u0074",
1309 b"TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g"),
1310
1311 #(L) 3<nen>B<gumi><kinpachi><sensei>
1312 ("\u0033\u5E74\u0042\u7D44\u91D1\u516B\u5148\u751F",
1313 b"3B-ww4c5e180e575a65lsy2b"),
1314
1315 # (M) <amuro><namie>-with-SUPER-MONKEYS
1316 ("\u5B89\u5BA4\u5948\u7F8E\u6075\u002D\u0077\u0069\u0074"
1317 "\u0068\u002D\u0053\u0055\u0050\u0045\u0052\u002D\u004D"
1318 "\u004F\u004E\u004B\u0045\u0059\u0053",
1319 b"-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n"),
1320
1321 # (N) Hello-Another-Way-<sorezore><no><basho>
1322 ("\u0048\u0065\u006C\u006C\u006F\u002D\u0041\u006E\u006F"
1323 "\u0074\u0068\u0065\u0072\u002D\u0057\u0061\u0079\u002D"
1324 "\u305D\u308C\u305E\u308C\u306E\u5834\u6240",
1325 b"Hello-Another-Way--fc4qua05auwb3674vfr0b"),
1326
1327 # (O) <hitotsu><yane><no><shita>2
1328 ("\u3072\u3068\u3064\u5C4B\u6839\u306E\u4E0B\u0032",
1329 b"2-u9tlzr9756bt3uc0v"),
1330
1331 # (P) Maji<de>Koi<suru>5<byou><mae>
1332 ("\u004D\u0061\u006A\u0069\u3067\u004B\u006F\u0069\u3059"
1333 "\u308B\u0035\u79D2\u524D",
1334 b"MajiKoi5-783gue6qz075azm5e"),
1335
1336 # (Q) <pafii>de<runba>
1337 ("\u30D1\u30D5\u30A3\u30FC\u0064\u0065\u30EB\u30F3\u30D0",
1338 b"de-jg4avhby1noc0d"),
1339
1340 # (R) <sono><supiido><de>
1341 ("\u305D\u306E\u30B9\u30D4\u30FC\u30C9\u3067",
1342 b"d9juau41awczczp"),
1343
1344 # (S) -> $1.00 <-
1345 ("\u002D\u003E\u0020\u0024\u0031\u002E\u0030\u0030\u0020"
1346 "\u003C\u002D",
1347 b"-> $1.00 <--")
1348 ]
1349
1350 for i in punycode_testcases:
1351 if len(i)!=2:
1352 print(repr(i))
1353
1354
1355 class PunycodeTest(unittest.TestCase):
1356 def test_encode(self):
1357 for uni, puny in punycode_testcases:
1358 # Need to convert both strings to lower case, since
1359 # some of the extended encodings use upper case, but our
1360 # code produces only lower case. Converting just puny to
1361 # lower is also insufficient, since some of the input characters
1362 # are upper case.
1363 self.assertEqual(
1364 str(uni.encode("punycode"), "ascii").lower(),
1365 str(puny, "ascii").lower()
1366 )
1367
1368 def test_decode(self):
1369 for uni, puny in punycode_testcases:
1370 self.assertEqual(uni, puny.decode("punycode"))
1371 puny = puny.decode("ascii").encode("ascii")
1372 self.assertEqual(uni, puny.decode("punycode"))
1373
1374 def test_decode_invalid(self):
1375 testcases = [
1376 (b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")),
1377 (b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")),
1378 (b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")),
1379 (b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")),
1380 (b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")),
1381 (b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")),
1382 (b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")),
1383 (b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")),
1384 (b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")),
1385 (b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")),
1386 (b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")),
1387 (b"xn--w&", "ignore", "xn-"),
1388 ]
1389 for puny, errors, expected in testcases:
1390 with self.subTest(puny=puny, errors=errors):
1391 if isinstance(expected, Exception):
1392 with self.assertRaises(UnicodeDecodeError) as cm:
1393 puny.decode("punycode", errors)
1394 exc = cm.exception
1395 self.assertEqual(exc.encoding, expected.encoding)
1396 self.assertEqual(exc.object, puny)
1397 self.assertEqual(exc.start, expected.start)
1398 self.assertEqual(exc.end, expected.end)
1399 else:
1400 self.assertEqual(puny.decode("punycode", errors), expected)
1401
1402
1403 # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html
1404 nameprep_tests = [
1405 # 3.1 Map to nothing.
1406 (b'foo\xc2\xad\xcd\x8f\xe1\xa0\x86\xe1\xa0\x8bbar'
1407 b'\xe2\x80\x8b\xe2\x81\xa0baz\xef\xb8\x80\xef\xb8\x88\xef'
1408 b'\xb8\x8f\xef\xbb\xbf',
1409 b'foobarbaz'),
1410 # 3.2 Case folding ASCII U+0043 U+0041 U+0046 U+0045.
1411 (b'CAFE',
1412 b'cafe'),
1413 # 3.3 Case folding 8bit U+00DF (german sharp s).
1414 # The original test case is bogus; it says \xc3\xdf
1415 (b'\xc3\x9f',
1416 b'ss'),
1417 # 3.4 Case folding U+0130 (turkish capital I with dot).
1418 (b'\xc4\xb0',
1419 b'i\xcc\x87'),
1420 # 3.5 Case folding multibyte U+0143 U+037A.
1421 (b'\xc5\x83\xcd\xba',
1422 b'\xc5\x84 \xce\xb9'),
1423 # 3.6 Case folding U+2121 U+33C6 U+1D7BB.
1424 # XXX: skip this as it fails in UCS-2 mode
1425 #('\xe2\x84\xa1\xe3\x8f\x86\xf0\x9d\x9e\xbb',
1426 # 'telc\xe2\x88\x95kg\xcf\x83'),
1427 (None, None),
1428 # 3.7 Normalization of U+006a U+030c U+00A0 U+00AA.
1429 (b'j\xcc\x8c\xc2\xa0\xc2\xaa',
1430 b'\xc7\xb0 a'),
1431 # 3.8 Case folding U+1FB7 and normalization.
1432 (b'\xe1\xbe\xb7',
1433 b'\xe1\xbe\xb6\xce\xb9'),
1434 # 3.9 Self-reverting case folding U+01F0 and normalization.
1435 # The original test case is bogus, it says `\xc7\xf0'
1436 (b'\xc7\xb0',
1437 b'\xc7\xb0'),
1438 # 3.10 Self-reverting case folding U+0390 and normalization.
1439 (b'\xce\x90',
1440 b'\xce\x90'),
1441 # 3.11 Self-reverting case folding U+03B0 and normalization.
1442 (b'\xce\xb0',
1443 b'\xce\xb0'),
1444 # 3.12 Self-reverting case folding U+1E96 and normalization.
1445 (b'\xe1\xba\x96',
1446 b'\xe1\xba\x96'),
1447 # 3.13 Self-reverting case folding U+1F56 and normalization.
1448 (b'\xe1\xbd\x96',
1449 b'\xe1\xbd\x96'),
1450 # 3.14 ASCII space character U+0020.
1451 (b' ',
1452 b' '),
1453 # 3.15 Non-ASCII 8bit space character U+00A0.
1454 (b'\xc2\xa0',
1455 b' '),
1456 # 3.16 Non-ASCII multibyte space character U+1680.
1457 (b'\xe1\x9a\x80',
1458 None),
1459 # 3.17 Non-ASCII multibyte space character U+2000.
1460 (b'\xe2\x80\x80',
1461 b' '),
1462 # 3.18 Zero Width Space U+200b.
1463 (b'\xe2\x80\x8b',
1464 b''),
1465 # 3.19 Non-ASCII multibyte space character U+3000.
1466 (b'\xe3\x80\x80',
1467 b' '),
1468 # 3.20 ASCII control characters U+0010 U+007F.
1469 (b'\x10\x7f',
1470 b'\x10\x7f'),
1471 # 3.21 Non-ASCII 8bit control character U+0085.
1472 (b'\xc2\x85',
1473 None),
1474 # 3.22 Non-ASCII multibyte control character U+180E.
1475 (b'\xe1\xa0\x8e',
1476 None),
1477 # 3.23 Zero Width No-Break Space U+FEFF.
1478 (b'\xef\xbb\xbf',
1479 b''),
1480 # 3.24 Non-ASCII control character U+1D175.
1481 (b'\xf0\x9d\x85\xb5',
1482 None),
1483 # 3.25 Plane 0 private use character U+F123.
1484 (b'\xef\x84\xa3',
1485 None),
1486 # 3.26 Plane 15 private use character U+F1234.
1487 (b'\xf3\xb1\x88\xb4',
1488 None),
1489 # 3.27 Plane 16 private use character U+10F234.
1490 (b'\xf4\x8f\x88\xb4',
1491 None),
1492 # 3.28 Non-character code point U+8FFFE.
1493 (b'\xf2\x8f\xbf\xbe',
1494 None),
1495 # 3.29 Non-character code point U+10FFFF.
1496 (b'\xf4\x8f\xbf\xbf',
1497 None),
1498 # 3.30 Surrogate code U+DF42.
1499 (b'\xed\xbd\x82',
1500 None),
1501 # 3.31 Non-plain text character U+FFFD.
1502 (b'\xef\xbf\xbd',
1503 None),
1504 # 3.32 Ideographic description character U+2FF5.
1505 (b'\xe2\xbf\xb5',
1506 None),
1507 # 3.33 Display property character U+0341.
1508 (b'\xcd\x81',
1509 b'\xcc\x81'),
1510 # 3.34 Left-to-right mark U+200E.
1511 (b'\xe2\x80\x8e',
1512 None),
1513 # 3.35 Deprecated U+202A.
1514 (b'\xe2\x80\xaa',
1515 None),
1516 # 3.36 Language tagging character U+E0001.
1517 (b'\xf3\xa0\x80\x81',
1518 None),
1519 # 3.37 Language tagging character U+E0042.
1520 (b'\xf3\xa0\x81\x82',
1521 None),
1522 # 3.38 Bidi: RandALCat character U+05BE and LCat characters.
1523 (b'foo\xd6\xbebar',
1524 None),
1525 # 3.39 Bidi: RandALCat character U+FD50 and LCat characters.
1526 (b'foo\xef\xb5\x90bar',
1527 None),
1528 # 3.40 Bidi: RandALCat character U+FB38 and LCat characters.
1529 (b'foo\xef\xb9\xb6bar',
1530 b'foo \xd9\x8ebar'),
1531 # 3.41 Bidi: RandALCat without trailing RandALCat U+0627 U+0031.
1532 (b'\xd8\xa71',
1533 None),
1534 # 3.42 Bidi: RandALCat character U+0627 U+0031 U+0628.
1535 (b'\xd8\xa71\xd8\xa8',
1536 b'\xd8\xa71\xd8\xa8'),
1537 # 3.43 Unassigned code point U+E0002.
1538 # Skip this test as we allow unassigned
1539 #(b'\xf3\xa0\x80\x82',
1540 # None),
1541 (None, None),
1542 # 3.44 Larger test (shrinking).
1543 # Original test case reads \xc3\xdf
1544 (b'X\xc2\xad\xc3\x9f\xc4\xb0\xe2\x84\xa1j\xcc\x8c\xc2\xa0\xc2'
1545 b'\xaa\xce\xb0\xe2\x80\x80',
1546 b'xssi\xcc\x87tel\xc7\xb0 a\xce\xb0 '),
1547 # 3.45 Larger test (expanding).
1548 # Original test case reads \xc3\x9f
1549 (b'X\xc3\x9f\xe3\x8c\x96\xc4\xb0\xe2\x84\xa1\xe2\x92\x9f\xe3\x8c'
1550 b'\x80',
1551 b'xss\xe3\x82\xad\xe3\x83\xad\xe3\x83\xa1\xe3\x83\xbc\xe3'
1552 b'\x83\x88\xe3\x83\xabi\xcc\x87tel\x28d\x29\xe3\x82'
1553 b'\xa2\xe3\x83\x91\xe3\x83\xbc\xe3\x83\x88')
1554 ]
1555
1556
1557 class NameprepTest(unittest.TestCase):
1558 def test_nameprep(self):
1559 from encodings.idna import nameprep
1560 for pos, (orig, prepped) in enumerate(nameprep_tests):
1561 if orig is None:
1562 # Skipped
1563 continue
1564 # The Unicode strings are given in UTF-8
1565 orig = str(orig, "utf-8", "surrogatepass")
1566 if prepped is None:
1567 # Input contains prohibited characters
1568 self.assertRaises(UnicodeEncodeError, nameprep, orig)
1569 else:
1570 prepped = str(prepped, "utf-8", "surrogatepass")
1571 try:
1572 self.assertEqual(nameprep(orig), prepped)
1573 except Exception as e:
1574 raise support.TestFailed("Test 3.%d: %s" % (pos+1, str(e)))
1575
1576
1577 class IDNACodecTest(unittest.TestCase):
1578
1579 invalid_decode_testcases = [
1580 (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFFpython.org", 0, 1, "")),
1581 (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFFhon.org", 3, 4, "")),
1582 (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF.org", 6, 7, "")),
1583 (b"python.\xFForg", UnicodeDecodeError("idna", b"python.\xFForg", 7, 8, "")),
1584 (b"python.o\xFFrg", UnicodeDecodeError("idna", b"python.o\xFFrg", 8, 9, "")),
1585 (b"python.org\xFF", UnicodeDecodeError("idna", b"python.org\xFF", 10, 11, "")),
1586 (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.org", 10, 11, "")),
1587 (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.org", 11, 12, "")),
1588 (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.org", 13, 14, "")),
1589 ]
1590 invalid_encode_testcases = [
1591 (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"foo.{'\xff'*60}", 4, 64, "")),
1592 ("あさ.\u034f", UnicodeEncodeError("idna", "あさ.\u034f", 3, 4, "")),
1593 ]
1594
1595 def test_builtin_decode(self):
1596 self.assertEqual(str(b"python.org", "idna"), "python.org")
1597 self.assertEqual(str(b"python.org.", "idna"), "python.org.")
1598 self.assertEqual(str(b"xn--pythn-mua.org", "idna"), "pyth\xf6n.org")
1599 self.assertEqual(str(b"xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.")
1600 self.assertEqual(str(b"XN--pythn-mua.org.", "idna"), "pyth\xf6n.org.")
1601 self.assertEqual(str(b"xN--pythn-mua.org.", "idna"), "pyth\xf6n.org.")
1602 self.assertEqual(str(b"Xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.")
1603 self.assertEqual(str(b"bugs.xn--pythn-mua.org.", "idna"),
1604 "bugs.pyth\xf6n.org.")
1605 self.assertEqual(str(b"bugs.XN--pythn-mua.org.", "idna"),
1606 "bugs.pyth\xf6n.org.")
1607
1608 def test_builtin_decode_invalid(self):
1609 for case, expected in self.invalid_decode_testcases:
1610 with self.subTest(case=case, expected=expected):
1611 with self.assertRaises(UnicodeDecodeError) as cm:
1612 case.decode("idna")
1613 exc = cm.exception
1614 self.assertEqual(exc.encoding, expected.encoding)
1615 self.assertEqual(exc.object, expected.object)
1616 self.assertEqual(exc.start, expected.start, msg=f'reason: {exc.reason}')
1617 self.assertEqual(exc.end, expected.end)
1618
1619 def test_builtin_encode(self):
1620 self.assertEqual("python.org".encode("idna"), b"python.org")
1621 self.assertEqual("python.org.".encode("idna"), b"python.org.")
1622 self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org")
1623 self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.")
1624
1625 def test_builtin_encode_invalid(self):
1626 for case, expected in self.invalid_encode_testcases:
1627 with self.subTest(case=case, expected=expected):
1628 with self.assertRaises(UnicodeEncodeError) as cm:
1629 case.encode("idna")
1630 exc = cm.exception
1631 self.assertEqual(exc.encoding, expected.encoding)
1632 self.assertEqual(exc.object, expected.object)
1633 self.assertEqual(exc.start, expected.start)
1634 self.assertEqual(exc.end, expected.end)
1635
1636 def test_builtin_decode_length_limit(self):
1637 with self.assertRaisesRegex(UnicodeDecodeError, "way too long"):
1638 (b"xn--016c"+b"a"*1100).decode("idna")
1639 with self.assertRaisesRegex(UnicodeDecodeError, "too long"):
1640 (b"xn--016c"+b"a"*70).decode("idna")
1641
1642 def test_stream(self):
1643 r = codecs.getreader("idna")(io.BytesIO(b"abc"))
1644 r.read(3)
1645 self.assertEqual(r.read(), "")
1646
1647 def test_incremental_decode(self):
1648 self.assertEqual(
1649 "".join(codecs.iterdecode((bytes([c]) for c in b"python.org"), "idna")),
1650 "python.org"
1651 )
1652 self.assertEqual(
1653 "".join(codecs.iterdecode((bytes([c]) for c in b"python.org."), "idna")),
1654 "python.org."
1655 )
1656 self.assertEqual(
1657 "".join(codecs.iterdecode((bytes([c]) for c in b"xn--pythn-mua.org."), "idna")),
1658 "pyth\xf6n.org."
1659 )
1660 self.assertEqual(
1661 "".join(codecs.iterdecode((bytes([c]) for c in b"xn--pythn-mua.org."), "idna")),
1662 "pyth\xf6n.org."
1663 )
1664
1665 decoder = codecs.getincrementaldecoder("idna")()
1666 self.assertEqual(decoder.decode(b"xn--xam", ), "")
1667 self.assertEqual(decoder.decode(b"ple-9ta.o", ), "\xe4xample.")
1668 self.assertEqual(decoder.decode(b"rg"), "")
1669 self.assertEqual(decoder.decode(b"", True), "org")
1670
1671 decoder.reset()
1672 self.assertEqual(decoder.decode(b"xn--xam", ), "")
1673 self.assertEqual(decoder.decode(b"ple-9ta.o", ), "\xe4xample.")
1674 self.assertEqual(decoder.decode(b"rg."), "org.")
1675 self.assertEqual(decoder.decode(b"", True), "")
1676
1677 def test_incremental_decode_invalid(self):
1678 iterdecode_testcases = [
1679 (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")),
1680 (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFF", 3, 4, "")),
1681 (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF", 6, 7, "")),
1682 (b"python.\xFForg", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")),
1683 (b"python.o\xFFrg", UnicodeDecodeError("idna", b"o\xFF", 1, 2, "")),
1684 (b"python.org\xFF", UnicodeDecodeError("idna", b"org\xFF", 3, 4, "")),
1685 (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.", 10, 11, "")),
1686 (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.", 11, 12, "")),
1687 (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.", 13, 14, "")),
1688 ]
1689 for case, expected in iterdecode_testcases:
1690 with self.subTest(case=case, expected=expected):
1691 with self.assertRaises(UnicodeDecodeError) as cm:
1692 list(codecs.iterdecode((bytes([c]) for c in case), "idna"))
1693 exc = cm.exception
1694 self.assertEqual(exc.encoding, expected.encoding)
1695 self.assertEqual(exc.object, expected.object)
1696 self.assertEqual(exc.start, expected.start)
1697 self.assertEqual(exc.end, expected.end)
1698
1699 decoder = codecs.getincrementaldecoder("idna")()
1700 for case, expected in self.invalid_decode_testcases:
1701 with self.subTest(case=case, expected=expected):
1702 with self.assertRaises(UnicodeDecodeError) as cm:
1703 decoder.decode(case)
1704 exc = cm.exception
1705 self.assertEqual(exc.encoding, expected.encoding)
1706 self.assertEqual(exc.object, expected.object)
1707 self.assertEqual(exc.start, expected.start)
1708 self.assertEqual(exc.end, expected.end)
1709
1710 def test_incremental_encode(self):
1711 self.assertEqual(
1712 b"".join(codecs.iterencode("python.org", "idna")),
1713 b"python.org"
1714 )
1715 self.assertEqual(
1716 b"".join(codecs.iterencode("python.org.", "idna")),
1717 b"python.org."
1718 )
1719 self.assertEqual(
1720 b"".join(codecs.iterencode("pyth\xf6n.org.", "idna")),
1721 b"xn--pythn-mua.org."
1722 )
1723 self.assertEqual(
1724 b"".join(codecs.iterencode("pyth\xf6n.org.", "idna")),
1725 b"xn--pythn-mua.org."
1726 )
1727
1728 encoder = codecs.getincrementalencoder("idna")()
1729 self.assertEqual(encoder.encode("\xe4x"), b"")
1730 self.assertEqual(encoder.encode("ample.org"), b"xn--xample-9ta.")
1731 self.assertEqual(encoder.encode("", True), b"org")
1732
1733 encoder.reset()
1734 self.assertEqual(encoder.encode("\xe4x"), b"")
1735 self.assertEqual(encoder.encode("ample.org."), b"xn--xample-9ta.org.")
1736 self.assertEqual(encoder.encode("", True), b"")
1737
1738 def test_incremental_encode_invalid(self):
1739 iterencode_testcases = [
1740 (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"{'\xff'*60}", 0, 60, "")),
1741 ("あさ.\u034f", UnicodeEncodeError("idna", "\u034f", 0, 1, "")),
1742 ]
1743 for case, expected in iterencode_testcases:
1744 with self.subTest(case=case, expected=expected):
1745 with self.assertRaises(UnicodeEncodeError) as cm:
1746 list(codecs.iterencode(case, "idna"))
1747 exc = cm.exception
1748 self.assertEqual(exc.encoding, expected.encoding)
1749 self.assertEqual(exc.object, expected.object)
1750 self.assertEqual(exc.start, expected.start)
1751 self.assertEqual(exc.end, expected.end)
1752
1753 # codecs.getincrementalencoder.encode() does not throw an error
1754
1755 def test_errors(self):
1756 """Only supports "strict" error handler"""
1757 "python.org".encode("idna", "strict")
1758 b"python.org".decode("idna", "strict")
1759 for errors in ("ignore", "replace", "backslashreplace",
1760 "surrogateescape"):
1761 self.assertRaises(Exception, "python.org".encode, "idna", errors)
1762 self.assertRaises(Exception,
1763 b"python.org".decode, "idna", errors)
1764
1765
1766 class CodecsModuleTest(unittest.TestCase):
1767
1768 def test_decode(self):
1769 self.assertEqual(codecs.decode(b'\xe4\xf6\xfc', 'latin-1'),
1770 '\xe4\xf6\xfc')
1771 self.assertRaises(TypeError, codecs.decode)
1772 self.assertEqual(codecs.decode(b'abc'), 'abc')
1773 self.assertRaises(UnicodeDecodeError, codecs.decode, b'\xff', 'ascii')
1774
1775 # test keywords
1776 self.assertEqual(codecs.decode(obj=b'\xe4\xf6\xfc', encoding='latin-1'),
1777 '\xe4\xf6\xfc')
1778 self.assertEqual(codecs.decode(b'[\xff]', 'ascii', errors='ignore'),
1779 '[]')
1780
1781 def test_encode(self):
1782 self.assertEqual(codecs.encode('\xe4\xf6\xfc', 'latin-1'),
1783 b'\xe4\xf6\xfc')
1784 self.assertRaises(TypeError, codecs.encode)
1785 self.assertRaises(LookupError, codecs.encode, "foo", "__spam__")
1786 self.assertEqual(codecs.encode('abc'), b'abc')
1787 self.assertRaises(UnicodeEncodeError, codecs.encode, '\xffff', 'ascii')
1788
1789 # test keywords
1790 self.assertEqual(codecs.encode(obj='\xe4\xf6\xfc', encoding='latin-1'),
1791 b'\xe4\xf6\xfc')
1792 self.assertEqual(codecs.encode('[\xff]', 'ascii', errors='ignore'),
1793 b'[]')
1794
1795 def test_register(self):
1796 self.assertRaises(TypeError, codecs.register)
1797 self.assertRaises(TypeError, codecs.register, 42)
1798
1799 def test_unregister(self):
1800 name = "nonexistent_codec_name"
1801 search_function = mock.Mock()
1802 codecs.register(search_function)
1803 self.assertRaises(TypeError, codecs.lookup, name)
1804 search_function.assert_called_with(name)
1805 search_function.reset_mock()
1806
1807 codecs.unregister(search_function)
1808 self.assertRaises(LookupError, codecs.lookup, name)
1809 search_function.assert_not_called()
1810
1811 def test_lookup(self):
1812 self.assertRaises(TypeError, codecs.lookup)
1813 self.assertRaises(LookupError, codecs.lookup, "__spam__")
1814 self.assertRaises(LookupError, codecs.lookup, " ")
1815
1816 def test_getencoder(self):
1817 self.assertRaises(TypeError, codecs.getencoder)
1818 self.assertRaises(LookupError, codecs.getencoder, "__spam__")
1819
1820 def test_getdecoder(self):
1821 self.assertRaises(TypeError, codecs.getdecoder)
1822 self.assertRaises(LookupError, codecs.getdecoder, "__spam__")
1823
1824 def test_getreader(self):
1825 self.assertRaises(TypeError, codecs.getreader)
1826 self.assertRaises(LookupError, codecs.getreader, "__spam__")
1827
1828 def test_getwriter(self):
1829 self.assertRaises(TypeError, codecs.getwriter)
1830 self.assertRaises(LookupError, codecs.getwriter, "__spam__")
1831
1832 @support.run_with_locale('LC_CTYPE', 'tr_TR')
1833 def test_lookup_issue1813(self):
1834 # Issue #1813: under Turkish locales, lookup of some codecs failed
1835 # because 'I' is lowercased as "ı" (dotless i)
1836 c = codecs.lookup('ASCII')
1837 self.assertEqual(c.name, 'ascii')
1838
1839 def test_all(self):
1840 api = (
1841 "encode", "decode",
1842 "register", "CodecInfo", "Codec", "IncrementalEncoder",
1843 "IncrementalDecoder", "StreamReader", "StreamWriter", "lookup",
1844 "getencoder", "getdecoder", "getincrementalencoder",
1845 "getincrementaldecoder", "getreader", "getwriter",
1846 "register_error", "lookup_error",
1847 "strict_errors", "replace_errors", "ignore_errors",
1848 "xmlcharrefreplace_errors", "backslashreplace_errors",
1849 "namereplace_errors",
1850 "open", "EncodedFile",
1851 "iterencode", "iterdecode",
1852 "BOM", "BOM_BE", "BOM_LE",
1853 "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_BE", "BOM_UTF16_LE",
1854 "BOM_UTF32", "BOM_UTF32_BE", "BOM_UTF32_LE",
1855 "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE", # Undocumented
1856 "StreamReaderWriter", "StreamRecoder",
1857 )
1858 self.assertCountEqual(api, codecs.__all__)
1859 for api in codecs.__all__:
1860 getattr(codecs, api)
1861
1862 def test_open(self):
1863 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
1864 for mode in ('w', 'r', 'r+', 'w+', 'a', 'a+'):
1865 with self.subTest(mode), self.assertWarns(DeprecationWarning):
1866 with codecs.open(os_helper.TESTFN, mode, 'ascii') as file:
1867 self.assertIsInstance(file, codecs.StreamReaderWriter)
1868
1869 def test_undefined(self):
1870 self.assertRaises(UnicodeError, codecs.encode, 'abc', 'undefined')
1871 self.assertRaises(UnicodeError, codecs.decode, b'abc', 'undefined')
1872 self.assertRaises(UnicodeError, codecs.encode, '', 'undefined')
1873 self.assertRaises(UnicodeError, codecs.decode, b'', 'undefined')
1874 for errors in ('strict', 'ignore', 'replace', 'backslashreplace'):
1875 self.assertRaises(UnicodeError,
1876 codecs.encode, 'abc', 'undefined', errors)
1877 self.assertRaises(UnicodeError,
1878 codecs.decode, b'abc', 'undefined', errors)
1879
1880 def test_file_closes_if_lookup_error_raised(self):
1881 mock_open = mock.mock_open()
1882 with mock.patch('builtins.open', mock_open) as file:
1883 with self.assertRaises(LookupError):
1884 codecs_open_no_warn(os_helper.TESTFN, 'wt', 'invalid-encoding')
1885
1886 file().close.assert_called()
1887
1888 def test_copy(self):
1889 orig = codecs.lookup('utf-8')
1890 dup = copy.copy(orig)
1891 self.assertIsNot(dup, orig)
1892 self.assertEqual(dup, orig)
1893 self.assertTrue(orig._is_text_encoding)
1894 self.assertEqual(dup.encode, orig.encode)
1895 self.assertEqual(dup.name, orig.name)
1896 self.assertEqual(dup.incrementalencoder, orig.incrementalencoder)
1897
1898 # Test a CodecInfo with _is_text_encoding equal to false.
1899 orig = codecs.lookup("base64")
1900 dup = copy.copy(orig)
1901 self.assertIsNot(dup, orig)
1902 self.assertEqual(dup, orig)
1903 self.assertFalse(orig._is_text_encoding)
1904 self.assertEqual(dup.encode, orig.encode)
1905 self.assertEqual(dup.name, orig.name)
1906 self.assertEqual(dup.incrementalencoder, orig.incrementalencoder)
1907
1908 def test_deepcopy(self):
1909 orig = codecs.lookup('utf-8')
1910 dup = copy.deepcopy(orig)
1911 self.assertIsNot(dup, orig)
1912 self.assertEqual(dup, orig)
1913 self.assertTrue(orig._is_text_encoding)
1914 self.assertEqual(dup.encode, orig.encode)
1915 self.assertEqual(dup.name, orig.name)
1916 self.assertEqual(dup.incrementalencoder, orig.incrementalencoder)
1917
1918 # Test a CodecInfo with _is_text_encoding equal to false.
1919 orig = codecs.lookup("base64")
1920 dup = copy.deepcopy(orig)
1921 self.assertIsNot(dup, orig)
1922 self.assertEqual(dup, orig)
1923 self.assertFalse(orig._is_text_encoding)
1924 self.assertEqual(dup.encode, orig.encode)
1925 self.assertEqual(dup.name, orig.name)
1926 self.assertEqual(dup.incrementalencoder, orig.incrementalencoder)
1927
1928 def test_pickle(self):
1929 codec_info = codecs.lookup('utf-8')
1930 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
1931 with self.subTest(protocol=proto):
1932 pickled_codec_info = pickle.dumps(codec_info)
1933 unpickled_codec_info = pickle.loads(pickled_codec_info)
1934 self.assertIsNot(codec_info, unpickled_codec_info)
1935 self.assertEqual(codec_info, unpickled_codec_info)
1936 self.assertEqual(codec_info.name, unpickled_codec_info.name)
1937 self.assertEqual(
1938 codec_info.incrementalencoder,
1939 unpickled_codec_info.incrementalencoder
1940 )
1941 self.assertTrue(unpickled_codec_info._is_text_encoding)
1942
1943 # Test a CodecInfo with _is_text_encoding equal to false.
1944 codec_info = codecs.lookup('base64')
1945 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
1946 with self.subTest(protocol=proto):
1947 pickled_codec_info = pickle.dumps(codec_info)
1948 unpickled_codec_info = pickle.loads(pickled_codec_info)
1949 self.assertIsNot(codec_info, unpickled_codec_info)
1950 self.assertEqual(codec_info, unpickled_codec_info)
1951 self.assertEqual(codec_info.name, unpickled_codec_info.name)
1952 self.assertEqual(
1953 codec_info.incrementalencoder,
1954 unpickled_codec_info.incrementalencoder
1955 )
1956 self.assertFalse(unpickled_codec_info._is_text_encoding)
1957
1958
1959 class StreamReaderTest(unittest.TestCase):
1960
1961 def setUp(self):
1962 self.reader = codecs.getreader('utf-8')
1963 self.stream = io.BytesIO(b'\xed\x95\x9c\n\xea\xb8\x80')
1964
1965 def test_readlines(self):
1966 f = self.reader(self.stream)
1967 self.assertEqual(f.readlines(), ['\ud55c\n', '\uae00'])
1968
1969 def test_copy(self):
1970 f = self.reader(Queue(b'\xed\x95\x9c\n\xea\xb8\x80'))
1971 with self.assertRaisesRegex(TypeError, 'StreamReader'):
1972 copy.copy(f)
1973 with self.assertRaisesRegex(TypeError, 'StreamReader'):
1974 copy.deepcopy(f)
1975
1976 def test_pickle(self):
1977 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
1978 with self.subTest(protocol=proto):
1979 f = self.reader(Queue(b'\xed\x95\x9c\n\xea\xb8\x80'))
1980 with self.assertRaisesRegex(TypeError, 'StreamReader'):
1981 pickle.dumps(f, proto)
1982
1983
1984 class StreamWriterTest(unittest.TestCase):
1985
1986 def setUp(self):
1987 self.writer = codecs.getwriter('utf-8')
1988
1989 def test_copy(self):
1990 f = self.writer(Queue(b''))
1991 with self.assertRaisesRegex(TypeError, 'StreamWriter'):
1992 copy.copy(f)
1993 with self.assertRaisesRegex(TypeError, 'StreamWriter'):
1994 copy.deepcopy(f)
1995
1996 def test_pickle(self):
1997 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
1998 with self.subTest(protocol=proto):
1999 f = self.writer(Queue(b''))
2000 with self.assertRaisesRegex(TypeError, 'StreamWriter'):
2001 pickle.dumps(f, proto)
2002
2003
2004 class StreamReaderWriterTest(unittest.TestCase):
2005
2006 def setUp(self):
2007 self.reader = codecs.getreader('latin1')
2008 self.writer = codecs.getwriter('utf-8')
2009
2010 def test_copy(self):
2011 f = codecs.StreamReaderWriter(Queue(b''), self.reader, self.writer)
2012 with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'):
2013 copy.copy(f)
2014 with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'):
2015 copy.deepcopy(f)
2016
2017 def test_pickle(self):
2018 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
2019 with self.subTest(protocol=proto):
2020 f = codecs.StreamReaderWriter(Queue(b''), self.reader, self.writer)
2021 with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'):
2022 pickle.dumps(f, proto)
2023
2024
2025 class EncodedFileTest(unittest.TestCase):
2026
2027 def test_basic(self):
2028 f = io.BytesIO(b'\xed\x95\x9c\n\xea\xb8\x80')
2029 ef = codecs.EncodedFile(f, 'utf-16-le', 'utf-8')
2030 self.assertEqual(ef.read(), b'\\\xd5\n\x00\x00\xae')
2031
2032 f = io.BytesIO()
2033 ef = codecs.EncodedFile(f, 'utf-8', 'latin-1')
2034 ef.write(b'\xc3\xbc')
2035 self.assertEqual(f.getvalue(), b'\xfc')
2036
2037 all_unicode_encodings = [
2038 "ascii",
2039 "big5",
2040 "big5hkscs",
2041 "charmap",
2042 "cp037",
2043 "cp1006",
2044 "cp1026",
2045 "cp1125",
2046 "cp1140",
2047 "cp1250",
2048 "cp1251",
2049 "cp1252",
2050 "cp1253",
2051 "cp1254",
2052 "cp1255",
2053 "cp1256",
2054 "cp1257",
2055 "cp1258",
2056 "cp424",
2057 "cp437",
2058 "cp500",
2059 "cp720",
2060 "cp737",
2061 "cp775",
2062 "cp850",
2063 "cp852",
2064 "cp855",
2065 "cp856",
2066 "cp857",
2067 "cp858",
2068 "cp860",
2069 "cp861",
2070 "cp862",
2071 "cp863",
2072 "cp864",
2073 "cp865",
2074 "cp866",
2075 "cp869",
2076 "cp874",
2077 "cp875",
2078 "cp932",
2079 "cp949",
2080 "cp950",
2081 "euc_jis_2004",
2082 "euc_jisx0213",
2083 "euc_jp",
2084 "euc_kr",
2085 "gb18030",
2086 "gb2312",
2087 "gbk",
2088 "hp_roman8",
2089 "hz",
2090 "idna",
2091 "iso2022_jp",
2092 "iso2022_jp_1",
2093 "iso2022_jp_2",
2094 "iso2022_jp_2004",
2095 "iso2022_jp_3",
2096 "iso2022_jp_ext",
2097 "iso2022_kr",
2098 "iso8859_1",
2099 "iso8859_10",
2100 "iso8859_11",
2101 "iso8859_13",
2102 "iso8859_14",
2103 "iso8859_15",
2104 "iso8859_16",
2105 "iso8859_2",
2106 "iso8859_3",
2107 "iso8859_4",
2108 "iso8859_5",
2109 "iso8859_6",
2110 "iso8859_7",
2111 "iso8859_8",
2112 "iso8859_9",
2113 "johab",
2114 "koi8_r",
2115 "koi8_t",
2116 "koi8_u",
2117 "kz1048",
2118 "latin_1",
2119 "mac_cyrillic",
2120 "mac_greek",
2121 "mac_iceland",
2122 "mac_latin2",
2123 "mac_roman",
2124 "mac_turkish",
2125 "palmos",
2126 "ptcp154",
2127 "punycode",
2128 "raw_unicode_escape",
2129 "shift_jis",
2130 "shift_jis_2004",
2131 "shift_jisx0213",
2132 "tis_620",
2133 "unicode_escape",
2134 "utf_16",
2135 "utf_16_be",
2136 "utf_16_le",
2137 "utf_7",
2138 "utf_8",
2139 ]
2140
2141 if hasattr(codecs, "mbcs_encode"):
2142 all_unicode_encodings.append("mbcs")
2143 if hasattr(codecs, "oem_encode"):
2144 all_unicode_encodings.append("oem")
2145
2146 # The following encoding is not tested, because it's not supposed
2147 # to work:
2148 # "undefined"
2149
2150 # The following encodings don't work in stateful mode
2151 broken_unicode_with_stateful = [
2152 "punycode",
2153 ]
2154
2155
2156 class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling):
2157 def test_basics(self):
2158 s = "abc123" # all codecs should be able to encode these
2159 for encoding in all_unicode_encodings:
2160 name = codecs.lookup(encoding).name
2161 if encoding.endswith("_codec"):
2162 name += "_codec"
2163 elif encoding == "latin_1":
2164 name = "latin_1"
2165 # Skip the mbcs alias on Windows
2166 if name != "mbcs":
2167 self.assertEqual(encoding.replace("_", "-"),
2168 name.replace("_", "-"))
2169
2170 (b, size) = codecs.getencoder(encoding)(s)
2171 self.assertEqual(size, len(s), "encoding=%r" % encoding)
2172 (chars, size) = codecs.getdecoder(encoding)(b)
2173 self.assertEqual(chars, s, "encoding=%r" % encoding)
2174
2175 if encoding not in broken_unicode_with_stateful:
2176 # check stream reader/writer
2177 q = Queue(b"")
2178 writer = codecs.getwriter(encoding)(q)
2179 encodedresult = b""
2180 for c in s:
2181 writer.write(c)
2182 chunk = q.read()
2183 self.assertTrue(type(chunk) is bytes, type(chunk))
2184 encodedresult += chunk
2185 q = Queue(b"")
2186 reader = codecs.getreader(encoding)(q)
2187 decodedresult = ""
2188 for c in encodedresult:
2189 q.write(bytes([c]))
2190 decodedresult += reader.read()
2191 self.assertEqual(decodedresult, s, "encoding=%r" % encoding)
2192
2193 if encoding not in broken_unicode_with_stateful:
2194 # check incremental decoder/encoder and iterencode()/iterdecode()
2195 try:
2196 encoder = codecs.getincrementalencoder(encoding)()
2197 except LookupError: # no IncrementalEncoder
2198 pass
2199 else:
2200 # check incremental decoder/encoder
2201 encodedresult = b""
2202 for c in s:
2203 encodedresult += encoder.encode(c)
2204 encodedresult += encoder.encode("", True)
2205 decoder = codecs.getincrementaldecoder(encoding)()
2206 decodedresult = ""
2207 for c in encodedresult:
2208 decodedresult += decoder.decode(bytes([c]))
2209 decodedresult += decoder.decode(b"", True)
2210 self.assertEqual(decodedresult, s,
2211 "encoding=%r" % encoding)
2212
2213 # check iterencode()/iterdecode()
2214 result = "".join(codecs.iterdecode(
2215 codecs.iterencode(s, encoding), encoding))
2216 self.assertEqual(result, s, "encoding=%r" % encoding)
2217
2218 # check iterencode()/iterdecode() with empty string
2219 result = "".join(codecs.iterdecode(
2220 codecs.iterencode("", encoding), encoding))
2221 self.assertEqual(result, "")
2222
2223 if encoding not in ("idna", "mbcs"):
2224 # check incremental decoder/encoder with errors argument
2225 try:
2226 encoder = codecs.getincrementalencoder(encoding)("ignore")
2227 except LookupError: # no IncrementalEncoder
2228 pass
2229 else:
2230 encodedresult = b"".join(encoder.encode(c) for c in s)
2231 decoder = codecs.getincrementaldecoder(encoding)("ignore")
2232 decodedresult = "".join(decoder.decode(bytes([c]))
2233 for c in encodedresult)
2234 self.assertEqual(decodedresult, s,
2235 "encoding=%r" % encoding)
2236
2237 @support.cpython_only
2238 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
2239 def test_basics_capi(self):
2240 s = "abc123" # all codecs should be able to encode these
2241 for encoding in all_unicode_encodings:
2242 if encoding not in broken_unicode_with_stateful:
2243 # check incremental decoder/encoder (fetched via the C API)
2244 try:
2245 cencoder = _testlimitedcapi.codec_incrementalencoder(encoding)
2246 except LookupError: # no IncrementalEncoder
2247 pass
2248 else:
2249 # check C API
2250 encodedresult = b""
2251 for c in s:
2252 encodedresult += cencoder.encode(c)
2253 encodedresult += cencoder.encode("", True)
2254 cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding)
2255 decodedresult = ""
2256 for c in encodedresult:
2257 decodedresult += cdecoder.decode(bytes([c]))
2258 decodedresult += cdecoder.decode(b"", True)
2259 self.assertEqual(decodedresult, s,
2260 "encoding=%r" % encoding)
2261
2262 if encoding not in ("idna", "mbcs"):
2263 # check incremental decoder/encoder with errors argument
2264 try:
2265 cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore")
2266 except LookupError: # no IncrementalEncoder
2267 pass
2268 else:
2269 encodedresult = b"".join(cencoder.encode(c) for c in s)
2270 cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore")
2271 decodedresult = "".join(cdecoder.decode(bytes([c]))
2272 for c in encodedresult)
2273 self.assertEqual(decodedresult, s,
2274 "encoding=%r" % encoding)
2275
2276 def test_seek(self):
2277 # all codecs should be able to encode these
2278 s = "%s\n%s\n" % (100*"abc123", 100*"def456")
2279 for encoding in all_unicode_encodings:
2280 if encoding == "idna": # FIXME: See SF bug #1163178
2281 continue
2282 if encoding in broken_unicode_with_stateful:
2283 continue
2284 reader = codecs.getreader(encoding)(io.BytesIO(s.encode(encoding)))
2285 for t in range(5):
2286 # Test that calling seek resets the internal codec state and buffers
2287 reader.seek(0, 0)
2288 data = reader.read()
2289 self.assertEqual(s, data)
2290
2291 def test_bad_decode_args(self):
2292 for encoding in all_unicode_encodings:
2293 decoder = codecs.getdecoder(encoding)
2294 self.assertRaises(TypeError, decoder)
2295 if encoding not in ("idna", "punycode"):
2296 self.assertRaises(TypeError, decoder, 42)
2297
2298 def test_bad_encode_args(self):
2299 for encoding in all_unicode_encodings:
2300 encoder = codecs.getencoder(encoding)
2301 self.assertRaises(TypeError, encoder)
2302
2303 def test_encoding_map_type_initialized(self):
2304 from encodings import cp1140
2305 # This used to crash, we are only verifying there's no crash.
2306 table_type = type(cp1140.encoding_table)
2307 self.assertEqual(table_type, table_type)
2308
2309 def test_decoder_state(self):
2310 # Check that getstate() and setstate() handle the state properly
2311 u = "abc123"
2312 for encoding in all_unicode_encodings:
2313 if encoding not in broken_unicode_with_stateful:
2314 self.check_state_handling_decode(encoding, u, u.encode(encoding))
2315 self.check_state_handling_encode(encoding, u, u.encode(encoding))
2316
2317
2318 class CharmapTest(unittest.TestCase):
2319 def test_decode_with_string_map(self):
2320 self.assertEqual(
2321 codecs.charmap_decode(b"\x00\x01\x02", "strict", "abc"),
2322 ("abc", 3)
2323 )
2324
2325 self.assertEqual(
2326 codecs.charmap_decode(b"\x00\x01\x02", "strict", "\U0010FFFFbc"),
2327 ("\U0010FFFFbc", 3)
2328 )
2329
2330 self.assertRaises(UnicodeDecodeError,
2331 codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab"
2332 )
2333
2334 self.assertRaises(UnicodeDecodeError,
2335 codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab\ufffe"
2336 )
2337
2338 self.assertEqual(
2339 codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab"),
2340 ("ab\ufffd", 3)
2341 )
2342
2343 self.assertEqual(
2344 codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab\ufffe"),
2345 ("ab\ufffd", 3)
2346 )
2347
2348 self.assertEqual(
2349 codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace", "ab"),
2350 ("ab\\x02", 3)
2351 )
2352
2353 self.assertEqual(
2354 codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace", "ab\ufffe"),
2355 ("ab\\x02", 3)
2356 )
2357
2358 self.assertEqual(
2359 codecs.charmap_decode(b"\x00\x01\x02", "ignore", "ab"),
2360 ("ab", 3)
2361 )
2362
2363 self.assertEqual(
2364 codecs.charmap_decode(b"\x00\x01\x02", "ignore", "ab\ufffe"),
2365 ("ab", 3)
2366 )
2367
2368 allbytes = bytes(range(256))
2369 self.assertEqual(
2370 codecs.charmap_decode(allbytes, "ignore", ""),
2371 ("", len(allbytes))
2372 )
2373
2374 def test_decode_with_int2str_map(self):
2375 self.assertEqual(
2376 codecs.charmap_decode(b"\x00\x01\x02", "strict",
2377 {0: 'a', 1: 'b', 2: 'c'}),
2378 ("abc", 3)
2379 )
2380
2381 self.assertEqual(
2382 codecs.charmap_decode(b"\x00\x01\x02", "strict",
2383 {0: 'Aa', 1: 'Bb', 2: 'Cc'}),
2384 ("AaBbCc", 3)
2385 )
2386
2387 self.assertEqual(
2388 codecs.charmap_decode(b"\x00\x01\x02", "strict",
2389 {0: '\U0010FFFF', 1: 'b', 2: 'c'}),
2390 ("\U0010FFFFbc", 3)
2391 )
2392
2393 self.assertEqual(
2394 codecs.charmap_decode(b"\x00\x01\x02", "strict",
2395 {0: 'a', 1: 'b', 2: ''}),
2396 ("ab", 3)
2397 )
2398
2399 self.assertRaises(UnicodeDecodeError,
2400 codecs.charmap_decode, b"\x00\x01\x02", "strict",
2401 {0: 'a', 1: 'b'}
2402 )
2403
2404 self.assertRaises(UnicodeDecodeError,
2405 codecs.charmap_decode, b"\x00\x01\x02", "strict",
2406 {0: 'a', 1: 'b', 2: None}
2407 )
2408
2409 # Issue #14850
2410 self.assertRaises(UnicodeDecodeError,
2411 codecs.charmap_decode, b"\x00\x01\x02", "strict",
2412 {0: 'a', 1: 'b', 2: '\ufffe'}
2413 )
2414
2415 self.assertEqual(
2416 codecs.charmap_decode(b"\x00\x01\x02", "replace",
2417 {0: 'a', 1: 'b'}),
2418 ("ab\ufffd", 3)
2419 )
2420
2421 self.assertEqual(
2422 codecs.charmap_decode(b"\x00\x01\x02", "replace",
2423 {0: 'a', 1: 'b', 2: None}),
2424 ("ab\ufffd", 3)
2425 )
2426
2427 # Issue #14850
2428 self.assertEqual(
2429 codecs.charmap_decode(b"\x00\x01\x02", "replace",
2430 {0: 'a', 1: 'b', 2: '\ufffe'}),
2431 ("ab\ufffd", 3)
2432 )
2433
2434 self.assertEqual(
2435 codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
2436 {0: 'a', 1: 'b'}),
2437 ("ab\\x02", 3)
2438 )
2439
2440 self.assertEqual(
2441 codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
2442 {0: 'a', 1: 'b', 2: None}),
2443 ("ab\\x02", 3)
2444 )
2445
2446 # Issue #14850
2447 self.assertEqual(
2448 codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
2449 {0: 'a', 1: 'b', 2: '\ufffe'}),
2450 ("ab\\x02", 3)
2451 )
2452
2453 self.assertEqual(
2454 codecs.charmap_decode(b"\x00\x01\x02", "ignore",
2455 {0: 'a', 1: 'b'}),
2456 ("ab", 3)
2457 )
2458
2459 self.assertEqual(
2460 codecs.charmap_decode(b"\x00\x01\x02", "ignore",
2461 {0: 'a', 1: 'b', 2: None}),
2462 ("ab", 3)
2463 )
2464
2465 # Issue #14850
2466 self.assertEqual(
2467 codecs.charmap_decode(b"\x00\x01\x02", "ignore",
2468 {0: 'a', 1: 'b', 2: '\ufffe'}),
2469 ("ab", 3)
2470 )
2471
2472 allbytes = bytes(range(256))
2473 self.assertEqual(
2474 codecs.charmap_decode(allbytes, "ignore", {}),
2475 ("", len(allbytes))
2476 )
2477
2478 self.assertRaisesRegex(TypeError,
2479 "character mapping must be in range\\(0x110000\\)",
2480 codecs.charmap_decode,
2481 b"\x00\x01\x02", "strict", {0: "A", 1: 'Bb', 2: -2}
2482 )
2483
2484 self.assertRaisesRegex(TypeError,
2485 "character mapping must be in range\\(0x110000\\)",
2486 codecs.charmap_decode,
2487 b"\x00\x01\x02", "strict", {0: "A", 1: 'Bb', 2: 999999999}
2488 )
2489
2490 def test_decode_with_int2int_map(self):
2491 a = ord('a')
2492 b = ord('b')
2493 c = ord('c')
2494
2495 self.assertEqual(
2496 codecs.charmap_decode(b"\x00\x01\x02", "strict",
2497 {0: a, 1: b, 2: c}),
2498 ("abc", 3)
2499 )
2500
2501 # Issue #15379
2502 self.assertEqual(
2503 codecs.charmap_decode(b"\x00\x01\x02", "strict",
2504 {0: 0x10FFFF, 1: b, 2: c}),
2505 ("\U0010FFFFbc", 3)
2506 )
2507
2508 self.assertEqual(
2509 codecs.charmap_decode(b"\x00\x01\x02", "strict",
2510 {0: sys.maxunicode, 1: b, 2: c}),
2511 (chr(sys.maxunicode) + "bc", 3)
2512 )
2513
2514 self.assertRaises(TypeError,
2515 codecs.charmap_decode, b"\x00\x01\x02", "strict",
2516 {0: sys.maxunicode + 1, 1: b, 2: c}
2517 )
2518
2519 self.assertRaises(UnicodeDecodeError,
2520 codecs.charmap_decode, b"\x00\x01\x02", "strict",
2521 {0: a, 1: b},
2522 )
2523
2524 self.assertRaises(UnicodeDecodeError,
2525 codecs.charmap_decode, b"\x00\x01\x02", "strict",
2526 {0: a, 1: b, 2: 0xFFFE},
2527 )
2528
2529 self.assertEqual(
2530 codecs.charmap_decode(b"\x00\x01\x02", "replace",
2531 {0: a, 1: b}),
2532 ("ab\ufffd", 3)
2533 )
2534
2535 self.assertEqual(
2536 codecs.charmap_decode(b"\x00\x01\x02", "replace",
2537 {0: a, 1: b, 2: 0xFFFE}),
2538 ("ab\ufffd", 3)
2539 )
2540
2541 self.assertEqual(
2542 codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
2543 {0: a, 1: b}),
2544 ("ab\\x02", 3)
2545 )
2546
2547 self.assertEqual(
2548 codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
2549 {0: a, 1: b, 2: 0xFFFE}),
2550 ("ab\\x02", 3)
2551 )
2552
2553 self.assertEqual(
2554 codecs.charmap_decode(b"\x00\x01\x02", "ignore",
2555 {0: a, 1: b}),
2556 ("ab", 3)
2557 )
2558
2559 self.assertEqual(
2560 codecs.charmap_decode(b"\x00\x01\x02", "ignore",
2561 {0: a, 1: b, 2: 0xFFFE}),
2562 ("ab", 3)
2563 )
2564
2565
2566 class WithStmtTest(unittest.TestCase):
2567 def test_encodedfile(self):
2568 f = io.BytesIO(b"\xc3\xbc")
2569 with codecs.EncodedFile(f, "latin-1", "utf-8") as ef:
2570 self.assertEqual(ef.read(), b"\xfc")
2571 self.assertTrue(f.closed)
2572
2573 def test_streamreaderwriter(self):
2574 f = io.BytesIO(b"\xc3\xbc")
2575 info = codecs.lookup("utf-8")
2576 with codecs.StreamReaderWriter(f, info.streamreader,
2577 info.streamwriter, 'strict') as srw:
2578 self.assertEqual(srw.read(), "\xfc")
2579
2580
2581 class TypesTest(unittest.TestCase):
2582 def test_decode_unicode(self):
2583 # Most decoders don't accept unicode input
2584 decoders = [
2585 codecs.utf_7_decode,
2586 codecs.utf_8_decode,
2587 codecs.utf_16_le_decode,
2588 codecs.utf_16_be_decode,
2589 codecs.utf_16_ex_decode,
2590 codecs.utf_32_decode,
2591 codecs.utf_32_le_decode,
2592 codecs.utf_32_be_decode,
2593 codecs.utf_32_ex_decode,
2594 codecs.latin_1_decode,
2595 codecs.ascii_decode,
2596 codecs.charmap_decode,
2597 ]
2598 if hasattr(codecs, "mbcs_decode"):
2599 decoders.append(codecs.mbcs_decode)
2600 for decoder in decoders:
2601 self.assertRaises(TypeError, decoder, "xxx")
2602
2603 def test_unicode_escape(self):
2604 # Escape-decoding a unicode string is supported and gives the same
2605 # result as decoding the equivalent ASCII bytes string.
2606 self.assertEqual(codecs.unicode_escape_decode(r"\u1234"), ("\u1234", 6))
2607 self.assertEqual(codecs.unicode_escape_decode(br"\u1234"), ("\u1234", 6))
2608 self.assertEqual(codecs.raw_unicode_escape_decode(r"\u1234"), ("\u1234", 6))
2609 self.assertEqual(codecs.raw_unicode_escape_decode(br"\u1234"), ("\u1234", 6))
2610
2611 self.assertRaises(UnicodeDecodeError, codecs.unicode_escape_decode, br"\U00110000")
2612 self.assertEqual(codecs.unicode_escape_decode(r"\U00110000", "replace"), ("\ufffd", 10))
2613 self.assertEqual(codecs.unicode_escape_decode(r"\U00110000", "backslashreplace"),
2614 (r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10))
2615
2616 self.assertRaises(UnicodeDecodeError, codecs.raw_unicode_escape_decode, br"\U00110000")
2617 self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000", "replace"), ("\ufffd", 10))
2618 self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000", "backslashreplace"),
2619 (r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10))
2620
2621
2622 class UnicodeEscapeTest(ReadTest, unittest.TestCase):
2623 encoding = "unicode-escape"
2624
2625 test_lone_surrogates = None
2626
2627 def test_empty(self):
2628 self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0))
2629 self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0))
2630
2631 def test_raw_encode(self):
2632 encode = codecs.unicode_escape_encode
2633 for b in range(32, 127):
2634 if b != b'\\'[0]:
2635 self.assertEqual(encode(chr(b)), (bytes([b]), 1))
2636
2637 def test_raw_decode(self):
2638 decode = codecs.unicode_escape_decode
2639 for b in range(256):
2640 if b != b'\\'[0]:
2641 self.assertEqual(decode(bytes([b]) + b'0'), (chr(b) + '0', 2))
2642
2643 def test_escape_encode(self):
2644 encode = codecs.unicode_escape_encode
2645 check = coding_checker(self, encode)
2646 check('\t', br'\t')
2647 check('\n', br'\n')
2648 check('\r', br'\r')
2649 check('\\', br'\\')
2650 for b in range(32):
2651 if chr(b) not in '\t\n\r':
2652 check(chr(b), ('\\x%02x' % b).encode())
2653 for b in range(127, 256):
2654 check(chr(b), ('\\x%02x' % b).encode())
2655 check('\u20ac', br'\u20ac')
2656 check('\U0001d120', br'\U0001d120')
2657
2658 def test_escape_decode(self):
2659 decode = codecs.unicode_escape_decode
2660 check = coding_checker(self, decode)
2661 check(b"[\\\n]", "[]")
2662 check(br'[\"]', '["]')
2663 check(br"[\']", "[']")
2664 check(br"[\\]", r"[\]")
2665 check(br"[\a]", "[\x07]")
2666 check(br"[\b]", "[\x08]")
2667 check(br"[\t]", "[\x09]")
2668 check(br"[\n]", "[\x0a]")
2669 check(br"[\v]", "[\x0b]")
2670 check(br"[\f]", "[\x0c]")
2671 check(br"[\r]", "[\x0d]")
2672 check(br"[\7]", "[\x07]")
2673 check(br"[\78]", "[\x078]")
2674 check(br"[\41]", "[!]")
2675 check(br"[\418]", "[!8]")
2676 check(br"[\101]", "[A]")
2677 check(br"[\1010]", "[A0]")
2678 check(br"[\x41]", "[A]")
2679 check(br"[\x410]", "[A0]")
2680 check(br"\u20ac", "\u20ac")
2681 check(br"\U0001d120", "\U0001d120")
2682
2683 def test_decode_warnings(self):
2684 decode = codecs.unicode_escape_decode
2685 check = coding_checker(self, decode)
2686 for i in range(97, 123):
2687 b = bytes([i])
2688 if b not in b'abfnrtuvx':
2689 with self.assertWarnsRegex(DeprecationWarning,
2690 r'"\\%c" is an invalid escape sequence' % i):
2691 check(b"\\" + b, "\\" + chr(i))
2692 if b.upper() not in b'UN':
2693 with self.assertWarnsRegex(DeprecationWarning,
2694 r'"\\%c" is an invalid escape sequence' % (i-32)):
2695 check(b"\\" + b.upper(), "\\" + chr(i-32))
2696 with self.assertWarnsRegex(DeprecationWarning,
2697 r'"\\8" is an invalid escape sequence'):
2698 check(br"\8", "\\8")
2699 with self.assertWarns(DeprecationWarning):
2700 check(br"\9", "\\9")
2701 with self.assertWarnsRegex(DeprecationWarning,
2702 r'"\\\xfa" is an invalid escape sequence') as cm:
2703 check(b"\\\xfa", "\\\xfa")
2704 for i in range(0o400, 0o1000):
2705 with self.assertWarnsRegex(DeprecationWarning,
2706 r'"\\%o" is an invalid octal escape sequence' % i):
2707 check(rb'\%o' % i, chr(i))
2708
2709 with self.assertWarnsRegex(DeprecationWarning,
2710 r'"\\z" is an invalid escape sequence'):
2711 self.assertEqual(decode(br'\x\z', 'ignore'), ('\\z', 4))
2712 with self.assertWarnsRegex(DeprecationWarning,
2713 r'"\\501" is an invalid octal escape sequence'):
2714 self.assertEqual(decode(br'\x\501', 'ignore'), ('\u0141', 6))
2715
2716 def test_decode_errors(self):
2717 decode = codecs.unicode_escape_decode
2718 for c, d in (b'x', 2), (b'u', 4), (b'U', 4):
2719 for i in range(d):
2720 self.assertRaises(UnicodeDecodeError, decode,
2721 b"\\" + c + b"0"*i)
2722 self.assertRaises(UnicodeDecodeError, decode,
2723 b"[\\" + c + b"0"*i + b"]")
2724 data = b"[\\" + c + b"0"*i + b"]\\" + c + b"0"*i
2725 self.assertEqual(decode(data, "ignore"), ("[]", len(data)))
2726 self.assertEqual(decode(data, "replace"),
2727 ("[\ufffd]\ufffd", len(data)))
2728 self.assertRaises(UnicodeDecodeError, decode, br"\U00110000")
2729 self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10))
2730 self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10))
2731
2732 def test_partial(self):
2733 self.check_partial(
2734 "\x00\t\n\r\\\xff\uffff\U00010000",
2735 [
2736 '',
2737 '',
2738 '',
2739 '\x00',
2740 '\x00',
2741 '\x00\t',
2742 '\x00\t',
2743 '\x00\t\n',
2744 '\x00\t\n',
2745 '\x00\t\n\r',
2746 '\x00\t\n\r',
2747 '\x00\t\n\r\\',
2748 '\x00\t\n\r\\',
2749 '\x00\t\n\r\\',
2750 '\x00\t\n\r\\',
2751 '\x00\t\n\r\\\xff',
2752 '\x00\t\n\r\\\xff',
2753 '\x00\t\n\r\\\xff',
2754 '\x00\t\n\r\\\xff',
2755 '\x00\t\n\r\\\xff',
2756 '\x00\t\n\r\\\xff',
2757 '\x00\t\n\r\\\xff\uffff',
2758 '\x00\t\n\r\\\xff\uffff',
2759 '\x00\t\n\r\\\xff\uffff',
2760 '\x00\t\n\r\\\xff\uffff',
2761 '\x00\t\n\r\\\xff\uffff',
2762 '\x00\t\n\r\\\xff\uffff',
2763 '\x00\t\n\r\\\xff\uffff',
2764 '\x00\t\n\r\\\xff\uffff',
2765 '\x00\t\n\r\\\xff\uffff',
2766 '\x00\t\n\r\\\xff\uffff',
2767 '\x00\t\n\r\\\xff\uffff\U00010000',
2768 ]
2769 )
2770
2771 class RawUnicodeEscapeTest(ReadTest, unittest.TestCase):
2772 encoding = "raw-unicode-escape"
2773
2774 test_lone_surrogates = None
2775
2776 def test_empty(self):
2777 self.assertEqual(codecs.raw_unicode_escape_encode(""), (b"", 0))
2778 self.assertEqual(codecs.raw_unicode_escape_decode(b""), ("", 0))
2779
2780 def test_raw_encode(self):
2781 encode = codecs.raw_unicode_escape_encode
2782 for b in range(256):
2783 self.assertEqual(encode(chr(b)), (bytes([b]), 1))
2784
2785 def test_raw_decode(self):
2786 decode = codecs.raw_unicode_escape_decode
2787 for b in range(256):
2788 self.assertEqual(decode(bytes([b]) + b'0'), (chr(b) + '0', 2))
2789
2790 def test_escape_encode(self):
2791 encode = codecs.raw_unicode_escape_encode
2792 check = coding_checker(self, encode)
2793 for b in range(256):
2794 if b not in b'uU':
2795 check('\\' + chr(b), b'\\' + bytes([b]))
2796 check('\u20ac', br'\u20ac')
2797 check('\U0001d120', br'\U0001d120')
2798
2799 def test_escape_decode(self):
2800 decode = codecs.raw_unicode_escape_decode
2801 check = coding_checker(self, decode)
2802 for b in range(256):
2803 if b not in b'uU':
2804 check(b'\\' + bytes([b]), '\\' + chr(b))
2805 check(br"\u20ac", "\u20ac")
2806 check(br"\U0001d120", "\U0001d120")
2807
2808 def test_decode_errors(self):
2809 decode = codecs.raw_unicode_escape_decode
2810 for c, d in (b'u', 4), (b'U', 4):
2811 for i in range(d):
2812 self.assertRaises(UnicodeDecodeError, decode,
2813 b"\\" + c + b"0"*i)
2814 self.assertRaises(UnicodeDecodeError, decode,
2815 b"[\\" + c + b"0"*i + b"]")
2816 data = b"[\\" + c + b"0"*i + b"]\\" + c + b"0"*i
2817 self.assertEqual(decode(data, "ignore"), ("[]", len(data)))
2818 self.assertEqual(decode(data, "replace"),
2819 ("[\ufffd]\ufffd", len(data)))
2820 self.assertRaises(UnicodeDecodeError, decode, br"\U00110000")
2821 self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10))
2822 self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10))
2823
2824 def test_partial(self):
2825 self.check_partial(
2826 "\x00\t\n\r\\\xff\uffff\U00010000",
2827 [
2828 '\x00',
2829 '\x00\t',
2830 '\x00\t\n',
2831 '\x00\t\n\r',
2832 '\x00\t\n\r',
2833 '\x00\t\n\r\\\xff',
2834 '\x00\t\n\r\\\xff',
2835 '\x00\t\n\r\\\xff',
2836 '\x00\t\n\r\\\xff',
2837 '\x00\t\n\r\\\xff',
2838 '\x00\t\n\r\\\xff',
2839 '\x00\t\n\r\\\xff\uffff',
2840 '\x00\t\n\r\\\xff\uffff',
2841 '\x00\t\n\r\\\xff\uffff',
2842 '\x00\t\n\r\\\xff\uffff',
2843 '\x00\t\n\r\\\xff\uffff',
2844 '\x00\t\n\r\\\xff\uffff',
2845 '\x00\t\n\r\\\xff\uffff',
2846 '\x00\t\n\r\\\xff\uffff',
2847 '\x00\t\n\r\\\xff\uffff',
2848 '\x00\t\n\r\\\xff\uffff',
2849 '\x00\t\n\r\\\xff\uffff\U00010000',
2850 ]
2851 )
2852
2853
2854 class EscapeEncodeTest(unittest.TestCase):
2855
2856 def test_escape_encode(self):
2857 tests = [
2858 (b'', (b'', 0)),
2859 (b'foobar', (b'foobar', 6)),
2860 (b'spam\0eggs', (b'spam\\x00eggs', 9)),
2861 (b'a\'b', (b"a\\'b", 3)),
2862 (b'b\\c', (b'b\\\\c', 3)),
2863 (b'c\nd', (b'c\\nd', 3)),
2864 (b'd\re', (b'd\\re', 3)),
2865 (b'f\x7fg', (b'f\\x7fg', 3)),
2866 ]
2867 for data, output in tests:
2868 with self.subTest(data=data):
2869 self.assertEqual(codecs.escape_encode(data), output)
2870 self.assertRaises(TypeError, codecs.escape_encode, 'spam')
2871 self.assertRaises(TypeError, codecs.escape_encode, bytearray(b'spam'))
2872
2873
2874 class SurrogateEscapeTest(unittest.TestCase):
2875
2876 def test_utf8(self):
2877 # Bad byte
2878 self.assertEqual(b"foo\x80bar".decode("utf-8", "surrogateescape"),
2879 "foo\udc80bar")
2880 self.assertEqual("foo\udc80bar".encode("utf-8", "surrogateescape"),
2881 b"foo\x80bar")
2882 # bad-utf-8 encoded surrogate
2883 self.assertEqual(b"\xed\xb0\x80".decode("utf-8", "surrogateescape"),
2884 "\udced\udcb0\udc80")
2885 self.assertEqual("\udced\udcb0\udc80".encode("utf-8", "surrogateescape"),
2886 b"\xed\xb0\x80")
2887
2888 def test_ascii(self):
2889 # bad byte
2890 self.assertEqual(b"foo\x80bar".decode("ascii", "surrogateescape"),
2891 "foo\udc80bar")
2892 self.assertEqual("foo\udc80bar".encode("ascii", "surrogateescape"),
2893 b"foo\x80bar")
2894
2895 def test_charmap(self):
2896 # bad byte: \xa5 is unmapped in iso-8859-3
2897 self.assertEqual(b"foo\xa5bar".decode("iso-8859-3", "surrogateescape"),
2898 "foo\udca5bar")
2899 self.assertEqual("foo\udca5bar".encode("iso-8859-3", "surrogateescape"),
2900 b"foo\xa5bar")
2901
2902 def test_latin1(self):
2903 # Issue6373
2904 self.assertEqual("\udce4\udceb\udcef\udcf6\udcfc".encode("latin-1", "surrogateescape"),
2905 b"\xe4\xeb\xef\xf6\xfc")
2906
2907
2908 class BomTest(unittest.TestCase):
2909 def test_seek0(self):
2910 data = "1234567890"
2911 tests = ("utf-16",
2912 "utf-16-le",
2913 "utf-16-be",
2914 "utf-32",
2915 "utf-32-le",
2916 "utf-32-be")
2917 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
2918 for encoding in tests:
2919 # Check if the BOM is written only once
2920 with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f:
2921 f.write(data)
2922 f.write(data)
2923 f.seek(0)
2924 self.assertEqual(f.read(), data * 2)
2925 f.seek(0)
2926 self.assertEqual(f.read(), data * 2)
2927
2928 # Check that the BOM is written after a seek(0)
2929 with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f:
2930 f.write(data[0])
2931 self.assertNotEqual(f.tell(), 0)
2932 f.seek(0)
2933 f.write(data)
2934 f.seek(0)
2935 self.assertEqual(f.read(), data)
2936
2937 # (StreamWriter) Check that the BOM is written after a seek(0)
2938 with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f:
2939 f.writer.write(data[0])
2940 self.assertNotEqual(f.writer.tell(), 0)
2941 f.writer.seek(0)
2942 f.writer.write(data)
2943 f.seek(0)
2944 self.assertEqual(f.read(), data)
2945
2946 # Check that the BOM is not written after a seek() at a position
2947 # different than the start
2948 with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f:
2949 f.write(data)
2950 f.seek(f.tell())
2951 f.write(data)
2952 f.seek(0)
2953 self.assertEqual(f.read(), data * 2)
2954
2955 # (StreamWriter) Check that the BOM is not written after a seek()
2956 # at a position different than the start
2957 with codecs_open_no_warn(os_helper.TESTFN, 'w+', encoding=encoding) as f:
2958 f.writer.write(data)
2959 f.writer.seek(f.writer.tell())
2960 f.writer.write(data)
2961 f.seek(0)
2962 self.assertEqual(f.read(), data * 2)
2963
2964
2965 bytes_transform_encodings = [
2966 "base64_codec",
2967 "uu_codec",
2968 "quopri_codec",
2969 "hex_codec",
2970 ]
2971
2972 transform_aliases = {
2973 "base64_codec": ["base64", "base_64"],
2974 "uu_codec": ["uu"],
2975 "quopri_codec": ["quopri", "quoted_printable", "quotedprintable"],
2976 "hex_codec": ["hex"],
2977 "rot_13": ["rot13"],
2978 }
2979
2980 try:
2981 import zlib
2982 except ImportError:
2983 zlib = None
2984 else:
2985 bytes_transform_encodings.append("zlib_codec")
2986 transform_aliases["zlib_codec"] = ["zip", "zlib"]
2987 try:
2988 import bz2 # noqa: F401
2989 except ImportError:
2990 pass
2991 else:
2992 bytes_transform_encodings.append("bz2_codec")
2993 transform_aliases["bz2_codec"] = ["bz2"]
2994
2995
2996 class TransformCodecTest(unittest.TestCase):
2997
2998 def test_basics(self):
2999 binput = bytes(range(256))
3000 for encoding in bytes_transform_encodings:
3001 with self.subTest(encoding=encoding):
3002 # generic codecs interface
3003 (o, size) = codecs.getencoder(encoding)(binput)
3004 self.assertEqual(size, len(binput))
3005 (i, size) = codecs.getdecoder(encoding)(o)
3006 self.assertEqual(size, len(o))
3007 self.assertEqual(i, binput)
3008
3009 def test_read(self):
3010 for encoding in bytes_transform_encodings:
3011 with self.subTest(encoding=encoding):
3012 sin = codecs.encode(b"\x80", encoding)
3013 reader = codecs.getreader(encoding)(io.BytesIO(sin))
3014 sout = reader.read()
3015 self.assertEqual(sout, b"\x80")
3016
3017 def test_readline(self):
3018 for encoding in bytes_transform_encodings:
3019 with self.subTest(encoding=encoding):
3020 sin = codecs.encode(b"\x80", encoding)
3021 reader = codecs.getreader(encoding)(io.BytesIO(sin))
3022 sout = reader.readline()
3023 self.assertEqual(sout, b"\x80")
3024
3025 def test_buffer_api_usage(self):
3026 # We check all the transform codecs accept memoryview input
3027 # for encoding and decoding
3028 # and also that they roundtrip correctly
3029 original = b"12345\x80"
3030 for encoding in bytes_transform_encodings:
3031 with self.subTest(encoding=encoding):
3032 data = original
3033 view = memoryview(data)
3034 data = codecs.encode(data, encoding)
3035 view_encoded = codecs.encode(view, encoding)
3036 self.assertEqual(view_encoded, data)
3037 view = memoryview(data)
3038 data = codecs.decode(data, encoding)
3039 self.assertEqual(data, original)
3040 view_decoded = codecs.decode(view, encoding)
3041 self.assertEqual(view_decoded, data)
3042
3043 def test_text_to_binary_denylists_binary_transforms(self):
3044 # Check binary -> binary codecs give a good error for str input
3045 bad_input = "bad input type"
3046 for encoding in bytes_transform_encodings:
3047 with self.subTest(encoding=encoding):
3048 fmt = (r"{!r} is not a text encoding; "
3049 r"use codecs.encode\(\) to handle arbitrary codecs")
3050 msg = fmt.format(encoding)
3051 with self.assertRaisesRegex(LookupError, msg) as failure:
3052 bad_input.encode(encoding)
3053 self.assertIsNone(failure.exception.__cause__)
3054
3055 def test_text_to_binary_denylists_text_transforms(self):
3056 # Check str.encode gives a good error message for str -> str codecs
3057 msg = (r"^'rot_13' is not a text encoding; "
3058 r"use codecs.encode\(\) to handle arbitrary codecs")
3059 with self.assertRaisesRegex(LookupError, msg):
3060 "just an example message".encode("rot_13")
3061
3062 def test_binary_to_text_denylists_binary_transforms(self):
3063 # Check bytes.decode and bytearray.decode give a good error
3064 # message for binary -> binary codecs
3065 data = b"encode first to ensure we meet any format restrictions"
3066 for encoding in bytes_transform_encodings:
3067 with self.subTest(encoding=encoding):
3068 encoded_data = codecs.encode(data, encoding)
3069 fmt = (r"{!r} is not a text encoding; "
3070 r"use codecs.decode\(\) to handle arbitrary codecs")
3071 msg = fmt.format(encoding)
3072 with self.assertRaisesRegex(LookupError, msg):
3073 encoded_data.decode(encoding)
3074 with self.assertRaisesRegex(LookupError, msg):
3075 bytearray(encoded_data).decode(encoding)
3076
3077 def test_binary_to_text_denylists_text_transforms(self):
3078 # Check str -> str codec gives a good error for binary input
3079 for bad_input in (b"immutable", bytearray(b"mutable")):
3080 with self.subTest(bad_input=bad_input):
3081 msg = (r"^'rot_13' is not a text encoding; "
3082 r"use codecs.decode\(\) to handle arbitrary codecs")
3083 with self.assertRaisesRegex(LookupError, msg) as failure:
3084 bad_input.decode("rot_13")
3085 self.assertIsNone(failure.exception.__cause__)
3086
3087 @unittest.skipUnless(zlib, "Requires zlib support")
3088 def test_custom_zlib_error_is_noted(self):
3089 # Check zlib codec gives a good error for malformed input
3090 msg = "decoding with 'zlib_codec' codec failed"
3091 with self.assertRaises(zlib.error) as failure:
3092 codecs.decode(b"hello", "zlib_codec")
3093 self.assertEqual(msg, failure.exception.__notes__[0])
3094
3095 def test_custom_hex_error_is_noted(self):
3096 # Check hex codec gives a good error for malformed input
3097 import binascii
3098 msg = "decoding with 'hex_codec' codec failed"
3099 with self.assertRaises(binascii.Error) as failure:
3100 codecs.decode(b"hello", "hex_codec")
3101 self.assertEqual(msg, failure.exception.__notes__[0])
3102
3103 # Ensure codec aliases from http://bugs.python.org/issue7475 work
3104 def test_aliases(self):
3105 for codec_name, aliases in transform_aliases.items():
3106 expected_name = codecs.lookup(codec_name).name
3107 for alias in aliases:
3108 with self.subTest(alias=alias):
3109 info = codecs.lookup(alias)
3110 self.assertEqual(info.name, expected_name)
3111
3112 def test_alias_modules_exist(self):
3113 encodings_dir = os.path.dirname(encodings.__file__)
3114 for value in encodings.aliases.aliases.values():
3115 codec_mod = f"encodings.{value}"
3116 self.assertIsNotNone(importlib.util.find_spec(codec_mod),
3117 f"Codec module not found: {codec_mod}")
3118
3119 def test_quopri_stateless(self):
3120 # Should encode with quotetabs=True
3121 encoded = codecs.encode(b"space tab\teol \n", "quopri-codec")
3122 self.assertEqual(encoded, b"space=20tab=09eol=20\n")
3123 # But should still support unescaped tabs and spaces
3124 unescaped = b"space tab eol\n"
3125 self.assertEqual(codecs.decode(unescaped, "quopri-codec"), unescaped)
3126
3127 def test_uu_invalid(self):
3128 # Missing "begin" line
3129 self.assertRaises(ValueError, codecs.decode, b"", "uu-codec")
3130
3131
3132 # The codec system tries to add notes to exceptions in order to ensure
3133 # the error mentions the operation being performed and the codec involved.
3134
3135 # Use a local codec registry to avoid appearing to leak objects when
3136 # registering multiple search functions
3137 _TEST_CODECS = {}
3138
3139 def _get_test_codec(codec_name):
3140 return _TEST_CODECS.get(codec_name)
3141
3142
3143 class ExceptionNotesTest(unittest.TestCase):
3144
3145 def setUp(self):
3146 self.codec_name = 'exception_notes_test'
3147 codecs.register(_get_test_codec)
3148 self.addCleanup(codecs.unregister, _get_test_codec)
3149
3150 # We store the object to raise on the instance because of a bad
3151 # interaction between the codec caching (which means we can't
3152 # recreate the codec entry) and regrtest refleak hunting (which
3153 # runs the same test instance multiple times). This means we
3154 # need to ensure the codecs call back in to the instance to find
3155 # out which exception to raise rather than binding them in a
3156 # closure to an object that may change on the next run
3157 self.obj_to_raise = RuntimeError
3158
3159 def tearDown(self):
3160 _TEST_CODECS.pop(self.codec_name, None)
3161 # Issue #22166: Also pop from caches to avoid appearance of ref leaks
3162 encodings._cache.pop(self.codec_name, None)
3163
3164 def set_codec(self, encode, decode):
3165 codec_info = codecs.CodecInfo(encode, decode,
3166 name=self.codec_name)
3167 _TEST_CODECS[self.codec_name] = codec_info
3168
3169 @contextlib.contextmanager
3170 def assertNoted(self, operation, exc_type, msg):
3171 full_msg = r"{} with {!r} codec failed".format(
3172 operation, self.codec_name)
3173 with self.assertRaises(exc_type) as caught:
3174 yield caught
3175 self.assertIn(full_msg, caught.exception.__notes__[0])
3176 caught.exception.__notes__.clear()
3177
3178 def raise_obj(self, *args, **kwds):
3179 # Helper to dynamically change the object raised by a test codec
3180 raise self.obj_to_raise
3181
3182 def check_note(self, obj_to_raise, msg, exc_type=RuntimeError):
3183 self.obj_to_raise = obj_to_raise
3184 self.set_codec(self.raise_obj, self.raise_obj)
3185 with self.assertNoted("encoding", exc_type, msg):
3186 "str_input".encode(self.codec_name)
3187 with self.assertNoted("encoding", exc_type, msg):
3188 codecs.encode("str_input", self.codec_name)
3189 with self.assertNoted("decoding", exc_type, msg):
3190 b"bytes input".decode(self.codec_name)
3191 with self.assertNoted("decoding", exc_type, msg):
3192 codecs.decode(b"bytes input", self.codec_name)
3193
3194 def test_raise_by_type(self):
3195 self.check_note(RuntimeError, "")
3196
3197 def test_raise_by_value(self):
3198 msg = "This should be noted"
3199 self.check_note(RuntimeError(msg), msg)
3200
3201 def test_raise_grandchild_subclass_exact_size(self):
3202 msg = "This should be noted"
3203 class MyRuntimeError(RuntimeError):
3204 __slots__ = ()
3205 self.check_note(MyRuntimeError(msg), msg, MyRuntimeError)
3206
3207 def test_raise_subclass_with_weakref_support(self):
3208 msg = "This should be noted"
3209 class MyRuntimeError(RuntimeError):
3210 pass
3211 self.check_note(MyRuntimeError(msg), msg, MyRuntimeError)
3212
3213 def test_init_override(self):
3214 class CustomInit(RuntimeError):
3215 def __init__(self):
3216 pass
3217 self.check_note(CustomInit, "")
3218
3219 def test_new_override(self):
3220 class CustomNew(RuntimeError):
3221 def __new__(cls):
3222 return super().__new__(cls)
3223 self.check_note(CustomNew, "")
3224
3225 def test_instance_attribute(self):
3226 msg = "This should be noted"
3227 exc = RuntimeError(msg)
3228 exc.attr = 1
3229 self.check_note(exc, "^{}$".format(msg))
3230
3231 def test_non_str_arg(self):
3232 self.check_note(RuntimeError(1), "1")
3233
3234 def test_multiple_args(self):
3235 msg_re = r"^\('a', 'b', 'c'\)$"
3236 self.check_note(RuntimeError('a', 'b', 'c'), msg_re)
3237
3238 # http://bugs.python.org/issue19609
3239 def test_codec_lookup_failure(self):
3240 msg = "^unknown encoding: {}$".format(self.codec_name)
3241 with self.assertRaisesRegex(LookupError, msg):
3242 "str input".encode(self.codec_name)
3243 with self.assertRaisesRegex(LookupError, msg):
3244 codecs.encode("str input", self.codec_name)
3245 with self.assertRaisesRegex(LookupError, msg):
3246 b"bytes input".decode(self.codec_name)
3247 with self.assertRaisesRegex(LookupError, msg):
3248 codecs.decode(b"bytes input", self.codec_name)
3249
3250 def test_unflagged_non_text_codec_handling(self):
3251 # The stdlib non-text codecs are now marked so they're
3252 # pre-emptively skipped by the text model related methods
3253 # However, third party codecs won't be flagged, so we still make
3254 # sure the case where an inappropriate output type is produced is
3255 # handled appropriately
3256 def encode_to_str(*args, **kwds):
3257 return "not bytes!", 0
3258 def decode_to_bytes(*args, **kwds):
3259 return b"not str!", 0
3260 self.set_codec(encode_to_str, decode_to_bytes)
3261 # No input or output type checks on the codecs module functions
3262 encoded = codecs.encode(None, self.codec_name)
3263 self.assertEqual(encoded, "not bytes!")
3264 decoded = codecs.decode(None, self.codec_name)
3265 self.assertEqual(decoded, b"not str!")
3266 # Text model methods should complain
3267 fmt = (r"^{!r} encoder returned 'str' instead of 'bytes'; "
3268 r"use codecs.encode\(\) to encode to arbitrary types$")
3269 msg = fmt.format(self.codec_name)
3270 with self.assertRaisesRegex(TypeError, msg):
3271 "str_input".encode(self.codec_name)
3272 fmt = (r"^{!r} decoder returned 'bytes' instead of 'str'; "
3273 r"use codecs.decode\(\) to decode to arbitrary types$")
3274 msg = fmt.format(self.codec_name)
3275 with self.assertRaisesRegex(TypeError, msg):
3276 b"bytes input".decode(self.codec_name)
3277
3278
3279
3280 @unittest.skipUnless(sys.platform == 'win32',
3281 'code pages are specific to Windows')
3282 class CodePageTest(unittest.TestCase):
3283 CP_UTF8 = 65001
3284
3285 def test_invalid_code_page(self):
3286 self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a')
3287 self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a')
3288 self.assertRaises(OSError, codecs.code_page_encode, 123, 'a')
3289 self.assertRaises(OSError, codecs.code_page_decode, 123, b'a')
3290
3291 def test_code_page_name(self):
3292 self.assertRaisesRegex(UnicodeEncodeError, 'cp932',
3293 codecs.code_page_encode, 932, '\xff')
3294 self.assertRaisesRegex(UnicodeDecodeError, 'cp932',
3295 codecs.code_page_decode, 932, b'\x81\x00', 'strict', True)
3296 self.assertRaisesRegex(UnicodeDecodeError, 'cp65001',
3297 codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True)
3298
3299 def check_decode(self, cp, tests):
3300 for raw, errors, expected, *rest in tests:
3301 if rest:
3302 altexpected, = rest
3303 else:
3304 altexpected = expected
3305 if expected is not None:
3306 try:
3307 decoded = codecs.code_page_decode(cp, raw, errors, True)
3308 except UnicodeDecodeError as err:
3309 self.fail('Unable to decode %a from "cp%s" with '
3310 'errors=%r: %s' % (raw, cp, errors, err))
3311 self.assertEqual(decoded[0], expected,
3312 '%a.decode("cp%s", %r)=%a != %a'
3313 % (raw, cp, errors, decoded[0], expected))
3314 # assert 0 <= decoded[1] <= len(raw)
3315 self.assertGreaterEqual(decoded[1], 0)
3316 self.assertLessEqual(decoded[1], len(raw))
3317 else:
3318 self.assertRaises(UnicodeDecodeError,
3319 codecs.code_page_decode, cp, raw, errors, True)
3320
3321 if altexpected is not None:
3322 decoded = raw.decode(f'cp{cp}', errors)
3323 self.assertEqual(decoded, altexpected,
3324 '%a.decode("cp%s", %r)=%a != %a'
3325 % (raw, cp, errors, decoded, altexpected))
3326 else:
3327 self.assertRaises(UnicodeDecodeError,
3328 raw.decode, f'cp{cp}', errors)
3329
3330 def check_encode(self, cp, tests):
3331 for text, errors, expected, *rest in tests:
3332 if rest:
3333 altexpected, = rest
3334 else:
3335 altexpected = expected
3336 if expected is not None:
3337 try:
3338 encoded = codecs.code_page_encode(cp, text, errors)
3339 except UnicodeEncodeError as err:
3340 self.fail('Unable to encode %a to "cp%s" with '
3341 'errors=%r: %s' % (text, cp, errors, err))
3342 self.assertEqual(encoded[0], expected,
3343 '%a.encode("cp%s", %r)=%a != %a'
3344 % (text, cp, errors, encoded[0], expected))
3345 self.assertEqual(encoded[1], len(text))
3346
3347 encoded = text.encode(f'cp{cp}', errors)
3348 self.assertEqual(encoded, altexpected,
3349 '%a.encode("cp%s", %r)=%a != %a'
3350 % (text, cp, errors, encoded, altexpected))
3351 else:
3352 self.assertRaises(UnicodeEncodeError,
3353 codecs.code_page_encode, cp, text, errors)
3354 self.assertRaises(UnicodeEncodeError,
3355 text.encode, f'cp{cp}', errors)
3356
3357 def test_cp932(self):
3358 self.check_encode(932, (
3359 ('abc', 'strict', b'abc'),
3360 ('\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'),
3361 ('\uf8f3', 'strict', b'\xff'),
3362 # test error handlers
3363 ('\xff', 'strict', None),
3364 ('[\xff]', 'ignore', b'[]'),
3365 ('[\xff]', 'replace', b'[y]', b'[?]'),
3366 ('[\u20ac]', 'replace', b'[?]'),
3367 ('[\xff]', 'backslashreplace', b'[\\xff]'),
3368 ('[\xff]', 'namereplace',
3369 b'[\\N{LATIN SMALL LETTER Y WITH DIAERESIS}]'),
3370 ('[\xff]', 'xmlcharrefreplace', b'[&#255;]'),
3371 ('\udcff', 'strict', None),
3372 ('[\udcff]', 'surrogateescape', b'[\xff]'),
3373 ('[\udcff]', 'surrogatepass', None),
3374 ))
3375 self.check_decode(932, (
3376 (b'abc', 'strict', 'abc'),
3377 (b'\x82\x84\xe9\x80', 'strict', '\uff44\u9a3e'),
3378 # invalid bytes
3379 (b'[\xff]', 'strict', None, '[\uf8f3]'),
3380 (b'[\xff]', 'ignore', '[]', '[\uf8f3]'),
3381 (b'[\xff]', 'replace', '[\ufffd]', '[\uf8f3]'),
3382 (b'[\xff]', 'backslashreplace', '[\\xff]', '[\uf8f3]'),
3383 (b'[\xff]', 'surrogateescape', '[\udcff]', '[\uf8f3]'),
3384 (b'[\xff]', 'surrogatepass', None, '[\uf8f3]'),
3385 (b'\x81\x00abc', 'strict', None),
3386 (b'\x81\x00abc', 'ignore', '\x00abc'),
3387 (b'\x81\x00abc', 'replace', '\ufffd\x00abc'),
3388 (b'\x81\x00abc', 'backslashreplace', '\\x81\x00abc'),
3389 ))
3390
3391 def test_cp1252(self):
3392 self.check_encode(1252, (
3393 ('abc', 'strict', b'abc'),
3394 ('\xe9\u20ac', 'strict', b'\xe9\x80'),
3395 ('\xff', 'strict', b'\xff'),
3396 # test error handlers
3397 ('\u0141', 'strict', None),
3398 ('\u0141', 'ignore', b''),
3399 ('\u0141', 'replace', b'L', b'?'),
3400 ('\udc98', 'surrogateescape', b'\x98'),
3401 ('\udc98', 'surrogatepass', None),
3402 ))
3403 self.check_decode(1252, (
3404 (b'abc', 'strict', 'abc'),
3405 (b'\xe9\x80', 'strict', '\xe9\u20ac'),
3406 (b'\xff', 'strict', '\xff'),
3407 ))
3408
3409 def test_cp708(self):
3410 self.check_encode(708, (
3411 ('abc2%', 'strict', b'abc2%'),
3412 ('\u060c\u0621\u064a', 'strict', b'\xac\xc1\xea'),
3413 ('\u2562\xe7\xa0', 'strict', b'\x86\x87\xff'),
3414 ('\x9a\x9f', 'strict', b'\x9a\x9f'),
3415 ('\u256b', 'strict', b'\xc0'),
3416 # test error handlers
3417 ('[\u0662]', 'strict', None),
3418 ('[\u0662]', 'ignore', b'[]'),
3419 ('[\u0662]', 'replace', b'[?]'),
3420 ('\udca0', 'surrogateescape', b'\xa0'),
3421 ('\udca0', 'surrogatepass', None),
3422 ))
3423 self.check_decode(708, (
3424 (b'abc2%', 'strict', 'abc2%'),
3425 (b'\xac\xc1\xea', 'strict', '\u060c\u0621\u064a'),
3426 (b'\x86\x87\xff', 'strict', '\u2562\xe7\xa0'),
3427 (b'\x9a\x9f', 'strict', '\x9a\x9f'),
3428 (b'\xc0', 'strict', '\u256b'),
3429 # test error handlers
3430 (b'\xa0', 'strict', None),
3431 (b'[\xa0]', 'ignore', '[]'),
3432 (b'[\xa0]', 'replace', '[\ufffd]'),
3433 (b'[\xa0]', 'backslashreplace', '[\\xa0]'),
3434 (b'[\xa0]', 'surrogateescape', '[\udca0]'),
3435 (b'[\xa0]', 'surrogatepass', None),
3436 ))
3437
3438 def test_cp20106(self):
3439 self.check_encode(20106, (
3440 ('abc', 'strict', b'abc'),
3441 ('\xa7\xc4\xdf', 'strict', b'@[~'),
3442 # test error handlers
3443 ('@', 'strict', None),
3444 ('@', 'ignore', b''),
3445 ('@', 'replace', b'?'),
3446 ('\udcbf', 'surrogateescape', b'\xbf'),
3447 ('\udcbf', 'surrogatepass', None),
3448 ))
3449 self.check_decode(20106, (
3450 (b'abc', 'strict', 'abc'),
3451 (b'@[~', 'strict', '\xa7\xc4\xdf'),
3452 (b'\xe1\xfe', 'strict', 'a\xdf'),
3453 # test error handlers
3454 (b'(\xbf)', 'strict', None),
3455 (b'(\xbf)', 'ignore', '()'),
3456 (b'(\xbf)', 'replace', '(\ufffd)'),
3457 (b'(\xbf)', 'backslashreplace', '(\\xbf)'),
3458 (b'(\xbf)', 'surrogateescape', '(\udcbf)'),
3459 (b'(\xbf)', 'surrogatepass', None),
3460 ))
3461
3462 def test_cp_utf7(self):
3463 cp = 65000
3464 self.check_encode(cp, (
3465 ('abc', 'strict', b'abc'),
3466 ('\xe9\u20ac', 'strict', b'+AOkgrA-'),
3467 ('\U0010ffff', 'strict', b'+2//f/w-'),
3468 ('\udc80', 'strict', b'+3IA-'),
3469 ('\ufffd', 'strict', b'+//0-'),
3470 ))
3471 self.check_decode(cp, (
3472 (b'abc', 'strict', 'abc'),
3473 (b'+AOkgrA-', 'strict', '\xe9\u20ac'),
3474 (b'+2//f/w-', 'strict', '\U0010ffff'),
3475 (b'+3IA-', 'strict', '\udc80'),
3476 (b'+//0-', 'strict', '\ufffd'),
3477 # invalid bytes
3478 (b'[+/]', 'strict', '[]'),
3479 (b'[\xff]', 'strict', '[\xff]'),
3480 ))
3481
3482 def test_multibyte_encoding(self):
3483 self.check_decode(932, (
3484 (b'\x84\xe9\x80', 'ignore', '\u9a3e'),
3485 (b'\x84\xe9\x80', 'replace', '\ufffd\u9a3e'),
3486 ))
3487 self.check_decode(self.CP_UTF8, (
3488 (b'\xff\xf4\x8f\xbf\xbf', 'ignore', '\U0010ffff'),
3489 (b'\xff\xf4\x8f\xbf\xbf', 'replace', '\ufffd\U0010ffff'),
3490 ))
3491 self.check_encode(self.CP_UTF8, (
3492 ('[\U0010ffff\uDC80]', 'ignore', b'[\xf4\x8f\xbf\xbf]'),
3493 ('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'),
3494 ))
3495
3496 def test_code_page_decode_flags(self):
3497 # Issue #36312: For some code pages (e.g. UTF-7) flags for
3498 # MultiByteToWideChar() must be set to 0.
3499 if support.verbose:
3500 sys.stdout.write('\n')
3501 for cp in (50220, 50221, 50222, 50225, 50227, 50229,
3502 *range(57002, 57011+1), 65000):
3503 # On small versions of Windows like Windows IoT
3504 # not all codepages are present.
3505 # A missing codepage causes an OSError exception
3506 # so check for the codepage before decoding
3507 if is_code_page_present(cp):
3508 self.assertEqual(codecs.code_page_decode(cp, b'abc'), ('abc', 3), f'cp{cp}')
3509 else:
3510 if support.verbose:
3511 print(f" skipping cp={cp}")
3512 self.assertEqual(codecs.code_page_decode(42, b'abc'),
3513 ('\uf061\uf062\uf063', 3))
3514
3515 def test_incremental(self):
3516 decoded = codecs.code_page_decode(932, b'\x82', 'strict', False)
3517 self.assertEqual(decoded, ('', 0))
3518
3519 decoded = codecs.code_page_decode(932,
3520 b'\xe9\x80\xe9', 'strict',
3521 False)
3522 self.assertEqual(decoded, ('\u9a3e', 2))
3523
3524 decoded = codecs.code_page_decode(932,
3525 b'\xe9\x80\xe9\x80', 'strict',
3526 False)
3527 self.assertEqual(decoded, ('\u9a3e\u9a3e', 4))
3528
3529 decoded = codecs.code_page_decode(932,
3530 b'abc', 'strict',
3531 False)
3532 self.assertEqual(decoded, ('abc', 3))
3533
3534 def test_mbcs_code_page(self):
3535 # Check that codec for the current Windows (ANSII) code page is
3536 # always available.
3537 try:
3538 from _winapi import GetACP
3539 except ImportError:
3540 self.skipTest('requires _winapi.GetACP')
3541 cp = GetACP()
3542 codecs.lookup(f'cp{cp}')
3543
3544 @support.bigmemtest(size=2**31, memuse=7, dry_run=False)
3545 def test_large_input(self, size):
3546 # Test input longer than INT_MAX.
3547 # Input should contain undecodable bytes before and after
3548 # the INT_MAX limit.
3549 encoded = (b'01234567' * ((size//8)-1) +
3550 b'\x85\x86\xea\xeb\xec\xef\xfc\xfd\xfe\xff')
3551 self.assertEqual(len(encoded), size+2)
3552 decoded = codecs.code_page_decode(932, encoded, 'surrogateescape', True)
3553 self.assertEqual(decoded[1], len(encoded))
3554 del encoded
3555 self.assertEqual(len(decoded[0]), decoded[1])
3556 self.assertEqual(decoded[0][:10], '0123456701')
3557 self.assertEqual(decoded[0][-20:],
3558 '6701234567'
3559 '\udc85\udc86\udcea\udceb\udcec'
3560 '\udcef\udcfc\udcfd\udcfe\udcff')
3561
3562 @support.bigmemtest(size=2**31, memuse=6, dry_run=False)
3563 def test_large_utf8_input(self, size):
3564 # Test input longer than INT_MAX.
3565 # Input should contain a decodable multi-byte character
3566 # surrounding INT_MAX
3567 encoded = (b'0123456\xed\x84\x80' * (size//8))
3568 self.assertEqual(len(encoded), size // 8 * 10)
3569 decoded = codecs.code_page_decode(65001, encoded, 'ignore', True)
3570 self.assertEqual(decoded[1], len(encoded))
3571 del encoded
3572 self.assertEqual(len(decoded[0]), size)
3573 self.assertEqual(decoded[0][:10], '0123456\ud10001')
3574 self.assertEqual(decoded[0][-11:], '56\ud1000123456\ud100')
3575
3576
3577 class ASCIITest(unittest.TestCase):
3578 def test_encode(self):
3579 self.assertEqual('abc123'.encode('ascii'), b'abc123')
3580
3581 def test_encode_error(self):
3582 for data, error_handler, expected in (
3583 ('[\x80\xff\u20ac]', 'ignore', b'[]'),
3584 ('[\x80\xff\u20ac]', 'replace', b'[???]'),
3585 ('[\x80\xff\u20ac]', 'xmlcharrefreplace', b'[&#128;&#255;&#8364;]'),
3586 ('[\x80\xff\u20ac\U000abcde]', 'backslashreplace',
3587 b'[\\x80\\xff\\u20ac\\U000abcde]'),
3588 ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'),
3589 ):
3590 with self.subTest(data=data, error_handler=error_handler,
3591 expected=expected):
3592 self.assertEqual(data.encode('ascii', error_handler),
3593 expected)
3594
3595 def test_encode_surrogateescape_error(self):
3596 with self.assertRaises(UnicodeEncodeError):
3597 # the first character can be decoded, but not the second
3598 '\udc80\xff'.encode('ascii', 'surrogateescape')
3599
3600 def test_decode(self):
3601 self.assertEqual(b'abc'.decode('ascii'), 'abc')
3602
3603 def test_decode_error(self):
3604 for data, error_handler, expected in (
3605 (b'[\x80\xff]', 'ignore', '[]'),
3606 (b'[\x80\xff]', 'replace', '[\ufffd\ufffd]'),
3607 (b'[\x80\xff]', 'surrogateescape', '[\udc80\udcff]'),
3608 (b'[\x80\xff]', 'backslashreplace', '[\\x80\\xff]'),
3609 ):
3610 with self.subTest(data=data, error_handler=error_handler,
3611 expected=expected):
3612 self.assertEqual(data.decode('ascii', error_handler),
3613 expected)
3614
3615
3616 class Latin1Test(unittest.TestCase):
3617 def test_encode(self):
3618 for data, expected in (
3619 ('abc', b'abc'),
3620 ('\x80\xe9\xff', b'\x80\xe9\xff'),
3621 ):
3622 with self.subTest(data=data, expected=expected):
3623 self.assertEqual(data.encode('latin1'), expected)
3624
3625 def test_encode_errors(self):
3626 for data, error_handler, expected in (
3627 ('[\u20ac\udc80]', 'ignore', b'[]'),
3628 ('[\u20ac\udc80]', 'replace', b'[??]'),
3629 ('[\u20ac\U000abcde]', 'backslashreplace',
3630 b'[\\u20ac\\U000abcde]'),
3631 ('[\u20ac\udc80]', 'xmlcharrefreplace', b'[&#8364;&#56448;]'),
3632 ('[\udc80\udcff]', 'surrogateescape', b'[\x80\xff]'),
3633 ):
3634 with self.subTest(data=data, error_handler=error_handler,
3635 expected=expected):
3636 self.assertEqual(data.encode('latin1', error_handler),
3637 expected)
3638
3639 def test_encode_surrogateescape_error(self):
3640 with self.assertRaises(UnicodeEncodeError):
3641 # the first character can be decoded, but not the second
3642 '\udc80\u20ac'.encode('latin1', 'surrogateescape')
3643
3644 def test_decode(self):
3645 for data, expected in (
3646 (b'abc', 'abc'),
3647 (b'[\x80\xff]', '[\x80\xff]'),
3648 ):
3649 with self.subTest(data=data, expected=expected):
3650 self.assertEqual(data.decode('latin1'), expected)
3651
3652
3653 class StreamRecoderTest(unittest.TestCase):
3654 def test_writelines(self):
3655 bio = io.BytesIO()
3656 codec = codecs.lookup('ascii')
3657 sr = codecs.StreamRecoder(bio, codec.encode, codec.decode,
3658 encodings.ascii.StreamReader, encodings.ascii.StreamWriter)
3659 sr.writelines([b'a', b'b'])
3660 self.assertEqual(bio.getvalue(), b'ab')
3661
3662 def test_write(self):
3663 bio = io.BytesIO()
3664 codec = codecs.lookup('latin1')
3665 # Recode from Latin-1 to utf-8.
3666 sr = codecs.StreamRecoder(bio, codec.encode, codec.decode,
3667 encodings.utf_8.StreamReader, encodings.utf_8.StreamWriter)
3668
3669 text = 'àñé'
3670 sr.write(text.encode('latin1'))
3671 self.assertEqual(bio.getvalue(), text.encode('utf-8'))
3672
3673 def test_seeking_read(self):
3674 bio = io.BytesIO('line1\nline2\nline3\n'.encode('utf-16-le'))
3675 sr = codecs.EncodedFile(bio, 'utf-8', 'utf-16-le')
3676
3677 self.assertEqual(sr.readline(), b'line1\n')
3678 sr.seek(0)
3679 self.assertEqual(sr.readline(), b'line1\n')
3680 self.assertEqual(sr.readline(), b'line2\n')
3681 self.assertEqual(sr.readline(), b'line3\n')
3682 self.assertEqual(sr.readline(), b'')
3683
3684 def test_seeking_write(self):
3685 bio = io.BytesIO('123456789\n'.encode('utf-16-le'))
3686 sr = codecs.EncodedFile(bio, 'utf-8', 'utf-16-le')
3687
3688 # Test that seek() only resets its internal buffer when offset
3689 # and whence are zero.
3690 sr.seek(2)
3691 sr.write(b'\nabc\n')
3692 self.assertEqual(sr.readline(), b'789\n')
3693 sr.seek(0)
3694 self.assertEqual(sr.readline(), b'1\n')
3695 self.assertEqual(sr.readline(), b'abc\n')
3696 self.assertEqual(sr.readline(), b'789\n')
3697
3698 def test_copy(self):
3699 bio = io.BytesIO()
3700 codec = codecs.lookup('ascii')
3701 sr = codecs.StreamRecoder(bio, codec.encode, codec.decode,
3702 encodings.ascii.StreamReader, encodings.ascii.StreamWriter)
3703
3704 with self.assertRaisesRegex(TypeError, 'StreamRecoder'):
3705 copy.copy(sr)
3706 with self.assertRaisesRegex(TypeError, 'StreamRecoder'):
3707 copy.deepcopy(sr)
3708
3709 def test_pickle(self):
3710 q = Queue(b'')
3711 codec = codecs.lookup('ascii')
3712 sr = codecs.StreamRecoder(q, codec.encode, codec.decode,
3713 encodings.ascii.StreamReader, encodings.ascii.StreamWriter)
3714
3715 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
3716 with self.subTest(protocol=proto):
3717 with self.assertRaisesRegex(TypeError, 'StreamRecoder'):
3718 pickle.dumps(sr, proto)
3719
3720
3721 @unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module')
3722 class LocaleCodecTest(unittest.TestCase):
3723 """
3724 Test indirectly _Py_DecodeUTF8Ex() and _Py_EncodeUTF8Ex().
3725 """
3726 ENCODING = sys.getfilesystemencoding()
3727 STRINGS = ("ascii", "ulatin1:\xa7\xe9",
3728 "u255:\xff",
3729 "UCS:\xe9\u20ac\U0010ffff",
3730 "surrogates:\uDC80\uDCFF")
3731 BYTES_STRINGS = (b"blatin1:\xa7\xe9", b"b255:\xff")
3732 SURROGATES = "\uDC80\uDCFF"
3733
3734 def encode(self, text, errors="strict"):
3735 return _testinternalcapi.EncodeLocaleEx(text, 0, errors)
3736
3737 def check_encode_strings(self, errors):
3738 for text in self.STRINGS:
3739 with self.subTest(text=text):
3740 try:
3741 expected = text.encode(self.ENCODING, errors)
3742 except UnicodeEncodeError:
3743 with self.assertRaises(RuntimeError) as cm:
3744 self.encode(text, errors)
3745 errmsg = str(cm.exception)
3746 self.assertRegex(errmsg, r"encode error: pos=[0-9]+, reason=")
3747 else:
3748 encoded = self.encode(text, errors)
3749 self.assertEqual(encoded, expected)
3750
3751 def test_encode_strict(self):
3752 self.check_encode_strings("strict")
3753
3754 def test_encode_surrogateescape(self):
3755 self.check_encode_strings("surrogateescape")
3756
3757 def test_encode_surrogatepass(self):
3758 try:
3759 self.encode('', 'surrogatepass')
3760 except ValueError as exc:
3761 if str(exc) == 'unsupported error handler':
3762 self.skipTest(f"{self.ENCODING!r} encoder doesn't support "
3763 f"surrogatepass error handler")
3764 else:
3765 raise
3766
3767 self.check_encode_strings("surrogatepass")
3768
3769 def test_encode_unsupported_error_handler(self):
3770 with self.assertRaises(ValueError) as cm:
3771 self.encode('', 'backslashreplace')
3772 self.assertEqual(str(cm.exception), 'unsupported error handler')
3773
3774 def decode(self, encoded, errors="strict"):
3775 return _testinternalcapi.DecodeLocaleEx(encoded, 0, errors)
3776
3777 def check_decode_strings(self, errors):
3778 is_utf8 = (self.ENCODING == "utf-8")
3779 if is_utf8:
3780 encode_errors = 'surrogateescape'
3781 else:
3782 encode_errors = 'strict'
3783
3784 strings = list(self.BYTES_STRINGS)
3785 for text in self.STRINGS:
3786 try:
3787 encoded = text.encode(self.ENCODING, encode_errors)
3788 if encoded not in strings:
3789 strings.append(encoded)
3790 except UnicodeEncodeError:
3791 encoded = None
3792
3793 if is_utf8:
3794 encoded2 = text.encode(self.ENCODING, 'surrogatepass')
3795 if encoded2 != encoded:
3796 strings.append(encoded2)
3797
3798 for encoded in strings:
3799 with self.subTest(encoded=encoded):
3800 try:
3801 expected = encoded.decode(self.ENCODING, errors)
3802 except UnicodeDecodeError:
3803 with self.assertRaises(RuntimeError) as cm:
3804 self.decode(encoded, errors)
3805 errmsg = str(cm.exception)
3806 self.assertStartsWith(errmsg, "decode error: ")
3807 else:
3808 decoded = self.decode(encoded, errors)
3809 self.assertEqual(decoded, expected)
3810
3811 def test_decode_strict(self):
3812 self.check_decode_strings("strict")
3813
3814 def test_decode_surrogateescape(self):
3815 self.check_decode_strings("surrogateescape")
3816
3817 def test_decode_surrogatepass(self):
3818 try:
3819 self.decode(b'', 'surrogatepass')
3820 except ValueError as exc:
3821 if str(exc) == 'unsupported error handler':
3822 self.skipTest(f"{self.ENCODING!r} decoder doesn't support "
3823 f"surrogatepass error handler")
3824 else:
3825 raise
3826
3827 self.check_decode_strings("surrogatepass")
3828
3829 def test_decode_unsupported_error_handler(self):
3830 with self.assertRaises(ValueError) as cm:
3831 self.decode(b'', 'backslashreplace')
3832 self.assertEqual(str(cm.exception), 'unsupported error handler')
3833
3834
3835 class Rot13Test(unittest.TestCase):
3836 """Test the educational ROT-13 codec."""
3837 def test_encode(self):
3838 ciphertext = codecs.encode("Caesar liked ciphers", 'rot-13')
3839 self.assertEqual(ciphertext, 'Pnrfne yvxrq pvcuref')
3840
3841 def test_decode(self):
3842 plaintext = codecs.decode('Rg gh, Oehgr?', 'rot-13')
3843 self.assertEqual(plaintext, 'Et tu, Brute?')
3844
3845 def test_incremental_encode(self):
3846 encoder = codecs.getincrementalencoder('rot-13')()
3847 ciphertext = encoder.encode('ABBA nag Cheryl Baker')
3848 self.assertEqual(ciphertext, 'NOON ant Purely Onxre')
3849
3850 def test_incremental_decode(self):
3851 decoder = codecs.getincrementaldecoder('rot-13')()
3852 plaintext = decoder.decode('terra Ares envy tha')
3853 self.assertEqual(plaintext, 'green Nerf rail gun')
3854
3855
3856 class Rot13UtilTest(unittest.TestCase):
3857 """Test the ROT-13 codec via rot13 function,
3858 i.e. the user has done something like:
3859 $ echo "Hello World" | python -m encodings.rot_13
3860 """
3861 def test_rot13_func(self):
3862 from encodings.rot_13 import rot13
3863 infile = io.StringIO('Gb or, be abg gb or, gung vf gur dhrfgvba')
3864 outfile = io.StringIO()
3865 rot13(infile, outfile)
3866 outfile.seek(0)
3867 plain_text = outfile.read()
3868 self.assertEqual(
3869 plain_text,
3870 'To be, or not to be, that is the question')
3871
3872
3873 class CodecNameNormalizationTest(unittest.TestCase):
3874 """Test codec name normalization"""
3875 def test_codecs_lookup(self):
3876 FOUND = (1, 2, 3, 4)
3877 NOT_FOUND = (None, None, None, None)
3878 def search_function(encoding):
3879 if encoding == "aaa_8":
3880 return FOUND
3881 else:
3882 return NOT_FOUND
3883
3884 codecs.register(search_function)
3885 self.addCleanup(codecs.unregister, search_function)
3886 self.assertEqual(FOUND, codecs.lookup('aaa_8'))
3887 self.assertEqual(FOUND, codecs.lookup('AAA-8'))
3888 self.assertEqual(FOUND, codecs.lookup('AAA---8'))
3889 self.assertEqual(FOUND, codecs.lookup('AAA 8'))
3890 self.assertEqual(FOUND, codecs.lookup('aaa\xe9\u20ac-8'))
3891 self.assertEqual(NOT_FOUND, codecs.lookup('AAA.8'))
3892 self.assertEqual(NOT_FOUND, codecs.lookup('AAA...8'))
3893 self.assertEqual(NOT_FOUND, codecs.lookup('BBB-8'))
3894 self.assertEqual(NOT_FOUND, codecs.lookup('BBB.8'))
3895 self.assertEqual(NOT_FOUND, codecs.lookup('a\xe9\u20ac-8'))
3896
3897 def test_encodings_normalize_encoding(self):
3898 # encodings.normalize_encoding() ignores non-ASCII characters.
3899 normalize = encodings.normalize_encoding
3900 self.assertEqual(normalize('utf_8'), 'utf_8')
3901 self.assertEqual(normalize('utf\xE9\u20AC\U0010ffff-8'), 'utf_8')
3902 self.assertEqual(normalize('utf 8'), 'utf_8')
3903 # encodings.normalize_encoding() doesn't convert
3904 # characters to lower case.
3905 self.assertEqual(normalize('UTF 8'), 'UTF_8')
3906 self.assertEqual(normalize('utf.8'), 'utf.8')
3907 self.assertEqual(normalize('utf...8'), 'utf...8')
3908
3909
3910 class CodecCacheTest(unittest.TestCase):
3911 def test_cache_bounded(self):
3912 for i in range(encodings._MAXCACHE + 1000):
3913 try:
3914 b'x'.decode(f'nonexist_{i}')
3915 except LookupError:
3916 pass
3917
3918 self.assertLessEqual(len(encodings._cache), encodings._MAXCACHE)
3919
3920
3921 if __name__ == "__main__":
3922 unittest.main()