Debian Codegraph @

Lib/test/test_codeccallbacks.py

raw
1 from _codecs import _unregister_error as _codecs_unregister_error
2 import codecs
3 import html.entities
4 import itertools
5 import re
6 import sys
7 import unicodedata
8 import unittest
9
10
11 class PosReturn:
12 # this can be used for configurable callbacks
13
14 def __init__(self):
15 self.pos = 0
16
17 def handle(self, exc):
18 oldpos = self.pos
19 realpos = oldpos
20 if realpos<0:
21 realpos = len(exc.object) + realpos
22 # if we don't advance this time, terminate on the next call
23 # otherwise we'd get an endless loop
24 if realpos <= exc.start:
25 self.pos = len(exc.object)
26 return ("<?>", oldpos)
27
28 class RepeatedPosReturn:
29 def __init__(self, repl="<?>"):
30 self.repl = repl
31 self.pos = 0
32 self.count = 0
33
34 def handle(self, exc):
35 if self.count > 0:
36 self.count -= 1
37 return (self.repl, self.pos)
38 return (self.repl, exc.end)
39
40 # A UnicodeEncodeError object with a bad start attribute
41 class BadStartUnicodeEncodeError(UnicodeEncodeError):
42 def __init__(self):
43 UnicodeEncodeError.__init__(self, "ascii", "", 0, 1, "bad")
44 self.start = []
45
46 # A UnicodeEncodeError object with a bad object attribute
47 class BadObjectUnicodeEncodeError(UnicodeEncodeError):
48 def __init__(self):
49 UnicodeEncodeError.__init__(self, "ascii", "", 0, 1, "bad")
50 self.object = []
51
52 # A UnicodeDecodeError object without an end attribute
53 class NoEndUnicodeDecodeError(UnicodeDecodeError):
54 def __init__(self):
55 UnicodeDecodeError.__init__(self, "ascii", bytearray(b""), 0, 1, "bad")
56 del self.end
57
58 # A UnicodeDecodeError object with a bad object attribute
59 class BadObjectUnicodeDecodeError(UnicodeDecodeError):
60 def __init__(self):
61 UnicodeDecodeError.__init__(self, "ascii", bytearray(b""), 0, 1, "bad")
62 self.object = []
63
64 # A UnicodeTranslateError object without a start attribute
65 class NoStartUnicodeTranslateError(UnicodeTranslateError):
66 def __init__(self):
67 UnicodeTranslateError.__init__(self, "", 0, 1, "bad")
68 del self.start
69
70 # A UnicodeTranslateError object without an end attribute
71 class NoEndUnicodeTranslateError(UnicodeTranslateError):
72 def __init__(self):
73 UnicodeTranslateError.__init__(self, "", 0, 1, "bad")
74 del self.end
75
76 # A UnicodeTranslateError object without an object attribute
77 class NoObjectUnicodeTranslateError(UnicodeTranslateError):
78 def __init__(self):
79 UnicodeTranslateError.__init__(self, "", 0, 1, "bad")
80 del self.object
81
82 class CodecCallbackTest(unittest.TestCase):
83
84 def test_xmlcharrefreplace(self):
85 # replace unencodable characters which numeric character entities.
86 # For ascii, latin-1 and charmaps this is completely implemented
87 # in C and should be reasonably fast.
88 s = "\u30b9\u30d1\u30e2 \xe4nd eggs"
89 self.assertEqual(
90 s.encode("ascii", "xmlcharrefreplace"),
91 b"&#12473;&#12497;&#12514; &#228;nd eggs"
92 )
93 self.assertEqual(
94 s.encode("latin-1", "xmlcharrefreplace"),
95 b"&#12473;&#12497;&#12514; \xe4nd eggs"
96 )
97
98 def test_xmlcharnamereplace(self):
99 # This time use a named character entity for unencodable
100 # characters, if one is available.
101
102 def xmlcharnamereplace(exc):
103 if not isinstance(exc, UnicodeEncodeError):
104 raise TypeError("don't know how to handle %r" % exc)
105 l = []
106 for c in exc.object[exc.start:exc.end]:
107 try:
108 l.append("&%s;" % html.entities.codepoint2name[ord(c)])
109 except KeyError:
110 l.append("&#%d;" % ord(c))
111 return ("".join(l), exc.end)
112
113 codecs.register_error(
114 "test.xmlcharnamereplace", xmlcharnamereplace)
115
116 sin = "\xab\u211c\xbb = \u2329\u1234\u20ac\u232a"
117 sout = b"&laquo;&real;&raquo; = &lang;&#4660;&euro;&rang;"
118 self.assertEqual(sin.encode("ascii", "test.xmlcharnamereplace"), sout)
119 sout = b"\xab&real;\xbb = &lang;&#4660;&euro;&rang;"
120 self.assertEqual(sin.encode("latin-1", "test.xmlcharnamereplace"), sout)
121 sout = b"\xab&real;\xbb = &lang;&#4660;\xa4&rang;"
122 self.assertEqual(sin.encode("iso-8859-15", "test.xmlcharnamereplace"), sout)
123
124 def test_uninamereplace(self):
125 # We're using the names from the unicode database this time,
126 # and we're doing "syntax highlighting" here, i.e. we include
127 # the replaced text in ANSI escape sequences. For this it is
128 # useful that the error handler is not called for every single
129 # unencodable character, but for a complete sequence of
130 # unencodable characters, otherwise we would output many
131 # unnecessary escape sequences.
132
133 def uninamereplace(exc):
134 if not isinstance(exc, UnicodeEncodeError):
135 raise TypeError("don't know how to handle %r" % exc)
136 l = []
137 for c in exc.object[exc.start:exc.end]:
138 l.append(unicodedata.name(c, "0x%x" % ord(c)))
139 return ("\033[1m%s\033[0m" % ", ".join(l), exc.end)
140
141 codecs.register_error(
142 "test.uninamereplace", uninamereplace)
143
144 sin = "\xac\u1234\u20ac\u8000"
145 sout = b"\033[1mNOT SIGN, ETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m"
146 self.assertEqual(sin.encode("ascii", "test.uninamereplace"), sout)
147
148 sout = b"\xac\033[1mETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m"
149 self.assertEqual(sin.encode("latin-1", "test.uninamereplace"), sout)
150
151 sout = b"\xac\033[1mETHIOPIC SYLLABLE SEE\033[0m\xa4\033[1mCJK UNIFIED IDEOGRAPH-8000\033[0m"
152 self.assertEqual(sin.encode("iso-8859-15", "test.uninamereplace"), sout)
153
154 def test_backslashescape(self):
155 # Does the same as the "unicode-escape" encoding, but with different
156 # base encodings.
157 sin = "a\xac\u1234\u20ac\u8000\U0010ffff"
158 sout = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff"
159 self.assertEqual(sin.encode("ascii", "backslashreplace"), sout)
160
161 sout = b"a\xac\\u1234\\u20ac\\u8000\\U0010ffff"
162 self.assertEqual(sin.encode("latin-1", "backslashreplace"), sout)
163
164 sout = b"a\xac\\u1234\xa4\\u8000\\U0010ffff"
165 self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout)
166
167 def test_nameescape(self):
168 # Does the same as backslashescape, but prefers ``\N{...}`` escape
169 # sequences.
170 sin = "a\xac\u1234\u20ac\u8000\U0010ffff"
171 sout = (b'a\\N{NOT SIGN}\\N{ETHIOPIC SYLLABLE SEE}\\N{EURO SIGN}'
172 b'\\N{CJK UNIFIED IDEOGRAPH-8000}\\U0010ffff')
173 self.assertEqual(sin.encode("ascii", "namereplace"), sout)
174
175 sout = (b'a\xac\\N{ETHIOPIC SYLLABLE SEE}\\N{EURO SIGN}'
176 b'\\N{CJK UNIFIED IDEOGRAPH-8000}\\U0010ffff')
177 self.assertEqual(sin.encode("latin-1", "namereplace"), sout)
178
179 sout = (b'a\xac\\N{ETHIOPIC SYLLABLE SEE}\xa4'
180 b'\\N{CJK UNIFIED IDEOGRAPH-8000}\\U0010ffff')
181 self.assertEqual(sin.encode("iso-8859-15", "namereplace"), sout)
182
183 def test_decoding_callbacks(self):
184 # This is a test for a decoding callback handler
185 # that allows the decoding of the invalid sequence
186 # "\xc0\x80" and returns "\x00" instead of raising an error.
187 # All other illegal sequences will be handled strictly.
188 def relaxedutf8(exc):
189 if not isinstance(exc, UnicodeDecodeError):
190 raise TypeError("don't know how to handle %r" % exc)
191 if exc.object[exc.start:exc.start+2] == b"\xc0\x80":
192 return ("\x00", exc.start+2) # retry after two bytes
193 else:
194 raise exc
195
196 codecs.register_error("test.relaxedutf8", relaxedutf8)
197
198 # all the "\xc0\x80" will be decoded to "\x00"
199 sin = b"a\x00b\xc0\x80c\xc3\xbc\xc0\x80\xc0\x80"
200 sout = "a\x00b\x00c\xfc\x00\x00"
201 self.assertEqual(sin.decode("utf-8", "test.relaxedutf8"), sout)
202
203 # "\xc0\x81" is not valid and a UnicodeDecodeError will be raised
204 sin = b"\xc0\x80\xc0\x81"
205 self.assertRaises(UnicodeDecodeError, sin.decode,
206 "utf-8", "test.relaxedutf8")
207
208 def test_charmapencode(self):
209 # For charmap encodings the replacement string will be
210 # mapped through the encoding again. This means, that
211 # to be able to use e.g. the "replace" handler, the
212 # charmap has to have a mapping for "?".
213 charmap = dict((ord(c), bytes(2*c.upper(), 'ascii')) for c in "abcdefgh")
214 sin = "abc"
215 sout = b"AABBCC"
216 self.assertEqual(codecs.charmap_encode(sin, "strict", charmap)[0], sout)
217
218 sin = "abcA"
219 self.assertRaises(UnicodeError, codecs.charmap_encode, sin, "strict", charmap)
220
221 charmap[ord("?")] = b"XYZ"
222 sin = "abcDEF"
223 sout = b"AABBCCXYZXYZXYZ"
224 self.assertEqual(codecs.charmap_encode(sin, "replace", charmap)[0], sout)
225
226 charmap[ord("?")] = "XYZ" # wrong type in mapping
227 self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap)
228
229 def test_callbacks(self):
230 def handler1(exc):
231 r = range(exc.start, exc.end)
232 if isinstance(exc, UnicodeEncodeError):
233 l = ["<%d>" % ord(exc.object[pos]) for pos in r]
234 elif isinstance(exc, UnicodeDecodeError):
235 l = ["<%d>" % exc.object[pos] for pos in r]
236 else:
237 raise TypeError("don't know how to handle %r" % exc)
238 return ("[%s]" % "".join(l), exc.end)
239
240 codecs.register_error("test.handler1", handler1)
241
242 def handler2(exc):
243 if not isinstance(exc, UnicodeDecodeError):
244 raise TypeError("don't know how to handle %r" % exc)
245 l = ["<%d>" % exc.object[pos] for pos in range(exc.start, exc.end)]
246 return ("[%s]" % "".join(l), exc.end+1) # skip one character
247
248 codecs.register_error("test.handler2", handler2)
249
250 s = b"\x00\x81\x7f\x80\xff"
251
252 self.assertEqual(
253 s.decode("ascii", "test.handler1"),
254 "\x00[<129>]\x7f[<128>][<255>]"
255 )
256 self.assertEqual(
257 s.decode("ascii", "test.handler2"),
258 "\x00[<129>][<128>]"
259 )
260
261 self.assertEqual(
262 b"\\u3042\\u3xxx".decode("unicode-escape", "test.handler1"),
263 "\u3042[<92><117><51>]xxx"
264 )
265
266 self.assertEqual(
267 b"\\u3042\\u3xx".decode("unicode-escape", "test.handler1"),
268 "\u3042[<92><117><51>]xx"
269 )
270
271 self.assertEqual(
272 codecs.charmap_decode(b"abc", "test.handler1", {ord("a"): "z"})[0],
273 "z[<98>][<99>]"
274 )
275
276 self.assertEqual(
277 "g\xfc\xdfrk".encode("ascii", "test.handler1"),
278 b"g[<252><223>]rk"
279 )
280
281 self.assertEqual(
282 "g\xfc\xdf".encode("ascii", "test.handler1"),
283 b"g[<252><223>]"
284 )
285
286 def test_longstrings(self):
287 # test long strings to check for memory overflow problems
288 errors = [ "strict", "ignore", "replace", "xmlcharrefreplace",
289 "backslashreplace", "namereplace"]
290 # register the handlers under different names,
291 # to prevent the codec from recognizing the name
292 for err in errors:
293 codecs.register_error("test." + err, codecs.lookup_error(err))
294 l = 1000
295 errors += [ "test." + err for err in errors ]
296 for uni in [ s*l for s in ("x", "\u3042", "a\xe4") ]:
297 for enc in ("ascii", "latin-1", "iso-8859-1", "iso-8859-15",
298 "utf-8", "utf-7", "utf-16", "utf-32"):
299 for err in errors:
300 try:
301 uni.encode(enc, err)
302 except UnicodeError:
303 pass
304
305 def check_exceptionobjectargs(self, exctype, args, msg):
306 # Test UnicodeError subclasses: construction, attribute assignment and __str__ conversion
307 # check with one missing argument
308 self.assertRaises(TypeError, exctype, *args[:-1])
309 # check with one argument too much
310 self.assertRaises(TypeError, exctype, *(args + ["too much"]))
311 # check with one argument of the wrong type
312 wrongargs = [ "spam", b"eggs", b"spam", 42, 1.0, None ]
313 for i in range(len(args)):
314 for wrongarg in wrongargs:
315 if type(wrongarg) is type(args[i]):
316 continue
317 # build argument array
318 callargs = []
319 for j in range(len(args)):
320 if i==j:
321 callargs.append(wrongarg)
322 else:
323 callargs.append(args[i])
324 self.assertRaises(TypeError, exctype, *callargs)
325
326 # check with the correct number and type of arguments
327 exc = exctype(*args)
328 self.assertEqual(str(exc), msg)
329
330 def test_unicodeencodeerror(self):
331 self.check_exceptionobjectargs(
332 UnicodeEncodeError,
333 ["ascii", "g\xfcrk", 1, 2, "ouch"],
334 "'ascii' codec can't encode character '\\xfc' in position 1: ouch"
335 )
336 self.check_exceptionobjectargs(
337 UnicodeEncodeError,
338 ["ascii", "g\xfcrk", 1, 4, "ouch"],
339 "'ascii' codec can't encode characters in position 1-3: ouch"
340 )
341 self.check_exceptionobjectargs(
342 UnicodeEncodeError,
343 ["ascii", "\xfcx", 0, 1, "ouch"],
344 "'ascii' codec can't encode character '\\xfc' in position 0: ouch"
345 )
346 self.check_exceptionobjectargs(
347 UnicodeEncodeError,
348 ["ascii", "\u0100x", 0, 1, "ouch"],
349 "'ascii' codec can't encode character '\\u0100' in position 0: ouch"
350 )
351 self.check_exceptionobjectargs(
352 UnicodeEncodeError,
353 ["ascii", "\uffffx", 0, 1, "ouch"],
354 "'ascii' codec can't encode character '\\uffff' in position 0: ouch"
355 )
356 self.check_exceptionobjectargs(
357 UnicodeEncodeError,
358 ["ascii", "\U00010000x", 0, 1, "ouch"],
359 "'ascii' codec can't encode character '\\U00010000' in position 0: ouch"
360 )
361
362 def test_unicodedecodeerror(self):
363 self.check_exceptionobjectargs(
364 UnicodeDecodeError,
365 ["ascii", bytearray(b"g\xfcrk"), 1, 2, "ouch"],
366 "'ascii' codec can't decode byte 0xfc in position 1: ouch"
367 )
368 self.check_exceptionobjectargs(
369 UnicodeDecodeError,
370 ["ascii", bytearray(b"g\xfcrk"), 1, 3, "ouch"],
371 "'ascii' codec can't decode bytes in position 1-2: ouch"
372 )
373
374 def test_unicodetranslateerror(self):
375 self.check_exceptionobjectargs(
376 UnicodeTranslateError,
377 ["g\xfcrk", 1, 2, "ouch"],
378 "can't translate character '\\xfc' in position 1: ouch"
379 )
380 self.check_exceptionobjectargs(
381 UnicodeTranslateError,
382 ["g\u0100rk", 1, 2, "ouch"],
383 "can't translate character '\\u0100' in position 1: ouch"
384 )
385 self.check_exceptionobjectargs(
386 UnicodeTranslateError,
387 ["g\uffffrk", 1, 2, "ouch"],
388 "can't translate character '\\uffff' in position 1: ouch"
389 )
390 self.check_exceptionobjectargs(
391 UnicodeTranslateError,
392 ["g\U00010000rk", 1, 2, "ouch"],
393 "can't translate character '\\U00010000' in position 1: ouch"
394 )
395 self.check_exceptionobjectargs(
396 UnicodeTranslateError,
397 ["g\xfcrk", 1, 3, "ouch"],
398 "can't translate characters in position 1-2: ouch"
399 )
400
401 def test_badandgoodstrictexceptions(self):
402 # "strict" complains about a non-exception passed in
403 self.assertRaises(
404 TypeError,
405 codecs.strict_errors,
406 42
407 )
408 # "strict" complains about the wrong exception type
409 self.assertRaises(
410 Exception,
411 codecs.strict_errors,
412 Exception("ouch")
413 )
414
415 # If the correct exception is passed in, "strict" raises it
416 self.assertRaises(
417 UnicodeEncodeError,
418 codecs.strict_errors,
419 UnicodeEncodeError("ascii", "\u3042", 0, 1, "ouch")
420 )
421 self.assertRaises(
422 UnicodeDecodeError,
423 codecs.strict_errors,
424 UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch")
425 )
426 self.assertRaises(
427 UnicodeTranslateError,
428 codecs.strict_errors,
429 UnicodeTranslateError("\u3042", 0, 1, "ouch")
430 )
431
432 def test_badandgoodignoreexceptions(self):
433 # "ignore" complains about a non-exception passed in
434 self.assertRaises(
435 TypeError,
436 codecs.ignore_errors,
437 42
438 )
439 # "ignore" complains about the wrong exception type
440 self.assertRaises(
441 TypeError,
442 codecs.ignore_errors,
443 UnicodeError("ouch")
444 )
445 # If the correct exception is passed in, "ignore" returns an empty replacement
446 self.assertEqual(
447 codecs.ignore_errors(
448 UnicodeEncodeError("ascii", "a\u3042b", 1, 2, "ouch")),
449 ("", 2)
450 )
451 self.assertEqual(
452 codecs.ignore_errors(
453 UnicodeDecodeError("ascii", bytearray(b"a\xffb"), 1, 2, "ouch")),
454 ("", 2)
455 )
456 self.assertEqual(
457 codecs.ignore_errors(
458 UnicodeTranslateError("a\u3042b", 1, 2, "ouch")),
459 ("", 2)
460 )
461
462 def test_badandgoodreplaceexceptions(self):
463 # "replace" complains about a non-exception passed in
464 self.assertRaises(
465 TypeError,
466 codecs.replace_errors,
467 42
468 )
469 # "replace" complains about the wrong exception type
470 self.assertRaises(
471 TypeError,
472 codecs.replace_errors,
473 UnicodeError("ouch")
474 )
475 self.assertRaises(
476 TypeError,
477 codecs.replace_errors,
478 BadObjectUnicodeEncodeError()
479 )
480 self.assertRaises(
481 TypeError,
482 codecs.replace_errors,
483 BadObjectUnicodeDecodeError()
484 )
485 # With the correct exception, "replace" returns an "?" or "\ufffd" replacement
486 self.assertEqual(
487 codecs.replace_errors(
488 UnicodeEncodeError("ascii", "a\u3042b", 1, 2, "ouch")),
489 ("?", 2)
490 )
491 self.assertEqual(
492 codecs.replace_errors(
493 UnicodeDecodeError("ascii", bytearray(b"a\xffb"), 1, 2, "ouch")),
494 ("\ufffd", 2)
495 )
496 self.assertEqual(
497 codecs.replace_errors(
498 UnicodeTranslateError("a\u3042b", 1, 2, "ouch")),
499 ("\ufffd", 2)
500 )
501
502 def test_badandgoodxmlcharrefreplaceexceptions(self):
503 # "xmlcharrefreplace" complains about a non-exception passed in
504 self.assertRaises(
505 TypeError,
506 codecs.xmlcharrefreplace_errors,
507 42
508 )
509 # "xmlcharrefreplace" complains about the wrong exception types
510 self.assertRaises(
511 TypeError,
512 codecs.xmlcharrefreplace_errors,
513 UnicodeError("ouch")
514 )
515 # "xmlcharrefreplace" can only be used for encoding
516 self.assertRaises(
517 TypeError,
518 codecs.xmlcharrefreplace_errors,
519 UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch")
520 )
521 self.assertRaises(
522 TypeError,
523 codecs.xmlcharrefreplace_errors,
524 UnicodeTranslateError("\u3042", 0, 1, "ouch")
525 )
526 # Use the correct exception
527 cs = (0, 1, 9, 10, 99, 100, 999, 1000, 9999, 10000, 99999, 100000,
528 999999, 1000000)
529 cs += (0xd800, 0xdfff)
530 s = "".join(chr(c) for c in cs)
531 self.assertEqual(
532 codecs.xmlcharrefreplace_errors(
533 UnicodeEncodeError("ascii", "a" + s + "b",
534 1, 1 + len(s), "ouch")
535 ),
536 ("".join("&#%d;" % c for c in cs), 1 + len(s))
537 )
538
539 def test_badandgoodbackslashreplaceexceptions(self):
540 # "backslashreplace" complains about a non-exception passed in
541 self.assertRaises(
542 TypeError,
543 codecs.backslashreplace_errors,
544 42
545 )
546 # "backslashreplace" complains about the wrong exception types
547 self.assertRaises(
548 TypeError,
549 codecs.backslashreplace_errors,
550 UnicodeError("ouch")
551 )
552 # Use the correct exception
553 tests = [
554 ("\u3042", "\\u3042"),
555 ("\n", "\\x0a"),
556 ("a", "\\x61"),
557 ("\x00", "\\x00"),
558 ("\xff", "\\xff"),
559 ("\u0100", "\\u0100"),
560 ("\uffff", "\\uffff"),
561 ("\U00010000", "\\U00010000"),
562 ("\U0010ffff", "\\U0010ffff"),
563 # Lone surrogates
564 ("\ud800", "\\ud800"),
565 ("\udfff", "\\udfff"),
566 ("\ud800\udfff", "\\ud800\\udfff"),
567 ]
568 for s, r in tests:
569 with self.subTest(str=s):
570 self.assertEqual(
571 codecs.backslashreplace_errors(
572 UnicodeEncodeError("ascii", "a" + s + "b",
573 1, 1 + len(s), "ouch")),
574 (r, 1 + len(s))
575 )
576 self.assertEqual(
577 codecs.backslashreplace_errors(
578 UnicodeTranslateError("a" + s + "b",
579 1, 1 + len(s), "ouch")),
580 (r, 1 + len(s))
581 )
582 tests = [
583 (b"a", "\\x61"),
584 (b"\n", "\\x0a"),
585 (b"\x00", "\\x00"),
586 (b"\xff", "\\xff"),
587 ]
588 for b, r in tests:
589 with self.subTest(bytes=b):
590 self.assertEqual(
591 codecs.backslashreplace_errors(
592 UnicodeDecodeError("ascii", bytearray(b"a" + b + b"b"),
593 1, 2, "ouch")),
594 (r, 2)
595 )
596
597 def test_badandgoodnamereplaceexceptions(self):
598 # "namereplace" complains about a non-exception passed in
599 self.assertRaises(
600 TypeError,
601 codecs.namereplace_errors,
602 42
603 )
604 # "namereplace" complains about the wrong exception types
605 self.assertRaises(
606 TypeError,
607 codecs.namereplace_errors,
608 UnicodeError("ouch")
609 )
610 # "namereplace" can only be used for encoding
611 self.assertRaises(
612 TypeError,
613 codecs.namereplace_errors,
614 UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch")
615 )
616 self.assertRaises(
617 TypeError,
618 codecs.namereplace_errors,
619 UnicodeTranslateError("\u3042", 0, 1, "ouch")
620 )
621 # Use the correct exception
622 tests = [
623 ("\u3042", "\\N{HIRAGANA LETTER A}"),
624 ("\x00", "\\x00"),
625 ("\ufbf9", "\\N{ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH "
626 "HAMZA ABOVE WITH ALEF MAKSURA ISOLATED FORM}"),
627 ("\U000e007f", "\\N{CANCEL TAG}"),
628 ("\U0010ffff", "\\U0010ffff"),
629 # Lone surrogates
630 ("\ud800", "\\ud800"),
631 ("\udfff", "\\udfff"),
632 ("\ud800\udfff", "\\ud800\\udfff"),
633 ]
634 for s, r in tests:
635 with self.subTest(str=s):
636 self.assertEqual(
637 codecs.namereplace_errors(
638 UnicodeEncodeError("ascii", "a" + s + "b",
639 1, 1 + len(s), "ouch")),
640 (r, 1 + len(s))
641 )
642
643 def test_badandgoodsurrogateescapeexceptions(self):
644 surrogateescape_errors = codecs.lookup_error('surrogateescape')
645 # "surrogateescape" complains about a non-exception passed in
646 self.assertRaises(
647 TypeError,
648 surrogateescape_errors,
649 42
650 )
651 # "surrogateescape" complains about the wrong exception types
652 self.assertRaises(
653 TypeError,
654 surrogateescape_errors,
655 UnicodeError("ouch")
656 )
657 # "surrogateescape" can not be used for translating
658 self.assertRaises(
659 TypeError,
660 surrogateescape_errors,
661 UnicodeTranslateError("\udc80", 0, 1, "ouch")
662 )
663 # Use the correct exception
664 for s in ("a", "\udc7f", "\udd00"):
665 with self.subTest(str=s):
666 self.assertRaises(
667 UnicodeEncodeError,
668 surrogateescape_errors,
669 UnicodeEncodeError("ascii", s, 0, 1, "ouch")
670 )
671 self.assertEqual(
672 surrogateescape_errors(
673 UnicodeEncodeError("ascii", "a\udc80b", 1, 2, "ouch")),
674 (b"\x80", 2)
675 )
676 self.assertRaises(
677 UnicodeDecodeError,
678 surrogateescape_errors,
679 UnicodeDecodeError("ascii", bytearray(b"a"), 0, 1, "ouch")
680 )
681 self.assertEqual(
682 surrogateescape_errors(
683 UnicodeDecodeError("ascii", bytearray(b"a\x80b"), 1, 2, "ouch")),
684 ("\udc80", 2)
685 )
686
687 def test_badandgoodsurrogatepassexceptions(self):
688 surrogatepass_errors = codecs.lookup_error('surrogatepass')
689 # "surrogatepass" complains about a non-exception passed in
690 self.assertRaises(
691 TypeError,
692 surrogatepass_errors,
693 42
694 )
695 # "surrogatepass" complains about the wrong exception types
696 self.assertRaises(
697 TypeError,
698 surrogatepass_errors,
699 UnicodeError("ouch")
700 )
701 # "surrogatepass" can not be used for translating
702 self.assertRaises(
703 TypeError,
704 surrogatepass_errors,
705 UnicodeTranslateError("\ud800", 0, 1, "ouch")
706 )
707 # Use the correct exception
708 for enc in ("utf-8", "utf-16le", "utf-16be", "utf-32le", "utf-32be"):
709 with self.subTest(encoding=enc):
710 self.assertRaises(
711 UnicodeEncodeError,
712 surrogatepass_errors,
713 UnicodeEncodeError(enc, "a", 0, 1, "ouch")
714 )
715 self.assertRaises(
716 UnicodeDecodeError,
717 surrogatepass_errors,
718 UnicodeDecodeError(enc, "a".encode(enc), 0, 1, "ouch")
719 )
720 for s in ("\ud800", "\udfff", "\ud800\udfff"):
721 with self.subTest(str=s):
722 self.assertRaises(
723 UnicodeEncodeError,
724 surrogatepass_errors,
725 UnicodeEncodeError("ascii", s, 0, len(s), "ouch")
726 )
727 tests = [
728 ("utf-8", "\ud800", b'\xed\xa0\x80', 3),
729 ("utf-16le", "\ud800", b'\x00\xd8', 2),
730 ("utf-16be", "\ud800", b'\xd8\x00', 2),
731 ("utf-32le", "\ud800", b'\x00\xd8\x00\x00', 4),
732 ("utf-32be", "\ud800", b'\x00\x00\xd8\x00', 4),
733 ("utf-8", "\udfff", b'\xed\xbf\xbf', 3),
734 ("utf-16le", "\udfff", b'\xff\xdf', 2),
735 ("utf-16be", "\udfff", b'\xdf\xff', 2),
736 ("utf-32le", "\udfff", b'\xff\xdf\x00\x00', 4),
737 ("utf-32be", "\udfff", b'\x00\x00\xdf\xff', 4),
738 ("utf-8", "\ud800\udfff", b'\xed\xa0\x80\xed\xbf\xbf', 3),
739 ("utf-16le", "\ud800\udfff", b'\x00\xd8\xff\xdf', 2),
740 ("utf-16be", "\ud800\udfff", b'\xd8\x00\xdf\xff', 2),
741 ("utf-32le", "\ud800\udfff", b'\x00\xd8\x00\x00\xff\xdf\x00\x00', 4),
742 ("utf-32be", "\ud800\udfff", b'\x00\x00\xd8\x00\x00\x00\xdf\xff', 4),
743 ]
744 for enc, s, b, n in tests:
745 with self.subTest(encoding=enc, str=s, bytes=b):
746 self.assertEqual(
747 surrogatepass_errors(
748 UnicodeEncodeError(enc, "a" + s + "b",
749 1, 1 + len(s), "ouch")),
750 (b, 1 + len(s))
751 )
752 self.assertEqual(
753 surrogatepass_errors(
754 UnicodeDecodeError(enc, bytearray(b"a" + b[:n] + b"b"),
755 1, 1 + n, "ouch")),
756 (s[:1], 1 + n)
757 )
758
759 def test_badhandlerresults(self):
760 results = ( 42, "foo", (1,2,3), ("foo", 1, 3), ("foo", None), ("foo",), ("foo", 1, 3), ("foo", None), ("foo",) )
761 encs = ("ascii", "latin-1", "iso-8859-1", "iso-8859-15")
762
763 for res in results:
764 codecs.register_error("test.badhandler", lambda x: res)
765 for enc in encs:
766 self.assertRaises(
767 TypeError,
768 "\u3042".encode,
769 enc,
770 "test.badhandler"
771 )
772 for (enc, bytes) in (
773 ("ascii", b"\xff"),
774 ("utf-8", b"\xff"),
775 ("utf-7", b"+x-"),
776 ):
777 self.assertRaises(
778 TypeError,
779 bytes.decode,
780 enc,
781 "test.badhandler"
782 )
783
784 def test_lookup(self):
785 self.assertEqual(codecs.strict_errors, codecs.lookup_error("strict"))
786 self.assertEqual(codecs.ignore_errors, codecs.lookup_error("ignore"))
787 self.assertEqual(codecs.strict_errors, codecs.lookup_error("strict"))
788 self.assertEqual(
789 codecs.xmlcharrefreplace_errors,
790 codecs.lookup_error("xmlcharrefreplace")
791 )
792 self.assertEqual(
793 codecs.backslashreplace_errors,
794 codecs.lookup_error("backslashreplace")
795 )
796 self.assertEqual(
797 codecs.namereplace_errors,
798 codecs.lookup_error("namereplace")
799 )
800
801 def test_encode_nonascii_replacement(self):
802 def handle(exc):
803 if isinstance(exc, UnicodeEncodeError):
804 return (repl, exc.end)
805 raise TypeError("don't know how to handle %r" % exc)
806 codecs.register_error("test.replacing", handle)
807
808 for enc, input, repl in (
809 ("ascii", "[¤]", "abc"),
810 ("iso-8859-1", "[€]", "½¾"),
811 ("iso-8859-15", "[¤]", "œŸ"),
812 ):
813 res = input.encode(enc, "test.replacing")
814 self.assertEqual(res, ("[" + repl + "]").encode(enc))
815
816 for enc, input, repl in (
817 ("utf-8", "[\udc80]", "\U0001f40d"),
818 ("utf-16", "[\udc80]", "\U0001f40d"),
819 ("utf-32", "[\udc80]", "\U0001f40d"),
820 ):
821 with self.subTest(encoding=enc):
822 with self.assertRaises(UnicodeEncodeError) as cm:
823 input.encode(enc, "test.replacing")
824 exc = cm.exception
825 self.assertEqual(exc.start, 1)
826 self.assertEqual(exc.end, 2)
827 self.assertEqual(exc.object, input)
828
829 def test_encode_unencodable_replacement(self):
830 def unencrepl(exc):
831 if isinstance(exc, UnicodeEncodeError):
832 return (repl, exc.end)
833 else:
834 raise TypeError("don't know how to handle %r" % exc)
835 codecs.register_error("test.unencreplhandler", unencrepl)
836
837 for enc, input, repl in (
838 ("ascii", "[¤]", "½"),
839 ("iso-8859-1", "[€]", "œ"),
840 ("iso-8859-15", "[¤]", "½"),
841 ("utf-8", "[\udc80]", "\udcff"),
842 ("utf-16", "[\udc80]", "\udcff"),
843 ("utf-32", "[\udc80]", "\udcff"),
844 ):
845 with self.subTest(encoding=enc):
846 with self.assertRaises(UnicodeEncodeError) as cm:
847 input.encode(enc, "test.unencreplhandler")
848 exc = cm.exception
849 self.assertEqual(exc.start, 1)
850 self.assertEqual(exc.end, 2)
851 self.assertEqual(exc.object, input)
852
853 def test_encode_bytes_replacement(self):
854 def handle(exc):
855 if isinstance(exc, UnicodeEncodeError):
856 return (repl, exc.end)
857 raise TypeError("don't know how to handle %r" % exc)
858 codecs.register_error("test.replacing", handle)
859
860 # It works even if the bytes sequence is not decodable.
861 for enc, input, repl in (
862 ("ascii", "[¤]", b"\xbd\xbe"),
863 ("iso-8859-1", "[€]", b"\xbd\xbe"),
864 ("iso-8859-15", "[¤]", b"\xbd\xbe"),
865 ("utf-8", "[\udc80]", b"\xbd\xbe"),
866 ("utf-16le", "[\udc80]", b"\xbd\xbe"),
867 ("utf-16be", "[\udc80]", b"\xbd\xbe"),
868 ("utf-32le", "[\udc80]", b"\xbc\xbd\xbe\xbf"),
869 ("utf-32be", "[\udc80]", b"\xbc\xbd\xbe\xbf"),
870 ):
871 with self.subTest(encoding=enc):
872 res = input.encode(enc, "test.replacing")
873 self.assertEqual(res, "[".encode(enc) + repl + "]".encode(enc))
874
875 def test_encode_odd_bytes_replacement(self):
876 def handle(exc):
877 if isinstance(exc, UnicodeEncodeError):
878 return (repl, exc.end)
879 raise TypeError("don't know how to handle %r" % exc)
880 codecs.register_error("test.replacing", handle)
881
882 input = "[\udc80]"
883 # Tests in which the replacement bytestring contains not whole number
884 # of code units.
885 for enc, repl in (
886 *itertools.product(("utf-16le", "utf-16be"),
887 [b"a", b"abc"]),
888 *itertools.product(("utf-32le", "utf-32be"),
889 [b"a", b"ab", b"abc", b"abcde"]),
890 ):
891 with self.subTest(encoding=enc, repl=repl):
892 with self.assertRaises(UnicodeEncodeError) as cm:
893 input.encode(enc, "test.replacing")
894 exc = cm.exception
895 self.assertEqual(exc.start, 1)
896 self.assertEqual(exc.end, 2)
897 self.assertEqual(exc.object, input)
898 self.assertEqual(exc.reason, "surrogates not allowed")
899
900 def test_badregistercall(self):
901 # enhance coverage of:
902 # Modules/_codecsmodule.c::register_error()
903 # Python/codecs.c::PyCodec_RegisterError()
904 self.assertRaises(TypeError, codecs.register_error, 42)
905 self.assertRaises(TypeError, codecs.register_error, "test.dummy", 42)
906
907 def test_badlookupcall(self):
908 # enhance coverage of:
909 # Modules/_codecsmodule.c::lookup_error()
910 self.assertRaises(TypeError, codecs.lookup_error)
911
912 def test_unknownhandler(self):
913 # enhance coverage of:
914 # Modules/_codecsmodule.c::lookup_error()
915 self.assertRaises(LookupError, codecs.lookup_error, "test.unknown")
916
917 def test_xmlcharrefvalues(self):
918 # enhance coverage of:
919 # Python/codecs.c::PyCodec_XMLCharRefReplaceErrors()
920 # and inline implementations
921 v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000,
922 500000, 1000000)
923 s = "".join([chr(x) for x in v])
924 codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors)
925 for enc in ("ascii", "iso-8859-15"):
926 for err in ("xmlcharrefreplace", "test.xmlcharrefreplace"):
927 s.encode(enc, err)
928
929 def test_decodehelper(self):
930 # enhance coverage of:
931 # Objects/unicodeobject.c::unicode_decode_call_errorhandler()
932 # and callers
933 self.assertRaises(LookupError, b"\xff".decode, "ascii", "test.unknown")
934
935 def baddecodereturn1(exc):
936 return 42
937 codecs.register_error("test.baddecodereturn1", baddecodereturn1)
938 self.assertRaises(TypeError, b"\xff".decode, "ascii", "test.baddecodereturn1")
939 self.assertRaises(TypeError, b"\\".decode, "unicode-escape", "test.baddecodereturn1")
940 self.assertRaises(TypeError, b"\\x0".decode, "unicode-escape", "test.baddecodereturn1")
941 self.assertRaises(TypeError, b"\\x0y".decode, "unicode-escape", "test.baddecodereturn1")
942 self.assertRaises(TypeError, b"\\Uffffeeee".decode, "unicode-escape", "test.baddecodereturn1")
943 self.assertRaises(TypeError, b"\\uyyyy".decode, "raw-unicode-escape", "test.baddecodereturn1")
944
945 def baddecodereturn2(exc):
946 return ("?", None)
947 codecs.register_error("test.baddecodereturn2", baddecodereturn2)
948 self.assertRaises(TypeError, b"\xff".decode, "ascii", "test.baddecodereturn2")
949
950 handler = PosReturn()
951 codecs.register_error("test.posreturn", handler.handle)
952
953 # Valid negative position
954 handler.pos = -1
955 self.assertEqual(b"\xff0".decode("ascii", "test.posreturn"), "<?>0")
956
957 # Valid negative position
958 handler.pos = -2
959 self.assertEqual(b"\xff0".decode("ascii", "test.posreturn"), "<?><?>")
960
961 # Negative position out of bounds
962 handler.pos = -3
963 self.assertRaises(IndexError, b"\xff0".decode, "ascii", "test.posreturn")
964
965 # Valid positive position
966 handler.pos = 1
967 self.assertEqual(b"\xff0".decode("ascii", "test.posreturn"), "<?>0")
968
969 # Largest valid positive position (one beyond end of input)
970 handler.pos = 2
971 self.assertEqual(b"\xff0".decode("ascii", "test.posreturn"), "<?>")
972
973 # Invalid positive position
974 handler.pos = 3
975 self.assertRaises(IndexError, b"\xff0".decode, "ascii", "test.posreturn")
976
977 # Restart at the "0"
978 handler.pos = 6
979 self.assertEqual(b"\\uyyyy0".decode("raw-unicode-escape", "test.posreturn"), "<?>0")
980
981 class D(dict):
982 def __getitem__(self, key):
983 raise ValueError
984 self.assertRaises(UnicodeError, codecs.charmap_decode, b"\xff", "strict", {0xff: None})
985 self.assertRaises(ValueError, codecs.charmap_decode, b"\xff", "strict", D())
986 self.assertRaises(TypeError, codecs.charmap_decode, b"\xff", "strict", {0xff: sys.maxunicode+1})
987
988 def test_encodehelper(self):
989 # enhance coverage of:
990 # Objects/unicodeobject.c::unicode_encode_call_errorhandler()
991 # and callers
992 self.assertRaises(LookupError, "\xff".encode, "ascii", "test.unknown")
993
994 def badencodereturn1(exc):
995 return 42
996 codecs.register_error("test.badencodereturn1", badencodereturn1)
997 self.assertRaises(TypeError, "\xff".encode, "ascii", "test.badencodereturn1")
998
999 def badencodereturn2(exc):
1000 return ("?", None)
1001 codecs.register_error("test.badencodereturn2", badencodereturn2)
1002 self.assertRaises(TypeError, "\xff".encode, "ascii", "test.badencodereturn2")
1003
1004 handler = PosReturn()
1005 codecs.register_error("test.posreturn", handler.handle)
1006
1007 # Valid negative position
1008 handler.pos = -1
1009 self.assertEqual("\xff0".encode("ascii", "test.posreturn"), b"<?>0")
1010
1011 # Valid negative position
1012 handler.pos = -2
1013 self.assertEqual("\xff0".encode("ascii", "test.posreturn"), b"<?><?>")
1014
1015 # Negative position out of bounds
1016 handler.pos = -3
1017 self.assertRaises(IndexError, "\xff0".encode, "ascii", "test.posreturn")
1018
1019 # Valid positive position
1020 handler.pos = 1
1021 self.assertEqual("\xff0".encode("ascii", "test.posreturn"), b"<?>0")
1022
1023 # Largest valid positive position (one beyond end of input
1024 handler.pos = 2
1025 self.assertEqual("\xff0".encode("ascii", "test.posreturn"), b"<?>")
1026
1027 # Invalid positive position
1028 handler.pos = 3
1029 self.assertRaises(IndexError, "\xff0".encode, "ascii", "test.posreturn")
1030
1031 handler.pos = 0
1032
1033 class D(dict):
1034 def __getitem__(self, key):
1035 raise ValueError
1036 for err in ("strict", "replace", "xmlcharrefreplace",
1037 "backslashreplace", "namereplace", "test.posreturn"):
1038 self.assertRaises(UnicodeError, codecs.charmap_encode, "\xff", err, {0xff: None})
1039 self.assertRaises(ValueError, codecs.charmap_encode, "\xff", err, D())
1040 self.assertRaises(TypeError, codecs.charmap_encode, "\xff", err, {0xff: 300})
1041
1042 def test_decodehelper_bug36819(self):
1043 handler = RepeatedPosReturn("x")
1044 codecs.register_error("test.bug36819", handler.handle)
1045
1046 testcases = [
1047 ("ascii", b"\xff"),
1048 ("utf-8", b"\xff"),
1049 ("utf-16be", b'\xdc\x80'),
1050 ("utf-32be", b'\x00\x00\xdc\x80'),
1051 ("iso-8859-6", b"\xff"),
1052 ]
1053 for enc, bad in testcases:
1054 input = "abcd".encode(enc) + bad
1055 with self.subTest(encoding=enc):
1056 handler.count = 50
1057 decoded = input.decode(enc, "test.bug36819")
1058 self.assertEqual(decoded, 'abcdx' * 51)
1059
1060 def test_encodehelper_bug36819(self):
1061 handler = RepeatedPosReturn()
1062 codecs.register_error("test.bug36819", handler.handle)
1063
1064 input = "abcd\udc80"
1065 encodings = ["ascii", "latin1", "utf-8", "utf-16", "utf-32"] # built-in
1066 encodings += ["iso-8859-15"] # charmap codec
1067 if sys.platform == 'win32':
1068 encodings = ["mbcs", "oem"] # code page codecs
1069
1070 handler.repl = "\udcff"
1071 for enc in encodings:
1072 with self.subTest(encoding=enc):
1073 handler.count = 50
1074 with self.assertRaises(UnicodeEncodeError) as cm:
1075 input.encode(enc, "test.bug36819")
1076 exc = cm.exception
1077 self.assertEqual(exc.start, 4)
1078 self.assertEqual(exc.end, 5)
1079 self.assertEqual(exc.object, input)
1080 if sys.platform == "win32":
1081 handler.count = 50
1082 with self.assertRaises(UnicodeEncodeError) as cm:
1083 codecs.code_page_encode(437, input, "test.bug36819")
1084 exc = cm.exception
1085 self.assertEqual(exc.start, 4)
1086 self.assertEqual(exc.end, 5)
1087 self.assertEqual(exc.object, input)
1088
1089 handler.repl = "x"
1090 for enc in encodings:
1091 with self.subTest(encoding=enc):
1092 # The interpreter should segfault after a handful of attempts.
1093 # 50 was chosen to try to ensure a segfault without a fix,
1094 # but not OOM a machine with one.
1095 handler.count = 50
1096 encoded = input.encode(enc, "test.bug36819")
1097 self.assertEqual(encoded.decode(enc), "abcdx" * 51)
1098 if sys.platform == "win32":
1099 handler.count = 50
1100 encoded = codecs.code_page_encode(437, input, "test.bug36819")
1101 self.assertEqual(encoded[0].decode(), "abcdx" * 51)
1102 self.assertEqual(encoded[1], len(input))
1103
1104 def test_translatehelper(self):
1105 # enhance coverage of:
1106 # Objects/unicodeobject.c::unicode_encode_call_errorhandler()
1107 # and callers
1108 # (Unfortunately the errors argument is not directly accessible
1109 # from Python, so we can't test that much)
1110 class D(dict):
1111 def __getitem__(self, key):
1112 raise ValueError
1113 #self.assertRaises(ValueError, "\xff".translate, D())
1114 self.assertRaises(ValueError, "\xff".translate, {0xff: sys.maxunicode+1})
1115 self.assertRaises(TypeError, "\xff".translate, {0xff: ()})
1116
1117 def test_bug828737(self):
1118 charmap = {
1119 ord("&"): "&amp;",
1120 ord("<"): "&lt;",
1121 ord(">"): "&gt;",
1122 ord('"'): "&quot;",
1123 }
1124
1125 for n in (1, 10, 100, 1000):
1126 text = 'abc<def>ghi'*n
1127 text.translate(charmap)
1128
1129 def test_mutating_decode_handler(self):
1130 baddata = [
1131 ("ascii", b"\xff"),
1132 ("utf-7", b"++"),
1133 ("utf-8", b"\xff"),
1134 ("utf-16", b"\xff"),
1135 ("utf-32", b"\xff"),
1136 ("unicode-escape", b"\\u123g"),
1137 ("raw-unicode-escape", b"\\u123g"),
1138 ]
1139
1140 def replacing(exc):
1141 if isinstance(exc, UnicodeDecodeError):
1142 exc.object = 42
1143 return ("\u4242", 0)
1144 else:
1145 raise TypeError("don't know how to handle %r" % exc)
1146 codecs.register_error("test.replacing", replacing)
1147
1148 for (encoding, data) in baddata:
1149 with self.assertRaises(TypeError):
1150 data.decode(encoding, "test.replacing")
1151
1152 def mutating(exc):
1153 if isinstance(exc, UnicodeDecodeError):
1154 exc.object = b""
1155 return ("\u4242", 0)
1156 else:
1157 raise TypeError("don't know how to handle %r" % exc)
1158 codecs.register_error("test.mutating", mutating)
1159 # If the decoder doesn't pick up the modified input the following
1160 # will lead to an endless loop
1161 for (encoding, data) in baddata:
1162 self.assertEqual(data.decode(encoding, "test.mutating"), "\u4242")
1163
1164 def test_mutating_decode_handler_unicode_escape(self):
1165 decode = codecs.unicode_escape_decode
1166 def mutating(exc):
1167 if isinstance(exc, UnicodeDecodeError):
1168 r = data.get(exc.object[:exc.end])
1169 if r is not None:
1170 exc.object = r[0] + exc.object[exc.end:]
1171 return ('\u0404', r[1])
1172 raise AssertionError("don't know how to handle %r" % exc)
1173
1174 codecs.register_error('test.mutating2', mutating)
1175 data = {
1176 br'\x0': (b'\\', 0),
1177 br'\x3': (b'xxx\\', 3),
1178 br'\x5': (b'x\\', 1),
1179 }
1180 def check(input, expected, msg):
1181 with self.assertWarns(DeprecationWarning) as cm:
1182 self.assertEqual(decode(input, 'test.mutating2'), (expected, len(input)))
1183 self.assertIn(msg, str(cm.warning))
1184
1185 check(br'\x0n\z', '\u0404\n\\z', r'"\z" is an invalid escape sequence')
1186 check(br'\x0n\501', '\u0404\n\u0141', r'"\501" is an invalid octal escape sequence')
1187 check(br'\x0z', '\u0404\\z', r'"\z" is an invalid escape sequence')
1188
1189 check(br'\x3n\zr', '\u0404\n\\zr', r'"\z" is an invalid escape sequence')
1190 check(br'\x3zr', '\u0404\\zr', r'"\z" is an invalid escape sequence')
1191 check(br'\x3z5', '\u0404\\z5', r'"\z" is an invalid escape sequence')
1192 check(memoryview(br'\x3z5x')[:-1], '\u0404\\z5', r'"\z" is an invalid escape sequence')
1193 check(memoryview(br'\x3z5xy')[:-2], '\u0404\\z5', r'"\z" is an invalid escape sequence')
1194
1195 check(br'\x5n\z', '\u0404\n\\z', r'"\z" is an invalid escape sequence')
1196 check(br'\x5n\501', '\u0404\n\u0141', r'"\501" is an invalid octal escape sequence')
1197 check(br'\x5z', '\u0404\\z', r'"\z" is an invalid escape sequence')
1198 check(memoryview(br'\x5zy')[:-1], '\u0404\\z', r'"\z" is an invalid escape sequence')
1199
1200 # issue32583
1201 def test_crashing_decode_handler(self):
1202 # better generating one more character to fill the extra space slot
1203 # so in debug build it can steadily fail
1204 def forward_shorter_than_end(exc):
1205 if isinstance(exc, UnicodeDecodeError):
1206 # size one character, 0 < forward < exc.end
1207 return ('\ufffd', exc.start+1)
1208 else:
1209 raise TypeError("don't know how to handle %r" % exc)
1210 codecs.register_error(
1211 "test.forward_shorter_than_end", forward_shorter_than_end)
1212
1213 self.assertEqual(
1214 b'\xd8\xd8\xd8\xd8\xd8\x00\x00\x00'.decode(
1215 'utf-16-le', 'test.forward_shorter_than_end'),
1216 '\ufffd\ufffd\ufffd\ufffd\xd8\x00'
1217 )
1218 self.assertEqual(
1219 b'\xd8\xd8\xd8\xd8\x00\xd8\x00\x00'.decode(
1220 'utf-16-be', 'test.forward_shorter_than_end'),
1221 '\ufffd\ufffd\ufffd\ufffd\xd8\x00'
1222 )
1223 self.assertEqual(
1224 b'\x11\x11\x11\x11\x11\x00\x00\x00\x00\x00\x00'.decode(
1225 'utf-32-le', 'test.forward_shorter_than_end'),
1226 '\ufffd\ufffd\ufffd\u1111\x00'
1227 )
1228 self.assertEqual(
1229 b'\x11\x11\x11\x00\x00\x11\x11\x00\x00\x00\x00'.decode(
1230 'utf-32-be', 'test.forward_shorter_than_end'),
1231 '\ufffd\ufffd\ufffd\u1111\x00'
1232 )
1233
1234 def replace_with_long(exc):
1235 if isinstance(exc, UnicodeDecodeError):
1236 exc.object = b"\x00" * 8
1237 return ('\ufffd', exc.start)
1238 else:
1239 raise TypeError("don't know how to handle %r" % exc)
1240 codecs.register_error("test.replace_with_long", replace_with_long)
1241
1242 self.assertEqual(
1243 b'\x00'.decode('utf-16', 'test.replace_with_long'),
1244 '\ufffd\x00\x00\x00\x00'
1245 )
1246 self.assertEqual(
1247 b'\x00'.decode('utf-32', 'test.replace_with_long'),
1248 '\ufffd\x00\x00'
1249 )
1250
1251 def test_fake_error_class(self):
1252 handlers = [
1253 codecs.strict_errors,
1254 codecs.ignore_errors,
1255 codecs.replace_errors,
1256 codecs.backslashreplace_errors,
1257 codecs.namereplace_errors,
1258 codecs.xmlcharrefreplace_errors,
1259 codecs.lookup_error('surrogateescape'),
1260 codecs.lookup_error('surrogatepass'),
1261 ]
1262 for cls in UnicodeEncodeError, UnicodeDecodeError, UnicodeTranslateError:
1263 class FakeUnicodeError(str):
1264 __class__ = cls
1265 for handler in handlers:
1266 with self.subTest(handler=handler, error_class=cls):
1267 self.assertRaises(TypeError, handler, FakeUnicodeError())
1268 class FakeUnicodeError(Exception):
1269 __class__ = cls
1270 for handler in handlers:
1271 with self.subTest(handler=handler, error_class=cls):
1272 with self.assertRaises((TypeError, FakeUnicodeError)):
1273 handler(FakeUnicodeError())
1274
1275 def test_reject_unregister_builtin_error_handler(self):
1276 for name in [
1277 'strict', 'ignore', 'replace', 'backslashreplace', 'namereplace',
1278 'xmlcharrefreplace', 'surrogateescape', 'surrogatepass',
1279 ]:
1280 with self.subTest(name):
1281 self.assertRaises(ValueError, _codecs_unregister_error, name)
1282
1283 def test_unregister_custom_error_handler(self):
1284 def custom_handler(exc):
1285 raise exc
1286
1287 custom_name = 'test.test_unregister_custom_error_handler'
1288 self.assertRaises(LookupError, codecs.lookup_error, custom_name)
1289 codecs.register_error(custom_name, custom_handler)
1290 self.assertIs(codecs.lookup_error(custom_name), custom_handler)
1291 self.assertTrue(_codecs_unregister_error(custom_name))
1292 self.assertRaises(LookupError, codecs.lookup_error, custom_name)
1293
1294 def test_unregister_custom_unknown_error_handler(self):
1295 unknown_name = 'test.test_unregister_custom_unknown_error_handler'
1296 self.assertRaises(LookupError, codecs.lookup_error, unknown_name)
1297 self.assertFalse(_codecs_unregister_error(unknown_name))
1298 self.assertRaises(LookupError, codecs.lookup_error, unknown_name)
1299
1300
1301 if __name__ == "__main__":
1302 unittest.main()