| 1 |
from _codecs import _unregister_error as _codecs_unregister_error |
| 2 |
import codecs |
| 3 |
import html.entities |
| 4 |
import itertools |
| 5 |
import re |
| 6 |
import sys |
| 7 |
import unicodedata |
| 8 |
import unittest |
| 9 |
|
| 10 |
|
| 11 |
class PosReturn: |
| 12 |
# this can be used for configurable callbacks |
| 13 |
|
| 14 |
def __init__(self): |
| 15 |
self.pos = 0 |
| 16 |
|
| 17 |
def handle(self, exc): |
| 18 |
oldpos = self.pos |
| 19 |
realpos = oldpos |
| 20 |
if realpos<0: |
| 21 |
realpos = len(exc.object) + realpos |
| 22 |
# if we don't advance this time, terminate on the next call |
| 23 |
# otherwise we'd get an endless loop |
| 24 |
if realpos <= exc.start: |
| 25 |
self.pos = len(exc.object) |
| 26 |
return ("<?>", oldpos) |
| 27 |
|
| 28 |
class RepeatedPosReturn: |
| 29 |
def __init__(self, repl="<?>"): |
| 30 |
self.repl = repl |
| 31 |
self.pos = 0 |
| 32 |
self.count = 0 |
| 33 |
|
| 34 |
def handle(self, exc): |
| 35 |
if self.count > 0: |
| 36 |
self.count -= 1 |
| 37 |
return (self.repl, self.pos) |
| 38 |
return (self.repl, exc.end) |
| 39 |
|
| 40 |
# A UnicodeEncodeError object with a bad start attribute |
| 41 |
class BadStartUnicodeEncodeError(UnicodeEncodeError): |
| 42 |
def __init__(self): |
| 43 |
UnicodeEncodeError.__init__(self, "ascii", "", 0, 1, "bad") |
| 44 |
self.start = [] |
| 45 |
|
| 46 |
# A UnicodeEncodeError object with a bad object attribute |
| 47 |
class BadObjectUnicodeEncodeError(UnicodeEncodeError): |
| 48 |
def __init__(self): |
| 49 |
UnicodeEncodeError.__init__(self, "ascii", "", 0, 1, "bad") |
| 50 |
self.object = [] |
| 51 |
|
| 52 |
# A UnicodeDecodeError object without an end attribute |
| 53 |
class NoEndUnicodeDecodeError(UnicodeDecodeError): |
| 54 |
def __init__(self): |
| 55 |
UnicodeDecodeError.__init__(self, "ascii", bytearray(b""), 0, 1, "bad") |
| 56 |
del self.end |
| 57 |
|
| 58 |
# A UnicodeDecodeError object with a bad object attribute |
| 59 |
class BadObjectUnicodeDecodeError(UnicodeDecodeError): |
| 60 |
def __init__(self): |
| 61 |
UnicodeDecodeError.__init__(self, "ascii", bytearray(b""), 0, 1, "bad") |
| 62 |
self.object = [] |
| 63 |
|
| 64 |
# A UnicodeTranslateError object without a start attribute |
| 65 |
class NoStartUnicodeTranslateError(UnicodeTranslateError): |
| 66 |
def __init__(self): |
| 67 |
UnicodeTranslateError.__init__(self, "", 0, 1, "bad") |
| 68 |
del self.start |
| 69 |
|
| 70 |
# A UnicodeTranslateError object without an end attribute |
| 71 |
class NoEndUnicodeTranslateError(UnicodeTranslateError): |
| 72 |
def __init__(self): |
| 73 |
UnicodeTranslateError.__init__(self, "", 0, 1, "bad") |
| 74 |
del self.end |
| 75 |
|
| 76 |
# A UnicodeTranslateError object without an object attribute |
| 77 |
class NoObjectUnicodeTranslateError(UnicodeTranslateError): |
| 78 |
def __init__(self): |
| 79 |
UnicodeTranslateError.__init__(self, "", 0, 1, "bad") |
| 80 |
del self.object |
| 81 |
|
| 82 |
class CodecCallbackTest(unittest.TestCase): |
| 83 |
|
| 84 |
def test_xmlcharrefreplace(self): |
| 85 |
# replace unencodable characters which numeric character entities. |
| 86 |
# For ascii, latin-1 and charmaps this is completely implemented |
| 87 |
# in C and should be reasonably fast. |
| 88 |
s = "\u30b9\u30d1\u30e2 \xe4nd eggs" |
| 89 |
self.assertEqual( |
| 90 |
s.encode("ascii", "xmlcharrefreplace"), |
| 91 |
b"スパモ änd eggs" |
| 92 |
) |
| 93 |
self.assertEqual( |
| 94 |
s.encode("latin-1", "xmlcharrefreplace"), |
| 95 |
b"スパモ \xe4nd eggs" |
| 96 |
) |
| 97 |
|
| 98 |
def test_xmlcharnamereplace(self): |
| 99 |
# This time use a named character entity for unencodable |
| 100 |
# characters, if one is available. |
| 101 |
|
| 102 |
def xmlcharnamereplace(exc): |
| 103 |
if not isinstance(exc, UnicodeEncodeError): |
| 104 |
raise TypeError("don't know how to handle %r" % exc) |
| 105 |
l = [] |
| 106 |
for c in exc.object[exc.start:exc.end]: |
| 107 |
try: |
| 108 |
l.append("&%s;" % html.entities.codepoint2name[ord(c)]) |
| 109 |
except KeyError: |
| 110 |
l.append("&#%d;" % ord(c)) |
| 111 |
return ("".join(l), exc.end) |
| 112 |
|
| 113 |
codecs.register_error( |
| 114 |
"test.xmlcharnamereplace", xmlcharnamereplace) |
| 115 |
|
| 116 |
sin = "\xab\u211c\xbb = \u2329\u1234\u20ac\u232a" |
| 117 |
sout = b"«ℜ» = ⟨ሴ€⟩" |
| 118 |
self.assertEqual(sin.encode("ascii", "test.xmlcharnamereplace"), sout) |
| 119 |
sout = b"\xabℜ\xbb = ⟨ሴ€⟩" |
| 120 |
self.assertEqual(sin.encode("latin-1", "test.xmlcharnamereplace"), sout) |
| 121 |
sout = b"\xabℜ\xbb = ⟨ሴ\xa4⟩" |
| 122 |
self.assertEqual(sin.encode("iso-8859-15", "test.xmlcharnamereplace"), sout) |
| 123 |
|
| 124 |
def test_uninamereplace(self): |
| 125 |
# We're using the names from the unicode database this time, |
| 126 |
# and we're doing "syntax highlighting" here, i.e. we include |
| 127 |
# the replaced text in ANSI escape sequences. For this it is |
| 128 |
# useful that the error handler is not called for every single |
| 129 |
# unencodable character, but for a complete sequence of |
| 130 |
# unencodable characters, otherwise we would output many |
| 131 |
# unnecessary escape sequences. |
| 132 |
|
| 133 |
def uninamereplace(exc): |
| 134 |
if not isinstance(exc, UnicodeEncodeError): |
| 135 |
raise TypeError("don't know how to handle %r" % exc) |
| 136 |
l = [] |
| 137 |
for c in exc.object[exc.start:exc.end]: |
| 138 |
l.append(unicodedata.name(c, "0x%x" % ord(c))) |
| 139 |
return ("\033[1m%s\033[0m" % ", ".join(l), exc.end) |
| 140 |
|
| 141 |
codecs.register_error( |
| 142 |
"test.uninamereplace", uninamereplace) |
| 143 |
|
| 144 |
sin = "\xac\u1234\u20ac\u8000" |
| 145 |
sout = b"\033[1mNOT SIGN, ETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m" |
| 146 |
self.assertEqual(sin.encode("ascii", "test.uninamereplace"), sout) |
| 147 |
|
| 148 |
sout = b"\xac\033[1mETHIOPIC SYLLABLE SEE, EURO SIGN, CJK UNIFIED IDEOGRAPH-8000\033[0m" |
| 149 |
self.assertEqual(sin.encode("latin-1", "test.uninamereplace"), sout) |
| 150 |
|
| 151 |
sout = b"\xac\033[1mETHIOPIC SYLLABLE SEE\033[0m\xa4\033[1mCJK UNIFIED IDEOGRAPH-8000\033[0m" |
| 152 |
self.assertEqual(sin.encode("iso-8859-15", "test.uninamereplace"), sout) |
| 153 |
|
| 154 |
def test_backslashescape(self): |
| 155 |
# Does the same as the "unicode-escape" encoding, but with different |
| 156 |
# base encodings. |
| 157 |
sin = "a\xac\u1234\u20ac\u8000\U0010ffff" |
| 158 |
sout = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" |
| 159 |
self.assertEqual(sin.encode("ascii", "backslashreplace"), sout) |
| 160 |
|
| 161 |
sout = b"a\xac\\u1234\\u20ac\\u8000\\U0010ffff" |
| 162 |
self.assertEqual(sin.encode("latin-1", "backslashreplace"), sout) |
| 163 |
|
| 164 |
sout = b"a\xac\\u1234\xa4\\u8000\\U0010ffff" |
| 165 |
self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout) |
| 166 |
|
| 167 |
def test_nameescape(self): |
| 168 |
# Does the same as backslashescape, but prefers ``\N{...}`` escape |
| 169 |
# sequences. |
| 170 |
sin = "a\xac\u1234\u20ac\u8000\U0010ffff" |
| 171 |
sout = (b'a\\N{NOT SIGN}\\N{ETHIOPIC SYLLABLE SEE}\\N{EURO SIGN}' |
| 172 |
b'\\N{CJK UNIFIED IDEOGRAPH-8000}\\U0010ffff') |
| 173 |
self.assertEqual(sin.encode("ascii", "namereplace"), sout) |
| 174 |
|
| 175 |
sout = (b'a\xac\\N{ETHIOPIC SYLLABLE SEE}\\N{EURO SIGN}' |
| 176 |
b'\\N{CJK UNIFIED IDEOGRAPH-8000}\\U0010ffff') |
| 177 |
self.assertEqual(sin.encode("latin-1", "namereplace"), sout) |
| 178 |
|
| 179 |
sout = (b'a\xac\\N{ETHIOPIC SYLLABLE SEE}\xa4' |
| 180 |
b'\\N{CJK UNIFIED IDEOGRAPH-8000}\\U0010ffff') |
| 181 |
self.assertEqual(sin.encode("iso-8859-15", "namereplace"), sout) |
| 182 |
|
| 183 |
def test_decoding_callbacks(self): |
| 184 |
# This is a test for a decoding callback handler |
| 185 |
# that allows the decoding of the invalid sequence |
| 186 |
# "\xc0\x80" and returns "\x00" instead of raising an error. |
| 187 |
# All other illegal sequences will be handled strictly. |
| 188 |
def relaxedutf8(exc): |
| 189 |
if not isinstance(exc, UnicodeDecodeError): |
| 190 |
raise TypeError("don't know how to handle %r" % exc) |
| 191 |
if exc.object[exc.start:exc.start+2] == b"\xc0\x80": |
| 192 |
return ("\x00", exc.start+2) # retry after two bytes |
| 193 |
else: |
| 194 |
raise exc |
| 195 |
|
| 196 |
codecs.register_error("test.relaxedutf8", relaxedutf8) |
| 197 |
|
| 198 |
# all the "\xc0\x80" will be decoded to "\x00" |
| 199 |
sin = b"a\x00b\xc0\x80c\xc3\xbc\xc0\x80\xc0\x80" |
| 200 |
sout = "a\x00b\x00c\xfc\x00\x00" |
| 201 |
self.assertEqual(sin.decode("utf-8", "test.relaxedutf8"), sout) |
| 202 |
|
| 203 |
# "\xc0\x81" is not valid and a UnicodeDecodeError will be raised |
| 204 |
sin = b"\xc0\x80\xc0\x81" |
| 205 |
self.assertRaises(UnicodeDecodeError, sin.decode, |
| 206 |
"utf-8", "test.relaxedutf8") |
| 207 |
|
| 208 |
def test_charmapencode(self): |
| 209 |
# For charmap encodings the replacement string will be |
| 210 |
# mapped through the encoding again. This means, that |
| 211 |
# to be able to use e.g. the "replace" handler, the |
| 212 |
# charmap has to have a mapping for "?". |
| 213 |
charmap = dict((ord(c), bytes(2*c.upper(), 'ascii')) for c in "abcdefgh") |
| 214 |
sin = "abc" |
| 215 |
sout = b"AABBCC" |
| 216 |
self.assertEqual(codecs.charmap_encode(sin, "strict", charmap)[0], sout) |
| 217 |
|
| 218 |
sin = "abcA" |
| 219 |
self.assertRaises(UnicodeError, codecs.charmap_encode, sin, "strict", charmap) |
| 220 |
|
| 221 |
charmap[ord("?")] = b"XYZ" |
| 222 |
sin = "abcDEF" |
| 223 |
sout = b"AABBCCXYZXYZXYZ" |
| 224 |
self.assertEqual(codecs.charmap_encode(sin, "replace", charmap)[0], sout) |
| 225 |
|
| 226 |
charmap[ord("?")] = "XYZ" # wrong type in mapping |
| 227 |
self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap) |
| 228 |
|
| 229 |
def test_callbacks(self): |
| 230 |
def handler1(exc): |
| 231 |
r = range(exc.start, exc.end) |
| 232 |
if isinstance(exc, UnicodeEncodeError): |
| 233 |
l = ["<%d>" % ord(exc.object[pos]) for pos in r] |
| 234 |
elif isinstance(exc, UnicodeDecodeError): |
| 235 |
l = ["<%d>" % exc.object[pos] for pos in r] |
| 236 |
else: |
| 237 |
raise TypeError("don't know how to handle %r" % exc) |
| 238 |
return ("[%s]" % "".join(l), exc.end) |
| 239 |
|
| 240 |
codecs.register_error("test.handler1", handler1) |
| 241 |
|
| 242 |
def handler2(exc): |
| 243 |
if not isinstance(exc, UnicodeDecodeError): |
| 244 |
raise TypeError("don't know how to handle %r" % exc) |
| 245 |
l = ["<%d>" % exc.object[pos] for pos in range(exc.start, exc.end)] |
| 246 |
return ("[%s]" % "".join(l), exc.end+1) # skip one character |
| 247 |
|
| 248 |
codecs.register_error("test.handler2", handler2) |
| 249 |
|
| 250 |
s = b"\x00\x81\x7f\x80\xff" |
| 251 |
|
| 252 |
self.assertEqual( |
| 253 |
s.decode("ascii", "test.handler1"), |
| 254 |
"\x00[<129>]\x7f[<128>][<255>]" |
| 255 |
) |
| 256 |
self.assertEqual( |
| 257 |
s.decode("ascii", "test.handler2"), |
| 258 |
"\x00[<129>][<128>]" |
| 259 |
) |
| 260 |
|
| 261 |
self.assertEqual( |
| 262 |
b"\\u3042\\u3xxx".decode("unicode-escape", "test.handler1"), |
| 263 |
"\u3042[<92><117><51>]xxx" |
| 264 |
) |
| 265 |
|
| 266 |
self.assertEqual( |
| 267 |
b"\\u3042\\u3xx".decode("unicode-escape", "test.handler1"), |
| 268 |
"\u3042[<92><117><51>]xx" |
| 269 |
) |
| 270 |
|
| 271 |
self.assertEqual( |
| 272 |
codecs.charmap_decode(b"abc", "test.handler1", {ord("a"): "z"})[0], |
| 273 |
"z[<98>][<99>]" |
| 274 |
) |
| 275 |
|
| 276 |
self.assertEqual( |
| 277 |
"g\xfc\xdfrk".encode("ascii", "test.handler1"), |
| 278 |
b"g[<252><223>]rk" |
| 279 |
) |
| 280 |
|
| 281 |
self.assertEqual( |
| 282 |
"g\xfc\xdf".encode("ascii", "test.handler1"), |
| 283 |
b"g[<252><223>]" |
| 284 |
) |
| 285 |
|
| 286 |
def test_longstrings(self): |
| 287 |
# test long strings to check for memory overflow problems |
| 288 |
errors = [ "strict", "ignore", "replace", "xmlcharrefreplace", |
| 289 |
"backslashreplace", "namereplace"] |
| 290 |
# register the handlers under different names, |
| 291 |
# to prevent the codec from recognizing the name |
| 292 |
for err in errors: |
| 293 |
codecs.register_error("test." + err, codecs.lookup_error(err)) |
| 294 |
l = 1000 |
| 295 |
errors += [ "test." + err for err in errors ] |
| 296 |
for uni in [ s*l for s in ("x", "\u3042", "a\xe4") ]: |
| 297 |
for enc in ("ascii", "latin-1", "iso-8859-1", "iso-8859-15", |
| 298 |
"utf-8", "utf-7", "utf-16", "utf-32"): |
| 299 |
for err in errors: |
| 300 |
try: |
| 301 |
uni.encode(enc, err) |
| 302 |
except UnicodeError: |
| 303 |
pass |
| 304 |
|
| 305 |
def check_exceptionobjectargs(self, exctype, args, msg): |
| 306 |
# Test UnicodeError subclasses: construction, attribute assignment and __str__ conversion |
| 307 |
# check with one missing argument |
| 308 |
self.assertRaises(TypeError, exctype, *args[:-1]) |
| 309 |
# check with one argument too much |
| 310 |
self.assertRaises(TypeError, exctype, *(args + ["too much"])) |
| 311 |
# check with one argument of the wrong type |
| 312 |
wrongargs = [ "spam", b"eggs", b"spam", 42, 1.0, None ] |
| 313 |
for i in range(len(args)): |
| 314 |
for wrongarg in wrongargs: |
| 315 |
if type(wrongarg) is type(args[i]): |
| 316 |
continue |
| 317 |
# build argument array |
| 318 |
callargs = [] |
| 319 |
for j in range(len(args)): |
| 320 |
if i==j: |
| 321 |
callargs.append(wrongarg) |
| 322 |
else: |
| 323 |
callargs.append(args[i]) |
| 324 |
self.assertRaises(TypeError, exctype, *callargs) |
| 325 |
|
| 326 |
# check with the correct number and type of arguments |
| 327 |
exc = exctype(*args) |
| 328 |
self.assertEqual(str(exc), msg) |
| 329 |
|
| 330 |
def test_unicodeencodeerror(self): |
| 331 |
self.check_exceptionobjectargs( |
| 332 |
UnicodeEncodeError, |
| 333 |
["ascii", "g\xfcrk", 1, 2, "ouch"], |
| 334 |
"'ascii' codec can't encode character '\\xfc' in position 1: ouch" |
| 335 |
) |
| 336 |
self.check_exceptionobjectargs( |
| 337 |
UnicodeEncodeError, |
| 338 |
["ascii", "g\xfcrk", 1, 4, "ouch"], |
| 339 |
"'ascii' codec can't encode characters in position 1-3: ouch" |
| 340 |
) |
| 341 |
self.check_exceptionobjectargs( |
| 342 |
UnicodeEncodeError, |
| 343 |
["ascii", "\xfcx", 0, 1, "ouch"], |
| 344 |
"'ascii' codec can't encode character '\\xfc' in position 0: ouch" |
| 345 |
) |
| 346 |
self.check_exceptionobjectargs( |
| 347 |
UnicodeEncodeError, |
| 348 |
["ascii", "\u0100x", 0, 1, "ouch"], |
| 349 |
"'ascii' codec can't encode character '\\u0100' in position 0: ouch" |
| 350 |
) |
| 351 |
self.check_exceptionobjectargs( |
| 352 |
UnicodeEncodeError, |
| 353 |
["ascii", "\uffffx", 0, 1, "ouch"], |
| 354 |
"'ascii' codec can't encode character '\\uffff' in position 0: ouch" |
| 355 |
) |
| 356 |
self.check_exceptionobjectargs( |
| 357 |
UnicodeEncodeError, |
| 358 |
["ascii", "\U00010000x", 0, 1, "ouch"], |
| 359 |
"'ascii' codec can't encode character '\\U00010000' in position 0: ouch" |
| 360 |
) |
| 361 |
|
| 362 |
def test_unicodedecodeerror(self): |
| 363 |
self.check_exceptionobjectargs( |
| 364 |
UnicodeDecodeError, |
| 365 |
["ascii", bytearray(b"g\xfcrk"), 1, 2, "ouch"], |
| 366 |
"'ascii' codec can't decode byte 0xfc in position 1: ouch" |
| 367 |
) |
| 368 |
self.check_exceptionobjectargs( |
| 369 |
UnicodeDecodeError, |
| 370 |
["ascii", bytearray(b"g\xfcrk"), 1, 3, "ouch"], |
| 371 |
"'ascii' codec can't decode bytes in position 1-2: ouch" |
| 372 |
) |
| 373 |
|
| 374 |
def test_unicodetranslateerror(self): |
| 375 |
self.check_exceptionobjectargs( |
| 376 |
UnicodeTranslateError, |
| 377 |
["g\xfcrk", 1, 2, "ouch"], |
| 378 |
"can't translate character '\\xfc' in position 1: ouch" |
| 379 |
) |
| 380 |
self.check_exceptionobjectargs( |
| 381 |
UnicodeTranslateError, |
| 382 |
["g\u0100rk", 1, 2, "ouch"], |
| 383 |
"can't translate character '\\u0100' in position 1: ouch" |
| 384 |
) |
| 385 |
self.check_exceptionobjectargs( |
| 386 |
UnicodeTranslateError, |
| 387 |
["g\uffffrk", 1, 2, "ouch"], |
| 388 |
"can't translate character '\\uffff' in position 1: ouch" |
| 389 |
) |
| 390 |
self.check_exceptionobjectargs( |
| 391 |
UnicodeTranslateError, |
| 392 |
["g\U00010000rk", 1, 2, "ouch"], |
| 393 |
"can't translate character '\\U00010000' in position 1: ouch" |
| 394 |
) |
| 395 |
self.check_exceptionobjectargs( |
| 396 |
UnicodeTranslateError, |
| 397 |
["g\xfcrk", 1, 3, "ouch"], |
| 398 |
"can't translate characters in position 1-2: ouch" |
| 399 |
) |
| 400 |
|
| 401 |
def test_badandgoodstrictexceptions(self): |
| 402 |
# "strict" complains about a non-exception passed in |
| 403 |
self.assertRaises( |
| 404 |
TypeError, |
| 405 |
codecs.strict_errors, |
| 406 |
42 |
| 407 |
) |
| 408 |
# "strict" complains about the wrong exception type |
| 409 |
self.assertRaises( |
| 410 |
Exception, |
| 411 |
codecs.strict_errors, |
| 412 |
Exception("ouch") |
| 413 |
) |
| 414 |
|
| 415 |
# If the correct exception is passed in, "strict" raises it |
| 416 |
self.assertRaises( |
| 417 |
UnicodeEncodeError, |
| 418 |
codecs.strict_errors, |
| 419 |
UnicodeEncodeError("ascii", "\u3042", 0, 1, "ouch") |
| 420 |
) |
| 421 |
self.assertRaises( |
| 422 |
UnicodeDecodeError, |
| 423 |
codecs.strict_errors, |
| 424 |
UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch") |
| 425 |
) |
| 426 |
self.assertRaises( |
| 427 |
UnicodeTranslateError, |
| 428 |
codecs.strict_errors, |
| 429 |
UnicodeTranslateError("\u3042", 0, 1, "ouch") |
| 430 |
) |
| 431 |
|
| 432 |
def test_badandgoodignoreexceptions(self): |
| 433 |
# "ignore" complains about a non-exception passed in |
| 434 |
self.assertRaises( |
| 435 |
TypeError, |
| 436 |
codecs.ignore_errors, |
| 437 |
42 |
| 438 |
) |
| 439 |
# "ignore" complains about the wrong exception type |
| 440 |
self.assertRaises( |
| 441 |
TypeError, |
| 442 |
codecs.ignore_errors, |
| 443 |
UnicodeError("ouch") |
| 444 |
) |
| 445 |
# If the correct exception is passed in, "ignore" returns an empty replacement |
| 446 |
self.assertEqual( |
| 447 |
codecs.ignore_errors( |
| 448 |
UnicodeEncodeError("ascii", "a\u3042b", 1, 2, "ouch")), |
| 449 |
("", 2) |
| 450 |
) |
| 451 |
self.assertEqual( |
| 452 |
codecs.ignore_errors( |
| 453 |
UnicodeDecodeError("ascii", bytearray(b"a\xffb"), 1, 2, "ouch")), |
| 454 |
("", 2) |
| 455 |
) |
| 456 |
self.assertEqual( |
| 457 |
codecs.ignore_errors( |
| 458 |
UnicodeTranslateError("a\u3042b", 1, 2, "ouch")), |
| 459 |
("", 2) |
| 460 |
) |
| 461 |
|
| 462 |
def test_badandgoodreplaceexceptions(self): |
| 463 |
# "replace" complains about a non-exception passed in |
| 464 |
self.assertRaises( |
| 465 |
TypeError, |
| 466 |
codecs.replace_errors, |
| 467 |
42 |
| 468 |
) |
| 469 |
# "replace" complains about the wrong exception type |
| 470 |
self.assertRaises( |
| 471 |
TypeError, |
| 472 |
codecs.replace_errors, |
| 473 |
UnicodeError("ouch") |
| 474 |
) |
| 475 |
self.assertRaises( |
| 476 |
TypeError, |
| 477 |
codecs.replace_errors, |
| 478 |
BadObjectUnicodeEncodeError() |
| 479 |
) |
| 480 |
self.assertRaises( |
| 481 |
TypeError, |
| 482 |
codecs.replace_errors, |
| 483 |
BadObjectUnicodeDecodeError() |
| 484 |
) |
| 485 |
# With the correct exception, "replace" returns an "?" or "\ufffd" replacement |
| 486 |
self.assertEqual( |
| 487 |
codecs.replace_errors( |
| 488 |
UnicodeEncodeError("ascii", "a\u3042b", 1, 2, "ouch")), |
| 489 |
("?", 2) |
| 490 |
) |
| 491 |
self.assertEqual( |
| 492 |
codecs.replace_errors( |
| 493 |
UnicodeDecodeError("ascii", bytearray(b"a\xffb"), 1, 2, "ouch")), |
| 494 |
("\ufffd", 2) |
| 495 |
) |
| 496 |
self.assertEqual( |
| 497 |
codecs.replace_errors( |
| 498 |
UnicodeTranslateError("a\u3042b", 1, 2, "ouch")), |
| 499 |
("\ufffd", 2) |
| 500 |
) |
| 501 |
|
| 502 |
def test_badandgoodxmlcharrefreplaceexceptions(self): |
| 503 |
# "xmlcharrefreplace" complains about a non-exception passed in |
| 504 |
self.assertRaises( |
| 505 |
TypeError, |
| 506 |
codecs.xmlcharrefreplace_errors, |
| 507 |
42 |
| 508 |
) |
| 509 |
# "xmlcharrefreplace" complains about the wrong exception types |
| 510 |
self.assertRaises( |
| 511 |
TypeError, |
| 512 |
codecs.xmlcharrefreplace_errors, |
| 513 |
UnicodeError("ouch") |
| 514 |
) |
| 515 |
# "xmlcharrefreplace" can only be used for encoding |
| 516 |
self.assertRaises( |
| 517 |
TypeError, |
| 518 |
codecs.xmlcharrefreplace_errors, |
| 519 |
UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch") |
| 520 |
) |
| 521 |
self.assertRaises( |
| 522 |
TypeError, |
| 523 |
codecs.xmlcharrefreplace_errors, |
| 524 |
UnicodeTranslateError("\u3042", 0, 1, "ouch") |
| 525 |
) |
| 526 |
# Use the correct exception |
| 527 |
cs = (0, 1, 9, 10, 99, 100, 999, 1000, 9999, 10000, 99999, 100000, |
| 528 |
999999, 1000000) |
| 529 |
cs += (0xd800, 0xdfff) |
| 530 |
s = "".join(chr(c) for c in cs) |
| 531 |
self.assertEqual( |
| 532 |
codecs.xmlcharrefreplace_errors( |
| 533 |
UnicodeEncodeError("ascii", "a" + s + "b", |
| 534 |
1, 1 + len(s), "ouch") |
| 535 |
), |
| 536 |
("".join("&#%d;" % c for c in cs), 1 + len(s)) |
| 537 |
) |
| 538 |
|
| 539 |
def test_badandgoodbackslashreplaceexceptions(self): |
| 540 |
# "backslashreplace" complains about a non-exception passed in |
| 541 |
self.assertRaises( |
| 542 |
TypeError, |
| 543 |
codecs.backslashreplace_errors, |
| 544 |
42 |
| 545 |
) |
| 546 |
# "backslashreplace" complains about the wrong exception types |
| 547 |
self.assertRaises( |
| 548 |
TypeError, |
| 549 |
codecs.backslashreplace_errors, |
| 550 |
UnicodeError("ouch") |
| 551 |
) |
| 552 |
# Use the correct exception |
| 553 |
tests = [ |
| 554 |
("\u3042", "\\u3042"), |
| 555 |
("\n", "\\x0a"), |
| 556 |
("a", "\\x61"), |
| 557 |
("\x00", "\\x00"), |
| 558 |
("\xff", "\\xff"), |
| 559 |
("\u0100", "\\u0100"), |
| 560 |
("\uffff", "\\uffff"), |
| 561 |
("\U00010000", "\\U00010000"), |
| 562 |
("\U0010ffff", "\\U0010ffff"), |
| 563 |
# Lone surrogates |
| 564 |
("\ud800", "\\ud800"), |
| 565 |
("\udfff", "\\udfff"), |
| 566 |
("\ud800\udfff", "\\ud800\\udfff"), |
| 567 |
] |
| 568 |
for s, r in tests: |
| 569 |
with self.subTest(str=s): |
| 570 |
self.assertEqual( |
| 571 |
codecs.backslashreplace_errors( |
| 572 |
UnicodeEncodeError("ascii", "a" + s + "b", |
| 573 |
1, 1 + len(s), "ouch")), |
| 574 |
(r, 1 + len(s)) |
| 575 |
) |
| 576 |
self.assertEqual( |
| 577 |
codecs.backslashreplace_errors( |
| 578 |
UnicodeTranslateError("a" + s + "b", |
| 579 |
1, 1 + len(s), "ouch")), |
| 580 |
(r, 1 + len(s)) |
| 581 |
) |
| 582 |
tests = [ |
| 583 |
(b"a", "\\x61"), |
| 584 |
(b"\n", "\\x0a"), |
| 585 |
(b"\x00", "\\x00"), |
| 586 |
(b"\xff", "\\xff"), |
| 587 |
] |
| 588 |
for b, r in tests: |
| 589 |
with self.subTest(bytes=b): |
| 590 |
self.assertEqual( |
| 591 |
codecs.backslashreplace_errors( |
| 592 |
UnicodeDecodeError("ascii", bytearray(b"a" + b + b"b"), |
| 593 |
1, 2, "ouch")), |
| 594 |
(r, 2) |
| 595 |
) |
| 596 |
|
| 597 |
def test_badandgoodnamereplaceexceptions(self): |
| 598 |
# "namereplace" complains about a non-exception passed in |
| 599 |
self.assertRaises( |
| 600 |
TypeError, |
| 601 |
codecs.namereplace_errors, |
| 602 |
42 |
| 603 |
) |
| 604 |
# "namereplace" complains about the wrong exception types |
| 605 |
self.assertRaises( |
| 606 |
TypeError, |
| 607 |
codecs.namereplace_errors, |
| 608 |
UnicodeError("ouch") |
| 609 |
) |
| 610 |
# "namereplace" can only be used for encoding |
| 611 |
self.assertRaises( |
| 612 |
TypeError, |
| 613 |
codecs.namereplace_errors, |
| 614 |
UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch") |
| 615 |
) |
| 616 |
self.assertRaises( |
| 617 |
TypeError, |
| 618 |
codecs.namereplace_errors, |
| 619 |
UnicodeTranslateError("\u3042", 0, 1, "ouch") |
| 620 |
) |
| 621 |
# Use the correct exception |
| 622 |
tests = [ |
| 623 |
("\u3042", "\\N{HIRAGANA LETTER A}"), |
| 624 |
("\x00", "\\x00"), |
| 625 |
("\ufbf9", "\\N{ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH " |
| 626 |
"HAMZA ABOVE WITH ALEF MAKSURA ISOLATED FORM}"), |
| 627 |
("\U000e007f", "\\N{CANCEL TAG}"), |
| 628 |
("\U0010ffff", "\\U0010ffff"), |
| 629 |
# Lone surrogates |
| 630 |
("\ud800", "\\ud800"), |
| 631 |
("\udfff", "\\udfff"), |
| 632 |
("\ud800\udfff", "\\ud800\\udfff"), |
| 633 |
] |
| 634 |
for s, r in tests: |
| 635 |
with self.subTest(str=s): |
| 636 |
self.assertEqual( |
| 637 |
codecs.namereplace_errors( |
| 638 |
UnicodeEncodeError("ascii", "a" + s + "b", |
| 639 |
1, 1 + len(s), "ouch")), |
| 640 |
(r, 1 + len(s)) |
| 641 |
) |
| 642 |
|
| 643 |
def test_badandgoodsurrogateescapeexceptions(self): |
| 644 |
surrogateescape_errors = codecs.lookup_error('surrogateescape') |
| 645 |
# "surrogateescape" complains about a non-exception passed in |
| 646 |
self.assertRaises( |
| 647 |
TypeError, |
| 648 |
surrogateescape_errors, |
| 649 |
42 |
| 650 |
) |
| 651 |
# "surrogateescape" complains about the wrong exception types |
| 652 |
self.assertRaises( |
| 653 |
TypeError, |
| 654 |
surrogateescape_errors, |
| 655 |
UnicodeError("ouch") |
| 656 |
) |
| 657 |
# "surrogateescape" can not be used for translating |
| 658 |
self.assertRaises( |
| 659 |
TypeError, |
| 660 |
surrogateescape_errors, |
| 661 |
UnicodeTranslateError("\udc80", 0, 1, "ouch") |
| 662 |
) |
| 663 |
# Use the correct exception |
| 664 |
for s in ("a", "\udc7f", "\udd00"): |
| 665 |
with self.subTest(str=s): |
| 666 |
self.assertRaises( |
| 667 |
UnicodeEncodeError, |
| 668 |
surrogateescape_errors, |
| 669 |
UnicodeEncodeError("ascii", s, 0, 1, "ouch") |
| 670 |
) |
| 671 |
self.assertEqual( |
| 672 |
surrogateescape_errors( |
| 673 |
UnicodeEncodeError("ascii", "a\udc80b", 1, 2, "ouch")), |
| 674 |
(b"\x80", 2) |
| 675 |
) |
| 676 |
self.assertRaises( |
| 677 |
UnicodeDecodeError, |
| 678 |
surrogateescape_errors, |
| 679 |
UnicodeDecodeError("ascii", bytearray(b"a"), 0, 1, "ouch") |
| 680 |
) |
| 681 |
self.assertEqual( |
| 682 |
surrogateescape_errors( |
| 683 |
UnicodeDecodeError("ascii", bytearray(b"a\x80b"), 1, 2, "ouch")), |
| 684 |
("\udc80", 2) |
| 685 |
) |
| 686 |
|
| 687 |
def test_badandgoodsurrogatepassexceptions(self): |
| 688 |
surrogatepass_errors = codecs.lookup_error('surrogatepass') |
| 689 |
# "surrogatepass" complains about a non-exception passed in |
| 690 |
self.assertRaises( |
| 691 |
TypeError, |
| 692 |
surrogatepass_errors, |
| 693 |
42 |
| 694 |
) |
| 695 |
# "surrogatepass" complains about the wrong exception types |
| 696 |
self.assertRaises( |
| 697 |
TypeError, |
| 698 |
surrogatepass_errors, |
| 699 |
UnicodeError("ouch") |
| 700 |
) |
| 701 |
# "surrogatepass" can not be used for translating |
| 702 |
self.assertRaises( |
| 703 |
TypeError, |
| 704 |
surrogatepass_errors, |
| 705 |
UnicodeTranslateError("\ud800", 0, 1, "ouch") |
| 706 |
) |
| 707 |
# Use the correct exception |
| 708 |
for enc in ("utf-8", "utf-16le", "utf-16be", "utf-32le", "utf-32be"): |
| 709 |
with self.subTest(encoding=enc): |
| 710 |
self.assertRaises( |
| 711 |
UnicodeEncodeError, |
| 712 |
surrogatepass_errors, |
| 713 |
UnicodeEncodeError(enc, "a", 0, 1, "ouch") |
| 714 |
) |
| 715 |
self.assertRaises( |
| 716 |
UnicodeDecodeError, |
| 717 |
surrogatepass_errors, |
| 718 |
UnicodeDecodeError(enc, "a".encode(enc), 0, 1, "ouch") |
| 719 |
) |
| 720 |
for s in ("\ud800", "\udfff", "\ud800\udfff"): |
| 721 |
with self.subTest(str=s): |
| 722 |
self.assertRaises( |
| 723 |
UnicodeEncodeError, |
| 724 |
surrogatepass_errors, |
| 725 |
UnicodeEncodeError("ascii", s, 0, len(s), "ouch") |
| 726 |
) |
| 727 |
tests = [ |
| 728 |
("utf-8", "\ud800", b'\xed\xa0\x80', 3), |
| 729 |
("utf-16le", "\ud800", b'\x00\xd8', 2), |
| 730 |
("utf-16be", "\ud800", b'\xd8\x00', 2), |
| 731 |
("utf-32le", "\ud800", b'\x00\xd8\x00\x00', 4), |
| 732 |
("utf-32be", "\ud800", b'\x00\x00\xd8\x00', 4), |
| 733 |
("utf-8", "\udfff", b'\xed\xbf\xbf', 3), |
| 734 |
("utf-16le", "\udfff", b'\xff\xdf', 2), |
| 735 |
("utf-16be", "\udfff", b'\xdf\xff', 2), |
| 736 |
("utf-32le", "\udfff", b'\xff\xdf\x00\x00', 4), |
| 737 |
("utf-32be", "\udfff", b'\x00\x00\xdf\xff', 4), |
| 738 |
("utf-8", "\ud800\udfff", b'\xed\xa0\x80\xed\xbf\xbf', 3), |
| 739 |
("utf-16le", "\ud800\udfff", b'\x00\xd8\xff\xdf', 2), |
| 740 |
("utf-16be", "\ud800\udfff", b'\xd8\x00\xdf\xff', 2), |
| 741 |
("utf-32le", "\ud800\udfff", b'\x00\xd8\x00\x00\xff\xdf\x00\x00', 4), |
| 742 |
("utf-32be", "\ud800\udfff", b'\x00\x00\xd8\x00\x00\x00\xdf\xff', 4), |
| 743 |
] |
| 744 |
for enc, s, b, n in tests: |
| 745 |
with self.subTest(encoding=enc, str=s, bytes=b): |
| 746 |
self.assertEqual( |
| 747 |
surrogatepass_errors( |
| 748 |
UnicodeEncodeError(enc, "a" + s + "b", |
| 749 |
1, 1 + len(s), "ouch")), |
| 750 |
(b, 1 + len(s)) |
| 751 |
) |
| 752 |
self.assertEqual( |
| 753 |
surrogatepass_errors( |
| 754 |
UnicodeDecodeError(enc, bytearray(b"a" + b[:n] + b"b"), |
| 755 |
1, 1 + n, "ouch")), |
| 756 |
(s[:1], 1 + n) |
| 757 |
) |
| 758 |
|
| 759 |
def test_badhandlerresults(self): |
| 760 |
results = ( 42, "foo", (1,2,3), ("foo", 1, 3), ("foo", None), ("foo",), ("foo", 1, 3), ("foo", None), ("foo",) ) |
| 761 |
encs = ("ascii", "latin-1", "iso-8859-1", "iso-8859-15") |
| 762 |
|
| 763 |
for res in results: |
| 764 |
codecs.register_error("test.badhandler", lambda x: res) |
| 765 |
for enc in encs: |
| 766 |
self.assertRaises( |
| 767 |
TypeError, |
| 768 |
"\u3042".encode, |
| 769 |
enc, |
| 770 |
"test.badhandler" |
| 771 |
) |
| 772 |
for (enc, bytes) in ( |
| 773 |
("ascii", b"\xff"), |
| 774 |
("utf-8", b"\xff"), |
| 775 |
("utf-7", b"+x-"), |
| 776 |
): |
| 777 |
self.assertRaises( |
| 778 |
TypeError, |
| 779 |
bytes.decode, |
| 780 |
enc, |
| 781 |
"test.badhandler" |
| 782 |
) |
| 783 |
|
| 784 |
def test_lookup(self): |
| 785 |
self.assertEqual(codecs.strict_errors, codecs.lookup_error("strict")) |
| 786 |
self.assertEqual(codecs.ignore_errors, codecs.lookup_error("ignore")) |
| 787 |
self.assertEqual(codecs.strict_errors, codecs.lookup_error("strict")) |
| 788 |
self.assertEqual( |
| 789 |
codecs.xmlcharrefreplace_errors, |
| 790 |
codecs.lookup_error("xmlcharrefreplace") |
| 791 |
) |
| 792 |
self.assertEqual( |
| 793 |
codecs.backslashreplace_errors, |
| 794 |
codecs.lookup_error("backslashreplace") |
| 795 |
) |
| 796 |
self.assertEqual( |
| 797 |
codecs.namereplace_errors, |
| 798 |
codecs.lookup_error("namereplace") |
| 799 |
) |
| 800 |
|
| 801 |
def test_encode_nonascii_replacement(self): |
| 802 |
def handle(exc): |
| 803 |
if isinstance(exc, UnicodeEncodeError): |
| 804 |
return (repl, exc.end) |
| 805 |
raise TypeError("don't know how to handle %r" % exc) |
| 806 |
codecs.register_error("test.replacing", handle) |
| 807 |
|
| 808 |
for enc, input, repl in ( |
| 809 |
("ascii", "[¤]", "abc"), |
| 810 |
("iso-8859-1", "[€]", "½¾"), |
| 811 |
("iso-8859-15", "[¤]", "œŸ"), |
| 812 |
): |
| 813 |
res = input.encode(enc, "test.replacing") |
| 814 |
self.assertEqual(res, ("[" + repl + "]").encode(enc)) |
| 815 |
|
| 816 |
for enc, input, repl in ( |
| 817 |
("utf-8", "[\udc80]", "\U0001f40d"), |
| 818 |
("utf-16", "[\udc80]", "\U0001f40d"), |
| 819 |
("utf-32", "[\udc80]", "\U0001f40d"), |
| 820 |
): |
| 821 |
with self.subTest(encoding=enc): |
| 822 |
with self.assertRaises(UnicodeEncodeError) as cm: |
| 823 |
input.encode(enc, "test.replacing") |
| 824 |
exc = cm.exception |
| 825 |
self.assertEqual(exc.start, 1) |
| 826 |
self.assertEqual(exc.end, 2) |
| 827 |
self.assertEqual(exc.object, input) |
| 828 |
|
| 829 |
def test_encode_unencodable_replacement(self): |
| 830 |
def unencrepl(exc): |
| 831 |
if isinstance(exc, UnicodeEncodeError): |
| 832 |
return (repl, exc.end) |
| 833 |
else: |
| 834 |
raise TypeError("don't know how to handle %r" % exc) |
| 835 |
codecs.register_error("test.unencreplhandler", unencrepl) |
| 836 |
|
| 837 |
for enc, input, repl in ( |
| 838 |
("ascii", "[¤]", "½"), |
| 839 |
("iso-8859-1", "[€]", "œ"), |
| 840 |
("iso-8859-15", "[¤]", "½"), |
| 841 |
("utf-8", "[\udc80]", "\udcff"), |
| 842 |
("utf-16", "[\udc80]", "\udcff"), |
| 843 |
("utf-32", "[\udc80]", "\udcff"), |
| 844 |
): |
| 845 |
with self.subTest(encoding=enc): |
| 846 |
with self.assertRaises(UnicodeEncodeError) as cm: |
| 847 |
input.encode(enc, "test.unencreplhandler") |
| 848 |
exc = cm.exception |
| 849 |
self.assertEqual(exc.start, 1) |
| 850 |
self.assertEqual(exc.end, 2) |
| 851 |
self.assertEqual(exc.object, input) |
| 852 |
|
| 853 |
def test_encode_bytes_replacement(self): |
| 854 |
def handle(exc): |
| 855 |
if isinstance(exc, UnicodeEncodeError): |
| 856 |
return (repl, exc.end) |
| 857 |
raise TypeError("don't know how to handle %r" % exc) |
| 858 |
codecs.register_error("test.replacing", handle) |
| 859 |
|
| 860 |
# It works even if the bytes sequence is not decodable. |
| 861 |
for enc, input, repl in ( |
| 862 |
("ascii", "[¤]", b"\xbd\xbe"), |
| 863 |
("iso-8859-1", "[€]", b"\xbd\xbe"), |
| 864 |
("iso-8859-15", "[¤]", b"\xbd\xbe"), |
| 865 |
("utf-8", "[\udc80]", b"\xbd\xbe"), |
| 866 |
("utf-16le", "[\udc80]", b"\xbd\xbe"), |
| 867 |
("utf-16be", "[\udc80]", b"\xbd\xbe"), |
| 868 |
("utf-32le", "[\udc80]", b"\xbc\xbd\xbe\xbf"), |
| 869 |
("utf-32be", "[\udc80]", b"\xbc\xbd\xbe\xbf"), |
| 870 |
): |
| 871 |
with self.subTest(encoding=enc): |
| 872 |
res = input.encode(enc, "test.replacing") |
| 873 |
self.assertEqual(res, "[".encode(enc) + repl + "]".encode(enc)) |
| 874 |
|
| 875 |
def test_encode_odd_bytes_replacement(self): |
| 876 |
def handle(exc): |
| 877 |
if isinstance(exc, UnicodeEncodeError): |
| 878 |
return (repl, exc.end) |
| 879 |
raise TypeError("don't know how to handle %r" % exc) |
| 880 |
codecs.register_error("test.replacing", handle) |
| 881 |
|
| 882 |
input = "[\udc80]" |
| 883 |
# Tests in which the replacement bytestring contains not whole number |
| 884 |
# of code units. |
| 885 |
for enc, repl in ( |
| 886 |
*itertools.product(("utf-16le", "utf-16be"), |
| 887 |
[b"a", b"abc"]), |
| 888 |
*itertools.product(("utf-32le", "utf-32be"), |
| 889 |
[b"a", b"ab", b"abc", b"abcde"]), |
| 890 |
): |
| 891 |
with self.subTest(encoding=enc, repl=repl): |
| 892 |
with self.assertRaises(UnicodeEncodeError) as cm: |
| 893 |
input.encode(enc, "test.replacing") |
| 894 |
exc = cm.exception |
| 895 |
self.assertEqual(exc.start, 1) |
| 896 |
self.assertEqual(exc.end, 2) |
| 897 |
self.assertEqual(exc.object, input) |
| 898 |
self.assertEqual(exc.reason, "surrogates not allowed") |
| 899 |
|
| 900 |
def test_badregistercall(self): |
| 901 |
# enhance coverage of: |
| 902 |
# Modules/_codecsmodule.c::register_error() |
| 903 |
# Python/codecs.c::PyCodec_RegisterError() |
| 904 |
self.assertRaises(TypeError, codecs.register_error, 42) |
| 905 |
self.assertRaises(TypeError, codecs.register_error, "test.dummy", 42) |
| 906 |
|
| 907 |
def test_badlookupcall(self): |
| 908 |
# enhance coverage of: |
| 909 |
# Modules/_codecsmodule.c::lookup_error() |
| 910 |
self.assertRaises(TypeError, codecs.lookup_error) |
| 911 |
|
| 912 |
def test_unknownhandler(self): |
| 913 |
# enhance coverage of: |
| 914 |
# Modules/_codecsmodule.c::lookup_error() |
| 915 |
self.assertRaises(LookupError, codecs.lookup_error, "test.unknown") |
| 916 |
|
| 917 |
def test_xmlcharrefvalues(self): |
| 918 |
# enhance coverage of: |
| 919 |
# Python/codecs.c::PyCodec_XMLCharRefReplaceErrors() |
| 920 |
# and inline implementations |
| 921 |
v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000, 100000, |
| 922 |
500000, 1000000) |
| 923 |
s = "".join([chr(x) for x in v]) |
| 924 |
codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors) |
| 925 |
for enc in ("ascii", "iso-8859-15"): |
| 926 |
for err in ("xmlcharrefreplace", "test.xmlcharrefreplace"): |
| 927 |
s.encode(enc, err) |
| 928 |
|
| 929 |
def test_decodehelper(self): |
| 930 |
# enhance coverage of: |
| 931 |
# Objects/unicodeobject.c::unicode_decode_call_errorhandler() |
| 932 |
# and callers |
| 933 |
self.assertRaises(LookupError, b"\xff".decode, "ascii", "test.unknown") |
| 934 |
|
| 935 |
def baddecodereturn1(exc): |
| 936 |
return 42 |
| 937 |
codecs.register_error("test.baddecodereturn1", baddecodereturn1) |
| 938 |
self.assertRaises(TypeError, b"\xff".decode, "ascii", "test.baddecodereturn1") |
| 939 |
self.assertRaises(TypeError, b"\\".decode, "unicode-escape", "test.baddecodereturn1") |
| 940 |
self.assertRaises(TypeError, b"\\x0".decode, "unicode-escape", "test.baddecodereturn1") |
| 941 |
self.assertRaises(TypeError, b"\\x0y".decode, "unicode-escape", "test.baddecodereturn1") |
| 942 |
self.assertRaises(TypeError, b"\\Uffffeeee".decode, "unicode-escape", "test.baddecodereturn1") |
| 943 |
self.assertRaises(TypeError, b"\\uyyyy".decode, "raw-unicode-escape", "test.baddecodereturn1") |
| 944 |
|
| 945 |
def baddecodereturn2(exc): |
| 946 |
return ("?", None) |
| 947 |
codecs.register_error("test.baddecodereturn2", baddecodereturn2) |
| 948 |
self.assertRaises(TypeError, b"\xff".decode, "ascii", "test.baddecodereturn2") |
| 949 |
|
| 950 |
handler = PosReturn() |
| 951 |
codecs.register_error("test.posreturn", handler.handle) |
| 952 |
|
| 953 |
# Valid negative position |
| 954 |
handler.pos = -1 |
| 955 |
self.assertEqual(b"\xff0".decode("ascii", "test.posreturn"), "<?>0") |
| 956 |
|
| 957 |
# Valid negative position |
| 958 |
handler.pos = -2 |
| 959 |
self.assertEqual(b"\xff0".decode("ascii", "test.posreturn"), "<?><?>") |
| 960 |
|
| 961 |
# Negative position out of bounds |
| 962 |
handler.pos = -3 |
| 963 |
self.assertRaises(IndexError, b"\xff0".decode, "ascii", "test.posreturn") |
| 964 |
|
| 965 |
# Valid positive position |
| 966 |
handler.pos = 1 |
| 967 |
self.assertEqual(b"\xff0".decode("ascii", "test.posreturn"), "<?>0") |
| 968 |
|
| 969 |
# Largest valid positive position (one beyond end of input) |
| 970 |
handler.pos = 2 |
| 971 |
self.assertEqual(b"\xff0".decode("ascii", "test.posreturn"), "<?>") |
| 972 |
|
| 973 |
# Invalid positive position |
| 974 |
handler.pos = 3 |
| 975 |
self.assertRaises(IndexError, b"\xff0".decode, "ascii", "test.posreturn") |
| 976 |
|
| 977 |
# Restart at the "0" |
| 978 |
handler.pos = 6 |
| 979 |
self.assertEqual(b"\\uyyyy0".decode("raw-unicode-escape", "test.posreturn"), "<?>0") |
| 980 |
|
| 981 |
class D(dict): |
| 982 |
def __getitem__(self, key): |
| 983 |
raise ValueError |
| 984 |
self.assertRaises(UnicodeError, codecs.charmap_decode, b"\xff", "strict", {0xff: None}) |
| 985 |
self.assertRaises(ValueError, codecs.charmap_decode, b"\xff", "strict", D()) |
| 986 |
self.assertRaises(TypeError, codecs.charmap_decode, b"\xff", "strict", {0xff: sys.maxunicode+1}) |
| 987 |
|
| 988 |
def test_encodehelper(self): |
| 989 |
# enhance coverage of: |
| 990 |
# Objects/unicodeobject.c::unicode_encode_call_errorhandler() |
| 991 |
# and callers |
| 992 |
self.assertRaises(LookupError, "\xff".encode, "ascii", "test.unknown") |
| 993 |
|
| 994 |
def badencodereturn1(exc): |
| 995 |
return 42 |
| 996 |
codecs.register_error("test.badencodereturn1", badencodereturn1) |
| 997 |
self.assertRaises(TypeError, "\xff".encode, "ascii", "test.badencodereturn1") |
| 998 |
|
| 999 |
def badencodereturn2(exc): |
| 1000 |
return ("?", None) |
| 1001 |
codecs.register_error("test.badencodereturn2", badencodereturn2) |
| 1002 |
self.assertRaises(TypeError, "\xff".encode, "ascii", "test.badencodereturn2") |
| 1003 |
|
| 1004 |
handler = PosReturn() |
| 1005 |
codecs.register_error("test.posreturn", handler.handle) |
| 1006 |
|
| 1007 |
# Valid negative position |
| 1008 |
handler.pos = -1 |
| 1009 |
self.assertEqual("\xff0".encode("ascii", "test.posreturn"), b"<?>0") |
| 1010 |
|
| 1011 |
# Valid negative position |
| 1012 |
handler.pos = -2 |
| 1013 |
self.assertEqual("\xff0".encode("ascii", "test.posreturn"), b"<?><?>") |
| 1014 |
|
| 1015 |
# Negative position out of bounds |
| 1016 |
handler.pos = -3 |
| 1017 |
self.assertRaises(IndexError, "\xff0".encode, "ascii", "test.posreturn") |
| 1018 |
|
| 1019 |
# Valid positive position |
| 1020 |
handler.pos = 1 |
| 1021 |
self.assertEqual("\xff0".encode("ascii", "test.posreturn"), b"<?>0") |
| 1022 |
|
| 1023 |
# Largest valid positive position (one beyond end of input |
| 1024 |
handler.pos = 2 |
| 1025 |
self.assertEqual("\xff0".encode("ascii", "test.posreturn"), b"<?>") |
| 1026 |
|
| 1027 |
# Invalid positive position |
| 1028 |
handler.pos = 3 |
| 1029 |
self.assertRaises(IndexError, "\xff0".encode, "ascii", "test.posreturn") |
| 1030 |
|
| 1031 |
handler.pos = 0 |
| 1032 |
|
| 1033 |
class D(dict): |
| 1034 |
def __getitem__(self, key): |
| 1035 |
raise ValueError |
| 1036 |
for err in ("strict", "replace", "xmlcharrefreplace", |
| 1037 |
"backslashreplace", "namereplace", "test.posreturn"): |
| 1038 |
self.assertRaises(UnicodeError, codecs.charmap_encode, "\xff", err, {0xff: None}) |
| 1039 |
self.assertRaises(ValueError, codecs.charmap_encode, "\xff", err, D()) |
| 1040 |
self.assertRaises(TypeError, codecs.charmap_encode, "\xff", err, {0xff: 300}) |
| 1041 |
|
| 1042 |
def test_decodehelper_bug36819(self): |
| 1043 |
handler = RepeatedPosReturn("x") |
| 1044 |
codecs.register_error("test.bug36819", handler.handle) |
| 1045 |
|
| 1046 |
testcases = [ |
| 1047 |
("ascii", b"\xff"), |
| 1048 |
("utf-8", b"\xff"), |
| 1049 |
("utf-16be", b'\xdc\x80'), |
| 1050 |
("utf-32be", b'\x00\x00\xdc\x80'), |
| 1051 |
("iso-8859-6", b"\xff"), |
| 1052 |
] |
| 1053 |
for enc, bad in testcases: |
| 1054 |
input = "abcd".encode(enc) + bad |
| 1055 |
with self.subTest(encoding=enc): |
| 1056 |
handler.count = 50 |
| 1057 |
decoded = input.decode(enc, "test.bug36819") |
| 1058 |
self.assertEqual(decoded, 'abcdx' * 51) |
| 1059 |
|
| 1060 |
def test_encodehelper_bug36819(self): |
| 1061 |
handler = RepeatedPosReturn() |
| 1062 |
codecs.register_error("test.bug36819", handler.handle) |
| 1063 |
|
| 1064 |
input = "abcd\udc80" |
| 1065 |
encodings = ["ascii", "latin1", "utf-8", "utf-16", "utf-32"] # built-in |
| 1066 |
encodings += ["iso-8859-15"] # charmap codec |
| 1067 |
if sys.platform == 'win32': |
| 1068 |
encodings = ["mbcs", "oem"] # code page codecs |
| 1069 |
|
| 1070 |
handler.repl = "\udcff" |
| 1071 |
for enc in encodings: |
| 1072 |
with self.subTest(encoding=enc): |
| 1073 |
handler.count = 50 |
| 1074 |
with self.assertRaises(UnicodeEncodeError) as cm: |
| 1075 |
input.encode(enc, "test.bug36819") |
| 1076 |
exc = cm.exception |
| 1077 |
self.assertEqual(exc.start, 4) |
| 1078 |
self.assertEqual(exc.end, 5) |
| 1079 |
self.assertEqual(exc.object, input) |
| 1080 |
if sys.platform == "win32": |
| 1081 |
handler.count = 50 |
| 1082 |
with self.assertRaises(UnicodeEncodeError) as cm: |
| 1083 |
codecs.code_page_encode(437, input, "test.bug36819") |
| 1084 |
exc = cm.exception |
| 1085 |
self.assertEqual(exc.start, 4) |
| 1086 |
self.assertEqual(exc.end, 5) |
| 1087 |
self.assertEqual(exc.object, input) |
| 1088 |
|
| 1089 |
handler.repl = "x" |
| 1090 |
for enc in encodings: |
| 1091 |
with self.subTest(encoding=enc): |
| 1092 |
# The interpreter should segfault after a handful of attempts. |
| 1093 |
# 50 was chosen to try to ensure a segfault without a fix, |
| 1094 |
# but not OOM a machine with one. |
| 1095 |
handler.count = 50 |
| 1096 |
encoded = input.encode(enc, "test.bug36819") |
| 1097 |
self.assertEqual(encoded.decode(enc), "abcdx" * 51) |
| 1098 |
if sys.platform == "win32": |
| 1099 |
handler.count = 50 |
| 1100 |
encoded = codecs.code_page_encode(437, input, "test.bug36819") |
| 1101 |
self.assertEqual(encoded[0].decode(), "abcdx" * 51) |
| 1102 |
self.assertEqual(encoded[1], len(input)) |
| 1103 |
|
| 1104 |
def test_translatehelper(self): |
| 1105 |
# enhance coverage of: |
| 1106 |
# Objects/unicodeobject.c::unicode_encode_call_errorhandler() |
| 1107 |
# and callers |
| 1108 |
# (Unfortunately the errors argument is not directly accessible |
| 1109 |
# from Python, so we can't test that much) |
| 1110 |
class D(dict): |
| 1111 |
def __getitem__(self, key): |
| 1112 |
raise ValueError |
| 1113 |
#self.assertRaises(ValueError, "\xff".translate, D()) |
| 1114 |
self.assertRaises(ValueError, "\xff".translate, {0xff: sys.maxunicode+1}) |
| 1115 |
self.assertRaises(TypeError, "\xff".translate, {0xff: ()}) |
| 1116 |
|
| 1117 |
def test_bug828737(self): |
| 1118 |
charmap = { |
| 1119 |
ord("&"): "&", |
| 1120 |
ord("<"): "<", |
| 1121 |
ord(">"): ">", |
| 1122 |
ord('"'): """, |
| 1123 |
} |
| 1124 |
|
| 1125 |
for n in (1, 10, 100, 1000): |
| 1126 |
text = 'abc<def>ghi'*n |
| 1127 |
text.translate(charmap) |
| 1128 |
|
| 1129 |
def test_mutating_decode_handler(self): |
| 1130 |
baddata = [ |
| 1131 |
("ascii", b"\xff"), |
| 1132 |
("utf-7", b"++"), |
| 1133 |
("utf-8", b"\xff"), |
| 1134 |
("utf-16", b"\xff"), |
| 1135 |
("utf-32", b"\xff"), |
| 1136 |
("unicode-escape", b"\\u123g"), |
| 1137 |
("raw-unicode-escape", b"\\u123g"), |
| 1138 |
] |
| 1139 |
|
| 1140 |
def replacing(exc): |
| 1141 |
if isinstance(exc, UnicodeDecodeError): |
| 1142 |
exc.object = 42 |
| 1143 |
return ("\u4242", 0) |
| 1144 |
else: |
| 1145 |
raise TypeError("don't know how to handle %r" % exc) |
| 1146 |
codecs.register_error("test.replacing", replacing) |
| 1147 |
|
| 1148 |
for (encoding, data) in baddata: |
| 1149 |
with self.assertRaises(TypeError): |
| 1150 |
data.decode(encoding, "test.replacing") |
| 1151 |
|
| 1152 |
def mutating(exc): |
| 1153 |
if isinstance(exc, UnicodeDecodeError): |
| 1154 |
exc.object = b"" |
| 1155 |
return ("\u4242", 0) |
| 1156 |
else: |
| 1157 |
raise TypeError("don't know how to handle %r" % exc) |
| 1158 |
codecs.register_error("test.mutating", mutating) |
| 1159 |
# If the decoder doesn't pick up the modified input the following |
| 1160 |
# will lead to an endless loop |
| 1161 |
for (encoding, data) in baddata: |
| 1162 |
self.assertEqual(data.decode(encoding, "test.mutating"), "\u4242") |
| 1163 |
|
| 1164 |
def test_mutating_decode_handler_unicode_escape(self): |
| 1165 |
decode = codecs.unicode_escape_decode |
| 1166 |
def mutating(exc): |
| 1167 |
if isinstance(exc, UnicodeDecodeError): |
| 1168 |
r = data.get(exc.object[:exc.end]) |
| 1169 |
if r is not None: |
| 1170 |
exc.object = r[0] + exc.object[exc.end:] |
| 1171 |
return ('\u0404', r[1]) |
| 1172 |
raise AssertionError("don't know how to handle %r" % exc) |
| 1173 |
|
| 1174 |
codecs.register_error('test.mutating2', mutating) |
| 1175 |
data = { |
| 1176 |
br'\x0': (b'\\', 0), |
| 1177 |
br'\x3': (b'xxx\\', 3), |
| 1178 |
br'\x5': (b'x\\', 1), |
| 1179 |
} |
| 1180 |
def check(input, expected, msg): |
| 1181 |
with self.assertWarns(DeprecationWarning) as cm: |
| 1182 |
self.assertEqual(decode(input, 'test.mutating2'), (expected, len(input))) |
| 1183 |
self.assertIn(msg, str(cm.warning)) |
| 1184 |
|
| 1185 |
check(br'\x0n\z', '\u0404\n\\z', r'"\z" is an invalid escape sequence') |
| 1186 |
check(br'\x0n\501', '\u0404\n\u0141', r'"\501" is an invalid octal escape sequence') |
| 1187 |
check(br'\x0z', '\u0404\\z', r'"\z" is an invalid escape sequence') |
| 1188 |
|
| 1189 |
check(br'\x3n\zr', '\u0404\n\\zr', r'"\z" is an invalid escape sequence') |
| 1190 |
check(br'\x3zr', '\u0404\\zr', r'"\z" is an invalid escape sequence') |
| 1191 |
check(br'\x3z5', '\u0404\\z5', r'"\z" is an invalid escape sequence') |
| 1192 |
check(memoryview(br'\x3z5x')[:-1], '\u0404\\z5', r'"\z" is an invalid escape sequence') |
| 1193 |
check(memoryview(br'\x3z5xy')[:-2], '\u0404\\z5', r'"\z" is an invalid escape sequence') |
| 1194 |
|
| 1195 |
check(br'\x5n\z', '\u0404\n\\z', r'"\z" is an invalid escape sequence') |
| 1196 |
check(br'\x5n\501', '\u0404\n\u0141', r'"\501" is an invalid octal escape sequence') |
| 1197 |
check(br'\x5z', '\u0404\\z', r'"\z" is an invalid escape sequence') |
| 1198 |
check(memoryview(br'\x5zy')[:-1], '\u0404\\z', r'"\z" is an invalid escape sequence') |
| 1199 |
|
| 1200 |
# issue32583 |
| 1201 |
def test_crashing_decode_handler(self): |
| 1202 |
# better generating one more character to fill the extra space slot |
| 1203 |
# so in debug build it can steadily fail |
| 1204 |
def forward_shorter_than_end(exc): |
| 1205 |
if isinstance(exc, UnicodeDecodeError): |
| 1206 |
# size one character, 0 < forward < exc.end |
| 1207 |
return ('\ufffd', exc.start+1) |
| 1208 |
else: |
| 1209 |
raise TypeError("don't know how to handle %r" % exc) |
| 1210 |
codecs.register_error( |
| 1211 |
"test.forward_shorter_than_end", forward_shorter_than_end) |
| 1212 |
|
| 1213 |
self.assertEqual( |
| 1214 |
b'\xd8\xd8\xd8\xd8\xd8\x00\x00\x00'.decode( |
| 1215 |
'utf-16-le', 'test.forward_shorter_than_end'), |
| 1216 |
'\ufffd\ufffd\ufffd\ufffd\xd8\x00' |
| 1217 |
) |
| 1218 |
self.assertEqual( |
| 1219 |
b'\xd8\xd8\xd8\xd8\x00\xd8\x00\x00'.decode( |
| 1220 |
'utf-16-be', 'test.forward_shorter_than_end'), |
| 1221 |
'\ufffd\ufffd\ufffd\ufffd\xd8\x00' |
| 1222 |
) |
| 1223 |
self.assertEqual( |
| 1224 |
b'\x11\x11\x11\x11\x11\x00\x00\x00\x00\x00\x00'.decode( |
| 1225 |
'utf-32-le', 'test.forward_shorter_than_end'), |
| 1226 |
'\ufffd\ufffd\ufffd\u1111\x00' |
| 1227 |
) |
| 1228 |
self.assertEqual( |
| 1229 |
b'\x11\x11\x11\x00\x00\x11\x11\x00\x00\x00\x00'.decode( |
| 1230 |
'utf-32-be', 'test.forward_shorter_than_end'), |
| 1231 |
'\ufffd\ufffd\ufffd\u1111\x00' |
| 1232 |
) |
| 1233 |
|
| 1234 |
def replace_with_long(exc): |
| 1235 |
if isinstance(exc, UnicodeDecodeError): |
| 1236 |
exc.object = b"\x00" * 8 |
| 1237 |
return ('\ufffd', exc.start) |
| 1238 |
else: |
| 1239 |
raise TypeError("don't know how to handle %r" % exc) |
| 1240 |
codecs.register_error("test.replace_with_long", replace_with_long) |
| 1241 |
|
| 1242 |
self.assertEqual( |
| 1243 |
b'\x00'.decode('utf-16', 'test.replace_with_long'), |
| 1244 |
'\ufffd\x00\x00\x00\x00' |
| 1245 |
) |
| 1246 |
self.assertEqual( |
| 1247 |
b'\x00'.decode('utf-32', 'test.replace_with_long'), |
| 1248 |
'\ufffd\x00\x00' |
| 1249 |
) |
| 1250 |
|
| 1251 |
def test_fake_error_class(self): |
| 1252 |
handlers = [ |
| 1253 |
codecs.strict_errors, |
| 1254 |
codecs.ignore_errors, |
| 1255 |
codecs.replace_errors, |
| 1256 |
codecs.backslashreplace_errors, |
| 1257 |
codecs.namereplace_errors, |
| 1258 |
codecs.xmlcharrefreplace_errors, |
| 1259 |
codecs.lookup_error('surrogateescape'), |
| 1260 |
codecs.lookup_error('surrogatepass'), |
| 1261 |
] |
| 1262 |
for cls in UnicodeEncodeError, UnicodeDecodeError, UnicodeTranslateError: |
| 1263 |
class FakeUnicodeError(str): |
| 1264 |
__class__ = cls |
| 1265 |
for handler in handlers: |
| 1266 |
with self.subTest(handler=handler, error_class=cls): |
| 1267 |
self.assertRaises(TypeError, handler, FakeUnicodeError()) |
| 1268 |
class FakeUnicodeError(Exception): |
| 1269 |
__class__ = cls |
| 1270 |
for handler in handlers: |
| 1271 |
with self.subTest(handler=handler, error_class=cls): |
| 1272 |
with self.assertRaises((TypeError, FakeUnicodeError)): |
| 1273 |
handler(FakeUnicodeError()) |
| 1274 |
|
| 1275 |
def test_reject_unregister_builtin_error_handler(self): |
| 1276 |
for name in [ |
| 1277 |
'strict', 'ignore', 'replace', 'backslashreplace', 'namereplace', |
| 1278 |
'xmlcharrefreplace', 'surrogateescape', 'surrogatepass', |
| 1279 |
]: |
| 1280 |
with self.subTest(name): |
| 1281 |
self.assertRaises(ValueError, _codecs_unregister_error, name) |
| 1282 |
|
| 1283 |
def test_unregister_custom_error_handler(self): |
| 1284 |
def custom_handler(exc): |
| 1285 |
raise exc |
| 1286 |
|
| 1287 |
custom_name = 'test.test_unregister_custom_error_handler' |
| 1288 |
self.assertRaises(LookupError, codecs.lookup_error, custom_name) |
| 1289 |
codecs.register_error(custom_name, custom_handler) |
| 1290 |
self.assertIs(codecs.lookup_error(custom_name), custom_handler) |
| 1291 |
self.assertTrue(_codecs_unregister_error(custom_name)) |
| 1292 |
self.assertRaises(LookupError, codecs.lookup_error, custom_name) |
| 1293 |
|
| 1294 |
def test_unregister_custom_unknown_error_handler(self): |
| 1295 |
unknown_name = 'test.test_unregister_custom_unknown_error_handler' |
| 1296 |
self.assertRaises(LookupError, codecs.lookup_error, unknown_name) |
| 1297 |
self.assertFalse(_codecs_unregister_error(unknown_name)) |
| 1298 |
self.assertRaises(LookupError, codecs.lookup_error, unknown_name) |
| 1299 |
|
| 1300 |
|
| 1301 |
if __name__ == "__main__": |
| 1302 |
unittest.main() |