| 1 |
import codecs |
| 2 |
import contextlib |
| 3 |
import io |
| 4 |
import re |
| 5 |
import sys |
| 6 |
import unittest |
| 7 |
import unittest.mock as mock |
| 8 |
import _testcapi |
| 9 |
from test.support import import_helper |
| 10 |
|
| 11 |
_testlimitedcapi = import_helper.import_module('_testlimitedcapi') |
| 12 |
|
| 13 |
NULL = None |
| 14 |
BAD_ARGUMENT = re.escape('bad argument type for built-in operation') |
| 15 |
|
| 16 |
|
| 17 |
class CAPIUnicodeTest(unittest.TestCase): |
| 18 |
# TODO: Test the following functions: |
| 19 |
# |
| 20 |
# PyUnicode_BuildEncodingMap |
| 21 |
# PyUnicode_FSConverter |
| 22 |
# PyUnicode_FSDecoder |
| 23 |
# PyUnicode_DecodeMBCS |
| 24 |
# PyUnicode_DecodeMBCSStateful |
| 25 |
# PyUnicode_DecodeCodePageStateful |
| 26 |
# PyUnicode_AsMBCSString |
| 27 |
# PyUnicode_EncodeCodePage |
| 28 |
# PyUnicode_DecodeLocaleAndSize |
| 29 |
# PyUnicode_DecodeLocale |
| 30 |
# PyUnicode_EncodeLocale |
| 31 |
# PyUnicode_DecodeFSDefault |
| 32 |
# PyUnicode_DecodeFSDefaultAndSize |
| 33 |
# PyUnicode_EncodeFSDefault |
| 34 |
|
| 35 |
def test_fromencodedobject(self): |
| 36 |
"""Test PyUnicode_FromEncodedObject()""" |
| 37 |
fromencodedobject = _testlimitedcapi.unicode_fromencodedobject |
| 38 |
|
| 39 |
self.assertEqual(fromencodedobject(b'abc', NULL), 'abc') |
| 40 |
self.assertEqual(fromencodedobject(b'abc', 'ascii'), 'abc') |
| 41 |
b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80' |
| 42 |
s = 'a\xa1\u4f60\U0001f600' |
| 43 |
self.assertEqual(fromencodedobject(b, NULL), s) |
| 44 |
self.assertEqual(fromencodedobject(b, 'utf-8'), s) |
| 45 |
self.assertEqual(fromencodedobject(b, 'latin1'), b.decode('latin1')) |
| 46 |
self.assertRaises(UnicodeDecodeError, fromencodedobject, b, 'ascii') |
| 47 |
self.assertEqual(fromencodedobject(b, 'ascii', 'replace'), |
| 48 |
'a' + '\ufffd'*9) |
| 49 |
self.assertEqual(fromencodedobject(bytearray(b), NULL), s) |
| 50 |
self.assertEqual(fromencodedobject(bytearray(b), 'utf-8'), s) |
| 51 |
self.assertRaises(LookupError, fromencodedobject, b'abc', 'foo') |
| 52 |
self.assertRaises(LookupError, fromencodedobject, b, 'ascii', 'foo') |
| 53 |
self.assertRaises(TypeError, fromencodedobject, 'abc', NULL) |
| 54 |
self.assertRaises(TypeError, fromencodedobject, 'abc', 'ascii') |
| 55 |
self.assertRaises(TypeError, fromencodedobject, [], NULL) |
| 56 |
self.assertRaises(TypeError, fromencodedobject, [], 'ascii') |
| 57 |
self.assertRaises(SystemError, fromencodedobject, NULL, NULL) |
| 58 |
self.assertRaises(SystemError, fromencodedobject, NULL, 'ascii') |
| 59 |
|
| 60 |
def test_decode(self): |
| 61 |
"""Test PyUnicode_Decode()""" |
| 62 |
decode = _testlimitedcapi.unicode_decode |
| 63 |
|
| 64 |
self.assertEqual(decode(b'[\xe2\x82\xac]', 'utf-8'), '[\u20ac]') |
| 65 |
self.assertEqual(decode(b'[\xa4]', 'iso8859-15'), '[\u20ac]') |
| 66 |
self.assertEqual(decode(b'[\xa4]', 'iso8859-15', 'strict'), '[\u20ac]') |
| 67 |
self.assertRaises(UnicodeDecodeError, decode, b'[\xa4]', 'utf-8') |
| 68 |
self.assertEqual(decode(b'[\xa4]', 'utf-8', 'replace'), '[\ufffd]') |
| 69 |
|
| 70 |
self.assertEqual(decode(b'[\xe2\x82\xac]', NULL), '[\u20ac]') |
| 71 |
self.assertEqual(decode(b'[\xa4]', NULL, 'replace'), '[\ufffd]') |
| 72 |
|
| 73 |
self.assertRaises(LookupError, decode, b'\xa4', 'foo') |
| 74 |
self.assertRaises(LookupError, decode, b'\xa4', 'utf-8', 'foo') |
| 75 |
# TODO: Test PyUnicode_Decode() with NULL as data and |
| 76 |
# negative size. |
| 77 |
|
| 78 |
def test_asencodedstring(self): |
| 79 |
"""Test PyUnicode_AsEncodedString()""" |
| 80 |
asencodedstring = _testlimitedcapi.unicode_asencodedstring |
| 81 |
|
| 82 |
self.assertEqual(asencodedstring('abc', NULL), b'abc') |
| 83 |
self.assertEqual(asencodedstring('abc', 'ascii'), b'abc') |
| 84 |
s = 'a\xa1\u4f60\U0001f600' |
| 85 |
b = b'a\xc2\xa1\xe4\xbd\xa0\xf0\x9f\x98\x80' |
| 86 |
self.assertEqual(asencodedstring(s, NULL), b) |
| 87 |
self.assertEqual(asencodedstring(s, 'utf-8'), b) |
| 88 |
self.assertEqual(asencodedstring('\xa1\xa2', 'latin1'), b'\xa1\xa2') |
| 89 |
self.assertRaises(UnicodeEncodeError, asencodedstring, '\xa1\xa2', 'ascii') |
| 90 |
self.assertEqual(asencodedstring(s, 'ascii', 'replace'), b'a???') |
| 91 |
|
| 92 |
self.assertRaises(LookupError, asencodedstring, 'abc', 'foo') |
| 93 |
self.assertRaises(LookupError, asencodedstring, s, 'ascii', 'foo') |
| 94 |
self.assertRaises(TypeError, asencodedstring, b'abc', NULL) |
| 95 |
self.assertRaises(TypeError, asencodedstring, b'abc', 'ascii') |
| 96 |
self.assertRaises(TypeError, asencodedstring, [], NULL) |
| 97 |
self.assertRaises(TypeError, asencodedstring, [], 'ascii') |
| 98 |
# CRASHES asencodedstring(NULL, NULL) |
| 99 |
# CRASHES asencodedstring(NULL, 'ascii') |
| 100 |
|
| 101 |
def test_decodeutf8(self): |
| 102 |
"""Test PyUnicode_DecodeUTF8()""" |
| 103 |
decodeutf8 = _testlimitedcapi.unicode_decodeutf8 |
| 104 |
|
| 105 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: |
| 106 |
b = s.encode('utf-8') |
| 107 |
self.assertEqual(decodeutf8(b), s) |
| 108 |
self.assertEqual(decodeutf8(b, 'strict'), s) |
| 109 |
|
| 110 |
self.assertRaises(UnicodeDecodeError, decodeutf8, b'\x80') |
| 111 |
self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xc0') |
| 112 |
self.assertRaises(UnicodeDecodeError, decodeutf8, b'\xff') |
| 113 |
self.assertRaises(UnicodeDecodeError, decodeutf8, b'a\xf0\x9f') |
| 114 |
self.assertEqual(decodeutf8(b'a\xf0\x9f', 'replace'), 'a\ufffd') |
| 115 |
self.assertEqual(decodeutf8(b'a\xf0\x9fb', 'replace'), 'a\ufffdb') |
| 116 |
|
| 117 |
self.assertRaises(LookupError, decodeutf8, b'a\x80', 'foo') |
| 118 |
# TODO: Test PyUnicode_DecodeUTF8() with NULL as data and |
| 119 |
# negative size. |
| 120 |
|
| 121 |
def test_decodeutf8stateful(self): |
| 122 |
"""Test PyUnicode_DecodeUTF8Stateful()""" |
| 123 |
decodeutf8stateful = _testlimitedcapi.unicode_decodeutf8stateful |
| 124 |
|
| 125 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: |
| 126 |
b = s.encode('utf-8') |
| 127 |
self.assertEqual(decodeutf8stateful(b), (s, len(b))) |
| 128 |
self.assertEqual(decodeutf8stateful(b, 'strict'), (s, len(b))) |
| 129 |
|
| 130 |
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\x80') |
| 131 |
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xc0') |
| 132 |
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'\xff') |
| 133 |
self.assertEqual(decodeutf8stateful(b'a\xf0\x9f'), ('a', 1)) |
| 134 |
self.assertEqual(decodeutf8stateful(b'a\xf0\x9f', 'replace'), ('a', 1)) |
| 135 |
self.assertRaises(UnicodeDecodeError, decodeutf8stateful, b'a\xf0\x9fb') |
| 136 |
self.assertEqual(decodeutf8stateful(b'a\xf0\x9fb', 'replace'), ('a\ufffdb', 4)) |
| 137 |
|
| 138 |
self.assertRaises(LookupError, decodeutf8stateful, b'a\x80', 'foo') |
| 139 |
# TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as data and |
| 140 |
# negative size. |
| 141 |
# TODO: Test PyUnicode_DecodeUTF8Stateful() with NULL as the address of |
| 142 |
# "consumed". |
| 143 |
|
| 144 |
def test_asutf8string(self): |
| 145 |
"""Test PyUnicode_AsUTF8String()""" |
| 146 |
asutf8string = _testlimitedcapi.unicode_asutf8string |
| 147 |
|
| 148 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: |
| 149 |
self.assertEqual(asutf8string(s), s.encode('utf-8')) |
| 150 |
|
| 151 |
self.assertRaises(UnicodeEncodeError, asutf8string, '\ud8ff') |
| 152 |
self.assertRaises(TypeError, asutf8string, b'abc') |
| 153 |
self.assertRaises(TypeError, asutf8string, []) |
| 154 |
# CRASHES asutf8string(NULL) |
| 155 |
|
| 156 |
def test_decodeutf16(self): |
| 157 |
"""Test PyUnicode_DecodeUTF16()""" |
| 158 |
decodeutf16 = _testlimitedcapi.unicode_decodeutf16 |
| 159 |
|
| 160 |
naturalbyteorder = -1 if sys.byteorder == 'little' else 1 |
| 161 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: |
| 162 |
b = s.encode('utf-16') |
| 163 |
self.assertEqual(decodeutf16(0, b), (naturalbyteorder, s)) |
| 164 |
b = s.encode('utf-16le') |
| 165 |
self.assertEqual(decodeutf16(-1, b), (-1, s)) |
| 166 |
self.assertEqual(decodeutf16(0, b'\xff\xfe'+b), (-1, s)) |
| 167 |
b = s.encode('utf-16be') |
| 168 |
self.assertEqual(decodeutf16(1, b), (1, s)) |
| 169 |
self.assertEqual(decodeutf16(0, b'\xfe\xff'+b), (1, s)) |
| 170 |
|
| 171 |
self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'a') |
| 172 |
self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'a') |
| 173 |
self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xff\xfea') |
| 174 |
self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xfe\xffa') |
| 175 |
|
| 176 |
self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x00\xde') |
| 177 |
self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xde\x00') |
| 178 |
self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xde\xde') |
| 179 |
self.assertEqual(decodeutf16(-1, b'\x00\xde', 'replace'), (-1, '\ufffd')) |
| 180 |
self.assertEqual(decodeutf16(1, b'\xde\x00', 'replace'), (1, '\ufffd')) |
| 181 |
self.assertEqual(decodeutf16(0, b'\xde\xde', 'replace'), (0, '\ufffd')) |
| 182 |
self.assertEqual(decodeutf16(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd')) |
| 183 |
self.assertEqual(decodeutf16(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd')) |
| 184 |
|
| 185 |
self.assertRaises(UnicodeDecodeError, decodeutf16, -1, b'\x3d\xd8') |
| 186 |
self.assertRaises(UnicodeDecodeError, decodeutf16, 1, b'\xd8\x3d') |
| 187 |
self.assertRaises(UnicodeDecodeError, decodeutf16, 0, b'\xd8\xd8') |
| 188 |
self.assertEqual(decodeutf16(-1, b'\x3d\xd8', 'replace'), (-1, '\ufffd')) |
| 189 |
self.assertEqual(decodeutf16(1, b'\xd8\x3d', 'replace'), (1, '\ufffd')) |
| 190 |
self.assertEqual(decodeutf16(0, b'\xd8\xd8', 'replace'), (0, '\ufffd')) |
| 191 |
self.assertEqual(decodeutf16(0, b'\xff\xfe\x3d\xd8', 'replace'), (-1, '\ufffd')) |
| 192 |
self.assertEqual(decodeutf16(0, b'\xfe\xff\xd8\x3d', 'replace'), (1, '\ufffd')) |
| 193 |
|
| 194 |
self.assertRaises(LookupError, decodeutf16, -1, b'\x00\xde', 'foo') |
| 195 |
self.assertRaises(LookupError, decodeutf16, 1, b'\xde\x00', 'foo') |
| 196 |
self.assertRaises(LookupError, decodeutf16, 0, b'\xde\xde', 'foo') |
| 197 |
# TODO: Test PyUnicode_DecodeUTF16() with NULL as data and |
| 198 |
# negative size. |
| 199 |
|
| 200 |
def test_decodeutf16stateful(self): |
| 201 |
"""Test PyUnicode_DecodeUTF16Stateful()""" |
| 202 |
decodeutf16stateful = _testlimitedcapi.unicode_decodeutf16stateful |
| 203 |
|
| 204 |
naturalbyteorder = -1 if sys.byteorder == 'little' else 1 |
| 205 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: |
| 206 |
b = s.encode('utf-16') |
| 207 |
self.assertEqual(decodeutf16stateful(0, b), (naturalbyteorder, s, len(b))) |
| 208 |
b = s.encode('utf-16le') |
| 209 |
self.assertEqual(decodeutf16stateful(-1, b), (-1, s, len(b))) |
| 210 |
self.assertEqual(decodeutf16stateful(0, b'\xff\xfe'+b), (-1, s, len(b)+2)) |
| 211 |
b = s.encode('utf-16be') |
| 212 |
self.assertEqual(decodeutf16stateful(1, b), (1, s, len(b))) |
| 213 |
self.assertEqual(decodeutf16stateful(0, b'\xfe\xff'+b), (1, s, len(b)+2)) |
| 214 |
|
| 215 |
self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d'), (-1, 'a', 2)) |
| 216 |
self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8'), (-1, 'a', 2)) |
| 217 |
self.assertEqual(decodeutf16stateful(-1, b'\x61\x00\x3d\xd8\x00'), (-1, 'a', 2)) |
| 218 |
self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8'), (1, 'a', 2)) |
| 219 |
self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d'), (1, 'a', 2)) |
| 220 |
self.assertEqual(decodeutf16stateful(1, b'\x00\x61\xd8\x3d\xde'), (1, 'a', 2)) |
| 221 |
self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x61\x00\x3d\xd8\x00'), (-1, 'a', 4)) |
| 222 |
self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\x00\x61\xd8\x3d\xde'), (1, 'a', 4)) |
| 223 |
|
| 224 |
self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x00\xde') |
| 225 |
self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xde\x00') |
| 226 |
self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 0, b'\xde\xde') |
| 227 |
self.assertEqual(decodeutf16stateful(-1, b'\x00\xde', 'replace'), (-1, '\ufffd', 2)) |
| 228 |
self.assertEqual(decodeutf16stateful(1, b'\xde\x00', 'replace'), (1, '\ufffd', 2)) |
| 229 |
self.assertEqual(decodeutf16stateful(0, b'\xde\xde', 'replace'), (0, '\ufffd', 2)) |
| 230 |
self.assertEqual(decodeutf16stateful(0, b'\xff\xfe\x00\xde', 'replace'), (-1, '\ufffd', 4)) |
| 231 |
self.assertEqual(decodeutf16stateful(0, b'\xfe\xff\xde\x00', 'replace'), (1, '\ufffd', 4)) |
| 232 |
|
| 233 |
self.assertRaises(UnicodeDecodeError, decodeutf16stateful, -1, b'\x3d\xd8\x61\x00') |
| 234 |
self.assertEqual(decodeutf16stateful(-1, b'\x3d\xd8\x61\x00', 'replace'), (-1, '\ufffda', 4)) |
| 235 |
self.assertRaises(UnicodeDecodeError, decodeutf16stateful, 1, b'\xd8\x3d\x00\x61') |
| 236 |
self.assertEqual(decodeutf16stateful(1, b'\xd8\x3d\x00\x61', 'replace'), (1, '\ufffda', 4)) |
| 237 |
|
| 238 |
self.assertRaises(LookupError, decodeutf16stateful, -1, b'\x00\xde', 'foo') |
| 239 |
self.assertRaises(LookupError, decodeutf16stateful, 1, b'\xde\x00', 'foo') |
| 240 |
self.assertRaises(LookupError, decodeutf16stateful, 0, b'\xde\xde', 'foo') |
| 241 |
# TODO: Test PyUnicode_DecodeUTF16Stateful() with NULL as data and |
| 242 |
# negative size. |
| 243 |
# TODO: Test PyUnicode_DecodeUTF16Stateful() with NULL as the address of |
| 244 |
# "consumed". |
| 245 |
|
| 246 |
def test_asutf16string(self): |
| 247 |
"""Test PyUnicode_AsUTF16String()""" |
| 248 |
asutf16string = _testlimitedcapi.unicode_asutf16string |
| 249 |
|
| 250 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: |
| 251 |
self.assertEqual(asutf16string(s), s.encode('utf-16')) |
| 252 |
|
| 253 |
self.assertRaises(UnicodeEncodeError, asutf16string, '\ud8ff') |
| 254 |
self.assertRaises(TypeError, asutf16string, b'abc') |
| 255 |
self.assertRaises(TypeError, asutf16string, []) |
| 256 |
# CRASHES asutf16string(NULL) |
| 257 |
|
| 258 |
def test_decodeutf32(self): |
| 259 |
"""Test PyUnicode_DecodeUTF8()""" |
| 260 |
decodeutf32 = _testlimitedcapi.unicode_decodeutf32 |
| 261 |
|
| 262 |
naturalbyteorder = -1 if sys.byteorder == 'little' else 1 |
| 263 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: |
| 264 |
b = s.encode('utf-32') |
| 265 |
self.assertEqual(decodeutf32(0, b), (naturalbyteorder, s)) |
| 266 |
b = s.encode('utf-32le') |
| 267 |
self.assertEqual(decodeutf32(-1, b), (-1, s)) |
| 268 |
self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00'+b), (-1, s)) |
| 269 |
b = s.encode('utf-32be') |
| 270 |
self.assertEqual(decodeutf32(1, b), (1, s)) |
| 271 |
self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff'+b), (1, s)) |
| 272 |
|
| 273 |
self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x61\x00\x00\x00\x00') |
| 274 |
self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\x00\x61\x00') |
| 275 |
self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00') |
| 276 |
self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00') |
| 277 |
|
| 278 |
self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\xff\xff\xff\xff') |
| 279 |
self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\xff\xff\xff\xff') |
| 280 |
self.assertRaises(UnicodeDecodeError, decodeutf32, 0, b'\xff\xff\xff\xff') |
| 281 |
self.assertEqual(decodeutf32(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd')) |
| 282 |
self.assertEqual(decodeutf32(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd')) |
| 283 |
self.assertEqual(decodeutf32(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd')) |
| 284 |
self.assertEqual(decodeutf32(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd')) |
| 285 |
self.assertEqual(decodeutf32(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd')) |
| 286 |
|
| 287 |
self.assertRaises(UnicodeDecodeError, decodeutf32, -1, b'\x3d\xd8\x00\x00') |
| 288 |
self.assertEqual(decodeutf32(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd')) |
| 289 |
self.assertRaises(UnicodeDecodeError, decodeutf32, 1, b'\x00\x00\xd8\x3d') |
| 290 |
self.assertEqual(decodeutf32(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd')) |
| 291 |
|
| 292 |
self.assertRaises(LookupError, decodeutf32, -1, b'\xff\xff\xff\xff', 'foo') |
| 293 |
self.assertRaises(LookupError, decodeutf32, 1, b'\xff\xff\xff\xff', 'foo') |
| 294 |
self.assertRaises(LookupError, decodeutf32, 0, b'\xff\xff\xff\xff', 'foo') |
| 295 |
# TODO: Test PyUnicode_DecodeUTF32() with NULL as data and |
| 296 |
# negative size. |
| 297 |
|
| 298 |
def test_decodeutf32stateful(self): |
| 299 |
"""Test PyUnicode_DecodeUTF32Stateful()""" |
| 300 |
decodeutf32stateful = _testlimitedcapi.unicode_decodeutf32stateful |
| 301 |
|
| 302 |
naturalbyteorder = -1 if sys.byteorder == 'little' else 1 |
| 303 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: |
| 304 |
b = s.encode('utf-32') |
| 305 |
self.assertEqual(decodeutf32stateful(0, b), (naturalbyteorder, s, len(b))) |
| 306 |
b = s.encode('utf-32le') |
| 307 |
self.assertEqual(decodeutf32stateful(-1, b), (-1, s, len(b))) |
| 308 |
self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, s, len(b)+4)) |
| 309 |
b = s.encode('utf-32be') |
| 310 |
self.assertEqual(decodeutf32stateful(1, b), (1, s, len(b))) |
| 311 |
self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, s, len(b)+4)) |
| 312 |
|
| 313 |
self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00'), (-1, 'a', 4)) |
| 314 |
self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6'), (-1, 'a', 4)) |
| 315 |
self.assertEqual(decodeutf32stateful(-1, b'\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 4)) |
| 316 |
self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00'), (1, 'a', 4)) |
| 317 |
self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01'), (1, 'a', 4)) |
| 318 |
self.assertEqual(decodeutf32stateful(1, b'\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 4)) |
| 319 |
self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\x61\x00\x00\x00\x00\xf6\x01'), (-1, 'a', 8)) |
| 320 |
self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\x00\x00\x00\x61\x00\x01\xf6'), (1, 'a', 8)) |
| 321 |
|
| 322 |
for b in b'\xff', b'\xff\xff', b'\xff\xff\xff': |
| 323 |
self.assertEqual(decodeutf32stateful(-1, b), (-1, '', 0)) |
| 324 |
self.assertEqual(decodeutf32stateful(1, b), (1, '', 0)) |
| 325 |
self.assertEqual(decodeutf32stateful(0, b), (0, '', 0)) |
| 326 |
self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00'+b), (-1, '', 4)) |
| 327 |
self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff'+b), (1, '', 4)) |
| 328 |
self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\xff\xff\xff\xff') |
| 329 |
self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\xff\xff\xff\xff') |
| 330 |
self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 0, b'\xff\xff\xff\xff') |
| 331 |
self.assertEqual(decodeutf32stateful(-1, b'\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 4)) |
| 332 |
self.assertEqual(decodeutf32stateful(1, b'\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 4)) |
| 333 |
self.assertEqual(decodeutf32stateful(0, b'\xff\xff\xff\xff', 'replace'), (0, '\ufffd', 4)) |
| 334 |
self.assertEqual(decodeutf32stateful(0, b'\xff\xfe\x00\x00\xff\xff\xff\xff', 'replace'), (-1, '\ufffd', 8)) |
| 335 |
self.assertEqual(decodeutf32stateful(0, b'\x00\x00\xfe\xff\xff\xff\xff\xff', 'replace'), (1, '\ufffd', 8)) |
| 336 |
|
| 337 |
self.assertRaises(UnicodeDecodeError, decodeutf32stateful, -1, b'\x3d\xd8\x00\x00') |
| 338 |
self.assertEqual(decodeutf32stateful(-1, b'\x3d\xd8\x00\x00', 'replace'), (-1, '\ufffd', 4)) |
| 339 |
self.assertRaises(UnicodeDecodeError, decodeutf32stateful, 1, b'\x00\x00\xd8\x3d') |
| 340 |
self.assertEqual(decodeutf32stateful(1, b'\x00\x00\xd8\x3d', 'replace'), (1, '\ufffd', 4)) |
| 341 |
|
| 342 |
self.assertRaises(LookupError, decodeutf32stateful, -1, b'\xff\xff\xff\xff', 'foo') |
| 343 |
self.assertRaises(LookupError, decodeutf32stateful, 1, b'\xff\xff\xff\xff', 'foo') |
| 344 |
self.assertRaises(LookupError, decodeutf32stateful, 0, b'\xff\xff\xff\xff', 'foo') |
| 345 |
# TODO: Test PyUnicode_DecodeUTF32Stateful() with NULL as data and |
| 346 |
# negative size. |
| 347 |
# TODO: Test PyUnicode_DecodeUTF32Stateful() with NULL as the address of |
| 348 |
# "consumed". |
| 349 |
|
| 350 |
def test_asutf32string(self): |
| 351 |
"""Test PyUnicode_AsUTF32String()""" |
| 352 |
asutf32string = _testlimitedcapi.unicode_asutf32string |
| 353 |
|
| 354 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600']: |
| 355 |
self.assertEqual(asutf32string(s), s.encode('utf-32')) |
| 356 |
|
| 357 |
self.assertRaises(UnicodeEncodeError, asutf32string, '\ud8ff') |
| 358 |
self.assertRaises(TypeError, asutf32string, b'abc') |
| 359 |
self.assertRaises(TypeError, asutf32string, []) |
| 360 |
# CRASHES asutf32string(NULL) |
| 361 |
|
| 362 |
def test_decodelatin1(self): |
| 363 |
"""Test PyUnicode_DecodeLatin1()""" |
| 364 |
decodelatin1 = _testlimitedcapi.unicode_decodelatin1 |
| 365 |
|
| 366 |
self.assertEqual(decodelatin1(b'abc'), 'abc') |
| 367 |
self.assertEqual(decodelatin1(b'abc', 'strict'), 'abc') |
| 368 |
self.assertEqual(decodelatin1(b'\xa1\xa2'), '\xa1\xa2') |
| 369 |
self.assertEqual(decodelatin1(b'\xa1\xa2', 'strict'), '\xa1\xa2') |
| 370 |
# TODO: Test PyUnicode_DecodeLatin1() with NULL as data and |
| 371 |
# negative size. |
| 372 |
|
| 373 |
def test_aslatin1string(self): |
| 374 |
"""Test PyUnicode_AsLatin1String()""" |
| 375 |
aslatin1string = _testlimitedcapi.unicode_aslatin1string |
| 376 |
|
| 377 |
self.assertEqual(aslatin1string('abc'), b'abc') |
| 378 |
self.assertEqual(aslatin1string('\xa1\xa2'), b'\xa1\xa2') |
| 379 |
|
| 380 |
self.assertRaises(UnicodeEncodeError, aslatin1string, '\u4f60') |
| 381 |
self.assertRaises(TypeError, aslatin1string, b'abc') |
| 382 |
self.assertRaises(TypeError, aslatin1string, []) |
| 383 |
# CRASHES aslatin1string(NULL) |
| 384 |
|
| 385 |
def test_decodeascii(self): |
| 386 |
"""Test PyUnicode_DecodeASCII()""" |
| 387 |
decodeascii = _testlimitedcapi.unicode_decodeascii |
| 388 |
|
| 389 |
self.assertEqual(decodeascii(b'abc'), 'abc') |
| 390 |
self.assertEqual(decodeascii(b'abc', 'strict'), 'abc') |
| 391 |
|
| 392 |
self.assertRaises(UnicodeDecodeError, decodeascii, b'\xff') |
| 393 |
self.assertEqual(decodeascii(b'a\xff', 'replace'), 'a\ufffd') |
| 394 |
self.assertEqual(decodeascii(b'a\xffb', 'replace'), 'a\ufffdb') |
| 395 |
|
| 396 |
self.assertRaises(LookupError, decodeascii, b'a\xff', 'foo') |
| 397 |
# TODO: Test PyUnicode_DecodeASCII() with NULL as data and |
| 398 |
# negative size. |
| 399 |
|
| 400 |
def test_asasciistring(self): |
| 401 |
"""Test PyUnicode_AsASCIIString()""" |
| 402 |
asasciistring = _testlimitedcapi.unicode_asasciistring |
| 403 |
|
| 404 |
self.assertEqual(asasciistring('abc'), b'abc') |
| 405 |
|
| 406 |
self.assertRaises(UnicodeEncodeError, asasciistring, '\x80') |
| 407 |
self.assertRaises(TypeError, asasciistring, b'abc') |
| 408 |
self.assertRaises(TypeError, asasciistring, []) |
| 409 |
# CRASHES asasciistring(NULL) |
| 410 |
|
| 411 |
def test_decodecharmap(self): |
| 412 |
"""Test PyUnicode_DecodeCharmap()""" |
| 413 |
decodecharmap = _testlimitedcapi.unicode_decodecharmap |
| 414 |
|
| 415 |
self.assertEqual(decodecharmap(b'\3\0\7', {0: 'a', 3: 'b', 7: 'c'}), 'bac') |
| 416 |
self.assertEqual(decodecharmap(b'\1\0\2', ['a', 'b', 'c']), 'bac') |
| 417 |
self.assertEqual(decodecharmap(b'\1\0\2', 'abc'), 'bac') |
| 418 |
self.assertEqual(decodecharmap(b'\1\0\2', ['\xa1', '\xa2', '\xa3']), '\xa2\xa1\xa3') |
| 419 |
self.assertEqual(decodecharmap(b'\1\0\2', ['\u4f60', '\u597d', '\u4e16']), '\u597d\u4f60\u4e16') |
| 420 |
self.assertEqual(decodecharmap(b'\1\0\2', ['\U0001f600', '\U0001f601', '\U0001f602']), '\U0001f601\U0001f600\U0001f602') |
| 421 |
|
| 422 |
self.assertEqual(decodecharmap(b'\1\0\2', [97, 98, 99]), 'bac') |
| 423 |
self.assertEqual(decodecharmap(b'\1\0\2', ['', 'b', 'cd']), 'bcd') |
| 424 |
|
| 425 |
self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {}) |
| 426 |
self.assertRaises(UnicodeDecodeError, decodecharmap, b'\0', {0: None}) |
| 427 |
self.assertEqual(decodecharmap(b'\1\0\2', [None, 'b', 'c'], 'replace'), 'b\ufffdc') |
| 428 |
self.assertEqual(decodecharmap(b'\1\0\2\xff', NULL), '\1\0\2\xff') |
| 429 |
self.assertRaises(TypeError, decodecharmap, b'\0', 42) |
| 430 |
|
| 431 |
# TODO: Test PyUnicode_DecodeCharmap() with NULL as data and |
| 432 |
# negative size. |
| 433 |
|
| 434 |
def test_ascharmapstring(self): |
| 435 |
"""Test PyUnicode_AsCharmapString()""" |
| 436 |
ascharmapstring = _testlimitedcapi.unicode_ascharmapstring |
| 437 |
|
| 438 |
self.assertEqual(ascharmapstring('abc', {97: 3, 98: 0, 99: 7}), b'\3\0\7') |
| 439 |
self.assertEqual(ascharmapstring('\xa1\xa2\xa3', {0xa1: 3, 0xa2: 0, 0xa3: 7}), b'\3\0\7') |
| 440 |
self.assertEqual(ascharmapstring('\u4f60\u597d\u4e16', {0x4f60: 3, 0x597d: 0, 0x4e16: 7}), b'\3\0\7') |
| 441 |
self.assertEqual(ascharmapstring('\U0001f600\U0001f601\U0001f602', {0x1f600: 3, 0x1f601: 0, 0x1f602: 7}), b'\3\0\7') |
| 442 |
self.assertEqual(ascharmapstring('abc', {97: 3, 98: b'', 99: b'spam'}), b'\3spam') |
| 443 |
|
| 444 |
self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {}) |
| 445 |
self.assertRaises(UnicodeEncodeError, ascharmapstring, 'a', {97: None}) |
| 446 |
self.assertRaises(TypeError, ascharmapstring, b'a', {}) |
| 447 |
self.assertRaises(TypeError, ascharmapstring, [], {}) |
| 448 |
self.assertRaises(TypeError, ascharmapstring, 'a', NULL) |
| 449 |
# CRASHES ascharmapstring(NULL, {}) |
| 450 |
|
| 451 |
def test_decodeunicodeescape(self): |
| 452 |
"""Test PyUnicode_DecodeUnicodeEscape()""" |
| 453 |
decodeunicodeescape = _testlimitedcapi.unicode_decodeunicodeescape |
| 454 |
|
| 455 |
self.assertEqual(decodeunicodeescape(b'abc'), 'abc') |
| 456 |
self.assertEqual(decodeunicodeescape(br'\t\n\r\x0b\x0c\x00\\'), '\t\n\r\v\f\0\\') |
| 457 |
self.assertEqual(decodeunicodeescape(b'\t\n\r\x0b\x0c\x00'), '\t\n\r\v\f\0') |
| 458 |
self.assertEqual(decodeunicodeescape(br'\xa1\xa2'), '\xa1\xa2') |
| 459 |
self.assertEqual(decodeunicodeescape(b'\xa1\xa2'), '\xa1\xa2') |
| 460 |
self.assertEqual(decodeunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d') |
| 461 |
self.assertEqual(decodeunicodeescape(br'\U0001f600'), '\U0001f600') |
| 462 |
with self.assertWarns(DeprecationWarning): |
| 463 |
self.assertEqual(decodeunicodeescape(br'\z'), r'\z') |
| 464 |
|
| 465 |
for b in b'\\', br'\xa', br'\u4f6', br'\U0001f60': |
| 466 |
self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b) |
| 467 |
self.assertRaises(UnicodeDecodeError, decodeunicodeescape, b, 'strict') |
| 468 |
self.assertEqual(decodeunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd') |
| 469 |
self.assertEqual(decodeunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy') |
| 470 |
|
| 471 |
self.assertRaises(LookupError, decodeunicodeescape, b'\\', 'foo') |
| 472 |
# TODO: Test PyUnicode_DecodeUnicodeEscape() with NULL as data and |
| 473 |
# negative size. |
| 474 |
|
| 475 |
def test_asunicodeescapestring(self): |
| 476 |
"""Test PyUnicode_AsUnicodeEscapeString()""" |
| 477 |
asunicodeescapestring = _testlimitedcapi.unicode_asunicodeescapestring |
| 478 |
|
| 479 |
self.assertEqual(asunicodeescapestring('abc'), b'abc') |
| 480 |
self.assertEqual(asunicodeescapestring('\t\n\r\v\f\0\\'), br'\t\n\r\x0b\x0c\x00\\') |
| 481 |
self.assertEqual(asunicodeescapestring('\xa1\xa2'), br'\xa1\xa2') |
| 482 |
self.assertEqual(asunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d') |
| 483 |
self.assertEqual(asunicodeescapestring('\U0001f600'), br'\U0001f600') |
| 484 |
|
| 485 |
self.assertRaises(TypeError, asunicodeescapestring, b'abc') |
| 486 |
self.assertRaises(TypeError, asunicodeescapestring, []) |
| 487 |
# CRASHES asunicodeescapestring(NULL) |
| 488 |
|
| 489 |
def test_decoderawunicodeescape(self): |
| 490 |
"""Test PyUnicode_DecodeRawUnicodeEscape()""" |
| 491 |
decoderawunicodeescape = _testlimitedcapi.unicode_decoderawunicodeescape |
| 492 |
|
| 493 |
self.assertEqual(decoderawunicodeescape(b'abc'), 'abc') |
| 494 |
self.assertEqual(decoderawunicodeescape(b'\t\n\r\v\f\0\\'), '\t\n\r\v\f\0\\') |
| 495 |
self.assertEqual(decoderawunicodeescape(b'\xa1\xa2'), '\xa1\xa2') |
| 496 |
self.assertEqual(decoderawunicodeescape(br'\u4f60\u597d'), '\u4f60\u597d') |
| 497 |
self.assertEqual(decoderawunicodeescape(br'\U0001f600'), '\U0001f600') |
| 498 |
self.assertEqual(decoderawunicodeescape(br'\xa1\xa2'), r'\xa1\xa2') |
| 499 |
self.assertEqual(decoderawunicodeescape(br'\z'), r'\z') |
| 500 |
|
| 501 |
for b in br'\u4f6', br'\U0001f60': |
| 502 |
self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b) |
| 503 |
self.assertRaises(UnicodeDecodeError, decoderawunicodeescape, b, 'strict') |
| 504 |
self.assertEqual(decoderawunicodeescape(br'x\U0001f60', 'replace'), 'x\ufffd') |
| 505 |
self.assertEqual(decoderawunicodeescape(br'x\U0001f60y', 'replace'), 'x\ufffdy') |
| 506 |
|
| 507 |
self.assertRaises(LookupError, decoderawunicodeescape, br'\U0001f60', 'foo') |
| 508 |
# TODO: Test PyUnicode_DecodeRawUnicodeEscape() with NULL as data and |
| 509 |
# negative size. |
| 510 |
|
| 511 |
def test_asrawunicodeescapestring(self): |
| 512 |
"""Test PyUnicode_AsRawUnicodeEscapeString()""" |
| 513 |
asrawunicodeescapestring = _testlimitedcapi.unicode_asrawunicodeescapestring |
| 514 |
|
| 515 |
self.assertEqual(asrawunicodeescapestring('abc'), b'abc') |
| 516 |
self.assertEqual(asrawunicodeescapestring('\t\n\r\v\f\0\\'), b'\t\n\r\v\f\0\\') |
| 517 |
self.assertEqual(asrawunicodeescapestring('\xa1\xa2'), b'\xa1\xa2') |
| 518 |
self.assertEqual(asrawunicodeescapestring('\u4f60\u597d'), br'\u4f60\u597d') |
| 519 |
self.assertEqual(asrawunicodeescapestring('\U0001f600'), br'\U0001f600') |
| 520 |
|
| 521 |
self.assertRaises(TypeError, asrawunicodeescapestring, b'abc') |
| 522 |
self.assertRaises(TypeError, asrawunicodeescapestring, []) |
| 523 |
# CRASHES asrawunicodeescapestring(NULL) |
| 524 |
|
| 525 |
|
| 526 |
class CAPICodecs(unittest.TestCase): |
| 527 |
|
| 528 |
def setUp(self): |
| 529 |
# Encoding names are normalized internally by converting them |
| 530 |
# to lowercase and their hyphens are replaced by underscores. |
| 531 |
self.encoding_name = 'test.test_capi.test_codecs.codec_reversed' |
| 532 |
# Make sure that our custom codec is not already registered (that |
| 533 |
# way we know whether we correctly unregistered the custom codec |
| 534 |
# after a test or not). |
| 535 |
self.assertRaises(LookupError, codecs.lookup, self.encoding_name) |
| 536 |
# create the search function without registering yet |
| 537 |
self._create_custom_codec() |
| 538 |
|
| 539 |
def _create_custom_codec(self): |
| 540 |
def codec_encoder(m, errors='strict'): |
| 541 |
return (type(m)().join(reversed(m)), len(m)) |
| 542 |
|
| 543 |
def codec_decoder(c, errors='strict'): |
| 544 |
return (type(c)().join(reversed(c)), len(c)) |
| 545 |
|
| 546 |
class IncrementalEncoder(codecs.IncrementalEncoder): |
| 547 |
def encode(self, input, final=False): |
| 548 |
return codec_encoder(input) |
| 549 |
|
| 550 |
class IncrementalDecoder(codecs.IncrementalDecoder): |
| 551 |
def decode(self, input, final=False): |
| 552 |
return codec_decoder(input) |
| 553 |
|
| 554 |
class StreamReader(codecs.StreamReader): |
| 555 |
def encode(self, input, errors='strict'): |
| 556 |
return codec_encoder(input, errors=errors) |
| 557 |
|
| 558 |
def decode(self, input, errors='strict'): |
| 559 |
return codec_decoder(input, errors=errors) |
| 560 |
|
| 561 |
class StreamWriter(codecs.StreamWriter): |
| 562 |
def encode(self, input, errors='strict'): |
| 563 |
return codec_encoder(input, errors=errors) |
| 564 |
|
| 565 |
def decode(self, input, errors='strict'): |
| 566 |
return codec_decoder(input, errors=errors) |
| 567 |
|
| 568 |
info = codecs.CodecInfo( |
| 569 |
encode=codec_encoder, |
| 570 |
decode=codec_decoder, |
| 571 |
streamreader=StreamReader, |
| 572 |
streamwriter=StreamWriter, |
| 573 |
incrementalencoder=IncrementalEncoder, |
| 574 |
incrementaldecoder=IncrementalDecoder, |
| 575 |
name=self.encoding_name |
| 576 |
) |
| 577 |
|
| 578 |
def search_function(encoding): |
| 579 |
if encoding == self.encoding_name: |
| 580 |
return info |
| 581 |
return None |
| 582 |
|
| 583 |
self.codec_info = info |
| 584 |
self.search_function = search_function |
| 585 |
|
| 586 |
@contextlib.contextmanager |
| 587 |
def use_custom_encoder(self): |
| 588 |
self.assertRaises(LookupError, codecs.lookup, self.encoding_name) |
| 589 |
codecs.register(self.search_function) |
| 590 |
yield |
| 591 |
codecs.unregister(self.search_function) |
| 592 |
self.assertRaises(LookupError, codecs.lookup, self.encoding_name) |
| 593 |
|
| 594 |
def test_codec_register(self): |
| 595 |
search_function, encoding = self.search_function, self.encoding_name |
| 596 |
# register the search function using the C API |
| 597 |
self.assertIsNone(_testcapi.codec_register(search_function)) |
| 598 |
# in case the test failed before cleaning up |
| 599 |
self.addCleanup(codecs.unregister, self.search_function) |
| 600 |
self.assertIs(codecs.lookup(encoding), search_function(encoding)) |
| 601 |
self.assertEqual(codecs.encode('123', encoding=encoding), '321') |
| 602 |
# unregister the search function using the regular API |
| 603 |
codecs.unregister(search_function) |
| 604 |
self.assertRaises(LookupError, codecs.lookup, encoding) |
| 605 |
|
| 606 |
def test_codec_unregister(self): |
| 607 |
search_function, encoding = self.search_function, self.encoding_name |
| 608 |
self.assertRaises(LookupError, codecs.lookup, encoding) |
| 609 |
# register the search function using the regular API |
| 610 |
codecs.register(search_function) |
| 611 |
# in case the test failed before cleaning up |
| 612 |
self.addCleanup(codecs.unregister, self.search_function) |
| 613 |
self.assertIsNotNone(codecs.lookup(encoding)) |
| 614 |
# unregister the search function using the C API |
| 615 |
self.assertIsNone(_testcapi.codec_unregister(search_function)) |
| 616 |
self.assertRaises(LookupError, codecs.lookup, encoding) |
| 617 |
|
| 618 |
def test_codec_known_encoding(self): |
| 619 |
self.assertRaises(LookupError, codecs.lookup, 'unknown-codec') |
| 620 |
self.assertFalse(_testcapi.codec_known_encoding('unknown-codec')) |
| 621 |
self.assertFalse(_testcapi.codec_known_encoding('unknown_codec')) |
| 622 |
self.assertFalse(_testcapi.codec_known_encoding('UNKNOWN-codec')) |
| 623 |
|
| 624 |
encoding_name = self.encoding_name |
| 625 |
self.assertRaises(LookupError, codecs.lookup, encoding_name) |
| 626 |
|
| 627 |
codecs.register(self.search_function) |
| 628 |
self.addCleanup(codecs.unregister, self.search_function) |
| 629 |
|
| 630 |
for name in [ |
| 631 |
encoding_name, |
| 632 |
encoding_name.upper(), |
| 633 |
encoding_name.replace('_', '-'), |
| 634 |
]: |
| 635 |
with self.subTest(name): |
| 636 |
self.assertTrue(_testcapi.codec_known_encoding(name)) |
| 637 |
|
| 638 |
def test_codec_encode(self): |
| 639 |
encode = _testcapi.codec_encode |
| 640 |
self.assertEqual(encode('a', 'utf-8', NULL), b'a') |
| 641 |
self.assertEqual(encode('a', 'utf-8', 'strict'), b'a') |
| 642 |
self.assertEqual(encode('[é]', 'ascii', 'ignore'), b'[]') |
| 643 |
|
| 644 |
self.assertRaises(TypeError, encode, NULL, 'ascii', 'strict') |
| 645 |
with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): |
| 646 |
encode('a', NULL, 'strict') |
| 647 |
|
| 648 |
def test_codec_decode(self): |
| 649 |
decode = _testcapi.codec_decode |
| 650 |
|
| 651 |
s = 'a\xa1\u4f60\U0001f600' |
| 652 |
b = s.encode() |
| 653 |
|
| 654 |
self.assertEqual(decode(b, 'utf-8', 'strict'), s) |
| 655 |
self.assertEqual(decode(b, 'utf-8', NULL), s) |
| 656 |
self.assertEqual(decode(b, 'latin1', 'strict'), b.decode('latin1')) |
| 657 |
self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', 'strict') |
| 658 |
self.assertRaises(UnicodeDecodeError, decode, b, 'ascii', NULL) |
| 659 |
self.assertEqual(decode(b, 'ascii', 'replace'), 'a' + '\ufffd'*9) |
| 660 |
|
| 661 |
# _codecs.decode() only reports an unknown error handling name when |
| 662 |
# the corresponding error handling function is used; this difers |
| 663 |
# from PyUnicode_Decode() which checks that both the encoding and |
| 664 |
# the error handling name are recognized before even attempting to |
| 665 |
# call the decoder. |
| 666 |
self.assertEqual(decode(b'', 'utf-8', 'unknown-error-handler'), '') |
| 667 |
self.assertEqual(decode(b'a', 'utf-8', 'unknown-error-handler'), 'a') |
| 668 |
|
| 669 |
self.assertRaises(TypeError, decode, NULL, 'ascii', 'strict') |
| 670 |
with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): |
| 671 |
decode(b, NULL, 'strict') |
| 672 |
|
| 673 |
def test_codec_encoder(self): |
| 674 |
codec_encoder = _testcapi.codec_encoder |
| 675 |
|
| 676 |
with self.use_custom_encoder(): |
| 677 |
encoder = codec_encoder(self.encoding_name) |
| 678 |
self.assertIs(encoder, self.codec_info.encode) |
| 679 |
|
| 680 |
with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): |
| 681 |
codec_encoder(NULL) |
| 682 |
|
| 683 |
def test_codec_decoder(self): |
| 684 |
codec_decoder = _testcapi.codec_decoder |
| 685 |
|
| 686 |
with self.use_custom_encoder(): |
| 687 |
decoder = codec_decoder(self.encoding_name) |
| 688 |
self.assertIs(decoder, self.codec_info.decode) |
| 689 |
|
| 690 |
with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): |
| 691 |
codec_decoder(NULL) |
| 692 |
|
| 693 |
def test_codec_incremental_encoder(self): |
| 694 |
codec_incremental_encoder = _testcapi.codec_incremental_encoder |
| 695 |
|
| 696 |
with self.use_custom_encoder(): |
| 697 |
encoding = self.encoding_name |
| 698 |
|
| 699 |
for errors in ['strict', NULL]: |
| 700 |
with self.subTest(errors): |
| 701 |
encoder = codec_incremental_encoder(encoding, errors) |
| 702 |
self.assertIsInstance(encoder, self.codec_info.incrementalencoder) |
| 703 |
|
| 704 |
with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): |
| 705 |
codec_incremental_encoder(NULL, 'strict') |
| 706 |
|
| 707 |
def test_codec_incremental_decoder(self): |
| 708 |
codec_incremental_decoder = _testcapi.codec_incremental_decoder |
| 709 |
|
| 710 |
with self.use_custom_encoder(): |
| 711 |
encoding = self.encoding_name |
| 712 |
|
| 713 |
for errors in ['strict', NULL]: |
| 714 |
with self.subTest(errors): |
| 715 |
decoder = codec_incremental_decoder(encoding, errors) |
| 716 |
self.assertIsInstance(decoder, self.codec_info.incrementaldecoder) |
| 717 |
|
| 718 |
with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): |
| 719 |
codec_incremental_decoder(NULL, 'strict') |
| 720 |
|
| 721 |
def test_codec_stream_reader(self): |
| 722 |
codec_stream_reader = _testcapi.codec_stream_reader |
| 723 |
|
| 724 |
with self.use_custom_encoder(): |
| 725 |
encoding, stream = self.encoding_name, io.StringIO() |
| 726 |
for errors in ['strict', NULL]: |
| 727 |
with self.subTest(errors): |
| 728 |
writer = codec_stream_reader(encoding, stream, errors) |
| 729 |
self.assertIsInstance(writer, self.codec_info.streamreader) |
| 730 |
|
| 731 |
with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): |
| 732 |
codec_stream_reader(NULL, stream, 'strict') |
| 733 |
|
| 734 |
def test_codec_stream_writer(self): |
| 735 |
codec_stream_writer = _testcapi.codec_stream_writer |
| 736 |
|
| 737 |
with self.use_custom_encoder(): |
| 738 |
encoding, stream = self.encoding_name, io.StringIO() |
| 739 |
for errors in ['strict', NULL]: |
| 740 |
with self.subTest(errors): |
| 741 |
writer = codec_stream_writer(encoding, stream, errors) |
| 742 |
self.assertIsInstance(writer, self.codec_info.streamwriter) |
| 743 |
|
| 744 |
with self.assertRaisesRegex(TypeError, BAD_ARGUMENT): |
| 745 |
codec_stream_writer(NULL, stream, 'strict') |
| 746 |
|
| 747 |
|
| 748 |
class CAPICodecErrors(unittest.TestCase): |
| 749 |
|
| 750 |
@classmethod |
| 751 |
def _generate_exception_args(cls): |
| 752 |
for objlen in range(5): |
| 753 |
maxind = 2 * max(2, objlen) |
| 754 |
for start in range(-maxind, maxind + 1): |
| 755 |
for end in range(-maxind, maxind + 1): |
| 756 |
yield objlen, start, end |
| 757 |
|
| 758 |
@classmethod |
| 759 |
def generate_encode_errors(cls): |
| 760 |
return tuple( |
| 761 |
UnicodeEncodeError('utf-8', '0' * objlen, start, end, 'why') |
| 762 |
for objlen, start, end in cls._generate_exception_args() |
| 763 |
) |
| 764 |
|
| 765 |
@classmethod |
| 766 |
def generate_decode_errors(cls): |
| 767 |
return tuple( |
| 768 |
UnicodeDecodeError('utf-8', b'0' * objlen, start, end, 'why') |
| 769 |
for objlen, start, end in cls._generate_exception_args() |
| 770 |
) |
| 771 |
|
| 772 |
@classmethod |
| 773 |
def generate_translate_errors(cls): |
| 774 |
return tuple( |
| 775 |
UnicodeTranslateError('0' * objlen, start, end, 'why') |
| 776 |
for objlen, start, end in cls._generate_exception_args() |
| 777 |
) |
| 778 |
|
| 779 |
@classmethod |
| 780 |
def setUpClass(cls): |
| 781 |
cls.unicode_encode_errors = cls.generate_encode_errors() |
| 782 |
cls.unicode_decode_errors = cls.generate_decode_errors() |
| 783 |
cls.unicode_translate_errors = cls.generate_translate_errors() |
| 784 |
cls.all_unicode_errors = ( |
| 785 |
cls.unicode_encode_errors |
| 786 |
+ cls.unicode_decode_errors |
| 787 |
+ cls.unicode_translate_errors |
| 788 |
) |
| 789 |
cls.bad_unicode_errors = ( |
| 790 |
ValueError(), |
| 791 |
) |
| 792 |
|
| 793 |
def test_codec_register_error(self): |
| 794 |
# for cleaning up between tests |
| 795 |
from _codecs import _unregister_error as _codecs_unregister_error |
| 796 |
|
| 797 |
self.assertRaises(LookupError, _testcapi.codec_lookup_error, 'custom') |
| 798 |
|
| 799 |
def custom_error_handler(exc): |
| 800 |
raise exc |
| 801 |
|
| 802 |
error_handler = mock.Mock(wraps=custom_error_handler) |
| 803 |
_testcapi.codec_register_error('custom', error_handler) |
| 804 |
self.addCleanup(_codecs_unregister_error, 'custom') |
| 805 |
|
| 806 |
self.assertRaises(UnicodeEncodeError, codecs.encode, |
| 807 |
'\xff', 'ascii', errors='custom') |
| 808 |
error_handler.assert_called_once() |
| 809 |
error_handler.reset_mock() |
| 810 |
|
| 811 |
self.assertRaises(UnicodeDecodeError, codecs.decode, |
| 812 |
b'\xff', 'ascii', errors='custom') |
| 813 |
error_handler.assert_called_once() |
| 814 |
|
| 815 |
# _codecs._unregister_error directly delegates to the internal C |
| 816 |
# function so a Python-level function test is sufficient (it is |
| 817 |
# tested in test_codeccallbacks). |
| 818 |
|
| 819 |
def test_codec_lookup_error(self): |
| 820 |
codec_lookup_error = _testcapi.codec_lookup_error |
| 821 |
self.assertIs(codec_lookup_error(NULL), codecs.strict_errors) |
| 822 |
self.assertIs(codec_lookup_error('strict'), codecs.strict_errors) |
| 823 |
self.assertIs(codec_lookup_error('ignore'), codecs.ignore_errors) |
| 824 |
self.assertIs(codec_lookup_error('replace'), codecs.replace_errors) |
| 825 |
self.assertIs(codec_lookup_error('xmlcharrefreplace'), codecs.xmlcharrefreplace_errors) |
| 826 |
self.assertIs(codec_lookup_error('backslashreplace'), codecs.backslashreplace_errors) |
| 827 |
self.assertIs(codec_lookup_error('namereplace'), codecs.namereplace_errors) |
| 828 |
self.assertRaises(LookupError, codec_lookup_error, 'unknown') |
| 829 |
|
| 830 |
def test_codec_strict_errors_handler(self): |
| 831 |
handler = _testcapi.codec_strict_errors |
| 832 |
for exc in self.all_unicode_errors + self.bad_unicode_errors: |
| 833 |
with self.subTest(handler=handler, exc=exc): |
| 834 |
self.assertRaises(type(exc), handler, exc) |
| 835 |
|
| 836 |
def test_codec_ignore_errors_handler(self): |
| 837 |
handler = _testcapi.codec_ignore_errors |
| 838 |
self.do_test_codec_errors_handler(handler, self.all_unicode_errors) |
| 839 |
|
| 840 |
def test_codec_replace_errors_handler(self): |
| 841 |
handler = _testcapi.codec_replace_errors |
| 842 |
self.do_test_codec_errors_handler(handler, self.all_unicode_errors) |
| 843 |
|
| 844 |
def test_codec_xmlcharrefreplace_errors_handler(self): |
| 845 |
handler = _testcapi.codec_xmlcharrefreplace_errors |
| 846 |
self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) |
| 847 |
|
| 848 |
def test_codec_backslashreplace_errors_handler(self): |
| 849 |
handler = _testcapi.codec_backslashreplace_errors |
| 850 |
self.do_test_codec_errors_handler(handler, self.all_unicode_errors) |
| 851 |
|
| 852 |
def test_codec_namereplace_errors_handler(self): |
| 853 |
handler = _testlimitedcapi.codec_namereplace_errors |
| 854 |
self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) |
| 855 |
|
| 856 |
def do_test_codec_errors_handler(self, handler, exceptions): |
| 857 |
self.assertNotEqual(len(exceptions), 0) |
| 858 |
for exc in exceptions: |
| 859 |
with self.subTest(handler=handler, exc=exc): |
| 860 |
# test that the handler does not crash |
| 861 |
res = handler(exc) |
| 862 |
self.assertIsInstance(res, tuple) |
| 863 |
self.assertEqual(len(res), 2) |
| 864 |
replacement, continue_from = res |
| 865 |
self.assertIsInstance(replacement, str) |
| 866 |
self.assertIsInstance(continue_from, int) |
| 867 |
self.assertGreaterEqual(continue_from, 0) |
| 868 |
self.assertLessEqual(continue_from, len(exc.object)) |
| 869 |
|
| 870 |
for bad_exc in ( |
| 871 |
self.bad_unicode_errors |
| 872 |
+ tuple(e for e in self.all_unicode_errors if e not in exceptions) |
| 873 |
): |
| 874 |
with self.subTest('bad type', handler=handler, exc=bad_exc): |
| 875 |
self.assertRaises(TypeError, handler, bad_exc) |
| 876 |
|
| 877 |
|
| 878 |
if __name__ == "__main__": |
| 879 |
unittest.main() |