| 1 |
import unittest |
| 2 |
import sys |
| 3 |
from test import support |
| 4 |
from test.support import threading_helper |
| 5 |
|
| 6 |
try: |
| 7 |
import _testcapi |
| 8 |
from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX |
| 9 |
except ImportError: |
| 10 |
_testcapi = None |
| 11 |
try: |
| 12 |
import _testlimitedcapi |
| 13 |
except ImportError: |
| 14 |
_testlimitedcapi = None |
| 15 |
try: |
| 16 |
import _testinternalcapi |
| 17 |
except ImportError: |
| 18 |
_testinternalcapi = None |
| 19 |
try: |
| 20 |
import ctypes |
| 21 |
except ImportError: |
| 22 |
ctypes = None |
| 23 |
|
| 24 |
|
| 25 |
NULL = None |
| 26 |
|
| 27 |
class Str(str): |
| 28 |
pass |
| 29 |
|
| 30 |
|
| 31 |
class CAPITest(unittest.TestCase): |
| 32 |
|
| 33 |
@support.cpython_only |
| 34 |
@unittest.skipIf(_testcapi is None, 'need _testcapi module') |
| 35 |
def test_new(self): |
| 36 |
"""Test PyUnicode_New()""" |
| 37 |
from _testcapi import unicode_new as new |
| 38 |
|
| 39 |
for maxchar in 0, 0x61, 0xa1, 0x4f60, 0x1f600, 0x10ffff: |
| 40 |
self.assertEqual(new(0, maxchar), '') |
| 41 |
self.assertEqual(new(5, maxchar), chr(maxchar)*5) |
| 42 |
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX, maxchar) |
| 43 |
self.assertEqual(new(0, 0x110000), '') |
| 44 |
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x4f60) |
| 45 |
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x4f60) |
| 46 |
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x1f600) |
| 47 |
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x1f600) |
| 48 |
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4, 0x1f600) |
| 49 |
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4+1, 0x1f600) |
| 50 |
self.assertRaises(SystemError, new, 5, 0x110000) |
| 51 |
self.assertRaises(SystemError, new, -1, 0) |
| 52 |
self.assertRaises(SystemError, new, PY_SSIZE_T_MIN, 0) |
| 53 |
|
| 54 |
@support.cpython_only |
| 55 |
@unittest.skipIf(_testcapi is None, 'need _testcapi module') |
| 56 |
def test_fill(self): |
| 57 |
"""Test PyUnicode_Fill()""" |
| 58 |
from _testcapi import unicode_fill as fill |
| 59 |
|
| 60 |
strings = [ |
| 61 |
# all strings have exactly 5 characters |
| 62 |
'abcde', '\xa1\xa2\xa3\xa4\xa5', |
| 63 |
'\u4f60\u597d\u4e16\u754c\uff01', |
| 64 |
'\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' |
| 65 |
] |
| 66 |
chars = [0x78, 0xa9, 0x20ac, 0x1f638] |
| 67 |
|
| 68 |
for idx, fill_char in enumerate(chars): |
| 69 |
# wide -> narrow: exceed maxchar limitation |
| 70 |
for to in strings[:idx]: |
| 71 |
self.assertRaises(ValueError, fill, to, 0, 0, fill_char) |
| 72 |
for to in strings[idx:]: |
| 73 |
for start in [*range(7), PY_SSIZE_T_MAX]: |
| 74 |
for length in [*range(-1, 7 - start), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]: |
| 75 |
filled = max(min(length, 5 - start), 0) |
| 76 |
if filled == 5 and to != strings[idx]: |
| 77 |
# narrow -> wide |
| 78 |
# Tests omitted since this creates invalid strings. |
| 79 |
continue |
| 80 |
expected = to[:start] + chr(fill_char) * filled + to[start + filled:] |
| 81 |
self.assertEqual(fill(to, start, length, fill_char), |
| 82 |
(expected, filled)) |
| 83 |
|
| 84 |
s = strings[0] |
| 85 |
self.assertRaises(IndexError, fill, s, -1, 0, 0x78) |
| 86 |
self.assertRaises(IndexError, fill, s, PY_SSIZE_T_MIN, 0, 0x78) |
| 87 |
self.assertRaises(ValueError, fill, s, 0, 0, 0x110000) |
| 88 |
self.assertRaises(SystemError, fill, b'abc', 0, 0, 0x78) |
| 89 |
self.assertRaises(SystemError, fill, [], 0, 0, 0x78) |
| 90 |
# CRASHES fill(s, 0, NULL, 0, 0) |
| 91 |
# CRASHES fill(NULL, 0, 0, 0x78) |
| 92 |
# TODO: Test PyUnicode_Fill() with non-modifiable unicode. |
| 93 |
|
| 94 |
@support.cpython_only |
| 95 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 96 |
def test_writechar(self): |
| 97 |
"""Test PyUnicode_WriteChar()""" |
| 98 |
from _testlimitedcapi import unicode_writechar as writechar |
| 99 |
|
| 100 |
strings = [ |
| 101 |
# one string for every kind |
| 102 |
'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', |
| 103 |
'\U0001f600\U0001f601\U0001f602' |
| 104 |
] |
| 105 |
# one character for every kind + out of range code |
| 106 |
chars = [0x78, 0xa9, 0x20ac, 0x1f638, 0x110000] |
| 107 |
for i, s in enumerate(strings): |
| 108 |
for j, c in enumerate(chars): |
| 109 |
if j <= i: |
| 110 |
self.assertEqual(writechar(s, 1, c), |
| 111 |
(s[:1] + chr(c) + s[2:], 0)) |
| 112 |
else: |
| 113 |
self.assertRaises(ValueError, writechar, s, 1, c) |
| 114 |
|
| 115 |
self.assertRaises(IndexError, writechar, 'abc', 3, 0x78) |
| 116 |
self.assertRaises(IndexError, writechar, 'abc', -1, 0x78) |
| 117 |
self.assertRaises(IndexError, writechar, 'abc', PY_SSIZE_T_MAX, 0x78) |
| 118 |
self.assertRaises(IndexError, writechar, 'abc', PY_SSIZE_T_MIN, 0x78) |
| 119 |
self.assertRaises(TypeError, writechar, b'abc', 0, 0x78) |
| 120 |
self.assertRaises(TypeError, writechar, [], 0, 0x78) |
| 121 |
# CRASHES writechar(NULL, 0, 0x78) |
| 122 |
# TODO: Test PyUnicode_WriteChar() with non-modifiable and legacy |
| 123 |
# unicode. |
| 124 |
|
| 125 |
@support.cpython_only |
| 126 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 127 |
def test_resize(self): |
| 128 |
"""Test PyUnicode_Resize()""" |
| 129 |
from _testlimitedcapi import unicode_resize as resize |
| 130 |
|
| 131 |
strings = [ |
| 132 |
# all strings have exactly 3 characters |
| 133 |
'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', |
| 134 |
'\U0001f600\U0001f601\U0001f602' |
| 135 |
] |
| 136 |
for s in strings: |
| 137 |
self.assertEqual(resize(s, 3), (s, 0)) |
| 138 |
self.assertEqual(resize(s, 2), (s[:2], 0)) |
| 139 |
self.assertEqual(resize(s, 4), (s + '\0', 0)) |
| 140 |
self.assertEqual(resize(s, 10), (s + '\0'*7, 0)) |
| 141 |
self.assertEqual(resize(s, 0), ('', 0)) |
| 142 |
self.assertRaises(MemoryError, resize, s, PY_SSIZE_T_MAX) |
| 143 |
self.assertRaises(SystemError, resize, s, -1) |
| 144 |
self.assertRaises(SystemError, resize, s, PY_SSIZE_T_MIN) |
| 145 |
self.assertRaises(SystemError, resize, b'abc', 0) |
| 146 |
self.assertRaises(SystemError, resize, [], 0) |
| 147 |
self.assertRaises(SystemError, resize, NULL, 0) |
| 148 |
# TODO: Test PyUnicode_Resize() with non-modifiable and legacy unicode |
| 149 |
# and with NULL as the address. |
| 150 |
|
| 151 |
@support.cpython_only |
| 152 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 153 |
def test_append(self): |
| 154 |
"""Test PyUnicode_Append()""" |
| 155 |
from _testlimitedcapi import unicode_append as append |
| 156 |
|
| 157 |
strings = [ |
| 158 |
'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', |
| 159 |
'\U0001f600\U0001f601\U0001f602' |
| 160 |
] |
| 161 |
for left in strings: |
| 162 |
left = left[::-1] |
| 163 |
for right in strings: |
| 164 |
expected = left + right |
| 165 |
self.assertEqual(append(left, right), expected) |
| 166 |
|
| 167 |
self.assertRaises(SystemError, append, 'abc', b'abc') |
| 168 |
self.assertRaises(SystemError, append, b'abc', 'abc') |
| 169 |
self.assertRaises(SystemError, append, b'abc', b'abc') |
| 170 |
self.assertRaises(SystemError, append, 'abc', []) |
| 171 |
self.assertRaises(SystemError, append, [], 'abc') |
| 172 |
self.assertRaises(SystemError, append, [], []) |
| 173 |
self.assertRaises(SystemError, append, NULL, 'abc') |
| 174 |
self.assertRaises(SystemError, append, 'abc', NULL) |
| 175 |
# TODO: Test PyUnicode_Append() with modifiable unicode |
| 176 |
# and with NULL as the address. |
| 177 |
# TODO: Check reference counts. |
| 178 |
|
| 179 |
@support.cpython_only |
| 180 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 181 |
def test_appendanddel(self): |
| 182 |
"""Test PyUnicode_AppendAndDel()""" |
| 183 |
from _testlimitedcapi import unicode_appendanddel as appendanddel |
| 184 |
|
| 185 |
strings = [ |
| 186 |
'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', |
| 187 |
'\U0001f600\U0001f601\U0001f602' |
| 188 |
] |
| 189 |
for left in strings: |
| 190 |
left = left[::-1] |
| 191 |
for right in strings: |
| 192 |
self.assertEqual(appendanddel(left, right), left + right) |
| 193 |
|
| 194 |
self.assertRaises(SystemError, appendanddel, 'abc', b'abc') |
| 195 |
self.assertRaises(SystemError, appendanddel, b'abc', 'abc') |
| 196 |
self.assertRaises(SystemError, appendanddel, b'abc', b'abc') |
| 197 |
self.assertRaises(SystemError, appendanddel, 'abc', []) |
| 198 |
self.assertRaises(SystemError, appendanddel, [], 'abc') |
| 199 |
self.assertRaises(SystemError, appendanddel, [], []) |
| 200 |
self.assertRaises(SystemError, appendanddel, NULL, 'abc') |
| 201 |
self.assertRaises(SystemError, appendanddel, 'abc', NULL) |
| 202 |
# TODO: Test PyUnicode_AppendAndDel() with modifiable unicode |
| 203 |
# and with NULL as the address. |
| 204 |
# TODO: Check reference counts. |
| 205 |
|
| 206 |
@support.cpython_only |
| 207 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 208 |
def test_fromstringandsize(self): |
| 209 |
"""Test PyUnicode_FromStringAndSize()""" |
| 210 |
from _testlimitedcapi import unicode_fromstringandsize as fromstringandsize |
| 211 |
|
| 212 |
self.assertEqual(fromstringandsize(b'abc'), 'abc') |
| 213 |
self.assertEqual(fromstringandsize(b'abc', 2), 'ab') |
| 214 |
self.assertEqual(fromstringandsize(b'abc\0def'), 'abc\0def') |
| 215 |
self.assertEqual(fromstringandsize(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') |
| 216 |
self.assertEqual(fromstringandsize(b'\xe4\xbd\xa0'), '\u4f60') |
| 217 |
self.assertEqual(fromstringandsize(b'\xf0\x9f\x98\x80'), '\U0001f600') |
| 218 |
self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xc2\xa1', 1) |
| 219 |
self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xa1', 1) |
| 220 |
self.assertEqual(fromstringandsize(b'', 0), '') |
| 221 |
self.assertEqual(fromstringandsize(NULL, 0), '') |
| 222 |
|
| 223 |
self.assertRaises(MemoryError, fromstringandsize, b'abc', PY_SSIZE_T_MAX) |
| 224 |
self.assertRaises(SystemError, fromstringandsize, b'abc', -1) |
| 225 |
self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN) |
| 226 |
self.assertRaises(SystemError, fromstringandsize, NULL, -1) |
| 227 |
self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MIN) |
| 228 |
self.assertRaises(SystemError, fromstringandsize, NULL, 3) |
| 229 |
self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MAX) |
| 230 |
|
| 231 |
@support.cpython_only |
| 232 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 233 |
def test_fromstring(self): |
| 234 |
"""Test PyUnicode_FromString()""" |
| 235 |
from _testlimitedcapi import unicode_fromstring as fromstring |
| 236 |
|
| 237 |
self.assertEqual(fromstring(b'abc'), 'abc') |
| 238 |
self.assertEqual(fromstring(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') |
| 239 |
self.assertEqual(fromstring(b'\xe4\xbd\xa0'), '\u4f60') |
| 240 |
self.assertEqual(fromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') |
| 241 |
self.assertRaises(UnicodeDecodeError, fromstring, b'\xc2') |
| 242 |
self.assertRaises(UnicodeDecodeError, fromstring, b'\xa1') |
| 243 |
self.assertEqual(fromstring(b''), '') |
| 244 |
|
| 245 |
# CRASHES fromstring(NULL) |
| 246 |
|
| 247 |
@support.cpython_only |
| 248 |
@unittest.skipIf(_testcapi is None, 'need _testcapi module') |
| 249 |
def test_fromkindanddata(self): |
| 250 |
"""Test PyUnicode_FromKindAndData()""" |
| 251 |
from _testcapi import unicode_fromkindanddata as fromkindanddata |
| 252 |
|
| 253 |
strings = [ |
| 254 |
'abcde', '\xa1\xa2\xa3\xa4\xa5', |
| 255 |
'\u4f60\u597d\u4e16\u754c\uff01', |
| 256 |
'\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' |
| 257 |
] |
| 258 |
enc1 = 'latin1' |
| 259 |
for s in strings[:2]: |
| 260 |
self.assertEqual(fromkindanddata(1, s.encode(enc1)), s) |
| 261 |
enc2 = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' |
| 262 |
for s in strings[:3]: |
| 263 |
self.assertEqual(fromkindanddata(2, s.encode(enc2)), s) |
| 264 |
enc4 = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' |
| 265 |
for s in strings: |
| 266 |
self.assertEqual(fromkindanddata(4, s.encode(enc4)), s) |
| 267 |
self.assertEqual(fromkindanddata(2, '\U0001f600'.encode(enc2)), |
| 268 |
'\ud83d\ude00') |
| 269 |
for kind in 1, 2, 4: |
| 270 |
self.assertEqual(fromkindanddata(kind, b''), '') |
| 271 |
self.assertEqual(fromkindanddata(kind, b'\0'*kind), '\0') |
| 272 |
self.assertEqual(fromkindanddata(kind, NULL, 0), '') |
| 273 |
|
| 274 |
for kind in -1, 0, 3, 5, 8: |
| 275 |
self.assertRaises(SystemError, fromkindanddata, kind, b'') |
| 276 |
self.assertRaises(ValueError, fromkindanddata, 1, b'abc', -1) |
| 277 |
self.assertRaises(ValueError, fromkindanddata, 1, b'abc', PY_SSIZE_T_MIN) |
| 278 |
self.assertRaises(ValueError, fromkindanddata, 1, NULL, -1) |
| 279 |
self.assertRaises(ValueError, fromkindanddata, 1, NULL, PY_SSIZE_T_MIN) |
| 280 |
# CRASHES fromkindanddata(1, NULL, 1) |
| 281 |
# CRASHES fromkindanddata(4, b'\xff\xff\xff\xff') |
| 282 |
|
| 283 |
@support.cpython_only |
| 284 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 285 |
def test_substring(self): |
| 286 |
"""Test PyUnicode_Substring()""" |
| 287 |
from _testlimitedcapi import unicode_substring as substring |
| 288 |
|
| 289 |
strings = [ |
| 290 |
'ab', 'ab\xa1\xa2', |
| 291 |
'ab\xa1\xa2\u4f60\u597d', |
| 292 |
'ab\xa1\xa2\u4f60\u597d\U0001f600\U0001f601' |
| 293 |
] |
| 294 |
for s in strings: |
| 295 |
for start in [*range(0, len(s) + 2), PY_SSIZE_T_MAX]: |
| 296 |
for end in [*range(max(start-1, 0), len(s) + 2), PY_SSIZE_T_MAX]: |
| 297 |
self.assertEqual(substring(s, start, end), s[start:end]) |
| 298 |
|
| 299 |
self.assertRaises(IndexError, substring, 'abc', -1, 0) |
| 300 |
self.assertRaises(IndexError, substring, 'abc', PY_SSIZE_T_MIN, 0) |
| 301 |
self.assertRaises(IndexError, substring, 'abc', 0, -1) |
| 302 |
self.assertRaises(IndexError, substring, 'abc', 0, PY_SSIZE_T_MIN) |
| 303 |
# CRASHES substring(b'abc', 0, 0) |
| 304 |
# CRASHES substring([], 0, 0) |
| 305 |
# CRASHES substring(NULL, 0, 0) |
| 306 |
|
| 307 |
@support.cpython_only |
| 308 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 309 |
def test_getlength(self): |
| 310 |
"""Test PyUnicode_GetLength()""" |
| 311 |
from _testlimitedcapi import unicode_getlength as getlength |
| 312 |
|
| 313 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', |
| 314 |
'a\ud800b\udfffc', '\ud834\udd1e']: |
| 315 |
self.assertEqual(getlength(s), len(s)) |
| 316 |
|
| 317 |
self.assertRaises(TypeError, getlength, b'abc') |
| 318 |
self.assertRaises(TypeError, getlength, []) |
| 319 |
# CRASHES getlength(NULL) |
| 320 |
|
| 321 |
@support.cpython_only |
| 322 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 323 |
def test_readchar(self): |
| 324 |
"""Test PyUnicode_ReadChar()""" |
| 325 |
from _testlimitedcapi import unicode_readchar as readchar |
| 326 |
|
| 327 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', |
| 328 |
'a\ud800b\udfffc', '\ud834\udd1e']: |
| 329 |
for i, c in enumerate(s): |
| 330 |
self.assertEqual(readchar(s, i), ord(c)) |
| 331 |
self.assertRaises(IndexError, readchar, s, len(s)) |
| 332 |
self.assertRaises(IndexError, readchar, s, PY_SSIZE_T_MAX) |
| 333 |
self.assertRaises(IndexError, readchar, s, -1) |
| 334 |
self.assertRaises(IndexError, readchar, s, PY_SSIZE_T_MIN) |
| 335 |
|
| 336 |
self.assertRaises(TypeError, readchar, b'abc', 0) |
| 337 |
self.assertRaises(TypeError, readchar, [], 0) |
| 338 |
# CRASHES readchar(NULL, 0) |
| 339 |
|
| 340 |
@support.cpython_only |
| 341 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 342 |
def test_fromobject(self): |
| 343 |
"""Test PyUnicode_FromObject()""" |
| 344 |
from _testlimitedcapi import unicode_fromobject as fromobject |
| 345 |
|
| 346 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', |
| 347 |
'a\ud800b\udfffc', '\ud834\udd1e']: |
| 348 |
self.assertEqual(fromobject(s), s) |
| 349 |
o = Str(s) |
| 350 |
s2 = fromobject(o) |
| 351 |
self.assertEqual(s2, s) |
| 352 |
self.assertIs(type(s2), str) |
| 353 |
self.assertIsNot(s2, s) |
| 354 |
|
| 355 |
self.assertRaises(TypeError, fromobject, b'abc') |
| 356 |
self.assertRaises(TypeError, fromobject, []) |
| 357 |
# CRASHES fromobject(NULL) |
| 358 |
|
| 359 |
@unittest.skipIf(ctypes is None, 'need ctypes') |
| 360 |
def test_from_format(self): |
| 361 |
"""Test PyUnicode_FromFormat()""" |
| 362 |
# Length modifiers "j" and "t" are not tested here because ctypes does |
| 363 |
# not expose types for intmax_t and ptrdiff_t. |
| 364 |
# _testlimitedcapi.test_string_from_format() has a wider coverage of all |
| 365 |
# formats. |
| 366 |
from ctypes import ( |
| 367 |
c_char_p, |
| 368 |
pythonapi, py_object, sizeof, |
| 369 |
c_int, c_long, c_longlong, c_ssize_t, |
| 370 |
c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p, |
| 371 |
c_wchar, c_wchar_p) |
| 372 |
name = "PyUnicode_FromFormat" |
| 373 |
_PyUnicode_FromFormat = getattr(pythonapi, name) |
| 374 |
_PyUnicode_FromFormat.argtypes = (c_char_p,) |
| 375 |
_PyUnicode_FromFormat.restype = py_object |
| 376 |
|
| 377 |
def PyUnicode_FromFormat(format, *args): |
| 378 |
cargs = tuple( |
| 379 |
py_object(arg) if isinstance(arg, str) else arg |
| 380 |
for arg in args) |
| 381 |
return _PyUnicode_FromFormat(format, *cargs) |
| 382 |
|
| 383 |
def check_format(expected, format, *args): |
| 384 |
text = PyUnicode_FromFormat(format, *args) |
| 385 |
self.assertEqual(expected, text) |
| 386 |
|
| 387 |
# ascii format, non-ascii argument |
| 388 |
check_format('ascii\x7f=unicode\xe9', |
| 389 |
b'ascii\x7f=%U', 'unicode\xe9') |
| 390 |
|
| 391 |
# non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV() |
| 392 |
# raises an error |
| 393 |
self.assertRaisesRegex(ValueError, |
| 394 |
r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format ' |
| 395 |
'string, got a non-ASCII byte: 0xe9$', |
| 396 |
PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') |
| 397 |
|
| 398 |
# test "%c" |
| 399 |
check_format('\uabcd', |
| 400 |
b'%c', c_int(0xabcd)) |
| 401 |
check_format('\U0010ffff', |
| 402 |
b'%c', c_int(0x10ffff)) |
| 403 |
with self.assertRaises(OverflowError): |
| 404 |
PyUnicode_FromFormat(b'%c', c_int(0x110000)) |
| 405 |
# Issue #18183 |
| 406 |
check_format('\U00010000\U00100000', |
| 407 |
b'%c%c', c_int(0x10000), c_int(0x100000)) |
| 408 |
|
| 409 |
# test "%" |
| 410 |
check_format('%', |
| 411 |
b'%%') |
| 412 |
check_format('%s', |
| 413 |
b'%%s') |
| 414 |
check_format('[%]', |
| 415 |
b'[%%]') |
| 416 |
check_format('%abc', |
| 417 |
b'%%%s', b'abc') |
| 418 |
|
| 419 |
# truncated string |
| 420 |
check_format('abc', |
| 421 |
b'%.3s', b'abcdef') |
| 422 |
check_format('abc[', |
| 423 |
b'%.6s', 'abc[\u20ac]'.encode('utf8')) |
| 424 |
check_format('abc[\u20ac', |
| 425 |
b'%.7s', 'abc[\u20ac]'.encode('utf8')) |
| 426 |
check_format('abc[\ufffd', |
| 427 |
b'%.5s', b'abc[\xff]') |
| 428 |
check_format('abc[', |
| 429 |
b'%.6s', b'abc[\xe2\x82]') |
| 430 |
check_format('abc[\ufffd]', |
| 431 |
b'%.7s', b'abc[\xe2\x82]') |
| 432 |
check_format('abc[\ufffd', |
| 433 |
b'%.7s', b'abc[\xe2\x82\0') |
| 434 |
check_format(' abc[', |
| 435 |
b'%10.6s', 'abc[\u20ac]'.encode('utf8')) |
| 436 |
check_format(' abc[\u20ac', |
| 437 |
b'%10.7s', 'abc[\u20ac]'.encode('utf8')) |
| 438 |
check_format(' abc[\ufffd', |
| 439 |
b'%10.5s', b'abc[\xff]') |
| 440 |
check_format(' abc[', |
| 441 |
b'%10.6s', b'abc[\xe2\x82]') |
| 442 |
check_format(' abc[\ufffd]', |
| 443 |
b'%10.7s', b'abc[\xe2\x82]') |
| 444 |
|
| 445 |
check_format("'\\u20acABC'", |
| 446 |
b'%A', '\u20acABC') |
| 447 |
check_format("'\\u20", |
| 448 |
b'%.5A', '\u20acABCDEF') |
| 449 |
check_format("'\u20acABC'", |
| 450 |
b'%R', '\u20acABC') |
| 451 |
check_format("'\u20acA", |
| 452 |
b'%.3R', '\u20acABCDEF') |
| 453 |
check_format('\u20acAB', |
| 454 |
b'%.3S', '\u20acABCDEF') |
| 455 |
check_format('\u20acAB', |
| 456 |
b'%.3U', '\u20acABCDEF') |
| 457 |
|
| 458 |
check_format('\u20acAB', |
| 459 |
b'%.3V', '\u20acABCDEF', None) |
| 460 |
check_format('abc[', |
| 461 |
b'%.6V', None, 'abc[\u20ac]'.encode('utf8')) |
| 462 |
check_format('abc[\u20ac', |
| 463 |
b'%.7V', None, 'abc[\u20ac]'.encode('utf8')) |
| 464 |
check_format('abc[\ufffd', |
| 465 |
b'%.5V', None, b'abc[\xff]') |
| 466 |
check_format('abc[', |
| 467 |
b'%.6V', None, b'abc[\xe2\x82]') |
| 468 |
check_format('abc[\ufffd]', |
| 469 |
b'%.7V', None, b'abc[\xe2\x82]') |
| 470 |
check_format(' abc[', |
| 471 |
b'%10.6V', None, 'abc[\u20ac]'.encode('utf8')) |
| 472 |
check_format(' abc[\u20ac', |
| 473 |
b'%10.7V', None, 'abc[\u20ac]'.encode('utf8')) |
| 474 |
check_format(' abc[\ufffd', |
| 475 |
b'%10.5V', None, b'abc[\xff]') |
| 476 |
check_format(' abc[', |
| 477 |
b'%10.6V', None, b'abc[\xe2\x82]') |
| 478 |
check_format(' abc[\ufffd]', |
| 479 |
b'%10.7V', None, b'abc[\xe2\x82]') |
| 480 |
check_format(' abc[\ufffd', |
| 481 |
b'%10.7V', None, b'abc[\xe2\x82\0') |
| 482 |
|
| 483 |
# following tests comes from #7330 |
| 484 |
# test width modifier and precision modifier with %S |
| 485 |
check_format("repr= abc", |
| 486 |
b'repr=%5S', 'abc') |
| 487 |
check_format("repr=ab", |
| 488 |
b'repr=%.2S', 'abc') |
| 489 |
check_format("repr= ab", |
| 490 |
b'repr=%5.2S', 'abc') |
| 491 |
|
| 492 |
# test width modifier and precision modifier with %R |
| 493 |
check_format("repr= 'abc'", |
| 494 |
b'repr=%8R', 'abc') |
| 495 |
check_format("repr='ab", |
| 496 |
b'repr=%.3R', 'abc') |
| 497 |
check_format("repr= 'ab", |
| 498 |
b'repr=%5.3R', 'abc') |
| 499 |
|
| 500 |
# test width modifier and precision modifier with %A |
| 501 |
check_format("repr= 'abc'", |
| 502 |
b'repr=%8A', 'abc') |
| 503 |
check_format("repr='ab", |
| 504 |
b'repr=%.3A', 'abc') |
| 505 |
check_format("repr= 'ab", |
| 506 |
b'repr=%5.3A', 'abc') |
| 507 |
|
| 508 |
# test width modifier and precision modifier with %s |
| 509 |
check_format("repr= abc", |
| 510 |
b'repr=%5s', b'abc') |
| 511 |
check_format("repr=ab", |
| 512 |
b'repr=%.2s', b'abc') |
| 513 |
check_format("repr= ab", |
| 514 |
b'repr=%5.2s', b'abc') |
| 515 |
|
| 516 |
# test width modifier and precision modifier with %U |
| 517 |
check_format("repr= abc", |
| 518 |
b'repr=%5U', 'abc') |
| 519 |
check_format("repr=ab", |
| 520 |
b'repr=%.2U', 'abc') |
| 521 |
check_format("repr= ab", |
| 522 |
b'repr=%5.2U', 'abc') |
| 523 |
|
| 524 |
# test width modifier and precision modifier with %V |
| 525 |
check_format("repr= abc", |
| 526 |
b'repr=%5V', 'abc', b'123') |
| 527 |
check_format("repr=ab", |
| 528 |
b'repr=%.2V', 'abc', b'123') |
| 529 |
check_format("repr= ab", |
| 530 |
b'repr=%5.2V', 'abc', b'123') |
| 531 |
check_format("repr= 123", |
| 532 |
b'repr=%5V', None, b'123') |
| 533 |
check_format("repr=12", |
| 534 |
b'repr=%.2V', None, b'123') |
| 535 |
check_format("repr= 12", |
| 536 |
b'repr=%5.2V', None, b'123') |
| 537 |
|
| 538 |
# test integer formats (%i, %d, %u, %o, %x, %X) |
| 539 |
check_format('010', |
| 540 |
b'%03i', c_int(10)) |
| 541 |
check_format('0010', |
| 542 |
b'%0.4i', c_int(10)) |
| 543 |
for conv, signed, value, expected in [ |
| 544 |
(b'i', True, -123, '-123'), |
| 545 |
(b'd', True, -123, '-123'), |
| 546 |
(b'u', False, 123, '123'), |
| 547 |
(b'o', False, 0o123, '123'), |
| 548 |
(b'x', False, 0xabc, 'abc'), |
| 549 |
(b'X', False, 0xabc, 'ABC'), |
| 550 |
]: |
| 551 |
for mod, ctype in [ |
| 552 |
(b'', c_int if signed else c_uint), |
| 553 |
(b'l', c_long if signed else c_ulong), |
| 554 |
(b'll', c_longlong if signed else c_ulonglong), |
| 555 |
(b'z', c_ssize_t if signed else c_size_t), |
| 556 |
]: |
| 557 |
with self.subTest(format=b'%' + mod + conv): |
| 558 |
check_format(expected, |
| 559 |
b'%' + mod + conv, ctype(value)) |
| 560 |
|
| 561 |
# test long output |
| 562 |
min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1)) |
| 563 |
max_longlong = -min_longlong - 1 |
| 564 |
check_format(str(min_longlong), |
| 565 |
b'%lld', c_longlong(min_longlong)) |
| 566 |
check_format(str(max_longlong), |
| 567 |
b'%lld', c_longlong(max_longlong)) |
| 568 |
max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1 |
| 569 |
check_format(str(max_ulonglong), |
| 570 |
b'%llu', c_ulonglong(max_ulonglong)) |
| 571 |
PyUnicode_FromFormat(b'%p', c_void_p(-1)) |
| 572 |
|
| 573 |
# test padding (width and/or precision) |
| 574 |
check_format('123', b'%2i', c_int(123)) |
| 575 |
check_format(' 123', b'%10i', c_int(123)) |
| 576 |
check_format('0000000123', b'%010i', c_int(123)) |
| 577 |
check_format('123 ', b'%-10i', c_int(123)) |
| 578 |
check_format('123 ', b'%-010i', c_int(123)) |
| 579 |
check_format('123', b'%.2i', c_int(123)) |
| 580 |
check_format('0000123', b'%.7i', c_int(123)) |
| 581 |
check_format(' 123', b'%10.2i', c_int(123)) |
| 582 |
check_format(' 0000123', b'%10.7i', c_int(123)) |
| 583 |
check_format('0000000123', b'%010.7i', c_int(123)) |
| 584 |
check_format('0000123 ', b'%-10.7i', c_int(123)) |
| 585 |
check_format('0000123 ', b'%-010.7i', c_int(123)) |
| 586 |
|
| 587 |
check_format('-123', b'%2i', c_int(-123)) |
| 588 |
check_format(' -123', b'%10i', c_int(-123)) |
| 589 |
check_format('-000000123', b'%010i', c_int(-123)) |
| 590 |
check_format('-123 ', b'%-10i', c_int(-123)) |
| 591 |
check_format('-123 ', b'%-010i', c_int(-123)) |
| 592 |
check_format('-123', b'%.2i', c_int(-123)) |
| 593 |
check_format('-0000123', b'%.7i', c_int(-123)) |
| 594 |
check_format(' -123', b'%10.2i', c_int(-123)) |
| 595 |
check_format(' -0000123', b'%10.7i', c_int(-123)) |
| 596 |
check_format('-000000123', b'%010.7i', c_int(-123)) |
| 597 |
check_format('-0000123 ', b'%-10.7i', c_int(-123)) |
| 598 |
check_format('-0000123 ', b'%-010.7i', c_int(-123)) |
| 599 |
|
| 600 |
check_format('123', b'%2u', c_uint(123)) |
| 601 |
check_format(' 123', b'%10u', c_uint(123)) |
| 602 |
check_format('0000000123', b'%010u', c_uint(123)) |
| 603 |
check_format('123 ', b'%-10u', c_uint(123)) |
| 604 |
check_format('123 ', b'%-010u', c_uint(123)) |
| 605 |
check_format('123', b'%.2u', c_uint(123)) |
| 606 |
check_format('0000123', b'%.7u', c_uint(123)) |
| 607 |
check_format(' 123', b'%10.2u', c_uint(123)) |
| 608 |
check_format(' 0000123', b'%10.7u', c_uint(123)) |
| 609 |
check_format('0000000123', b'%010.7u', c_uint(123)) |
| 610 |
check_format('0000123 ', b'%-10.7u', c_uint(123)) |
| 611 |
check_format('0000123 ', b'%-010.7u', c_uint(123)) |
| 612 |
|
| 613 |
check_format('123', b'%2o', c_uint(0o123)) |
| 614 |
check_format(' 123', b'%10o', c_uint(0o123)) |
| 615 |
check_format('0000000123', b'%010o', c_uint(0o123)) |
| 616 |
check_format('123 ', b'%-10o', c_uint(0o123)) |
| 617 |
check_format('123 ', b'%-010o', c_uint(0o123)) |
| 618 |
check_format('123', b'%.2o', c_uint(0o123)) |
| 619 |
check_format('0000123', b'%.7o', c_uint(0o123)) |
| 620 |
check_format(' 123', b'%10.2o', c_uint(0o123)) |
| 621 |
check_format(' 0000123', b'%10.7o', c_uint(0o123)) |
| 622 |
check_format('0000000123', b'%010.7o', c_uint(0o123)) |
| 623 |
check_format('0000123 ', b'%-10.7o', c_uint(0o123)) |
| 624 |
check_format('0000123 ', b'%-010.7o', c_uint(0o123)) |
| 625 |
|
| 626 |
check_format('abc', b'%2x', c_uint(0xabc)) |
| 627 |
check_format(' abc', b'%10x', c_uint(0xabc)) |
| 628 |
check_format('0000000abc', b'%010x', c_uint(0xabc)) |
| 629 |
check_format('abc ', b'%-10x', c_uint(0xabc)) |
| 630 |
check_format('abc ', b'%-010x', c_uint(0xabc)) |
| 631 |
check_format('abc', b'%.2x', c_uint(0xabc)) |
| 632 |
check_format('0000abc', b'%.7x', c_uint(0xabc)) |
| 633 |
check_format(' abc', b'%10.2x', c_uint(0xabc)) |
| 634 |
check_format(' 0000abc', b'%10.7x', c_uint(0xabc)) |
| 635 |
check_format('0000000abc', b'%010.7x', c_uint(0xabc)) |
| 636 |
check_format('0000abc ', b'%-10.7x', c_uint(0xabc)) |
| 637 |
check_format('0000abc ', b'%-010.7x', c_uint(0xabc)) |
| 638 |
|
| 639 |
check_format('ABC', b'%2X', c_uint(0xabc)) |
| 640 |
check_format(' ABC', b'%10X', c_uint(0xabc)) |
| 641 |
check_format('0000000ABC', b'%010X', c_uint(0xabc)) |
| 642 |
check_format('ABC ', b'%-10X', c_uint(0xabc)) |
| 643 |
check_format('ABC ', b'%-010X', c_uint(0xabc)) |
| 644 |
check_format('ABC', b'%.2X', c_uint(0xabc)) |
| 645 |
check_format('0000ABC', b'%.7X', c_uint(0xabc)) |
| 646 |
check_format(' ABC', b'%10.2X', c_uint(0xabc)) |
| 647 |
check_format(' 0000ABC', b'%10.7X', c_uint(0xabc)) |
| 648 |
check_format('0000000ABC', b'%010.7X', c_uint(0xabc)) |
| 649 |
check_format('0000ABC ', b'%-10.7X', c_uint(0xabc)) |
| 650 |
check_format('0000ABC ', b'%-010.7X', c_uint(0xabc)) |
| 651 |
|
| 652 |
# test %A |
| 653 |
check_format(r"%A:'abc\xe9\uabcd\U0010ffff'", |
| 654 |
b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') |
| 655 |
|
| 656 |
# test %V |
| 657 |
check_format('abc', |
| 658 |
b'%V', 'abc', b'xyz') |
| 659 |
check_format('xyz', |
| 660 |
b'%V', None, b'xyz') |
| 661 |
|
| 662 |
# test %ls |
| 663 |
check_format('abc', b'%ls', c_wchar_p('abc')) |
| 664 |
check_format('\u4eba\u6c11', b'%ls', c_wchar_p('\u4eba\u6c11')) |
| 665 |
check_format('\U0001f4bb+\U0001f40d', |
| 666 |
b'%ls', c_wchar_p('\U0001f4bb+\U0001f40d')) |
| 667 |
check_format(' ab', b'%5.2ls', c_wchar_p('abc')) |
| 668 |
check_format(' \u4eba\u6c11', b'%5ls', c_wchar_p('\u4eba\u6c11')) |
| 669 |
check_format(' \U0001f4bb+\U0001f40d', |
| 670 |
b'%5ls', c_wchar_p('\U0001f4bb+\U0001f40d')) |
| 671 |
check_format('\u4eba', b'%.1ls', c_wchar_p('\u4eba\u6c11')) |
| 672 |
check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d', |
| 673 |
b'%.1ls', c_wchar_p('\U0001f4bb+\U0001f40d')) |
| 674 |
check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', |
| 675 |
b'%.2ls', c_wchar_p('\U0001f4bb+\U0001f40d')) |
| 676 |
|
| 677 |
# test %lV |
| 678 |
check_format('abc', |
| 679 |
b'%lV', 'abc', c_wchar_p('xyz')) |
| 680 |
check_format('xyz', |
| 681 |
b'%lV', None, c_wchar_p('xyz')) |
| 682 |
check_format('\u4eba\u6c11', |
| 683 |
b'%lV', None, c_wchar_p('\u4eba\u6c11')) |
| 684 |
check_format('\U0001f4bb+\U0001f40d', |
| 685 |
b'%lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) |
| 686 |
check_format(' ab', |
| 687 |
b'%5.2lV', None, c_wchar_p('abc')) |
| 688 |
check_format(' \u4eba\u6c11', |
| 689 |
b'%5lV', None, c_wchar_p('\u4eba\u6c11')) |
| 690 |
check_format(' \U0001f4bb+\U0001f40d', |
| 691 |
b'%5lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) |
| 692 |
check_format('\u4eba', |
| 693 |
b'%.1lV', None, c_wchar_p('\u4eba\u6c11')) |
| 694 |
check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d', |
| 695 |
b'%.1lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) |
| 696 |
check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb', |
| 697 |
b'%.2lV', None, c_wchar_p('\U0001f4bb+\U0001f40d')) |
| 698 |
|
| 699 |
# test %T |
| 700 |
check_format('type: str', |
| 701 |
b'type: %T', py_object("abc")) |
| 702 |
check_format(f'type: st', |
| 703 |
b'type: %.2T', py_object("abc")) |
| 704 |
check_format(f'type: str', |
| 705 |
b'type: %10T', py_object("abc")) |
| 706 |
|
| 707 |
class LocalType: |
| 708 |
pass |
| 709 |
obj = LocalType() |
| 710 |
fullname = f'{__name__}.{LocalType.__qualname__}' |
| 711 |
check_format(f'type: {fullname}', |
| 712 |
b'type: %T', py_object(obj)) |
| 713 |
fullname_alt = f'{__name__}:{LocalType.__qualname__}' |
| 714 |
check_format(f'type: {fullname_alt}', |
| 715 |
b'type: %#T', py_object(obj)) |
| 716 |
|
| 717 |
# test %N |
| 718 |
check_format('type: str', |
| 719 |
b'type: %N', py_object(str)) |
| 720 |
check_format(f'type: st', |
| 721 |
b'type: %.2N', py_object(str)) |
| 722 |
check_format(f'type: str', |
| 723 |
b'type: %10N', py_object(str)) |
| 724 |
|
| 725 |
check_format(f'type: {fullname}', |
| 726 |
b'type: %N', py_object(type(obj))) |
| 727 |
check_format(f'type: {fullname_alt}', |
| 728 |
b'type: %#N', py_object(type(obj))) |
| 729 |
with self.assertRaisesRegex(TypeError, "%N argument must be a type"): |
| 730 |
check_format('type: str', |
| 731 |
b'type: %N', py_object("abc")) |
| 732 |
|
| 733 |
# test variable width and precision |
| 734 |
check_format(' abc', b'%*s', c_int(5), b'abc') |
| 735 |
check_format('ab', b'%.*s', c_int(2), b'abc') |
| 736 |
check_format(' ab', b'%*.*s', c_int(5), c_int(2), b'abc') |
| 737 |
check_format(' abc', b'%*U', c_int(5), 'abc') |
| 738 |
check_format('ab', b'%.*U', c_int(2), 'abc') |
| 739 |
check_format(' ab', b'%*.*U', c_int(5), c_int(2), 'abc') |
| 740 |
check_format(' ab', b'%*.*V', c_int(5), c_int(2), None, b'abc') |
| 741 |
check_format(' ab', b'%*.*lV', c_int(5), c_int(2), |
| 742 |
None, c_wchar_p('abc')) |
| 743 |
check_format(' 123', b'%*i', c_int(8), c_int(123)) |
| 744 |
check_format('00123', b'%.*i', c_int(5), c_int(123)) |
| 745 |
check_format(' 00123', b'%*.*i', c_int(8), c_int(5), c_int(123)) |
| 746 |
|
| 747 |
# test %p |
| 748 |
# We cannot test the exact result, |
| 749 |
# because it returns a hex representation of a C pointer, |
| 750 |
# which is going to be different each time. But, we can test the format. |
| 751 |
p_format_regex = r'^0x[a-zA-Z0-9]{3,}$' |
| 752 |
p_format1 = PyUnicode_FromFormat(b'%p', 'abc') |
| 753 |
self.assertIsInstance(p_format1, str) |
| 754 |
self.assertRegex(p_format1, p_format_regex) |
| 755 |
|
| 756 |
p_format2 = PyUnicode_FromFormat(b'%p %p', '123456', b'xyz') |
| 757 |
self.assertIsInstance(p_format2, str) |
| 758 |
self.assertRegex(p_format2, |
| 759 |
r'0x[a-zA-Z0-9]{3,} 0x[a-zA-Z0-9]{3,}') |
| 760 |
|
| 761 |
# Extra args are ignored: |
| 762 |
p_format3 = PyUnicode_FromFormat(b'%p', '123456', None, b'xyz') |
| 763 |
self.assertIsInstance(p_format3, str) |
| 764 |
self.assertRegex(p_format3, p_format_regex) |
| 765 |
|
| 766 |
# Test string decode from parameter of %s using utf-8. |
| 767 |
# b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of |
| 768 |
# '\u4eba\u6c11' |
| 769 |
check_format('repr=\u4eba\u6c11', |
| 770 |
b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91') |
| 771 |
|
| 772 |
#Test replace error handler. |
| 773 |
check_format('repr=abc\ufffd', |
| 774 |
b'repr=%V', None, b'abc\xff') |
| 775 |
|
| 776 |
# Issue #33817: empty strings |
| 777 |
check_format('', |
| 778 |
b'') |
| 779 |
check_format('', |
| 780 |
b'%s', b'') |
| 781 |
|
| 782 |
# test invalid format strings. these tests are just here |
| 783 |
# to check for crashes and should not be considered as specifications |
| 784 |
for fmt in (b'%', b'%0', b'%01', b'%.', b'%.1', |
| 785 |
b'%0%s', b'%1%s', b'%.%s', b'%.1%s', b'%1abc', |
| 786 |
b'%l', b'%ll', b'%z', b'%lls', b'%zs'): |
| 787 |
with self.subTest(fmt=fmt): |
| 788 |
self.assertRaisesRegex(SystemError, 'invalid format string', |
| 789 |
PyUnicode_FromFormat, fmt, b'abc') |
| 790 |
self.assertRaisesRegex(SystemError, 'invalid format string', |
| 791 |
PyUnicode_FromFormat, b'%+i', c_int(10)) |
| 792 |
|
| 793 |
@support.cpython_only |
| 794 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 795 |
def test_interninplace(self): |
| 796 |
"""Test PyUnicode_InternInPlace()""" |
| 797 |
from _testlimitedcapi import unicode_interninplace as interninplace |
| 798 |
|
| 799 |
s = b'abc'.decode() |
| 800 |
r = interninplace(s) |
| 801 |
self.assertEqual(r, 'abc') |
| 802 |
|
| 803 |
# CRASHES interninplace(b'abc') |
| 804 |
# CRASHES interninplace(NULL) |
| 805 |
|
| 806 |
@support.cpython_only |
| 807 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 808 |
def test_internfromstring(self): |
| 809 |
"""Test PyUnicode_InternFromString()""" |
| 810 |
from _testlimitedcapi import unicode_internfromstring as internfromstring |
| 811 |
|
| 812 |
self.assertEqual(internfromstring(b'abc'), 'abc') |
| 813 |
self.assertEqual(internfromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') |
| 814 |
self.assertRaises(UnicodeDecodeError, internfromstring, b'\xc2') |
| 815 |
self.assertRaises(UnicodeDecodeError, internfromstring, b'\xa1') |
| 816 |
self.assertEqual(internfromstring(b''), '') |
| 817 |
|
| 818 |
# CRASHES internfromstring(NULL) |
| 819 |
|
| 820 |
@support.cpython_only |
| 821 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 822 |
def test_fromwidechar(self): |
| 823 |
"""Test PyUnicode_FromWideChar()""" |
| 824 |
from _testlimitedcapi import unicode_fromwidechar as fromwidechar |
| 825 |
from _testcapi import SIZEOF_WCHAR_T |
| 826 |
|
| 827 |
if SIZEOF_WCHAR_T == 2: |
| 828 |
encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' |
| 829 |
elif SIZEOF_WCHAR_T == 4: |
| 830 |
encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' |
| 831 |
|
| 832 |
for s in '', 'abc', '\xa1\xa2', '\u4f60', '\U0001f600': |
| 833 |
b = s.encode(encoding) |
| 834 |
self.assertEqual(fromwidechar(b), s) |
| 835 |
self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s) |
| 836 |
for s in '\ud83d', '\ude00': |
| 837 |
b = s.encode(encoding, 'surrogatepass') |
| 838 |
self.assertEqual(fromwidechar(b), s) |
| 839 |
self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s) |
| 840 |
|
| 841 |
self.assertEqual(fromwidechar('abc'.encode(encoding), 2), 'ab') |
| 842 |
if SIZEOF_WCHAR_T == 2: |
| 843 |
self.assertEqual(fromwidechar('a\U0001f600'.encode(encoding), 2), 'a\ud83d') |
| 844 |
|
| 845 |
self.assertRaises(MemoryError, fromwidechar, b'', PY_SSIZE_T_MAX) |
| 846 |
self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, -2) |
| 847 |
self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, PY_SSIZE_T_MIN) |
| 848 |
self.assertEqual(fromwidechar(NULL, 0), '') |
| 849 |
self.assertRaises(SystemError, fromwidechar, NULL, 1) |
| 850 |
self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MAX) |
| 851 |
self.assertRaises(SystemError, fromwidechar, NULL, -1) |
| 852 |
self.assertRaises(SystemError, fromwidechar, NULL, -2) |
| 853 |
self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MIN) |
| 854 |
|
| 855 |
@support.cpython_only |
| 856 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 857 |
def test_aswidechar(self): |
| 858 |
"""Test PyUnicode_AsWideChar()""" |
| 859 |
from _testlimitedcapi import unicode_aswidechar |
| 860 |
from _testlimitedcapi import unicode_aswidechar_null |
| 861 |
from _testcapi import SIZEOF_WCHAR_T |
| 862 |
|
| 863 |
wchar, size = unicode_aswidechar('abcdef', 2) |
| 864 |
self.assertEqual(size, 2) |
| 865 |
self.assertEqual(wchar, 'ab') |
| 866 |
|
| 867 |
wchar, size = unicode_aswidechar('abc', 3) |
| 868 |
self.assertEqual(size, 3) |
| 869 |
self.assertEqual(wchar, 'abc') |
| 870 |
self.assertEqual(unicode_aswidechar_null('abc', 10), 4) |
| 871 |
self.assertEqual(unicode_aswidechar_null('abc', 0), 4) |
| 872 |
|
| 873 |
wchar, size = unicode_aswidechar('abc', 4) |
| 874 |
self.assertEqual(size, 3) |
| 875 |
self.assertEqual(wchar, 'abc\0') |
| 876 |
|
| 877 |
wchar, size = unicode_aswidechar('abc', 10) |
| 878 |
self.assertEqual(size, 3) |
| 879 |
self.assertEqual(wchar, 'abc\0') |
| 880 |
|
| 881 |
wchar, size = unicode_aswidechar('abc\0def', 20) |
| 882 |
self.assertEqual(size, 7) |
| 883 |
self.assertEqual(wchar, 'abc\0def\0') |
| 884 |
self.assertEqual(unicode_aswidechar_null('abc\0def', 20), 8) |
| 885 |
|
| 886 |
nonbmp = chr(0x10ffff) |
| 887 |
if SIZEOF_WCHAR_T == 2: |
| 888 |
nchar = 2 |
| 889 |
else: # SIZEOF_WCHAR_T == 4 |
| 890 |
nchar = 1 |
| 891 |
wchar, size = unicode_aswidechar(nonbmp, 10) |
| 892 |
self.assertEqual(size, nchar) |
| 893 |
self.assertEqual(wchar, nonbmp + '\0') |
| 894 |
self.assertEqual(unicode_aswidechar_null(nonbmp, 10), nchar + 1) |
| 895 |
|
| 896 |
self.assertRaises(TypeError, unicode_aswidechar, b'abc', 10) |
| 897 |
self.assertRaises(TypeError, unicode_aswidechar, [], 10) |
| 898 |
self.assertRaises(SystemError, unicode_aswidechar, NULL, 10) |
| 899 |
self.assertRaises(TypeError, unicode_aswidechar_null, b'abc', 10) |
| 900 |
self.assertRaises(TypeError, unicode_aswidechar_null, [], 10) |
| 901 |
self.assertRaises(SystemError, unicode_aswidechar_null, NULL, 10) |
| 902 |
|
| 903 |
@support.cpython_only |
| 904 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 905 |
def test_aswidecharstring(self): |
| 906 |
"""Test PyUnicode_AsWideCharString()""" |
| 907 |
from _testlimitedcapi import unicode_aswidecharstring |
| 908 |
from _testlimitedcapi import unicode_aswidecharstring_null |
| 909 |
from _testcapi import SIZEOF_WCHAR_T |
| 910 |
|
| 911 |
wchar, size = unicode_aswidecharstring('abc') |
| 912 |
self.assertEqual(size, 3) |
| 913 |
self.assertEqual(wchar, 'abc\0') |
| 914 |
self.assertEqual(unicode_aswidecharstring_null('abc'), 'abc') |
| 915 |
|
| 916 |
wchar, size = unicode_aswidecharstring('abc\0def') |
| 917 |
self.assertEqual(size, 7) |
| 918 |
self.assertEqual(wchar, 'abc\0def\0') |
| 919 |
self.assertRaises(ValueError, unicode_aswidecharstring_null, 'abc\0def') |
| 920 |
|
| 921 |
nonbmp = chr(0x10ffff) |
| 922 |
if SIZEOF_WCHAR_T == 2: |
| 923 |
nchar = 2 |
| 924 |
else: # SIZEOF_WCHAR_T == 4 |
| 925 |
nchar = 1 |
| 926 |
wchar, size = unicode_aswidecharstring(nonbmp) |
| 927 |
self.assertEqual(size, nchar) |
| 928 |
self.assertEqual(wchar, nonbmp + '\0') |
| 929 |
self.assertEqual(unicode_aswidecharstring_null(nonbmp), nonbmp) |
| 930 |
|
| 931 |
self.assertRaises(TypeError, unicode_aswidecharstring, b'abc') |
| 932 |
self.assertRaises(TypeError, unicode_aswidecharstring, []) |
| 933 |
self.assertRaises(SystemError, unicode_aswidecharstring, NULL) |
| 934 |
self.assertRaises(TypeError, unicode_aswidecharstring_null, b'abc') |
| 935 |
self.assertRaises(TypeError, unicode_aswidecharstring_null, []) |
| 936 |
self.assertRaises(SystemError, unicode_aswidecharstring_null, NULL) |
| 937 |
|
| 938 |
@support.cpython_only |
| 939 |
@unittest.skipIf(_testcapi is None, 'need _testcapi module') |
| 940 |
def test_asucs4(self): |
| 941 |
"""Test PyUnicode_AsUCS4()""" |
| 942 |
from _testcapi import unicode_asucs4 |
| 943 |
|
| 944 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', |
| 945 |
'a\ud800b\udfffc', '\ud834\udd1e']: |
| 946 |
l = len(s) |
| 947 |
self.assertEqual(unicode_asucs4(s, l, 1), s+'\0') |
| 948 |
self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff') |
| 949 |
self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff') |
| 950 |
self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff') |
| 951 |
self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1) |
| 952 |
self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0) |
| 953 |
s = '\0'.join([s, s]) |
| 954 |
self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0') |
| 955 |
self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff') |
| 956 |
|
| 957 |
# CRASHES unicode_asucs4(b'abc', 1, 0) |
| 958 |
# CRASHES unicode_asucs4(b'abc', 1, 1) |
| 959 |
# CRASHES unicode_asucs4([], 1, 1) |
| 960 |
# CRASHES unicode_asucs4(NULL, 1, 0) |
| 961 |
# CRASHES unicode_asucs4(NULL, 1, 1) |
| 962 |
|
| 963 |
@support.cpython_only |
| 964 |
@unittest.skipIf(_testcapi is None, 'need _testcapi module') |
| 965 |
def test_asucs4copy(self): |
| 966 |
"""Test PyUnicode_AsUCS4Copy()""" |
| 967 |
from _testcapi import unicode_asucs4copy as asucs4copy |
| 968 |
|
| 969 |
for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', |
| 970 |
'a\ud800b\udfffc', '\ud834\udd1e']: |
| 971 |
self.assertEqual(asucs4copy(s), s+'\0') |
| 972 |
s = '\0'.join([s, s]) |
| 973 |
self.assertEqual(asucs4copy(s), s+'\0') |
| 974 |
|
| 975 |
# CRASHES asucs4copy(b'abc') |
| 976 |
# CRASHES asucs4copy([]) |
| 977 |
# CRASHES asucs4copy(NULL) |
| 978 |
|
| 979 |
@support.cpython_only |
| 980 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 981 |
def test_fromordinal(self): |
| 982 |
"""Test PyUnicode_FromOrdinal()""" |
| 983 |
from _testlimitedcapi import unicode_fromordinal as fromordinal |
| 984 |
|
| 985 |
self.assertEqual(fromordinal(0x61), 'a') |
| 986 |
self.assertEqual(fromordinal(0x20ac), '\u20ac') |
| 987 |
self.assertEqual(fromordinal(0x1f600), '\U0001f600') |
| 988 |
|
| 989 |
self.assertRaises(ValueError, fromordinal, 0x110000) |
| 990 |
self.assertRaises(ValueError, fromordinal, -1) |
| 991 |
|
| 992 |
@support.cpython_only |
| 993 |
@unittest.skipIf(_testcapi is None, 'need _testcapi module') |
| 994 |
def test_asutf8(self): |
| 995 |
"""Test PyUnicode_AsUTF8()""" |
| 996 |
from _testcapi import unicode_asutf8 |
| 997 |
|
| 998 |
self.assertEqual(unicode_asutf8('abc', 4), b'abc\0') |
| 999 |
self.assertEqual(unicode_asutf8('абв', 7), b'\xd0\xb0\xd0\xb1\xd0\xb2\0') |
| 1000 |
self.assertEqual(unicode_asutf8('\U0001f600', 5), b'\xf0\x9f\x98\x80\0') |
| 1001 |
self.assertEqual(unicode_asutf8('abc\0def', 8), b'abc\0def\0') |
| 1002 |
|
| 1003 |
self.assertRaises(UnicodeEncodeError, unicode_asutf8, '\ud8ff', 0) |
| 1004 |
self.assertRaises(TypeError, unicode_asutf8, b'abc', 0) |
| 1005 |
self.assertRaises(TypeError, unicode_asutf8, [], 0) |
| 1006 |
# CRASHES unicode_asutf8(NULL, 0) |
| 1007 |
|
| 1008 |
@unittest.skipIf(_testcapi is None, 'need _testcapi module') |
| 1009 |
@threading_helper.requires_working_threading() |
| 1010 |
def test_asutf8_race(self): |
| 1011 |
"""Test that there's no race condition in PyUnicode_AsUTF8()""" |
| 1012 |
unicode_asutf8 = _testcapi.unicode_asutf8 |
| 1013 |
from threading import Thread |
| 1014 |
|
| 1015 |
data = "😊" |
| 1016 |
|
| 1017 |
def worker(): |
| 1018 |
for _ in range(1000): |
| 1019 |
self.assertEqual(unicode_asutf8(data, 5), b'\xf0\x9f\x98\x8a\0') |
| 1020 |
|
| 1021 |
threads = [Thread(target=worker) for _ in range(10)] |
| 1022 |
with threading_helper.start_threads(threads): |
| 1023 |
pass |
| 1024 |
|
| 1025 |
|
| 1026 |
@support.cpython_only |
| 1027 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1028 |
def test_asutf8andsize(self): |
| 1029 |
"""Test PyUnicode_AsUTF8AndSize()""" |
| 1030 |
from _testlimitedcapi import unicode_asutf8andsize |
| 1031 |
from _testlimitedcapi import unicode_asutf8andsize_null |
| 1032 |
|
| 1033 |
self.assertEqual(unicode_asutf8andsize('abc', 4), (b'abc\0', 3)) |
| 1034 |
self.assertEqual(unicode_asutf8andsize('абв', 7), (b'\xd0\xb0\xd0\xb1\xd0\xb2\0', 6)) |
| 1035 |
self.assertEqual(unicode_asutf8andsize('\U0001f600', 5), (b'\xf0\x9f\x98\x80\0', 4)) |
| 1036 |
self.assertEqual(unicode_asutf8andsize('abc\0def', 8), (b'abc\0def\0', 7)) |
| 1037 |
self.assertEqual(unicode_asutf8andsize_null('abc', 4), b'abc\0') |
| 1038 |
self.assertEqual(unicode_asutf8andsize_null('abc\0def', 8), b'abc\0def\0') |
| 1039 |
|
| 1040 |
self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, '\ud8ff', 0) |
| 1041 |
self.assertRaises(TypeError, unicode_asutf8andsize, b'abc', 0) |
| 1042 |
self.assertRaises(TypeError, unicode_asutf8andsize, [], 0) |
| 1043 |
self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize_null, '\ud8ff', 0) |
| 1044 |
self.assertRaises(TypeError, unicode_asutf8andsize_null, b'abc', 0) |
| 1045 |
self.assertRaises(TypeError, unicode_asutf8andsize_null, [], 0) |
| 1046 |
# CRASHES unicode_asutf8andsize(NULL, 0) |
| 1047 |
# CRASHES unicode_asutf8andsize_null(NULL, 0) |
| 1048 |
|
| 1049 |
@support.cpython_only |
| 1050 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1051 |
def test_getdefaultencoding(self): |
| 1052 |
"""Test PyUnicode_GetDefaultEncoding()""" |
| 1053 |
from _testlimitedcapi import unicode_getdefaultencoding as getdefaultencoding |
| 1054 |
|
| 1055 |
self.assertEqual(getdefaultencoding(), b'utf-8') |
| 1056 |
|
| 1057 |
@support.cpython_only |
| 1058 |
@unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module') |
| 1059 |
def test_transform_decimal_and_space(self): |
| 1060 |
"""Test _PyUnicode_TransformDecimalAndSpaceToASCII()""" |
| 1061 |
from _testinternalcapi import _PyUnicode_TransformDecimalAndSpaceToASCII as transform_decimal |
| 1062 |
|
| 1063 |
self.assertEqual(transform_decimal('123'), |
| 1064 |
'123') |
| 1065 |
self.assertEqual(transform_decimal('\u0663.\u0661\u0664'), |
| 1066 |
'3.14') |
| 1067 |
self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"), |
| 1068 |
" 3.14 ") |
| 1069 |
self.assertEqual(transform_decimal('12\u20ac3'), |
| 1070 |
'12?') |
| 1071 |
self.assertEqual(transform_decimal(''), '') |
| 1072 |
|
| 1073 |
self.assertRaises(SystemError, transform_decimal, b'123') |
| 1074 |
self.assertRaises(SystemError, transform_decimal, []) |
| 1075 |
# CRASHES transform_decimal(NULL) |
| 1076 |
|
| 1077 |
@support.cpython_only |
| 1078 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1079 |
def test_concat(self): |
| 1080 |
"""Test PyUnicode_Concat()""" |
| 1081 |
from _testlimitedcapi import unicode_concat as concat |
| 1082 |
|
| 1083 |
self.assertEqual(concat('abc', 'def'), 'abcdef') |
| 1084 |
self.assertEqual(concat('abc', 'где'), 'abcгде') |
| 1085 |
self.assertEqual(concat('абв', 'def'), 'абвdef') |
| 1086 |
self.assertEqual(concat('абв', 'где'), 'абвгде') |
| 1087 |
self.assertEqual(concat('a\0b', 'c\0d'), 'a\0bc\0d') |
| 1088 |
|
| 1089 |
self.assertRaises(TypeError, concat, b'abc', 'def') |
| 1090 |
self.assertRaises(TypeError, concat, 'abc', b'def') |
| 1091 |
self.assertRaises(TypeError, concat, b'abc', b'def') |
| 1092 |
self.assertRaises(TypeError, concat, [], 'def') |
| 1093 |
self.assertRaises(TypeError, concat, 'abc', []) |
| 1094 |
self.assertRaises(TypeError, concat, [], []) |
| 1095 |
# CRASHES concat(NULL, 'def') |
| 1096 |
# CRASHES concat('abc', NULL) |
| 1097 |
|
| 1098 |
@support.cpython_only |
| 1099 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1100 |
def test_split(self): |
| 1101 |
"""Test PyUnicode_Split()""" |
| 1102 |
from _testlimitedcapi import unicode_split as split |
| 1103 |
|
| 1104 |
self.assertEqual(split('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) |
| 1105 |
self.assertEqual(split('a|b|c|d', '|', 2), ['a', 'b', 'c|d']) |
| 1106 |
self.assertEqual(split('a|b|c|d', '|', PY_SSIZE_T_MAX), |
| 1107 |
['a', 'b', 'c', 'd']) |
| 1108 |
self.assertEqual(split('a|b|c|d', '|', -1), ['a', 'b', 'c', 'd']) |
| 1109 |
self.assertEqual(split('a|b|c|d', '|', PY_SSIZE_T_MIN), |
| 1110 |
['a', 'b', 'c', 'd']) |
| 1111 |
self.assertEqual(split('a|b|c|d', '\u20ac'), ['a|b|c|d']) |
| 1112 |
self.assertEqual(split('a||b|c||d', '||'), ['a', 'b|c', 'd']) |
| 1113 |
self.assertEqual(split('а|б|в|г', '|'), ['а', 'б', 'в', 'г']) |
| 1114 |
self.assertEqual(split('абабагаламага', 'а'), |
| 1115 |
['', 'б', 'б', 'г', 'л', 'м', 'г', '']) |
| 1116 |
self.assertEqual(split(' a\tb\nc\rd\ve\f', NULL), |
| 1117 |
['a', 'b', 'c', 'd', 'e']) |
| 1118 |
self.assertEqual(split('a\x85b\xa0c\u1680d\u2000e', NULL), |
| 1119 |
['a', 'b', 'c', 'd', 'e']) |
| 1120 |
|
| 1121 |
self.assertRaises(ValueError, split, 'a|b|c|d', '') |
| 1122 |
self.assertRaises(TypeError, split, 'a|b|c|d', ord('|')) |
| 1123 |
self.assertRaises(TypeError, split, [], '|') |
| 1124 |
# CRASHES split(NULL, '|') |
| 1125 |
|
| 1126 |
@support.cpython_only |
| 1127 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1128 |
def test_rsplit(self): |
| 1129 |
"""Test PyUnicode_RSplit()""" |
| 1130 |
from _testlimitedcapi import unicode_rsplit as rsplit |
| 1131 |
|
| 1132 |
self.assertEqual(rsplit('a|b|c|d', '|'), ['a', 'b', 'c', 'd']) |
| 1133 |
self.assertEqual(rsplit('a|b|c|d', '|', 2), ['a|b', 'c', 'd']) |
| 1134 |
self.assertEqual(rsplit('a|b|c|d', '|', PY_SSIZE_T_MAX), |
| 1135 |
['a', 'b', 'c', 'd']) |
| 1136 |
self.assertEqual(rsplit('a|b|c|d', '|', -1), ['a', 'b', 'c', 'd']) |
| 1137 |
self.assertEqual(rsplit('a|b|c|d', '|', PY_SSIZE_T_MIN), |
| 1138 |
['a', 'b', 'c', 'd']) |
| 1139 |
self.assertEqual(rsplit('a|b|c|d', '\u20ac'), ['a|b|c|d']) |
| 1140 |
self.assertEqual(rsplit('a||b|c||d', '||'), ['a', 'b|c', 'd']) |
| 1141 |
self.assertEqual(rsplit('а|б|в|г', '|'), ['а', 'б', 'в', 'г']) |
| 1142 |
self.assertEqual(rsplit('абабагаламага', 'а'), |
| 1143 |
['', 'б', 'б', 'г', 'л', 'м', 'г', '']) |
| 1144 |
self.assertEqual(rsplit('aжbжcжd', 'ж'), ['a', 'b', 'c', 'd']) |
| 1145 |
self.assertEqual(rsplit(' a\tb\nc\rd\ve\f', NULL), |
| 1146 |
['a', 'b', 'c', 'd', 'e']) |
| 1147 |
self.assertEqual(rsplit('a\x85b\xa0c\u1680d\u2000e', NULL), |
| 1148 |
['a', 'b', 'c', 'd', 'e']) |
| 1149 |
|
| 1150 |
self.assertRaises(ValueError, rsplit, 'a|b|c|d', '') |
| 1151 |
self.assertRaises(TypeError, rsplit, 'a|b|c|d', ord('|')) |
| 1152 |
self.assertRaises(TypeError, rsplit, [], '|') |
| 1153 |
# CRASHES rsplit(NULL, '|') |
| 1154 |
|
| 1155 |
@support.cpython_only |
| 1156 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1157 |
def test_partition(self): |
| 1158 |
"""Test PyUnicode_Partition()""" |
| 1159 |
from _testlimitedcapi import unicode_partition as partition |
| 1160 |
|
| 1161 |
self.assertEqual(partition('a|b|c', '|'), ('a', '|', 'b|c')) |
| 1162 |
self.assertEqual(partition('a||b||c', '||'), ('a', '||', 'b||c')) |
| 1163 |
self.assertEqual(partition('а|б|в', '|'), ('а', '|', 'б|в')) |
| 1164 |
self.assertEqual(partition('кабан', 'а'), ('к', 'а', 'бан')) |
| 1165 |
self.assertEqual(partition('aжbжc', 'ж'), ('a', 'ж', 'bжc')) |
| 1166 |
|
| 1167 |
self.assertRaises(ValueError, partition, 'a|b|c', '') |
| 1168 |
self.assertRaises(TypeError, partition, b'a|b|c', '|') |
| 1169 |
self.assertRaises(TypeError, partition, 'a|b|c', b'|') |
| 1170 |
self.assertRaises(TypeError, partition, 'a|b|c', ord('|')) |
| 1171 |
self.assertRaises(TypeError, partition, [], '|') |
| 1172 |
# CRASHES partition(NULL, '|') |
| 1173 |
# CRASHES partition('a|b|c', NULL) |
| 1174 |
|
| 1175 |
@support.cpython_only |
| 1176 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1177 |
def test_rpartition(self): |
| 1178 |
"""Test PyUnicode_RPartition()""" |
| 1179 |
from _testlimitedcapi import unicode_rpartition as rpartition |
| 1180 |
|
| 1181 |
self.assertEqual(rpartition('a|b|c', '|'), ('a|b', '|', 'c')) |
| 1182 |
self.assertEqual(rpartition('a||b||c', '||'), ('a||b', '||', 'c')) |
| 1183 |
self.assertEqual(rpartition('а|б|в', '|'), ('а|б', '|', 'в')) |
| 1184 |
self.assertEqual(rpartition('кабан', 'а'), ('каб', 'а', 'н')) |
| 1185 |
self.assertEqual(rpartition('aжbжc', 'ж'), ('aжb', 'ж', 'c')) |
| 1186 |
|
| 1187 |
self.assertRaises(ValueError, rpartition, 'a|b|c', '') |
| 1188 |
self.assertRaises(TypeError, rpartition, b'a|b|c', '|') |
| 1189 |
self.assertRaises(TypeError, rpartition, 'a|b|c', b'|') |
| 1190 |
self.assertRaises(TypeError, rpartition, 'a|b|c', ord('|')) |
| 1191 |
self.assertRaises(TypeError, rpartition, [], '|') |
| 1192 |
# CRASHES rpartition(NULL, '|') |
| 1193 |
# CRASHES rpartition('a|b|c', NULL) |
| 1194 |
|
| 1195 |
@support.cpython_only |
| 1196 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1197 |
def test_splitlines(self): |
| 1198 |
"""Test PyUnicode_SplitLines()""" |
| 1199 |
from _testlimitedcapi import unicode_splitlines as splitlines |
| 1200 |
|
| 1201 |
self.assertEqual(splitlines('a\nb\rc\r\nd'), ['a', 'b', 'c', 'd']) |
| 1202 |
self.assertEqual(splitlines('a\nb\rc\r\nd', True), |
| 1203 |
['a\n', 'b\r', 'c\r\n', 'd']) |
| 1204 |
self.assertEqual(splitlines('a\x85b\u2028c\u2029d'), |
| 1205 |
['a', 'b', 'c', 'd']) |
| 1206 |
self.assertEqual(splitlines('a\x85b\u2028c\u2029d', True), |
| 1207 |
['a\x85', 'b\u2028', 'c\u2029', 'd']) |
| 1208 |
self.assertEqual(splitlines('а\nб\rв\r\nг'), ['а', 'б', 'в', 'г']) |
| 1209 |
|
| 1210 |
self.assertRaises(TypeError, splitlines, b'a\nb\rc\r\nd') |
| 1211 |
# CRASHES splitlines(NULL) |
| 1212 |
|
| 1213 |
@support.cpython_only |
| 1214 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1215 |
def test_translate(self): |
| 1216 |
"""Test PyUnicode_Translate()""" |
| 1217 |
from _testlimitedcapi import unicode_translate as translate |
| 1218 |
|
| 1219 |
self.assertEqual(translate('abcd', {ord('a'): 'A', ord('b'): ord('B'), ord('c'): '<>'}), 'AB<>d') |
| 1220 |
self.assertEqual(translate('абвг', {ord('а'): 'А', ord('б'): ord('Б'), ord('в'): '<>'}), 'АБ<>г') |
| 1221 |
self.assertEqual(translate('abc', {}), 'abc') |
| 1222 |
self.assertEqual(translate('abc', []), 'abc') |
| 1223 |
self.assertRaises(UnicodeTranslateError, translate, 'abc', {ord('b'): None}) |
| 1224 |
self.assertRaises(UnicodeTranslateError, translate, 'abc', {ord('b'): None}, 'strict') |
| 1225 |
self.assertRaises(LookupError, translate, 'abc', {ord('b'): None}, 'foo') |
| 1226 |
self.assertEqual(translate('abc', {ord('b'): None}, 'ignore'), 'ac') |
| 1227 |
self.assertEqual(translate('abc', {ord('b'): None}, 'replace'), 'a\ufffdc') |
| 1228 |
self.assertEqual(translate('abc', {ord('b'): None}, 'backslashreplace'), r'a\x62c') |
| 1229 |
# XXX Other error handlers do not support UnicodeTranslateError |
| 1230 |
self.assertRaises(TypeError, translate, b'abc', []) |
| 1231 |
self.assertRaises(TypeError, translate, 123, []) |
| 1232 |
self.assertRaises(TypeError, translate, 'abc', {ord('a'): b'A'}) |
| 1233 |
self.assertRaises(TypeError, translate, 'abc', 123) |
| 1234 |
self.assertRaises(TypeError, translate, 'abc', NULL) |
| 1235 |
self.assertRaises(LookupError, translate, 'abc', {ord('b'): None}, 'foo') |
| 1236 |
# CRASHES translate(NULL, []) |
| 1237 |
|
| 1238 |
@support.cpython_only |
| 1239 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1240 |
def test_join(self): |
| 1241 |
"""Test PyUnicode_Join()""" |
| 1242 |
from _testlimitedcapi import unicode_join as join |
| 1243 |
self.assertEqual(join('|', ['a', 'b', 'c']), 'a|b|c') |
| 1244 |
self.assertEqual(join('|', ['a', '', 'c']), 'a||c') |
| 1245 |
self.assertEqual(join('', ['a', 'b', 'c']), 'abc') |
| 1246 |
self.assertEqual(join(NULL, ['a', 'b', 'c']), 'a b c') |
| 1247 |
self.assertEqual(join('|', ['а', 'б', 'в']), 'а|б|в') |
| 1248 |
self.assertEqual(join('ж', ['а', 'б', 'в']), 'ажбжв') |
| 1249 |
self.assertRaises(TypeError, join, b'|', ['a', 'b', 'c']) |
| 1250 |
self.assertRaises(TypeError, join, '|', [b'a', b'b', b'c']) |
| 1251 |
self.assertRaises(TypeError, join, NULL, [b'a', b'b', b'c']) |
| 1252 |
self.assertRaises(TypeError, join, '|', b'123') |
| 1253 |
self.assertRaises(TypeError, join, '|', 123) |
| 1254 |
self.assertRaises(SystemError, join, '|', NULL) |
| 1255 |
|
| 1256 |
@support.cpython_only |
| 1257 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1258 |
def test_count(self): |
| 1259 |
"""Test PyUnicode_Count()""" |
| 1260 |
from _testlimitedcapi import unicode_count |
| 1261 |
|
| 1262 |
for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": |
| 1263 |
for i, ch in enumerate(str): |
| 1264 |
self.assertEqual(unicode_count(str, ch, 0, len(str)), 1) |
| 1265 |
|
| 1266 |
str = "!>_<!" |
| 1267 |
self.assertEqual(unicode_count(str, 'z', 0, len(str)), 0) |
| 1268 |
self.assertEqual(unicode_count(str, '', 0, len(str)), len(str)+1) |
| 1269 |
# start < end |
| 1270 |
self.assertEqual(unicode_count(str, '!', 1, len(str)+1), 1) |
| 1271 |
self.assertEqual(unicode_count(str, '!', 1, PY_SSIZE_T_MAX), 1) |
| 1272 |
# start >= end |
| 1273 |
self.assertEqual(unicode_count(str, '!', 0, 0), 0) |
| 1274 |
self.assertEqual(unicode_count(str, '!', len(str), 0), 0) |
| 1275 |
# negative |
| 1276 |
self.assertEqual(unicode_count(str, '!', -len(str), -1), 1) |
| 1277 |
self.assertEqual(unicode_count(str, '!', -len(str)-1, -1), 1) |
| 1278 |
self.assertEqual(unicode_count(str, '!', PY_SSIZE_T_MIN, -1), 1) |
| 1279 |
# bad arguments |
| 1280 |
self.assertRaises(TypeError, unicode_count, str, b'!', 0, len(str)) |
| 1281 |
self.assertRaises(TypeError, unicode_count, b"!>_<!", '!', 0, len(str)) |
| 1282 |
self.assertRaises(TypeError, unicode_count, str, ord('!'), 0, len(str)) |
| 1283 |
self.assertRaises(TypeError, unicode_count, [], '!', 0, len(str), 1) |
| 1284 |
# CRASHES unicode_count(NULL, '!', 0, len(str)) |
| 1285 |
# CRASHES unicode_count(str, NULL, 0, len(str)) |
| 1286 |
|
| 1287 |
@support.cpython_only |
| 1288 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1289 |
def test_tailmatch(self): |
| 1290 |
"""Test PyUnicode_Tailmatch()""" |
| 1291 |
from _testlimitedcapi import unicode_tailmatch as tailmatch |
| 1292 |
|
| 1293 |
str = 'ababahalamaha' |
| 1294 |
self.assertEqual(tailmatch(str, 'aba', 0, len(str), -1), 1) |
| 1295 |
self.assertEqual(tailmatch(str, 'aha', 0, len(str), 1), 1) |
| 1296 |
|
| 1297 |
self.assertEqual(tailmatch(str, 'aba', 0, PY_SSIZE_T_MAX, -1), 1) |
| 1298 |
self.assertEqual(tailmatch(str, 'aba', -len(str), PY_SSIZE_T_MAX, -1), 1) |
| 1299 |
self.assertEqual(tailmatch(str, 'aba', PY_SSIZE_T_MIN, len(str), -1), 1) |
| 1300 |
self.assertEqual(tailmatch(str, 'aha', 0, PY_SSIZE_T_MAX, 1), 1) |
| 1301 |
self.assertEqual(tailmatch(str, 'aha', PY_SSIZE_T_MIN, len(str), 1), 1) |
| 1302 |
|
| 1303 |
self.assertEqual(tailmatch(str, 'z', 0, len(str), 1), 0) |
| 1304 |
self.assertEqual(tailmatch(str, 'z', 0, len(str), -1), 0) |
| 1305 |
self.assertEqual(tailmatch(str, '', 0, len(str), 1), 1) |
| 1306 |
self.assertEqual(tailmatch(str, '', 0, len(str), -1), 1) |
| 1307 |
|
| 1308 |
self.assertEqual(tailmatch(str, 'ba', 0, len(str)-1, -1), 0) |
| 1309 |
self.assertEqual(tailmatch(str, 'ba', 1, len(str)-1, -1), 1) |
| 1310 |
self.assertEqual(tailmatch(str, 'aba', 1, len(str)-1, -1), 0) |
| 1311 |
self.assertEqual(tailmatch(str, 'ba', -len(str)+1, -1, -1), 1) |
| 1312 |
self.assertEqual(tailmatch(str, 'ah', 0, len(str), 1), 0) |
| 1313 |
self.assertEqual(tailmatch(str, 'ah', 0, len(str)-1, 1), 1) |
| 1314 |
self.assertEqual(tailmatch(str, 'ah', -len(str), -1, 1), 1) |
| 1315 |
|
| 1316 |
# bad arguments |
| 1317 |
self.assertRaises(TypeError, tailmatch, str, ('aba', 'aha'), 0, len(str), -1) |
| 1318 |
self.assertRaises(TypeError, tailmatch, str, ('aba', 'aha'), 0, len(str), 1) |
| 1319 |
# CRASHES tailmatch(NULL, 'aba', 0, len(str), -1) |
| 1320 |
# CRASHES tailmatch(str, NULL, 0, len(str), -1) |
| 1321 |
|
| 1322 |
@support.cpython_only |
| 1323 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1324 |
def test_find(self): |
| 1325 |
"""Test PyUnicode_Find()""" |
| 1326 |
from _testlimitedcapi import unicode_find as find |
| 1327 |
|
| 1328 |
for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": |
| 1329 |
for i, ch in enumerate(str): |
| 1330 |
self.assertEqual(find(str, ch, 0, len(str), 1), i) |
| 1331 |
self.assertEqual(find(str, ch, 0, len(str), -1), i) |
| 1332 |
|
| 1333 |
str = "!>_<!" |
| 1334 |
self.assertEqual(find(str, 'z', 0, len(str), 1), -1) |
| 1335 |
self.assertEqual(find(str, 'z', 0, len(str), -1), -1) |
| 1336 |
self.assertEqual(find(str, '', 0, len(str), 1), 0) |
| 1337 |
self.assertEqual(find(str, '', 0, len(str), -1), len(str)) |
| 1338 |
# start < end |
| 1339 |
self.assertEqual(find(str, '!', 1, len(str)+1, 1), 4) |
| 1340 |
self.assertEqual(find(str, '!', 1, PY_SSIZE_T_MAX, 1), 4) |
| 1341 |
self.assertEqual(find(str, '!', 0, len(str)+1, -1), 4) |
| 1342 |
self.assertEqual(find(str, '!', 0, PY_SSIZE_T_MAX, -1), 4) |
| 1343 |
# start >= end |
| 1344 |
self.assertEqual(find(str, '!', 0, 0, 1), -1) |
| 1345 |
self.assertEqual(find(str, '!', 0, 0, -1), -1) |
| 1346 |
self.assertEqual(find(str, '!', len(str), 0, 1), -1) |
| 1347 |
self.assertEqual(find(str, '!', len(str), 0, -1), -1) |
| 1348 |
# negative |
| 1349 |
self.assertEqual(find(str, '!', -len(str), -1, 1), 0) |
| 1350 |
self.assertEqual(find(str, '!', -len(str), -1, -1), 0) |
| 1351 |
self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, -1, 1), 0) |
| 1352 |
self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, -1, -1), 0) |
| 1353 |
self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, 1), 0) |
| 1354 |
self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1), 4) |
| 1355 |
# bad arguments |
| 1356 |
self.assertRaises(TypeError, find, str, b'!', 0, len(str), 1) |
| 1357 |
self.assertRaises(TypeError, find, b"!>_<!", '!', 0, len(str), 1) |
| 1358 |
self.assertRaises(TypeError, find, str, ord('!'), 0, len(str), 1) |
| 1359 |
self.assertRaises(TypeError, find, [], '!', 0, len(str), 1) |
| 1360 |
# CRASHES find(NULL, '!', 0, len(str), 1) |
| 1361 |
# CRASHES find(str, NULL, 0, len(str), 1) |
| 1362 |
|
| 1363 |
@support.cpython_only |
| 1364 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1365 |
def test_findchar(self): |
| 1366 |
"""Test PyUnicode_FindChar()""" |
| 1367 |
from _testlimitedcapi import unicode_findchar |
| 1368 |
|
| 1369 |
for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1": |
| 1370 |
for i, ch in enumerate(str): |
| 1371 |
self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), 1), i) |
| 1372 |
self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i) |
| 1373 |
|
| 1374 |
str = "!>_<!" |
| 1375 |
self.assertEqual(unicode_findchar(str, 0x110000, 0, len(str), 1), -1) |
| 1376 |
self.assertEqual(unicode_findchar(str, 0x110000, 0, len(str), -1), -1) |
| 1377 |
# start < end |
| 1378 |
self.assertEqual(unicode_findchar(str, ord('!'), 1, len(str)+1, 1), 4) |
| 1379 |
self.assertEqual(unicode_findchar(str, ord('!'), 1, PY_SSIZE_T_MAX, 1), 4) |
| 1380 |
self.assertEqual(unicode_findchar(str, ord('!'), 0, len(str)+1, -1), 4) |
| 1381 |
self.assertEqual(unicode_findchar(str, ord('!'), 0, PY_SSIZE_T_MAX, -1), 4) |
| 1382 |
# start >= end |
| 1383 |
self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1) |
| 1384 |
self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, -1), -1) |
| 1385 |
self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1) |
| 1386 |
self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, -1), -1) |
| 1387 |
# negative |
| 1388 |
self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0) |
| 1389 |
self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0) |
| 1390 |
self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, -1, 1), 0) |
| 1391 |
self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, -1, -1), 0) |
| 1392 |
self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, 1), 0) |
| 1393 |
self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1), 4) |
| 1394 |
# bad arguments |
| 1395 |
# CRASHES unicode_findchar(b"!>_<!", ord('!'), 0, len(str), 1) |
| 1396 |
# CRASHES unicode_findchar([], ord('!'), 0, len(str), 1) |
| 1397 |
# CRASHES unicode_findchar(NULL, ord('!'), 0, len(str), 1), 1) |
| 1398 |
|
| 1399 |
@support.cpython_only |
| 1400 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1401 |
def test_replace(self): |
| 1402 |
"""Test PyUnicode_Replace()""" |
| 1403 |
from _testlimitedcapi import unicode_replace as replace |
| 1404 |
|
| 1405 |
str = 'abracadabra' |
| 1406 |
self.assertEqual(replace(str, 'a', '='), '=br=c=d=br=') |
| 1407 |
self.assertEqual(replace(str, 'a', '<>'), '<>br<>c<>d<>br<>') |
| 1408 |
self.assertEqual(replace(str, 'abra', '='), '=cad=') |
| 1409 |
self.assertEqual(replace(str, 'a', '=', 2), '=br=cadabra') |
| 1410 |
self.assertEqual(replace(str, 'a', '=', 0), str) |
| 1411 |
self.assertEqual(replace(str, 'a', '=', PY_SSIZE_T_MAX), '=br=c=d=br=') |
| 1412 |
self.assertEqual(replace(str, 'a', '=', -1), '=br=c=d=br=') |
| 1413 |
self.assertEqual(replace(str, 'a', '=', PY_SSIZE_T_MIN), '=br=c=d=br=') |
| 1414 |
self.assertEqual(replace(str, 'z', '='), str) |
| 1415 |
self.assertEqual(replace(str, '', '='), '=a=b=r=a=c=a=d=a=b=r=a=') |
| 1416 |
self.assertEqual(replace(str, 'a', 'ж'), 'жbrжcжdжbrж') |
| 1417 |
self.assertEqual(replace('абабагаламага', 'а', '='), '=б=б=г=л=м=г=') |
| 1418 |
self.assertEqual(replace('Баден-Баден', 'Баден', 'Baden'), 'Baden-Baden') |
| 1419 |
# bad arguments |
| 1420 |
self.assertRaises(TypeError, replace, 'a', 'a', b'=') |
| 1421 |
self.assertRaises(TypeError, replace, 'a', b'a', '=') |
| 1422 |
self.assertRaises(TypeError, replace, b'a', 'a', '=') |
| 1423 |
self.assertRaises(TypeError, replace, 'a', 'a', ord('=')) |
| 1424 |
self.assertRaises(TypeError, replace, 'a', ord('a'), '=') |
| 1425 |
self.assertRaises(TypeError, replace, [], 'a', '=') |
| 1426 |
# CRASHES replace('a', 'a', NULL) |
| 1427 |
# CRASHES replace('a', NULL, '=') |
| 1428 |
# CRASHES replace(NULL, 'a', '=') |
| 1429 |
|
| 1430 |
@support.cpython_only |
| 1431 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1432 |
def test_compare(self): |
| 1433 |
"""Test PyUnicode_Compare()""" |
| 1434 |
from _testlimitedcapi import unicode_compare as compare |
| 1435 |
|
| 1436 |
self.assertEqual(compare('abc', 'abc'), 0) |
| 1437 |
self.assertEqual(compare('abc', 'def'), -1) |
| 1438 |
self.assertEqual(compare('def', 'abc'), 1) |
| 1439 |
self.assertEqual(compare('abc', 'abc\0def'), -1) |
| 1440 |
self.assertEqual(compare('abc\0def', 'abc\0def'), 0) |
| 1441 |
self.assertEqual(compare('абв', 'abc'), 1) |
| 1442 |
|
| 1443 |
self.assertRaises(TypeError, compare, b'abc', 'abc') |
| 1444 |
self.assertRaises(TypeError, compare, 'abc', b'abc') |
| 1445 |
self.assertRaises(TypeError, compare, b'abc', b'abc') |
| 1446 |
self.assertRaises(TypeError, compare, [], 'abc') |
| 1447 |
self.assertRaises(TypeError, compare, 'abc', []) |
| 1448 |
self.assertRaises(TypeError, compare, [], []) |
| 1449 |
# CRASHES compare(NULL, 'abc') |
| 1450 |
# CRASHES compare('abc', NULL) |
| 1451 |
|
| 1452 |
@support.cpython_only |
| 1453 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1454 |
def test_comparewithasciistring(self): |
| 1455 |
"""Test PyUnicode_CompareWithASCIIString()""" |
| 1456 |
from _testlimitedcapi import unicode_comparewithasciistring as comparewithasciistring |
| 1457 |
|
| 1458 |
self.assertEqual(comparewithasciistring('abc', b'abc'), 0) |
| 1459 |
self.assertEqual(comparewithasciistring('abc', b'def'), -1) |
| 1460 |
self.assertEqual(comparewithasciistring('def', b'abc'), 1) |
| 1461 |
self.assertEqual(comparewithasciistring('abc', b'abc\0def'), 0) |
| 1462 |
self.assertEqual(comparewithasciistring('abc\0def', b'abc\0def'), 1) |
| 1463 |
self.assertEqual(comparewithasciistring('абв', b'abc'), 1) |
| 1464 |
|
| 1465 |
# CRASHES comparewithasciistring(b'abc', b'abc') |
| 1466 |
# CRASHES comparewithasciistring([], b'abc') |
| 1467 |
# CRASHES comparewithasciistring(NULL, b'abc') |
| 1468 |
|
| 1469 |
@support.cpython_only |
| 1470 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1471 |
def test_equaltoutf8(self): |
| 1472 |
# Test PyUnicode_EqualToUTF8() |
| 1473 |
from _testlimitedcapi import unicode_equaltoutf8 as equaltoutf8 |
| 1474 |
from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize |
| 1475 |
|
| 1476 |
strings = [ |
| 1477 |
'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', |
| 1478 |
'\U0001f600\U0001f601\U0001f602', |
| 1479 |
'\U0010ffff', |
| 1480 |
] |
| 1481 |
for s in strings: |
| 1482 |
# Call PyUnicode_AsUTF8AndSize() which creates the UTF-8 |
| 1483 |
# encoded string cached in the Unicode object. |
| 1484 |
asutf8andsize(s, 0) |
| 1485 |
b = s.encode() |
| 1486 |
self.assertEqual(equaltoutf8(s, b), 1) # Use the UTF-8 cache. |
| 1487 |
s2 = b.decode() # New Unicode object without the UTF-8 cache. |
| 1488 |
self.assertEqual(equaltoutf8(s2, b), 1) |
| 1489 |
self.assertEqual(equaltoutf8(s + 'x', b + b'x'), 1) |
| 1490 |
self.assertEqual(equaltoutf8(s + 'x', b + b'y'), 0) |
| 1491 |
self.assertEqual(equaltoutf8(s, b + b'\0'), 1) |
| 1492 |
self.assertEqual(equaltoutf8(s2, b + b'\0'), 1) |
| 1493 |
self.assertEqual(equaltoutf8(s + '\0', b + b'\0'), 0) |
| 1494 |
self.assertEqual(equaltoutf8(s + '\0', b), 0) |
| 1495 |
self.assertEqual(equaltoutf8(s2, b + b'x'), 0) |
| 1496 |
self.assertEqual(equaltoutf8(s2, b[:-1]), 0) |
| 1497 |
self.assertEqual(equaltoutf8(s2, b[:-1] + b'x'), 0) |
| 1498 |
|
| 1499 |
self.assertEqual(equaltoutf8('', b''), 1) |
| 1500 |
self.assertEqual(equaltoutf8('', b'\0'), 1) |
| 1501 |
|
| 1502 |
# embedded null chars/bytes |
| 1503 |
self.assertEqual(equaltoutf8('abc', b'abc\0def\0'), 1) |
| 1504 |
self.assertEqual(equaltoutf8('a\0bc', b'abc'), 0) |
| 1505 |
self.assertEqual(equaltoutf8('abc', b'a\0bc'), 0) |
| 1506 |
|
| 1507 |
# Surrogate characters are always treated as not equal |
| 1508 |
self.assertEqual(equaltoutf8('\udcfe', |
| 1509 |
'\udcfe'.encode("utf8", "surrogateescape")), 0) |
| 1510 |
self.assertEqual(equaltoutf8('\udcfe', |
| 1511 |
'\udcfe'.encode("utf8", "surrogatepass")), 0) |
| 1512 |
self.assertEqual(equaltoutf8('\ud801', |
| 1513 |
'\ud801'.encode("utf8", "surrogatepass")), 0) |
| 1514 |
|
| 1515 |
@support.cpython_only |
| 1516 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1517 |
def test_equaltoutf8andsize(self): |
| 1518 |
# Test PyUnicode_EqualToUTF8AndSize() |
| 1519 |
from _testlimitedcapi import unicode_equaltoutf8andsize as equaltoutf8andsize |
| 1520 |
from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize |
| 1521 |
|
| 1522 |
strings = [ |
| 1523 |
'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', |
| 1524 |
'\U0001f600\U0001f601\U0001f602', |
| 1525 |
'\U0010ffff', |
| 1526 |
] |
| 1527 |
for s in strings: |
| 1528 |
# Call PyUnicode_AsUTF8AndSize() which creates the UTF-8 |
| 1529 |
# encoded string cached in the Unicode object. |
| 1530 |
asutf8andsize(s, 0) |
| 1531 |
b = s.encode() |
| 1532 |
self.assertEqual(equaltoutf8andsize(s, b), 1) # Use the UTF-8 cache. |
| 1533 |
s2 = b.decode() # New Unicode object without the UTF-8 cache. |
| 1534 |
self.assertEqual(equaltoutf8andsize(s2, b), 1) |
| 1535 |
self.assertEqual(equaltoutf8andsize(s + 'x', b + b'x'), 1) |
| 1536 |
self.assertEqual(equaltoutf8andsize(s + 'x', b + b'y'), 0) |
| 1537 |
self.assertEqual(equaltoutf8andsize(s, b + b'\0'), 0) |
| 1538 |
self.assertEqual(equaltoutf8andsize(s2, b + b'\0'), 0) |
| 1539 |
self.assertEqual(equaltoutf8andsize(s + '\0', b + b'\0'), 1) |
| 1540 |
self.assertEqual(equaltoutf8andsize(s + '\0', b), 0) |
| 1541 |
self.assertEqual(equaltoutf8andsize(s2, b + b'x'), 0) |
| 1542 |
self.assertEqual(equaltoutf8andsize(s2, b[:-1]), 0) |
| 1543 |
self.assertEqual(equaltoutf8andsize(s2, b[:-1] + b'x'), 0) |
| 1544 |
# Not null-terminated, |
| 1545 |
self.assertEqual(equaltoutf8andsize(s, b + b'x', len(b)), 1) |
| 1546 |
self.assertEqual(equaltoutf8andsize(s2, b + b'x', len(b)), 1) |
| 1547 |
self.assertEqual(equaltoutf8andsize(s + '\0', b + b'\0x', len(b) + 1), 1) |
| 1548 |
self.assertEqual(equaltoutf8andsize(s2, b, len(b) - 1), 0) |
| 1549 |
self.assertEqual(equaltoutf8andsize(s, b, -1), 0) |
| 1550 |
self.assertEqual(equaltoutf8andsize(s, b, PY_SSIZE_T_MAX), 0) |
| 1551 |
self.assertEqual(equaltoutf8andsize(s, b, PY_SSIZE_T_MIN), 0) |
| 1552 |
|
| 1553 |
self.assertEqual(equaltoutf8andsize('', b''), 1) |
| 1554 |
self.assertEqual(equaltoutf8andsize('', b'\0'), 0) |
| 1555 |
self.assertEqual(equaltoutf8andsize('', b'x', 0), 1) |
| 1556 |
|
| 1557 |
# embedded null chars/bytes |
| 1558 |
self.assertEqual(equaltoutf8andsize('abc\0def', b'abc\0def'), 1) |
| 1559 |
self.assertEqual(equaltoutf8andsize('abc\0def\0', b'abc\0def\0'), 1) |
| 1560 |
|
| 1561 |
# Surrogate characters are always treated as not equal |
| 1562 |
self.assertEqual(equaltoutf8andsize('\udcfe', |
| 1563 |
'\udcfe'.encode("utf8", "surrogateescape")), 0) |
| 1564 |
self.assertEqual(equaltoutf8andsize('\udcfe', |
| 1565 |
'\udcfe'.encode("utf8", "surrogatepass")), 0) |
| 1566 |
self.assertEqual(equaltoutf8andsize('\ud801', |
| 1567 |
'\ud801'.encode("utf8", "surrogatepass")), 0) |
| 1568 |
|
| 1569 |
def check_not_equal_encoding(text, encoding): |
| 1570 |
self.assertEqual(equaltoutf8andsize(text, text.encode(encoding)), 0) |
| 1571 |
self.assertNotEqual(text.encode(encoding), text.encode("utf8")) |
| 1572 |
|
| 1573 |
# Strings encoded to other encodings are not equal to expected UTF8-encoding string |
| 1574 |
check_not_equal_encoding('Stéphane', 'latin1') |
| 1575 |
check_not_equal_encoding('Stéphane', 'utf-16-le') # embedded null characters |
| 1576 |
check_not_equal_encoding('北京市', 'gbk') |
| 1577 |
|
| 1578 |
# CRASHES equaltoutf8andsize('abc', b'abc', -1) |
| 1579 |
# CRASHES equaltoutf8andsize(b'abc', b'abc') |
| 1580 |
# CRASHES equaltoutf8andsize([], b'abc') |
| 1581 |
# CRASHES equaltoutf8andsize(NULL, b'abc') |
| 1582 |
# CRASHES equaltoutf8andsize('abc', NULL) |
| 1583 |
|
| 1584 |
@support.cpython_only |
| 1585 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1586 |
def test_richcompare(self): |
| 1587 |
"""Test PyUnicode_RichCompare()""" |
| 1588 |
from _testlimitedcapi import unicode_richcompare as richcompare |
| 1589 |
|
| 1590 |
LT, LE, EQ, NE, GT, GE = range(6) |
| 1591 |
strings = ('abc', 'абв', '\U0001f600', 'abc\0') |
| 1592 |
for s1 in strings: |
| 1593 |
for s2 in strings: |
| 1594 |
self.assertIs(richcompare(s1, s2, LT), s1 < s2) |
| 1595 |
self.assertIs(richcompare(s1, s2, LE), s1 <= s2) |
| 1596 |
self.assertIs(richcompare(s1, s2, EQ), s1 == s2) |
| 1597 |
self.assertIs(richcompare(s1, s2, NE), s1 != s2) |
| 1598 |
self.assertIs(richcompare(s1, s2, GT), s1 > s2) |
| 1599 |
self.assertIs(richcompare(s1, s2, GE), s1 >= s2) |
| 1600 |
|
| 1601 |
for op in LT, LE, EQ, NE, GT, GE: |
| 1602 |
self.assertIs(richcompare(b'abc', 'abc', op), NotImplemented) |
| 1603 |
self.assertIs(richcompare('abc', b'abc', op), NotImplemented) |
| 1604 |
self.assertIs(richcompare(b'abc', b'abc', op), NotImplemented) |
| 1605 |
self.assertIs(richcompare([], 'abc', op), NotImplemented) |
| 1606 |
self.assertIs(richcompare('abc', [], op), NotImplemented) |
| 1607 |
self.assertIs(richcompare([], [], op), NotImplemented) |
| 1608 |
|
| 1609 |
# CRASHES richcompare(NULL, 'abc', op) |
| 1610 |
# CRASHES richcompare('abc', NULL, op) |
| 1611 |
|
| 1612 |
@support.cpython_only |
| 1613 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1614 |
def test_format(self): |
| 1615 |
"""Test PyUnicode_Format()""" |
| 1616 |
from _testlimitedcapi import unicode_format as format |
| 1617 |
|
| 1618 |
self.assertEqual(format('x=%d!', 42), 'x=42!') |
| 1619 |
self.assertEqual(format('x=%d!', (42,)), 'x=42!') |
| 1620 |
self.assertEqual(format('x=%d y=%s!', (42, [])), 'x=42 y=[]!') |
| 1621 |
|
| 1622 |
self.assertRaises(SystemError, format, 'x=%d!', NULL) |
| 1623 |
self.assertRaises(SystemError, format, NULL, 42) |
| 1624 |
|
| 1625 |
@support.cpython_only |
| 1626 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1627 |
def test_contains(self): |
| 1628 |
"""Test PyUnicode_Contains()""" |
| 1629 |
from _testlimitedcapi import unicode_contains as contains |
| 1630 |
|
| 1631 |
self.assertEqual(contains('abcd', ''), 1) |
| 1632 |
self.assertEqual(contains('abcd', 'b'), 1) |
| 1633 |
self.assertEqual(contains('abcd', 'x'), 0) |
| 1634 |
self.assertEqual(contains('abcd', 'ж'), 0) |
| 1635 |
self.assertEqual(contains('abcd', '\0'), 0) |
| 1636 |
self.assertEqual(contains('abc\0def', '\0'), 1) |
| 1637 |
self.assertEqual(contains('abcd', 'bc'), 1) |
| 1638 |
|
| 1639 |
self.assertRaises(TypeError, contains, b'abcd', 'b') |
| 1640 |
self.assertRaises(TypeError, contains, 'abcd', b'b') |
| 1641 |
self.assertRaises(TypeError, contains, b'abcd', b'b') |
| 1642 |
self.assertRaises(TypeError, contains, [], 'b') |
| 1643 |
self.assertRaises(TypeError, contains, 'abcd', ord('b')) |
| 1644 |
# CRASHES contains(NULL, 'b') |
| 1645 |
# CRASHES contains('abcd', NULL) |
| 1646 |
|
| 1647 |
@support.cpython_only |
| 1648 |
@unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') |
| 1649 |
def test_isidentifier(self): |
| 1650 |
"""Test PyUnicode_IsIdentifier()""" |
| 1651 |
from _testlimitedcapi import unicode_isidentifier as isidentifier |
| 1652 |
|
| 1653 |
self.assertEqual(isidentifier("a"), 1) |
| 1654 |
self.assertEqual(isidentifier("b0"), 1) |
| 1655 |
self.assertEqual(isidentifier("µ"), 1) |
| 1656 |
self.assertEqual(isidentifier("𝔘𝔫𝔦𝔠𝔬𝔡𝔢"), 1) |
| 1657 |
|
| 1658 |
self.assertEqual(isidentifier(""), 0) |
| 1659 |
self.assertEqual(isidentifier(" "), 0) |
| 1660 |
self.assertEqual(isidentifier("["), 0) |
| 1661 |
self.assertEqual(isidentifier("©"), 0) |
| 1662 |
self.assertEqual(isidentifier("0"), 0) |
| 1663 |
self.assertEqual(isidentifier("32M"), 0) |
| 1664 |
|
| 1665 |
# CRASHES isidentifier(b"a") |
| 1666 |
# CRASHES isidentifier([]) |
| 1667 |
# CRASHES isidentifier(NULL) |
| 1668 |
|
| 1669 |
@support.cpython_only |
| 1670 |
@unittest.skipIf(_testcapi is None, 'need _testcapi module') |
| 1671 |
def test_copycharacters(self): |
| 1672 |
"""Test PyUnicode_CopyCharacters()""" |
| 1673 |
from _testcapi import unicode_copycharacters |
| 1674 |
|
| 1675 |
strings = [ |
| 1676 |
# all strings have exactly 5 characters |
| 1677 |
'abcde', '\xa1\xa2\xa3\xa4\xa5', |
| 1678 |
'\u4f60\u597d\u4e16\u754c\uff01', |
| 1679 |
'\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' |
| 1680 |
] |
| 1681 |
|
| 1682 |
for idx, from_ in enumerate(strings): |
| 1683 |
# wide -> narrow: exceed maxchar limitation |
| 1684 |
for to in strings[:idx]: |
| 1685 |
self.assertRaises( |
| 1686 |
SystemError, |
| 1687 |
unicode_copycharacters, to, 0, from_, 0, 5 |
| 1688 |
) |
| 1689 |
# same kind |
| 1690 |
for from_start in range(5): |
| 1691 |
self.assertEqual( |
| 1692 |
unicode_copycharacters(from_, 0, from_, from_start, 5), |
| 1693 |
(from_[from_start:from_start+5].ljust(5, '\0'), |
| 1694 |
5-from_start) |
| 1695 |
) |
| 1696 |
for to_start in range(5): |
| 1697 |
self.assertEqual( |
| 1698 |
unicode_copycharacters(from_, to_start, from_, to_start, 5), |
| 1699 |
(from_[to_start:to_start+5].rjust(5, '\0'), |
| 1700 |
5-to_start) |
| 1701 |
) |
| 1702 |
# narrow -> wide |
| 1703 |
# Tests omitted since this creates invalid strings. |
| 1704 |
|
| 1705 |
s = strings[0] |
| 1706 |
self.assertRaises(IndexError, unicode_copycharacters, s, 6, s, 0, 5) |
| 1707 |
self.assertRaises(IndexError, unicode_copycharacters, s, PY_SSIZE_T_MAX, s, 0, 5) |
| 1708 |
self.assertRaises(IndexError, unicode_copycharacters, s, -1, s, 0, 5) |
| 1709 |
self.assertRaises(IndexError, unicode_copycharacters, s, PY_SSIZE_T_MIN, s, 0, 5) |
| 1710 |
self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, 6, 5) |
| 1711 |
self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, PY_SSIZE_T_MAX, 5) |
| 1712 |
self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, -1, 5) |
| 1713 |
self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, PY_SSIZE_T_MIN, 5) |
| 1714 |
self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5) |
| 1715 |
self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, PY_SSIZE_T_MAX) |
| 1716 |
self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1) |
| 1717 |
self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, PY_SSIZE_T_MIN) |
| 1718 |
self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) |
| 1719 |
self.assertRaises(SystemError, unicode_copycharacters, s, 0, [], 0, 0) |
| 1720 |
# CRASHES unicode_copycharacters(s, 0, NULL, 0, 0) |
| 1721 |
# TODO: Test PyUnicode_CopyCharacters() with non-unicode and |
| 1722 |
# non-modifiable unicode as "to". |
| 1723 |
|
| 1724 |
@support.cpython_only |
| 1725 |
@unittest.skipIf(_testcapi is None, 'need _testcapi module') |
| 1726 |
def test_pep393_utf8_caching_bug(self): |
| 1727 |
# Issue #25709: Problem with string concatenation and utf-8 cache |
| 1728 |
from _testcapi import getargs_s_hash |
| 1729 |
for k in 0x24, 0xa4, 0x20ac, 0x1f40d: |
| 1730 |
s = '' |
| 1731 |
for i in range(5): |
| 1732 |
# Due to CPython specific optimization the 's' string can be |
| 1733 |
# resized in-place. |
| 1734 |
s += chr(k) |
| 1735 |
# Parsing with the "s#" format code calls indirectly |
| 1736 |
# PyUnicode_AsUTF8AndSize() which creates the UTF-8 |
| 1737 |
# encoded string cached in the Unicode object. |
| 1738 |
self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) |
| 1739 |
# Check that the second call returns the same result |
| 1740 |
self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1)) |
| 1741 |
|
| 1742 |
|
| 1743 |
class PyUnicodeWriterTest(unittest.TestCase): |
| 1744 |
def create_writer(self, size): |
| 1745 |
return _testcapi.PyUnicodeWriter(size) |
| 1746 |
|
| 1747 |
def test_basic(self): |
| 1748 |
writer = self.create_writer(100) |
| 1749 |
|
| 1750 |
# test PyUnicodeWriter_WriteUTF8() |
| 1751 |
writer.write_utf8(b'var', -1) |
| 1752 |
|
| 1753 |
# test PyUnicodeWriter_WriteChar() |
| 1754 |
writer.write_char(ord('=')) |
| 1755 |
|
| 1756 |
# test PyUnicodeWriter_WriteSubstring() |
| 1757 |
writer.write_substring("[long]", 1, 5) |
| 1758 |
# CRASHES writer.write_substring(NULL, 0, 0) |
| 1759 |
|
| 1760 |
# test PyUnicodeWriter_WriteStr() |
| 1761 |
writer.write_str(" value ") |
| 1762 |
# CRASHES writer.write_str(NULL) |
| 1763 |
|
| 1764 |
# test PyUnicodeWriter_WriteRepr() |
| 1765 |
writer.write_repr("repr") |
| 1766 |
|
| 1767 |
self.assertEqual(writer.finish(), |
| 1768 |
"var=long value 'repr'") |
| 1769 |
|
| 1770 |
def test_repr_null(self): |
| 1771 |
writer = self.create_writer(0) |
| 1772 |
writer.write_utf8(b'var=', -1) |
| 1773 |
writer.write_repr(NULL) |
| 1774 |
self.assertEqual(writer.finish(), |
| 1775 |
"var=<NULL>") |
| 1776 |
|
| 1777 |
def test_write_char(self): |
| 1778 |
writer = self.create_writer(0) |
| 1779 |
writer.write_char(0) |
| 1780 |
writer.write_char(ord('$')) |
| 1781 |
writer.write_char(0x20ac) |
| 1782 |
writer.write_char(0x10_ffff) |
| 1783 |
self.assertRaises(ValueError, writer.write_char, 0x11_0000) |
| 1784 |
self.assertRaises(ValueError, writer.write_char, 0xFFFF_FFFF) |
| 1785 |
self.assertEqual(writer.finish(), |
| 1786 |
"\0$\u20AC\U0010FFFF") |
| 1787 |
|
| 1788 |
def test_utf8(self): |
| 1789 |
writer = self.create_writer(0) |
| 1790 |
writer.write_utf8(b"ascii", -1) |
| 1791 |
writer.write_char(ord('-')) |
| 1792 |
writer.write_utf8(b"latin1=\xC3\xA9", -1) |
| 1793 |
writer.write_char(ord('-')) |
| 1794 |
writer.write_utf8(b"euro=\xE2\x82\xAC", -1) |
| 1795 |
writer.write_char(ord('.')) |
| 1796 |
writer.write_utf8(NULL, 0) |
| 1797 |
# CRASHES writer.write_utf8(NULL, 1) |
| 1798 |
# CRASHES writer.write_utf8(NULL, -1) |
| 1799 |
self.assertEqual(writer.finish(), |
| 1800 |
"ascii-latin1=\xE9-euro=\u20AC.") |
| 1801 |
|
| 1802 |
def test_ascii(self): |
| 1803 |
writer = self.create_writer(0) |
| 1804 |
writer.write_ascii(b"Hello ", -1) |
| 1805 |
writer.write_ascii(b"", 0) |
| 1806 |
writer.write_ascii(NULL, 0) |
| 1807 |
# CRASHES writer.write_ascii(NULL, 1) |
| 1808 |
# CRASHES writer.write_ascii(NULL, -1) |
| 1809 |
writer.write_ascii(b"Python! <truncated>", 6) |
| 1810 |
self.assertEqual(writer.finish(), "Hello Python") |
| 1811 |
|
| 1812 |
def test_invalid_utf8(self): |
| 1813 |
writer = self.create_writer(0) |
| 1814 |
with self.assertRaises(UnicodeDecodeError): |
| 1815 |
writer.write_utf8(b"invalid=\xFF", -1) |
| 1816 |
|
| 1817 |
def test_recover_utf8_error(self): |
| 1818 |
# test recovering from PyUnicodeWriter_WriteUTF8() error |
| 1819 |
writer = self.create_writer(0) |
| 1820 |
writer.write_utf8(b"value=", -1) |
| 1821 |
|
| 1822 |
# write fails with an invalid string |
| 1823 |
with self.assertRaises(UnicodeDecodeError): |
| 1824 |
writer.write_utf8(b"invalid\xFF", -1) |
| 1825 |
with self.assertRaises(UnicodeDecodeError): |
| 1826 |
s = "truncated\u20AC".encode() |
| 1827 |
writer.write_utf8(s, len(s) - 1) |
| 1828 |
|
| 1829 |
# retry write with a valid string |
| 1830 |
writer.write_utf8(b"valid", -1) |
| 1831 |
|
| 1832 |
self.assertEqual(writer.finish(), |
| 1833 |
"value=valid") |
| 1834 |
|
| 1835 |
def test_decode_utf8(self): |
| 1836 |
# test PyUnicodeWriter_DecodeUTF8Stateful() |
| 1837 |
writer = self.create_writer(0) |
| 1838 |
writer.decodeutf8stateful(b"ign\xFFore", -1, b"ignore") |
| 1839 |
writer.write_char(ord('-')) |
| 1840 |
writer.decodeutf8stateful(b"replace\xFF", -1, b"replace") |
| 1841 |
writer.write_char(ord('-')) |
| 1842 |
|
| 1843 |
# incomplete trailing UTF-8 sequence |
| 1844 |
writer.decodeutf8stateful(b"incomplete\xC3", -1, b"replace") |
| 1845 |
|
| 1846 |
writer.decodeutf8stateful(NULL, 0, b"replace") |
| 1847 |
# CRASHES writer.decodeutf8stateful(NULL, 1, b"replace") |
| 1848 |
# CRASHES writer.decodeutf8stateful(NULL, -1, b"replace") |
| 1849 |
with self.assertRaises(UnicodeDecodeError): |
| 1850 |
writer.decodeutf8stateful(b"default\xFF", -1, NULL) |
| 1851 |
|
| 1852 |
self.assertEqual(writer.finish(), |
| 1853 |
"ignore-replace\uFFFD-incomplete\uFFFD") |
| 1854 |
|
| 1855 |
def test_decode_utf8_consumed(self): |
| 1856 |
# test PyUnicodeWriter_DecodeUTF8Stateful() with consumed |
| 1857 |
writer = self.create_writer(0) |
| 1858 |
|
| 1859 |
# valid string |
| 1860 |
consumed = writer.decodeutf8stateful(b"text", -1, b"strict", True) |
| 1861 |
self.assertEqual(consumed, 4) |
| 1862 |
writer.write_char(ord('-')) |
| 1863 |
|
| 1864 |
# non-ASCII |
| 1865 |
consumed = writer.decodeutf8stateful(b"\xC3\xA9-\xE2\x82\xAC", 6, b"strict", True) |
| 1866 |
self.assertEqual(consumed, 6) |
| 1867 |
writer.write_char(ord('-')) |
| 1868 |
|
| 1869 |
# invalid UTF-8 (consumed is 0 on error) |
| 1870 |
with self.assertRaises(UnicodeDecodeError): |
| 1871 |
writer.decodeutf8stateful(b"invalid\xFF", -1, b"strict", True) |
| 1872 |
|
| 1873 |
# ignore error handler |
| 1874 |
consumed = writer.decodeutf8stateful(b"more\xFF", -1, b"ignore", True) |
| 1875 |
self.assertEqual(consumed, 5) |
| 1876 |
writer.write_char(ord('-')) |
| 1877 |
|
| 1878 |
# incomplete trailing UTF-8 sequence |
| 1879 |
consumed = writer.decodeutf8stateful(b"incomplete\xC3", -1, b"ignore", True) |
| 1880 |
self.assertEqual(consumed, 10) |
| 1881 |
writer.write_char(ord('-')) |
| 1882 |
|
| 1883 |
consumed = writer.decodeutf8stateful(NULL, 0, b"replace", True) |
| 1884 |
self.assertEqual(consumed, 0) |
| 1885 |
# CRASHES writer.decodeutf8stateful(NULL, 1, b"replace", True) |
| 1886 |
# CRASHES writer.decodeutf8stateful(NULL, -1, b"replace", True) |
| 1887 |
consumed = writer.decodeutf8stateful(b"default\xC3", -1, NULL, True) |
| 1888 |
self.assertEqual(consumed, 7) |
| 1889 |
|
| 1890 |
self.assertEqual(writer.finish(), "text-\xE9-\u20AC-more-incomplete-default") |
| 1891 |
|
| 1892 |
def test_widechar(self): |
| 1893 |
from _testcapi import SIZEOF_WCHAR_T |
| 1894 |
|
| 1895 |
if SIZEOF_WCHAR_T == 2: |
| 1896 |
encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' |
| 1897 |
elif SIZEOF_WCHAR_T == 4: |
| 1898 |
encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' |
| 1899 |
|
| 1900 |
writer = self.create_writer(0) |
| 1901 |
writer.write_widechar("latin1=\xE9".encode(encoding)) |
| 1902 |
writer.write_char(ord("-")) |
| 1903 |
writer.write_widechar("euro=\u20AC".encode(encoding)) |
| 1904 |
writer.write_char(ord("-")) |
| 1905 |
writer.write_widechar("max=\U0010ffff".encode(encoding)) |
| 1906 |
writer.write_char(ord("-")) |
| 1907 |
writer.write_widechar("zeroes=".encode(encoding).ljust(SIZEOF_WCHAR_T * 10, b'\0'), |
| 1908 |
10) |
| 1909 |
writer.write_char(ord('.')) |
| 1910 |
|
| 1911 |
if SIZEOF_WCHAR_T == 4: |
| 1912 |
invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else |
| 1913 |
b'\x00\x11\x00\x00') |
| 1914 |
with self.assertRaises(ValueError): |
| 1915 |
writer.write_widechar("invalid=".encode(encoding) + invalid) |
| 1916 |
writer.write_widechar(b'', -5) |
| 1917 |
writer.write_widechar(NULL, 0) |
| 1918 |
# CRASHES writer.write_widechar(NULL, 1) |
| 1919 |
# CRASHES writer.write_widechar(NULL, -1) |
| 1920 |
|
| 1921 |
self.assertEqual(writer.finish(), |
| 1922 |
"latin1=\xE9-euro=\u20AC-max=\U0010ffff-zeroes=\0\0\0.") |
| 1923 |
|
| 1924 |
def test_ucs4(self): |
| 1925 |
encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' |
| 1926 |
|
| 1927 |
writer = self.create_writer(0) |
| 1928 |
writer.write_ucs4("ascii IGNORED".encode(encoding), 5) |
| 1929 |
writer.write_char(ord("-")) |
| 1930 |
writer.write_ucs4("latin1=\xe9".encode(encoding)) |
| 1931 |
writer.write_char(ord("-")) |
| 1932 |
writer.write_ucs4("euro=\u20ac".encode(encoding)) |
| 1933 |
writer.write_char(ord("-")) |
| 1934 |
writer.write_ucs4("max=\U0010ffff".encode(encoding)) |
| 1935 |
writer.write_char(ord(".")) |
| 1936 |
self.assertEqual(writer.finish(), |
| 1937 |
"ascii-latin1=\xE9-euro=\u20AC-max=\U0010ffff.") |
| 1938 |
|
| 1939 |
# Test some special characters |
| 1940 |
writer = self.create_writer(0) |
| 1941 |
# Lone surrogate character |
| 1942 |
writer.write_ucs4("lone\uDC80".encode(encoding, 'surrogatepass')) |
| 1943 |
writer.write_char(ord("-")) |
| 1944 |
# Surrogate pair |
| 1945 |
writer.write_ucs4("pair\uD83D\uDC0D".encode(encoding, 'surrogatepass')) |
| 1946 |
writer.write_char(ord("-")) |
| 1947 |
writer.write_ucs4("null[\0]".encode(encoding), 7) |
| 1948 |
invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else |
| 1949 |
b'\x00\x11\x00\x00') |
| 1950 |
# CRASHES writer.write_ucs4("invalid".encode(encoding) + invalid) |
| 1951 |
writer.write_ucs4(NULL, 0) |
| 1952 |
# CRASHES writer.write_ucs4(NULL, 1) |
| 1953 |
self.assertEqual(writer.finish(), |
| 1954 |
"lone\udc80-pair\ud83d\udc0d-null[\x00]") |
| 1955 |
|
| 1956 |
# invalid size |
| 1957 |
writer = self.create_writer(0) |
| 1958 |
with self.assertRaises(ValueError): |
| 1959 |
writer.write_ucs4("text".encode(encoding), -1) |
| 1960 |
self.assertRaises(ValueError, writer.write_ucs4, b'', -1) |
| 1961 |
self.assertRaises(ValueError, writer.write_ucs4, NULL, -1) |
| 1962 |
|
| 1963 |
def test_substring_empty(self): |
| 1964 |
writer = self.create_writer(0) |
| 1965 |
writer.write_substring("abc", 1, 1) |
| 1966 |
self.assertEqual(writer.finish(), '') |
| 1967 |
|
| 1968 |
|
| 1969 |
@unittest.skipIf(ctypes is None, 'need ctypes') |
| 1970 |
class PyUnicodeWriterFormatTest(unittest.TestCase): |
| 1971 |
def create_writer(self, size): |
| 1972 |
return _testcapi.PyUnicodeWriter(size) |
| 1973 |
|
| 1974 |
def writer_format(self, writer, *args): |
| 1975 |
from ctypes import c_char_p, pythonapi, c_int, c_void_p |
| 1976 |
_PyUnicodeWriter_Format = getattr(pythonapi, "PyUnicodeWriter_Format") |
| 1977 |
_PyUnicodeWriter_Format.argtypes = (c_void_p, c_char_p,) |
| 1978 |
_PyUnicodeWriter_Format.restype = c_int |
| 1979 |
|
| 1980 |
if _PyUnicodeWriter_Format(writer.get_pointer(), *args) < 0: |
| 1981 |
raise ValueError("PyUnicodeWriter_Format failed") |
| 1982 |
|
| 1983 |
def test_format(self): |
| 1984 |
from ctypes import c_int |
| 1985 |
writer = self.create_writer(0) |
| 1986 |
self.writer_format(writer, b'%s %i', b'abc', c_int(123)) |
| 1987 |
writer.write_char(ord('.')) |
| 1988 |
self.assertEqual(writer.finish(), 'abc 123.') |
| 1989 |
|
| 1990 |
def test_recover_error(self): |
| 1991 |
# test recovering from PyUnicodeWriter_Format() error |
| 1992 |
writer = self.create_writer(0) |
| 1993 |
self.writer_format(writer, b"%s ", b"Hello") |
| 1994 |
|
| 1995 |
# PyUnicodeWriter_Format() fails with an invalid format string |
| 1996 |
with self.assertRaises(ValueError): |
| 1997 |
self.writer_format(writer, b"%s\xff", b"World") |
| 1998 |
|
| 1999 |
# Retry PyUnicodeWriter_Format() with a valid format string |
| 2000 |
self.writer_format(writer, b"%s.", b"World") |
| 2001 |
|
| 2002 |
self.assertEqual(writer.finish(), 'Hello World.') |
| 2003 |
|
| 2004 |
def test_unicode_equal(self): |
| 2005 |
unicode_equal = _testlimitedcapi.unicode_equal |
| 2006 |
|
| 2007 |
def copy(text): |
| 2008 |
return text.encode().decode() |
| 2009 |
|
| 2010 |
self.assertTrue(unicode_equal("", "")) |
| 2011 |
self.assertTrue(unicode_equal("abc", "abc")) |
| 2012 |
self.assertTrue(unicode_equal("abc", copy("abc"))) |
| 2013 |
self.assertTrue(unicode_equal("\u20ac", copy("\u20ac"))) |
| 2014 |
self.assertTrue(unicode_equal("\U0010ffff", copy("\U0010ffff"))) |
| 2015 |
|
| 2016 |
self.assertFalse(unicode_equal("abc", "abcd")) |
| 2017 |
self.assertFalse(unicode_equal("\u20ac", "\u20ad")) |
| 2018 |
self.assertFalse(unicode_equal("\U0010ffff", "\U0010fffe")) |
| 2019 |
|
| 2020 |
# str subclass |
| 2021 |
self.assertTrue(unicode_equal("abc", Str("abc"))) |
| 2022 |
self.assertTrue(unicode_equal(Str("abc"), "abc")) |
| 2023 |
self.assertFalse(unicode_equal("abc", Str("abcd"))) |
| 2024 |
self.assertFalse(unicode_equal(Str("abc"), "abcd")) |
| 2025 |
|
| 2026 |
# invalid type |
| 2027 |
for invalid_type in (b'bytes', 123, ("tuple",)): |
| 2028 |
with self.subTest(invalid_type=invalid_type): |
| 2029 |
with self.assertRaises(TypeError): |
| 2030 |
unicode_equal("abc", invalid_type) |
| 2031 |
with self.assertRaises(TypeError): |
| 2032 |
unicode_equal(invalid_type, "abc") |
| 2033 |
|
| 2034 |
# CRASHES unicode_equal("abc", NULL) |
| 2035 |
# CRASHES unicode_equal(NULL, "abc") |
| 2036 |
|
| 2037 |
|
| 2038 |
if __name__ == "__main__": |
| 2039 |
unittest.main() |