Debian Codegraph @

Lib/test/test_capi/test_unicode.py

raw
1 import unittest
2 import sys
3 from test import support
4 from test.support import threading_helper
5
6 try:
7 import _testcapi
8 from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX
9 except ImportError:
10 _testcapi = None
11 try:
12 import _testlimitedcapi
13 except ImportError:
14 _testlimitedcapi = None
15 try:
16 import _testinternalcapi
17 except ImportError:
18 _testinternalcapi = None
19 try:
20 import ctypes
21 except ImportError:
22 ctypes = None
23
24
25 NULL = None
26
27 class Str(str):
28 pass
29
30
31 class CAPITest(unittest.TestCase):
32
33 @support.cpython_only
34 @unittest.skipIf(_testcapi is None, 'need _testcapi module')
35 def test_new(self):
36 """Test PyUnicode_New()"""
37 from _testcapi import unicode_new as new
38
39 for maxchar in 0, 0x61, 0xa1, 0x4f60, 0x1f600, 0x10ffff:
40 self.assertEqual(new(0, maxchar), '')
41 self.assertEqual(new(5, maxchar), chr(maxchar)*5)
42 self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX, maxchar)
43 self.assertEqual(new(0, 0x110000), '')
44 self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x4f60)
45 self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x4f60)
46 self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x1f600)
47 self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x1f600)
48 self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4, 0x1f600)
49 self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4+1, 0x1f600)
50 self.assertRaises(SystemError, new, 5, 0x110000)
51 self.assertRaises(SystemError, new, -1, 0)
52 self.assertRaises(SystemError, new, PY_SSIZE_T_MIN, 0)
53
54 @support.cpython_only
55 @unittest.skipIf(_testcapi is None, 'need _testcapi module')
56 def test_fill(self):
57 """Test PyUnicode_Fill()"""
58 from _testcapi import unicode_fill as fill
59
60 strings = [
61 # all strings have exactly 5 characters
62 'abcde', '\xa1\xa2\xa3\xa4\xa5',
63 '\u4f60\u597d\u4e16\u754c\uff01',
64 '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604'
65 ]
66 chars = [0x78, 0xa9, 0x20ac, 0x1f638]
67
68 for idx, fill_char in enumerate(chars):
69 # wide -> narrow: exceed maxchar limitation
70 for to in strings[:idx]:
71 self.assertRaises(ValueError, fill, to, 0, 0, fill_char)
72 for to in strings[idx:]:
73 for start in [*range(7), PY_SSIZE_T_MAX]:
74 for length in [*range(-1, 7 - start), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX]:
75 filled = max(min(length, 5 - start), 0)
76 if filled == 5 and to != strings[idx]:
77 # narrow -> wide
78 # Tests omitted since this creates invalid strings.
79 continue
80 expected = to[:start] + chr(fill_char) * filled + to[start + filled:]
81 self.assertEqual(fill(to, start, length, fill_char),
82 (expected, filled))
83
84 s = strings[0]
85 self.assertRaises(IndexError, fill, s, -1, 0, 0x78)
86 self.assertRaises(IndexError, fill, s, PY_SSIZE_T_MIN, 0, 0x78)
87 self.assertRaises(ValueError, fill, s, 0, 0, 0x110000)
88 self.assertRaises(SystemError, fill, b'abc', 0, 0, 0x78)
89 self.assertRaises(SystemError, fill, [], 0, 0, 0x78)
90 # CRASHES fill(s, 0, NULL, 0, 0)
91 # CRASHES fill(NULL, 0, 0, 0x78)
92 # TODO: Test PyUnicode_Fill() with non-modifiable unicode.
93
94 @support.cpython_only
95 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
96 def test_writechar(self):
97 """Test PyUnicode_WriteChar()"""
98 from _testlimitedcapi import unicode_writechar as writechar
99
100 strings = [
101 # one string for every kind
102 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16',
103 '\U0001f600\U0001f601\U0001f602'
104 ]
105 # one character for every kind + out of range code
106 chars = [0x78, 0xa9, 0x20ac, 0x1f638, 0x110000]
107 for i, s in enumerate(strings):
108 for j, c in enumerate(chars):
109 if j <= i:
110 self.assertEqual(writechar(s, 1, c),
111 (s[:1] + chr(c) + s[2:], 0))
112 else:
113 self.assertRaises(ValueError, writechar, s, 1, c)
114
115 self.assertRaises(IndexError, writechar, 'abc', 3, 0x78)
116 self.assertRaises(IndexError, writechar, 'abc', -1, 0x78)
117 self.assertRaises(IndexError, writechar, 'abc', PY_SSIZE_T_MAX, 0x78)
118 self.assertRaises(IndexError, writechar, 'abc', PY_SSIZE_T_MIN, 0x78)
119 self.assertRaises(TypeError, writechar, b'abc', 0, 0x78)
120 self.assertRaises(TypeError, writechar, [], 0, 0x78)
121 # CRASHES writechar(NULL, 0, 0x78)
122 # TODO: Test PyUnicode_WriteChar() with non-modifiable and legacy
123 # unicode.
124
125 @support.cpython_only
126 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
127 def test_resize(self):
128 """Test PyUnicode_Resize()"""
129 from _testlimitedcapi import unicode_resize as resize
130
131 strings = [
132 # all strings have exactly 3 characters
133 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16',
134 '\U0001f600\U0001f601\U0001f602'
135 ]
136 for s in strings:
137 self.assertEqual(resize(s, 3), (s, 0))
138 self.assertEqual(resize(s, 2), (s[:2], 0))
139 self.assertEqual(resize(s, 4), (s + '\0', 0))
140 self.assertEqual(resize(s, 10), (s + '\0'*7, 0))
141 self.assertEqual(resize(s, 0), ('', 0))
142 self.assertRaises(MemoryError, resize, s, PY_SSIZE_T_MAX)
143 self.assertRaises(SystemError, resize, s, -1)
144 self.assertRaises(SystemError, resize, s, PY_SSIZE_T_MIN)
145 self.assertRaises(SystemError, resize, b'abc', 0)
146 self.assertRaises(SystemError, resize, [], 0)
147 self.assertRaises(SystemError, resize, NULL, 0)
148 # TODO: Test PyUnicode_Resize() with non-modifiable and legacy unicode
149 # and with NULL as the address.
150
151 @support.cpython_only
152 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
153 def test_append(self):
154 """Test PyUnicode_Append()"""
155 from _testlimitedcapi import unicode_append as append
156
157 strings = [
158 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16',
159 '\U0001f600\U0001f601\U0001f602'
160 ]
161 for left in strings:
162 left = left[::-1]
163 for right in strings:
164 expected = left + right
165 self.assertEqual(append(left, right), expected)
166
167 self.assertRaises(SystemError, append, 'abc', b'abc')
168 self.assertRaises(SystemError, append, b'abc', 'abc')
169 self.assertRaises(SystemError, append, b'abc', b'abc')
170 self.assertRaises(SystemError, append, 'abc', [])
171 self.assertRaises(SystemError, append, [], 'abc')
172 self.assertRaises(SystemError, append, [], [])
173 self.assertRaises(SystemError, append, NULL, 'abc')
174 self.assertRaises(SystemError, append, 'abc', NULL)
175 # TODO: Test PyUnicode_Append() with modifiable unicode
176 # and with NULL as the address.
177 # TODO: Check reference counts.
178
179 @support.cpython_only
180 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
181 def test_appendanddel(self):
182 """Test PyUnicode_AppendAndDel()"""
183 from _testlimitedcapi import unicode_appendanddel as appendanddel
184
185 strings = [
186 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16',
187 '\U0001f600\U0001f601\U0001f602'
188 ]
189 for left in strings:
190 left = left[::-1]
191 for right in strings:
192 self.assertEqual(appendanddel(left, right), left + right)
193
194 self.assertRaises(SystemError, appendanddel, 'abc', b'abc')
195 self.assertRaises(SystemError, appendanddel, b'abc', 'abc')
196 self.assertRaises(SystemError, appendanddel, b'abc', b'abc')
197 self.assertRaises(SystemError, appendanddel, 'abc', [])
198 self.assertRaises(SystemError, appendanddel, [], 'abc')
199 self.assertRaises(SystemError, appendanddel, [], [])
200 self.assertRaises(SystemError, appendanddel, NULL, 'abc')
201 self.assertRaises(SystemError, appendanddel, 'abc', NULL)
202 # TODO: Test PyUnicode_AppendAndDel() with modifiable unicode
203 # and with NULL as the address.
204 # TODO: Check reference counts.
205
206 @support.cpython_only
207 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
208 def test_fromstringandsize(self):
209 """Test PyUnicode_FromStringAndSize()"""
210 from _testlimitedcapi import unicode_fromstringandsize as fromstringandsize
211
212 self.assertEqual(fromstringandsize(b'abc'), 'abc')
213 self.assertEqual(fromstringandsize(b'abc', 2), 'ab')
214 self.assertEqual(fromstringandsize(b'abc\0def'), 'abc\0def')
215 self.assertEqual(fromstringandsize(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2')
216 self.assertEqual(fromstringandsize(b'\xe4\xbd\xa0'), '\u4f60')
217 self.assertEqual(fromstringandsize(b'\xf0\x9f\x98\x80'), '\U0001f600')
218 self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xc2\xa1', 1)
219 self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xa1', 1)
220 self.assertEqual(fromstringandsize(b'', 0), '')
221 self.assertEqual(fromstringandsize(NULL, 0), '')
222
223 self.assertRaises(MemoryError, fromstringandsize, b'abc', PY_SSIZE_T_MAX)
224 self.assertRaises(SystemError, fromstringandsize, b'abc', -1)
225 self.assertRaises(SystemError, fromstringandsize, b'abc', PY_SSIZE_T_MIN)
226 self.assertRaises(SystemError, fromstringandsize, NULL, -1)
227 self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MIN)
228 self.assertRaises(SystemError, fromstringandsize, NULL, 3)
229 self.assertRaises(SystemError, fromstringandsize, NULL, PY_SSIZE_T_MAX)
230
231 @support.cpython_only
232 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
233 def test_fromstring(self):
234 """Test PyUnicode_FromString()"""
235 from _testlimitedcapi import unicode_fromstring as fromstring
236
237 self.assertEqual(fromstring(b'abc'), 'abc')
238 self.assertEqual(fromstring(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2')
239 self.assertEqual(fromstring(b'\xe4\xbd\xa0'), '\u4f60')
240 self.assertEqual(fromstring(b'\xf0\x9f\x98\x80'), '\U0001f600')
241 self.assertRaises(UnicodeDecodeError, fromstring, b'\xc2')
242 self.assertRaises(UnicodeDecodeError, fromstring, b'\xa1')
243 self.assertEqual(fromstring(b''), '')
244
245 # CRASHES fromstring(NULL)
246
247 @support.cpython_only
248 @unittest.skipIf(_testcapi is None, 'need _testcapi module')
249 def test_fromkindanddata(self):
250 """Test PyUnicode_FromKindAndData()"""
251 from _testcapi import unicode_fromkindanddata as fromkindanddata
252
253 strings = [
254 'abcde', '\xa1\xa2\xa3\xa4\xa5',
255 '\u4f60\u597d\u4e16\u754c\uff01',
256 '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604'
257 ]
258 enc1 = 'latin1'
259 for s in strings[:2]:
260 self.assertEqual(fromkindanddata(1, s.encode(enc1)), s)
261 enc2 = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be'
262 for s in strings[:3]:
263 self.assertEqual(fromkindanddata(2, s.encode(enc2)), s)
264 enc4 = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be'
265 for s in strings:
266 self.assertEqual(fromkindanddata(4, s.encode(enc4)), s)
267 self.assertEqual(fromkindanddata(2, '\U0001f600'.encode(enc2)),
268 '\ud83d\ude00')
269 for kind in 1, 2, 4:
270 self.assertEqual(fromkindanddata(kind, b''), '')
271 self.assertEqual(fromkindanddata(kind, b'\0'*kind), '\0')
272 self.assertEqual(fromkindanddata(kind, NULL, 0), '')
273
274 for kind in -1, 0, 3, 5, 8:
275 self.assertRaises(SystemError, fromkindanddata, kind, b'')
276 self.assertRaises(ValueError, fromkindanddata, 1, b'abc', -1)
277 self.assertRaises(ValueError, fromkindanddata, 1, b'abc', PY_SSIZE_T_MIN)
278 self.assertRaises(ValueError, fromkindanddata, 1, NULL, -1)
279 self.assertRaises(ValueError, fromkindanddata, 1, NULL, PY_SSIZE_T_MIN)
280 # CRASHES fromkindanddata(1, NULL, 1)
281 # CRASHES fromkindanddata(4, b'\xff\xff\xff\xff')
282
283 @support.cpython_only
284 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
285 def test_substring(self):
286 """Test PyUnicode_Substring()"""
287 from _testlimitedcapi import unicode_substring as substring
288
289 strings = [
290 'ab', 'ab\xa1\xa2',
291 'ab\xa1\xa2\u4f60\u597d',
292 'ab\xa1\xa2\u4f60\u597d\U0001f600\U0001f601'
293 ]
294 for s in strings:
295 for start in [*range(0, len(s) + 2), PY_SSIZE_T_MAX]:
296 for end in [*range(max(start-1, 0), len(s) + 2), PY_SSIZE_T_MAX]:
297 self.assertEqual(substring(s, start, end), s[start:end])
298
299 self.assertRaises(IndexError, substring, 'abc', -1, 0)
300 self.assertRaises(IndexError, substring, 'abc', PY_SSIZE_T_MIN, 0)
301 self.assertRaises(IndexError, substring, 'abc', 0, -1)
302 self.assertRaises(IndexError, substring, 'abc', 0, PY_SSIZE_T_MIN)
303 # CRASHES substring(b'abc', 0, 0)
304 # CRASHES substring([], 0, 0)
305 # CRASHES substring(NULL, 0, 0)
306
307 @support.cpython_only
308 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
309 def test_getlength(self):
310 """Test PyUnicode_GetLength()"""
311 from _testlimitedcapi import unicode_getlength as getlength
312
313 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600',
314 'a\ud800b\udfffc', '\ud834\udd1e']:
315 self.assertEqual(getlength(s), len(s))
316
317 self.assertRaises(TypeError, getlength, b'abc')
318 self.assertRaises(TypeError, getlength, [])
319 # CRASHES getlength(NULL)
320
321 @support.cpython_only
322 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
323 def test_readchar(self):
324 """Test PyUnicode_ReadChar()"""
325 from _testlimitedcapi import unicode_readchar as readchar
326
327 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600',
328 'a\ud800b\udfffc', '\ud834\udd1e']:
329 for i, c in enumerate(s):
330 self.assertEqual(readchar(s, i), ord(c))
331 self.assertRaises(IndexError, readchar, s, len(s))
332 self.assertRaises(IndexError, readchar, s, PY_SSIZE_T_MAX)
333 self.assertRaises(IndexError, readchar, s, -1)
334 self.assertRaises(IndexError, readchar, s, PY_SSIZE_T_MIN)
335
336 self.assertRaises(TypeError, readchar, b'abc', 0)
337 self.assertRaises(TypeError, readchar, [], 0)
338 # CRASHES readchar(NULL, 0)
339
340 @support.cpython_only
341 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
342 def test_fromobject(self):
343 """Test PyUnicode_FromObject()"""
344 from _testlimitedcapi import unicode_fromobject as fromobject
345
346 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600',
347 'a\ud800b\udfffc', '\ud834\udd1e']:
348 self.assertEqual(fromobject(s), s)
349 o = Str(s)
350 s2 = fromobject(o)
351 self.assertEqual(s2, s)
352 self.assertIs(type(s2), str)
353 self.assertIsNot(s2, s)
354
355 self.assertRaises(TypeError, fromobject, b'abc')
356 self.assertRaises(TypeError, fromobject, [])
357 # CRASHES fromobject(NULL)
358
359 @unittest.skipIf(ctypes is None, 'need ctypes')
360 def test_from_format(self):
361 """Test PyUnicode_FromFormat()"""
362 # Length modifiers "j" and "t" are not tested here because ctypes does
363 # not expose types for intmax_t and ptrdiff_t.
364 # _testlimitedcapi.test_string_from_format() has a wider coverage of all
365 # formats.
366 from ctypes import (
367 c_char_p,
368 pythonapi, py_object, sizeof,
369 c_int, c_long, c_longlong, c_ssize_t,
370 c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p,
371 c_wchar, c_wchar_p)
372 name = "PyUnicode_FromFormat"
373 _PyUnicode_FromFormat = getattr(pythonapi, name)
374 _PyUnicode_FromFormat.argtypes = (c_char_p,)
375 _PyUnicode_FromFormat.restype = py_object
376
377 def PyUnicode_FromFormat(format, *args):
378 cargs = tuple(
379 py_object(arg) if isinstance(arg, str) else arg
380 for arg in args)
381 return _PyUnicode_FromFormat(format, *cargs)
382
383 def check_format(expected, format, *args):
384 text = PyUnicode_FromFormat(format, *args)
385 self.assertEqual(expected, text)
386
387 # ascii format, non-ascii argument
388 check_format('ascii\x7f=unicode\xe9',
389 b'ascii\x7f=%U', 'unicode\xe9')
390
391 # non-ascii format, ascii argument: ensure that PyUnicode_FromFormatV()
392 # raises an error
393 self.assertRaisesRegex(ValueError,
394 r'^PyUnicode_FromFormatV\(\) expects an ASCII-encoded format '
395 'string, got a non-ASCII byte: 0xe9$',
396 PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii')
397
398 # test "%c"
399 check_format('\uabcd',
400 b'%c', c_int(0xabcd))
401 check_format('\U0010ffff',
402 b'%c', c_int(0x10ffff))
403 with self.assertRaises(OverflowError):
404 PyUnicode_FromFormat(b'%c', c_int(0x110000))
405 # Issue #18183
406 check_format('\U00010000\U00100000',
407 b'%c%c', c_int(0x10000), c_int(0x100000))
408
409 # test "%"
410 check_format('%',
411 b'%%')
412 check_format('%s',
413 b'%%s')
414 check_format('[%]',
415 b'[%%]')
416 check_format('%abc',
417 b'%%%s', b'abc')
418
419 # truncated string
420 check_format('abc',
421 b'%.3s', b'abcdef')
422 check_format('abc[',
423 b'%.6s', 'abc[\u20ac]'.encode('utf8'))
424 check_format('abc[\u20ac',
425 b'%.7s', 'abc[\u20ac]'.encode('utf8'))
426 check_format('abc[\ufffd',
427 b'%.5s', b'abc[\xff]')
428 check_format('abc[',
429 b'%.6s', b'abc[\xe2\x82]')
430 check_format('abc[\ufffd]',
431 b'%.7s', b'abc[\xe2\x82]')
432 check_format('abc[\ufffd',
433 b'%.7s', b'abc[\xe2\x82\0')
434 check_format(' abc[',
435 b'%10.6s', 'abc[\u20ac]'.encode('utf8'))
436 check_format(' abc[\u20ac',
437 b'%10.7s', 'abc[\u20ac]'.encode('utf8'))
438 check_format(' abc[\ufffd',
439 b'%10.5s', b'abc[\xff]')
440 check_format(' abc[',
441 b'%10.6s', b'abc[\xe2\x82]')
442 check_format(' abc[\ufffd]',
443 b'%10.7s', b'abc[\xe2\x82]')
444
445 check_format("'\\u20acABC'",
446 b'%A', '\u20acABC')
447 check_format("'\\u20",
448 b'%.5A', '\u20acABCDEF')
449 check_format("'\u20acABC'",
450 b'%R', '\u20acABC')
451 check_format("'\u20acA",
452 b'%.3R', '\u20acABCDEF')
453 check_format('\u20acAB',
454 b'%.3S', '\u20acABCDEF')
455 check_format('\u20acAB',
456 b'%.3U', '\u20acABCDEF')
457
458 check_format('\u20acAB',
459 b'%.3V', '\u20acABCDEF', None)
460 check_format('abc[',
461 b'%.6V', None, 'abc[\u20ac]'.encode('utf8'))
462 check_format('abc[\u20ac',
463 b'%.7V', None, 'abc[\u20ac]'.encode('utf8'))
464 check_format('abc[\ufffd',
465 b'%.5V', None, b'abc[\xff]')
466 check_format('abc[',
467 b'%.6V', None, b'abc[\xe2\x82]')
468 check_format('abc[\ufffd]',
469 b'%.7V', None, b'abc[\xe2\x82]')
470 check_format(' abc[',
471 b'%10.6V', None, 'abc[\u20ac]'.encode('utf8'))
472 check_format(' abc[\u20ac',
473 b'%10.7V', None, 'abc[\u20ac]'.encode('utf8'))
474 check_format(' abc[\ufffd',
475 b'%10.5V', None, b'abc[\xff]')
476 check_format(' abc[',
477 b'%10.6V', None, b'abc[\xe2\x82]')
478 check_format(' abc[\ufffd]',
479 b'%10.7V', None, b'abc[\xe2\x82]')
480 check_format(' abc[\ufffd',
481 b'%10.7V', None, b'abc[\xe2\x82\0')
482
483 # following tests comes from #7330
484 # test width modifier and precision modifier with %S
485 check_format("repr= abc",
486 b'repr=%5S', 'abc')
487 check_format("repr=ab",
488 b'repr=%.2S', 'abc')
489 check_format("repr= ab",
490 b'repr=%5.2S', 'abc')
491
492 # test width modifier and precision modifier with %R
493 check_format("repr= 'abc'",
494 b'repr=%8R', 'abc')
495 check_format("repr='ab",
496 b'repr=%.3R', 'abc')
497 check_format("repr= 'ab",
498 b'repr=%5.3R', 'abc')
499
500 # test width modifier and precision modifier with %A
501 check_format("repr= 'abc'",
502 b'repr=%8A', 'abc')
503 check_format("repr='ab",
504 b'repr=%.3A', 'abc')
505 check_format("repr= 'ab",
506 b'repr=%5.3A', 'abc')
507
508 # test width modifier and precision modifier with %s
509 check_format("repr= abc",
510 b'repr=%5s', b'abc')
511 check_format("repr=ab",
512 b'repr=%.2s', b'abc')
513 check_format("repr= ab",
514 b'repr=%5.2s', b'abc')
515
516 # test width modifier and precision modifier with %U
517 check_format("repr= abc",
518 b'repr=%5U', 'abc')
519 check_format("repr=ab",
520 b'repr=%.2U', 'abc')
521 check_format("repr= ab",
522 b'repr=%5.2U', 'abc')
523
524 # test width modifier and precision modifier with %V
525 check_format("repr= abc",
526 b'repr=%5V', 'abc', b'123')
527 check_format("repr=ab",
528 b'repr=%.2V', 'abc', b'123')
529 check_format("repr= ab",
530 b'repr=%5.2V', 'abc', b'123')
531 check_format("repr= 123",
532 b'repr=%5V', None, b'123')
533 check_format("repr=12",
534 b'repr=%.2V', None, b'123')
535 check_format("repr= 12",
536 b'repr=%5.2V', None, b'123')
537
538 # test integer formats (%i, %d, %u, %o, %x, %X)
539 check_format('010',
540 b'%03i', c_int(10))
541 check_format('0010',
542 b'%0.4i', c_int(10))
543 for conv, signed, value, expected in [
544 (b'i', True, -123, '-123'),
545 (b'd', True, -123, '-123'),
546 (b'u', False, 123, '123'),
547 (b'o', False, 0o123, '123'),
548 (b'x', False, 0xabc, 'abc'),
549 (b'X', False, 0xabc, 'ABC'),
550 ]:
551 for mod, ctype in [
552 (b'', c_int if signed else c_uint),
553 (b'l', c_long if signed else c_ulong),
554 (b'll', c_longlong if signed else c_ulonglong),
555 (b'z', c_ssize_t if signed else c_size_t),
556 ]:
557 with self.subTest(format=b'%' + mod + conv):
558 check_format(expected,
559 b'%' + mod + conv, ctype(value))
560
561 # test long output
562 min_longlong = -(2 ** (8 * sizeof(c_longlong) - 1))
563 max_longlong = -min_longlong - 1
564 check_format(str(min_longlong),
565 b'%lld', c_longlong(min_longlong))
566 check_format(str(max_longlong),
567 b'%lld', c_longlong(max_longlong))
568 max_ulonglong = 2 ** (8 * sizeof(c_ulonglong)) - 1
569 check_format(str(max_ulonglong),
570 b'%llu', c_ulonglong(max_ulonglong))
571 PyUnicode_FromFormat(b'%p', c_void_p(-1))
572
573 # test padding (width and/or precision)
574 check_format('123', b'%2i', c_int(123))
575 check_format(' 123', b'%10i', c_int(123))
576 check_format('0000000123', b'%010i', c_int(123))
577 check_format('123 ', b'%-10i', c_int(123))
578 check_format('123 ', b'%-010i', c_int(123))
579 check_format('123', b'%.2i', c_int(123))
580 check_format('0000123', b'%.7i', c_int(123))
581 check_format(' 123', b'%10.2i', c_int(123))
582 check_format(' 0000123', b'%10.7i', c_int(123))
583 check_format('0000000123', b'%010.7i', c_int(123))
584 check_format('0000123 ', b'%-10.7i', c_int(123))
585 check_format('0000123 ', b'%-010.7i', c_int(123))
586
587 check_format('-123', b'%2i', c_int(-123))
588 check_format(' -123', b'%10i', c_int(-123))
589 check_format('-000000123', b'%010i', c_int(-123))
590 check_format('-123 ', b'%-10i', c_int(-123))
591 check_format('-123 ', b'%-010i', c_int(-123))
592 check_format('-123', b'%.2i', c_int(-123))
593 check_format('-0000123', b'%.7i', c_int(-123))
594 check_format(' -123', b'%10.2i', c_int(-123))
595 check_format(' -0000123', b'%10.7i', c_int(-123))
596 check_format('-000000123', b'%010.7i', c_int(-123))
597 check_format('-0000123 ', b'%-10.7i', c_int(-123))
598 check_format('-0000123 ', b'%-010.7i', c_int(-123))
599
600 check_format('123', b'%2u', c_uint(123))
601 check_format(' 123', b'%10u', c_uint(123))
602 check_format('0000000123', b'%010u', c_uint(123))
603 check_format('123 ', b'%-10u', c_uint(123))
604 check_format('123 ', b'%-010u', c_uint(123))
605 check_format('123', b'%.2u', c_uint(123))
606 check_format('0000123', b'%.7u', c_uint(123))
607 check_format(' 123', b'%10.2u', c_uint(123))
608 check_format(' 0000123', b'%10.7u', c_uint(123))
609 check_format('0000000123', b'%010.7u', c_uint(123))
610 check_format('0000123 ', b'%-10.7u', c_uint(123))
611 check_format('0000123 ', b'%-010.7u', c_uint(123))
612
613 check_format('123', b'%2o', c_uint(0o123))
614 check_format(' 123', b'%10o', c_uint(0o123))
615 check_format('0000000123', b'%010o', c_uint(0o123))
616 check_format('123 ', b'%-10o', c_uint(0o123))
617 check_format('123 ', b'%-010o', c_uint(0o123))
618 check_format('123', b'%.2o', c_uint(0o123))
619 check_format('0000123', b'%.7o', c_uint(0o123))
620 check_format(' 123', b'%10.2o', c_uint(0o123))
621 check_format(' 0000123', b'%10.7o', c_uint(0o123))
622 check_format('0000000123', b'%010.7o', c_uint(0o123))
623 check_format('0000123 ', b'%-10.7o', c_uint(0o123))
624 check_format('0000123 ', b'%-010.7o', c_uint(0o123))
625
626 check_format('abc', b'%2x', c_uint(0xabc))
627 check_format(' abc', b'%10x', c_uint(0xabc))
628 check_format('0000000abc', b'%010x', c_uint(0xabc))
629 check_format('abc ', b'%-10x', c_uint(0xabc))
630 check_format('abc ', b'%-010x', c_uint(0xabc))
631 check_format('abc', b'%.2x', c_uint(0xabc))
632 check_format('0000abc', b'%.7x', c_uint(0xabc))
633 check_format(' abc', b'%10.2x', c_uint(0xabc))
634 check_format(' 0000abc', b'%10.7x', c_uint(0xabc))
635 check_format('0000000abc', b'%010.7x', c_uint(0xabc))
636 check_format('0000abc ', b'%-10.7x', c_uint(0xabc))
637 check_format('0000abc ', b'%-010.7x', c_uint(0xabc))
638
639 check_format('ABC', b'%2X', c_uint(0xabc))
640 check_format(' ABC', b'%10X', c_uint(0xabc))
641 check_format('0000000ABC', b'%010X', c_uint(0xabc))
642 check_format('ABC ', b'%-10X', c_uint(0xabc))
643 check_format('ABC ', b'%-010X', c_uint(0xabc))
644 check_format('ABC', b'%.2X', c_uint(0xabc))
645 check_format('0000ABC', b'%.7X', c_uint(0xabc))
646 check_format(' ABC', b'%10.2X', c_uint(0xabc))
647 check_format(' 0000ABC', b'%10.7X', c_uint(0xabc))
648 check_format('0000000ABC', b'%010.7X', c_uint(0xabc))
649 check_format('0000ABC ', b'%-10.7X', c_uint(0xabc))
650 check_format('0000ABC ', b'%-010.7X', c_uint(0xabc))
651
652 # test %A
653 check_format(r"%A:'abc\xe9\uabcd\U0010ffff'",
654 b'%%A:%A', 'abc\xe9\uabcd\U0010ffff')
655
656 # test %V
657 check_format('abc',
658 b'%V', 'abc', b'xyz')
659 check_format('xyz',
660 b'%V', None, b'xyz')
661
662 # test %ls
663 check_format('abc', b'%ls', c_wchar_p('abc'))
664 check_format('\u4eba\u6c11', b'%ls', c_wchar_p('\u4eba\u6c11'))
665 check_format('\U0001f4bb+\U0001f40d',
666 b'%ls', c_wchar_p('\U0001f4bb+\U0001f40d'))
667 check_format(' ab', b'%5.2ls', c_wchar_p('abc'))
668 check_format(' \u4eba\u6c11', b'%5ls', c_wchar_p('\u4eba\u6c11'))
669 check_format(' \U0001f4bb+\U0001f40d',
670 b'%5ls', c_wchar_p('\U0001f4bb+\U0001f40d'))
671 check_format('\u4eba', b'%.1ls', c_wchar_p('\u4eba\u6c11'))
672 check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d',
673 b'%.1ls', c_wchar_p('\U0001f4bb+\U0001f40d'))
674 check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb',
675 b'%.2ls', c_wchar_p('\U0001f4bb+\U0001f40d'))
676
677 # test %lV
678 check_format('abc',
679 b'%lV', 'abc', c_wchar_p('xyz'))
680 check_format('xyz',
681 b'%lV', None, c_wchar_p('xyz'))
682 check_format('\u4eba\u6c11',
683 b'%lV', None, c_wchar_p('\u4eba\u6c11'))
684 check_format('\U0001f4bb+\U0001f40d',
685 b'%lV', None, c_wchar_p('\U0001f4bb+\U0001f40d'))
686 check_format(' ab',
687 b'%5.2lV', None, c_wchar_p('abc'))
688 check_format(' \u4eba\u6c11',
689 b'%5lV', None, c_wchar_p('\u4eba\u6c11'))
690 check_format(' \U0001f4bb+\U0001f40d',
691 b'%5lV', None, c_wchar_p('\U0001f4bb+\U0001f40d'))
692 check_format('\u4eba',
693 b'%.1lV', None, c_wchar_p('\u4eba\u6c11'))
694 check_format('\U0001f4bb' if sizeof(c_wchar) > 2 else '\ud83d',
695 b'%.1lV', None, c_wchar_p('\U0001f4bb+\U0001f40d'))
696 check_format('\U0001f4bb+' if sizeof(c_wchar) > 2 else '\U0001f4bb',
697 b'%.2lV', None, c_wchar_p('\U0001f4bb+\U0001f40d'))
698
699 # test %T
700 check_format('type: str',
701 b'type: %T', py_object("abc"))
702 check_format(f'type: st',
703 b'type: %.2T', py_object("abc"))
704 check_format(f'type: str',
705 b'type: %10T', py_object("abc"))
706
707 class LocalType:
708 pass
709 obj = LocalType()
710 fullname = f'{__name__}.{LocalType.__qualname__}'
711 check_format(f'type: {fullname}',
712 b'type: %T', py_object(obj))
713 fullname_alt = f'{__name__}:{LocalType.__qualname__}'
714 check_format(f'type: {fullname_alt}',
715 b'type: %#T', py_object(obj))
716
717 # test %N
718 check_format('type: str',
719 b'type: %N', py_object(str))
720 check_format(f'type: st',
721 b'type: %.2N', py_object(str))
722 check_format(f'type: str',
723 b'type: %10N', py_object(str))
724
725 check_format(f'type: {fullname}',
726 b'type: %N', py_object(type(obj)))
727 check_format(f'type: {fullname_alt}',
728 b'type: %#N', py_object(type(obj)))
729 with self.assertRaisesRegex(TypeError, "%N argument must be a type"):
730 check_format('type: str',
731 b'type: %N', py_object("abc"))
732
733 # test variable width and precision
734 check_format(' abc', b'%*s', c_int(5), b'abc')
735 check_format('ab', b'%.*s', c_int(2), b'abc')
736 check_format(' ab', b'%*.*s', c_int(5), c_int(2), b'abc')
737 check_format(' abc', b'%*U', c_int(5), 'abc')
738 check_format('ab', b'%.*U', c_int(2), 'abc')
739 check_format(' ab', b'%*.*U', c_int(5), c_int(2), 'abc')
740 check_format(' ab', b'%*.*V', c_int(5), c_int(2), None, b'abc')
741 check_format(' ab', b'%*.*lV', c_int(5), c_int(2),
742 None, c_wchar_p('abc'))
743 check_format(' 123', b'%*i', c_int(8), c_int(123))
744 check_format('00123', b'%.*i', c_int(5), c_int(123))
745 check_format(' 00123', b'%*.*i', c_int(8), c_int(5), c_int(123))
746
747 # test %p
748 # We cannot test the exact result,
749 # because it returns a hex representation of a C pointer,
750 # which is going to be different each time. But, we can test the format.
751 p_format_regex = r'^0x[a-zA-Z0-9]{3,}$'
752 p_format1 = PyUnicode_FromFormat(b'%p', 'abc')
753 self.assertIsInstance(p_format1, str)
754 self.assertRegex(p_format1, p_format_regex)
755
756 p_format2 = PyUnicode_FromFormat(b'%p %p', '123456', b'xyz')
757 self.assertIsInstance(p_format2, str)
758 self.assertRegex(p_format2,
759 r'0x[a-zA-Z0-9]{3,} 0x[a-zA-Z0-9]{3,}')
760
761 # Extra args are ignored:
762 p_format3 = PyUnicode_FromFormat(b'%p', '123456', None, b'xyz')
763 self.assertIsInstance(p_format3, str)
764 self.assertRegex(p_format3, p_format_regex)
765
766 # Test string decode from parameter of %s using utf-8.
767 # b'\xe4\xba\xba\xe6\xb0\x91' is utf-8 encoded byte sequence of
768 # '\u4eba\u6c11'
769 check_format('repr=\u4eba\u6c11',
770 b'repr=%V', None, b'\xe4\xba\xba\xe6\xb0\x91')
771
772 #Test replace error handler.
773 check_format('repr=abc\ufffd',
774 b'repr=%V', None, b'abc\xff')
775
776 # Issue #33817: empty strings
777 check_format('',
778 b'')
779 check_format('',
780 b'%s', b'')
781
782 # test invalid format strings. these tests are just here
783 # to check for crashes and should not be considered as specifications
784 for fmt in (b'%', b'%0', b'%01', b'%.', b'%.1',
785 b'%0%s', b'%1%s', b'%.%s', b'%.1%s', b'%1abc',
786 b'%l', b'%ll', b'%z', b'%lls', b'%zs'):
787 with self.subTest(fmt=fmt):
788 self.assertRaisesRegex(SystemError, 'invalid format string',
789 PyUnicode_FromFormat, fmt, b'abc')
790 self.assertRaisesRegex(SystemError, 'invalid format string',
791 PyUnicode_FromFormat, b'%+i', c_int(10))
792
793 @support.cpython_only
794 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
795 def test_interninplace(self):
796 """Test PyUnicode_InternInPlace()"""
797 from _testlimitedcapi import unicode_interninplace as interninplace
798
799 s = b'abc'.decode()
800 r = interninplace(s)
801 self.assertEqual(r, 'abc')
802
803 # CRASHES interninplace(b'abc')
804 # CRASHES interninplace(NULL)
805
806 @support.cpython_only
807 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
808 def test_internfromstring(self):
809 """Test PyUnicode_InternFromString()"""
810 from _testlimitedcapi import unicode_internfromstring as internfromstring
811
812 self.assertEqual(internfromstring(b'abc'), 'abc')
813 self.assertEqual(internfromstring(b'\xf0\x9f\x98\x80'), '\U0001f600')
814 self.assertRaises(UnicodeDecodeError, internfromstring, b'\xc2')
815 self.assertRaises(UnicodeDecodeError, internfromstring, b'\xa1')
816 self.assertEqual(internfromstring(b''), '')
817
818 # CRASHES internfromstring(NULL)
819
820 @support.cpython_only
821 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
822 def test_fromwidechar(self):
823 """Test PyUnicode_FromWideChar()"""
824 from _testlimitedcapi import unicode_fromwidechar as fromwidechar
825 from _testcapi import SIZEOF_WCHAR_T
826
827 if SIZEOF_WCHAR_T == 2:
828 encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be'
829 elif SIZEOF_WCHAR_T == 4:
830 encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be'
831
832 for s in '', 'abc', '\xa1\xa2', '\u4f60', '\U0001f600':
833 b = s.encode(encoding)
834 self.assertEqual(fromwidechar(b), s)
835 self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s)
836 for s in '\ud83d', '\ude00':
837 b = s.encode(encoding, 'surrogatepass')
838 self.assertEqual(fromwidechar(b), s)
839 self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s)
840
841 self.assertEqual(fromwidechar('abc'.encode(encoding), 2), 'ab')
842 if SIZEOF_WCHAR_T == 2:
843 self.assertEqual(fromwidechar('a\U0001f600'.encode(encoding), 2), 'a\ud83d')
844
845 self.assertRaises(MemoryError, fromwidechar, b'', PY_SSIZE_T_MAX)
846 self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, -2)
847 self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, PY_SSIZE_T_MIN)
848 self.assertEqual(fromwidechar(NULL, 0), '')
849 self.assertRaises(SystemError, fromwidechar, NULL, 1)
850 self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MAX)
851 self.assertRaises(SystemError, fromwidechar, NULL, -1)
852 self.assertRaises(SystemError, fromwidechar, NULL, -2)
853 self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MIN)
854
855 @support.cpython_only
856 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
857 def test_aswidechar(self):
858 """Test PyUnicode_AsWideChar()"""
859 from _testlimitedcapi import unicode_aswidechar
860 from _testlimitedcapi import unicode_aswidechar_null
861 from _testcapi import SIZEOF_WCHAR_T
862
863 wchar, size = unicode_aswidechar('abcdef', 2)
864 self.assertEqual(size, 2)
865 self.assertEqual(wchar, 'ab')
866
867 wchar, size = unicode_aswidechar('abc', 3)
868 self.assertEqual(size, 3)
869 self.assertEqual(wchar, 'abc')
870 self.assertEqual(unicode_aswidechar_null('abc', 10), 4)
871 self.assertEqual(unicode_aswidechar_null('abc', 0), 4)
872
873 wchar, size = unicode_aswidechar('abc', 4)
874 self.assertEqual(size, 3)
875 self.assertEqual(wchar, 'abc\0')
876
877 wchar, size = unicode_aswidechar('abc', 10)
878 self.assertEqual(size, 3)
879 self.assertEqual(wchar, 'abc\0')
880
881 wchar, size = unicode_aswidechar('abc\0def', 20)
882 self.assertEqual(size, 7)
883 self.assertEqual(wchar, 'abc\0def\0')
884 self.assertEqual(unicode_aswidechar_null('abc\0def', 20), 8)
885
886 nonbmp = chr(0x10ffff)
887 if SIZEOF_WCHAR_T == 2:
888 nchar = 2
889 else: # SIZEOF_WCHAR_T == 4
890 nchar = 1
891 wchar, size = unicode_aswidechar(nonbmp, 10)
892 self.assertEqual(size, nchar)
893 self.assertEqual(wchar, nonbmp + '\0')
894 self.assertEqual(unicode_aswidechar_null(nonbmp, 10), nchar + 1)
895
896 self.assertRaises(TypeError, unicode_aswidechar, b'abc', 10)
897 self.assertRaises(TypeError, unicode_aswidechar, [], 10)
898 self.assertRaises(SystemError, unicode_aswidechar, NULL, 10)
899 self.assertRaises(TypeError, unicode_aswidechar_null, b'abc', 10)
900 self.assertRaises(TypeError, unicode_aswidechar_null, [], 10)
901 self.assertRaises(SystemError, unicode_aswidechar_null, NULL, 10)
902
903 @support.cpython_only
904 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
905 def test_aswidecharstring(self):
906 """Test PyUnicode_AsWideCharString()"""
907 from _testlimitedcapi import unicode_aswidecharstring
908 from _testlimitedcapi import unicode_aswidecharstring_null
909 from _testcapi import SIZEOF_WCHAR_T
910
911 wchar, size = unicode_aswidecharstring('abc')
912 self.assertEqual(size, 3)
913 self.assertEqual(wchar, 'abc\0')
914 self.assertEqual(unicode_aswidecharstring_null('abc'), 'abc')
915
916 wchar, size = unicode_aswidecharstring('abc\0def')
917 self.assertEqual(size, 7)
918 self.assertEqual(wchar, 'abc\0def\0')
919 self.assertRaises(ValueError, unicode_aswidecharstring_null, 'abc\0def')
920
921 nonbmp = chr(0x10ffff)
922 if SIZEOF_WCHAR_T == 2:
923 nchar = 2
924 else: # SIZEOF_WCHAR_T == 4
925 nchar = 1
926 wchar, size = unicode_aswidecharstring(nonbmp)
927 self.assertEqual(size, nchar)
928 self.assertEqual(wchar, nonbmp + '\0')
929 self.assertEqual(unicode_aswidecharstring_null(nonbmp), nonbmp)
930
931 self.assertRaises(TypeError, unicode_aswidecharstring, b'abc')
932 self.assertRaises(TypeError, unicode_aswidecharstring, [])
933 self.assertRaises(SystemError, unicode_aswidecharstring, NULL)
934 self.assertRaises(TypeError, unicode_aswidecharstring_null, b'abc')
935 self.assertRaises(TypeError, unicode_aswidecharstring_null, [])
936 self.assertRaises(SystemError, unicode_aswidecharstring_null, NULL)
937
938 @support.cpython_only
939 @unittest.skipIf(_testcapi is None, 'need _testcapi module')
940 def test_asucs4(self):
941 """Test PyUnicode_AsUCS4()"""
942 from _testcapi import unicode_asucs4
943
944 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600',
945 'a\ud800b\udfffc', '\ud834\udd1e']:
946 l = len(s)
947 self.assertEqual(unicode_asucs4(s, l, 1), s+'\0')
948 self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff')
949 self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff')
950 self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff')
951 self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1)
952 self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0)
953 s = '\0'.join([s, s])
954 self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0')
955 self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff')
956
957 # CRASHES unicode_asucs4(b'abc', 1, 0)
958 # CRASHES unicode_asucs4(b'abc', 1, 1)
959 # CRASHES unicode_asucs4([], 1, 1)
960 # CRASHES unicode_asucs4(NULL, 1, 0)
961 # CRASHES unicode_asucs4(NULL, 1, 1)
962
963 @support.cpython_only
964 @unittest.skipIf(_testcapi is None, 'need _testcapi module')
965 def test_asucs4copy(self):
966 """Test PyUnicode_AsUCS4Copy()"""
967 from _testcapi import unicode_asucs4copy as asucs4copy
968
969 for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600',
970 'a\ud800b\udfffc', '\ud834\udd1e']:
971 self.assertEqual(asucs4copy(s), s+'\0')
972 s = '\0'.join([s, s])
973 self.assertEqual(asucs4copy(s), s+'\0')
974
975 # CRASHES asucs4copy(b'abc')
976 # CRASHES asucs4copy([])
977 # CRASHES asucs4copy(NULL)
978
979 @support.cpython_only
980 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
981 def test_fromordinal(self):
982 """Test PyUnicode_FromOrdinal()"""
983 from _testlimitedcapi import unicode_fromordinal as fromordinal
984
985 self.assertEqual(fromordinal(0x61), 'a')
986 self.assertEqual(fromordinal(0x20ac), '\u20ac')
987 self.assertEqual(fromordinal(0x1f600), '\U0001f600')
988
989 self.assertRaises(ValueError, fromordinal, 0x110000)
990 self.assertRaises(ValueError, fromordinal, -1)
991
992 @support.cpython_only
993 @unittest.skipIf(_testcapi is None, 'need _testcapi module')
994 def test_asutf8(self):
995 """Test PyUnicode_AsUTF8()"""
996 from _testcapi import unicode_asutf8
997
998 self.assertEqual(unicode_asutf8('abc', 4), b'abc\0')
999 self.assertEqual(unicode_asutf8('абв', 7), b'\xd0\xb0\xd0\xb1\xd0\xb2\0')
1000 self.assertEqual(unicode_asutf8('\U0001f600', 5), b'\xf0\x9f\x98\x80\0')
1001 self.assertEqual(unicode_asutf8('abc\0def', 8), b'abc\0def\0')
1002
1003 self.assertRaises(UnicodeEncodeError, unicode_asutf8, '\ud8ff', 0)
1004 self.assertRaises(TypeError, unicode_asutf8, b'abc', 0)
1005 self.assertRaises(TypeError, unicode_asutf8, [], 0)
1006 # CRASHES unicode_asutf8(NULL, 0)
1007
1008 @unittest.skipIf(_testcapi is None, 'need _testcapi module')
1009 @threading_helper.requires_working_threading()
1010 def test_asutf8_race(self):
1011 """Test that there's no race condition in PyUnicode_AsUTF8()"""
1012 unicode_asutf8 = _testcapi.unicode_asutf8
1013 from threading import Thread
1014
1015 data = "😊"
1016
1017 def worker():
1018 for _ in range(1000):
1019 self.assertEqual(unicode_asutf8(data, 5), b'\xf0\x9f\x98\x8a\0')
1020
1021 threads = [Thread(target=worker) for _ in range(10)]
1022 with threading_helper.start_threads(threads):
1023 pass
1024
1025
1026 @support.cpython_only
1027 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1028 def test_asutf8andsize(self):
1029 """Test PyUnicode_AsUTF8AndSize()"""
1030 from _testlimitedcapi import unicode_asutf8andsize
1031 from _testlimitedcapi import unicode_asutf8andsize_null
1032
1033 self.assertEqual(unicode_asutf8andsize('abc', 4), (b'abc\0', 3))
1034 self.assertEqual(unicode_asutf8andsize('абв', 7), (b'\xd0\xb0\xd0\xb1\xd0\xb2\0', 6))
1035 self.assertEqual(unicode_asutf8andsize('\U0001f600', 5), (b'\xf0\x9f\x98\x80\0', 4))
1036 self.assertEqual(unicode_asutf8andsize('abc\0def', 8), (b'abc\0def\0', 7))
1037 self.assertEqual(unicode_asutf8andsize_null('abc', 4), b'abc\0')
1038 self.assertEqual(unicode_asutf8andsize_null('abc\0def', 8), b'abc\0def\0')
1039
1040 self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, '\ud8ff', 0)
1041 self.assertRaises(TypeError, unicode_asutf8andsize, b'abc', 0)
1042 self.assertRaises(TypeError, unicode_asutf8andsize, [], 0)
1043 self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize_null, '\ud8ff', 0)
1044 self.assertRaises(TypeError, unicode_asutf8andsize_null, b'abc', 0)
1045 self.assertRaises(TypeError, unicode_asutf8andsize_null, [], 0)
1046 # CRASHES unicode_asutf8andsize(NULL, 0)
1047 # CRASHES unicode_asutf8andsize_null(NULL, 0)
1048
1049 @support.cpython_only
1050 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1051 def test_getdefaultencoding(self):
1052 """Test PyUnicode_GetDefaultEncoding()"""
1053 from _testlimitedcapi import unicode_getdefaultencoding as getdefaultencoding
1054
1055 self.assertEqual(getdefaultencoding(), b'utf-8')
1056
1057 @support.cpython_only
1058 @unittest.skipIf(_testinternalcapi is None, 'need _testinternalcapi module')
1059 def test_transform_decimal_and_space(self):
1060 """Test _PyUnicode_TransformDecimalAndSpaceToASCII()"""
1061 from _testinternalcapi import _PyUnicode_TransformDecimalAndSpaceToASCII as transform_decimal
1062
1063 self.assertEqual(transform_decimal('123'),
1064 '123')
1065 self.assertEqual(transform_decimal('\u0663.\u0661\u0664'),
1066 '3.14')
1067 self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"),
1068 " 3.14 ")
1069 self.assertEqual(transform_decimal('12\u20ac3'),
1070 '12?')
1071 self.assertEqual(transform_decimal(''), '')
1072
1073 self.assertRaises(SystemError, transform_decimal, b'123')
1074 self.assertRaises(SystemError, transform_decimal, [])
1075 # CRASHES transform_decimal(NULL)
1076
1077 @support.cpython_only
1078 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1079 def test_concat(self):
1080 """Test PyUnicode_Concat()"""
1081 from _testlimitedcapi import unicode_concat as concat
1082
1083 self.assertEqual(concat('abc', 'def'), 'abcdef')
1084 self.assertEqual(concat('abc', 'где'), 'abcгде')
1085 self.assertEqual(concat('абв', 'def'), 'абвdef')
1086 self.assertEqual(concat('абв', 'где'), 'абвгде')
1087 self.assertEqual(concat('a\0b', 'c\0d'), 'a\0bc\0d')
1088
1089 self.assertRaises(TypeError, concat, b'abc', 'def')
1090 self.assertRaises(TypeError, concat, 'abc', b'def')
1091 self.assertRaises(TypeError, concat, b'abc', b'def')
1092 self.assertRaises(TypeError, concat, [], 'def')
1093 self.assertRaises(TypeError, concat, 'abc', [])
1094 self.assertRaises(TypeError, concat, [], [])
1095 # CRASHES concat(NULL, 'def')
1096 # CRASHES concat('abc', NULL)
1097
1098 @support.cpython_only
1099 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1100 def test_split(self):
1101 """Test PyUnicode_Split()"""
1102 from _testlimitedcapi import unicode_split as split
1103
1104 self.assertEqual(split('a|b|c|d', '|'), ['a', 'b', 'c', 'd'])
1105 self.assertEqual(split('a|b|c|d', '|', 2), ['a', 'b', 'c|d'])
1106 self.assertEqual(split('a|b|c|d', '|', PY_SSIZE_T_MAX),
1107 ['a', 'b', 'c', 'd'])
1108 self.assertEqual(split('a|b|c|d', '|', -1), ['a', 'b', 'c', 'd'])
1109 self.assertEqual(split('a|b|c|d', '|', PY_SSIZE_T_MIN),
1110 ['a', 'b', 'c', 'd'])
1111 self.assertEqual(split('a|b|c|d', '\u20ac'), ['a|b|c|d'])
1112 self.assertEqual(split('a||b|c||d', '||'), ['a', 'b|c', 'd'])
1113 self.assertEqual(split('а|б|в|г', '|'), ['а', 'б', 'в', 'г'])
1114 self.assertEqual(split('абабагаламага', 'а'),
1115 ['', 'б', 'б', 'г', 'л', 'м', 'г', ''])
1116 self.assertEqual(split(' a\tb\nc\rd\ve\f', NULL),
1117 ['a', 'b', 'c', 'd', 'e'])
1118 self.assertEqual(split('a\x85b\xa0c\u1680d\u2000e', NULL),
1119 ['a', 'b', 'c', 'd', 'e'])
1120
1121 self.assertRaises(ValueError, split, 'a|b|c|d', '')
1122 self.assertRaises(TypeError, split, 'a|b|c|d', ord('|'))
1123 self.assertRaises(TypeError, split, [], '|')
1124 # CRASHES split(NULL, '|')
1125
1126 @support.cpython_only
1127 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1128 def test_rsplit(self):
1129 """Test PyUnicode_RSplit()"""
1130 from _testlimitedcapi import unicode_rsplit as rsplit
1131
1132 self.assertEqual(rsplit('a|b|c|d', '|'), ['a', 'b', 'c', 'd'])
1133 self.assertEqual(rsplit('a|b|c|d', '|', 2), ['a|b', 'c', 'd'])
1134 self.assertEqual(rsplit('a|b|c|d', '|', PY_SSIZE_T_MAX),
1135 ['a', 'b', 'c', 'd'])
1136 self.assertEqual(rsplit('a|b|c|d', '|', -1), ['a', 'b', 'c', 'd'])
1137 self.assertEqual(rsplit('a|b|c|d', '|', PY_SSIZE_T_MIN),
1138 ['a', 'b', 'c', 'd'])
1139 self.assertEqual(rsplit('a|b|c|d', '\u20ac'), ['a|b|c|d'])
1140 self.assertEqual(rsplit('a||b|c||d', '||'), ['a', 'b|c', 'd'])
1141 self.assertEqual(rsplit('а|б|в|г', '|'), ['а', 'б', 'в', 'г'])
1142 self.assertEqual(rsplit('абабагаламага', 'а'),
1143 ['', 'б', 'б', 'г', 'л', 'м', 'г', ''])
1144 self.assertEqual(rsplit('aжbжcжd', 'ж'), ['a', 'b', 'c', 'd'])
1145 self.assertEqual(rsplit(' a\tb\nc\rd\ve\f', NULL),
1146 ['a', 'b', 'c', 'd', 'e'])
1147 self.assertEqual(rsplit('a\x85b\xa0c\u1680d\u2000e', NULL),
1148 ['a', 'b', 'c', 'd', 'e'])
1149
1150 self.assertRaises(ValueError, rsplit, 'a|b|c|d', '')
1151 self.assertRaises(TypeError, rsplit, 'a|b|c|d', ord('|'))
1152 self.assertRaises(TypeError, rsplit, [], '|')
1153 # CRASHES rsplit(NULL, '|')
1154
1155 @support.cpython_only
1156 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1157 def test_partition(self):
1158 """Test PyUnicode_Partition()"""
1159 from _testlimitedcapi import unicode_partition as partition
1160
1161 self.assertEqual(partition('a|b|c', '|'), ('a', '|', 'b|c'))
1162 self.assertEqual(partition('a||b||c', '||'), ('a', '||', 'b||c'))
1163 self.assertEqual(partition('а|б|в', '|'), ('а', '|', 'б|в'))
1164 self.assertEqual(partition('кабан', 'а'), ('к', 'а', 'бан'))
1165 self.assertEqual(partition('aжbжc', 'ж'), ('a', 'ж', 'bжc'))
1166
1167 self.assertRaises(ValueError, partition, 'a|b|c', '')
1168 self.assertRaises(TypeError, partition, b'a|b|c', '|')
1169 self.assertRaises(TypeError, partition, 'a|b|c', b'|')
1170 self.assertRaises(TypeError, partition, 'a|b|c', ord('|'))
1171 self.assertRaises(TypeError, partition, [], '|')
1172 # CRASHES partition(NULL, '|')
1173 # CRASHES partition('a|b|c', NULL)
1174
1175 @support.cpython_only
1176 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1177 def test_rpartition(self):
1178 """Test PyUnicode_RPartition()"""
1179 from _testlimitedcapi import unicode_rpartition as rpartition
1180
1181 self.assertEqual(rpartition('a|b|c', '|'), ('a|b', '|', 'c'))
1182 self.assertEqual(rpartition('a||b||c', '||'), ('a||b', '||', 'c'))
1183 self.assertEqual(rpartition('а|б|в', '|'), ('а|б', '|', 'в'))
1184 self.assertEqual(rpartition('кабан', 'а'), ('каб', 'а', 'н'))
1185 self.assertEqual(rpartition('aжbжc', 'ж'), ('aжb', 'ж', 'c'))
1186
1187 self.assertRaises(ValueError, rpartition, 'a|b|c', '')
1188 self.assertRaises(TypeError, rpartition, b'a|b|c', '|')
1189 self.assertRaises(TypeError, rpartition, 'a|b|c', b'|')
1190 self.assertRaises(TypeError, rpartition, 'a|b|c', ord('|'))
1191 self.assertRaises(TypeError, rpartition, [], '|')
1192 # CRASHES rpartition(NULL, '|')
1193 # CRASHES rpartition('a|b|c', NULL)
1194
1195 @support.cpython_only
1196 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1197 def test_splitlines(self):
1198 """Test PyUnicode_SplitLines()"""
1199 from _testlimitedcapi import unicode_splitlines as splitlines
1200
1201 self.assertEqual(splitlines('a\nb\rc\r\nd'), ['a', 'b', 'c', 'd'])
1202 self.assertEqual(splitlines('a\nb\rc\r\nd', True),
1203 ['a\n', 'b\r', 'c\r\n', 'd'])
1204 self.assertEqual(splitlines('a\x85b\u2028c\u2029d'),
1205 ['a', 'b', 'c', 'd'])
1206 self.assertEqual(splitlines('a\x85b\u2028c\u2029d', True),
1207 ['a\x85', 'b\u2028', 'c\u2029', 'd'])
1208 self.assertEqual(splitlines('а\nб\rв\r\nг'), ['а', 'б', 'в', 'г'])
1209
1210 self.assertRaises(TypeError, splitlines, b'a\nb\rc\r\nd')
1211 # CRASHES splitlines(NULL)
1212
1213 @support.cpython_only
1214 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1215 def test_translate(self):
1216 """Test PyUnicode_Translate()"""
1217 from _testlimitedcapi import unicode_translate as translate
1218
1219 self.assertEqual(translate('abcd', {ord('a'): 'A', ord('b'): ord('B'), ord('c'): '<>'}), 'AB<>d')
1220 self.assertEqual(translate('абвг', {ord('а'): 'А', ord('б'): ord('Б'), ord('в'): '<>'}), 'АБ<>г')
1221 self.assertEqual(translate('abc', {}), 'abc')
1222 self.assertEqual(translate('abc', []), 'abc')
1223 self.assertRaises(UnicodeTranslateError, translate, 'abc', {ord('b'): None})
1224 self.assertRaises(UnicodeTranslateError, translate, 'abc', {ord('b'): None}, 'strict')
1225 self.assertRaises(LookupError, translate, 'abc', {ord('b'): None}, 'foo')
1226 self.assertEqual(translate('abc', {ord('b'): None}, 'ignore'), 'ac')
1227 self.assertEqual(translate('abc', {ord('b'): None}, 'replace'), 'a\ufffdc')
1228 self.assertEqual(translate('abc', {ord('b'): None}, 'backslashreplace'), r'a\x62c')
1229 # XXX Other error handlers do not support UnicodeTranslateError
1230 self.assertRaises(TypeError, translate, b'abc', [])
1231 self.assertRaises(TypeError, translate, 123, [])
1232 self.assertRaises(TypeError, translate, 'abc', {ord('a'): b'A'})
1233 self.assertRaises(TypeError, translate, 'abc', 123)
1234 self.assertRaises(TypeError, translate, 'abc', NULL)
1235 self.assertRaises(LookupError, translate, 'abc', {ord('b'): None}, 'foo')
1236 # CRASHES translate(NULL, [])
1237
1238 @support.cpython_only
1239 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1240 def test_join(self):
1241 """Test PyUnicode_Join()"""
1242 from _testlimitedcapi import unicode_join as join
1243 self.assertEqual(join('|', ['a', 'b', 'c']), 'a|b|c')
1244 self.assertEqual(join('|', ['a', '', 'c']), 'a||c')
1245 self.assertEqual(join('', ['a', 'b', 'c']), 'abc')
1246 self.assertEqual(join(NULL, ['a', 'b', 'c']), 'a b c')
1247 self.assertEqual(join('|', ['а', 'б', 'в']), 'а|б|в')
1248 self.assertEqual(join('ж', ['а', 'б', 'в']), 'ажбжв')
1249 self.assertRaises(TypeError, join, b'|', ['a', 'b', 'c'])
1250 self.assertRaises(TypeError, join, '|', [b'a', b'b', b'c'])
1251 self.assertRaises(TypeError, join, NULL, [b'a', b'b', b'c'])
1252 self.assertRaises(TypeError, join, '|', b'123')
1253 self.assertRaises(TypeError, join, '|', 123)
1254 self.assertRaises(SystemError, join, '|', NULL)
1255
1256 @support.cpython_only
1257 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1258 def test_count(self):
1259 """Test PyUnicode_Count()"""
1260 from _testlimitedcapi import unicode_count
1261
1262 for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1":
1263 for i, ch in enumerate(str):
1264 self.assertEqual(unicode_count(str, ch, 0, len(str)), 1)
1265
1266 str = "!>_<!"
1267 self.assertEqual(unicode_count(str, 'z', 0, len(str)), 0)
1268 self.assertEqual(unicode_count(str, '', 0, len(str)), len(str)+1)
1269 # start < end
1270 self.assertEqual(unicode_count(str, '!', 1, len(str)+1), 1)
1271 self.assertEqual(unicode_count(str, '!', 1, PY_SSIZE_T_MAX), 1)
1272 # start >= end
1273 self.assertEqual(unicode_count(str, '!', 0, 0), 0)
1274 self.assertEqual(unicode_count(str, '!', len(str), 0), 0)
1275 # negative
1276 self.assertEqual(unicode_count(str, '!', -len(str), -1), 1)
1277 self.assertEqual(unicode_count(str, '!', -len(str)-1, -1), 1)
1278 self.assertEqual(unicode_count(str, '!', PY_SSIZE_T_MIN, -1), 1)
1279 # bad arguments
1280 self.assertRaises(TypeError, unicode_count, str, b'!', 0, len(str))
1281 self.assertRaises(TypeError, unicode_count, b"!>_<!", '!', 0, len(str))
1282 self.assertRaises(TypeError, unicode_count, str, ord('!'), 0, len(str))
1283 self.assertRaises(TypeError, unicode_count, [], '!', 0, len(str), 1)
1284 # CRASHES unicode_count(NULL, '!', 0, len(str))
1285 # CRASHES unicode_count(str, NULL, 0, len(str))
1286
1287 @support.cpython_only
1288 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1289 def test_tailmatch(self):
1290 """Test PyUnicode_Tailmatch()"""
1291 from _testlimitedcapi import unicode_tailmatch as tailmatch
1292
1293 str = 'ababahalamaha'
1294 self.assertEqual(tailmatch(str, 'aba', 0, len(str), -1), 1)
1295 self.assertEqual(tailmatch(str, 'aha', 0, len(str), 1), 1)
1296
1297 self.assertEqual(tailmatch(str, 'aba', 0, PY_SSIZE_T_MAX, -1), 1)
1298 self.assertEqual(tailmatch(str, 'aba', -len(str), PY_SSIZE_T_MAX, -1), 1)
1299 self.assertEqual(tailmatch(str, 'aba', PY_SSIZE_T_MIN, len(str), -1), 1)
1300 self.assertEqual(tailmatch(str, 'aha', 0, PY_SSIZE_T_MAX, 1), 1)
1301 self.assertEqual(tailmatch(str, 'aha', PY_SSIZE_T_MIN, len(str), 1), 1)
1302
1303 self.assertEqual(tailmatch(str, 'z', 0, len(str), 1), 0)
1304 self.assertEqual(tailmatch(str, 'z', 0, len(str), -1), 0)
1305 self.assertEqual(tailmatch(str, '', 0, len(str), 1), 1)
1306 self.assertEqual(tailmatch(str, '', 0, len(str), -1), 1)
1307
1308 self.assertEqual(tailmatch(str, 'ba', 0, len(str)-1, -1), 0)
1309 self.assertEqual(tailmatch(str, 'ba', 1, len(str)-1, -1), 1)
1310 self.assertEqual(tailmatch(str, 'aba', 1, len(str)-1, -1), 0)
1311 self.assertEqual(tailmatch(str, 'ba', -len(str)+1, -1, -1), 1)
1312 self.assertEqual(tailmatch(str, 'ah', 0, len(str), 1), 0)
1313 self.assertEqual(tailmatch(str, 'ah', 0, len(str)-1, 1), 1)
1314 self.assertEqual(tailmatch(str, 'ah', -len(str), -1, 1), 1)
1315
1316 # bad arguments
1317 self.assertRaises(TypeError, tailmatch, str, ('aba', 'aha'), 0, len(str), -1)
1318 self.assertRaises(TypeError, tailmatch, str, ('aba', 'aha'), 0, len(str), 1)
1319 # CRASHES tailmatch(NULL, 'aba', 0, len(str), -1)
1320 # CRASHES tailmatch(str, NULL, 0, len(str), -1)
1321
1322 @support.cpython_only
1323 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1324 def test_find(self):
1325 """Test PyUnicode_Find()"""
1326 from _testlimitedcapi import unicode_find as find
1327
1328 for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1":
1329 for i, ch in enumerate(str):
1330 self.assertEqual(find(str, ch, 0, len(str), 1), i)
1331 self.assertEqual(find(str, ch, 0, len(str), -1), i)
1332
1333 str = "!>_<!"
1334 self.assertEqual(find(str, 'z', 0, len(str), 1), -1)
1335 self.assertEqual(find(str, 'z', 0, len(str), -1), -1)
1336 self.assertEqual(find(str, '', 0, len(str), 1), 0)
1337 self.assertEqual(find(str, '', 0, len(str), -1), len(str))
1338 # start < end
1339 self.assertEqual(find(str, '!', 1, len(str)+1, 1), 4)
1340 self.assertEqual(find(str, '!', 1, PY_SSIZE_T_MAX, 1), 4)
1341 self.assertEqual(find(str, '!', 0, len(str)+1, -1), 4)
1342 self.assertEqual(find(str, '!', 0, PY_SSIZE_T_MAX, -1), 4)
1343 # start >= end
1344 self.assertEqual(find(str, '!', 0, 0, 1), -1)
1345 self.assertEqual(find(str, '!', 0, 0, -1), -1)
1346 self.assertEqual(find(str, '!', len(str), 0, 1), -1)
1347 self.assertEqual(find(str, '!', len(str), 0, -1), -1)
1348 # negative
1349 self.assertEqual(find(str, '!', -len(str), -1, 1), 0)
1350 self.assertEqual(find(str, '!', -len(str), -1, -1), 0)
1351 self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, -1, 1), 0)
1352 self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, -1, -1), 0)
1353 self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, 1), 0)
1354 self.assertEqual(find(str, '!', PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1), 4)
1355 # bad arguments
1356 self.assertRaises(TypeError, find, str, b'!', 0, len(str), 1)
1357 self.assertRaises(TypeError, find, b"!>_<!", '!', 0, len(str), 1)
1358 self.assertRaises(TypeError, find, str, ord('!'), 0, len(str), 1)
1359 self.assertRaises(TypeError, find, [], '!', 0, len(str), 1)
1360 # CRASHES find(NULL, '!', 0, len(str), 1)
1361 # CRASHES find(str, NULL, 0, len(str), 1)
1362
1363 @support.cpython_only
1364 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1365 def test_findchar(self):
1366 """Test PyUnicode_FindChar()"""
1367 from _testlimitedcapi import unicode_findchar
1368
1369 for str in "\xa1", "\u8000\u8080", "\ud800\udc02", "\U0001f100\U0001f1f1":
1370 for i, ch in enumerate(str):
1371 self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), 1), i)
1372 self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i)
1373
1374 str = "!>_<!"
1375 self.assertEqual(unicode_findchar(str, 0x110000, 0, len(str), 1), -1)
1376 self.assertEqual(unicode_findchar(str, 0x110000, 0, len(str), -1), -1)
1377 # start < end
1378 self.assertEqual(unicode_findchar(str, ord('!'), 1, len(str)+1, 1), 4)
1379 self.assertEqual(unicode_findchar(str, ord('!'), 1, PY_SSIZE_T_MAX, 1), 4)
1380 self.assertEqual(unicode_findchar(str, ord('!'), 0, len(str)+1, -1), 4)
1381 self.assertEqual(unicode_findchar(str, ord('!'), 0, PY_SSIZE_T_MAX, -1), 4)
1382 # start >= end
1383 self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, 1), -1)
1384 self.assertEqual(unicode_findchar(str, ord('!'), 0, 0, -1), -1)
1385 self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, 1), -1)
1386 self.assertEqual(unicode_findchar(str, ord('!'), len(str), 0, -1), -1)
1387 # negative
1388 self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, 1), 0)
1389 self.assertEqual(unicode_findchar(str, ord('!'), -len(str), -1, -1), 0)
1390 self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, -1, 1), 0)
1391 self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, -1, -1), 0)
1392 self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, 1), 0)
1393 self.assertEqual(unicode_findchar(str, ord('!'), PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, -1), 4)
1394 # bad arguments
1395 # CRASHES unicode_findchar(b"!>_<!", ord('!'), 0, len(str), 1)
1396 # CRASHES unicode_findchar([], ord('!'), 0, len(str), 1)
1397 # CRASHES unicode_findchar(NULL, ord('!'), 0, len(str), 1), 1)
1398
1399 @support.cpython_only
1400 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1401 def test_replace(self):
1402 """Test PyUnicode_Replace()"""
1403 from _testlimitedcapi import unicode_replace as replace
1404
1405 str = 'abracadabra'
1406 self.assertEqual(replace(str, 'a', '='), '=br=c=d=br=')
1407 self.assertEqual(replace(str, 'a', '<>'), '<>br<>c<>d<>br<>')
1408 self.assertEqual(replace(str, 'abra', '='), '=cad=')
1409 self.assertEqual(replace(str, 'a', '=', 2), '=br=cadabra')
1410 self.assertEqual(replace(str, 'a', '=', 0), str)
1411 self.assertEqual(replace(str, 'a', '=', PY_SSIZE_T_MAX), '=br=c=d=br=')
1412 self.assertEqual(replace(str, 'a', '=', -1), '=br=c=d=br=')
1413 self.assertEqual(replace(str, 'a', '=', PY_SSIZE_T_MIN), '=br=c=d=br=')
1414 self.assertEqual(replace(str, 'z', '='), str)
1415 self.assertEqual(replace(str, '', '='), '=a=b=r=a=c=a=d=a=b=r=a=')
1416 self.assertEqual(replace(str, 'a', 'ж'), 'жbrжcжdжbrж')
1417 self.assertEqual(replace('абабагаламага', 'а', '='), '=б=б=г=л=м=г=')
1418 self.assertEqual(replace('Баден-Баден', 'Баден', 'Baden'), 'Baden-Baden')
1419 # bad arguments
1420 self.assertRaises(TypeError, replace, 'a', 'a', b'=')
1421 self.assertRaises(TypeError, replace, 'a', b'a', '=')
1422 self.assertRaises(TypeError, replace, b'a', 'a', '=')
1423 self.assertRaises(TypeError, replace, 'a', 'a', ord('='))
1424 self.assertRaises(TypeError, replace, 'a', ord('a'), '=')
1425 self.assertRaises(TypeError, replace, [], 'a', '=')
1426 # CRASHES replace('a', 'a', NULL)
1427 # CRASHES replace('a', NULL, '=')
1428 # CRASHES replace(NULL, 'a', '=')
1429
1430 @support.cpython_only
1431 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1432 def test_compare(self):
1433 """Test PyUnicode_Compare()"""
1434 from _testlimitedcapi import unicode_compare as compare
1435
1436 self.assertEqual(compare('abc', 'abc'), 0)
1437 self.assertEqual(compare('abc', 'def'), -1)
1438 self.assertEqual(compare('def', 'abc'), 1)
1439 self.assertEqual(compare('abc', 'abc\0def'), -1)
1440 self.assertEqual(compare('abc\0def', 'abc\0def'), 0)
1441 self.assertEqual(compare('абв', 'abc'), 1)
1442
1443 self.assertRaises(TypeError, compare, b'abc', 'abc')
1444 self.assertRaises(TypeError, compare, 'abc', b'abc')
1445 self.assertRaises(TypeError, compare, b'abc', b'abc')
1446 self.assertRaises(TypeError, compare, [], 'abc')
1447 self.assertRaises(TypeError, compare, 'abc', [])
1448 self.assertRaises(TypeError, compare, [], [])
1449 # CRASHES compare(NULL, 'abc')
1450 # CRASHES compare('abc', NULL)
1451
1452 @support.cpython_only
1453 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1454 def test_comparewithasciistring(self):
1455 """Test PyUnicode_CompareWithASCIIString()"""
1456 from _testlimitedcapi import unicode_comparewithasciistring as comparewithasciistring
1457
1458 self.assertEqual(comparewithasciistring('abc', b'abc'), 0)
1459 self.assertEqual(comparewithasciistring('abc', b'def'), -1)
1460 self.assertEqual(comparewithasciistring('def', b'abc'), 1)
1461 self.assertEqual(comparewithasciistring('abc', b'abc\0def'), 0)
1462 self.assertEqual(comparewithasciistring('abc\0def', b'abc\0def'), 1)
1463 self.assertEqual(comparewithasciistring('абв', b'abc'), 1)
1464
1465 # CRASHES comparewithasciistring(b'abc', b'abc')
1466 # CRASHES comparewithasciistring([], b'abc')
1467 # CRASHES comparewithasciistring(NULL, b'abc')
1468
1469 @support.cpython_only
1470 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1471 def test_equaltoutf8(self):
1472 # Test PyUnicode_EqualToUTF8()
1473 from _testlimitedcapi import unicode_equaltoutf8 as equaltoutf8
1474 from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize
1475
1476 strings = [
1477 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16',
1478 '\U0001f600\U0001f601\U0001f602',
1479 '\U0010ffff',
1480 ]
1481 for s in strings:
1482 # Call PyUnicode_AsUTF8AndSize() which creates the UTF-8
1483 # encoded string cached in the Unicode object.
1484 asutf8andsize(s, 0)
1485 b = s.encode()
1486 self.assertEqual(equaltoutf8(s, b), 1) # Use the UTF-8 cache.
1487 s2 = b.decode() # New Unicode object without the UTF-8 cache.
1488 self.assertEqual(equaltoutf8(s2, b), 1)
1489 self.assertEqual(equaltoutf8(s + 'x', b + b'x'), 1)
1490 self.assertEqual(equaltoutf8(s + 'x', b + b'y'), 0)
1491 self.assertEqual(equaltoutf8(s, b + b'\0'), 1)
1492 self.assertEqual(equaltoutf8(s2, b + b'\0'), 1)
1493 self.assertEqual(equaltoutf8(s + '\0', b + b'\0'), 0)
1494 self.assertEqual(equaltoutf8(s + '\0', b), 0)
1495 self.assertEqual(equaltoutf8(s2, b + b'x'), 0)
1496 self.assertEqual(equaltoutf8(s2, b[:-1]), 0)
1497 self.assertEqual(equaltoutf8(s2, b[:-1] + b'x'), 0)
1498
1499 self.assertEqual(equaltoutf8('', b''), 1)
1500 self.assertEqual(equaltoutf8('', b'\0'), 1)
1501
1502 # embedded null chars/bytes
1503 self.assertEqual(equaltoutf8('abc', b'abc\0def\0'), 1)
1504 self.assertEqual(equaltoutf8('a\0bc', b'abc'), 0)
1505 self.assertEqual(equaltoutf8('abc', b'a\0bc'), 0)
1506
1507 # Surrogate characters are always treated as not equal
1508 self.assertEqual(equaltoutf8('\udcfe',
1509 '\udcfe'.encode("utf8", "surrogateescape")), 0)
1510 self.assertEqual(equaltoutf8('\udcfe',
1511 '\udcfe'.encode("utf8", "surrogatepass")), 0)
1512 self.assertEqual(equaltoutf8('\ud801',
1513 '\ud801'.encode("utf8", "surrogatepass")), 0)
1514
1515 @support.cpython_only
1516 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1517 def test_equaltoutf8andsize(self):
1518 # Test PyUnicode_EqualToUTF8AndSize()
1519 from _testlimitedcapi import unicode_equaltoutf8andsize as equaltoutf8andsize
1520 from _testlimitedcapi import unicode_asutf8andsize as asutf8andsize
1521
1522 strings = [
1523 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16',
1524 '\U0001f600\U0001f601\U0001f602',
1525 '\U0010ffff',
1526 ]
1527 for s in strings:
1528 # Call PyUnicode_AsUTF8AndSize() which creates the UTF-8
1529 # encoded string cached in the Unicode object.
1530 asutf8andsize(s, 0)
1531 b = s.encode()
1532 self.assertEqual(equaltoutf8andsize(s, b), 1) # Use the UTF-8 cache.
1533 s2 = b.decode() # New Unicode object without the UTF-8 cache.
1534 self.assertEqual(equaltoutf8andsize(s2, b), 1)
1535 self.assertEqual(equaltoutf8andsize(s + 'x', b + b'x'), 1)
1536 self.assertEqual(equaltoutf8andsize(s + 'x', b + b'y'), 0)
1537 self.assertEqual(equaltoutf8andsize(s, b + b'\0'), 0)
1538 self.assertEqual(equaltoutf8andsize(s2, b + b'\0'), 0)
1539 self.assertEqual(equaltoutf8andsize(s + '\0', b + b'\0'), 1)
1540 self.assertEqual(equaltoutf8andsize(s + '\0', b), 0)
1541 self.assertEqual(equaltoutf8andsize(s2, b + b'x'), 0)
1542 self.assertEqual(equaltoutf8andsize(s2, b[:-1]), 0)
1543 self.assertEqual(equaltoutf8andsize(s2, b[:-1] + b'x'), 0)
1544 # Not null-terminated,
1545 self.assertEqual(equaltoutf8andsize(s, b + b'x', len(b)), 1)
1546 self.assertEqual(equaltoutf8andsize(s2, b + b'x', len(b)), 1)
1547 self.assertEqual(equaltoutf8andsize(s + '\0', b + b'\0x', len(b) + 1), 1)
1548 self.assertEqual(equaltoutf8andsize(s2, b, len(b) - 1), 0)
1549 self.assertEqual(equaltoutf8andsize(s, b, -1), 0)
1550 self.assertEqual(equaltoutf8andsize(s, b, PY_SSIZE_T_MAX), 0)
1551 self.assertEqual(equaltoutf8andsize(s, b, PY_SSIZE_T_MIN), 0)
1552
1553 self.assertEqual(equaltoutf8andsize('', b''), 1)
1554 self.assertEqual(equaltoutf8andsize('', b'\0'), 0)
1555 self.assertEqual(equaltoutf8andsize('', b'x', 0), 1)
1556
1557 # embedded null chars/bytes
1558 self.assertEqual(equaltoutf8andsize('abc\0def', b'abc\0def'), 1)
1559 self.assertEqual(equaltoutf8andsize('abc\0def\0', b'abc\0def\0'), 1)
1560
1561 # Surrogate characters are always treated as not equal
1562 self.assertEqual(equaltoutf8andsize('\udcfe',
1563 '\udcfe'.encode("utf8", "surrogateescape")), 0)
1564 self.assertEqual(equaltoutf8andsize('\udcfe',
1565 '\udcfe'.encode("utf8", "surrogatepass")), 0)
1566 self.assertEqual(equaltoutf8andsize('\ud801',
1567 '\ud801'.encode("utf8", "surrogatepass")), 0)
1568
1569 def check_not_equal_encoding(text, encoding):
1570 self.assertEqual(equaltoutf8andsize(text, text.encode(encoding)), 0)
1571 self.assertNotEqual(text.encode(encoding), text.encode("utf8"))
1572
1573 # Strings encoded to other encodings are not equal to expected UTF8-encoding string
1574 check_not_equal_encoding('Stéphane', 'latin1')
1575 check_not_equal_encoding('Stéphane', 'utf-16-le') # embedded null characters
1576 check_not_equal_encoding('北京市', 'gbk')
1577
1578 # CRASHES equaltoutf8andsize('abc', b'abc', -1)
1579 # CRASHES equaltoutf8andsize(b'abc', b'abc')
1580 # CRASHES equaltoutf8andsize([], b'abc')
1581 # CRASHES equaltoutf8andsize(NULL, b'abc')
1582 # CRASHES equaltoutf8andsize('abc', NULL)
1583
1584 @support.cpython_only
1585 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1586 def test_richcompare(self):
1587 """Test PyUnicode_RichCompare()"""
1588 from _testlimitedcapi import unicode_richcompare as richcompare
1589
1590 LT, LE, EQ, NE, GT, GE = range(6)
1591 strings = ('abc', 'абв', '\U0001f600', 'abc\0')
1592 for s1 in strings:
1593 for s2 in strings:
1594 self.assertIs(richcompare(s1, s2, LT), s1 < s2)
1595 self.assertIs(richcompare(s1, s2, LE), s1 <= s2)
1596 self.assertIs(richcompare(s1, s2, EQ), s1 == s2)
1597 self.assertIs(richcompare(s1, s2, NE), s1 != s2)
1598 self.assertIs(richcompare(s1, s2, GT), s1 > s2)
1599 self.assertIs(richcompare(s1, s2, GE), s1 >= s2)
1600
1601 for op in LT, LE, EQ, NE, GT, GE:
1602 self.assertIs(richcompare(b'abc', 'abc', op), NotImplemented)
1603 self.assertIs(richcompare('abc', b'abc', op), NotImplemented)
1604 self.assertIs(richcompare(b'abc', b'abc', op), NotImplemented)
1605 self.assertIs(richcompare([], 'abc', op), NotImplemented)
1606 self.assertIs(richcompare('abc', [], op), NotImplemented)
1607 self.assertIs(richcompare([], [], op), NotImplemented)
1608
1609 # CRASHES richcompare(NULL, 'abc', op)
1610 # CRASHES richcompare('abc', NULL, op)
1611
1612 @support.cpython_only
1613 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1614 def test_format(self):
1615 """Test PyUnicode_Format()"""
1616 from _testlimitedcapi import unicode_format as format
1617
1618 self.assertEqual(format('x=%d!', 42), 'x=42!')
1619 self.assertEqual(format('x=%d!', (42,)), 'x=42!')
1620 self.assertEqual(format('x=%d y=%s!', (42, [])), 'x=42 y=[]!')
1621
1622 self.assertRaises(SystemError, format, 'x=%d!', NULL)
1623 self.assertRaises(SystemError, format, NULL, 42)
1624
1625 @support.cpython_only
1626 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1627 def test_contains(self):
1628 """Test PyUnicode_Contains()"""
1629 from _testlimitedcapi import unicode_contains as contains
1630
1631 self.assertEqual(contains('abcd', ''), 1)
1632 self.assertEqual(contains('abcd', 'b'), 1)
1633 self.assertEqual(contains('abcd', 'x'), 0)
1634 self.assertEqual(contains('abcd', 'ж'), 0)
1635 self.assertEqual(contains('abcd', '\0'), 0)
1636 self.assertEqual(contains('abc\0def', '\0'), 1)
1637 self.assertEqual(contains('abcd', 'bc'), 1)
1638
1639 self.assertRaises(TypeError, contains, b'abcd', 'b')
1640 self.assertRaises(TypeError, contains, 'abcd', b'b')
1641 self.assertRaises(TypeError, contains, b'abcd', b'b')
1642 self.assertRaises(TypeError, contains, [], 'b')
1643 self.assertRaises(TypeError, contains, 'abcd', ord('b'))
1644 # CRASHES contains(NULL, 'b')
1645 # CRASHES contains('abcd', NULL)
1646
1647 @support.cpython_only
1648 @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module')
1649 def test_isidentifier(self):
1650 """Test PyUnicode_IsIdentifier()"""
1651 from _testlimitedcapi import unicode_isidentifier as isidentifier
1652
1653 self.assertEqual(isidentifier("a"), 1)
1654 self.assertEqual(isidentifier("b0"), 1)
1655 self.assertEqual(isidentifier("µ"), 1)
1656 self.assertEqual(isidentifier("𝔘𝔫𝔦𝔠𝔬𝔡𝔢"), 1)
1657
1658 self.assertEqual(isidentifier(""), 0)
1659 self.assertEqual(isidentifier(" "), 0)
1660 self.assertEqual(isidentifier("["), 0)
1661 self.assertEqual(isidentifier("©"), 0)
1662 self.assertEqual(isidentifier("0"), 0)
1663 self.assertEqual(isidentifier("32M"), 0)
1664
1665 # CRASHES isidentifier(b"a")
1666 # CRASHES isidentifier([])
1667 # CRASHES isidentifier(NULL)
1668
1669 @support.cpython_only
1670 @unittest.skipIf(_testcapi is None, 'need _testcapi module')
1671 def test_copycharacters(self):
1672 """Test PyUnicode_CopyCharacters()"""
1673 from _testcapi import unicode_copycharacters
1674
1675 strings = [
1676 # all strings have exactly 5 characters
1677 'abcde', '\xa1\xa2\xa3\xa4\xa5',
1678 '\u4f60\u597d\u4e16\u754c\uff01',
1679 '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604'
1680 ]
1681
1682 for idx, from_ in enumerate(strings):
1683 # wide -> narrow: exceed maxchar limitation
1684 for to in strings[:idx]:
1685 self.assertRaises(
1686 SystemError,
1687 unicode_copycharacters, to, 0, from_, 0, 5
1688 )
1689 # same kind
1690 for from_start in range(5):
1691 self.assertEqual(
1692 unicode_copycharacters(from_, 0, from_, from_start, 5),
1693 (from_[from_start:from_start+5].ljust(5, '\0'),
1694 5-from_start)
1695 )
1696 for to_start in range(5):
1697 self.assertEqual(
1698 unicode_copycharacters(from_, to_start, from_, to_start, 5),
1699 (from_[to_start:to_start+5].rjust(5, '\0'),
1700 5-to_start)
1701 )
1702 # narrow -> wide
1703 # Tests omitted since this creates invalid strings.
1704
1705 s = strings[0]
1706 self.assertRaises(IndexError, unicode_copycharacters, s, 6, s, 0, 5)
1707 self.assertRaises(IndexError, unicode_copycharacters, s, PY_SSIZE_T_MAX, s, 0, 5)
1708 self.assertRaises(IndexError, unicode_copycharacters, s, -1, s, 0, 5)
1709 self.assertRaises(IndexError, unicode_copycharacters, s, PY_SSIZE_T_MIN, s, 0, 5)
1710 self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, 6, 5)
1711 self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, PY_SSIZE_T_MAX, 5)
1712 self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, -1, 5)
1713 self.assertRaises(IndexError, unicode_copycharacters, s, 0, s, PY_SSIZE_T_MIN, 5)
1714 self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5)
1715 self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, PY_SSIZE_T_MAX)
1716 self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1)
1717 self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, PY_SSIZE_T_MIN)
1718 self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0)
1719 self.assertRaises(SystemError, unicode_copycharacters, s, 0, [], 0, 0)
1720 # CRASHES unicode_copycharacters(s, 0, NULL, 0, 0)
1721 # TODO: Test PyUnicode_CopyCharacters() with non-unicode and
1722 # non-modifiable unicode as "to".
1723
1724 @support.cpython_only
1725 @unittest.skipIf(_testcapi is None, 'need _testcapi module')
1726 def test_pep393_utf8_caching_bug(self):
1727 # Issue #25709: Problem with string concatenation and utf-8 cache
1728 from _testcapi import getargs_s_hash
1729 for k in 0x24, 0xa4, 0x20ac, 0x1f40d:
1730 s = ''
1731 for i in range(5):
1732 # Due to CPython specific optimization the 's' string can be
1733 # resized in-place.
1734 s += chr(k)
1735 # Parsing with the "s#" format code calls indirectly
1736 # PyUnicode_AsUTF8AndSize() which creates the UTF-8
1737 # encoded string cached in the Unicode object.
1738 self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1))
1739 # Check that the second call returns the same result
1740 self.assertEqual(getargs_s_hash(s), chr(k).encode() * (i + 1))
1741
1742
1743 class PyUnicodeWriterTest(unittest.TestCase):
1744 def create_writer(self, size):
1745 return _testcapi.PyUnicodeWriter(size)
1746
1747 def test_basic(self):
1748 writer = self.create_writer(100)
1749
1750 # test PyUnicodeWriter_WriteUTF8()
1751 writer.write_utf8(b'var', -1)
1752
1753 # test PyUnicodeWriter_WriteChar()
1754 writer.write_char(ord('='))
1755
1756 # test PyUnicodeWriter_WriteSubstring()
1757 writer.write_substring("[long]", 1, 5)
1758 # CRASHES writer.write_substring(NULL, 0, 0)
1759
1760 # test PyUnicodeWriter_WriteStr()
1761 writer.write_str(" value ")
1762 # CRASHES writer.write_str(NULL)
1763
1764 # test PyUnicodeWriter_WriteRepr()
1765 writer.write_repr("repr")
1766
1767 self.assertEqual(writer.finish(),
1768 "var=long value 'repr'")
1769
1770 def test_repr_null(self):
1771 writer = self.create_writer(0)
1772 writer.write_utf8(b'var=', -1)
1773 writer.write_repr(NULL)
1774 self.assertEqual(writer.finish(),
1775 "var=<NULL>")
1776
1777 def test_write_char(self):
1778 writer = self.create_writer(0)
1779 writer.write_char(0)
1780 writer.write_char(ord('$'))
1781 writer.write_char(0x20ac)
1782 writer.write_char(0x10_ffff)
1783 self.assertRaises(ValueError, writer.write_char, 0x11_0000)
1784 self.assertRaises(ValueError, writer.write_char, 0xFFFF_FFFF)
1785 self.assertEqual(writer.finish(),
1786 "\0$\u20AC\U0010FFFF")
1787
1788 def test_utf8(self):
1789 writer = self.create_writer(0)
1790 writer.write_utf8(b"ascii", -1)
1791 writer.write_char(ord('-'))
1792 writer.write_utf8(b"latin1=\xC3\xA9", -1)
1793 writer.write_char(ord('-'))
1794 writer.write_utf8(b"euro=\xE2\x82\xAC", -1)
1795 writer.write_char(ord('.'))
1796 writer.write_utf8(NULL, 0)
1797 # CRASHES writer.write_utf8(NULL, 1)
1798 # CRASHES writer.write_utf8(NULL, -1)
1799 self.assertEqual(writer.finish(),
1800 "ascii-latin1=\xE9-euro=\u20AC.")
1801
1802 def test_ascii(self):
1803 writer = self.create_writer(0)
1804 writer.write_ascii(b"Hello ", -1)
1805 writer.write_ascii(b"", 0)
1806 writer.write_ascii(NULL, 0)
1807 # CRASHES writer.write_ascii(NULL, 1)
1808 # CRASHES writer.write_ascii(NULL, -1)
1809 writer.write_ascii(b"Python! <truncated>", 6)
1810 self.assertEqual(writer.finish(), "Hello Python")
1811
1812 def test_invalid_utf8(self):
1813 writer = self.create_writer(0)
1814 with self.assertRaises(UnicodeDecodeError):
1815 writer.write_utf8(b"invalid=\xFF", -1)
1816
1817 def test_recover_utf8_error(self):
1818 # test recovering from PyUnicodeWriter_WriteUTF8() error
1819 writer = self.create_writer(0)
1820 writer.write_utf8(b"value=", -1)
1821
1822 # write fails with an invalid string
1823 with self.assertRaises(UnicodeDecodeError):
1824 writer.write_utf8(b"invalid\xFF", -1)
1825 with self.assertRaises(UnicodeDecodeError):
1826 s = "truncated\u20AC".encode()
1827 writer.write_utf8(s, len(s) - 1)
1828
1829 # retry write with a valid string
1830 writer.write_utf8(b"valid", -1)
1831
1832 self.assertEqual(writer.finish(),
1833 "value=valid")
1834
1835 def test_decode_utf8(self):
1836 # test PyUnicodeWriter_DecodeUTF8Stateful()
1837 writer = self.create_writer(0)
1838 writer.decodeutf8stateful(b"ign\xFFore", -1, b"ignore")
1839 writer.write_char(ord('-'))
1840 writer.decodeutf8stateful(b"replace\xFF", -1, b"replace")
1841 writer.write_char(ord('-'))
1842
1843 # incomplete trailing UTF-8 sequence
1844 writer.decodeutf8stateful(b"incomplete\xC3", -1, b"replace")
1845
1846 writer.decodeutf8stateful(NULL, 0, b"replace")
1847 # CRASHES writer.decodeutf8stateful(NULL, 1, b"replace")
1848 # CRASHES writer.decodeutf8stateful(NULL, -1, b"replace")
1849 with self.assertRaises(UnicodeDecodeError):
1850 writer.decodeutf8stateful(b"default\xFF", -1, NULL)
1851
1852 self.assertEqual(writer.finish(),
1853 "ignore-replace\uFFFD-incomplete\uFFFD")
1854
1855 def test_decode_utf8_consumed(self):
1856 # test PyUnicodeWriter_DecodeUTF8Stateful() with consumed
1857 writer = self.create_writer(0)
1858
1859 # valid string
1860 consumed = writer.decodeutf8stateful(b"text", -1, b"strict", True)
1861 self.assertEqual(consumed, 4)
1862 writer.write_char(ord('-'))
1863
1864 # non-ASCII
1865 consumed = writer.decodeutf8stateful(b"\xC3\xA9-\xE2\x82\xAC", 6, b"strict", True)
1866 self.assertEqual(consumed, 6)
1867 writer.write_char(ord('-'))
1868
1869 # invalid UTF-8 (consumed is 0 on error)
1870 with self.assertRaises(UnicodeDecodeError):
1871 writer.decodeutf8stateful(b"invalid\xFF", -1, b"strict", True)
1872
1873 # ignore error handler
1874 consumed = writer.decodeutf8stateful(b"more\xFF", -1, b"ignore", True)
1875 self.assertEqual(consumed, 5)
1876 writer.write_char(ord('-'))
1877
1878 # incomplete trailing UTF-8 sequence
1879 consumed = writer.decodeutf8stateful(b"incomplete\xC3", -1, b"ignore", True)
1880 self.assertEqual(consumed, 10)
1881 writer.write_char(ord('-'))
1882
1883 consumed = writer.decodeutf8stateful(NULL, 0, b"replace", True)
1884 self.assertEqual(consumed, 0)
1885 # CRASHES writer.decodeutf8stateful(NULL, 1, b"replace", True)
1886 # CRASHES writer.decodeutf8stateful(NULL, -1, b"replace", True)
1887 consumed = writer.decodeutf8stateful(b"default\xC3", -1, NULL, True)
1888 self.assertEqual(consumed, 7)
1889
1890 self.assertEqual(writer.finish(), "text-\xE9-\u20AC-more-incomplete-default")
1891
1892 def test_widechar(self):
1893 from _testcapi import SIZEOF_WCHAR_T
1894
1895 if SIZEOF_WCHAR_T == 2:
1896 encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be'
1897 elif SIZEOF_WCHAR_T == 4:
1898 encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be'
1899
1900 writer = self.create_writer(0)
1901 writer.write_widechar("latin1=\xE9".encode(encoding))
1902 writer.write_char(ord("-"))
1903 writer.write_widechar("euro=\u20AC".encode(encoding))
1904 writer.write_char(ord("-"))
1905 writer.write_widechar("max=\U0010ffff".encode(encoding))
1906 writer.write_char(ord("-"))
1907 writer.write_widechar("zeroes=".encode(encoding).ljust(SIZEOF_WCHAR_T * 10, b'\0'),
1908 10)
1909 writer.write_char(ord('.'))
1910
1911 if SIZEOF_WCHAR_T == 4:
1912 invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else
1913 b'\x00\x11\x00\x00')
1914 with self.assertRaises(ValueError):
1915 writer.write_widechar("invalid=".encode(encoding) + invalid)
1916 writer.write_widechar(b'', -5)
1917 writer.write_widechar(NULL, 0)
1918 # CRASHES writer.write_widechar(NULL, 1)
1919 # CRASHES writer.write_widechar(NULL, -1)
1920
1921 self.assertEqual(writer.finish(),
1922 "latin1=\xE9-euro=\u20AC-max=\U0010ffff-zeroes=\0\0\0.")
1923
1924 def test_ucs4(self):
1925 encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be'
1926
1927 writer = self.create_writer(0)
1928 writer.write_ucs4("ascii IGNORED".encode(encoding), 5)
1929 writer.write_char(ord("-"))
1930 writer.write_ucs4("latin1=\xe9".encode(encoding))
1931 writer.write_char(ord("-"))
1932 writer.write_ucs4("euro=\u20ac".encode(encoding))
1933 writer.write_char(ord("-"))
1934 writer.write_ucs4("max=\U0010ffff".encode(encoding))
1935 writer.write_char(ord("."))
1936 self.assertEqual(writer.finish(),
1937 "ascii-latin1=\xE9-euro=\u20AC-max=\U0010ffff.")
1938
1939 # Test some special characters
1940 writer = self.create_writer(0)
1941 # Lone surrogate character
1942 writer.write_ucs4("lone\uDC80".encode(encoding, 'surrogatepass'))
1943 writer.write_char(ord("-"))
1944 # Surrogate pair
1945 writer.write_ucs4("pair\uD83D\uDC0D".encode(encoding, 'surrogatepass'))
1946 writer.write_char(ord("-"))
1947 writer.write_ucs4("null[\0]".encode(encoding), 7)
1948 invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else
1949 b'\x00\x11\x00\x00')
1950 # CRASHES writer.write_ucs4("invalid".encode(encoding) + invalid)
1951 writer.write_ucs4(NULL, 0)
1952 # CRASHES writer.write_ucs4(NULL, 1)
1953 self.assertEqual(writer.finish(),
1954 "lone\udc80-pair\ud83d\udc0d-null[\x00]")
1955
1956 # invalid size
1957 writer = self.create_writer(0)
1958 with self.assertRaises(ValueError):
1959 writer.write_ucs4("text".encode(encoding), -1)
1960 self.assertRaises(ValueError, writer.write_ucs4, b'', -1)
1961 self.assertRaises(ValueError, writer.write_ucs4, NULL, -1)
1962
1963 def test_substring_empty(self):
1964 writer = self.create_writer(0)
1965 writer.write_substring("abc", 1, 1)
1966 self.assertEqual(writer.finish(), '')
1967
1968
1969 @unittest.skipIf(ctypes is None, 'need ctypes')
1970 class PyUnicodeWriterFormatTest(unittest.TestCase):
1971 def create_writer(self, size):
1972 return _testcapi.PyUnicodeWriter(size)
1973
1974 def writer_format(self, writer, *args):
1975 from ctypes import c_char_p, pythonapi, c_int, c_void_p
1976 _PyUnicodeWriter_Format = getattr(pythonapi, "PyUnicodeWriter_Format")
1977 _PyUnicodeWriter_Format.argtypes = (c_void_p, c_char_p,)
1978 _PyUnicodeWriter_Format.restype = c_int
1979
1980 if _PyUnicodeWriter_Format(writer.get_pointer(), *args) < 0:
1981 raise ValueError("PyUnicodeWriter_Format failed")
1982
1983 def test_format(self):
1984 from ctypes import c_int
1985 writer = self.create_writer(0)
1986 self.writer_format(writer, b'%s %i', b'abc', c_int(123))
1987 writer.write_char(ord('.'))
1988 self.assertEqual(writer.finish(), 'abc 123.')
1989
1990 def test_recover_error(self):
1991 # test recovering from PyUnicodeWriter_Format() error
1992 writer = self.create_writer(0)
1993 self.writer_format(writer, b"%s ", b"Hello")
1994
1995 # PyUnicodeWriter_Format() fails with an invalid format string
1996 with self.assertRaises(ValueError):
1997 self.writer_format(writer, b"%s\xff", b"World")
1998
1999 # Retry PyUnicodeWriter_Format() with a valid format string
2000 self.writer_format(writer, b"%s.", b"World")
2001
2002 self.assertEqual(writer.finish(), 'Hello World.')
2003
2004 def test_unicode_equal(self):
2005 unicode_equal = _testlimitedcapi.unicode_equal
2006
2007 def copy(text):
2008 return text.encode().decode()
2009
2010 self.assertTrue(unicode_equal("", ""))
2011 self.assertTrue(unicode_equal("abc", "abc"))
2012 self.assertTrue(unicode_equal("abc", copy("abc")))
2013 self.assertTrue(unicode_equal("\u20ac", copy("\u20ac")))
2014 self.assertTrue(unicode_equal("\U0010ffff", copy("\U0010ffff")))
2015
2016 self.assertFalse(unicode_equal("abc", "abcd"))
2017 self.assertFalse(unicode_equal("\u20ac", "\u20ad"))
2018 self.assertFalse(unicode_equal("\U0010ffff", "\U0010fffe"))
2019
2020 # str subclass
2021 self.assertTrue(unicode_equal("abc", Str("abc")))
2022 self.assertTrue(unicode_equal(Str("abc"), "abc"))
2023 self.assertFalse(unicode_equal("abc", Str("abcd")))
2024 self.assertFalse(unicode_equal(Str("abc"), "abcd"))
2025
2026 # invalid type
2027 for invalid_type in (b'bytes', 123, ("tuple",)):
2028 with self.subTest(invalid_type=invalid_type):
2029 with self.assertRaises(TypeError):
2030 unicode_equal("abc", invalid_type)
2031 with self.assertRaises(TypeError):
2032 unicode_equal(invalid_type, "abc")
2033
2034 # CRASHES unicode_equal("abc", NULL)
2035 # CRASHES unicode_equal(NULL, "abc")
2036
2037
2038 if __name__ == "__main__":
2039 unittest.main()