Debian Codegraph @

Lib/test/test_unicodedata.py

raw
1 """ Tests for the unicodedata module.
2
3 Written by Marc-Andre Lemburg ([email protected]).
4
5 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
6
7 """
8
9 from functools import partial
10 import hashlib
11 from http.client import HTTPException
12 import sys
13 import unicodedata
14 import unittest
15 from test.support import (
16 open_urlresource,
17 requires_resource,
18 script_helper,
19 cpython_only,
20 check_disallow_instantiation,
21 force_not_colorized,
22 is_resource_enabled,
23 findfile,
24 )
25
26
27 quicktest = not is_resource_enabled('cpu')
28
29 def iterallchars():
30 maxunicode = 0xffff if quicktest else sys.maxunicode
31 return map(chr, range(maxunicode + 1))
32
33 class UnicodeMethodsTest(unittest.TestCase):
34
35 # update this, if the database changes
36 expectedchecksum = ('486bf97d506d0ccf0e463fd1f40c51029805af5a'
37 if quicktest else
38 '9e43ee3929471739680c0e705482b4ae1c4122e4')
39
40 def test_method_checksum(self):
41 h = hashlib.sha1()
42 for char in iterallchars():
43 s1 = char + 'abc'
44 s2 = char + 'ABC'
45 s3 = char + '123'
46 data = (
47 # Predicates (single char)
48 "01"[char.isalnum()],
49 "01"[char.isalpha()],
50 "01"[char.isdecimal()],
51 "01"[char.isdigit()],
52 "01"[char.islower()],
53 "01"[char.isnumeric()],
54 "01"[char.isspace()],
55 "01"[char.istitle()],
56 "01"[char.isupper()],
57
58 # Predicates (multiple chars)
59 "01"[s1.isalnum()],
60 "01"[s1.isalpha()],
61 "01"[s3.isdecimal()],
62 "01"[s3.isdigit()],
63 "01"[s1.islower()],
64 "01"[s3.isnumeric()],
65 "01"[(char + ' \t').isspace()],
66 "01"[s1.istitle()],
67 "01"[s2.isupper()],
68
69 # Mappings (single char)
70 char.lower(),
71 char.upper(),
72 char.title(),
73
74 # Mappings (multiple chars)
75 s1.lower(),
76 s2.upper(),
77 s1.title(),
78 s2.title(),
79
80 )
81 h.update(''.join(data).encode('utf-8', 'surrogatepass'))
82 result = h.hexdigest()
83 self.assertEqual(result, self.expectedchecksum)
84
85
86 class UnicodeFunctionsTest(unittest.TestCase):
87 db = unicodedata
88 old = False
89
90 # Update this if the database changes. Make sure to do a full rebuild
91 # (e.g. 'make distclean && make') to get the correct checksum.
92 expectedchecksum = ('1ba453ec456896f1190d849b6e9b7c2e1a4128e0'
93 if quicktest else
94 '46ca89d9fe34881d0be3a4a4b29f5aa8c019640c')
95
96 def test_function_checksum(self):
97 db = self.db
98 data = []
99 h = hashlib.sha1()
100
101 for char in iterallchars():
102 data = "%.12g%.12g%.12g%s%s%s%s%s%s%s" % (
103 # Properties
104 db.digit(char, -1),
105 db.numeric(char, -1),
106 db.decimal(char, -1),
107 db.category(char),
108 db.bidirectional(char),
109 db.decomposition(char),
110 db.mirrored(char),
111 db.combining(char),
112 db.east_asian_width(char),
113 db.name(char, ""),
114 )
115 h.update(data.encode("ascii"))
116 result = h.hexdigest()
117 self.assertEqual(result, self.expectedchecksum)
118
119 def test_name(self):
120 name = self.db.name
121 self.assertRaises(ValueError, name, '\0')
122 self.assertRaises(ValueError, name, '\n')
123 self.assertRaises(ValueError, name, '\x1F')
124 self.assertRaises(ValueError, name, '\x7F')
125 self.assertRaises(ValueError, name, '\x9F')
126 self.assertRaises(ValueError, name, '\uFFFE')
127 self.assertRaises(ValueError, name, '\uFFFF')
128 self.assertRaises(ValueError, name, '\U0010FFFF')
129 self.assertEqual(name('\U0010FFFF', 42), 42)
130
131 self.assertEqual(name(' '), 'SPACE')
132 self.assertEqual(name('1'), 'DIGIT ONE')
133 self.assertEqual(name('A'), 'LATIN CAPITAL LETTER A')
134 self.assertEqual(name('\xA0'), 'NO-BREAK SPACE')
135 self.assertEqual(name('\u0221', None), None if self.old else
136 'LATIN SMALL LETTER D WITH CURL')
137 self.assertEqual(name('\u3400'), 'CJK UNIFIED IDEOGRAPH-3400')
138 self.assertEqual(name('\u9FA5'), 'CJK UNIFIED IDEOGRAPH-9FA5')
139 self.assertEqual(name('\uAC00'), 'HANGUL SYLLABLE GA')
140 self.assertEqual(name('\uD7A3'), 'HANGUL SYLLABLE HIH')
141 self.assertEqual(name('\uF900'), 'CJK COMPATIBILITY IDEOGRAPH-F900')
142 self.assertEqual(name('\uFA6A'), 'CJK COMPATIBILITY IDEOGRAPH-FA6A')
143 self.assertEqual(name('\uFBF9'),
144 'ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA '
145 'ABOVE WITH ALEF MAKSURA ISOLATED FORM')
146 self.assertEqual(name('\U00013460', None), None if self.old else
147 'EGYPTIAN HIEROGLYPH-13460')
148 self.assertEqual(name('\U000143FA', None), None if self.old else
149 'EGYPTIAN HIEROGLYPH-143FA')
150 self.assertEqual(name('\U00018B00', None), None if self.old else
151 'KHITAN SMALL SCRIPT CHARACTER-18B00')
152 self.assertEqual(name('\U00018CD5', None), None if self.old else
153 'KHITAN SMALL SCRIPT CHARACTER-18CD5')
154 self.assertEqual(name('\U00018CFF', None), None if self.old else
155 'KHITAN SMALL SCRIPT CHARACTER-18CFF')
156 self.assertEqual(name('\U0001B170', None), None if self.old else
157 'NUSHU CHARACTER-1B170')
158 self.assertEqual(name('\U0001B2FB', None), None if self.old else
159 'NUSHU CHARACTER-1B2FB')
160 self.assertEqual(name('\U0001FBA8', None), None if self.old else
161 'BOX DRAWINGS LIGHT DIAGONAL UPPER CENTRE TO '
162 'MIDDLE LEFT AND MIDDLE RIGHT TO LOWER CENTRE')
163 self.assertEqual(name('\U0002A6D6'), 'CJK UNIFIED IDEOGRAPH-2A6D6')
164 self.assertEqual(name('\U0002FA1D'), 'CJK COMPATIBILITY IDEOGRAPH-2FA1D')
165 self.assertEqual(name('\U000323AF', None), None if self.old else
166 'CJK UNIFIED IDEOGRAPH-323AF')
167
168 @requires_resource('cpu')
169 def test_name_inverse_lookup(self):
170 for char in iterallchars():
171 looked_name = self.db.name(char, None)
172 if looked_name is not None:
173 self.assertEqual(self.db.lookup(looked_name), char)
174
175 def test_no_names_in_pua(self):
176 puas = [*range(0xe000, 0xf8ff),
177 *range(0xf0000, 0xfffff),
178 *range(0x100000, 0x10ffff)]
179 for i in puas:
180 char = chr(i)
181 self.assertRaises(ValueError, self.db.name, char)
182
183 def test_lookup_nonexistant(self):
184 # just make sure that lookup can fail
185 for nonexistent in [
186 "LATIN SMLL LETR A",
187 "OPEN HANDS SIGHS",
188 "DREGS",
189 "HANDBUG",
190 "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK",
191 "???",
192 "CJK UNIFIED IDEOGRAPH-03400",
193 "CJK UNIFIED IDEOGRAPH-020000",
194 "CJK UNIFIED IDEOGRAPH-33FF",
195 "CJK UNIFIED IDEOGRAPH-F900",
196 "CJK UNIFIED IDEOGRAPH-13460",
197 "CJK UNIFIED IDEOGRAPH-17000",
198 "CJK UNIFIED IDEOGRAPH-18B00",
199 "CJK UNIFIED IDEOGRAPH-1B170",
200 "CJK COMPATIBILITY IDEOGRAPH-3400",
201 "TANGUT IDEOGRAPH-3400",
202 "HANGUL SYLLABLE AC00",
203 ]:
204 self.assertRaises(KeyError, self.db.lookup, nonexistent)
205
206 def test_digit(self):
207 self.assertEqual(self.db.digit('A', None), None)
208 self.assertEqual(self.db.digit('9'), 9)
209 self.assertEqual(self.db.digit('\u215b', None), None)
210 self.assertEqual(self.db.digit('\u2468'), 9)
211 self.assertEqual(self.db.digit('\U00020000', None), None)
212 self.assertEqual(self.db.digit('\U0001D7FD'), 7)
213
214 # New in 13.0.0
215 self.assertEqual(self.db.digit('\U0001fbf9', None), 9)
216 # New in 14.0.0
217 self.assertEqual(self.db.digit('\U00016ac9', None), 9)
218 # New in 15.0.0
219 self.assertEqual(self.db.digit('\U0001e4f9', None), 9)
220
221 self.assertRaises(TypeError, self.db.digit)
222 self.assertRaises(TypeError, self.db.digit, 'xx')
223 self.assertRaises(ValueError, self.db.digit, 'x')
224
225 def test_numeric(self):
226 self.assertEqual(self.db.numeric('A',None), None)
227 self.assertEqual(self.db.numeric('9'), 9)
228 self.assertEqual(self.db.numeric('\u215b'), 0.125)
229 self.assertEqual(self.db.numeric('\u2468'), 9.0)
230 self.assertEqual(self.db.numeric('\U00020000', None), None)
231
232 # New in 4.1.0
233 self.assertEqual(self.db.numeric('\U0001012A', None), None if self.old else 9000)
234 # Changed in 4.1.0
235 self.assertEqual(self.db.numeric('\u5793', None), 1e20 if self.old else None)
236 # New in 5.0.0
237 self.assertEqual(self.db.numeric('\u07c0', None), None if self.old else 0.0)
238 # New in 5.1.0
239 self.assertEqual(self.db.numeric('\ua627', None), None if self.old else 7.0)
240 # Changed in 5.2.0
241 self.assertEqual(self.db.numeric('\u09f6'), 3.0 if self.old else 3/16)
242 # New in 6.0.0
243 self.assertEqual(self.db.numeric('\u0b72', None), None if self.old else 0.25)
244 # New in 12.0.0
245 self.assertEqual(self.db.numeric('\U0001ed3c', None), None if self.old else 0.5)
246 # New in 13.0.0
247 self.assertEqual(self.db.numeric('\U0001fbf9', None), None if self.old else 9)
248 # New in 14.0.0
249 self.assertEqual(self.db.numeric('\U00016ac9', None), None if self.old else 9)
250 # New in 15.0.0
251 self.assertEqual(self.db.numeric('\U0001e4f9', None), None if self.old else 9)
252
253 self.assertRaises(TypeError, self.db.numeric)
254 self.assertRaises(TypeError, self.db.numeric, 'xx')
255 self.assertRaises(ValueError, self.db.numeric, 'x')
256
257 def test_decimal(self):
258 self.assertEqual(self.db.decimal('A',None), None)
259 self.assertEqual(self.db.decimal('9'), 9)
260 self.assertEqual(self.db.decimal('\u215b', None), None)
261 self.assertEqual(self.db.decimal('\u2468', None), None)
262 self.assertEqual(self.db.decimal('\U00020000', None), None)
263 self.assertEqual(self.db.decimal('\U0001D7FD'), 7)
264
265 # New in 4.1.0
266 self.assertEqual(self.db.decimal('\xb2', None), 2 if self.old else None)
267 self.assertEqual(self.db.decimal('\u1369', None), 1 if self.old else None)
268 # New in 5.0.0
269 self.assertEqual(self.db.decimal('\u07c0', None), None if self.old else 0)
270 # New in 13.0.0
271 self.assertEqual(self.db.decimal('\U0001fbf9', None), None if self.old else 9)
272 # New in 14.0.0
273 self.assertEqual(self.db.decimal('\U00016ac9', None), None if self.old else 9)
274 # New in 15.0.0
275 self.assertEqual(self.db.decimal('\U0001e4f9', None), None if self.old else 9)
276
277 self.assertRaises(TypeError, self.db.decimal)
278 self.assertRaises(TypeError, self.db.decimal, 'xx')
279 self.assertRaises(ValueError, self.db.decimal, 'x')
280
281 def test_category(self):
282 self.assertEqual(self.db.category('\uFFFE'), 'Cn')
283 self.assertEqual(self.db.category('a'), 'Ll')
284 self.assertEqual(self.db.category('A'), 'Lu')
285 self.assertEqual(self.db.category('\U00020000'), 'Lo')
286
287 # New in 4.1.0
288 self.assertEqual(self.db.category('\U0001012A'), 'Cn' if self.old else 'No')
289 self.assertEqual(self.db.category('\U000e01ef'), 'Cn' if self.old else 'Mn')
290 # New in 5.1.0
291 self.assertEqual(self.db.category('\u0374'), 'Sk' if self.old else 'Lm')
292 # Changed in 13.0.0
293 self.assertEqual(self.db.category('\u0b55'), 'Cn' if self.old else 'Mn')
294 self.assertEqual(self.db.category('\U0003134a'), 'Cn' if self.old else 'Lo')
295 # Changed in 14.0.0
296 self.assertEqual(self.db.category('\u061d'), 'Cn' if self.old else 'Po')
297 self.assertEqual(self.db.category('\U0002b738'), 'Cn' if self.old else 'Lo')
298 # Changed in 15.0.0
299 self.assertEqual(self.db.category('\u0cf3'), 'Cn' if self.old else 'Mc')
300 self.assertEqual(self.db.category('\U000323af'), 'Cn' if self.old else 'Lo')
301
302 self.assertRaises(TypeError, self.db.category)
303 self.assertRaises(TypeError, self.db.category, 'xx')
304
305 def test_bidirectional(self):
306 self.assertEqual(self.db.bidirectional('\uFFFE'), '')
307 self.assertEqual(self.db.bidirectional(' '), 'WS')
308 self.assertEqual(self.db.bidirectional('A'), 'L')
309 self.assertEqual(self.db.bidirectional('\U00020000'), 'L')
310
311 # New in 4.1.0
312 self.assertEqual(self.db.bidirectional('+'), 'ET' if self.old else 'ES')
313 self.assertEqual(self.db.bidirectional('\u0221'), '' if self.old else 'L')
314 self.assertEqual(self.db.bidirectional('\U000e01ef'), '' if self.old else 'NSM')
315 # New in 13.0.0
316 self.assertEqual(self.db.bidirectional('\u0b55'), '' if self.old else 'NSM')
317 self.assertEqual(self.db.bidirectional('\U0003134a'), '' if self.old else 'L')
318 # New in 14.0.0
319 self.assertEqual(self.db.bidirectional('\u061d'), '' if self.old else 'AL')
320 self.assertEqual(self.db.bidirectional('\U0002b738'), '' if self.old else 'L')
321 # New in 15.0.0
322 self.assertEqual(self.db.bidirectional('\u0cf3'), '' if self.old else 'L')
323 self.assertEqual(self.db.bidirectional('\U000323af'), '' if self.old else 'L')
324 # New in 16.0.0
325 self.assertEqual(self.db.bidirectional('\u0897'), '' if self.old else 'NSM')
326 self.assertEqual(self.db.bidirectional('\U0001fbef'), '' if self.old else 'ON')
327
328 self.assertRaises(TypeError, self.db.bidirectional)
329 self.assertRaises(TypeError, self.db.bidirectional, 'xx')
330
331 def test_decomposition(self):
332 self.assertEqual(self.db.decomposition('\uFFFE'),'')
333 self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034')
334
335 # New in 4.1.0
336 self.assertEqual(self.db.decomposition('\u03f9'), '' if self.old else '<compat> 03A3')
337 # New in 13.0.0
338 self.assertEqual(self.db.decomposition('\uab69'), '' if self.old else '<super> 028D')
339 self.assertEqual(self.db.decomposition('\U00011938'), '' if self.old else '11935 11930')
340 self.assertEqual(self.db.decomposition('\U0001fbf9'), '' if self.old else '<font> 0039')
341 # New in 14.0.0
342 self.assertEqual(self.db.decomposition('\ua7f2'), '' if self.old else '<super> 0043')
343 self.assertEqual(self.db.decomposition('\U000107ba'), '' if self.old else '<super> 1DF1E')
344 # New in 15.0.0
345 self.assertEqual(self.db.decomposition('\U0001e06d'), '' if self.old else '<super> 04B1')
346 # New in 16.0.0
347 self.assertEqual(self.db.decomposition('\U0001CCD6'), '' if self.old else '<font> 0041')
348
349 # Hangul characters
350 self.assertEqual(self.db.decomposition('\uAC00'), '1100 1161')
351 self.assertEqual(self.db.decomposition('\uD4DB'), '1111 1171 11B6')
352 self.assertEqual(self.db.decomposition('\uC2F8'), '110A 1161')
353 self.assertEqual(self.db.decomposition('\uD7A3'), '1112 1175 11C2')
354
355 self.assertRaises(TypeError, self.db.decomposition)
356 self.assertRaises(TypeError, self.db.decomposition, 'xx')
357
358 def test_mirrored(self):
359 self.assertEqual(self.db.mirrored('\uFFFE'), 0)
360 self.assertEqual(self.db.mirrored('a'), 0)
361 self.assertEqual(self.db.mirrored('\u2201'), 1)
362 self.assertEqual(self.db.mirrored('\U00020000'), 0)
363
364 # New in 5.0.0
365 self.assertEqual(self.db.mirrored('\u0f3a'), 0 if self.old else 1)
366 self.assertEqual(self.db.mirrored('\U0001d7c3'), 0 if self.old else 1)
367 # New in 11.0.0
368 self.assertEqual(self.db.mirrored('\u29a1'), 1 if self.old else 0)
369 # New in 14.0.0
370 self.assertEqual(self.db.mirrored('\u2e5c'), 0 if self.old else 1)
371 # New in 16.0.0
372 self.assertEqual(self.db.mirrored('\u226D'), 0 if self.old else 1)
373
374 self.assertRaises(TypeError, self.db.mirrored)
375 self.assertRaises(TypeError, self.db.mirrored, 'xx')
376
377 def test_combining(self):
378 self.assertEqual(self.db.combining('\uFFFE'), 0)
379 self.assertEqual(self.db.combining('a'), 0)
380 self.assertEqual(self.db.combining('\u20e1'), 230)
381 self.assertEqual(self.db.combining('\U00020000'), 0)
382
383 # New in 4.1.0
384 self.assertEqual(self.db.combining('\u0350'), 0 if self.old else 230)
385 # New in 9.0.0
386 self.assertEqual(self.db.combining('\U0001e94a'), 0 if self.old else 7)
387 # New in 13.0.0
388 self.assertEqual(self.db.combining('\u1abf'), 0 if self.old else 220)
389 self.assertEqual(self.db.combining('\U00016ff1'), 0 if self.old else 6)
390 # New in 14.0.0
391 self.assertEqual(self.db.combining('\u0c3c'), 0 if self.old else 7)
392 self.assertEqual(self.db.combining('\U0001e2ae'), 0 if self.old else 230)
393 # New in 15.0.0
394 self.assertEqual(self.db.combining('\U00010efd'), 0 if self.old else 220)
395 # New in 16.0.0
396 self.assertEqual(self.db.combining('\u0897'), 0 if self.old else 230)
397
398 self.assertRaises(TypeError, self.db.combining)
399 self.assertRaises(TypeError, self.db.combining, 'xx')
400
401 def test_normalization(self):
402 # Test normalize() and is_normalized()
403 def check(ch, expected):
404 if isinstance(expected, str):
405 expected = [expected]*4
406 forms = ('NFC', 'NFD', 'NFKC', 'NFKD')
407 result = [self.db.normalize(form, ch) for form in forms]
408 self.assertEqual(ascii(result), ascii(list(expected)))
409 self.assertEqual([self.db.is_normalized(form, ch) for form in forms],
410 [ch == y for x, y in zip(result, expected)])
411
412 check('', '')
413 check('A', 'A')
414 check(' ', ' ')
415 check('\U0010ffff', '\U0010ffff')
416 check('abc', 'abc')
417 # Broken in 4.0.0
418 check('\u0340', '\u0300')
419 check('\u0300', '\u0300')
420 check('\U0002fa1d', '\U0002a600')
421 check('\U0002a600', '\U0002a600')
422 check('\u0344', '\u0308\u0301')
423 check('\u0308\u0301', '\u0308\u0301')
424 # Broken in 4.0.0 and 4.0.1
425 check('\U0001d1bc', '\U0001d1ba\U0001d165')
426 check('\U0001d1ba\U0001d165', '\U0001d1ba\U0001d165')
427 check('\ufb2c', '\u05e9\u05bc\u05c1')
428 check('\u05e9\u05bc\u05c1', '\u05e9\u05bc\u05c1')
429 check('\U0001d1c0', '\U0001d1ba\U0001d165\U0001d16f')
430 check('\U0001d1ba\U0001d165\U0001d16f', '\U0001d1ba\U0001d165\U0001d16f')
431
432 # Broken in 4.0.0
433 check('\xa0', ['\xa0', '\xa0', ' ', ' '])
434 check('\u2003', ['\u2003', '\u2003', ' ', ' '])
435 check('\U0001d7ff', ['\U0001d7ff', '\U0001d7ff', '9', '9'])
436
437 check('\xa8', ['\xa8', '\xa8', ' \u0308', ' \u0308'])
438 check(' \u0308', ' \u0308')
439
440 check('\xc0', ['\xc0', 'A\u0300']*2)
441 check('A\u0300', ['\xc0', 'A\u0300']*2)
442
443 check('\ud7a3', ['\ud7a3', '\u1112\u1175\u11c2']*2)
444 check('\u1112\u1175\u11c2', ['\ud7a3', '\u1112\u1175\u11c2']*2)
445
446 check('\xb4', ['\xb4', '\xb4', ' \u0301', ' \u0301'])
447 check('\u1ffd', ['\xb4', '\xb4', ' \u0301', ' \u0301'])
448 check(' \u0301', ' \u0301')
449
450 check('\xc5', ['\xc5', 'A\u030a']*2)
451 check('\u212b', ['\xc5', 'A\u030a']*2)
452 check('A\u030a', ['\xc5', 'A\u030a']*2)
453
454 check('\u1f71', ['\u03ac', '\u03b1\u0301']*2)
455 check('\u03ac', ['\u03ac', '\u03b1\u0301']*2)
456 check('\u03b1\u0301', ['\u03ac', '\u03b1\u0301']*2)
457
458 check('\u01c4', ['\u01c4', '\u01c4', 'D\u017d', 'DZ\u030c'])
459 check('D\u017d', ['D\u017d', 'DZ\u030c']*2)
460 check('DZ\u030c', ['D\u017d', 'DZ\u030c']*2)
461
462 check('\u1fed', ['\u1fed', '\xa8\u0300', ' \u0308\u0300', ' \u0308\u0300'])
463 check('\xa8\u0300', ['\u1fed', '\xa8\u0300', ' \u0308\u0300', ' \u0308\u0300'])
464 check(' \u0308\u0300', ' \u0308\u0300')
465
466 check('\u326e', ['\u326e', '\u326e', '\uac00', '\u1100\u1161'])
467 check('\u320e', ['\u320e', '\u320e', '(\uac00)', '(\u1100\u1161)'])
468 check('(\uac00)', ['(\uac00)', '(\u1100\u1161)']*2)
469 check('(\u1100\u1161)', ['(\uac00)', '(\u1100\u1161)']*2)
470
471 check('\u0385', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301'])
472 check('\u1fee', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301'])
473 check('\xa8\u0301', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301'])
474 check(' \u0308\u0301', ' \u0308\u0301')
475
476 check('\u1fdf', ['\u1fdf', '\u1ffe\u0342', ' \u0314\u0342', ' \u0314\u0342'])
477 check('\u1ffe\u0342', ['\u1fdf', '\u1ffe\u0342', ' \u0314\u0342', ' \u0314\u0342'])
478 check('\u1ffe', ['\u1ffe', '\u1ffe', ' \u0314', ' \u0314'])
479 check(' \u0314\u0342', ' \u0314\u0342')
480
481 check('\u03d3', ['\u03d3', '\u03d2\u0301', '\u038e', '\u03a5\u0301'])
482 check('\u03d2\u0301', ['\u03d3', '\u03d2\u0301', '\u038e', '\u03a5\u0301'])
483 check('\u038e', ['\u038e', '\u03a5\u0301']*2)
484 check('\u1feb', ['\u038e', '\u03a5\u0301']*2)
485 check('\u03a5\u0301', ['\u038e', '\u03a5\u0301']*2)
486
487 check('\u0626', ['\u0626', '\u064a\u0654']*2)
488 check('\u064a\u0654', ['\u0626', '\u064a\u0654']*2)
489 check('\ufe89', ['\ufe89', '\ufe89', '\u0626', '\u064a\u0654'])
490 check('\ufe8a', ['\ufe8a', '\ufe8a', '\u0626', '\u064a\u0654'])
491 check('\ufe8b', ['\ufe8b', '\ufe8b', '\u0626', '\u064a\u0654'])
492 check('\ufe8c', ['\ufe8c', '\ufe8c', '\u0626', '\u064a\u0654'])
493
494 check('\ufef9', ['\ufef9', '\ufef9', '\u0644\u0625', '\u0644\u0627\u0655'])
495 check('\ufefa', ['\ufefa', '\ufefa', '\u0644\u0625', '\u0644\u0627\u0655'])
496 check('\ufefb', ['\ufefb', '\ufefb', '\u0644\u0627', '\u0644\u0627'])
497 check('\ufefc', ['\ufefc', '\ufefc', '\u0644\u0627', '\u0644\u0627'])
498 check('\u0644\u0625', ['\u0644\u0625', '\u0644\u0627\u0655']*2)
499 check('\u0644\u0627\u0655', ['\u0644\u0625', '\u0644\u0627\u0655']*2)
500 check('\u0644\u0627', '\u0644\u0627')
501
502 # Broken in 4.0.0
503 check('\u327c', '\u327c' if self.old else
504 ['\u327c', '\u327c', '\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169'])
505 check('\ucc38\uace0', ['\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']*2)
506 check('\ucc38', ['\ucc38', '\u110e\u1161\u11b7']*2)
507 check('\u110e\u1161\u11b7\u1100\u1169',
508 ['\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']*2)
509 check('\u110e\u1161\u11b7\u1100',
510 ['\ucc38\u1100', '\u110e\u1161\u11b7\u1100']*2)
511 check('\u110e\u1161\u11b7',
512 ['\ucc38', '\u110e\u1161\u11b7']*2)
513 check('\u110e\u1161',
514 ['\ucc28', '\u110e\u1161']*2)
515 check('\u110e', '\u110e')
516 # Broken in 4.0.0-12.0.0
517 check('\U00011938', '\U00011938' if self.old else
518 ['\U00011938', '\U00011935\U00011930']*2)
519 check('\U00011935\U00011930', ['\U00011938', '\U00011935\U00011930']*2)
520 # New in 4.0.1
521 check('\u321d', '\u321d' if self.old else
522 ['\u321d', '\u321d', '(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)'])
523 check('(\uc624\uc804)',
524 ['(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']*2)
525 check('(\u110b\u1169\u110c\u1165\u11ab)',
526 ['(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']*2)
527 check('\u4d57', '\u4d57')
528 check('\u45d7', '\u45d7' if self.old else '\u45d7')
529 check('\U0002f9bf', '\u4d57' if self.old else '\u45d7')
530 # New in 4.1.0
531 check('\u03a3', '\u03a3')
532 check('\u03f9', '\u03f9' if self.old else
533 ['\u03f9', '\u03f9', '\u03a3', '\u03a3'])
534 # New in 5.0.0
535 check('\u1b06', '\u1b06' if self.old else ['\u1b06', '\u1b05\u1b35']*2)
536 # New in 5.2.0
537 check('\U0001f213', '\U0001f213' if self.old else
538 ['\U0001f213', '\U0001f213', '\u30c7', '\u30c6\u3099'])
539 # New in 6.1.0
540 check('\ufa2e', '\ufa2e' if self.old else '\u90de')
541 # New in 13.0.0
542 check('\U00011938', '\U00011938' if self.old else
543 ['\U00011938', '\U00011935\U00011930', '\U00011938', '\U00011935\U00011930'])
544 check('\U0001fbf9', '\U0001fbf9' if self.old else
545 ['\U0001fbf9', '\U0001fbf9', '9', '9'])
546 # New in 14.0.0
547 check('\U000107ba', '\U000107ba' if self.old else
548 ['\U000107ba', '\U000107ba', '\U0001df1e', '\U0001df1e'])
549 # New in 15.0.0
550 check('\U0001e06d', '\U0001e06d' if self.old else
551 ['\U0001e06d', '\U0001e06d', '\u04b1', '\u04b1'])
552 # New in 16.0.0
553 check('\U0001ccd6', '\U0001ccd6' if self.old else
554 ['\U0001ccd6', '\U0001ccd6', 'A', 'A'])
555
556 self.assertRaises(TypeError, self.db.normalize)
557 self.assertRaises(TypeError, self.db.normalize, 'NFC')
558 self.assertRaises(ValueError, self.db.normalize, 'SPAM', 'A')
559
560 self.assertRaises(TypeError, self.db.is_normalized)
561 self.assertRaises(TypeError, self.db.is_normalized, 'NFC')
562 self.assertRaises(ValueError, self.db.is_normalized, 'SPAM', 'A')
563
564 def test_pr29(self):
565 # https://www.unicode.org/review/pr-29.html
566 # See issues #1054943 and #10254.
567 composed = ("\u0b47\u0300\u0b3e", "\u1100\u0300\u1161",
568 'Li\u030dt-s\u1e73\u0301',
569 '\u092e\u093e\u0930\u094d\u0915 \u091c\u093c'
570 + '\u0941\u0915\u0947\u0930\u092c\u0930\u094d\u0917',
571 '\u0915\u093f\u0930\u094d\u0917\u093f\u091c\u093c'
572 + '\u0938\u094d\u0924\u093e\u0928')
573 for text in composed:
574 self.assertEqual(self.db.normalize('NFC', text), text)
575
576 def test_issue10254(self):
577 # Crash reported in #10254
578 # New in 4.1.0
579 a = 'C\u0338' * 20 + 'C\u0327'
580 b = 'C\u0338' * 20 + '\xC7'
581 self.assertEqual(self.db.normalize('NFC', a), b)
582
583 def test_long_combining_mark_run(self):
584 # gh-149079: avoid quadratic canonical ordering.
585 payload = "a" + ("\u0300\u0327" * 32)
586 nfd = "a" + ("\u0327" * 32) + ("\u0300" * 32)
587 nfc = "\u00e0" + ("\u0327" * 32) + ("\u0300" * 31)
588
589 self.assertEqual(self.db.normalize("NFD", payload), nfd)
590 self.assertEqual(self.db.normalize("NFKD", payload), nfd)
591 self.assertEqual(self.db.normalize("NFC", payload), nfc)
592 self.assertEqual(self.db.normalize("NFKC", payload), nfc)
593
594 def test_combining_mark_run_fast_paths(self):
595 # gh-149079: cover short runs and already-sorted long runs.
596 short_payload = "a" + ("\u0300\u0327" * 9) + "\u0300"
597 short_nfd = "a" + ("\u0327" * 9) + ("\u0300" * 10)
598 short_nfc = "\u00e0" + ("\u0327" * 9) + ("\u0300" * 9)
599 long_sorted = "a" + ("\u0327" * 30) + ("\u0300" * 30)
600 long_sorted_nfc = "\u00e0" + ("\u0327" * 30) + ("\u0300" * 29)
601
602 self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd)
603 self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd)
604 self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc)
605 self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc)
606 self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted)
607 self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted)
608 self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc)
609 self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc)
610
611 def test_issue29456(self):
612 # Fix #29456
613 u1176_str_a = '\u1100\u1176\u11a8'
614 u1176_str_b = '\u1100\u1176\u11a8'
615 u11a7_str_a = '\u1100\u1175\u11a7'
616 u11a7_str_b = '\uae30\u11a7'
617 u11c3_str_a = '\u1100\u1175\u11c3'
618 u11c3_str_b = '\uae30\u11c3'
619 self.assertEqual(self.db.normalize('NFC', u1176_str_a), u1176_str_b)
620 # New in 4.1.0
621 self.assertEqual(self.db.normalize('NFC', u11a7_str_a), u11a7_str_b)
622 self.assertEqual(self.db.normalize('NFC', u11c3_str_a), u11c3_str_b)
623
624 def test_east_asian_width(self):
625 eaw = self.db.east_asian_width
626 self.assertRaises(TypeError, eaw, b'a')
627 self.assertRaises(TypeError, eaw, bytearray())
628 self.assertRaises(TypeError, eaw, '')
629 self.assertRaises(TypeError, eaw, 'ra')
630 self.assertEqual(eaw('\x1e'), 'N')
631 self.assertEqual(eaw('\x20'), 'Na')
632 self.assertEqual(eaw('\uC894'), 'W')
633 self.assertEqual(eaw('\uFF66'), 'H')
634 self.assertEqual(eaw('\uFF1F'), 'F')
635 self.assertEqual(eaw('\u2010'), 'A')
636 self.assertEqual(eaw('\U00020000'), 'W')
637
638 # New in 4.1.0
639 self.assertEqual(eaw('\u0350'), 'N' if self.old else 'A')
640 self.assertEqual(eaw('\U000e01ef'), 'N' if self.old else 'A')
641 # New in 5.2.0
642 self.assertEqual(eaw('\u115a'), 'N' if self.old else 'W')
643 # New in 9.0.0
644 self.assertEqual(eaw('\u231a'), 'N' if self.old else 'W')
645 self.assertEqual(eaw('\u2614'), 'N' if self.old else 'W')
646 self.assertEqual(eaw('\U0001f19a'), 'N' if self.old else 'W')
647 self.assertEqual(eaw('\U0001f991'), 'N' if self.old else 'W')
648 self.assertEqual(eaw('\U0001f9c0'), 'N' if self.old else 'W')
649 # New in 12.0.0
650 self.assertEqual(eaw('\u32ff'), 'N' if self.old else 'W')
651 self.assertEqual(eaw('\U0001fa95'), 'N' if self.old else 'W')
652 # New in 13.0.0
653 self.assertEqual(eaw('\u31bb'), 'N' if self.old else 'W')
654 self.assertEqual(eaw('\U0003134a'), 'N' if self.old else 'W')
655 # New in 14.0.0
656 self.assertEqual(eaw('\u9ffd'), 'N' if self.old else 'W')
657 self.assertEqual(eaw('\U0002b738'), 'N' if self.old else 'W')
658 # New in 15.0.0
659 self.assertEqual(eaw('\U000323af'), 'N' if self.old else 'W')
660 # New in 16.0.0
661 self.assertEqual(eaw('\u2630'), 'N' if self.old else 'W')
662 self.assertEqual(eaw('\U0001FAE9'), 'N' if self.old else 'W')
663
664 def test_east_asian_width_unassigned(self):
665 eaw = self.db.east_asian_width
666 # unassigned
667 for char in '\u0530\u0ecf\u10c6\u20fc\uaaca\U000107bd\U000115f2':
668 self.assertEqual(eaw(char), 'N')
669 self.assertIs(self.db.name(char, None), None)
670
671 # unassigned but reserved for CJK
672 for char in ('\U0002A6E0\U0002FA20\U0003134B\U0003FFFD'
673 '\uFA6E\uFADA'): # New in 5.2.0
674 self.assertEqual(eaw(char), 'W')
675 self.assertIs(self.db.name(char, None), None)
676
677 # private use areas
678 for char in '\uE000\uF800\U000F0000\U000FFFEE\U00100000\U0010FFF0':
679 self.assertEqual(eaw(char), 'A')
680 self.assertIs(self.db.name(char, None), None)
681
682
683 class Unicode_3_2_0_FunctionsTest(UnicodeFunctionsTest):
684 db = unicodedata.ucd_3_2_0
685 old = True
686 expectedchecksum = ('883824cb6c0ccf994e4451ebf281e2d6d479af47'
687 if quicktest else
688 'caf1a7f2f380f927461837f1901ef20683f98683')
689
690
691 class UnicodeMiscTest(unittest.TestCase):
692 db = unicodedata
693
694 @cpython_only
695 def test_disallow_instantiation(self):
696 # Ensure that the type disallows instantiation (bpo-43916)
697 check_disallow_instantiation(self, unicodedata.UCD)
698
699 @force_not_colorized
700 def test_failed_import_during_compiling(self):
701 # Issue 4367
702 # Decoding \N escapes requires the unicodedata module. If it can't be
703 # imported, we shouldn't segfault.
704
705 # This program should raise a SyntaxError in the eval.
706 code = "import sys;" \
707 "sys.modules['unicodedata'] = None;" \
708 """eval("'\\\\N{SOFT HYPHEN}'")"""
709 # We use a separate process because the unicodedata module may already
710 # have been loaded in this process.
711 result = script_helper.assert_python_failure("-c", code)
712 error = "SyntaxError: (unicode error) \\N escapes not supported " \
713 "(can't load unicodedata module)"
714 self.assertIn(error, result.err.decode("ascii"))
715
716 def test_unicodedata_unload_reload(self):
717 # gh-149449: dropping unicodedata and running gc must not leave the
718 # cached _ucnhash_CAPI pointer dangling.
719 code = (
720 "import gc, sys\n"
721 "assert '\\N{GRINNING FACE}'.encode("
722 " 'ascii', errors='namereplace') == b'\\\\N{GRINNING FACE}'\n"
723 "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER A}'\", '<x>', 'exec')\n"
724 "del sys.modules['unicodedata']\n"
725 "gc.collect()\n"
726 "assert '\\N{WINKING FACE}'.encode("
727 " 'ascii', errors='namereplace') == b'\\\\N{WINKING FACE}'\n"
728 "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER B}'\", '<x>', 'exec')\n"
729 )
730 script_helper.assert_python_ok("-c", code)
731
732 def test_decimal_numeric_consistent(self):
733 # Test that decimal and numeric are consistent,
734 # i.e. if a character has a decimal value,
735 # its numeric value should be the same.
736 count = 0
737 for c in iterallchars():
738 dec = self.db.decimal(c, -1)
739 if dec != -1:
740 self.assertEqual(dec, self.db.numeric(c))
741 count += 1
742 self.assertTrue(count >= 10, count) # should have tested at least the ASCII digits
743
744 def test_digit_numeric_consistent(self):
745 # Test that digit and numeric are consistent,
746 # i.e. if a character has a digit value,
747 # its numeric value should be the same.
748 count = 0
749 for c in iterallchars():
750 dec = self.db.digit(c, -1)
751 if dec != -1:
752 self.assertEqual(dec, self.db.numeric(c))
753 count += 1
754 self.assertTrue(count >= 10, count) # should have tested at least the ASCII digits
755
756 def test_normalize_consistent(self):
757 allchars = list(iterallchars())
758 for form in ('NFC', 'NFD', 'NFKC', 'NFKD'):
759 for c in allchars:
760 norm = self.db.normalize(form, c)
761 self.assertEqual(self.db.is_normalized(form, c), norm == c)
762 if norm != c:
763 self.assertEqual(self.db.normalize(form, norm), norm)
764 self.assertTrue(self.db.is_normalized(form, norm))
765
766 def test_bug_1704793(self):
767 self.assertEqual(self.db.lookup("GOTHIC LETTER FAIHU"), '\U00010346')
768
769 def test_ucd_510(self):
770 # In UCD 5.1.0, a mirrored property changed wrt. UCD 3.2.0
771 self.assertTrue(unicodedata.mirrored("\u0f3a"))
772 self.assertTrue(not unicodedata.ucd_3_2_0.mirrored("\u0f3a"))
773 # Also, we now have two ways of representing
774 # the upper-case mapping: as delta, or as absolute value
775 self.assertTrue("a".upper()=='A')
776 self.assertTrue("\u1d79".upper()=='\ua77d')
777 self.assertTrue(".".upper()=='.')
778
779 @requires_resource('cpu')
780 def test_bug_5828(self):
781 self.assertEqual("\u1d79".lower(), "\u1d79")
782 # Only U+0000 should have U+0000 as its upper/lower/titlecase variant
783 self.assertEqual(
784 [
785 c for c in iterallchars()
786 if "\x00" in (c.lower(), c.upper(), c.title())
787 ],
788 ["\x00"]
789 )
790
791 def test_bug_4971(self):
792 # LETTER DZ WITH CARON: DZ, Dz, dz
793 self.assertEqual("\u01c4".title(), "\u01c5")
794 self.assertEqual("\u01c5".title(), "\u01c5")
795 self.assertEqual("\u01c6".title(), "\u01c5")
796
797 def test_linebreak_7643(self):
798 for c in iterallchars():
799 lines = (c + 'A').splitlines()
800 if c in ('\x0a', '\x0b', '\x0c', '\x0d', '\x85',
801 '\x1c', '\x1d', '\x1e', '\u2028', '\u2029'):
802 self.assertEqual(len(lines), 2,
803 r"%a should be a linebreak" % c)
804 else:
805 self.assertEqual(len(lines), 1,
806 r"%a should not be a linebreak" % c)
807
808
809 class NormalizationTest(unittest.TestCase):
810 @staticmethod
811 def check_version(testfile):
812 hdr = testfile.readline()
813 return unicodedata.unidata_version in hdr
814
815 @staticmethod
816 def unistr(data):
817 data = [int(x, 16) for x in data.split(" ")]
818 return "".join([chr(x) for x in data])
819
820 @requires_resource('network')
821 @requires_resource('cpu')
822 def test_normalization(self):
823 TESTDATAFILE = "NormalizationTest.txt"
824 TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{TESTDATAFILE}"
825
826 # Hit the exception early
827 try:
828 testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
829 check=self.check_version)
830 except PermissionError:
831 self.skipTest(f"Permission error when downloading {TESTDATAURL} "
832 f"into the test data directory")
833 except (OSError, HTTPException) as exc:
834 self.skipTest(f"Failed to download {TESTDATAURL}: {exc}")
835
836 with testdata:
837 self.run_normalization_tests(testdata, unicodedata)
838
839 @requires_resource('cpu')
840 def test_normalization_3_2_0(self):
841 testdatafile = findfile('NormalizationTest-3.2.0.txt')
842 with open(testdatafile, encoding='utf-8') as testdata:
843 self.run_normalization_tests(testdata, unicodedata.ucd_3_2_0)
844
845 def run_normalization_tests(self, testdata, ucd):
846 part = None
847 part1_data = set()
848
849 NFC = partial(ucd.normalize, "NFC")
850 NFKC = partial(ucd.normalize, "NFKC")
851 NFD = partial(ucd.normalize, "NFD")
852 NFKD = partial(ucd.normalize, "NFKD")
853 is_normalized = ucd.is_normalized
854
855 for line in testdata:
856 if '#' in line:
857 line = line.split('#')[0]
858 line = line.strip()
859 if not line:
860 continue
861 if line.startswith("@Part"):
862 part = line.split()[0]
863 continue
864 c1,c2,c3,c4,c5 = [self.unistr(x) for x in line.split(';')[:-1]]
865
866 # Perform tests
867 self.assertTrue(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
868 self.assertTrue(c4 == NFC(c4) == NFC(c5), line)
869 self.assertTrue(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
870 self.assertTrue(c5 == NFD(c4) == NFD(c5), line)
871 self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \
872 NFKC(c3) == NFKC(c4) == NFKC(c5),
873 line)
874 self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \
875 NFKD(c3) == NFKD(c4) == NFKD(c5),
876 line)
877
878 self.assertTrue(is_normalized("NFC", c2))
879 self.assertTrue(is_normalized("NFC", c4))
880
881 self.assertTrue(is_normalized("NFD", c3))
882 self.assertTrue(is_normalized("NFD", c5))
883
884 self.assertTrue(is_normalized("NFKC", c4))
885 self.assertTrue(is_normalized("NFKD", c5))
886
887 # Record part 1 data
888 if part == "@Part1":
889 part1_data.add(c1)
890
891 # Perform tests for all other data
892 for X in iterallchars():
893 if X in part1_data:
894 continue
895 self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), ord(X))
896
897 def test_edge_cases(self):
898 self.assertRaises(TypeError, unicodedata.normalize)
899 self.assertRaises(ValueError, unicodedata.normalize, 'unknown', 'xx')
900 self.assertEqual(unicodedata.normalize('NFKC', ''), '')
901
902 def test_bug_834676(self):
903 # Check for bug 834676
904 unicodedata.normalize('NFC', '\ud55c\uae00')
905
906 def test_normalize_return_type(self):
907 # gh-129569: normalize() return type must always be str
908 normalize = unicodedata.normalize
909
910 class MyStr(str):
911 pass
912
913 normalization_forms = ("NFC", "NFKC", "NFD", "NFKD")
914 input_strings = (
915 # normalized strings
916 "",
917 "ascii",
918 # unnormalized strings
919 "\u1e0b\u0323",
920 "\u0071\u0307\u0323",
921 )
922
923 for form in normalization_forms:
924 for input_str in input_strings:
925 with self.subTest(form=form, input_str=input_str):
926 self.assertIs(type(normalize(form, input_str)), str)
927 self.assertIs(type(normalize(form, MyStr(input_str))), str)
928
929
930 if __name__ == "__main__":
931 unittest.main()