Debian Codegraph @

Lib/test/test_unicodedata.py

raw
1 """ Tests for the unicodedata module.
2
3 Written by Marc-Andre Lemburg ([email protected]).
4
5 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
6
7 """
8
9 from functools import partial
10 import hashlib
11 from http.client import HTTPException
12 import sys
13 import unicodedata
14 import unittest
15 from test.support import (
16 open_urlresource,
17 requires_resource,
18 script_helper,
19 cpython_only,
20 check_disallow_instantiation,
21 force_not_colorized,
22 is_resource_enabled,
23 findfile,
24 )
25
26
27 quicktest = not is_resource_enabled('cpu')
28
29 def iterallchars():
30 maxunicode = 0xffff if quicktest else sys.maxunicode
31 return map(chr, range(maxunicode + 1))
32
33 class UnicodeMethodsTest(unittest.TestCase):
34
35 # update this, if the database changes
36 expectedchecksum = ('486bf97d506d0ccf0e463fd1f40c51029805af5a'
37 if quicktest else
38 '9e43ee3929471739680c0e705482b4ae1c4122e4')
39
40 def test_method_checksum(self):
41 h = hashlib.sha1()
42 for char in iterallchars():
43 s1 = char + 'abc'
44 s2 = char + 'ABC'
45 s3 = char + '123'
46 data = (
47 # Predicates (single char)
48 "01"[char.isalnum()],
49 "01"[char.isalpha()],
50 "01"[char.isdecimal()],
51 "01"[char.isdigit()],
52 "01"[char.islower()],
53 "01"[char.isnumeric()],
54 "01"[char.isspace()],
55 "01"[char.istitle()],
56 "01"[char.isupper()],
57
58 # Predicates (multiple chars)
59 "01"[s1.isalnum()],
60 "01"[s1.isalpha()],
61 "01"[s3.isdecimal()],
62 "01"[s3.isdigit()],
63 "01"[s1.islower()],
64 "01"[s3.isnumeric()],
65 "01"[(char + ' \t').isspace()],
66 "01"[s1.istitle()],
67 "01"[s2.isupper()],
68
69 # Mappings (single char)
70 char.lower(),
71 char.upper(),
72 char.title(),
73
74 # Mappings (multiple chars)
75 s1.lower(),
76 s2.upper(),
77 s1.title(),
78 s2.title(),
79
80 )
81 h.update(''.join(data).encode('utf-8', 'surrogatepass'))
82 result = h.hexdigest()
83 self.assertEqual(result, self.expectedchecksum)
84
85
86 class UnicodeFunctionsTest(unittest.TestCase):
87 db = unicodedata
88 old = False
89
90 # Update this if the database changes. Make sure to do a full rebuild
91 # (e.g. 'make distclean && make') to get the correct checksum.
92 expectedchecksum = ('1ba453ec456896f1190d849b6e9b7c2e1a4128e0'
93 if quicktest else
94 '46ca89d9fe34881d0be3a4a4b29f5aa8c019640c')
95
96 def test_function_checksum(self):
97 db = self.db
98 data = []
99 h = hashlib.sha1()
100
101 for char in iterallchars():
102 data = "%.12g%.12g%.12g%s%s%s%s%s%s%s" % (
103 # Properties
104 db.digit(char, -1),
105 db.numeric(char, -1),
106 db.decimal(char, -1),
107 db.category(char),
108 db.bidirectional(char),
109 db.decomposition(char),
110 db.mirrored(char),
111 db.combining(char),
112 db.east_asian_width(char),
113 db.name(char, ""),
114 )
115 h.update(data.encode("ascii"))
116 result = h.hexdigest()
117 self.assertEqual(result, self.expectedchecksum)
118
119 def test_name(self):
120 name = self.db.name
121 self.assertRaises(ValueError, name, '\0')
122 self.assertRaises(ValueError, name, '\n')
123 self.assertRaises(ValueError, name, '\x1F')
124 self.assertRaises(ValueError, name, '\x7F')
125 self.assertRaises(ValueError, name, '\x9F')
126 self.assertRaises(ValueError, name, '\uFFFE')
127 self.assertRaises(ValueError, name, '\uFFFF')
128 self.assertRaises(ValueError, name, '\U0010FFFF')
129 self.assertEqual(name('\U0010FFFF', 42), 42)
130
131 self.assertEqual(name(' '), 'SPACE')
132 self.assertEqual(name('1'), 'DIGIT ONE')
133 self.assertEqual(name('A'), 'LATIN CAPITAL LETTER A')
134 self.assertEqual(name('\xA0'), 'NO-BREAK SPACE')
135 self.assertEqual(name('\u0221', None), None if self.old else
136 'LATIN SMALL LETTER D WITH CURL')
137 self.assertEqual(name('\u3400'), 'CJK UNIFIED IDEOGRAPH-3400')
138 self.assertEqual(name('\u9FA5'), 'CJK UNIFIED IDEOGRAPH-9FA5')
139 self.assertEqual(name('\uAC00'), 'HANGUL SYLLABLE GA')
140 self.assertEqual(name('\uD7A3'), 'HANGUL SYLLABLE HIH')
141 self.assertEqual(name('\uF900'), 'CJK COMPATIBILITY IDEOGRAPH-F900')
142 self.assertEqual(name('\uFA6A'), 'CJK COMPATIBILITY IDEOGRAPH-FA6A')
143 self.assertEqual(name('\uFBF9'),
144 'ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA '
145 'ABOVE WITH ALEF MAKSURA ISOLATED FORM')
146 self.assertEqual(name('\U00013460', None), None if self.old else
147 'EGYPTIAN HIEROGLYPH-13460')
148 self.assertEqual(name('\U000143FA', None), None if self.old else
149 'EGYPTIAN HIEROGLYPH-143FA')
150 self.assertEqual(name('\U00018B00', None), None if self.old else
151 'KHITAN SMALL SCRIPT CHARACTER-18B00')
152 self.assertEqual(name('\U00018CD5', None), None if self.old else
153 'KHITAN SMALL SCRIPT CHARACTER-18CD5')
154 self.assertEqual(name('\U00018CFF', None), None if self.old else
155 'KHITAN SMALL SCRIPT CHARACTER-18CFF')
156 self.assertEqual(name('\U0001B170', None), None if self.old else
157 'NUSHU CHARACTER-1B170')
158 self.assertEqual(name('\U0001B2FB', None), None if self.old else
159 'NUSHU CHARACTER-1B2FB')
160 self.assertEqual(name('\U0001FBA8', None), None if self.old else
161 'BOX DRAWINGS LIGHT DIAGONAL UPPER CENTRE TO '
162 'MIDDLE LEFT AND MIDDLE RIGHT TO LOWER CENTRE')
163 self.assertEqual(name('\U0002A6D6'), 'CJK UNIFIED IDEOGRAPH-2A6D6')
164 self.assertEqual(name('\U0002FA1D'), 'CJK COMPATIBILITY IDEOGRAPH-2FA1D')
165 self.assertEqual(name('\U000323AF', None), None if self.old else
166 'CJK UNIFIED IDEOGRAPH-323AF')
167
168 @requires_resource('cpu')
169 def test_name_inverse_lookup(self):
170 for char in iterallchars():
171 looked_name = self.db.name(char, None)
172 if looked_name is not None:
173 self.assertEqual(self.db.lookup(looked_name), char)
174
175 def test_no_names_in_pua(self):
176 puas = [*range(0xe000, 0xf8ff),
177 *range(0xf0000, 0xfffff),
178 *range(0x100000, 0x10ffff)]
179 for i in puas:
180 char = chr(i)
181 self.assertRaises(ValueError, self.db.name, char)
182
183 def test_lookup_nonexistant(self):
184 # just make sure that lookup can fail
185 for nonexistent in [
186 "LATIN SMLL LETR A",
187 "OPEN HANDS SIGHS",
188 "DREGS",
189 "HANDBUG",
190 "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK",
191 "???",
192 "CJK UNIFIED IDEOGRAPH-03400",
193 "CJK UNIFIED IDEOGRAPH-020000",
194 "CJK UNIFIED IDEOGRAPH-33FF",
195 "CJK UNIFIED IDEOGRAPH-F900",
196 "CJK UNIFIED IDEOGRAPH-13460",
197 "CJK UNIFIED IDEOGRAPH-17000",
198 "CJK UNIFIED IDEOGRAPH-18B00",
199 "CJK UNIFIED IDEOGRAPH-1B170",
200 "CJK COMPATIBILITY IDEOGRAPH-3400",
201 "TANGUT IDEOGRAPH-3400",
202 "HANGUL SYLLABLE AC00",
203 ]:
204 self.assertRaises(KeyError, self.db.lookup, nonexistent)
205
206 def test_digit(self):
207 self.assertEqual(self.db.digit('A', None), None)
208 self.assertEqual(self.db.digit('9'), 9)
209 self.assertEqual(self.db.digit('\u215b', None), None)
210 self.assertEqual(self.db.digit('\u2468'), 9)
211 self.assertEqual(self.db.digit('\U00020000', None), None)
212 self.assertEqual(self.db.digit('\U0001D7FD'), 7)
213
214 # New in 13.0.0
215 self.assertEqual(self.db.digit('\U0001fbf9', None), 9)
216 # New in 14.0.0
217 self.assertEqual(self.db.digit('\U00016ac9', None), 9)
218 # New in 15.0.0
219 self.assertEqual(self.db.digit('\U0001e4f9', None), 9)
220
221 self.assertRaises(TypeError, self.db.digit)
222 self.assertRaises(TypeError, self.db.digit, 'xx')
223 self.assertRaises(ValueError, self.db.digit, 'x')
224
225 def test_numeric(self):
226 self.assertEqual(self.db.numeric('A',None), None)
227 self.assertEqual(self.db.numeric('9'), 9)
228 self.assertEqual(self.db.numeric('\u215b'), 0.125)
229 self.assertEqual(self.db.numeric('\u2468'), 9.0)
230 self.assertEqual(self.db.numeric('\U00020000', None), None)
231
232 # New in 4.1.0
233 self.assertEqual(self.db.numeric('\U0001012A', None), None if self.old else 9000)
234 # Changed in 4.1.0
235 self.assertEqual(self.db.numeric('\u5793', None), 1e20 if self.old else None)
236 # New in 5.0.0
237 self.assertEqual(self.db.numeric('\u07c0', None), None if self.old else 0.0)
238 # New in 5.1.0
239 self.assertEqual(self.db.numeric('\ua627', None), None if self.old else 7.0)
240 # Changed in 5.2.0
241 self.assertEqual(self.db.numeric('\u09f6'), 3.0 if self.old else 3/16)
242 # New in 6.0.0
243 self.assertEqual(self.db.numeric('\u0b72', None), None if self.old else 0.25)
244 # New in 12.0.0
245 self.assertEqual(self.db.numeric('\U0001ed3c', None), None if self.old else 0.5)
246 # New in 13.0.0
247 self.assertEqual(self.db.numeric('\U0001fbf9', None), None if self.old else 9)
248 # New in 14.0.0
249 self.assertEqual(self.db.numeric('\U00016ac9', None), None if self.old else 9)
250 # New in 15.0.0
251 self.assertEqual(self.db.numeric('\U0001e4f9', None), None if self.old else 9)
252
253 self.assertRaises(TypeError, self.db.numeric)
254 self.assertRaises(TypeError, self.db.numeric, 'xx')
255 self.assertRaises(ValueError, self.db.numeric, 'x')
256
257 def test_decimal(self):
258 self.assertEqual(self.db.decimal('A',None), None)
259 self.assertEqual(self.db.decimal('9'), 9)
260 self.assertEqual(self.db.decimal('\u215b', None), None)
261 self.assertEqual(self.db.decimal('\u2468', None), None)
262 self.assertEqual(self.db.decimal('\U00020000', None), None)
263 self.assertEqual(self.db.decimal('\U0001D7FD'), 7)
264
265 # New in 4.1.0
266 self.assertEqual(self.db.decimal('\xb2', None), 2 if self.old else None)
267 self.assertEqual(self.db.decimal('\u1369', None), 1 if self.old else None)
268 # New in 5.0.0
269 self.assertEqual(self.db.decimal('\u07c0', None), None if self.old else 0)
270 # New in 13.0.0
271 self.assertEqual(self.db.decimal('\U0001fbf9', None), None if self.old else 9)
272 # New in 14.0.0
273 self.assertEqual(self.db.decimal('\U00016ac9', None), None if self.old else 9)
274 # New in 15.0.0
275 self.assertEqual(self.db.decimal('\U0001e4f9', None), None if self.old else 9)
276
277 self.assertRaises(TypeError, self.db.decimal)
278 self.assertRaises(TypeError, self.db.decimal, 'xx')
279 self.assertRaises(ValueError, self.db.decimal, 'x')
280
281 def test_category(self):
282 self.assertEqual(self.db.category('\uFFFE'), 'Cn')
283 self.assertEqual(self.db.category('a'), 'Ll')
284 self.assertEqual(self.db.category('A'), 'Lu')
285 self.assertEqual(self.db.category('\U00020000'), 'Lo')
286
287 # New in 4.1.0
288 self.assertEqual(self.db.category('\U0001012A'), 'Cn' if self.old else 'No')
289 self.assertEqual(self.db.category('\U000e01ef'), 'Cn' if self.old else 'Mn')
290 # New in 5.1.0
291 self.assertEqual(self.db.category('\u0374'), 'Sk' if self.old else 'Lm')
292 # Changed in 13.0.0
293 self.assertEqual(self.db.category('\u0b55'), 'Cn' if self.old else 'Mn')
294 self.assertEqual(self.db.category('\U0003134a'), 'Cn' if self.old else 'Lo')
295 # Changed in 14.0.0
296 self.assertEqual(self.db.category('\u061d'), 'Cn' if self.old else 'Po')
297 self.assertEqual(self.db.category('\U0002b738'), 'Cn' if self.old else 'Lo')
298 # Changed in 15.0.0
299 self.assertEqual(self.db.category('\u0cf3'), 'Cn' if self.old else 'Mc')
300 self.assertEqual(self.db.category('\U000323af'), 'Cn' if self.old else 'Lo')
301
302 self.assertRaises(TypeError, self.db.category)
303 self.assertRaises(TypeError, self.db.category, 'xx')
304
305 def test_bidirectional(self):
306 self.assertEqual(self.db.bidirectional('\uFFFE'), '')
307 self.assertEqual(self.db.bidirectional(' '), 'WS')
308 self.assertEqual(self.db.bidirectional('A'), 'L')
309 self.assertEqual(self.db.bidirectional('\U00020000'), 'L')
310
311 # New in 4.1.0
312 self.assertEqual(self.db.bidirectional('+'), 'ET' if self.old else 'ES')
313 self.assertEqual(self.db.bidirectional('\u0221'), '' if self.old else 'L')
314 self.assertEqual(self.db.bidirectional('\U000e01ef'), '' if self.old else 'NSM')
315 # New in 13.0.0
316 self.assertEqual(self.db.bidirectional('\u0b55'), '' if self.old else 'NSM')
317 self.assertEqual(self.db.bidirectional('\U0003134a'), '' if self.old else 'L')
318 # New in 14.0.0
319 self.assertEqual(self.db.bidirectional('\u061d'), '' if self.old else 'AL')
320 self.assertEqual(self.db.bidirectional('\U0002b738'), '' if self.old else 'L')
321 # New in 15.0.0
322 self.assertEqual(self.db.bidirectional('\u0cf3'), '' if self.old else 'L')
323 self.assertEqual(self.db.bidirectional('\U000323af'), '' if self.old else 'L')
324 # New in 16.0.0
325 self.assertEqual(self.db.bidirectional('\u0897'), '' if self.old else 'NSM')
326 self.assertEqual(self.db.bidirectional('\U0001fbef'), '' if self.old else 'ON')
327
328 self.assertRaises(TypeError, self.db.bidirectional)
329 self.assertRaises(TypeError, self.db.bidirectional, 'xx')
330
331 def test_decomposition(self):
332 self.assertEqual(self.db.decomposition('\uFFFE'),'')
333 self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034')
334
335 # New in 4.1.0
336 self.assertEqual(self.db.decomposition('\u03f9'), '' if self.old else '<compat> 03A3')
337 # New in 13.0.0
338 self.assertEqual(self.db.decomposition('\uab69'), '' if self.old else '<super> 028D')
339 self.assertEqual(self.db.decomposition('\U00011938'), '' if self.old else '11935 11930')
340 self.assertEqual(self.db.decomposition('\U0001fbf9'), '' if self.old else '<font> 0039')
341 # New in 14.0.0
342 self.assertEqual(self.db.decomposition('\ua7f2'), '' if self.old else '<super> 0043')
343 self.assertEqual(self.db.decomposition('\U000107ba'), '' if self.old else '<super> 1DF1E')
344 # New in 15.0.0
345 self.assertEqual(self.db.decomposition('\U0001e06d'), '' if self.old else '<super> 04B1')
346 # New in 16.0.0
347 self.assertEqual(self.db.decomposition('\U0001CCD6'), '' if self.old else '<font> 0041')
348
349 # Hangul characters
350 self.assertEqual(self.db.decomposition('\uAC00'), '1100 1161')
351 self.assertEqual(self.db.decomposition('\uD4DB'), '1111 1171 11B6')
352 self.assertEqual(self.db.decomposition('\uC2F8'), '110A 1161')
353 self.assertEqual(self.db.decomposition('\uD7A3'), '1112 1175 11C2')
354
355 self.assertRaises(TypeError, self.db.decomposition)
356 self.assertRaises(TypeError, self.db.decomposition, 'xx')
357
358 def test_mirrored(self):
359 self.assertEqual(self.db.mirrored('\uFFFE'), 0)
360 self.assertEqual(self.db.mirrored('a'), 0)
361 self.assertEqual(self.db.mirrored('\u2201'), 1)
362 self.assertEqual(self.db.mirrored('\U00020000'), 0)
363
364 # New in 5.0.0
365 self.assertEqual(self.db.mirrored('\u0f3a'), 0 if self.old else 1)
366 self.assertEqual(self.db.mirrored('\U0001d7c3'), 0 if self.old else 1)
367 # New in 11.0.0
368 self.assertEqual(self.db.mirrored('\u29a1'), 1 if self.old else 0)
369 # New in 14.0.0
370 self.assertEqual(self.db.mirrored('\u2e5c'), 0 if self.old else 1)
371 # New in 16.0.0
372 self.assertEqual(self.db.mirrored('\u226D'), 0 if self.old else 1)
373
374 self.assertRaises(TypeError, self.db.mirrored)
375 self.assertRaises(TypeError, self.db.mirrored, 'xx')
376
377 def test_combining(self):
378 self.assertEqual(self.db.combining('\uFFFE'), 0)
379 self.assertEqual(self.db.combining('a'), 0)
380 self.assertEqual(self.db.combining('\u20e1'), 230)
381 self.assertEqual(self.db.combining('\U00020000'), 0)
382
383 # New in 4.1.0
384 self.assertEqual(self.db.combining('\u0350'), 0 if self.old else 230)
385 # New in 9.0.0
386 self.assertEqual(self.db.combining('\U0001e94a'), 0 if self.old else 7)
387 # New in 13.0.0
388 self.assertEqual(self.db.combining('\u1abf'), 0 if self.old else 220)
389 self.assertEqual(self.db.combining('\U00016ff1'), 0 if self.old else 6)
390 # New in 14.0.0
391 self.assertEqual(self.db.combining('\u0c3c'), 0 if self.old else 7)
392 self.assertEqual(self.db.combining('\U0001e2ae'), 0 if self.old else 230)
393 # New in 15.0.0
394 self.assertEqual(self.db.combining('\U00010efd'), 0 if self.old else 220)
395 # New in 16.0.0
396 self.assertEqual(self.db.combining('\u0897'), 0 if self.old else 230)
397
398 self.assertRaises(TypeError, self.db.combining)
399 self.assertRaises(TypeError, self.db.combining, 'xx')
400
401 def test_normalization(self):
402 # Test normalize() and is_normalized()
403 def check(ch, expected):
404 if isinstance(expected, str):
405 expected = [expected]*4
406 forms = ('NFC', 'NFD', 'NFKC', 'NFKD')
407 result = [self.db.normalize(form, ch) for form in forms]
408 self.assertEqual(ascii(result), ascii(list(expected)))
409 self.assertEqual([self.db.is_normalized(form, ch) for form in forms],
410 [ch == y for x, y in zip(result, expected)])
411
412 check('', '')
413 check('A', 'A')
414 check(' ', ' ')
415 check('\U0010ffff', '\U0010ffff')
416 check('abc', 'abc')
417 # Broken in 4.0.0
418 check('\u0340', '\u0300')
419 check('\u0300', '\u0300')
420 check('\U0002fa1d', '\U0002a600')
421 check('\U0002a600', '\U0002a600')
422 check('\u0344', '\u0308\u0301')
423 check('\u0308\u0301', '\u0308\u0301')
424 # Broken in 4.0.0 and 4.0.1
425 check('\U0001d1bc', '\U0001d1ba\U0001d165')
426 check('\U0001d1ba\U0001d165', '\U0001d1ba\U0001d165')
427 check('\ufb2c', '\u05e9\u05bc\u05c1')
428 check('\u05e9\u05bc\u05c1', '\u05e9\u05bc\u05c1')
429 check('\U0001d1c0', '\U0001d1ba\U0001d165\U0001d16f')
430 check('\U0001d1ba\U0001d165\U0001d16f', '\U0001d1ba\U0001d165\U0001d16f')
431
432 # Broken in 4.0.0
433 check('\xa0', ['\xa0', '\xa0', ' ', ' '])
434 check('\u2003', ['\u2003', '\u2003', ' ', ' '])
435 check('\U0001d7ff', ['\U0001d7ff', '\U0001d7ff', '9', '9'])
436
437 check('\xa8', ['\xa8', '\xa8', ' \u0308', ' \u0308'])
438 check(' \u0308', ' \u0308')
439
440 check('\xc0', ['\xc0', 'A\u0300']*2)
441 check('A\u0300', ['\xc0', 'A\u0300']*2)
442
443 check('\ud7a3', ['\ud7a3', '\u1112\u1175\u11c2']*2)
444 check('\u1112\u1175\u11c2', ['\ud7a3', '\u1112\u1175\u11c2']*2)
445
446 check('\xb4', ['\xb4', '\xb4', ' \u0301', ' \u0301'])
447 check('\u1ffd', ['\xb4', '\xb4', ' \u0301', ' \u0301'])
448 check(' \u0301', ' \u0301')
449
450 check('\xc5', ['\xc5', 'A\u030a']*2)
451 check('\u212b', ['\xc5', 'A\u030a']*2)
452 check('A\u030a', ['\xc5', 'A\u030a']*2)
453
454 check('\u1f71', ['\u03ac', '\u03b1\u0301']*2)
455 check('\u03ac', ['\u03ac', '\u03b1\u0301']*2)
456 check('\u03b1\u0301', ['\u03ac', '\u03b1\u0301']*2)
457
458 check('\u01c4', ['\u01c4', '\u01c4', 'D\u017d', 'DZ\u030c'])
459 check('D\u017d', ['D\u017d', 'DZ\u030c']*2)
460 check('DZ\u030c', ['D\u017d', 'DZ\u030c']*2)
461
462 check('\u1fed', ['\u1fed', '\xa8\u0300', ' \u0308\u0300', ' \u0308\u0300'])
463 check('\xa8\u0300', ['\u1fed', '\xa8\u0300', ' \u0308\u0300', ' \u0308\u0300'])
464 check(' \u0308\u0300', ' \u0308\u0300')
465
466 check('\u326e', ['\u326e', '\u326e', '\uac00', '\u1100\u1161'])
467 check('\u320e', ['\u320e', '\u320e', '(\uac00)', '(\u1100\u1161)'])
468 check('(\uac00)', ['(\uac00)', '(\u1100\u1161)']*2)
469 check('(\u1100\u1161)', ['(\uac00)', '(\u1100\u1161)']*2)
470
471 check('\u0385', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301'])
472 check('\u1fee', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301'])
473 check('\xa8\u0301', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301'])
474 check(' \u0308\u0301', ' \u0308\u0301')
475
476 check('\u1fdf', ['\u1fdf', '\u1ffe\u0342', ' \u0314\u0342', ' \u0314\u0342'])
477 check('\u1ffe\u0342', ['\u1fdf', '\u1ffe\u0342', ' \u0314\u0342', ' \u0314\u0342'])
478 check('\u1ffe', ['\u1ffe', '\u1ffe', ' \u0314', ' \u0314'])
479 check(' \u0314\u0342', ' \u0314\u0342')
480
481 check('\u03d3', ['\u03d3', '\u03d2\u0301', '\u038e', '\u03a5\u0301'])
482 check('\u03d2\u0301', ['\u03d3', '\u03d2\u0301', '\u038e', '\u03a5\u0301'])
483 check('\u038e', ['\u038e', '\u03a5\u0301']*2)
484 check('\u1feb', ['\u038e', '\u03a5\u0301']*2)
485 check('\u03a5\u0301', ['\u038e', '\u03a5\u0301']*2)
486
487 check('\u0626', ['\u0626', '\u064a\u0654']*2)
488 check('\u064a\u0654', ['\u0626', '\u064a\u0654']*2)
489 check('\ufe89', ['\ufe89', '\ufe89', '\u0626', '\u064a\u0654'])
490 check('\ufe8a', ['\ufe8a', '\ufe8a', '\u0626', '\u064a\u0654'])
491 check('\ufe8b', ['\ufe8b', '\ufe8b', '\u0626', '\u064a\u0654'])
492 check('\ufe8c', ['\ufe8c', '\ufe8c', '\u0626', '\u064a\u0654'])
493
494 check('\ufef9', ['\ufef9', '\ufef9', '\u0644\u0625', '\u0644\u0627\u0655'])
495 check('\ufefa', ['\ufefa', '\ufefa', '\u0644\u0625', '\u0644\u0627\u0655'])
496 check('\ufefb', ['\ufefb', '\ufefb', '\u0644\u0627', '\u0644\u0627'])
497 check('\ufefc', ['\ufefc', '\ufefc', '\u0644\u0627', '\u0644\u0627'])
498 check('\u0644\u0625', ['\u0644\u0625', '\u0644\u0627\u0655']*2)
499 check('\u0644\u0627\u0655', ['\u0644\u0625', '\u0644\u0627\u0655']*2)
500 check('\u0644\u0627', '\u0644\u0627')
501
502 # Broken in 4.0.0
503 check('\u327c', '\u327c' if self.old else
504 ['\u327c', '\u327c', '\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169'])
505 check('\ucc38\uace0', ['\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']*2)
506 check('\ucc38', ['\ucc38', '\u110e\u1161\u11b7']*2)
507 check('\u110e\u1161\u11b7\u1100\u1169',
508 ['\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']*2)
509 check('\u110e\u1161\u11b7\u1100',
510 ['\ucc38\u1100', '\u110e\u1161\u11b7\u1100']*2)
511 check('\u110e\u1161\u11b7',
512 ['\ucc38', '\u110e\u1161\u11b7']*2)
513 check('\u110e\u1161',
514 ['\ucc28', '\u110e\u1161']*2)
515 check('\u110e', '\u110e')
516 # Broken in 4.0.0-12.0.0
517 check('\U00011938', '\U00011938' if self.old else
518 ['\U00011938', '\U00011935\U00011930']*2)
519 check('\U00011935\U00011930', ['\U00011938', '\U00011935\U00011930']*2)
520 # New in 4.0.1
521 check('\u321d', '\u321d' if self.old else
522 ['\u321d', '\u321d', '(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)'])
523 check('(\uc624\uc804)',
524 ['(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']*2)
525 check('(\u110b\u1169\u110c\u1165\u11ab)',
526 ['(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']*2)
527 check('\u4d57', '\u4d57')
528 check('\u45d7', '\u45d7' if self.old else '\u45d7')
529 check('\U0002f9bf', '\u4d57' if self.old else '\u45d7')
530 # New in 4.1.0
531 check('\u03a3', '\u03a3')
532 check('\u03f9', '\u03f9' if self.old else
533 ['\u03f9', '\u03f9', '\u03a3', '\u03a3'])
534 # New in 5.0.0
535 check('\u1b06', '\u1b06' if self.old else ['\u1b06', '\u1b05\u1b35']*2)
536 # New in 5.2.0
537 check('\U0001f213', '\U0001f213' if self.old else
538 ['\U0001f213', '\U0001f213', '\u30c7', '\u30c6\u3099'])
539 # New in 6.1.0
540 check('\ufa2e', '\ufa2e' if self.old else '\u90de')
541 # New in 13.0.0
542 check('\U00011938', '\U00011938' if self.old else
543 ['\U00011938', '\U00011935\U00011930', '\U00011938', '\U00011935\U00011930'])
544 check('\U0001fbf9', '\U0001fbf9' if self.old else
545 ['\U0001fbf9', '\U0001fbf9', '9', '9'])
546 # New in 14.0.0
547 check('\U000107ba', '\U000107ba' if self.old else
548 ['\U000107ba', '\U000107ba', '\U0001df1e', '\U0001df1e'])
549 # New in 15.0.0
550 check('\U0001e06d', '\U0001e06d' if self.old else
551 ['\U0001e06d', '\U0001e06d', '\u04b1', '\u04b1'])
552 # New in 16.0.0
553 check('\U0001ccd6', '\U0001ccd6' if self.old else
554 ['\U0001ccd6', '\U0001ccd6', 'A', 'A'])
555
556 self.assertRaises(TypeError, self.db.normalize)
557 self.assertRaises(TypeError, self.db.normalize, 'NFC')
558 self.assertRaises(ValueError, self.db.normalize, 'SPAM', 'A')
559
560 self.assertRaises(TypeError, self.db.is_normalized)
561 self.assertRaises(TypeError, self.db.is_normalized, 'NFC')
562 self.assertRaises(ValueError, self.db.is_normalized, 'SPAM', 'A')
563
564 def test_pr29(self):
565 # https://www.unicode.org/review/pr-29.html
566 # See issues #1054943 and #10254.
567 composed = ("\u0b47\u0300\u0b3e", "\u1100\u0300\u1161",
568 'Li\u030dt-s\u1e73\u0301',
569 '\u092e\u093e\u0930\u094d\u0915 \u091c\u093c'
570 + '\u0941\u0915\u0947\u0930\u092c\u0930\u094d\u0917',
571 '\u0915\u093f\u0930\u094d\u0917\u093f\u091c\u093c'
572 + '\u0938\u094d\u0924\u093e\u0928')
573 for text in composed:
574 self.assertEqual(self.db.normalize('NFC', text), text)
575
576 def test_issue10254(self):
577 # Crash reported in #10254
578 # New in 4.1.0
579 a = 'C\u0338' * 20 + 'C\u0327'
580 b = 'C\u0338' * 20 + '\xC7'
581 self.assertEqual(self.db.normalize('NFC', a), b)
582
583 def test_issue29456(self):
584 # Fix #29456
585 u1176_str_a = '\u1100\u1176\u11a8'
586 u1176_str_b = '\u1100\u1176\u11a8'
587 u11a7_str_a = '\u1100\u1175\u11a7'
588 u11a7_str_b = '\uae30\u11a7'
589 u11c3_str_a = '\u1100\u1175\u11c3'
590 u11c3_str_b = '\uae30\u11c3'
591 self.assertEqual(self.db.normalize('NFC', u1176_str_a), u1176_str_b)
592 # New in 4.1.0
593 self.assertEqual(self.db.normalize('NFC', u11a7_str_a), u11a7_str_b)
594 self.assertEqual(self.db.normalize('NFC', u11c3_str_a), u11c3_str_b)
595
596 def test_east_asian_width(self):
597 eaw = self.db.east_asian_width
598 self.assertRaises(TypeError, eaw, b'a')
599 self.assertRaises(TypeError, eaw, bytearray())
600 self.assertRaises(TypeError, eaw, '')
601 self.assertRaises(TypeError, eaw, 'ra')
602 self.assertEqual(eaw('\x1e'), 'N')
603 self.assertEqual(eaw('\x20'), 'Na')
604 self.assertEqual(eaw('\uC894'), 'W')
605 self.assertEqual(eaw('\uFF66'), 'H')
606 self.assertEqual(eaw('\uFF1F'), 'F')
607 self.assertEqual(eaw('\u2010'), 'A')
608 self.assertEqual(eaw('\U00020000'), 'W')
609
610 # New in 4.1.0
611 self.assertEqual(eaw('\u0350'), 'N' if self.old else 'A')
612 self.assertEqual(eaw('\U000e01ef'), 'N' if self.old else 'A')
613 # New in 5.2.0
614 self.assertEqual(eaw('\u115a'), 'N' if self.old else 'W')
615 # New in 9.0.0
616 self.assertEqual(eaw('\u231a'), 'N' if self.old else 'W')
617 self.assertEqual(eaw('\u2614'), 'N' if self.old else 'W')
618 self.assertEqual(eaw('\U0001f19a'), 'N' if self.old else 'W')
619 self.assertEqual(eaw('\U0001f991'), 'N' if self.old else 'W')
620 self.assertEqual(eaw('\U0001f9c0'), 'N' if self.old else 'W')
621 # New in 12.0.0
622 self.assertEqual(eaw('\u32ff'), 'N' if self.old else 'W')
623 self.assertEqual(eaw('\U0001fa95'), 'N' if self.old else 'W')
624 # New in 13.0.0
625 self.assertEqual(eaw('\u31bb'), 'N' if self.old else 'W')
626 self.assertEqual(eaw('\U0003134a'), 'N' if self.old else 'W')
627 # New in 14.0.0
628 self.assertEqual(eaw('\u9ffd'), 'N' if self.old else 'W')
629 self.assertEqual(eaw('\U0002b738'), 'N' if self.old else 'W')
630 # New in 15.0.0
631 self.assertEqual(eaw('\U000323af'), 'N' if self.old else 'W')
632 # New in 16.0.0
633 self.assertEqual(eaw('\u2630'), 'N' if self.old else 'W')
634 self.assertEqual(eaw('\U0001FAE9'), 'N' if self.old else 'W')
635
636 def test_east_asian_width_unassigned(self):
637 eaw = self.db.east_asian_width
638 # unassigned
639 for char in '\u0530\u0ecf\u10c6\u20fc\uaaca\U000107bd\U000115f2':
640 self.assertEqual(eaw(char), 'N')
641 self.assertIs(self.db.name(char, None), None)
642
643 # unassigned but reserved for CJK
644 for char in ('\U0002A6E0\U0002FA20\U0003134B\U0003FFFD'
645 '\uFA6E\uFADA'): # New in 5.2.0
646 self.assertEqual(eaw(char), 'W')
647 self.assertIs(self.db.name(char, None), None)
648
649 # private use areas
650 for char in '\uE000\uF800\U000F0000\U000FFFEE\U00100000\U0010FFF0':
651 self.assertEqual(eaw(char), 'A')
652 self.assertIs(self.db.name(char, None), None)
653
654
655 class Unicode_3_2_0_FunctionsTest(UnicodeFunctionsTest):
656 db = unicodedata.ucd_3_2_0
657 old = True
658 expectedchecksum = ('883824cb6c0ccf994e4451ebf281e2d6d479af47'
659 if quicktest else
660 'caf1a7f2f380f927461837f1901ef20683f98683')
661
662
663 class UnicodeMiscTest(unittest.TestCase):
664 db = unicodedata
665
666 @cpython_only
667 def test_disallow_instantiation(self):
668 # Ensure that the type disallows instantiation (bpo-43916)
669 check_disallow_instantiation(self, unicodedata.UCD)
670
671 @force_not_colorized
672 def test_failed_import_during_compiling(self):
673 # Issue 4367
674 # Decoding \N escapes requires the unicodedata module. If it can't be
675 # imported, we shouldn't segfault.
676
677 # This program should raise a SyntaxError in the eval.
678 code = "import sys;" \
679 "sys.modules['unicodedata'] = None;" \
680 """eval("'\\\\N{SOFT HYPHEN}'")"""
681 # We use a separate process because the unicodedata module may already
682 # have been loaded in this process.
683 result = script_helper.assert_python_failure("-c", code)
684 error = "SyntaxError: (unicode error) \\N escapes not supported " \
685 "(can't load unicodedata module)"
686 self.assertIn(error, result.err.decode("ascii"))
687
688 def test_decimal_numeric_consistent(self):
689 # Test that decimal and numeric are consistent,
690 # i.e. if a character has a decimal value,
691 # its numeric value should be the same.
692 count = 0
693 for c in iterallchars():
694 dec = self.db.decimal(c, -1)
695 if dec != -1:
696 self.assertEqual(dec, self.db.numeric(c))
697 count += 1
698 self.assertTrue(count >= 10, count) # should have tested at least the ASCII digits
699
700 def test_digit_numeric_consistent(self):
701 # Test that digit and numeric are consistent,
702 # i.e. if a character has a digit value,
703 # its numeric value should be the same.
704 count = 0
705 for c in iterallchars():
706 dec = self.db.digit(c, -1)
707 if dec != -1:
708 self.assertEqual(dec, self.db.numeric(c))
709 count += 1
710 self.assertTrue(count >= 10, count) # should have tested at least the ASCII digits
711
712 def test_normalize_consistent(self):
713 allchars = list(iterallchars())
714 for form in ('NFC', 'NFD', 'NFKC', 'NFKD'):
715 for c in allchars:
716 norm = self.db.normalize(form, c)
717 self.assertEqual(self.db.is_normalized(form, c), norm == c)
718 if norm != c:
719 self.assertEqual(self.db.normalize(form, norm), norm)
720 self.assertTrue(self.db.is_normalized(form, norm))
721
722 def test_bug_1704793(self):
723 self.assertEqual(self.db.lookup("GOTHIC LETTER FAIHU"), '\U00010346')
724
725 def test_ucd_510(self):
726 # In UCD 5.1.0, a mirrored property changed wrt. UCD 3.2.0
727 self.assertTrue(unicodedata.mirrored("\u0f3a"))
728 self.assertTrue(not unicodedata.ucd_3_2_0.mirrored("\u0f3a"))
729 # Also, we now have two ways of representing
730 # the upper-case mapping: as delta, or as absolute value
731 self.assertTrue("a".upper()=='A')
732 self.assertTrue("\u1d79".upper()=='\ua77d')
733 self.assertTrue(".".upper()=='.')
734
735 @requires_resource('cpu')
736 def test_bug_5828(self):
737 self.assertEqual("\u1d79".lower(), "\u1d79")
738 # Only U+0000 should have U+0000 as its upper/lower/titlecase variant
739 self.assertEqual(
740 [
741 c for c in iterallchars()
742 if "\x00" in (c.lower(), c.upper(), c.title())
743 ],
744 ["\x00"]
745 )
746
747 def test_bug_4971(self):
748 # LETTER DZ WITH CARON: DZ, Dz, dz
749 self.assertEqual("\u01c4".title(), "\u01c5")
750 self.assertEqual("\u01c5".title(), "\u01c5")
751 self.assertEqual("\u01c6".title(), "\u01c5")
752
753 def test_linebreak_7643(self):
754 for c in iterallchars():
755 lines = (c + 'A').splitlines()
756 if c in ('\x0a', '\x0b', '\x0c', '\x0d', '\x85',
757 '\x1c', '\x1d', '\x1e', '\u2028', '\u2029'):
758 self.assertEqual(len(lines), 2,
759 r"%a should be a linebreak" % c)
760 else:
761 self.assertEqual(len(lines), 1,
762 r"%a should not be a linebreak" % c)
763
764
765 class NormalizationTest(unittest.TestCase):
766 @staticmethod
767 def check_version(testfile):
768 hdr = testfile.readline()
769 return unicodedata.unidata_version in hdr
770
771 @staticmethod
772 def unistr(data):
773 data = [int(x, 16) for x in data.split(" ")]
774 return "".join([chr(x) for x in data])
775
776 @requires_resource('network')
777 @requires_resource('cpu')
778 def test_normalization(self):
779 TESTDATAFILE = "NormalizationTest.txt"
780 TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{TESTDATAFILE}"
781
782 # Hit the exception early
783 try:
784 testdata = open_urlresource(TESTDATAURL, encoding="utf-8",
785 check=self.check_version)
786 except PermissionError:
787 self.skipTest(f"Permission error when downloading {TESTDATAURL} "
788 f"into the test data directory")
789 except (OSError, HTTPException) as exc:
790 self.skipTest(f"Failed to download {TESTDATAURL}: {exc}")
791
792 with testdata:
793 self.run_normalization_tests(testdata, unicodedata)
794
795 @requires_resource('cpu')
796 def test_normalization_3_2_0(self):
797 testdatafile = findfile('NormalizationTest-3.2.0.txt')
798 with open(testdatafile, encoding='utf-8') as testdata:
799 self.run_normalization_tests(testdata, unicodedata.ucd_3_2_0)
800
801 def run_normalization_tests(self, testdata, ucd):
802 part = None
803 part1_data = set()
804
805 NFC = partial(ucd.normalize, "NFC")
806 NFKC = partial(ucd.normalize, "NFKC")
807 NFD = partial(ucd.normalize, "NFD")
808 NFKD = partial(ucd.normalize, "NFKD")
809 is_normalized = ucd.is_normalized
810
811 for line in testdata:
812 if '#' in line:
813 line = line.split('#')[0]
814 line = line.strip()
815 if not line:
816 continue
817 if line.startswith("@Part"):
818 part = line.split()[0]
819 continue
820 c1,c2,c3,c4,c5 = [self.unistr(x) for x in line.split(';')[:-1]]
821
822 # Perform tests
823 self.assertTrue(c2 == NFC(c1) == NFC(c2) == NFC(c3), line)
824 self.assertTrue(c4 == NFC(c4) == NFC(c5), line)
825 self.assertTrue(c3 == NFD(c1) == NFD(c2) == NFD(c3), line)
826 self.assertTrue(c5 == NFD(c4) == NFD(c5), line)
827 self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \
828 NFKC(c3) == NFKC(c4) == NFKC(c5),
829 line)
830 self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \
831 NFKD(c3) == NFKD(c4) == NFKD(c5),
832 line)
833
834 self.assertTrue(is_normalized("NFC", c2))
835 self.assertTrue(is_normalized("NFC", c4))
836
837 self.assertTrue(is_normalized("NFD", c3))
838 self.assertTrue(is_normalized("NFD", c5))
839
840 self.assertTrue(is_normalized("NFKC", c4))
841 self.assertTrue(is_normalized("NFKD", c5))
842
843 # Record part 1 data
844 if part == "@Part1":
845 part1_data.add(c1)
846
847 # Perform tests for all other data
848 for X in iterallchars():
849 if X in part1_data:
850 continue
851 self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), ord(X))
852
853 def test_edge_cases(self):
854 self.assertRaises(TypeError, unicodedata.normalize)
855 self.assertRaises(ValueError, unicodedata.normalize, 'unknown', 'xx')
856 self.assertEqual(unicodedata.normalize('NFKC', ''), '')
857
858 def test_bug_834676(self):
859 # Check for bug 834676
860 unicodedata.normalize('NFC', '\ud55c\uae00')
861
862 def test_normalize_return_type(self):
863 # gh-129569: normalize() return type must always be str
864 normalize = unicodedata.normalize
865
866 class MyStr(str):
867 pass
868
869 normalization_forms = ("NFC", "NFKC", "NFD", "NFKD")
870 input_strings = (
871 # normalized strings
872 "",
873 "ascii",
874 # unnormalized strings
875 "\u1e0b\u0323",
876 "\u0071\u0307\u0323",
877 )
878
879 for form in normalization_forms:
880 for input_str in input_strings:
881 with self.subTest(form=form, input_str=input_str):
882 self.assertIs(type(normalize(form, input_str)), str)
883 self.assertIs(type(normalize(form, MyStr(input_str))), str)
884
885
886 if __name__ == "__main__":
887 unittest.main()