| 1 |
""" Tests for the unicodedata module. |
| 2 |
|
| 3 |
Written by Marc-Andre Lemburg ([email protected]). |
| 4 |
|
| 5 |
(c) Copyright CNRI, All Rights Reserved. NO WARRANTY. |
| 6 |
|
| 7 |
""" |
| 8 |
|
| 9 |
from functools import partial |
| 10 |
import hashlib |
| 11 |
from http.client import HTTPException |
| 12 |
import sys |
| 13 |
import unicodedata |
| 14 |
import unittest |
| 15 |
from test.support import ( |
| 16 |
open_urlresource, |
| 17 |
requires_resource, |
| 18 |
script_helper, |
| 19 |
cpython_only, |
| 20 |
check_disallow_instantiation, |
| 21 |
force_not_colorized, |
| 22 |
is_resource_enabled, |
| 23 |
findfile, |
| 24 |
) |
| 25 |
|
| 26 |
|
| 27 |
quicktest = not is_resource_enabled('cpu') |
| 28 |
|
| 29 |
def iterallchars(): |
| 30 |
maxunicode = 0xffff if quicktest else sys.maxunicode |
| 31 |
return map(chr, range(maxunicode + 1)) |
| 32 |
|
| 33 |
class UnicodeMethodsTest(unittest.TestCase): |
| 34 |
|
| 35 |
# update this, if the database changes |
| 36 |
expectedchecksum = ('486bf97d506d0ccf0e463fd1f40c51029805af5a' |
| 37 |
if quicktest else |
| 38 |
'9e43ee3929471739680c0e705482b4ae1c4122e4') |
| 39 |
|
| 40 |
def test_method_checksum(self): |
| 41 |
h = hashlib.sha1() |
| 42 |
for char in iterallchars(): |
| 43 |
s1 = char + 'abc' |
| 44 |
s2 = char + 'ABC' |
| 45 |
s3 = char + '123' |
| 46 |
data = ( |
| 47 |
# Predicates (single char) |
| 48 |
"01"[char.isalnum()], |
| 49 |
"01"[char.isalpha()], |
| 50 |
"01"[char.isdecimal()], |
| 51 |
"01"[char.isdigit()], |
| 52 |
"01"[char.islower()], |
| 53 |
"01"[char.isnumeric()], |
| 54 |
"01"[char.isspace()], |
| 55 |
"01"[char.istitle()], |
| 56 |
"01"[char.isupper()], |
| 57 |
|
| 58 |
# Predicates (multiple chars) |
| 59 |
"01"[s1.isalnum()], |
| 60 |
"01"[s1.isalpha()], |
| 61 |
"01"[s3.isdecimal()], |
| 62 |
"01"[s3.isdigit()], |
| 63 |
"01"[s1.islower()], |
| 64 |
"01"[s3.isnumeric()], |
| 65 |
"01"[(char + ' \t').isspace()], |
| 66 |
"01"[s1.istitle()], |
| 67 |
"01"[s2.isupper()], |
| 68 |
|
| 69 |
# Mappings (single char) |
| 70 |
char.lower(), |
| 71 |
char.upper(), |
| 72 |
char.title(), |
| 73 |
|
| 74 |
# Mappings (multiple chars) |
| 75 |
s1.lower(), |
| 76 |
s2.upper(), |
| 77 |
s1.title(), |
| 78 |
s2.title(), |
| 79 |
|
| 80 |
) |
| 81 |
h.update(''.join(data).encode('utf-8', 'surrogatepass')) |
| 82 |
result = h.hexdigest() |
| 83 |
self.assertEqual(result, self.expectedchecksum) |
| 84 |
|
| 85 |
|
| 86 |
class UnicodeFunctionsTest(unittest.TestCase): |
| 87 |
db = unicodedata |
| 88 |
old = False |
| 89 |
|
| 90 |
# Update this if the database changes. Make sure to do a full rebuild |
| 91 |
# (e.g. 'make distclean && make') to get the correct checksum. |
| 92 |
expectedchecksum = ('1ba453ec456896f1190d849b6e9b7c2e1a4128e0' |
| 93 |
if quicktest else |
| 94 |
'46ca89d9fe34881d0be3a4a4b29f5aa8c019640c') |
| 95 |
|
| 96 |
def test_function_checksum(self): |
| 97 |
db = self.db |
| 98 |
data = [] |
| 99 |
h = hashlib.sha1() |
| 100 |
|
| 101 |
for char in iterallchars(): |
| 102 |
data = "%.12g%.12g%.12g%s%s%s%s%s%s%s" % ( |
| 103 |
# Properties |
| 104 |
db.digit(char, -1), |
| 105 |
db.numeric(char, -1), |
| 106 |
db.decimal(char, -1), |
| 107 |
db.category(char), |
| 108 |
db.bidirectional(char), |
| 109 |
db.decomposition(char), |
| 110 |
db.mirrored(char), |
| 111 |
db.combining(char), |
| 112 |
db.east_asian_width(char), |
| 113 |
db.name(char, ""), |
| 114 |
) |
| 115 |
h.update(data.encode("ascii")) |
| 116 |
result = h.hexdigest() |
| 117 |
self.assertEqual(result, self.expectedchecksum) |
| 118 |
|
| 119 |
def test_name(self): |
| 120 |
name = self.db.name |
| 121 |
self.assertRaises(ValueError, name, '\0') |
| 122 |
self.assertRaises(ValueError, name, '\n') |
| 123 |
self.assertRaises(ValueError, name, '\x1F') |
| 124 |
self.assertRaises(ValueError, name, '\x7F') |
| 125 |
self.assertRaises(ValueError, name, '\x9F') |
| 126 |
self.assertRaises(ValueError, name, '\uFFFE') |
| 127 |
self.assertRaises(ValueError, name, '\uFFFF') |
| 128 |
self.assertRaises(ValueError, name, '\U0010FFFF') |
| 129 |
self.assertEqual(name('\U0010FFFF', 42), 42) |
| 130 |
|
| 131 |
self.assertEqual(name(' '), 'SPACE') |
| 132 |
self.assertEqual(name('1'), 'DIGIT ONE') |
| 133 |
self.assertEqual(name('A'), 'LATIN CAPITAL LETTER A') |
| 134 |
self.assertEqual(name('\xA0'), 'NO-BREAK SPACE') |
| 135 |
self.assertEqual(name('\u0221', None), None if self.old else |
| 136 |
'LATIN SMALL LETTER D WITH CURL') |
| 137 |
self.assertEqual(name('\u3400'), 'CJK UNIFIED IDEOGRAPH-3400') |
| 138 |
self.assertEqual(name('\u9FA5'), 'CJK UNIFIED IDEOGRAPH-9FA5') |
| 139 |
self.assertEqual(name('\uAC00'), 'HANGUL SYLLABLE GA') |
| 140 |
self.assertEqual(name('\uD7A3'), 'HANGUL SYLLABLE HIH') |
| 141 |
self.assertEqual(name('\uF900'), 'CJK COMPATIBILITY IDEOGRAPH-F900') |
| 142 |
self.assertEqual(name('\uFA6A'), 'CJK COMPATIBILITY IDEOGRAPH-FA6A') |
| 143 |
self.assertEqual(name('\uFBF9'), |
| 144 |
'ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA ' |
| 145 |
'ABOVE WITH ALEF MAKSURA ISOLATED FORM') |
| 146 |
self.assertEqual(name('\U00013460', None), None if self.old else |
| 147 |
'EGYPTIAN HIEROGLYPH-13460') |
| 148 |
self.assertEqual(name('\U000143FA', None), None if self.old else |
| 149 |
'EGYPTIAN HIEROGLYPH-143FA') |
| 150 |
self.assertEqual(name('\U00018B00', None), None if self.old else |
| 151 |
'KHITAN SMALL SCRIPT CHARACTER-18B00') |
| 152 |
self.assertEqual(name('\U00018CD5', None), None if self.old else |
| 153 |
'KHITAN SMALL SCRIPT CHARACTER-18CD5') |
| 154 |
self.assertEqual(name('\U00018CFF', None), None if self.old else |
| 155 |
'KHITAN SMALL SCRIPT CHARACTER-18CFF') |
| 156 |
self.assertEqual(name('\U0001B170', None), None if self.old else |
| 157 |
'NUSHU CHARACTER-1B170') |
| 158 |
self.assertEqual(name('\U0001B2FB', None), None if self.old else |
| 159 |
'NUSHU CHARACTER-1B2FB') |
| 160 |
self.assertEqual(name('\U0001FBA8', None), None if self.old else |
| 161 |
'BOX DRAWINGS LIGHT DIAGONAL UPPER CENTRE TO ' |
| 162 |
'MIDDLE LEFT AND MIDDLE RIGHT TO LOWER CENTRE') |
| 163 |
self.assertEqual(name('\U0002A6D6'), 'CJK UNIFIED IDEOGRAPH-2A6D6') |
| 164 |
self.assertEqual(name('\U0002FA1D'), 'CJK COMPATIBILITY IDEOGRAPH-2FA1D') |
| 165 |
self.assertEqual(name('\U000323AF', None), None if self.old else |
| 166 |
'CJK UNIFIED IDEOGRAPH-323AF') |
| 167 |
|
| 168 |
@requires_resource('cpu') |
| 169 |
def test_name_inverse_lookup(self): |
| 170 |
for char in iterallchars(): |
| 171 |
looked_name = self.db.name(char, None) |
| 172 |
if looked_name is not None: |
| 173 |
self.assertEqual(self.db.lookup(looked_name), char) |
| 174 |
|
| 175 |
def test_no_names_in_pua(self): |
| 176 |
puas = [*range(0xe000, 0xf8ff), |
| 177 |
*range(0xf0000, 0xfffff), |
| 178 |
*range(0x100000, 0x10ffff)] |
| 179 |
for i in puas: |
| 180 |
char = chr(i) |
| 181 |
self.assertRaises(ValueError, self.db.name, char) |
| 182 |
|
| 183 |
def test_lookup_nonexistant(self): |
| 184 |
# just make sure that lookup can fail |
| 185 |
for nonexistent in [ |
| 186 |
"LATIN SMLL LETR A", |
| 187 |
"OPEN HANDS SIGHS", |
| 188 |
"DREGS", |
| 189 |
"HANDBUG", |
| 190 |
"MODIFIER LETTER CYRILLIC SMALL QUESTION MARK", |
| 191 |
"???", |
| 192 |
"CJK UNIFIED IDEOGRAPH-03400", |
| 193 |
"CJK UNIFIED IDEOGRAPH-020000", |
| 194 |
"CJK UNIFIED IDEOGRAPH-33FF", |
| 195 |
"CJK UNIFIED IDEOGRAPH-F900", |
| 196 |
"CJK UNIFIED IDEOGRAPH-13460", |
| 197 |
"CJK UNIFIED IDEOGRAPH-17000", |
| 198 |
"CJK UNIFIED IDEOGRAPH-18B00", |
| 199 |
"CJK UNIFIED IDEOGRAPH-1B170", |
| 200 |
"CJK COMPATIBILITY IDEOGRAPH-3400", |
| 201 |
"TANGUT IDEOGRAPH-3400", |
| 202 |
"HANGUL SYLLABLE AC00", |
| 203 |
]: |
| 204 |
self.assertRaises(KeyError, self.db.lookup, nonexistent) |
| 205 |
|
| 206 |
def test_digit(self): |
| 207 |
self.assertEqual(self.db.digit('A', None), None) |
| 208 |
self.assertEqual(self.db.digit('9'), 9) |
| 209 |
self.assertEqual(self.db.digit('\u215b', None), None) |
| 210 |
self.assertEqual(self.db.digit('\u2468'), 9) |
| 211 |
self.assertEqual(self.db.digit('\U00020000', None), None) |
| 212 |
self.assertEqual(self.db.digit('\U0001D7FD'), 7) |
| 213 |
|
| 214 |
# New in 13.0.0 |
| 215 |
self.assertEqual(self.db.digit('\U0001fbf9', None), 9) |
| 216 |
# New in 14.0.0 |
| 217 |
self.assertEqual(self.db.digit('\U00016ac9', None), 9) |
| 218 |
# New in 15.0.0 |
| 219 |
self.assertEqual(self.db.digit('\U0001e4f9', None), 9) |
| 220 |
|
| 221 |
self.assertRaises(TypeError, self.db.digit) |
| 222 |
self.assertRaises(TypeError, self.db.digit, 'xx') |
| 223 |
self.assertRaises(ValueError, self.db.digit, 'x') |
| 224 |
|
| 225 |
def test_numeric(self): |
| 226 |
self.assertEqual(self.db.numeric('A',None), None) |
| 227 |
self.assertEqual(self.db.numeric('9'), 9) |
| 228 |
self.assertEqual(self.db.numeric('\u215b'), 0.125) |
| 229 |
self.assertEqual(self.db.numeric('\u2468'), 9.0) |
| 230 |
self.assertEqual(self.db.numeric('\U00020000', None), None) |
| 231 |
|
| 232 |
# New in 4.1.0 |
| 233 |
self.assertEqual(self.db.numeric('\U0001012A', None), None if self.old else 9000) |
| 234 |
# Changed in 4.1.0 |
| 235 |
self.assertEqual(self.db.numeric('\u5793', None), 1e20 if self.old else None) |
| 236 |
# New in 5.0.0 |
| 237 |
self.assertEqual(self.db.numeric('\u07c0', None), None if self.old else 0.0) |
| 238 |
# New in 5.1.0 |
| 239 |
self.assertEqual(self.db.numeric('\ua627', None), None if self.old else 7.0) |
| 240 |
# Changed in 5.2.0 |
| 241 |
self.assertEqual(self.db.numeric('\u09f6'), 3.0 if self.old else 3/16) |
| 242 |
# New in 6.0.0 |
| 243 |
self.assertEqual(self.db.numeric('\u0b72', None), None if self.old else 0.25) |
| 244 |
# New in 12.0.0 |
| 245 |
self.assertEqual(self.db.numeric('\U0001ed3c', None), None if self.old else 0.5) |
| 246 |
# New in 13.0.0 |
| 247 |
self.assertEqual(self.db.numeric('\U0001fbf9', None), None if self.old else 9) |
| 248 |
# New in 14.0.0 |
| 249 |
self.assertEqual(self.db.numeric('\U00016ac9', None), None if self.old else 9) |
| 250 |
# New in 15.0.0 |
| 251 |
self.assertEqual(self.db.numeric('\U0001e4f9', None), None if self.old else 9) |
| 252 |
|
| 253 |
self.assertRaises(TypeError, self.db.numeric) |
| 254 |
self.assertRaises(TypeError, self.db.numeric, 'xx') |
| 255 |
self.assertRaises(ValueError, self.db.numeric, 'x') |
| 256 |
|
| 257 |
def test_decimal(self): |
| 258 |
self.assertEqual(self.db.decimal('A',None), None) |
| 259 |
self.assertEqual(self.db.decimal('9'), 9) |
| 260 |
self.assertEqual(self.db.decimal('\u215b', None), None) |
| 261 |
self.assertEqual(self.db.decimal('\u2468', None), None) |
| 262 |
self.assertEqual(self.db.decimal('\U00020000', None), None) |
| 263 |
self.assertEqual(self.db.decimal('\U0001D7FD'), 7) |
| 264 |
|
| 265 |
# New in 4.1.0 |
| 266 |
self.assertEqual(self.db.decimal('\xb2', None), 2 if self.old else None) |
| 267 |
self.assertEqual(self.db.decimal('\u1369', None), 1 if self.old else None) |
| 268 |
# New in 5.0.0 |
| 269 |
self.assertEqual(self.db.decimal('\u07c0', None), None if self.old else 0) |
| 270 |
# New in 13.0.0 |
| 271 |
self.assertEqual(self.db.decimal('\U0001fbf9', None), None if self.old else 9) |
| 272 |
# New in 14.0.0 |
| 273 |
self.assertEqual(self.db.decimal('\U00016ac9', None), None if self.old else 9) |
| 274 |
# New in 15.0.0 |
| 275 |
self.assertEqual(self.db.decimal('\U0001e4f9', None), None if self.old else 9) |
| 276 |
|
| 277 |
self.assertRaises(TypeError, self.db.decimal) |
| 278 |
self.assertRaises(TypeError, self.db.decimal, 'xx') |
| 279 |
self.assertRaises(ValueError, self.db.decimal, 'x') |
| 280 |
|
| 281 |
def test_category(self): |
| 282 |
self.assertEqual(self.db.category('\uFFFE'), 'Cn') |
| 283 |
self.assertEqual(self.db.category('a'), 'Ll') |
| 284 |
self.assertEqual(self.db.category('A'), 'Lu') |
| 285 |
self.assertEqual(self.db.category('\U00020000'), 'Lo') |
| 286 |
|
| 287 |
# New in 4.1.0 |
| 288 |
self.assertEqual(self.db.category('\U0001012A'), 'Cn' if self.old else 'No') |
| 289 |
self.assertEqual(self.db.category('\U000e01ef'), 'Cn' if self.old else 'Mn') |
| 290 |
# New in 5.1.0 |
| 291 |
self.assertEqual(self.db.category('\u0374'), 'Sk' if self.old else 'Lm') |
| 292 |
# Changed in 13.0.0 |
| 293 |
self.assertEqual(self.db.category('\u0b55'), 'Cn' if self.old else 'Mn') |
| 294 |
self.assertEqual(self.db.category('\U0003134a'), 'Cn' if self.old else 'Lo') |
| 295 |
# Changed in 14.0.0 |
| 296 |
self.assertEqual(self.db.category('\u061d'), 'Cn' if self.old else 'Po') |
| 297 |
self.assertEqual(self.db.category('\U0002b738'), 'Cn' if self.old else 'Lo') |
| 298 |
# Changed in 15.0.0 |
| 299 |
self.assertEqual(self.db.category('\u0cf3'), 'Cn' if self.old else 'Mc') |
| 300 |
self.assertEqual(self.db.category('\U000323af'), 'Cn' if self.old else 'Lo') |
| 301 |
|
| 302 |
self.assertRaises(TypeError, self.db.category) |
| 303 |
self.assertRaises(TypeError, self.db.category, 'xx') |
| 304 |
|
| 305 |
def test_bidirectional(self): |
| 306 |
self.assertEqual(self.db.bidirectional('\uFFFE'), '') |
| 307 |
self.assertEqual(self.db.bidirectional(' '), 'WS') |
| 308 |
self.assertEqual(self.db.bidirectional('A'), 'L') |
| 309 |
self.assertEqual(self.db.bidirectional('\U00020000'), 'L') |
| 310 |
|
| 311 |
# New in 4.1.0 |
| 312 |
self.assertEqual(self.db.bidirectional('+'), 'ET' if self.old else 'ES') |
| 313 |
self.assertEqual(self.db.bidirectional('\u0221'), '' if self.old else 'L') |
| 314 |
self.assertEqual(self.db.bidirectional('\U000e01ef'), '' if self.old else 'NSM') |
| 315 |
# New in 13.0.0 |
| 316 |
self.assertEqual(self.db.bidirectional('\u0b55'), '' if self.old else 'NSM') |
| 317 |
self.assertEqual(self.db.bidirectional('\U0003134a'), '' if self.old else 'L') |
| 318 |
# New in 14.0.0 |
| 319 |
self.assertEqual(self.db.bidirectional('\u061d'), '' if self.old else 'AL') |
| 320 |
self.assertEqual(self.db.bidirectional('\U0002b738'), '' if self.old else 'L') |
| 321 |
# New in 15.0.0 |
| 322 |
self.assertEqual(self.db.bidirectional('\u0cf3'), '' if self.old else 'L') |
| 323 |
self.assertEqual(self.db.bidirectional('\U000323af'), '' if self.old else 'L') |
| 324 |
# New in 16.0.0 |
| 325 |
self.assertEqual(self.db.bidirectional('\u0897'), '' if self.old else 'NSM') |
| 326 |
self.assertEqual(self.db.bidirectional('\U0001fbef'), '' if self.old else 'ON') |
| 327 |
|
| 328 |
self.assertRaises(TypeError, self.db.bidirectional) |
| 329 |
self.assertRaises(TypeError, self.db.bidirectional, 'xx') |
| 330 |
|
| 331 |
def test_decomposition(self): |
| 332 |
self.assertEqual(self.db.decomposition('\uFFFE'),'') |
| 333 |
self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034') |
| 334 |
|
| 335 |
# New in 4.1.0 |
| 336 |
self.assertEqual(self.db.decomposition('\u03f9'), '' if self.old else '<compat> 03A3') |
| 337 |
# New in 13.0.0 |
| 338 |
self.assertEqual(self.db.decomposition('\uab69'), '' if self.old else '<super> 028D') |
| 339 |
self.assertEqual(self.db.decomposition('\U00011938'), '' if self.old else '11935 11930') |
| 340 |
self.assertEqual(self.db.decomposition('\U0001fbf9'), '' if self.old else '<font> 0039') |
| 341 |
# New in 14.0.0 |
| 342 |
self.assertEqual(self.db.decomposition('\ua7f2'), '' if self.old else '<super> 0043') |
| 343 |
self.assertEqual(self.db.decomposition('\U000107ba'), '' if self.old else '<super> 1DF1E') |
| 344 |
# New in 15.0.0 |
| 345 |
self.assertEqual(self.db.decomposition('\U0001e06d'), '' if self.old else '<super> 04B1') |
| 346 |
# New in 16.0.0 |
| 347 |
self.assertEqual(self.db.decomposition('\U0001CCD6'), '' if self.old else '<font> 0041') |
| 348 |
|
| 349 |
# Hangul characters |
| 350 |
self.assertEqual(self.db.decomposition('\uAC00'), '1100 1161') |
| 351 |
self.assertEqual(self.db.decomposition('\uD4DB'), '1111 1171 11B6') |
| 352 |
self.assertEqual(self.db.decomposition('\uC2F8'), '110A 1161') |
| 353 |
self.assertEqual(self.db.decomposition('\uD7A3'), '1112 1175 11C2') |
| 354 |
|
| 355 |
self.assertRaises(TypeError, self.db.decomposition) |
| 356 |
self.assertRaises(TypeError, self.db.decomposition, 'xx') |
| 357 |
|
| 358 |
def test_mirrored(self): |
| 359 |
self.assertEqual(self.db.mirrored('\uFFFE'), 0) |
| 360 |
self.assertEqual(self.db.mirrored('a'), 0) |
| 361 |
self.assertEqual(self.db.mirrored('\u2201'), 1) |
| 362 |
self.assertEqual(self.db.mirrored('\U00020000'), 0) |
| 363 |
|
| 364 |
# New in 5.0.0 |
| 365 |
self.assertEqual(self.db.mirrored('\u0f3a'), 0 if self.old else 1) |
| 366 |
self.assertEqual(self.db.mirrored('\U0001d7c3'), 0 if self.old else 1) |
| 367 |
# New in 11.0.0 |
| 368 |
self.assertEqual(self.db.mirrored('\u29a1'), 1 if self.old else 0) |
| 369 |
# New in 14.0.0 |
| 370 |
self.assertEqual(self.db.mirrored('\u2e5c'), 0 if self.old else 1) |
| 371 |
# New in 16.0.0 |
| 372 |
self.assertEqual(self.db.mirrored('\u226D'), 0 if self.old else 1) |
| 373 |
|
| 374 |
self.assertRaises(TypeError, self.db.mirrored) |
| 375 |
self.assertRaises(TypeError, self.db.mirrored, 'xx') |
| 376 |
|
| 377 |
def test_combining(self): |
| 378 |
self.assertEqual(self.db.combining('\uFFFE'), 0) |
| 379 |
self.assertEqual(self.db.combining('a'), 0) |
| 380 |
self.assertEqual(self.db.combining('\u20e1'), 230) |
| 381 |
self.assertEqual(self.db.combining('\U00020000'), 0) |
| 382 |
|
| 383 |
# New in 4.1.0 |
| 384 |
self.assertEqual(self.db.combining('\u0350'), 0 if self.old else 230) |
| 385 |
# New in 9.0.0 |
| 386 |
self.assertEqual(self.db.combining('\U0001e94a'), 0 if self.old else 7) |
| 387 |
# New in 13.0.0 |
| 388 |
self.assertEqual(self.db.combining('\u1abf'), 0 if self.old else 220) |
| 389 |
self.assertEqual(self.db.combining('\U00016ff1'), 0 if self.old else 6) |
| 390 |
# New in 14.0.0 |
| 391 |
self.assertEqual(self.db.combining('\u0c3c'), 0 if self.old else 7) |
| 392 |
self.assertEqual(self.db.combining('\U0001e2ae'), 0 if self.old else 230) |
| 393 |
# New in 15.0.0 |
| 394 |
self.assertEqual(self.db.combining('\U00010efd'), 0 if self.old else 220) |
| 395 |
# New in 16.0.0 |
| 396 |
self.assertEqual(self.db.combining('\u0897'), 0 if self.old else 230) |
| 397 |
|
| 398 |
self.assertRaises(TypeError, self.db.combining) |
| 399 |
self.assertRaises(TypeError, self.db.combining, 'xx') |
| 400 |
|
| 401 |
def test_normalization(self): |
| 402 |
# Test normalize() and is_normalized() |
| 403 |
def check(ch, expected): |
| 404 |
if isinstance(expected, str): |
| 405 |
expected = [expected]*4 |
| 406 |
forms = ('NFC', 'NFD', 'NFKC', 'NFKD') |
| 407 |
result = [self.db.normalize(form, ch) for form in forms] |
| 408 |
self.assertEqual(ascii(result), ascii(list(expected))) |
| 409 |
self.assertEqual([self.db.is_normalized(form, ch) for form in forms], |
| 410 |
[ch == y for x, y in zip(result, expected)]) |
| 411 |
|
| 412 |
check('', '') |
| 413 |
check('A', 'A') |
| 414 |
check(' ', ' ') |
| 415 |
check('\U0010ffff', '\U0010ffff') |
| 416 |
check('abc', 'abc') |
| 417 |
# Broken in 4.0.0 |
| 418 |
check('\u0340', '\u0300') |
| 419 |
check('\u0300', '\u0300') |
| 420 |
check('\U0002fa1d', '\U0002a600') |
| 421 |
check('\U0002a600', '\U0002a600') |
| 422 |
check('\u0344', '\u0308\u0301') |
| 423 |
check('\u0308\u0301', '\u0308\u0301') |
| 424 |
# Broken in 4.0.0 and 4.0.1 |
| 425 |
check('\U0001d1bc', '\U0001d1ba\U0001d165') |
| 426 |
check('\U0001d1ba\U0001d165', '\U0001d1ba\U0001d165') |
| 427 |
check('\ufb2c', '\u05e9\u05bc\u05c1') |
| 428 |
check('\u05e9\u05bc\u05c1', '\u05e9\u05bc\u05c1') |
| 429 |
check('\U0001d1c0', '\U0001d1ba\U0001d165\U0001d16f') |
| 430 |
check('\U0001d1ba\U0001d165\U0001d16f', '\U0001d1ba\U0001d165\U0001d16f') |
| 431 |
|
| 432 |
# Broken in 4.0.0 |
| 433 |
check('\xa0', ['\xa0', '\xa0', ' ', ' ']) |
| 434 |
check('\u2003', ['\u2003', '\u2003', ' ', ' ']) |
| 435 |
check('\U0001d7ff', ['\U0001d7ff', '\U0001d7ff', '9', '9']) |
| 436 |
|
| 437 |
check('\xa8', ['\xa8', '\xa8', ' \u0308', ' \u0308']) |
| 438 |
check(' \u0308', ' \u0308') |
| 439 |
|
| 440 |
check('\xc0', ['\xc0', 'A\u0300']*2) |
| 441 |
check('A\u0300', ['\xc0', 'A\u0300']*2) |
| 442 |
|
| 443 |
check('\ud7a3', ['\ud7a3', '\u1112\u1175\u11c2']*2) |
| 444 |
check('\u1112\u1175\u11c2', ['\ud7a3', '\u1112\u1175\u11c2']*2) |
| 445 |
|
| 446 |
check('\xb4', ['\xb4', '\xb4', ' \u0301', ' \u0301']) |
| 447 |
check('\u1ffd', ['\xb4', '\xb4', ' \u0301', ' \u0301']) |
| 448 |
check(' \u0301', ' \u0301') |
| 449 |
|
| 450 |
check('\xc5', ['\xc5', 'A\u030a']*2) |
| 451 |
check('\u212b', ['\xc5', 'A\u030a']*2) |
| 452 |
check('A\u030a', ['\xc5', 'A\u030a']*2) |
| 453 |
|
| 454 |
check('\u1f71', ['\u03ac', '\u03b1\u0301']*2) |
| 455 |
check('\u03ac', ['\u03ac', '\u03b1\u0301']*2) |
| 456 |
check('\u03b1\u0301', ['\u03ac', '\u03b1\u0301']*2) |
| 457 |
|
| 458 |
check('\u01c4', ['\u01c4', '\u01c4', 'D\u017d', 'DZ\u030c']) |
| 459 |
check('D\u017d', ['D\u017d', 'DZ\u030c']*2) |
| 460 |
check('DZ\u030c', ['D\u017d', 'DZ\u030c']*2) |
| 461 |
|
| 462 |
check('\u1fed', ['\u1fed', '\xa8\u0300', ' \u0308\u0300', ' \u0308\u0300']) |
| 463 |
check('\xa8\u0300', ['\u1fed', '\xa8\u0300', ' \u0308\u0300', ' \u0308\u0300']) |
| 464 |
check(' \u0308\u0300', ' \u0308\u0300') |
| 465 |
|
| 466 |
check('\u326e', ['\u326e', '\u326e', '\uac00', '\u1100\u1161']) |
| 467 |
check('\u320e', ['\u320e', '\u320e', '(\uac00)', '(\u1100\u1161)']) |
| 468 |
check('(\uac00)', ['(\uac00)', '(\u1100\u1161)']*2) |
| 469 |
check('(\u1100\u1161)', ['(\uac00)', '(\u1100\u1161)']*2) |
| 470 |
|
| 471 |
check('\u0385', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301']) |
| 472 |
check('\u1fee', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301']) |
| 473 |
check('\xa8\u0301', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301']) |
| 474 |
check(' \u0308\u0301', ' \u0308\u0301') |
| 475 |
|
| 476 |
check('\u1fdf', ['\u1fdf', '\u1ffe\u0342', ' \u0314\u0342', ' \u0314\u0342']) |
| 477 |
check('\u1ffe\u0342', ['\u1fdf', '\u1ffe\u0342', ' \u0314\u0342', ' \u0314\u0342']) |
| 478 |
check('\u1ffe', ['\u1ffe', '\u1ffe', ' \u0314', ' \u0314']) |
| 479 |
check(' \u0314\u0342', ' \u0314\u0342') |
| 480 |
|
| 481 |
check('\u03d3', ['\u03d3', '\u03d2\u0301', '\u038e', '\u03a5\u0301']) |
| 482 |
check('\u03d2\u0301', ['\u03d3', '\u03d2\u0301', '\u038e', '\u03a5\u0301']) |
| 483 |
check('\u038e', ['\u038e', '\u03a5\u0301']*2) |
| 484 |
check('\u1feb', ['\u038e', '\u03a5\u0301']*2) |
| 485 |
check('\u03a5\u0301', ['\u038e', '\u03a5\u0301']*2) |
| 486 |
|
| 487 |
check('\u0626', ['\u0626', '\u064a\u0654']*2) |
| 488 |
check('\u064a\u0654', ['\u0626', '\u064a\u0654']*2) |
| 489 |
check('\ufe89', ['\ufe89', '\ufe89', '\u0626', '\u064a\u0654']) |
| 490 |
check('\ufe8a', ['\ufe8a', '\ufe8a', '\u0626', '\u064a\u0654']) |
| 491 |
check('\ufe8b', ['\ufe8b', '\ufe8b', '\u0626', '\u064a\u0654']) |
| 492 |
check('\ufe8c', ['\ufe8c', '\ufe8c', '\u0626', '\u064a\u0654']) |
| 493 |
|
| 494 |
check('\ufef9', ['\ufef9', '\ufef9', '\u0644\u0625', '\u0644\u0627\u0655']) |
| 495 |
check('\ufefa', ['\ufefa', '\ufefa', '\u0644\u0625', '\u0644\u0627\u0655']) |
| 496 |
check('\ufefb', ['\ufefb', '\ufefb', '\u0644\u0627', '\u0644\u0627']) |
| 497 |
check('\ufefc', ['\ufefc', '\ufefc', '\u0644\u0627', '\u0644\u0627']) |
| 498 |
check('\u0644\u0625', ['\u0644\u0625', '\u0644\u0627\u0655']*2) |
| 499 |
check('\u0644\u0627\u0655', ['\u0644\u0625', '\u0644\u0627\u0655']*2) |
| 500 |
check('\u0644\u0627', '\u0644\u0627') |
| 501 |
|
| 502 |
# Broken in 4.0.0 |
| 503 |
check('\u327c', '\u327c' if self.old else |
| 504 |
['\u327c', '\u327c', '\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']) |
| 505 |
check('\ucc38\uace0', ['\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']*2) |
| 506 |
check('\ucc38', ['\ucc38', '\u110e\u1161\u11b7']*2) |
| 507 |
check('\u110e\u1161\u11b7\u1100\u1169', |
| 508 |
['\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']*2) |
| 509 |
check('\u110e\u1161\u11b7\u1100', |
| 510 |
['\ucc38\u1100', '\u110e\u1161\u11b7\u1100']*2) |
| 511 |
check('\u110e\u1161\u11b7', |
| 512 |
['\ucc38', '\u110e\u1161\u11b7']*2) |
| 513 |
check('\u110e\u1161', |
| 514 |
['\ucc28', '\u110e\u1161']*2) |
| 515 |
check('\u110e', '\u110e') |
| 516 |
# Broken in 4.0.0-12.0.0 |
| 517 |
check('\U00011938', '\U00011938' if self.old else |
| 518 |
['\U00011938', '\U00011935\U00011930']*2) |
| 519 |
check('\U00011935\U00011930', ['\U00011938', '\U00011935\U00011930']*2) |
| 520 |
# New in 4.0.1 |
| 521 |
check('\u321d', '\u321d' if self.old else |
| 522 |
['\u321d', '\u321d', '(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']) |
| 523 |
check('(\uc624\uc804)', |
| 524 |
['(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']*2) |
| 525 |
check('(\u110b\u1169\u110c\u1165\u11ab)', |
| 526 |
['(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']*2) |
| 527 |
check('\u4d57', '\u4d57') |
| 528 |
check('\u45d7', '\u45d7' if self.old else '\u45d7') |
| 529 |
check('\U0002f9bf', '\u4d57' if self.old else '\u45d7') |
| 530 |
# New in 4.1.0 |
| 531 |
check('\u03a3', '\u03a3') |
| 532 |
check('\u03f9', '\u03f9' if self.old else |
| 533 |
['\u03f9', '\u03f9', '\u03a3', '\u03a3']) |
| 534 |
# New in 5.0.0 |
| 535 |
check('\u1b06', '\u1b06' if self.old else ['\u1b06', '\u1b05\u1b35']*2) |
| 536 |
# New in 5.2.0 |
| 537 |
check('\U0001f213', '\U0001f213' if self.old else |
| 538 |
['\U0001f213', '\U0001f213', '\u30c7', '\u30c6\u3099']) |
| 539 |
# New in 6.1.0 |
| 540 |
check('\ufa2e', '\ufa2e' if self.old else '\u90de') |
| 541 |
# New in 13.0.0 |
| 542 |
check('\U00011938', '\U00011938' if self.old else |
| 543 |
['\U00011938', '\U00011935\U00011930', '\U00011938', '\U00011935\U00011930']) |
| 544 |
check('\U0001fbf9', '\U0001fbf9' if self.old else |
| 545 |
['\U0001fbf9', '\U0001fbf9', '9', '9']) |
| 546 |
# New in 14.0.0 |
| 547 |
check('\U000107ba', '\U000107ba' if self.old else |
| 548 |
['\U000107ba', '\U000107ba', '\U0001df1e', '\U0001df1e']) |
| 549 |
# New in 15.0.0 |
| 550 |
check('\U0001e06d', '\U0001e06d' if self.old else |
| 551 |
['\U0001e06d', '\U0001e06d', '\u04b1', '\u04b1']) |
| 552 |
# New in 16.0.0 |
| 553 |
check('\U0001ccd6', '\U0001ccd6' if self.old else |
| 554 |
['\U0001ccd6', '\U0001ccd6', 'A', 'A']) |
| 555 |
|
| 556 |
self.assertRaises(TypeError, self.db.normalize) |
| 557 |
self.assertRaises(TypeError, self.db.normalize, 'NFC') |
| 558 |
self.assertRaises(ValueError, self.db.normalize, 'SPAM', 'A') |
| 559 |
|
| 560 |
self.assertRaises(TypeError, self.db.is_normalized) |
| 561 |
self.assertRaises(TypeError, self.db.is_normalized, 'NFC') |
| 562 |
self.assertRaises(ValueError, self.db.is_normalized, 'SPAM', 'A') |
| 563 |
|
| 564 |
def test_pr29(self): |
| 565 |
# https://www.unicode.org/review/pr-29.html |
| 566 |
# See issues #1054943 and #10254. |
| 567 |
composed = ("\u0b47\u0300\u0b3e", "\u1100\u0300\u1161", |
| 568 |
'Li\u030dt-s\u1e73\u0301', |
| 569 |
'\u092e\u093e\u0930\u094d\u0915 \u091c\u093c' |
| 570 |
+ '\u0941\u0915\u0947\u0930\u092c\u0930\u094d\u0917', |
| 571 |
'\u0915\u093f\u0930\u094d\u0917\u093f\u091c\u093c' |
| 572 |
+ '\u0938\u094d\u0924\u093e\u0928') |
| 573 |
for text in composed: |
| 574 |
self.assertEqual(self.db.normalize('NFC', text), text) |
| 575 |
|
| 576 |
def test_issue10254(self): |
| 577 |
# Crash reported in #10254 |
| 578 |
# New in 4.1.0 |
| 579 |
a = 'C\u0338' * 20 + 'C\u0327' |
| 580 |
b = 'C\u0338' * 20 + '\xC7' |
| 581 |
self.assertEqual(self.db.normalize('NFC', a), b) |
| 582 |
|
| 583 |
def test_long_combining_mark_run(self): |
| 584 |
# gh-149079: avoid quadratic canonical ordering. |
| 585 |
payload = "a" + ("\u0300\u0327" * 32) |
| 586 |
nfd = "a" + ("\u0327" * 32) + ("\u0300" * 32) |
| 587 |
nfc = "\u00e0" + ("\u0327" * 32) + ("\u0300" * 31) |
| 588 |
|
| 589 |
self.assertEqual(self.db.normalize("NFD", payload), nfd) |
| 590 |
self.assertEqual(self.db.normalize("NFKD", payload), nfd) |
| 591 |
self.assertEqual(self.db.normalize("NFC", payload), nfc) |
| 592 |
self.assertEqual(self.db.normalize("NFKC", payload), nfc) |
| 593 |
|
| 594 |
def test_combining_mark_run_fast_paths(self): |
| 595 |
# gh-149079: cover short runs and already-sorted long runs. |
| 596 |
short_payload = "a" + ("\u0300\u0327" * 9) + "\u0300" |
| 597 |
short_nfd = "a" + ("\u0327" * 9) + ("\u0300" * 10) |
| 598 |
short_nfc = "\u00e0" + ("\u0327" * 9) + ("\u0300" * 9) |
| 599 |
long_sorted = "a" + ("\u0327" * 30) + ("\u0300" * 30) |
| 600 |
long_sorted_nfc = "\u00e0" + ("\u0327" * 30) + ("\u0300" * 29) |
| 601 |
|
| 602 |
self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd) |
| 603 |
self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd) |
| 604 |
self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc) |
| 605 |
self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc) |
| 606 |
self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted) |
| 607 |
self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted) |
| 608 |
self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc) |
| 609 |
self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc) |
| 610 |
|
| 611 |
def test_issue29456(self): |
| 612 |
# Fix #29456 |
| 613 |
u1176_str_a = '\u1100\u1176\u11a8' |
| 614 |
u1176_str_b = '\u1100\u1176\u11a8' |
| 615 |
u11a7_str_a = '\u1100\u1175\u11a7' |
| 616 |
u11a7_str_b = '\uae30\u11a7' |
| 617 |
u11c3_str_a = '\u1100\u1175\u11c3' |
| 618 |
u11c3_str_b = '\uae30\u11c3' |
| 619 |
self.assertEqual(self.db.normalize('NFC', u1176_str_a), u1176_str_b) |
| 620 |
# New in 4.1.0 |
| 621 |
self.assertEqual(self.db.normalize('NFC', u11a7_str_a), u11a7_str_b) |
| 622 |
self.assertEqual(self.db.normalize('NFC', u11c3_str_a), u11c3_str_b) |
| 623 |
|
| 624 |
def test_east_asian_width(self): |
| 625 |
eaw = self.db.east_asian_width |
| 626 |
self.assertRaises(TypeError, eaw, b'a') |
| 627 |
self.assertRaises(TypeError, eaw, bytearray()) |
| 628 |
self.assertRaises(TypeError, eaw, '') |
| 629 |
self.assertRaises(TypeError, eaw, 'ra') |
| 630 |
self.assertEqual(eaw('\x1e'), 'N') |
| 631 |
self.assertEqual(eaw('\x20'), 'Na') |
| 632 |
self.assertEqual(eaw('\uC894'), 'W') |
| 633 |
self.assertEqual(eaw('\uFF66'), 'H') |
| 634 |
self.assertEqual(eaw('\uFF1F'), 'F') |
| 635 |
self.assertEqual(eaw('\u2010'), 'A') |
| 636 |
self.assertEqual(eaw('\U00020000'), 'W') |
| 637 |
|
| 638 |
# New in 4.1.0 |
| 639 |
self.assertEqual(eaw('\u0350'), 'N' if self.old else 'A') |
| 640 |
self.assertEqual(eaw('\U000e01ef'), 'N' if self.old else 'A') |
| 641 |
# New in 5.2.0 |
| 642 |
self.assertEqual(eaw('\u115a'), 'N' if self.old else 'W') |
| 643 |
# New in 9.0.0 |
| 644 |
self.assertEqual(eaw('\u231a'), 'N' if self.old else 'W') |
| 645 |
self.assertEqual(eaw('\u2614'), 'N' if self.old else 'W') |
| 646 |
self.assertEqual(eaw('\U0001f19a'), 'N' if self.old else 'W') |
| 647 |
self.assertEqual(eaw('\U0001f991'), 'N' if self.old else 'W') |
| 648 |
self.assertEqual(eaw('\U0001f9c0'), 'N' if self.old else 'W') |
| 649 |
# New in 12.0.0 |
| 650 |
self.assertEqual(eaw('\u32ff'), 'N' if self.old else 'W') |
| 651 |
self.assertEqual(eaw('\U0001fa95'), 'N' if self.old else 'W') |
| 652 |
# New in 13.0.0 |
| 653 |
self.assertEqual(eaw('\u31bb'), 'N' if self.old else 'W') |
| 654 |
self.assertEqual(eaw('\U0003134a'), 'N' if self.old else 'W') |
| 655 |
# New in 14.0.0 |
| 656 |
self.assertEqual(eaw('\u9ffd'), 'N' if self.old else 'W') |
| 657 |
self.assertEqual(eaw('\U0002b738'), 'N' if self.old else 'W') |
| 658 |
# New in 15.0.0 |
| 659 |
self.assertEqual(eaw('\U000323af'), 'N' if self.old else 'W') |
| 660 |
# New in 16.0.0 |
| 661 |
self.assertEqual(eaw('\u2630'), 'N' if self.old else 'W') |
| 662 |
self.assertEqual(eaw('\U0001FAE9'), 'N' if self.old else 'W') |
| 663 |
|
| 664 |
def test_east_asian_width_unassigned(self): |
| 665 |
eaw = self.db.east_asian_width |
| 666 |
# unassigned |
| 667 |
for char in '\u0530\u0ecf\u10c6\u20fc\uaaca\U000107bd\U000115f2': |
| 668 |
self.assertEqual(eaw(char), 'N') |
| 669 |
self.assertIs(self.db.name(char, None), None) |
| 670 |
|
| 671 |
# unassigned but reserved for CJK |
| 672 |
for char in ('\U0002A6E0\U0002FA20\U0003134B\U0003FFFD' |
| 673 |
'\uFA6E\uFADA'): # New in 5.2.0 |
| 674 |
self.assertEqual(eaw(char), 'W') |
| 675 |
self.assertIs(self.db.name(char, None), None) |
| 676 |
|
| 677 |
# private use areas |
| 678 |
for char in '\uE000\uF800\U000F0000\U000FFFEE\U00100000\U0010FFF0': |
| 679 |
self.assertEqual(eaw(char), 'A') |
| 680 |
self.assertIs(self.db.name(char, None), None) |
| 681 |
|
| 682 |
|
| 683 |
class Unicode_3_2_0_FunctionsTest(UnicodeFunctionsTest): |
| 684 |
db = unicodedata.ucd_3_2_0 |
| 685 |
old = True |
| 686 |
expectedchecksum = ('883824cb6c0ccf994e4451ebf281e2d6d479af47' |
| 687 |
if quicktest else |
| 688 |
'caf1a7f2f380f927461837f1901ef20683f98683') |
| 689 |
|
| 690 |
|
| 691 |
class UnicodeMiscTest(unittest.TestCase): |
| 692 |
db = unicodedata |
| 693 |
|
| 694 |
@cpython_only |
| 695 |
def test_disallow_instantiation(self): |
| 696 |
# Ensure that the type disallows instantiation (bpo-43916) |
| 697 |
check_disallow_instantiation(self, unicodedata.UCD) |
| 698 |
|
| 699 |
@force_not_colorized |
| 700 |
def test_failed_import_during_compiling(self): |
| 701 |
# Issue 4367 |
| 702 |
# Decoding \N escapes requires the unicodedata module. If it can't be |
| 703 |
# imported, we shouldn't segfault. |
| 704 |
|
| 705 |
# This program should raise a SyntaxError in the eval. |
| 706 |
code = "import sys;" \ |
| 707 |
"sys.modules['unicodedata'] = None;" \ |
| 708 |
"""eval("'\\\\N{SOFT HYPHEN}'")""" |
| 709 |
# We use a separate process because the unicodedata module may already |
| 710 |
# have been loaded in this process. |
| 711 |
result = script_helper.assert_python_failure("-c", code) |
| 712 |
error = "SyntaxError: (unicode error) \\N escapes not supported " \ |
| 713 |
"(can't load unicodedata module)" |
| 714 |
self.assertIn(error, result.err.decode("ascii")) |
| 715 |
|
| 716 |
def test_unicodedata_unload_reload(self): |
| 717 |
# gh-149449: dropping unicodedata and running gc must not leave the |
| 718 |
# cached _ucnhash_CAPI pointer dangling. |
| 719 |
code = ( |
| 720 |
"import gc, sys\n" |
| 721 |
"assert '\\N{GRINNING FACE}'.encode(" |
| 722 |
" 'ascii', errors='namereplace') == b'\\\\N{GRINNING FACE}'\n" |
| 723 |
"compile(r\"x = '\\\\N{LATIN CAPITAL LETTER A}'\", '<x>', 'exec')\n" |
| 724 |
"del sys.modules['unicodedata']\n" |
| 725 |
"gc.collect()\n" |
| 726 |
"assert '\\N{WINKING FACE}'.encode(" |
| 727 |
" 'ascii', errors='namereplace') == b'\\\\N{WINKING FACE}'\n" |
| 728 |
"compile(r\"x = '\\\\N{LATIN CAPITAL LETTER B}'\", '<x>', 'exec')\n" |
| 729 |
) |
| 730 |
script_helper.assert_python_ok("-c", code) |
| 731 |
|
| 732 |
def test_decimal_numeric_consistent(self): |
| 733 |
# Test that decimal and numeric are consistent, |
| 734 |
# i.e. if a character has a decimal value, |
| 735 |
# its numeric value should be the same. |
| 736 |
count = 0 |
| 737 |
for c in iterallchars(): |
| 738 |
dec = self.db.decimal(c, -1) |
| 739 |
if dec != -1: |
| 740 |
self.assertEqual(dec, self.db.numeric(c)) |
| 741 |
count += 1 |
| 742 |
self.assertTrue(count >= 10, count) # should have tested at least the ASCII digits |
| 743 |
|
| 744 |
def test_digit_numeric_consistent(self): |
| 745 |
# Test that digit and numeric are consistent, |
| 746 |
# i.e. if a character has a digit value, |
| 747 |
# its numeric value should be the same. |
| 748 |
count = 0 |
| 749 |
for c in iterallchars(): |
| 750 |
dec = self.db.digit(c, -1) |
| 751 |
if dec != -1: |
| 752 |
self.assertEqual(dec, self.db.numeric(c)) |
| 753 |
count += 1 |
| 754 |
self.assertTrue(count >= 10, count) # should have tested at least the ASCII digits |
| 755 |
|
| 756 |
def test_normalize_consistent(self): |
| 757 |
allchars = list(iterallchars()) |
| 758 |
for form in ('NFC', 'NFD', 'NFKC', 'NFKD'): |
| 759 |
for c in allchars: |
| 760 |
norm = self.db.normalize(form, c) |
| 761 |
self.assertEqual(self.db.is_normalized(form, c), norm == c) |
| 762 |
if norm != c: |
| 763 |
self.assertEqual(self.db.normalize(form, norm), norm) |
| 764 |
self.assertTrue(self.db.is_normalized(form, norm)) |
| 765 |
|
| 766 |
def test_bug_1704793(self): |
| 767 |
self.assertEqual(self.db.lookup("GOTHIC LETTER FAIHU"), '\U00010346') |
| 768 |
|
| 769 |
def test_ucd_510(self): |
| 770 |
# In UCD 5.1.0, a mirrored property changed wrt. UCD 3.2.0 |
| 771 |
self.assertTrue(unicodedata.mirrored("\u0f3a")) |
| 772 |
self.assertTrue(not unicodedata.ucd_3_2_0.mirrored("\u0f3a")) |
| 773 |
# Also, we now have two ways of representing |
| 774 |
# the upper-case mapping: as delta, or as absolute value |
| 775 |
self.assertTrue("a".upper()=='A') |
| 776 |
self.assertTrue("\u1d79".upper()=='\ua77d') |
| 777 |
self.assertTrue(".".upper()=='.') |
| 778 |
|
| 779 |
@requires_resource('cpu') |
| 780 |
def test_bug_5828(self): |
| 781 |
self.assertEqual("\u1d79".lower(), "\u1d79") |
| 782 |
# Only U+0000 should have U+0000 as its upper/lower/titlecase variant |
| 783 |
self.assertEqual( |
| 784 |
[ |
| 785 |
c for c in iterallchars() |
| 786 |
if "\x00" in (c.lower(), c.upper(), c.title()) |
| 787 |
], |
| 788 |
["\x00"] |
| 789 |
) |
| 790 |
|
| 791 |
def test_bug_4971(self): |
| 792 |
# LETTER DZ WITH CARON: DZ, Dz, dz |
| 793 |
self.assertEqual("\u01c4".title(), "\u01c5") |
| 794 |
self.assertEqual("\u01c5".title(), "\u01c5") |
| 795 |
self.assertEqual("\u01c6".title(), "\u01c5") |
| 796 |
|
| 797 |
def test_linebreak_7643(self): |
| 798 |
for c in iterallchars(): |
| 799 |
lines = (c + 'A').splitlines() |
| 800 |
if c in ('\x0a', '\x0b', '\x0c', '\x0d', '\x85', |
| 801 |
'\x1c', '\x1d', '\x1e', '\u2028', '\u2029'): |
| 802 |
self.assertEqual(len(lines), 2, |
| 803 |
r"%a should be a linebreak" % c) |
| 804 |
else: |
| 805 |
self.assertEqual(len(lines), 1, |
| 806 |
r"%a should not be a linebreak" % c) |
| 807 |
|
| 808 |
|
| 809 |
class NormalizationTest(unittest.TestCase): |
| 810 |
@staticmethod |
| 811 |
def check_version(testfile): |
| 812 |
hdr = testfile.readline() |
| 813 |
return unicodedata.unidata_version in hdr |
| 814 |
|
| 815 |
@staticmethod |
| 816 |
def unistr(data): |
| 817 |
data = [int(x, 16) for x in data.split(" ")] |
| 818 |
return "".join([chr(x) for x in data]) |
| 819 |
|
| 820 |
@requires_resource('network') |
| 821 |
@requires_resource('cpu') |
| 822 |
def test_normalization(self): |
| 823 |
TESTDATAFILE = "NormalizationTest.txt" |
| 824 |
TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{TESTDATAFILE}" |
| 825 |
|
| 826 |
# Hit the exception early |
| 827 |
try: |
| 828 |
testdata = open_urlresource(TESTDATAURL, encoding="utf-8", |
| 829 |
check=self.check_version) |
| 830 |
except PermissionError: |
| 831 |
self.skipTest(f"Permission error when downloading {TESTDATAURL} " |
| 832 |
f"into the test data directory") |
| 833 |
except (OSError, HTTPException) as exc: |
| 834 |
self.skipTest(f"Failed to download {TESTDATAURL}: {exc}") |
| 835 |
|
| 836 |
with testdata: |
| 837 |
self.run_normalization_tests(testdata, unicodedata) |
| 838 |
|
| 839 |
@requires_resource('cpu') |
| 840 |
def test_normalization_3_2_0(self): |
| 841 |
testdatafile = findfile('NormalizationTest-3.2.0.txt') |
| 842 |
with open(testdatafile, encoding='utf-8') as testdata: |
| 843 |
self.run_normalization_tests(testdata, unicodedata.ucd_3_2_0) |
| 844 |
|
| 845 |
def run_normalization_tests(self, testdata, ucd): |
| 846 |
part = None |
| 847 |
part1_data = set() |
| 848 |
|
| 849 |
NFC = partial(ucd.normalize, "NFC") |
| 850 |
NFKC = partial(ucd.normalize, "NFKC") |
| 851 |
NFD = partial(ucd.normalize, "NFD") |
| 852 |
NFKD = partial(ucd.normalize, "NFKD") |
| 853 |
is_normalized = ucd.is_normalized |
| 854 |
|
| 855 |
for line in testdata: |
| 856 |
if '#' in line: |
| 857 |
line = line.split('#')[0] |
| 858 |
line = line.strip() |
| 859 |
if not line: |
| 860 |
continue |
| 861 |
if line.startswith("@Part"): |
| 862 |
part = line.split()[0] |
| 863 |
continue |
| 864 |
c1,c2,c3,c4,c5 = [self.unistr(x) for x in line.split(';')[:-1]] |
| 865 |
|
| 866 |
# Perform tests |
| 867 |
self.assertTrue(c2 == NFC(c1) == NFC(c2) == NFC(c3), line) |
| 868 |
self.assertTrue(c4 == NFC(c4) == NFC(c5), line) |
| 869 |
self.assertTrue(c3 == NFD(c1) == NFD(c2) == NFD(c3), line) |
| 870 |
self.assertTrue(c5 == NFD(c4) == NFD(c5), line) |
| 871 |
self.assertTrue(c4 == NFKC(c1) == NFKC(c2) == \ |
| 872 |
NFKC(c3) == NFKC(c4) == NFKC(c5), |
| 873 |
line) |
| 874 |
self.assertTrue(c5 == NFKD(c1) == NFKD(c2) == \ |
| 875 |
NFKD(c3) == NFKD(c4) == NFKD(c5), |
| 876 |
line) |
| 877 |
|
| 878 |
self.assertTrue(is_normalized("NFC", c2)) |
| 879 |
self.assertTrue(is_normalized("NFC", c4)) |
| 880 |
|
| 881 |
self.assertTrue(is_normalized("NFD", c3)) |
| 882 |
self.assertTrue(is_normalized("NFD", c5)) |
| 883 |
|
| 884 |
self.assertTrue(is_normalized("NFKC", c4)) |
| 885 |
self.assertTrue(is_normalized("NFKD", c5)) |
| 886 |
|
| 887 |
# Record part 1 data |
| 888 |
if part == "@Part1": |
| 889 |
part1_data.add(c1) |
| 890 |
|
| 891 |
# Perform tests for all other data |
| 892 |
for X in iterallchars(): |
| 893 |
if X in part1_data: |
| 894 |
continue |
| 895 |
self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), ord(X)) |
| 896 |
|
| 897 |
def test_edge_cases(self): |
| 898 |
self.assertRaises(TypeError, unicodedata.normalize) |
| 899 |
self.assertRaises(ValueError, unicodedata.normalize, 'unknown', 'xx') |
| 900 |
self.assertEqual(unicodedata.normalize('NFKC', ''), '') |
| 901 |
|
| 902 |
def test_bug_834676(self): |
| 903 |
# Check for bug 834676 |
| 904 |
unicodedata.normalize('NFC', '\ud55c\uae00') |
| 905 |
|
| 906 |
def test_normalize_return_type(self): |
| 907 |
# gh-129569: normalize() return type must always be str |
| 908 |
normalize = unicodedata.normalize |
| 909 |
|
| 910 |
class MyStr(str): |
| 911 |
pass |
| 912 |
|
| 913 |
normalization_forms = ("NFC", "NFKC", "NFD", "NFKD") |
| 914 |
input_strings = ( |
| 915 |
# normalized strings |
| 916 |
"", |
| 917 |
"ascii", |
| 918 |
# unnormalized strings |
| 919 |
"\u1e0b\u0323", |
| 920 |
"\u0071\u0307\u0323", |
| 921 |
) |
| 922 |
|
| 923 |
for form in normalization_forms: |
| 924 |
for input_str in input_strings: |
| 925 |
with self.subTest(form=form, input_str=input_str): |
| 926 |
self.assertIs(type(normalize(form, input_str)), str) |
| 927 |
self.assertIs(type(normalize(form, MyStr(input_str))), str) |
| 928 |
|
| 929 |
|
| 930 |
if __name__ == "__main__": |
| 931 |
unittest.main() |