| 1 |
""" Test script for the Unicode implementation. |
| 2 |
|
| 3 |
Written by Bill Tutt. |
| 4 |
Modified for Python 2.0 by Fredrik Lundh ([email protected]) |
| 5 |
|
| 6 |
(c) Copyright CNRI, All Rights Reserved. NO WARRANTY. |
| 7 |
|
| 8 |
"""#" |
| 9 |
|
| 10 |
import ast |
| 11 |
import unittest |
| 12 |
import unicodedata |
| 13 |
import urllib.error |
| 14 |
|
| 15 |
from test import support |
| 16 |
from http.client import HTTPException |
| 17 |
|
| 18 |
try: |
| 19 |
from _testcapi import INT_MAX, PY_SSIZE_T_MAX, UINT_MAX |
| 20 |
except ImportError: |
| 21 |
INT_MAX = PY_SSIZE_T_MAX = UINT_MAX = 2**64 - 1 |
| 22 |
|
| 23 |
class UnicodeNamesTest(unittest.TestCase): |
| 24 |
|
| 25 |
def checkletter(self, name, code): |
| 26 |
# Helper that put all \N escapes inside eval'd raw strings, |
| 27 |
# to make sure this script runs even if the compiler |
| 28 |
# chokes on \N escapes |
| 29 |
res = ast.literal_eval(r'"\N{%s}"' % name) |
| 30 |
self.assertEqual(res, code) |
| 31 |
return res |
| 32 |
|
| 33 |
def test_general(self): |
| 34 |
# General and case insensitivity test: |
| 35 |
chars = [ |
| 36 |
"LATIN CAPITAL LETTER T", |
| 37 |
"LATIN SMALL LETTER H", |
| 38 |
"LATIN SMALL LETTER E", |
| 39 |
"SPACE", |
| 40 |
"LATIN SMALL LETTER R", |
| 41 |
"LATIN CAPITAL LETTER E", |
| 42 |
"LATIN SMALL LETTER D", |
| 43 |
"SPACE", |
| 44 |
"LATIN SMALL LETTER f", |
| 45 |
"LATIN CAPITAL LeTtEr o", |
| 46 |
"LATIN SMaLl LETTER x", |
| 47 |
"SPACE", |
| 48 |
"LATIN SMALL LETTER A", |
| 49 |
"LATIN SMALL LETTER T", |
| 50 |
"LATIN SMALL LETTER E", |
| 51 |
"SPACE", |
| 52 |
"LATIN SMALL LETTER T", |
| 53 |
"LATIN SMALL LETTER H", |
| 54 |
"LATIN SMALL LETTER E", |
| 55 |
"SpAcE", |
| 56 |
"LATIN SMALL LETTER S", |
| 57 |
"LATIN SMALL LETTER H", |
| 58 |
"LATIN small LETTER e", |
| 59 |
"LATIN small LETTER e", |
| 60 |
"LATIN SMALL LETTER P", |
| 61 |
"FULL STOP" |
| 62 |
] |
| 63 |
string = "The rEd fOx ate the sheep." |
| 64 |
|
| 65 |
self.assertEqual( |
| 66 |
"".join([self.checkletter(*args) for args in zip(chars, string)]), |
| 67 |
string |
| 68 |
) |
| 69 |
|
| 70 |
def test_ascii_letters(self): |
| 71 |
for char in "".join(map(chr, range(ord("a"), ord("z")))): |
| 72 |
name = "LATIN SMALL LETTER %s" % char.upper() |
| 73 |
code = unicodedata.lookup(name) |
| 74 |
self.assertEqual(unicodedata.name(code), name) |
| 75 |
|
| 76 |
def test_hangul_syllables(self): |
| 77 |
self.checkletter("HANGUL SYLLABLE GA", "\uac00") |
| 78 |
self.checkletter("HANGUL SYLLABLE GGWEOSS", "\uafe8") |
| 79 |
self.checkletter("HANGUL SYLLABLE DOLS", "\ub3d0") |
| 80 |
self.checkletter("HANGUL SYLLABLE RYAN", "\ub7b8") |
| 81 |
self.checkletter("HANGUL SYLLABLE MWIK", "\ubba0") |
| 82 |
self.checkletter("HANGUL SYLLABLE BBWAEM", "\ubf88") |
| 83 |
self.checkletter("HANGUL SYLLABLE SSEOL", "\uc370") |
| 84 |
self.checkletter("HANGUL SYLLABLE YI", "\uc758") |
| 85 |
self.checkletter("HANGUL SYLLABLE JJYOSS", "\ucb40") |
| 86 |
self.checkletter("HANGUL SYLLABLE KYEOLS", "\ucf28") |
| 87 |
self.checkletter("HANGUL SYLLABLE PAN", "\ud310") |
| 88 |
self.checkletter("HANGUL SYLLABLE HWEOK", "\ud6f8") |
| 89 |
self.checkletter("HANGUL SYLLABLE HIH", "\ud7a3") |
| 90 |
|
| 91 |
self.checkletter("haNGul SYllABle WAe", '\uc65c') |
| 92 |
self.checkletter("HAngUL syLLabLE waE", '\uc65c') |
| 93 |
|
| 94 |
self.assertRaises(ValueError, unicodedata.name, "\ud7a4") |
| 95 |
|
| 96 |
def test_cjk_unified_ideographs(self): |
| 97 |
self.checkletter("CJK UNIFIED IDEOGRAPH-3400", "\u3400") |
| 98 |
self.checkletter("CJK UNIFIED IDEOGRAPH-4DB5", "\u4db5") |
| 99 |
self.checkletter("CJK UNIFIED IDEOGRAPH-4E00", "\u4e00") |
| 100 |
self.checkletter("CJK UNIFIED IDEOGRAPH-9FCB", "\u9fCB") |
| 101 |
self.checkletter("CJK UNIFIED IDEOGRAPH-20000", "\U00020000") |
| 102 |
self.checkletter("CJK UNIFIED IDEOGRAPH-2A6D6", "\U0002a6d6") |
| 103 |
self.checkletter("CJK UNIFIED IDEOGRAPH-2A700", "\U0002A700") |
| 104 |
self.checkletter("CJK UNIFIED IDEOGRAPH-2B734", "\U0002B734") |
| 105 |
self.checkletter("CJK UNIFIED IDEOGRAPH-2B740", "\U0002B740") |
| 106 |
self.checkletter("CJK UNIFIED IDEOGRAPH-2B81D", "\U0002B81D") |
| 107 |
self.checkletter("CJK UNIFIED IDEOGRAPH-3134A", "\U0003134A") |
| 108 |
|
| 109 |
self.checkletter("cjK UniFIeD idEogRAph-3aBc", "\u3abc") |
| 110 |
self.checkletter("CJk uNIfiEd IDeOGraPH-3AbC", "\u3abc") |
| 111 |
self.checkletter("cjK UniFIeD idEogRAph-2aBcD", "\U0002abcd") |
| 112 |
self.checkletter("CJk uNIfiEd IDeOGraPH-2AbCd", "\U0002abcd") |
| 113 |
|
| 114 |
def test_tangut_ideographs(self): |
| 115 |
self.checkletter("TANGUT IDEOGRAPH-17000", "\U00017000") |
| 116 |
self.checkletter("TANGUT IDEOGRAPH-187F7", "\U000187f7") |
| 117 |
self.checkletter("TANGUT IDEOGRAPH-18D00", "\U00018D00") |
| 118 |
self.checkletter("TANGUT IDEOGRAPH-18D08", "\U00018d08") |
| 119 |
self.checkletter("tangut ideograph-18d08", "\U00018d08") |
| 120 |
|
| 121 |
def test_egyptian_hieroglyphs(self): |
| 122 |
self.checkletter("EGYPTIAN HIEROGLYPH-13460", "\U00013460") |
| 123 |
self.checkletter("EGYPTIAN HIEROGLYPH-143FA", "\U000143fa") |
| 124 |
self.checkletter("egyptian hieroglyph-143fa", "\U000143fa") |
| 125 |
|
| 126 |
def test_khitan_small_script_characters(self): |
| 127 |
self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18B00", "\U00018b00") |
| 128 |
self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CD5", "\U00018cd5") |
| 129 |
self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CFF", "\U00018cff") |
| 130 |
self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CFF", "\U00018cff") |
| 131 |
self.checkletter("khitan small script character-18cff", "\U00018cff") |
| 132 |
|
| 133 |
def test_nushu_characters(self): |
| 134 |
self.checkletter("NUSHU CHARACTER-1B170", "\U0001b170") |
| 135 |
self.checkletter("NUSHU CHARACTER-1B2FB", "\U0001b2fb") |
| 136 |
self.checkletter("nushu character-1b2fb", "\U0001b2fb") |
| 137 |
|
| 138 |
def test_bmp_characters(self): |
| 139 |
for code in range(0x10000): |
| 140 |
char = chr(code) |
| 141 |
name = unicodedata.name(char, None) |
| 142 |
if name is not None: |
| 143 |
self.assertEqual(unicodedata.lookup(name), char) |
| 144 |
|
| 145 |
def test_misc_symbols(self): |
| 146 |
self.checkletter("PILCROW SIGN", "\u00b6") |
| 147 |
self.checkletter("REPLACEMENT CHARACTER", "\uFFFD") |
| 148 |
self.checkletter("HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK", "\uFF9F") |
| 149 |
self.checkletter("FULLWIDTH LATIN SMALL LETTER A", "\uFF41") |
| 150 |
|
| 151 |
def test_aliases(self): |
| 152 |
# Check that the aliases defined in the NameAliases.txt file work. |
| 153 |
# This should be updated when new aliases are added or the file |
| 154 |
# should be downloaded and parsed instead. See #12753. |
| 155 |
aliases = [ |
| 156 |
('LATIN CAPITAL LETTER GHA', 0x01A2), |
| 157 |
('LATIN SMALL LETTER GHA', 0x01A3), |
| 158 |
('KANNADA LETTER LLLA', 0x0CDE), |
| 159 |
('LAO LETTER FO FON', 0x0E9D), |
| 160 |
('LAO LETTER FO FAY', 0x0E9F), |
| 161 |
('LAO LETTER RO', 0x0EA3), |
| 162 |
('LAO LETTER LO', 0x0EA5), |
| 163 |
('TIBETAN MARK BKA- SHOG GI MGO RGYAN', 0x0FD0), |
| 164 |
('YI SYLLABLE ITERATION MARK', 0xA015), |
| 165 |
('PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET', 0xFE18), |
| 166 |
('BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS', 0x1D0C5) |
| 167 |
] |
| 168 |
for alias, codepoint in aliases: |
| 169 |
self.checkletter(alias, chr(codepoint)) |
| 170 |
name = unicodedata.name(chr(codepoint)) |
| 171 |
self.assertNotEqual(name, alias) |
| 172 |
self.assertEqual(unicodedata.lookup(alias), |
| 173 |
unicodedata.lookup(name)) |
| 174 |
with self.assertRaises(KeyError): |
| 175 |
unicodedata.ucd_3_2_0.lookup(alias) |
| 176 |
|
| 177 |
def test_aliases_names_in_pua_range(self): |
| 178 |
# We are storing aliases in the PUA 15, but their names shouldn't leak |
| 179 |
for cp in range(0xf0000, 0xf0100): |
| 180 |
with self.assertRaises(ValueError) as cm: |
| 181 |
unicodedata.name(chr(cp)) |
| 182 |
self.assertEqual(str(cm.exception), 'no such name') |
| 183 |
|
| 184 |
def test_named_sequences_names_in_pua_range(self): |
| 185 |
# We are storing named seq in the PUA 15, but their names shouldn't leak |
| 186 |
for cp in range(0xf0100, 0xf0fff): |
| 187 |
with self.assertRaises(ValueError) as cm: |
| 188 |
unicodedata.name(chr(cp)) |
| 189 |
self.assertEqual(str(cm.exception), 'no such name') |
| 190 |
|
| 191 |
def test_named_sequences_sample(self): |
| 192 |
# Check a few named sequences. See #12753. |
| 193 |
sequences = [ |
| 194 |
('LATIN SMALL LETTER R WITH TILDE', '\u0072\u0303'), |
| 195 |
('TAMIL SYLLABLE SAI', '\u0BB8\u0BC8'), |
| 196 |
('TAMIL SYLLABLE MOO', '\u0BAE\u0BCB'), |
| 197 |
('TAMIL SYLLABLE NNOO', '\u0BA3\u0BCB'), |
| 198 |
('TAMIL CONSONANT KSS', '\u0B95\u0BCD\u0BB7\u0BCD'), |
| 199 |
] |
| 200 |
for seqname, codepoints in sequences: |
| 201 |
self.assertEqual(unicodedata.lookup(seqname), codepoints) |
| 202 |
with self.assertRaises(SyntaxError): |
| 203 |
self.checkletter(seqname, None) |
| 204 |
with self.assertRaises(KeyError): |
| 205 |
unicodedata.ucd_3_2_0.lookup(seqname) |
| 206 |
|
| 207 |
def test_named_sequences_full(self): |
| 208 |
# Check all the named sequences |
| 209 |
def check_version(testfile): |
| 210 |
hdr = testfile.readline() |
| 211 |
return unicodedata.unidata_version in hdr |
| 212 |
url = ("http://www.pythontest.net/unicode/%s/NamedSequences.txt" % |
| 213 |
unicodedata.unidata_version) |
| 214 |
try: |
| 215 |
testdata = support.open_urlresource(url, encoding="utf-8", |
| 216 |
check=check_version) |
| 217 |
except urllib.error.HTTPError as exc: |
| 218 |
exc.close() |
| 219 |
self.skipTest(f"Could not retrieve {url}: {exc!r}") |
| 220 |
except (OSError, HTTPException) as exc: |
| 221 |
self.skipTest(f"Could not retrieve {url}: {exc!r}") |
| 222 |
with testdata: |
| 223 |
for line in testdata: |
| 224 |
line = line.strip() |
| 225 |
if not line or line.startswith('#'): |
| 226 |
continue |
| 227 |
seqname, codepoints = line.split(';') |
| 228 |
codepoints = ''.join(chr(int(cp, 16)) for cp in codepoints.split()) |
| 229 |
self.assertEqual(unicodedata.lookup(seqname), codepoints) |
| 230 |
with self.assertRaises(SyntaxError): |
| 231 |
self.checkletter(seqname, None) |
| 232 |
with self.assertRaises(KeyError): |
| 233 |
unicodedata.ucd_3_2_0.lookup(seqname) |
| 234 |
|
| 235 |
def test_errors(self): |
| 236 |
self.assertRaises(TypeError, unicodedata.name) |
| 237 |
self.assertRaises(TypeError, unicodedata.name, 'xx') |
| 238 |
self.assertRaises(TypeError, unicodedata.lookup) |
| 239 |
self.assertRaises(KeyError, unicodedata.lookup, 'unknown') |
| 240 |
|
| 241 |
def test_strict_error_handling(self): |
| 242 |
# bogus character name |
| 243 |
self.assertRaises( |
| 244 |
UnicodeError, |
| 245 |
str, b"\\N{blah}", 'unicode-escape', 'strict' |
| 246 |
) |
| 247 |
# long bogus character name |
| 248 |
self.assertRaises( |
| 249 |
UnicodeError, |
| 250 |
str, bytes("\\N{%s}" % ("x" * 100000), "ascii"), 'unicode-escape', 'strict' |
| 251 |
) |
| 252 |
# missing closing brace |
| 253 |
self.assertRaises( |
| 254 |
UnicodeError, |
| 255 |
str, b"\\N{SPACE", 'unicode-escape', 'strict' |
| 256 |
) |
| 257 |
# missing opening brace |
| 258 |
self.assertRaises( |
| 259 |
UnicodeError, |
| 260 |
str, b"\\NSPACE", 'unicode-escape', 'strict' |
| 261 |
) |
| 262 |
|
| 263 |
@support.cpython_only |
| 264 |
@unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, "needs UINT_MAX < SIZE_MAX") |
| 265 |
@support.bigmemtest(size=UINT_MAX + 1, memuse=2 + 1, dry_run=False) |
| 266 |
def test_issue16335(self, size): |
| 267 |
# very very long bogus character name |
| 268 |
x = b'\\N{SPACE' + b'x' * (UINT_MAX + 1) + b'}' |
| 269 |
self.assertEqual(len(x), len(b'\\N{SPACE}') + (UINT_MAX + 1)) |
| 270 |
self.assertRaisesRegex(UnicodeError, |
| 271 |
'unknown Unicode character name', |
| 272 |
x.decode, 'unicode-escape' |
| 273 |
) |
| 274 |
|
| 275 |
|
| 276 |
if __name__ == "__main__": |
| 277 |
unittest.main() |