Debian Codegraph @

Lib/test/test_ucn.py

raw
1 """ Test script for the Unicode implementation.
2
3 Written by Bill Tutt.
4 Modified for Python 2.0 by Fredrik Lundh ([email protected])
5
6 (c) Copyright CNRI, All Rights Reserved. NO WARRANTY.
7
8 """#"
9
10 import ast
11 import unittest
12 import unicodedata
13 import urllib.error
14
15 from test import support
16 from http.client import HTTPException
17
18 try:
19 from _testcapi import INT_MAX, PY_SSIZE_T_MAX, UINT_MAX
20 except ImportError:
21 INT_MAX = PY_SSIZE_T_MAX = UINT_MAX = 2**64 - 1
22
23 class UnicodeNamesTest(unittest.TestCase):
24
25 def checkletter(self, name, code):
26 # Helper that put all \N escapes inside eval'd raw strings,
27 # to make sure this script runs even if the compiler
28 # chokes on \N escapes
29 res = ast.literal_eval(r'"\N{%s}"' % name)
30 self.assertEqual(res, code)
31 return res
32
33 def test_general(self):
34 # General and case insensitivity test:
35 chars = [
36 "LATIN CAPITAL LETTER T",
37 "LATIN SMALL LETTER H",
38 "LATIN SMALL LETTER E",
39 "SPACE",
40 "LATIN SMALL LETTER R",
41 "LATIN CAPITAL LETTER E",
42 "LATIN SMALL LETTER D",
43 "SPACE",
44 "LATIN SMALL LETTER f",
45 "LATIN CAPITAL LeTtEr o",
46 "LATIN SMaLl LETTER x",
47 "SPACE",
48 "LATIN SMALL LETTER A",
49 "LATIN SMALL LETTER T",
50 "LATIN SMALL LETTER E",
51 "SPACE",
52 "LATIN SMALL LETTER T",
53 "LATIN SMALL LETTER H",
54 "LATIN SMALL LETTER E",
55 "SpAcE",
56 "LATIN SMALL LETTER S",
57 "LATIN SMALL LETTER H",
58 "LATIN small LETTER e",
59 "LATIN small LETTER e",
60 "LATIN SMALL LETTER P",
61 "FULL STOP"
62 ]
63 string = "The rEd fOx ate the sheep."
64
65 self.assertEqual(
66 "".join([self.checkletter(*args) for args in zip(chars, string)]),
67 string
68 )
69
70 def test_ascii_letters(self):
71 for char in "".join(map(chr, range(ord("a"), ord("z")))):
72 name = "LATIN SMALL LETTER %s" % char.upper()
73 code = unicodedata.lookup(name)
74 self.assertEqual(unicodedata.name(code), name)
75
76 def test_hangul_syllables(self):
77 self.checkletter("HANGUL SYLLABLE GA", "\uac00")
78 self.checkletter("HANGUL SYLLABLE GGWEOSS", "\uafe8")
79 self.checkletter("HANGUL SYLLABLE DOLS", "\ub3d0")
80 self.checkletter("HANGUL SYLLABLE RYAN", "\ub7b8")
81 self.checkletter("HANGUL SYLLABLE MWIK", "\ubba0")
82 self.checkletter("HANGUL SYLLABLE BBWAEM", "\ubf88")
83 self.checkletter("HANGUL SYLLABLE SSEOL", "\uc370")
84 self.checkletter("HANGUL SYLLABLE YI", "\uc758")
85 self.checkletter("HANGUL SYLLABLE JJYOSS", "\ucb40")
86 self.checkletter("HANGUL SYLLABLE KYEOLS", "\ucf28")
87 self.checkletter("HANGUL SYLLABLE PAN", "\ud310")
88 self.checkletter("HANGUL SYLLABLE HWEOK", "\ud6f8")
89 self.checkletter("HANGUL SYLLABLE HIH", "\ud7a3")
90
91 self.checkletter("haNGul SYllABle WAe", '\uc65c')
92 self.checkletter("HAngUL syLLabLE waE", '\uc65c')
93
94 self.assertRaises(ValueError, unicodedata.name, "\ud7a4")
95
96 def test_cjk_unified_ideographs(self):
97 self.checkletter("CJK UNIFIED IDEOGRAPH-3400", "\u3400")
98 self.checkletter("CJK UNIFIED IDEOGRAPH-4DB5", "\u4db5")
99 self.checkletter("CJK UNIFIED IDEOGRAPH-4E00", "\u4e00")
100 self.checkletter("CJK UNIFIED IDEOGRAPH-9FCB", "\u9fCB")
101 self.checkletter("CJK UNIFIED IDEOGRAPH-20000", "\U00020000")
102 self.checkletter("CJK UNIFIED IDEOGRAPH-2A6D6", "\U0002a6d6")
103 self.checkletter("CJK UNIFIED IDEOGRAPH-2A700", "\U0002A700")
104 self.checkletter("CJK UNIFIED IDEOGRAPH-2B734", "\U0002B734")
105 self.checkletter("CJK UNIFIED IDEOGRAPH-2B740", "\U0002B740")
106 self.checkletter("CJK UNIFIED IDEOGRAPH-2B81D", "\U0002B81D")
107 self.checkletter("CJK UNIFIED IDEOGRAPH-3134A", "\U0003134A")
108
109 self.checkletter("cjK UniFIeD idEogRAph-3aBc", "\u3abc")
110 self.checkletter("CJk uNIfiEd IDeOGraPH-3AbC", "\u3abc")
111 self.checkletter("cjK UniFIeD idEogRAph-2aBcD", "\U0002abcd")
112 self.checkletter("CJk uNIfiEd IDeOGraPH-2AbCd", "\U0002abcd")
113
114 def test_tangut_ideographs(self):
115 self.checkletter("TANGUT IDEOGRAPH-17000", "\U00017000")
116 self.checkletter("TANGUT IDEOGRAPH-187F7", "\U000187f7")
117 self.checkletter("TANGUT IDEOGRAPH-18D00", "\U00018D00")
118 self.checkletter("TANGUT IDEOGRAPH-18D08", "\U00018d08")
119 self.checkletter("tangut ideograph-18d08", "\U00018d08")
120
121 def test_egyptian_hieroglyphs(self):
122 self.checkletter("EGYPTIAN HIEROGLYPH-13460", "\U00013460")
123 self.checkletter("EGYPTIAN HIEROGLYPH-143FA", "\U000143fa")
124 self.checkletter("egyptian hieroglyph-143fa", "\U000143fa")
125
126 def test_khitan_small_script_characters(self):
127 self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18B00", "\U00018b00")
128 self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CD5", "\U00018cd5")
129 self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CFF", "\U00018cff")
130 self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CFF", "\U00018cff")
131 self.checkletter("khitan small script character-18cff", "\U00018cff")
132
133 def test_nushu_characters(self):
134 self.checkletter("NUSHU CHARACTER-1B170", "\U0001b170")
135 self.checkletter("NUSHU CHARACTER-1B2FB", "\U0001b2fb")
136 self.checkletter("nushu character-1b2fb", "\U0001b2fb")
137
138 def test_bmp_characters(self):
139 for code in range(0x10000):
140 char = chr(code)
141 name = unicodedata.name(char, None)
142 if name is not None:
143 self.assertEqual(unicodedata.lookup(name), char)
144
145 def test_misc_symbols(self):
146 self.checkletter("PILCROW SIGN", "\u00b6")
147 self.checkletter("REPLACEMENT CHARACTER", "\uFFFD")
148 self.checkletter("HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK", "\uFF9F")
149 self.checkletter("FULLWIDTH LATIN SMALL LETTER A", "\uFF41")
150
151 def test_aliases(self):
152 # Check that the aliases defined in the NameAliases.txt file work.
153 # This should be updated when new aliases are added or the file
154 # should be downloaded and parsed instead. See #12753.
155 aliases = [
156 ('LATIN CAPITAL LETTER GHA', 0x01A2),
157 ('LATIN SMALL LETTER GHA', 0x01A3),
158 ('KANNADA LETTER LLLA', 0x0CDE),
159 ('LAO LETTER FO FON', 0x0E9D),
160 ('LAO LETTER FO FAY', 0x0E9F),
161 ('LAO LETTER RO', 0x0EA3),
162 ('LAO LETTER LO', 0x0EA5),
163 ('TIBETAN MARK BKA- SHOG GI MGO RGYAN', 0x0FD0),
164 ('YI SYLLABLE ITERATION MARK', 0xA015),
165 ('PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET', 0xFE18),
166 ('BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS', 0x1D0C5)
167 ]
168 for alias, codepoint in aliases:
169 self.checkletter(alias, chr(codepoint))
170 name = unicodedata.name(chr(codepoint))
171 self.assertNotEqual(name, alias)
172 self.assertEqual(unicodedata.lookup(alias),
173 unicodedata.lookup(name))
174 with self.assertRaises(KeyError):
175 unicodedata.ucd_3_2_0.lookup(alias)
176
177 def test_aliases_names_in_pua_range(self):
178 # We are storing aliases in the PUA 15, but their names shouldn't leak
179 for cp in range(0xf0000, 0xf0100):
180 with self.assertRaises(ValueError) as cm:
181 unicodedata.name(chr(cp))
182 self.assertEqual(str(cm.exception), 'no such name')
183
184 def test_named_sequences_names_in_pua_range(self):
185 # We are storing named seq in the PUA 15, but their names shouldn't leak
186 for cp in range(0xf0100, 0xf0fff):
187 with self.assertRaises(ValueError) as cm:
188 unicodedata.name(chr(cp))
189 self.assertEqual(str(cm.exception), 'no such name')
190
191 def test_named_sequences_sample(self):
192 # Check a few named sequences. See #12753.
193 sequences = [
194 ('LATIN SMALL LETTER R WITH TILDE', '\u0072\u0303'),
195 ('TAMIL SYLLABLE SAI', '\u0BB8\u0BC8'),
196 ('TAMIL SYLLABLE MOO', '\u0BAE\u0BCB'),
197 ('TAMIL SYLLABLE NNOO', '\u0BA3\u0BCB'),
198 ('TAMIL CONSONANT KSS', '\u0B95\u0BCD\u0BB7\u0BCD'),
199 ]
200 for seqname, codepoints in sequences:
201 self.assertEqual(unicodedata.lookup(seqname), codepoints)
202 with self.assertRaises(SyntaxError):
203 self.checkletter(seqname, None)
204 with self.assertRaises(KeyError):
205 unicodedata.ucd_3_2_0.lookup(seqname)
206
207 def test_named_sequences_full(self):
208 # Check all the named sequences
209 def check_version(testfile):
210 hdr = testfile.readline()
211 return unicodedata.unidata_version in hdr
212 url = ("http://www.pythontest.net/unicode/%s/NamedSequences.txt" %
213 unicodedata.unidata_version)
214 try:
215 testdata = support.open_urlresource(url, encoding="utf-8",
216 check=check_version)
217 except urllib.error.HTTPError as exc:
218 exc.close()
219 self.skipTest(f"Could not retrieve {url}: {exc!r}")
220 except (OSError, HTTPException) as exc:
221 self.skipTest(f"Could not retrieve {url}: {exc!r}")
222 with testdata:
223 for line in testdata:
224 line = line.strip()
225 if not line or line.startswith('#'):
226 continue
227 seqname, codepoints = line.split(';')
228 codepoints = ''.join(chr(int(cp, 16)) for cp in codepoints.split())
229 self.assertEqual(unicodedata.lookup(seqname), codepoints)
230 with self.assertRaises(SyntaxError):
231 self.checkletter(seqname, None)
232 with self.assertRaises(KeyError):
233 unicodedata.ucd_3_2_0.lookup(seqname)
234
235 def test_errors(self):
236 self.assertRaises(TypeError, unicodedata.name)
237 self.assertRaises(TypeError, unicodedata.name, 'xx')
238 self.assertRaises(TypeError, unicodedata.lookup)
239 self.assertRaises(KeyError, unicodedata.lookup, 'unknown')
240
241 def test_strict_error_handling(self):
242 # bogus character name
243 self.assertRaises(
244 UnicodeError,
245 str, b"\\N{blah}", 'unicode-escape', 'strict'
246 )
247 # long bogus character name
248 self.assertRaises(
249 UnicodeError,
250 str, bytes("\\N{%s}" % ("x" * 100000), "ascii"), 'unicode-escape', 'strict'
251 )
252 # missing closing brace
253 self.assertRaises(
254 UnicodeError,
255 str, b"\\N{SPACE", 'unicode-escape', 'strict'
256 )
257 # missing opening brace
258 self.assertRaises(
259 UnicodeError,
260 str, b"\\NSPACE", 'unicode-escape', 'strict'
261 )
262
263 @support.cpython_only
264 @unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, "needs UINT_MAX < SIZE_MAX")
265 @support.bigmemtest(size=UINT_MAX + 1, memuse=2 + 1, dry_run=False)
266 def test_issue16335(self, size):
267 # very very long bogus character name
268 x = b'\\N{SPACE' + b'x' * (UINT_MAX + 1) + b'}'
269 self.assertEqual(len(x), len(b'\\N{SPACE}') + (UINT_MAX + 1))
270 self.assertRaisesRegex(UnicodeError,
271 'unknown Unicode character name',
272 x.decode, 'unicode-escape'
273 )
274
275
276 if __name__ == "__main__":
277 unittest.main()