| 1 |
# |
| 2 |
# (re)generate unicode property and type databases |
| 3 |
# |
| 4 |
# This script converts Unicode database files to Modules/unicodedata_db.h, |
| 5 |
# Modules/unicodename_db.h, and Objects/unicodetype_db.h |
| 6 |
# |
| 7 |
# history: |
| 8 |
# 2000-09-24 fl created (based on bits and pieces from unidb) |
| 9 |
# 2000-09-25 fl merged tim's splitbin fixes, separate decomposition table |
| 10 |
# 2000-09-25 fl added character type table |
| 11 |
# 2000-09-26 fl added LINEBREAK, DECIMAL, and DIGIT flags/fields (2.0) |
| 12 |
# 2000-11-03 fl expand first/last ranges |
| 13 |
# 2001-01-19 fl added character name tables (2.1) |
| 14 |
# 2001-01-21 fl added decomp compression; dynamic phrasebook threshold |
| 15 |
# 2002-09-11 wd use string methods |
| 16 |
# 2002-10-18 mvl update to Unicode 3.2 |
| 17 |
# 2002-10-22 mvl generate NFC tables |
| 18 |
# 2002-11-24 mvl expand all ranges, sort names version-independently |
| 19 |
# 2002-11-25 mvl add UNIDATA_VERSION |
| 20 |
# 2004-05-29 perky add east asian width information |
| 21 |
# 2006-03-10 mvl update to Unicode 4.1; add UCD 3.2 delta |
| 22 |
# 2008-06-11 gb add PRINTABLE_MASK for Atsuo Ishimoto's ascii() patch |
| 23 |
# 2011-10-21 ezio add support for name aliases and named sequences |
| 24 |
# 2012-01 benjamin add full case mappings |
| 25 |
# |
| 26 |
# written by Fredrik Lundh ([email protected]) |
| 27 |
# |
| 28 |
|
| 29 |
import dataclasses |
| 30 |
import os |
| 31 |
import sys |
| 32 |
import zipfile |
| 33 |
|
| 34 |
from functools import partial |
| 35 |
from textwrap import dedent |
| 36 |
from typing import Iterator, List, Optional, Set, Tuple |
| 37 |
|
| 38 |
SCRIPT = os.path.normpath(sys.argv[0]) |
| 39 |
VERSION = "3.3" |
| 40 |
|
| 41 |
# The Unicode Database |
| 42 |
# -------------------- |
| 43 |
# When changing UCD version please update |
| 44 |
# * Doc/library/stdtypes.rst, and |
| 45 |
# * Doc/library/unicodedata.rst |
| 46 |
# * Doc/reference/lexical_analysis.rst (three occurrences) |
| 47 |
UNIDATA_VERSION = "16.0.0" |
| 48 |
UNICODE_DATA = "UnicodeData%s.txt" |
| 49 |
COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt" |
| 50 |
EASTASIAN_WIDTH = "EastAsianWidth%s.txt" |
| 51 |
UNIHAN = "Unihan%s.zip" |
| 52 |
DERIVED_CORE_PROPERTIES = "DerivedCoreProperties%s.txt" |
| 53 |
DERIVEDNORMALIZATION_PROPS = "DerivedNormalizationProps%s.txt" |
| 54 |
LINE_BREAK = "LineBreak%s.txt" |
| 55 |
NAME_ALIASES = "NameAliases%s.txt" |
| 56 |
NAMED_SEQUENCES = "NamedSequences%s.txt" |
| 57 |
SPECIAL_CASING = "SpecialCasing%s.txt" |
| 58 |
CASE_FOLDING = "CaseFolding%s.txt" |
| 59 |
|
| 60 |
# Private Use Areas -- in planes 1, 15, 16 |
| 61 |
PUA_1 = range(0xE000, 0xF900) |
| 62 |
PUA_15 = range(0xF0000, 0xFFFFE) |
| 63 |
PUA_16 = range(0x100000, 0x10FFFE) |
| 64 |
|
| 65 |
# we use this ranges of PUA_15 to store name aliases and named sequences |
| 66 |
NAME_ALIASES_START = 0xF0000 |
| 67 |
NAMED_SEQUENCES_START = 0xF0200 |
| 68 |
|
| 69 |
old_versions = ["3.2.0"] |
| 70 |
|
| 71 |
CATEGORY_NAMES = [ "Cn", "Lu", "Ll", "Lt", "Mn", "Mc", "Me", "Nd", |
| 72 |
"Nl", "No", "Zs", "Zl", "Zp", "Cc", "Cf", "Cs", "Co", "Cn", "Lm", |
| 73 |
"Lo", "Pc", "Pd", "Ps", "Pe", "Pi", "Pf", "Po", "Sm", "Sc", "Sk", |
| 74 |
"So" ] |
| 75 |
|
| 76 |
BIDIRECTIONAL_NAMES = [ "", "L", "LRE", "LRO", "R", "AL", "RLE", "RLO", |
| 77 |
"PDF", "EN", "ES", "ET", "AN", "CS", "NSM", "BN", "B", "S", "WS", |
| 78 |
"ON", "LRI", "RLI", "FSI", "PDI" ] |
| 79 |
|
| 80 |
# "N" needs to be the first entry, see the comment in makeunicodedata |
| 81 |
EASTASIANWIDTH_NAMES = [ "N", "H", "W", "Na", "A", "F" ] |
| 82 |
|
| 83 |
MANDATORY_LINE_BREAKS = [ "BK", "CR", "LF", "NL" ] |
| 84 |
|
| 85 |
# note: should match definitions in Objects/unicodectype.c |
| 86 |
ALPHA_MASK = 0x01 |
| 87 |
DECIMAL_MASK = 0x02 |
| 88 |
DIGIT_MASK = 0x04 |
| 89 |
LOWER_MASK = 0x08 |
| 90 |
LINEBREAK_MASK = 0x10 |
| 91 |
SPACE_MASK = 0x20 |
| 92 |
TITLE_MASK = 0x40 |
| 93 |
UPPER_MASK = 0x80 |
| 94 |
XID_START_MASK = 0x100 |
| 95 |
XID_CONTINUE_MASK = 0x200 |
| 96 |
PRINTABLE_MASK = 0x400 |
| 97 |
NUMERIC_MASK = 0x800 |
| 98 |
CASE_IGNORABLE_MASK = 0x1000 |
| 99 |
CASED_MASK = 0x2000 |
| 100 |
EXTENDED_CASE_MASK = 0x4000 |
| 101 |
|
| 102 |
# Maps the range names in UnicodeData.txt to prefixes for |
| 103 |
# derived names specified by rule NR2. |
| 104 |
# Hangul should always be at index 0, since it uses special format. |
| 105 |
derived_name_range_names = [ |
| 106 |
("Hangul Syllable", "HANGUL SYLLABLE "), |
| 107 |
("CJK Ideograph", "CJK UNIFIED IDEOGRAPH-"), |
| 108 |
("Tangut Ideograph", "TANGUT IDEOGRAPH-"), |
| 109 |
] |
| 110 |
|
| 111 |
|
| 112 |
def maketables(trace=0): |
| 113 |
|
| 114 |
print("--- Reading", UNICODE_DATA % "", "...") |
| 115 |
|
| 116 |
unicode = UnicodeData(UNIDATA_VERSION) |
| 117 |
|
| 118 |
print(len(list(filter(None, unicode.table))), "characters") |
| 119 |
|
| 120 |
for version in old_versions: |
| 121 |
print("--- Reading", UNICODE_DATA % ("-"+version), "...") |
| 122 |
old_unicode = UnicodeData(version, ideograph_check=False) |
| 123 |
print(len(list(filter(None, old_unicode.table))), "characters") |
| 124 |
merge_old_version(version, unicode, old_unicode) |
| 125 |
|
| 126 |
makeunicodename(unicode, trace) |
| 127 |
makeunicodedata(unicode, trace) |
| 128 |
makeunicodetype(unicode, trace) |
| 129 |
|
| 130 |
|
| 131 |
# -------------------------------------------------------------------- |
| 132 |
# unicode character properties |
| 133 |
|
| 134 |
def makeunicodedata(unicode, trace): |
| 135 |
|
| 136 |
# the default value of east_asian_width is "N", for unassigned code points |
| 137 |
# not mentioned in EastAsianWidth.txt |
| 138 |
# in addition there are some reserved but unassigned code points in CJK |
| 139 |
# ranges that are classified as "W". code points in private use areas |
| 140 |
# have a width of "A". both of these have entries in |
| 141 |
# EastAsianWidth.txt |
| 142 |
# see https://unicode.org/reports/tr11/#Unassigned |
| 143 |
assert EASTASIANWIDTH_NAMES[0] == "N" |
| 144 |
dummy = (0, 0, 0, 0, 0, 0) |
| 145 |
table = [dummy] |
| 146 |
cache = {0: dummy} |
| 147 |
index = [0] * len(unicode.chars) |
| 148 |
|
| 149 |
FILE = "Modules/unicodedata_db.h" |
| 150 |
|
| 151 |
print("--- Preparing", FILE, "...") |
| 152 |
|
| 153 |
# 1) database properties |
| 154 |
|
| 155 |
for char in unicode.chars: |
| 156 |
record = unicode.table[char] |
| 157 |
if record: |
| 158 |
# extract database properties |
| 159 |
category = CATEGORY_NAMES.index(record.general_category) |
| 160 |
combining = int(record.canonical_combining_class) |
| 161 |
bidirectional = BIDIRECTIONAL_NAMES.index(record.bidi_class) |
| 162 |
mirrored = record.bidi_mirrored == "Y" |
| 163 |
eastasianwidth = EASTASIANWIDTH_NAMES.index(record.east_asian_width) |
| 164 |
normalizationquickcheck = record.quick_check |
| 165 |
item = ( |
| 166 |
category, combining, bidirectional, mirrored, eastasianwidth, |
| 167 |
normalizationquickcheck |
| 168 |
) |
| 169 |
elif unicode.widths[char] is not None: |
| 170 |
# an unassigned but reserved character, with a known |
| 171 |
# east_asian_width |
| 172 |
eastasianwidth = EASTASIANWIDTH_NAMES.index(unicode.widths[char]) |
| 173 |
item = (0, 0, 0, 0, eastasianwidth, 0) |
| 174 |
else: |
| 175 |
continue |
| 176 |
|
| 177 |
# add entry to index and item tables |
| 178 |
i = cache.get(item) |
| 179 |
if i is None: |
| 180 |
cache[item] = i = len(table) |
| 181 |
table.append(item) |
| 182 |
index[char] = i |
| 183 |
|
| 184 |
# 2) decomposition data |
| 185 |
|
| 186 |
decomp_data_cache = {} |
| 187 |
decomp_data = [0] |
| 188 |
decomp_prefix = [""] |
| 189 |
decomp_index = [0] * len(unicode.chars) |
| 190 |
decomp_size = 0 |
| 191 |
|
| 192 |
comp_pairs = [] |
| 193 |
comp_first = [None] * len(unicode.chars) |
| 194 |
comp_last = [None] * len(unicode.chars) |
| 195 |
|
| 196 |
for char in unicode.chars: |
| 197 |
record = unicode.table[char] |
| 198 |
if record: |
| 199 |
if record.decomposition_type: |
| 200 |
decomp = record.decomposition_type.split() |
| 201 |
if len(decomp) > 19: |
| 202 |
raise Exception("character %x has a decomposition too large for nfd_nfkd" % char) |
| 203 |
# prefix |
| 204 |
if decomp[0][0] == "<": |
| 205 |
prefix = decomp.pop(0) |
| 206 |
else: |
| 207 |
prefix = "" |
| 208 |
try: |
| 209 |
i = decomp_prefix.index(prefix) |
| 210 |
except ValueError: |
| 211 |
i = len(decomp_prefix) |
| 212 |
decomp_prefix.append(prefix) |
| 213 |
prefix = i |
| 214 |
assert prefix < 256 |
| 215 |
# content |
| 216 |
decomp = [prefix + (len(decomp)<<8)] + [int(s, 16) for s in decomp] |
| 217 |
# Collect NFC pairs |
| 218 |
if not prefix and len(decomp) == 3 and \ |
| 219 |
char not in unicode.exclusions and \ |
| 220 |
unicode.table[decomp[1]].canonical_combining_class == "0": |
| 221 |
p, l, r = decomp |
| 222 |
comp_first[l] = 1 |
| 223 |
comp_last[r] = 1 |
| 224 |
comp_pairs.append((l,r,char)) |
| 225 |
key = tuple(decomp) |
| 226 |
i = decomp_data_cache.get(key, -1) |
| 227 |
if i == -1: |
| 228 |
i = len(decomp_data) |
| 229 |
decomp_data.extend(decomp) |
| 230 |
decomp_size = decomp_size + len(decomp) * 2 |
| 231 |
decomp_data_cache[key] = i |
| 232 |
else: |
| 233 |
assert decomp_data[i:i+len(decomp)] == decomp |
| 234 |
else: |
| 235 |
i = 0 |
| 236 |
decomp_index[char] = i |
| 237 |
|
| 238 |
f = l = 0 |
| 239 |
comp_first_ranges = [] |
| 240 |
comp_last_ranges = [] |
| 241 |
prev_f = prev_l = None |
| 242 |
for i in unicode.chars: |
| 243 |
if comp_first[i] is not None: |
| 244 |
comp_first[i] = f |
| 245 |
f += 1 |
| 246 |
if prev_f is None: |
| 247 |
prev_f = (i,i) |
| 248 |
elif prev_f[1]+1 == i: |
| 249 |
prev_f = prev_f[0],i |
| 250 |
else: |
| 251 |
comp_first_ranges.append(prev_f) |
| 252 |
prev_f = (i,i) |
| 253 |
if comp_last[i] is not None: |
| 254 |
comp_last[i] = l |
| 255 |
l += 1 |
| 256 |
if prev_l is None: |
| 257 |
prev_l = (i,i) |
| 258 |
elif prev_l[1]+1 == i: |
| 259 |
prev_l = prev_l[0],i |
| 260 |
else: |
| 261 |
comp_last_ranges.append(prev_l) |
| 262 |
prev_l = (i,i) |
| 263 |
comp_first_ranges.append(prev_f) |
| 264 |
comp_last_ranges.append(prev_l) |
| 265 |
total_first = f |
| 266 |
total_last = l |
| 267 |
|
| 268 |
comp_data = [0]*(total_first*total_last) |
| 269 |
for f,l,char in comp_pairs: |
| 270 |
f = comp_first[f] |
| 271 |
l = comp_last[l] |
| 272 |
comp_data[f*total_last+l] = char |
| 273 |
|
| 274 |
print(len(table), "unique properties") |
| 275 |
print(len(decomp_prefix), "unique decomposition prefixes") |
| 276 |
print(len(decomp_data), "unique decomposition entries:", end=' ') |
| 277 |
print(decomp_size, "bytes") |
| 278 |
print(total_first, "first characters in NFC") |
| 279 |
print(total_last, "last characters in NFC") |
| 280 |
print(len(comp_pairs), "NFC pairs") |
| 281 |
|
| 282 |
print("--- Writing", FILE, "...") |
| 283 |
|
| 284 |
with open(FILE, "w") as fp: |
| 285 |
fprint = partial(print, file=fp) |
| 286 |
|
| 287 |
fprint("/* this file was generated by %s %s */" % (SCRIPT, VERSION)) |
| 288 |
fprint() |
| 289 |
fprint('#define UNIDATA_VERSION "%s"' % UNIDATA_VERSION) |
| 290 |
fprint("/* a list of unique database records */") |
| 291 |
fprint("const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = {") |
| 292 |
for item in table: |
| 293 |
fprint(" {%d, %d, %d, %d, %d, %d}," % item) |
| 294 |
fprint("};") |
| 295 |
fprint() |
| 296 |
|
| 297 |
fprint("/* Reindexing of NFC first characters. */") |
| 298 |
fprint("#define TOTAL_FIRST",total_first) |
| 299 |
fprint("#define TOTAL_LAST",total_last) |
| 300 |
fprint("struct reindex{int start;short count,index;};") |
| 301 |
fprint("static struct reindex nfc_first[] = {") |
| 302 |
for start,end in comp_first_ranges: |
| 303 |
fprint(" { %d, %d, %d}," % (start,end-start,comp_first[start])) |
| 304 |
fprint(" {0,0,0}") |
| 305 |
fprint("};\n") |
| 306 |
fprint("static struct reindex nfc_last[] = {") |
| 307 |
for start,end in comp_last_ranges: |
| 308 |
fprint(" { %d, %d, %d}," % (start,end-start,comp_last[start])) |
| 309 |
fprint(" {0,0,0}") |
| 310 |
fprint("};\n") |
| 311 |
|
| 312 |
# FIXME: <fl> the following tables could be made static, and |
| 313 |
# the support code moved into unicodedatabase.c |
| 314 |
|
| 315 |
fprint("/* string literals */") |
| 316 |
fprint("const char *_PyUnicode_CategoryNames[] = {") |
| 317 |
for name in CATEGORY_NAMES: |
| 318 |
fprint(" \"%s\"," % name) |
| 319 |
fprint(" NULL") |
| 320 |
fprint("};") |
| 321 |
|
| 322 |
fprint("const char *_PyUnicode_BidirectionalNames[] = {") |
| 323 |
for name in BIDIRECTIONAL_NAMES: |
| 324 |
fprint(" \"%s\"," % name) |
| 325 |
fprint(" NULL") |
| 326 |
fprint("};") |
| 327 |
|
| 328 |
fprint("const char *_PyUnicode_EastAsianWidthNames[] = {") |
| 329 |
for name in EASTASIANWIDTH_NAMES: |
| 330 |
fprint(" \"%s\"," % name) |
| 331 |
fprint(" NULL") |
| 332 |
fprint("};") |
| 333 |
|
| 334 |
fprint("static const char *decomp_prefix[] = {") |
| 335 |
for name in decomp_prefix: |
| 336 |
fprint(" \"%s\"," % name) |
| 337 |
fprint(" NULL") |
| 338 |
fprint("};") |
| 339 |
|
| 340 |
# split record index table |
| 341 |
index1, index2, shift = splitbins(index, trace) |
| 342 |
|
| 343 |
fprint("/* index tables for the database records */") |
| 344 |
fprint("#define SHIFT", shift) |
| 345 |
Array("index1", index1).dump(fp, trace) |
| 346 |
Array("index2", index2).dump(fp, trace) |
| 347 |
|
| 348 |
# split decomposition index table |
| 349 |
index1, index2, shift = splitbins(decomp_index, trace) |
| 350 |
|
| 351 |
fprint("/* decomposition data */") |
| 352 |
Array("decomp_data", decomp_data).dump(fp, trace) |
| 353 |
|
| 354 |
fprint("/* index tables for the decomposition data */") |
| 355 |
fprint("#define DECOMP_SHIFT", shift) |
| 356 |
Array("decomp_index1", index1).dump(fp, trace) |
| 357 |
Array("decomp_index2", index2).dump(fp, trace) |
| 358 |
|
| 359 |
index, index2, shift = splitbins(comp_data, trace) |
| 360 |
fprint("/* NFC pairs */") |
| 361 |
fprint("#define COMP_SHIFT", shift) |
| 362 |
Array("comp_index", index).dump(fp, trace) |
| 363 |
Array("comp_data", index2).dump(fp, trace) |
| 364 |
|
| 365 |
# Generate delta tables for old versions |
| 366 |
for version, table, normalization in unicode.changed: |
| 367 |
cversion = version.replace(".","_") |
| 368 |
records = [table[0]] |
| 369 |
cache = {table[0]:0} |
| 370 |
index = [0] * len(table) |
| 371 |
for i, record in enumerate(table): |
| 372 |
try: |
| 373 |
index[i] = cache[record] |
| 374 |
except KeyError: |
| 375 |
index[i] = cache[record] = len(records) |
| 376 |
records.append(record) |
| 377 |
index1, index2, shift = splitbins(index, trace) |
| 378 |
fprint("static const change_record change_records_%s[] = {" % cversion) |
| 379 |
for record in records: |
| 380 |
fprint(" { %s }," % ", ".join(map(str,record))) |
| 381 |
fprint("};") |
| 382 |
Array("changes_%s_index" % cversion, index1).dump(fp, trace) |
| 383 |
Array("changes_%s_data" % cversion, index2).dump(fp, trace) |
| 384 |
fprint("static const change_record* get_change_%s(Py_UCS4 n)" % cversion) |
| 385 |
fprint("{") |
| 386 |
fprint(" int index;") |
| 387 |
fprint(" if (n >= 0x110000) index = 0;") |
| 388 |
fprint(" else {") |
| 389 |
fprint(" index = changes_%s_index[n>>%d];" % (cversion, shift)) |
| 390 |
fprint(" index = changes_%s_data[(index<<%d)+(n & %d)];" % \ |
| 391 |
(cversion, shift, ((1<<shift)-1))) |
| 392 |
fprint(" }") |
| 393 |
fprint(" return change_records_%s+index;" % cversion) |
| 394 |
fprint("}\n") |
| 395 |
fprint("static Py_UCS4 normalization_%s(Py_UCS4 n)" % cversion) |
| 396 |
fprint("{") |
| 397 |
fprint(" switch(n) {") |
| 398 |
for k, v in normalization: |
| 399 |
fprint(" case %s: return 0x%s;" % (hex(k), v)) |
| 400 |
fprint(" default: return 0;") |
| 401 |
fprint(" }\n}\n") |
| 402 |
|
| 403 |
|
| 404 |
# -------------------------------------------------------------------- |
| 405 |
# unicode character type tables |
| 406 |
|
| 407 |
def makeunicodetype(unicode, trace): |
| 408 |
|
| 409 |
FILE = "Objects/unicodetype_db.h" |
| 410 |
|
| 411 |
print("--- Preparing", FILE, "...") |
| 412 |
|
| 413 |
# extract unicode types |
| 414 |
dummy = (0, 0, 0, 0, 0, 0) |
| 415 |
table = [dummy] |
| 416 |
cache = {dummy: 0} |
| 417 |
index = [0] * len(unicode.chars) |
| 418 |
numeric = {} |
| 419 |
spaces = [] |
| 420 |
linebreaks = [] |
| 421 |
extra_casing = [] |
| 422 |
|
| 423 |
for char in unicode.chars: |
| 424 |
record = unicode.table[char] |
| 425 |
if record: |
| 426 |
# extract database properties |
| 427 |
category = record.general_category |
| 428 |
bidirectional = record.bidi_class |
| 429 |
properties = record.binary_properties |
| 430 |
flags = 0 |
| 431 |
if category in ["Lm", "Lt", "Lu", "Ll", "Lo"]: |
| 432 |
flags |= ALPHA_MASK |
| 433 |
if "Lowercase" in properties: |
| 434 |
flags |= LOWER_MASK |
| 435 |
if 'Line_Break' in properties or bidirectional == "B": |
| 436 |
flags |= LINEBREAK_MASK |
| 437 |
linebreaks.append(char) |
| 438 |
if category == "Zs" or bidirectional in ("WS", "B", "S"): |
| 439 |
flags |= SPACE_MASK |
| 440 |
spaces.append(char) |
| 441 |
if category == "Lt": |
| 442 |
flags |= TITLE_MASK |
| 443 |
if "Uppercase" in properties: |
| 444 |
flags |= UPPER_MASK |
| 445 |
if char == ord(" ") or category[0] not in ("C", "Z"): |
| 446 |
flags |= PRINTABLE_MASK |
| 447 |
if "XID_Start" in properties: |
| 448 |
flags |= XID_START_MASK |
| 449 |
if "XID_Continue" in properties: |
| 450 |
flags |= XID_CONTINUE_MASK |
| 451 |
if "Cased" in properties: |
| 452 |
flags |= CASED_MASK |
| 453 |
if "Case_Ignorable" in properties: |
| 454 |
flags |= CASE_IGNORABLE_MASK |
| 455 |
sc = unicode.special_casing.get(char) |
| 456 |
cf = unicode.case_folding.get(char, [char]) |
| 457 |
if record.simple_uppercase_mapping: |
| 458 |
upper = int(record.simple_uppercase_mapping, 16) |
| 459 |
else: |
| 460 |
upper = char |
| 461 |
if record.simple_lowercase_mapping: |
| 462 |
lower = int(record.simple_lowercase_mapping, 16) |
| 463 |
else: |
| 464 |
lower = char |
| 465 |
if record.simple_titlecase_mapping: |
| 466 |
title = int(record.simple_titlecase_mapping, 16) |
| 467 |
else: |
| 468 |
title = upper |
| 469 |
if sc is None and cf != [lower]: |
| 470 |
sc = ([lower], [title], [upper]) |
| 471 |
if sc is None: |
| 472 |
if upper == lower == title: |
| 473 |
upper = lower = title = 0 |
| 474 |
else: |
| 475 |
upper = upper - char |
| 476 |
lower = lower - char |
| 477 |
title = title - char |
| 478 |
assert (abs(upper) <= 2147483647 and |
| 479 |
abs(lower) <= 2147483647 and |
| 480 |
abs(title) <= 2147483647) |
| 481 |
else: |
| 482 |
# This happens either when some character maps to more than one |
| 483 |
# character in uppercase, lowercase, or titlecase or the |
| 484 |
# casefolded version of the character is different from the |
| 485 |
# lowercase. The extra characters are stored in a different |
| 486 |
# array. |
| 487 |
flags |= EXTENDED_CASE_MASK |
| 488 |
lower = len(extra_casing) | (len(sc[0]) << 24) |
| 489 |
extra_casing.extend(sc[0]) |
| 490 |
if cf != sc[0]: |
| 491 |
lower |= len(cf) << 20 |
| 492 |
extra_casing.extend(cf) |
| 493 |
upper = len(extra_casing) | (len(sc[2]) << 24) |
| 494 |
extra_casing.extend(sc[2]) |
| 495 |
# Title is probably equal to upper. |
| 496 |
if sc[1] == sc[2]: |
| 497 |
title = upper |
| 498 |
else: |
| 499 |
title = len(extra_casing) | (len(sc[1]) << 24) |
| 500 |
extra_casing.extend(sc[1]) |
| 501 |
# decimal digit, integer digit |
| 502 |
decimal = 0 |
| 503 |
if record.decomposition_mapping: |
| 504 |
flags |= DECIMAL_MASK |
| 505 |
decimal = int(record.decomposition_mapping) |
| 506 |
digit = 0 |
| 507 |
if record.numeric_type: |
| 508 |
flags |= DIGIT_MASK |
| 509 |
digit = int(record.numeric_type) |
| 510 |
if record.numeric_value: |
| 511 |
flags |= NUMERIC_MASK |
| 512 |
numeric.setdefault(record.numeric_value, []).append(char) |
| 513 |
item = ( |
| 514 |
upper, lower, title, decimal, digit, flags |
| 515 |
) |
| 516 |
# add entry to index and item tables |
| 517 |
i = cache.get(item) |
| 518 |
if i is None: |
| 519 |
cache[item] = i = len(table) |
| 520 |
table.append(item) |
| 521 |
index[char] = i |
| 522 |
|
| 523 |
print(len(table), "unique character type entries") |
| 524 |
print(sum(map(len, numeric.values())), "numeric code points") |
| 525 |
print(len(spaces), "whitespace code points") |
| 526 |
print(len(linebreaks), "linebreak code points") |
| 527 |
print(len(extra_casing), "extended case array") |
| 528 |
|
| 529 |
print("--- Writing", FILE, "...") |
| 530 |
|
| 531 |
with open(FILE, "w") as fp: |
| 532 |
fprint = partial(print, file=fp) |
| 533 |
|
| 534 |
fprint("/* this file was generated by %s %s */" % (SCRIPT, VERSION)) |
| 535 |
fprint() |
| 536 |
fprint("/* a list of unique character type descriptors */") |
| 537 |
fprint("const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = {") |
| 538 |
for item in table: |
| 539 |
fprint(" {%d, %d, %d, %d, %d, %d}," % item) |
| 540 |
fprint("};") |
| 541 |
fprint() |
| 542 |
|
| 543 |
fprint("/* extended case mappings */") |
| 544 |
fprint() |
| 545 |
fprint("const Py_UCS4 _PyUnicode_ExtendedCase[] = {") |
| 546 |
for c in extra_casing: |
| 547 |
fprint(" %d," % c) |
| 548 |
fprint("};") |
| 549 |
fprint() |
| 550 |
|
| 551 |
# split decomposition index table |
| 552 |
index1, index2, shift = splitbins(index, trace) |
| 553 |
|
| 554 |
fprint("/* type indexes */") |
| 555 |
fprint("#define SHIFT", shift) |
| 556 |
Array("index1", index1).dump(fp, trace) |
| 557 |
Array("index2", index2).dump(fp, trace) |
| 558 |
|
| 559 |
# Generate code for _PyUnicode_ToNumeric() |
| 560 |
numeric_items = sorted(numeric.items()) |
| 561 |
fprint('/* Returns the numeric value as double for Unicode characters') |
| 562 |
fprint(' * having this property, -1.0 otherwise.') |
| 563 |
fprint(' */') |
| 564 |
fprint('double _PyUnicode_ToNumeric(Py_UCS4 ch)') |
| 565 |
fprint('{') |
| 566 |
fprint(' switch (ch) {') |
| 567 |
for value, codepoints in numeric_items: |
| 568 |
# Turn text into float literals |
| 569 |
parts = value.split('/') |
| 570 |
parts = [repr(float(part)) for part in parts] |
| 571 |
value = '/'.join(parts) |
| 572 |
|
| 573 |
codepoints.sort() |
| 574 |
for codepoint in codepoints: |
| 575 |
fprint(' case 0x%04X:' % (codepoint,)) |
| 576 |
fprint(' return (double) %s;' % (value,)) |
| 577 |
fprint(' }') |
| 578 |
fprint(' return -1.0;') |
| 579 |
fprint('}') |
| 580 |
fprint() |
| 581 |
|
| 582 |
# Generate code for _PyUnicode_IsWhitespace() |
| 583 |
fprint("/* Returns 1 for Unicode characters having the bidirectional") |
| 584 |
fprint(" * type 'WS', 'B' or 'S' or the category 'Zs', 0 otherwise.") |
| 585 |
fprint(" */") |
| 586 |
fprint('int _PyUnicode_IsWhitespace(const Py_UCS4 ch)') |
| 587 |
fprint('{') |
| 588 |
fprint(' switch (ch) {') |
| 589 |
|
| 590 |
for codepoint in sorted(spaces): |
| 591 |
fprint(' case 0x%04X:' % (codepoint,)) |
| 592 |
fprint(' return 1;') |
| 593 |
|
| 594 |
fprint(' }') |
| 595 |
fprint(' return 0;') |
| 596 |
fprint('}') |
| 597 |
fprint() |
| 598 |
|
| 599 |
# Generate code for _PyUnicode_IsLinebreak() |
| 600 |
fprint("/* Returns 1 for Unicode characters having the line break") |
| 601 |
fprint(" * property 'BK', 'CR', 'LF' or 'NL' or having bidirectional") |
| 602 |
fprint(" * type 'B', 0 otherwise.") |
| 603 |
fprint(" */") |
| 604 |
fprint('int _PyUnicode_IsLinebreak(const Py_UCS4 ch)') |
| 605 |
fprint('{') |
| 606 |
fprint(' switch (ch) {') |
| 607 |
for codepoint in sorted(linebreaks): |
| 608 |
fprint(' case 0x%04X:' % (codepoint,)) |
| 609 |
fprint(' return 1;') |
| 610 |
|
| 611 |
fprint(' }') |
| 612 |
fprint(' return 0;') |
| 613 |
fprint('}') |
| 614 |
fprint() |
| 615 |
|
| 616 |
|
| 617 |
# -------------------------------------------------------------------- |
| 618 |
# unicode name database |
| 619 |
|
| 620 |
def makeunicodename(unicode, trace): |
| 621 |
from dawg import build_compression_dawg |
| 622 |
|
| 623 |
FILE = "Modules/unicodename_db.h" |
| 624 |
|
| 625 |
print("--- Preparing", FILE, "...") |
| 626 |
|
| 627 |
# unicode name hash table |
| 628 |
|
| 629 |
# extract names |
| 630 |
data = [] |
| 631 |
for char in unicode.chars: |
| 632 |
record = unicode.table[char] |
| 633 |
if record: |
| 634 |
name = record.name.strip() |
| 635 |
if name and name[0] != "<": |
| 636 |
data.append((name, char)) |
| 637 |
|
| 638 |
print("--- Writing", FILE, "...") |
| 639 |
|
| 640 |
with open(FILE, "w") as fp: |
| 641 |
fprint = partial(print, file=fp) |
| 642 |
|
| 643 |
fprint("/* this file was generated by %s %s */" % (SCRIPT, VERSION)) |
| 644 |
fprint() |
| 645 |
fprint("#define NAME_MAXLEN", 256) |
| 646 |
assert max(len(x) for x in data) < 256 |
| 647 |
fprint() |
| 648 |
|
| 649 |
fprint("/* name->code dictionary */") |
| 650 |
packed_dawg, pos_to_codepoint = build_compression_dawg(data) |
| 651 |
notfound = len(pos_to_codepoint) |
| 652 |
inverse_list = [notfound] * len(unicode.chars) |
| 653 |
for pos, codepoint in enumerate(pos_to_codepoint): |
| 654 |
inverse_list[codepoint] = pos |
| 655 |
Array("packed_name_dawg", list(packed_dawg)).dump(fp, trace) |
| 656 |
Array("dawg_pos_to_codepoint", pos_to_codepoint).dump(fp, trace) |
| 657 |
index1, index2, shift = splitbins(inverse_list, trace) |
| 658 |
fprint("#define DAWG_CODEPOINT_TO_POS_SHIFT", shift) |
| 659 |
fprint("#define DAWG_CODEPOINT_TO_POS_NOTFOUND", notfound) |
| 660 |
Array("dawg_codepoint_to_pos_index1", index1).dump(fp, trace) |
| 661 |
Array("dawg_codepoint_to_pos_index2", index2).dump(fp, trace) |
| 662 |
|
| 663 |
fprint() |
| 664 |
fprint('static const unsigned int aliases_start = %#x;' % |
| 665 |
NAME_ALIASES_START) |
| 666 |
fprint('static const unsigned int aliases_end = %#x;' % |
| 667 |
(NAME_ALIASES_START + len(unicode.aliases))) |
| 668 |
|
| 669 |
fprint('static const unsigned int name_aliases[] = {') |
| 670 |
for name, codepoint in unicode.aliases: |
| 671 |
fprint(' 0x%04X,' % codepoint) |
| 672 |
fprint('};') |
| 673 |
|
| 674 |
# In Unicode 6.0.0, the sequences contain at most 4 BMP chars, |
| 675 |
# so we are using Py_UCS2 seq[4]. This needs to be updated if longer |
| 676 |
# sequences or sequences with non-BMP chars are added. |
| 677 |
# unicodedata_lookup should be adapted too. |
| 678 |
fprint(dedent(""" |
| 679 |
typedef struct NamedSequence { |
| 680 |
int seqlen; |
| 681 |
Py_UCS2 seq[4]; |
| 682 |
} named_sequence; |
| 683 |
""")) |
| 684 |
|
| 685 |
fprint('static const unsigned int named_sequences_start = %#x;' % |
| 686 |
NAMED_SEQUENCES_START) |
| 687 |
fprint('static const unsigned int named_sequences_end = %#x;' % |
| 688 |
(NAMED_SEQUENCES_START + len(unicode.named_sequences))) |
| 689 |
|
| 690 |
fprint('static const named_sequence named_sequences[] = {') |
| 691 |
for name, sequence in unicode.named_sequences: |
| 692 |
seq_str = ', '.join('0x%04X' % cp for cp in sequence) |
| 693 |
fprint(' {%d, {%s}},' % (len(sequence), seq_str)) |
| 694 |
fprint('};') |
| 695 |
|
| 696 |
fprint(dedent(""" |
| 697 |
typedef struct { |
| 698 |
Py_UCS4 first; |
| 699 |
Py_UCS4 last; |
| 700 |
int prefixid; |
| 701 |
} derived_name_range; |
| 702 |
""")) |
| 703 |
|
| 704 |
fprint('static const derived_name_range derived_name_ranges[] = {') |
| 705 |
for name_range in unicode.derived_name_ranges: |
| 706 |
fprint(' {0x%s, 0x%s, %d},' % name_range) |
| 707 |
fprint('};') |
| 708 |
|
| 709 |
fprint('static const char * const derived_name_prefixes[] = {') |
| 710 |
for _, prefix in derived_name_range_names: |
| 711 |
fprint(' "%s",' % prefix) |
| 712 |
fprint('};') |
| 713 |
|
| 714 |
def merge_old_version(version, new, old): |
| 715 |
# Changes to exclusion file not implemented yet |
| 716 |
if old.exclusions != new.exclusions: |
| 717 |
raise NotImplementedError("exclusions differ") |
| 718 |
|
| 719 |
# In these change records, 0xFF means "no change" |
| 720 |
bidir_changes = [0xFF]*0x110000 |
| 721 |
category_changes = [0xFF]*0x110000 |
| 722 |
decimal_changes = [0xFF]*0x110000 |
| 723 |
mirrored_changes = [0xFF]*0x110000 |
| 724 |
east_asian_width_changes = [0xFF]*0x110000 |
| 725 |
# In numeric data, 0 means "no change", |
| 726 |
# -1 means "did not have a numeric value |
| 727 |
numeric_changes = [0] * 0x110000 |
| 728 |
# normalization_changes is a list of key-value pairs |
| 729 |
normalization_changes = [] |
| 730 |
for i in range(0x110000): |
| 731 |
if new.table[i] is None: |
| 732 |
# Characters unassigned in the new version ought to |
| 733 |
# be unassigned in the old one |
| 734 |
assert old.table[i] is None |
| 735 |
continue |
| 736 |
# check characters unassigned in the old version |
| 737 |
if old.table[i] is None: |
| 738 |
# category 0 is "unassigned" |
| 739 |
category_changes[i] = 0 |
| 740 |
continue |
| 741 |
# check characters that differ |
| 742 |
if old.table[i] != new.table[i]: |
| 743 |
for k, field in enumerate(dataclasses.fields(UcdRecord)): |
| 744 |
value = getattr(old.table[i], field.name) |
| 745 |
new_value = getattr(new.table[i], field.name) |
| 746 |
if value != new_value: |
| 747 |
if k == 1 and i in PUA_15: |
| 748 |
# the name is not set in the old.table, but in the |
| 749 |
# new.table we are using it for aliases and named seq |
| 750 |
assert value == '' |
| 751 |
elif k == 2: |
| 752 |
category_changes[i] = CATEGORY_NAMES.index(value) |
| 753 |
elif k == 4: |
| 754 |
bidir_changes[i] = BIDIRECTIONAL_NAMES.index(value) |
| 755 |
elif k == 5: |
| 756 |
# We assume that all normalization changes are in 1:1 mappings |
| 757 |
assert " " not in value |
| 758 |
normalization_changes.append((i, value)) |
| 759 |
elif k == 6: |
| 760 |
# we only support changes where the old value is a single digit |
| 761 |
assert value in "0123456789" |
| 762 |
decimal_changes[i] = int(value) |
| 763 |
elif k == 8: |
| 764 |
# Since 0 encodes "no change", the old value is better not 0 |
| 765 |
if not value: |
| 766 |
numeric_changes[i] = -1 |
| 767 |
else: |
| 768 |
numeric_changes[i] = float(value) |
| 769 |
assert numeric_changes[i] not in (0, -1) |
| 770 |
elif k == 9: |
| 771 |
if value == 'Y': |
| 772 |
mirrored_changes[i] = '1' |
| 773 |
else: |
| 774 |
mirrored_changes[i] = '0' |
| 775 |
elif k == 11: |
| 776 |
# change to ISO comment, ignore |
| 777 |
pass |
| 778 |
elif k == 12: |
| 779 |
# change to simple uppercase mapping; ignore |
| 780 |
pass |
| 781 |
elif k == 13: |
| 782 |
# change to simple lowercase mapping; ignore |
| 783 |
pass |
| 784 |
elif k == 14: |
| 785 |
# change to simple titlecase mapping; ignore |
| 786 |
pass |
| 787 |
elif k == 15: |
| 788 |
# change to east asian width |
| 789 |
east_asian_width_changes[i] = EASTASIANWIDTH_NAMES.index(value) |
| 790 |
elif k == 16: |
| 791 |
# derived property changes; not yet |
| 792 |
pass |
| 793 |
elif k == 17: |
| 794 |
# normalization quickchecks are not performed |
| 795 |
# for older versions |
| 796 |
pass |
| 797 |
else: |
| 798 |
class Difference(Exception):pass |
| 799 |
raise Difference(hex(i), k, old.table[i], new.table[i]) |
| 800 |
new.changed.append((version, list(zip(bidir_changes, category_changes, |
| 801 |
decimal_changes, mirrored_changes, |
| 802 |
east_asian_width_changes, |
| 803 |
numeric_changes)), |
| 804 |
normalization_changes)) |
| 805 |
|
| 806 |
|
| 807 |
DATA_DIR = os.path.join('Tools', 'unicode', 'data') |
| 808 |
|
| 809 |
def open_data(template, version): |
| 810 |
local = os.path.join(DATA_DIR, template % ('-'+version,)) |
| 811 |
if not os.path.exists(local): |
| 812 |
import urllib.request |
| 813 |
if version == '3.2.0': |
| 814 |
# irregular url structure |
| 815 |
url = ('https://www.unicode.org/Public/3.2-Update/'+template) % ('-'+version,) |
| 816 |
else: |
| 817 |
url = ('https://www.unicode.org/Public/%s/ucd/'+template) % (version, '') |
| 818 |
os.makedirs(DATA_DIR, exist_ok=True) |
| 819 |
urllib.request.urlretrieve(url, filename=local) |
| 820 |
if local.endswith('.txt'): |
| 821 |
return open(local, encoding='utf-8') |
| 822 |
else: |
| 823 |
# Unihan.zip |
| 824 |
return open(local, 'rb') |
| 825 |
|
| 826 |
|
| 827 |
def expand_range(char_range: str) -> Iterator[int]: |
| 828 |
''' |
| 829 |
Parses ranges of code points, as described in UAX #44: |
| 830 |
https://www.unicode.org/reports/tr44/#Code_Point_Ranges |
| 831 |
''' |
| 832 |
if '..' in char_range: |
| 833 |
first, last = [int(c, 16) for c in char_range.split('..')] |
| 834 |
else: |
| 835 |
first = last = int(char_range, 16) |
| 836 |
for char in range(first, last+1): |
| 837 |
yield char |
| 838 |
|
| 839 |
|
| 840 |
class UcdFile: |
| 841 |
''' |
| 842 |
A file in the standard format of the UCD. |
| 843 |
|
| 844 |
See: https://www.unicode.org/reports/tr44/#Format_Conventions |
| 845 |
|
| 846 |
Note that, as described there, the Unihan data files have their |
| 847 |
own separate format. |
| 848 |
''' |
| 849 |
|
| 850 |
def __init__(self, template: str, version: str) -> None: |
| 851 |
self.template = template |
| 852 |
self.version = version |
| 853 |
|
| 854 |
def records(self) -> Iterator[List[str]]: |
| 855 |
with open_data(self.template, self.version) as file: |
| 856 |
for line in file: |
| 857 |
line = line.split('#', 1)[0].strip() |
| 858 |
if not line: |
| 859 |
continue |
| 860 |
yield [field.strip() for field in line.split(';')] |
| 861 |
|
| 862 |
def __iter__(self) -> Iterator[List[str]]: |
| 863 |
return self.records() |
| 864 |
|
| 865 |
def expanded(self) -> Iterator[Tuple[int, List[str]]]: |
| 866 |
for record in self.records(): |
| 867 |
char_range, rest = record[0], record[1:] |
| 868 |
for char in expand_range(char_range): |
| 869 |
yield char, rest |
| 870 |
|
| 871 |
|
| 872 |
@dataclasses.dataclass |
| 873 |
class UcdRecord: |
| 874 |
# 15 fields from UnicodeData.txt . See: |
| 875 |
# https://www.unicode.org/reports/tr44/#UnicodeData.txt |
| 876 |
codepoint: str |
| 877 |
name: str |
| 878 |
general_category: str |
| 879 |
canonical_combining_class: str |
| 880 |
bidi_class: str |
| 881 |
decomposition_type: str |
| 882 |
decomposition_mapping: str |
| 883 |
numeric_type: str |
| 884 |
numeric_value: str |
| 885 |
bidi_mirrored: str |
| 886 |
unicode_1_name: str # obsolete |
| 887 |
iso_comment: str # obsolete |
| 888 |
simple_uppercase_mapping: str |
| 889 |
simple_lowercase_mapping: str |
| 890 |
simple_titlecase_mapping: str |
| 891 |
|
| 892 |
# https://www.unicode.org/reports/tr44/#EastAsianWidth.txt |
| 893 |
east_asian_width: Optional[str] |
| 894 |
|
| 895 |
# Binary properties, as a set of those that are true. |
| 896 |
# Taken from multiple files: |
| 897 |
# https://www.unicode.org/reports/tr44/#DerivedCoreProperties.txt |
| 898 |
# https://www.unicode.org/reports/tr44/#LineBreak.txt |
| 899 |
binary_properties: Set[str] |
| 900 |
|
| 901 |
# The Quick_Check properties related to normalization: |
| 902 |
# https://www.unicode.org/reports/tr44/#Decompositions_and_Normalization |
| 903 |
# We store them as a bitmask. |
| 904 |
quick_check: int |
| 905 |
|
| 906 |
|
| 907 |
def from_row(row: List[str]) -> UcdRecord: |
| 908 |
return UcdRecord(*row, None, set(), 0) |
| 909 |
|
| 910 |
|
| 911 |
# -------------------------------------------------------------------- |
| 912 |
# the following support code is taken from the unidb utilities |
| 913 |
# Copyright (c) 1999-2000 by Secret Labs AB |
| 914 |
|
| 915 |
# load a unicode-data file from disk |
| 916 |
|
| 917 |
class UnicodeData: |
| 918 |
# table: List[Optional[UcdRecord]] # index is codepoint; None means unassigned |
| 919 |
|
| 920 |
def __init__(self, version, ideograph_check=True): |
| 921 |
self.changed = [] |
| 922 |
table = [None] * 0x110000 |
| 923 |
for s in UcdFile(UNICODE_DATA, version): |
| 924 |
char = int(s[0], 16) |
| 925 |
table[char] = from_row(s) |
| 926 |
|
| 927 |
self.derived_name_ranges = [] |
| 928 |
|
| 929 |
# expand first-last ranges |
| 930 |
field = None |
| 931 |
for i in range(0, 0x110000): |
| 932 |
# The file UnicodeData.txt has its own distinct way of |
| 933 |
# expressing ranges. See: |
| 934 |
# https://www.unicode.org/reports/tr44/#Code_Point_Ranges |
| 935 |
s = table[i] |
| 936 |
if s: |
| 937 |
if s.name[-6:] == "First>": |
| 938 |
s.name = "" |
| 939 |
field = dataclasses.astuple(s)[:15] |
| 940 |
elif s.name[-5:] == "Last>": |
| 941 |
for j, (rangename, _) in enumerate(derived_name_range_names): |
| 942 |
if s.name.startswith("<" + rangename): |
| 943 |
self.derived_name_ranges.append( |
| 944 |
(field[0], s.codepoint, j)) |
| 945 |
break |
| 946 |
s.name = "" |
| 947 |
field = None |
| 948 |
elif field: |
| 949 |
table[i] = from_row(('%X' % i,) + field[1:]) |
| 950 |
|
| 951 |
# public attributes |
| 952 |
self.filename = UNICODE_DATA % '' |
| 953 |
self.table = table |
| 954 |
self.chars = list(range(0x110000)) # unicode 3.2 |
| 955 |
|
| 956 |
# check for name aliases and named sequences, see #12753 |
| 957 |
# aliases and named sequences are not in 3.2.0 |
| 958 |
if version != '3.2.0': |
| 959 |
self.aliases = [] |
| 960 |
# store aliases in the Private Use Area 15, in range U+F0000..U+F00FF, |
| 961 |
# in order to take advantage of the compression and lookup |
| 962 |
# algorithms used for the other characters |
| 963 |
pua_index = NAME_ALIASES_START |
| 964 |
for char, name, abbrev in UcdFile(NAME_ALIASES, version): |
| 965 |
char = int(char, 16) |
| 966 |
self.aliases.append((name, char)) |
| 967 |
# also store the name in the PUA 1 |
| 968 |
self.table[pua_index].name = name |
| 969 |
pua_index += 1 |
| 970 |
assert pua_index - NAME_ALIASES_START == len(self.aliases) |
| 971 |
|
| 972 |
self.named_sequences = [] |
| 973 |
# store named sequences in the PUA 1, in range U+F0100.., |
| 974 |
# in order to take advantage of the compression and lookup |
| 975 |
# algorithms used for the other characters. |
| 976 |
|
| 977 |
assert pua_index < NAMED_SEQUENCES_START |
| 978 |
pua_index = NAMED_SEQUENCES_START |
| 979 |
for name, chars in UcdFile(NAMED_SEQUENCES, version): |
| 980 |
chars = tuple(int(char, 16) for char in chars.split()) |
| 981 |
# check that the structure defined in makeunicodename is OK |
| 982 |
assert 2 <= len(chars) <= 4, "change the Py_UCS2 array size" |
| 983 |
assert all(c <= 0xFFFF for c in chars), ("use Py_UCS4 in " |
| 984 |
"the NamedSequence struct and in unicodedata_lookup") |
| 985 |
self.named_sequences.append((name, chars)) |
| 986 |
# also store these in the PUA 1 |
| 987 |
self.table[pua_index].name = name |
| 988 |
pua_index += 1 |
| 989 |
assert pua_index - NAMED_SEQUENCES_START == len(self.named_sequences) |
| 990 |
|
| 991 |
self.exclusions = {} |
| 992 |
for char, in UcdFile(COMPOSITION_EXCLUSIONS, version): |
| 993 |
char = int(char, 16) |
| 994 |
self.exclusions[char] = 1 |
| 995 |
|
| 996 |
widths = [None] * 0x110000 |
| 997 |
for char, (width,) in UcdFile(EASTASIAN_WIDTH, version).expanded(): |
| 998 |
widths[char] = width |
| 999 |
|
| 1000 |
for i in range(0, 0x110000): |
| 1001 |
if table[i] is not None: |
| 1002 |
table[i].east_asian_width = widths[i] |
| 1003 |
self.widths = widths |
| 1004 |
|
| 1005 |
for char, (propname, *propinfo) in UcdFile(DERIVED_CORE_PROPERTIES, version).expanded(): |
| 1006 |
if propinfo: |
| 1007 |
# this is not a binary property, ignore it |
| 1008 |
continue |
| 1009 |
|
| 1010 |
if table[char]: |
| 1011 |
# Some properties (e.g. Default_Ignorable_Code_Point) |
| 1012 |
# apply to unassigned code points; ignore them |
| 1013 |
table[char].binary_properties.add(propname) |
| 1014 |
|
| 1015 |
for char_range, value in UcdFile(LINE_BREAK, version): |
| 1016 |
if value not in MANDATORY_LINE_BREAKS: |
| 1017 |
continue |
| 1018 |
for char in expand_range(char_range): |
| 1019 |
table[char].binary_properties.add('Line_Break') |
| 1020 |
|
| 1021 |
# We only want the quickcheck properties |
| 1022 |
# Format: NF?_QC; Y(es)/N(o)/M(aybe) |
| 1023 |
# Yes is the default, hence only N and M occur |
| 1024 |
# In 3.2.0, the format was different (NF?_NO) |
| 1025 |
# The parsing will incorrectly determine these as |
| 1026 |
# "yes", however, unicodedata.c will not perform quickchecks |
| 1027 |
# for older versions, and no delta records will be created. |
| 1028 |
quickchecks = [0] * 0x110000 |
| 1029 |
qc_order = 'NFD_QC NFKD_QC NFC_QC NFKC_QC'.split() |
| 1030 |
for s in UcdFile(DERIVEDNORMALIZATION_PROPS, version): |
| 1031 |
if len(s) < 2 or s[1] not in qc_order: |
| 1032 |
continue |
| 1033 |
quickcheck = 'MN'.index(s[2]) + 1 # Maybe or No |
| 1034 |
quickcheck_shift = qc_order.index(s[1])*2 |
| 1035 |
quickcheck <<= quickcheck_shift |
| 1036 |
for char in expand_range(s[0]): |
| 1037 |
assert not (quickchecks[char]>>quickcheck_shift)&3 |
| 1038 |
quickchecks[char] |= quickcheck |
| 1039 |
for i in range(0, 0x110000): |
| 1040 |
if table[i] is not None: |
| 1041 |
table[i].quick_check = quickchecks[i] |
| 1042 |
|
| 1043 |
with open_data(UNIHAN, version) as file: |
| 1044 |
zip = zipfile.ZipFile(file) |
| 1045 |
if version == '3.2.0': |
| 1046 |
data = zip.open('Unihan-3.2.0.txt').read() |
| 1047 |
else: |
| 1048 |
data = zip.open('Unihan_NumericValues.txt').read() |
| 1049 |
for line in data.decode("utf-8").splitlines(): |
| 1050 |
if not line.startswith('U+'): |
| 1051 |
continue |
| 1052 |
code, tag, value = line.split(None, 3)[:3] |
| 1053 |
if tag not in ('kAccountingNumeric', 'kPrimaryNumeric', |
| 1054 |
'kOtherNumeric'): |
| 1055 |
continue |
| 1056 |
value = value.strip().replace(',', '') |
| 1057 |
i = int(code[2:], 16) |
| 1058 |
# Patch the numeric field |
| 1059 |
if table[i] is not None: |
| 1060 |
table[i].numeric_value = value |
| 1061 |
|
| 1062 |
sc = self.special_casing = {} |
| 1063 |
for data in UcdFile(SPECIAL_CASING, version): |
| 1064 |
if data[4]: |
| 1065 |
# We ignore all conditionals (since they depend on |
| 1066 |
# languages) except for one, which is hardcoded. See |
| 1067 |
# handle_capital_sigma in unicodeobject.c. |
| 1068 |
continue |
| 1069 |
c = int(data[0], 16) |
| 1070 |
lower = [int(char, 16) for char in data[1].split()] |
| 1071 |
title = [int(char, 16) for char in data[2].split()] |
| 1072 |
upper = [int(char, 16) for char in data[3].split()] |
| 1073 |
sc[c] = (lower, title, upper) |
| 1074 |
|
| 1075 |
cf = self.case_folding = {} |
| 1076 |
if version != '3.2.0': |
| 1077 |
for data in UcdFile(CASE_FOLDING, version): |
| 1078 |
if data[1] in "CF": |
| 1079 |
c = int(data[0], 16) |
| 1080 |
cf[c] = [int(char, 16) for char in data[2].split()] |
| 1081 |
|
| 1082 |
def uselatin1(self): |
| 1083 |
# restrict character range to ISO Latin 1 |
| 1084 |
self.chars = list(range(256)) |
| 1085 |
|
| 1086 |
|
| 1087 |
|
| 1088 |
# stuff to deal with arrays of unsigned integers |
| 1089 |
|
| 1090 |
class Array: |
| 1091 |
|
| 1092 |
def __init__(self, name, data): |
| 1093 |
self.name = name |
| 1094 |
self.data = data |
| 1095 |
|
| 1096 |
def dump(self, file, trace=0): |
| 1097 |
# write data to file, as a C array |
| 1098 |
size = getsize(self.data) |
| 1099 |
if trace: |
| 1100 |
print(self.name+":", size*len(self.data), "bytes", file=sys.stderr) |
| 1101 |
file.write("static const ") |
| 1102 |
if size == 1: |
| 1103 |
file.write("unsigned char") |
| 1104 |
elif size == 2: |
| 1105 |
file.write("unsigned short") |
| 1106 |
else: |
| 1107 |
file.write("unsigned int") |
| 1108 |
file.write(" " + self.name + "[] = {\n") |
| 1109 |
if self.data: |
| 1110 |
s = " " |
| 1111 |
for item in self.data: |
| 1112 |
i = str(item) + ", " |
| 1113 |
if len(s) + len(i) > 78: |
| 1114 |
file.write(s.rstrip() + "\n") |
| 1115 |
s = " " + i |
| 1116 |
else: |
| 1117 |
s = s + i |
| 1118 |
if s.strip(): |
| 1119 |
file.write(s.rstrip() + "\n") |
| 1120 |
file.write("};\n\n") |
| 1121 |
|
| 1122 |
|
| 1123 |
def getsize(data): |
| 1124 |
# return smallest possible integer size for the given array |
| 1125 |
maxdata = max(data) |
| 1126 |
if maxdata < 256: |
| 1127 |
return 1 |
| 1128 |
elif maxdata < 65536: |
| 1129 |
return 2 |
| 1130 |
else: |
| 1131 |
return 4 |
| 1132 |
|
| 1133 |
|
| 1134 |
def splitbins(t, trace=0): |
| 1135 |
"""t, trace=0 -> (t1, t2, shift). Split a table to save space. |
| 1136 |
|
| 1137 |
t is a sequence of ints. This function can be useful to save space if |
| 1138 |
many of the ints are the same. t1 and t2 are lists of ints, and shift |
| 1139 |
is an int, chosen to minimize the combined size of t1 and t2 (in C |
| 1140 |
code), and where for each i in range(len(t)), |
| 1141 |
t[i] == t2[(t1[i >> shift] << shift) + (i & mask)] |
| 1142 |
where mask is a bitmask isolating the last "shift" bits. |
| 1143 |
|
| 1144 |
If optional arg trace is non-zero (default zero), progress info |
| 1145 |
is printed to sys.stderr. The higher the value, the more info |
| 1146 |
you'll get. |
| 1147 |
""" |
| 1148 |
|
| 1149 |
if trace: |
| 1150 |
def dump(t1, t2, shift, bytes): |
| 1151 |
print("%d+%d bins at shift %d; %d bytes" % ( |
| 1152 |
len(t1), len(t2), shift, bytes), file=sys.stderr) |
| 1153 |
print("Size of original table:", len(t)*getsize(t), "bytes", |
| 1154 |
file=sys.stderr) |
| 1155 |
n = len(t)-1 # last valid index |
| 1156 |
maxshift = 0 # the most we can shift n and still have something left |
| 1157 |
if n > 0: |
| 1158 |
while n >> 1: |
| 1159 |
n >>= 1 |
| 1160 |
maxshift += 1 |
| 1161 |
del n |
| 1162 |
bytes = sys.maxsize # smallest total size so far |
| 1163 |
t = tuple(t) # so slices can be dict keys |
| 1164 |
for shift in range(maxshift + 1): |
| 1165 |
t1 = [] |
| 1166 |
t2 = [] |
| 1167 |
size = 2**shift |
| 1168 |
bincache = {} |
| 1169 |
for i in range(0, len(t), size): |
| 1170 |
bin = t[i:i+size] |
| 1171 |
index = bincache.get(bin) |
| 1172 |
if index is None: |
| 1173 |
index = len(t2) |
| 1174 |
bincache[bin] = index |
| 1175 |
t2.extend(bin) |
| 1176 |
t1.append(index >> shift) |
| 1177 |
# determine memory size |
| 1178 |
b = len(t1)*getsize(t1) + len(t2)*getsize(t2) |
| 1179 |
if trace > 1: |
| 1180 |
dump(t1, t2, shift, b) |
| 1181 |
if b < bytes: |
| 1182 |
best = t1, t2, shift |
| 1183 |
bytes = b |
| 1184 |
t1, t2, shift = best |
| 1185 |
if trace: |
| 1186 |
print("Best:", end=' ', file=sys.stderr) |
| 1187 |
dump(t1, t2, shift, bytes) |
| 1188 |
if __debug__: |
| 1189 |
# exhaustively verify that the decomposition is correct |
| 1190 |
mask = ~((~0) << shift) # i.e., low-bit mask of shift bits |
| 1191 |
for i in range(len(t)): |
| 1192 |
assert t[i] == t2[(t1[i >> shift] << shift) + (i & mask)] |
| 1193 |
return best |
| 1194 |
|
| 1195 |
|
| 1196 |
if __name__ == "__main__": |
| 1197 |
maketables(1) |