Debian Codegraph @

Tools/unicode/makeunicodedata.py

raw
1 #
2 # (re)generate unicode property and type databases
3 #
4 # This script converts Unicode database files to Modules/unicodedata_db.h,
5 # Modules/unicodename_db.h, and Objects/unicodetype_db.h
6 #
7 # history:
8 # 2000-09-24 fl created (based on bits and pieces from unidb)
9 # 2000-09-25 fl merged tim's splitbin fixes, separate decomposition table
10 # 2000-09-25 fl added character type table
11 # 2000-09-26 fl added LINEBREAK, DECIMAL, and DIGIT flags/fields (2.0)
12 # 2000-11-03 fl expand first/last ranges
13 # 2001-01-19 fl added character name tables (2.1)
14 # 2001-01-21 fl added decomp compression; dynamic phrasebook threshold
15 # 2002-09-11 wd use string methods
16 # 2002-10-18 mvl update to Unicode 3.2
17 # 2002-10-22 mvl generate NFC tables
18 # 2002-11-24 mvl expand all ranges, sort names version-independently
19 # 2002-11-25 mvl add UNIDATA_VERSION
20 # 2004-05-29 perky add east asian width information
21 # 2006-03-10 mvl update to Unicode 4.1; add UCD 3.2 delta
22 # 2008-06-11 gb add PRINTABLE_MASK for Atsuo Ishimoto's ascii() patch
23 # 2011-10-21 ezio add support for name aliases and named sequences
24 # 2012-01 benjamin add full case mappings
25 #
26 # written by Fredrik Lundh ([email protected])
27 #
28
29 import dataclasses
30 import os
31 import sys
32 import zipfile
33
34 from functools import partial
35 from textwrap import dedent
36 from typing import Iterator, List, Optional, Set, Tuple
37
38 SCRIPT = os.path.normpath(sys.argv[0])
39 VERSION = "3.3"
40
41 # The Unicode Database
42 # --------------------
43 # When changing UCD version please update
44 # * Doc/library/stdtypes.rst, and
45 # * Doc/library/unicodedata.rst
46 # * Doc/reference/lexical_analysis.rst (three occurrences)
47 UNIDATA_VERSION = "16.0.0"
48 UNICODE_DATA = "UnicodeData%s.txt"
49 COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt"
50 EASTASIAN_WIDTH = "EastAsianWidth%s.txt"
51 UNIHAN = "Unihan%s.zip"
52 DERIVED_CORE_PROPERTIES = "DerivedCoreProperties%s.txt"
53 DERIVEDNORMALIZATION_PROPS = "DerivedNormalizationProps%s.txt"
54 LINE_BREAK = "LineBreak%s.txt"
55 NAME_ALIASES = "NameAliases%s.txt"
56 NAMED_SEQUENCES = "NamedSequences%s.txt"
57 SPECIAL_CASING = "SpecialCasing%s.txt"
58 CASE_FOLDING = "CaseFolding%s.txt"
59
60 # Private Use Areas -- in planes 1, 15, 16
61 PUA_1 = range(0xE000, 0xF900)
62 PUA_15 = range(0xF0000, 0xFFFFE)
63 PUA_16 = range(0x100000, 0x10FFFE)
64
65 # we use this ranges of PUA_15 to store name aliases and named sequences
66 NAME_ALIASES_START = 0xF0000
67 NAMED_SEQUENCES_START = 0xF0200
68
69 old_versions = ["3.2.0"]
70
71 CATEGORY_NAMES = [ "Cn", "Lu", "Ll", "Lt", "Mn", "Mc", "Me", "Nd",
72 "Nl", "No", "Zs", "Zl", "Zp", "Cc", "Cf", "Cs", "Co", "Cn", "Lm",
73 "Lo", "Pc", "Pd", "Ps", "Pe", "Pi", "Pf", "Po", "Sm", "Sc", "Sk",
74 "So" ]
75
76 BIDIRECTIONAL_NAMES = [ "", "L", "LRE", "LRO", "R", "AL", "RLE", "RLO",
77 "PDF", "EN", "ES", "ET", "AN", "CS", "NSM", "BN", "B", "S", "WS",
78 "ON", "LRI", "RLI", "FSI", "PDI" ]
79
80 # "N" needs to be the first entry, see the comment in makeunicodedata
81 EASTASIANWIDTH_NAMES = [ "N", "H", "W", "Na", "A", "F" ]
82
83 MANDATORY_LINE_BREAKS = [ "BK", "CR", "LF", "NL" ]
84
85 # note: should match definitions in Objects/unicodectype.c
86 ALPHA_MASK = 0x01
87 DECIMAL_MASK = 0x02
88 DIGIT_MASK = 0x04
89 LOWER_MASK = 0x08
90 LINEBREAK_MASK = 0x10
91 SPACE_MASK = 0x20
92 TITLE_MASK = 0x40
93 UPPER_MASK = 0x80
94 XID_START_MASK = 0x100
95 XID_CONTINUE_MASK = 0x200
96 PRINTABLE_MASK = 0x400
97 NUMERIC_MASK = 0x800
98 CASE_IGNORABLE_MASK = 0x1000
99 CASED_MASK = 0x2000
100 EXTENDED_CASE_MASK = 0x4000
101
102 # Maps the range names in UnicodeData.txt to prefixes for
103 # derived names specified by rule NR2.
104 # Hangul should always be at index 0, since it uses special format.
105 derived_name_range_names = [
106 ("Hangul Syllable", "HANGUL SYLLABLE "),
107 ("CJK Ideograph", "CJK UNIFIED IDEOGRAPH-"),
108 ("Tangut Ideograph", "TANGUT IDEOGRAPH-"),
109 ]
110
111
112 def maketables(trace=0):
113
114 print("--- Reading", UNICODE_DATA % "", "...")
115
116 unicode = UnicodeData(UNIDATA_VERSION)
117
118 print(len(list(filter(None, unicode.table))), "characters")
119
120 for version in old_versions:
121 print("--- Reading", UNICODE_DATA % ("-"+version), "...")
122 old_unicode = UnicodeData(version, ideograph_check=False)
123 print(len(list(filter(None, old_unicode.table))), "characters")
124 merge_old_version(version, unicode, old_unicode)
125
126 makeunicodename(unicode, trace)
127 makeunicodedata(unicode, trace)
128 makeunicodetype(unicode, trace)
129
130
131 # --------------------------------------------------------------------
132 # unicode character properties
133
134 def makeunicodedata(unicode, trace):
135
136 # the default value of east_asian_width is "N", for unassigned code points
137 # not mentioned in EastAsianWidth.txt
138 # in addition there are some reserved but unassigned code points in CJK
139 # ranges that are classified as "W". code points in private use areas
140 # have a width of "A". both of these have entries in
141 # EastAsianWidth.txt
142 # see https://unicode.org/reports/tr11/#Unassigned
143 assert EASTASIANWIDTH_NAMES[0] == "N"
144 dummy = (0, 0, 0, 0, 0, 0)
145 table = [dummy]
146 cache = {0: dummy}
147 index = [0] * len(unicode.chars)
148
149 FILE = "Modules/unicodedata_db.h"
150
151 print("--- Preparing", FILE, "...")
152
153 # 1) database properties
154
155 for char in unicode.chars:
156 record = unicode.table[char]
157 if record:
158 # extract database properties
159 category = CATEGORY_NAMES.index(record.general_category)
160 combining = int(record.canonical_combining_class)
161 bidirectional = BIDIRECTIONAL_NAMES.index(record.bidi_class)
162 mirrored = record.bidi_mirrored == "Y"
163 eastasianwidth = EASTASIANWIDTH_NAMES.index(record.east_asian_width)
164 normalizationquickcheck = record.quick_check
165 item = (
166 category, combining, bidirectional, mirrored, eastasianwidth,
167 normalizationquickcheck
168 )
169 elif unicode.widths[char] is not None:
170 # an unassigned but reserved character, with a known
171 # east_asian_width
172 eastasianwidth = EASTASIANWIDTH_NAMES.index(unicode.widths[char])
173 item = (0, 0, 0, 0, eastasianwidth, 0)
174 else:
175 continue
176
177 # add entry to index and item tables
178 i = cache.get(item)
179 if i is None:
180 cache[item] = i = len(table)
181 table.append(item)
182 index[char] = i
183
184 # 2) decomposition data
185
186 decomp_data_cache = {}
187 decomp_data = [0]
188 decomp_prefix = [""]
189 decomp_index = [0] * len(unicode.chars)
190 decomp_size = 0
191
192 comp_pairs = []
193 comp_first = [None] * len(unicode.chars)
194 comp_last = [None] * len(unicode.chars)
195
196 for char in unicode.chars:
197 record = unicode.table[char]
198 if record:
199 if record.decomposition_type:
200 decomp = record.decomposition_type.split()
201 if len(decomp) > 19:
202 raise Exception("character %x has a decomposition too large for nfd_nfkd" % char)
203 # prefix
204 if decomp[0][0] == "<":
205 prefix = decomp.pop(0)
206 else:
207 prefix = ""
208 try:
209 i = decomp_prefix.index(prefix)
210 except ValueError:
211 i = len(decomp_prefix)
212 decomp_prefix.append(prefix)
213 prefix = i
214 assert prefix < 256
215 # content
216 decomp = [prefix + (len(decomp)<<8)] + [int(s, 16) for s in decomp]
217 # Collect NFC pairs
218 if not prefix and len(decomp) == 3 and \
219 char not in unicode.exclusions and \
220 unicode.table[decomp[1]].canonical_combining_class == "0":
221 p, l, r = decomp
222 comp_first[l] = 1
223 comp_last[r] = 1
224 comp_pairs.append((l,r,char))
225 key = tuple(decomp)
226 i = decomp_data_cache.get(key, -1)
227 if i == -1:
228 i = len(decomp_data)
229 decomp_data.extend(decomp)
230 decomp_size = decomp_size + len(decomp) * 2
231 decomp_data_cache[key] = i
232 else:
233 assert decomp_data[i:i+len(decomp)] == decomp
234 else:
235 i = 0
236 decomp_index[char] = i
237
238 f = l = 0
239 comp_first_ranges = []
240 comp_last_ranges = []
241 prev_f = prev_l = None
242 for i in unicode.chars:
243 if comp_first[i] is not None:
244 comp_first[i] = f
245 f += 1
246 if prev_f is None:
247 prev_f = (i,i)
248 elif prev_f[1]+1 == i:
249 prev_f = prev_f[0],i
250 else:
251 comp_first_ranges.append(prev_f)
252 prev_f = (i,i)
253 if comp_last[i] is not None:
254 comp_last[i] = l
255 l += 1
256 if prev_l is None:
257 prev_l = (i,i)
258 elif prev_l[1]+1 == i:
259 prev_l = prev_l[0],i
260 else:
261 comp_last_ranges.append(prev_l)
262 prev_l = (i,i)
263 comp_first_ranges.append(prev_f)
264 comp_last_ranges.append(prev_l)
265 total_first = f
266 total_last = l
267
268 comp_data = [0]*(total_first*total_last)
269 for f,l,char in comp_pairs:
270 f = comp_first[f]
271 l = comp_last[l]
272 comp_data[f*total_last+l] = char
273
274 print(len(table), "unique properties")
275 print(len(decomp_prefix), "unique decomposition prefixes")
276 print(len(decomp_data), "unique decomposition entries:", end=' ')
277 print(decomp_size, "bytes")
278 print(total_first, "first characters in NFC")
279 print(total_last, "last characters in NFC")
280 print(len(comp_pairs), "NFC pairs")
281
282 print("--- Writing", FILE, "...")
283
284 with open(FILE, "w") as fp:
285 fprint = partial(print, file=fp)
286
287 fprint("/* this file was generated by %s %s */" % (SCRIPT, VERSION))
288 fprint()
289 fprint('#define UNIDATA_VERSION "%s"' % UNIDATA_VERSION)
290 fprint("/* a list of unique database records */")
291 fprint("const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = {")
292 for item in table:
293 fprint(" {%d, %d, %d, %d, %d, %d}," % item)
294 fprint("};")
295 fprint()
296
297 fprint("/* Reindexing of NFC first characters. */")
298 fprint("#define TOTAL_FIRST",total_first)
299 fprint("#define TOTAL_LAST",total_last)
300 fprint("struct reindex{int start;short count,index;};")
301 fprint("static struct reindex nfc_first[] = {")
302 for start,end in comp_first_ranges:
303 fprint(" { %d, %d, %d}," % (start,end-start,comp_first[start]))
304 fprint(" {0,0,0}")
305 fprint("};\n")
306 fprint("static struct reindex nfc_last[] = {")
307 for start,end in comp_last_ranges:
308 fprint(" { %d, %d, %d}," % (start,end-start,comp_last[start]))
309 fprint(" {0,0,0}")
310 fprint("};\n")
311
312 # FIXME: <fl> the following tables could be made static, and
313 # the support code moved into unicodedatabase.c
314
315 fprint("/* string literals */")
316 fprint("const char *_PyUnicode_CategoryNames[] = {")
317 for name in CATEGORY_NAMES:
318 fprint(" \"%s\"," % name)
319 fprint(" NULL")
320 fprint("};")
321
322 fprint("const char *_PyUnicode_BidirectionalNames[] = {")
323 for name in BIDIRECTIONAL_NAMES:
324 fprint(" \"%s\"," % name)
325 fprint(" NULL")
326 fprint("};")
327
328 fprint("const char *_PyUnicode_EastAsianWidthNames[] = {")
329 for name in EASTASIANWIDTH_NAMES:
330 fprint(" \"%s\"," % name)
331 fprint(" NULL")
332 fprint("};")
333
334 fprint("static const char *decomp_prefix[] = {")
335 for name in decomp_prefix:
336 fprint(" \"%s\"," % name)
337 fprint(" NULL")
338 fprint("};")
339
340 # split record index table
341 index1, index2, shift = splitbins(index, trace)
342
343 fprint("/* index tables for the database records */")
344 fprint("#define SHIFT", shift)
345 Array("index1", index1).dump(fp, trace)
346 Array("index2", index2).dump(fp, trace)
347
348 # split decomposition index table
349 index1, index2, shift = splitbins(decomp_index, trace)
350
351 fprint("/* decomposition data */")
352 Array("decomp_data", decomp_data).dump(fp, trace)
353
354 fprint("/* index tables for the decomposition data */")
355 fprint("#define DECOMP_SHIFT", shift)
356 Array("decomp_index1", index1).dump(fp, trace)
357 Array("decomp_index2", index2).dump(fp, trace)
358
359 index, index2, shift = splitbins(comp_data, trace)
360 fprint("/* NFC pairs */")
361 fprint("#define COMP_SHIFT", shift)
362 Array("comp_index", index).dump(fp, trace)
363 Array("comp_data", index2).dump(fp, trace)
364
365 # Generate delta tables for old versions
366 for version, table, normalization in unicode.changed:
367 cversion = version.replace(".","_")
368 records = [table[0]]
369 cache = {table[0]:0}
370 index = [0] * len(table)
371 for i, record in enumerate(table):
372 try:
373 index[i] = cache[record]
374 except KeyError:
375 index[i] = cache[record] = len(records)
376 records.append(record)
377 index1, index2, shift = splitbins(index, trace)
378 fprint("static const change_record change_records_%s[] = {" % cversion)
379 for record in records:
380 fprint(" { %s }," % ", ".join(map(str,record)))
381 fprint("};")
382 Array("changes_%s_index" % cversion, index1).dump(fp, trace)
383 Array("changes_%s_data" % cversion, index2).dump(fp, trace)
384 fprint("static const change_record* get_change_%s(Py_UCS4 n)" % cversion)
385 fprint("{")
386 fprint(" int index;")
387 fprint(" if (n >= 0x110000) index = 0;")
388 fprint(" else {")
389 fprint(" index = changes_%s_index[n>>%d];" % (cversion, shift))
390 fprint(" index = changes_%s_data[(index<<%d)+(n & %d)];" % \
391 (cversion, shift, ((1<<shift)-1)))
392 fprint(" }")
393 fprint(" return change_records_%s+index;" % cversion)
394 fprint("}\n")
395 fprint("static Py_UCS4 normalization_%s(Py_UCS4 n)" % cversion)
396 fprint("{")
397 fprint(" switch(n) {")
398 for k, v in normalization:
399 fprint(" case %s: return 0x%s;" % (hex(k), v))
400 fprint(" default: return 0;")
401 fprint(" }\n}\n")
402
403
404 # --------------------------------------------------------------------
405 # unicode character type tables
406
407 def makeunicodetype(unicode, trace):
408
409 FILE = "Objects/unicodetype_db.h"
410
411 print("--- Preparing", FILE, "...")
412
413 # extract unicode types
414 dummy = (0, 0, 0, 0, 0, 0)
415 table = [dummy]
416 cache = {dummy: 0}
417 index = [0] * len(unicode.chars)
418 numeric = {}
419 spaces = []
420 linebreaks = []
421 extra_casing = []
422
423 for char in unicode.chars:
424 record = unicode.table[char]
425 if record:
426 # extract database properties
427 category = record.general_category
428 bidirectional = record.bidi_class
429 properties = record.binary_properties
430 flags = 0
431 if category in ["Lm", "Lt", "Lu", "Ll", "Lo"]:
432 flags |= ALPHA_MASK
433 if "Lowercase" in properties:
434 flags |= LOWER_MASK
435 if 'Line_Break' in properties or bidirectional == "B":
436 flags |= LINEBREAK_MASK
437 linebreaks.append(char)
438 if category == "Zs" or bidirectional in ("WS", "B", "S"):
439 flags |= SPACE_MASK
440 spaces.append(char)
441 if category == "Lt":
442 flags |= TITLE_MASK
443 if "Uppercase" in properties:
444 flags |= UPPER_MASK
445 if char == ord(" ") or category[0] not in ("C", "Z"):
446 flags |= PRINTABLE_MASK
447 if "XID_Start" in properties:
448 flags |= XID_START_MASK
449 if "XID_Continue" in properties:
450 flags |= XID_CONTINUE_MASK
451 if "Cased" in properties:
452 flags |= CASED_MASK
453 if "Case_Ignorable" in properties:
454 flags |= CASE_IGNORABLE_MASK
455 sc = unicode.special_casing.get(char)
456 cf = unicode.case_folding.get(char, [char])
457 if record.simple_uppercase_mapping:
458 upper = int(record.simple_uppercase_mapping, 16)
459 else:
460 upper = char
461 if record.simple_lowercase_mapping:
462 lower = int(record.simple_lowercase_mapping, 16)
463 else:
464 lower = char
465 if record.simple_titlecase_mapping:
466 title = int(record.simple_titlecase_mapping, 16)
467 else:
468 title = upper
469 if sc is None and cf != [lower]:
470 sc = ([lower], [title], [upper])
471 if sc is None:
472 if upper == lower == title:
473 upper = lower = title = 0
474 else:
475 upper = upper - char
476 lower = lower - char
477 title = title - char
478 assert (abs(upper) <= 2147483647 and
479 abs(lower) <= 2147483647 and
480 abs(title) <= 2147483647)
481 else:
482 # This happens either when some character maps to more than one
483 # character in uppercase, lowercase, or titlecase or the
484 # casefolded version of the character is different from the
485 # lowercase. The extra characters are stored in a different
486 # array.
487 flags |= EXTENDED_CASE_MASK
488 lower = len(extra_casing) | (len(sc[0]) << 24)
489 extra_casing.extend(sc[0])
490 if cf != sc[0]:
491 lower |= len(cf) << 20
492 extra_casing.extend(cf)
493 upper = len(extra_casing) | (len(sc[2]) << 24)
494 extra_casing.extend(sc[2])
495 # Title is probably equal to upper.
496 if sc[1] == sc[2]:
497 title = upper
498 else:
499 title = len(extra_casing) | (len(sc[1]) << 24)
500 extra_casing.extend(sc[1])
501 # decimal digit, integer digit
502 decimal = 0
503 if record.decomposition_mapping:
504 flags |= DECIMAL_MASK
505 decimal = int(record.decomposition_mapping)
506 digit = 0
507 if record.numeric_type:
508 flags |= DIGIT_MASK
509 digit = int(record.numeric_type)
510 if record.numeric_value:
511 flags |= NUMERIC_MASK
512 numeric.setdefault(record.numeric_value, []).append(char)
513 item = (
514 upper, lower, title, decimal, digit, flags
515 )
516 # add entry to index and item tables
517 i = cache.get(item)
518 if i is None:
519 cache[item] = i = len(table)
520 table.append(item)
521 index[char] = i
522
523 print(len(table), "unique character type entries")
524 print(sum(map(len, numeric.values())), "numeric code points")
525 print(len(spaces), "whitespace code points")
526 print(len(linebreaks), "linebreak code points")
527 print(len(extra_casing), "extended case array")
528
529 print("--- Writing", FILE, "...")
530
531 with open(FILE, "w") as fp:
532 fprint = partial(print, file=fp)
533
534 fprint("/* this file was generated by %s %s */" % (SCRIPT, VERSION))
535 fprint()
536 fprint("/* a list of unique character type descriptors */")
537 fprint("const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = {")
538 for item in table:
539 fprint(" {%d, %d, %d, %d, %d, %d}," % item)
540 fprint("};")
541 fprint()
542
543 fprint("/* extended case mappings */")
544 fprint()
545 fprint("const Py_UCS4 _PyUnicode_ExtendedCase[] = {")
546 for c in extra_casing:
547 fprint(" %d," % c)
548 fprint("};")
549 fprint()
550
551 # split decomposition index table
552 index1, index2, shift = splitbins(index, trace)
553
554 fprint("/* type indexes */")
555 fprint("#define SHIFT", shift)
556 Array("index1", index1).dump(fp, trace)
557 Array("index2", index2).dump(fp, trace)
558
559 # Generate code for _PyUnicode_ToNumeric()
560 numeric_items = sorted(numeric.items())
561 fprint('/* Returns the numeric value as double for Unicode characters')
562 fprint(' * having this property, -1.0 otherwise.')
563 fprint(' */')
564 fprint('double _PyUnicode_ToNumeric(Py_UCS4 ch)')
565 fprint('{')
566 fprint(' switch (ch) {')
567 for value, codepoints in numeric_items:
568 # Turn text into float literals
569 parts = value.split('/')
570 parts = [repr(float(part)) for part in parts]
571 value = '/'.join(parts)
572
573 codepoints.sort()
574 for codepoint in codepoints:
575 fprint(' case 0x%04X:' % (codepoint,))
576 fprint(' return (double) %s;' % (value,))
577 fprint(' }')
578 fprint(' return -1.0;')
579 fprint('}')
580 fprint()
581
582 # Generate code for _PyUnicode_IsWhitespace()
583 fprint("/* Returns 1 for Unicode characters having the bidirectional")
584 fprint(" * type 'WS', 'B' or 'S' or the category 'Zs', 0 otherwise.")
585 fprint(" */")
586 fprint('int _PyUnicode_IsWhitespace(const Py_UCS4 ch)')
587 fprint('{')
588 fprint(' switch (ch) {')
589
590 for codepoint in sorted(spaces):
591 fprint(' case 0x%04X:' % (codepoint,))
592 fprint(' return 1;')
593
594 fprint(' }')
595 fprint(' return 0;')
596 fprint('}')
597 fprint()
598
599 # Generate code for _PyUnicode_IsLinebreak()
600 fprint("/* Returns 1 for Unicode characters having the line break")
601 fprint(" * property 'BK', 'CR', 'LF' or 'NL' or having bidirectional")
602 fprint(" * type 'B', 0 otherwise.")
603 fprint(" */")
604 fprint('int _PyUnicode_IsLinebreak(const Py_UCS4 ch)')
605 fprint('{')
606 fprint(' switch (ch) {')
607 for codepoint in sorted(linebreaks):
608 fprint(' case 0x%04X:' % (codepoint,))
609 fprint(' return 1;')
610
611 fprint(' }')
612 fprint(' return 0;')
613 fprint('}')
614 fprint()
615
616
617 # --------------------------------------------------------------------
618 # unicode name database
619
620 def makeunicodename(unicode, trace):
621 from dawg import build_compression_dawg
622
623 FILE = "Modules/unicodename_db.h"
624
625 print("--- Preparing", FILE, "...")
626
627 # unicode name hash table
628
629 # extract names
630 data = []
631 for char in unicode.chars:
632 record = unicode.table[char]
633 if record:
634 name = record.name.strip()
635 if name and name[0] != "<":
636 data.append((name, char))
637
638 print("--- Writing", FILE, "...")
639
640 with open(FILE, "w") as fp:
641 fprint = partial(print, file=fp)
642
643 fprint("/* this file was generated by %s %s */" % (SCRIPT, VERSION))
644 fprint()
645 fprint("#define NAME_MAXLEN", 256)
646 assert max(len(x) for x in data) < 256
647 fprint()
648
649 fprint("/* name->code dictionary */")
650 packed_dawg, pos_to_codepoint = build_compression_dawg(data)
651 notfound = len(pos_to_codepoint)
652 inverse_list = [notfound] * len(unicode.chars)
653 for pos, codepoint in enumerate(pos_to_codepoint):
654 inverse_list[codepoint] = pos
655 Array("packed_name_dawg", list(packed_dawg)).dump(fp, trace)
656 Array("dawg_pos_to_codepoint", pos_to_codepoint).dump(fp, trace)
657 index1, index2, shift = splitbins(inverse_list, trace)
658 fprint("#define DAWG_CODEPOINT_TO_POS_SHIFT", shift)
659 fprint("#define DAWG_CODEPOINT_TO_POS_NOTFOUND", notfound)
660 Array("dawg_codepoint_to_pos_index1", index1).dump(fp, trace)
661 Array("dawg_codepoint_to_pos_index2", index2).dump(fp, trace)
662
663 fprint()
664 fprint('static const unsigned int aliases_start = %#x;' %
665 NAME_ALIASES_START)
666 fprint('static const unsigned int aliases_end = %#x;' %
667 (NAME_ALIASES_START + len(unicode.aliases)))
668
669 fprint('static const unsigned int name_aliases[] = {')
670 for name, codepoint in unicode.aliases:
671 fprint(' 0x%04X,' % codepoint)
672 fprint('};')
673
674 # In Unicode 6.0.0, the sequences contain at most 4 BMP chars,
675 # so we are using Py_UCS2 seq[4]. This needs to be updated if longer
676 # sequences or sequences with non-BMP chars are added.
677 # unicodedata_lookup should be adapted too.
678 fprint(dedent("""
679 typedef struct NamedSequence {
680 int seqlen;
681 Py_UCS2 seq[4];
682 } named_sequence;
683 """))
684
685 fprint('static const unsigned int named_sequences_start = %#x;' %
686 NAMED_SEQUENCES_START)
687 fprint('static const unsigned int named_sequences_end = %#x;' %
688 (NAMED_SEQUENCES_START + len(unicode.named_sequences)))
689
690 fprint('static const named_sequence named_sequences[] = {')
691 for name, sequence in unicode.named_sequences:
692 seq_str = ', '.join('0x%04X' % cp for cp in sequence)
693 fprint(' {%d, {%s}},' % (len(sequence), seq_str))
694 fprint('};')
695
696 fprint(dedent("""
697 typedef struct {
698 Py_UCS4 first;
699 Py_UCS4 last;
700 int prefixid;
701 } derived_name_range;
702 """))
703
704 fprint('static const derived_name_range derived_name_ranges[] = {')
705 for name_range in unicode.derived_name_ranges:
706 fprint(' {0x%s, 0x%s, %d},' % name_range)
707 fprint('};')
708
709 fprint('static const char * const derived_name_prefixes[] = {')
710 for _, prefix in derived_name_range_names:
711 fprint(' "%s",' % prefix)
712 fprint('};')
713
714 def merge_old_version(version, new, old):
715 # Changes to exclusion file not implemented yet
716 if old.exclusions != new.exclusions:
717 raise NotImplementedError("exclusions differ")
718
719 # In these change records, 0xFF means "no change"
720 bidir_changes = [0xFF]*0x110000
721 category_changes = [0xFF]*0x110000
722 decimal_changes = [0xFF]*0x110000
723 mirrored_changes = [0xFF]*0x110000
724 east_asian_width_changes = [0xFF]*0x110000
725 # In numeric data, 0 means "no change",
726 # -1 means "did not have a numeric value
727 numeric_changes = [0] * 0x110000
728 # normalization_changes is a list of key-value pairs
729 normalization_changes = []
730 for i in range(0x110000):
731 if new.table[i] is None:
732 # Characters unassigned in the new version ought to
733 # be unassigned in the old one
734 assert old.table[i] is None
735 continue
736 # check characters unassigned in the old version
737 if old.table[i] is None:
738 # category 0 is "unassigned"
739 category_changes[i] = 0
740 continue
741 # check characters that differ
742 if old.table[i] != new.table[i]:
743 for k, field in enumerate(dataclasses.fields(UcdRecord)):
744 value = getattr(old.table[i], field.name)
745 new_value = getattr(new.table[i], field.name)
746 if value != new_value:
747 if k == 1 and i in PUA_15:
748 # the name is not set in the old.table, but in the
749 # new.table we are using it for aliases and named seq
750 assert value == ''
751 elif k == 2:
752 category_changes[i] = CATEGORY_NAMES.index(value)
753 elif k == 4:
754 bidir_changes[i] = BIDIRECTIONAL_NAMES.index(value)
755 elif k == 5:
756 # We assume that all normalization changes are in 1:1 mappings
757 assert " " not in value
758 normalization_changes.append((i, value))
759 elif k == 6:
760 # we only support changes where the old value is a single digit
761 assert value in "0123456789"
762 decimal_changes[i] = int(value)
763 elif k == 8:
764 # Since 0 encodes "no change", the old value is better not 0
765 if not value:
766 numeric_changes[i] = -1
767 else:
768 numeric_changes[i] = float(value)
769 assert numeric_changes[i] not in (0, -1)
770 elif k == 9:
771 if value == 'Y':
772 mirrored_changes[i] = '1'
773 else:
774 mirrored_changes[i] = '0'
775 elif k == 11:
776 # change to ISO comment, ignore
777 pass
778 elif k == 12:
779 # change to simple uppercase mapping; ignore
780 pass
781 elif k == 13:
782 # change to simple lowercase mapping; ignore
783 pass
784 elif k == 14:
785 # change to simple titlecase mapping; ignore
786 pass
787 elif k == 15:
788 # change to east asian width
789 east_asian_width_changes[i] = EASTASIANWIDTH_NAMES.index(value)
790 elif k == 16:
791 # derived property changes; not yet
792 pass
793 elif k == 17:
794 # normalization quickchecks are not performed
795 # for older versions
796 pass
797 else:
798 class Difference(Exception):pass
799 raise Difference(hex(i), k, old.table[i], new.table[i])
800 new.changed.append((version, list(zip(bidir_changes, category_changes,
801 decimal_changes, mirrored_changes,
802 east_asian_width_changes,
803 numeric_changes)),
804 normalization_changes))
805
806
807 DATA_DIR = os.path.join('Tools', 'unicode', 'data')
808
809 def open_data(template, version):
810 local = os.path.join(DATA_DIR, template % ('-'+version,))
811 if not os.path.exists(local):
812 import urllib.request
813 if version == '3.2.0':
814 # irregular url structure
815 url = ('https://www.unicode.org/Public/3.2-Update/'+template) % ('-'+version,)
816 else:
817 url = ('https://www.unicode.org/Public/%s/ucd/'+template) % (version, '')
818 os.makedirs(DATA_DIR, exist_ok=True)
819 urllib.request.urlretrieve(url, filename=local)
820 if local.endswith('.txt'):
821 return open(local, encoding='utf-8')
822 else:
823 # Unihan.zip
824 return open(local, 'rb')
825
826
827 def expand_range(char_range: str) -> Iterator[int]:
828 '''
829 Parses ranges of code points, as described in UAX #44:
830 https://www.unicode.org/reports/tr44/#Code_Point_Ranges
831 '''
832 if '..' in char_range:
833 first, last = [int(c, 16) for c in char_range.split('..')]
834 else:
835 first = last = int(char_range, 16)
836 for char in range(first, last+1):
837 yield char
838
839
840 class UcdFile:
841 '''
842 A file in the standard format of the UCD.
843
844 See: https://www.unicode.org/reports/tr44/#Format_Conventions
845
846 Note that, as described there, the Unihan data files have their
847 own separate format.
848 '''
849
850 def __init__(self, template: str, version: str) -> None:
851 self.template = template
852 self.version = version
853
854 def records(self) -> Iterator[List[str]]:
855 with open_data(self.template, self.version) as file:
856 for line in file:
857 line = line.split('#', 1)[0].strip()
858 if not line:
859 continue
860 yield [field.strip() for field in line.split(';')]
861
862 def __iter__(self) -> Iterator[List[str]]:
863 return self.records()
864
865 def expanded(self) -> Iterator[Tuple[int, List[str]]]:
866 for record in self.records():
867 char_range, rest = record[0], record[1:]
868 for char in expand_range(char_range):
869 yield char, rest
870
871
872 @dataclasses.dataclass
873 class UcdRecord:
874 # 15 fields from UnicodeData.txt . See:
875 # https://www.unicode.org/reports/tr44/#UnicodeData.txt
876 codepoint: str
877 name: str
878 general_category: str
879 canonical_combining_class: str
880 bidi_class: str
881 decomposition_type: str
882 decomposition_mapping: str
883 numeric_type: str
884 numeric_value: str
885 bidi_mirrored: str
886 unicode_1_name: str # obsolete
887 iso_comment: str # obsolete
888 simple_uppercase_mapping: str
889 simple_lowercase_mapping: str
890 simple_titlecase_mapping: str
891
892 # https://www.unicode.org/reports/tr44/#EastAsianWidth.txt
893 east_asian_width: Optional[str]
894
895 # Binary properties, as a set of those that are true.
896 # Taken from multiple files:
897 # https://www.unicode.org/reports/tr44/#DerivedCoreProperties.txt
898 # https://www.unicode.org/reports/tr44/#LineBreak.txt
899 binary_properties: Set[str]
900
901 # The Quick_Check properties related to normalization:
902 # https://www.unicode.org/reports/tr44/#Decompositions_and_Normalization
903 # We store them as a bitmask.
904 quick_check: int
905
906
907 def from_row(row: List[str]) -> UcdRecord:
908 return UcdRecord(*row, None, set(), 0)
909
910
911 # --------------------------------------------------------------------
912 # the following support code is taken from the unidb utilities
913 # Copyright (c) 1999-2000 by Secret Labs AB
914
915 # load a unicode-data file from disk
916
917 class UnicodeData:
918 # table: List[Optional[UcdRecord]] # index is codepoint; None means unassigned
919
920 def __init__(self, version, ideograph_check=True):
921 self.changed = []
922 table = [None] * 0x110000
923 for s in UcdFile(UNICODE_DATA, version):
924 char = int(s[0], 16)
925 table[char] = from_row(s)
926
927 self.derived_name_ranges = []
928
929 # expand first-last ranges
930 field = None
931 for i in range(0, 0x110000):
932 # The file UnicodeData.txt has its own distinct way of
933 # expressing ranges. See:
934 # https://www.unicode.org/reports/tr44/#Code_Point_Ranges
935 s = table[i]
936 if s:
937 if s.name[-6:] == "First>":
938 s.name = ""
939 field = dataclasses.astuple(s)[:15]
940 elif s.name[-5:] == "Last>":
941 for j, (rangename, _) in enumerate(derived_name_range_names):
942 if s.name.startswith("<" + rangename):
943 self.derived_name_ranges.append(
944 (field[0], s.codepoint, j))
945 break
946 s.name = ""
947 field = None
948 elif field:
949 table[i] = from_row(('%X' % i,) + field[1:])
950
951 # public attributes
952 self.filename = UNICODE_DATA % ''
953 self.table = table
954 self.chars = list(range(0x110000)) # unicode 3.2
955
956 # check for name aliases and named sequences, see #12753
957 # aliases and named sequences are not in 3.2.0
958 if version != '3.2.0':
959 self.aliases = []
960 # store aliases in the Private Use Area 15, in range U+F0000..U+F00FF,
961 # in order to take advantage of the compression and lookup
962 # algorithms used for the other characters
963 pua_index = NAME_ALIASES_START
964 for char, name, abbrev in UcdFile(NAME_ALIASES, version):
965 char = int(char, 16)
966 self.aliases.append((name, char))
967 # also store the name in the PUA 1
968 self.table[pua_index].name = name
969 pua_index += 1
970 assert pua_index - NAME_ALIASES_START == len(self.aliases)
971
972 self.named_sequences = []
973 # store named sequences in the PUA 1, in range U+F0100..,
974 # in order to take advantage of the compression and lookup
975 # algorithms used for the other characters.
976
977 assert pua_index < NAMED_SEQUENCES_START
978 pua_index = NAMED_SEQUENCES_START
979 for name, chars in UcdFile(NAMED_SEQUENCES, version):
980 chars = tuple(int(char, 16) for char in chars.split())
981 # check that the structure defined in makeunicodename is OK
982 assert 2 <= len(chars) <= 4, "change the Py_UCS2 array size"
983 assert all(c <= 0xFFFF for c in chars), ("use Py_UCS4 in "
984 "the NamedSequence struct and in unicodedata_lookup")
985 self.named_sequences.append((name, chars))
986 # also store these in the PUA 1
987 self.table[pua_index].name = name
988 pua_index += 1
989 assert pua_index - NAMED_SEQUENCES_START == len(self.named_sequences)
990
991 self.exclusions = {}
992 for char, in UcdFile(COMPOSITION_EXCLUSIONS, version):
993 char = int(char, 16)
994 self.exclusions[char] = 1
995
996 widths = [None] * 0x110000
997 for char, (width,) in UcdFile(EASTASIAN_WIDTH, version).expanded():
998 widths[char] = width
999
1000 for i in range(0, 0x110000):
1001 if table[i] is not None:
1002 table[i].east_asian_width = widths[i]
1003 self.widths = widths
1004
1005 for char, (propname, *propinfo) in UcdFile(DERIVED_CORE_PROPERTIES, version).expanded():
1006 if propinfo:
1007 # this is not a binary property, ignore it
1008 continue
1009
1010 if table[char]:
1011 # Some properties (e.g. Default_Ignorable_Code_Point)
1012 # apply to unassigned code points; ignore them
1013 table[char].binary_properties.add(propname)
1014
1015 for char_range, value in UcdFile(LINE_BREAK, version):
1016 if value not in MANDATORY_LINE_BREAKS:
1017 continue
1018 for char in expand_range(char_range):
1019 table[char].binary_properties.add('Line_Break')
1020
1021 # We only want the quickcheck properties
1022 # Format: NF?_QC; Y(es)/N(o)/M(aybe)
1023 # Yes is the default, hence only N and M occur
1024 # In 3.2.0, the format was different (NF?_NO)
1025 # The parsing will incorrectly determine these as
1026 # "yes", however, unicodedata.c will not perform quickchecks
1027 # for older versions, and no delta records will be created.
1028 quickchecks = [0] * 0x110000
1029 qc_order = 'NFD_QC NFKD_QC NFC_QC NFKC_QC'.split()
1030 for s in UcdFile(DERIVEDNORMALIZATION_PROPS, version):
1031 if len(s) < 2 or s[1] not in qc_order:
1032 continue
1033 quickcheck = 'MN'.index(s[2]) + 1 # Maybe or No
1034 quickcheck_shift = qc_order.index(s[1])*2
1035 quickcheck <<= quickcheck_shift
1036 for char in expand_range(s[0]):
1037 assert not (quickchecks[char]>>quickcheck_shift)&3
1038 quickchecks[char] |= quickcheck
1039 for i in range(0, 0x110000):
1040 if table[i] is not None:
1041 table[i].quick_check = quickchecks[i]
1042
1043 with open_data(UNIHAN, version) as file:
1044 zip = zipfile.ZipFile(file)
1045 if version == '3.2.0':
1046 data = zip.open('Unihan-3.2.0.txt').read()
1047 else:
1048 data = zip.open('Unihan_NumericValues.txt').read()
1049 for line in data.decode("utf-8").splitlines():
1050 if not line.startswith('U+'):
1051 continue
1052 code, tag, value = line.split(None, 3)[:3]
1053 if tag not in ('kAccountingNumeric', 'kPrimaryNumeric',
1054 'kOtherNumeric'):
1055 continue
1056 value = value.strip().replace(',', '')
1057 i = int(code[2:], 16)
1058 # Patch the numeric field
1059 if table[i] is not None:
1060 table[i].numeric_value = value
1061
1062 sc = self.special_casing = {}
1063 for data in UcdFile(SPECIAL_CASING, version):
1064 if data[4]:
1065 # We ignore all conditionals (since they depend on
1066 # languages) except for one, which is hardcoded. See
1067 # handle_capital_sigma in unicodeobject.c.
1068 continue
1069 c = int(data[0], 16)
1070 lower = [int(char, 16) for char in data[1].split()]
1071 title = [int(char, 16) for char in data[2].split()]
1072 upper = [int(char, 16) for char in data[3].split()]
1073 sc[c] = (lower, title, upper)
1074
1075 cf = self.case_folding = {}
1076 if version != '3.2.0':
1077 for data in UcdFile(CASE_FOLDING, version):
1078 if data[1] in "CF":
1079 c = int(data[0], 16)
1080 cf[c] = [int(char, 16) for char in data[2].split()]
1081
1082 def uselatin1(self):
1083 # restrict character range to ISO Latin 1
1084 self.chars = list(range(256))
1085
1086
1087
1088 # stuff to deal with arrays of unsigned integers
1089
1090 class Array:
1091
1092 def __init__(self, name, data):
1093 self.name = name
1094 self.data = data
1095
1096 def dump(self, file, trace=0):
1097 # write data to file, as a C array
1098 size = getsize(self.data)
1099 if trace:
1100 print(self.name+":", size*len(self.data), "bytes", file=sys.stderr)
1101 file.write("static const ")
1102 if size == 1:
1103 file.write("unsigned char")
1104 elif size == 2:
1105 file.write("unsigned short")
1106 else:
1107 file.write("unsigned int")
1108 file.write(" " + self.name + "[] = {\n")
1109 if self.data:
1110 s = " "
1111 for item in self.data:
1112 i = str(item) + ", "
1113 if len(s) + len(i) > 78:
1114 file.write(s.rstrip() + "\n")
1115 s = " " + i
1116 else:
1117 s = s + i
1118 if s.strip():
1119 file.write(s.rstrip() + "\n")
1120 file.write("};\n\n")
1121
1122
1123 def getsize(data):
1124 # return smallest possible integer size for the given array
1125 maxdata = max(data)
1126 if maxdata < 256:
1127 return 1
1128 elif maxdata < 65536:
1129 return 2
1130 else:
1131 return 4
1132
1133
1134 def splitbins(t, trace=0):
1135 """t, trace=0 -> (t1, t2, shift). Split a table to save space.
1136
1137 t is a sequence of ints. This function can be useful to save space if
1138 many of the ints are the same. t1 and t2 are lists of ints, and shift
1139 is an int, chosen to minimize the combined size of t1 and t2 (in C
1140 code), and where for each i in range(len(t)),
1141 t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
1142 where mask is a bitmask isolating the last "shift" bits.
1143
1144 If optional arg trace is non-zero (default zero), progress info
1145 is printed to sys.stderr. The higher the value, the more info
1146 you'll get.
1147 """
1148
1149 if trace:
1150 def dump(t1, t2, shift, bytes):
1151 print("%d+%d bins at shift %d; %d bytes" % (
1152 len(t1), len(t2), shift, bytes), file=sys.stderr)
1153 print("Size of original table:", len(t)*getsize(t), "bytes",
1154 file=sys.stderr)
1155 n = len(t)-1 # last valid index
1156 maxshift = 0 # the most we can shift n and still have something left
1157 if n > 0:
1158 while n >> 1:
1159 n >>= 1
1160 maxshift += 1
1161 del n
1162 bytes = sys.maxsize # smallest total size so far
1163 t = tuple(t) # so slices can be dict keys
1164 for shift in range(maxshift + 1):
1165 t1 = []
1166 t2 = []
1167 size = 2**shift
1168 bincache = {}
1169 for i in range(0, len(t), size):
1170 bin = t[i:i+size]
1171 index = bincache.get(bin)
1172 if index is None:
1173 index = len(t2)
1174 bincache[bin] = index
1175 t2.extend(bin)
1176 t1.append(index >> shift)
1177 # determine memory size
1178 b = len(t1)*getsize(t1) + len(t2)*getsize(t2)
1179 if trace > 1:
1180 dump(t1, t2, shift, b)
1181 if b < bytes:
1182 best = t1, t2, shift
1183 bytes = b
1184 t1, t2, shift = best
1185 if trace:
1186 print("Best:", end=' ', file=sys.stderr)
1187 dump(t1, t2, shift, bytes)
1188 if __debug__:
1189 # exhaustively verify that the decomposition is correct
1190 mask = ~((~0) << shift) # i.e., low-bit mask of shift bits
1191 for i in range(len(t)):
1192 assert t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
1193 return best
1194
1195
1196 if __name__ == "__main__":
1197 maketables(1)