Debian Codegraph @

scripts/build_folding_map.py

raw
1 #!/usr/bin/env python3
2
3 import os
4 import sys
5 import textwrap
6
7
8 self_path = os.path.dirname(os.path.realpath(__file__));
9 f = open(self_path + "/unicode/CaseFolding.txt", "r")
10
11 status_list = [ "C", "F" ]
12
13 folding_list = [ dict(), dict(), dict() ]
14
15 # Filter the foldings for "full" folding.
16 for line in f:
17 comment_off = line.find("#")
18 if comment_off >= 0:
19 line = line[:comment_off]
20 line = line.strip()
21 if not line:
22 continue
23
24 raw_codepoint, status, raw_mapping, ignored_tail = line.split(";", 3)
25 if not status.strip() in status_list:
26 continue
27 codepoint = int(raw_codepoint.strip(), 16)
28 mapping = [int(it, 16) for it in raw_mapping.strip().split(" ")]
29 mapping_len = len(mapping)
30
31 if mapping_len in range(1, 4):
32 folding_list[mapping_len-1][codepoint] = mapping
33 else:
34 assert(False)
35 f.close()
36
37
38 # If we assume that (index0 ... index-1) makes a range (as defined below),
39 # check that the newly provided index is compatible with the range too; i.e.
40 # verify that the range can be extended without breaking its properties.
41 #
42 # Currently, we can handle ranges which:
43 #
44 # (1) either form consecutive sequence of codepoints and which map that range
45 # to other consecutive range of codepoints (of the same length);
46 #
47 # (2) or a consecutive sequence of codepoints with step 2 where each codepoint
48 # CP is mapped to the codepoint CP+1
49 # (e.g. 0x1234 -> 0x1235; 0x1236 -> 0x1237; 0x1238 -> 0x1239; ...).
50 #
51 # Note: When the codepoints in the range are mapped to multiple codepoints,
52 # only the 1st mapped codepoint is considered. All the other ones have to be
53 # shared by all the mappings covered by the range.
54 def is_range_compatible(folding, codepoint_list, index0, index):
55 N = index - index0
56 codepoint0 = codepoint_list[index0]
57 codepoint1 = codepoint_list[index0+1]
58 codepointN = codepoint_list[index]
59 mapping0 = folding[codepoint0]
60 mapping1 = folding[codepoint1]
61 mappingN = folding[codepointN]
62
63 # Check the range type (1):
64 if codepoint1 - codepoint0 == 1 and codepointN - codepoint0 == N \
65 and mapping1[0] - mapping0[0] == 1 and mapping1[1:] == mapping0[1:] \
66 and mappingN[0] - mapping0[0] == N and mappingN[1:] == mapping0[1:]:
67 return True
68
69 # Check the range type (2):
70 if codepoint1 - codepoint0 == 2 and codepointN - codepoint0 == 2 * N \
71 and mapping0[0] - codepoint0 == 1 \
72 and mapping1[0] - codepoint1 == 1 and mapping1[1:] == mapping0[1:] \
73 and mappingN[0] - codepointN == 1 and mappingN[1:] == mapping0[1:]:
74 return True
75
76 return False
77
78
79 def mapping_str(list, mapping):
80 return ",".join("0x{:04x}".format(x) for x in mapping)
81
82 for mapping_len in range(1, 4):
83 folding = folding_list[mapping_len-1]
84 codepoint_list = list(folding)
85
86 index0 = 0
87 count = len(folding)
88
89 records = list()
90 data_records = list()
91
92 while index0 < count:
93 index1 = index0 + 1
94 while index1 < count and is_range_compatible(folding, codepoint_list, index0, index1):
95 index1 += 1
96
97 if index1 - index0 > 2:
98 # Range of codepoints
99 records.append("R(0x{:04x},0x{:04x})".format(codepoint_list[index0], codepoint_list[index1-1]))
100 data_records.append(mapping_str(data_records, folding[codepoint_list[index0]]))
101 data_records.append(mapping_str(data_records, folding[codepoint_list[index1-1]]))
102 index0 = index1
103 else:
104 # Single codepoint
105 records.append("S(0x{:04x})".format(codepoint_list[index0]))
106 data_records.append(mapping_str(data_records, folding[codepoint_list[index0]]))
107 index0 += 1
108
109 sys.stdout.write("static const unsigned FOLD_MAP_{}[] = {{\n".format(mapping_len))
110 sys.stdout.write("\n".join(textwrap.wrap(", ".join(records), 110,
111 initial_indent = " ", subsequent_indent=" ")))
112 sys.stdout.write("\n};\n")
113
114 sys.stdout.write("static const unsigned FOLD_MAP_{}_DATA[] = {{\n".format(mapping_len))
115 sys.stdout.write("\n".join(textwrap.wrap(", ".join(data_records), 110,
116 initial_indent = " ", subsequent_indent=" ")))
117 sys.stdout.write("\n};\n")
118
119
120