Debian Codegraph @

Tools/unicode/genmap_tchinese.py

raw
1 #
2 # genmap_tchinese.py: Traditional Chinese Codecs Map Generator
3 #
4 # Original Author: Hye-Shik Chang <[email protected]>
5 #
6 import os
7
8 from genmap_support import *
9
10
11 # ranges for (lead byte, follower byte)
12 BIG5_C1 = (0xa1, 0xfe)
13 BIG5_C2 = (0x40, 0xfe)
14 BIG5HKSCS_C1 = (0x87, 0xfe)
15 BIG5HKSCS_C2 = (0x40, 0xfe)
16
17 MAPPINGS_BIG5 = 'https://unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/OTHER/BIG5.TXT'
18 MAPPINGS_CP950 = 'https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP950.TXT'
19
20 HKSCS_VERSION = '2004'
21 # The files for HKSCS mappings are available under a restrictive license.
22 # Users of the script need to download the files from the HKSARG CCLI website:
23 MAPPINGS_HKSCS = f'https://www.ccli.gov.hk/en/archive/terms_hkscs-{HKSCS_VERSION}-big5-iso.html'
24
25
26 def bh2s(code):
27 return ((code >> 8) - 0x87) * (0xfe - 0x40 + 1) + ((code & 0xff) - 0x40)
28
29
30 def split_bytes(code):
31 """Split 0xABCD into 0xAB, 0xCD"""
32 return code >> 8, code & 0xff
33
34
35 def parse_hkscs_map(fo):
36 fo.seek(0, 0)
37 table = []
38 for line in fo:
39 line = line.split('#', 1)[0].strip()
40 # We expect 4 columns in supported HKSCS files:
41 # [1999]: unsupported
42 # [2001]: unsupported
43 # [2004]: Big-5; iso10646-1:1993; iso10646-1:2000; iso10646:2003+amd1
44 # [2008]: Big-5; iso10646-1:1993; iso10646-1:2000; iso10646:2003+amd6
45 # [2016]: not supported here--uses a json file instead
46 #
47 # In both supported cases, we only need the first and last column:
48 # * Big-5 is a hex string (always 4 digits)
49 # * iso10646:2003 is either a hex string (4 or 5 digits) or a sequence
50 # of hex strings like: `<code_point1,code_point2>`
51 try:
52 hkscs_col, _, _, uni_col = line.split()
53 hkscs = int(hkscs_col, 16)
54 seq = tuple(int(cp, 16) for cp in uni_col.strip('<>').split(','))
55 except ValueError:
56 continue
57 table.append((hkscs, seq))
58 return table
59
60
61 def make_hkscs_map(table):
62 decode_map = {}
63 encode_map_bmp, encode_map_notbmp = {}, {}
64 is_bmp_map = {}
65 sequences = []
66 beginnings = {}
67 single_cp_table = []
68 # Determine multi-codepoint sequences, and sequence beginnings that encode
69 # multiple multibyte (i.e. Big-5) codes.
70 for mbcode, cp_seq in table:
71 cp, *_ = cp_seq
72 if len(cp_seq) == 1:
73 single_cp_table.append((mbcode, cp))
74 else:
75 sequences.append((mbcode, cp_seq))
76 beginnings.setdefault(cp, []).append(mbcode)
77 # Decode table only cares about single code points (no sequences) currently
78 for mbcode, cp in single_cp_table:
79 b1, b2 = split_bytes(mbcode)
80 decode_map.setdefault(b1, {})
81 decode_map[b1][b2] = cp & 0xffff
82 # Encode table needs to mark code points beginning a sequence as tuples.
83 for cp, mbcodes in beginnings.items():
84 plane = cp >> 16
85 if plane == 0:
86 encode_map = encode_map_bmp
87 elif plane == 2:
88 encode_map = encode_map_notbmp
89 is_bmp_map[bh2s(mbcodes[0])] = 1
90 else:
91 assert False, 'only plane 0 (BMP) and plane 2 (SIP) allowed'
92 if len(mbcodes) == 1:
93 encode_value = mbcodes[0]
94 else:
95 encode_value = tuple(mbcodes)
96 uni_b1, uni_b2 = split_bytes(cp & 0xffff)
97 encode_map.setdefault(uni_b1, {})
98 encode_map[uni_b1][uni_b2] = encode_value
99
100 return decode_map, encode_map_bmp, encode_map_notbmp, is_bmp_map
101
102
103 def load_big5_map():
104 mapfile = open_mapping_file('python-mappings/BIG5.txt', MAPPINGS_BIG5)
105 with mapfile:
106 big5decmap = loadmap(mapfile)
107 # big5 mapping fix: use the cp950 mapping for these characters as the file
108 # provided by unicode.org doesn't define a mapping. See notes in BIG5.txt.
109 # Since U+5341, U+5345, U+FF0F, U+FF3C already have a big5 mapping, no
110 # roundtrip compatibility is guaranteed for those.
111 for m in """\
112 0xA15A 0x2574
113 0xA1C3 0xFFE3
114 0xA1C5 0x02CD
115 0xA1FE 0xFF0F
116 0xA240 0xFF3C
117 0xA2CC 0x5341
118 0xA2CE 0x5345""".splitlines():
119 bcode, ucode = list(map(eval, m.split()))
120 big5decmap[bcode >> 8][bcode & 0xff] = ucode
121 # encoding map
122 big5encmap = {}
123 for c1, m in list(big5decmap.items()):
124 for c2, code in list(m.items()):
125 big5encmap.setdefault(code >> 8, {})
126 if code & 0xff not in big5encmap[code >> 8]:
127 big5encmap[code >> 8][code & 0xff] = c1 << 8 | c2
128 # fix unicode->big5 priority for the above-mentioned duplicate characters
129 big5encmap[0xFF][0x0F] = 0xA241
130 big5encmap[0xFF][0x3C] = 0xA242
131 big5encmap[0x53][0x41] = 0xA451
132 big5encmap[0x53][0x45] = 0xA4CA
133
134 return big5decmap, big5encmap
135
136
137 def load_cp950_map():
138 mapfile = open_mapping_file('python-mappings/CP950.TXT', MAPPINGS_CP950)
139 with mapfile:
140 cp950decmap = loadmap(mapfile)
141 cp950encmap = {}
142 for c1, m in list(cp950decmap.items()):
143 for c2, code in list(m.items()):
144 cp950encmap.setdefault(code >> 8, {})
145 if code & 0xff not in cp950encmap[code >> 8]:
146 cp950encmap[code >> 8][code & 0xff] = c1 << 8 | c2
147 # fix unicode->big5 duplicated mapping priority
148 cp950encmap[0x53][0x41] = 0xA451
149 cp950encmap[0x53][0x45] = 0xA4CA
150 return cp950decmap, cp950encmap
151
152
153 def main_tw():
154 big5decmap, big5encmap = load_big5_map()
155 cp950decmap, cp950encmap = load_cp950_map()
156
157 # CP950 extends Big5, and the codec can use the Big5 lookup tables
158 # for most entries. So the CP950 tables should only include entries
159 # that are not in Big5:
160 for c1, m in list(cp950encmap.items()):
161 for c2, code in list(m.items()):
162 if (c1 in big5encmap and c2 in big5encmap[c1]
163 and big5encmap[c1][c2] == code):
164 del cp950encmap[c1][c2]
165 for c1, m in list(cp950decmap.items()):
166 for c2, code in list(m.items()):
167 if (c1 in big5decmap and c2 in big5decmap[c1]
168 and big5decmap[c1][c2] == code):
169 del cp950decmap[c1][c2]
170
171 with open('mappings_tw.h', 'w') as fp:
172 print_autogen(fp, os.path.basename(__file__))
173 write_big5_maps(fp, 'BIG5', 'big5', big5decmap, big5encmap)
174 write_big5_maps(fp, 'CP950', 'cp950ext', cp950decmap, cp950encmap)
175
176
177 def write_big5_maps(fp, display_name, table_name, decode_map, encode_map):
178 print(f'Generating {display_name} decode map...')
179 writer = DecodeMapWriter(fp, table_name, decode_map)
180 writer.update_decode_map(BIG5_C1, BIG5_C2)
181 writer.generate()
182 print(f'Generating {display_name} encode map...')
183 writer = EncodeMapWriter(fp, table_name, encode_map)
184 writer.generate()
185
186
187 class HintsWriter:
188 filler_class = BufferedFiller
189
190 def __init__(self, fp, prefix, isbmpmap):
191 self.fp = fp
192 self.prefix = prefix
193 self.isbmpmap = isbmpmap
194 self.filler = self.filler_class()
195
196 def fillhints(self, hintfrom, hintto):
197 name = f'{self.prefix}_phint_{hintfrom}'
198 self.fp.write(f'static const unsigned char {name}[] = {{\n')
199 for msbcode in range(hintfrom, hintto+1, 8):
200 v = 0
201 for c in range(msbcode, msbcode+8):
202 v |= self.isbmpmap.get(c, 0) << (c - msbcode)
203 self.filler.write('%d,' % v)
204 self.filler.printout(self.fp)
205 self.fp.write('};\n\n')
206
207
208 def main_hkscs():
209 filename = f'python-mappings/hkscs-{HKSCS_VERSION}-big5-iso.txt'
210 with open_mapping_file(filename, MAPPINGS_HKSCS) as f:
211 table = parse_hkscs_map(f)
212 hkscsdecmap, hkscsencmap_bmp, hkscsencmap_nonbmp, isbmpmap = (
213 make_hkscs_map(table)
214 )
215 with open('mappings_hk.h', 'w') as fp:
216 print('Generating BIG5HKSCS decode map...')
217 print_autogen(fp, os.path.basename(__file__))
218 writer = DecodeMapWriter(fp, 'big5hkscs', hkscsdecmap)
219 writer.update_decode_map(BIG5HKSCS_C1, BIG5HKSCS_C2)
220 writer.generate()
221
222 print('Generating BIG5HKSCS decode map Unicode plane hints...')
223 writer = HintsWriter(fp, 'big5hkscs', isbmpmap)
224 writer.fillhints(bh2s(0x8740), bh2s(0xa0fe))
225 writer.fillhints(bh2s(0xc6a1), bh2s(0xc8fe))
226 writer.fillhints(bh2s(0xf9d6), bh2s(0xfefe))
227
228 print('Generating BIG5HKSCS encode map (BMP)...')
229 writer = EncodeMapWriter(fp, 'big5hkscs_bmp', hkscsencmap_bmp)
230 writer.generate()
231
232 print('Generating BIG5HKSCS encode map (non-BMP)...')
233 writer = EncodeMapWriter(fp, 'big5hkscs_nonbmp', hkscsencmap_nonbmp)
234 writer.generate()
235
236
237 if __name__ == '__main__':
238 main_tw()
239 main_hkscs()