| 1 |
# |
| 2 |
# genmap_tchinese.py: Traditional Chinese Codecs Map Generator |
| 3 |
# |
| 4 |
# Original Author: Hye-Shik Chang <[email protected]> |
| 5 |
# |
| 6 |
import os |
| 7 |
|
| 8 |
from genmap_support import * |
| 9 |
|
| 10 |
|
| 11 |
# ranges for (lead byte, follower byte) |
| 12 |
BIG5_C1 = (0xa1, 0xfe) |
| 13 |
BIG5_C2 = (0x40, 0xfe) |
| 14 |
BIG5HKSCS_C1 = (0x87, 0xfe) |
| 15 |
BIG5HKSCS_C2 = (0x40, 0xfe) |
| 16 |
|
| 17 |
MAPPINGS_BIG5 = 'https://unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/OTHER/BIG5.TXT' |
| 18 |
MAPPINGS_CP950 = 'https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP950.TXT' |
| 19 |
|
| 20 |
HKSCS_VERSION = '2004' |
| 21 |
# The files for HKSCS mappings are available under a restrictive license. |
| 22 |
# Users of the script need to download the files from the HKSARG CCLI website: |
| 23 |
MAPPINGS_HKSCS = f'https://www.ccli.gov.hk/en/archive/terms_hkscs-{HKSCS_VERSION}-big5-iso.html' |
| 24 |
|
| 25 |
|
| 26 |
def bh2s(code): |
| 27 |
return ((code >> 8) - 0x87) * (0xfe - 0x40 + 1) + ((code & 0xff) - 0x40) |
| 28 |
|
| 29 |
|
| 30 |
def split_bytes(code): |
| 31 |
"""Split 0xABCD into 0xAB, 0xCD""" |
| 32 |
return code >> 8, code & 0xff |
| 33 |
|
| 34 |
|
| 35 |
def parse_hkscs_map(fo): |
| 36 |
fo.seek(0, 0) |
| 37 |
table = [] |
| 38 |
for line in fo: |
| 39 |
line = line.split('#', 1)[0].strip() |
| 40 |
# We expect 4 columns in supported HKSCS files: |
| 41 |
# [1999]: unsupported |
| 42 |
# [2001]: unsupported |
| 43 |
# [2004]: Big-5; iso10646-1:1993; iso10646-1:2000; iso10646:2003+amd1 |
| 44 |
# [2008]: Big-5; iso10646-1:1993; iso10646-1:2000; iso10646:2003+amd6 |
| 45 |
# [2016]: not supported here--uses a json file instead |
| 46 |
# |
| 47 |
# In both supported cases, we only need the first and last column: |
| 48 |
# * Big-5 is a hex string (always 4 digits) |
| 49 |
# * iso10646:2003 is either a hex string (4 or 5 digits) or a sequence |
| 50 |
# of hex strings like: `<code_point1,code_point2>` |
| 51 |
try: |
| 52 |
hkscs_col, _, _, uni_col = line.split() |
| 53 |
hkscs = int(hkscs_col, 16) |
| 54 |
seq = tuple(int(cp, 16) for cp in uni_col.strip('<>').split(',')) |
| 55 |
except ValueError: |
| 56 |
continue |
| 57 |
table.append((hkscs, seq)) |
| 58 |
return table |
| 59 |
|
| 60 |
|
| 61 |
def make_hkscs_map(table): |
| 62 |
decode_map = {} |
| 63 |
encode_map_bmp, encode_map_notbmp = {}, {} |
| 64 |
is_bmp_map = {} |
| 65 |
sequences = [] |
| 66 |
beginnings = {} |
| 67 |
single_cp_table = [] |
| 68 |
# Determine multi-codepoint sequences, and sequence beginnings that encode |
| 69 |
# multiple multibyte (i.e. Big-5) codes. |
| 70 |
for mbcode, cp_seq in table: |
| 71 |
cp, *_ = cp_seq |
| 72 |
if len(cp_seq) == 1: |
| 73 |
single_cp_table.append((mbcode, cp)) |
| 74 |
else: |
| 75 |
sequences.append((mbcode, cp_seq)) |
| 76 |
beginnings.setdefault(cp, []).append(mbcode) |
| 77 |
# Decode table only cares about single code points (no sequences) currently |
| 78 |
for mbcode, cp in single_cp_table: |
| 79 |
b1, b2 = split_bytes(mbcode) |
| 80 |
decode_map.setdefault(b1, {}) |
| 81 |
decode_map[b1][b2] = cp & 0xffff |
| 82 |
# Encode table needs to mark code points beginning a sequence as tuples. |
| 83 |
for cp, mbcodes in beginnings.items(): |
| 84 |
plane = cp >> 16 |
| 85 |
if plane == 0: |
| 86 |
encode_map = encode_map_bmp |
| 87 |
elif plane == 2: |
| 88 |
encode_map = encode_map_notbmp |
| 89 |
is_bmp_map[bh2s(mbcodes[0])] = 1 |
| 90 |
else: |
| 91 |
assert False, 'only plane 0 (BMP) and plane 2 (SIP) allowed' |
| 92 |
if len(mbcodes) == 1: |
| 93 |
encode_value = mbcodes[0] |
| 94 |
else: |
| 95 |
encode_value = tuple(mbcodes) |
| 96 |
uni_b1, uni_b2 = split_bytes(cp & 0xffff) |
| 97 |
encode_map.setdefault(uni_b1, {}) |
| 98 |
encode_map[uni_b1][uni_b2] = encode_value |
| 99 |
|
| 100 |
return decode_map, encode_map_bmp, encode_map_notbmp, is_bmp_map |
| 101 |
|
| 102 |
|
| 103 |
def load_big5_map(): |
| 104 |
mapfile = open_mapping_file('python-mappings/BIG5.txt', MAPPINGS_BIG5) |
| 105 |
with mapfile: |
| 106 |
big5decmap = loadmap(mapfile) |
| 107 |
# big5 mapping fix: use the cp950 mapping for these characters as the file |
| 108 |
# provided by unicode.org doesn't define a mapping. See notes in BIG5.txt. |
| 109 |
# Since U+5341, U+5345, U+FF0F, U+FF3C already have a big5 mapping, no |
| 110 |
# roundtrip compatibility is guaranteed for those. |
| 111 |
for m in """\ |
| 112 |
0xA15A 0x2574 |
| 113 |
0xA1C3 0xFFE3 |
| 114 |
0xA1C5 0x02CD |
| 115 |
0xA1FE 0xFF0F |
| 116 |
0xA240 0xFF3C |
| 117 |
0xA2CC 0x5341 |
| 118 |
0xA2CE 0x5345""".splitlines(): |
| 119 |
bcode, ucode = list(map(eval, m.split())) |
| 120 |
big5decmap[bcode >> 8][bcode & 0xff] = ucode |
| 121 |
# encoding map |
| 122 |
big5encmap = {} |
| 123 |
for c1, m in list(big5decmap.items()): |
| 124 |
for c2, code in list(m.items()): |
| 125 |
big5encmap.setdefault(code >> 8, {}) |
| 126 |
if code & 0xff not in big5encmap[code >> 8]: |
| 127 |
big5encmap[code >> 8][code & 0xff] = c1 << 8 | c2 |
| 128 |
# fix unicode->big5 priority for the above-mentioned duplicate characters |
| 129 |
big5encmap[0xFF][0x0F] = 0xA241 |
| 130 |
big5encmap[0xFF][0x3C] = 0xA242 |
| 131 |
big5encmap[0x53][0x41] = 0xA451 |
| 132 |
big5encmap[0x53][0x45] = 0xA4CA |
| 133 |
|
| 134 |
return big5decmap, big5encmap |
| 135 |
|
| 136 |
|
| 137 |
def load_cp950_map(): |
| 138 |
mapfile = open_mapping_file('python-mappings/CP950.TXT', MAPPINGS_CP950) |
| 139 |
with mapfile: |
| 140 |
cp950decmap = loadmap(mapfile) |
| 141 |
cp950encmap = {} |
| 142 |
for c1, m in list(cp950decmap.items()): |
| 143 |
for c2, code in list(m.items()): |
| 144 |
cp950encmap.setdefault(code >> 8, {}) |
| 145 |
if code & 0xff not in cp950encmap[code >> 8]: |
| 146 |
cp950encmap[code >> 8][code & 0xff] = c1 << 8 | c2 |
| 147 |
# fix unicode->big5 duplicated mapping priority |
| 148 |
cp950encmap[0x53][0x41] = 0xA451 |
| 149 |
cp950encmap[0x53][0x45] = 0xA4CA |
| 150 |
return cp950decmap, cp950encmap |
| 151 |
|
| 152 |
|
| 153 |
def main_tw(): |
| 154 |
big5decmap, big5encmap = load_big5_map() |
| 155 |
cp950decmap, cp950encmap = load_cp950_map() |
| 156 |
|
| 157 |
# CP950 extends Big5, and the codec can use the Big5 lookup tables |
| 158 |
# for most entries. So the CP950 tables should only include entries |
| 159 |
# that are not in Big5: |
| 160 |
for c1, m in list(cp950encmap.items()): |
| 161 |
for c2, code in list(m.items()): |
| 162 |
if (c1 in big5encmap and c2 in big5encmap[c1] |
| 163 |
and big5encmap[c1][c2] == code): |
| 164 |
del cp950encmap[c1][c2] |
| 165 |
for c1, m in list(cp950decmap.items()): |
| 166 |
for c2, code in list(m.items()): |
| 167 |
if (c1 in big5decmap and c2 in big5decmap[c1] |
| 168 |
and big5decmap[c1][c2] == code): |
| 169 |
del cp950decmap[c1][c2] |
| 170 |
|
| 171 |
with open('mappings_tw.h', 'w') as fp: |
| 172 |
print_autogen(fp, os.path.basename(__file__)) |
| 173 |
write_big5_maps(fp, 'BIG5', 'big5', big5decmap, big5encmap) |
| 174 |
write_big5_maps(fp, 'CP950', 'cp950ext', cp950decmap, cp950encmap) |
| 175 |
|
| 176 |
|
| 177 |
def write_big5_maps(fp, display_name, table_name, decode_map, encode_map): |
| 178 |
print(f'Generating {display_name} decode map...') |
| 179 |
writer = DecodeMapWriter(fp, table_name, decode_map) |
| 180 |
writer.update_decode_map(BIG5_C1, BIG5_C2) |
| 181 |
writer.generate() |
| 182 |
print(f'Generating {display_name} encode map...') |
| 183 |
writer = EncodeMapWriter(fp, table_name, encode_map) |
| 184 |
writer.generate() |
| 185 |
|
| 186 |
|
| 187 |
class HintsWriter: |
| 188 |
filler_class = BufferedFiller |
| 189 |
|
| 190 |
def __init__(self, fp, prefix, isbmpmap): |
| 191 |
self.fp = fp |
| 192 |
self.prefix = prefix |
| 193 |
self.isbmpmap = isbmpmap |
| 194 |
self.filler = self.filler_class() |
| 195 |
|
| 196 |
def fillhints(self, hintfrom, hintto): |
| 197 |
name = f'{self.prefix}_phint_{hintfrom}' |
| 198 |
self.fp.write(f'static const unsigned char {name}[] = {{\n') |
| 199 |
for msbcode in range(hintfrom, hintto+1, 8): |
| 200 |
v = 0 |
| 201 |
for c in range(msbcode, msbcode+8): |
| 202 |
v |= self.isbmpmap.get(c, 0) << (c - msbcode) |
| 203 |
self.filler.write('%d,' % v) |
| 204 |
self.filler.printout(self.fp) |
| 205 |
self.fp.write('};\n\n') |
| 206 |
|
| 207 |
|
| 208 |
def main_hkscs(): |
| 209 |
filename = f'python-mappings/hkscs-{HKSCS_VERSION}-big5-iso.txt' |
| 210 |
with open_mapping_file(filename, MAPPINGS_HKSCS) as f: |
| 211 |
table = parse_hkscs_map(f) |
| 212 |
hkscsdecmap, hkscsencmap_bmp, hkscsencmap_nonbmp, isbmpmap = ( |
| 213 |
make_hkscs_map(table) |
| 214 |
) |
| 215 |
with open('mappings_hk.h', 'w') as fp: |
| 216 |
print('Generating BIG5HKSCS decode map...') |
| 217 |
print_autogen(fp, os.path.basename(__file__)) |
| 218 |
writer = DecodeMapWriter(fp, 'big5hkscs', hkscsdecmap) |
| 219 |
writer.update_decode_map(BIG5HKSCS_C1, BIG5HKSCS_C2) |
| 220 |
writer.generate() |
| 221 |
|
| 222 |
print('Generating BIG5HKSCS decode map Unicode plane hints...') |
| 223 |
writer = HintsWriter(fp, 'big5hkscs', isbmpmap) |
| 224 |
writer.fillhints(bh2s(0x8740), bh2s(0xa0fe)) |
| 225 |
writer.fillhints(bh2s(0xc6a1), bh2s(0xc8fe)) |
| 226 |
writer.fillhints(bh2s(0xf9d6), bh2s(0xfefe)) |
| 227 |
|
| 228 |
print('Generating BIG5HKSCS encode map (BMP)...') |
| 229 |
writer = EncodeMapWriter(fp, 'big5hkscs_bmp', hkscsencmap_bmp) |
| 230 |
writer.generate() |
| 231 |
|
| 232 |
print('Generating BIG5HKSCS encode map (non-BMP)...') |
| 233 |
writer = EncodeMapWriter(fp, 'big5hkscs_nonbmp', hkscsencmap_nonbmp) |
| 234 |
writer.generate() |
| 235 |
|
| 236 |
|
| 237 |
if __name__ == '__main__': |
| 238 |
main_tw() |
| 239 |
main_hkscs() |