| 1 |
#!/usr/bin/env python3 |
| 2 |
|
| 3 |
import os |
| 4 |
import sys |
| 5 |
import textwrap |
| 6 |
|
| 7 |
|
| 8 |
self_path = os.path.dirname(os.path.realpath(__file__)); |
| 9 |
f = open(self_path + "/unicode/CaseFolding.txt", "r") |
| 10 |
|
| 11 |
status_list = [ "C", "F" ] |
| 12 |
|
| 13 |
folding_list = [ dict(), dict(), dict() ] |
| 14 |
|
| 15 |
# Filter the foldings for "full" folding. |
| 16 |
for line in f: |
| 17 |
comment_off = line.find("#") |
| 18 |
if comment_off >= 0: |
| 19 |
line = line[:comment_off] |
| 20 |
line = line.strip() |
| 21 |
if not line: |
| 22 |
continue |
| 23 |
|
| 24 |
raw_codepoint, status, raw_mapping, ignored_tail = line.split(";", 3) |
| 25 |
if not status.strip() in status_list: |
| 26 |
continue |
| 27 |
codepoint = int(raw_codepoint.strip(), 16) |
| 28 |
mapping = [int(it, 16) for it in raw_mapping.strip().split(" ")] |
| 29 |
mapping_len = len(mapping) |
| 30 |
|
| 31 |
if mapping_len in range(1, 4): |
| 32 |
folding_list[mapping_len-1][codepoint] = mapping |
| 33 |
else: |
| 34 |
assert(False) |
| 35 |
f.close() |
| 36 |
|
| 37 |
|
| 38 |
# If we assume that (index0 ... index-1) makes a range (as defined below), |
| 39 |
# check that the newly provided index is compatible with the range too; i.e. |
| 40 |
# verify that the range can be extended without breaking its properties. |
| 41 |
# |
| 42 |
# Currently, we can handle ranges which: |
| 43 |
# |
| 44 |
# (1) either form consecutive sequence of codepoints and which map that range |
| 45 |
# to other consecutive range of codepoints (of the same length); |
| 46 |
# |
| 47 |
# (2) or a consecutive sequence of codepoints with step 2 where each codepoint |
| 48 |
# CP is mapped to the codepoint CP+1 |
| 49 |
# (e.g. 0x1234 -> 0x1235; 0x1236 -> 0x1237; 0x1238 -> 0x1239; ...). |
| 50 |
# |
| 51 |
# Note: When the codepoints in the range are mapped to multiple codepoints, |
| 52 |
# only the 1st mapped codepoint is considered. All the other ones have to be |
| 53 |
# shared by all the mappings covered by the range. |
| 54 |
def is_range_compatible(folding, codepoint_list, index0, index): |
| 55 |
N = index - index0 |
| 56 |
codepoint0 = codepoint_list[index0] |
| 57 |
codepoint1 = codepoint_list[index0+1] |
| 58 |
codepointN = codepoint_list[index] |
| 59 |
mapping0 = folding[codepoint0] |
| 60 |
mapping1 = folding[codepoint1] |
| 61 |
mappingN = folding[codepointN] |
| 62 |
|
| 63 |
# Check the range type (1): |
| 64 |
if codepoint1 - codepoint0 == 1 and codepointN - codepoint0 == N \ |
| 65 |
and mapping1[0] - mapping0[0] == 1 and mapping1[1:] == mapping0[1:] \ |
| 66 |
and mappingN[0] - mapping0[0] == N and mappingN[1:] == mapping0[1:]: |
| 67 |
return True |
| 68 |
|
| 69 |
# Check the range type (2): |
| 70 |
if codepoint1 - codepoint0 == 2 and codepointN - codepoint0 == 2 * N \ |
| 71 |
and mapping0[0] - codepoint0 == 1 \ |
| 72 |
and mapping1[0] - codepoint1 == 1 and mapping1[1:] == mapping0[1:] \ |
| 73 |
and mappingN[0] - codepointN == 1 and mappingN[1:] == mapping0[1:]: |
| 74 |
return True |
| 75 |
|
| 76 |
return False |
| 77 |
|
| 78 |
|
| 79 |
def mapping_str(list, mapping): |
| 80 |
return ",".join("0x{:04x}".format(x) for x in mapping) |
| 81 |
|
| 82 |
for mapping_len in range(1, 4): |
| 83 |
folding = folding_list[mapping_len-1] |
| 84 |
codepoint_list = list(folding) |
| 85 |
|
| 86 |
index0 = 0 |
| 87 |
count = len(folding) |
| 88 |
|
| 89 |
records = list() |
| 90 |
data_records = list() |
| 91 |
|
| 92 |
while index0 < count: |
| 93 |
index1 = index0 + 1 |
| 94 |
while index1 < count and is_range_compatible(folding, codepoint_list, index0, index1): |
| 95 |
index1 += 1 |
| 96 |
|
| 97 |
if index1 - index0 > 2: |
| 98 |
# Range of codepoints |
| 99 |
records.append("R(0x{:04x},0x{:04x})".format(codepoint_list[index0], codepoint_list[index1-1])) |
| 100 |
data_records.append(mapping_str(data_records, folding[codepoint_list[index0]])) |
| 101 |
data_records.append(mapping_str(data_records, folding[codepoint_list[index1-1]])) |
| 102 |
index0 = index1 |
| 103 |
else: |
| 104 |
# Single codepoint |
| 105 |
records.append("S(0x{:04x})".format(codepoint_list[index0])) |
| 106 |
data_records.append(mapping_str(data_records, folding[codepoint_list[index0]])) |
| 107 |
index0 += 1 |
| 108 |
|
| 109 |
sys.stdout.write("static const unsigned FOLD_MAP_{}[] = {{\n".format(mapping_len)) |
| 110 |
sys.stdout.write("\n".join(textwrap.wrap(", ".join(records), 110, |
| 111 |
initial_indent = " ", subsequent_indent=" "))) |
| 112 |
sys.stdout.write("\n};\n") |
| 113 |
|
| 114 |
sys.stdout.write("static const unsigned FOLD_MAP_{}_DATA[] = {{\n".format(mapping_len)) |
| 115 |
sys.stdout.write("\n".join(textwrap.wrap(", ".join(data_records), 110, |
| 116 |
initial_indent = " ", subsequent_indent=" "))) |
| 117 |
sys.stdout.write("\n};\n") |
| 118 |
|
| 119 |
|
| 120 |
|