| 1 |
#/** |
| 2 |
# ******************************************************************************* |
| 3 |
# * Copyright (C) 2016 and later: Unicode, Inc. and others. * |
| 4 |
# * License & terms of use: http://www.unicode.org/copyright.html * |
| 5 |
# ******************************************************************************* |
| 6 |
# ******************************************************************************* |
| 7 |
# * Copyright (C) 2000-2004, International Business Machines Corporation and * |
| 8 |
# * others. All Rights Reserved. * |
| 9 |
# ******************************************************************************* |
| 10 |
# */ |
| 11 |
|
| 12 |
#!perl |
| 13 |
|
| 14 |
# Usage - $0 <remap file> |
| 15 |
# e.g. - indic indic.txt |
| 16 |
# The input file should be a subset of the Unicode data file containing |
| 17 |
# the blocks of interest. |
| 18 |
# |
| 19 |
# The remap file should have lines of the form |
| 20 |
# "\u0D01>\u0D02;" |
| 21 |
# including the quotes. These will be interpreted as saying that the |
| 22 |
# undefined code point U+D01 (derived via mapping from InterIndic) |
| 23 |
# can be remapped to U+D02. |
| 24 |
# |
| 25 |
# The purpose of this script is to process the Indic script data into |
| 26 |
# a form usable by the IndicTransliterator, that is, the Indic-Indic |
| 27 |
# transliterator. The transliterator needs two things: A mapping of |
| 28 |
# the code points in common, and a list of the exceptions. |
| 29 |
|
| 30 |
# Assume we are located in icu4j/src/com/ibm/tools/translit/. |
| 31 |
# We want the Unicode DB in icu4j/src/data/unicode/. |
| 32 |
$UNICODE_DB = "../../../../data/unicode/UnicodeData.txt"; |
| 33 |
$EXCEPTIONS_FILE = shift; |
| 34 |
|
| 35 |
# Assume we are located in icu4j/src/com/ibm/tools/translit/. |
| 36 |
# We want to output files to icu4j/src/com/ibm/text/resources/. |
| 37 |
# Output directory |
| 38 |
$OUTDIR = "../../text/resources"; |
| 39 |
|
| 40 |
# The template file should contain java code that can be used |
| 41 |
# to generate RuleBasedTransliterator resource files. The template |
| 42 |
# should contain the following embedded symbols, which this script |
| 43 |
# will replace: |
| 44 |
# $TOOL - name of generating tool |
| 45 |
# $DATE - date of generation |
| 46 |
# $SCRIPTFROM - name of source script |
| 47 |
# $SCRIPTTO - name of target script |
| 48 |
# $RULES - rules |
| 49 |
$RBT_TEMPLATE = 'rbtTemplate.txt'; |
| 50 |
|
| 51 |
# Name of this tool in generated RBT files |
| 52 |
$RBT_GEN_TOOL = 'icu4j/src/com/ibm/tools/translit/indic.pl'; |
| 53 |
|
| 54 |
$DUMP = 0; # If 1, dump out internal data |
| 55 |
|
| 56 |
$DO_HEURISTIC_REMAP = 0; # If 1, do automatic heuristic remapping |
| 57 |
$DO_DECOMP_REMAP = 0; # If 1, do decomp remapping |
| 58 |
|
| 59 |
open(UNICODE_DB); |
| 60 |
while (<UNICODE_DB>) { |
| 61 |
next if (m|^0[0-8]|); # Skip up to Devanagari block (0900) |
| 62 |
last if (m|^0D[8-F]|i); # Bail out after Malayam block (0D00) |
| 63 |
# 0D39;MALAYALAM LETTER HA;Lo;0;L;;;;;N;;;;; |
| 64 |
my @data = split(/;/); |
| 65 |
my $fullCode = hex($data[0]); # e.g., 0x093F |
| 66 |
my $code = $fullCode & 0x7F; # e.g., 0x3F |
| 67 |
my ($script, $name) = ($data[1] =~ /(\w+)\s+(.+)/); |
| 68 |
die "Can't parse $_" unless ($name); |
| 69 |
# e.g., $code/$script/$name = 3F/MALAYALAM/VOWEL SIGN I |
| 70 |
|
| 71 |
# Titlecase the script |
| 72 |
$script = ucfirst(lc($script)); |
| 73 |
|
| 74 |
# Fix a couple inconsistencies in the 3.0 data |
| 75 |
# REVISIT: Is this okay to do? |
| 76 |
if ($DO_HEURISTIC_REMAP) { |
| 77 |
if ($script eq 'Gujarati' && $code >= 5 && $code <= 0x14) { |
| 78 |
$name =~ s/^VOWEL/LETTER/; |
| 79 |
} |
| 80 |
} |
| 81 |
|
| 82 |
# Keep track of all script names we encounter. We also note the |
| 83 |
# base of the block. |
| 84 |
my $base = $fullCode & ~0x7F; # e.g., 0x900; |
| 85 |
if (exists $SCRIPT_TO_BASE{$script}) { |
| 86 |
die "Script base mismatch for $script: $base vs. $SCRIPT_TO_BASE{$script}" |
| 87 |
if ($SCRIPT_TO_BASE{$script} ne $base); |
| 88 |
} else { |
| 89 |
$SCRIPT_TO_BASE{$script} = $base; |
| 90 |
} |
| 91 |
|
| 92 |
# Build up a mapping by name. For each name, keep a hash keyed by |
| 93 |
# code point. For each code point, keep an array of script names. |
| 94 |
# Also keep a total use count for each name. |
| 95 |
push @{$NAME_CODE_TO_SCRIPTS{$name}{$code}}, $script; |
| 96 |
++$NAME_CODE_TO_SCRIPTS{$name}{count}; |
| 97 |
|
| 98 |
# Build a map that looks like this: |
| 99 |
# $SCRIPT_NAME_TO_CODE{<script>}{<name>} = <code> |
| 100 |
# or undef if there is no mapping. |
| 101 |
$SCRIPT_NAME_TO_CODE{$script}{$name} = $code; |
| 102 |
|
| 103 |
# Build a map that looks like this: |
| 104 |
$SCRIPT_CODE_TO_NAME{$script}{$code} = $name; |
| 105 |
|
| 106 |
# And a map from the fullCode point to the name |
| 107 |
$FULLCODE_TO_NAME{$fullCode} = $name; |
| 108 |
|
| 109 |
# Map code (0..7F) to name. This is usually a 1-1 mapping, but |
| 110 |
# is 1-n in a few cases. |
| 111 |
if (exists $CODE_TO_NAME{$code}) { |
| 112 |
if ($name ne $CODE_TO_NAME{$code}) { |
| 113 |
# For multiple names on a code offset, use the format |
| 114 |
# (a/b), (a/b/c), etc. |
| 115 |
local $_ = $CODE_TO_NAME{$code}; |
| 116 |
if (m|^\(|) { |
| 117 |
if (!m|[\(\)/]$name[\(\)/]|) { |
| 118 |
s|\)$|/$name\)|; |
| 119 |
} |
| 120 |
} else { |
| 121 |
$_ = "($_/$name)"; |
| 122 |
} |
| 123 |
$CODE_TO_NAME{$code} = $_; |
| 124 |
} |
| 125 |
} else { |
| 126 |
$CODE_TO_NAME{$code} = $name; |
| 127 |
} |
| 128 |
} |
| 129 |
close(UNICODE_DB); |
| 130 |
|
| 131 |
# Read and parse the manual remapping file. This contains lines |
| 132 |
# of the form: |
| 133 |
|
| 134 |
# |"\u0956>\u0948;" // AI Length Mark -> Devanagari Vowel Sign AI |
| 135 |
|
| 136 |
# The left hand side contains a non-existent full code value. It |
| 137 |
# should be a single value. The right hand side contains one or more |
| 138 |
# real full code values. The idea is that when a mapping from another |
| 139 |
# script ends up at the non-existent code point on the left, the |
| 140 |
# sequence on the right should be substituted. In this example, |
| 141 |
# Devanagari has no AI Length Mark. So, if transliterating from |
| 142 |
# Oriya, then the character 0B56 (Oriya AI Length Mark) will remap to |
| 143 |
# the non-existent 0956, and that remaps to 0948, our chosen |
| 144 |
# Devanagari equivalent. For our purposes, the left hand side should |
| 145 |
# be taken to mean its equivalent point in the InterIndic range. In |
| 146 |
# this example, what it really says is E056>0948 in the |
| 147 |
# InterIndic-Devanagari transliterator. |
| 148 |
|
| 149 |
if ($EXCEPTIONS_FILE) { |
| 150 |
open(EXCEPTIONS_FILE) or die; |
| 151 |
while (<EXCEPTIONS_FILE>) { |
| 152 |
if (m|^\s*\"([^\"]*?)\"|) { |
| 153 |
my $line = $_; |
| 154 |
$_ = $1; |
| 155 |
if (/^(.*)>(.*);$/) { |
| 156 |
my ($rawFrom, $rawTo) = ($1, $2); |
| 157 |
my @from = parseUnicodeEscape($rawFrom); |
| 158 |
my @to = parseUnicodeEscape($rawTo); |
| 159 |
my $from = hexArray(@from); |
| 160 |
# Some entries look like this: |
| 161 |
# |"\u0955>\u0955;" |
| 162 |
# these do nothing; ignore them. |
| 163 |
if (intArraysEqual(\@from, \@to)) { |
| 164 |
#print STDERR "Ignoring NOOP remap of $from\n"; |
| 165 |
} elsif (exists $EXCEPTIONS{$from}) { |
| 166 |
print STDERR "ERROR in $EXCEPTIONS_FILE - Duplicate remap entries for $from\n"; |
| 167 |
} elsif (scalar @from > 1) { |
| 168 |
print STDERR "ERROR in $EXCEPTIONS_FILE - Ignoring multichar remap: ", hexArray(@from), "->", hexArray(@to), "\n"; |
| 169 |
} else { |
| 170 |
# Check this for validity. Full code on the left |
| 171 |
# should NOT exist. Full code seq on the right should. |
| 172 |
if (exists $FULLCODE_TO_NAME{$from[0]}) { |
| 173 |
print STDERR "ERROR in $EXCEPTIONS_FILE - Invalid remap; left side defined: ", hexArray(@from), "->", hexArray(@to), "\n"; |
| 174 |
} elsif (grep(! exists $FULLCODE_TO_NAME{$_}, @to)) { |
| 175 |
print STDERR "ERROR in $EXCEPTIONS_FILE - Invalid remap; right side undefined: ", hexArray(@from), "->", hexArray(@to), "\n"; |
| 176 |
} else { |
| 177 |
$EXCEPTIONS{$from[0]} = \@to; |
| 178 |
} |
| 179 |
} |
| 180 |
} else { die "ERROR in $EXCEPTIONS_FILE - Can't parse \"$_\" in line $line"; } |
| 181 |
} |
| 182 |
} |
| 183 |
close(EXCEPTIONS_FILE); |
| 184 |
print STDERR "$EXCEPTIONS_FILE: Loaded ", scalar keys %EXCEPTIONS, " remappings\n"; |
| 185 |
} |
| 186 |
|
| 187 |
if ($DO_DECOMP_REMAP) { |
| 188 |
# Read the NamesList.txt file. This contains decomposition data. |
| 189 |
# Gather these into %DECOMP, which maps a name to n1.n2..., where n1 |
| 190 |
# etc. are decomposed names. E.g. $DECOMP{'LETTER RRA'} -> 'LETTER |
| 191 |
# RA.SIGN NUKTA'. There may be different mappings in different script |
| 192 |
# blocks (LETTER RRA is mapped differently in Devanagari and Bengali), |
| 193 |
# in which case the name goes into %DECOMP_MISMATCH, and is removed |
| 194 |
# from %DECOMP. |
| 195 |
$NAMES = "NamesList.txt"; |
| 196 |
open(NAMES); |
| 197 |
while (<NAMES>) { |
| 198 |
# Skip to start of DEVANAGARI block |
| 199 |
last if (/^\@\@\s+0900/); |
| 200 |
} |
| 201 |
while (<NAMES>) { |
| 202 |
# Continue until start of SINHALA block |
| 203 |
last if (/^\@\@\s+0D80/); |
| 204 |
if (/^([0-9A-Z]{4})/i) { |
| 205 |
$code = $1; |
| 206 |
} elsif (/^\s+:\s*(.+)/) { |
| 207 |
# We've found a mapping of the form: |
| 208 |
# 0929 DEVANAGARI LETTER NNNA |
| 209 |
# * for transcribing Dravidian alveolar n |
| 210 |
# : 0928 093C |
| 211 |
my $from = $FULLCODE_TO_NAME{hex($code)}; |
| 212 |
my @to = map($FULLCODE_TO_NAME{hex($_)}, split(/\s+/, $1)); |
| 213 |
if (exists $DECOMP{$from}) { |
| 214 |
my $aref = $DECOMP{$from}; |
| 215 |
if (join(".", @$aref) ne join(".", @to)) { |
| 216 |
print STDERR "ERROR: Decomp mismatch for $from\n"; |
| 217 |
print STDERR " : $from = ", join(".", @$aref), "\n"; |
| 218 |
print STDERR " : $from = ", join(".", @to), "\n"; |
| 219 |
$DECOMP_MISMATCH{$from} = 1; |
| 220 |
} |
| 221 |
} else { |
| 222 |
$DECOMP{$from} = \@to; |
| 223 |
} |
| 224 |
} |
| 225 |
} |
| 226 |
close(NAMES); |
| 227 |
# Remove mismatches |
| 228 |
foreach (keys %DECOMP_MISMATCH) { |
| 229 |
delete $DECOMP{$_}; |
| 230 |
} |
| 231 |
if ($DUMP) { |
| 232 |
foreach (keys %DECOMP) { |
| 233 |
print "$_ = ", join(" + ", @{$DECOMP{$_}}), "\n"; |
| 234 |
} |
| 235 |
} |
| 236 |
} |
| 237 |
|
| 238 |
# Count the total number of scripts |
| 239 |
|
| 240 |
$SCRIPT_COUNT = scalar keys %SCRIPT_TO_BASE; |
| 241 |
#print join("\n", sort keys %SCRIPT_TO_BASE), "\n"; |
| 242 |
|
| 243 |
# Dump out the %NAME_CODE_TO_SCRIPTS map. |
| 244 |
|
| 245 |
if ($DUMP) { |
| 246 |
print "\nBY NAME:\n"; |
| 247 |
foreach my $pass ((1, 2)) { |
| 248 |
print "\nBY NAME - SINGLETONS:\n" if ($pass eq 2); |
| 249 |
foreach my $name (sort keys %NAME_CODE_TO_SCRIPTS) { |
| 250 |
if ($pass eq 1) { |
| 251 |
next if (1 >= $NAME_CODE_TO_SCRIPTS{$name}{count}); |
| 252 |
} else { |
| 253 |
next if (1 < $NAME_CODE_TO_SCRIPTS{$name}{count}); |
| 254 |
} |
| 255 |
print "$name:"; |
| 256 |
my $href = $NAME_CODE_TO_SCRIPTS{$name}; |
| 257 |
foreach my $code (sort {$a <=> $b} keys %$href) { |
| 258 |
next if ($code eq 'count'); |
| 259 |
my $aref = $href->{$code}; |
| 260 |
print " ", hex2($code), " (", formatScriptList($aref), ")"; |
| 261 |
} |
| 262 |
print "\n"; |
| 263 |
} |
| 264 |
} |
| 265 |
} |
| 266 |
|
| 267 |
# Create some transliterators, based on the scripts and the %NAME_CODE_TO_SCRIPTS |
| 268 |
# map. Only use %NAME_CODE_TO_SCRIPTS entries with a count of 2 or more, that is, |
| 269 |
# names that occur in two or more scripts. For those scripts where |
| 270 |
# the names occur, map both up to the InterIndic range, and down to |
| 271 |
# the target script. |
| 272 |
|
| 273 |
$INTERINDIC = 0xE000; |
| 274 |
$INTERINDIC_EXTRA = 0xE080; |
| 275 |
$INTERINDIC_EXTRA_NEXT = $INTERINDIC_EXTRA; |
| 276 |
|
| 277 |
# For each script, create a hash. The hash has a key for each |
| 278 |
# code point, either within its block, or in the InterIndic block. |
| 279 |
# the value of the key is the mapping. |
| 280 |
|
| 281 |
# The script hashes are named %DEVANAGARI, etc., and referenced |
| 282 |
# with symbolic refs. |
| 283 |
|
| 284 |
@REMAP = ('s/\bSHORT\s+//i', |
| 285 |
's/\bCANDRA\s+//i', |
| 286 |
's/\bQA$/KA/i', |
| 287 |
's/\bKHHA$/KHA/i', |
| 288 |
's/\bGHHA$/GA/i', |
| 289 |
's/\bZA$/JA/i', |
| 290 |
's/\bFA$/PHA/i', |
| 291 |
's/\bVA$/BA/i', |
| 292 |
's/\bNNNA$/NA/i', |
| 293 |
's/\bRRA$/RA/i', |
| 294 |
's/\bLLLA$/LLA/i', |
| 295 |
's/\bLLLA$/LA/i', |
| 296 |
's/\bLLA$/LA/i', |
| 297 |
's/^A(.) LENGTH MARK$/VOWEL SIGN A$1/i', |
| 298 |
's/CANDRABINDU/BINDI/i', |
| 299 |
's/BINDI/CANDRABINDU/i', |
| 300 |
); |
| 301 |
|
| 302 |
# Do this so we see zero counts: |
| 303 |
foreach my $remap (@REMAP) { $REMAP{$remap} = 0; } |
| 304 |
|
| 305 |
# This loop iterates over the names in the NAME_CODE_TO_SCRIPTS hash. |
| 306 |
# These names are things like "LETTER NNNA". For each name, it then |
| 307 |
# creates script mappings up to the InterIndic area, and back down |
| 308 |
# to the script areas. If a name maps to more than one offset, |
| 309 |
# then it uses the InterIndic extra range. Either way, it picks |
| 310 |
# a single InterIndic point, either an offset point or something in |
| 311 |
# the extra range, and maps up and down from that point. |
| 312 |
foreach my $name (sort keys %NAME_CODE_TO_SCRIPTS) { |
| 313 |
next if (1 >= $NAME_CODE_TO_SCRIPTS{$name}{count}); |
| 314 |
my $href = $NAME_CODE_TO_SCRIPTS{$name}; |
| 315 |
# Count the number of different codes assigned to this name. |
| 316 |
# Usually 1, but 2 for a handful of names. |
| 317 |
my $codeCount = (keys %{$NAME_CODE_TO_SCRIPTS{$name}}) - 1; # less 1: {count} |
| 318 |
# If $codeCount is 1, then map directly up to the $INTERINDIC |
| 319 |
# base. If $codeCount is 2, then map into unused spots starting |
| 320 |
# at $INTERINDIC_EXTRA. |
| 321 |
my $interIndicCode; |
| 322 |
if ($codeCount > 1) { |
| 323 |
# Map into the InterIndic extra range |
| 324 |
$interIndicCode = $INTERINDIC_EXTRA_NEXT++; |
| 325 |
} |
| 326 |
my %seen; |
| 327 |
foreach my $code (sort {$a ne 'count' && $b ne 'count' && $a <=> $b} keys %$href) { |
| 328 |
next if ($code eq 'count'); |
| 329 |
my $aref = $href->{$code}; # Ref to array of scripts |
| 330 |
if ($codeCount == 1) { |
| 331 |
# Map directly |
| 332 |
$interIndicCode = $INTERINDIC + $code; |
| 333 |
} |
| 334 |
# Keep track of the names of the extra InterIndic points |
| 335 |
$INTERINDIC_NAME_TO_FULLCODE{$name} = $interIndicCode; |
| 336 |
|
| 337 |
foreach my $scr (@$aref) { |
| 338 |
$seen{$scr} = 1; |
| 339 |
my $fullCode = $SCRIPT_TO_BASE{$scr} + $code; |
| 340 |
$ {$scr}{$fullCode} = hex4($interIndicCode) . "; // $name"; |
| 341 |
$ {$scr}{$interIndicCode} = hex4($fullCode) . "; // $name"; |
| 342 |
} |
| 343 |
} |
| 344 |
# Now handle InterIndic->Script unmapped points. For each name, |
| 345 |
# some of the scripts will be left out -- will have no mappings |
| 346 |
# to that name. For these scripts, we can either leave them |
| 347 |
# unmapped (so the InterIndic->Local mapping is empty), or |
| 348 |
# try to remap. |
| 349 |
unmappedScript: |
| 350 |
foreach my $scr (keys %SCRIPT_TO_BASE) { |
| 351 |
next if ($seen{$scr}); |
| 352 |
|
| 353 |
if ($DO_HEURISTIC_REMAP) { |
| 354 |
# Try to remap through the known equivalences in our |
| 355 |
# remapping table |
| 356 |
foreach my $remapRE (@REMAP) { |
| 357 |
local $_ = $name; |
| 358 |
if (eval($remapRE)) { |
| 359 |
if (exists $SCRIPT_NAME_TO_CODE{$scr}{$_}) { |
| 360 |
$ {$scr}{$interIndicCode} = |
| 361 |
hex4($SCRIPT_TO_BASE{$scr} + $SCRIPT_NAME_TO_CODE{$scr}{$_}) . |
| 362 |
"; // REMAP: $name -> $_"; |
| 363 |
++$REMAP{$remapRE}; |
| 364 |
next unmappedScript; |
| 365 |
} |
| 366 |
} |
| 367 |
} |
| 368 |
} |
| 369 |
|
| 370 |
# Try to remap through the file. This contains remappings of |
| 371 |
# the form 0991->0993. That is, it contains local remappings |
| 372 |
# that we can substitute and try again with. |
| 373 |
#|GURMUKHI-InterIndic ------------------------------ |
| 374 |
#|// 0A02>; // UNMAPPED INTERNAL: SIGN BINDI |
| 375 |
#|InterIndic-GURMUKHI ------------------------------ |
| 376 |
#|// E001>; // UNMAPPED EXTERNAL: SIGN CANDRABINDU |
| 377 |
#|"\u0A01>\u0A02;" |
| 378 |
# In this example, the remapping tells us that the non-existent |
| 379 |
# character A01 should be considered equivalent to the real |
| 380 |
# character A02. |
| 381 |
# We implement this by adding two mappings; one from |
| 382 |
# the InterIndic equivalent of A01, that is, E001, to A02, |
| 383 |
# and one from A02, which otherwise has no mapping, to E001. |
| 384 |
if ($EXCEPTIONS_FILE && $interIndicCode < $INTERINDIC_EXTRA) { |
| 385 |
# Try to map this InterIndic character back to a the spot |
| 386 |
# it would occupy in this script if it had a mapping. |
| 387 |
my $code = $interIndicCode & 0x7F; |
| 388 |
my $pseudoFullCode = $SCRIPT_TO_BASE{$scr} + $code; |
| 389 |
if (exists $EXCEPTIONS{$pseudoFullCode}) { |
| 390 |
my $fullCodeArray = $EXCEPTIONS{$pseudoFullCode}; |
| 391 |
my $comment; |
| 392 |
foreach my $c (@$fullCodeArray) { |
| 393 |
$comment .= "." if ($comment); |
| 394 |
$comment .= $FULLCODE_TO_NAME{$c}; |
| 395 |
} |
| 396 |
$comment = "; // REMAP ($EXCEPTIONS_FILE): " . |
| 397 |
hex4($pseudoFullCode) . ">" . hexArray(@$fullCodeArray) . " = " . |
| 398 |
$CODE_TO_NAME{$code} . ">" . $comment; |
| 399 |
$ {$scr}{$interIndicCode} = hexArray(@$fullCodeArray) . $comment; |
| 400 |
if (scalar @$fullCodeArray == 1) { |
| 401 |
if (exists $ {$scr}{$fullCodeArray->[0]}) { |
| 402 |
# There's already a proper mapping; no need to fill |
| 403 |
# in reverse |
| 404 |
} else { |
| 405 |
$ {$scr}{$fullCodeArray->[0]} = hex4($interIndicCode) . $comment; |
| 406 |
} |
| 407 |
} |
| 408 |
next unmappedScript; |
| 409 |
} |
| 410 |
} |
| 411 |
|
| 412 |
$SCRIPT_FULLCODE_TO_IS_UNMAPPED{$scr}{$interIndicCode} = 1; |
| 413 |
local $_ = "; // UNMAPPED InterIndic-$scr: $name"; |
| 414 |
if (exists $SCRIPT_CODE_TO_NAME{$scr}{$interIndicCode & 0x7F}) { |
| 415 |
my $fullCode = $SCRIPT_TO_BASE{$scr} + ($interIndicCode & 0x7F); |
| 416 |
$_ .= " (" . hex4($fullCode) . " = " . $FULLCODE_TO_NAME{$fullCode} . ")"; |
| 417 |
} |
| 418 |
$ {$scr}{$interIndicCode} = $_; |
| 419 |
} |
| 420 |
} |
| 421 |
|
| 422 |
# Add in unmapped entries for each script |
| 423 |
foreach my $scr (keys %SCRIPT_TO_BASE) { |
| 424 |
my $base = $SCRIPT_TO_BASE{$scr}; |
| 425 |
unmappedInt: |
| 426 |
foreach my $code (keys %{$SCRIPT_CODE_TO_NAME{$scr}}) { |
| 427 |
my $fullCode = $code + $base; |
| 428 |
next if (exists $ {$scr}{$fullCode}); |
| 429 |
my $name = $SCRIPT_CODE_TO_NAME{$scr}{$code}; |
| 430 |
|
| 431 |
if ($DO_HEURISTIC_REMAP) { |
| 432 |
foreach my $remapRE (@REMAP) { |
| 433 |
local $_ = $name; |
| 434 |
if (eval($remapRE)) { |
| 435 |
if (exists $INTERINDIC_NAME_TO_FULLCODE{$_}) { |
| 436 |
$ {$scr}{$fullCode} = |
| 437 |
hex4($INTERINDIC_NAME_TO_FULLCODE{$_}) . |
| 438 |
"; // REMAP: $name -> $_"; |
| 439 |
++$REMAP{$remapRE}; |
| 440 |
next unmappedInt; |
| 441 |
} |
| 442 |
} |
| 443 |
} |
| 444 |
} |
| 445 |
|
| 446 |
# Now try the decomp table |
| 447 |
if ($DO_DECOMP_REMAP && exists $DECOMP{$name}) { |
| 448 |
my $x; |
| 449 |
my $cmt = "; // DECOMP: $name -> "; |
| 450 |
foreach my $n (@{$DECOMP{$name}}) { |
| 451 |
if (exists $SCRIPT_NAME_TO_CODE{$scr}{$n}) { |
| 452 |
$x .= hex4($SCRIPT_TO_BASE{$scr} + $SCRIPT_NAME_TO_CODE{$scr}{$n}); |
| 453 |
$cmt .= $n . " + "; |
| 454 |
} else { |
| 455 |
$cmt = 0; |
| 456 |
last; |
| 457 |
} |
| 458 |
} |
| 459 |
if ($cmt) { |
| 460 |
$ {$scr}{$fullCode} = $x . $cmt; |
| 461 |
next unmappedInt; |
| 462 |
} |
| 463 |
} |
| 464 |
|
| 465 |
$SCRIPT_FULLCODE_TO_IS_UNMAPPED{$scr}{$fullCode} = 1; |
| 466 |
$ {$scr}{$fullCode} = "; // UNMAPPED $scr-InterIndic: $name"; |
| 467 |
} |
| 468 |
} |
| 469 |
|
| 470 |
# GUR |
| 471 |
# E00B>; // UNMAPPED EXTERNAL: LETTER VOCALIC R "\u0A0B>\u0A30\u0A3F;" |
| 472 |
# E00C>; // UNMAPPED EXTERNAL: LETTER VOCALIC L "\u0A0C>\u0A07;" |
| 473 |
# E00D>; // UNMAPPED EXTERNAL: LETTER CANDRA E "\u0A0D>\u0A10;" |
| 474 |
# E011>; // UNMAPPED EXTERNAL: LETTER CANDRA O "\u0A11>\u0A14;" |
| 475 |
# E037>; // UNMAPPED EXTERNAL: LETTER SSA "\u0A37>\u0A36;" |
| 476 |
# E045>; // UNMAPPED EXTERNAL: VOWEL SIGN CANDRA E "\u0A45>\u0A48;" |
| 477 |
# E049>; // UNMAPPED EXTERNAL: VOWEL SIGN CANDRA O "\u0A49>\u0A4C;" |
| 478 |
# Fix QA too |
| 479 |
|
| 480 |
# Dump out script maps |
| 481 |
foreach my $scr (sort keys %SCRIPT_TO_BASE) { |
| 482 |
## next unless ($scr eq 'TELUGU'); # Debugging |
| 483 |
my @rules; |
| 484 |
my $flag = 1; |
| 485 |
foreach my $fullCode (sort {$a <=> $b} keys %{$scr}) { |
| 486 |
if ($flag && $fullCode >= $INTERINDIC) { |
| 487 |
# We have the complete <scr>-InterIndic rules; dump |
| 488 |
# them out. |
| 489 |
generateRBT($scr, "InterIndic", \@rules, $OUTDIR); |
| 490 |
@rules = (); |
| 491 |
$flag = 0; |
| 492 |
} |
| 493 |
if (exists $SCRIPT_FULLCODE_TO_IS_UNMAPPED{$scr}{$fullCode}) { |
| 494 |
push @rules, "// " . hex4($fullCode) . ">" . $ {$scr}{$fullCode}; |
| 495 |
} else { |
| 496 |
push @rules, hex4($fullCode) . ">" . $ {$scr}{$fullCode}; |
| 497 |
} |
| 498 |
} |
| 499 |
# Now generate the InterIndic-<scr> rules. |
| 500 |
generateRBT("InterIndic", $scr, \@rules, $OUTDIR); |
| 501 |
|
| 502 |
# print "$scr-InterIndic ------------------------------\n"; |
| 503 |
# my $flag = 1; |
| 504 |
# foreach my $fullCode (sort {$a <=> $b} keys %{$scr}) { |
| 505 |
# if ($flag && $fullCode >= $INTERINDIC) { |
| 506 |
# print "InterIndic-$scr ------------------------------\n"; |
| 507 |
# $flag = 0; |
| 508 |
# } |
| 509 |
# if (exists $SCRIPT_FULLCODE_TO_IS_UNMAPPED{$scr}{$fullCode}) { |
| 510 |
# print "// ", hex4($fullCode), ">", $ {$scr}{$fullCode}, "\n"; |
| 511 |
# } else { |
| 512 |
# print hex4($fullCode), ">", $ {$scr}{$fullCode}, "\n"; |
| 513 |
# } |
| 514 |
# } |
| 515 |
} |
| 516 |
|
| 517 |
# List successful remappings |
| 518 |
if ($DO_HEURISTIC_REMAP) { |
| 519 |
foreach my $remap (sort keys %REMAP) { |
| 520 |
print STDERR "REMAP ", $REMAP{$remap}, " x $remap\n"; |
| 521 |
} |
| 522 |
} |
| 523 |
|
| 524 |
#---------------------------------------------------------------------- |
| 525 |
# SUBROUTINES |
| 526 |
|
| 527 |
# Return a listing of an array of scripts |
| 528 |
# Param: array ref |
| 529 |
sub formatScriptList { |
| 530 |
my $aref = shift; |
| 531 |
if ($SCRIPT_COUNT == @$aref) { |
| 532 |
return "all"; |
| 533 |
} elsif (($SCRIPT_COUNT - 3) <= @$aref) { |
| 534 |
my $s = "all but"; |
| 535 |
my %temp; |
| 536 |
foreach (@$aref) { $temp{$_} = 1; } |
| 537 |
foreach (sort keys %SCRIPT_TO_BASE) { |
| 538 |
$s .= " $_" unless exists $temp{$_}; |
| 539 |
} |
| 540 |
return $s; |
| 541 |
} else { |
| 542 |
return join(" ", @$aref); |
| 543 |
} |
| 544 |
} |
| 545 |
|
| 546 |
# Format as %02X hex |
| 547 |
sub hex2 { |
| 548 |
sprintf("%02X", $_[0]); |
| 549 |
} |
| 550 |
|
| 551 |
# Format as %04X hex |
| 552 |
sub hex4 { |
| 553 |
sprintf("\\u%04X", $_[0]); |
| 554 |
} |
| 555 |
|
| 556 |
# Format an array as %04X hex, delimited by "."s |
| 557 |
sub hexArray { |
| 558 |
join("", map { hex4($_); } @_); |
| 559 |
} |
| 560 |
|
| 561 |
# Parse a string of the form "\u0D01" to an array of integers. |
| 562 |
# Must ONLY contain escapes. |
| 563 |
# Return the array. |
| 564 |
sub parseUnicodeEscape { |
| 565 |
local $_ = shift; |
| 566 |
my $orig = $_; |
| 567 |
my @result; |
| 568 |
while (length($_)) { |
| 569 |
if (/^\\u([0-9a-f]{4})(.*)/i) { |
| 570 |
push @result, hex($1); |
| 571 |
$_ = $2; |
| 572 |
} else { |
| 573 |
die "Can't parse Unicode escape $orig\n"; |
| 574 |
} |
| 575 |
} |
| 576 |
if (0 == @result) { |
| 577 |
die "Can't parse Unicode escape $orig\n"; |
| 578 |
} |
| 579 |
@result; |
| 580 |
} |
| 581 |
|
| 582 |
# Return 1 if the two arrays of ints are equal. |
| 583 |
# Param: ref to array of ints |
| 584 |
# Param: ref to array of ints |
| 585 |
sub intArraysEqual { |
| 586 |
my $a = shift; |
| 587 |
my $b = shift; |
| 588 |
if (scalar @$a == scalar @$b) { |
| 589 |
for (my $i=0; $i<@$a; ++$i) { |
| 590 |
if ($a->[$i] != $b->[$i]) { |
| 591 |
return 0; |
| 592 |
} |
| 593 |
} |
| 594 |
return 1; |
| 595 |
} |
| 596 |
return 0; |
| 597 |
} |
| 598 |
|
| 599 |
# Given a rule, possibly with trailing // comment, |
| 600 |
# quote the rule part and add a trailing "+" after |
| 601 |
# it. |
| 602 |
sub quoteRule { |
| 603 |
my $cmt; |
| 604 |
$cmt = $1 if (s|(\s*//.*)||); # isolate trailing // comment |
| 605 |
s/^(.*;)/\"$1\"+/; |
| 606 |
s/$/$cmt/; |
| 607 |
$_; |
| 608 |
} |
| 609 |
|
| 610 |
# Given the name of the source script, name of the target script, |
| 611 |
# and array of rule strings, return a string containing the source |
| 612 |
# for a RuleBasedTransliterator file. |
| 613 |
# Param: source script name |
| 614 |
# Param: target script name |
| 615 |
# Param: ref to array of rules. These rules are unquoted, without |
| 616 |
# concatenators between them, but do have trailing ';' separators. |
| 617 |
# Param: name of output directory |
| 618 |
sub generateRBT { |
| 619 |
# $TOOL - name of generating tool |
| 620 |
# $DATE - date of generation |
| 621 |
# $SCRIPTFROM - name of source script |
| 622 |
# $SCRIPTTO - name of target script |
| 623 |
# $RULES - rules |
| 624 |
my ($source, $target, $rules, $outdir) = @_; |
| 625 |
my $text; |
| 626 |
$outdir =~ s|[/\\]$||; # Delete trailing / or \ |
| 627 |
my $OUT = "$outdir/TransliterationRule_${source}_$target.java"; |
| 628 |
open(RBT_TEMPLATE) or die; |
| 629 |
open(OUT, ">$OUT") or die; |
| 630 |
while (<RBT_TEMPLATE>) { |
| 631 |
while (/\$([A-Za-z0-9]+)/) { |
| 632 |
my $tag = $1; |
| 633 |
my $sub; |
| 634 |
if ($tag eq 'TOOL') { |
| 635 |
$sub = $RBT_GEN_TOOL; |
| 636 |
} elsif ($tag eq 'DATE') { |
| 637 |
$sub = localtime; |
| 638 |
} elsif ($tag eq 'SCRIPTFROM') { |
| 639 |
$sub = $source; |
| 640 |
} elsif ($tag eq 'SCRIPTTO') { |
| 641 |
$sub = $target; |
| 642 |
} elsif ($tag eq 'RULES') { |
| 643 |
# Get any whitespace-only indent off the front of this tag |
| 644 |
my $indent; |
| 645 |
$indent = $1 if (/^(\s+)\$$tag/); |
| 646 |
|
| 647 |
# The rules in the array are not quoted. We need to quote |
| 648 |
# them and add '+' operators between them. We do NOT need |
| 649 |
# to add ';' separators. We DO need to separate trailing |
| 650 |
# // comments and handle them. |
| 651 |
$sub = join("\n$indent", map("eRule, @$rules)) . |
| 652 |
"\n$indent\"\""; |
| 653 |
} else { |
| 654 |
print STDERR "ERROR in $RBT_TEMPLATE: Unknown tag $tag\n"; |
| 655 |
$sub = "[ERROR:Unknown tag \$$tag]"; |
| 656 |
} |
| 657 |
s/\$$tag/$sub/; |
| 658 |
} |
| 659 |
print OUT; |
| 660 |
} |
| 661 |
close(OUT); |
| 662 |
close(RBT_TEMPLATE); |
| 663 |
print STDERR "Written: $OUT\n"; |
| 664 |
} |
| 665 |
|
| 666 |
__END__ |