Debian Codegraph @

tools/misc/src/main/resources/com/ibm/icu/dev/tool/translit/indic.pl

raw
1 #/**
2 # *******************************************************************************
3 # * Copyright (C) 2016 and later: Unicode, Inc. and others. *
4 # * License & terms of use: http://www.unicode.org/copyright.html *
5 # *******************************************************************************
6 # *******************************************************************************
7 # * Copyright (C) 2000-2004, International Business Machines Corporation and *
8 # * others. All Rights Reserved. *
9 # *******************************************************************************
10 # */
11
12 #!perl
13
14 # Usage - $0 <remap file>
15 # e.g. - indic indic.txt
16 # The input file should be a subset of the Unicode data file containing
17 # the blocks of interest.
18 #
19 # The remap file should have lines of the form
20 # "\u0D01>\u0D02;"
21 # including the quotes. These will be interpreted as saying that the
22 # undefined code point U+D01 (derived via mapping from InterIndic)
23 # can be remapped to U+D02.
24 #
25 # The purpose of this script is to process the Indic script data into
26 # a form usable by the IndicTransliterator, that is, the Indic-Indic
27 # transliterator. The transliterator needs two things: A mapping of
28 # the code points in common, and a list of the exceptions.
29
30 # Assume we are located in icu4j/src/com/ibm/tools/translit/.
31 # We want the Unicode DB in icu4j/src/data/unicode/.
32 $UNICODE_DB = "../../../../data/unicode/UnicodeData.txt";
33 $EXCEPTIONS_FILE = shift;
34
35 # Assume we are located in icu4j/src/com/ibm/tools/translit/.
36 # We want to output files to icu4j/src/com/ibm/text/resources/.
37 # Output directory
38 $OUTDIR = "../../text/resources";
39
40 # The template file should contain java code that can be used
41 # to generate RuleBasedTransliterator resource files. The template
42 # should contain the following embedded symbols, which this script
43 # will replace:
44 # $TOOL - name of generating tool
45 # $DATE - date of generation
46 # $SCRIPTFROM - name of source script
47 # $SCRIPTTO - name of target script
48 # $RULES - rules
49 $RBT_TEMPLATE = 'rbtTemplate.txt';
50
51 # Name of this tool in generated RBT files
52 $RBT_GEN_TOOL = 'icu4j/src/com/ibm/tools/translit/indic.pl';
53
54 $DUMP = 0; # If 1, dump out internal data
55
56 $DO_HEURISTIC_REMAP = 0; # If 1, do automatic heuristic remapping
57 $DO_DECOMP_REMAP = 0; # If 1, do decomp remapping
58
59 open(UNICODE_DB);
60 while (<UNICODE_DB>) {
61 next if (m|^0[0-8]|); # Skip up to Devanagari block (0900)
62 last if (m|^0D[8-F]|i); # Bail out after Malayam block (0D00)
63 # 0D39;MALAYALAM LETTER HA;Lo;0;L;;;;;N;;;;;
64 my @data = split(/;/);
65 my $fullCode = hex($data[0]); # e.g., 0x093F
66 my $code = $fullCode & 0x7F; # e.g., 0x3F
67 my ($script, $name) = ($data[1] =~ /(\w+)\s+(.+)/);
68 die "Can't parse $_" unless ($name);
69 # e.g., $code/$script/$name = 3F/MALAYALAM/VOWEL SIGN I
70
71 # Titlecase the script
72 $script = ucfirst(lc($script));
73
74 # Fix a couple inconsistencies in the 3.0 data
75 # REVISIT: Is this okay to do?
76 if ($DO_HEURISTIC_REMAP) {
77 if ($script eq 'Gujarati' && $code >= 5 && $code <= 0x14) {
78 $name =~ s/^VOWEL/LETTER/;
79 }
80 }
81
82 # Keep track of all script names we encounter. We also note the
83 # base of the block.
84 my $base = $fullCode & ~0x7F; # e.g., 0x900;
85 if (exists $SCRIPT_TO_BASE{$script}) {
86 die "Script base mismatch for $script: $base vs. $SCRIPT_TO_BASE{$script}"
87 if ($SCRIPT_TO_BASE{$script} ne $base);
88 } else {
89 $SCRIPT_TO_BASE{$script} = $base;
90 }
91
92 # Build up a mapping by name. For each name, keep a hash keyed by
93 # code point. For each code point, keep an array of script names.
94 # Also keep a total use count for each name.
95 push @{$NAME_CODE_TO_SCRIPTS{$name}{$code}}, $script;
96 ++$NAME_CODE_TO_SCRIPTS{$name}{count};
97
98 # Build a map that looks like this:
99 # $SCRIPT_NAME_TO_CODE{<script>}{<name>} = <code>
100 # or undef if there is no mapping.
101 $SCRIPT_NAME_TO_CODE{$script}{$name} = $code;
102
103 # Build a map that looks like this:
104 $SCRIPT_CODE_TO_NAME{$script}{$code} = $name;
105
106 # And a map from the fullCode point to the name
107 $FULLCODE_TO_NAME{$fullCode} = $name;
108
109 # Map code (0..7F) to name. This is usually a 1-1 mapping, but
110 # is 1-n in a few cases.
111 if (exists $CODE_TO_NAME{$code}) {
112 if ($name ne $CODE_TO_NAME{$code}) {
113 # For multiple names on a code offset, use the format
114 # (a/b), (a/b/c), etc.
115 local $_ = $CODE_TO_NAME{$code};
116 if (m|^\(|) {
117 if (!m|[\(\)/]$name[\(\)/]|) {
118 s|\)$|/$name\)|;
119 }
120 } else {
121 $_ = "($_/$name)";
122 }
123 $CODE_TO_NAME{$code} = $_;
124 }
125 } else {
126 $CODE_TO_NAME{$code} = $name;
127 }
128 }
129 close(UNICODE_DB);
130
131 # Read and parse the manual remapping file. This contains lines
132 # of the form:
133
134 # |"\u0956>\u0948;" // AI Length Mark -> Devanagari Vowel Sign AI
135
136 # The left hand side contains a non-existent full code value. It
137 # should be a single value. The right hand side contains one or more
138 # real full code values. The idea is that when a mapping from another
139 # script ends up at the non-existent code point on the left, the
140 # sequence on the right should be substituted. In this example,
141 # Devanagari has no AI Length Mark. So, if transliterating from
142 # Oriya, then the character 0B56 (Oriya AI Length Mark) will remap to
143 # the non-existent 0956, and that remaps to 0948, our chosen
144 # Devanagari equivalent. For our purposes, the left hand side should
145 # be taken to mean its equivalent point in the InterIndic range. In
146 # this example, what it really says is E056>0948 in the
147 # InterIndic-Devanagari transliterator.
148
149 if ($EXCEPTIONS_FILE) {
150 open(EXCEPTIONS_FILE) or die;
151 while (<EXCEPTIONS_FILE>) {
152 if (m|^\s*\"([^\"]*?)\"|) {
153 my $line = $_;
154 $_ = $1;
155 if (/^(.*)>(.*);$/) {
156 my ($rawFrom, $rawTo) = ($1, $2);
157 my @from = parseUnicodeEscape($rawFrom);
158 my @to = parseUnicodeEscape($rawTo);
159 my $from = hexArray(@from);
160 # Some entries look like this:
161 # |"\u0955>\u0955;"
162 # these do nothing; ignore them.
163 if (intArraysEqual(\@from, \@to)) {
164 #print STDERR "Ignoring NOOP remap of $from\n";
165 } elsif (exists $EXCEPTIONS{$from}) {
166 print STDERR "ERROR in $EXCEPTIONS_FILE - Duplicate remap entries for $from\n";
167 } elsif (scalar @from > 1) {
168 print STDERR "ERROR in $EXCEPTIONS_FILE - Ignoring multichar remap: ", hexArray(@from), "->", hexArray(@to), "\n";
169 } else {
170 # Check this for validity. Full code on the left
171 # should NOT exist. Full code seq on the right should.
172 if (exists $FULLCODE_TO_NAME{$from[0]}) {
173 print STDERR "ERROR in $EXCEPTIONS_FILE - Invalid remap; left side defined: ", hexArray(@from), "->", hexArray(@to), "\n";
174 } elsif (grep(! exists $FULLCODE_TO_NAME{$_}, @to)) {
175 print STDERR "ERROR in $EXCEPTIONS_FILE - Invalid remap; right side undefined: ", hexArray(@from), "->", hexArray(@to), "\n";
176 } else {
177 $EXCEPTIONS{$from[0]} = \@to;
178 }
179 }
180 } else { die "ERROR in $EXCEPTIONS_FILE - Can't parse \"$_\" in line $line"; }
181 }
182 }
183 close(EXCEPTIONS_FILE);
184 print STDERR "$EXCEPTIONS_FILE: Loaded ", scalar keys %EXCEPTIONS, " remappings\n";
185 }
186
187 if ($DO_DECOMP_REMAP) {
188 # Read the NamesList.txt file. This contains decomposition data.
189 # Gather these into %DECOMP, which maps a name to n1.n2..., where n1
190 # etc. are decomposed names. E.g. $DECOMP{'LETTER RRA'} -> 'LETTER
191 # RA.SIGN NUKTA'. There may be different mappings in different script
192 # blocks (LETTER RRA is mapped differently in Devanagari and Bengali),
193 # in which case the name goes into %DECOMP_MISMATCH, and is removed
194 # from %DECOMP.
195 $NAMES = "NamesList.txt";
196 open(NAMES);
197 while (<NAMES>) {
198 # Skip to start of DEVANAGARI block
199 last if (/^\@\@\s+0900/);
200 }
201 while (<NAMES>) {
202 # Continue until start of SINHALA block
203 last if (/^\@\@\s+0D80/);
204 if (/^([0-9A-Z]{4})/i) {
205 $code = $1;
206 } elsif (/^\s+:\s*(.+)/) {
207 # We've found a mapping of the form:
208 # 0929 DEVANAGARI LETTER NNNA
209 # * for transcribing Dravidian alveolar n
210 # : 0928 093C
211 my $from = $FULLCODE_TO_NAME{hex($code)};
212 my @to = map($FULLCODE_TO_NAME{hex($_)}, split(/\s+/, $1));
213 if (exists $DECOMP{$from}) {
214 my $aref = $DECOMP{$from};
215 if (join(".", @$aref) ne join(".", @to)) {
216 print STDERR "ERROR: Decomp mismatch for $from\n";
217 print STDERR " : $from = ", join(".", @$aref), "\n";
218 print STDERR " : $from = ", join(".", @to), "\n";
219 $DECOMP_MISMATCH{$from} = 1;
220 }
221 } else {
222 $DECOMP{$from} = \@to;
223 }
224 }
225 }
226 close(NAMES);
227 # Remove mismatches
228 foreach (keys %DECOMP_MISMATCH) {
229 delete $DECOMP{$_};
230 }
231 if ($DUMP) {
232 foreach (keys %DECOMP) {
233 print "$_ = ", join(" + ", @{$DECOMP{$_}}), "\n";
234 }
235 }
236 }
237
238 # Count the total number of scripts
239
240 $SCRIPT_COUNT = scalar keys %SCRIPT_TO_BASE;
241 #print join("\n", sort keys %SCRIPT_TO_BASE), "\n";
242
243 # Dump out the %NAME_CODE_TO_SCRIPTS map.
244
245 if ($DUMP) {
246 print "\nBY NAME:\n";
247 foreach my $pass ((1, 2)) {
248 print "\nBY NAME - SINGLETONS:\n" if ($pass eq 2);
249 foreach my $name (sort keys %NAME_CODE_TO_SCRIPTS) {
250 if ($pass eq 1) {
251 next if (1 >= $NAME_CODE_TO_SCRIPTS{$name}{count});
252 } else {
253 next if (1 < $NAME_CODE_TO_SCRIPTS{$name}{count});
254 }
255 print "$name:";
256 my $href = $NAME_CODE_TO_SCRIPTS{$name};
257 foreach my $code (sort {$a <=> $b} keys %$href) {
258 next if ($code eq 'count');
259 my $aref = $href->{$code};
260 print " ", hex2($code), " (", formatScriptList($aref), ")";
261 }
262 print "\n";
263 }
264 }
265 }
266
267 # Create some transliterators, based on the scripts and the %NAME_CODE_TO_SCRIPTS
268 # map. Only use %NAME_CODE_TO_SCRIPTS entries with a count of 2 or more, that is,
269 # names that occur in two or more scripts. For those scripts where
270 # the names occur, map both up to the InterIndic range, and down to
271 # the target script.
272
273 $INTERINDIC = 0xE000;
274 $INTERINDIC_EXTRA = 0xE080;
275 $INTERINDIC_EXTRA_NEXT = $INTERINDIC_EXTRA;
276
277 # For each script, create a hash. The hash has a key for each
278 # code point, either within its block, or in the InterIndic block.
279 # the value of the key is the mapping.
280
281 # The script hashes are named %DEVANAGARI, etc., and referenced
282 # with symbolic refs.
283
284 @REMAP = ('s/\bSHORT\s+//i',
285 's/\bCANDRA\s+//i',
286 's/\bQA$/KA/i',
287 's/\bKHHA$/KHA/i',
288 's/\bGHHA$/GA/i',
289 's/\bZA$/JA/i',
290 's/\bFA$/PHA/i',
291 's/\bVA$/BA/i',
292 's/\bNNNA$/NA/i',
293 's/\bRRA$/RA/i',
294 's/\bLLLA$/LLA/i',
295 's/\bLLLA$/LA/i',
296 's/\bLLA$/LA/i',
297 's/^A(.) LENGTH MARK$/VOWEL SIGN A$1/i',
298 's/CANDRABINDU/BINDI/i',
299 's/BINDI/CANDRABINDU/i',
300 );
301
302 # Do this so we see zero counts:
303 foreach my $remap (@REMAP) { $REMAP{$remap} = 0; }
304
305 # This loop iterates over the names in the NAME_CODE_TO_SCRIPTS hash.
306 # These names are things like "LETTER NNNA". For each name, it then
307 # creates script mappings up to the InterIndic area, and back down
308 # to the script areas. If a name maps to more than one offset,
309 # then it uses the InterIndic extra range. Either way, it picks
310 # a single InterIndic point, either an offset point or something in
311 # the extra range, and maps up and down from that point.
312 foreach my $name (sort keys %NAME_CODE_TO_SCRIPTS) {
313 next if (1 >= $NAME_CODE_TO_SCRIPTS{$name}{count});
314 my $href = $NAME_CODE_TO_SCRIPTS{$name};
315 # Count the number of different codes assigned to this name.
316 # Usually 1, but 2 for a handful of names.
317 my $codeCount = (keys %{$NAME_CODE_TO_SCRIPTS{$name}}) - 1; # less 1: {count}
318 # If $codeCount is 1, then map directly up to the $INTERINDIC
319 # base. If $codeCount is 2, then map into unused spots starting
320 # at $INTERINDIC_EXTRA.
321 my $interIndicCode;
322 if ($codeCount > 1) {
323 # Map into the InterIndic extra range
324 $interIndicCode = $INTERINDIC_EXTRA_NEXT++;
325 }
326 my %seen;
327 foreach my $code (sort {$a ne 'count' && $b ne 'count' && $a <=> $b} keys %$href) {
328 next if ($code eq 'count');
329 my $aref = $href->{$code}; # Ref to array of scripts
330 if ($codeCount == 1) {
331 # Map directly
332 $interIndicCode = $INTERINDIC + $code;
333 }
334 # Keep track of the names of the extra InterIndic points
335 $INTERINDIC_NAME_TO_FULLCODE{$name} = $interIndicCode;
336
337 foreach my $scr (@$aref) {
338 $seen{$scr} = 1;
339 my $fullCode = $SCRIPT_TO_BASE{$scr} + $code;
340 $ {$scr}{$fullCode} = hex4($interIndicCode) . "; // $name";
341 $ {$scr}{$interIndicCode} = hex4($fullCode) . "; // $name";
342 }
343 }
344 # Now handle InterIndic->Script unmapped points. For each name,
345 # some of the scripts will be left out -- will have no mappings
346 # to that name. For these scripts, we can either leave them
347 # unmapped (so the InterIndic->Local mapping is empty), or
348 # try to remap.
349 unmappedScript:
350 foreach my $scr (keys %SCRIPT_TO_BASE) {
351 next if ($seen{$scr});
352
353 if ($DO_HEURISTIC_REMAP) {
354 # Try to remap through the known equivalences in our
355 # remapping table
356 foreach my $remapRE (@REMAP) {
357 local $_ = $name;
358 if (eval($remapRE)) {
359 if (exists $SCRIPT_NAME_TO_CODE{$scr}{$_}) {
360 $ {$scr}{$interIndicCode} =
361 hex4($SCRIPT_TO_BASE{$scr} + $SCRIPT_NAME_TO_CODE{$scr}{$_}) .
362 "; // REMAP: $name -> $_";
363 ++$REMAP{$remapRE};
364 next unmappedScript;
365 }
366 }
367 }
368 }
369
370 # Try to remap through the file. This contains remappings of
371 # the form 0991->0993. That is, it contains local remappings
372 # that we can substitute and try again with.
373 #|GURMUKHI-InterIndic ------------------------------
374 #|// 0A02>; // UNMAPPED INTERNAL: SIGN BINDI
375 #|InterIndic-GURMUKHI ------------------------------
376 #|// E001>; // UNMAPPED EXTERNAL: SIGN CANDRABINDU
377 #|"\u0A01>\u0A02;"
378 # In this example, the remapping tells us that the non-existent
379 # character A01 should be considered equivalent to the real
380 # character A02.
381 # We implement this by adding two mappings; one from
382 # the InterIndic equivalent of A01, that is, E001, to A02,
383 # and one from A02, which otherwise has no mapping, to E001.
384 if ($EXCEPTIONS_FILE && $interIndicCode < $INTERINDIC_EXTRA) {
385 # Try to map this InterIndic character back to a the spot
386 # it would occupy in this script if it had a mapping.
387 my $code = $interIndicCode & 0x7F;
388 my $pseudoFullCode = $SCRIPT_TO_BASE{$scr} + $code;
389 if (exists $EXCEPTIONS{$pseudoFullCode}) {
390 my $fullCodeArray = $EXCEPTIONS{$pseudoFullCode};
391 my $comment;
392 foreach my $c (@$fullCodeArray) {
393 $comment .= "." if ($comment);
394 $comment .= $FULLCODE_TO_NAME{$c};
395 }
396 $comment = "; // REMAP ($EXCEPTIONS_FILE): " .
397 hex4($pseudoFullCode) . ">" . hexArray(@$fullCodeArray) . " = " .
398 $CODE_TO_NAME{$code} . ">" . $comment;
399 $ {$scr}{$interIndicCode} = hexArray(@$fullCodeArray) . $comment;
400 if (scalar @$fullCodeArray == 1) {
401 if (exists $ {$scr}{$fullCodeArray->[0]}) {
402 # There's already a proper mapping; no need to fill
403 # in reverse
404 } else {
405 $ {$scr}{$fullCodeArray->[0]} = hex4($interIndicCode) . $comment;
406 }
407 }
408 next unmappedScript;
409 }
410 }
411
412 $SCRIPT_FULLCODE_TO_IS_UNMAPPED{$scr}{$interIndicCode} = 1;
413 local $_ = "; // UNMAPPED InterIndic-$scr: $name";
414 if (exists $SCRIPT_CODE_TO_NAME{$scr}{$interIndicCode & 0x7F}) {
415 my $fullCode = $SCRIPT_TO_BASE{$scr} + ($interIndicCode & 0x7F);
416 $_ .= " (" . hex4($fullCode) . " = " . $FULLCODE_TO_NAME{$fullCode} . ")";
417 }
418 $ {$scr}{$interIndicCode} = $_;
419 }
420 }
421
422 # Add in unmapped entries for each script
423 foreach my $scr (keys %SCRIPT_TO_BASE) {
424 my $base = $SCRIPT_TO_BASE{$scr};
425 unmappedInt:
426 foreach my $code (keys %{$SCRIPT_CODE_TO_NAME{$scr}}) {
427 my $fullCode = $code + $base;
428 next if (exists $ {$scr}{$fullCode});
429 my $name = $SCRIPT_CODE_TO_NAME{$scr}{$code};
430
431 if ($DO_HEURISTIC_REMAP) {
432 foreach my $remapRE (@REMAP) {
433 local $_ = $name;
434 if (eval($remapRE)) {
435 if (exists $INTERINDIC_NAME_TO_FULLCODE{$_}) {
436 $ {$scr}{$fullCode} =
437 hex4($INTERINDIC_NAME_TO_FULLCODE{$_}) .
438 "; // REMAP: $name -> $_";
439 ++$REMAP{$remapRE};
440 next unmappedInt;
441 }
442 }
443 }
444 }
445
446 # Now try the decomp table
447 if ($DO_DECOMP_REMAP && exists $DECOMP{$name}) {
448 my $x;
449 my $cmt = "; // DECOMP: $name -> ";
450 foreach my $n (@{$DECOMP{$name}}) {
451 if (exists $SCRIPT_NAME_TO_CODE{$scr}{$n}) {
452 $x .= hex4($SCRIPT_TO_BASE{$scr} + $SCRIPT_NAME_TO_CODE{$scr}{$n});
453 $cmt .= $n . " + ";
454 } else {
455 $cmt = 0;
456 last;
457 }
458 }
459 if ($cmt) {
460 $ {$scr}{$fullCode} = $x . $cmt;
461 next unmappedInt;
462 }
463 }
464
465 $SCRIPT_FULLCODE_TO_IS_UNMAPPED{$scr}{$fullCode} = 1;
466 $ {$scr}{$fullCode} = "; // UNMAPPED $scr-InterIndic: $name";
467 }
468 }
469
470 # GUR
471 # E00B>; // UNMAPPED EXTERNAL: LETTER VOCALIC R "\u0A0B>\u0A30\u0A3F;"
472 # E00C>; // UNMAPPED EXTERNAL: LETTER VOCALIC L "\u0A0C>\u0A07;"
473 # E00D>; // UNMAPPED EXTERNAL: LETTER CANDRA E "\u0A0D>\u0A10;"
474 # E011>; // UNMAPPED EXTERNAL: LETTER CANDRA O "\u0A11>\u0A14;"
475 # E037>; // UNMAPPED EXTERNAL: LETTER SSA "\u0A37>\u0A36;"
476 # E045>; // UNMAPPED EXTERNAL: VOWEL SIGN CANDRA E "\u0A45>\u0A48;"
477 # E049>; // UNMAPPED EXTERNAL: VOWEL SIGN CANDRA O "\u0A49>\u0A4C;"
478 # Fix QA too
479
480 # Dump out script maps
481 foreach my $scr (sort keys %SCRIPT_TO_BASE) {
482 ## next unless ($scr eq 'TELUGU'); # Debugging
483 my @rules;
484 my $flag = 1;
485 foreach my $fullCode (sort {$a <=> $b} keys %{$scr}) {
486 if ($flag && $fullCode >= $INTERINDIC) {
487 # We have the complete <scr>-InterIndic rules; dump
488 # them out.
489 generateRBT($scr, "InterIndic", \@rules, $OUTDIR);
490 @rules = ();
491 $flag = 0;
492 }
493 if (exists $SCRIPT_FULLCODE_TO_IS_UNMAPPED{$scr}{$fullCode}) {
494 push @rules, "// " . hex4($fullCode) . ">" . $ {$scr}{$fullCode};
495 } else {
496 push @rules, hex4($fullCode) . ">" . $ {$scr}{$fullCode};
497 }
498 }
499 # Now generate the InterIndic-<scr> rules.
500 generateRBT("InterIndic", $scr, \@rules, $OUTDIR);
501
502 # print "$scr-InterIndic ------------------------------\n";
503 # my $flag = 1;
504 # foreach my $fullCode (sort {$a <=> $b} keys %{$scr}) {
505 # if ($flag && $fullCode >= $INTERINDIC) {
506 # print "InterIndic-$scr ------------------------------\n";
507 # $flag = 0;
508 # }
509 # if (exists $SCRIPT_FULLCODE_TO_IS_UNMAPPED{$scr}{$fullCode}) {
510 # print "// ", hex4($fullCode), ">", $ {$scr}{$fullCode}, "\n";
511 # } else {
512 # print hex4($fullCode), ">", $ {$scr}{$fullCode}, "\n";
513 # }
514 # }
515 }
516
517 # List successful remappings
518 if ($DO_HEURISTIC_REMAP) {
519 foreach my $remap (sort keys %REMAP) {
520 print STDERR "REMAP ", $REMAP{$remap}, " x $remap\n";
521 }
522 }
523
524 #----------------------------------------------------------------------
525 # SUBROUTINES
526
527 # Return a listing of an array of scripts
528 # Param: array ref
529 sub formatScriptList {
530 my $aref = shift;
531 if ($SCRIPT_COUNT == @$aref) {
532 return "all";
533 } elsif (($SCRIPT_COUNT - 3) <= @$aref) {
534 my $s = "all but";
535 my %temp;
536 foreach (@$aref) { $temp{$_} = 1; }
537 foreach (sort keys %SCRIPT_TO_BASE) {
538 $s .= " $_" unless exists $temp{$_};
539 }
540 return $s;
541 } else {
542 return join(" ", @$aref);
543 }
544 }
545
546 # Format as %02X hex
547 sub hex2 {
548 sprintf("%02X", $_[0]);
549 }
550
551 # Format as %04X hex
552 sub hex4 {
553 sprintf("\\u%04X", $_[0]);
554 }
555
556 # Format an array as %04X hex, delimited by "."s
557 sub hexArray {
558 join("", map { hex4($_); } @_);
559 }
560
561 # Parse a string of the form "\u0D01" to an array of integers.
562 # Must ONLY contain escapes.
563 # Return the array.
564 sub parseUnicodeEscape {
565 local $_ = shift;
566 my $orig = $_;
567 my @result;
568 while (length($_)) {
569 if (/^\\u([0-9a-f]{4})(.*)/i) {
570 push @result, hex($1);
571 $_ = $2;
572 } else {
573 die "Can't parse Unicode escape $orig\n";
574 }
575 }
576 if (0 == @result) {
577 die "Can't parse Unicode escape $orig\n";
578 }
579 @result;
580 }
581
582 # Return 1 if the two arrays of ints are equal.
583 # Param: ref to array of ints
584 # Param: ref to array of ints
585 sub intArraysEqual {
586 my $a = shift;
587 my $b = shift;
588 if (scalar @$a == scalar @$b) {
589 for (my $i=0; $i<@$a; ++$i) {
590 if ($a->[$i] != $b->[$i]) {
591 return 0;
592 }
593 }
594 return 1;
595 }
596 return 0;
597 }
598
599 # Given a rule, possibly with trailing // comment,
600 # quote the rule part and add a trailing "+" after
601 # it.
602 sub quoteRule {
603 my $cmt;
604 $cmt = $1 if (s|(\s*//.*)||); # isolate trailing // comment
605 s/^(.*;)/\"$1\"+/;
606 s/$/$cmt/;
607 $_;
608 }
609
610 # Given the name of the source script, name of the target script,
611 # and array of rule strings, return a string containing the source
612 # for a RuleBasedTransliterator file.
613 # Param: source script name
614 # Param: target script name
615 # Param: ref to array of rules. These rules are unquoted, without
616 # concatenators between them, but do have trailing ';' separators.
617 # Param: name of output directory
618 sub generateRBT {
619 # $TOOL - name of generating tool
620 # $DATE - date of generation
621 # $SCRIPTFROM - name of source script
622 # $SCRIPTTO - name of target script
623 # $RULES - rules
624 my ($source, $target, $rules, $outdir) = @_;
625 my $text;
626 $outdir =~ s|[/\\]$||; # Delete trailing / or \
627 my $OUT = "$outdir/TransliterationRule_${source}_$target.java";
628 open(RBT_TEMPLATE) or die;
629 open(OUT, ">$OUT") or die;
630 while (<RBT_TEMPLATE>) {
631 while (/\$([A-Za-z0-9]+)/) {
632 my $tag = $1;
633 my $sub;
634 if ($tag eq 'TOOL') {
635 $sub = $RBT_GEN_TOOL;
636 } elsif ($tag eq 'DATE') {
637 $sub = localtime;
638 } elsif ($tag eq 'SCRIPTFROM') {
639 $sub = $source;
640 } elsif ($tag eq 'SCRIPTTO') {
641 $sub = $target;
642 } elsif ($tag eq 'RULES') {
643 # Get any whitespace-only indent off the front of this tag
644 my $indent;
645 $indent = $1 if (/^(\s+)\$$tag/);
646
647 # The rules in the array are not quoted. We need to quote
648 # them and add '+' operators between them. We do NOT need
649 # to add ';' separators. We DO need to separate trailing
650 # // comments and handle them.
651 $sub = join("\n$indent", map(&quoteRule, @$rules)) .
652 "\n$indent\"\"";
653 } else {
654 print STDERR "ERROR in $RBT_TEMPLATE: Unknown tag $tag\n";
655 $sub = "[ERROR:Unknown tag \$$tag]";
656 }
657 s/\$$tag/$sub/;
658 }
659 print OUT;
660 }
661 close(OUT);
662 close(RBT_TEMPLATE);
663 print STDERR "Written: $OUT\n";
664 }
665
666 __END__