Debian Codegraph @

tools/misc/src/main/resources/com/ibm/icu/dev/tool/translit/dumpICUrules.bat

raw
1 # ************************************************************************
2 # Copyright (C) 2016 and later: Unicode, Inc. and others.
3 # License & terms of use: http://www.unicode.org/copyright.html
4 # ************************************************************************
5 @rem = '--*-Perl-*--
6 @echo off
7 if "%OS%" == "Windows_NT" goto WinNT
8 perl -x -S "%0" %1 %2 %3 %4 %5 %6 %7 %8 %9
9 goto endofperl
10 :WinNT
11 perl -x -S "%0" %*
12 if NOT "%COMSPEC%" == "%SystemRoot%\system32\cmd.exe" goto endofperl
13 if %errorlevel% == 9009 echo You do not have Perl in your PATH.
14 goto endofperl
15 @rem ';
16 #!perl
17 #line 14
18 # ************************************************************************
19 # Copyright (C) 2000-2004, International Business Machines Corporation and
20 # others. All Rights Reserved.
21 # ************************************************************************
22
23 # This perl script creates ICU transliterator data files, that live
24 # in icu/data, from ICU4J UTF8 transliterator data files, in
25 # icu4j/src/com/ibm/icu/impl/data/.
26 #
27 # The transformation that is done is very minimal. The script assumes
28 # that the input files use only # comments
29 # and that they follow a rigid format.
30 #
31 # The output files are named according to ICU conventions (see NAME_MAP
32 # below) and created in the current directory. They should be manually
33 # checked and then copied into the icu/data/tranlit directory.
34 # An ICU build must
35 # then be initiated, and the standard suite of ICU transliterator tests
36 # should be run after that.
37 #
38 # Alan Liu 5/19/00 2/27/01
39
40 use Getopt::Long;
41 use strict;
42
43 use vars qw(%USED_FILES);
44
45 my $DIR = "../../../impl/data";
46 my $ID = '';
47
48 GetOptions('dir=s' => \$DIR,
49 'id=s' => \$ID,
50 '<>' => \&usage) || die;
51
52 usage() if (@ARGV);
53
54 my $ID =~ s/-/_/;
55 if (! -d $DIR) {
56 print STDERR "$DIR is not a directory\n";
57 usage();
58 }
59
60 sub usage {
61 my $me = $0;
62 $me =~ s|.+[/\\]||;
63 print "Usage: $me [-dir <dir>] [-id <id>]\n";
64 print " --dir <dir> Specify the directory containing the\n";
65 print " Transliterator_*.txt files\n";
66 print " --id <id> Specify a single ID to transform, e.g.\n";
67 print " Fullwidth-Halfwidth\n";
68 exit(1);
69 }
70
71 my $JAVA_ONLY = '-';
72
73 my $OUTDIR = "icu4c";
74 mkdir($OUTDIR,0777);
75
76 # Mapping from Java file names to ICU file names
77 # Constraints on ICU4C file name: icudt20b_
78 # |--9 (prefix)---|---18(name with distinguisher,e.g. "t_" )----|
79 # --4 ("."+extn)--| = 31 characters total.
80 # That is, must have length(%NAME_MAP{x}) <= 16
81
82 my $MAX_ICU4C_FILENAME_LEN = 18;
83
84 # -- HISTORY -- If not marked, then pre 2.2.
85 # All InterIndic are pre 2.2.
86 # Any_Accents
87 # Any_Publishing
88 # Arabic_Latin * 2.2
89 # Cyrillic_Latin
90 # Fullwidth_Halfwidth
91 # Greek_Latin
92 # Greek_Latin_UNGEGN * 2.2 (moved from el.txt)
93 # Han_Latin * 2.2
94 # Han_Latin_Definition * 2.2
95 # Han_Latin_EDICT * 2.2 J only
96 # Hebrew_Latin * 2.2
97 # Hiragana_Katakana
98 # Hiragana_Latin
99 # Latin_Jamo
100 # Latin_Katakana
101 # ThaiLogical_Latin * 2.2 J only
102 # Thai_ThaiLogical * 2.2 J only
103 # Thai_ThaiSemi * 2.2 J only
104
105 my %NAME_MAP = (
106 # An ICU name of "" means the ICU name == the ID
107
108 # We filter names based on what is in use in the index file.
109
110 # Flag a rule as JAVA_ONLY if it exists and we use it in Java,
111 # but we don't use it in C.
112
113 # Use official script abbreviations where possible.
114
115 # |..............| |..............|
116 # 1234567890123456 1234567890123456
117 Any_Accents => "",
118 Any_Publishing => "",
119 Cyrillic_Latin => "Cyrl_Latn",
120 Fullwidth_Halfwidth => "FWidth_HWidth",
121 Greek_Latin => "Grek_Latn",
122 Hiragana_Katakana => "Hira_Kana",
123 Hiragana_Latin => "Hira_Latn",
124 Latin_Jamo => "Latn_Jamo",
125 Latin_Katakana => "Latn_Kana",
126
127 Arabic_Latin => "Arab_Latn",
128 Greek_Latin_UNGEGN => "Grek_Latn_UNGEGN",
129 Han_Latin => "Hani_Latn",
130 Han_Latin_Definition => "Hani_Latn_Def",
131 Han_Latin_EDICT => "Hani_Latn_EDICT",
132 Hebrew_Latin => "Hebr_Latn",
133 ThaiLogical_Latin => $JAVA_ONLY, # "ThaiLog_Latn",
134 Thai_ThaiLogical => $JAVA_ONLY, # "Thai_ThaiLog",
135 Thai_ThaiSemi => $JAVA_ONLY, # "Thai_ThaiSemi",
136
137 InterIndic_Bengali => "InterIndic_Beng",
138 InterIndic_Devanagari => "InterIndic_Deva",
139 InterIndic_Gujarati => "InterIndic_Gujr",
140 InterIndic_Gurmukhi => "InterIndic_Guru",
141 InterIndic_Kannada => "InterIndic_Knda",
142 InterIndic_Latin => "InterIndic_Latn",
143 InterIndic_Malayalam => "InterIndic_Mlym",
144 InterIndic_Oriya => "InterIndic_Orya",
145 InterIndic_Tamil => "InterIndic_Taml",
146 InterIndic_Telugu => "InterIndic_Telu",
147
148 Bengali_InterIndic => "Beng_InterIndic",
149 Devanagari_InterIndic => "Deva_InterIndic",
150 Gujarati_InterIndic => "Gujr_InterIndic",
151 Gurmukhi_InterIndic => "Guru_InterIndic",
152 Kannada_InterIndic => "Knda_InterIndic",
153 Latin_InterIndic => "Latn_InterIndic",
154 Malayalam_InterIndic => "Mlym_InterIndic",
155 Oriya_InterIndic => "Orya_InterIndic",
156 Tamil_InterIndic => "Taml_InterIndic",
157 Telugu_InterIndic => "Telu_InterIndic",
158
159 Han_Pinyin => $JAVA_ONLY,
160 Kanji_English => $JAVA_ONLY,
161 Kanji_OnRomaji => $JAVA_ONLY,
162
163 Latin_NumericPinyin => "Latn_NPinyn",
164 Tone_Digit => "Tone_Digit",
165 Han_Spacedhan => "Hani_SpHan",
166 );
167
168 my ($x,$x,$x,$x,$x,$THIS_YEAR) = localtime();
169 $THIS_YEAR += 1900;
170
171 # Header blocks of text written at start of ICU output files
172 my $HEADER1 = <<END;
173 //--------------------------------------------------------------------
174 // Copyright (c) 1999-$THIS_YEAR, International Business Machines
175 // Corporation and others. All Rights Reserved.
176 //--------------------------------------------------------------------
177 // THIS IS A MACHINE-GENERATED FILE
178 END
179 my $HEADER2 = <<END;
180 //--------------------------------------------------------------------
181 END
182
183 my $TOOL = $0;
184
185 # Convert the index first; this tells us which rule files are in use.
186 convertIndex();
187
188 # print "In use:\n", join("\n", sort keys(%USED_FILES)), "\n";
189
190 # Iterate over all Java RBT rule files
191 foreach (<$DIR/Transliterator_*.txt>) {
192 next if (/~$/); # Ignore emacs backups
193 next if (/_index\.txt$/); # The index file was processed above
194 # Select either the command-line arg, if there was one, or
195 # any files mentioned in the index.
196 my $leaf = $_;
197 $leaf =~ s|.+[/\\]||;
198 if (($ID && $leaf =~ /$ID/) || exists $USED_FILES{$leaf}) {
199 my ($out, $id) = convertFileName($_);
200 if ($out) {
201 if ($out eq $JAVA_ONLY) {
202 print STDERR "*** $id skipped: Java only ***\n";
203 next;
204 }
205 file($id, $_, $out);
206 }
207 } elsif (!$ID) {
208 print "*** $leaf skipped: not in use ***\n";
209 }
210 }
211
212 ######################################################################
213 # Convert a Java file name to C
214 # Param: Java file name of the form m|Transliterator_(.+)\.utf8\.txt$|
215 # Return: A C file name (e.g., ldevan.txt) or the empty string,
216 # if there is no mapping, or $JAVA_ONLY if the given file isn't
217 # intended to be incorporated into C.
218 sub convertFileName {
219 local $_ = shift;
220 my $id;
221 if (m|Transliterator_(.+)\.utf8\.txt$| ||
222 m|Transliterator_(.+)\.txt$|) {
223 $id = $1;
224 } else { die "Can't parse Java file name $_"; }
225 if (!exists $NAME_MAP{$id}) {
226 print STDERR "ERROR: $id not in map; please update $0\n";
227 return '';
228 }
229 my $out = $NAME_MAP{$id};
230 if ($out eq '') {
231 $out = $id;
232 }
233 if ($out ne $JAVA_ONLY) {
234 $out = 't_' . $out;
235 }
236 if (length($out) > $MAX_ICU4C_FILENAME_LEN) {
237 print STDERR "ERROR: ICU4C file name \"$out\" too long; please update $0\n";
238 return '';
239 }
240 return ($out, $id);
241 }
242
243 ######################################################################
244 # Convert the index file from Java to C format
245 sub convertIndex {
246 my $JAVA_INDEX = "Transliterator_index.txt";
247 my $C_INDEX = "translit_index.txt";
248 open(JAVA_INDEX, "$DIR/$JAVA_INDEX") or die;
249 open(C_INDEX, ">$OUTDIR/$C_INDEX") or die;
250
251 header(\*C_INDEX, $JAVA_INDEX);
252
253 print C_INDEX <<END;
254 //--------------------------------------------------------------------
255 // N.B.: This file has been generated mechanically from the
256 // corresponding ICU4J file, which is the primary file that receives
257 // primary updates. The colon-delimited fields have been split into
258 // separate strings. For 'file' and 'internal' lines, the encoding
259 // field has been deleted, since the encoding is processed at build
260 // time in ICU4C. Certain large rule sets not intended for general
261 // use have been commented out with the notation "Java only".
262 //--------------------------------------------------------------------
263
264 translit_index {
265 RuleBasedTransliteratorIDs {
266 END
267
268 while (<JAVA_INDEX>) {
269 # ignore CVS keyword substitutions
270 next if /\$(Source|Revision|Date)/;
271
272 # we have printed out the copyright info ... ignore one in Java version
273 next if /Copyright/ ;
274 next if /Corporation/;
275
276 # Comments; change # to //
277 if (s|^(\s*)\#|$1//|) {
278 print C_INDEX;
279 next;
280 }
281 # Blank lines
282 if (!/\S/) {
283 print C_INDEX;
284 next;
285 }
286 # Content lines
287 chomp;
288 my $prefix = '';
289 #replace \p with \\p
290 $_=~ s/\\p/\\\\p/g;
291 my @a = split(':', $_);
292 if ($a[1] eq 'file' || $a[1] eq 'internal') {
293 # Convert the file name
294 my $id;
295 # Record file names in use
296 $USED_FILES{$a[2]} = 1;
297 ($a[2], $id) = convertFileName($a[2]);
298 if ($a[2] eq $JAVA_ONLY) {
299 $prefix = '// Java only: ';
300 }
301
302 # Delete the encoding field
303 splice(@a, 3, 1);
304 } elsif ($a[1] eq 'alias') {
305 # Pad out with extra blank fields to make the
306 # 2-d array square
307 push @a, "";
308 } else {
309 die "Can't parse $_";
310 }
311 print C_INDEX
312 $prefix, "{ ",
313 join(", ", map("\"$_\"", @a)),
314 " },\n";
315 }
316
317 print C_INDEX <<END;
318 }
319 }
320 END
321
322 close(C_INDEX);
323 close(JAVA_INDEX);
324 print STDERR "$JAVA_INDEX -> $C_INDEX\n";
325 }
326
327 ######################################################################
328 # Output a header
329 # Param: Filehandle
330 sub header {
331 my $out = shift;
332 my $in = shift;
333 print $out $HEADER1;
334 print $out "// Tool: $TOOL\n// Source: $in\n";
335 print $out "// Date: ", scalar localtime, "\n";
336 print $out $HEADER2;
337 print $out "\n";
338 }
339
340 ######################################################################
341 # Process one file
342 # Param: ID, e.g. Fullwidth-Halfwidth
343 # Param: Java input file name, e.g.
344 # f:/icu4j/src/com/ibm/text/resources/Transliterator_Fullwidth_Halfwidth.utf8.txt
345 # Param: ICU output file name, e.g. fullhalf
346 sub file {
347 my $id = shift;
348 my $IN = shift;
349 my $out = shift;
350
351 my $OUT = "$out.txt";
352
353 # Show input size. Show output size later -- useful for quick sanity check.
354 print "$id (", -s $IN, ") -> $OUT (";
355
356 # Open file, write UTF8 marker, close it, and reopen in text mode
357 open(OUT, ">$OUTDIR/$OUT") or die;
358 binmode OUT; # Must do this so we can write our UTF8 marker
359 print OUT pack("C3", 0xEF, 0xBB, 0xBF); # Write UTF8 marker
360 close(OUT);
361
362 open(OUT, ">>$OUTDIR/$OUT") or die;
363 print OUT " // -*- Coding: utf-8; -*-\n";
364
365 header(\*OUT, $IN);
366 print OUT "// $id\n";
367 print OUT "\n";
368 print OUT "$out {\n";
369 print OUT " Rule {\n";
370
371 open(IN, $IN) or die;
372 binmode IN; # IN is a UTF8 file
373
374 my $first = 1;
375 my $BOM = pack("C3", 239, 187, 191); # a UTF8 byte order mark
376
377 # Process each line by changing # comments to // comments
378 # and taking other text and enclosing it in double quotes
379 while (<IN>) {
380 my $raw = $_;
381 # ignore CVS keyword substitutions
382 next if /\$(Source|Revision|Date)/;
383
384 # we have printed out the copyright info ... ignore one in Java version
385 next if /Copyright/ ;
386 next if /Corporation/;
387
388 # Look for and delete BOM
389 if ($first) {
390 s/^$BOM//;
391 $first = 0;
392 }
393
394 # Clean the eol junk up
395 s/[\x0D\x0A]+$//;
396
397 # If there is a trailing backslash, then delete it -- we don't
398 # need line continuation in C, since adjacent strings are
399 # concatenated. Count trailing backslashes; if they are odd,
400 # one is trailing.
401 if (m|(\\+)$|) {
402 if ((length($1) % 2) == 1) {
403 s|\\$||;
404 }
405 }
406
407 # Transform escaped characters
408 hideEscapes();
409
410 if (/^(\s*)(\#.*)$/) {
411 # Comment-only line
412 my ($white, $cmt) = ($1, $2);
413 $cmt =~ s|\#|//|;
414 $_ = $white . $cmt;
415
416 } elsif (!/\S/) {
417 # Blank line -- leave as-is
418
419 } else {
420 # Remove single-quoted matter
421 my @quotes;
422 my $nquotes = 0;
423 my $x = $_;
424 while (s/^([^\']*)(\'[^\']*\')/$1<<x$nquotes>>/) {
425 push @quotes, $2;
426 ++$nquotes;
427 }
428
429 # Extract comment
430 my $cmt = '';
431 if (s|\#(.*)||) {
432 $cmt = '//' . $1;
433 }
434
435 # Add quotes
436 s|^(\s*)(\S.*?)(\s*)$|$1\"$2\"$3|;
437
438 # Restore single-quoted matter
439 for (my $i=0; $i<$nquotes; ++$i) {
440 s|<<x$i>>|$quotes[$i]|;
441 }
442
443 # Restore comment
444 $_ .= $cmt;
445 }
446
447 # Restore escaped characters
448 restoreEscapes();
449
450 print OUT $_, "\n";
451 }
452
453 # Finish up
454 close(IN);
455 print OUT " }\n";
456 print OUT "}\n";
457 close(OUT);
458
459 # Write output file size for sanity check
460 print -s "$OUTDIR/$OUT", ")\n";
461 }
462
463 ######################################################################
464 sub hideEscapes {
465 # Transform escaped characters
466 s|\\\\|<<bs>>|g; # DO THIS FIRST Transform backslashes
467 s|\\u([a-zA-Z0-9]{4})|<<u$1>>|g; # Transform Unicode escapes
468 s|\\\"|<<dq>>|g; # Transform backslash double quote
469 s|\\\'|<<sq>>|g; # Transform backslash single quote
470 s|\\\#|<<lb>>|g; # Transform backslash pound
471 s|\\(.)|<<q$1>>|g; # Transform backslash escapes
472 }
473
474 ######################################################################
475 sub restoreEscapes {
476 # Restore escaped characters
477 s|<<bs>>|\\\\|g;
478 s|<<dq>>|\\\\\\\"|g;
479 s|<<sq>>|\\\\\\\'|g;
480 s|<<lb>>|\\\\\\\#|g;
481 s|<<q(.)>>|\\\\\\$1|g;
482 s|<<u0000>>|\\\\u0000|g; # Double escape U+0000
483 s|<<u(....)>>|\\u$1|g;
484 }
485
486 __END__
487 :endofperl