| 1 |
# ************************************************************************ |
| 2 |
# Copyright (C) 2016 and later: Unicode, Inc. and others. |
| 3 |
# License & terms of use: http://www.unicode.org/copyright.html |
| 4 |
# ************************************************************************ |
| 5 |
@rem = '--*-Perl-*-- |
| 6 |
@echo off |
| 7 |
if "%OS%" == "Windows_NT" goto WinNT |
| 8 |
perl -x -S "%0" %1 %2 %3 %4 %5 %6 %7 %8 %9 |
| 9 |
goto endofperl |
| 10 |
:WinNT |
| 11 |
perl -x -S "%0" %* |
| 12 |
if NOT "%COMSPEC%" == "%SystemRoot%\system32\cmd.exe" goto endofperl |
| 13 |
if %errorlevel% == 9009 echo You do not have Perl in your PATH. |
| 14 |
goto endofperl |
| 15 |
@rem '; |
| 16 |
#!perl |
| 17 |
#line 14 |
| 18 |
# ************************************************************************ |
| 19 |
# Copyright (C) 2000-2004, International Business Machines Corporation and |
| 20 |
# others. All Rights Reserved. |
| 21 |
# ************************************************************************ |
| 22 |
|
| 23 |
# This perl script creates ICU transliterator data files, that live |
| 24 |
# in icu/data, from ICU4J UTF8 transliterator data files, in |
| 25 |
# icu4j/src/com/ibm/icu/impl/data/. |
| 26 |
# |
| 27 |
# The transformation that is done is very minimal. The script assumes |
| 28 |
# that the input files use only # comments |
| 29 |
# and that they follow a rigid format. |
| 30 |
# |
| 31 |
# The output files are named according to ICU conventions (see NAME_MAP |
| 32 |
# below) and created in the current directory. They should be manually |
| 33 |
# checked and then copied into the icu/data/tranlit directory. |
| 34 |
# An ICU build must |
| 35 |
# then be initiated, and the standard suite of ICU transliterator tests |
| 36 |
# should be run after that. |
| 37 |
# |
| 38 |
# Alan Liu 5/19/00 2/27/01 |
| 39 |
|
| 40 |
use Getopt::Long; |
| 41 |
use strict; |
| 42 |
|
| 43 |
use vars qw(%USED_FILES); |
| 44 |
|
| 45 |
my $DIR = "../../../impl/data"; |
| 46 |
my $ID = ''; |
| 47 |
|
| 48 |
GetOptions('dir=s' => \$DIR, |
| 49 |
'id=s' => \$ID, |
| 50 |
'<>' => \&usage) || die; |
| 51 |
|
| 52 |
usage() if (@ARGV); |
| 53 |
|
| 54 |
my $ID =~ s/-/_/; |
| 55 |
if (! -d $DIR) { |
| 56 |
print STDERR "$DIR is not a directory\n"; |
| 57 |
usage(); |
| 58 |
} |
| 59 |
|
| 60 |
sub usage { |
| 61 |
my $me = $0; |
| 62 |
$me =~ s|.+[/\\]||; |
| 63 |
print "Usage: $me [-dir <dir>] [-id <id>]\n"; |
| 64 |
print " --dir <dir> Specify the directory containing the\n"; |
| 65 |
print " Transliterator_*.txt files\n"; |
| 66 |
print " --id <id> Specify a single ID to transform, e.g.\n"; |
| 67 |
print " Fullwidth-Halfwidth\n"; |
| 68 |
exit(1); |
| 69 |
} |
| 70 |
|
| 71 |
my $JAVA_ONLY = '-'; |
| 72 |
|
| 73 |
my $OUTDIR = "icu4c"; |
| 74 |
mkdir($OUTDIR,0777); |
| 75 |
|
| 76 |
# Mapping from Java file names to ICU file names |
| 77 |
# Constraints on ICU4C file name: icudt20b_ |
| 78 |
# |--9 (prefix)---|---18(name with distinguisher,e.g. "t_" )----| |
| 79 |
# --4 ("."+extn)--| = 31 characters total. |
| 80 |
# That is, must have length(%NAME_MAP{x}) <= 16 |
| 81 |
|
| 82 |
my $MAX_ICU4C_FILENAME_LEN = 18; |
| 83 |
|
| 84 |
# -- HISTORY -- If not marked, then pre 2.2. |
| 85 |
# All InterIndic are pre 2.2. |
| 86 |
# Any_Accents |
| 87 |
# Any_Publishing |
| 88 |
# Arabic_Latin * 2.2 |
| 89 |
# Cyrillic_Latin |
| 90 |
# Fullwidth_Halfwidth |
| 91 |
# Greek_Latin |
| 92 |
# Greek_Latin_UNGEGN * 2.2 (moved from el.txt) |
| 93 |
# Han_Latin * 2.2 |
| 94 |
# Han_Latin_Definition * 2.2 |
| 95 |
# Han_Latin_EDICT * 2.2 J only |
| 96 |
# Hebrew_Latin * 2.2 |
| 97 |
# Hiragana_Katakana |
| 98 |
# Hiragana_Latin |
| 99 |
# Latin_Jamo |
| 100 |
# Latin_Katakana |
| 101 |
# ThaiLogical_Latin * 2.2 J only |
| 102 |
# Thai_ThaiLogical * 2.2 J only |
| 103 |
# Thai_ThaiSemi * 2.2 J only |
| 104 |
|
| 105 |
my %NAME_MAP = ( |
| 106 |
# An ICU name of "" means the ICU name == the ID |
| 107 |
|
| 108 |
# We filter names based on what is in use in the index file. |
| 109 |
|
| 110 |
# Flag a rule as JAVA_ONLY if it exists and we use it in Java, |
| 111 |
# but we don't use it in C. |
| 112 |
|
| 113 |
# Use official script abbreviations where possible. |
| 114 |
|
| 115 |
# |..............| |..............| |
| 116 |
# 1234567890123456 1234567890123456 |
| 117 |
Any_Accents => "", |
| 118 |
Any_Publishing => "", |
| 119 |
Cyrillic_Latin => "Cyrl_Latn", |
| 120 |
Fullwidth_Halfwidth => "FWidth_HWidth", |
| 121 |
Greek_Latin => "Grek_Latn", |
| 122 |
Hiragana_Katakana => "Hira_Kana", |
| 123 |
Hiragana_Latin => "Hira_Latn", |
| 124 |
Latin_Jamo => "Latn_Jamo", |
| 125 |
Latin_Katakana => "Latn_Kana", |
| 126 |
|
| 127 |
Arabic_Latin => "Arab_Latn", |
| 128 |
Greek_Latin_UNGEGN => "Grek_Latn_UNGEGN", |
| 129 |
Han_Latin => "Hani_Latn", |
| 130 |
Han_Latin_Definition => "Hani_Latn_Def", |
| 131 |
Han_Latin_EDICT => "Hani_Latn_EDICT", |
| 132 |
Hebrew_Latin => "Hebr_Latn", |
| 133 |
ThaiLogical_Latin => $JAVA_ONLY, # "ThaiLog_Latn", |
| 134 |
Thai_ThaiLogical => $JAVA_ONLY, # "Thai_ThaiLog", |
| 135 |
Thai_ThaiSemi => $JAVA_ONLY, # "Thai_ThaiSemi", |
| 136 |
|
| 137 |
InterIndic_Bengali => "InterIndic_Beng", |
| 138 |
InterIndic_Devanagari => "InterIndic_Deva", |
| 139 |
InterIndic_Gujarati => "InterIndic_Gujr", |
| 140 |
InterIndic_Gurmukhi => "InterIndic_Guru", |
| 141 |
InterIndic_Kannada => "InterIndic_Knda", |
| 142 |
InterIndic_Latin => "InterIndic_Latn", |
| 143 |
InterIndic_Malayalam => "InterIndic_Mlym", |
| 144 |
InterIndic_Oriya => "InterIndic_Orya", |
| 145 |
InterIndic_Tamil => "InterIndic_Taml", |
| 146 |
InterIndic_Telugu => "InterIndic_Telu", |
| 147 |
|
| 148 |
Bengali_InterIndic => "Beng_InterIndic", |
| 149 |
Devanagari_InterIndic => "Deva_InterIndic", |
| 150 |
Gujarati_InterIndic => "Gujr_InterIndic", |
| 151 |
Gurmukhi_InterIndic => "Guru_InterIndic", |
| 152 |
Kannada_InterIndic => "Knda_InterIndic", |
| 153 |
Latin_InterIndic => "Latn_InterIndic", |
| 154 |
Malayalam_InterIndic => "Mlym_InterIndic", |
| 155 |
Oriya_InterIndic => "Orya_InterIndic", |
| 156 |
Tamil_InterIndic => "Taml_InterIndic", |
| 157 |
Telugu_InterIndic => "Telu_InterIndic", |
| 158 |
|
| 159 |
Han_Pinyin => $JAVA_ONLY, |
| 160 |
Kanji_English => $JAVA_ONLY, |
| 161 |
Kanji_OnRomaji => $JAVA_ONLY, |
| 162 |
|
| 163 |
Latin_NumericPinyin => "Latn_NPinyn", |
| 164 |
Tone_Digit => "Tone_Digit", |
| 165 |
Han_Spacedhan => "Hani_SpHan", |
| 166 |
); |
| 167 |
|
| 168 |
my ($x,$x,$x,$x,$x,$THIS_YEAR) = localtime(); |
| 169 |
$THIS_YEAR += 1900; |
| 170 |
|
| 171 |
# Header blocks of text written at start of ICU output files |
| 172 |
my $HEADER1 = <<END; |
| 173 |
//-------------------------------------------------------------------- |
| 174 |
// Copyright (c) 1999-$THIS_YEAR, International Business Machines |
| 175 |
// Corporation and others. All Rights Reserved. |
| 176 |
//-------------------------------------------------------------------- |
| 177 |
// THIS IS A MACHINE-GENERATED FILE |
| 178 |
END |
| 179 |
my $HEADER2 = <<END; |
| 180 |
//-------------------------------------------------------------------- |
| 181 |
END |
| 182 |
|
| 183 |
my $TOOL = $0; |
| 184 |
|
| 185 |
# Convert the index first; this tells us which rule files are in use. |
| 186 |
convertIndex(); |
| 187 |
|
| 188 |
# print "In use:\n", join("\n", sort keys(%USED_FILES)), "\n"; |
| 189 |
|
| 190 |
# Iterate over all Java RBT rule files |
| 191 |
foreach (<$DIR/Transliterator_*.txt>) { |
| 192 |
next if (/~$/); # Ignore emacs backups |
| 193 |
next if (/_index\.txt$/); # The index file was processed above |
| 194 |
# Select either the command-line arg, if there was one, or |
| 195 |
# any files mentioned in the index. |
| 196 |
my $leaf = $_; |
| 197 |
$leaf =~ s|.+[/\\]||; |
| 198 |
if (($ID && $leaf =~ /$ID/) || exists $USED_FILES{$leaf}) { |
| 199 |
my ($out, $id) = convertFileName($_); |
| 200 |
if ($out) { |
| 201 |
if ($out eq $JAVA_ONLY) { |
| 202 |
print STDERR "*** $id skipped: Java only ***\n"; |
| 203 |
next; |
| 204 |
} |
| 205 |
file($id, $_, $out); |
| 206 |
} |
| 207 |
} elsif (!$ID) { |
| 208 |
print "*** $leaf skipped: not in use ***\n"; |
| 209 |
} |
| 210 |
} |
| 211 |
|
| 212 |
###################################################################### |
| 213 |
# Convert a Java file name to C |
| 214 |
# Param: Java file name of the form m|Transliterator_(.+)\.utf8\.txt$| |
| 215 |
# Return: A C file name (e.g., ldevan.txt) or the empty string, |
| 216 |
# if there is no mapping, or $JAVA_ONLY if the given file isn't |
| 217 |
# intended to be incorporated into C. |
| 218 |
sub convertFileName { |
| 219 |
local $_ = shift; |
| 220 |
my $id; |
| 221 |
if (m|Transliterator_(.+)\.utf8\.txt$| || |
| 222 |
m|Transliterator_(.+)\.txt$|) { |
| 223 |
$id = $1; |
| 224 |
} else { die "Can't parse Java file name $_"; } |
| 225 |
if (!exists $NAME_MAP{$id}) { |
| 226 |
print STDERR "ERROR: $id not in map; please update $0\n"; |
| 227 |
return ''; |
| 228 |
} |
| 229 |
my $out = $NAME_MAP{$id}; |
| 230 |
if ($out eq '') { |
| 231 |
$out = $id; |
| 232 |
} |
| 233 |
if ($out ne $JAVA_ONLY) { |
| 234 |
$out = 't_' . $out; |
| 235 |
} |
| 236 |
if (length($out) > $MAX_ICU4C_FILENAME_LEN) { |
| 237 |
print STDERR "ERROR: ICU4C file name \"$out\" too long; please update $0\n"; |
| 238 |
return ''; |
| 239 |
} |
| 240 |
return ($out, $id); |
| 241 |
} |
| 242 |
|
| 243 |
###################################################################### |
| 244 |
# Convert the index file from Java to C format |
| 245 |
sub convertIndex { |
| 246 |
my $JAVA_INDEX = "Transliterator_index.txt"; |
| 247 |
my $C_INDEX = "translit_index.txt"; |
| 248 |
open(JAVA_INDEX, "$DIR/$JAVA_INDEX") or die; |
| 249 |
open(C_INDEX, ">$OUTDIR/$C_INDEX") or die; |
| 250 |
|
| 251 |
header(\*C_INDEX, $JAVA_INDEX); |
| 252 |
|
| 253 |
print C_INDEX <<END; |
| 254 |
//-------------------------------------------------------------------- |
| 255 |
// N.B.: This file has been generated mechanically from the |
| 256 |
// corresponding ICU4J file, which is the primary file that receives |
| 257 |
// primary updates. The colon-delimited fields have been split into |
| 258 |
// separate strings. For 'file' and 'internal' lines, the encoding |
| 259 |
// field has been deleted, since the encoding is processed at build |
| 260 |
// time in ICU4C. Certain large rule sets not intended for general |
| 261 |
// use have been commented out with the notation "Java only". |
| 262 |
//-------------------------------------------------------------------- |
| 263 |
|
| 264 |
translit_index { |
| 265 |
RuleBasedTransliteratorIDs { |
| 266 |
END |
| 267 |
|
| 268 |
while (<JAVA_INDEX>) { |
| 269 |
# ignore CVS keyword substitutions |
| 270 |
next if /\$(Source|Revision|Date)/; |
| 271 |
|
| 272 |
# we have printed out the copyright info ... ignore one in Java version |
| 273 |
next if /Copyright/ ; |
| 274 |
next if /Corporation/; |
| 275 |
|
| 276 |
# Comments; change # to // |
| 277 |
if (s|^(\s*)\#|$1//|) { |
| 278 |
print C_INDEX; |
| 279 |
next; |
| 280 |
} |
| 281 |
# Blank lines |
| 282 |
if (!/\S/) { |
| 283 |
print C_INDEX; |
| 284 |
next; |
| 285 |
} |
| 286 |
# Content lines |
| 287 |
chomp; |
| 288 |
my $prefix = ''; |
| 289 |
#replace \p with \\p |
| 290 |
$_=~ s/\\p/\\\\p/g; |
| 291 |
my @a = split(':', $_); |
| 292 |
if ($a[1] eq 'file' || $a[1] eq 'internal') { |
| 293 |
# Convert the file name |
| 294 |
my $id; |
| 295 |
# Record file names in use |
| 296 |
$USED_FILES{$a[2]} = 1; |
| 297 |
($a[2], $id) = convertFileName($a[2]); |
| 298 |
if ($a[2] eq $JAVA_ONLY) { |
| 299 |
$prefix = '// Java only: '; |
| 300 |
} |
| 301 |
|
| 302 |
# Delete the encoding field |
| 303 |
splice(@a, 3, 1); |
| 304 |
} elsif ($a[1] eq 'alias') { |
| 305 |
# Pad out with extra blank fields to make the |
| 306 |
# 2-d array square |
| 307 |
push @a, ""; |
| 308 |
} else { |
| 309 |
die "Can't parse $_"; |
| 310 |
} |
| 311 |
print C_INDEX |
| 312 |
$prefix, "{ ", |
| 313 |
join(", ", map("\"$_\"", @a)), |
| 314 |
" },\n"; |
| 315 |
} |
| 316 |
|
| 317 |
print C_INDEX <<END; |
| 318 |
} |
| 319 |
} |
| 320 |
END |
| 321 |
|
| 322 |
close(C_INDEX); |
| 323 |
close(JAVA_INDEX); |
| 324 |
print STDERR "$JAVA_INDEX -> $C_INDEX\n"; |
| 325 |
} |
| 326 |
|
| 327 |
###################################################################### |
| 328 |
# Output a header |
| 329 |
# Param: Filehandle |
| 330 |
sub header { |
| 331 |
my $out = shift; |
| 332 |
my $in = shift; |
| 333 |
print $out $HEADER1; |
| 334 |
print $out "// Tool: $TOOL\n// Source: $in\n"; |
| 335 |
print $out "// Date: ", scalar localtime, "\n"; |
| 336 |
print $out $HEADER2; |
| 337 |
print $out "\n"; |
| 338 |
} |
| 339 |
|
| 340 |
###################################################################### |
| 341 |
# Process one file |
| 342 |
# Param: ID, e.g. Fullwidth-Halfwidth |
| 343 |
# Param: Java input file name, e.g. |
| 344 |
# f:/icu4j/src/com/ibm/text/resources/Transliterator_Fullwidth_Halfwidth.utf8.txt |
| 345 |
# Param: ICU output file name, e.g. fullhalf |
| 346 |
sub file { |
| 347 |
my $id = shift; |
| 348 |
my $IN = shift; |
| 349 |
my $out = shift; |
| 350 |
|
| 351 |
my $OUT = "$out.txt"; |
| 352 |
|
| 353 |
# Show input size. Show output size later -- useful for quick sanity check. |
| 354 |
print "$id (", -s $IN, ") -> $OUT ("; |
| 355 |
|
| 356 |
# Open file, write UTF8 marker, close it, and reopen in text mode |
| 357 |
open(OUT, ">$OUTDIR/$OUT") or die; |
| 358 |
binmode OUT; # Must do this so we can write our UTF8 marker |
| 359 |
print OUT pack("C3", 0xEF, 0xBB, 0xBF); # Write UTF8 marker |
| 360 |
close(OUT); |
| 361 |
|
| 362 |
open(OUT, ">>$OUTDIR/$OUT") or die; |
| 363 |
print OUT " // -*- Coding: utf-8; -*-\n"; |
| 364 |
|
| 365 |
header(\*OUT, $IN); |
| 366 |
print OUT "// $id\n"; |
| 367 |
print OUT "\n"; |
| 368 |
print OUT "$out {\n"; |
| 369 |
print OUT " Rule {\n"; |
| 370 |
|
| 371 |
open(IN, $IN) or die; |
| 372 |
binmode IN; # IN is a UTF8 file |
| 373 |
|
| 374 |
my $first = 1; |
| 375 |
my $BOM = pack("C3", 239, 187, 191); # a UTF8 byte order mark |
| 376 |
|
| 377 |
# Process each line by changing # comments to // comments |
| 378 |
# and taking other text and enclosing it in double quotes |
| 379 |
while (<IN>) { |
| 380 |
my $raw = $_; |
| 381 |
# ignore CVS keyword substitutions |
| 382 |
next if /\$(Source|Revision|Date)/; |
| 383 |
|
| 384 |
# we have printed out the copyright info ... ignore one in Java version |
| 385 |
next if /Copyright/ ; |
| 386 |
next if /Corporation/; |
| 387 |
|
| 388 |
# Look for and delete BOM |
| 389 |
if ($first) { |
| 390 |
s/^$BOM//; |
| 391 |
$first = 0; |
| 392 |
} |
| 393 |
|
| 394 |
# Clean the eol junk up |
| 395 |
s/[\x0D\x0A]+$//; |
| 396 |
|
| 397 |
# If there is a trailing backslash, then delete it -- we don't |
| 398 |
# need line continuation in C, since adjacent strings are |
| 399 |
# concatenated. Count trailing backslashes; if they are odd, |
| 400 |
# one is trailing. |
| 401 |
if (m|(\\+)$|) { |
| 402 |
if ((length($1) % 2) == 1) { |
| 403 |
s|\\$||; |
| 404 |
} |
| 405 |
} |
| 406 |
|
| 407 |
# Transform escaped characters |
| 408 |
hideEscapes(); |
| 409 |
|
| 410 |
if (/^(\s*)(\#.*)$/) { |
| 411 |
# Comment-only line |
| 412 |
my ($white, $cmt) = ($1, $2); |
| 413 |
$cmt =~ s|\#|//|; |
| 414 |
$_ = $white . $cmt; |
| 415 |
|
| 416 |
} elsif (!/\S/) { |
| 417 |
# Blank line -- leave as-is |
| 418 |
|
| 419 |
} else { |
| 420 |
# Remove single-quoted matter |
| 421 |
my @quotes; |
| 422 |
my $nquotes = 0; |
| 423 |
my $x = $_; |
| 424 |
while (s/^([^\']*)(\'[^\']*\')/$1<<x$nquotes>>/) { |
| 425 |
push @quotes, $2; |
| 426 |
++$nquotes; |
| 427 |
} |
| 428 |
|
| 429 |
# Extract comment |
| 430 |
my $cmt = ''; |
| 431 |
if (s|\#(.*)||) { |
| 432 |
$cmt = '//' . $1; |
| 433 |
} |
| 434 |
|
| 435 |
# Add quotes |
| 436 |
s|^(\s*)(\S.*?)(\s*)$|$1\"$2\"$3|; |
| 437 |
|
| 438 |
# Restore single-quoted matter |
| 439 |
for (my $i=0; $i<$nquotes; ++$i) { |
| 440 |
s|<<x$i>>|$quotes[$i]|; |
| 441 |
} |
| 442 |
|
| 443 |
# Restore comment |
| 444 |
$_ .= $cmt; |
| 445 |
} |
| 446 |
|
| 447 |
# Restore escaped characters |
| 448 |
restoreEscapes(); |
| 449 |
|
| 450 |
print OUT $_, "\n"; |
| 451 |
} |
| 452 |
|
| 453 |
# Finish up |
| 454 |
close(IN); |
| 455 |
print OUT " }\n"; |
| 456 |
print OUT "}\n"; |
| 457 |
close(OUT); |
| 458 |
|
| 459 |
# Write output file size for sanity check |
| 460 |
print -s "$OUTDIR/$OUT", ")\n"; |
| 461 |
} |
| 462 |
|
| 463 |
###################################################################### |
| 464 |
sub hideEscapes { |
| 465 |
# Transform escaped characters |
| 466 |
s|\\\\|<<bs>>|g; # DO THIS FIRST Transform backslashes |
| 467 |
s|\\u([a-zA-Z0-9]{4})|<<u$1>>|g; # Transform Unicode escapes |
| 468 |
s|\\\"|<<dq>>|g; # Transform backslash double quote |
| 469 |
s|\\\'|<<sq>>|g; # Transform backslash single quote |
| 470 |
s|\\\#|<<lb>>|g; # Transform backslash pound |
| 471 |
s|\\(.)|<<q$1>>|g; # Transform backslash escapes |
| 472 |
} |
| 473 |
|
| 474 |
###################################################################### |
| 475 |
sub restoreEscapes { |
| 476 |
# Restore escaped characters |
| 477 |
s|<<bs>>|\\\\|g; |
| 478 |
s|<<dq>>|\\\\\\\"|g; |
| 479 |
s|<<sq>>|\\\\\\\'|g; |
| 480 |
s|<<lb>>|\\\\\\\#|g; |
| 481 |
s|<<q(.)>>|\\\\\\$1|g; |
| 482 |
s|<<u0000>>|\\\\u0000|g; # Double escape U+0000 |
| 483 |
s|<<u(....)>>|\\u$1|g; |
| 484 |
} |
| 485 |
|
| 486 |
__END__ |
| 487 |
:endofperl |