| 1 |
#!/usr/bin/perl -s |
| 2 |
|
| 3 |
# PODNAME: tmx2tmx |
| 4 |
# ABSTRACT: Utility to convert and filter TMX files |
| 5 |
|
| 6 |
use strict; |
| 7 |
use warnings; |
| 8 |
|
| 9 |
use XML::TMX::Reader; |
| 10 |
use XML::TMX::Writer; |
| 11 |
|
| 12 |
use XML::DT; |
| 13 |
|
| 14 |
our ( |
| 15 |
$toTrados, ## Fix some issues in the TMX file the old trados software did not cope with |
| 16 |
$clean, ## Remove empty UTs and UTs with only junk characters |
| 17 |
$cat, ## Concatenates TMX files |
| 18 |
$select, ## Filters the TMX to the selected languages |
| 19 |
); |
| 20 |
|
| 21 |
# here we must take care that only one of the options is being used. |
| 22 |
|
| 23 |
|
| 24 |
toTrados() if $toTrados; |
| 25 |
clean() if $clean; |
| 26 |
cat() if $cat; |
| 27 |
selectl() if $select; |
| 28 |
|
| 29 |
die "No option supplied\n"; |
| 30 |
|
| 31 |
|
| 32 |
|
| 33 |
|
| 34 |
#--( AUX FUNCS )----------- |
| 35 |
sub trim { |
| 36 |
my $x = shift; |
| 37 |
for ($x) { |
| 38 |
s!^\s+!!; |
| 39 |
s!\s+$!! |
| 40 |
} |
| 41 |
$x |
| 42 |
} |
| 43 |
|
| 44 |
|
| 45 |
sub toTrados { |
| 46 |
$/ = "</tu>"; |
| 47 |
|
| 48 |
my $header = <>; |
| 49 |
|
| 50 |
# <header/> -> <header></header> |
| 51 |
$header =~ s!<header([^/>]+)/>!<header$1></header>!; |
| 52 |
|
| 53 |
# <body version...> -> <body> |
| 54 |
$header =~ s!<body[^>]+>!<body>!; |
| 55 |
|
| 56 |
# tmx version -> <tmx version="1.1"> |
| 57 |
$header =~ s!<tmx version[^>]+>!<tmx version="1.1">!; |
| 58 |
|
| 59 |
# srclang -> srclang="foo" |
| 60 |
if ($header =~ m!(xml:)?lang=(["'])([^"']+)\2!) { |
| 61 |
my $lang = $3; |
| 62 |
$header =~ s!srclang=(["'])[^"']+\1!srclang="$lang"!; |
| 63 |
} |
| 64 |
|
| 65 |
$header =~ s/xml:lang/lang/g; |
| 66 |
|
| 67 |
# xml:lang -> lang |
| 68 |
print $header; |
| 69 |
|
| 70 |
while (<>) { |
| 71 |
s/xml:lang/lang/g; |
| 72 |
print |
| 73 |
} |
| 74 |
|
| 75 |
# ugly, but prevents us from forgetting an else somewhere. |
| 76 |
exit; |
| 77 |
} |
| 78 |
|
| 79 |
|
| 80 |
sub clean { |
| 81 |
my $file; |
| 82 |
my $reader; |
| 83 |
if ($file = shift) { |
| 84 |
# isto tv fosse mais rápido em sax... |
| 85 |
print dt( $file, |
| 86 |
-default => sub{ toxml() }, |
| 87 |
seg => sub{ |
| 88 |
$c = trim($c); toxml |
| 89 |
}, |
| 90 |
tu => sub{ |
| 91 |
# remove entries with junk, only |
| 92 |
return "" if $c =~ m!<seg>(\s|[\|\!@#\$%^&\-])*</seg>!; |
| 93 |
toxml() |
| 94 |
}, |
| 95 |
); |
| 96 |
} else { |
| 97 |
die "At the moment, we do not handle files from stdin"; |
| 98 |
# TODO... aqui fazia jeito que o XML::TMX::Reader lêsse de um |
| 99 |
# filehandle já aberto (STDIN, por exemplo); |
| 100 |
} |
| 101 |
exit; |
| 102 |
} |
| 103 |
|
| 104 |
sub _join_and_print_headers { |
| 105 |
my $header; |
| 106 |
|
| 107 |
while (@_) { |
| 108 |
my $file = shift @_; |
| 109 |
my $reader = XML::TMX::Reader->new($file); |
| 110 |
for my $key (keys %{$reader->{header}}) { |
| 111 |
if ($key eq "-prop") { |
| 112 |
for my $type (keys %{$reader->{header}{$key}}) { |
| 113 |
push @{$header->{$key}{$type}}, @{$reader->{header}{$key}{$type}} |
| 114 |
} |
| 115 |
} |
| 116 |
elsif ($key eq "-note") { |
| 117 |
push @{$header->{$key}}, @{$reader->{header}{$key}} |
| 118 |
} |
| 119 |
else { |
| 120 |
$header->{$key} = $reader->{header}{$key} |
| 121 |
} |
| 122 |
} |
| 123 |
} |
| 124 |
|
| 125 |
my $writer = XML::TMX::Writer->new(); |
| 126 |
$writer->start_tmx(%$header); |
| 127 |
} |
| 128 |
|
| 129 |
sub open_tmx_with_correct_encoding { |
| 130 |
# in the future it might be a good idea to use the guess_encoding method from Reader.pm |
| 131 |
my $filename = shift; |
| 132 |
|
| 133 |
open my $f, $filename or die "Can't open file $filename"; |
| 134 |
my $header = <$f>; |
| 135 |
close $f; |
| 136 |
|
| 137 |
my $fh; |
| 138 |
if ($header =~ /encoding\s*=\s*["']([^"']+)["']/) { |
| 139 |
open $fh, "<:encoding($1)", $filename or die "Can't open file $filename"; |
| 140 |
} |
| 141 |
else { |
| 142 |
open $fh, "<:utf8", $filename or die "Can't open file $filename"; |
| 143 |
} |
| 144 |
return $fh; |
| 145 |
} |
| 146 |
|
| 147 |
sub cat { |
| 148 |
my @files ; |
| 149 |
if (@ARGV) { |
| 150 |
@files = @ARGV; |
| 151 |
} else { |
| 152 |
@files = map { chomp; $_ } <>; |
| 153 |
} |
| 154 |
|
| 155 |
binmode STDOUT, ":utf8"; |
| 156 |
_join_and_print_headers(@files); |
| 157 |
|
| 158 |
my $fh; |
| 159 |
my $file = shift @files; |
| 160 |
$fh = open_tmx_with_correct_encoding($file); |
| 161 |
# open $fh, "<:utf8", $file or die "Cannot open file: $file\n"; |
| 162 |
while (<$fh>) { |
| 163 |
last if m!<body!; |
| 164 |
} |
| 165 |
print "<!-- FILE: $file -->\n"; |
| 166 |
while (<$fh>) { |
| 167 |
s!</\s*(body|tmx)\s*>!!; |
| 168 |
print; |
| 169 |
} |
| 170 |
close $fh; |
| 171 |
|
| 172 |
for $file (@files) { |
| 173 |
$fh = open_tmx_with_correct_encoding($file); |
| 174 |
# open $fh, "<:utf8", $file or die "Cannot open file: $file\n"; |
| 175 |
print "<!-- FILE: $file -->\n"; |
| 176 |
while (<$fh>) { |
| 177 |
last if m!<body!; |
| 178 |
} |
| 179 |
s/<body[^>]*>//g; |
| 180 |
print; |
| 181 |
while (<$fh>) { |
| 182 |
s!</\s*(body|tmx)\s*>!!; |
| 183 |
print; |
| 184 |
} |
| 185 |
close $fh; |
| 186 |
} |
| 187 |
print "</body>\n"; |
| 188 |
print "</tmx>\n"; |
| 189 |
|
| 190 |
exit; |
| 191 |
} |
| 192 |
|
| 193 |
sub selectl { |
| 194 |
my @files; |
| 195 |
if (@ARGV) { |
| 196 |
@files = @ARGV; |
| 197 |
} else { |
| 198 |
@files = map { chomp; $_ } <>; |
| 199 |
} |
| 200 |
|
| 201 |
_join_and_print_headers(@files); |
| 202 |
my %languages = map { (lc $_ => 1) } split( /,/, $select ); |
| 203 |
|
| 204 |
my $tmx; |
| 205 |
while ($tmx = shift @files) { |
| 206 |
die "Can't find $tmx" if (!$tmx || !-f$tmx); |
| 207 |
|
| 208 |
print STDERR "$tmx\n"; |
| 209 |
my $reader = XML::TMX::Reader->new($tmx); |
| 210 |
$reader->for_tu( |
| 211 |
{ |
| 212 |
-verbose => 1, |
| 213 |
-output => 1, |
| 214 |
-header => 0 |
| 215 |
}, |
| 216 |
sub { |
| 217 |
my $tu = shift; |
| 218 |
my $l = 0; |
| 219 |
for my $k (keys %$tu) { |
| 220 |
next if $k =~ /^-/; |
| 221 |
if (exists $languages{lc $k}) { |
| 222 |
$l++; |
| 223 |
} else { |
| 224 |
delete $tu->{$k} |
| 225 |
} |
| 226 |
} |
| 227 |
if ($l == keys %languages) { |
| 228 |
return $tu; |
| 229 |
} else { |
| 230 |
return undef; |
| 231 |
} |
| 232 |
}); |
| 233 |
} |
| 234 |
print "</body>\n</tmx>\n"; |
| 235 |
exit; |
| 236 |
} |
| 237 |
|
| 238 |
__END__ |
| 239 |
|
| 240 |
=pod |
| 241 |
|
| 242 |
=encoding utf-8 |
| 243 |
|
| 244 |
=head1 NAME |
| 245 |
|
| 246 |
tmx2tmx - Utility to convert and filter TMX files |
| 247 |
|
| 248 |
=head1 VERSION |
| 249 |
|
| 250 |
version 0.39 |
| 251 |
|
| 252 |
=head1 DESCRIPTION |
| 253 |
|
| 254 |
This utility processes TMX documents and return TMX documents. Tasks |
| 255 |
done with this utility include conversion between TMX versions and TMX |
| 256 |
cleaning. |
| 257 |
|
| 258 |
=head2 TRADOS conversion |
| 259 |
|
| 260 |
As you maybe know, TRADOS is a company producing computer software for |
| 261 |
translators. It includes WorkBench which imports TMX |
| 262 |
files. Unfortunately, the version I used do not import TMX version |
| 263 |
1.4. |
| 264 |
|
| 265 |
This process is done using the switch C<-toTrados>: |
| 266 |
|
| 267 |
tmx2tmx -toTrados file.tmx > trados.tmx |
| 268 |
|
| 269 |
=head2 TMX Cleaning |
| 270 |
|
| 271 |
Specially when translation memories are created from anotated text, or |
| 272 |
extracted directly from the Internet using any automatic method. This |
| 273 |
switch is used to remove junk in translation units. |
| 274 |
|
| 275 |
This option tries to remove junk from TMX files, like empty pairs |
| 276 |
where one of the sides is empty, or removing other junk type. |
| 277 |
|
| 278 |
Use it this way: |
| 279 |
|
| 280 |
tmx2tmx -clean file.tmx > file2.tmx |
| 281 |
|
| 282 |
=head2 Concatenating TMX |
| 283 |
|
| 284 |
tmx2tmx -cat file1.tmx ... filen.tmx > file.tmx |
| 285 |
ls | grep '.tmx$' | tmx2tmx -cat > file.tmx |
| 286 |
|
| 287 |
=head2 Select languages |
| 288 |
|
| 289 |
Select a bunch of languages: |
| 290 |
|
| 291 |
tmx2tmx -select=PT,EN,FR huge.tmx > pt-en-fr.tmx |
| 292 |
|
| 293 |
=head1 SYNOPSYS |
| 294 |
|
| 295 |
tmx2tmx -cat file1.tmx ... filen.tmx > file.tmx |
| 296 |
|
| 297 |
tmx2tmx -toTrados file1.tmx > file2.tmx |
| 298 |
|
| 299 |
tmx2tmx -clean file1.tmx > file2.tmx |
| 300 |
|
| 301 |
tmx2tmx -select=PT,EN multilingual.tmx > pt-en.tmx |
| 302 |
|
| 303 |
=head1 SEE ALSO |
| 304 |
|
| 305 |
tmx2html, po2tmx, XML::TMX |
| 306 |
|
| 307 |
=head1 AUTHORS |
| 308 |
|
| 309 |
=over 4 |
| 310 |
|
| 311 |
=item * |
| 312 |
|
| 313 |
Alberto Simões <[email protected]> |
| 314 |
|
| 315 |
=item * |
| 316 |
|
| 317 |
José João Almeida <[email protected]> |
| 318 |
|
| 319 |
=back |
| 320 |
|
| 321 |
=head1 COPYRIGHT AND LICENSE |
| 322 |
|
| 323 |
This software is copyright (c) 2010-2017 by Projeto Natura <[email protected]>. |
| 324 |
|
| 325 |
This is free software; you can redistribute it and/or modify it under |
| 326 |
the same terms as the Perl 5 programming language system itself. |
| 327 |
|
| 328 |
=cut |