| 1 |
#!/usr/bin/perl -s |
| 2 |
|
| 3 |
# PODNAME: tmx-tokenize |
| 4 |
# ABSTRACT: Tokenizes translation units on a tmx file. |
| 5 |
|
| 6 |
use strict; |
| 7 |
use warnings; |
| 8 |
|
| 9 |
our $o; |
| 10 |
|
| 11 |
eval { require FL3 }; |
| 12 |
die "This XML::TMX script requires Lingua::FreeLing3 to be installed\n" if $@; |
| 13 |
|
| 14 |
FL3->import(); |
| 15 |
use XML::TMX::Reader '0.25'; |
| 16 |
|
| 17 |
my $file = shift or die "You must supply the name of the file to tokenize"; |
| 18 |
|
| 19 |
my $reader = XML::TMX::Reader->new($file); |
| 20 |
|
| 21 |
my $output = "t_$file"; |
| 22 |
$output = $o if $o; |
| 23 |
|
| 24 |
binmode STDOUT, ":utf8"; |
| 25 |
$reader->for_tu( { |
| 26 |
-output => $output, |
| 27 |
-prop => { tokenized => "true" }, |
| 28 |
verbose => 1 |
| 29 |
}, |
| 30 |
sub { |
| 31 |
my $tu = shift; |
| 32 |
for my $lang (keys %$tu) { |
| 33 |
if ($lang =~ /(pt|es|it|ru|en|gl)/i) { |
| 34 |
my $ln = lc $1; |
| 35 |
my $txt = $tu->{$lang}{-seg}; |
| 36 |
if ($txt !~ /^\s*$/) { |
| 37 |
$txt = join(" ", |
| 38 |
@{ tokenizer($ln)->tokenize($txt, |
| 39 |
to_text => 1)}); |
| 40 |
} |
| 41 |
$tu->{$lang}{-seg} = $txt; |
| 42 |
} |
| 43 |
} |
| 44 |
return $tu; |
| 45 |
}); |
| 46 |
|
| 47 |
__END__ |
| 48 |
|
| 49 |
=pod |
| 50 |
|
| 51 |
=encoding UTF-8 |
| 52 |
|
| 53 |
=head1 NAME |
| 54 |
|
| 55 |
tmx-tokenize - Tokenizes translation units on a tmx file. |
| 56 |
|
| 57 |
=head1 VERSION |
| 58 |
|
| 59 |
version 0.39 |
| 60 |
|
| 61 |
=head1 SYNOPSIS |
| 62 |
|
| 63 |
tmx-tokenize file.tmx # creates t_file.tmx |
| 64 |
|
| 65 |
tmx-tokenize -o=out.tmx file.tmx |
| 66 |
|
| 67 |
=head1 DESCRIPTION |
| 68 |
|
| 69 |
Although this script is bundled in C<XML::TMX>, it has a soft |
| 70 |
dependency on C<Lingua::FreeLing3>. Soft means that the dependency is |
| 71 |
not ensured at install time, and other features of the module can |
| 72 |
still be used without C<Lingua::FreeLing3>. Nevertheless, if you want |
| 73 |
to use this tool you should install that module. |
| 74 |
|
| 75 |
At the moment the supported languages are the same as supported by |
| 76 |
FreeLing3: English, Spanish, Russian, Portuguese and Italian. |
| 77 |
|
| 78 |
It your TMX file includes any other language, they will be maintained |
| 79 |
without a change. This behavior can change in the future, as a basic |
| 80 |
regexp based tokenizer might be implemented. |
| 81 |
|
| 82 |
=head1 SEE ALSO |
| 83 |
|
| 84 |
XML::TMX, Lingua::FreeLing3 |
| 85 |
|
| 86 |
=head1 AUTHORS |
| 87 |
|
| 88 |
=over 4 |
| 89 |
|
| 90 |
=item * |
| 91 |
|
| 92 |
Alberto Simões <[email protected]> |
| 93 |
|
| 94 |
=item * |
| 95 |
|
| 96 |
José João Almeida <[email protected]> |
| 97 |
|
| 98 |
=back |
| 99 |
|
| 100 |
=head1 COPYRIGHT AND LICENSE |
| 101 |
|
| 102 |
This software is copyright (c) 2010-2017 by Projeto Natura <[email protected]>. |
| 103 |
|
| 104 |
This is free software; you can redistribute it and/or modify it under |
| 105 |
the same terms as the Perl 5 programming language system itself. |
| 106 |
|
| 107 |
=cut |