| 1 |
#!/usr/bin/perl -s |
| 2 |
|
| 3 |
# PODNAME: tmx-POStagger |
| 4 |
# ABSTRACT: POStaggers translation units on a tmx file. |
| 5 |
|
| 6 |
use strict; |
| 7 |
use warnings; |
| 8 |
use File::Temp qw/ :POSIX /; |
| 9 |
|
| 10 |
my $tmpName = tmpnam(); |
| 11 |
|
| 12 |
our ( |
| 13 |
$o, # output filename |
| 14 |
$mwe, # tag mwe -- STILL NOT SUPPORTED |
| 15 |
$dates, # do dates analysis... |
| 16 |
$ner, # try to do NER.... |
| 17 |
$compact, # compact form |
| 18 |
$s, # use <s> tags (on by default) |
| 19 |
); |
| 20 |
|
| 21 |
$s = 1 unless defined $s; |
| 22 |
|
| 23 |
eval { require FL3 }; |
| 24 |
die "This XML::TMX script requires Lingua::FreeLing3 to be installed\n" if $@; |
| 25 |
FL3->import(); |
| 26 |
|
| 27 |
eval { require Lingua::FreeLing3::Word }; |
| 28 |
die "This XML::TMX script requires Lingua::FreeLing3 to be installed\n" if $@; |
| 29 |
Lingua::FreeLing3::Word->import(); |
| 30 |
|
| 31 |
my %initted = (); |
| 32 |
|
| 33 |
use XML::TMX::Reader '0.25'; |
| 34 |
|
| 35 |
my $file = shift or die "You must supply the name of the file to tokenize"; |
| 36 |
|
| 37 |
my $reader = XML::TMX::Reader->new($file); |
| 38 |
|
| 39 |
my $output = "pos_$file"; |
| 40 |
$output = $o if $o; |
| 41 |
|
| 42 |
my $pos_languages = {}; |
| 43 |
|
| 44 |
binmode STDOUT, ":utf8"; |
| 45 |
$reader->for_tu( |
| 46 |
{ |
| 47 |
-output => $tmpName, |
| 48 |
-verbose => 1, |
| 49 |
-verbatim => 1, |
| 50 |
}, |
| 51 |
sub { |
| 52 |
my $tu = shift; |
| 53 |
for my $lang (keys %$tu) { |
| 54 |
if ($lang =~ /(pt|es|gl|it|fr|ru|en)/i) { |
| 55 |
$pos_languages->{$1} = 1; |
| 56 |
|
| 57 |
my $ln = lc $1; |
| 58 |
my $seg = $compact ? "" : "<![CDATA["; |
| 59 |
|
| 60 |
_init_ma($ln) unless exists $initted{$ln}; |
| 61 |
|
| 62 |
my $txt = $tu->{$lang}{-seg}; |
| 63 |
my @tokens = map { Lingua::FreeLing3::Word->new($_) } split /\s+/, $txt; |
| 64 |
my $sentences = splitter($ln)->split(\@tokens); |
| 65 |
$sentences = morph($ln)->analyze($sentences); |
| 66 |
$sentences = hmm($ln)->tag($sentences); |
| 67 |
for my $stc (@$sentences) { |
| 68 |
$seg .= "<s>\n" if $s; |
| 69 |
$seg .= $compact ? _dump_compact($stc->words) |
| 70 |
: _dump_words($stc->words); |
| 71 |
$seg .= "</s>\n" if $s; |
| 72 |
} |
| 73 |
$tu->{$lang}{-iscdata} = $compact ? 0 : 1; |
| 74 |
$seg .= "]]>" unless $compact; |
| 75 |
$tu->{$lang}{-seg} = $seg; |
| 76 |
} |
| 77 |
} |
| 78 |
return $tu; |
| 79 |
}); |
| 80 |
|
| 81 |
$reader = XML::TMX::Reader->new($tmpName); |
| 82 |
|
| 83 |
$reader->for_tu({ |
| 84 |
-output => $output, |
| 85 |
-verbatim => 1, |
| 86 |
-raw => 1, |
| 87 |
-prop => ($compact ? {} : { |
| 88 |
'pos-tagged' => join(",",keys %$pos_languages), |
| 89 |
'pos-fields' => "word,lemma,pos", |
| 90 |
'pos-s-attributes' => ['s'], |
| 91 |
})}, |
| 92 |
sub { |
| 93 |
$_ = $_[0]; |
| 94 |
if (!/\[CDATA\[/) { |
| 95 |
s{(<s>)(.*?)(</s>)} |
| 96 |
{my ($before,$middle,$after) = ($1,$2,$3); |
| 97 |
for ($middle) { |
| 98 |
s/&/&/g;s/</</g;s/>/>/g; |
| 99 |
}; |
| 100 |
$before.$middle.$after |
| 101 |
}ge; |
| 102 |
} |
| 103 |
return $_; }); |
| 104 |
|
| 105 |
#print STDERR "$tmpName\n"; |
| 106 |
unlink $tmpName if -e $tmpName; |
| 107 |
|
| 108 |
sub _init_ma { |
| 109 |
my $lang = shift; |
| 110 |
|
| 111 |
morph($lang, |
| 112 |
QuantitiesDetection => 0, |
| 113 |
MultiwordsDetection => ($mwe // 0), |
| 114 |
NumbersDetection => 0, |
| 115 |
DatesDetection => ($dates // 0), |
| 116 |
NERecognition => ($ner // 0)); |
| 117 |
|
| 118 |
$initted{$lang}++; |
| 119 |
} |
| 120 |
|
| 121 |
sub _dump_words { |
| 122 |
my @words = @_; |
| 123 |
my $seg = ""; |
| 124 |
for my $t (@words) { |
| 125 |
if ($t->is_multiword) { |
| 126 |
$seg .= sprintf("<mwe lema=\"%s\" pos=\"%s\">\n", $t->lemma(), $t->tag()); |
| 127 |
$seg .= _dump_words($t->get_mw_words); |
| 128 |
$seg .= "</mwe>\n"; |
| 129 |
} else { |
| 130 |
$seg .= $t->form() ."\t". $t->lemma() ."\t". $t->tag() ."\n"; |
| 131 |
} |
| 132 |
} |
| 133 |
return $seg; |
| 134 |
} |
| 135 |
|
| 136 |
sub _dump_compact { |
| 137 |
my $seg = join(" " => map { |
| 138 |
my $t = $_; |
| 139 |
($t->is_multiword ? join("_", map { $_->lemma() || $_->lc_form() } $t->get_mw_words) |
| 140 |
: ($t->lemma() || $t->lc_form())) |
| 141 |
. "/" . substr($t->tag(), 0, 1); |
| 142 |
} @_) . "\n"; |
| 143 |
$seg =~ s/&/&/g; |
| 144 |
$seg =~ s/</</g; |
| 145 |
$seg =~ s/>/>/g; |
| 146 |
return $seg; |
| 147 |
} |
| 148 |
|
| 149 |
__END__ |
| 150 |
|
| 151 |
=pod |
| 152 |
|
| 153 |
=encoding UTF-8 |
| 154 |
|
| 155 |
=head1 NAME |
| 156 |
|
| 157 |
tmx-POStagger - POStaggers translation units on a tmx file. |
| 158 |
|
| 159 |
=head1 VERSION |
| 160 |
|
| 161 |
version 0.39 |
| 162 |
|
| 163 |
=head1 SYNOPSIS |
| 164 |
|
| 165 |
tmx-POStagger file.tmx # creates pos_file.tmx |
| 166 |
|
| 167 |
tmx-POStagger -o=out.tmx file.tmx |
| 168 |
-ner ... tags multiword named entities |
| 169 |
-dates |
| 170 |
-compact |
| 171 |
|
| 172 |
=head1 DESCRIPTION |
| 173 |
|
| 174 |
Although this script is bundled in C<XML::TMX>, it has a soft |
| 175 |
dependency on C<Lingua::FreeLing3>. Soft means that the dependency is |
| 176 |
not ensured at install time, and other features of the module can |
| 177 |
still be used without C<Lingua::FreeLing3>. Nevertheless, if you want |
| 178 |
to use this tool you should install that module. |
| 179 |
|
| 180 |
At the moment the supported languages are the same as supported by |
| 181 |
FreeLing3: English, Spanish, Russian, Portuguese and Italian. |
| 182 |
|
| 183 |
It your TMX file includes any other language, they will be maintained |
| 184 |
without a change. This behavior can change in the future, as a basic |
| 185 |
regexp based POStaggerr might be implemented. |
| 186 |
|
| 187 |
=head2 Options |
| 188 |
|
| 189 |
-ner -- groups Proper names with tag C<mwe> |
| 190 |
|
| 191 |
which which WDT |
| 192 |
includes include VBZ |
| 193 |
<mwe lema="edward_cole" pos="NP00000"> |
| 194 |
Edward |
| 195 |
Cole |
| 196 |
</mwe> |
| 197 |
|
| 198 |
-compact |
| 199 |
|
| 200 |
-dates |
| 201 |
|
| 202 |
=head1 SEE ALSO |
| 203 |
|
| 204 |
XML::TMX, Lingua::FreeLing3 |
| 205 |
|
| 206 |
=head1 AUTHORS |
| 207 |
|
| 208 |
=over 4 |
| 209 |
|
| 210 |
=item * |
| 211 |
|
| 212 |
Alberto Simões <[email protected]> |
| 213 |
|
| 214 |
=item * |
| 215 |
|
| 216 |
José João Almeida <[email protected]> |
| 217 |
|
| 218 |
=back |
| 219 |
|
| 220 |
=head1 COPYRIGHT AND LICENSE |
| 221 |
|
| 222 |
This software is copyright (c) 2010-2017 by Projeto Natura <[email protected]>. |
| 223 |
|
| 224 |
This is free software; you can redistribute it and/or modify it under |
| 225 |
the same terms as the Perl 5 programming language system itself. |
| 226 |
|
| 227 |
=cut |