Debian Codegraph @

scripts/tmx-POStagger

raw
1 #!/usr/bin/perl -s
2
3 # PODNAME: tmx-POStagger
4 # ABSTRACT: POStaggers translation units on a tmx file.
5
6 use strict;
7 use warnings;
8 use File::Temp qw/ :POSIX /;
9
10 my $tmpName = tmpnam();
11
12 our (
13 $o, # output filename
14 $mwe, # tag mwe -- STILL NOT SUPPORTED
15 $dates, # do dates analysis...
16 $ner, # try to do NER....
17 $compact, # compact form
18 $s, # use <s> tags (on by default)
19 );
20
21 $s = 1 unless defined $s;
22
23 eval { require FL3 };
24 die "This XML::TMX script requires Lingua::FreeLing3 to be installed\n" if $@;
25 FL3->import();
26
27 eval { require Lingua::FreeLing3::Word };
28 die "This XML::TMX script requires Lingua::FreeLing3 to be installed\n" if $@;
29 Lingua::FreeLing3::Word->import();
30
31 my %initted = ();
32
33 use XML::TMX::Reader '0.25';
34
35 my $file = shift or die "You must supply the name of the file to tokenize";
36
37 my $reader = XML::TMX::Reader->new($file);
38
39 my $output = "pos_$file";
40 $output = $o if $o;
41
42 my $pos_languages = {};
43
44 binmode STDOUT, ":utf8";
45 $reader->for_tu(
46 {
47 -output => $tmpName,
48 -verbose => 1,
49 -verbatim => 1,
50 },
51 sub {
52 my $tu = shift;
53 for my $lang (keys %$tu) {
54 if ($lang =~ /(pt|es|gl|it|fr|ru|en)/i) {
55 $pos_languages->{$1} = 1;
56
57 my $ln = lc $1;
58 my $seg = $compact ? "" : "<![CDATA[";
59
60 _init_ma($ln) unless exists $initted{$ln};
61
62 my $txt = $tu->{$lang}{-seg};
63 my @tokens = map { Lingua::FreeLing3::Word->new($_) } split /\s+/, $txt;
64 my $sentences = splitter($ln)->split(\@tokens);
65 $sentences = morph($ln)->analyze($sentences);
66 $sentences = hmm($ln)->tag($sentences);
67 for my $stc (@$sentences) {
68 $seg .= "<s>\n" if $s;
69 $seg .= $compact ? _dump_compact($stc->words)
70 : _dump_words($stc->words);
71 $seg .= "</s>\n" if $s;
72 }
73 $tu->{$lang}{-iscdata} = $compact ? 0 : 1;
74 $seg .= "]]>" unless $compact;
75 $tu->{$lang}{-seg} = $seg;
76 }
77 }
78 return $tu;
79 });
80
81 $reader = XML::TMX::Reader->new($tmpName);
82
83 $reader->for_tu({
84 -output => $output,
85 -verbatim => 1,
86 -raw => 1,
87 -prop => ($compact ? {} : {
88 'pos-tagged' => join(",",keys %$pos_languages),
89 'pos-fields' => "word,lemma,pos",
90 'pos-s-attributes' => ['s'],
91 })},
92 sub {
93 $_ = $_[0];
94 if (!/\[CDATA\[/) {
95 s{(<s>)(.*?)(</s>)}
96 {my ($before,$middle,$after) = ($1,$2,$3);
97 for ($middle) {
98 s/&/&amp;/g;s/</&lt;/g;s/>/&gt;/g;
99 };
100 $before.$middle.$after
101 }ge;
102 }
103 return $_; });
104
105 #print STDERR "$tmpName\n";
106 unlink $tmpName if -e $tmpName;
107
108 sub _init_ma {
109 my $lang = shift;
110
111 morph($lang,
112 QuantitiesDetection => 0,
113 MultiwordsDetection => ($mwe // 0),
114 NumbersDetection => 0,
115 DatesDetection => ($dates // 0),
116 NERecognition => ($ner // 0));
117
118 $initted{$lang}++;
119 }
120
121 sub _dump_words {
122 my @words = @_;
123 my $seg = "";
124 for my $t (@words) {
125 if ($t->is_multiword) {
126 $seg .= sprintf("<mwe lema=\"%s\" pos=\"%s\">\n", $t->lemma(), $t->tag());
127 $seg .= _dump_words($t->get_mw_words);
128 $seg .= "</mwe>\n";
129 } else {
130 $seg .= $t->form() ."\t". $t->lemma() ."\t". $t->tag() ."\n";
131 }
132 }
133 return $seg;
134 }
135
136 sub _dump_compact {
137 my $seg = join(" " => map {
138 my $t = $_;
139 ($t->is_multiword ? join("_", map { $_->lemma() || $_->lc_form() } $t->get_mw_words)
140 : ($t->lemma() || $t->lc_form()))
141 . "/" . substr($t->tag(), 0, 1);
142 } @_) . "\n";
143 $seg =~ s/&/&amp;/g;
144 $seg =~ s/</&lt;/g;
145 $seg =~ s/>/&gt;/g;
146 return $seg;
147 }
148
149 __END__
150
151 =pod
152
153 =encoding UTF-8
154
155 =head1 NAME
156
157 tmx-POStagger - POStaggers translation units on a tmx file.
158
159 =head1 VERSION
160
161 version 0.39
162
163 =head1 SYNOPSIS
164
165 tmx-POStagger file.tmx # creates pos_file.tmx
166
167 tmx-POStagger -o=out.tmx file.tmx
168 -ner ... tags multiword named entities
169 -dates
170 -compact
171
172 =head1 DESCRIPTION
173
174 Although this script is bundled in C<XML::TMX>, it has a soft
175 dependency on C<Lingua::FreeLing3>. Soft means that the dependency is
176 not ensured at install time, and other features of the module can
177 still be used without C<Lingua::FreeLing3>. Nevertheless, if you want
178 to use this tool you should install that module.
179
180 At the moment the supported languages are the same as supported by
181 FreeLing3: English, Spanish, Russian, Portuguese and Italian.
182
183 It your TMX file includes any other language, they will be maintained
184 without a change. This behavior can change in the future, as a basic
185 regexp based POStaggerr might be implemented.
186
187 =head2 Options
188
189 -ner -- groups Proper names with tag C<mwe>
190
191 which which WDT
192 includes include VBZ
193 <mwe lema="edward_cole" pos="NP00000">
194 Edward
195 Cole
196 </mwe>
197
198 -compact
199
200 -dates
201
202 =head1 SEE ALSO
203
204 XML::TMX, Lingua::FreeLing3
205
206 =head1 AUTHORS
207
208 =over 4
209
210 =item *
211
212 Alberto Simões <[email protected]>
213
214 =item *
215
216 José João Almeida <[email protected]>
217
218 =back
219
220 =head1 COPYRIGHT AND LICENSE
221
222 This software is copyright (c) 2010-2017 by Projeto Natura <[email protected]>.
223
224 This is free software; you can redistribute it and/or modify it under
225 the same terms as the Perl 5 programming language system itself.
226
227 =cut