Debian Codegraph @

scripts/tmx-tokenize

raw
1 #!/usr/bin/perl -s
2
3 # PODNAME: tmx-tokenize
4 # ABSTRACT: Tokenizes translation units on a tmx file.
5
6 use strict;
7 use warnings;
8
9 our $o;
10
11 eval { require FL3 };
12 die "This XML::TMX script requires Lingua::FreeLing3 to be installed\n" if $@;
13
14 FL3->import();
15 use XML::TMX::Reader '0.25';
16
17 my $file = shift or die "You must supply the name of the file to tokenize";
18
19 my $reader = XML::TMX::Reader->new($file);
20
21 my $output = "t_$file";
22 $output = $o if $o;
23
24 binmode STDOUT, ":utf8";
25 $reader->for_tu( {
26 -output => $output,
27 -prop => { tokenized => "true" },
28 verbose => 1
29 },
30 sub {
31 my $tu = shift;
32 for my $lang (keys %$tu) {
33 if ($lang =~ /(pt|es|it|ru|en|gl)/i) {
34 my $ln = lc $1;
35 my $txt = $tu->{$lang}{-seg};
36 if ($txt !~ /^\s*$/) {
37 $txt = join(" ",
38 @{ tokenizer($ln)->tokenize($txt,
39 to_text => 1)});
40 }
41 $tu->{$lang}{-seg} = $txt;
42 }
43 }
44 return $tu;
45 });
46
47 __END__
48
49 =pod
50
51 =encoding UTF-8
52
53 =head1 NAME
54
55 tmx-tokenize - Tokenizes translation units on a tmx file.
56
57 =head1 VERSION
58
59 version 0.39
60
61 =head1 SYNOPSIS
62
63 tmx-tokenize file.tmx # creates t_file.tmx
64
65 tmx-tokenize -o=out.tmx file.tmx
66
67 =head1 DESCRIPTION
68
69 Although this script is bundled in C<XML::TMX>, it has a soft
70 dependency on C<Lingua::FreeLing3>. Soft means that the dependency is
71 not ensured at install time, and other features of the module can
72 still be used without C<Lingua::FreeLing3>. Nevertheless, if you want
73 to use this tool you should install that module.
74
75 At the moment the supported languages are the same as supported by
76 FreeLing3: English, Spanish, Russian, Portuguese and Italian.
77
78 It your TMX file includes any other language, they will be maintained
79 without a change. This behavior can change in the future, as a basic
80 regexp based tokenizer might be implemented.
81
82 =head1 SEE ALSO
83
84 XML::TMX, Lingua::FreeLing3
85
86 =head1 AUTHORS
87
88 =over 4
89
90 =item *
91
92 Alberto Simões <[email protected]>
93
94 =item *
95
96 José João Almeida <[email protected]>
97
98 =back
99
100 =head1 COPYRIGHT AND LICENSE
101
102 This software is copyright (c) 2010-2017 by Projeto Natura <[email protected]>.
103
104 This is free software; you can redistribute it and/or modify it under
105 the same terms as the Perl 5 programming language system itself.
106
107 =cut