Debian Codegraph @

scripts/tmxuniq

raw
1 #!/usr/bin/perl -s
2
3 # PODNAME: tmxuniq
4 # ABSTRACT: removes duplicated translation units from TMXs
5
6 BEGIN {
7 eval "require Lingua::PT::PLNbase";
8 if ($@) {
9 print STDERR
10 "This script requires Lingua::PT::PLNbase to run. We suggest installing\n",
11 "it using CPAN: http://www.cpan.org/modules/INSTALL.html\n";
12 exit(0);
13 }
14 }
15
16 use DB_File;
17 use Fcntl ;
18 use XML::TMX::Reader;
19 use Digest::MD5 qw(md5_hex);
20 use Encode;
21 use Lingua::PT::PLNbase
22
23 our ($cont,$id,$dig,$tok,$o,$fast);
24
25 if ($cont) {
26 tie %dic, 'DB_File', "__tmxuniq_$$.db", O_RDWR|O_CREAT , 0640, $DB_BTREE;
27 } else {
28 tie %dic, 'DB_File', "__tmxuniq_$$.db", O_RDWR|O_CREAT|O_TRUNC , 0640, $DB_BTREE;
29 }
30
31 my $cid = 0;
32
33 for my $file (@ARGV){
34 my $tm = XML::TMX::Reader->new($file);
35
36 print STDERR "Processing...";
37
38 $tm->for_tu
39 (
40 { output => $o || "$file._" },
41 sub {
42 my $tu = shift;
43 $cid++;
44 $tu->{-prop}{id} = $cid if $id;
45
46 my $key = join("|||", map { n($tu->{$_}{-seg}) } sort grep { !/^-/ } keys %$tu);
47 my $digest = md5_hex(encode_utf8($key));
48
49 unless ($cid % 10000) {
50 my $size = -s "__tmxuniq_$$.db";
51 printf STDERR
52 "\rTotal: %10d Removed: %8d (%.2f%%) Database size: %10d bytes",
53 $cid, $rem, (100*$rem/$cid), $size;
54 }
55
56 if ($dic{$digest}) {
57 $dic{$digest} .= "$cid;" unless $fast;
58 $rem ++;
59 return undef
60 } else {
61 $dic{$digest} = "$cid;";
62 $tu->{-prop}{digest} = $digest if $dig;
63 return {%$tu} ; # used clone.. no idea why
64 }
65 }
66 );
67
68 my $size = -s "__tmxuniq_$$.db";
69 if ($cid) {
70 printf STDERR "\rTotal: %10d Removed: %8d (%.2f%%) Database size: %10d bytes\n",
71 $cid, $rem, (100*$rem/$cid), $size;
72 } else {
73 printf STDERR "\rHuh.. empty TMX?\n";
74 }
75 undef $tm;
76 }
77 untie %h;
78
79 sub n {
80 my $a = shift;
81
82 $a =~ s/\.{6,}/...../g;
83
84 $a = tokenize( { rs => ' ' } => $a ) if $tok;
85
86 $a =~ s/\s+/ /g;
87 $a =~ s/ $//;
88 $a =~ s/^ //;
89 return $a;
90 }
91
92 __END__
93
94 =pod
95
96 =encoding UTF-8
97
98 =head1 NAME
99
100 tmxuniq - removes duplicated translation units from TMXs
101
102 =head1 VERSION
103
104 version 0.39
105
106 =head1 SYNOPSIS
107
108 tmxuniq [options] -l=en:pt tmx1 ...
109
110 =head1 DESCRIPTION
111
112 Removes duplicated translation units from a set of TMX (Translation
113 Memory eXange format).
114
115 =head1 OPTIONS
116
117 -id -- insert a uniq id property in each TU
118 -dig -- insert a digest property in each TU
119 -tok -- tokenize/normalize text
120 -o=out.tmx -- (with 1 argument) redefine output (default = input._)
121
122 =head1 SEE ALSO
123
124 perl(1).
125
126 =head1 AUTHORS
127
128 =over 4
129
130 =item *
131
132 Alberto Simões <[email protected]>
133
134 =item *
135
136 José João Almeida <[email protected]>
137
138 =back
139
140 =head1 COPYRIGHT AND LICENSE
141
142 This software is copyright (c) 2010-2017 by Projeto Natura <[email protected]>.
143
144 This is free software; you can redistribute it and/or modify it under
145 the same terms as the Perl 5 programming language system itself.
146
147 =cut