| 1 |
#!/usr/bin/perl -s |
| 2 |
|
| 3 |
# PODNAME: tmx-explode |
| 4 |
# ABSTRACT: Explodes tmx files in a file per language |
| 5 |
|
| 6 |
use strict ; |
| 7 |
use Lingua::NATools::Client; |
| 8 |
use Data::Dumper; |
| 9 |
$Data::Dumper::Indent=1; |
| 10 |
$Data::Dumper::Terse=1; |
| 11 |
|
| 12 |
my $Gn1={}; |
| 13 |
my $Gn2={}; |
| 14 |
my $Geq=0; |
| 15 |
my $Geq2=0; |
| 16 |
my $t; |
| 17 |
my $perfeitas =0; |
| 18 |
my $nonumber =0; |
| 19 |
my $pessimas =0; |
| 20 |
my $delnumbers=0; |
| 21 |
my $desiguaist=0; # size(bigger) > 2(size(shorter)) |
| 22 |
my $iguais=0; |
| 23 |
our $debug = 1; |
| 24 |
my $tmx = shift or die; |
| 25 |
|
| 26 |
use XML::TMX::Reader; |
| 27 |
|
| 28 |
my $t = XML::TMX::Reader->new( $tmx ); |
| 29 |
|
| 30 |
my $tun = 0; |
| 31 |
print "\n================($tmx)========\n"; |
| 32 |
|
| 33 |
$t->for_tu( \&mycmp ); |
| 34 |
|
| 35 |
print "\n comon numbers = ",MScardinter($Gn1,$Gn2)*2, |
| 36 |
"\n eq-same-TU numbers = ", $Geq*2 , |
| 37 |
"\n total numbers = ", $t= MScard($Gn1)+MScard($Gn2), |
| 38 |
"\n TU = $tun", |
| 39 |
"\n good TU (Num>0) = $perfeitas", |
| 40 |
"\n good TU (Num=0) = $nonumber", |
| 41 |
"\n very bad TU = $pessimas ($delnumbers)", |
| 42 |
"\n equal TU = $iguais", |
| 43 |
"\n size too diff TU = $desiguaist", |
| 44 |
(($Geq*2.4 < $t )? "\n PROBLEMS - numbers too diff." : ""), |
| 45 |
(($pessimas / $tun > 0.2 )? "\n PROBLEMS - too many bad TU" : ""), |
| 46 |
"\n After del. bad TU:". $Geq2*2, "/" , $t-$delnumbers,"=", |
| 47 |
($Geq2*2)/( $t-$delnumbers), |
| 48 |
"\n\n"; |
| 49 |
|
| 50 |
sub mycmp{ |
| 51 |
$tun++; |
| 52 |
my $tu = shift; |
| 53 |
if(length($tu->{en}) > 30 && (length($tu->{en}) > 2 * length($tu->{pt}) || |
| 54 |
length($tu->{pt}) > 2 * length($tu->{en}))){ |
| 55 |
print "remove $tun = size is too different\n" if $debug ; |
| 56 |
$desiguaist ++ ; |
| 57 |
} |
| 58 |
elsif(length($tu->{en}) > 30 && $tu->{en} eq $tu->{pt}){ |
| 59 |
print "remove $tun = the TUs are equal.\n" if $debug ; |
| 60 |
$iguais ++ ; |
| 61 |
} |
| 62 |
my $n1 = getn($tu->{en}); |
| 63 |
my $n2 = getn($tu->{pt}); |
| 64 |
my $n3 = MScardinter($n1,$n2); |
| 65 |
$Geq += $n3; |
| 66 |
$Geq2 += $n3; |
| 67 |
my $n4 = MScard($n1)+MScard($n2) ; |
| 68 |
$perfeitas ++ if $n4 == $n3*2 && $n4 > 0; |
| 69 |
$nonumber ++ if $n4 == $n3*2 && $n4 == 0; |
| 70 |
if((3+ $n3) * 2.4 < 3+$n4 ) { |
| 71 |
$delnumbers += $n4; |
| 72 |
$Geq2 -= $n3; |
| 73 |
$pessimas ++; |
| 74 |
print "remove $tun = Number in this TUs are too different\n" if $debug; |
| 75 |
print " ***EN\t$tu->{en}\n ***PT\t$tu->{pt}\n" if $debug; |
| 76 |
} |
| 77 |
MSaddto($Gn1,$n1); |
| 78 |
MSaddto($Gn2,$n2); |
| 79 |
} |
| 80 |
|
| 81 |
sub MSaddto{ # a = a U b |
| 82 |
my ($m1,$m2)=@_; |
| 83 |
for(keys %$m2){$m1->{$_} += $m2->{$_}} |
| 84 |
} |
| 85 |
|
| 86 |
sub MScardinter{ # #(a /\ b) |
| 87 |
my ($m1,$m2)=@_; |
| 88 |
my $com = 0; |
| 89 |
for(keys %$m2){ $com += $m1->{$_} > $m2->{$_} ? $m2->{$_} : $m1->{$_}} |
| 90 |
$com |
| 91 |
} |
| 92 |
|
| 93 |
sub MScard{ # Multiset cardinal |
| 94 |
my ($m1)=@_; |
| 95 |
my $c = 0; |
| 96 |
for(values %$m1){ $c += $_ } |
| 97 |
$c |
| 98 |
} |
| 99 |
|
| 100 |
sub getn{ # get the multiset of the numbers present in the sentence |
| 101 |
my $s=shift; |
| 102 |
my %a=(); |
| 103 |
for( $s =~ m/(\d+)/g){$a{$_}++} |
| 104 |
\%a |
| 105 |
} |
| 106 |
|
| 107 |
|
| 108 |
|
| 109 |
__END__ |
| 110 |
|