Debian Codegraph @

scripts/tmxclean

raw
1 #!/usr/bin/perl -s
2
3 # PODNAME: tmxclean
4 # ABSTRACT: Simple tool to clean TMX files
5
6 use v5.10;
7 use strict;
8 use warnings;
9
10 use XML::TMX::Reader;
11
12 our (
13 $junk, # remove if one of the languages just have junk
14 $o , $output, # output filename
15 $eq , # remove if seg(l1) = seg(l2)
16 $len , # remove if len(li) > 50 ∧ len(lj) > 2len(li)
17 $v, $verbose,
18 $all,
19 );
20
21 my $cleaned = 0;
22 my $processed = 0;
23 my $tmx = shift or help();
24 my $reader = XML::TMX::Reader->new($tmx);
25
26 $junk//=1;
27 $len //=1 if $all;
28 $eq //=1 if $all;
29
30 $output ||= $o || "_cleaned_$tmx";
31
32 print STDERR "loading..." if $v;
33
34 $reader->for_tu( {output => $output},
35 \&cleaner);
36
37 printf STDERR "\rRemoved %d/%d (%.3f%%).\n",
38 $cleaned, $processed, 100*$cleaned/$processed if $v;
39
40 sub cleaner {
41 my $langs = shift;
42 $processed++;
43 my $remove = 0;
44 my %seg=();
45 my @len=();
46 for my $k (keys %$langs) {
47 next if $k =~ /^-/;
48 $remove = 1 if $eq && $seg{$langs->{$k}{-seg}}++;
49
50 $remove = 1 if $junk && $langs->{$k}{-seg} =~ /^[-.,0-9\s]+$/;
51 $remove = 1 if $junk && $langs->{$k}{-seg} =~ /^\W*$/;
52 push(@len, length($langs->{$k}{-seg}));
53 }
54 @len = sort{$a <=> $b} @len;
55 $remove = 1 if $len && $len[-1] > 50 && $len[0]*2< $len[-1];
56
57 $cleaned++ if $remove;
58 printf STDERR "\rRemoved %d/%d (%.3f%%)...", $cleaned, $processed,
59 100*$cleaned/$processed if $v && $processed%1000==0;
60 return $remove ? undef : $langs;
61 }
62
63 sub help {
64 print " tmxclean [-junk=1] <file.tmx>\n";
65 exit 1;
66 }
67
68 __END__
69
70 =pod
71
72 =encoding UTF-8
73
74 =head1 NAME
75
76 tmxclean - Simple tool to clean TMX files
77
78 =head1 VERSION
79
80 version 0.39
81
82 =head1 SYNOPSIS
83
84 $ tmx-clean file.tmx
85 -junk remove if one of the languages just have junk (def: True)
86 -eq remove if seg(l1) = seg(l2)
87 -len remove if len(li) > 50 ∧ len(lj) > 2len(li)
88 -output=out.tmx
89 -v, -verbose
90 -all same as -junk -eq -len
91
92 =head1 DESCRIPTION
93
94 Removes the translation units (TU) that
95
96 1. have no letters... (unless -junk=0)
97 2. seg(L1) = seg(L2) (if -eq)
98
99 =head1 SEE ALSO
100
101 XML::TMX
102
103 =head1 AUTHORS
104
105 =over 4
106
107 =item *
108
109 Alberto Simões <[email protected]>
110
111 =item *
112
113 José João Almeida <[email protected]>
114
115 =back
116
117 =head1 COPYRIGHT AND LICENSE
118
119 This software is copyright (c) 2010-2017 by Projeto Natura <[email protected]>.
120
121 This is free software; you can redistribute it and/or modify it under
122 the same terms as the Perl 5 programming language system itself.
123
124 =cut