Debian Codegraph @

data/ru/tools/make-hash-db.pl

raw
1 #!/usr/bin/perl -w
2
3 use strict;
4 use warnings;
5 use BerkeleyDB;
6 use vars qw( %file_tail %file_rule %rule %file_root %file_para %para );
7 use Encode;
8
9 # mrd format docs
10 # https://code.google.com/p/opencorpora/wiki/MRDFileFormat
11 # http://aot.ru/docs/sokirko/Dialog2004.htm
12
13 system("rm -f rule.db tail.db root.db para.db");
14
15 tie %file_rule, 'BerkeleyDB::Hash', -Filename => "rule.db", -Flags => DB_CREATE or die "cant\n";
16
17 print "Preparing rules:\n";
18 open(F, "rgramtab20030306.tab") or die("cant");
19 while(<F>) {
20 if(/^\s*$/) { next; }
21 if(/^\/\/.*$/) { next; }
22 chomp; # Ancode, ,PartOfSpeech, Grammems
23 my @a = split(/ /,du($_));
24 my $s;
25 # if($#a>3) {$s=":".$a[4];} else {$s="";}
26 $file_rule{ek($a[0])} = ek($a[2].",".$a[3]);
27 $rule{$a[0]} = $a[2].",".$a[3];
28 }
29 close F;
30 untie %file_rule;
31
32 $| = 1;
33
34 tie %file_para, 'BerkeleyDB::Hash', -Filename => "para.db", -Flags => DB_CREATE or die "cant\n";
35 tie %file_tail, 'BerkeleyDB::Hash', -Filename => "tail.db", -Flags => DB_CREATE or die "cant\n";
36 open(F, "morphs20030410.mrd") or die("cant");
37 my $nt = <F>; chomp $nt;
38 print "Preparing tails:\n";
39 for(my $i=0; $i<$nt; $i++) {
40 my $s = <F>;
41 chomp $s;
42 $file_para{ek($i)} = ek(du($s));
43 $para{$i} = du($s);
44 my @a = split(/%/, du($s));
45 shift @a;
46 my @rulestr = ();
47 my $state = 0;
48 for my $a (@a) {
49 my ($tail ,$rulestr) = split /\*/, $a;
50 $tail = lc $tail;
51 $file_tail{ek("$i:$tail")} = ek($rulestr);
52 }
53 print "." if ($i % 100 == 0);
54 }
55 untie %file_tail;
56 untie %file_para;
57
58 my %examples = (); my %para_freq = ();
59
60 my $nr = <F>; chomp $nr;
61 print "\nPreparing roots:\n";
62 tie %file_root, 'BerkeleyDB::Hash', -Filename => "root.db", -Flags => DB_CREATE or die "cant\n";
63 my $ni=0;
64 while(<F>) {
65 chomp;
66 my $s = lc(du($_));
67 my ($str, $num) = split(/ /, $s);
68 $str = "" if $str eq "#";
69 $file_root{ek($str)} .= ek(":$num");
70 $para_freq{$num}++;
71 print "." if ( $ni++ % 1000 eq 0 );
72 my $type = $para{$num};
73 my @tpl = map { $str.$_ } split(/%/,$type); shift @tpl;
74 my @gra = @tpl;
75 s/\*.+?$// foreach @tpl;
76 s/^.+?\*// foreach @gra;
77 foreach my $i ( 0 .. $#gra ) {
78 my $c = $gra[$i];
79 my @anlist = unpack "A2" x (length($c)/2), $c;
80 foreach my $ancode ( @anlist ) {
81 my @cases = defined $examples{$ancode} ? @{ $examples{$ancode} } : ();
82 if ( $#cases < 5 ) {
83 push @{ $examples{$ancode} }, lc($tpl[$i]);
84 }
85 }
86 }
87 }
88 untie %file_root;
89
90 open(EXM,"|sort -k 2 > morph-examp.txt") or die("cant");
91 foreach my $ancode ( keys %examples ) {
92 my @cases = @{ $examples{$ancode} };
93 my $rl = $rule{$ancode};
94 print EXM eu("$ancode\t$rl\t".join("\t",@cases))."\n";
95 }
96 close(EXM);
97
98 open(ANC,">ancode-not-defined.txt") or die("cant");
99 foreach my $ancode ( sort { $rule{$a} cmp $rule{$b} } keys %rule ) {
100 if ( ! defined $examples{$ancode} ) {
101 print ANC eu("$ancode\t$rule{$ancode}\n");
102 }
103 }
104 close(ANC);
105
106 open(PAR,">paradigm-not-used.txt") or die("cant");
107 foreach my $i ( sort { $a <=> $b } keys %para ) {
108 if ( ! defined $para_freq{$i} ) {
109 print PAR eu("$i\t$para{$i}\n");
110 }
111 }
112 close(PAR);
113
114 print "\ntails:$nt\troots:$nr\n";
115
116 sub eu {
117 return encode("utf-8",shift);
118 }
119
120 sub du {
121 return decode("utf-8",shift);
122 }
123
124 sub ek {
125 return encode("koi8-r",shift);
126 }
127
128 sub dk {
129 return decode("koi8-r",shift);
130 }
131