| 1 |
#!/usr/bin/perl -w |
| 2 |
|
| 3 |
use strict; |
| 4 |
use warnings; |
| 5 |
use BerkeleyDB; |
| 6 |
use vars qw( %file_tail %file_rule %rule %file_root %file_para %para ); |
| 7 |
use Encode; |
| 8 |
|
| 9 |
# mrd format docs |
| 10 |
# https://code.google.com/p/opencorpora/wiki/MRDFileFormat |
| 11 |
# http://aot.ru/docs/sokirko/Dialog2004.htm |
| 12 |
|
| 13 |
system("rm -f rule.db tail.db root.db para.db"); |
| 14 |
|
| 15 |
tie %file_rule, 'BerkeleyDB::Hash', -Filename => "rule.db", -Flags => DB_CREATE or die "cant\n"; |
| 16 |
|
| 17 |
print "Preparing rules:\n"; |
| 18 |
open(F, "rgramtab20030306.tab") or die("cant"); |
| 19 |
while(<F>) { |
| 20 |
if(/^\s*$/) { next; } |
| 21 |
if(/^\/\/.*$/) { next; } |
| 22 |
chomp; # Ancode, ,PartOfSpeech, Grammems |
| 23 |
my @a = split(/ /,du($_)); |
| 24 |
my $s; |
| 25 |
# if($#a>3) {$s=":".$a[4];} else {$s="";} |
| 26 |
$file_rule{ek($a[0])} = ek($a[2].",".$a[3]); |
| 27 |
$rule{$a[0]} = $a[2].",".$a[3]; |
| 28 |
} |
| 29 |
close F; |
| 30 |
untie %file_rule; |
| 31 |
|
| 32 |
$| = 1; |
| 33 |
|
| 34 |
tie %file_para, 'BerkeleyDB::Hash', -Filename => "para.db", -Flags => DB_CREATE or die "cant\n"; |
| 35 |
tie %file_tail, 'BerkeleyDB::Hash', -Filename => "tail.db", -Flags => DB_CREATE or die "cant\n"; |
| 36 |
open(F, "morphs20030410.mrd") or die("cant"); |
| 37 |
my $nt = <F>; chomp $nt; |
| 38 |
print "Preparing tails:\n"; |
| 39 |
for(my $i=0; $i<$nt; $i++) { |
| 40 |
my $s = <F>; |
| 41 |
chomp $s; |
| 42 |
$file_para{ek($i)} = ek(du($s)); |
| 43 |
$para{$i} = du($s); |
| 44 |
my @a = split(/%/, du($s)); |
| 45 |
shift @a; |
| 46 |
my @rulestr = (); |
| 47 |
my $state = 0; |
| 48 |
for my $a (@a) { |
| 49 |
my ($tail ,$rulestr) = split /\*/, $a; |
| 50 |
$tail = lc $tail; |
| 51 |
$file_tail{ek("$i:$tail")} = ek($rulestr); |
| 52 |
} |
| 53 |
print "." if ($i % 100 == 0); |
| 54 |
} |
| 55 |
untie %file_tail; |
| 56 |
untie %file_para; |
| 57 |
|
| 58 |
my %examples = (); my %para_freq = (); |
| 59 |
|
| 60 |
my $nr = <F>; chomp $nr; |
| 61 |
print "\nPreparing roots:\n"; |
| 62 |
tie %file_root, 'BerkeleyDB::Hash', -Filename => "root.db", -Flags => DB_CREATE or die "cant\n"; |
| 63 |
my $ni=0; |
| 64 |
while(<F>) { |
| 65 |
chomp; |
| 66 |
my $s = lc(du($_)); |
| 67 |
my ($str, $num) = split(/ /, $s); |
| 68 |
$str = "" if $str eq "#"; |
| 69 |
$file_root{ek($str)} .= ek(":$num"); |
| 70 |
$para_freq{$num}++; |
| 71 |
print "." if ( $ni++ % 1000 eq 0 ); |
| 72 |
my $type = $para{$num}; |
| 73 |
my @tpl = map { $str.$_ } split(/%/,$type); shift @tpl; |
| 74 |
my @gra = @tpl; |
| 75 |
s/\*.+?$// foreach @tpl; |
| 76 |
s/^.+?\*// foreach @gra; |
| 77 |
foreach my $i ( 0 .. $#gra ) { |
| 78 |
my $c = $gra[$i]; |
| 79 |
my @anlist = unpack "A2" x (length($c)/2), $c; |
| 80 |
foreach my $ancode ( @anlist ) { |
| 81 |
my @cases = defined $examples{$ancode} ? @{ $examples{$ancode} } : (); |
| 82 |
if ( $#cases < 5 ) { |
| 83 |
push @{ $examples{$ancode} }, lc($tpl[$i]); |
| 84 |
} |
| 85 |
} |
| 86 |
} |
| 87 |
} |
| 88 |
untie %file_root; |
| 89 |
|
| 90 |
open(EXM,"|sort -k 2 > morph-examp.txt") or die("cant"); |
| 91 |
foreach my $ancode ( keys %examples ) { |
| 92 |
my @cases = @{ $examples{$ancode} }; |
| 93 |
my $rl = $rule{$ancode}; |
| 94 |
print EXM eu("$ancode\t$rl\t".join("\t",@cases))."\n"; |
| 95 |
} |
| 96 |
close(EXM); |
| 97 |
|
| 98 |
open(ANC,">ancode-not-defined.txt") or die("cant"); |
| 99 |
foreach my $ancode ( sort { $rule{$a} cmp $rule{$b} } keys %rule ) { |
| 100 |
if ( ! defined $examples{$ancode} ) { |
| 101 |
print ANC eu("$ancode\t$rule{$ancode}\n"); |
| 102 |
} |
| 103 |
} |
| 104 |
close(ANC); |
| 105 |
|
| 106 |
open(PAR,">paradigm-not-used.txt") or die("cant"); |
| 107 |
foreach my $i ( sort { $a <=> $b } keys %para ) { |
| 108 |
if ( ! defined $para_freq{$i} ) { |
| 109 |
print PAR eu("$i\t$para{$i}\n"); |
| 110 |
} |
| 111 |
} |
| 112 |
close(PAR); |
| 113 |
|
| 114 |
print "\ntails:$nt\troots:$nr\n"; |
| 115 |
|
| 116 |
sub eu { |
| 117 |
return encode("utf-8",shift); |
| 118 |
} |
| 119 |
|
| 120 |
sub du { |
| 121 |
return decode("utf-8",shift); |
| 122 |
} |
| 123 |
|
| 124 |
sub ek { |
| 125 |
return encode("koi8-r",shift); |
| 126 |
} |
| 127 |
|
| 128 |
sub dk { |
| 129 |
return decode("koi8-r",shift); |
| 130 |
} |
| 131 |
|