Debian Codegraph @

data/ru/tools/mkdict.pl

raw
1 #!/usr/bin/perl
2 #
3 # Generate Russian dictionaries for Link Grammar.
4 # Based on aot.ru opensource files
5 #
6 # Copyright (c) 2004, 2012, 2013 Sergey Protasov
7 # Copyright (c) 2012, 2013 Linas Vepstas
8
9 use strict;
10 use warnings;
11
12 use BerkeleyDB;
13 use Encode;
14 use utf8;
15 use vars qw( %rule %tail %root );
16
17 tie %rule, "BerkeleyDB::Hash", -Filename => "rule.db", -Flags => DB_RDONLY or die "run ./make-hash-db.pl first!";
18 tie %tail, "BerkeleyDB::Hash", -Filename => "tail.db", -Flags => DB_RDONLY or die "run ./make-hash-db.pl first!";
19 tie %root, "BerkeleyDB::Hash", -Filename => "root.db", -Flags => DB_RDONLY or die "run ./make-hash-db.pl first!";
20
21 sub lgtrim($) {
22 my ($cur) = @_;
23 $_ = $cur;
24 chomp;
25 s/имя,//;
26 s/фам,//;
27 s/Н,относ/Н/;
28 s/Н,вопр>/Н/;
29 return $_;
30 }
31
32 sub alnum($) {
33 my ($i) = @_;
34
35 my $a = int($i/26); my $va = $i % 26;
36 my $b = int($a/26); my $vb = $a % 26;
37 my $c = int($b/26); my $vc = $b % 26;
38
39 my @lets = ('A'..'Z');
40
41 my $la = $lets[$va];
42 my $lb = $lets[$vb];
43 my $lc = $lets[$vc];
44 return "$lc$lb$la";
45 }
46
47 # These words are defined in the base.dict ( 4.0.dict )
48 my @nonwords1 = ("бы.p", "когда.i", "если.i", "и.i", "как.e",
49 "не.p", "отныне.e", "отовсюду.e", "задом.e", "вполоборота.e",
50 "боком.e", "велик.amss");
51 my %skipwords = (); $skipwords{$_}++ foreach @nonwords1;
52
53 my @nonwords2 = qw(не как где и если когда два две оба обе три четыре);
54 my $rHskipstemcon = ();
55 foreach my $nw1 ( @nonwords2 ) {
56 for(my $i=0; $i <= length $nw1; $i++) {
57 my ($left, $right) = (substr($nw1,0,$i), substr($nw1,$i));
58 if(my $d = $root{ek($left)}) {
59 my @num = split(/:/, $d);
60 shift @num;
61 foreach my $num(@num){
62 if(my $c = dk($tail{ek("$num:$right")})) {
63 my $alcon = "LL".alnum($num)."+";
64 $rHskipstemcon->{$left}->{$num}++;
65 $rHskipstemcon->{"=".$right}->{$num}++;
66 #print eu("$i\t$nw1\t$left\t$right\t$num\t$alcon\t$c\n");
67 }
68 }
69 }
70 }
71 }
72
73 my %nums_for_empty_stem = ();
74 while (my ($stem, $val) = each(%root)){
75 next unless $stem eq "";
76 chomp $val; $val =~ s/^://;
77 my @nums = split(/:/, $val);
78 $nums_for_empty_stem{$_}++ foreach @nums;
79 }
80
81 print scalar(keys %rule) . " n rules\n";
82
83 my $tcnt=0; my %uniqon = (); my %lefties = (); my %righties = (); my %suftags = ();
84 my %alltags = (); my $rHse = {};
85
86 while (my ($numsuf, $modestr) = each(%tail)){
87 $numsuf = dk($numsuf);
88
89 chomp $numsuf;
90 my ($lcon, $suf) = split(/:/, $numsuf);
91 $suf = "=".$suf;
92 $uniqon{$lcon} += 1; # 439
93
94 my @modeli = unpack "A2" x (length($modestr)/2), dk($modestr);
95 my @res = ();
96 foreach my $mode ( @modeli ) { #
97 my $pos = defined $rule{ek($mode)} ? dk($rule{ek($mode)}) : "U";
98 chomp $pos;
99 print eu($mode) . " U\n" if ( $pos eq "U" );
100 #print eu($pos)."\n"; # Г,св,пе,прч,прш,дст,ед,мр,тв
101 # copied straight from TestLem.pm
102 if ( $pos =~ /мр-жр/ ) {
103 $pos =~ s/мр-жр/мр/; push @res, $pos;
104 $pos =~ s/мр/жр/; push @res, $pos;
105 } else {
106 push @res, $pos;
107 }
108 #print eu($pos) . "\n";
109 }
110
111 # copied straight from TestLem.pm
112 # проверка винительного одушевленного
113 for (my $i=0; $i <= $#res; $i++) {
114 if($res[$i] =~ /П(.*мр.*)вн.*/ || $res[$i] =~ /П(.*мн.*)вн.*/) {
115 my $rr = $1;
116 my $l = 0; my $n = 0;
117 foreach (@res) {
118 my $rrr = $rr."рд";
119 my $rri = $rr."им";
120 $l = 1 if /П$rrr.*/; # correct
121 $n = 1 if /П$rri.*/;
122 #$l = 1 if /П$rrрд.*/; # incorrect, but match with TestLem.pm
123 #$n = 1 if /П$rrим.*/;
124 }
125 $res[$i] .= ",од" if $l;
126 $res[$i] .= ",но" if $n;
127 }
128 #$rHse->{$lcon}->{$res[$i]}++ if ($suf eq "="); # 439
129 #print "$lcon ".eu($res[$i])."\n" if ($suf eq "=");
130 }
131
132 # copied straight from TestLem.pm
133 # Get rid of duplicates
134 my %dres = ();
135 foreach my $pos(@res) {
136 my $tpos = lgtrim($pos);
137 $rHse->{$lcon}->{$tpos}++ if ($suf eq "=");
138 #print "$lcon ".eu($tpos)."\n" if ($suf eq "=");
139 $dres{$tpos} = 1;
140 }
141 my @uni = keys %dres;
142
143 foreach my $pos(@uni) {
144 $alltags{$pos}++;
145 my $suftag = "$suf.".parse($pos);
146
147 if ( not defined $nums_for_empty_stem{$lcon} ) {
148 my $alcon = alnum($lcon);
149 my $lcontor = "LL$alcon-";
150 if ( not defined $rHskipstemcon->{$suf}->{$lcon} ) {
151 $lefties{$suftag} .= $lcontor . " ";
152 }
153 } else {
154 $suftag =~ s/^=//;
155 }
156
157 my $rcontor = "<morph-" . $pos . ">";
158 $righties{$suftag} .= $rcontor . " ";
159 #print eu($suftag)." .=\t$rcontor\n";
160
161 $suftags{$suftag} = 1;
162 }
163 $tcnt++;
164 }
165
166 my $scnt = scalar(keys %suftags);
167
168 print "gen suffix\n";
169 open(SUF,">suffix.dict") or die("cant");
170 print SUF "%\n";
171 print SUF "% Tagged Suffix Table\n";
172 print SUF "%\n";
173 print SUF "% Total number of suffixes = $tcnt\n";
174 print SUF "% Total number of tagged suffixes = $scnt\n";
175 print "% Total number of suffixes = $tcnt\n";
176 print "% Total number of tagged suffixes = $scnt\n";
177
178 my $un = scalar(keys %uniqon);
179 print SUF "% Number of unique stem-to-suffix connectors = $un\n";
180 print SUF "%\n";
181
182 my @suftagli = keys %suftags;
183 my @ssuftagli = sort @suftagli;
184
185 my %common=();
186 my $nsf = 0;
187 foreach my $suftag ( @ssuftagli ) {
188 my $left = defined $lefties{$suftag} ? $lefties{$suftag} : "";
189 my $right = defined $righties{$suftag} ? $righties{$suftag} : "";
190
191 chomp $left; chomp $right;
192 #print eu("$suftag\t[$left][$right]\n");
193
194 my @rightli = split / /, $right;
195 my %runiq = map { $_ => 1 } @rightli;
196 my @uright = keys %runiq;
197 if (1 < $#uright) {
198 print "% Heyyyyy multi!\n";
199 die "oh no Mr. Billl!";
200 }
201 my @leftli = split / /, $left;
202 my @sleftli = sort @leftli;
203
204 next if ( $#sleftli eq -1 and $suftag =~ m/^\=/ );
205 next if ( $suftag =~ m/^\=\./ );
206
207 # Compute the connection string....
208 # suftag:=а.cmv constr:(LLDTX- or LLDTY-)
209 my $constr = "(".join(" or ", @sleftli).")\n";
210
211 $constr .= " & " . $uright[0];
212
213 $common{$constr} .= "\n" . $suftag;
214
215 $nsf++;
216 }
217
218 print SUF "%\n% Total number of suftags = $nsf\n%\n";
219 print "% Total number of suftags = $nsf\n";
220
221 $nsf = scalar(keys %common);
222 print SUF "%\n% Total number of entries = $nsf\n%\n";
223 print "% Total number of entries = $nsf\n";
224
225
226 # reverse the thin, so that we can alphabetize it.
227 my %rev = ();
228 while ((my $constr, my $sufs) = each(%common)) {
229 $rev{$sufs} = $constr;
230 # print "$sufs :\n $constr;\n";
231 }
232
233 # place into alphabetical order
234 foreach my $sufs( sort keys %rev ) {
235 my $constr = $rev{$sufs};
236 print SUF eu($sufs)." :\n ".eu($constr).";\n";
237 }
238 close(SUF);
239
240 # mkmorph
241
242 my %cat2 = (); my %uni = (); my %ord = (); my %cmnt = ();
243
244 # load the morphology info.
245 my $comment = "";
246 my $cnt = 0;
247 open FILE, "4.0.morph";
248 while (<FILE>) {
249 next unless /^\%/;
250
251 # Save the inline comments for later re-printing.
252 if (!/morph/) { $comment .= du($_); }
253
254 next unless /morph/;
255 my $str = du($_);
256 chomp $str;
257
258 #% morph{"ПРЕДК,нст"} ="<глагол-пе-пр-sub> & (<глагол-pr> or <макро-глагол-pr>)";
259 (my $nr, my $rl) = split(/\=/, $str);
260 $nr =~ s/^\%\ +morph\{\"//; $nr =~ s/\"\}\ *$//;
261 $rl =~ s/^\ ?\"//; $rl =~ s/\"\;\ ?$//;
262
263 $cat2{$nr} = $rl;
264
265 # Keep them in the same order.
266 $cnt++;
267 $uni{$nr} = $cnt;
268 $ord{$cnt} = $nr;
269 $cmnt{$cnt} = $comment;
270 $comment = "";
271 }
272 close FILE;
273
274 print "gen morph\n";
275 open(MRF,">morph.dict") or die("cant");
276 print MRF <<EOF;
277 %%
278 %% Link Grammar for Russian -- Morphology rules.
279 %%
280 %% based on aot.ru opensource files
281 %%
282 %% Copyright (c) 2004, 2012 Sergey Protasov
283 %% http://sz.ru/parser/
284 %% svp dot zuzino.net.ru, svp dot tj.ru
285 %%
286 %% Below uses the utf8 encoding
287 EOF
288
289 my $mcnt = scalar( keys %cat2 );
290 print MRF "%\n% Total number of rules = $mcnt\n%\n";
291 print "% Total number of rules = $mcnt\n";
292
293 my $rcnt=1; my %posex = ();
294 foreach my $cnt ( sort { $a <=> $b } values %uni ) {
295 my $pos = $ord{$cnt};
296 $posex{$pos}++;
297 my $rule = $cat2{$pos};
298 if ($rcnt != $cnt) {
299 # print " Elided: $cnt\n"; $rcnt++;
300 }
301 $rcnt++;
302
303 # Print the in-line comments, if any.
304 if ($cnt > 1) { print MRF eu($cmnt{$cnt}); }
305 print MRF "<morph-".eu($pos)."> :\n ".eu($rule).";\n\n";
306 }
307
308 print MRF "%\n% Undefined ..... TODO XXX FIXME\n%\n";
309 foreach my $tag ( sort keys %alltags ) {
310 next if ( defined $posex{$tag} );
311 print MRF "<morph-".eu($tag).">\n";
312 }
313 print MRF " : XXX+;\n";
314 close(MRF);
315
316 system("mkdir ./words") if ( ! -d "./words");
317
318 # mkroot mkstem
319
320 $rcnt = 0; my %uniq = (); %rev = ();
321
322 while (my ($stem, $val) = each(%root)){
323 chomp $val;
324 $stem = dk($stem);
325 $val = dk($val);
326 $val =~ s/^://;
327
328 my @nums = split(/:/, $val);
329 foreach my $num (@nums){
330 $uniq{$num} = 1;
331 }
332 $rcnt++;
333 $rev{$val} .= " " . $stem;
334 #print eu("duuude $stem is $val\n");
335 }
336 # There are 135775 stems according to this...
337 # and something like 2997 connectors! WoW! why so many connectors ???
338 # Worse, the stems are divided into 6955 classes! Ouch
339
340 print "gen stem\n";
341 $un = scalar( keys %uniq );
342 my $revcnt = scalar( keys %rev );
343
344 open(RTS,">stem.dict") or die("cant");
345 print RTS "%\n% Stems (roots)\n%\n% Total number of stems = $rcnt\n";
346 print RTS "% Number of stem connectors = $un\n";
347 print RTS "% Number of stem classes = $revcnt\n";
348 print RTS "%\n";
349 print "% Total number of stems = $rcnt\n";
350 print "% Number of stem connectors = $un\n";
351 print "% Number of stem classes = $revcnt\n";
352
353 # put back in an order where we can sort alphabetically
354 my %fwd= ();
355 while (my ($nums, $stems) = each(%rev)){
356 #print eu("$stems\t\t$nums\n");
357 chomp $stems; $stems =~ s/^\s+|\s+$//g;
358 # вист грипп блеф кейф кикс флирт 16:1383
359 $fwd{$stems} = $nums;
360 }
361
362 # The actual alphabetical sort
363
364 $revcnt = 0; my $filecnt = 1;
365
366 foreach my $stems ( sort keys %fwd ) {
367 $stems =~ s/^\s+|\s+$//g;
368 next if ( $stems eq "" ); # skip nums_for_empty_stem
369 next unless defined $fwd{$stems};
370 my @words = sort split(/\s+/, $stems);
371 my @paradigms = split /:/, $fwd{$stems};
372 next unless $#paradigms > -1;
373 #print eu("$stems\n\t\t$fwd{$stems}\n");
374
375 my %emptysuflinks = ();
376 foreach my $snum ( @paradigms ) {
377 if ( defined $rHse->{$snum} ) {
378 foreach my $key ( keys %{$rHse->{$snum}} ) {
379 next unless defined $posex{$key};
380 $emptysuflinks{$key}++;
381 }
382 }
383 }
384
385 # write inline if less than 40 words, else dump to a file
386 if ( $#words < 40 ) {
387 my $cnt = 0;
388 foreach my $w ( @words ) {
389 print RTS eu($w).".= ";
390 $cnt++;
391 if (5 < $cnt) {
392 print RTS "\n";
393 $cnt = 0;
394 }
395 }
396 print RTS ":\n";
397 } else {
398 print RTS "/ru/words/words.$filecnt:\n";
399 open WFILE, ">words/words.$filecnt";
400 foreach my $w ( @words ) {
401 print WFILE eu($w).".=\n";
402 }
403 close WFILE;
404 $filecnt++;
405 }
406
407 my @links = map { "LL".alnum($_)."+" } @paradigms;
408 print RTS " ".eu(join(" or ", @links)).";\n\n";
409 $revcnt ++;
410
411 next unless scalar( keys %emptysuflinks ) > 0;
412
413 # write inline if less than 40 words, else dump to a file
414 foreach my $key (keys %emptysuflinks ) {
415 my $cnt = 0;
416 my $tot = 0;
417 foreach my $w ( @words ) {
418 my $wkey = $w.".".parse($key);
419 next if ( defined $skipwords{$wkey} );
420 next if ( defined $skipwords{$w} );
421 print RTS eu($wkey)." ";
422 $cnt++;
423 $tot++;
424 if (5 < $cnt) {
425 print RTS "\n";
426 $cnt = 0;
427 }
428 }
429 print RTS ": ".eu("<morph-$key>").";\n\n" if $tot > 0;
430 }
431 }
432 # print "% duude revers=$revcnt\n";
433 close(RTS);
434
435 if ( -d "../words" ) {
436 system("cp *.dict ../");
437 system("rm -f ../words/words.*");
438 system("cp words/words.* ../words/");
439 }
440
441
442 sub parse
443 {
444 my $s = shift;
445 #return $s;
446 my $res = "";
447 $res .= "n" if $s =~ /^С(,|$)/; # существительное noun
448 $res .= "a" if $s =~ /^П(,|$)/; # прилагательное adjective
449 $res .= "v" if $s =~ /^Г(,|$)/; # глагол verb
450 $res .= "e" if $s =~ /^Н(,|$)/; # наречие
451 $res .= "p" if $s =~ /^ЧАСТ(,|$)/; # частица
452 $res .= "c" if $s =~ /^ЧИСЛ(,|$)/; # числительное
453 $res .= "k" if $s =~ /^ЧИСЛ-П(,|$)/;
454 $res .= "i" if $s =~ /^СОЮЗ(,|$)/;
455 $res .= "j" if $s =~ /^ПРЕДЛ(,|$)/; # предлог
456 $res .= "o" if $s =~ /^ПОСЛ(,|$)/;
457 $res .= "q" if $s =~ /^МЕЖД(,|$)/; # междометие
458 $res .= "x" if $s =~ /^ПРЕДК(,|$)/; # предикатив
459 $res .= "y" if $s =~ /^ВВОДН(,|$)/;
460 $res .= "f" if $s =~ /^ФРАЗ(,|$)/; # фразеологизм
461 $res .= "m" if $s =~ /^МС(,|$)/; # местоимение
462 $res .= "w" if $s =~ /^МС-П(,|$)/; # местоимение-предлог
463 $res .= "u" if $s =~ /^МС-ПРЕДК(,|$)/; # местоимение-предикатив
464 $res .= "n" if $s =~ /,нс(,|$)/; # несовершенное imperfective
465 $res .= "s" if $s =~ /,св(,|$)/; # совершенное perfective
466 $res .= "n" if $s =~ /,нп(,|$)/; # непереходный
467 $res .= "p" if $s =~ /,пе(,|$)/; # переходный
468 $res .= "p" if $s =~ /,прч(,|$)/; # причастие
469 $res .= "d" if $s =~ /,дст(,|$)/; # действительный залог active voice
470 $res .= "s" if $s =~ /,стр(,|$)/; # страдательный залог passive voice
471 $res .= "d" if $s =~ /,дпр(,|$)/; # деепричастие
472 $res .= "i" if $s =~ /,инф(,|$)/; # инфинитив
473 $res .= "n" if $s =~ /,нст(,|$)/; # настоящее present
474 $res .= "f" if $s =~ /,буд(,|$)/; # будущее future
475 $res .= "p" if $s =~ /,прш(,|$)/; # прошлое past
476 $res .= "v" if $s =~ /,пвл(,|$)/; # повелительное наклонение
477 $res .= "1" if $s =~ /,1л(,|$)/; # 1-ое лицо
478 $res .= "2" if $s =~ /,2л(,|$)/; # 2-ое лицо
479 $res .= "3" if $s =~ /,3л(,|$)/; # 3-е лицо
480
481 $res .= "a" if $s =~ /,аббр(,|$)/; # аббревиатура
482 $res .= "d" if $s =~ /,но(,|$)/; # неодушевленное
483 $res .= "l" if $s =~ /,од(,|$)/; # одушевленное
484 $res .= "m" if $s =~ /,мр(,|$)/; # мужской род masculine gender
485 $res .= "f" if $s =~ /,жр(,|$)/; # женский род feminine
486 $res .= "n" if $s =~ /,ср(,|$)/; # средний род neuter gender
487 $res .= "s" if $s =~ /,ед(,|$)/; # единственное singular
488 $res .= "p" if $s =~ /,мн(,|$)/; # множественное plural
489 $res .= "i" if $s =~ /,им(,|$)/; # именительный common case
490 $res .= "g" if $s =~ /,рд(,|$)/; # родительный genitive case
491 $res .= "d" if $s =~ /,дт(,|$)/; # дательный dative case
492 $res .= "t" if $s =~ /,тв(,|$)/; # творительный instrumental case (Ablative case)
493 $res .= "v" if $s =~ /,вн(,|$)/; # винительный objective case (or Accusative case)
494 $res .= "p" if $s =~ /,пр(,|$)/; # предложный prepositional case
495 $res .= "s" if $s =~ /,кр(,|$)/; # краткая форма
496 $res .= "s" if $s =~ /,сравн(,|$)/; # сравнительная степень
497 $res .= "f" if $s =~ /,фам(,|$)/; # фамилия
498 $res .= "n" if $s =~ /,имя(,|$)/; # имя
499 return $res;
500 }
501
502 sub eu {
503 return encode("utf-8",shift);
504 }
505
506 sub du {
507 return decode("utf-8",shift);
508 }
509
510 sub ek {
511 return encode("koi8-r",shift);
512 }
513
514 sub dk {
515 return decode("koi8-r",shift);
516 }
517