Debian Codegraph @

lib/Dict.pm

raw
1 # V1.5 5/2004
2 # * generate warnings on empty headwords that appear after
3 # punctuation removal
4 # V1.4 4/2004
5 # * trim leading spaces after punctuation removal from headwords
6 # * made warnings of non-acsii characters in non utf8 index work
7 # V1.3 2/2004
8 # * use LC_ALL instead of LANG environment variable
9 # for setting locale for 'sort'
10 # * don't use -df switches to 'sort' when generating
11 # 00-database-utf8 headword
12 # * do headword mangling in the same way (hopefully) like dictd
13 # does in its tolower_alnumspace_utf8() function
14 # V1.2 5/2003
15 # * gave optional argument 'locale' to open_dict()
16 # V1.1 improved by Michael Bunk, <kleinerwurm at gmx.net>
17 # * This API is undocumented & cruel! But at least I added
18 # some comments here.
19 #
20 # V1.0 Copyright (C) 2000 Horst Eyermann <[email protected]>
21 #
22 # This program is free software; you can redistribute it and/or modify
23 # it under the terms of the GNU General Public License as published by
24 # the Free Software Foundation; either version 2 of the License, or
25 # (at your option) any later version.
26 #
27 # This program is distributed in the hope that it will be useful,
28 # but WITHOUT ANY WARRANTY; without even the implied warranty of
29 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
30 # GNU General Public License for more details.
31 #
32 # You should have received a copy of the GNU General Public License
33 # along with this program; if not, visit
34 # <http://www.gnu.org/licenses/gpl-2.0.txt>
35
36 package Dict;
37
38 use strict;
39 use Text::Wrap;# for fill() or wrap()
40 use Encode 'decode_utf8';
41 use utf8;
42
43 our ($headwords, $prev_word, $start_article, $head, $fill2, $end_article,
44 @hwords, $text, $all_text, $utf8mode);
45
46 # there is CPAN module MIME::Base64 for this! but anyway,
47 # without it we are independent
48 our $b64_list = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
49 sub b64_encode {
50 my $val = shift @_;
51 my $result = "";
52
53 my $tempo = ($val & 0xC0000000) >> 30;
54 if ( $tempo != 0) {
55 $result = substr $b64_list, $tempo, 1;
56 }
57
58 $tempo = ($val & 0x3f000000) >> 24;
59
60 if ( $result ne "" || $tempo != 0) {
61 $result .= substr $b64_list, $tempo, 1;
62 }
63
64 $tempo = ($val & 0x00fc0000) >> 18;
65
66 if ( $result ne "" || $tempo != 0) {
67 $result .= substr $b64_list, $tempo, 1;
68 }
69
70 $tempo = ($val & 0x0003f000) >> 12;
71
72 if ( $result ne "" || $tempo != 0) {
73 $result .= substr $b64_list, $tempo, 1;
74 }
75
76 $tempo = ($val & 0x00000fc0) >> 6;
77
78 if ( $result ne "" || $tempo != 0) {
79 $result .= substr $b64_list, $tempo, 1;
80 }
81
82 $tempo = ($val & 0x0000003f);
83 $result .= substr $b64_list, $tempo, 1;
84
85 return $result;
86 }
87
88
89 sub open_dict{
90 my ($this, $name, $locale, $d, $sortcmd);
91
92 $this = shift @_;
93 $name = shift @_;
94 $locale = shift @_;
95 $utf8mode = shift @_;
96
97 # set locale
98 # otherwise collating sequence (sorting oder) might be
99 # wrong (space after letters and things like that)
100 printf STDERR "Dict.pm: Using LC_ALL: $locale\n";
101 $ENV{'LC_ALL'}=$locale;
102
103 # open file where definitions will go
104 open DATA, ">$name.dict";
105
106 # open a pipe and output the index in a sorted way
107
108 # dictfmt uses only -df (and no options in utf8 / 8bit mode)
109 # in bash we enter: sort -t $'\t' -k 1,1bdf
110 # the sort options:
111 # -t: give field separator, the TAB
112 # -k: only use first field for sorting
113 # b: ignore trailing blanks (might not be needed or even harmful)
114 # df: as usual (dictionary order, ignore case)
115
116 # don't give -df option when we generate 00-database-utf8 index
117 $d = ($utf8mode) ? "" : "df";
118 $sortcmd = "|sort -t \"\t\" -k 1,1b".$d." >$name.index";
119 warn "using sort command: '$sortcmd'\n";
120 open INDEX, $sortcmd;
121 if ($utf8mode) { binmode(INDEX, ":utf8"); }
122 warn "\nopened: $name.dict and \n $name.index\n";
123
124 $headwords = 0;
125 $prev_word = "";
126 $start_article = 0;
127 $head = 0;
128 $fill2 = "";
129 }
130
131 sub write_newline {
132 # directly write into data file
133 # newlines in the INDEX don't make sense anyway
134 print DATA "\n";
135 }
136
137
138 sub set_headword {
139 # Here a marker is set for the next entries being headwords
140
141 # But how does it work for the first entry? You cannot call set_headword
142 # before having one??
143
144 # For the first time set_headword is called, NOTHING is written to index!
145 # At second call, first entry's headword is written.
146
147 # this also means that for the very last entry to appear in the index,
148 # after outputting the last entry we once again have to call set_headword!
149 # couldn't this be done in a better way? yes, new api!
150
151 $head = 1;
152 $end_article = tell DATA;
153
154
155 # Output status
156 if (($headwords % 100) == 0)
157 {
158 printf STDERR "%8d Headwords.\r", $headwords
159 }
160
161 # that hwords is an array means multiple headwords
162 # can reference the same definition! good.
163 if(@hwords != 0) {
164 foreach (@hwords) {
165 # warn about non-ascii chars if not in utf8 mode
166 if(!$utf8mode && /\P{IsASCII}/)
167 {
168 warn "Non ASCII char in headword: \"$_\"\n";
169 # btw, dictfmt quits here!
170 }
171
172 # make perl believe it is utf8
173 # see 'man perluniintro'
174 if($utf8mode) { $_ = decode_utf8($_); }
175
176 # generate two headwords for "00-database-*":
177 # first with, second without "-" characters
178 if(/^00-database-/) {
179 print INDEX "$_\t" . b64_encode($start_article);
180 print INDEX "\t" . b64_encode($end_article-$start_article) . "\n"; }
181
182 # do headword mangling in the same way (hopefully) like dictd
183 # does in its str.c:tolower_alnumspace_utf8() function:
184 # if not in allchars mode (we can't be since we don't support it),
185 # - any whitespace characters are translated to simple space
186 # - only alphanumeric characters are considered
187 # AlexeyCheusov> Minus should also be removed unless you use "allchars" mode.
188 # AlexeyCheusov> In last case, ALL characters from the query should be matched.
189 s/\s/ /g;
190 s/[^\s\p{IsAlnum}]//g;
191
192 # thereafter characters are translated to lower case (required
193 # even in allchars mode!)
194 $_ = lc $_;
195
196 # sanity checks
197 if(/^\s+(.+)/)
198 {
199 warn "\nWarning: Trimmed leading space(s) from headword '$_'\n";
200 $_ = $1;
201 warn "Now it is '$_'\n";
202 }
203
204 if(/^\s*$/)
205 {
206 warn "\nSkipping empty or whitespace-only headword!\n";
207 warn "\t\@hwords: ", @hwords, "\n";
208 }
209 else
210 {
211 print INDEX "$_\t" . b64_encode($start_article);
212 print INDEX "\t" . b64_encode($end_article-$start_article) . "\n"; }
213 }
214 }
215
216 $start_article = $end_article;
217 @hwords = ();
218 }
219
220 sub end_headword {
221 $head = 0;
222 }
223
224
225 sub add_text{
226 $text = shift @_;
227
228 # FIXME api <-> layout not independent!
229 # connect multiple headwords with comma in dict file
230 if (($head) && (defined $hwords[0])) { $all_text .= ", ". $text}
231 # usual output
232 else { $all_text .= $text};
233 }
234
235
236 sub write_text{
237 $Text::Wrap::columns = 70;
238 print DATA fill("", $fill2, $all_text);
239 $all_text = "";
240 return if ((!defined $text) || ($text eq ""));
241
242 if ($head == 1) {
243 push @hwords, $text;# FIXME $text is an ugly global var!
244 $headwords += 1;
245 }
246
247 # FIXME $text is an ugly global var!
248 $text = "";
249 }
250
251 sub add_headword{
252 $text = shift @_;
253 return if ((!defined $text) || ($text eq ""));
254
255 if ($head == 1) {
256 push @hwords, $text;# FIXME $text is an ugly global var!
257 $headwords += 1;
258 }
259
260 $text = "";
261 }
262
263
264 sub write_direct{
265 $text = shift @_;
266 print DATA $text;
267 }
268
269 1;