Debian Codegraph @

modules/gcide/idxgcide.l

raw
1 %top {
2 /* This file is part of GNU Dico.
3 Copyright (C) 2012-2024 Sergey Poznyakoff
4
5 GNU Dico is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 3, or (at your option)
8 any later version.
9
10 GNU Dico is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with GNU Dico. If not, see <http://www.gnu.org/licenses/>. */
17
18 #include <config.h>
19 #include <dico.h>
20 #include <unistd.h>
21 #include <getopt.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #define obstack_chunk_alloc malloc
25 #define obstack_chunk_free free
26 #include <obstack.h>
27 #include <errno.h>
28 #include <sysexits.h>
29 #include <appi18n.h>
30 #include "gcide.h"
31 }
32 %{
33
34 int verbose_option;
35 int dry_run_option;
36 char *debug_letters;
37
38 char *dictdir, *idxdir, *infile, *suf;
39 char letters[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
40 int letno;
41 unsigned line;
42
43 char *idxname;
44 FILE *idxfile;
45 struct gcide_idx_header idx_header = {
46 GCIDE_IDX_MAGIC,
47 GCIDE_IDX_VERSION,
48 GCIDE_IDX_HEADER_PAGESIZE
49 };
50 struct gcide_idx_page *idx_page;
51
52 unsigned long char_position;
53 unsigned long file_position;
54 unsigned long headword_position;
55 struct obstack stk;
56 struct obstack refstk;
57 int retstate;
58 char *endtag;
59 int at_ent;
60
61 #define YY_USER_ACTION do { \
62 char_position = file_position; \
63 file_position += yyleng; \
64 } while (0);
65
66 static void
67 full_write(void *data, size_t size)
68 {
69 if (fwrite(data, size, 1, idxfile) != 1) {
70 DICO_LOG_ERRNO();
71 exit(EX_UNAVAILABLE);
72 }
73 }
74
75 static void
76 flush_page()
77 {
78 if (idx_page->ipg_header.hdr.phdr_numentries == 0 ||
79 idx_page->ipg_header.hdr.phdr_text_offset == 0) {
80 dico_log(L_ERR, 0, _("page too small, aborting"));
81 exit(EX_UNAVAILABLE);
82 }
83 full_write(idx_page, idx_header.ihdr_pagesize);
84 memset(idx_page, 0, idx_header.ihdr_pagesize);
85 idx_page->ipg_header.hdr.phdr_text_offset = idx_header.ihdr_pagesize / 2;
86 idx_header.ihdr_num_pages++;
87 }
88
89 static void
90 store_ref(struct gcide_ref *ref)
91 {
92 char *textarea;
93
94 if (idx_page->ipg_header.hdr.phdr_numentries == idx_header.ihdr_maxpageref
95 || idx_page->ipg_header.hdr.phdr_text_offset + ref->ref_hwbytelen >
96 idx_header.ihdr_pagesize)
97 flush_page();
98
99 ref->ref_hwoff = idx_page->ipg_header.hdr.phdr_text_offset;
100 textarea = (char*)idx_page + idx_page->ipg_header.hdr.phdr_text_offset;
101 memcpy(textarea, ref->ref_headword, ref->ref_hwbytelen);
102 idx_page->ipg_header.hdr.phdr_text_offset += ref->ref_hwbytelen;
103 memcpy(idx_page->ipg_ref + idx_page->ipg_header.hdr.phdr_numentries++,
104 ref, sizeof(ref[0]));
105 }
106
107 static int
108 refcmp(const void *a, const void *b)
109 {
110 struct gcide_ref const *aref = a;
111 struct gcide_ref const *bref = b;
112 int res = utf8_strcasecmp(aref->ref_headword, bref->ref_headword);
113 if (res == 0)
114 res = bref->ref_primary - aref->ref_primary;
115 return res;
116 }
117
118 static void
119 flush_refs()
120 {
121 int i;
122 struct gcide_ref *ref = obstack_finish(&refstk);
123
124 if (verbose_option)
125 dico_log(L_INFO, 0, _("Sorting references"));
126 qsort(ref, idx_header.ihdr_num_headwords, sizeof(ref[0]), refcmp);
127
128 if (verbose_option)
129 dico_log(L_INFO, 0, _("Writing references"));
130
131 for (i = 0; i < idx_header.ihdr_num_headwords; i++, ref++)
132 store_ref(ref);
133 if (idx_page->ipg_header.hdr.phdr_numentries)
134 flush_page();
135 }
136
137 static int
138 strrefcmp(void const *a, void const *b)
139 {
140 return utf8_strcasecmp(*(char**)a + 1, *(char**)b + 1);
141 }
142
143 static char *
144 wstrim(char *p)
145 {
146 int c = *p++;
147 p = utf8_space_trim(p);
148 *--p = c;
149 return p;
150 }
151
152 static void
153 flush_headwords(void)
154 {
155 char *p;
156 unsigned long size;
157 char *headword;
158
159 obstack_1grow(&stk, 0);
160 headword = obstack_finish(&stk);
161 if (headword[0]) {
162 char **hv;
163 int i, hc = 0;
164
165 for (p = headword; *p; ) {
166 /* Save pointer to next word. */
167 char *q = p + strlen(p) + 1;
168
169 /* Strip initial and trailing whitespace */
170 p = wstrim(p);
171 obstack_grow(&stk, &p, sizeof(p));
172 hc++;
173
174 /* Get next word. */
175 p = q;
176 }
177
178 hv = obstack_finish(&stk);
179 qsort(hv, hc, sizeof(hv[0]), strrefcmp);
180
181 size = char_position - headword_position;
182 for (i = 0; i < hc; i++) {
183 struct gcide_ref ref;
184
185 if (i > 0 && utf8_strcasecmp(hv[i] + 1, hv[i-1] + 1) == 0)
186 continue;
187
188 p = hv[i] + 1;
189 if (verbose_option > 1)
190 dico_log(L_INFO, 0, "%s: %c %lu %lu", p, letters[letno-1],
191 headword_position, size);
192 memset(&ref, 0, sizeof(ref));
193
194 ref.ref_hwbytelen = strlen(p) + 1;
195 ref.ref_hwlen = utf8_strlen(p);
196 ref.ref_letter = letters[letno-1];
197 ref.ref_offset = headword_position;
198 ref.ref_size = size;
199 ref.ref_headword = p;
200 ref.ref_primary = hv[i][0] - '0';
201 obstack_grow(&refstk, &ref, sizeof(ref));
202 idx_header.ihdr_num_headwords++;
203 }
204 idx_header.ihdr_num_defs++;
205 }
206 headword = NULL;
207 headword_position = char_position;
208 at_ent = 1;
209 }
210
211 static void
212 add_space(struct obstack *stk)
213 {
214 char *p = obstack_base(stk);
215 int n = obstack_object_size(stk);
216 if (!(n > 0 && p[n-1] == ' '))
217 obstack_1grow(stk, ' ');
218 }
219
220 static inline void
221 mark_primary(int prim)
222 {
223 obstack_1grow(&stk, prim ? '1' : '0');
224 }
225
226 #define SET_HWPOS() {\
227 if (at_ent) {\
228 at_ent = 0;\
229 headword_position = char_position;\
230 }\
231 }
232 #define BEGIN_HEADWORD(end, prim) \
233 { retstate = YYSTATE; \
234 endtag = (end); \
235 mark_primary(prim); \
236 SET_HWPOS(); \
237 BEGIN(HEADWORD); }
238
239 #define advance_line() \
240 { line++; \
241 if (yy_flex_debug) \
242 fprintf(stderr, "# %d %s\n", line, infile); \
243 }
244 %}
245 %option 8bit
246 %option nounput
247 %option noinput
248
249 %x HEADWORD COMMENT HTMLCOM
250
251 XD [0-9a-f]
252 %%
253 <INITIAL>{
254 "<--" { BEGIN(COMMENT); }
255 "<!" { BEGIN(HTMLCOM); }
256 ("\\'d8")?"<hw>" { BEGIN_HEADWORD("hw", 1); }
257 "<mhw>" SET_HWPOS();
258 "<p><ent>" flush_headwords();
259 "<asp>" BEGIN_HEADWORD("asp", 1);
260 "<altname>" BEGIN_HEADWORD("altname", 0);
261 "<decf>" BEGIN_HEADWORD("decf", 1);
262 "<col>" BEGIN_HEADWORD("col", 0);
263 "<colf>" BEGIN_HEADWORD("colf", 0);
264 "<conjf>" BEGIN_HEADWORD("conjf", 0);
265 . ;
266 \n advance_line();
267 }
268 <HEADWORD>{
269 "</"[a-zA-Z][a-zA-Z]*">" {
270 if (yyleng == strlen(endtag) + 3 &&
271 memcmp(endtag, yytext + 2, yyleng-3) == 0) {
272 BEGIN(retstate);
273 obstack_1grow(&stk, 0);
274 }
275 }
276 "<"[a-zA-Z?][a-zA-Z0-9]*"/" {
277 char const *s = gcide_entity_to_utf8(yytext);
278 if (s) {
279 if (strcmp(s, "<?>") == 0)
280 dico_log(L_WARN, 0,
281 _("%s:%u: unknown or illegible character in a headword"),
282 infile, line);
283 obstack_grow(&stk, s, strlen(s));
284 } else
285 dico_log(L_WARN, 0, _("%s:%u: unrecognized entity: %s"),
286 infile, line, yytext);
287 }
288 "<"[a-zA-Z][^/>]*">" /* ignore tag */;
289 ["*`] ;
290 "\\'d8" ; /* ignore double vertical bar */
291 "\\'"{XD}{XD} {
292 char const *s = gcide_escape_to_utf8(yytext+2);
293
294 if (s) {
295 int len = strlen(s);
296 obstack_grow(&stk, s, len);
297 } else {
298 obstack_grow(&stk, yytext, yyleng);
299 dico_log(L_WARN, 0,
300 _("%s:%u: unknown character sequence %s"),
301 infile, line, yytext);
302 }
303 }
304 [ \t] add_space(&stk);
305 . obstack_grow(&stk, yytext, yyleng);
306 \n {
307 add_space(&stk);
308 advance_line();
309 }
310 }
311 <COMMENT>{
312 "-->" { BEGIN(INITIAL); }
313 . ;
314 \n advance_line();
315 }
316 <HTMLCOM>{
317 "!>" { BEGIN(INITIAL); }
318 . ;
319 \n advance_line();
320 }
321 %%
322
323 int
324 yywrap(void)
325 {
326 flush_headwords();
327
328 if (letters[letno] == 0)
329 return 1;
330 yy_flex_debug = debug_letters && strchr(debug_letters, letters[letno]);
331 *suf = letters[letno++];
332 if (yyin)
333 fclose(yyin);
334 if (verbose_option)
335 dico_log(L_INFO, 0, _("Indexing %s"), infile);
336 yyin = fopen(infile, "r");
337 if (!yyin) {
338 dico_log(L_ERR, errno, _("cannot open file %s"), infile);
339 exit(EX_NOINPUT);
340 }
341 file_position = char_position = headword_position = 0;
342 line = 1;
343 return 0;
344 }
345
346 static int inherit_fd;
347 #include "idxgcide-cli.h"
348
349 /* Usage: idxgcide DICTDIR [IDXDIR] */
350 int
351 main(int argc, char **argv)
352 {
353 int index;
354
355 appi18n_init();
356 dico_set_program_name(argv[0]);
357 yy_flex_debug = 0;
358
359 get_options(argc, argv, &index);
360
361 idx_page = malloc(idx_header.ihdr_pagesize);
362 if (!idx_page) {
363 DICO_LOG_ERRNO();
364 exit(EX_UNAVAILABLE);
365 }
366 idx_page->ipg_header.hdr.phdr_numentries = 0;
367 idx_page->ipg_header.hdr.phdr_text_offset = idx_header.ihdr_pagesize / 2;
368
369 idx_header.ihdr_maxpageref = idx_header.ihdr_pagesize / 2 /
370 sizeof(struct gcide_ref);
371 if (idx_header.ihdr_maxpageref < 1) {
372 dico_log(L_ERR, 0, _("page size too small"));
373 exit(EX_USAGE);
374 }
375 idx_header.ihdr_maxpageref--;
376
377 argc -= index;
378 argv += index;
379
380 if (argc < 1 || argc > 2) {
381 dico_log(L_ERR, 0, _("bad number of arguments"));
382 exit(EX_USAGE);
383 }
384 dictdir = argv[0];
385 idxdir = argc == 2 ? argv[1] : dictdir;
386
387 infile = dico_full_file_name(dictdir, "CIDE.A");
388 if (!infile) {
389 DICO_LOG_MEMERR();
390 exit(EX_UNAVAILABLE);
391 }
392 suf = infile + strlen(infile) - 1;
393
394 if (dry_run_option && !inherit_fd)
395 idxname = "/dev/null";
396 else
397 idxname = dico_full_file_name(idxdir, GCIDE_IDX_FILE_NAME);
398 if (!idxname) {
399 DICO_LOG_MEMERR();
400 exit(EX_UNAVAILABLE);
401 }
402
403 if (inherit_fd) {
404 if (ftruncate(inherit_fd, 0)) {
405 dico_log(L_ERR, errno, _("cannot truncate index file `%s'"), idxname);
406 exit(EX_CANTCREAT);
407 }
408 idxfile = fdopen(inherit_fd, "w");
409 } else {
410 idxfile = fopen(idxname, "w");
411 }
412 if (!idxfile) {
413 dico_log(L_ERR, errno, _("cannot create index file `%s'"), idxname);
414 exit(EX_CANTCREAT);
415 }
416 fseek(idxfile, idx_header.ihdr_pagesize, SEEK_SET);
417
418 obstack_init(&stk);
419 obstack_init(&refstk);
420
421 letno = 0;
422 yywrap();
423 while (yylex ())
424 ;
425
426 flush_refs();
427
428 fseek(idxfile, 0, SEEK_SET);
429 full_write(&idx_header, sizeof(idx_header));
430 fclose(idxfile);
431
432 if (verbose_option)
433 dico_log(L_INFO, 0,
434 _("headwords=%lu, definitions=%lu, refs per page=%lu, "
435 "pages=%lu"),
436 idx_header.ihdr_num_headwords,
437 idx_header.ihdr_num_defs,
438 idx_header.ihdr_maxpageref,
439 idx_header.ihdr_num_pages);
440
441 exit (0);
442 }
443
444 /* Local Variables: */
445 /* mode: c */
446 /* eval: (setq buffer-file-coding-system (if window-system 'utf-8 'raw-text-unix)) */
447 /* End: */