| 1 |
%top { |
| 2 |
/* This file is part of GNU Dico. |
| 3 |
Copyright (C) 2012-2024 Sergey Poznyakoff |
| 4 |
|
| 5 |
GNU Dico is free software; you can redistribute it and/or modify |
| 6 |
it under the terms of the GNU General Public License as published by |
| 7 |
the Free Software Foundation; either version 3, or (at your option) |
| 8 |
any later version. |
| 9 |
|
| 10 |
GNU Dico is distributed in the hope that it will be useful, |
| 11 |
but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 |
GNU General Public License for more details. |
| 14 |
|
| 15 |
You should have received a copy of the GNU General Public License |
| 16 |
along with GNU Dico. If not, see <http://www.gnu.org/licenses/>. */ |
| 17 |
|
| 18 |
#include <config.h> |
| 19 |
#include <dico.h> |
| 20 |
#include <unistd.h> |
| 21 |
#include <getopt.h> |
| 22 |
#include <stdio.h> |
| 23 |
#include <stdlib.h> |
| 24 |
#define obstack_chunk_alloc malloc |
| 25 |
#define obstack_chunk_free free |
| 26 |
#include <obstack.h> |
| 27 |
#include <errno.h> |
| 28 |
#include <sysexits.h> |
| 29 |
#include <appi18n.h> |
| 30 |
#include "gcide.h" |
| 31 |
} |
| 32 |
%{ |
| 33 |
|
| 34 |
int verbose_option; |
| 35 |
int dry_run_option; |
| 36 |
char *debug_letters; |
| 37 |
|
| 38 |
char *dictdir, *idxdir, *infile, *suf; |
| 39 |
char letters[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; |
| 40 |
int letno; |
| 41 |
unsigned line; |
| 42 |
|
| 43 |
char *idxname; |
| 44 |
FILE *idxfile; |
| 45 |
struct gcide_idx_header idx_header = { |
| 46 |
GCIDE_IDX_MAGIC, |
| 47 |
GCIDE_IDX_VERSION, |
| 48 |
GCIDE_IDX_HEADER_PAGESIZE |
| 49 |
}; |
| 50 |
struct gcide_idx_page *idx_page; |
| 51 |
|
| 52 |
unsigned long char_position; |
| 53 |
unsigned long file_position; |
| 54 |
unsigned long headword_position; |
| 55 |
struct obstack stk; |
| 56 |
struct obstack refstk; |
| 57 |
int retstate; |
| 58 |
char *endtag; |
| 59 |
int at_ent; |
| 60 |
|
| 61 |
#define YY_USER_ACTION do { \ |
| 62 |
char_position = file_position; \ |
| 63 |
file_position += yyleng; \ |
| 64 |
} while (0); |
| 65 |
|
| 66 |
static void |
| 67 |
full_write(void *data, size_t size) |
| 68 |
{ |
| 69 |
if (fwrite(data, size, 1, idxfile) != 1) { |
| 70 |
DICO_LOG_ERRNO(); |
| 71 |
exit(EX_UNAVAILABLE); |
| 72 |
} |
| 73 |
} |
| 74 |
|
| 75 |
static void |
| 76 |
flush_page() |
| 77 |
{ |
| 78 |
if (idx_page->ipg_header.hdr.phdr_numentries == 0 || |
| 79 |
idx_page->ipg_header.hdr.phdr_text_offset == 0) { |
| 80 |
dico_log(L_ERR, 0, _("page too small, aborting")); |
| 81 |
exit(EX_UNAVAILABLE); |
| 82 |
} |
| 83 |
full_write(idx_page, idx_header.ihdr_pagesize); |
| 84 |
memset(idx_page, 0, idx_header.ihdr_pagesize); |
| 85 |
idx_page->ipg_header.hdr.phdr_text_offset = idx_header.ihdr_pagesize / 2; |
| 86 |
idx_header.ihdr_num_pages++; |
| 87 |
} |
| 88 |
|
| 89 |
static void |
| 90 |
store_ref(struct gcide_ref *ref) |
| 91 |
{ |
| 92 |
char *textarea; |
| 93 |
|
| 94 |
if (idx_page->ipg_header.hdr.phdr_numentries == idx_header.ihdr_maxpageref |
| 95 |
|| idx_page->ipg_header.hdr.phdr_text_offset + ref->ref_hwbytelen > |
| 96 |
idx_header.ihdr_pagesize) |
| 97 |
flush_page(); |
| 98 |
|
| 99 |
ref->ref_hwoff = idx_page->ipg_header.hdr.phdr_text_offset; |
| 100 |
textarea = (char*)idx_page + idx_page->ipg_header.hdr.phdr_text_offset; |
| 101 |
memcpy(textarea, ref->ref_headword, ref->ref_hwbytelen); |
| 102 |
idx_page->ipg_header.hdr.phdr_text_offset += ref->ref_hwbytelen; |
| 103 |
memcpy(idx_page->ipg_ref + idx_page->ipg_header.hdr.phdr_numentries++, |
| 104 |
ref, sizeof(ref[0])); |
| 105 |
} |
| 106 |
|
| 107 |
static int |
| 108 |
refcmp(const void *a, const void *b) |
| 109 |
{ |
| 110 |
struct gcide_ref const *aref = a; |
| 111 |
struct gcide_ref const *bref = b; |
| 112 |
int res = utf8_strcasecmp(aref->ref_headword, bref->ref_headword); |
| 113 |
if (res == 0) |
| 114 |
res = bref->ref_primary - aref->ref_primary; |
| 115 |
return res; |
| 116 |
} |
| 117 |
|
| 118 |
static void |
| 119 |
flush_refs() |
| 120 |
{ |
| 121 |
int i; |
| 122 |
struct gcide_ref *ref = obstack_finish(&refstk); |
| 123 |
|
| 124 |
if (verbose_option) |
| 125 |
dico_log(L_INFO, 0, _("Sorting references")); |
| 126 |
qsort(ref, idx_header.ihdr_num_headwords, sizeof(ref[0]), refcmp); |
| 127 |
|
| 128 |
if (verbose_option) |
| 129 |
dico_log(L_INFO, 0, _("Writing references")); |
| 130 |
|
| 131 |
for (i = 0; i < idx_header.ihdr_num_headwords; i++, ref++) |
| 132 |
store_ref(ref); |
| 133 |
if (idx_page->ipg_header.hdr.phdr_numentries) |
| 134 |
flush_page(); |
| 135 |
} |
| 136 |
|
| 137 |
static int |
| 138 |
strrefcmp(void const *a, void const *b) |
| 139 |
{ |
| 140 |
return utf8_strcasecmp(*(char**)a + 1, *(char**)b + 1); |
| 141 |
} |
| 142 |
|
| 143 |
static char * |
| 144 |
wstrim(char *p) |
| 145 |
{ |
| 146 |
int c = *p++; |
| 147 |
p = utf8_space_trim(p); |
| 148 |
*--p = c; |
| 149 |
return p; |
| 150 |
} |
| 151 |
|
| 152 |
static void |
| 153 |
flush_headwords(void) |
| 154 |
{ |
| 155 |
char *p; |
| 156 |
unsigned long size; |
| 157 |
char *headword; |
| 158 |
|
| 159 |
obstack_1grow(&stk, 0); |
| 160 |
headword = obstack_finish(&stk); |
| 161 |
if (headword[0]) { |
| 162 |
char **hv; |
| 163 |
int i, hc = 0; |
| 164 |
|
| 165 |
for (p = headword; *p; ) { |
| 166 |
/* Save pointer to next word. */ |
| 167 |
char *q = p + strlen(p) + 1; |
| 168 |
|
| 169 |
/* Strip initial and trailing whitespace */ |
| 170 |
p = wstrim(p); |
| 171 |
obstack_grow(&stk, &p, sizeof(p)); |
| 172 |
hc++; |
| 173 |
|
| 174 |
/* Get next word. */ |
| 175 |
p = q; |
| 176 |
} |
| 177 |
|
| 178 |
hv = obstack_finish(&stk); |
| 179 |
qsort(hv, hc, sizeof(hv[0]), strrefcmp); |
| 180 |
|
| 181 |
size = char_position - headword_position; |
| 182 |
for (i = 0; i < hc; i++) { |
| 183 |
struct gcide_ref ref; |
| 184 |
|
| 185 |
if (i > 0 && utf8_strcasecmp(hv[i] + 1, hv[i-1] + 1) == 0) |
| 186 |
continue; |
| 187 |
|
| 188 |
p = hv[i] + 1; |
| 189 |
if (verbose_option > 1) |
| 190 |
dico_log(L_INFO, 0, "%s: %c %lu %lu", p, letters[letno-1], |
| 191 |
headword_position, size); |
| 192 |
memset(&ref, 0, sizeof(ref)); |
| 193 |
|
| 194 |
ref.ref_hwbytelen = strlen(p) + 1; |
| 195 |
ref.ref_hwlen = utf8_strlen(p); |
| 196 |
ref.ref_letter = letters[letno-1]; |
| 197 |
ref.ref_offset = headword_position; |
| 198 |
ref.ref_size = size; |
| 199 |
ref.ref_headword = p; |
| 200 |
ref.ref_primary = hv[i][0] - '0'; |
| 201 |
obstack_grow(&refstk, &ref, sizeof(ref)); |
| 202 |
idx_header.ihdr_num_headwords++; |
| 203 |
} |
| 204 |
idx_header.ihdr_num_defs++; |
| 205 |
} |
| 206 |
headword = NULL; |
| 207 |
headword_position = char_position; |
| 208 |
at_ent = 1; |
| 209 |
} |
| 210 |
|
| 211 |
static void |
| 212 |
add_space(struct obstack *stk) |
| 213 |
{ |
| 214 |
char *p = obstack_base(stk); |
| 215 |
int n = obstack_object_size(stk); |
| 216 |
if (!(n > 0 && p[n-1] == ' ')) |
| 217 |
obstack_1grow(stk, ' '); |
| 218 |
} |
| 219 |
|
| 220 |
static inline void |
| 221 |
mark_primary(int prim) |
| 222 |
{ |
| 223 |
obstack_1grow(&stk, prim ? '1' : '0'); |
| 224 |
} |
| 225 |
|
| 226 |
#define SET_HWPOS() {\ |
| 227 |
if (at_ent) {\ |
| 228 |
at_ent = 0;\ |
| 229 |
headword_position = char_position;\ |
| 230 |
}\ |
| 231 |
} |
| 232 |
#define BEGIN_HEADWORD(end, prim) \ |
| 233 |
{ retstate = YYSTATE; \ |
| 234 |
endtag = (end); \ |
| 235 |
mark_primary(prim); \ |
| 236 |
SET_HWPOS(); \ |
| 237 |
BEGIN(HEADWORD); } |
| 238 |
|
| 239 |
#define advance_line() \ |
| 240 |
{ line++; \ |
| 241 |
if (yy_flex_debug) \ |
| 242 |
fprintf(stderr, "# %d %s\n", line, infile); \ |
| 243 |
} |
| 244 |
%} |
| 245 |
%option 8bit |
| 246 |
%option nounput |
| 247 |
%option noinput |
| 248 |
|
| 249 |
%x HEADWORD COMMENT HTMLCOM |
| 250 |
|
| 251 |
XD [0-9a-f] |
| 252 |
%% |
| 253 |
<INITIAL>{ |
| 254 |
"<--" { BEGIN(COMMENT); } |
| 255 |
"<!" { BEGIN(HTMLCOM); } |
| 256 |
("\\'d8")?"<hw>" { BEGIN_HEADWORD("hw", 1); } |
| 257 |
"<mhw>" SET_HWPOS(); |
| 258 |
"<p><ent>" flush_headwords(); |
| 259 |
"<asp>" BEGIN_HEADWORD("asp", 1); |
| 260 |
"<altname>" BEGIN_HEADWORD("altname", 0); |
| 261 |
"<decf>" BEGIN_HEADWORD("decf", 1); |
| 262 |
"<col>" BEGIN_HEADWORD("col", 0); |
| 263 |
"<colf>" BEGIN_HEADWORD("colf", 0); |
| 264 |
"<conjf>" BEGIN_HEADWORD("conjf", 0); |
| 265 |
. ; |
| 266 |
\n advance_line(); |
| 267 |
} |
| 268 |
<HEADWORD>{ |
| 269 |
"</"[a-zA-Z][a-zA-Z]*">" { |
| 270 |
if (yyleng == strlen(endtag) + 3 && |
| 271 |
memcmp(endtag, yytext + 2, yyleng-3) == 0) { |
| 272 |
BEGIN(retstate); |
| 273 |
obstack_1grow(&stk, 0); |
| 274 |
} |
| 275 |
} |
| 276 |
"<"[a-zA-Z?][a-zA-Z0-9]*"/" { |
| 277 |
char const *s = gcide_entity_to_utf8(yytext); |
| 278 |
if (s) { |
| 279 |
if (strcmp(s, "<?>") == 0) |
| 280 |
dico_log(L_WARN, 0, |
| 281 |
_("%s:%u: unknown or illegible character in a headword"), |
| 282 |
infile, line); |
| 283 |
obstack_grow(&stk, s, strlen(s)); |
| 284 |
} else |
| 285 |
dico_log(L_WARN, 0, _("%s:%u: unrecognized entity: %s"), |
| 286 |
infile, line, yytext); |
| 287 |
} |
| 288 |
"<"[a-zA-Z][^/>]*">" /* ignore tag */; |
| 289 |
["*`] ; |
| 290 |
"\\'d8" ; /* ignore double vertical bar */ |
| 291 |
"\\'"{XD}{XD} { |
| 292 |
char const *s = gcide_escape_to_utf8(yytext+2); |
| 293 |
|
| 294 |
if (s) { |
| 295 |
int len = strlen(s); |
| 296 |
obstack_grow(&stk, s, len); |
| 297 |
} else { |
| 298 |
obstack_grow(&stk, yytext, yyleng); |
| 299 |
dico_log(L_WARN, 0, |
| 300 |
_("%s:%u: unknown character sequence %s"), |
| 301 |
infile, line, yytext); |
| 302 |
} |
| 303 |
} |
| 304 |
[ \t] add_space(&stk); |
| 305 |
. obstack_grow(&stk, yytext, yyleng); |
| 306 |
\n { |
| 307 |
add_space(&stk); |
| 308 |
advance_line(); |
| 309 |
} |
| 310 |
} |
| 311 |
<COMMENT>{ |
| 312 |
"-->" { BEGIN(INITIAL); } |
| 313 |
. ; |
| 314 |
\n advance_line(); |
| 315 |
} |
| 316 |
<HTMLCOM>{ |
| 317 |
"!>" { BEGIN(INITIAL); } |
| 318 |
. ; |
| 319 |
\n advance_line(); |
| 320 |
} |
| 321 |
%% |
| 322 |
|
| 323 |
int |
| 324 |
yywrap(void) |
| 325 |
{ |
| 326 |
flush_headwords(); |
| 327 |
|
| 328 |
if (letters[letno] == 0) |
| 329 |
return 1; |
| 330 |
yy_flex_debug = debug_letters && strchr(debug_letters, letters[letno]); |
| 331 |
*suf = letters[letno++]; |
| 332 |
if (yyin) |
| 333 |
fclose(yyin); |
| 334 |
if (verbose_option) |
| 335 |
dico_log(L_INFO, 0, _("Indexing %s"), infile); |
| 336 |
yyin = fopen(infile, "r"); |
| 337 |
if (!yyin) { |
| 338 |
dico_log(L_ERR, errno, _("cannot open file %s"), infile); |
| 339 |
exit(EX_NOINPUT); |
| 340 |
} |
| 341 |
file_position = char_position = headword_position = 0; |
| 342 |
line = 1; |
| 343 |
return 0; |
| 344 |
} |
| 345 |
|
| 346 |
static int inherit_fd; |
| 347 |
#include "idxgcide-cli.h" |
| 348 |
|
| 349 |
/* Usage: idxgcide DICTDIR [IDXDIR] */ |
| 350 |
int |
| 351 |
main(int argc, char **argv) |
| 352 |
{ |
| 353 |
int index; |
| 354 |
|
| 355 |
appi18n_init(); |
| 356 |
dico_set_program_name(argv[0]); |
| 357 |
yy_flex_debug = 0; |
| 358 |
|
| 359 |
get_options(argc, argv, &index); |
| 360 |
|
| 361 |
idx_page = malloc(idx_header.ihdr_pagesize); |
| 362 |
if (!idx_page) { |
| 363 |
DICO_LOG_ERRNO(); |
| 364 |
exit(EX_UNAVAILABLE); |
| 365 |
} |
| 366 |
idx_page->ipg_header.hdr.phdr_numentries = 0; |
| 367 |
idx_page->ipg_header.hdr.phdr_text_offset = idx_header.ihdr_pagesize / 2; |
| 368 |
|
| 369 |
idx_header.ihdr_maxpageref = idx_header.ihdr_pagesize / 2 / |
| 370 |
sizeof(struct gcide_ref); |
| 371 |
if (idx_header.ihdr_maxpageref < 1) { |
| 372 |
dico_log(L_ERR, 0, _("page size too small")); |
| 373 |
exit(EX_USAGE); |
| 374 |
} |
| 375 |
idx_header.ihdr_maxpageref--; |
| 376 |
|
| 377 |
argc -= index; |
| 378 |
argv += index; |
| 379 |
|
| 380 |
if (argc < 1 || argc > 2) { |
| 381 |
dico_log(L_ERR, 0, _("bad number of arguments")); |
| 382 |
exit(EX_USAGE); |
| 383 |
} |
| 384 |
dictdir = argv[0]; |
| 385 |
idxdir = argc == 2 ? argv[1] : dictdir; |
| 386 |
|
| 387 |
infile = dico_full_file_name(dictdir, "CIDE.A"); |
| 388 |
if (!infile) { |
| 389 |
DICO_LOG_MEMERR(); |
| 390 |
exit(EX_UNAVAILABLE); |
| 391 |
} |
| 392 |
suf = infile + strlen(infile) - 1; |
| 393 |
|
| 394 |
if (dry_run_option && !inherit_fd) |
| 395 |
idxname = "/dev/null"; |
| 396 |
else |
| 397 |
idxname = dico_full_file_name(idxdir, GCIDE_IDX_FILE_NAME); |
| 398 |
if (!idxname) { |
| 399 |
DICO_LOG_MEMERR(); |
| 400 |
exit(EX_UNAVAILABLE); |
| 401 |
} |
| 402 |
|
| 403 |
if (inherit_fd) { |
| 404 |
if (ftruncate(inherit_fd, 0)) { |
| 405 |
dico_log(L_ERR, errno, _("cannot truncate index file `%s'"), idxname); |
| 406 |
exit(EX_CANTCREAT); |
| 407 |
} |
| 408 |
idxfile = fdopen(inherit_fd, "w"); |
| 409 |
} else { |
| 410 |
idxfile = fopen(idxname, "w"); |
| 411 |
} |
| 412 |
if (!idxfile) { |
| 413 |
dico_log(L_ERR, errno, _("cannot create index file `%s'"), idxname); |
| 414 |
exit(EX_CANTCREAT); |
| 415 |
} |
| 416 |
fseek(idxfile, idx_header.ihdr_pagesize, SEEK_SET); |
| 417 |
|
| 418 |
obstack_init(&stk); |
| 419 |
obstack_init(&refstk); |
| 420 |
|
| 421 |
letno = 0; |
| 422 |
yywrap(); |
| 423 |
while (yylex ()) |
| 424 |
; |
| 425 |
|
| 426 |
flush_refs(); |
| 427 |
|
| 428 |
fseek(idxfile, 0, SEEK_SET); |
| 429 |
full_write(&idx_header, sizeof(idx_header)); |
| 430 |
fclose(idxfile); |
| 431 |
|
| 432 |
if (verbose_option) |
| 433 |
dico_log(L_INFO, 0, |
| 434 |
_("headwords=%lu, definitions=%lu, refs per page=%lu, " |
| 435 |
"pages=%lu"), |
| 436 |
idx_header.ihdr_num_headwords, |
| 437 |
idx_header.ihdr_num_defs, |
| 438 |
idx_header.ihdr_maxpageref, |
| 439 |
idx_header.ihdr_num_pages); |
| 440 |
|
| 441 |
exit (0); |
| 442 |
} |
| 443 |
|
| 444 |
/* Local Variables: */ |
| 445 |
/* mode: c */ |
| 446 |
/* eval: (setq buffer-file-coding-system (if window-system 'utf-8 'raw-text-unix)) */ |
| 447 |
/* End: */ |