Debian Codegraph @

src/backend/tsearch/spell.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * spell.c
4 * Normalizing word with ISpell
5 *
6 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
7 *
8 * Ispell dictionary
9 * -----------------
10 *
11 * Rules of dictionaries are defined in two files with .affix and .dict
12 * extensions. They are used by spell checker programs Ispell and Hunspell.
13 *
14 * An .affix file declares morphological rules to get a basic form of words.
15 * The format of an .affix file has different structure for Ispell and Hunspell
16 * dictionaries. The Hunspell format is more complicated. But when an .affix
17 * file is imported and compiled, it is stored in the same structure AffixNode.
18 *
19 * A .dict file stores a list of basic forms of words with references to
20 * affix rules. The format of a .dict file has the same structure for Ispell
21 * and Hunspell dictionaries.
22 *
23 * Compilation of a dictionary
24 * ---------------------------
25 *
26 * A compiled dictionary is stored in the IspellDict structure. Compilation of
27 * a dictionary is divided into the several steps:
28 * - NIImportDictionary() - stores each word of a .dict file in the
29 * temporary Spell field.
30 * - NIImportAffixes() - stores affix rules of an .affix file in the
31 * Affix field (not temporary) if an .affix file has the Ispell format.
32 * -> NIImportOOAffixes() - stores affix rules if an .affix file has the
33 * Hunspell format. The AffixData field is initialized if AF parameter
34 * is defined.
35 * - NISortDictionary() - builds a prefix tree (Trie) from the words list
36 * and stores it in the Dictionary field. The words list is got from the
37 * Spell field. The AffixData field is initialized if AF parameter is not
38 * defined.
39 * - NISortAffixes():
40 * - builds a list of compound affixes from the affix list and stores it
41 * in the CompoundAffix.
42 * - builds prefix trees (Trie) from the affix list for prefixes and suffixes
43 * and stores them in Suffix and Prefix fields.
44 * The affix list is got from the Affix field.
45 *
46 * Memory management
47 * -----------------
48 *
49 * The IspellDict structure has the Spell field which is used only in compile
50 * time. The Spell field stores a words list. It can take a lot of memory.
51 * Therefore when a dictionary is compiled this field is cleared by
52 * NIFinishBuild().
53 *
54 * All resources which should cleared by NIFinishBuild() is initialized using
55 * tmpalloc() and tmpalloc0().
56 *
57 * IDENTIFICATION
58 * src/backend/tsearch/spell.c
59 *
60 *-------------------------------------------------------------------------
61 */
62
63 #include "postgres.h"
64
65 #include "catalog/pg_collation.h"
66 #include "miscadmin.h"
67 #include "tsearch/dicts/spell.h"
68 #include "tsearch/ts_locale.h"
69 #include "utils/formatting.h"
70 #include "utils/memutils.h"
71
72
73 /*
74 * Initialization requires a lot of memory that's not needed
75 * after the initialization is done. During initialization,
76 * CurrentMemoryContext is the long-lived memory context associated
77 * with the dictionary cache entry. We keep the short-lived stuff
78 * in the Conf->buildCxt context.
79 */
80 #define tmpalloc(sz) MemoryContextAlloc(Conf->buildCxt, (sz))
81 #define tmpalloc0(sz) MemoryContextAllocZero(Conf->buildCxt, (sz))
82
83 /*
84 * Prepare for constructing an ISpell dictionary.
85 *
86 * The IspellDict struct is assumed to be zeroed when allocated.
87 */
88 void
89 NIStartBuild(IspellDict *Conf)
90 {
91 /*
92 * The temp context is a child of CurTransactionContext, so that it will
93 * go away automatically on error.
94 */
95 Conf->buildCxt = AllocSetContextCreate(CurTransactionContext,
96 "Ispell dictionary init context",
97 ALLOCSET_DEFAULT_SIZES);
98 }
99
100 /*
101 * Clean up when dictionary construction is complete.
102 */
103 void
104 NIFinishBuild(IspellDict *Conf)
105 {
106 /* Release no-longer-needed temp memory */
107 MemoryContextDelete(Conf->buildCxt);
108 /* Just for cleanliness, zero the now-dangling pointers */
109 Conf->buildCxt = NULL;
110 Conf->Spell = NULL;
111 Conf->firstfree = NULL;
112 Conf->CompoundAffixFlags = NULL;
113 }
114
115
116 /*
117 * "Compact" palloc: allocate without extra palloc overhead.
118 *
119 * Since we have no need to free the ispell data items individually, there's
120 * not much value in the per-chunk overhead normally consumed by palloc.
121 * Getting rid of it is helpful since ispell can allocate a lot of small nodes.
122 *
123 * We currently pre-zero all data allocated this way, even though some of it
124 * doesn't need that. The cpalloc and cpalloc0 macros are just documentation
125 * to indicate which allocations actually require zeroing.
126 */
127 #define COMPACT_ALLOC_CHUNK 8192 /* amount to get from palloc at once */
128 #define COMPACT_MAX_REQ 1024 /* must be < COMPACT_ALLOC_CHUNK */
129
130 static void *
131 compact_palloc0(IspellDict *Conf, size_t size)
132 {
133 void *result;
134
135 /* Should only be called during init */
136 Assert(Conf->buildCxt != NULL);
137
138 /* No point in this for large chunks */
139 if (size > COMPACT_MAX_REQ)
140 return palloc0(size);
141
142 /* Keep everything maxaligned */
143 size = MAXALIGN(size);
144
145 /* Need more space? */
146 if (size > Conf->avail)
147 {
148 Conf->firstfree = palloc0(COMPACT_ALLOC_CHUNK);
149 Conf->avail = COMPACT_ALLOC_CHUNK;
150 }
151
152 result = Conf->firstfree;
153 Conf->firstfree += size;
154 Conf->avail -= size;
155
156 return result;
157 }
158
159 #define cpalloc(size) compact_palloc0(Conf, size)
160 #define cpalloc0(size) compact_palloc0(Conf, size)
161
162 static char *
163 cpstrdup(IspellDict *Conf, const char *str)
164 {
165 char *res = cpalloc(strlen(str) + 1);
166
167 strcpy(res, str);
168 return res;
169 }
170
171
172 /*
173 * Apply str_tolower(), producing a temporary result (in the buildCxt).
174 */
175 static char *
176 lowerstr_ctx(IspellDict *Conf, const char *src)
177 {
178 MemoryContext saveCtx;
179 char *dst;
180
181 saveCtx = MemoryContextSwitchTo(Conf->buildCxt);
182 dst = str_tolower(src, strlen(src), DEFAULT_COLLATION_OID);
183 MemoryContextSwitchTo(saveCtx);
184
185 return dst;
186 }
187
188 #define MAX_NORM 1024
189 #define MAXNORMLEN 256
190
191 #define STRNCMP(s,p) strncmp( (s), (p), strlen(p) )
192 #define GETWCHAR(W,L,N,T) ( ((const uint8*)(W))[ ((T)==FF_PREFIX) ? (N) : ( (L) - 1 - (N) ) ] )
193 #define GETCHAR(A,N,T) GETWCHAR( (A)->repl, (A)->replen, N, T )
194
195 static const char *VoidString = "";
196
197 static int
198 cmpspell(const void *s1, const void *s2)
199 {
200 return strcmp((*(SPELL *const *) s1)->word, (*(SPELL *const *) s2)->word);
201 }
202
203 static int
204 cmpspellaffix(const void *s1, const void *s2)
205 {
206 return strcmp((*(SPELL *const *) s1)->p.flag,
207 (*(SPELL *const *) s2)->p.flag);
208 }
209
210 static int
211 cmpcmdflag(const void *f1, const void *f2)
212 {
213 CompoundAffixFlag *fv1 = (CompoundAffixFlag *) f1,
214 *fv2 = (CompoundAffixFlag *) f2;
215
216 Assert(fv1->flagMode == fv2->flagMode);
217
218 if (fv1->flagMode == FM_NUM)
219 {
220 if (fv1->flag.i == fv2->flag.i)
221 return 0;
222
223 return (fv1->flag.i > fv2->flag.i) ? 1 : -1;
224 }
225
226 return strcmp(fv1->flag.s, fv2->flag.s);
227 }
228
229 static char *
230 findchar(char *str, int c)
231 {
232 while (*str)
233 {
234 if (t_iseq(str, c))
235 return str;
236 str += pg_mblen_cstr(str);
237 }
238
239 return NULL;
240 }
241
242 static char *
243 findchar2(char *str, int c1, int c2)
244 {
245 while (*str)
246 {
247 if (t_iseq(str, c1) || t_iseq(str, c2))
248 return str;
249 str += pg_mblen_cstr(str);
250 }
251
252 return NULL;
253 }
254
255
256 /* backward string compare for suffix tree operations */
257 static int
258 strbcmp(const unsigned char *s1, const unsigned char *s2)
259 {
260 int l1 = strlen((const char *) s1) - 1,
261 l2 = strlen((const char *) s2) - 1;
262
263 while (l1 >= 0 && l2 >= 0)
264 {
265 if (s1[l1] < s2[l2])
266 return -1;
267 if (s1[l1] > s2[l2])
268 return 1;
269 l1--;
270 l2--;
271 }
272 if (l1 < l2)
273 return -1;
274 if (l1 > l2)
275 return 1;
276
277 return 0;
278 }
279
280 static int
281 strbncmp(const unsigned char *s1, const unsigned char *s2, size_t count)
282 {
283 int l1 = strlen((const char *) s1) - 1,
284 l2 = strlen((const char *) s2) - 1,
285 l = count;
286
287 while (l1 >= 0 && l2 >= 0 && l > 0)
288 {
289 if (s1[l1] < s2[l2])
290 return -1;
291 if (s1[l1] > s2[l2])
292 return 1;
293 l1--;
294 l2--;
295 l--;
296 }
297 if (l == 0)
298 return 0;
299 if (l1 < l2)
300 return -1;
301 if (l1 > l2)
302 return 1;
303 return 0;
304 }
305
306 /*
307 * Compares affixes.
308 * First compares the type of an affix. Prefixes should go before affixes.
309 * If types are equal then compares replaceable string.
310 */
311 static int
312 cmpaffix(const void *s1, const void *s2)
313 {
314 const AFFIX *a1 = (const AFFIX *) s1;
315 const AFFIX *a2 = (const AFFIX *) s2;
316
317 if (a1->type < a2->type)
318 return -1;
319 if (a1->type > a2->type)
320 return 1;
321 if (a1->type == FF_PREFIX)
322 return strcmp(a1->repl, a2->repl);
323 else
324 return strbcmp((const unsigned char *) a1->repl,
325 (const unsigned char *) a2->repl);
326 }
327
328 /*
329 * Gets an affix flag from the set of affix flags (sflagset).
330 *
331 * Several flags can be stored in a single string. Flags can be represented by:
332 * - 1 character (FM_CHAR). A character may be Unicode.
333 * - 2 characters (FM_LONG). A character may be Unicode.
334 * - numbers from 1 to 65000 (FM_NUM).
335 *
336 * Depending on the flagMode an affix string can have the following format:
337 * - FM_CHAR: ABCD
338 * Here we have 4 flags: A, B, C and D
339 * - FM_LONG: ABCDE*
340 * Here we have 3 flags: AB, CD and E*
341 * - FM_NUM: 200,205,50
342 * Here we have 3 flags: 200, 205 and 50
343 *
344 * Conf: current dictionary.
345 * sflagset: the set of affix flags. Returns a reference to the start of a next
346 * affix flag.
347 * sflag: returns an affix flag from sflagset.
348 */
349 static void
350 getNextFlagFromString(IspellDict *Conf, const char **sflagset, char *sflag)
351 {
352 int32 s;
353 char *next;
354 const char *sbuf = *sflagset;
355 int maxstep;
356 int clen;
357 bool stop = false;
358 bool met_comma = false;
359
360 maxstep = (Conf->flagMode == FM_LONG) ? 2 : 1;
361
362 while (**sflagset)
363 {
364 switch (Conf->flagMode)
365 {
366 case FM_LONG:
367 case FM_CHAR:
368 clen = ts_copychar_cstr(sflag, *sflagset);
369 sflag += clen;
370
371 /* Go to start of the next flag */
372 *sflagset += clen;
373
374 /* Check if we get all characters of flag */
375 maxstep--;
376 stop = (maxstep == 0);
377 break;
378 case FM_NUM:
379 errno = 0;
380 s = strtol(*sflagset, &next, 10);
381 if (*sflagset == next || errno == ERANGE)
382 ereport(ERROR,
383 (errcode(ERRCODE_CONFIG_FILE_ERROR),
384 errmsg("invalid affix flag \"%s\"", *sflagset)));
385 if (s < 0 || s > FLAGNUM_MAXSIZE)
386 ereport(ERROR,
387 (errcode(ERRCODE_CONFIG_FILE_ERROR),
388 errmsg("affix flag \"%s\" is out of range",
389 *sflagset)));
390 sflag += sprintf(sflag, "%0d", s);
391
392 /* Go to start of the next flag */
393 *sflagset = next;
394 while (**sflagset)
395 {
396 if (isdigit((unsigned char) **sflagset))
397 {
398 if (!met_comma)
399 ereport(ERROR,
400 (errcode(ERRCODE_CONFIG_FILE_ERROR),
401 errmsg("invalid affix flag \"%s\"",
402 *sflagset)));
403 break;
404 }
405 else if (t_iseq(*sflagset, ','))
406 {
407 if (met_comma)
408 ereport(ERROR,
409 (errcode(ERRCODE_CONFIG_FILE_ERROR),
410 errmsg("invalid affix flag \"%s\"",
411 *sflagset)));
412 met_comma = true;
413 }
414 else if (!isspace((unsigned char) **sflagset))
415 {
416 ereport(ERROR,
417 (errcode(ERRCODE_CONFIG_FILE_ERROR),
418 errmsg("invalid character in affix flag \"%s\"",
419 *sflagset)));
420 }
421
422 *sflagset += pg_mblen_cstr(*sflagset);
423 }
424 stop = true;
425 break;
426 default:
427 elog(ERROR, "unrecognized type of Conf->flagMode: %d",
428 Conf->flagMode);
429 }
430
431 if (stop)
432 break;
433 }
434
435 if (Conf->flagMode == FM_LONG && maxstep > 0)
436 ereport(ERROR,
437 (errcode(ERRCODE_CONFIG_FILE_ERROR),
438 errmsg("invalid affix flag \"%s\" with \"long\" flag value",
439 sbuf)));
440
441 *sflag = '\0';
442 }
443
444 /*
445 * Checks if the affix set Conf->AffixData[affix] contains affixflag.
446 * Conf->AffixData[affix] does not contain affixflag if this flag is not used
447 * actually by the .dict file.
448 *
449 * Conf: current dictionary.
450 * affix: index of the Conf->AffixData array.
451 * affixflag: the affix flag.
452 *
453 * Returns true if the string Conf->AffixData[affix] contains affixflag,
454 * otherwise returns false.
455 */
456 static bool
457 IsAffixFlagInUse(IspellDict *Conf, int affix, const char *affixflag)
458 {
459 const char *flagcur;
460 char flag[BUFSIZ];
461
462 if (*affixflag == 0)
463 return true;
464
465 Assert(affix < Conf->nAffixData);
466
467 flagcur = Conf->AffixData[affix];
468
469 while (*flagcur)
470 {
471 getNextFlagFromString(Conf, &flagcur, flag);
472 /* Compare first affix flag in flagcur with affixflag */
473 if (strcmp(flag, affixflag) == 0)
474 return true;
475 }
476
477 /* Could not find affixflag */
478 return false;
479 }
480
481 /*
482 * Adds the new word into the temporary array Spell.
483 *
484 * Conf: current dictionary.
485 * word: new word.
486 * flag: set of affix flags. Single flag can be get by getNextFlagFromString().
487 */
488 static void
489 NIAddSpell(IspellDict *Conf, const char *word, const char *flag)
490 {
491 if (Conf->nspell >= Conf->mspell)
492 {
493 if (Conf->mspell)
494 {
495 Conf->mspell *= 2;
496 Conf->Spell = (SPELL **) repalloc(Conf->Spell, Conf->mspell * sizeof(SPELL *));
497 }
498 else
499 {
500 Conf->mspell = 1024 * 20;
501 Conf->Spell = (SPELL **) tmpalloc(Conf->mspell * sizeof(SPELL *));
502 }
503 }
504 Conf->Spell[Conf->nspell] = (SPELL *) tmpalloc(SPELLHDRSZ + strlen(word) + 1);
505 strcpy(Conf->Spell[Conf->nspell]->word, word);
506 Conf->Spell[Conf->nspell]->p.flag = (*flag != '\0')
507 ? cpstrdup(Conf, flag) : VoidString;
508 Conf->nspell++;
509 }
510
511 /*
512 * Imports dictionary into the temporary array Spell.
513 *
514 * Note caller must already have applied get_tsearch_config_filename.
515 *
516 * Conf: current dictionary.
517 * filename: path to the .dict file.
518 */
519 void
520 NIImportDictionary(IspellDict *Conf, const char *filename)
521 {
522 tsearch_readline_state trst;
523 char *line;
524
525 if (!tsearch_readline_begin(&trst, filename))
526 ereport(ERROR,
527 (errcode(ERRCODE_CONFIG_FILE_ERROR),
528 errmsg("could not open dictionary file \"%s\": %m",
529 filename)));
530
531 while ((line = tsearch_readline(&trst)) != NULL)
532 {
533 char *s,
534 *pstr;
535
536 /* Set of affix flags */
537 const char *flag;
538
539 /* Extract flag from the line */
540 flag = NULL;
541 if ((s = findchar(line, '/')))
542 {
543 *s++ = '\0';
544 flag = s;
545 while (*s)
546 {
547 /* we allow only single encoded flags for faster works */
548 if (pg_mblen_cstr(s) == 1 && isprint((unsigned char) *s) && !isspace((unsigned char) *s))
549 s++;
550 else
551 {
552 *s = '\0';
553 break;
554 }
555 }
556 }
557 else
558 flag = "";
559
560 /* Remove trailing spaces */
561 s = line;
562 while (*s)
563 {
564 if (isspace((unsigned char) *s))
565 {
566 *s = '\0';
567 break;
568 }
569 s += pg_mblen_cstr(s);
570 }
571 pstr = lowerstr_ctx(Conf, line);
572
573 NIAddSpell(Conf, pstr, flag);
574 pfree(pstr);
575
576 pfree(line);
577 }
578 tsearch_readline_end(&trst);
579 }
580
581 /*
582 * Searches a basic form of word in the prefix tree. This word was generated
583 * using an affix rule. This rule may not be presented in an affix set of
584 * a basic form of word.
585 *
586 * For example, we have the entry in the .dict file:
587 * meter/GMD
588 *
589 * The affix rule with the flag S:
590 * SFX S y ies [^aeiou]y
591 * is not presented here.
592 *
593 * The affix rule with the flag M:
594 * SFX M 0 's .
595 * is presented here.
596 *
597 * Conf: current dictionary.
598 * word: basic form of word.
599 * affixflag: affix flag, by which a basic form of word was generated.
600 * flag: compound flag used to compare with StopMiddle->compoundflag.
601 *
602 * Returns 1 if the word was found in the prefix tree, else returns 0.
603 */
604 static int
605 FindWord(IspellDict *Conf, const char *word, const char *affixflag, int flag)
606 {
607 SPNode *node = Conf->Dictionary;
608 SPNodeData *StopLow,
609 *StopHigh,
610 *StopMiddle;
611 const uint8 *ptr = (const uint8 *) word;
612
613 flag &= FF_COMPOUNDFLAGMASK;
614
615 while (node && *ptr)
616 {
617 StopLow = node->data;
618 StopHigh = node->data + node->length;
619 while (StopLow < StopHigh)
620 {
621 StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
622 if (StopMiddle->val == *ptr)
623 {
624 if (*(ptr + 1) == '\0' && StopMiddle->isword)
625 {
626 if (flag == 0)
627 {
628 /*
629 * The word can be formed only with another word. And
630 * in the flag parameter there is not a sign that we
631 * search compound words.
632 */
633 if (StopMiddle->compoundflag & FF_COMPOUNDONLY)
634 return 0;
635 }
636 else if ((flag & StopMiddle->compoundflag) == 0)
637 return 0;
638
639 /*
640 * Check if this affix rule is presented in the affix set
641 * with index StopMiddle->affix.
642 */
643 if (IsAffixFlagInUse(Conf, StopMiddle->affix, affixflag))
644 return 1;
645 }
646 node = StopMiddle->node;
647 ptr++;
648 break;
649 }
650 else if (StopMiddle->val < *ptr)
651 StopLow = StopMiddle + 1;
652 else
653 StopHigh = StopMiddle;
654 }
655 if (StopLow >= StopHigh)
656 break;
657 }
658 return 0;
659 }
660
661 /*
662 * Adds a new affix rule to the Affix field.
663 *
664 * Conf: current dictionary.
665 * flag: affix flag ('\' in the below example).
666 * flagflags: set of flags from the flagval field for this affix rule. This set
667 * is listed after '/' character in the added string (repl).
668 *
669 * For example L flag in the hunspell_sample.affix:
670 * SFX \ 0 Y/L [^Y]
671 *
672 * mask: condition for search ('[^Y]' in the above example).
673 * find: stripping characters from beginning (at prefix) or end (at suffix)
674 * of the word ('0' in the above example, 0 means that there is not
675 * stripping character).
676 * repl: adding string after stripping ('Y' in the above example).
677 * type: FF_SUFFIX or FF_PREFIX.
678 */
679 static void
680 NIAddAffix(IspellDict *Conf, const char *flag, char flagflags, const char *mask,
681 const char *find, const char *repl, int type)
682 {
683 AFFIX *Affix;
684
685 if (Conf->naffixes >= Conf->maffixes)
686 {
687 if (Conf->maffixes)
688 {
689 Conf->maffixes *= 2;
690 Conf->Affix = (AFFIX *) repalloc(Conf->Affix, Conf->maffixes * sizeof(AFFIX));
691 }
692 else
693 {
694 Conf->maffixes = 16;
695 Conf->Affix = (AFFIX *) palloc(Conf->maffixes * sizeof(AFFIX));
696 }
697 }
698
699 Affix = Conf->Affix + Conf->naffixes;
700
701 /* This affix rule can be applied for words with any ending */
702 if (strcmp(mask, ".") == 0 || *mask == '\0')
703 {
704 Affix->issimple = 1;
705 Affix->isregis = 0;
706 }
707 /* This affix rule will use regis to search word ending */
708 else if (RS_isRegis(mask))
709 {
710 Affix->issimple = 0;
711 Affix->isregis = 1;
712 RS_compile(&(Affix->reg.regis), (type == FF_SUFFIX),
713 *mask ? mask : VoidString);
714 }
715 /* This affix rule will use regex_t to search word ending */
716 else
717 {
718 int masklen;
719 int wmasklen;
720 int err;
721 pg_wchar *wmask;
722 char *tmask;
723
724 Affix->issimple = 0;
725 Affix->isregis = 0;
726 tmask = (char *) tmpalloc(strlen(mask) + 3);
727 if (type == FF_SUFFIX)
728 sprintf(tmask, "%s$", mask);
729 else
730 sprintf(tmask, "^%s", mask);
731
732 masklen = strlen(tmask);
733 wmask = (pg_wchar *) tmpalloc((masklen + 1) * sizeof(pg_wchar));
734 wmasklen = pg_mb2wchar_with_len(tmask, wmask, masklen);
735
736 /*
737 * The regex and all internal state created by pg_regcomp are
738 * allocated in the dictionary's memory context, and will be freed
739 * automatically when it is destroyed.
740 */
741 Affix->reg.pregex = palloc(sizeof(regex_t));
742 err = pg_regcomp(Affix->reg.pregex, wmask, wmasklen,
743 REG_ADVANCED | REG_NOSUB,
744 DEFAULT_COLLATION_OID);
745 if (err)
746 {
747 char errstr[100];
748
749 pg_regerror(err, Affix->reg.pregex, errstr, sizeof(errstr));
750 ereport(ERROR,
751 (errcode(ERRCODE_INVALID_REGULAR_EXPRESSION),
752 errmsg("invalid regular expression: %s", errstr)));
753 }
754 }
755
756 Affix->flagflags = flagflags;
757 if ((Affix->flagflags & FF_COMPOUNDONLY) || (Affix->flagflags & FF_COMPOUNDPERMITFLAG))
758 {
759 if ((Affix->flagflags & FF_COMPOUNDFLAG) == 0)
760 Affix->flagflags |= FF_COMPOUNDFLAG;
761 }
762 Affix->flag = cpstrdup(Conf, flag);
763 Affix->type = type;
764
765 Affix->find = (find && *find) ? cpstrdup(Conf, find) : VoidString;
766 if ((Affix->replen = strlen(repl)) > 0)
767 Affix->repl = cpstrdup(Conf, repl);
768 else
769 Affix->repl = VoidString;
770 Conf->naffixes++;
771 }
772
773 /* Parsing states for parse_affentry() and friends */
774 #define PAE_WAIT_MASK 0
775 #define PAE_INMASK 1
776 #define PAE_WAIT_FIND 2
777 #define PAE_INFIND 3
778 #define PAE_WAIT_REPL 4
779 #define PAE_INREPL 5
780 #define PAE_WAIT_TYPE 6
781 #define PAE_WAIT_FLAG 7
782
783 /*
784 * Parse next space-separated field of an .affix file line.
785 *
786 * *str is the input pointer (will be advanced past field)
787 * next is where to copy the field value to, with null termination
788 *
789 * The buffer at "next" must be of size BUFSIZ; we truncate the input to fit.
790 *
791 * Returns true if we found a field, false if not.
792 */
793 static bool
794 get_nextfield(char **str, char *next)
795 {
796 int state = PAE_WAIT_MASK;
797 int avail = BUFSIZ;
798
799 while (**str)
800 {
801 int clen = pg_mblen_cstr(*str);
802
803 if (state == PAE_WAIT_MASK)
804 {
805 if (t_iseq(*str, '#'))
806 return false;
807 else if (!isspace((unsigned char) **str))
808 {
809 if (clen < avail)
810 {
811 ts_copychar_with_len(next, *str, clen);
812 next += clen;
813 avail -= clen;
814 }
815 state = PAE_INMASK;
816 }
817 }
818 else /* state == PAE_INMASK */
819 {
820 if (isspace((unsigned char) **str))
821 {
822 *next = '\0';
823 return true;
824 }
825 else
826 {
827 if (clen < avail)
828 {
829 ts_copychar_with_len(next, *str, clen);
830 next += clen;
831 avail -= clen;
832 }
833 }
834 }
835 *str += clen;
836 }
837
838 *next = '\0';
839
840 return (state == PAE_INMASK); /* OK if we got a nonempty field */
841 }
842
843 /*
844 * Parses entry of an .affix file of MySpell or Hunspell format.
845 *
846 * An .affix file entry has the following format:
847 * - header
848 * <type> <flag> <cross_flag> <flag_count>
849 * - fields after header:
850 * <type> <flag> <find> <replace> <mask>
851 *
852 * str is the input line
853 * field values are returned to type etc, which must be buffers of size BUFSIZ.
854 *
855 * Returns number of fields found; any omitted fields are set to empty strings.
856 */
857 static int
858 parse_ooaffentry(char *str, char *type, char *flag, char *find,
859 char *repl, char *mask)
860 {
861 int state = PAE_WAIT_TYPE;
862 int fields_read = 0;
863 bool valid = false;
864
865 *type = *flag = *find = *repl = *mask = '\0';
866
867 while (*str)
868 {
869 switch (state)
870 {
871 case PAE_WAIT_TYPE:
872 valid = get_nextfield(&str, type);
873 state = PAE_WAIT_FLAG;
874 break;
875 case PAE_WAIT_FLAG:
876 valid = get_nextfield(&str, flag);
877 state = PAE_WAIT_FIND;
878 break;
879 case PAE_WAIT_FIND:
880 valid = get_nextfield(&str, find);
881 state = PAE_WAIT_REPL;
882 break;
883 case PAE_WAIT_REPL:
884 valid = get_nextfield(&str, repl);
885 state = PAE_WAIT_MASK;
886 break;
887 case PAE_WAIT_MASK:
888 valid = get_nextfield(&str, mask);
889 state = -1; /* force loop exit */
890 break;
891 default:
892 elog(ERROR, "unrecognized state in parse_ooaffentry: %d",
893 state);
894 break;
895 }
896 if (valid)
897 fields_read++;
898 else
899 break; /* early EOL */
900 if (state < 0)
901 break; /* got all fields */
902 }
903
904 return fields_read;
905 }
906
907 /*
908 * Parses entry of an .affix file of Ispell format
909 *
910 * An .affix file entry has the following format:
911 * <mask> > [-<find>,]<replace>
912 *
913 * Output buffers mask, find, repl must be of length BUFSIZ;
914 * we truncate the input to fit.
915 */
916 static bool
917 parse_affentry(const char *str, char *mask, char *find, char *repl)
918 {
919 int state = PAE_WAIT_MASK;
920 char *pmask = mask,
921 *pfind = find,
922 *prepl = repl;
923 char *emask = mask + BUFSIZ;
924 char *efind = find + BUFSIZ;
925 char *erepl = repl + BUFSIZ;
926
927 *mask = *find = *repl = '\0';
928
929 while (*str)
930 {
931 int clen = pg_mblen_cstr(str);
932
933 if (state == PAE_WAIT_MASK)
934 {
935 if (t_iseq(str, '#'))
936 return false;
937 else if (!isspace((unsigned char) *str))
938 {
939 if (pmask < emask - clen)
940 pmask += ts_copychar_with_len(pmask, str, clen);
941 state = PAE_INMASK;
942 }
943 }
944 else if (state == PAE_INMASK)
945 {
946 if (t_iseq(str, '>'))
947 {
948 *pmask = '\0';
949 state = PAE_WAIT_FIND;
950 }
951 else if (!isspace((unsigned char) *str))
952 {
953 if (pmask < emask - clen)
954 pmask += ts_copychar_with_len(pmask, str, clen);
955 }
956 }
957 else if (state == PAE_WAIT_FIND)
958 {
959 if (t_iseq(str, '-'))
960 {
961 state = PAE_INFIND;
962 }
963 else if (t_isalpha_cstr(str) || t_iseq(str, '\'') /* english 's */ )
964 {
965 if (prepl < erepl - clen)
966 prepl += ts_copychar_with_len(prepl, str, clen);
967 state = PAE_INREPL;
968 }
969 else if (!isspace((unsigned char) *str))
970 ereport(ERROR,
971 (errcode(ERRCODE_CONFIG_FILE_ERROR),
972 errmsg("syntax error")));
973 }
974 else if (state == PAE_INFIND)
975 {
976 if (t_iseq(str, ','))
977 {
978 *pfind = '\0';
979 state = PAE_WAIT_REPL;
980 }
981 else if (t_isalpha_cstr(str))
982 {
983 if (pfind < efind - clen)
984 pfind += ts_copychar_with_len(pfind, str, clen);
985 }
986 else if (!isspace((unsigned char) *str))
987 ereport(ERROR,
988 (errcode(ERRCODE_CONFIG_FILE_ERROR),
989 errmsg("syntax error")));
990 }
991 else if (state == PAE_WAIT_REPL)
992 {
993 if (t_iseq(str, '-'))
994 {
995 break; /* void repl */
996 }
997 else if (t_isalpha_cstr(str))
998 {
999 if (prepl < erepl - clen)
1000 prepl += ts_copychar_with_len(prepl, str, clen);
1001 state = PAE_INREPL;
1002 }
1003 else if (!isspace((unsigned char) *str))
1004 ereport(ERROR,
1005 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1006 errmsg("syntax error")));
1007 }
1008 else if (state == PAE_INREPL)
1009 {
1010 if (t_iseq(str, '#'))
1011 {
1012 *prepl = '\0';
1013 break;
1014 }
1015 else if (t_isalpha_cstr(str))
1016 {
1017 if (prepl < erepl - clen)
1018 prepl += ts_copychar_with_len(prepl, str, clen);
1019 }
1020 else if (!isspace((unsigned char) *str))
1021 ereport(ERROR,
1022 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1023 errmsg("syntax error")));
1024 }
1025 else
1026 elog(ERROR, "unrecognized state in parse_affentry: %d", state);
1027
1028 str += clen;
1029 }
1030
1031 *pmask = *pfind = *prepl = '\0';
1032
1033 return (*mask && (*find || *repl));
1034 }
1035
1036 /*
1037 * Sets a Hunspell options depending on flag type.
1038 */
1039 static void
1040 setCompoundAffixFlagValue(IspellDict *Conf, CompoundAffixFlag *entry,
1041 char *s, uint32 val)
1042 {
1043 if (Conf->flagMode == FM_NUM)
1044 {
1045 char *next;
1046 int i;
1047
1048 errno = 0;
1049 i = strtol(s, &next, 10);
1050 if (s == next || errno == ERANGE)
1051 ereport(ERROR,
1052 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1053 errmsg("invalid affix flag \"%s\"", s)));
1054 if (i < 0 || i > FLAGNUM_MAXSIZE)
1055 ereport(ERROR,
1056 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1057 errmsg("affix flag \"%s\" is out of range", s)));
1058
1059 entry->flag.i = i;
1060 }
1061 else
1062 entry->flag.s = cpstrdup(Conf, s);
1063
1064 entry->flagMode = Conf->flagMode;
1065 entry->value = val;
1066 }
1067
1068 /*
1069 * Sets up a correspondence for the affix parameter with the affix flag.
1070 *
1071 * Conf: current dictionary.
1072 * s: affix flag in string.
1073 * val: affix parameter.
1074 */
1075 static void
1076 addCompoundAffixFlagValue(IspellDict *Conf, const char *s, uint32 val)
1077 {
1078 CompoundAffixFlag *newValue;
1079 char sbuf[BUFSIZ];
1080 char *sflag;
1081
1082 while (*s && isspace((unsigned char) *s))
1083 s += pg_mblen_cstr(s);
1084
1085 if (!*s)
1086 ereport(ERROR,
1087 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1088 errmsg("syntax error")));
1089
1090 /* Get flag without \n */
1091 sflag = sbuf;
1092 while (*s && !isspace((unsigned char) *s) && *s != '\n')
1093 {
1094 int clen = pg_mblen_cstr(s);
1095
1096 /* Truncate the input to fit in BUFSIZ */
1097 if (sflag < sbuf + BUFSIZ - clen)
1098 sflag += ts_copychar_with_len(sflag, s, clen);
1099 s += clen;
1100 }
1101 *sflag = '\0';
1102
1103 /* Resize array or allocate memory for array CompoundAffixFlag */
1104 if (Conf->nCompoundAffixFlag >= Conf->mCompoundAffixFlag)
1105 {
1106 if (Conf->mCompoundAffixFlag)
1107 {
1108 Conf->mCompoundAffixFlag *= 2;
1109 Conf->CompoundAffixFlags = (CompoundAffixFlag *)
1110 repalloc(Conf->CompoundAffixFlags,
1111 Conf->mCompoundAffixFlag * sizeof(CompoundAffixFlag));
1112 }
1113 else
1114 {
1115 Conf->mCompoundAffixFlag = 10;
1116 Conf->CompoundAffixFlags = (CompoundAffixFlag *)
1117 tmpalloc(Conf->mCompoundAffixFlag * sizeof(CompoundAffixFlag));
1118 }
1119 }
1120
1121 newValue = Conf->CompoundAffixFlags + Conf->nCompoundAffixFlag;
1122
1123 setCompoundAffixFlagValue(Conf, newValue, sbuf, val);
1124
1125 Conf->usecompound = true;
1126 Conf->nCompoundAffixFlag++;
1127 }
1128
1129 /*
1130 * Returns a set of affix parameters which correspondence to the set of affix
1131 * flags s.
1132 */
1133 static int
1134 getCompoundAffixFlagValue(IspellDict *Conf, const char *s)
1135 {
1136 uint32 flag = 0;
1137 CompoundAffixFlag *found,
1138 key;
1139 char sflag[BUFSIZ];
1140 const char *flagcur;
1141
1142 if (Conf->nCompoundAffixFlag == 0)
1143 return 0;
1144
1145 flagcur = s;
1146 while (*flagcur)
1147 {
1148 getNextFlagFromString(Conf, &flagcur, sflag);
1149 setCompoundAffixFlagValue(Conf, &key, sflag, 0);
1150
1151 found = (CompoundAffixFlag *)
1152 bsearch(&key, Conf->CompoundAffixFlags,
1153 Conf->nCompoundAffixFlag, sizeof(CompoundAffixFlag),
1154 cmpcmdflag);
1155 if (found != NULL)
1156 flag |= found->value;
1157 }
1158
1159 return flag;
1160 }
1161
1162 /*
1163 * Returns a flag set using the s parameter.
1164 *
1165 * If Conf->useFlagAliases is true then the s parameter is index of the
1166 * Conf->AffixData array and function returns its entry.
1167 * Else function returns the s parameter.
1168 */
1169 static const char *
1170 getAffixFlagSet(IspellDict *Conf, char *s)
1171 {
1172 if (Conf->useFlagAliases && *s != '\0')
1173 {
1174 int curaffix;
1175 char *end;
1176
1177 errno = 0;
1178 curaffix = strtol(s, &end, 10);
1179 if (s == end || errno == ERANGE)
1180 ereport(ERROR,
1181 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1182 errmsg("invalid affix alias \"%s\"", s)));
1183
1184 if (curaffix > 0 && curaffix < Conf->nAffixData)
1185 {
1186 if (Conf->AffixData[curaffix] == NULL)
1187 ereport(ERROR,
1188 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1189 errmsg("invalid affix alias \"%s\"", s)));
1190
1191 /*
1192 * Do not subtract 1 from curaffix because empty string was added
1193 * in NIImportOOAffixes
1194 */
1195 return Conf->AffixData[curaffix];
1196 }
1197 else if (curaffix > Conf->nAffixData)
1198 ereport(ERROR,
1199 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1200 errmsg("invalid affix alias \"%s\"", s)));
1201 return VoidString;
1202 }
1203 else
1204 return s;
1205 }
1206
1207 /*
1208 * Import an affix file that follows MySpell or Hunspell format.
1209 *
1210 * Conf: current dictionary.
1211 * filename: path to the .affix file.
1212 */
1213 static void
1214 NIImportOOAffixes(IspellDict *Conf, const char *filename)
1215 {
1216 char type[BUFSIZ],
1217 *ptype = NULL;
1218 char sflag[BUFSIZ];
1219 char mask[BUFSIZ],
1220 *pmask;
1221 char find[BUFSIZ],
1222 *pfind;
1223 char repl[BUFSIZ],
1224 *prepl;
1225 bool isSuffix = false;
1226 int naffix = 0,
1227 curaffix = 0;
1228 int sflaglen = 0;
1229 char flagflags = 0;
1230 tsearch_readline_state trst;
1231 char *recoded;
1232
1233 /* read file to find any flag */
1234 Conf->usecompound = false;
1235 Conf->useFlagAliases = false;
1236 Conf->flagMode = FM_CHAR;
1237
1238 if (!tsearch_readline_begin(&trst, filename))
1239 ereport(ERROR,
1240 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1241 errmsg("could not open affix file \"%s\": %m",
1242 filename)));
1243
1244 while ((recoded = tsearch_readline(&trst)) != NULL)
1245 {
1246 if (*recoded == '\0' || isspace((unsigned char) *recoded) || t_iseq(recoded, '#'))
1247 {
1248 pfree(recoded);
1249 continue;
1250 }
1251
1252 if (STRNCMP(recoded, "COMPOUNDFLAG") == 0)
1253 addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDFLAG"),
1254 FF_COMPOUNDFLAG);
1255 else if (STRNCMP(recoded, "COMPOUNDBEGIN") == 0)
1256 addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDBEGIN"),
1257 FF_COMPOUNDBEGIN);
1258 else if (STRNCMP(recoded, "COMPOUNDLAST") == 0)
1259 addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDLAST"),
1260 FF_COMPOUNDLAST);
1261 /* COMPOUNDLAST and COMPOUNDEND are synonyms */
1262 else if (STRNCMP(recoded, "COMPOUNDEND") == 0)
1263 addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDEND"),
1264 FF_COMPOUNDLAST);
1265 else if (STRNCMP(recoded, "COMPOUNDMIDDLE") == 0)
1266 addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDMIDDLE"),
1267 FF_COMPOUNDMIDDLE);
1268 else if (STRNCMP(recoded, "ONLYINCOMPOUND") == 0)
1269 addCompoundAffixFlagValue(Conf, recoded + strlen("ONLYINCOMPOUND"),
1270 FF_COMPOUNDONLY);
1271 else if (STRNCMP(recoded, "COMPOUNDPERMITFLAG") == 0)
1272 addCompoundAffixFlagValue(Conf,
1273 recoded + strlen("COMPOUNDPERMITFLAG"),
1274 FF_COMPOUNDPERMITFLAG);
1275 else if (STRNCMP(recoded, "COMPOUNDFORBIDFLAG") == 0)
1276 addCompoundAffixFlagValue(Conf,
1277 recoded + strlen("COMPOUNDFORBIDFLAG"),
1278 FF_COMPOUNDFORBIDFLAG);
1279 else if (STRNCMP(recoded, "FLAG") == 0)
1280 {
1281 char *s = recoded + strlen("FLAG");
1282
1283 while (*s && isspace((unsigned char) *s))
1284 s += pg_mblen_cstr(s);
1285
1286 if (*s)
1287 {
1288 if (STRNCMP(s, "long") == 0)
1289 Conf->flagMode = FM_LONG;
1290 else if (STRNCMP(s, "num") == 0)
1291 Conf->flagMode = FM_NUM;
1292 else if (STRNCMP(s, "default") != 0)
1293 ereport(ERROR,
1294 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1295 errmsg("Ispell dictionary supports only "
1296 "\"default\", \"long\", "
1297 "and \"num\" flag values")));
1298 }
1299 }
1300
1301 pfree(recoded);
1302 }
1303 tsearch_readline_end(&trst);
1304
1305 if (Conf->nCompoundAffixFlag > 1)
1306 qsort(Conf->CompoundAffixFlags, Conf->nCompoundAffixFlag,
1307 sizeof(CompoundAffixFlag), cmpcmdflag);
1308
1309 if (!tsearch_readline_begin(&trst, filename))
1310 ereport(ERROR,
1311 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1312 errmsg("could not open affix file \"%s\": %m",
1313 filename)));
1314
1315 while ((recoded = tsearch_readline(&trst)) != NULL)
1316 {
1317 int fields_read;
1318
1319 if (*recoded == '\0' || isspace((unsigned char) *recoded) || t_iseq(recoded, '#'))
1320 goto nextline;
1321
1322 fields_read = parse_ooaffentry(recoded, type, sflag, find, repl, mask);
1323
1324 if (ptype)
1325 pfree(ptype);
1326 ptype = lowerstr_ctx(Conf, type);
1327
1328 /* First try to parse AF parameter (alias compression) */
1329 if (STRNCMP(ptype, "af") == 0)
1330 {
1331 /* First line is the number of aliases */
1332 if (!Conf->useFlagAliases)
1333 {
1334 Conf->useFlagAliases = true;
1335 naffix = atoi(sflag);
1336 if (naffix <= 0)
1337 ereport(ERROR,
1338 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1339 errmsg("invalid number of flag vector aliases")));
1340
1341 /* Also reserve place for empty flag set */
1342 naffix++;
1343
1344 Conf->AffixData = (const char **) palloc0(naffix * sizeof(char *));
1345 Conf->lenAffixData = Conf->nAffixData = naffix;
1346
1347 /* Add empty flag set into AffixData */
1348 Conf->AffixData[curaffix] = VoidString;
1349 curaffix++;
1350 }
1351 /* Other lines are aliases */
1352 else
1353 {
1354 if (curaffix < naffix)
1355 {
1356 Conf->AffixData[curaffix] = cpstrdup(Conf, sflag);
1357 curaffix++;
1358 }
1359 else
1360 ereport(ERROR,
1361 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1362 errmsg("number of aliases exceeds specified number %d",
1363 naffix - 1)));
1364 }
1365 goto nextline;
1366 }
1367 /* Else try to parse prefixes and suffixes */
1368 if (fields_read < 4 ||
1369 (STRNCMP(ptype, "sfx") != 0 && STRNCMP(ptype, "pfx") != 0))
1370 goto nextline;
1371
1372 sflaglen = strlen(sflag);
1373 if (sflaglen == 0
1374 || (sflaglen > 1 && Conf->flagMode == FM_CHAR)
1375 || (sflaglen > 2 && Conf->flagMode == FM_LONG))
1376 goto nextline;
1377
1378 /*--------
1379 * Affix header. For example:
1380 * SFX \ N 1
1381 *--------
1382 */
1383 if (fields_read == 4)
1384 {
1385 isSuffix = (STRNCMP(ptype, "sfx") == 0);
1386 if (t_iseq(find, 'y') || t_iseq(find, 'Y'))
1387 flagflags = FF_CROSSPRODUCT;
1388 else
1389 flagflags = 0;
1390 }
1391 /*--------
1392 * Affix fields. For example:
1393 * SFX \ 0 Y/L [^Y]
1394 *--------
1395 */
1396 else
1397 {
1398 char *ptr;
1399 int aflg = 0;
1400
1401 /* Get flags after '/' (flags are case sensitive) */
1402 if ((ptr = strchr(repl, '/')) != NULL)
1403 aflg |= getCompoundAffixFlagValue(Conf,
1404 getAffixFlagSet(Conf,
1405 ptr + 1));
1406 /* Get lowercased version of string before '/' */
1407 prepl = lowerstr_ctx(Conf, repl);
1408 if ((ptr = strchr(prepl, '/')) != NULL)
1409 *ptr = '\0';
1410 pfind = lowerstr_ctx(Conf, find);
1411 pmask = lowerstr_ctx(Conf, mask);
1412 if (t_iseq(find, '0'))
1413 *pfind = '\0';
1414 if (t_iseq(repl, '0'))
1415 *prepl = '\0';
1416
1417 NIAddAffix(Conf, sflag, flagflags | aflg, pmask, pfind, prepl,
1418 isSuffix ? FF_SUFFIX : FF_PREFIX);
1419 pfree(prepl);
1420 pfree(pfind);
1421 pfree(pmask);
1422 }
1423
1424 nextline:
1425 pfree(recoded);
1426 }
1427
1428 tsearch_readline_end(&trst);
1429 if (ptype)
1430 pfree(ptype);
1431
1432 /* Reject incomplete AF alias table. */
1433 if (Conf->useFlagAliases && curaffix != naffix)
1434 ereport(ERROR,
1435 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1436 errmsg("number of aliases is less than specified number %d",
1437 naffix - 1)));
1438 }
1439
1440 /*
1441 * import affixes
1442 *
1443 * Note caller must already have applied get_tsearch_config_filename
1444 *
1445 * This function is responsible for parsing ispell ("old format") affix files.
1446 * If we realize that the file contains new-format commands, we pass off the
1447 * work to NIImportOOAffixes(), which will re-read the whole file.
1448 */
1449 void
1450 NIImportAffixes(IspellDict *Conf, const char *filename)
1451 {
1452 char *pstr = NULL;
1453 char flag[BUFSIZ];
1454 char mask[BUFSIZ];
1455 char find[BUFSIZ];
1456 char repl[BUFSIZ];
1457 char *s;
1458 bool suffixes = false;
1459 bool prefixes = false;
1460 char flagflags = 0;
1461 tsearch_readline_state trst;
1462 bool oldformat = false;
1463 char *recoded = NULL;
1464
1465 flag[0] = '\0'; /* no flag seen yet */
1466
1467 if (!tsearch_readline_begin(&trst, filename))
1468 ereport(ERROR,
1469 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1470 errmsg("could not open affix file \"%s\": %m",
1471 filename)));
1472
1473 Conf->usecompound = false;
1474 Conf->useFlagAliases = false;
1475 Conf->flagMode = FM_CHAR;
1476
1477 while ((recoded = tsearch_readline(&trst)) != NULL)
1478 {
1479 pstr = str_tolower(recoded, strlen(recoded), DEFAULT_COLLATION_OID);
1480
1481 /* Skip comments and empty lines */
1482 if (*pstr == '#' || *pstr == '\n')
1483 goto nextline;
1484
1485 if (STRNCMP(pstr, "compoundwords") == 0)
1486 {
1487 /* Find case-insensitive L flag in non-lowercased string */
1488 s = findchar2(recoded, 'l', 'L');
1489 if (s)
1490 {
1491 while (*s && !isspace((unsigned char) *s))
1492 s += pg_mblen_cstr(s);
1493 while (*s && isspace((unsigned char) *s))
1494 s += pg_mblen_cstr(s);
1495
1496 if (*s && pg_mblen_cstr(s) == 1)
1497 {
1498 addCompoundAffixFlagValue(Conf, s, FF_COMPOUNDFLAG);
1499 Conf->usecompound = true;
1500 }
1501 oldformat = true;
1502 goto nextline;
1503 }
1504 }
1505 if (STRNCMP(pstr, "suffixes") == 0)
1506 {
1507 suffixes = true;
1508 prefixes = false;
1509 oldformat = true;
1510 goto nextline;
1511 }
1512 if (STRNCMP(pstr, "prefixes") == 0)
1513 {
1514 suffixes = false;
1515 prefixes = true;
1516 oldformat = true;
1517 goto nextline;
1518 }
1519 if (STRNCMP(pstr, "flag") == 0)
1520 {
1521 s = recoded + 4; /* we need non-lowercased string */
1522 flagflags = 0;
1523
1524 while (*s && isspace((unsigned char) *s))
1525 s += pg_mblen_cstr(s);
1526
1527 if (*s == '*')
1528 {
1529 flagflags |= FF_CROSSPRODUCT;
1530 s++;
1531 }
1532 else if (*s == '~')
1533 {
1534 flagflags |= FF_COMPOUNDONLY;
1535 s++;
1536 }
1537
1538 if (*s == '\\')
1539 s++;
1540
1541 /*
1542 * An old-format flag is a single ASCII character; we expect it to
1543 * be followed by EOL, whitespace, or ':'. Otherwise this is a
1544 * new-format flag command.
1545 */
1546 if (*s && pg_mblen_cstr(s) == 1)
1547 {
1548 flag[0] = *s++;
1549 flag[1] = '\0';
1550
1551 if (*s == '\0' || *s == '#' || *s == '\n' || *s == ':' ||
1552 isspace((unsigned char) *s))
1553 {
1554 oldformat = true;
1555 goto nextline;
1556 }
1557 }
1558 goto isnewformat;
1559 }
1560 if (STRNCMP(recoded, "COMPOUNDFLAG") == 0 ||
1561 STRNCMP(recoded, "COMPOUNDMIN") == 0 ||
1562 STRNCMP(recoded, "PFX") == 0 ||
1563 STRNCMP(recoded, "SFX") == 0)
1564 goto isnewformat;
1565
1566 if ((!suffixes) && (!prefixes))
1567 goto nextline;
1568
1569 if (!parse_affentry(pstr, mask, find, repl))
1570 goto nextline;
1571
1572 NIAddAffix(Conf, flag, flagflags, mask, find, repl, suffixes ? FF_SUFFIX : FF_PREFIX);
1573
1574 nextline:
1575 pfree(recoded);
1576 pfree(pstr);
1577 }
1578 tsearch_readline_end(&trst);
1579 return;
1580
1581 isnewformat:
1582 if (oldformat)
1583 ereport(ERROR,
1584 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1585 errmsg("affix file contains both old-style and new-style commands")));
1586 tsearch_readline_end(&trst);
1587
1588 NIImportOOAffixes(Conf, filename);
1589 }
1590
1591 /*
1592 * Merges two affix flag sets and stores a new affix flag set into
1593 * Conf->AffixData.
1594 *
1595 * Returns index of a new affix flag set.
1596 */
1597 static int
1598 MergeAffix(IspellDict *Conf, int a1, int a2)
1599 {
1600 const char **ptr;
1601
1602 Assert(a1 < Conf->nAffixData && a2 < Conf->nAffixData);
1603
1604 /* Do not merge affix flags if one of affix flags is empty */
1605 if (*Conf->AffixData[a1] == '\0')
1606 return a2;
1607 else if (*Conf->AffixData[a2] == '\0')
1608 return a1;
1609
1610 /* Double the size of AffixData if there's not enough space */
1611 if (Conf->nAffixData + 1 >= Conf->lenAffixData)
1612 {
1613 Conf->lenAffixData *= 2;
1614 Conf->AffixData = (const char **) repalloc(Conf->AffixData,
1615 sizeof(char *) * Conf->lenAffixData);
1616 }
1617
1618 ptr = Conf->AffixData + Conf->nAffixData;
1619 if (Conf->flagMode == FM_NUM)
1620 {
1621 char *p = cpalloc(strlen(Conf->AffixData[a1]) +
1622 strlen(Conf->AffixData[a2]) +
1623 1 /* comma */ + 1 /* \0 */ );
1624
1625 sprintf(p, "%s,%s", Conf->AffixData[a1], Conf->AffixData[a2]);
1626 *ptr = p;
1627 }
1628 else
1629 {
1630 char *p = cpalloc(strlen(Conf->AffixData[a1]) +
1631 strlen(Conf->AffixData[a2]) +
1632 1 /* \0 */ );
1633
1634 sprintf(p, "%s%s", Conf->AffixData[a1], Conf->AffixData[a2]);
1635 *ptr = p;
1636 }
1637 ptr++;
1638 *ptr = NULL;
1639 Conf->nAffixData++;
1640
1641 return Conf->nAffixData - 1;
1642 }
1643
1644 /*
1645 * Returns a set of affix parameters which correspondence to the set of affix
1646 * flags with the given index.
1647 */
1648 static uint32
1649 makeCompoundFlags(IspellDict *Conf, int affix)
1650 {
1651 Assert(affix < Conf->nAffixData);
1652
1653 return (getCompoundAffixFlagValue(Conf, Conf->AffixData[affix]) &
1654 FF_COMPOUNDFLAGMASK);
1655 }
1656
1657 /*
1658 * Makes a prefix tree for the given level.
1659 *
1660 * Conf: current dictionary.
1661 * low: lower index of the Conf->Spell array.
1662 * high: upper index of the Conf->Spell array.
1663 * level: current prefix tree level.
1664 */
1665 static SPNode *
1666 mkSPNode(IspellDict *Conf, int low, int high, int level)
1667 {
1668 int i;
1669 int nchar = 0;
1670 char lastchar = '\0';
1671 SPNode *rs;
1672 SPNodeData *data;
1673 int lownew = low;
1674
1675 for (i = low; i < high; i++)
1676 if (Conf->Spell[i]->p.d.len > level && lastchar != Conf->Spell[i]->word[level])
1677 {
1678 nchar++;
1679 lastchar = Conf->Spell[i]->word[level];
1680 }
1681
1682 if (!nchar)
1683 return NULL;
1684
1685 rs = (SPNode *) cpalloc0(SPNHDRSZ + nchar * sizeof(SPNodeData));
1686 rs->length = nchar;
1687 data = rs->data;
1688
1689 lastchar = '\0';
1690 for (i = low; i < high; i++)
1691 if (Conf->Spell[i]->p.d.len > level)
1692 {
1693 if (lastchar != Conf->Spell[i]->word[level])
1694 {
1695 if (lastchar)
1696 {
1697 /* Next level of the prefix tree */
1698 data->node = mkSPNode(Conf, lownew, i, level + 1);
1699 lownew = i;
1700 data++;
1701 }
1702 lastchar = Conf->Spell[i]->word[level];
1703 }
1704 data->val = ((uint8 *) (Conf->Spell[i]->word))[level];
1705 if (Conf->Spell[i]->p.d.len == level + 1)
1706 {
1707 bool clearCompoundOnly = false;
1708
1709 if (data->isword && data->affix != Conf->Spell[i]->p.d.affix)
1710 {
1711 /*
1712 * MergeAffix called a few times. If one of word is
1713 * allowed to be in compound word and another isn't, then
1714 * clear FF_COMPOUNDONLY flag.
1715 */
1716
1717 clearCompoundOnly = (FF_COMPOUNDONLY & data->compoundflag
1718 & makeCompoundFlags(Conf, Conf->Spell[i]->p.d.affix))
1719 ? false : true;
1720 data->affix = MergeAffix(Conf, data->affix, Conf->Spell[i]->p.d.affix);
1721 }
1722 else
1723 data->affix = Conf->Spell[i]->p.d.affix;
1724 data->isword = 1;
1725
1726 data->compoundflag = makeCompoundFlags(Conf, data->affix);
1727
1728 if ((data->compoundflag & FF_COMPOUNDONLY) &&
1729 (data->compoundflag & FF_COMPOUNDFLAG) == 0)
1730 data->compoundflag |= FF_COMPOUNDFLAG;
1731
1732 if (clearCompoundOnly)
1733 data->compoundflag &= ~FF_COMPOUNDONLY;
1734 }
1735 }
1736
1737 /* Next level of the prefix tree */
1738 data->node = mkSPNode(Conf, lownew, high, level + 1);
1739
1740 return rs;
1741 }
1742
1743 /*
1744 * Builds the Conf->Dictionary tree and AffixData from the imported dictionary
1745 * and affixes.
1746 */
1747 void
1748 NISortDictionary(IspellDict *Conf)
1749 {
1750 int i;
1751 int naffix;
1752 int curaffix;
1753
1754 /* compress affixes */
1755
1756 /*
1757 * If we use flag aliases then we need to use Conf->AffixData filled in
1758 * the NIImportOOAffixes().
1759 */
1760 if (Conf->useFlagAliases)
1761 {
1762 for (i = 0; i < Conf->nspell; i++)
1763 {
1764 char *end;
1765
1766 if (*Conf->Spell[i]->p.flag != '\0')
1767 {
1768 errno = 0;
1769 curaffix = strtol(Conf->Spell[i]->p.flag, &end, 10);
1770 if (Conf->Spell[i]->p.flag == end || errno == ERANGE)
1771 ereport(ERROR,
1772 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1773 errmsg("invalid affix alias \"%s\"",
1774 Conf->Spell[i]->p.flag)));
1775 if (curaffix < 0 || curaffix >= Conf->nAffixData)
1776 ereport(ERROR,
1777 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1778 errmsg("invalid affix alias \"%s\"",
1779 Conf->Spell[i]->p.flag)));
1780 if (*end != '\0' && !isdigit((unsigned char) *end) && !isspace((unsigned char) *end))
1781 ereport(ERROR,
1782 (errcode(ERRCODE_CONFIG_FILE_ERROR),
1783 errmsg("invalid affix alias \"%s\"",
1784 Conf->Spell[i]->p.flag)));
1785 }
1786 else
1787 {
1788 /*
1789 * If Conf->Spell[i]->p.flag is empty, then get empty value of
1790 * Conf->AffixData (0 index).
1791 */
1792 curaffix = 0;
1793 }
1794
1795 Conf->Spell[i]->p.d.affix = curaffix;
1796 Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word);
1797 }
1798 }
1799 /* Otherwise fill Conf->AffixData here */
1800 else
1801 {
1802 /* Count the number of different flags used in the dictionary */
1803 qsort(Conf->Spell, Conf->nspell, sizeof(SPELL *),
1804 cmpspellaffix);
1805
1806 naffix = 0;
1807 for (i = 0; i < Conf->nspell; i++)
1808 {
1809 if (i == 0 ||
1810 strcmp(Conf->Spell[i]->p.flag, Conf->Spell[i - 1]->p.flag) != 0)
1811 naffix++;
1812 }
1813
1814 /*
1815 * Fill in Conf->AffixData with the affixes that were used in the
1816 * dictionary. Replace textual flag-field of Conf->Spell entries with
1817 * indexes into Conf->AffixData array.
1818 */
1819 Conf->AffixData = (const char **) palloc0(naffix * sizeof(const char *));
1820
1821 curaffix = -1;
1822 for (i = 0; i < Conf->nspell; i++)
1823 {
1824 if (i == 0 ||
1825 strcmp(Conf->Spell[i]->p.flag, Conf->AffixData[curaffix]) != 0)
1826 {
1827 curaffix++;
1828 Assert(curaffix < naffix);
1829 Conf->AffixData[curaffix] = cpstrdup(Conf,
1830 Conf->Spell[i]->p.flag);
1831 }
1832
1833 Conf->Spell[i]->p.d.affix = curaffix;
1834 Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word);
1835 }
1836
1837 Conf->lenAffixData = Conf->nAffixData = naffix;
1838 }
1839
1840 /* Start build a prefix tree */
1841 qsort(Conf->Spell, Conf->nspell, sizeof(SPELL *), cmpspell);
1842 Conf->Dictionary = mkSPNode(Conf, 0, Conf->nspell, 0);
1843 }
1844
1845 /*
1846 * Makes a prefix tree for the given level using the repl string of an affix
1847 * rule. Affixes with empty replace string do not include in the prefix tree.
1848 * This affixes are included by mkVoidAffix().
1849 *
1850 * Conf: current dictionary.
1851 * low: lower index of the Conf->Affix array.
1852 * high: upper index of the Conf->Affix array.
1853 * level: current prefix tree level.
1854 * type: FF_SUFFIX or FF_PREFIX.
1855 */
1856 static AffixNode *
1857 mkANode(IspellDict *Conf, int low, int high, int level, int type)
1858 {
1859 int i;
1860 int nchar = 0;
1861 uint8 lastchar = '\0';
1862 AffixNode *rs;
1863 AffixNodeData *data;
1864 int lownew = low;
1865 int naff;
1866 AFFIX **aff;
1867
1868 for (i = low; i < high; i++)
1869 if (Conf->Affix[i].replen > level && lastchar != GETCHAR(Conf->Affix + i, level, type))
1870 {
1871 nchar++;
1872 lastchar = GETCHAR(Conf->Affix + i, level, type);
1873 }
1874
1875 if (!nchar)
1876 return NULL;
1877
1878 aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1));
1879 naff = 0;
1880
1881 rs = (AffixNode *) cpalloc0(ANHRDSZ + nchar * sizeof(AffixNodeData));
1882 rs->length = nchar;
1883 data = rs->data;
1884
1885 lastchar = '\0';
1886 for (i = low; i < high; i++)
1887 if (Conf->Affix[i].replen > level)
1888 {
1889 if (lastchar != GETCHAR(Conf->Affix + i, level, type))
1890 {
1891 if (lastchar)
1892 {
1893 /* Next level of the prefix tree */
1894 data->node = mkANode(Conf, lownew, i, level + 1, type);
1895 if (naff)
1896 {
1897 data->naff = naff;
1898 data->aff = (AFFIX **) cpalloc(sizeof(AFFIX *) * naff);
1899 memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
1900 naff = 0;
1901 }
1902 data++;
1903 lownew = i;
1904 }
1905 lastchar = GETCHAR(Conf->Affix + i, level, type);
1906 }
1907 data->val = GETCHAR(Conf->Affix + i, level, type);
1908 if (Conf->Affix[i].replen == level + 1)
1909 { /* affix stopped */
1910 aff[naff++] = Conf->Affix + i;
1911 }
1912 }
1913
1914 /* Next level of the prefix tree */
1915 data->node = mkANode(Conf, lownew, high, level + 1, type);
1916 if (naff)
1917 {
1918 data->naff = naff;
1919 data->aff = (AFFIX **) cpalloc(sizeof(AFFIX *) * naff);
1920 memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
1921 naff = 0;
1922 }
1923
1924 pfree(aff);
1925
1926 return rs;
1927 }
1928
1929 /*
1930 * Makes the root void node in the prefix tree. The root void node is created
1931 * for affixes which have empty replace string ("repl" field).
1932 */
1933 static void
1934 mkVoidAffix(IspellDict *Conf, bool issuffix, int startsuffix)
1935 {
1936 int i,
1937 cnt = 0;
1938 int start = (issuffix) ? startsuffix : 0;
1939 int end = (issuffix) ? Conf->naffixes : startsuffix;
1940 AffixNode *Affix = (AffixNode *) palloc0(ANHRDSZ + sizeof(AffixNodeData));
1941
1942 Affix->length = 1;
1943 Affix->isvoid = 1;
1944
1945 if (issuffix)
1946 {
1947 Affix->data->node = Conf->Suffix;
1948 Conf->Suffix = Affix;
1949 }
1950 else
1951 {
1952 Affix->data->node = Conf->Prefix;
1953 Conf->Prefix = Affix;
1954 }
1955
1956 /* Count affixes with empty replace string */
1957 for (i = start; i < end; i++)
1958 if (Conf->Affix[i].replen == 0)
1959 cnt++;
1960
1961 /* There is not affixes with empty replace string */
1962 if (cnt == 0)
1963 return;
1964
1965 Affix->data->aff = (AFFIX **) cpalloc(sizeof(AFFIX *) * cnt);
1966 Affix->data->naff = (uint32) cnt;
1967
1968 cnt = 0;
1969 for (i = start; i < end; i++)
1970 if (Conf->Affix[i].replen == 0)
1971 {
1972 Affix->data->aff[cnt] = Conf->Affix + i;
1973 cnt++;
1974 }
1975 }
1976
1977 /*
1978 * Checks if the affixflag is used by dictionary. Conf->AffixData does not
1979 * contain affixflag if this flag is not used actually by the .dict file.
1980 *
1981 * Conf: current dictionary.
1982 * affixflag: affix flag.
1983 *
1984 * Returns true if the Conf->AffixData array contains affixflag, otherwise
1985 * returns false.
1986 */
1987 static bool
1988 isAffixInUse(IspellDict *Conf, const char *affixflag)
1989 {
1990 int i;
1991
1992 for (i = 0; i < Conf->nAffixData; i++)
1993 if (IsAffixFlagInUse(Conf, i, affixflag))
1994 return true;
1995
1996 return false;
1997 }
1998
1999 /*
2000 * Builds Conf->Prefix and Conf->Suffix trees from the imported affixes.
2001 */
2002 void
2003 NISortAffixes(IspellDict *Conf)
2004 {
2005 AFFIX *Affix;
2006 size_t i;
2007 CMPDAffix *ptr;
2008 int firstsuffix = Conf->naffixes;
2009
2010 if (Conf->naffixes == 0)
2011 return;
2012
2013 /* Store compound affixes in the Conf->CompoundAffix array */
2014 if (Conf->naffixes > 1)
2015 qsort(Conf->Affix, Conf->naffixes, sizeof(AFFIX), cmpaffix);
2016 /* +1 for terminator */
2017 Conf->CompoundAffix = ptr = palloc_array(CMPDAffix, Conf->naffixes + 1);
2018 ptr->affix = NULL;
2019
2020 for (i = 0; i < Conf->naffixes; i++)
2021 {
2022 Affix = &(((AFFIX *) Conf->Affix)[i]);
2023 if (Affix->type == FF_SUFFIX && i < firstsuffix)
2024 firstsuffix = i;
2025
2026 if ((Affix->flagflags & FF_COMPOUNDFLAG) && Affix->replen > 0 &&
2027 isAffixInUse(Conf, Affix->flag))
2028 {
2029 bool issuffix = (Affix->type == FF_SUFFIX);
2030
2031 if (ptr == Conf->CompoundAffix ||
2032 issuffix != (ptr - 1)->issuffix ||
2033 strbncmp((const unsigned char *) (ptr - 1)->affix,
2034 (const unsigned char *) Affix->repl,
2035 (ptr - 1)->len))
2036 {
2037 /* leave only unique and minimal suffixes */
2038 ptr->affix = Affix->repl;
2039 ptr->len = Affix->replen;
2040 ptr->issuffix = issuffix;
2041 ptr++;
2042 }
2043 }
2044 }
2045 ptr->affix = NULL;
2046 Conf->CompoundAffix = (CMPDAffix *) repalloc(Conf->CompoundAffix, sizeof(CMPDAffix) * (ptr - Conf->CompoundAffix + 1));
2047
2048 /* Start build a prefix tree */
2049 Conf->Prefix = mkANode(Conf, 0, firstsuffix, 0, FF_PREFIX);
2050 Conf->Suffix = mkANode(Conf, firstsuffix, Conf->naffixes, 0, FF_SUFFIX);
2051 mkVoidAffix(Conf, true, firstsuffix);
2052 mkVoidAffix(Conf, false, firstsuffix);
2053 }
2054
2055 static AffixNodeData *
2056 FindAffixes(AffixNode *node, const char *word, int wrdlen, int *level, int type)
2057 {
2058 AffixNodeData *StopLow,
2059 *StopHigh,
2060 *StopMiddle;
2061 uint8 symbol;
2062
2063 if (node->isvoid)
2064 { /* search void affixes */
2065 if (node->data->naff)
2066 return node->data;
2067 node = node->data->node;
2068 }
2069
2070 while (node && *level < wrdlen)
2071 {
2072 StopLow = node->data;
2073 StopHigh = node->data + node->length;
2074 while (StopLow < StopHigh)
2075 {
2076 StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
2077 symbol = GETWCHAR(word, wrdlen, *level, type);
2078
2079 if (StopMiddle->val == symbol)
2080 {
2081 (*level)++;
2082 if (StopMiddle->naff)
2083 return StopMiddle;
2084 node = StopMiddle->node;
2085 break;
2086 }
2087 else if (StopMiddle->val < symbol)
2088 StopLow = StopMiddle + 1;
2089 else
2090 StopHigh = StopMiddle;
2091 }
2092 if (StopLow >= StopHigh)
2093 break;
2094 }
2095 return NULL;
2096 }
2097
2098 /*
2099 * Checks to see if affix applies to word, transforms word if so.
2100 * The transformation consists of replacing Affix->replen leading or
2101 * trailing bytes with the Affix->find string.
2102 *
2103 * word: input word
2104 * len: length of input word
2105 * Affix: affix to consider
2106 * flagflags: context flags showing whether we are handling a compound word
2107 * newword: output buffer (MUST be of length 2 * MAXNORMLEN)
2108 * baselen: input/output argument
2109 *
2110 * If baselen isn't NULL, then *baselen is used to return the length of
2111 * the non-changed part of the word when applying a suffix, and is used
2112 * to detect whether the input contained only a prefix and suffix when
2113 * later applying a prefix.
2114 *
2115 * Returns newword on success, or NULL if the affix can't be applied.
2116 * On success, the modified word is stored into newword.
2117 */
2118 static char *
2119 CheckAffix(const char *word, size_t len, AFFIX *Affix, int flagflags, char *newword, int *baselen)
2120 {
2121 size_t keeplen,
2122 findlen;
2123
2124 /*
2125 * Check compound allow flags
2126 */
2127
2128 if (flagflags == 0)
2129 {
2130 if (Affix->flagflags & FF_COMPOUNDONLY)
2131 return NULL;
2132 }
2133 else if (flagflags & FF_COMPOUNDBEGIN)
2134 {
2135 if (Affix->flagflags & FF_COMPOUNDFORBIDFLAG)
2136 return NULL;
2137 if ((Affix->flagflags & FF_COMPOUNDBEGIN) == 0)
2138 if (Affix->type == FF_SUFFIX)
2139 return NULL;
2140 }
2141 else if (flagflags & FF_COMPOUNDMIDDLE)
2142 {
2143 if ((Affix->flagflags & FF_COMPOUNDMIDDLE) == 0 ||
2144 (Affix->flagflags & FF_COMPOUNDFORBIDFLAG))
2145 return NULL;
2146 }
2147 else if (flagflags & FF_COMPOUNDLAST)
2148 {
2149 if (Affix->flagflags & FF_COMPOUNDFORBIDFLAG)
2150 return NULL;
2151 if ((Affix->flagflags & FF_COMPOUNDLAST) == 0)
2152 if (Affix->type == FF_PREFIX)
2153 return NULL;
2154 }
2155
2156 /*
2157 * Protect against output buffer overrun (len < Affix->replen would be
2158 * caller error, but check anyway)
2159 */
2160 Assert(len == strlen(word));
2161 if (len < Affix->replen)
2162 return NULL;
2163 keeplen = len - Affix->replen; /* how much of word we will keep */
2164 findlen = strlen(Affix->find);
2165 if (keeplen + findlen >= 2 * MAXNORMLEN)
2166 return NULL;
2167
2168 /*
2169 * make replace pattern of affix
2170 */
2171 if (Affix->type == FF_SUFFIX)
2172 {
2173 memcpy(newword, word, keeplen);
2174 strcpy(newword + keeplen, Affix->find);
2175 if (baselen) /* store length of non-changed part of word */
2176 *baselen = keeplen;
2177 }
2178 else
2179 {
2180 /*
2181 * if prefix is an all non-changed part's length then all word
2182 * contains only prefix and suffix, so out
2183 */
2184 if (baselen && *baselen + findlen <= Affix->replen)
2185 return NULL;
2186 memcpy(newword, Affix->find, findlen);
2187 strcpy(newword + findlen, word + Affix->replen);
2188 }
2189
2190 /*
2191 * check resulting word
2192 */
2193 if (Affix->issimple)
2194 return newword;
2195 else if (Affix->isregis)
2196 {
2197 if (RS_execute(&(Affix->reg.regis), newword))
2198 return newword;
2199 }
2200 else
2201 {
2202 pg_wchar *data;
2203 size_t data_len;
2204 int newword_len;
2205
2206 /* Convert data string to wide characters */
2207 newword_len = strlen(newword);
2208 data = (pg_wchar *) palloc((newword_len + 1) * sizeof(pg_wchar));
2209 data_len = pg_mb2wchar_with_len(newword, data, newword_len);
2210
2211 if (pg_regexec(Affix->reg.pregex, data, data_len,
2212 0, NULL, 0, NULL, 0) == REG_OKAY)
2213 {
2214 pfree(data);
2215 return newword;
2216 }
2217 pfree(data);
2218 }
2219
2220 return NULL;
2221 }
2222
2223 static int
2224 addToResult(char **forms, char **cur, char *word)
2225 {
2226 if (cur - forms >= MAX_NORM - 1)
2227 return 0;
2228 if (forms == cur || strcmp(word, *(cur - 1)) != 0)
2229 {
2230 *cur = pstrdup(word);
2231 *(cur + 1) = NULL;
2232 return 1;
2233 }
2234
2235 return 0;
2236 }
2237
2238 static char **
2239 NormalizeSubWord(IspellDict *Conf, const char *word, int flag)
2240 {
2241 AffixNodeData *suffix = NULL,
2242 *prefix = NULL;
2243 int slevel = 0,
2244 plevel = 0;
2245 int wrdlen = strlen(word),
2246 swrdlen;
2247 char **forms;
2248 char **cur;
2249 char newword[2 * MAXNORMLEN] = "";
2250 char pnewword[2 * MAXNORMLEN] = "";
2251 AffixNode *snode = Conf->Suffix,
2252 *pnode;
2253 int i,
2254 j;
2255
2256 if (wrdlen > MAXNORMLEN)
2257 return NULL;
2258 cur = forms = (char **) palloc(MAX_NORM * sizeof(char *));
2259 *cur = NULL;
2260
2261
2262 /* Check that the word itself is normal form */
2263 if (FindWord(Conf, word, VoidString, flag))
2264 {
2265 *cur = pstrdup(word);
2266 cur++;
2267 *cur = NULL;
2268 }
2269
2270 /* Find all other NORMAL forms of the 'word' (check only prefix) */
2271 pnode = Conf->Prefix;
2272 plevel = 0;
2273 while (pnode)
2274 {
2275 prefix = FindAffixes(pnode, word, wrdlen, &plevel, FF_PREFIX);
2276 if (!prefix)
2277 break;
2278 for (j = 0; j < prefix->naff; j++)
2279 {
2280 if (CheckAffix(word, wrdlen, prefix->aff[j], flag, newword, NULL))
2281 {
2282 /* prefix success */
2283 if (FindWord(Conf, newword, prefix->aff[j]->flag, flag))
2284 cur += addToResult(forms, cur, newword);
2285 }
2286 }
2287 pnode = prefix->node;
2288 }
2289
2290 /*
2291 * Find all other NORMAL forms of the 'word' (check suffix and then
2292 * prefix)
2293 */
2294 while (snode)
2295 {
2296 int baselen = 0;
2297
2298 /* find possible suffix */
2299 suffix = FindAffixes(snode, word, wrdlen, &slevel, FF_SUFFIX);
2300 if (!suffix)
2301 break;
2302 /* foreach suffix check affix */
2303 for (i = 0; i < suffix->naff; i++)
2304 {
2305 if (CheckAffix(word, wrdlen, suffix->aff[i], flag, newword, &baselen))
2306 {
2307 /* suffix success */
2308 if (FindWord(Conf, newword, suffix->aff[i]->flag, flag))
2309 cur += addToResult(forms, cur, newword);
2310
2311 /* now we will look changed word with prefixes */
2312 pnode = Conf->Prefix;
2313 plevel = 0;
2314 swrdlen = strlen(newword);
2315 while (pnode)
2316 {
2317 prefix = FindAffixes(pnode, newword, swrdlen, &plevel, FF_PREFIX);
2318 if (!prefix)
2319 break;
2320 for (j = 0; j < prefix->naff; j++)
2321 {
2322 if (CheckAffix(newword, swrdlen, prefix->aff[j], flag, pnewword, &baselen))
2323 {
2324 /* prefix success */
2325 const char *ff = (prefix->aff[j]->flagflags & suffix->aff[i]->flagflags & FF_CROSSPRODUCT) ?
2326 VoidString : prefix->aff[j]->flag;
2327
2328 if (FindWord(Conf, pnewword, ff, flag))
2329 cur += addToResult(forms, cur, pnewword);
2330 }
2331 }
2332 pnode = prefix->node;
2333 }
2334 }
2335 }
2336
2337 snode = suffix->node;
2338 }
2339
2340 if (cur == forms)
2341 {
2342 pfree(forms);
2343 return NULL;
2344 }
2345 return forms;
2346 }
2347
2348 typedef struct SplitVar
2349 {
2350 int nstem;
2351 int lenstem;
2352 char **stem;
2353 struct SplitVar *next;
2354 } SplitVar;
2355
2356 static int
2357 CheckCompoundAffixes(CMPDAffix **ptr, const char *word, int len, bool CheckInPlace)
2358 {
2359 bool issuffix;
2360
2361 /* in case CompoundAffix is null: */
2362 if (*ptr == NULL)
2363 return -1;
2364
2365 if (CheckInPlace)
2366 {
2367 while ((*ptr)->affix)
2368 {
2369 if (len > (*ptr)->len && strncmp((*ptr)->affix, word, (*ptr)->len) == 0)
2370 {
2371 len = (*ptr)->len;
2372 issuffix = (*ptr)->issuffix;
2373 (*ptr)++;
2374 return (issuffix) ? len : 0;
2375 }
2376 (*ptr)++;
2377 }
2378 }
2379 else
2380 {
2381 const char *affbegin;
2382
2383 while ((*ptr)->affix)
2384 {
2385 if (len > (*ptr)->len && (affbegin = strstr(word, (*ptr)->affix)) != NULL)
2386 {
2387 len = (*ptr)->len + (affbegin - word);
2388 issuffix = (*ptr)->issuffix;
2389 (*ptr)++;
2390 return (issuffix) ? len : 0;
2391 }
2392 (*ptr)++;
2393 }
2394 }
2395 return -1;
2396 }
2397
2398 static SplitVar *
2399 CopyVar(SplitVar *s, int makedup)
2400 {
2401 SplitVar *v = (SplitVar *) palloc(sizeof(SplitVar));
2402
2403 v->next = NULL;
2404 if (s)
2405 {
2406 int i;
2407
2408 v->lenstem = s->lenstem;
2409 v->stem = (char **) palloc(sizeof(char *) * v->lenstem);
2410 v->nstem = s->nstem;
2411 for (i = 0; i < s->nstem; i++)
2412 v->stem[i] = (makedup) ? pstrdup(s->stem[i]) : s->stem[i];
2413 }
2414 else
2415 {
2416 v->lenstem = 16;
2417 v->stem = (char **) palloc(sizeof(char *) * v->lenstem);
2418 v->nstem = 0;
2419 }
2420 return v;
2421 }
2422
2423 static void
2424 AddStem(SplitVar *v, char *word)
2425 {
2426 if (v->nstem >= v->lenstem)
2427 {
2428 v->lenstem *= 2;
2429 v->stem = (char **) repalloc(v->stem, sizeof(char *) * v->lenstem);
2430 }
2431
2432 v->stem[v->nstem] = word;
2433 v->nstem++;
2434 }
2435
2436 static SplitVar *
2437 SplitToVariants(IspellDict *Conf, SPNode *snode, SplitVar *orig, const char *word, int wordlen, int startpos, int minpos)
2438 {
2439 SplitVar *var = NULL;
2440 SPNodeData *StopLow,
2441 *StopHigh,
2442 *StopMiddle = NULL;
2443 SPNode *node = (snode) ? snode : Conf->Dictionary;
2444 int level = (snode) ? minpos : startpos; /* recursive
2445 * minpos==level */
2446 int lenaff;
2447 CMPDAffix *caff;
2448 char *notprobed;
2449 int compoundflag = 0;
2450
2451 /* since this function recurses, it could be driven to stack overflow */
2452 check_stack_depth();
2453
2454 notprobed = (char *) palloc(wordlen);
2455 memset(notprobed, 1, wordlen);
2456 var = CopyVar(orig, 1);
2457
2458 while (level < wordlen)
2459 {
2460 /* find word with epenthetic or/and compound affix */
2461 caff = Conf->CompoundAffix;
2462 while (level > startpos && (lenaff = CheckCompoundAffixes(&caff, word + level, wordlen - level, (node) ? true : false)) >= 0)
2463 {
2464 /*
2465 * there is one of compound affixes, so check word for existings
2466 */
2467 char buf[MAXNORMLEN];
2468 char **subres;
2469
2470 lenaff = level - startpos + lenaff;
2471
2472 if (!notprobed[startpos + lenaff - 1])
2473 continue;
2474
2475 if (level + lenaff - 1 <= minpos)
2476 continue;
2477
2478 if (lenaff >= MAXNORMLEN)
2479 continue; /* skip too big value */
2480 if (lenaff > 0)
2481 memcpy(buf, word + startpos, lenaff);
2482 buf[lenaff] = '\0';
2483
2484 if (level == 0)
2485 compoundflag = FF_COMPOUNDBEGIN;
2486 else if (level == wordlen - 1)
2487 compoundflag = FF_COMPOUNDLAST;
2488 else
2489 compoundflag = FF_COMPOUNDMIDDLE;
2490 subres = NormalizeSubWord(Conf, buf, compoundflag);
2491 if (subres)
2492 {
2493 /* Yes, it was a word from dictionary */
2494 SplitVar *new = CopyVar(var, 0);
2495 SplitVar *ptr = var;
2496 char **sptr = subres;
2497
2498 notprobed[startpos + lenaff - 1] = 0;
2499
2500 while (*sptr)
2501 {
2502 AddStem(new, *sptr);
2503 sptr++;
2504 }
2505 pfree(subres);
2506
2507 while (ptr->next)
2508 ptr = ptr->next;
2509 ptr->next = SplitToVariants(Conf, NULL, new, word, wordlen, startpos + lenaff, startpos + lenaff);
2510
2511 pfree(new->stem);
2512 pfree(new);
2513 }
2514 }
2515
2516 if (!node)
2517 break;
2518
2519 StopLow = node->data;
2520 StopHigh = node->data + node->length;
2521 while (StopLow < StopHigh)
2522 {
2523 StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
2524 if (StopMiddle->val == ((uint8 *) (word))[level])
2525 break;
2526 else if (StopMiddle->val < ((uint8 *) (word))[level])
2527 StopLow = StopMiddle + 1;
2528 else
2529 StopHigh = StopMiddle;
2530 }
2531
2532 if (StopLow < StopHigh)
2533 {
2534 if (startpos == 0)
2535 compoundflag = FF_COMPOUNDBEGIN;
2536 else if (level == wordlen - 1)
2537 compoundflag = FF_COMPOUNDLAST;
2538 else
2539 compoundflag = FF_COMPOUNDMIDDLE;
2540
2541 /* find infinitive */
2542 if (StopMiddle->isword &&
2543 (StopMiddle->compoundflag & compoundflag) &&
2544 notprobed[level])
2545 {
2546 /* ok, we found full compoundallowed word */
2547 if (level > minpos)
2548 {
2549 /* and its length more than minimal */
2550 if (wordlen == level + 1)
2551 {
2552 /* well, it was last word */
2553 AddStem(var, pnstrdup(word + startpos, wordlen - startpos));
2554 pfree(notprobed);
2555 return var;
2556 }
2557 else
2558 {
2559 /* then we will search more big word at the same point */
2560 SplitVar *ptr = var;
2561
2562 while (ptr->next)
2563 ptr = ptr->next;
2564 ptr->next = SplitToVariants(Conf, node, var, word, wordlen, startpos, level);
2565 /* we can find next word */
2566 level++;
2567 AddStem(var, pnstrdup(word + startpos, level - startpos));
2568 node = Conf->Dictionary;
2569 startpos = level;
2570 continue;
2571 }
2572 }
2573 }
2574 node = StopMiddle->node;
2575 }
2576 else
2577 node = NULL;
2578 level++;
2579 }
2580
2581 AddStem(var, pnstrdup(word + startpos, wordlen - startpos));
2582 pfree(notprobed);
2583 return var;
2584 }
2585
2586 static void
2587 addNorm(TSLexeme **lres, TSLexeme **lcur, char *word, int flags, uint16 NVariant)
2588 {
2589 if (*lres == NULL)
2590 *lcur = *lres = (TSLexeme *) palloc(MAX_NORM * sizeof(TSLexeme));
2591
2592 if (*lcur - *lres < MAX_NORM - 1)
2593 {
2594 (*lcur)->lexeme = word;
2595 (*lcur)->flags = flags;
2596 (*lcur)->nvariant = NVariant;
2597 (*lcur)++;
2598 (*lcur)->lexeme = NULL;
2599 }
2600 }
2601
2602 TSLexeme *
2603 NINormalizeWord(IspellDict *Conf, const char *word)
2604 {
2605 char **res;
2606 TSLexeme *lcur = NULL,
2607 *lres = NULL;
2608 uint16 NVariant = 1;
2609
2610 res = NormalizeSubWord(Conf, word, 0);
2611
2612 if (res)
2613 {
2614 char **ptr = res;
2615
2616 while (*ptr && (lcur - lres) < MAX_NORM)
2617 {
2618 addNorm(&lres, &lcur, *ptr, 0, NVariant++);
2619 ptr++;
2620 }
2621 pfree(res);
2622 }
2623
2624 if (Conf->usecompound)
2625 {
2626 int wordlen = strlen(word);
2627 SplitVar *ptr,
2628 *var = SplitToVariants(Conf, NULL, NULL, word, wordlen, 0, -1);
2629 int i;
2630
2631 while (var)
2632 {
2633 if (var->nstem > 1)
2634 {
2635 char **subres = NormalizeSubWord(Conf, var->stem[var->nstem - 1], FF_COMPOUNDLAST);
2636
2637 if (subres)
2638 {
2639 char **subptr = subres;
2640
2641 while (*subptr)
2642 {
2643 for (i = 0; i < var->nstem - 1; i++)
2644 {
2645 addNorm(&lres, &lcur, (subptr == subres) ? var->stem[i] : pstrdup(var->stem[i]), 0, NVariant);
2646 }
2647
2648 addNorm(&lres, &lcur, *subptr, 0, NVariant);
2649 subptr++;
2650 NVariant++;
2651 }
2652
2653 pfree(subres);
2654 var->stem[0] = NULL;
2655 pfree(var->stem[var->nstem - 1]);
2656 }
2657 }
2658
2659 for (i = 0; i < var->nstem && var->stem[i]; i++)
2660 pfree(var->stem[i]);
2661 ptr = var->next;
2662 pfree(var->stem);
2663 pfree(var);
2664 var = ptr;
2665 }
2666 }
2667
2668 return lres;
2669 }