Debian Codegraph @

src/backend/tsearch/ts_parse.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * ts_parse.c
4 * main parse functions for tsearch
5 *
6 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
7 *
8 *
9 * IDENTIFICATION
10 * src/backend/tsearch/ts_parse.c
11 *
12 *-------------------------------------------------------------------------
13 */
14
15 #include "postgres.h"
16
17 #include "tsearch/ts_cache.h"
18 #include "tsearch/ts_utils.h"
19 #include "varatt.h"
20
21 #define IGNORE_LONGLEXEME 1
22
23 /*
24 * Lexize subsystem
25 */
26
27 typedef struct ParsedLex
28 {
29 int type;
30 char *lemm;
31 int lenlemm;
32 struct ParsedLex *next;
33 } ParsedLex;
34
35 typedef struct ListParsedLex
36 {
37 ParsedLex *head;
38 ParsedLex *tail;
39 } ListParsedLex;
40
41 typedef struct
42 {
43 TSConfigCacheEntry *cfg;
44 Oid curDictId;
45 int posDict;
46 DictSubState dictState;
47 ParsedLex *curSub;
48 ListParsedLex towork; /* current list to work */
49 ListParsedLex waste; /* list of lexemes that already lexized */
50
51 /*
52 * fields to store last variant to lexize (basically, thesaurus or similar
53 * to, which wants several lexemes
54 */
55
56 ParsedLex *lastRes;
57 TSLexeme *tmpRes;
58 } LexizeData;
59
60 static void
61 LexizeInit(LexizeData *ld, TSConfigCacheEntry *cfg)
62 {
63 ld->cfg = cfg;
64 ld->curDictId = InvalidOid;
65 ld->posDict = 0;
66 ld->towork.head = ld->towork.tail = ld->curSub = NULL;
67 ld->waste.head = ld->waste.tail = NULL;
68 ld->lastRes = NULL;
69 ld->tmpRes = NULL;
70 }
71
72 static void
73 LPLAddTail(ListParsedLex *list, ParsedLex *newpl)
74 {
75 if (list->tail)
76 {
77 list->tail->next = newpl;
78 list->tail = newpl;
79 }
80 else
81 list->head = list->tail = newpl;
82 newpl->next = NULL;
83 }
84
85 static ParsedLex *
86 LPLRemoveHead(ListParsedLex *list)
87 {
88 ParsedLex *res = list->head;
89
90 if (list->head)
91 list->head = list->head->next;
92
93 if (list->head == NULL)
94 list->tail = NULL;
95
96 return res;
97 }
98
99 static void
100 LexizeAddLemm(LexizeData *ld, int type, char *lemm, int lenlemm)
101 {
102 ParsedLex *newpl = (ParsedLex *) palloc(sizeof(ParsedLex));
103
104 newpl->type = type;
105 newpl->lemm = lemm;
106 newpl->lenlemm = lenlemm;
107 LPLAddTail(&ld->towork, newpl);
108 ld->curSub = ld->towork.tail;
109 }
110
111 static void
112 RemoveHead(LexizeData *ld)
113 {
114 LPLAddTail(&ld->waste, LPLRemoveHead(&ld->towork));
115
116 ld->posDict = 0;
117 }
118
119 static void
120 setCorrLex(LexizeData *ld, ParsedLex **correspondLexem)
121 {
122 if (correspondLexem)
123 {
124 *correspondLexem = ld->waste.head;
125 }
126 else
127 {
128 ParsedLex *tmp,
129 *ptr = ld->waste.head;
130
131 while (ptr)
132 {
133 tmp = ptr->next;
134 pfree(ptr);
135 ptr = tmp;
136 }
137 }
138 ld->waste.head = ld->waste.tail = NULL;
139 }
140
141 static void
142 moveToWaste(LexizeData *ld, ParsedLex *stop)
143 {
144 bool go = true;
145
146 while (ld->towork.head && go)
147 {
148 if (ld->towork.head == stop)
149 {
150 ld->curSub = stop->next;
151 go = false;
152 }
153 RemoveHead(ld);
154 }
155 }
156
157 static void
158 setNewTmpRes(LexizeData *ld, ParsedLex *lex, TSLexeme *res)
159 {
160 if (ld->tmpRes)
161 {
162 TSLexeme *ptr;
163
164 for (ptr = ld->tmpRes; ptr->lexeme; ptr++)
165 pfree(ptr->lexeme);
166 pfree(ld->tmpRes);
167 }
168 ld->tmpRes = res;
169 ld->lastRes = lex;
170 }
171
172 static TSLexeme *
173 LexizeExec(LexizeData *ld, ParsedLex **correspondLexem)
174 {
175 int i;
176 ListDictionary *map;
177 TSDictionaryCacheEntry *dict;
178 TSLexeme *res;
179
180 if (ld->curDictId == InvalidOid)
181 {
182 /*
183 * usual mode: dictionary wants only one word, but we should keep in
184 * mind that we should go through all stack
185 */
186
187 while (ld->towork.head)
188 {
189 ParsedLex *curVal = ld->towork.head;
190 char *curValLemm = curVal->lemm;
191 int curValLenLemm = curVal->lenlemm;
192
193 map = ld->cfg->map + curVal->type;
194
195 if (curVal->type == 0 || curVal->type >= ld->cfg->lenmap || map->len == 0)
196 {
197 /* skip this type of lexeme */
198 RemoveHead(ld);
199 continue;
200 }
201
202 for (i = ld->posDict; i < map->len; i++)
203 {
204 dict = lookup_ts_dictionary_cache(map->dictIds[i]);
205
206 ld->dictState.isend = ld->dictState.getnext = false;
207 ld->dictState.private_state = NULL;
208 res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(dict->lexize),
209 PointerGetDatum(dict->dictData),
210 PointerGetDatum(curValLemm),
211 Int32GetDatum(curValLenLemm),
212 PointerGetDatum(&ld->dictState)));
213
214 if (ld->dictState.getnext)
215 {
216 /*
217 * dictionary wants next word, so setup and store current
218 * position and go to multiword mode
219 */
220
221 ld->curDictId = DatumGetObjectId(map->dictIds[i]);
222 ld->posDict = i + 1;
223 ld->curSub = curVal->next;
224 if (res)
225 setNewTmpRes(ld, curVal, res);
226 return LexizeExec(ld, correspondLexem);
227 }
228
229 if (!res) /* dictionary doesn't know this lexeme */
230 continue;
231
232 if (res->flags & TSL_FILTER)
233 {
234 curValLemm = res->lexeme;
235 curValLenLemm = strlen(res->lexeme);
236 continue;
237 }
238
239 RemoveHead(ld);
240 setCorrLex(ld, correspondLexem);
241 return res;
242 }
243
244 RemoveHead(ld);
245 }
246 }
247 else
248 { /* curDictId is valid */
249 dict = lookup_ts_dictionary_cache(ld->curDictId);
250
251 /*
252 * Dictionary ld->curDictId asks us about following words
253 */
254
255 while (ld->curSub)
256 {
257 ParsedLex *curVal = ld->curSub;
258
259 map = ld->cfg->map + curVal->type;
260
261 if (curVal->type != 0)
262 {
263 bool dictExists = false;
264
265 if (curVal->type >= ld->cfg->lenmap || map->len == 0)
266 {
267 /* skip this type of lexeme */
268 ld->curSub = curVal->next;
269 continue;
270 }
271
272 /*
273 * We should be sure that current type of lexeme is recognized
274 * by our dictionary: we just check is it exist in list of
275 * dictionaries ?
276 */
277 for (i = 0; i < map->len && !dictExists; i++)
278 if (ld->curDictId == DatumGetObjectId(map->dictIds[i]))
279 dictExists = true;
280
281 if (!dictExists)
282 {
283 /*
284 * Dictionary can't work with current type of lexeme,
285 * return to basic mode and redo all stored lexemes
286 */
287 ld->curDictId = InvalidOid;
288 return LexizeExec(ld, correspondLexem);
289 }
290 }
291
292 ld->dictState.isend = (curVal->type == 0);
293 ld->dictState.getnext = false;
294
295 res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(dict->lexize),
296 PointerGetDatum(dict->dictData),
297 PointerGetDatum(curVal->lemm),
298 Int32GetDatum(curVal->lenlemm),
299 PointerGetDatum(&ld->dictState)));
300
301 if (ld->dictState.getnext)
302 {
303 /* Dictionary wants one more */
304 ld->curSub = curVal->next;
305 if (res)
306 setNewTmpRes(ld, curVal, res);
307 continue;
308 }
309
310 if (res || ld->tmpRes)
311 {
312 /*
313 * Dictionary normalizes lexemes, so we remove from stack all
314 * used lexemes, return to basic mode and redo end of stack
315 * (if it exists)
316 */
317 if (res)
318 {
319 moveToWaste(ld, ld->curSub);
320 }
321 else
322 {
323 res = ld->tmpRes;
324 moveToWaste(ld, ld->lastRes);
325 }
326
327 /* reset to initial state */
328 ld->curDictId = InvalidOid;
329 ld->posDict = 0;
330 ld->lastRes = NULL;
331 ld->tmpRes = NULL;
332 setCorrLex(ld, correspondLexem);
333 return res;
334 }
335
336 /*
337 * Dict don't want next lexem and didn't recognize anything, redo
338 * from ld->towork.head
339 */
340 ld->curDictId = InvalidOid;
341 return LexizeExec(ld, correspondLexem);
342 }
343 }
344
345 setCorrLex(ld, correspondLexem);
346 return NULL;
347 }
348
349 /*
350 * Parse string and lexize words.
351 *
352 * prs will be filled in.
353 */
354 void
355 parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen)
356 {
357 int type,
358 lenlemm = 0; /* silence compiler warning */
359 char *lemm = NULL;
360 LexizeData ldata;
361 TSLexeme *norms;
362 TSConfigCacheEntry *cfg;
363 TSParserCacheEntry *prsobj;
364 void *prsdata;
365
366 cfg = lookup_ts_config_cache(cfgId);
367 prsobj = lookup_ts_parser_cache(cfg->prsId);
368
369 prsdata = DatumGetPointer(FunctionCall2(&prsobj->prsstart,
370 PointerGetDatum(buf),
371 Int32GetDatum(buflen)));
372
373 LexizeInit(&ldata, cfg);
374
375 do
376 {
377 type = DatumGetInt32(FunctionCall3(&(prsobj->prstoken),
378 PointerGetDatum(prsdata),
379 PointerGetDatum(&lemm),
380 PointerGetDatum(&lenlemm)));
381
382 if (type > 0 && lenlemm >= MAXSTRLEN)
383 {
384 #ifdef IGNORE_LONGLEXEME
385 ereport(NOTICE,
386 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
387 errmsg("word is too long to be indexed"),
388 errdetail("Words longer than %d characters are ignored.",
389 MAXSTRLEN)));
390 continue;
391 #else
392 ereport(ERROR,
393 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
394 errmsg("word is too long to be indexed"),
395 errdetail("Words longer than %d characters are ignored.",
396 MAXSTRLEN)));
397 #endif
398 }
399
400 LexizeAddLemm(&ldata, type, lemm, lenlemm);
401
402 while ((norms = LexizeExec(&ldata, NULL)) != NULL)
403 {
404 prs->pos++; /* set pos */
405
406 for (TSLexeme *ptr = norms; ptr->lexeme; ptr++)
407 {
408 size_t lexeme_len = strlen(ptr->lexeme);
409
410 if (lexeme_len > MAXSTRLEN)
411 {
412 #ifdef IGNORE_LONGLEXEME
413 ereport(NOTICE,
414 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
415 errmsg("word is too long to be indexed"),
416 errdetail("Words longer than %d characters are ignored.",
417 MAXSTRLEN)));
418 continue;
419 #else
420 ereport(ERROR,
421 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
422 errmsg("word is too long to be indexed"),
423 errdetail("Words longer than %d characters are ignored.",
424 MAXSTRLEN)));
425 #endif
426 }
427
428 if (prs->curwords == prs->lenwords)
429 {
430 prs->lenwords *= 2;
431 prs->words = (ParsedWord *) repalloc(prs->words, prs->lenwords * sizeof(ParsedWord));
432 }
433
434 if (ptr->flags & TSL_ADDPOS)
435 prs->pos++;
436 prs->words[prs->curwords].len = lexeme_len;
437 prs->words[prs->curwords].word = ptr->lexeme;
438 prs->words[prs->curwords].nvariant = ptr->nvariant;
439 prs->words[prs->curwords].flags = ptr->flags & TSL_PREFIX;
440 prs->words[prs->curwords].alen = 0;
441 prs->words[prs->curwords].pos.pos = LIMITPOS(prs->pos);
442 prs->curwords++;
443 }
444 pfree(norms);
445 }
446 } while (type > 0);
447
448 FunctionCall1(&(prsobj->prsend), PointerGetDatum(prsdata));
449 }
450
451 /*
452 * Headline framework
453 */
454
455 /* Add a word to prs->words[] */
456 static void
457 hladdword(HeadlineParsedText *prs, char *buf, int buflen, int type)
458 {
459 if (prs->curwords >= prs->lenwords)
460 {
461 prs->lenwords *= 2;
462 prs->words = (HeadlineWordEntry *) repalloc(prs->words, prs->lenwords * sizeof(HeadlineWordEntry));
463 }
464 memset(&(prs->words[prs->curwords]), 0, sizeof(HeadlineWordEntry));
465 prs->words[prs->curwords].type = (uint8) type;
466 prs->words[prs->curwords].len = buflen;
467 prs->words[prs->curwords].word = palloc(buflen);
468 memcpy(prs->words[prs->curwords].word, buf, buflen);
469 prs->curwords++;
470 }
471
472 /*
473 * Add pos and matching-query-item data to the just-added word.
474 * Here, buf/buflen represent a processed lexeme, not raw token text.
475 *
476 * If the query contains more than one matching item, we replicate
477 * the last-added word so that each item can be pointed to. The
478 * duplicate entries are marked with repeated = 1.
479 */
480 static void
481 hlfinditem(HeadlineParsedText *prs, TSQuery query, int32 pos, char *buf, int buflen)
482 {
483 int i;
484 QueryItem *item = GETQUERY(query);
485 HeadlineWordEntry *word;
486
487 while (prs->curwords + query->size >= prs->lenwords)
488 {
489 prs->lenwords *= 2;
490 prs->words = (HeadlineWordEntry *) repalloc(prs->words, prs->lenwords * sizeof(HeadlineWordEntry));
491 }
492
493 word = &(prs->words[prs->curwords - 1]);
494 word->pos = LIMITPOS(pos);
495 for (i = 0; i < query->size; i++)
496 {
497 if (item->type == QI_VAL &&
498 tsCompareString(GETOPERAND(query) + item->qoperand.distance, item->qoperand.length,
499 buf, buflen, item->qoperand.prefix) == 0)
500 {
501 if (word->item)
502 {
503 memcpy(&(prs->words[prs->curwords]), word, sizeof(HeadlineWordEntry));
504 prs->words[prs->curwords].item = &item->qoperand;
505 prs->words[prs->curwords].repeated = 1;
506 prs->curwords++;
507 }
508 else
509 word->item = &item->qoperand;
510 }
511 item++;
512 }
513 }
514
515 static void
516 addHLParsedLex(HeadlineParsedText *prs, TSQuery query, ParsedLex *lexs, TSLexeme *norms)
517 {
518 ParsedLex *tmplexs;
519 TSLexeme *ptr;
520 int32 savedpos;
521
522 while (lexs)
523 {
524 if (lexs->type > 0)
525 hladdword(prs, lexs->lemm, lexs->lenlemm, lexs->type);
526
527 ptr = norms;
528 savedpos = prs->vectorpos;
529 while (ptr && ptr->lexeme)
530 {
531 if (ptr->flags & TSL_ADDPOS)
532 savedpos++;
533 hlfinditem(prs, query, savedpos, ptr->lexeme, strlen(ptr->lexeme));
534 ptr++;
535 }
536
537 tmplexs = lexs->next;
538 pfree(lexs);
539 lexs = tmplexs;
540 }
541
542 if (norms)
543 {
544 ptr = norms;
545 while (ptr->lexeme)
546 {
547 if (ptr->flags & TSL_ADDPOS)
548 prs->vectorpos++;
549 pfree(ptr->lexeme);
550 ptr++;
551 }
552 pfree(norms);
553 }
554 }
555
556 void
557 hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query, char *buf, int buflen)
558 {
559 int type,
560 lenlemm = 0; /* silence compiler warning */
561 char *lemm = NULL;
562 LexizeData ldata;
563 TSLexeme *norms;
564 ParsedLex *lexs;
565 TSConfigCacheEntry *cfg;
566 TSParserCacheEntry *prsobj;
567 void *prsdata;
568
569 cfg = lookup_ts_config_cache(cfgId);
570 prsobj = lookup_ts_parser_cache(cfg->prsId);
571
572 prsdata = DatumGetPointer(FunctionCall2(&(prsobj->prsstart),
573 PointerGetDatum(buf),
574 Int32GetDatum(buflen)));
575
576 LexizeInit(&ldata, cfg);
577
578 do
579 {
580 type = DatumGetInt32(FunctionCall3(&(prsobj->prstoken),
581 PointerGetDatum(prsdata),
582 PointerGetDatum(&lemm),
583 PointerGetDatum(&lenlemm)));
584
585 if (type > 0 && lenlemm >= MAXSTRLEN)
586 {
587 #ifdef IGNORE_LONGLEXEME
588 ereport(NOTICE,
589 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
590 errmsg("word is too long to be indexed"),
591 errdetail("Words longer than %d characters are ignored.",
592 MAXSTRLEN)));
593 continue;
594 #else
595 ereport(ERROR,
596 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
597 errmsg("word is too long to be indexed"),
598 errdetail("Words longer than %d characters are ignored.",
599 MAXSTRLEN)));
600 #endif
601 }
602
603 LexizeAddLemm(&ldata, type, lemm, lenlemm);
604
605 do
606 {
607 if ((norms = LexizeExec(&ldata, &lexs)) != NULL)
608 {
609 prs->vectorpos++;
610 addHLParsedLex(prs, query, lexs, norms);
611 }
612 else
613 addHLParsedLex(prs, query, lexs, NULL);
614 } while (norms);
615 } while (type > 0);
616
617 FunctionCall1(&(prsobj->prsend), PointerGetDatum(prsdata));
618 }
619
620 /*
621 * Generate the headline, as a text object, from HeadlineParsedText.
622 */
623 text *
624 generateHeadline(HeadlineParsedText *prs)
625 {
626 text *out;
627 char *ptr;
628 int len = 128;
629 int numfragments = 0;
630 int16 infrag = 0;
631
632 HeadlineWordEntry *wrd = prs->words;
633
634 out = (text *) palloc(len);
635 ptr = ((char *) out) + VARHDRSZ;
636
637 while (wrd - prs->words < prs->curwords)
638 {
639 while (wrd->len + prs->stopsellen + prs->startsellen + prs->fragdelimlen + (ptr - ((char *) out)) >= len)
640 {
641 int dist = ptr - ((char *) out);
642
643 len *= 2;
644 out = (text *) repalloc(out, len);
645 ptr = ((char *) out) + dist;
646 }
647
648 if (wrd->in && !wrd->repeated)
649 {
650 if (!infrag)
651 {
652
653 /* start of a new fragment */
654 infrag = 1;
655 numfragments++;
656 /* add a fragment delimiter if this is after the first one */
657 if (numfragments > 1)
658 {
659 memcpy(ptr, prs->fragdelim, prs->fragdelimlen);
660 ptr += prs->fragdelimlen;
661 }
662 }
663 if (wrd->replace)
664 {
665 *ptr = ' ';
666 ptr++;
667 }
668 else if (!wrd->skip)
669 {
670 if (wrd->selected)
671 {
672 memcpy(ptr, prs->startsel, prs->startsellen);
673 ptr += prs->startsellen;
674 }
675 memcpy(ptr, wrd->word, wrd->len);
676 ptr += wrd->len;
677 if (wrd->selected)
678 {
679 memcpy(ptr, prs->stopsel, prs->stopsellen);
680 ptr += prs->stopsellen;
681 }
682 }
683 }
684 else if (!wrd->repeated)
685 {
686 if (infrag)
687 infrag = 0;
688 pfree(wrd->word);
689 }
690
691 wrd++;
692 }
693
694 SET_VARSIZE(out, ptr - ((char *) out));
695 return out;
696 }