Debian Codegraph @

src/backend/tsearch/to_tsany.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * to_tsany.c
4 * to_ts* function definitions
5 *
6 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
7 *
8 *
9 * IDENTIFICATION
10 * src/backend/tsearch/to_tsany.c
11 *
12 *-------------------------------------------------------------------------
13 */
14 #include "postgres.h"
15
16 #include "tsearch/ts_cache.h"
17 #include "tsearch/ts_utils.h"
18 #include "utils/builtins.h"
19 #include "utils/jsonfuncs.h"
20
21
22 /*
23 * Opaque data structure, which is passed by parse_tsquery() to pushval_morph().
24 */
25 typedef struct MorphOpaque
26 {
27 Oid cfg_id;
28
29 /*
30 * Single tsquery morph could be parsed into multiple words. When these
31 * words reside in adjacent positions, they are connected using this
32 * operator. Usually, that is OP_PHRASE, which requires word positions of
33 * a complex morph to exactly match the tsvector.
34 */
35 int qoperator;
36 } MorphOpaque;
37
38 typedef struct TSVectorBuildState
39 {
40 ParsedText *prs;
41 Oid cfgId;
42 } TSVectorBuildState;
43
44 static void add_to_tsvector(void *_state, char *elem_value, int elem_len);
45
46
47 Datum
48 get_current_ts_config(PG_FUNCTION_ARGS)
49 {
50 PG_RETURN_OID(getTSCurrentConfig(true));
51 }
52
53 /*
54 * to_tsvector
55 */
56 static int
57 compareWORD(const void *a, const void *b)
58 {
59 int res;
60
61 res = tsCompareString(((const ParsedWord *) a)->word, ((const ParsedWord *) a)->len,
62 ((const ParsedWord *) b)->word, ((const ParsedWord *) b)->len,
63 false);
64
65 if (res == 0)
66 {
67 if (((const ParsedWord *) a)->pos.pos == ((const ParsedWord *) b)->pos.pos)
68 return 0;
69
70 res = (((const ParsedWord *) a)->pos.pos > ((const ParsedWord *) b)->pos.pos) ? 1 : -1;
71 }
72
73 return res;
74 }
75
76 static int
77 uniqueWORD(ParsedWord *a, int32 l)
78 {
79 ParsedWord *ptr,
80 *res;
81 int tmppos;
82
83 if (l == 1)
84 {
85 tmppos = LIMITPOS(a->pos.pos);
86 a->alen = 2;
87 a->pos.apos = (uint16 *) palloc(sizeof(uint16) * a->alen);
88 a->pos.apos[0] = 1;
89 a->pos.apos[1] = tmppos;
90 return l;
91 }
92
93 res = a;
94 ptr = a + 1;
95
96 /*
97 * Sort words with its positions
98 */
99 qsort(a, l, sizeof(ParsedWord), compareWORD);
100
101 /*
102 * Initialize first word and its first position
103 */
104 tmppos = LIMITPOS(a->pos.pos);
105 a->alen = 2;
106 a->pos.apos = (uint16 *) palloc(sizeof(uint16) * a->alen);
107 a->pos.apos[0] = 1;
108 a->pos.apos[1] = tmppos;
109
110 /*
111 * Summarize position information for each word
112 */
113 while (ptr - a < l)
114 {
115 if (!(ptr->len == res->len &&
116 strncmp(ptr->word, res->word, res->len) == 0))
117 {
118 /*
119 * Got a new word, so put it in result
120 */
121 res++;
122 res->len = ptr->len;
123 res->word = ptr->word;
124 tmppos = LIMITPOS(ptr->pos.pos);
125 res->alen = 2;
126 res->pos.apos = (uint16 *) palloc(sizeof(uint16) * res->alen);
127 res->pos.apos[0] = 1;
128 res->pos.apos[1] = tmppos;
129 }
130 else
131 {
132 /*
133 * The word already exists, so adjust position information. But
134 * before we should check size of position's array, max allowed
135 * value for position and uniqueness of position
136 */
137 pfree(ptr->word);
138 if (res->pos.apos[0] < MAXNUMPOS - 1 && res->pos.apos[res->pos.apos[0]] != MAXENTRYPOS - 1 &&
139 res->pos.apos[res->pos.apos[0]] != LIMITPOS(ptr->pos.pos))
140 {
141 if (res->pos.apos[0] + 1 >= res->alen)
142 {
143 res->alen *= 2;
144 res->pos.apos = (uint16 *) repalloc(res->pos.apos, sizeof(uint16) * res->alen);
145 }
146 if (res->pos.apos[0] == 0 || res->pos.apos[res->pos.apos[0]] != LIMITPOS(ptr->pos.pos))
147 {
148 res->pos.apos[res->pos.apos[0] + 1] = LIMITPOS(ptr->pos.pos);
149 res->pos.apos[0]++;
150 }
151 }
152 }
153 ptr++;
154 }
155
156 return res + 1 - a;
157 }
158
159 /*
160 * make value of tsvector, given parsed text
161 *
162 * Note: frees prs->words and subsidiary data.
163 */
164 TSVector
165 make_tsvector(ParsedText *prs)
166 {
167 int i,
168 j;
169 size_t lenstr = 0,
170 totallen;
171 TSVector in;
172 WordEntry *ptr;
173 char *str;
174 int stroff;
175
176 /* Merge duplicate words */
177 if (prs->curwords > 0)
178 prs->curwords = uniqueWORD(prs->words, prs->curwords);
179
180 /*
181 * Determine space needed. Since what we are calculating is equivalent to
182 * the size of a portion of the input data structure, lenstr surely can't
183 * overflow size_t.
184 */
185 for (i = 0; i < prs->curwords; i++)
186 {
187 int toklen = prs->words[i].len;
188
189 /* Double-check that caller passed only lexemes of valid lengths */
190 if (toklen <= 0 || toklen > MAXSTRLEN)
191 ereport(ERROR,
192 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
193 errmsg("lexeme is too long for tsvector (%zu bytes, max %zu bytes)",
194 (size_t) toklen, (size_t) MAXSTRLEN)));
195 lenstr += toklen;
196 if (prs->words[i].alen)
197 {
198 lenstr = SHORTALIGN(lenstr);
199 lenstr += sizeof(uint16) + prs->words[i].pos.apos[0] * sizeof(WordEntryPos);
200 }
201 }
202
203 if (lenstr > MAXSTRPOS)
204 ereport(ERROR,
205 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
206 errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)",
207 lenstr, (size_t) MAXSTRPOS)));
208
209 totallen = CALCDATASIZE(prs->curwords, lenstr);
210 in = (TSVector) palloc0(totallen);
211 SET_VARSIZE(in, totallen);
212 in->size = prs->curwords;
213
214 ptr = ARRPTR(in);
215 str = STRPTR(in);
216 stroff = 0;
217 for (i = 0; i < prs->curwords; i++)
218 {
219 ptr->len = prs->words[i].len;
220 ptr->pos = stroff;
221 memcpy(str + stroff, prs->words[i].word, prs->words[i].len);
222 stroff += prs->words[i].len;
223 pfree(prs->words[i].word);
224 if (prs->words[i].alen)
225 {
226 int k = prs->words[i].pos.apos[0];
227 WordEntryPos *wptr;
228
229 if (k > 0xFFFF)
230 elog(ERROR, "positions array too long");
231
232 ptr->haspos = 1;
233 stroff = SHORTALIGN(stroff);
234 *(uint16 *) (str + stroff) = (uint16) k;
235 wptr = POSDATAPTR(in, ptr);
236 for (j = 0; j < k; j++)
237 {
238 WEP_SETWEIGHT(wptr[j], 0);
239 WEP_SETPOS(wptr[j], prs->words[i].pos.apos[j + 1]);
240 }
241 stroff += sizeof(uint16) + k * sizeof(WordEntryPos);
242 pfree(prs->words[i].pos.apos);
243 }
244 else
245 ptr->haspos = 0;
246 ptr++;
247 }
248
249 if (prs->words)
250 pfree(prs->words);
251
252 return in;
253 }
254
255 Datum
256 to_tsvector_byid(PG_FUNCTION_ARGS)
257 {
258 Oid cfgId = PG_GETARG_OID(0);
259 text *in = PG_GETARG_TEXT_PP(1);
260 ParsedText prs;
261 TSVector out;
262
263 prs.lenwords = VARSIZE_ANY_EXHDR(in) / 6; /* just estimation of word's
264 * number */
265 if (prs.lenwords < 2)
266 prs.lenwords = 2;
267 else if (prs.lenwords > MaxAllocSize / sizeof(ParsedWord))
268 prs.lenwords = MaxAllocSize / sizeof(ParsedWord);
269 prs.curwords = 0;
270 prs.pos = 0;
271 prs.words = (ParsedWord *) palloc(sizeof(ParsedWord) * prs.lenwords);
272
273 parsetext(cfgId, &prs, VARDATA_ANY(in), VARSIZE_ANY_EXHDR(in));
274
275 PG_FREE_IF_COPY(in, 1);
276
277 out = make_tsvector(&prs);
278
279 PG_RETURN_TSVECTOR(out);
280 }
281
282 Datum
283 to_tsvector(PG_FUNCTION_ARGS)
284 {
285 text *in = PG_GETARG_TEXT_PP(0);
286 Oid cfgId;
287
288 cfgId = getTSCurrentConfig(true);
289 PG_RETURN_DATUM(DirectFunctionCall2(to_tsvector_byid,
290 ObjectIdGetDatum(cfgId),
291 PointerGetDatum(in)));
292 }
293
294 /*
295 * Worker function for jsonb(_string)_to_tsvector(_byid)
296 */
297 static TSVector
298 jsonb_to_tsvector_worker(Oid cfgId, Jsonb *jb, uint32 flags)
299 {
300 TSVectorBuildState state;
301 ParsedText prs;
302
303 prs.words = NULL;
304 prs.curwords = 0;
305 state.prs = &prs;
306 state.cfgId = cfgId;
307
308 iterate_jsonb_values(jb, flags, &state, add_to_tsvector);
309
310 return make_tsvector(&prs);
311 }
312
313 Datum
314 jsonb_string_to_tsvector_byid(PG_FUNCTION_ARGS)
315 {
316 Oid cfgId = PG_GETARG_OID(0);
317 Jsonb *jb = PG_GETARG_JSONB_P(1);
318 TSVector result;
319
320 result = jsonb_to_tsvector_worker(cfgId, jb, jtiString);
321 PG_FREE_IF_COPY(jb, 1);
322
323 PG_RETURN_TSVECTOR(result);
324 }
325
326 Datum
327 jsonb_string_to_tsvector(PG_FUNCTION_ARGS)
328 {
329 Jsonb *jb = PG_GETARG_JSONB_P(0);
330 Oid cfgId;
331 TSVector result;
332
333 cfgId = getTSCurrentConfig(true);
334 result = jsonb_to_tsvector_worker(cfgId, jb, jtiString);
335 PG_FREE_IF_COPY(jb, 0);
336
337 PG_RETURN_TSVECTOR(result);
338 }
339
340 Datum
341 jsonb_to_tsvector_byid(PG_FUNCTION_ARGS)
342 {
343 Oid cfgId = PG_GETARG_OID(0);
344 Jsonb *jb = PG_GETARG_JSONB_P(1);
345 Jsonb *jbFlags = PG_GETARG_JSONB_P(2);
346 TSVector result;
347 uint32 flags = parse_jsonb_index_flags(jbFlags);
348
349 result = jsonb_to_tsvector_worker(cfgId, jb, flags);
350 PG_FREE_IF_COPY(jb, 1);
351 PG_FREE_IF_COPY(jbFlags, 2);
352
353 PG_RETURN_TSVECTOR(result);
354 }
355
356 Datum
357 jsonb_to_tsvector(PG_FUNCTION_ARGS)
358 {
359 Jsonb *jb = PG_GETARG_JSONB_P(0);
360 Jsonb *jbFlags = PG_GETARG_JSONB_P(1);
361 Oid cfgId;
362 TSVector result;
363 uint32 flags = parse_jsonb_index_flags(jbFlags);
364
365 cfgId = getTSCurrentConfig(true);
366 result = jsonb_to_tsvector_worker(cfgId, jb, flags);
367 PG_FREE_IF_COPY(jb, 0);
368 PG_FREE_IF_COPY(jbFlags, 1);
369
370 PG_RETURN_TSVECTOR(result);
371 }
372
373 /*
374 * Worker function for json(_string)_to_tsvector(_byid)
375 */
376 static TSVector
377 json_to_tsvector_worker(Oid cfgId, text *json, uint32 flags)
378 {
379 TSVectorBuildState state;
380 ParsedText prs;
381
382 prs.words = NULL;
383 prs.curwords = 0;
384 state.prs = &prs;
385 state.cfgId = cfgId;
386
387 iterate_json_values(json, flags, &state, add_to_tsvector);
388
389 return make_tsvector(&prs);
390 }
391
392 Datum
393 json_string_to_tsvector_byid(PG_FUNCTION_ARGS)
394 {
395 Oid cfgId = PG_GETARG_OID(0);
396 text *json = PG_GETARG_TEXT_P(1);
397 TSVector result;
398
399 result = json_to_tsvector_worker(cfgId, json, jtiString);
400 PG_FREE_IF_COPY(json, 1);
401
402 PG_RETURN_TSVECTOR(result);
403 }
404
405 Datum
406 json_string_to_tsvector(PG_FUNCTION_ARGS)
407 {
408 text *json = PG_GETARG_TEXT_P(0);
409 Oid cfgId;
410 TSVector result;
411
412 cfgId = getTSCurrentConfig(true);
413 result = json_to_tsvector_worker(cfgId, json, jtiString);
414 PG_FREE_IF_COPY(json, 0);
415
416 PG_RETURN_TSVECTOR(result);
417 }
418
419 Datum
420 json_to_tsvector_byid(PG_FUNCTION_ARGS)
421 {
422 Oid cfgId = PG_GETARG_OID(0);
423 text *json = PG_GETARG_TEXT_P(1);
424 Jsonb *jbFlags = PG_GETARG_JSONB_P(2);
425 TSVector result;
426 uint32 flags = parse_jsonb_index_flags(jbFlags);
427
428 result = json_to_tsvector_worker(cfgId, json, flags);
429 PG_FREE_IF_COPY(json, 1);
430 PG_FREE_IF_COPY(jbFlags, 2);
431
432 PG_RETURN_TSVECTOR(result);
433 }
434
435 Datum
436 json_to_tsvector(PG_FUNCTION_ARGS)
437 {
438 text *json = PG_GETARG_TEXT_P(0);
439 Jsonb *jbFlags = PG_GETARG_JSONB_P(1);
440 Oid cfgId;
441 TSVector result;
442 uint32 flags = parse_jsonb_index_flags(jbFlags);
443
444 cfgId = getTSCurrentConfig(true);
445 result = json_to_tsvector_worker(cfgId, json, flags);
446 PG_FREE_IF_COPY(json, 0);
447 PG_FREE_IF_COPY(jbFlags, 1);
448
449 PG_RETURN_TSVECTOR(result);
450 }
451
452 /*
453 * Parse lexemes in an element of a json(b) value, add to TSVectorBuildState.
454 */
455 static void
456 add_to_tsvector(void *_state, char *elem_value, int elem_len)
457 {
458 TSVectorBuildState *state = (TSVectorBuildState *) _state;
459 ParsedText *prs = state->prs;
460 int32 prevwords;
461
462 if (prs->words == NULL)
463 {
464 /*
465 * First time through: initialize words array to a reasonable size.
466 * (parsetext() will realloc it bigger as needed.)
467 */
468 prs->lenwords = 16;
469 prs->words = (ParsedWord *) palloc(sizeof(ParsedWord) * prs->lenwords);
470 prs->curwords = 0;
471 prs->pos = 0;
472 }
473
474 prevwords = prs->curwords;
475
476 parsetext(state->cfgId, prs, elem_value, elem_len);
477
478 /*
479 * If we extracted any words from this JSON element, advance pos to create
480 * an artificial break between elements. This is because we don't want
481 * phrase searches to think that the last word in this element is adjacent
482 * to the first word in the next one.
483 */
484 if (prs->curwords > prevwords)
485 prs->pos += 1;
486 }
487
488
489 /*
490 * to_tsquery
491 */
492
493
494 /*
495 * This function is used for morph parsing.
496 *
497 * The value is passed to parsetext which will call the right dictionary to
498 * lexize the word. If it turns out to be a stopword, we push a QI_VALSTOP
499 * to the stack.
500 *
501 * All words belonging to the same variant are pushed as an ANDed list,
502 * and different variants are ORed together.
503 */
504 static void
505 pushval_morph(Datum opaque, TSQueryParserState state, char *strval, int lenval, int16 weight, bool prefix)
506 {
507 int32 count = 0;
508 ParsedText prs;
509 uint32 variant,
510 pos = 0,
511 cntvar = 0,
512 cntpos = 0,
513 cnt = 0;
514 MorphOpaque *data = (MorphOpaque *) DatumGetPointer(opaque);
515
516 prs.lenwords = 4;
517 prs.curwords = 0;
518 prs.pos = 0;
519 prs.words = (ParsedWord *) palloc(sizeof(ParsedWord) * prs.lenwords);
520
521 parsetext(data->cfg_id, &prs, strval, lenval);
522
523 if (prs.curwords > 0)
524 {
525 while (count < prs.curwords)
526 {
527 /*
528 * Were any stop words removed? If so, fill empty positions with
529 * placeholders linked by an appropriate operator.
530 */
531 if (pos > 0 && pos + 1 < prs.words[count].pos.pos)
532 {
533 while (pos + 1 < prs.words[count].pos.pos)
534 {
535 /* put placeholders for each missing stop word */
536 pushStop(state);
537 if (cntpos)
538 pushOperator(state, data->qoperator, 1);
539 cntpos++;
540 pos++;
541 }
542 }
543
544 /* save current word's position */
545 pos = prs.words[count].pos.pos;
546
547 /* Go through all variants obtained from this token */
548 cntvar = 0;
549 while (count < prs.curwords && pos == prs.words[count].pos.pos)
550 {
551 variant = prs.words[count].nvariant;
552
553 /* Push all words belonging to the same variant */
554 cnt = 0;
555 while (count < prs.curwords &&
556 pos == prs.words[count].pos.pos &&
557 variant == prs.words[count].nvariant)
558 {
559 pushValue(state,
560 prs.words[count].word,
561 prs.words[count].len,
562 weight,
563 ((prs.words[count].flags & TSL_PREFIX) || prefix));
564 pfree(prs.words[count].word);
565 if (cnt)
566 pushOperator(state, OP_AND, 0);
567 cnt++;
568 count++;
569 }
570
571 if (cntvar)
572 pushOperator(state, OP_OR, 0);
573 cntvar++;
574 }
575
576 if (cntpos)
577 {
578 /* distance may be useful */
579 pushOperator(state, data->qoperator, 1);
580 }
581
582 cntpos++;
583 }
584
585 pfree(prs.words);
586 }
587 else
588 pushStop(state);
589 }
590
591 Datum
592 to_tsquery_byid(PG_FUNCTION_ARGS)
593 {
594 text *in = PG_GETARG_TEXT_PP(1);
595 TSQuery query;
596 MorphOpaque data;
597
598 data.cfg_id = PG_GETARG_OID(0);
599
600 /*
601 * Passing OP_PHRASE as a qoperator makes tsquery require matching of word
602 * positions of a complex morph exactly match the tsvector. Also, when
603 * the complex morphs are connected with OP_PHRASE operator, we connect
604 * all their words into the OP_PHRASE sequence.
605 */
606 data.qoperator = OP_PHRASE;
607
608 query = parse_tsquery(text_to_cstring(in),
609 pushval_morph,
610 PointerGetDatum(&data),
611 0,
612 NULL);
613
614 PG_RETURN_TSQUERY(query);
615 }
616
617 Datum
618 to_tsquery(PG_FUNCTION_ARGS)
619 {
620 text *in = PG_GETARG_TEXT_PP(0);
621 Oid cfgId;
622
623 cfgId = getTSCurrentConfig(true);
624 PG_RETURN_DATUM(DirectFunctionCall2(to_tsquery_byid,
625 ObjectIdGetDatum(cfgId),
626 PointerGetDatum(in)));
627 }
628
629 Datum
630 plainto_tsquery_byid(PG_FUNCTION_ARGS)
631 {
632 text *in = PG_GETARG_TEXT_PP(1);
633 TSQuery query;
634 MorphOpaque data;
635
636 data.cfg_id = PG_GETARG_OID(0);
637
638 /*
639 * parse_tsquery() with P_TSQ_PLAIN flag takes the whole input text as a
640 * single morph. Passing OP_PHRASE as a qoperator makes tsquery require
641 * matching of all words independently on their positions.
642 */
643 data.qoperator = OP_AND;
644
645 query = parse_tsquery(text_to_cstring(in),
646 pushval_morph,
647 PointerGetDatum(&data),
648 P_TSQ_PLAIN,
649 NULL);
650
651 PG_RETURN_POINTER(query);
652 }
653
654 Datum
655 plainto_tsquery(PG_FUNCTION_ARGS)
656 {
657 text *in = PG_GETARG_TEXT_PP(0);
658 Oid cfgId;
659
660 cfgId = getTSCurrentConfig(true);
661 PG_RETURN_DATUM(DirectFunctionCall2(plainto_tsquery_byid,
662 ObjectIdGetDatum(cfgId),
663 PointerGetDatum(in)));
664 }
665
666
667 Datum
668 phraseto_tsquery_byid(PG_FUNCTION_ARGS)
669 {
670 text *in = PG_GETARG_TEXT_PP(1);
671 TSQuery query;
672 MorphOpaque data;
673
674 data.cfg_id = PG_GETARG_OID(0);
675
676 /*
677 * parse_tsquery() with P_TSQ_PLAIN flag takes the whole input text as a
678 * single morph. Passing OP_PHRASE as a qoperator makes tsquery require
679 * matching of word positions.
680 */
681 data.qoperator = OP_PHRASE;
682
683 query = parse_tsquery(text_to_cstring(in),
684 pushval_morph,
685 PointerGetDatum(&data),
686 P_TSQ_PLAIN,
687 NULL);
688
689 PG_RETURN_TSQUERY(query);
690 }
691
692 Datum
693 phraseto_tsquery(PG_FUNCTION_ARGS)
694 {
695 text *in = PG_GETARG_TEXT_PP(0);
696 Oid cfgId;
697
698 cfgId = getTSCurrentConfig(true);
699 PG_RETURN_DATUM(DirectFunctionCall2(phraseto_tsquery_byid,
700 ObjectIdGetDatum(cfgId),
701 PointerGetDatum(in)));
702 }
703
704 Datum
705 websearch_to_tsquery_byid(PG_FUNCTION_ARGS)
706 {
707 text *in = PG_GETARG_TEXT_PP(1);
708 MorphOpaque data;
709 TSQuery query = NULL;
710
711 data.cfg_id = PG_GETARG_OID(0);
712
713 /*
714 * Passing OP_PHRASE as a qoperator makes tsquery require matching of word
715 * positions of a complex morph exactly match the tsvector. Also, when
716 * the complex morphs are given in quotes, we connect all their words into
717 * the OP_PHRASE sequence.
718 */
719 data.qoperator = OP_PHRASE;
720
721 query = parse_tsquery(text_to_cstring(in),
722 pushval_morph,
723 PointerGetDatum(&data),
724 P_TSQ_WEB,
725 NULL);
726
727 PG_RETURN_TSQUERY(query);
728 }
729
730 Datum
731 websearch_to_tsquery(PG_FUNCTION_ARGS)
732 {
733 text *in = PG_GETARG_TEXT_PP(0);
734 Oid cfgId;
735
736 cfgId = getTSCurrentConfig(true);
737 PG_RETURN_DATUM(DirectFunctionCall2(websearch_to_tsquery_byid,
738 ObjectIdGetDatum(cfgId),
739 PointerGetDatum(in)));
740 }