Debian Codegraph @

src/backend/utils/adt/tsvector_op.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * tsvector_op.c
4 * operations over tsvector
5 *
6 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
7 *
8 *
9 * IDENTIFICATION
10 * src/backend/utils/adt/tsvector_op.c
11 *
12 *-------------------------------------------------------------------------
13 */
14 #include "postgres.h"
15
16 #include <limits.h>
17
18 #include "access/htup_details.h"
19 #include "catalog/namespace.h"
20 #include "catalog/pg_type.h"
21 #include "commands/trigger.h"
22 #include "common/int.h"
23 #include "executor/spi.h"
24 #include "funcapi.h"
25 #include "lib/qunique.h"
26 #include "mb/pg_wchar.h"
27 #include "miscadmin.h"
28 #include "parser/parse_coerce.h"
29 #include "tsearch/ts_utils.h"
30 #include "utils/array.h"
31 #include "utils/builtins.h"
32 #include "utils/regproc.h"
33 #include "utils/rel.h"
34
35
36 typedef struct
37 {
38 WordEntry *arrb;
39 WordEntry *arre;
40 char *values;
41 char *operand;
42 } CHKVAL;
43
44
45 typedef struct StatEntry
46 {
47 uint32 ndoc; /* zero indicates that we were already here
48 * while walking through the tree */
49 uint32 nentry;
50 struct StatEntry *left;
51 struct StatEntry *right;
52 uint32 lenlexeme;
53 char lexeme[FLEXIBLE_ARRAY_MEMBER];
54 } StatEntry;
55
56 #define STATENTRYHDRSZ (offsetof(StatEntry, lexeme))
57
58 typedef struct
59 {
60 int32 weight;
61
62 uint32 maxdepth;
63
64 StatEntry **stack;
65 uint32 stackpos;
66
67 StatEntry *root;
68 } TSVectorStat;
69
70
71 static TSTernaryValue TS_execute_recurse(QueryItem *curitem, void *arg,
72 uint32 flags,
73 TSExecuteCallback chkcond);
74 static bool TS_execute_locations_recurse(QueryItem *curitem,
75 void *arg,
76 TSExecuteCallback chkcond,
77 List **locations);
78 static int tsvector_bsearch(const TSVector tsv, char *lexeme, int lexeme_len);
79 static Datum tsvector_update_trigger(PG_FUNCTION_ARGS, bool config_column);
80
81
82 /*
83 * Order: haspos, len, word, for all positions (pos, weight)
84 */
85 static int
86 silly_cmp_tsvector(const TSVector a, const TSVector b)
87 {
88 if (VARSIZE(a) < VARSIZE(b))
89 return -1;
90 else if (VARSIZE(a) > VARSIZE(b))
91 return 1;
92 else if (a->size < b->size)
93 return -1;
94 else if (a->size > b->size)
95 return 1;
96 else
97 {
98 WordEntry *aptr = ARRPTR(a);
99 WordEntry *bptr = ARRPTR(b);
100 int i = 0;
101 int res;
102
103
104 for (i = 0; i < a->size; i++)
105 {
106 if (aptr->haspos != bptr->haspos)
107 {
108 return (aptr->haspos > bptr->haspos) ? -1 : 1;
109 }
110 else if ((res = tsCompareString(STRPTR(a) + aptr->pos, aptr->len, STRPTR(b) + bptr->pos, bptr->len, false)) != 0)
111 {
112 return res;
113 }
114 else if (aptr->haspos)
115 {
116 WordEntryPos *ap = POSDATAPTR(a, aptr);
117 WordEntryPos *bp = POSDATAPTR(b, bptr);
118 int j;
119
120 if (POSDATALEN(a, aptr) != POSDATALEN(b, bptr))
121 return (POSDATALEN(a, aptr) > POSDATALEN(b, bptr)) ? -1 : 1;
122
123 for (j = 0; j < POSDATALEN(a, aptr); j++)
124 {
125 if (WEP_GETPOS(*ap) != WEP_GETPOS(*bp))
126 {
127 return (WEP_GETPOS(*ap) > WEP_GETPOS(*bp)) ? -1 : 1;
128 }
129 else if (WEP_GETWEIGHT(*ap) != WEP_GETWEIGHT(*bp))
130 {
131 return (WEP_GETWEIGHT(*ap) > WEP_GETWEIGHT(*bp)) ? -1 : 1;
132 }
133 ap++, bp++;
134 }
135 }
136
137 aptr++;
138 bptr++;
139 }
140 }
141
142 return 0;
143 }
144
145 #define TSVECTORCMPFUNC( type, action, ret ) \
146 Datum \
147 tsvector_##type(PG_FUNCTION_ARGS) \
148 { \
149 TSVector a = PG_GETARG_TSVECTOR(0); \
150 TSVector b = PG_GETARG_TSVECTOR(1); \
151 int res = silly_cmp_tsvector(a, b); \
152 PG_FREE_IF_COPY(a,0); \
153 PG_FREE_IF_COPY(b,1); \
154 PG_RETURN_##ret( res action 0 ); \
155 } \
156 /* keep compiler quiet - no extra ; */ \
157 extern int no_such_variable
158
159 TSVECTORCMPFUNC(lt, <, BOOL);
160 TSVECTORCMPFUNC(le, <=, BOOL);
161 TSVECTORCMPFUNC(eq, ==, BOOL);
162 TSVECTORCMPFUNC(ge, >=, BOOL);
163 TSVECTORCMPFUNC(gt, >, BOOL);
164 TSVECTORCMPFUNC(ne, !=, BOOL);
165 TSVECTORCMPFUNC(cmp, +, INT32);
166
167 Datum
168 tsvector_strip(PG_FUNCTION_ARGS)
169 {
170 TSVector in = PG_GETARG_TSVECTOR(0);
171 TSVector out;
172 int i,
173 len = 0;
174 WordEntry *arrin = ARRPTR(in),
175 *arrout;
176 char *cur;
177
178 /* Output can't be bigger than input, so no need for overflow checks */
179 for (i = 0; i < in->size; i++)
180 len += arrin[i].len;
181
182 len = CALCDATASIZE(in->size, len);
183 out = (TSVector) palloc0(len);
184 SET_VARSIZE(out, len);
185 out->size = in->size;
186 arrout = ARRPTR(out);
187 cur = STRPTR(out);
188 for (i = 0; i < in->size; i++)
189 {
190 memcpy(cur, STRPTR(in) + arrin[i].pos, arrin[i].len);
191 arrout[i].haspos = 0;
192 arrout[i].len = arrin[i].len;
193 arrout[i].pos = cur - STRPTR(out);
194 cur += arrout[i].len;
195 }
196
197 PG_FREE_IF_COPY(in, 0);
198 PG_RETURN_POINTER(out);
199 }
200
201 Datum
202 tsvector_length(PG_FUNCTION_ARGS)
203 {
204 TSVector in = PG_GETARG_TSVECTOR(0);
205 int32 ret = in->size;
206
207 PG_FREE_IF_COPY(in, 0);
208 PG_RETURN_INT32(ret);
209 }
210
211 static int
212 parse_weight(char cw)
213 {
214 int w;
215
216 switch (cw)
217 {
218 case 'A':
219 case 'a':
220 w = 3;
221 break;
222 case 'B':
223 case 'b':
224 w = 2;
225 break;
226 case 'C':
227 case 'c':
228 w = 1;
229 break;
230 case 'D':
231 case 'd':
232 w = 0;
233 break;
234 default:
235 /* Avoid printing non-ASCII bytes, else we have encoding issues */
236 if (cw >= ' ' && cw < 0x7f)
237 ereport(ERROR,
238 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
239 errmsg("unrecognized weight: \"%c\"", cw)));
240 else /* use \ooo format, like charout() */
241 ereport(ERROR,
242 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
243 errmsg("unrecognized weight: \"\\%03o\"",
244 (unsigned char) cw)));
245 }
246 return w;
247 }
248
249
250 Datum
251 tsvector_setweight(PG_FUNCTION_ARGS)
252 {
253 TSVector in = PG_GETARG_TSVECTOR(0);
254 char cw = PG_GETARG_CHAR(1);
255 TSVector out;
256 int i,
257 j;
258 WordEntry *entry;
259 WordEntryPos *p;
260 int w = parse_weight(cw);
261
262 out = (TSVector) palloc(VARSIZE(in));
263 memcpy(out, in, VARSIZE(in));
264 entry = ARRPTR(out);
265 i = out->size;
266 while (i--)
267 {
268 if ((j = POSDATALEN(out, entry)) != 0)
269 {
270 p = POSDATAPTR(out, entry);
271 while (j--)
272 {
273 WEP_SETWEIGHT(*p, w);
274 p++;
275 }
276 }
277 entry++;
278 }
279
280 PG_FREE_IF_COPY(in, 0);
281 PG_RETURN_POINTER(out);
282 }
283
284 /*
285 * setweight(tsin tsvector, char_weight "char", lexemes "text"[])
286 *
287 * Assign weight w to elements of tsin that are listed in lexemes.
288 */
289 Datum
290 tsvector_setweight_by_filter(PG_FUNCTION_ARGS)
291 {
292 TSVector tsin = PG_GETARG_TSVECTOR(0);
293 char char_weight = PG_GETARG_CHAR(1);
294 ArrayType *lexemes = PG_GETARG_ARRAYTYPE_P(2);
295
296 TSVector tsout;
297 int i,
298 j,
299 nlexemes,
300 weight;
301 WordEntry *entry;
302 Datum *dlexemes;
303 bool *nulls;
304
305 weight = parse_weight(char_weight);
306
307 tsout = (TSVector) palloc(VARSIZE(tsin));
308 memcpy(tsout, tsin, VARSIZE(tsin));
309 entry = ARRPTR(tsout);
310
311 deconstruct_array_builtin(lexemes, TEXTOID, &dlexemes, &nulls, &nlexemes);
312
313 /*
314 * Assuming that lexemes array is significantly shorter than tsvector we
315 * can iterate through lexemes performing binary search of each lexeme
316 * from lexemes in tsvector.
317 */
318 for (i = 0; i < nlexemes; i++)
319 {
320 char *lex;
321 int lex_len,
322 lex_pos;
323
324 /* Ignore null array elements, they surely don't match */
325 if (nulls[i])
326 continue;
327
328 lex = VARDATA(dlexemes[i]);
329 lex_len = VARSIZE(dlexemes[i]) - VARHDRSZ;
330 lex_pos = tsvector_bsearch(tsout, lex, lex_len);
331
332 if (lex_pos >= 0 && (j = POSDATALEN(tsout, entry + lex_pos)) != 0)
333 {
334 WordEntryPos *p = POSDATAPTR(tsout, entry + lex_pos);
335
336 while (j--)
337 {
338 WEP_SETWEIGHT(*p, weight);
339 p++;
340 }
341 }
342 }
343
344 PG_FREE_IF_COPY(tsin, 0);
345 PG_FREE_IF_COPY(lexemes, 2);
346
347 PG_RETURN_POINTER(tsout);
348 }
349
350 #define compareEntry(pa, a, pb, b) \
351 tsCompareString((pa) + (a)->pos, (a)->len, \
352 (pb) + (b)->pos, (b)->len, \
353 false)
354
355 /*
356 * Add positions from src to dest after offsetting them by maxpos.
357 * Return the number added (might be less than expected due to overflow)
358 */
359 static int32
360 add_pos(TSVector src, WordEntry *srcptr,
361 TSVector dest, WordEntry *destptr,
362 int32 maxpos)
363 {
364 uint16 *clen = &_POSVECPTR(dest, destptr)->npos;
365 int i;
366 uint16 slen = POSDATALEN(src, srcptr),
367 startlen;
368 WordEntryPos *spos = POSDATAPTR(src, srcptr),
369 *dpos = POSDATAPTR(dest, destptr);
370
371 if (!destptr->haspos)
372 *clen = 0;
373
374 startlen = *clen;
375 for (i = 0;
376 i < slen && *clen < MAXNUMPOS &&
377 (*clen == 0 || WEP_GETPOS(dpos[*clen - 1]) != MAXENTRYPOS - 1);
378 i++)
379 {
380 WEP_SETWEIGHT(dpos[*clen], WEP_GETWEIGHT(spos[i]));
381 WEP_SETPOS(dpos[*clen], LIMITPOS(WEP_GETPOS(spos[i]) + maxpos));
382 (*clen)++;
383 }
384
385 if (*clen != startlen)
386 destptr->haspos = 1;
387 return *clen - startlen;
388 }
389
390 /*
391 * Perform binary search of given lexeme in TSVector.
392 * Returns lexeme position in TSVector's entry array or -1 if lexeme wasn't
393 * found.
394 */
395 static int
396 tsvector_bsearch(const TSVector tsv, char *lexeme, int lexeme_len)
397 {
398 WordEntry *arrin = ARRPTR(tsv);
399 int StopLow = 0,
400 StopHigh = tsv->size,
401 StopMiddle,
402 cmp;
403
404 while (StopLow < StopHigh)
405 {
406 StopMiddle = (StopLow + StopHigh) / 2;
407
408 cmp = tsCompareString(lexeme, lexeme_len,
409 STRPTR(tsv) + arrin[StopMiddle].pos,
410 arrin[StopMiddle].len,
411 false);
412
413 if (cmp < 0)
414 StopHigh = StopMiddle;
415 else if (cmp > 0)
416 StopLow = StopMiddle + 1;
417 else /* found it */
418 return StopMiddle;
419 }
420
421 return -1;
422 }
423
424 /*
425 * qsort comparator functions
426 */
427
428 static int
429 compare_int(const void *va, const void *vb)
430 {
431 int a = *((const int *) va);
432 int b = *((const int *) vb);
433
434 return pg_cmp_s32(a, b);
435 }
436
437 static int
438 compare_text_lexemes(const void *va, const void *vb)
439 {
440 Datum a = *((const Datum *) va);
441 Datum b = *((const Datum *) vb);
442 char *alex = VARDATA_ANY(a);
443 int alex_len = VARSIZE_ANY_EXHDR(a);
444 char *blex = VARDATA_ANY(b);
445 int blex_len = VARSIZE_ANY_EXHDR(b);
446
447 return tsCompareString(alex, alex_len, blex, blex_len, false);
448 }
449
450 /*
451 * Internal routine to delete lexemes from TSVector by array of offsets.
452 *
453 * int *indices_to_delete -- array of lexeme offsets to delete (modified here!)
454 * int indices_count -- size of that array
455 *
456 * Returns new TSVector without given lexemes along with their positions
457 * and weights.
458 */
459 static TSVector
460 tsvector_delete_by_indices(TSVector tsv, int *indices_to_delete,
461 int indices_count)
462 {
463 TSVector tsout;
464 WordEntry *arrin = ARRPTR(tsv),
465 *arrout;
466 char *data = STRPTR(tsv),
467 *dataout;
468 int i, /* index in arrin */
469 j, /* index in arrout */
470 k, /* index in indices_to_delete */
471 curoff; /* index in dataout area */
472
473 /*
474 * Sort the filter array to simplify membership checks below. Also, get
475 * rid of any duplicate entries, so that we can assume that indices_count
476 * is exactly equal to the number of lexemes that will be removed.
477 */
478 if (indices_count > 1)
479 {
480 qsort(indices_to_delete, indices_count, sizeof(int), compare_int);
481 indices_count = qunique(indices_to_delete, indices_count, sizeof(int),
482 compare_int);
483 }
484
485 /*
486 * Here we overestimate tsout size, since we don't know how much space is
487 * used by the deleted lexeme(s). We will set exact size below.
488 */
489 tsout = (TSVector) palloc0(VARSIZE(tsv));
490
491 /* This count must be correct because STRPTR(tsout) relies on it. */
492 tsout->size = tsv->size - indices_count;
493
494 /*
495 * Copy tsv to tsout, skipping lexemes listed in indices_to_delete.
496 *
497 * Output can't be bigger than input, so no need for overflow checks.
498 */
499 arrout = ARRPTR(tsout);
500 dataout = STRPTR(tsout);
501 curoff = 0;
502 for (i = j = k = 0; i < tsv->size; i++)
503 {
504 /*
505 * If current i is present in indices_to_delete, skip this lexeme.
506 * Since indices_to_delete is already sorted, we only need to check
507 * the current (k'th) entry.
508 */
509 if (k < indices_count && i == indices_to_delete[k])
510 {
511 k++;
512 continue;
513 }
514
515 /* Copy lexeme and its positions and weights */
516 memcpy(dataout + curoff, data + arrin[i].pos, arrin[i].len);
517 arrout[j].haspos = arrin[i].haspos;
518 arrout[j].len = arrin[i].len;
519 arrout[j].pos = curoff;
520 curoff += arrin[i].len;
521 if (arrin[i].haspos)
522 {
523 int len = POSDATALEN(tsv, arrin + i) * sizeof(WordEntryPos)
524 + sizeof(uint16);
525
526 curoff = SHORTALIGN(curoff);
527 memcpy(dataout + curoff,
528 STRPTR(tsv) + SHORTALIGN(arrin[i].pos + arrin[i].len),
529 len);
530 curoff += len;
531 }
532
533 j++;
534 }
535
536 /*
537 * k should now be exactly equal to indices_count. If it isn't then the
538 * caller provided us with indices outside of [0, tsv->size) range and
539 * estimation of tsout's size is wrong.
540 */
541 Assert(k == indices_count);
542
543 SET_VARSIZE(tsout, CALCDATASIZE(tsout->size, curoff));
544 return tsout;
545 }
546
547 /*
548 * Delete given lexeme from tsvector.
549 * Implementation of user-level ts_delete(tsvector, text).
550 */
551 Datum
552 tsvector_delete_str(PG_FUNCTION_ARGS)
553 {
554 TSVector tsin = PG_GETARG_TSVECTOR(0),
555 tsout;
556 text *tlexeme = PG_GETARG_TEXT_PP(1);
557 char *lexeme = VARDATA_ANY(tlexeme);
558 int lexeme_len = VARSIZE_ANY_EXHDR(tlexeme),
559 skip_index;
560
561 if ((skip_index = tsvector_bsearch(tsin, lexeme, lexeme_len)) == -1)
562 PG_RETURN_POINTER(tsin);
563
564 tsout = tsvector_delete_by_indices(tsin, &skip_index, 1);
565
566 PG_FREE_IF_COPY(tsin, 0);
567 PG_FREE_IF_COPY(tlexeme, 1);
568 PG_RETURN_POINTER(tsout);
569 }
570
571 /*
572 * Delete given array of lexemes from tsvector.
573 * Implementation of user-level ts_delete(tsvector, text[]).
574 */
575 Datum
576 tsvector_delete_arr(PG_FUNCTION_ARGS)
577 {
578 TSVector tsin = PG_GETARG_TSVECTOR(0),
579 tsout;
580 ArrayType *lexemes = PG_GETARG_ARRAYTYPE_P(1);
581 int i,
582 nlex,
583 skip_count,
584 *skip_indices;
585 Datum *dlexemes;
586 bool *nulls;
587
588 deconstruct_array_builtin(lexemes, TEXTOID, &dlexemes, &nulls, &nlex);
589
590 /*
591 * In typical use case array of lexemes to delete is relatively small. So
592 * here we optimize things for that scenario: iterate through lexarr
593 * performing binary search of each lexeme from lexarr in tsvector.
594 */
595 skip_indices = palloc0(nlex * sizeof(int));
596 for (i = skip_count = 0; i < nlex; i++)
597 {
598 char *lex;
599 int lex_len,
600 lex_pos;
601
602 /* Ignore null array elements, they surely don't match */
603 if (nulls[i])
604 continue;
605
606 lex = VARDATA(dlexemes[i]);
607 lex_len = VARSIZE(dlexemes[i]) - VARHDRSZ;
608 lex_pos = tsvector_bsearch(tsin, lex, lex_len);
609
610 if (lex_pos >= 0)
611 skip_indices[skip_count++] = lex_pos;
612 }
613
614 tsout = tsvector_delete_by_indices(tsin, skip_indices, skip_count);
615
616 pfree(skip_indices);
617 PG_FREE_IF_COPY(tsin, 0);
618 PG_FREE_IF_COPY(lexemes, 1);
619
620 PG_RETURN_POINTER(tsout);
621 }
622
623 /*
624 * Expand tsvector as table with following columns:
625 * lexeme: lexeme text
626 * positions: integer array of lexeme positions
627 * weights: char array of weights corresponding to positions
628 */
629 Datum
630 tsvector_unnest(PG_FUNCTION_ARGS)
631 {
632 FuncCallContext *funcctx;
633 TSVector tsin;
634
635 if (SRF_IS_FIRSTCALL())
636 {
637 MemoryContext oldcontext;
638 TupleDesc tupdesc;
639
640 funcctx = SRF_FIRSTCALL_INIT();
641 oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
642
643 tupdesc = CreateTemplateTupleDesc(3);
644 TupleDescInitEntry(tupdesc, (AttrNumber) 1, "lexeme",
645 TEXTOID, -1, 0);
646 TupleDescInitEntry(tupdesc, (AttrNumber) 2, "positions",
647 INT2ARRAYOID, -1, 0);
648 TupleDescInitEntry(tupdesc, (AttrNumber) 3, "weights",
649 TEXTARRAYOID, -1, 0);
650 if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
651 elog(ERROR, "return type must be a row type");
652 funcctx->tuple_desc = tupdesc;
653
654 funcctx->user_fctx = PG_GETARG_TSVECTOR_COPY(0);
655
656 MemoryContextSwitchTo(oldcontext);
657 }
658
659 funcctx = SRF_PERCALL_SETUP();
660 tsin = (TSVector) funcctx->user_fctx;
661
662 if (funcctx->call_cntr < tsin->size)
663 {
664 WordEntry *arrin = ARRPTR(tsin);
665 char *data = STRPTR(tsin);
666 HeapTuple tuple;
667 int j,
668 i = funcctx->call_cntr;
669 bool nulls[] = {false, false, false};
670 Datum values[3];
671
672 values[0] = PointerGetDatum(cstring_to_text_with_len(data + arrin[i].pos, arrin[i].len));
673
674 if (arrin[i].haspos)
675 {
676 WordEntryPosVector *posv;
677 Datum *positions;
678 Datum *weights;
679 char weight;
680
681 /*
682 * Internally tsvector stores position and weight in the same
683 * uint16 (2 bits for weight, 14 for position). Here we extract
684 * that in two separate arrays.
685 */
686 posv = _POSVECPTR(tsin, arrin + i);
687 positions = palloc(posv->npos * sizeof(Datum));
688 weights = palloc(posv->npos * sizeof(Datum));
689 for (j = 0; j < posv->npos; j++)
690 {
691 positions[j] = Int16GetDatum(WEP_GETPOS(posv->pos[j]));
692 weight = 'D' - WEP_GETWEIGHT(posv->pos[j]);
693 weights[j] = PointerGetDatum(cstring_to_text_with_len(&weight,
694 1));
695 }
696
697 values[1] = PointerGetDatum(construct_array_builtin(positions, posv->npos, INT2OID));
698 values[2] = PointerGetDatum(construct_array_builtin(weights, posv->npos, TEXTOID));
699 }
700 else
701 {
702 nulls[1] = nulls[2] = true;
703 }
704
705 tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
706 SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple));
707 }
708 else
709 {
710 SRF_RETURN_DONE(funcctx);
711 }
712 }
713
714 /*
715 * Convert tsvector to array of lexemes.
716 */
717 Datum
718 tsvector_to_array(PG_FUNCTION_ARGS)
719 {
720 TSVector tsin = PG_GETARG_TSVECTOR(0);
721 WordEntry *arrin = ARRPTR(tsin);
722 Datum *elements;
723 int i;
724 ArrayType *array;
725
726 elements = palloc_array(Datum, tsin->size);
727
728 for (i = 0; i < tsin->size; i++)
729 {
730 elements[i] = PointerGetDatum(cstring_to_text_with_len(STRPTR(tsin) + arrin[i].pos,
731 arrin[i].len));
732 }
733
734 array = construct_array_builtin(elements, tsin->size, TEXTOID);
735
736 pfree(elements);
737 PG_FREE_IF_COPY(tsin, 0);
738 PG_RETURN_POINTER(array);
739 }
740
741 /*
742 * Build tsvector from array of lexemes.
743 */
744 Datum
745 array_to_tsvector(PG_FUNCTION_ARGS)
746 {
747 ArrayType *v = PG_GETARG_ARRAYTYPE_P(0);
748 TSVector tsout;
749 Datum *dlexemes;
750 WordEntry *arrout;
751 bool *nulls;
752 int nitems,
753 i,
754 tslen,
755 datalen = 0;
756 char *cur;
757
758 deconstruct_array_builtin(v, TEXTOID, &dlexemes, &nulls, &nitems);
759
760 /*
761 * Reject nulls and zero-length or over-length strings (maybe we should
762 * just ignore them, instead?)
763 */
764 for (i = 0; i < nitems; i++)
765 {
766 int toklen;
767
768 if (nulls[i])
769 ereport(ERROR,
770 (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
771 errmsg("lexeme array may not contain nulls")));
772
773 toklen = VARSIZE(dlexemes[i]) - VARHDRSZ;
774 if (toklen == 0)
775 ereport(ERROR,
776 (errcode(ERRCODE_ZERO_LENGTH_CHARACTER_STRING),
777 errmsg("lexeme array may not contain empty strings")));
778 if (toklen >= MAXSTRLEN)
779 ereport(ERROR,
780 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
781 errmsg("word is too long (%d bytes, max %d bytes)",
782 toklen,
783 MAXSTRLEN - 1)));
784 }
785
786 /* Sort and de-dup, because this is required for a valid tsvector. */
787 if (nitems > 1)
788 {
789 qsort(dlexemes, nitems, sizeof(Datum), compare_text_lexemes);
790 nitems = qunique(dlexemes, nitems, sizeof(Datum),
791 compare_text_lexemes);
792 }
793
794 /* Calculate space needed for surviving lexemes. */
795 for (i = 0; i < nitems; i++)
796 datalen += VARSIZE(dlexemes[i]) - VARHDRSZ;
797 if (datalen > MAXSTRPOS)
798 ereport(ERROR,
799 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
800 errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)",
801 (size_t) datalen, (size_t) MAXSTRPOS)));
802 tslen = CALCDATASIZE(nitems, datalen);
803
804 /* Allocate and fill tsvector. */
805 tsout = (TSVector) palloc0(tslen);
806 SET_VARSIZE(tsout, tslen);
807 tsout->size = nitems;
808
809 arrout = ARRPTR(tsout);
810 cur = STRPTR(tsout);
811 for (i = 0; i < nitems; i++)
812 {
813 char *lex = VARDATA(dlexemes[i]);
814 int lex_len = VARSIZE(dlexemes[i]) - VARHDRSZ;
815
816 memcpy(cur, lex, lex_len);
817 arrout[i].haspos = 0;
818 arrout[i].len = lex_len;
819 arrout[i].pos = cur - STRPTR(tsout);
820 cur += lex_len;
821 }
822
823 PG_FREE_IF_COPY(v, 0);
824 PG_RETURN_POINTER(tsout);
825 }
826
827 /*
828 * ts_filter(): keep only lexemes with given weights in tsvector.
829 */
830 Datum
831 tsvector_filter(PG_FUNCTION_ARGS)
832 {
833 TSVector tsin = PG_GETARG_TSVECTOR(0),
834 tsout;
835 ArrayType *weights = PG_GETARG_ARRAYTYPE_P(1);
836 WordEntry *arrin = ARRPTR(tsin),
837 *arrout;
838 char *datain = STRPTR(tsin),
839 *dataout;
840 Datum *dweights;
841 bool *nulls;
842 int nweights;
843 int i,
844 j;
845 int cur_pos = 0;
846 char mask = 0;
847
848 deconstruct_array_builtin(weights, CHAROID, &dweights, &nulls, &nweights);
849
850 for (i = 0; i < nweights; i++)
851 {
852 char char_weight;
853
854 if (nulls[i])
855 ereport(ERROR,
856 (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
857 errmsg("weight array may not contain nulls")));
858
859 char_weight = DatumGetChar(dweights[i]);
860 mask |= 1 << parse_weight(char_weight);
861 }
862
863 /*
864 * The output tsvector might be smaller than the input, but it can't be
865 * bigger, so VARSIZE(tsin) is surely enough space. Also, we don't need
866 * to worry about overflows below.
867 */
868 tsout = (TSVector) palloc0(VARSIZE(tsin));
869 tsout->size = tsin->size;
870 arrout = ARRPTR(tsout);
871 /* worst-case location of output's lexemes; we may need to adjust below */
872 dataout = STRPTR(tsout);
873
874 for (i = j = 0; i < tsin->size; i++)
875 {
876 WordEntryPosVector *posvin,
877 *posvout;
878 int npos = 0;
879 int k;
880
881 if (!arrin[i].haspos)
882 continue;
883
884 posvin = _POSVECPTR(tsin, arrin + i);
885 posvout = (WordEntryPosVector *)
886 (dataout + SHORTALIGN(cur_pos + arrin[i].len));
887
888 for (k = 0; k < posvin->npos; k++)
889 {
890 if (mask & (1 << WEP_GETWEIGHT(posvin->pos[k])))
891 posvout->pos[npos++] = posvin->pos[k];
892 }
893
894 /* if no satisfactory positions found, skip lexeme */
895 if (!npos)
896 continue;
897
898 arrout[j].haspos = true;
899 arrout[j].len = arrin[i].len;
900 arrout[j].pos = cur_pos;
901
902 memcpy(dataout + cur_pos, datain + arrin[i].pos, arrin[i].len);
903 posvout->npos = npos;
904 cur_pos += SHORTALIGN(arrin[i].len);
905 cur_pos += POSDATALEN(tsout, arrout + j) * sizeof(WordEntryPos) +
906 sizeof(uint16);
907 j++;
908 }
909
910 tsout->size = j;
911 if (dataout != STRPTR(tsout))
912 memmove(STRPTR(tsout), dataout, cur_pos);
913
914 SET_VARSIZE(tsout, CALCDATASIZE(tsout->size, cur_pos));
915
916 PG_FREE_IF_COPY(tsin, 0);
917 PG_RETURN_POINTER(tsout);
918 }
919
920 Datum
921 tsvector_concat(PG_FUNCTION_ARGS)
922 {
923 TSVector in1 = PG_GETARG_TSVECTOR(0);
924 TSVector in2 = PG_GETARG_TSVECTOR(1);
925 TSVector out;
926 WordEntry *ptr;
927 WordEntry *ptr1,
928 *ptr2;
929 WordEntryPos *p;
930 int maxpos = 0,
931 i,
932 j,
933 i1,
934 i2,
935 dataoff,
936 output_bytes,
937 output_size;
938 char *data,
939 *data1,
940 *data2;
941
942 /* Get max position in in1; we'll need this to offset in2's positions */
943 ptr = ARRPTR(in1);
944 i = in1->size;
945 while (i--)
946 {
947 if ((j = POSDATALEN(in1, ptr)) != 0)
948 {
949 p = POSDATAPTR(in1, ptr);
950 while (j--)
951 {
952 if (WEP_GETPOS(*p) > maxpos)
953 maxpos = WEP_GETPOS(*p);
954 p++;
955 }
956 }
957 ptr++;
958 }
959
960 ptr1 = ARRPTR(in1);
961 ptr2 = ARRPTR(in2);
962 data1 = STRPTR(in1);
963 data2 = STRPTR(in2);
964 i1 = in1->size;
965 i2 = in2->size;
966
967 /*
968 * Conservative estimate of space needed. We might need all the data in
969 * both inputs, and conceivably add a pad byte before position data for
970 * each item where there was none before.
971 *
972 * Note: since the MAXSTRPOS limit constrains each input tsvector to be
973 * considerably less than MaxAllocSize, we don't need to worry about
974 * integer overflow here, nor in the data-copying steps below. We do need
975 * to enforce that the result meets the MAXSTRPOS limit, but we check that
976 * once at the end.
977 */
978 output_bytes = VARSIZE(in1) + VARSIZE(in2) + i1 + i2;
979
980 out = (TSVector) palloc0(output_bytes);
981 SET_VARSIZE(out, output_bytes);
982
983 /*
984 * We must make out->size valid so that STRPTR(out) is sensible. We'll
985 * collapse out any unused space at the end.
986 */
987 out->size = in1->size + in2->size;
988
989 ptr = ARRPTR(out);
990 data = STRPTR(out);
991 dataoff = 0;
992 while (i1 && i2)
993 {
994 int cmp = compareEntry(data1, ptr1, data2, ptr2);
995
996 if (cmp < 0)
997 { /* in1 first */
998 ptr->haspos = ptr1->haspos;
999 ptr->len = ptr1->len;
1000 memcpy(data + dataoff, data1 + ptr1->pos, ptr1->len);
1001 ptr->pos = dataoff;
1002 dataoff += ptr1->len;
1003 if (ptr->haspos)
1004 {
1005 dataoff = SHORTALIGN(dataoff);
1006 memcpy(data + dataoff, _POSVECPTR(in1, ptr1), POSDATALEN(in1, ptr1) * sizeof(WordEntryPos) + sizeof(uint16));
1007 dataoff += POSDATALEN(in1, ptr1) * sizeof(WordEntryPos) + sizeof(uint16);
1008 }
1009
1010 ptr++;
1011 ptr1++;
1012 i1--;
1013 }
1014 else if (cmp > 0)
1015 { /* in2 first */
1016 ptr->haspos = ptr2->haspos;
1017 ptr->len = ptr2->len;
1018 memcpy(data + dataoff, data2 + ptr2->pos, ptr2->len);
1019 ptr->pos = dataoff;
1020 dataoff += ptr2->len;
1021 if (ptr->haspos)
1022 {
1023 int addlen = add_pos(in2, ptr2, out, ptr, maxpos);
1024
1025 if (addlen == 0)
1026 ptr->haspos = 0;
1027 else
1028 {
1029 dataoff = SHORTALIGN(dataoff);
1030 dataoff += addlen * sizeof(WordEntryPos) + sizeof(uint16);
1031 }
1032 }
1033
1034 ptr++;
1035 ptr2++;
1036 i2--;
1037 }
1038 else
1039 {
1040 ptr->haspos = ptr1->haspos | ptr2->haspos;
1041 ptr->len = ptr1->len;
1042 memcpy(data + dataoff, data1 + ptr1->pos, ptr1->len);
1043 ptr->pos = dataoff;
1044 dataoff += ptr1->len;
1045 if (ptr->haspos)
1046 {
1047 if (ptr1->haspos)
1048 {
1049 dataoff = SHORTALIGN(dataoff);
1050 memcpy(data + dataoff, _POSVECPTR(in1, ptr1), POSDATALEN(in1, ptr1) * sizeof(WordEntryPos) + sizeof(uint16));
1051 dataoff += POSDATALEN(in1, ptr1) * sizeof(WordEntryPos) + sizeof(uint16);
1052 if (ptr2->haspos)
1053 dataoff += add_pos(in2, ptr2, out, ptr, maxpos) * sizeof(WordEntryPos);
1054 }
1055 else /* must have ptr2->haspos */
1056 {
1057 int addlen = add_pos(in2, ptr2, out, ptr, maxpos);
1058
1059 if (addlen == 0)
1060 ptr->haspos = 0;
1061 else
1062 {
1063 dataoff = SHORTALIGN(dataoff);
1064 dataoff += addlen * sizeof(WordEntryPos) + sizeof(uint16);
1065 }
1066 }
1067 }
1068
1069 ptr++;
1070 ptr1++;
1071 ptr2++;
1072 i1--;
1073 i2--;
1074 }
1075 }
1076
1077 while (i1)
1078 {
1079 ptr->haspos = ptr1->haspos;
1080 ptr->len = ptr1->len;
1081 memcpy(data + dataoff, data1 + ptr1->pos, ptr1->len);
1082 ptr->pos = dataoff;
1083 dataoff += ptr1->len;
1084 if (ptr->haspos)
1085 {
1086 dataoff = SHORTALIGN(dataoff);
1087 memcpy(data + dataoff, _POSVECPTR(in1, ptr1), POSDATALEN(in1, ptr1) * sizeof(WordEntryPos) + sizeof(uint16));
1088 dataoff += POSDATALEN(in1, ptr1) * sizeof(WordEntryPos) + sizeof(uint16);
1089 }
1090
1091 ptr++;
1092 ptr1++;
1093 i1--;
1094 }
1095
1096 while (i2)
1097 {
1098 ptr->haspos = ptr2->haspos;
1099 ptr->len = ptr2->len;
1100 memcpy(data + dataoff, data2 + ptr2->pos, ptr2->len);
1101 ptr->pos = dataoff;
1102 dataoff += ptr2->len;
1103 if (ptr->haspos)
1104 {
1105 int addlen = add_pos(in2, ptr2, out, ptr, maxpos);
1106
1107 if (addlen == 0)
1108 ptr->haspos = 0;
1109 else
1110 {
1111 dataoff = SHORTALIGN(dataoff);
1112 dataoff += addlen * sizeof(WordEntryPos) + sizeof(uint16);
1113 }
1114 }
1115
1116 ptr++;
1117 ptr2++;
1118 i2--;
1119 }
1120
1121 /*
1122 * Instead of checking each offset individually, we check for overflow of
1123 * pos fields once at the end.
1124 */
1125 if (dataoff > MAXSTRPOS)
1126 ereport(ERROR,
1127 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1128 errmsg("string is too long for tsvector (%zu bytes, max %zu bytes)",
1129 (size_t) dataoff, (size_t) MAXSTRPOS)));
1130
1131 /*
1132 * Adjust sizes (asserting that we didn't overrun the original estimates)
1133 * and collapse out any unused array entries.
1134 */
1135 output_size = ptr - ARRPTR(out);
1136 Assert(output_size <= out->size);
1137 out->size = output_size;
1138 if (data != STRPTR(out))
1139 memmove(STRPTR(out), data, dataoff);
1140 output_bytes = CALCDATASIZE(out->size, dataoff);
1141 Assert(output_bytes <= VARSIZE(out));
1142 SET_VARSIZE(out, output_bytes);
1143
1144 PG_FREE_IF_COPY(in1, 0);
1145 PG_FREE_IF_COPY(in2, 1);
1146 PG_RETURN_POINTER(out);
1147 }
1148
1149 /*
1150 * Compare two strings by tsvector rules.
1151 *
1152 * if prefix = true then it returns zero value iff b has prefix a
1153 */
1154 int32
1155 tsCompareString(char *a, int lena, char *b, int lenb, bool prefix)
1156 {
1157 int cmp;
1158
1159 if (lena == 0)
1160 {
1161 if (prefix)
1162 cmp = 0; /* empty string is prefix of anything */
1163 else
1164 cmp = (lenb > 0) ? -1 : 0;
1165 }
1166 else if (lenb == 0)
1167 {
1168 cmp = (lena > 0) ? 1 : 0;
1169 }
1170 else
1171 {
1172 cmp = memcmp(a, b, Min((unsigned int) lena, (unsigned int) lenb));
1173
1174 if (prefix)
1175 {
1176 if (cmp == 0 && lena > lenb)
1177 cmp = 1; /* a is longer, so not a prefix of b */
1178 }
1179 else if (cmp == 0 && lena != lenb)
1180 {
1181 cmp = (lena < lenb) ? -1 : 1;
1182 }
1183 }
1184
1185 return cmp;
1186 }
1187
1188 /*
1189 * Check weight info or/and fill 'data' with the required positions
1190 */
1191 static TSTernaryValue
1192 checkclass_str(CHKVAL *chkval, WordEntry *entry, QueryOperand *val,
1193 ExecPhraseData *data)
1194 {
1195 TSTernaryValue result = TS_NO;
1196
1197 Assert(data == NULL || data->npos == 0);
1198
1199 if (entry->haspos)
1200 {
1201 WordEntryPosVector *posvec;
1202
1203 /*
1204 * We can't use the _POSVECPTR macro here because the pointer to the
1205 * tsvector's lexeme storage is already contained in chkval->values.
1206 */
1207 posvec = (WordEntryPosVector *)
1208 (chkval->values + SHORTALIGN(entry->pos + entry->len));
1209
1210 if (val->weight && data)
1211 {
1212 WordEntryPos *posvec_iter = posvec->pos;
1213 WordEntryPos *dptr;
1214
1215 /*
1216 * Filter position information by weights
1217 */
1218 dptr = data->pos = palloc(sizeof(WordEntryPos) * posvec->npos);
1219 data->allocated = true;
1220
1221 /* Is there a position with a matching weight? */
1222 while (posvec_iter < posvec->pos + posvec->npos)
1223 {
1224 /* If true, append this position to the data->pos */
1225 if (val->weight & (1 << WEP_GETWEIGHT(*posvec_iter)))
1226 {
1227 *dptr = WEP_GETPOS(*posvec_iter);
1228 dptr++;
1229 }
1230
1231 posvec_iter++;
1232 }
1233
1234 data->npos = dptr - data->pos;
1235
1236 if (data->npos > 0)
1237 result = TS_YES;
1238 else
1239 {
1240 pfree(data->pos);
1241 data->pos = NULL;
1242 data->allocated = false;
1243 }
1244 }
1245 else if (val->weight)
1246 {
1247 WordEntryPos *posvec_iter = posvec->pos;
1248
1249 /* Is there a position with a matching weight? */
1250 while (posvec_iter < posvec->pos + posvec->npos)
1251 {
1252 if (val->weight & (1 << WEP_GETWEIGHT(*posvec_iter)))
1253 {
1254 result = TS_YES;
1255 break; /* no need to go further */
1256 }
1257
1258 posvec_iter++;
1259 }
1260 }
1261 else if (data)
1262 {
1263 data->npos = posvec->npos;
1264 data->pos = posvec->pos;
1265 data->allocated = false;
1266 result = TS_YES;
1267 }
1268 else
1269 {
1270 /* simplest case: no weight check, positions not needed */
1271 result = TS_YES;
1272 }
1273 }
1274 else
1275 {
1276 /*
1277 * Position info is lacking, so if the caller requires it, we can only
1278 * say that maybe there is a match.
1279 *
1280 * Notice, however, that we *don't* check val->weight here.
1281 * Historically, stripped tsvectors are considered to match queries
1282 * whether or not the query has a weight restriction; that's a little
1283 * dubious but we'll preserve the behavior.
1284 */
1285 if (data)
1286 result = TS_MAYBE;
1287 else
1288 result = TS_YES;
1289 }
1290
1291 return result;
1292 }
1293
1294 /*
1295 * TS_execute callback for matching a tsquery operand to plain tsvector data
1296 */
1297 static TSTernaryValue
1298 checkcondition_str(void *checkval, QueryOperand *val, ExecPhraseData *data)
1299 {
1300 CHKVAL *chkval = (CHKVAL *) checkval;
1301 WordEntry *StopLow = chkval->arrb;
1302 WordEntry *StopHigh = chkval->arre;
1303 WordEntry *StopMiddle = StopHigh;
1304 TSTernaryValue res = TS_NO;
1305
1306 /* Loop invariant: StopLow <= val < StopHigh */
1307 while (StopLow < StopHigh)
1308 {
1309 int difference;
1310
1311 StopMiddle = StopLow + (StopHigh - StopLow) / 2;
1312 difference = tsCompareString(chkval->operand + val->distance,
1313 val->length,
1314 chkval->values + StopMiddle->pos,
1315 StopMiddle->len,
1316 false);
1317
1318 if (difference == 0)
1319 {
1320 /* Check weight info & fill 'data' with positions */
1321 res = checkclass_str(chkval, StopMiddle, val, data);
1322 break;
1323 }
1324 else if (difference > 0)
1325 StopLow = StopMiddle + 1;
1326 else
1327 StopHigh = StopMiddle;
1328 }
1329
1330 /*
1331 * If it's a prefix search, we should also consider lexemes that the
1332 * search term is a prefix of (which will necessarily immediately follow
1333 * the place we found in the above loop). But we can skip them if there
1334 * was a definite match on the exact term AND the caller doesn't need
1335 * position info.
1336 */
1337 if (val->prefix && (res != TS_YES || data))
1338 {
1339 WordEntryPos *allpos = NULL;
1340 int npos = 0,
1341 totalpos = 0;
1342
1343 /* adjust start position for corner case */
1344 if (StopLow >= StopHigh)
1345 StopMiddle = StopHigh;
1346
1347 /* we don't try to re-use any data from the initial match */
1348 if (data)
1349 {
1350 if (data->allocated)
1351 pfree(data->pos);
1352 data->pos = NULL;
1353 data->allocated = false;
1354 data->npos = 0;
1355 }
1356 res = TS_NO;
1357
1358 while ((res != TS_YES || data) &&
1359 StopMiddle < chkval->arre &&
1360 tsCompareString(chkval->operand + val->distance,
1361 val->length,
1362 chkval->values + StopMiddle->pos,
1363 StopMiddle->len,
1364 true) == 0)
1365 {
1366 TSTernaryValue subres;
1367
1368 subres = checkclass_str(chkval, StopMiddle, val, data);
1369
1370 if (subres != TS_NO)
1371 {
1372 if (data)
1373 {
1374 /*
1375 * We need to join position information
1376 */
1377 if (subres == TS_MAYBE)
1378 {
1379 /*
1380 * No position info for this match, so we must report
1381 * MAYBE overall.
1382 */
1383 res = TS_MAYBE;
1384 /* forget any previous positions */
1385 npos = 0;
1386 /* don't leak storage */
1387 if (allpos)
1388 pfree(allpos);
1389 break;
1390 }
1391
1392 while (npos + data->npos > totalpos)
1393 {
1394 if (totalpos == 0)
1395 {
1396 totalpos = 256;
1397 allpos = palloc(sizeof(WordEntryPos) * totalpos);
1398 }
1399 else
1400 {
1401 totalpos *= 2;
1402 allpos = repalloc(allpos, sizeof(WordEntryPos) * totalpos);
1403 }
1404 }
1405
1406 memcpy(allpos + npos, data->pos, sizeof(WordEntryPos) * data->npos);
1407 npos += data->npos;
1408
1409 /* don't leak storage from individual matches */
1410 if (data->allocated)
1411 pfree(data->pos);
1412 data->pos = NULL;
1413 data->allocated = false;
1414 /* it's important to reset data->npos before next loop */
1415 data->npos = 0;
1416 }
1417 else
1418 {
1419 /* Don't need positions, just handle YES/MAYBE */
1420 if (subres == TS_YES || res == TS_NO)
1421 res = subres;
1422 }
1423 }
1424
1425 StopMiddle++;
1426 }
1427
1428 if (data && npos > 0)
1429 {
1430 /* Sort and make unique array of found positions */
1431 data->pos = allpos;
1432 qsort(data->pos, npos, sizeof(WordEntryPos), compareWordEntryPos);
1433 data->npos = qunique(data->pos, npos, sizeof(WordEntryPos),
1434 compareWordEntryPos);
1435 data->allocated = true;
1436 res = TS_YES;
1437 }
1438 }
1439
1440 return res;
1441 }
1442
1443 /*
1444 * Compute output position list for a tsquery operator in phrase mode.
1445 *
1446 * Merge the position lists in Ldata and Rdata as specified by "emit",
1447 * returning the result list into *data. The input position lists must be
1448 * sorted and unique, and the output will be as well.
1449 *
1450 * data: pointer to initially-all-zeroes output struct, or NULL
1451 * Ldata, Rdata: input position lists
1452 * emit: bitmask of TSPO_XXX flags
1453 * Loffset: offset to be added to Ldata positions before comparing/outputting
1454 * Roffset: offset to be added to Rdata positions before comparing/outputting
1455 * max_npos: maximum possible required size of output position array
1456 *
1457 * Loffset and Roffset should not be negative, else we risk trying to output
1458 * negative positions, which won't fit into WordEntryPos.
1459 *
1460 * The result is boolean (TS_YES or TS_NO), but for the caller's convenience
1461 * we return it as TSTernaryValue.
1462 *
1463 * Returns TS_YES if any positions were emitted to *data; or if data is NULL,
1464 * returns TS_YES if any positions would have been emitted.
1465 */
1466 #define TSPO_L_ONLY 0x01 /* emit positions appearing only in L */
1467 #define TSPO_R_ONLY 0x02 /* emit positions appearing only in R */
1468 #define TSPO_BOTH 0x04 /* emit positions appearing in both L&R */
1469
1470 static TSTernaryValue
1471 TS_phrase_output(ExecPhraseData *data,
1472 ExecPhraseData *Ldata,
1473 ExecPhraseData *Rdata,
1474 int emit,
1475 int Loffset,
1476 int Roffset,
1477 int max_npos)
1478 {
1479 int Lindex,
1480 Rindex;
1481
1482 /* Loop until both inputs are exhausted */
1483 Lindex = Rindex = 0;
1484 while (Lindex < Ldata->npos || Rindex < Rdata->npos)
1485 {
1486 int Lpos,
1487 Rpos;
1488 int output_pos = 0;
1489
1490 /*
1491 * Fetch current values to compare. WEP_GETPOS() is needed because
1492 * ExecPhraseData->data can point to a tsvector's WordEntryPosVector.
1493 */
1494 if (Lindex < Ldata->npos)
1495 Lpos = WEP_GETPOS(Ldata->pos[Lindex]) + Loffset;
1496 else
1497 {
1498 /* L array exhausted, so we're done if R_ONLY isn't set */
1499 if (!(emit & TSPO_R_ONLY))
1500 break;
1501 Lpos = INT_MAX;
1502 }
1503 if (Rindex < Rdata->npos)
1504 Rpos = WEP_GETPOS(Rdata->pos[Rindex]) + Roffset;
1505 else
1506 {
1507 /* R array exhausted, so we're done if L_ONLY isn't set */
1508 if (!(emit & TSPO_L_ONLY))
1509 break;
1510 Rpos = INT_MAX;
1511 }
1512
1513 /* Merge-join the two input lists */
1514 if (Lpos < Rpos)
1515 {
1516 /* Lpos is not matched in Rdata, should we output it? */
1517 if (emit & TSPO_L_ONLY)
1518 output_pos = Lpos;
1519 Lindex++;
1520 }
1521 else if (Lpos == Rpos)
1522 {
1523 /* Lpos and Rpos match ... should we output it? */
1524 if (emit & TSPO_BOTH)
1525 output_pos = Rpos;
1526 Lindex++;
1527 Rindex++;
1528 }
1529 else /* Lpos > Rpos */
1530 {
1531 /* Rpos is not matched in Ldata, should we output it? */
1532 if (emit & TSPO_R_ONLY)
1533 output_pos = Rpos;
1534 Rindex++;
1535 }
1536
1537 if (output_pos > 0)
1538 {
1539 if (data)
1540 {
1541 /* Store position, first allocating output array if needed */
1542 if (data->pos == NULL)
1543 {
1544 data->pos = (WordEntryPos *)
1545 palloc(max_npos * sizeof(WordEntryPos));
1546 data->allocated = true;
1547 }
1548 data->pos[data->npos++] = output_pos;
1549 }
1550 else
1551 {
1552 /*
1553 * Exact positions not needed, so return TS_YES as soon as we
1554 * know there is at least one.
1555 */
1556 return TS_YES;
1557 }
1558 }
1559 }
1560
1561 if (data && data->npos > 0)
1562 {
1563 /* Let's assert we didn't overrun the array */
1564 Assert(data->npos <= max_npos);
1565 return TS_YES;
1566 }
1567 return TS_NO;
1568 }
1569
1570 /*
1571 * Execute tsquery at or below an OP_PHRASE operator.
1572 *
1573 * This handles tsquery execution at recursion levels where we need to care
1574 * about match locations.
1575 *
1576 * In addition to the same arguments used for TS_execute, the caller may pass
1577 * a preinitialized-to-zeroes ExecPhraseData struct, to be filled with lexeme
1578 * match position info on success. data == NULL if no position data need be
1579 * returned.
1580 * Note: the function assumes data != NULL for operators other than OP_PHRASE.
1581 * This is OK because an outside call always starts from an OP_PHRASE node,
1582 * and all internal recursion cases pass data != NULL.
1583 *
1584 * The detailed semantics of the match data, given that the function returned
1585 * TS_YES (successful match), are:
1586 *
1587 * npos > 0, negate = false:
1588 * query is matched at specified position(s) (and only those positions)
1589 * npos > 0, negate = true:
1590 * query is matched at all positions *except* specified position(s)
1591 * npos = 0, negate = true:
1592 * query is matched at all positions
1593 * npos = 0, negate = false:
1594 * disallowed (this should result in TS_NO or TS_MAYBE, as appropriate)
1595 *
1596 * Successful matches also return a "width" value which is the match width in
1597 * lexemes, less one. Hence, "width" is zero for simple one-lexeme matches,
1598 * and is the sum of the phrase operator distances for phrase matches. Note
1599 * that when width > 0, the listed positions represent the ends of matches not
1600 * the starts. (This unintuitive rule is needed to avoid possibly generating
1601 * negative positions, which wouldn't fit into the WordEntryPos arrays.)
1602 *
1603 * If the TSExecuteCallback function reports that an operand is present
1604 * but fails to provide position(s) for it, we will return TS_MAYBE when
1605 * it is possible but not certain that the query is matched.
1606 *
1607 * When the function returns TS_NO or TS_MAYBE, it must return npos = 0,
1608 * negate = false (which is the state initialized by the caller); but the
1609 * "width" output in such cases is undefined.
1610 */
1611 static TSTernaryValue
1612 TS_phrase_execute(QueryItem *curitem, void *arg, uint32 flags,
1613 TSExecuteCallback chkcond,
1614 ExecPhraseData *data)
1615 {
1616 ExecPhraseData Ldata,
1617 Rdata;
1618 TSTernaryValue lmatch,
1619 rmatch;
1620 int Loffset,
1621 Roffset,
1622 maxwidth;
1623
1624 /* since this function recurses, it could be driven to stack overflow */
1625 check_stack_depth();
1626
1627 /* ... and let's check for query cancel while we're at it */
1628 CHECK_FOR_INTERRUPTS();
1629
1630 if (curitem->type == QI_VAL)
1631 return chkcond(arg, (QueryOperand *) curitem, data);
1632
1633 switch (curitem->qoperator.oper)
1634 {
1635 case OP_NOT:
1636
1637 /*
1638 * We need not touch data->width, since a NOT operation does not
1639 * change the match width.
1640 */
1641 if (flags & TS_EXEC_SKIP_NOT)
1642 {
1643 /* with SKIP_NOT, report NOT as "match everywhere" */
1644 Assert(data->npos == 0 && !data->negate);
1645 data->negate = true;
1646 return TS_YES;
1647 }
1648 switch (TS_phrase_execute(curitem + 1, arg, flags, chkcond, data))
1649 {
1650 case TS_NO:
1651 /* change "match nowhere" to "match everywhere" */
1652 Assert(data->npos == 0 && !data->negate);
1653 data->negate = true;
1654 return TS_YES;
1655 case TS_YES:
1656 if (data->npos > 0)
1657 {
1658 /* we have some positions, invert negate flag */
1659 data->negate = !data->negate;
1660 return TS_YES;
1661 }
1662 else if (data->negate)
1663 {
1664 /* change "match everywhere" to "match nowhere" */
1665 data->negate = false;
1666 return TS_NO;
1667 }
1668 /* Should not get here if result was TS_YES */
1669 Assert(false);
1670 break;
1671 case TS_MAYBE:
1672 /* match positions are, and remain, uncertain */
1673 return TS_MAYBE;
1674 }
1675 break;
1676
1677 case OP_PHRASE:
1678 case OP_AND:
1679 memset(&Ldata, 0, sizeof(Ldata));
1680 memset(&Rdata, 0, sizeof(Rdata));
1681
1682 lmatch = TS_phrase_execute(curitem + curitem->qoperator.left,
1683 arg, flags, chkcond, &Ldata);
1684 if (lmatch == TS_NO)
1685 return TS_NO;
1686
1687 rmatch = TS_phrase_execute(curitem + 1,
1688 arg, flags, chkcond, &Rdata);
1689 if (rmatch == TS_NO)
1690 return TS_NO;
1691
1692 /*
1693 * If either operand has no position information, then we can't
1694 * return reliable position data, only a MAYBE result.
1695 */
1696 if (lmatch == TS_MAYBE || rmatch == TS_MAYBE)
1697 return TS_MAYBE;
1698
1699 if (curitem->qoperator.oper == OP_PHRASE)
1700 {
1701 /*
1702 * Compute Loffset and Roffset suitable for phrase match, and
1703 * compute overall width of whole phrase match.
1704 */
1705 Loffset = curitem->qoperator.distance + Rdata.width;
1706 Roffset = 0;
1707 if (data)
1708 data->width = curitem->qoperator.distance +
1709 Ldata.width + Rdata.width;
1710 }
1711 else
1712 {
1713 /*
1714 * For OP_AND, set output width and alignment like OP_OR (see
1715 * comment below)
1716 */
1717 maxwidth = Max(Ldata.width, Rdata.width);
1718 Loffset = maxwidth - Ldata.width;
1719 Roffset = maxwidth - Rdata.width;
1720 if (data)
1721 data->width = maxwidth;
1722 }
1723
1724 if (Ldata.negate && Rdata.negate)
1725 {
1726 /* !L & !R: treat as !(L | R) */
1727 (void) TS_phrase_output(data, &Ldata, &Rdata,
1728 TSPO_BOTH | TSPO_L_ONLY | TSPO_R_ONLY,
1729 Loffset, Roffset,
1730 Ldata.npos + Rdata.npos);
1731 if (data)
1732 data->negate = true;
1733 return TS_YES;
1734 }
1735 else if (Ldata.negate)
1736 {
1737 /* !L & R */
1738 return TS_phrase_output(data, &Ldata, &Rdata,
1739 TSPO_R_ONLY,
1740 Loffset, Roffset,
1741 Rdata.npos);
1742 }
1743 else if (Rdata.negate)
1744 {
1745 /* L & !R */
1746 return TS_phrase_output(data, &Ldata, &Rdata,
1747 TSPO_L_ONLY,
1748 Loffset, Roffset,
1749 Ldata.npos);
1750 }
1751 else
1752 {
1753 /* straight AND */
1754 return TS_phrase_output(data, &Ldata, &Rdata,
1755 TSPO_BOTH,
1756 Loffset, Roffset,
1757 Min(Ldata.npos, Rdata.npos));
1758 }
1759
1760 case OP_OR:
1761 memset(&Ldata, 0, sizeof(Ldata));
1762 memset(&Rdata, 0, sizeof(Rdata));
1763
1764 lmatch = TS_phrase_execute(curitem + curitem->qoperator.left,
1765 arg, flags, chkcond, &Ldata);
1766 rmatch = TS_phrase_execute(curitem + 1,
1767 arg, flags, chkcond, &Rdata);
1768
1769 if (lmatch == TS_NO && rmatch == TS_NO)
1770 return TS_NO;
1771
1772 /*
1773 * If either operand has no position information, then we can't
1774 * return reliable position data, only a MAYBE result.
1775 */
1776 if (lmatch == TS_MAYBE || rmatch == TS_MAYBE)
1777 return TS_MAYBE;
1778
1779 /*
1780 * Cope with undefined output width from failed submatch. (This
1781 * takes less code than trying to ensure that all failure returns
1782 * set data->width to zero.)
1783 */
1784 if (lmatch == TS_NO)
1785 Ldata.width = 0;
1786 if (rmatch == TS_NO)
1787 Rdata.width = 0;
1788
1789 /*
1790 * For OP_AND and OP_OR, report the width of the wider of the two
1791 * inputs, and align the narrower input's positions to the right
1792 * end of that width. This rule deals at least somewhat
1793 * reasonably with cases like "x <-> (y | z <-> q)".
1794 */
1795 maxwidth = Max(Ldata.width, Rdata.width);
1796 Loffset = maxwidth - Ldata.width;
1797 Roffset = maxwidth - Rdata.width;
1798 data->width = maxwidth;
1799
1800 if (Ldata.negate && Rdata.negate)
1801 {
1802 /* !L | !R: treat as !(L & R) */
1803 (void) TS_phrase_output(data, &Ldata, &Rdata,
1804 TSPO_BOTH,
1805 Loffset, Roffset,
1806 Min(Ldata.npos, Rdata.npos));
1807 data->negate = true;
1808 return TS_YES;
1809 }
1810 else if (Ldata.negate)
1811 {
1812 /* !L | R: treat as !(L & !R) */
1813 (void) TS_phrase_output(data, &Ldata, &Rdata,
1814 TSPO_L_ONLY,
1815 Loffset, Roffset,
1816 Ldata.npos);
1817 data->negate = true;
1818 return TS_YES;
1819 }
1820 else if (Rdata.negate)
1821 {
1822 /* L | !R: treat as !(!L & R) */
1823 (void) TS_phrase_output(data, &Ldata, &Rdata,
1824 TSPO_R_ONLY,
1825 Loffset, Roffset,
1826 Rdata.npos);
1827 data->negate = true;
1828 return TS_YES;
1829 }
1830 else
1831 {
1832 /* straight OR */
1833 return TS_phrase_output(data, &Ldata, &Rdata,
1834 TSPO_BOTH | TSPO_L_ONLY | TSPO_R_ONLY,
1835 Loffset, Roffset,
1836 Ldata.npos + Rdata.npos);
1837 }
1838
1839 default:
1840 elog(ERROR, "unrecognized operator: %d", curitem->qoperator.oper);
1841 }
1842
1843 /* not reachable, but keep compiler quiet */
1844 return TS_NO;
1845 }
1846
1847
1848 /*
1849 * Evaluate tsquery boolean expression.
1850 *
1851 * curitem: current tsquery item (initially, the first one)
1852 * arg: opaque value to pass through to callback function
1853 * flags: bitmask of flag bits shown in ts_utils.h
1854 * chkcond: callback function to check whether a primitive value is present
1855 */
1856 bool
1857 TS_execute(QueryItem *curitem, void *arg, uint32 flags,
1858 TSExecuteCallback chkcond)
1859 {
1860 /*
1861 * If we get TS_MAYBE from the recursion, return true. We could only see
1862 * that result if the caller passed TS_EXEC_PHRASE_NO_POS, so there's no
1863 * need to check again.
1864 */
1865 return TS_execute_recurse(curitem, arg, flags, chkcond) != TS_NO;
1866 }
1867
1868 /*
1869 * Evaluate tsquery boolean expression.
1870 *
1871 * This is the same as TS_execute except that TS_MAYBE is returned as-is.
1872 */
1873 TSTernaryValue
1874 TS_execute_ternary(QueryItem *curitem, void *arg, uint32 flags,
1875 TSExecuteCallback chkcond)
1876 {
1877 return TS_execute_recurse(curitem, arg, flags, chkcond);
1878 }
1879
1880 /*
1881 * TS_execute recursion for operators above any phrase operator. Here we do
1882 * not need to worry about lexeme positions. As soon as we hit an OP_PHRASE
1883 * operator, we pass it off to TS_phrase_execute which does worry.
1884 */
1885 static TSTernaryValue
1886 TS_execute_recurse(QueryItem *curitem, void *arg, uint32 flags,
1887 TSExecuteCallback chkcond)
1888 {
1889 TSTernaryValue lmatch;
1890
1891 /* since this function recurses, it could be driven to stack overflow */
1892 check_stack_depth();
1893
1894 /* ... and let's check for query cancel while we're at it */
1895 CHECK_FOR_INTERRUPTS();
1896
1897 if (curitem->type == QI_VAL)
1898 return chkcond(arg, (QueryOperand *) curitem,
1899 NULL /* don't need position info */ );
1900
1901 switch (curitem->qoperator.oper)
1902 {
1903 case OP_NOT:
1904 if (flags & TS_EXEC_SKIP_NOT)
1905 return TS_YES;
1906 switch (TS_execute_recurse(curitem + 1, arg, flags, chkcond))
1907 {
1908 case TS_NO:
1909 return TS_YES;
1910 case TS_YES:
1911 return TS_NO;
1912 case TS_MAYBE:
1913 return TS_MAYBE;
1914 }
1915 break;
1916
1917 case OP_AND:
1918 lmatch = TS_execute_recurse(curitem + curitem->qoperator.left, arg,
1919 flags, chkcond);
1920 if (lmatch == TS_NO)
1921 return TS_NO;
1922 switch (TS_execute_recurse(curitem + 1, arg, flags, chkcond))
1923 {
1924 case TS_NO:
1925 return TS_NO;
1926 case TS_YES:
1927 return lmatch;
1928 case TS_MAYBE:
1929 return TS_MAYBE;
1930 }
1931 break;
1932
1933 case OP_OR:
1934 lmatch = TS_execute_recurse(curitem + curitem->qoperator.left, arg,
1935 flags, chkcond);
1936 if (lmatch == TS_YES)
1937 return TS_YES;
1938 switch (TS_execute_recurse(curitem + 1, arg, flags, chkcond))
1939 {
1940 case TS_NO:
1941 return lmatch;
1942 case TS_YES:
1943 return TS_YES;
1944 case TS_MAYBE:
1945 return TS_MAYBE;
1946 }
1947 break;
1948
1949 case OP_PHRASE:
1950
1951 /*
1952 * If we get a MAYBE result, and the caller doesn't want that,
1953 * convert it to NO. It would be more consistent, perhaps, to
1954 * return the result of TS_phrase_execute() verbatim and then
1955 * convert MAYBE results at the top of the recursion. But
1956 * converting at the topmost phrase operator gives results that
1957 * are bug-compatible with the old implementation, so do it like
1958 * this for now.
1959 */
1960 switch (TS_phrase_execute(curitem, arg, flags, chkcond, NULL))
1961 {
1962 case TS_NO:
1963 return TS_NO;
1964 case TS_YES:
1965 return TS_YES;
1966 case TS_MAYBE:
1967 return (flags & TS_EXEC_PHRASE_NO_POS) ? TS_MAYBE : TS_NO;
1968 }
1969 break;
1970
1971 default:
1972 elog(ERROR, "unrecognized operator: %d", curitem->qoperator.oper);
1973 }
1974
1975 /* not reachable, but keep compiler quiet */
1976 return TS_NO;
1977 }
1978
1979 /*
1980 * Evaluate tsquery and report locations of matching terms.
1981 *
1982 * This is like TS_execute except that it returns match locations not just
1983 * success/failure status. The callback function is required to provide
1984 * position data (we report failure if it doesn't).
1985 *
1986 * On successful match, the result is a List of ExecPhraseData structs, one
1987 * for each AND'ed term or phrase operator in the query. Each struct includes
1988 * a sorted array of lexeme positions matching that term. (Recall that for
1989 * phrase operators, the match includes width+1 lexemes, and the recorded
1990 * position is that of the rightmost lexeme.)
1991 *
1992 * OR subexpressions are handled by union'ing their match locations into a
1993 * single List element, which is valid since any of those locations contains
1994 * a match. However, when some of the OR'ed terms are phrase operators, we
1995 * report the maximum width of any of the OR'ed terms, making such cases
1996 * slightly imprecise in the conservative direction. (For example, if the
1997 * tsquery is "(A <-> B) | C", an occurrence of C in the data would be
1998 * reported as though it includes the lexeme to the left of C.)
1999 *
2000 * Locations of NOT subexpressions are not reported. (Obviously, there can
2001 * be no successful NOT matches at top level, or the match would have failed.
2002 * So this amounts to ignoring NOTs underneath ORs.)
2003 *
2004 * The result is NIL if no match, or if position data was not returned.
2005 *
2006 * Arguments are the same as for TS_execute, although flags is currently
2007 * vestigial since none of the defined bits are sensible here.
2008 */
2009 List *
2010 TS_execute_locations(QueryItem *curitem, void *arg,
2011 uint32 flags,
2012 TSExecuteCallback chkcond)
2013 {
2014 List *result;
2015
2016 /* No flags supported, as yet */
2017 Assert(flags == TS_EXEC_EMPTY);
2018 if (TS_execute_locations_recurse(curitem, arg, chkcond, &result))
2019 return result;
2020 return NIL;
2021 }
2022
2023 /*
2024 * TS_execute_locations recursion for operators above any phrase operator.
2025 * OP_PHRASE subexpressions can be passed off to TS_phrase_execute.
2026 */
2027 static bool
2028 TS_execute_locations_recurse(QueryItem *curitem, void *arg,
2029 TSExecuteCallback chkcond,
2030 List **locations)
2031 {
2032 bool lmatch,
2033 rmatch;
2034 List *llocations,
2035 *rlocations;
2036 ExecPhraseData *data;
2037
2038 /* since this function recurses, it could be driven to stack overflow */
2039 check_stack_depth();
2040
2041 /* ... and let's check for query cancel while we're at it */
2042 CHECK_FOR_INTERRUPTS();
2043
2044 /* Default locations result is empty */
2045 *locations = NIL;
2046
2047 if (curitem->type == QI_VAL)
2048 {
2049 data = palloc0_object(ExecPhraseData);
2050 if (chkcond(arg, (QueryOperand *) curitem, data) == TS_YES)
2051 {
2052 *locations = list_make1(data);
2053 return true;
2054 }
2055 pfree(data);
2056 return false;
2057 }
2058
2059 switch (curitem->qoperator.oper)
2060 {
2061 case OP_NOT:
2062 if (!TS_execute_locations_recurse(curitem + 1, arg, chkcond,
2063 &llocations))
2064 return true; /* we don't pass back any locations */
2065 return false;
2066
2067 case OP_AND:
2068 if (!TS_execute_locations_recurse(curitem + curitem->qoperator.left,
2069 arg, chkcond,
2070 &llocations))
2071 return false;
2072 if (!TS_execute_locations_recurse(curitem + 1,
2073 arg, chkcond,
2074 &rlocations))
2075 return false;
2076 *locations = list_concat(llocations, rlocations);
2077 return true;
2078
2079 case OP_OR:
2080 lmatch = TS_execute_locations_recurse(curitem + curitem->qoperator.left,
2081 arg, chkcond,
2082 &llocations);
2083 rmatch = TS_execute_locations_recurse(curitem + 1,
2084 arg, chkcond,
2085 &rlocations);
2086 if (lmatch || rmatch)
2087 {
2088 /*
2089 * We generate an AND'able location struct from each
2090 * combination of sub-matches, following the disjunctive law
2091 * (A & B) | (C & D) = (A | C) & (A | D) & (B | C) & (B | D).
2092 *
2093 * However, if either input didn't produce locations (i.e., it
2094 * failed or was a NOT), we must just return the other list.
2095 */
2096 if (llocations == NIL)
2097 *locations = rlocations;
2098 else if (rlocations == NIL)
2099 *locations = llocations;
2100 else
2101 {
2102 ListCell *ll;
2103
2104 foreach(ll, llocations)
2105 {
2106 ExecPhraseData *ldata = (ExecPhraseData *) lfirst(ll);
2107 ListCell *lr;
2108
2109 foreach(lr, rlocations)
2110 {
2111 ExecPhraseData *rdata = (ExecPhraseData *) lfirst(lr);
2112
2113 data = palloc0_object(ExecPhraseData);
2114 (void) TS_phrase_output(data, ldata, rdata,
2115 TSPO_BOTH | TSPO_L_ONLY | TSPO_R_ONLY,
2116 0, 0,
2117 ldata->npos + rdata->npos);
2118 /* Report the larger width, as explained above. */
2119 data->width = Max(ldata->width, rdata->width);
2120 *locations = lappend(*locations, data);
2121 }
2122 }
2123 }
2124
2125 return true;
2126 }
2127 return false;
2128
2129 case OP_PHRASE:
2130 /* We can hand this off to TS_phrase_execute */
2131 data = palloc0_object(ExecPhraseData);
2132 if (TS_phrase_execute(curitem, arg, TS_EXEC_EMPTY, chkcond,
2133 data) == TS_YES)
2134 {
2135 if (!data->negate)
2136 *locations = list_make1(data);
2137 return true;
2138 }
2139 pfree(data);
2140 return false;
2141
2142 default:
2143 elog(ERROR, "unrecognized operator: %d", curitem->qoperator.oper);
2144 }
2145
2146 /* not reachable, but keep compiler quiet */
2147 return false;
2148 }
2149
2150 /*
2151 * Detect whether a tsquery boolean expression requires any positive matches
2152 * to values shown in the tsquery.
2153 *
2154 * This is needed to know whether a GIN index search requires full index scan.
2155 * For example, 'x & !y' requires a match of x, so it's sufficient to scan
2156 * entries for x; but 'x | !y' could match rows containing neither x nor y.
2157 */
2158 bool
2159 tsquery_requires_match(QueryItem *curitem)
2160 {
2161 /* since this function recurses, it could be driven to stack overflow */
2162 check_stack_depth();
2163
2164 if (curitem->type == QI_VAL)
2165 return true;
2166
2167 switch (curitem->qoperator.oper)
2168 {
2169 case OP_NOT:
2170
2171 /*
2172 * Assume there are no required matches underneath a NOT. For
2173 * some cases with nested NOTs, we could prove there's a required
2174 * match, but it seems unlikely to be worth the trouble.
2175 */
2176 return false;
2177
2178 case OP_PHRASE:
2179
2180 /*
2181 * Treat OP_PHRASE as OP_AND here
2182 */
2183 case OP_AND:
2184 /* If either side requires a match, we're good */
2185 if (tsquery_requires_match(curitem + curitem->qoperator.left))
2186 return true;
2187 else
2188 return tsquery_requires_match(curitem + 1);
2189
2190 case OP_OR:
2191 /* Both sides must require a match */
2192 if (tsquery_requires_match(curitem + curitem->qoperator.left))
2193 return tsquery_requires_match(curitem + 1);
2194 else
2195 return false;
2196
2197 default:
2198 elog(ERROR, "unrecognized operator: %d", curitem->qoperator.oper);
2199 }
2200
2201 /* not reachable, but keep compiler quiet */
2202 return false;
2203 }
2204
2205 /*
2206 * boolean operations
2207 */
2208 Datum
2209 ts_match_qv(PG_FUNCTION_ARGS)
2210 {
2211 PG_RETURN_DATUM(DirectFunctionCall2(ts_match_vq,
2212 PG_GETARG_DATUM(1),
2213 PG_GETARG_DATUM(0)));
2214 }
2215
2216 Datum
2217 ts_match_vq(PG_FUNCTION_ARGS)
2218 {
2219 TSVector val = PG_GETARG_TSVECTOR(0);
2220 TSQuery query = PG_GETARG_TSQUERY(1);
2221 CHKVAL chkval;
2222 bool result;
2223
2224 /* empty query matches nothing */
2225 if (!query->size)
2226 {
2227 PG_FREE_IF_COPY(val, 0);
2228 PG_FREE_IF_COPY(query, 1);
2229 PG_RETURN_BOOL(false);
2230 }
2231
2232 chkval.arrb = ARRPTR(val);
2233 chkval.arre = chkval.arrb + val->size;
2234 chkval.values = STRPTR(val);
2235 chkval.operand = GETOPERAND(query);
2236 result = TS_execute(GETQUERY(query),
2237 &chkval,
2238 TS_EXEC_EMPTY,
2239 checkcondition_str);
2240
2241 PG_FREE_IF_COPY(val, 0);
2242 PG_FREE_IF_COPY(query, 1);
2243 PG_RETURN_BOOL(result);
2244 }
2245
2246 Datum
2247 ts_match_tt(PG_FUNCTION_ARGS)
2248 {
2249 TSVector vector;
2250 TSQuery query;
2251 bool res;
2252
2253 vector = DatumGetTSVector(DirectFunctionCall1(to_tsvector,
2254 PG_GETARG_DATUM(0)));
2255 query = DatumGetTSQuery(DirectFunctionCall1(plainto_tsquery,
2256 PG_GETARG_DATUM(1)));
2257
2258 res = DatumGetBool(DirectFunctionCall2(ts_match_vq,
2259 TSVectorGetDatum(vector),
2260 TSQueryGetDatum(query)));
2261
2262 pfree(vector);
2263 pfree(query);
2264
2265 PG_RETURN_BOOL(res);
2266 }
2267
2268 Datum
2269 ts_match_tq(PG_FUNCTION_ARGS)
2270 {
2271 TSVector vector;
2272 TSQuery query = PG_GETARG_TSQUERY(1);
2273 bool res;
2274
2275 vector = DatumGetTSVector(DirectFunctionCall1(to_tsvector,
2276 PG_GETARG_DATUM(0)));
2277
2278 res = DatumGetBool(DirectFunctionCall2(ts_match_vq,
2279 TSVectorGetDatum(vector),
2280 TSQueryGetDatum(query)));
2281
2282 pfree(vector);
2283 PG_FREE_IF_COPY(query, 1);
2284
2285 PG_RETURN_BOOL(res);
2286 }
2287
2288 /*
2289 * ts_stat statistic function support
2290 */
2291
2292
2293 /*
2294 * Returns the number of positions in value 'wptr' within tsvector 'txt',
2295 * that have a weight equal to one of the weights in 'weight' bitmask.
2296 */
2297 static int
2298 check_weight(TSVector txt, WordEntry *wptr, int8 weight)
2299 {
2300 int len = POSDATALEN(txt, wptr);
2301 int num = 0;
2302 WordEntryPos *ptr = POSDATAPTR(txt, wptr);
2303
2304 while (len--)
2305 {
2306 if (weight & (1 << WEP_GETWEIGHT(*ptr)))
2307 num++;
2308 ptr++;
2309 }
2310 return num;
2311 }
2312
2313 #define compareStatWord(a,e,t) \
2314 tsCompareString((a)->lexeme, (a)->lenlexeme, \
2315 STRPTR(t) + (e)->pos, (e)->len, \
2316 false)
2317
2318 static void
2319 insertStatEntry(MemoryContext persistentContext, TSVectorStat *stat, TSVector txt, uint32 off)
2320 {
2321 WordEntry *we = ARRPTR(txt) + off;
2322 StatEntry *node = stat->root,
2323 *pnode = NULL;
2324 int n,
2325 res = 0;
2326 uint32 depth = 1;
2327
2328 if (stat->weight == 0)
2329 n = (we->haspos) ? POSDATALEN(txt, we) : 1;
2330 else
2331 n = (we->haspos) ? check_weight(txt, we, stat->weight) : 0;
2332
2333 if (n == 0)
2334 return; /* nothing to insert */
2335
2336 while (node)
2337 {
2338 res = compareStatWord(node, we, txt);
2339
2340 if (res == 0)
2341 {
2342 break;
2343 }
2344 else
2345 {
2346 pnode = node;
2347 node = (res < 0) ? node->left : node->right;
2348 }
2349 depth++;
2350 }
2351
2352 if (depth > stat->maxdepth)
2353 stat->maxdepth = depth;
2354
2355 if (node == NULL)
2356 {
2357 node = MemoryContextAlloc(persistentContext, STATENTRYHDRSZ + we->len);
2358 node->left = node->right = NULL;
2359 node->ndoc = 1;
2360 node->nentry = n;
2361 node->lenlexeme = we->len;
2362 memcpy(node->lexeme, STRPTR(txt) + we->pos, node->lenlexeme);
2363
2364 if (pnode == NULL)
2365 {
2366 stat->root = node;
2367 }
2368 else
2369 {
2370 if (res < 0)
2371 pnode->left = node;
2372 else
2373 pnode->right = node;
2374 }
2375 }
2376 else
2377 {
2378 node->ndoc++;
2379 node->nentry += n;
2380 }
2381 }
2382
2383 static void
2384 chooseNextStatEntry(MemoryContext persistentContext, TSVectorStat *stat, TSVector txt,
2385 uint32 low, uint32 high, uint32 offset)
2386 {
2387 uint32 pos;
2388 uint32 middle = (low + high) >> 1;
2389
2390 pos = (low + middle) >> 1;
2391 if (low != middle && pos >= offset && pos - offset < txt->size)
2392 insertStatEntry(persistentContext, stat, txt, pos - offset);
2393 pos = (high + middle + 1) >> 1;
2394 if (middle + 1 != high && pos >= offset && pos - offset < txt->size)
2395 insertStatEntry(persistentContext, stat, txt, pos - offset);
2396
2397 if (low != middle)
2398 chooseNextStatEntry(persistentContext, stat, txt, low, middle, offset);
2399 if (high != middle + 1)
2400 chooseNextStatEntry(persistentContext, stat, txt, middle + 1, high, offset);
2401 }
2402
2403 /*
2404 * This is written like a custom aggregate function, because the
2405 * original plan was to do just that. Unfortunately, an aggregate function
2406 * can't return a set, so that plan was abandoned. If that limitation is
2407 * lifted in the future, ts_stat could be a real aggregate function so that
2408 * you could use it like this:
2409 *
2410 * SELECT ts_stat(vector_column) FROM vector_table;
2411 *
2412 * where vector_column is a tsvector-type column in vector_table.
2413 */
2414
2415 static TSVectorStat *
2416 ts_accum(MemoryContext persistentContext, TSVectorStat *stat, Datum data)
2417 {
2418 TSVector txt = DatumGetTSVector(data);
2419 uint32 i,
2420 nbit = 0,
2421 offset;
2422
2423 if (stat == NULL)
2424 { /* Init in first */
2425 stat = MemoryContextAllocZero(persistentContext, sizeof(TSVectorStat));
2426 stat->maxdepth = 1;
2427 }
2428
2429 /* simple check of correctness */
2430 if (txt == NULL || txt->size == 0)
2431 {
2432 if (txt && txt != (TSVector) DatumGetPointer(data))
2433 pfree(txt);
2434 return stat;
2435 }
2436
2437 i = txt->size - 1;
2438 for (; i > 0; i >>= 1)
2439 nbit++;
2440
2441 nbit = 1 << nbit;
2442 offset = (nbit - txt->size) / 2;
2443
2444 insertStatEntry(persistentContext, stat, txt, (nbit >> 1) - offset);
2445 chooseNextStatEntry(persistentContext, stat, txt, 0, nbit, offset);
2446
2447 return stat;
2448 }
2449
2450 static void
2451 ts_setup_firstcall(FunctionCallInfo fcinfo, FuncCallContext *funcctx,
2452 TSVectorStat *stat)
2453 {
2454 TupleDesc tupdesc;
2455 MemoryContext oldcontext;
2456 StatEntry *node;
2457
2458 funcctx->user_fctx = stat;
2459
2460 oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
2461
2462 stat->stack = palloc0(sizeof(StatEntry *) * (stat->maxdepth + 1));
2463 stat->stackpos = 0;
2464
2465 node = stat->root;
2466 /* find leftmost value */
2467 if (node == NULL)
2468 stat->stack[stat->stackpos] = NULL;
2469 else
2470 for (;;)
2471 {
2472 stat->stack[stat->stackpos] = node;
2473 if (node->left)
2474 {
2475 stat->stackpos++;
2476 node = node->left;
2477 }
2478 else
2479 break;
2480 }
2481 Assert(stat->stackpos <= stat->maxdepth);
2482
2483 if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
2484 elog(ERROR, "return type must be a row type");
2485 funcctx->tuple_desc = tupdesc;
2486 funcctx->attinmeta = TupleDescGetAttInMetadata(tupdesc);
2487
2488 MemoryContextSwitchTo(oldcontext);
2489 }
2490
2491 static StatEntry *
2492 walkStatEntryTree(TSVectorStat *stat)
2493 {
2494 StatEntry *node = stat->stack[stat->stackpos];
2495
2496 if (node == NULL)
2497 return NULL;
2498
2499 if (node->ndoc != 0)
2500 {
2501 /* return entry itself: we already was at left sublink */
2502 return node;
2503 }
2504 else if (node->right && node->right != stat->stack[stat->stackpos + 1])
2505 {
2506 /* go on right sublink */
2507 stat->stackpos++;
2508 node = node->right;
2509
2510 /* find most-left value */
2511 for (;;)
2512 {
2513 stat->stack[stat->stackpos] = node;
2514 if (node->left)
2515 {
2516 stat->stackpos++;
2517 node = node->left;
2518 }
2519 else
2520 break;
2521 }
2522 Assert(stat->stackpos <= stat->maxdepth);
2523 }
2524 else
2525 {
2526 /* we already return all left subtree, itself and right subtree */
2527 if (stat->stackpos == 0)
2528 return NULL;
2529
2530 stat->stackpos--;
2531 return walkStatEntryTree(stat);
2532 }
2533
2534 return node;
2535 }
2536
2537 static Datum
2538 ts_process_call(FuncCallContext *funcctx)
2539 {
2540 TSVectorStat *st;
2541 StatEntry *entry;
2542
2543 st = (TSVectorStat *) funcctx->user_fctx;
2544
2545 entry = walkStatEntryTree(st);
2546
2547 if (entry != NULL)
2548 {
2549 Datum result;
2550 char *values[3];
2551 char ndoc[16];
2552 char nentry[16];
2553 HeapTuple tuple;
2554
2555 values[0] = palloc(entry->lenlexeme + 1);
2556 memcpy(values[0], entry->lexeme, entry->lenlexeme);
2557 (values[0])[entry->lenlexeme] = '\0';
2558 sprintf(ndoc, "%d", entry->ndoc);
2559 values[1] = ndoc;
2560 sprintf(nentry, "%d", entry->nentry);
2561 values[2] = nentry;
2562
2563 tuple = BuildTupleFromCStrings(funcctx->attinmeta, values);
2564 result = HeapTupleGetDatum(tuple);
2565
2566 pfree(values[0]);
2567
2568 /* mark entry as already visited */
2569 entry->ndoc = 0;
2570
2571 return result;
2572 }
2573
2574 return (Datum) 0;
2575 }
2576
2577 static TSVectorStat *
2578 ts_stat_sql(MemoryContext persistentContext, text *txt, text *ws)
2579 {
2580 char *query = text_to_cstring(txt);
2581 TSVectorStat *stat;
2582 bool isnull;
2583 Portal portal;
2584 SPIPlanPtr plan;
2585
2586 if ((plan = SPI_prepare(query, 0, NULL)) == NULL)
2587 /* internal error */
2588 elog(ERROR, "SPI_prepare(\"%s\") failed", query);
2589
2590 if ((portal = SPI_cursor_open(NULL, plan, NULL, NULL, true)) == NULL)
2591 /* internal error */
2592 elog(ERROR, "SPI_cursor_open(\"%s\") failed", query);
2593
2594 SPI_cursor_fetch(portal, true, 100);
2595
2596 if (SPI_tuptable == NULL ||
2597 SPI_tuptable->tupdesc->natts != 1 ||
2598 !IsBinaryCoercible(SPI_gettypeid(SPI_tuptable->tupdesc, 1),
2599 TSVECTOROID))
2600 ereport(ERROR,
2601 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2602 errmsg("ts_stat query must return one tsvector column")));
2603
2604 stat = MemoryContextAllocZero(persistentContext, sizeof(TSVectorStat));
2605 stat->maxdepth = 1;
2606
2607 if (ws)
2608 {
2609 char *buf;
2610 const char *end;
2611
2612 buf = VARDATA_ANY(ws);
2613 end = buf + VARSIZE_ANY_EXHDR(ws);
2614 while (buf < end)
2615 {
2616 int len = pg_mblen_range(buf, end);
2617
2618 if (len == 1)
2619 {
2620 switch (*buf)
2621 {
2622 case 'A':
2623 case 'a':
2624 stat->weight |= 1 << 3;
2625 break;
2626 case 'B':
2627 case 'b':
2628 stat->weight |= 1 << 2;
2629 break;
2630 case 'C':
2631 case 'c':
2632 stat->weight |= 1 << 1;
2633 break;
2634 case 'D':
2635 case 'd':
2636 stat->weight |= 1;
2637 break;
2638 default:
2639 stat->weight |= 0;
2640 }
2641 }
2642 buf += len;
2643 }
2644 }
2645
2646 while (SPI_processed > 0)
2647 {
2648 uint64 i;
2649
2650 for (i = 0; i < SPI_processed; i++)
2651 {
2652 Datum data = SPI_getbinval(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1, &isnull);
2653
2654 if (!isnull)
2655 stat = ts_accum(persistentContext, stat, data);
2656 }
2657
2658 SPI_freetuptable(SPI_tuptable);
2659 SPI_cursor_fetch(portal, true, 100);
2660 }
2661
2662 SPI_freetuptable(SPI_tuptable);
2663 SPI_cursor_close(portal);
2664 SPI_freeplan(plan);
2665 pfree(query);
2666
2667 return stat;
2668 }
2669
2670 Datum
2671 ts_stat1(PG_FUNCTION_ARGS)
2672 {
2673 FuncCallContext *funcctx;
2674 Datum result;
2675
2676 if (SRF_IS_FIRSTCALL())
2677 {
2678 TSVectorStat *stat;
2679 text *txt = PG_GETARG_TEXT_PP(0);
2680
2681 funcctx = SRF_FIRSTCALL_INIT();
2682 SPI_connect();
2683 stat = ts_stat_sql(funcctx->multi_call_memory_ctx, txt, NULL);
2684 PG_FREE_IF_COPY(txt, 0);
2685 ts_setup_firstcall(fcinfo, funcctx, stat);
2686 SPI_finish();
2687 }
2688
2689 funcctx = SRF_PERCALL_SETUP();
2690 if ((result = ts_process_call(funcctx)) != (Datum) 0)
2691 SRF_RETURN_NEXT(funcctx, result);
2692 SRF_RETURN_DONE(funcctx);
2693 }
2694
2695 Datum
2696 ts_stat2(PG_FUNCTION_ARGS)
2697 {
2698 FuncCallContext *funcctx;
2699 Datum result;
2700
2701 if (SRF_IS_FIRSTCALL())
2702 {
2703 TSVectorStat *stat;
2704 text *txt = PG_GETARG_TEXT_PP(0);
2705 text *ws = PG_GETARG_TEXT_PP(1);
2706
2707 funcctx = SRF_FIRSTCALL_INIT();
2708 SPI_connect();
2709 stat = ts_stat_sql(funcctx->multi_call_memory_ctx, txt, ws);
2710 PG_FREE_IF_COPY(txt, 0);
2711 PG_FREE_IF_COPY(ws, 1);
2712 ts_setup_firstcall(fcinfo, funcctx, stat);
2713 SPI_finish();
2714 }
2715
2716 funcctx = SRF_PERCALL_SETUP();
2717 if ((result = ts_process_call(funcctx)) != (Datum) 0)
2718 SRF_RETURN_NEXT(funcctx, result);
2719 SRF_RETURN_DONE(funcctx);
2720 }
2721
2722
2723 /*
2724 * Triggers for automatic update of a tsvector column from text column(s)
2725 *
2726 * Trigger arguments are either
2727 * name of tsvector col, name of tsconfig to use, name(s) of text col(s)
2728 * name of tsvector col, name of regconfig col, name(s) of text col(s)
2729 * ie, tsconfig can either be specified by name, or indirectly as the
2730 * contents of a regconfig field in the row. If the name is used, it must
2731 * be explicitly schema-qualified.
2732 */
2733 Datum
2734 tsvector_update_trigger_byid(PG_FUNCTION_ARGS)
2735 {
2736 return tsvector_update_trigger(fcinfo, false);
2737 }
2738
2739 Datum
2740 tsvector_update_trigger_bycolumn(PG_FUNCTION_ARGS)
2741 {
2742 return tsvector_update_trigger(fcinfo, true);
2743 }
2744
2745 static Datum
2746 tsvector_update_trigger(PG_FUNCTION_ARGS, bool config_column)
2747 {
2748 TriggerData *trigdata;
2749 Trigger *trigger;
2750 Relation rel;
2751 HeapTuple rettuple = NULL;
2752 int tsvector_attr_num,
2753 i;
2754 ParsedText prs;
2755 Datum datum;
2756 bool isnull;
2757 text *txt;
2758 Oid cfgId;
2759 bool update_needed;
2760
2761 /* Check call context */
2762 if (!CALLED_AS_TRIGGER(fcinfo)) /* internal error */
2763 elog(ERROR, "tsvector_update_trigger: not fired by trigger manager");
2764
2765 trigdata = (TriggerData *) fcinfo->context;
2766 if (!TRIGGER_FIRED_FOR_ROW(trigdata->tg_event))
2767 elog(ERROR, "tsvector_update_trigger: must be fired for row");
2768 if (!TRIGGER_FIRED_BEFORE(trigdata->tg_event))
2769 elog(ERROR, "tsvector_update_trigger: must be fired BEFORE event");
2770
2771 if (TRIGGER_FIRED_BY_INSERT(trigdata->tg_event))
2772 {
2773 rettuple = trigdata->tg_trigtuple;
2774 update_needed = true;
2775 }
2776 else if (TRIGGER_FIRED_BY_UPDATE(trigdata->tg_event))
2777 {
2778 rettuple = trigdata->tg_newtuple;
2779 update_needed = false; /* computed below */
2780 }
2781 else
2782 elog(ERROR, "tsvector_update_trigger: must be fired for INSERT or UPDATE");
2783
2784 trigger = trigdata->tg_trigger;
2785 rel = trigdata->tg_relation;
2786
2787 if (trigger->tgnargs < 3)
2788 elog(ERROR, "tsvector_update_trigger: arguments must be tsvector_field, ts_config, text_field1, ...)");
2789
2790 /* Find the target tsvector column */
2791 tsvector_attr_num = SPI_fnumber(rel->rd_att, trigger->tgargs[0]);
2792 if (tsvector_attr_num == SPI_ERROR_NOATTRIBUTE)
2793 ereport(ERROR,
2794 (errcode(ERRCODE_UNDEFINED_COLUMN),
2795 errmsg("tsvector column \"%s\" does not exist",
2796 trigger->tgargs[0])));
2797 /* This will effectively reject system columns, so no separate test: */
2798 if (!IsBinaryCoercible(SPI_gettypeid(rel->rd_att, tsvector_attr_num),
2799 TSVECTOROID))
2800 ereport(ERROR,
2801 (errcode(ERRCODE_DATATYPE_MISMATCH),
2802 errmsg("column \"%s\" is not of tsvector type",
2803 trigger->tgargs[0])));
2804
2805 /* Find the configuration to use */
2806 if (config_column)
2807 {
2808 int config_attr_num;
2809
2810 config_attr_num = SPI_fnumber(rel->rd_att, trigger->tgargs[1]);
2811 if (config_attr_num == SPI_ERROR_NOATTRIBUTE)
2812 ereport(ERROR,
2813 (errcode(ERRCODE_UNDEFINED_COLUMN),
2814 errmsg("configuration column \"%s\" does not exist",
2815 trigger->tgargs[1])));
2816 if (!IsBinaryCoercible(SPI_gettypeid(rel->rd_att, config_attr_num),
2817 REGCONFIGOID))
2818 ereport(ERROR,
2819 (errcode(ERRCODE_DATATYPE_MISMATCH),
2820 errmsg("column \"%s\" is not of regconfig type",
2821 trigger->tgargs[1])));
2822
2823 datum = SPI_getbinval(rettuple, rel->rd_att, config_attr_num, &isnull);
2824 if (isnull)
2825 ereport(ERROR,
2826 (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
2827 errmsg("configuration column \"%s\" must not be null",
2828 trigger->tgargs[1])));
2829 cfgId = DatumGetObjectId(datum);
2830 }
2831 else
2832 {
2833 List *names;
2834
2835 names = stringToQualifiedNameList(trigger->tgargs[1], NULL);
2836 /* require a schema so that results are not search path dependent */
2837 if (list_length(names) < 2)
2838 ereport(ERROR,
2839 (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2840 errmsg("text search configuration name \"%s\" must be schema-qualified",
2841 trigger->tgargs[1])));
2842 cfgId = get_ts_config_oid(names, false);
2843 }
2844
2845 /* initialize parse state */
2846 prs.lenwords = 32;
2847 prs.curwords = 0;
2848 prs.pos = 0;
2849 prs.words = (ParsedWord *) palloc(sizeof(ParsedWord) * prs.lenwords);
2850
2851 /* find all words in indexable column(s) */
2852 for (i = 2; i < trigger->tgnargs; i++)
2853 {
2854 int numattr;
2855
2856 numattr = SPI_fnumber(rel->rd_att, trigger->tgargs[i]);
2857 if (numattr == SPI_ERROR_NOATTRIBUTE)
2858 ereport(ERROR,
2859 (errcode(ERRCODE_UNDEFINED_COLUMN),
2860 errmsg("column \"%s\" does not exist",
2861 trigger->tgargs[i])));
2862 if (!IsBinaryCoercible(SPI_gettypeid(rel->rd_att, numattr), TEXTOID))
2863 ereport(ERROR,
2864 (errcode(ERRCODE_DATATYPE_MISMATCH),
2865 errmsg("column \"%s\" is not of a character type",
2866 trigger->tgargs[i])));
2867
2868 if (bms_is_member(numattr - FirstLowInvalidHeapAttributeNumber, trigdata->tg_updatedcols))
2869 update_needed = true;
2870
2871 datum = SPI_getbinval(rettuple, rel->rd_att, numattr, &isnull);
2872 if (isnull)
2873 continue;
2874
2875 txt = DatumGetTextPP(datum);
2876
2877 parsetext(cfgId, &prs, VARDATA_ANY(txt), VARSIZE_ANY_EXHDR(txt));
2878
2879 if (txt != (text *) DatumGetPointer(datum))
2880 pfree(txt);
2881 }
2882
2883 if (update_needed)
2884 {
2885 /* make tsvector value */
2886 datum = TSVectorGetDatum(make_tsvector(&prs));
2887 isnull = false;
2888
2889 /* and insert it into tuple */
2890 rettuple = heap_modify_tuple_by_cols(rettuple, rel->rd_att,
2891 1, &tsvector_attr_num,
2892 &datum, &isnull);
2893
2894 pfree(DatumGetPointer(datum));
2895 }
2896
2897 return PointerGetDatum(rettuple);
2898 }