| 1 |
/*------------------------------------------------------------------------- |
| 2 |
* |
| 3 |
* tsrank.c |
| 4 |
* rank tsvector by tsquery |
| 5 |
* |
| 6 |
* Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group |
| 7 |
* |
| 8 |
* |
| 9 |
* IDENTIFICATION |
| 10 |
* src/backend/utils/adt/tsrank.c |
| 11 |
* |
| 12 |
*------------------------------------------------------------------------- |
| 13 |
*/ |
| 14 |
#include "postgres.h" |
| 15 |
|
| 16 |
#include <limits.h> |
| 17 |
#include <math.h> |
| 18 |
|
| 19 |
#include "miscadmin.h" |
| 20 |
#include "tsearch/ts_utils.h" |
| 21 |
#include "utils/array.h" |
| 22 |
#include "utils/fmgrprotos.h" |
| 23 |
|
| 24 |
#define NUM_WEIGHTS 4 |
| 25 |
static const float default_weights[NUM_WEIGHTS] = {0.1f, 0.2f, 0.4f, 1.0f}; |
| 26 |
|
| 27 |
#define wpos(wep) ( w[ WEP_GETWEIGHT(wep) ] ) |
| 28 |
|
| 29 |
#define RANK_NO_NORM 0x00 |
| 30 |
#define RANK_NORM_LOGLENGTH 0x01 |
| 31 |
#define RANK_NORM_LENGTH 0x02 |
| 32 |
#define RANK_NORM_EXTDIST 0x04 |
| 33 |
#define RANK_NORM_UNIQ 0x08 |
| 34 |
#define RANK_NORM_LOGUNIQ 0x10 |
| 35 |
#define RANK_NORM_RDIVRPLUS1 0x20 |
| 36 |
#define DEF_NORM_METHOD RANK_NO_NORM |
| 37 |
|
| 38 |
static float calc_rank_or(const float *w, TSVector t, TSQuery q); |
| 39 |
static float calc_rank_and(const float *w, TSVector t, TSQuery q); |
| 40 |
|
| 41 |
/* |
| 42 |
* Returns a weight of a word collocation |
| 43 |
*/ |
| 44 |
static float4 |
| 45 |
word_distance(int32 w) |
| 46 |
{ |
| 47 |
if (w > 100) |
| 48 |
return 1e-30f; |
| 49 |
|
| 50 |
return 1.0 / (1.005 + 0.05 * exp(((float4) w) / 1.5 - 2)); |
| 51 |
} |
| 52 |
|
| 53 |
static int |
| 54 |
cnt_length(TSVector t) |
| 55 |
{ |
| 56 |
WordEntry *ptr = ARRPTR(t), |
| 57 |
*end = (WordEntry *) STRPTR(t); |
| 58 |
int len = 0; |
| 59 |
|
| 60 |
while (ptr < end) |
| 61 |
{ |
| 62 |
int clen = POSDATALEN(t, ptr); |
| 63 |
|
| 64 |
if (clen == 0) |
| 65 |
len += 1; |
| 66 |
else |
| 67 |
len += clen; |
| 68 |
|
| 69 |
ptr++; |
| 70 |
} |
| 71 |
|
| 72 |
return len; |
| 73 |
} |
| 74 |
|
| 75 |
|
| 76 |
#define WordECompareQueryItem(e,q,p,i,m) \ |
| 77 |
tsCompareString((q) + (i)->distance, (i)->length, \ |
| 78 |
(e) + (p)->pos, (p)->len, (m)) |
| 79 |
|
| 80 |
|
| 81 |
/* |
| 82 |
* Returns a pointer to a WordEntry's array corresponding to 'item' from |
| 83 |
* tsvector 't'. 'q' is the TSQuery containing 'item'. |
| 84 |
* Returns NULL if not found. |
| 85 |
*/ |
| 86 |
static WordEntry * |
| 87 |
find_wordentry(TSVector t, TSQuery q, QueryOperand *item, int32 *nitem) |
| 88 |
{ |
| 89 |
WordEntry *StopLow = ARRPTR(t); |
| 90 |
WordEntry *StopHigh = (WordEntry *) STRPTR(t); |
| 91 |
WordEntry *StopMiddle = StopHigh; |
| 92 |
int difference; |
| 93 |
|
| 94 |
*nitem = 0; |
| 95 |
|
| 96 |
/* Loop invariant: StopLow <= item < StopHigh */ |
| 97 |
while (StopLow < StopHigh) |
| 98 |
{ |
| 99 |
StopMiddle = StopLow + (StopHigh - StopLow) / 2; |
| 100 |
difference = WordECompareQueryItem(STRPTR(t), GETOPERAND(q), StopMiddle, item, false); |
| 101 |
if (difference == 0) |
| 102 |
{ |
| 103 |
StopHigh = StopMiddle; |
| 104 |
*nitem = 1; |
| 105 |
break; |
| 106 |
} |
| 107 |
else if (difference > 0) |
| 108 |
StopLow = StopMiddle + 1; |
| 109 |
else |
| 110 |
StopHigh = StopMiddle; |
| 111 |
} |
| 112 |
|
| 113 |
if (item->prefix) |
| 114 |
{ |
| 115 |
if (StopLow >= StopHigh) |
| 116 |
StopMiddle = StopHigh; |
| 117 |
|
| 118 |
*nitem = 0; |
| 119 |
|
| 120 |
while (StopMiddle < (WordEntry *) STRPTR(t) && |
| 121 |
WordECompareQueryItem(STRPTR(t), GETOPERAND(q), StopMiddle, item, true) == 0) |
| 122 |
{ |
| 123 |
(*nitem)++; |
| 124 |
StopMiddle++; |
| 125 |
} |
| 126 |
} |
| 127 |
|
| 128 |
return (*nitem > 0) ? StopHigh : NULL; |
| 129 |
} |
| 130 |
|
| 131 |
|
| 132 |
/* |
| 133 |
* sort QueryOperands by (length, word) |
| 134 |
*/ |
| 135 |
static int |
| 136 |
compareQueryOperand(const void *a, const void *b, void *arg) |
| 137 |
{ |
| 138 |
char *operand = (char *) arg; |
| 139 |
QueryOperand *qa = (*(QueryOperand *const *) a); |
| 140 |
QueryOperand *qb = (*(QueryOperand *const *) b); |
| 141 |
|
| 142 |
return tsCompareString(operand + qa->distance, qa->length, |
| 143 |
operand + qb->distance, qb->length, |
| 144 |
false); |
| 145 |
} |
| 146 |
|
| 147 |
/* |
| 148 |
* Returns a sorted, de-duplicated array of QueryOperands in a query. |
| 149 |
* The returned QueryOperands are pointers to the original QueryOperands |
| 150 |
* in the query. |
| 151 |
* |
| 152 |
* Length of the returned array is stored in *size |
| 153 |
*/ |
| 154 |
static QueryOperand ** |
| 155 |
SortAndUniqItems(TSQuery q, int *size) |
| 156 |
{ |
| 157 |
char *operand = GETOPERAND(q); |
| 158 |
QueryItem *item = GETQUERY(q); |
| 159 |
QueryOperand **res, |
| 160 |
**ptr, |
| 161 |
**prevptr; |
| 162 |
|
| 163 |
ptr = res = (QueryOperand **) palloc(sizeof(QueryOperand *) * *size); |
| 164 |
|
| 165 |
/* Collect all operands from the tree to res */ |
| 166 |
while ((*size)--) |
| 167 |
{ |
| 168 |
if (item->type == QI_VAL) |
| 169 |
{ |
| 170 |
*ptr = (QueryOperand *) item; |
| 171 |
ptr++; |
| 172 |
} |
| 173 |
item++; |
| 174 |
} |
| 175 |
|
| 176 |
*size = ptr - res; |
| 177 |
if (*size < 2) |
| 178 |
return res; |
| 179 |
|
| 180 |
qsort_arg(res, *size, sizeof(QueryOperand *), compareQueryOperand, operand); |
| 181 |
|
| 182 |
ptr = res + 1; |
| 183 |
prevptr = res; |
| 184 |
|
| 185 |
/* remove duplicates */ |
| 186 |
while (ptr - res < *size) |
| 187 |
{ |
| 188 |
if (compareQueryOperand(ptr, prevptr, operand) != 0) |
| 189 |
{ |
| 190 |
prevptr++; |
| 191 |
*prevptr = *ptr; |
| 192 |
} |
| 193 |
ptr++; |
| 194 |
} |
| 195 |
|
| 196 |
*size = prevptr + 1 - res; |
| 197 |
return res; |
| 198 |
} |
| 199 |
|
| 200 |
static float |
| 201 |
calc_rank_and(const float *w, TSVector t, TSQuery q) |
| 202 |
{ |
| 203 |
WordEntryPosVector **pos; |
| 204 |
WordEntryPosVector1 posnull; |
| 205 |
WordEntryPosVector *POSNULL; |
| 206 |
int i, |
| 207 |
k, |
| 208 |
l, |
| 209 |
p; |
| 210 |
WordEntry *entry, |
| 211 |
*firstentry; |
| 212 |
WordEntryPos *post, |
| 213 |
*ct; |
| 214 |
int32 dimt, |
| 215 |
lenct, |
| 216 |
dist, |
| 217 |
nitem; |
| 218 |
float res = -1.0; |
| 219 |
QueryOperand **item; |
| 220 |
int size = q->size; |
| 221 |
|
| 222 |
item = SortAndUniqItems(q, &size); |
| 223 |
if (size < 2) |
| 224 |
{ |
| 225 |
pfree(item); |
| 226 |
return calc_rank_or(w, t, q); |
| 227 |
} |
| 228 |
pos = (WordEntryPosVector **) palloc0(sizeof(WordEntryPosVector *) * q->size); |
| 229 |
|
| 230 |
/* A dummy WordEntryPos array to use when haspos is false */ |
| 231 |
posnull.npos = 1; |
| 232 |
posnull.pos[0] = 0; |
| 233 |
WEP_SETPOS(posnull.pos[0], MAXENTRYPOS - 1); |
| 234 |
POSNULL = (WordEntryPosVector *) &posnull; |
| 235 |
|
| 236 |
for (i = 0; i < size; i++) |
| 237 |
{ |
| 238 |
firstentry = entry = find_wordentry(t, q, item[i], &nitem); |
| 239 |
if (!entry) |
| 240 |
continue; |
| 241 |
|
| 242 |
while (entry - firstentry < nitem) |
| 243 |
{ |
| 244 |
if (entry->haspos) |
| 245 |
pos[i] = _POSVECPTR(t, entry); |
| 246 |
else |
| 247 |
pos[i] = POSNULL; |
| 248 |
|
| 249 |
dimt = pos[i]->npos; |
| 250 |
post = pos[i]->pos; |
| 251 |
for (k = 0; k < i; k++) |
| 252 |
{ |
| 253 |
if (!pos[k]) |
| 254 |
continue; |
| 255 |
lenct = pos[k]->npos; |
| 256 |
ct = pos[k]->pos; |
| 257 |
for (l = 0; l < dimt; l++) |
| 258 |
{ |
| 259 |
for (p = 0; p < lenct; p++) |
| 260 |
{ |
| 261 |
dist = abs((int) WEP_GETPOS(post[l]) - (int) WEP_GETPOS(ct[p])); |
| 262 |
if (dist || (dist == 0 && (pos[i] == POSNULL || pos[k] == POSNULL))) |
| 263 |
{ |
| 264 |
float curw; |
| 265 |
|
| 266 |
if (!dist) |
| 267 |
dist = MAXENTRYPOS; |
| 268 |
curw = sqrt(wpos(post[l]) * wpos(ct[p]) * word_distance(dist)); |
| 269 |
res = (res < 0) ? curw : 1.0 - (1.0 - res) * (1.0 - curw); |
| 270 |
} |
| 271 |
} |
| 272 |
} |
| 273 |
} |
| 274 |
|
| 275 |
entry++; |
| 276 |
} |
| 277 |
} |
| 278 |
pfree(pos); |
| 279 |
pfree(item); |
| 280 |
return res; |
| 281 |
} |
| 282 |
|
| 283 |
static float |
| 284 |
calc_rank_or(const float *w, TSVector t, TSQuery q) |
| 285 |
{ |
| 286 |
WordEntry *entry, |
| 287 |
*firstentry; |
| 288 |
WordEntryPosVector1 posnull; |
| 289 |
WordEntryPos *post; |
| 290 |
int32 dimt, |
| 291 |
j, |
| 292 |
i, |
| 293 |
nitem; |
| 294 |
float res = 0.0; |
| 295 |
QueryOperand **item; |
| 296 |
int size = q->size; |
| 297 |
|
| 298 |
/* A dummy WordEntryPos array to use when haspos is false */ |
| 299 |
posnull.npos = 1; |
| 300 |
posnull.pos[0] = 0; |
| 301 |
|
| 302 |
item = SortAndUniqItems(q, &size); |
| 303 |
|
| 304 |
for (i = 0; i < size; i++) |
| 305 |
{ |
| 306 |
float resj, |
| 307 |
wjm; |
| 308 |
int32 jm; |
| 309 |
|
| 310 |
firstentry = entry = find_wordentry(t, q, item[i], &nitem); |
| 311 |
if (!entry) |
| 312 |
continue; |
| 313 |
|
| 314 |
while (entry - firstentry < nitem) |
| 315 |
{ |
| 316 |
if (entry->haspos) |
| 317 |
{ |
| 318 |
dimt = POSDATALEN(t, entry); |
| 319 |
post = POSDATAPTR(t, entry); |
| 320 |
} |
| 321 |
else |
| 322 |
{ |
| 323 |
dimt = posnull.npos; |
| 324 |
post = posnull.pos; |
| 325 |
} |
| 326 |
|
| 327 |
resj = 0.0; |
| 328 |
wjm = -1.0; |
| 329 |
jm = 0; |
| 330 |
for (j = 0; j < dimt; j++) |
| 331 |
{ |
| 332 |
resj = resj + wpos(post[j]) / ((j + 1) * (j + 1)); |
| 333 |
if (wpos(post[j]) > wjm) |
| 334 |
{ |
| 335 |
wjm = wpos(post[j]); |
| 336 |
jm = j; |
| 337 |
} |
| 338 |
} |
| 339 |
/* |
| 340 |
limit (sum(1/i^2),i=1,inf) = pi^2/6 |
| 341 |
resj = sum(wi/i^2),i=1,noccurrence, |
| 342 |
wi - should be sorted desc, |
| 343 |
don't sort for now, just choose maximum weight. This should be corrected |
| 344 |
Oleg Bartunov |
| 345 |
*/ |
| 346 |
res = res + (wjm + resj - wjm / ((jm + 1) * (jm + 1))) / 1.64493406685; |
| 347 |
|
| 348 |
entry++; |
| 349 |
} |
| 350 |
} |
| 351 |
if (size > 0) |
| 352 |
res = res / size; |
| 353 |
pfree(item); |
| 354 |
return res; |
| 355 |
} |
| 356 |
|
| 357 |
static float |
| 358 |
calc_rank(const float *w, TSVector t, TSQuery q, int32 method) |
| 359 |
{ |
| 360 |
QueryItem *item = GETQUERY(q); |
| 361 |
float res = 0.0; |
| 362 |
int len; |
| 363 |
|
| 364 |
if (!t->size || !q->size) |
| 365 |
return 0.0; |
| 366 |
|
| 367 |
/* XXX: What about NOT? */ |
| 368 |
res = (item->type == QI_OPR && (item->qoperator.oper == OP_AND || |
| 369 |
item->qoperator.oper == OP_PHRASE)) ? |
| 370 |
calc_rank_and(w, t, q) : |
| 371 |
calc_rank_or(w, t, q); |
| 372 |
|
| 373 |
if (res < 0) |
| 374 |
res = 1e-20f; |
| 375 |
|
| 376 |
if ((method & RANK_NORM_LOGLENGTH) && t->size > 0) |
| 377 |
res /= log((double) (cnt_length(t) + 1)) / log(2.0); |
| 378 |
|
| 379 |
if (method & RANK_NORM_LENGTH) |
| 380 |
{ |
| 381 |
len = cnt_length(t); |
| 382 |
if (len > 0) |
| 383 |
res /= (float) len; |
| 384 |
} |
| 385 |
|
| 386 |
/* RANK_NORM_EXTDIST not applicable */ |
| 387 |
|
| 388 |
if ((method & RANK_NORM_UNIQ) && t->size > 0) |
| 389 |
res /= (float) (t->size); |
| 390 |
|
| 391 |
if ((method & RANK_NORM_LOGUNIQ) && t->size > 0) |
| 392 |
res /= log((double) (t->size + 1)) / log(2.0); |
| 393 |
|
| 394 |
if (method & RANK_NORM_RDIVRPLUS1) |
| 395 |
res /= (res + 1); |
| 396 |
|
| 397 |
return res; |
| 398 |
} |
| 399 |
|
| 400 |
/* |
| 401 |
* Extract weights from an array. The weights are stored in *ws, which must |
| 402 |
* have space for NUM_WEIGHTS elements. |
| 403 |
*/ |
| 404 |
static void |
| 405 |
getWeights(ArrayType *win, float *ws) |
| 406 |
{ |
| 407 |
int i; |
| 408 |
float4 *arrdata; |
| 409 |
|
| 410 |
Assert(win != NULL); |
| 411 |
|
| 412 |
if (ARR_NDIM(win) != 1) |
| 413 |
ereport(ERROR, |
| 414 |
(errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), |
| 415 |
errmsg("array of weight must be one-dimensional"))); |
| 416 |
|
| 417 |
if (ArrayGetNItems(ARR_NDIM(win), ARR_DIMS(win)) < NUM_WEIGHTS) |
| 418 |
ereport(ERROR, |
| 419 |
(errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR), |
| 420 |
errmsg("array of weight is too short"))); |
| 421 |
|
| 422 |
if (array_contains_nulls(win)) |
| 423 |
ereport(ERROR, |
| 424 |
(errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), |
| 425 |
errmsg("array of weight must not contain nulls"))); |
| 426 |
|
| 427 |
arrdata = (float4 *) ARR_DATA_PTR(win); |
| 428 |
for (i = 0; i < NUM_WEIGHTS; i++) |
| 429 |
{ |
| 430 |
ws[i] = (arrdata[i] >= 0) ? arrdata[i] : default_weights[i]; |
| 431 |
if (ws[i] > 1.0) |
| 432 |
ereport(ERROR, |
| 433 |
(errcode(ERRCODE_INVALID_PARAMETER_VALUE), |
| 434 |
errmsg("weight out of range"))); |
| 435 |
} |
| 436 |
} |
| 437 |
|
| 438 |
Datum |
| 439 |
ts_rank_wttf(PG_FUNCTION_ARGS) |
| 440 |
{ |
| 441 |
ArrayType *win = (ArrayType *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0)); |
| 442 |
TSVector txt = PG_GETARG_TSVECTOR(1); |
| 443 |
TSQuery query = PG_GETARG_TSQUERY(2); |
| 444 |
int method = PG_GETARG_INT32(3); |
| 445 |
float weights[NUM_WEIGHTS]; |
| 446 |
float res; |
| 447 |
|
| 448 |
getWeights(win, weights); |
| 449 |
res = calc_rank(weights, txt, query, method); |
| 450 |
|
| 451 |
PG_FREE_IF_COPY(win, 0); |
| 452 |
PG_FREE_IF_COPY(txt, 1); |
| 453 |
PG_FREE_IF_COPY(query, 2); |
| 454 |
PG_RETURN_FLOAT4(res); |
| 455 |
} |
| 456 |
|
| 457 |
Datum |
| 458 |
ts_rank_wtt(PG_FUNCTION_ARGS) |
| 459 |
{ |
| 460 |
ArrayType *win = (ArrayType *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0)); |
| 461 |
TSVector txt = PG_GETARG_TSVECTOR(1); |
| 462 |
TSQuery query = PG_GETARG_TSQUERY(2); |
| 463 |
float weights[NUM_WEIGHTS]; |
| 464 |
float res; |
| 465 |
|
| 466 |
getWeights(win, weights); |
| 467 |
res = calc_rank(weights, txt, query, DEF_NORM_METHOD); |
| 468 |
|
| 469 |
PG_FREE_IF_COPY(win, 0); |
| 470 |
PG_FREE_IF_COPY(txt, 1); |
| 471 |
PG_FREE_IF_COPY(query, 2); |
| 472 |
PG_RETURN_FLOAT4(res); |
| 473 |
} |
| 474 |
|
| 475 |
Datum |
| 476 |
ts_rank_ttf(PG_FUNCTION_ARGS) |
| 477 |
{ |
| 478 |
TSVector txt = PG_GETARG_TSVECTOR(0); |
| 479 |
TSQuery query = PG_GETARG_TSQUERY(1); |
| 480 |
int method = PG_GETARG_INT32(2); |
| 481 |
float res; |
| 482 |
|
| 483 |
res = calc_rank(default_weights, txt, query, method); |
| 484 |
|
| 485 |
PG_FREE_IF_COPY(txt, 0); |
| 486 |
PG_FREE_IF_COPY(query, 1); |
| 487 |
PG_RETURN_FLOAT4(res); |
| 488 |
} |
| 489 |
|
| 490 |
Datum |
| 491 |
ts_rank_tt(PG_FUNCTION_ARGS) |
| 492 |
{ |
| 493 |
TSVector txt = PG_GETARG_TSVECTOR(0); |
| 494 |
TSQuery query = PG_GETARG_TSQUERY(1); |
| 495 |
float res; |
| 496 |
|
| 497 |
res = calc_rank(default_weights, txt, query, DEF_NORM_METHOD); |
| 498 |
|
| 499 |
PG_FREE_IF_COPY(txt, 0); |
| 500 |
PG_FREE_IF_COPY(query, 1); |
| 501 |
PG_RETURN_FLOAT4(res); |
| 502 |
} |
| 503 |
|
| 504 |
typedef struct |
| 505 |
{ |
| 506 |
union |
| 507 |
{ |
| 508 |
struct |
| 509 |
{ /* compiled doc representation */ |
| 510 |
QueryItem **items; |
| 511 |
int16 nitem; |
| 512 |
} query; |
| 513 |
struct |
| 514 |
{ /* struct is used for preparing doc |
| 515 |
* representation */ |
| 516 |
QueryItem *item; |
| 517 |
WordEntry *entry; |
| 518 |
} map; |
| 519 |
} data; |
| 520 |
WordEntryPos pos; |
| 521 |
} DocRepresentation; |
| 522 |
|
| 523 |
static int |
| 524 |
compareDocR(const void *va, const void *vb) |
| 525 |
{ |
| 526 |
const DocRepresentation *a = (const DocRepresentation *) va; |
| 527 |
const DocRepresentation *b = (const DocRepresentation *) vb; |
| 528 |
|
| 529 |
if (WEP_GETPOS(a->pos) == WEP_GETPOS(b->pos)) |
| 530 |
{ |
| 531 |
if (WEP_GETWEIGHT(a->pos) == WEP_GETWEIGHT(b->pos)) |
| 532 |
{ |
| 533 |
if (a->data.map.entry == b->data.map.entry) |
| 534 |
return 0; |
| 535 |
|
| 536 |
return (a->data.map.entry > b->data.map.entry) ? 1 : -1; |
| 537 |
} |
| 538 |
|
| 539 |
return (WEP_GETWEIGHT(a->pos) > WEP_GETWEIGHT(b->pos)) ? 1 : -1; |
| 540 |
} |
| 541 |
|
| 542 |
return (WEP_GETPOS(a->pos) > WEP_GETPOS(b->pos)) ? 1 : -1; |
| 543 |
} |
| 544 |
|
| 545 |
#define MAXQROPOS MAXENTRYPOS |
| 546 |
typedef struct |
| 547 |
{ |
| 548 |
bool operandexists; |
| 549 |
bool reverseinsert; /* indicates insert order, true means |
| 550 |
* descending order */ |
| 551 |
uint32 npos; |
| 552 |
WordEntryPos pos[MAXQROPOS]; |
| 553 |
} QueryRepresentationOperand; |
| 554 |
|
| 555 |
typedef struct |
| 556 |
{ |
| 557 |
TSQuery query; |
| 558 |
QueryRepresentationOperand *operandData; |
| 559 |
} QueryRepresentation; |
| 560 |
|
| 561 |
#define QR_GET_OPERAND_DATA(q, v) \ |
| 562 |
( (q)->operandData + (((QueryItem*)(v)) - GETQUERY((q)->query)) ) |
| 563 |
|
| 564 |
/* |
| 565 |
* TS_execute callback for matching a tsquery operand to QueryRepresentation |
| 566 |
*/ |
| 567 |
static TSTernaryValue |
| 568 |
checkcondition_QueryOperand(void *checkval, QueryOperand *val, |
| 569 |
ExecPhraseData *data) |
| 570 |
{ |
| 571 |
QueryRepresentation *qr = (QueryRepresentation *) checkval; |
| 572 |
QueryRepresentationOperand *opData = QR_GET_OPERAND_DATA(qr, val); |
| 573 |
|
| 574 |
if (!opData->operandexists) |
| 575 |
return TS_NO; |
| 576 |
|
| 577 |
if (data) |
| 578 |
{ |
| 579 |
data->npos = opData->npos; |
| 580 |
data->pos = opData->pos; |
| 581 |
if (opData->reverseinsert) |
| 582 |
data->pos += MAXQROPOS - opData->npos; |
| 583 |
} |
| 584 |
|
| 585 |
return TS_YES; |
| 586 |
} |
| 587 |
|
| 588 |
typedef struct |
| 589 |
{ |
| 590 |
int pos; |
| 591 |
int p; |
| 592 |
int q; |
| 593 |
DocRepresentation *begin; |
| 594 |
DocRepresentation *end; |
| 595 |
} CoverExt; |
| 596 |
|
| 597 |
static void |
| 598 |
resetQueryRepresentation(QueryRepresentation *qr, bool reverseinsert) |
| 599 |
{ |
| 600 |
int i; |
| 601 |
|
| 602 |
for (i = 0; i < qr->query->size; i++) |
| 603 |
{ |
| 604 |
qr->operandData[i].operandexists = false; |
| 605 |
qr->operandData[i].reverseinsert = reverseinsert; |
| 606 |
qr->operandData[i].npos = 0; |
| 607 |
} |
| 608 |
} |
| 609 |
|
| 610 |
static void |
| 611 |
fillQueryRepresentationData(QueryRepresentation *qr, DocRepresentation *entry) |
| 612 |
{ |
| 613 |
int i; |
| 614 |
int lastPos; |
| 615 |
QueryRepresentationOperand *opData; |
| 616 |
|
| 617 |
for (i = 0; i < entry->data.query.nitem; i++) |
| 618 |
{ |
| 619 |
if (entry->data.query.items[i]->type != QI_VAL) |
| 620 |
continue; |
| 621 |
|
| 622 |
opData = QR_GET_OPERAND_DATA(qr, entry->data.query.items[i]); |
| 623 |
|
| 624 |
opData->operandexists = true; |
| 625 |
|
| 626 |
if (opData->npos == 0) |
| 627 |
{ |
| 628 |
lastPos = (opData->reverseinsert) ? (MAXQROPOS - 1) : 0; |
| 629 |
opData->pos[lastPos] = entry->pos; |
| 630 |
opData->npos++; |
| 631 |
continue; |
| 632 |
} |
| 633 |
|
| 634 |
lastPos = opData->reverseinsert ? |
| 635 |
(MAXQROPOS - opData->npos) : |
| 636 |
(opData->npos - 1); |
| 637 |
|
| 638 |
if (WEP_GETPOS(opData->pos[lastPos]) != WEP_GETPOS(entry->pos)) |
| 639 |
{ |
| 640 |
lastPos = opData->reverseinsert ? |
| 641 |
(MAXQROPOS - 1 - opData->npos) : |
| 642 |
(opData->npos); |
| 643 |
|
| 644 |
opData->pos[lastPos] = entry->pos; |
| 645 |
opData->npos++; |
| 646 |
} |
| 647 |
} |
| 648 |
} |
| 649 |
|
| 650 |
static bool |
| 651 |
Cover(DocRepresentation *doc, int len, QueryRepresentation *qr, CoverExt *ext) |
| 652 |
{ |
| 653 |
DocRepresentation *ptr; |
| 654 |
int lastpos = ext->pos; |
| 655 |
bool found = false; |
| 656 |
|
| 657 |
/* |
| 658 |
* since this function recurses, it could be driven to stack overflow. |
| 659 |
* (though any decent compiler will optimize away the tail-recursion. |
| 660 |
*/ |
| 661 |
check_stack_depth(); |
| 662 |
|
| 663 |
resetQueryRepresentation(qr, false); |
| 664 |
|
| 665 |
ext->p = INT_MAX; |
| 666 |
ext->q = 0; |
| 667 |
ptr = doc + ext->pos; |
| 668 |
|
| 669 |
/* find upper bound of cover from current position, move up */ |
| 670 |
while (ptr - doc < len) |
| 671 |
{ |
| 672 |
fillQueryRepresentationData(qr, ptr); |
| 673 |
|
| 674 |
if (TS_execute(GETQUERY(qr->query), qr, |
| 675 |
TS_EXEC_EMPTY, checkcondition_QueryOperand)) |
| 676 |
{ |
| 677 |
if (WEP_GETPOS(ptr->pos) > ext->q) |
| 678 |
{ |
| 679 |
ext->q = WEP_GETPOS(ptr->pos); |
| 680 |
ext->end = ptr; |
| 681 |
lastpos = ptr - doc; |
| 682 |
found = true; |
| 683 |
} |
| 684 |
break; |
| 685 |
} |
| 686 |
ptr++; |
| 687 |
} |
| 688 |
|
| 689 |
if (!found) |
| 690 |
return false; |
| 691 |
|
| 692 |
resetQueryRepresentation(qr, true); |
| 693 |
|
| 694 |
ptr = doc + lastpos; |
| 695 |
|
| 696 |
/* find lower bound of cover from found upper bound, move down */ |
| 697 |
while (ptr >= doc + ext->pos) |
| 698 |
{ |
| 699 |
/* |
| 700 |
* we scan doc from right to left, so pos info in reverse order! |
| 701 |
*/ |
| 702 |
fillQueryRepresentationData(qr, ptr); |
| 703 |
|
| 704 |
if (TS_execute(GETQUERY(qr->query), qr, |
| 705 |
TS_EXEC_EMPTY, checkcondition_QueryOperand)) |
| 706 |
{ |
| 707 |
if (WEP_GETPOS(ptr->pos) < ext->p) |
| 708 |
{ |
| 709 |
ext->begin = ptr; |
| 710 |
ext->p = WEP_GETPOS(ptr->pos); |
| 711 |
} |
| 712 |
break; |
| 713 |
} |
| 714 |
ptr--; |
| 715 |
} |
| 716 |
|
| 717 |
if (ext->p <= ext->q) |
| 718 |
{ |
| 719 |
/* |
| 720 |
* set position for next try to next lexeme after beginning of found |
| 721 |
* cover |
| 722 |
*/ |
| 723 |
ext->pos = (ptr - doc) + 1; |
| 724 |
return true; |
| 725 |
} |
| 726 |
|
| 727 |
ext->pos++; |
| 728 |
return Cover(doc, len, qr, ext); |
| 729 |
} |
| 730 |
|
| 731 |
static DocRepresentation * |
| 732 |
get_docrep(TSVector txt, QueryRepresentation *qr, int *doclen) |
| 733 |
{ |
| 734 |
QueryItem *item = GETQUERY(qr->query); |
| 735 |
WordEntry *entry, |
| 736 |
*firstentry; |
| 737 |
WordEntryPos *post; |
| 738 |
int32 dimt, /* number of 'post' items */ |
| 739 |
j, |
| 740 |
i, |
| 741 |
nitem; |
| 742 |
int len = qr->query->size * 4, |
| 743 |
cur = 0; |
| 744 |
DocRepresentation *doc; |
| 745 |
|
| 746 |
doc = (DocRepresentation *) palloc(sizeof(DocRepresentation) * len); |
| 747 |
|
| 748 |
/* |
| 749 |
* Iterate through query to make DocRepresentation for words and it's |
| 750 |
* entries satisfied by query |
| 751 |
*/ |
| 752 |
for (i = 0; i < qr->query->size; i++) |
| 753 |
{ |
| 754 |
QueryOperand *curoperand; |
| 755 |
|
| 756 |
if (item[i].type != QI_VAL) |
| 757 |
continue; |
| 758 |
|
| 759 |
curoperand = &item[i].qoperand; |
| 760 |
|
| 761 |
firstentry = entry = find_wordentry(txt, qr->query, curoperand, &nitem); |
| 762 |
if (!entry) |
| 763 |
continue; |
| 764 |
|
| 765 |
/* iterations over entries in tsvector */ |
| 766 |
while (entry - firstentry < nitem) |
| 767 |
{ |
| 768 |
if (entry->haspos) |
| 769 |
{ |
| 770 |
dimt = POSDATALEN(txt, entry); |
| 771 |
post = POSDATAPTR(txt, entry); |
| 772 |
} |
| 773 |
else |
| 774 |
{ |
| 775 |
/* ignore words without positions */ |
| 776 |
entry++; |
| 777 |
continue; |
| 778 |
} |
| 779 |
|
| 780 |
while (cur + dimt >= len) |
| 781 |
{ |
| 782 |
len *= 2; |
| 783 |
doc = (DocRepresentation *) repalloc(doc, sizeof(DocRepresentation) * len); |
| 784 |
} |
| 785 |
|
| 786 |
/* iterations over entry's positions */ |
| 787 |
for (j = 0; j < dimt; j++) |
| 788 |
{ |
| 789 |
if (curoperand->weight == 0 || |
| 790 |
curoperand->weight & (1 << WEP_GETWEIGHT(post[j]))) |
| 791 |
{ |
| 792 |
doc[cur].pos = post[j]; |
| 793 |
doc[cur].data.map.entry = entry; |
| 794 |
doc[cur].data.map.item = (QueryItem *) curoperand; |
| 795 |
cur++; |
| 796 |
} |
| 797 |
} |
| 798 |
|
| 799 |
entry++; |
| 800 |
} |
| 801 |
} |
| 802 |
|
| 803 |
if (cur > 0) |
| 804 |
{ |
| 805 |
DocRepresentation *rptr = doc + 1, |
| 806 |
*wptr = doc, |
| 807 |
storage; |
| 808 |
|
| 809 |
/* |
| 810 |
* Sort representation in ascending order by pos and entry |
| 811 |
*/ |
| 812 |
qsort(doc, cur, sizeof(DocRepresentation), compareDocR); |
| 813 |
|
| 814 |
/* |
| 815 |
* Join QueryItem per WordEntry and its position |
| 816 |
*/ |
| 817 |
storage.pos = doc->pos; |
| 818 |
storage.data.query.items = palloc(sizeof(QueryItem *) * qr->query->size); |
| 819 |
storage.data.query.items[0] = doc->data.map.item; |
| 820 |
storage.data.query.nitem = 1; |
| 821 |
|
| 822 |
while (rptr - doc < cur) |
| 823 |
{ |
| 824 |
if (rptr->pos == (rptr - 1)->pos && |
| 825 |
rptr->data.map.entry == (rptr - 1)->data.map.entry) |
| 826 |
{ |
| 827 |
storage.data.query.items[storage.data.query.nitem] = rptr->data.map.item; |
| 828 |
storage.data.query.nitem++; |
| 829 |
} |
| 830 |
else |
| 831 |
{ |
| 832 |
*wptr = storage; |
| 833 |
wptr++; |
| 834 |
storage.pos = rptr->pos; |
| 835 |
storage.data.query.items = palloc(sizeof(QueryItem *) * qr->query->size); |
| 836 |
storage.data.query.items[0] = rptr->data.map.item; |
| 837 |
storage.data.query.nitem = 1; |
| 838 |
} |
| 839 |
|
| 840 |
rptr++; |
| 841 |
} |
| 842 |
|
| 843 |
*wptr = storage; |
| 844 |
wptr++; |
| 845 |
|
| 846 |
*doclen = wptr - doc; |
| 847 |
return doc; |
| 848 |
} |
| 849 |
|
| 850 |
pfree(doc); |
| 851 |
return NULL; |
| 852 |
} |
| 853 |
|
| 854 |
static float4 |
| 855 |
calc_rank_cd(const float4 *arrdata, TSVector txt, TSQuery query, int method) |
| 856 |
{ |
| 857 |
DocRepresentation *doc; |
| 858 |
int len, |
| 859 |
i, |
| 860 |
doclen = 0; |
| 861 |
CoverExt ext; |
| 862 |
double Wdoc = 0.0; |
| 863 |
double invws[NUM_WEIGHTS]; |
| 864 |
double SumDist = 0.0, |
| 865 |
PrevExtPos = 0.0; |
| 866 |
int NExtent = 0; |
| 867 |
QueryRepresentation qr; |
| 868 |
|
| 869 |
|
| 870 |
for (i = 0; i < NUM_WEIGHTS; i++) |
| 871 |
{ |
| 872 |
invws[i] = ((double) ((arrdata[i] >= 0) ? arrdata[i] : default_weights[i])); |
| 873 |
if (invws[i] > 1.0) |
| 874 |
ereport(ERROR, |
| 875 |
(errcode(ERRCODE_INVALID_PARAMETER_VALUE), |
| 876 |
errmsg("weight out of range"))); |
| 877 |
invws[i] = 1.0 / invws[i]; |
| 878 |
} |
| 879 |
|
| 880 |
qr.query = query; |
| 881 |
qr.operandData = (QueryRepresentationOperand *) |
| 882 |
palloc0(sizeof(QueryRepresentationOperand) * query->size); |
| 883 |
|
| 884 |
doc = get_docrep(txt, &qr, &doclen); |
| 885 |
if (!doc) |
| 886 |
{ |
| 887 |
pfree(qr.operandData); |
| 888 |
return 0.0; |
| 889 |
} |
| 890 |
|
| 891 |
MemSet(&ext, 0, sizeof(CoverExt)); |
| 892 |
while (Cover(doc, doclen, &qr, &ext)) |
| 893 |
{ |
| 894 |
double Cpos = 0.0; |
| 895 |
double InvSum = 0.0; |
| 896 |
double CurExtPos; |
| 897 |
int nNoise; |
| 898 |
DocRepresentation *ptr = ext.begin; |
| 899 |
|
| 900 |
while (ptr <= ext.end) |
| 901 |
{ |
| 902 |
InvSum += invws[WEP_GETWEIGHT(ptr->pos)]; |
| 903 |
ptr++; |
| 904 |
} |
| 905 |
|
| 906 |
Cpos = ((double) (ext.end - ext.begin + 1)) / InvSum; |
| 907 |
|
| 908 |
/* |
| 909 |
* if doc are big enough then ext.q may be equal to ext.p due to limit |
| 910 |
* of positional information. In this case we approximate number of |
| 911 |
* noise word as half cover's length |
| 912 |
*/ |
| 913 |
nNoise = (ext.q - ext.p) - (ext.end - ext.begin); |
| 914 |
if (nNoise < 0) |
| 915 |
nNoise = (ext.end - ext.begin) / 2; |
| 916 |
Wdoc += Cpos / ((double) (1 + nNoise)); |
| 917 |
|
| 918 |
CurExtPos = ((double) (ext.q + ext.p)) / 2.0; |
| 919 |
if (NExtent > 0 && CurExtPos > PrevExtPos /* prevent division by |
| 920 |
* zero in a case of |
| 921 |
* multiple lexize */ ) |
| 922 |
SumDist += 1.0 / (CurExtPos - PrevExtPos); |
| 923 |
|
| 924 |
PrevExtPos = CurExtPos; |
| 925 |
NExtent++; |
| 926 |
} |
| 927 |
|
| 928 |
if ((method & RANK_NORM_LOGLENGTH) && txt->size > 0) |
| 929 |
Wdoc /= log((double) (cnt_length(txt) + 1)); |
| 930 |
|
| 931 |
if (method & RANK_NORM_LENGTH) |
| 932 |
{ |
| 933 |
len = cnt_length(txt); |
| 934 |
if (len > 0) |
| 935 |
Wdoc /= (double) len; |
| 936 |
} |
| 937 |
|
| 938 |
if ((method & RANK_NORM_EXTDIST) && NExtent > 0 && SumDist > 0) |
| 939 |
Wdoc /= ((double) NExtent) / SumDist; |
| 940 |
|
| 941 |
if ((method & RANK_NORM_UNIQ) && txt->size > 0) |
| 942 |
Wdoc /= (double) (txt->size); |
| 943 |
|
| 944 |
if ((method & RANK_NORM_LOGUNIQ) && txt->size > 0) |
| 945 |
Wdoc /= log((double) (txt->size + 1)) / log(2.0); |
| 946 |
|
| 947 |
if (method & RANK_NORM_RDIVRPLUS1) |
| 948 |
Wdoc /= (Wdoc + 1); |
| 949 |
|
| 950 |
pfree(doc); |
| 951 |
|
| 952 |
pfree(qr.operandData); |
| 953 |
|
| 954 |
return (float4) Wdoc; |
| 955 |
} |
| 956 |
|
| 957 |
Datum |
| 958 |
ts_rankcd_wttf(PG_FUNCTION_ARGS) |
| 959 |
{ |
| 960 |
ArrayType *win = (ArrayType *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0)); |
| 961 |
TSVector txt = PG_GETARG_TSVECTOR(1); |
| 962 |
TSQuery query = PG_GETARG_TSQUERY(2); |
| 963 |
int method = PG_GETARG_INT32(3); |
| 964 |
float weights[NUM_WEIGHTS]; |
| 965 |
float res; |
| 966 |
|
| 967 |
getWeights(win, weights); |
| 968 |
res = calc_rank_cd(weights, txt, query, method); |
| 969 |
|
| 970 |
PG_FREE_IF_COPY(win, 0); |
| 971 |
PG_FREE_IF_COPY(txt, 1); |
| 972 |
PG_FREE_IF_COPY(query, 2); |
| 973 |
PG_RETURN_FLOAT4(res); |
| 974 |
} |
| 975 |
|
| 976 |
Datum |
| 977 |
ts_rankcd_wtt(PG_FUNCTION_ARGS) |
| 978 |
{ |
| 979 |
ArrayType *win = (ArrayType *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0)); |
| 980 |
TSVector txt = PG_GETARG_TSVECTOR(1); |
| 981 |
TSQuery query = PG_GETARG_TSQUERY(2); |
| 982 |
float weights[NUM_WEIGHTS]; |
| 983 |
float res; |
| 984 |
|
| 985 |
getWeights(win, weights); |
| 986 |
res = calc_rank_cd(weights, txt, query, DEF_NORM_METHOD); |
| 987 |
|
| 988 |
PG_FREE_IF_COPY(win, 0); |
| 989 |
PG_FREE_IF_COPY(txt, 1); |
| 990 |
PG_FREE_IF_COPY(query, 2); |
| 991 |
PG_RETURN_FLOAT4(res); |
| 992 |
} |
| 993 |
|
| 994 |
Datum |
| 995 |
ts_rankcd_ttf(PG_FUNCTION_ARGS) |
| 996 |
{ |
| 997 |
TSVector txt = PG_GETARG_TSVECTOR(0); |
| 998 |
TSQuery query = PG_GETARG_TSQUERY(1); |
| 999 |
int method = PG_GETARG_INT32(2); |
| 1000 |
float res; |
| 1001 |
|
| 1002 |
res = calc_rank_cd(default_weights, txt, query, method); |
| 1003 |
|
| 1004 |
PG_FREE_IF_COPY(txt, 0); |
| 1005 |
PG_FREE_IF_COPY(query, 1); |
| 1006 |
PG_RETURN_FLOAT4(res); |
| 1007 |
} |
| 1008 |
|
| 1009 |
Datum |
| 1010 |
ts_rankcd_tt(PG_FUNCTION_ARGS) |
| 1011 |
{ |
| 1012 |
TSVector txt = PG_GETARG_TSVECTOR(0); |
| 1013 |
TSQuery query = PG_GETARG_TSQUERY(1); |
| 1014 |
float res; |
| 1015 |
|
| 1016 |
res = calc_rank_cd(default_weights, txt, query, DEF_NORM_METHOD); |
| 1017 |
|
| 1018 |
PG_FREE_IF_COPY(txt, 0); |
| 1019 |
PG_FREE_IF_COPY(query, 1); |
| 1020 |
PG_RETURN_FLOAT4(res); |
| 1021 |
} |