| 1 |
/*------------------------------------------------------------------------- |
| 2 |
* |
| 3 |
* ts_parse.c |
| 4 |
* main parse functions for tsearch |
| 5 |
* |
| 6 |
* Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group |
| 7 |
* |
| 8 |
* |
| 9 |
* IDENTIFICATION |
| 10 |
* src/backend/tsearch/ts_parse.c |
| 11 |
* |
| 12 |
*------------------------------------------------------------------------- |
| 13 |
*/ |
| 14 |
|
| 15 |
#include "postgres.h" |
| 16 |
|
| 17 |
#include "tsearch/ts_cache.h" |
| 18 |
#include "tsearch/ts_utils.h" |
| 19 |
#include "varatt.h" |
| 20 |
|
| 21 |
#define IGNORE_LONGLEXEME 1 |
| 22 |
|
| 23 |
/* |
| 24 |
* Lexize subsystem |
| 25 |
*/ |
| 26 |
|
| 27 |
typedef struct ParsedLex |
| 28 |
{ |
| 29 |
int type; |
| 30 |
char *lemm; |
| 31 |
int lenlemm; |
| 32 |
struct ParsedLex *next; |
| 33 |
} ParsedLex; |
| 34 |
|
| 35 |
typedef struct ListParsedLex |
| 36 |
{ |
| 37 |
ParsedLex *head; |
| 38 |
ParsedLex *tail; |
| 39 |
} ListParsedLex; |
| 40 |
|
| 41 |
typedef struct |
| 42 |
{ |
| 43 |
TSConfigCacheEntry *cfg; |
| 44 |
Oid curDictId; |
| 45 |
int posDict; |
| 46 |
DictSubState dictState; |
| 47 |
ParsedLex *curSub; |
| 48 |
ListParsedLex towork; /* current list to work */ |
| 49 |
ListParsedLex waste; /* list of lexemes that already lexized */ |
| 50 |
|
| 51 |
/* |
| 52 |
* fields to store last variant to lexize (basically, thesaurus or similar |
| 53 |
* to, which wants several lexemes |
| 54 |
*/ |
| 55 |
|
| 56 |
ParsedLex *lastRes; |
| 57 |
TSLexeme *tmpRes; |
| 58 |
} LexizeData; |
| 59 |
|
| 60 |
static void |
| 61 |
LexizeInit(LexizeData *ld, TSConfigCacheEntry *cfg) |
| 62 |
{ |
| 63 |
ld->cfg = cfg; |
| 64 |
ld->curDictId = InvalidOid; |
| 65 |
ld->posDict = 0; |
| 66 |
ld->towork.head = ld->towork.tail = ld->curSub = NULL; |
| 67 |
ld->waste.head = ld->waste.tail = NULL; |
| 68 |
ld->lastRes = NULL; |
| 69 |
ld->tmpRes = NULL; |
| 70 |
} |
| 71 |
|
| 72 |
static void |
| 73 |
LPLAddTail(ListParsedLex *list, ParsedLex *newpl) |
| 74 |
{ |
| 75 |
if (list->tail) |
| 76 |
{ |
| 77 |
list->tail->next = newpl; |
| 78 |
list->tail = newpl; |
| 79 |
} |
| 80 |
else |
| 81 |
list->head = list->tail = newpl; |
| 82 |
newpl->next = NULL; |
| 83 |
} |
| 84 |
|
| 85 |
static ParsedLex * |
| 86 |
LPLRemoveHead(ListParsedLex *list) |
| 87 |
{ |
| 88 |
ParsedLex *res = list->head; |
| 89 |
|
| 90 |
if (list->head) |
| 91 |
list->head = list->head->next; |
| 92 |
|
| 93 |
if (list->head == NULL) |
| 94 |
list->tail = NULL; |
| 95 |
|
| 96 |
return res; |
| 97 |
} |
| 98 |
|
| 99 |
static void |
| 100 |
LexizeAddLemm(LexizeData *ld, int type, char *lemm, int lenlemm) |
| 101 |
{ |
| 102 |
ParsedLex *newpl = (ParsedLex *) palloc(sizeof(ParsedLex)); |
| 103 |
|
| 104 |
newpl->type = type; |
| 105 |
newpl->lemm = lemm; |
| 106 |
newpl->lenlemm = lenlemm; |
| 107 |
LPLAddTail(&ld->towork, newpl); |
| 108 |
ld->curSub = ld->towork.tail; |
| 109 |
} |
| 110 |
|
| 111 |
static void |
| 112 |
RemoveHead(LexizeData *ld) |
| 113 |
{ |
| 114 |
LPLAddTail(&ld->waste, LPLRemoveHead(&ld->towork)); |
| 115 |
|
| 116 |
ld->posDict = 0; |
| 117 |
} |
| 118 |
|
| 119 |
static void |
| 120 |
setCorrLex(LexizeData *ld, ParsedLex **correspondLexem) |
| 121 |
{ |
| 122 |
if (correspondLexem) |
| 123 |
{ |
| 124 |
*correspondLexem = ld->waste.head; |
| 125 |
} |
| 126 |
else |
| 127 |
{ |
| 128 |
ParsedLex *tmp, |
| 129 |
*ptr = ld->waste.head; |
| 130 |
|
| 131 |
while (ptr) |
| 132 |
{ |
| 133 |
tmp = ptr->next; |
| 134 |
pfree(ptr); |
| 135 |
ptr = tmp; |
| 136 |
} |
| 137 |
} |
| 138 |
ld->waste.head = ld->waste.tail = NULL; |
| 139 |
} |
| 140 |
|
| 141 |
static void |
| 142 |
moveToWaste(LexizeData *ld, ParsedLex *stop) |
| 143 |
{ |
| 144 |
bool go = true; |
| 145 |
|
| 146 |
while (ld->towork.head && go) |
| 147 |
{ |
| 148 |
if (ld->towork.head == stop) |
| 149 |
{ |
| 150 |
ld->curSub = stop->next; |
| 151 |
go = false; |
| 152 |
} |
| 153 |
RemoveHead(ld); |
| 154 |
} |
| 155 |
} |
| 156 |
|
| 157 |
static void |
| 158 |
setNewTmpRes(LexizeData *ld, ParsedLex *lex, TSLexeme *res) |
| 159 |
{ |
| 160 |
if (ld->tmpRes) |
| 161 |
{ |
| 162 |
TSLexeme *ptr; |
| 163 |
|
| 164 |
for (ptr = ld->tmpRes; ptr->lexeme; ptr++) |
| 165 |
pfree(ptr->lexeme); |
| 166 |
pfree(ld->tmpRes); |
| 167 |
} |
| 168 |
ld->tmpRes = res; |
| 169 |
ld->lastRes = lex; |
| 170 |
} |
| 171 |
|
| 172 |
static TSLexeme * |
| 173 |
LexizeExec(LexizeData *ld, ParsedLex **correspondLexem) |
| 174 |
{ |
| 175 |
int i; |
| 176 |
ListDictionary *map; |
| 177 |
TSDictionaryCacheEntry *dict; |
| 178 |
TSLexeme *res; |
| 179 |
|
| 180 |
if (ld->curDictId == InvalidOid) |
| 181 |
{ |
| 182 |
/* |
| 183 |
* usual mode: dictionary wants only one word, but we should keep in |
| 184 |
* mind that we should go through all stack |
| 185 |
*/ |
| 186 |
|
| 187 |
while (ld->towork.head) |
| 188 |
{ |
| 189 |
ParsedLex *curVal = ld->towork.head; |
| 190 |
char *curValLemm = curVal->lemm; |
| 191 |
int curValLenLemm = curVal->lenlemm; |
| 192 |
|
| 193 |
map = ld->cfg->map + curVal->type; |
| 194 |
|
| 195 |
if (curVal->type == 0 || curVal->type >= ld->cfg->lenmap || map->len == 0) |
| 196 |
{ |
| 197 |
/* skip this type of lexeme */ |
| 198 |
RemoveHead(ld); |
| 199 |
continue; |
| 200 |
} |
| 201 |
|
| 202 |
for (i = ld->posDict; i < map->len; i++) |
| 203 |
{ |
| 204 |
dict = lookup_ts_dictionary_cache(map->dictIds[i]); |
| 205 |
|
| 206 |
ld->dictState.isend = ld->dictState.getnext = false; |
| 207 |
ld->dictState.private_state = NULL; |
| 208 |
res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(dict->lexize), |
| 209 |
PointerGetDatum(dict->dictData), |
| 210 |
PointerGetDatum(curValLemm), |
| 211 |
Int32GetDatum(curValLenLemm), |
| 212 |
PointerGetDatum(&ld->dictState))); |
| 213 |
|
| 214 |
if (ld->dictState.getnext) |
| 215 |
{ |
| 216 |
/* |
| 217 |
* dictionary wants next word, so setup and store current |
| 218 |
* position and go to multiword mode |
| 219 |
*/ |
| 220 |
|
| 221 |
ld->curDictId = DatumGetObjectId(map->dictIds[i]); |
| 222 |
ld->posDict = i + 1; |
| 223 |
ld->curSub = curVal->next; |
| 224 |
if (res) |
| 225 |
setNewTmpRes(ld, curVal, res); |
| 226 |
return LexizeExec(ld, correspondLexem); |
| 227 |
} |
| 228 |
|
| 229 |
if (!res) /* dictionary doesn't know this lexeme */ |
| 230 |
continue; |
| 231 |
|
| 232 |
if (res->flags & TSL_FILTER) |
| 233 |
{ |
| 234 |
curValLemm = res->lexeme; |
| 235 |
curValLenLemm = strlen(res->lexeme); |
| 236 |
continue; |
| 237 |
} |
| 238 |
|
| 239 |
RemoveHead(ld); |
| 240 |
setCorrLex(ld, correspondLexem); |
| 241 |
return res; |
| 242 |
} |
| 243 |
|
| 244 |
RemoveHead(ld); |
| 245 |
} |
| 246 |
} |
| 247 |
else |
| 248 |
{ /* curDictId is valid */ |
| 249 |
dict = lookup_ts_dictionary_cache(ld->curDictId); |
| 250 |
|
| 251 |
/* |
| 252 |
* Dictionary ld->curDictId asks us about following words |
| 253 |
*/ |
| 254 |
|
| 255 |
while (ld->curSub) |
| 256 |
{ |
| 257 |
ParsedLex *curVal = ld->curSub; |
| 258 |
|
| 259 |
map = ld->cfg->map + curVal->type; |
| 260 |
|
| 261 |
if (curVal->type != 0) |
| 262 |
{ |
| 263 |
bool dictExists = false; |
| 264 |
|
| 265 |
if (curVal->type >= ld->cfg->lenmap || map->len == 0) |
| 266 |
{ |
| 267 |
/* skip this type of lexeme */ |
| 268 |
ld->curSub = curVal->next; |
| 269 |
continue; |
| 270 |
} |
| 271 |
|
| 272 |
/* |
| 273 |
* We should be sure that current type of lexeme is recognized |
| 274 |
* by our dictionary: we just check is it exist in list of |
| 275 |
* dictionaries ? |
| 276 |
*/ |
| 277 |
for (i = 0; i < map->len && !dictExists; i++) |
| 278 |
if (ld->curDictId == DatumGetObjectId(map->dictIds[i])) |
| 279 |
dictExists = true; |
| 280 |
|
| 281 |
if (!dictExists) |
| 282 |
{ |
| 283 |
/* |
| 284 |
* Dictionary can't work with current type of lexeme, |
| 285 |
* return to basic mode and redo all stored lexemes |
| 286 |
*/ |
| 287 |
ld->curDictId = InvalidOid; |
| 288 |
return LexizeExec(ld, correspondLexem); |
| 289 |
} |
| 290 |
} |
| 291 |
|
| 292 |
ld->dictState.isend = (curVal->type == 0); |
| 293 |
ld->dictState.getnext = false; |
| 294 |
|
| 295 |
res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(dict->lexize), |
| 296 |
PointerGetDatum(dict->dictData), |
| 297 |
PointerGetDatum(curVal->lemm), |
| 298 |
Int32GetDatum(curVal->lenlemm), |
| 299 |
PointerGetDatum(&ld->dictState))); |
| 300 |
|
| 301 |
if (ld->dictState.getnext) |
| 302 |
{ |
| 303 |
/* Dictionary wants one more */ |
| 304 |
ld->curSub = curVal->next; |
| 305 |
if (res) |
| 306 |
setNewTmpRes(ld, curVal, res); |
| 307 |
continue; |
| 308 |
} |
| 309 |
|
| 310 |
if (res || ld->tmpRes) |
| 311 |
{ |
| 312 |
/* |
| 313 |
* Dictionary normalizes lexemes, so we remove from stack all |
| 314 |
* used lexemes, return to basic mode and redo end of stack |
| 315 |
* (if it exists) |
| 316 |
*/ |
| 317 |
if (res) |
| 318 |
{ |
| 319 |
moveToWaste(ld, ld->curSub); |
| 320 |
} |
| 321 |
else |
| 322 |
{ |
| 323 |
res = ld->tmpRes; |
| 324 |
moveToWaste(ld, ld->lastRes); |
| 325 |
} |
| 326 |
|
| 327 |
/* reset to initial state */ |
| 328 |
ld->curDictId = InvalidOid; |
| 329 |
ld->posDict = 0; |
| 330 |
ld->lastRes = NULL; |
| 331 |
ld->tmpRes = NULL; |
| 332 |
setCorrLex(ld, correspondLexem); |
| 333 |
return res; |
| 334 |
} |
| 335 |
|
| 336 |
/* |
| 337 |
* Dict don't want next lexem and didn't recognize anything, redo |
| 338 |
* from ld->towork.head |
| 339 |
*/ |
| 340 |
ld->curDictId = InvalidOid; |
| 341 |
return LexizeExec(ld, correspondLexem); |
| 342 |
} |
| 343 |
} |
| 344 |
|
| 345 |
setCorrLex(ld, correspondLexem); |
| 346 |
return NULL; |
| 347 |
} |
| 348 |
|
| 349 |
/* |
| 350 |
* Parse string and lexize words. |
| 351 |
* |
| 352 |
* prs will be filled in. |
| 353 |
*/ |
| 354 |
void |
| 355 |
parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen) |
| 356 |
{ |
| 357 |
int type, |
| 358 |
lenlemm = 0; /* silence compiler warning */ |
| 359 |
char *lemm = NULL; |
| 360 |
LexizeData ldata; |
| 361 |
TSLexeme *norms; |
| 362 |
TSConfigCacheEntry *cfg; |
| 363 |
TSParserCacheEntry *prsobj; |
| 364 |
void *prsdata; |
| 365 |
|
| 366 |
cfg = lookup_ts_config_cache(cfgId); |
| 367 |
prsobj = lookup_ts_parser_cache(cfg->prsId); |
| 368 |
|
| 369 |
prsdata = DatumGetPointer(FunctionCall2(&prsobj->prsstart, |
| 370 |
PointerGetDatum(buf), |
| 371 |
Int32GetDatum(buflen))); |
| 372 |
|
| 373 |
LexizeInit(&ldata, cfg); |
| 374 |
|
| 375 |
do |
| 376 |
{ |
| 377 |
type = DatumGetInt32(FunctionCall3(&(prsobj->prstoken), |
| 378 |
PointerGetDatum(prsdata), |
| 379 |
PointerGetDatum(&lemm), |
| 380 |
PointerGetDatum(&lenlemm))); |
| 381 |
|
| 382 |
if (type > 0 && lenlemm >= MAXSTRLEN) |
| 383 |
{ |
| 384 |
#ifdef IGNORE_LONGLEXEME |
| 385 |
ereport(NOTICE, |
| 386 |
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), |
| 387 |
errmsg("word is too long to be indexed"), |
| 388 |
errdetail("Words longer than %d characters are ignored.", |
| 389 |
MAXSTRLEN))); |
| 390 |
continue; |
| 391 |
#else |
| 392 |
ereport(ERROR, |
| 393 |
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), |
| 394 |
errmsg("word is too long to be indexed"), |
| 395 |
errdetail("Words longer than %d characters are ignored.", |
| 396 |
MAXSTRLEN))); |
| 397 |
#endif |
| 398 |
} |
| 399 |
|
| 400 |
LexizeAddLemm(&ldata, type, lemm, lenlemm); |
| 401 |
|
| 402 |
while ((norms = LexizeExec(&ldata, NULL)) != NULL) |
| 403 |
{ |
| 404 |
prs->pos++; /* set pos */ |
| 405 |
|
| 406 |
for (TSLexeme *ptr = norms; ptr->lexeme; ptr++) |
| 407 |
{ |
| 408 |
size_t lexeme_len = strlen(ptr->lexeme); |
| 409 |
|
| 410 |
if (lexeme_len > MAXSTRLEN) |
| 411 |
{ |
| 412 |
#ifdef IGNORE_LONGLEXEME |
| 413 |
ereport(NOTICE, |
| 414 |
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), |
| 415 |
errmsg("word is too long to be indexed"), |
| 416 |
errdetail("Words longer than %d characters are ignored.", |
| 417 |
MAXSTRLEN))); |
| 418 |
continue; |
| 419 |
#else |
| 420 |
ereport(ERROR, |
| 421 |
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), |
| 422 |
errmsg("word is too long to be indexed"), |
| 423 |
errdetail("Words longer than %d characters are ignored.", |
| 424 |
MAXSTRLEN))); |
| 425 |
#endif |
| 426 |
} |
| 427 |
|
| 428 |
if (prs->curwords == prs->lenwords) |
| 429 |
{ |
| 430 |
prs->lenwords *= 2; |
| 431 |
prs->words = (ParsedWord *) repalloc(prs->words, prs->lenwords * sizeof(ParsedWord)); |
| 432 |
} |
| 433 |
|
| 434 |
if (ptr->flags & TSL_ADDPOS) |
| 435 |
prs->pos++; |
| 436 |
prs->words[prs->curwords].len = lexeme_len; |
| 437 |
prs->words[prs->curwords].word = ptr->lexeme; |
| 438 |
prs->words[prs->curwords].nvariant = ptr->nvariant; |
| 439 |
prs->words[prs->curwords].flags = ptr->flags & TSL_PREFIX; |
| 440 |
prs->words[prs->curwords].alen = 0; |
| 441 |
prs->words[prs->curwords].pos.pos = LIMITPOS(prs->pos); |
| 442 |
prs->curwords++; |
| 443 |
} |
| 444 |
pfree(norms); |
| 445 |
} |
| 446 |
} while (type > 0); |
| 447 |
|
| 448 |
FunctionCall1(&(prsobj->prsend), PointerGetDatum(prsdata)); |
| 449 |
} |
| 450 |
|
| 451 |
/* |
| 452 |
* Headline framework |
| 453 |
*/ |
| 454 |
|
| 455 |
/* Add a word to prs->words[] */ |
| 456 |
static void |
| 457 |
hladdword(HeadlineParsedText *prs, char *buf, int buflen, int type) |
| 458 |
{ |
| 459 |
if (prs->curwords >= prs->lenwords) |
| 460 |
{ |
| 461 |
prs->lenwords *= 2; |
| 462 |
prs->words = (HeadlineWordEntry *) repalloc(prs->words, prs->lenwords * sizeof(HeadlineWordEntry)); |
| 463 |
} |
| 464 |
memset(&(prs->words[prs->curwords]), 0, sizeof(HeadlineWordEntry)); |
| 465 |
prs->words[prs->curwords].type = (uint8) type; |
| 466 |
prs->words[prs->curwords].len = buflen; |
| 467 |
prs->words[prs->curwords].word = palloc(buflen); |
| 468 |
memcpy(prs->words[prs->curwords].word, buf, buflen); |
| 469 |
prs->curwords++; |
| 470 |
} |
| 471 |
|
| 472 |
/* |
| 473 |
* Add pos and matching-query-item data to the just-added word. |
| 474 |
* Here, buf/buflen represent a processed lexeme, not raw token text. |
| 475 |
* |
| 476 |
* If the query contains more than one matching item, we replicate |
| 477 |
* the last-added word so that each item can be pointed to. The |
| 478 |
* duplicate entries are marked with repeated = 1. |
| 479 |
*/ |
| 480 |
static void |
| 481 |
hlfinditem(HeadlineParsedText *prs, TSQuery query, int32 pos, char *buf, int buflen) |
| 482 |
{ |
| 483 |
int i; |
| 484 |
QueryItem *item = GETQUERY(query); |
| 485 |
HeadlineWordEntry *word; |
| 486 |
|
| 487 |
while (prs->curwords + query->size >= prs->lenwords) |
| 488 |
{ |
| 489 |
prs->lenwords *= 2; |
| 490 |
prs->words = (HeadlineWordEntry *) repalloc(prs->words, prs->lenwords * sizeof(HeadlineWordEntry)); |
| 491 |
} |
| 492 |
|
| 493 |
word = &(prs->words[prs->curwords - 1]); |
| 494 |
word->pos = LIMITPOS(pos); |
| 495 |
for (i = 0; i < query->size; i++) |
| 496 |
{ |
| 497 |
if (item->type == QI_VAL && |
| 498 |
tsCompareString(GETOPERAND(query) + item->qoperand.distance, item->qoperand.length, |
| 499 |
buf, buflen, item->qoperand.prefix) == 0) |
| 500 |
{ |
| 501 |
if (word->item) |
| 502 |
{ |
| 503 |
memcpy(&(prs->words[prs->curwords]), word, sizeof(HeadlineWordEntry)); |
| 504 |
prs->words[prs->curwords].item = &item->qoperand; |
| 505 |
prs->words[prs->curwords].repeated = 1; |
| 506 |
prs->curwords++; |
| 507 |
} |
| 508 |
else |
| 509 |
word->item = &item->qoperand; |
| 510 |
} |
| 511 |
item++; |
| 512 |
} |
| 513 |
} |
| 514 |
|
| 515 |
static void |
| 516 |
addHLParsedLex(HeadlineParsedText *prs, TSQuery query, ParsedLex *lexs, TSLexeme *norms) |
| 517 |
{ |
| 518 |
ParsedLex *tmplexs; |
| 519 |
TSLexeme *ptr; |
| 520 |
int32 savedpos; |
| 521 |
|
| 522 |
while (lexs) |
| 523 |
{ |
| 524 |
if (lexs->type > 0) |
| 525 |
hladdword(prs, lexs->lemm, lexs->lenlemm, lexs->type); |
| 526 |
|
| 527 |
ptr = norms; |
| 528 |
savedpos = prs->vectorpos; |
| 529 |
while (ptr && ptr->lexeme) |
| 530 |
{ |
| 531 |
if (ptr->flags & TSL_ADDPOS) |
| 532 |
savedpos++; |
| 533 |
hlfinditem(prs, query, savedpos, ptr->lexeme, strlen(ptr->lexeme)); |
| 534 |
ptr++; |
| 535 |
} |
| 536 |
|
| 537 |
tmplexs = lexs->next; |
| 538 |
pfree(lexs); |
| 539 |
lexs = tmplexs; |
| 540 |
} |
| 541 |
|
| 542 |
if (norms) |
| 543 |
{ |
| 544 |
ptr = norms; |
| 545 |
while (ptr->lexeme) |
| 546 |
{ |
| 547 |
if (ptr->flags & TSL_ADDPOS) |
| 548 |
prs->vectorpos++; |
| 549 |
pfree(ptr->lexeme); |
| 550 |
ptr++; |
| 551 |
} |
| 552 |
pfree(norms); |
| 553 |
} |
| 554 |
} |
| 555 |
|
| 556 |
void |
| 557 |
hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query, char *buf, int buflen) |
| 558 |
{ |
| 559 |
int type, |
| 560 |
lenlemm = 0; /* silence compiler warning */ |
| 561 |
char *lemm = NULL; |
| 562 |
LexizeData ldata; |
| 563 |
TSLexeme *norms; |
| 564 |
ParsedLex *lexs; |
| 565 |
TSConfigCacheEntry *cfg; |
| 566 |
TSParserCacheEntry *prsobj; |
| 567 |
void *prsdata; |
| 568 |
|
| 569 |
cfg = lookup_ts_config_cache(cfgId); |
| 570 |
prsobj = lookup_ts_parser_cache(cfg->prsId); |
| 571 |
|
| 572 |
prsdata = DatumGetPointer(FunctionCall2(&(prsobj->prsstart), |
| 573 |
PointerGetDatum(buf), |
| 574 |
Int32GetDatum(buflen))); |
| 575 |
|
| 576 |
LexizeInit(&ldata, cfg); |
| 577 |
|
| 578 |
do |
| 579 |
{ |
| 580 |
type = DatumGetInt32(FunctionCall3(&(prsobj->prstoken), |
| 581 |
PointerGetDatum(prsdata), |
| 582 |
PointerGetDatum(&lemm), |
| 583 |
PointerGetDatum(&lenlemm))); |
| 584 |
|
| 585 |
if (type > 0 && lenlemm >= MAXSTRLEN) |
| 586 |
{ |
| 587 |
#ifdef IGNORE_LONGLEXEME |
| 588 |
ereport(NOTICE, |
| 589 |
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), |
| 590 |
errmsg("word is too long to be indexed"), |
| 591 |
errdetail("Words longer than %d characters are ignored.", |
| 592 |
MAXSTRLEN))); |
| 593 |
continue; |
| 594 |
#else |
| 595 |
ereport(ERROR, |
| 596 |
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), |
| 597 |
errmsg("word is too long to be indexed"), |
| 598 |
errdetail("Words longer than %d characters are ignored.", |
| 599 |
MAXSTRLEN))); |
| 600 |
#endif |
| 601 |
} |
| 602 |
|
| 603 |
LexizeAddLemm(&ldata, type, lemm, lenlemm); |
| 604 |
|
| 605 |
do |
| 606 |
{ |
| 607 |
if ((norms = LexizeExec(&ldata, &lexs)) != NULL) |
| 608 |
{ |
| 609 |
prs->vectorpos++; |
| 610 |
addHLParsedLex(prs, query, lexs, norms); |
| 611 |
} |
| 612 |
else |
| 613 |
addHLParsedLex(prs, query, lexs, NULL); |
| 614 |
} while (norms); |
| 615 |
} while (type > 0); |
| 616 |
|
| 617 |
FunctionCall1(&(prsobj->prsend), PointerGetDatum(prsdata)); |
| 618 |
} |
| 619 |
|
| 620 |
/* |
| 621 |
* Generate the headline, as a text object, from HeadlineParsedText. |
| 622 |
*/ |
| 623 |
text * |
| 624 |
generateHeadline(HeadlineParsedText *prs) |
| 625 |
{ |
| 626 |
text *out; |
| 627 |
char *ptr; |
| 628 |
int len = 128; |
| 629 |
int numfragments = 0; |
| 630 |
int16 infrag = 0; |
| 631 |
|
| 632 |
HeadlineWordEntry *wrd = prs->words; |
| 633 |
|
| 634 |
out = (text *) palloc(len); |
| 635 |
ptr = ((char *) out) + VARHDRSZ; |
| 636 |
|
| 637 |
while (wrd - prs->words < prs->curwords) |
| 638 |
{ |
| 639 |
while (wrd->len + prs->stopsellen + prs->startsellen + prs->fragdelimlen + (ptr - ((char *) out)) >= len) |
| 640 |
{ |
| 641 |
int dist = ptr - ((char *) out); |
| 642 |
|
| 643 |
len *= 2; |
| 644 |
out = (text *) repalloc(out, len); |
| 645 |
ptr = ((char *) out) + dist; |
| 646 |
} |
| 647 |
|
| 648 |
if (wrd->in && !wrd->repeated) |
| 649 |
{ |
| 650 |
if (!infrag) |
| 651 |
{ |
| 652 |
|
| 653 |
/* start of a new fragment */ |
| 654 |
infrag = 1; |
| 655 |
numfragments++; |
| 656 |
/* add a fragment delimiter if this is after the first one */ |
| 657 |
if (numfragments > 1) |
| 658 |
{ |
| 659 |
memcpy(ptr, prs->fragdelim, prs->fragdelimlen); |
| 660 |
ptr += prs->fragdelimlen; |
| 661 |
} |
| 662 |
} |
| 663 |
if (wrd->replace) |
| 664 |
{ |
| 665 |
*ptr = ' '; |
| 666 |
ptr++; |
| 667 |
} |
| 668 |
else if (!wrd->skip) |
| 669 |
{ |
| 670 |
if (wrd->selected) |
| 671 |
{ |
| 672 |
memcpy(ptr, prs->startsel, prs->startsellen); |
| 673 |
ptr += prs->startsellen; |
| 674 |
} |
| 675 |
memcpy(ptr, wrd->word, wrd->len); |
| 676 |
ptr += wrd->len; |
| 677 |
if (wrd->selected) |
| 678 |
{ |
| 679 |
memcpy(ptr, prs->stopsel, prs->stopsellen); |
| 680 |
ptr += prs->stopsellen; |
| 681 |
} |
| 682 |
} |
| 683 |
} |
| 684 |
else if (!wrd->repeated) |
| 685 |
{ |
| 686 |
if (infrag) |
| 687 |
infrag = 0; |
| 688 |
pfree(wrd->word); |
| 689 |
} |
| 690 |
|
| 691 |
wrd++; |
| 692 |
} |
| 693 |
|
| 694 |
SET_VARSIZE(out, ptr - ((char *) out)); |
| 695 |
return out; |
| 696 |
} |