Debian Codegraph @

ext/fts3/fts3_snippet.c

raw
1 /*
2 ** 2009 Oct 23
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 */
13
14 #include "fts3Int.h"
15 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
16
17 #include <string.h>
18 #include <assert.h>
19
20 /*
21 ** Characters that may appear in the second argument to matchinfo().
22 */
23 #define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */
24 #define FTS3_MATCHINFO_NCOL 'c' /* 1 value */
25 #define FTS3_MATCHINFO_NDOC 'n' /* 1 value */
26 #define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */
27 #define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */
28 #define FTS3_MATCHINFO_LCS 's' /* nCol values */
29 #define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */
30 #define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */
31 #define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */
32
33 /*
34 ** The default value for the second argument to matchinfo().
35 */
36 #define FTS3_MATCHINFO_DEFAULT "pcx"
37
38
39 /*
40 ** Used as an sqlite3Fts3ExprIterate() context when loading phrase doclists to
41 ** Fts3Expr.aDoclist[]/nDoclist.
42 */
43 typedef struct LoadDoclistCtx LoadDoclistCtx;
44 struct LoadDoclistCtx {
45 Fts3Cursor *pCsr; /* FTS3 Cursor */
46 int nPhrase; /* Number of phrases seen so far */
47 int nToken; /* Number of tokens seen so far */
48 };
49
50 /*
51 ** The following types are used as part of the implementation of the
52 ** fts3BestSnippet() routine.
53 */
54 typedef struct SnippetIter SnippetIter;
55 typedef struct SnippetPhrase SnippetPhrase;
56 typedef struct SnippetFragment SnippetFragment;
57
58 struct SnippetIter {
59 Fts3Cursor *pCsr; /* Cursor snippet is being generated from */
60 int iCol; /* Extract snippet from this column */
61 int nSnippet; /* Requested snippet length (in tokens) */
62 int nPhrase; /* Number of phrases in query */
63 SnippetPhrase *aPhrase; /* Array of size nPhrase */
64 int iCurrent; /* First token of current snippet */
65 };
66
67 struct SnippetPhrase {
68 int nToken; /* Number of tokens in phrase */
69 char *pList; /* Pointer to start of phrase position list */
70 i64 iHead; /* Next value in position list */
71 char *pHead; /* Position list data following iHead */
72 i64 iTail; /* Next value in trailing position list */
73 char *pTail; /* Position list data following iTail */
74 };
75
76 struct SnippetFragment {
77 int iCol; /* Column snippet is extracted from */
78 int iPos; /* Index of first token in snippet */
79 u64 covered; /* Mask of query phrases covered */
80 u64 hlmask; /* Mask of snippet terms to highlight */
81 };
82
83 /*
84 ** This type is used as an sqlite3Fts3ExprIterate() context object while
85 ** accumulating the data returned by the matchinfo() function.
86 */
87 typedef struct MatchInfo MatchInfo;
88 struct MatchInfo {
89 Fts3Cursor *pCursor; /* FTS3 Cursor */
90 int nCol; /* Number of columns in table */
91 int nPhrase; /* Number of matchable phrases in query */
92 sqlite3_int64 nDoc; /* Number of docs in database */
93 char flag;
94 u32 *aMatchinfo; /* Pre-allocated buffer */
95 };
96
97 /*
98 ** An instance of this structure is used to manage a pair of buffers, each
99 ** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below
100 ** for details.
101 */
102 struct MatchinfoBuffer {
103 u8 aRef[3];
104 int nElem;
105 int bGlobal; /* Set if global data is loaded */
106 char *zMatchinfo;
107 u32 aMI[FLEXARRAY];
108 };
109
110 /* Size (in bytes) of a MatchinfoBuffer sufficient for N elements */
111 #define SZ_MATCHINFOBUFFER(N) \
112 (offsetof(MatchinfoBuffer,aMI)+(((N)+1)/2)*sizeof(u64))
113
114
115 /*
116 ** The snippet() and offsets() functions both return text values. An instance
117 ** of the following structure is used to accumulate those values while the
118 ** functions are running. See fts3StringAppend() for details.
119 */
120 typedef struct StrBuffer StrBuffer;
121 struct StrBuffer {
122 char *z; /* Pointer to buffer containing string */
123 int n; /* Length of z in bytes (excl. nul-term) */
124 int nAlloc; /* Allocated size of buffer z in bytes */
125 };
126
127
128 /*************************************************************************
129 ** Start of MatchinfoBuffer code.
130 */
131
132 /*
133 ** Allocate a two-slot MatchinfoBuffer object.
134 */
135 static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){
136 MatchinfoBuffer *pRet;
137 sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1)
138 + SZ_MATCHINFOBUFFER(1);
139 sqlite3_int64 nStr = strlen(zMatchinfo);
140
141 pRet = sqlite3Fts3MallocZero(nByte + nStr+1);
142 if( pRet ){
143 pRet->aMI[0] = (u8*)(&pRet->aMI[1]) - (u8*)pRet;
144 pRet->aMI[1+nElem] = pRet->aMI[0]
145 + sizeof(u32)*((int)nElem+1);
146 pRet->nElem = (int)nElem;
147 pRet->zMatchinfo = ((char*)pRet) + nByte;
148 memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1);
149 pRet->aRef[0] = 1;
150 }
151
152 return pRet;
153 }
154
155 static void fts3MIBufferFree(void *p){
156 MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]);
157
158 assert( (u32*)p==&pBuf->aMI[1]
159 || (u32*)p==&pBuf->aMI[pBuf->nElem+2]
160 );
161 if( (u32*)p==&pBuf->aMI[1] ){
162 pBuf->aRef[1] = 0;
163 }else{
164 pBuf->aRef[2] = 0;
165 }
166
167 if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){
168 sqlite3_free(pBuf);
169 }
170 }
171
172 static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){
173 void (*xRet)(void*) = 0;
174 u32 *aOut = 0;
175
176 if( p->aRef[1]==0 ){
177 p->aRef[1] = 1;
178 aOut = &p->aMI[1];
179 xRet = fts3MIBufferFree;
180 }
181 else if( p->aRef[2]==0 ){
182 p->aRef[2] = 1;
183 aOut = &p->aMI[p->nElem+2];
184 xRet = fts3MIBufferFree;
185 }else{
186 aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32));
187 if( aOut ){
188 xRet = sqlite3_free;
189 if( p->bGlobal ) memcpy(aOut, &p->aMI[1], p->nElem*sizeof(u32));
190 }
191 }
192
193 *paOut = aOut;
194 return xRet;
195 }
196
197 static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){
198 p->bGlobal = 1;
199 memcpy(&p->aMI[2+p->nElem], &p->aMI[1], p->nElem*sizeof(u32));
200 }
201
202 /*
203 ** Free a MatchinfoBuffer object allocated using fts3MIBufferNew()
204 */
205 void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){
206 if( p ){
207 assert( p->aRef[0]==1 );
208 p->aRef[0] = 0;
209 if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){
210 sqlite3_free(p);
211 }
212 }
213 }
214
215 /*
216 ** End of MatchinfoBuffer code.
217 *************************************************************************/
218
219
220 /*
221 ** This function is used to help iterate through a position-list. A position
222 ** list is a list of unique integers, sorted from smallest to largest. Each
223 ** element of the list is represented by an FTS3 varint that takes the value
224 ** of the difference between the current element and the previous one plus
225 ** two. For example, to store the position-list:
226 **
227 ** 4 9 113
228 **
229 ** the three varints:
230 **
231 ** 6 7 106
232 **
233 ** are encoded.
234 **
235 ** When this function is called, *pp points to the start of an element of
236 ** the list. *piPos contains the value of the previous entry in the list.
237 ** After it returns, *piPos contains the value of the next element of the
238 ** list and *pp is advanced to the following varint.
239 */
240 static void fts3GetDeltaPosition(char **pp, i64 *piPos){
241 int iVal;
242 *pp += fts3GetVarint32(*pp, &iVal);
243 *piPos += (iVal-2);
244 }
245
246 /*
247 ** Helper function for sqlite3Fts3ExprIterate() (see below).
248 */
249 static int fts3ExprIterate2(
250 Fts3Expr *pExpr, /* Expression to iterate phrases of */
251 int *piPhrase, /* Pointer to phrase counter */
252 int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */
253 void *pCtx /* Second argument to pass to callback */
254 ){
255 int rc; /* Return code */
256 int eType = pExpr->eType; /* Type of expression node pExpr */
257
258 if( eType!=FTSQUERY_PHRASE ){
259 assert( pExpr->pLeft && pExpr->pRight );
260 rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx);
261 if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){
262 rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx);
263 }
264 }else{
265 rc = x(pExpr, *piPhrase, pCtx);
266 (*piPhrase)++;
267 }
268 return rc;
269 }
270
271 /*
272 ** Iterate through all phrase nodes in an FTS3 query, except those that
273 ** are part of a sub-tree that is the right-hand-side of a NOT operator.
274 ** For each phrase node found, the supplied callback function is invoked.
275 **
276 ** If the callback function returns anything other than SQLITE_OK,
277 ** the iteration is abandoned and the error code returned immediately.
278 ** Otherwise, SQLITE_OK is returned after a callback has been made for
279 ** all eligible phrase nodes.
280 */
281 int sqlite3Fts3ExprIterate(
282 Fts3Expr *pExpr, /* Expression to iterate phrases of */
283 int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */
284 void *pCtx /* Second argument to pass to callback */
285 ){
286 int iPhrase = 0; /* Variable used as the phrase counter */
287 return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
288 }
289
290 /*
291 ** This is an sqlite3Fts3ExprIterate() callback used while loading the
292 ** doclists for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
293 ** fts3ExprLoadDoclists().
294 */
295 static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
296 int rc = SQLITE_OK;
297 Fts3Phrase *pPhrase = pExpr->pPhrase;
298 LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
299
300 UNUSED_PARAMETER(iPhrase);
301
302 p->nPhrase++;
303 p->nToken += pPhrase->nToken;
304
305 return rc;
306 }
307
308 /*
309 ** Load the doclists for each phrase in the query associated with FTS3 cursor
310 ** pCsr.
311 **
312 ** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable
313 ** phrases in the expression (all phrases except those directly or
314 ** indirectly descended from the right-hand-side of a NOT operator). If
315 ** pnToken is not NULL, then it is set to the number of tokens in all
316 ** matchable phrases of the expression.
317 */
318 static int fts3ExprLoadDoclists(
319 Fts3Cursor *pCsr, /* Fts3 cursor for current query */
320 int *pnPhrase, /* OUT: Number of phrases in query */
321 int *pnToken /* OUT: Number of tokens in query */
322 ){
323 int rc; /* Return Code */
324 LoadDoclistCtx sCtx = {0,0,0}; /* Context for sqlite3Fts3ExprIterate() */
325 sCtx.pCsr = pCsr;
326 rc = sqlite3Fts3ExprIterate(pCsr->pExpr,fts3ExprLoadDoclistsCb,(void*)&sCtx);
327 if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
328 if( pnToken ) *pnToken = sCtx.nToken;
329 return rc;
330 }
331
332 static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
333 (*(int *)ctx)++;
334 pExpr->iPhrase = iPhrase;
335 return SQLITE_OK;
336 }
337 static int fts3ExprPhraseCount(Fts3Expr *pExpr){
338 int nPhrase = 0;
339 (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
340 return nPhrase;
341 }
342
343 /*
344 ** Advance the position list iterator specified by the first two
345 ** arguments so that it points to the first element with a value greater
346 ** than or equal to parameter iNext.
347 */
348 static void fts3SnippetAdvance(char **ppIter, i64 *piIter, int iNext){
349 char *pIter = *ppIter;
350 if( pIter ){
351 i64 iIter = *piIter;
352
353 while( iIter<iNext ){
354 if( 0==(*pIter & 0xFE) ){
355 iIter = -1;
356 pIter = 0;
357 break;
358 }
359 fts3GetDeltaPosition(&pIter, &iIter);
360 }
361
362 *piIter = iIter;
363 *ppIter = pIter;
364 }
365 }
366
367 /*
368 ** Advance the snippet iterator to the next candidate snippet.
369 */
370 static int fts3SnippetNextCandidate(SnippetIter *pIter){
371 int i; /* Loop counter */
372
373 if( pIter->iCurrent<0 ){
374 /* The SnippetIter object has just been initialized. The first snippet
375 ** candidate always starts at offset 0 (even if this candidate has a
376 ** score of 0.0).
377 */
378 pIter->iCurrent = 0;
379
380 /* Advance the 'head' iterator of each phrase to the first offset that
381 ** is greater than or equal to (iNext+nSnippet).
382 */
383 for(i=0; i<pIter->nPhrase; i++){
384 SnippetPhrase *pPhrase = &pIter->aPhrase[i];
385 fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet);
386 }
387 }else{
388 int iStart;
389 int iEnd = 0x7FFFFFFF;
390
391 for(i=0; i<pIter->nPhrase; i++){
392 SnippetPhrase *pPhrase = &pIter->aPhrase[i];
393 if( pPhrase->pHead && pPhrase->iHead<iEnd ){
394 iEnd = pPhrase->iHead;
395 }
396 }
397 if( iEnd==0x7FFFFFFF ){
398 return 1;
399 }
400
401 assert( pIter->nSnippet>=0 );
402 pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1;
403 for(i=0; i<pIter->nPhrase; i++){
404 SnippetPhrase *pPhrase = &pIter->aPhrase[i];
405 fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1);
406 fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart);
407 }
408 }
409
410 return 0;
411 }
412
413 /*
414 ** Retrieve information about the current candidate snippet of snippet
415 ** iterator pIter.
416 */
417 static void fts3SnippetDetails(
418 SnippetIter *pIter, /* Snippet iterator */
419 u64 mCovered, /* Bitmask of phrases already covered */
420 int *piToken, /* OUT: First token of proposed snippet */
421 int *piScore, /* OUT: "Score" for this snippet */
422 u64 *pmCover, /* OUT: Bitmask of phrases covered */
423 u64 *pmHighlight /* OUT: Bitmask of terms to highlight */
424 ){
425 int iStart = pIter->iCurrent; /* First token of snippet */
426 int iScore = 0; /* Score of this snippet */
427 int i; /* Loop counter */
428 u64 mCover = 0; /* Mask of phrases covered by this snippet */
429 u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */
430
431 for(i=0; i<pIter->nPhrase; i++){
432 SnippetPhrase *pPhrase = &pIter->aPhrase[i];
433 if( pPhrase->pTail ){
434 char *pCsr = pPhrase->pTail;
435 i64 iCsr = pPhrase->iTail;
436
437 while( iCsr<(iStart+pIter->nSnippet) && iCsr>=iStart ){
438 int j;
439 u64 mPhrase = (u64)1 << (i%64);
440 u64 mPos = (u64)1 << (iCsr - iStart);
441 assert( iCsr>=iStart && (iCsr - iStart)<=64 );
442 assert( i>=0 );
443 if( (mCover|mCovered)&mPhrase ){
444 iScore++;
445 }else{
446 iScore += 1000;
447 }
448 mCover |= mPhrase;
449
450 for(j=0; j<pPhrase->nToken && j<pIter->nSnippet; j++){
451 mHighlight |= (mPos>>j);
452 }
453
454 if( 0==(*pCsr & 0x0FE) ) break;
455 fts3GetDeltaPosition(&pCsr, &iCsr);
456 }
457 }
458 }
459
460 /* Set the output variables before returning. */
461 *piToken = iStart;
462 *piScore = iScore;
463 *pmCover = mCover;
464 *pmHighlight = mHighlight;
465 }
466
467 /*
468 ** This function is an sqlite3Fts3ExprIterate() callback used by
469 ** fts3BestSnippet(). Each invocation populates an element of the
470 ** SnippetIter.aPhrase[] array.
471 */
472 static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
473 SnippetIter *p = (SnippetIter *)ctx;
474 SnippetPhrase *pPhrase = &p->aPhrase[iPhrase];
475 char *pCsr;
476 int rc;
477
478 pPhrase->nToken = pExpr->pPhrase->nToken;
479 rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr);
480 assert( rc==SQLITE_OK || pCsr==0 );
481 if( pCsr ){
482 i64 iFirst = 0;
483 pPhrase->pList = pCsr;
484 fts3GetDeltaPosition(&pCsr, &iFirst);
485 if( iFirst<0 ){
486 rc = FTS_CORRUPT_VTAB;
487 }else{
488 pPhrase->pHead = pCsr;
489 pPhrase->pTail = pCsr;
490 pPhrase->iHead = iFirst;
491 pPhrase->iTail = iFirst;
492 }
493 }else{
494 assert( rc!=SQLITE_OK || (
495 pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0
496 ));
497 }
498
499 return rc;
500 }
501
502 /*
503 ** Select the fragment of text consisting of nFragment contiguous tokens
504 ** from column iCol that represent the "best" snippet. The best snippet
505 ** is the snippet with the highest score, where scores are calculated
506 ** by adding:
507 **
508 ** (a) +1 point for each occurrence of a matchable phrase in the snippet.
509 **
510 ** (b) +1000 points for the first occurrence of each matchable phrase in
511 ** the snippet for which the corresponding mCovered bit is not set.
512 **
513 ** The selected snippet parameters are stored in structure *pFragment before
514 ** returning. The score of the selected snippet is stored in *piScore
515 ** before returning.
516 */
517 static int fts3BestSnippet(
518 int nSnippet, /* Desired snippet length */
519 Fts3Cursor *pCsr, /* Cursor to create snippet for */
520 int iCol, /* Index of column to create snippet from */
521 u64 mCovered, /* Mask of phrases already covered */
522 u64 *pmSeen, /* IN/OUT: Mask of phrases seen */
523 SnippetFragment *pFragment, /* OUT: Best snippet found */
524 int *piScore /* OUT: Score of snippet pFragment */
525 ){
526 int rc; /* Return Code */
527 int nList; /* Number of phrases in expression */
528 SnippetIter sIter; /* Iterates through snippet candidates */
529 sqlite3_int64 nByte; /* Number of bytes of space to allocate */
530 int iBestScore = -1; /* Best snippet score found so far */
531 int i; /* Loop counter */
532
533 memset(&sIter, 0, sizeof(sIter));
534
535 /* Iterate through the phrases in the expression to count them. The same
536 ** callback makes sure the doclists are loaded for each phrase.
537 */
538 rc = fts3ExprLoadDoclists(pCsr, &nList, 0);
539 if( rc!=SQLITE_OK ){
540 return rc;
541 }
542
543 /* Now that it is known how many phrases there are, allocate and zero
544 ** the required space using malloc().
545 */
546 nByte = sizeof(SnippetPhrase) * nList;
547 sIter.aPhrase = (SnippetPhrase *)sqlite3Fts3MallocZero(nByte);
548 if( !sIter.aPhrase ){
549 return SQLITE_NOMEM;
550 }
551
552 /* Initialize the contents of the SnippetIter object. Then iterate through
553 ** the set of phrases in the expression to populate the aPhrase[] array.
554 */
555 sIter.pCsr = pCsr;
556 sIter.iCol = iCol;
557 sIter.nSnippet = nSnippet;
558 sIter.nPhrase = nList;
559 sIter.iCurrent = -1;
560 rc = sqlite3Fts3ExprIterate(
561 pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter
562 );
563 if( rc==SQLITE_OK ){
564
565 /* Set the *pmSeen output variable. */
566 for(i=0; i<nList; i++){
567 if( sIter.aPhrase[i].pHead ){
568 *pmSeen |= (u64)1 << (i%64);
569 }
570 }
571
572 /* Loop through all candidate snippets. Store the best snippet in
573 ** *pFragment. Store its associated 'score' in iBestScore.
574 */
575 pFragment->iCol = iCol;
576 while( !fts3SnippetNextCandidate(&sIter) ){
577 int iPos;
578 int iScore;
579 u64 mCover;
580 u64 mHighlite;
581 fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite);
582 assert( iScore>=0 );
583 if( iScore>iBestScore ){
584 pFragment->iPos = iPos;
585 pFragment->hlmask = mHighlite;
586 pFragment->covered = mCover;
587 iBestScore = iScore;
588 }
589 }
590
591 *piScore = iBestScore;
592 }
593 sqlite3_free(sIter.aPhrase);
594 return rc;
595 }
596
597
598 /*
599 ** Append a string to the string-buffer passed as the first argument.
600 **
601 ** If nAppend is negative, then the length of the string zAppend is
602 ** determined using strlen().
603 */
604 static int fts3StringAppend(
605 StrBuffer *pStr, /* Buffer to append to */
606 const char *zAppend, /* Pointer to data to append to buffer */
607 int nAppend /* Size of zAppend in bytes (or -1) */
608 ){
609 if( nAppend<0 ){
610 nAppend = (int)strlen(zAppend);
611 }
612
613 /* If there is insufficient space allocated at StrBuffer.z, use realloc()
614 ** to grow the buffer until so that it is big enough to accommodate the
615 ** appended data.
616 */
617 if( pStr->n+nAppend+1>=pStr->nAlloc ){
618 sqlite3_int64 nAlloc = pStr->nAlloc+(sqlite3_int64)nAppend+100;
619 char *zNew = sqlite3_realloc64(pStr->z, nAlloc);
620 if( !zNew ){
621 return SQLITE_NOMEM;
622 }
623 pStr->z = zNew;
624 pStr->nAlloc = nAlloc;
625 }
626 assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) );
627
628 /* Append the data to the string buffer. */
629 memcpy(&pStr->z[pStr->n], zAppend, nAppend);
630 pStr->n += nAppend;
631 pStr->z[pStr->n] = '\0';
632
633 return SQLITE_OK;
634 }
635
636 /*
637 ** The fts3BestSnippet() function often selects snippets that end with a
638 ** query term. That is, the final term of the snippet is always a term
639 ** that requires highlighting. For example, if 'X' is a highlighted term
640 ** and '.' is a non-highlighted term, BestSnippet() may select:
641 **
642 ** ........X.....X
643 **
644 ** This function "shifts" the beginning of the snippet forward in the
645 ** document so that there are approximately the same number of
646 ** non-highlighted terms to the right of the final highlighted term as there
647 ** are to the left of the first highlighted term. For example, to this:
648 **
649 ** ....X.....X....
650 **
651 ** This is done as part of extracting the snippet text, not when selecting
652 ** the snippet. Snippet selection is done based on doclists only, so there
653 ** is no way for fts3BestSnippet() to know whether or not the document
654 ** actually contains terms that follow the final highlighted term.
655 */
656 static int fts3SnippetShift(
657 Fts3Table *pTab, /* FTS3 table snippet comes from */
658 int iLangid, /* Language id to use in tokenizing */
659 int nSnippet, /* Number of tokens desired for snippet */
660 const char *zDoc, /* Document text to extract snippet from */
661 int nDoc, /* Size of buffer zDoc in bytes */
662 int *piPos, /* IN/OUT: First token of snippet */
663 u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */
664 ){
665 u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */
666
667 if( hlmask ){
668 int nLeft; /* Tokens to the left of first highlight */
669 int nRight; /* Tokens to the right of last highlight */
670 int nDesired; /* Ideal number of tokens to shift forward */
671
672 for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++);
673 for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++);
674 assert( (nSnippet-1-nRight)<=63 && (nSnippet-1-nRight)>=0 );
675 nDesired = (nLeft-nRight)/2;
676
677 /* Ideally, the start of the snippet should be pushed forward in the
678 ** document nDesired tokens. This block checks if there are actually
679 ** nDesired tokens to the right of the snippet. If so, *piPos and
680 ** *pHlMask are updated to shift the snippet nDesired tokens to the
681 ** right. Otherwise, the snippet is shifted by the number of tokens
682 ** available.
683 */
684 if( nDesired>0 ){
685 int nShift; /* Number of tokens to shift snippet by */
686 int iCurrent = 0; /* Token counter */
687 int rc; /* Return Code */
688 sqlite3_tokenizer_module *pMod;
689 sqlite3_tokenizer_cursor *pC;
690 pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
691
692 /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired)
693 ** or more tokens in zDoc/nDoc.
694 */
695 rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC);
696 if( rc!=SQLITE_OK ){
697 return rc;
698 }
699 while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){
700 const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0;
701 rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent);
702 }
703 pMod->xClose(pC);
704 if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; }
705
706 nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet;
707 assert( nShift<=nDesired );
708 if( nShift>0 ){
709 *piPos += nShift;
710 *pHlmask = hlmask >> nShift;
711 }
712 }
713 }
714 return SQLITE_OK;
715 }
716
717 /*
718 ** Extract the snippet text for fragment pFragment from cursor pCsr and
719 ** append it to string buffer pOut.
720 */
721 static int fts3SnippetText(
722 Fts3Cursor *pCsr, /* FTS3 Cursor */
723 SnippetFragment *pFragment, /* Snippet to extract */
724 int iFragment, /* Fragment number */
725 int isLast, /* True for final fragment in snippet */
726 int nSnippet, /* Number of tokens in extracted snippet */
727 const char *zOpen, /* String inserted before highlighted term */
728 const char *zClose, /* String inserted after highlighted term */
729 const char *zEllipsis, /* String inserted between snippets */
730 StrBuffer *pOut /* Write output here */
731 ){
732 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
733 int rc; /* Return code */
734 const char *zDoc; /* Document text to extract snippet from */
735 int nDoc; /* Size of zDoc in bytes */
736 int iCurrent = 0; /* Current token number of document */
737 int iEnd = 0; /* Byte offset of end of current token */
738 int isShiftDone = 0; /* True after snippet is shifted */
739 int iPos = pFragment->iPos; /* First token of snippet */
740 u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */
741 int iCol = pFragment->iCol+1; /* Query column to extract text from */
742 sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */
743 sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */
744
745 zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol);
746 if( zDoc==0 ){
747 if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){
748 return SQLITE_NOMEM;
749 }
750 return SQLITE_OK;
751 }
752 nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol);
753
754 /* Open a token cursor on the document. */
755 pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule;
756 rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC);
757 if( rc!=SQLITE_OK ){
758 return rc;
759 }
760
761 while( rc==SQLITE_OK ){
762 const char *ZDUMMY; /* Dummy argument used with tokenizer */
763 int DUMMY1 = -1; /* Dummy argument used with tokenizer */
764 int iBegin = 0; /* Offset in zDoc of start of token */
765 int iFin = 0; /* Offset in zDoc of end of token */
766 int isHighlight = 0; /* True for highlighted terms */
767
768 /* Variable DUMMY1 is initialized to a negative value above. Elsewhere
769 ** in the FTS code the variable that the third argument to xNext points to
770 ** is initialized to zero before the first (*but not necessarily
771 ** subsequent*) call to xNext(). This is done for a particular application
772 ** that needs to know whether or not the tokenizer is being used for
773 ** snippet generation or for some other purpose.
774 **
775 ** Extreme care is required when writing code to depend on this
776 ** initialization. It is not a documented part of the tokenizer interface.
777 ** If a tokenizer is used directly by any code outside of FTS, this
778 ** convention might not be respected. */
779 rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent);
780 if( rc!=SQLITE_OK ){
781 if( rc==SQLITE_DONE ){
782 /* Special case - the last token of the snippet is also the last token
783 ** of the column. Append any punctuation that occurred between the end
784 ** of the previous token and the end of the document to the output.
785 ** Then break out of the loop. */
786 rc = fts3StringAppend(pOut, &zDoc[iEnd], -1);
787 }
788 break;
789 }
790 if( iCurrent<iPos ){ continue; }
791
792 if( !isShiftDone ){
793 int n = nDoc - iBegin;
794 rc = fts3SnippetShift(
795 pTab, pCsr->iLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask
796 );
797 isShiftDone = 1;
798
799 /* Now that the shift has been done, check if the initial "..." are
800 ** required. They are required if (a) this is not the first fragment,
801 ** or (b) this fragment does not begin at position 0 of its column.
802 */
803 if( rc==SQLITE_OK ){
804 if( iPos>0 || iFragment>0 ){
805 rc = fts3StringAppend(pOut, zEllipsis, -1);
806 }else if( iBegin ){
807 rc = fts3StringAppend(pOut, zDoc, iBegin);
808 }
809 }
810 if( rc!=SQLITE_OK || iCurrent<iPos ) continue;
811 }
812
813 if( iCurrent>=(iPos+nSnippet) ){
814 if( isLast ){
815 rc = fts3StringAppend(pOut, zEllipsis, -1);
816 }
817 break;
818 }
819
820 /* Set isHighlight to true if this term should be highlighted. */
821 isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0;
822
823 if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd);
824 if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1);
825 if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin);
826 if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1);
827
828 iEnd = iFin;
829 }
830
831 pMod->xClose(pC);
832 return rc;
833 }
834
835
836 /*
837 ** This function is used to count the entries in a column-list (a
838 ** delta-encoded list of term offsets within a single column of a single
839 ** row). When this function is called, *ppCollist should point to the
840 ** beginning of the first varint in the column-list (the varint that
841 ** contains the position of the first matching term in the column data).
842 ** Before returning, *ppCollist is set to point to the first byte after
843 ** the last varint in the column-list (either the 0x00 signifying the end
844 ** of the position-list, or the 0x01 that precedes the column number of
845 ** the next column in the position-list).
846 **
847 ** The number of elements in the column-list is returned.
848 */
849 static int fts3ColumnlistCount(char **ppCollist){
850 char *pEnd = *ppCollist;
851 char c = 0;
852 int nEntry = 0;
853
854 /* A column-list is terminated by either a 0x01 or 0x00. */
855 while( 0xFE & (*pEnd | c) ){
856 c = *pEnd++ & 0x80;
857 if( !c ) nEntry++;
858 }
859
860 *ppCollist = pEnd;
861 return nEntry;
862 }
863
864 /*
865 ** This function gathers 'y' or 'b' data for a single phrase.
866 */
867 static int fts3ExprLHits(
868 Fts3Expr *pExpr, /* Phrase expression node */
869 MatchInfo *p /* Matchinfo context */
870 ){
871 Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
872 int iStart;
873 Fts3Phrase *pPhrase = pExpr->pPhrase;
874 char *pIter = pPhrase->doclist.pList;
875 int iCol = 0;
876
877 assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS );
878 if( p->flag==FTS3_MATCHINFO_LHITS ){
879 iStart = pExpr->iPhrase * p->nCol;
880 }else{
881 iStart = pExpr->iPhrase * ((p->nCol + 31) / 32);
882 }
883
884 if( pIter ) while( 1 ){
885 int nHit = fts3ColumnlistCount(&pIter);
886 if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){
887 if( p->flag==FTS3_MATCHINFO_LHITS ){
888 p->aMatchinfo[iStart + iCol] = (u32)nHit;
889 }else if( nHit ){
890 p->aMatchinfo[iStart + iCol/32] |= (1U << (iCol&0x1F));
891 }
892 }
893 assert( *pIter==0x00 || *pIter==0x01 );
894 if( *pIter!=0x01 ) break;
895 pIter++;
896 pIter += fts3GetVarint32(pIter, &iCol);
897 if( iCol>=p->nCol ) return FTS_CORRUPT_VTAB;
898 }
899 return SQLITE_OK;
900 }
901
902 /*
903 ** Gather the results for matchinfo directives 'y' and 'b'.
904 */
905 static int fts3ExprLHitGather(
906 Fts3Expr *pExpr,
907 MatchInfo *p
908 ){
909 int rc = SQLITE_OK;
910 assert( (pExpr->pLeft==0)==(pExpr->pRight==0) );
911 if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
912 if( pExpr->pLeft ){
913 rc = fts3ExprLHitGather(pExpr->pLeft, p);
914 if( rc==SQLITE_OK ) rc = fts3ExprLHitGather(pExpr->pRight, p);
915 }else{
916 rc = fts3ExprLHits(pExpr, p);
917 }
918 }
919 return rc;
920 }
921
922 /*
923 ** sqlite3Fts3ExprIterate() callback used to collect the "global" matchinfo
924 ** stats for a single query.
925 **
926 ** sqlite3Fts3ExprIterate() callback to load the 'global' elements of a
927 ** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements
928 ** of the matchinfo array that are constant for all rows returned by the
929 ** current query.
930 **
931 ** Argument pCtx is actually a pointer to a struct of type MatchInfo. This
932 ** function populates Matchinfo.aMatchinfo[] as follows:
933 **
934 ** for(iCol=0; iCol<nCol; iCol++){
935 ** aMatchinfo[3*iPhrase*nCol + 3*iCol + 1] = X;
936 ** aMatchinfo[3*iPhrase*nCol + 3*iCol + 2] = Y;
937 ** }
938 **
939 ** where X is the number of matches for phrase iPhrase is column iCol of all
940 ** rows of the table. Y is the number of rows for which column iCol contains
941 ** at least one instance of phrase iPhrase.
942 **
943 ** If the phrase pExpr consists entirely of deferred tokens, then all X and
944 ** Y values are set to nDoc, where nDoc is the number of documents in the
945 ** file system. This is done because the full-text index doclist is required
946 ** to calculate these values properly, and the full-text index doclist is
947 ** not available for deferred tokens.
948 */
949 static int fts3ExprGlobalHitsCb(
950 Fts3Expr *pExpr, /* Phrase expression node */
951 int iPhrase, /* Phrase number (numbered from zero) */
952 void *pCtx /* Pointer to MatchInfo structure */
953 ){
954 MatchInfo *p = (MatchInfo *)pCtx;
955 return sqlite3Fts3EvalPhraseStats(
956 p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol]
957 );
958 }
959
960 /*
961 ** sqlite3Fts3ExprIterate() callback used to collect the "local" part of the
962 ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the
963 ** array that are different for each row returned by the query.
964 */
965 static int fts3ExprLocalHitsCb(
966 Fts3Expr *pExpr, /* Phrase expression node */
967 int iPhrase, /* Phrase number */
968 void *pCtx /* Pointer to MatchInfo structure */
969 ){
970 int rc = SQLITE_OK;
971 MatchInfo *p = (MatchInfo *)pCtx;
972 int iStart = iPhrase * p->nCol * 3;
973 int i;
974
975 for(i=0; i<p->nCol && rc==SQLITE_OK; i++){
976 char *pCsr;
977 rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr);
978 if( pCsr ){
979 p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
980 }else{
981 p->aMatchinfo[iStart+i*3] = 0;
982 }
983 }
984
985 return rc;
986 }
987
988 static int fts3MatchinfoCheck(
989 Fts3Table *pTab,
990 char cArg,
991 char **pzErr
992 ){
993 if( (cArg==FTS3_MATCHINFO_NPHRASE)
994 || (cArg==FTS3_MATCHINFO_NCOL)
995 || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4)
996 || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4)
997 || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
998 || (cArg==FTS3_MATCHINFO_LCS)
999 || (cArg==FTS3_MATCHINFO_HITS)
1000 || (cArg==FTS3_MATCHINFO_LHITS)
1001 || (cArg==FTS3_MATCHINFO_LHITS_BM)
1002 ){
1003 return SQLITE_OK;
1004 }
1005 sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg);
1006 return SQLITE_ERROR;
1007 }
1008
1009 static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
1010 size_t nVal; /* Number of integers output by cArg */
1011
1012 switch( cArg ){
1013 case FTS3_MATCHINFO_NDOC:
1014 case FTS3_MATCHINFO_NPHRASE:
1015 case FTS3_MATCHINFO_NCOL:
1016 nVal = 1;
1017 break;
1018
1019 case FTS3_MATCHINFO_AVGLENGTH:
1020 case FTS3_MATCHINFO_LENGTH:
1021 case FTS3_MATCHINFO_LCS:
1022 nVal = pInfo->nCol;
1023 break;
1024
1025 case FTS3_MATCHINFO_LHITS:
1026 nVal = (size_t)pInfo->nCol * pInfo->nPhrase;
1027 break;
1028
1029 case FTS3_MATCHINFO_LHITS_BM:
1030 nVal = (size_t)pInfo->nPhrase * ((pInfo->nCol + 31) / 32);
1031 break;
1032
1033 default:
1034 assert( cArg==FTS3_MATCHINFO_HITS );
1035 nVal = (size_t)pInfo->nCol * pInfo->nPhrase * 3;
1036 break;
1037 }
1038
1039 return nVal;
1040 }
1041
1042 static int fts3MatchinfoSelectDoctotal(
1043 Fts3Table *pTab,
1044 sqlite3_stmt **ppStmt,
1045 sqlite3_int64 *pnDoc,
1046 const char **paLen,
1047 const char **ppEnd
1048 ){
1049 sqlite3_stmt *pStmt;
1050 const char *a;
1051 const char *pEnd;
1052 sqlite3_int64 nDoc;
1053 int n;
1054
1055
1056 if( !*ppStmt ){
1057 int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
1058 if( rc!=SQLITE_OK ) return rc;
1059 }
1060 pStmt = *ppStmt;
1061 assert( sqlite3_data_count(pStmt)==1 );
1062
1063 n = sqlite3_column_bytes(pStmt, 0);
1064 a = sqlite3_column_blob(pStmt, 0);
1065 if( a==0 ){
1066 return FTS_CORRUPT_VTAB;
1067 }
1068 pEnd = a + n;
1069 a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc);
1070 if( nDoc<=0 || a>pEnd ){
1071 return FTS_CORRUPT_VTAB;
1072 }
1073 *pnDoc = nDoc;
1074
1075 if( paLen ) *paLen = a;
1076 if( ppEnd ) *ppEnd = pEnd;
1077 return SQLITE_OK;
1078 }
1079
1080 /*
1081 ** An instance of the following structure is used to store state while
1082 ** iterating through a multi-column position-list corresponding to the
1083 ** hits for a single phrase on a single row in order to calculate the
1084 ** values for a matchinfo() FTS3_MATCHINFO_LCS request.
1085 */
1086 typedef struct LcsIterator LcsIterator;
1087 struct LcsIterator {
1088 Fts3Expr *pExpr; /* Pointer to phrase expression */
1089 int iPosOffset; /* Tokens count up to end of this phrase */
1090 char *pRead; /* Cursor used to iterate through aDoclist */
1091 int iPos; /* Current position */
1092 };
1093
1094 /*
1095 ** If LcsIterator.iCol is set to the following value, the iterator has
1096 ** finished iterating through all offsets for all columns.
1097 */
1098 #define LCS_ITERATOR_FINISHED 0x7FFFFFFF;
1099
1100 static int fts3MatchinfoLcsCb(
1101 Fts3Expr *pExpr, /* Phrase expression node */
1102 int iPhrase, /* Phrase number (numbered from zero) */
1103 void *pCtx /* Pointer to MatchInfo structure */
1104 ){
1105 LcsIterator *aIter = (LcsIterator *)pCtx;
1106 aIter[iPhrase].pExpr = pExpr;
1107 return SQLITE_OK;
1108 }
1109
1110 /*
1111 ** Advance the iterator passed as an argument to the next position. Return
1112 ** 1 if the iterator is at EOF or if it now points to the start of the
1113 ** position list for the next column.
1114 */
1115 static int fts3LcsIteratorAdvance(LcsIterator *pIter){
1116 char *pRead;
1117 sqlite3_int64 iRead;
1118 int rc = 0;
1119
1120 if( NEVER(pIter==0) ) return 1;
1121 pRead = pIter->pRead;
1122 pRead += sqlite3Fts3GetVarint(pRead, &iRead);
1123 if( iRead==0 || iRead==1 ){
1124 pRead = 0;
1125 rc = 1;
1126 }else{
1127 pIter->iPos += (int)(iRead-2);
1128 }
1129
1130 pIter->pRead = pRead;
1131 return rc;
1132 }
1133
1134 /*
1135 ** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag.
1136 **
1137 ** If the call is successful, the longest-common-substring lengths for each
1138 ** column are written into the first nCol elements of the pInfo->aMatchinfo[]
1139 ** array before returning. SQLITE_OK is returned in this case.
1140 **
1141 ** Otherwise, if an error occurs, an SQLite error code is returned and the
1142 ** data written to the first nCol elements of pInfo->aMatchinfo[] is
1143 ** undefined.
1144 */
1145 static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
1146 LcsIterator *aIter;
1147 int i;
1148 int iCol;
1149 int nToken = 0;
1150 int rc = SQLITE_OK;
1151
1152 /* Allocate and populate the array of LcsIterator objects. The array
1153 ** contains one element for each matchable phrase in the query.
1154 **/
1155 aIter = sqlite3Fts3MallocZero(sizeof(LcsIterator) * pCsr->nPhrase);
1156 if( !aIter ) return SQLITE_NOMEM;
1157 (void)sqlite3Fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
1158
1159 for(i=0; i<pInfo->nPhrase; i++){
1160 LcsIterator *pIter = &aIter[i];
1161 nToken -= pIter->pExpr->pPhrase->nToken;
1162 pIter->iPosOffset = nToken;
1163 }
1164
1165 for(iCol=0; iCol<pInfo->nCol; iCol++){
1166 int nLcs = 0; /* LCS value for this column */
1167 int nLive = 0; /* Number of iterators in aIter not at EOF */
1168
1169 for(i=0; i<pInfo->nPhrase; i++){
1170 LcsIterator *pIt = &aIter[i];
1171 rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead);
1172 if( rc!=SQLITE_OK ) goto matchinfo_lcs_out;
1173 if( pIt->pRead ){
1174 pIt->iPos = pIt->iPosOffset;
1175 fts3LcsIteratorAdvance(pIt);
1176 if( pIt->pRead==0 ){
1177 rc = FTS_CORRUPT_VTAB;
1178 goto matchinfo_lcs_out;
1179 }
1180 nLive++;
1181 }
1182 }
1183
1184 while( nLive>0 ){
1185 LcsIterator *pAdv = 0; /* The iterator to advance by one position */
1186 int nThisLcs = 0; /* LCS for the current iterator positions */
1187
1188 for(i=0; i<pInfo->nPhrase; i++){
1189 LcsIterator *pIter = &aIter[i];
1190 if( pIter->pRead==0 ){
1191 /* This iterator is already at EOF for this column. */
1192 nThisLcs = 0;
1193 }else{
1194 if( pAdv==0 || pIter->iPos<pAdv->iPos ){
1195 pAdv = pIter;
1196 }
1197 if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){
1198 nThisLcs++;
1199 }else{
1200 nThisLcs = 1;
1201 }
1202 if( nThisLcs>nLcs ) nLcs = nThisLcs;
1203 }
1204 }
1205 if( fts3LcsIteratorAdvance(pAdv) ) nLive--;
1206 }
1207
1208 pInfo->aMatchinfo[iCol] = nLcs;
1209 }
1210
1211 matchinfo_lcs_out:
1212 sqlite3_free(aIter);
1213 return rc;
1214 }
1215
1216 /*
1217 ** Populate the buffer pInfo->aMatchinfo[] with an array of integers to
1218 ** be returned by the matchinfo() function. Argument zArg contains the
1219 ** format string passed as the second argument to matchinfo (or the
1220 ** default value "pcx" if no second argument was specified). The format
1221 ** string has already been validated and the pInfo->aMatchinfo[] array
1222 ** is guaranteed to be large enough for the output.
1223 **
1224 ** If bGlobal is true, then populate all fields of the matchinfo() output.
1225 ** If it is false, then assume that those fields that do not change between
1226 ** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS)
1227 ** have already been populated.
1228 **
1229 ** Return SQLITE_OK if successful, or an SQLite error code if an error
1230 ** occurs. If a value other than SQLITE_OK is returned, the state the
1231 ** pInfo->aMatchinfo[] buffer is left in is undefined.
1232 */
1233 static int fts3MatchinfoValues(
1234 Fts3Cursor *pCsr, /* FTS3 cursor object */
1235 int bGlobal, /* True to grab the global stats */
1236 MatchInfo *pInfo, /* Matchinfo context object */
1237 const char *zArg /* Matchinfo format string */
1238 ){
1239 int rc = SQLITE_OK;
1240 int i;
1241 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
1242 sqlite3_stmt *pSelect = 0;
1243
1244 for(i=0; rc==SQLITE_OK && zArg[i]; i++){
1245 pInfo->flag = zArg[i];
1246 switch( zArg[i] ){
1247 case FTS3_MATCHINFO_NPHRASE:
1248 if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
1249 break;
1250
1251 case FTS3_MATCHINFO_NCOL:
1252 if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol;
1253 break;
1254
1255 case FTS3_MATCHINFO_NDOC:
1256 if( bGlobal ){
1257 sqlite3_int64 nDoc = 0;
1258 rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0, 0);
1259 pInfo->aMatchinfo[0] = (u32)nDoc;
1260 }
1261 break;
1262
1263 case FTS3_MATCHINFO_AVGLENGTH:
1264 if( bGlobal ){
1265 sqlite3_int64 nDoc; /* Number of rows in table */
1266 const char *a; /* Aggregate column length array */
1267 const char *pEnd; /* First byte past end of length array */
1268
1269 rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a, &pEnd);
1270 if( rc==SQLITE_OK ){
1271 int iCol;
1272 for(iCol=0; iCol<pInfo->nCol; iCol++){
1273 u32 iVal;
1274 sqlite3_int64 nToken;
1275 a += sqlite3Fts3GetVarint(a, &nToken);
1276 if( a>pEnd ){
1277 rc = SQLITE_CORRUPT_VTAB;
1278 break;
1279 }
1280 iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc);
1281 pInfo->aMatchinfo[iCol] = iVal;
1282 }
1283 }
1284 }
1285 break;
1286
1287 case FTS3_MATCHINFO_LENGTH: {
1288 sqlite3_stmt *pSelectDocsize = 0;
1289 rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize);
1290 if( rc==SQLITE_OK ){
1291 int iCol;
1292 const char *a = sqlite3_column_blob(pSelectDocsize, 0);
1293 const char *pEnd = a + sqlite3_column_bytes(pSelectDocsize, 0);
1294 for(iCol=0; iCol<pInfo->nCol; iCol++){
1295 sqlite3_int64 nToken;
1296 a += sqlite3Fts3GetVarintBounded(a, pEnd, &nToken);
1297 if( a>pEnd ){
1298 rc = SQLITE_CORRUPT_VTAB;
1299 break;
1300 }
1301 pInfo->aMatchinfo[iCol] = (u32)nToken;
1302 }
1303 }
1304 sqlite3_reset(pSelectDocsize);
1305 break;
1306 }
1307
1308 case FTS3_MATCHINFO_LCS:
1309 rc = fts3ExprLoadDoclists(pCsr, 0, 0);
1310 if( rc==SQLITE_OK ){
1311 rc = fts3MatchinfoLcs(pCsr, pInfo);
1312 }
1313 break;
1314
1315 case FTS3_MATCHINFO_LHITS_BM:
1316 case FTS3_MATCHINFO_LHITS: {
1317 size_t nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
1318 memset(pInfo->aMatchinfo, 0, nZero);
1319 rc = fts3ExprLHitGather(pCsr->pExpr, pInfo);
1320 break;
1321 }
1322
1323 default: {
1324 Fts3Expr *pExpr;
1325 assert( zArg[i]==FTS3_MATCHINFO_HITS );
1326 pExpr = pCsr->pExpr;
1327 rc = fts3ExprLoadDoclists(pCsr, 0, 0);
1328 if( rc!=SQLITE_OK ) break;
1329 if( bGlobal ){
1330 if( pCsr->pDeferred ){
1331 rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0);
1332 if( rc!=SQLITE_OK ) break;
1333 }
1334 rc = sqlite3Fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
1335 sqlite3Fts3EvalTestDeferred(pCsr, &rc);
1336 if( rc!=SQLITE_OK ) break;
1337 }
1338 (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
1339 break;
1340 }
1341 }
1342
1343 pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]);
1344 }
1345
1346 sqlite3_reset(pSelect);
1347 return rc;
1348 }
1349
1350
1351 /*
1352 ** Populate pCsr->aMatchinfo[] with data for the current row. The
1353 ** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
1354 */
1355 static void fts3GetMatchinfo(
1356 sqlite3_context *pCtx, /* Return results here */
1357 Fts3Cursor *pCsr, /* FTS3 Cursor object */
1358 const char *zArg /* Second argument to matchinfo() function */
1359 ){
1360 MatchInfo sInfo;
1361 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
1362 int rc = SQLITE_OK;
1363 int bGlobal = 0; /* Collect 'global' stats as well as local */
1364
1365 u32 *aOut = 0;
1366 void (*xDestroyOut)(void*) = 0;
1367
1368 memset(&sInfo, 0, sizeof(MatchInfo));
1369 sInfo.pCursor = pCsr;
1370 sInfo.nCol = pTab->nColumn;
1371
1372 /* If there is cached matchinfo() data, but the format string for the
1373 ** cache does not match the format string for this request, discard
1374 ** the cached data. */
1375 if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){
1376 sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
1377 pCsr->pMIBuffer = 0;
1378 }
1379
1380 /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the
1381 ** matchinfo function has been called for this query. In this case
1382 ** allocate the array used to accumulate the matchinfo data and
1383 ** initialize those elements that are constant for every row.
1384 */
1385 if( pCsr->pMIBuffer==0 ){
1386 size_t nMatchinfo = 0; /* Number of u32 elements in match-info */
1387 int i; /* Used to iterate through zArg */
1388
1389 /* Determine the number of phrases in the query */
1390 pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr);
1391 sInfo.nPhrase = pCsr->nPhrase;
1392
1393 /* Determine the number of integers in the buffer returned by this call. */
1394 for(i=0; zArg[i]; i++){
1395 char *zErr = 0;
1396 if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
1397 sqlite3_result_error(pCtx, zErr, -1);
1398 sqlite3_free(zErr);
1399 return;
1400 }
1401 nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
1402 }
1403
1404 /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
1405 pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg);
1406 if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM;
1407
1408 pCsr->isMatchinfoNeeded = 1;
1409 bGlobal = 1;
1410 }
1411
1412 if( rc==SQLITE_OK ){
1413 xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut);
1414 if( xDestroyOut==0 ){
1415 rc = SQLITE_NOMEM;
1416 }
1417 }
1418
1419 if( rc==SQLITE_OK ){
1420 sInfo.aMatchinfo = aOut;
1421 sInfo.nPhrase = pCsr->nPhrase;
1422 rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
1423 if( bGlobal ){
1424 fts3MIBufferSetGlobal(pCsr->pMIBuffer);
1425 }
1426 }
1427
1428 if( rc!=SQLITE_OK ){
1429 sqlite3_result_error_code(pCtx, rc);
1430 if( xDestroyOut ) xDestroyOut(aOut);
1431 }else{
1432 int n = pCsr->pMIBuffer->nElem * sizeof(u32);
1433 sqlite3_result_blob(pCtx, aOut, n, xDestroyOut);
1434 }
1435 }
1436
1437 /*
1438 ** Implementation of snippet() function.
1439 */
1440 void sqlite3Fts3Snippet(
1441 sqlite3_context *pCtx, /* SQLite function call context */
1442 Fts3Cursor *pCsr, /* Cursor object */
1443 const char *zStart, /* Snippet start text - "<b>" */
1444 const char *zEnd, /* Snippet end text - "</b>" */
1445 const char *zEllipsis, /* Snippet ellipsis text - "<b>...</b>" */
1446 int iCol, /* Extract snippet from this column */
1447 int nToken /* Approximate number of tokens in snippet */
1448 ){
1449 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
1450 int rc = SQLITE_OK;
1451 int i;
1452 StrBuffer res = {0, 0, 0};
1453
1454 /* The returned text includes up to four fragments of text extracted from
1455 ** the data in the current row. The first iteration of the for(...) loop
1456 ** below attempts to locate a single fragment of text nToken tokens in
1457 ** size that contains at least one instance of all phrases in the query
1458 ** expression that appear in the current row. If such a fragment of text
1459 ** cannot be found, the second iteration of the loop attempts to locate
1460 ** a pair of fragments, and so on.
1461 */
1462 int nSnippet = 0; /* Number of fragments in this snippet */
1463 SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */
1464 int nFToken = -1; /* Number of tokens in each fragment */
1465
1466 if( !pCsr->pExpr ){
1467 sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC);
1468 return;
1469 }
1470
1471 /* Limit the snippet length to 64 tokens. */
1472 if( nToken<-64 ) nToken = -64;
1473 if( nToken>+64 ) nToken = +64;
1474
1475 for(nSnippet=1; 1; nSnippet++){
1476
1477 int iSnip; /* Loop counter 0..nSnippet-1 */
1478 u64 mCovered = 0; /* Bitmask of phrases covered by snippet */
1479 u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */
1480
1481 if( nToken>=0 ){
1482 nFToken = (nToken+nSnippet-1) / nSnippet;
1483 }else{
1484 nFToken = -1 * nToken;
1485 }
1486
1487 for(iSnip=0; iSnip<nSnippet; iSnip++){
1488 int iBestScore = -1; /* Best score of columns checked so far */
1489 int iRead; /* Used to iterate through columns */
1490 SnippetFragment *pFragment = &aSnippet[iSnip];
1491
1492 memset(pFragment, 0, sizeof(*pFragment));
1493
1494 /* Loop through all columns of the table being considered for snippets.
1495 ** If the iCol argument to this function was negative, this means all
1496 ** columns of the FTS3 table. Otherwise, only column iCol is considered.
1497 */
1498 for(iRead=0; iRead<pTab->nColumn; iRead++){
1499 SnippetFragment sF = {0, 0, 0, 0};
1500 int iS = 0;
1501 if( iCol>=0 && iRead!=iCol ) continue;
1502
1503 /* Find the best snippet of nFToken tokens in column iRead. */
1504 rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS);
1505 if( rc!=SQLITE_OK ){
1506 goto snippet_out;
1507 }
1508 if( iS>iBestScore ){
1509 *pFragment = sF;
1510 iBestScore = iS;
1511 }
1512 }
1513
1514 mCovered |= pFragment->covered;
1515 }
1516
1517 /* If all query phrases seen by fts3BestSnippet() are present in at least
1518 ** one of the nSnippet snippet fragments, break out of the loop.
1519 */
1520 assert( (mCovered&mSeen)==mCovered );
1521 if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break;
1522 }
1523
1524 assert( nFToken>0 );
1525
1526 for(i=0; i<nSnippet && rc==SQLITE_OK; i++){
1527 rc = fts3SnippetText(pCsr, &aSnippet[i],
1528 i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res
1529 );
1530 }
1531
1532 snippet_out:
1533 sqlite3Fts3SegmentsClose(pTab);
1534 if( rc!=SQLITE_OK ){
1535 sqlite3_result_error_code(pCtx, rc);
1536 sqlite3_free(res.z);
1537 }else{
1538 sqlite3_result_text(pCtx, res.z, -1, sqlite3_free);
1539 }
1540 }
1541
1542
1543 typedef struct TermOffset TermOffset;
1544 typedef struct TermOffsetCtx TermOffsetCtx;
1545
1546 struct TermOffset {
1547 char *pList; /* Position-list */
1548 i64 iPos; /* Position just read from pList */
1549 i64 iOff; /* Offset of this term from read positions */
1550 };
1551
1552 struct TermOffsetCtx {
1553 Fts3Cursor *pCsr;
1554 int iCol; /* Column of table to populate aTerm for */
1555 int iTerm;
1556 sqlite3_int64 iDocid;
1557 TermOffset *aTerm;
1558 };
1559
1560 /*
1561 ** This function is an sqlite3Fts3ExprIterate() callback used by sqlite3Fts3Offsets().
1562 */
1563 static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
1564 TermOffsetCtx *p = (TermOffsetCtx *)ctx;
1565 int nTerm; /* Number of tokens in phrase */
1566 int iTerm; /* For looping through nTerm phrase terms */
1567 char *pList; /* Pointer to position list for phrase */
1568 i64 iPos = 0; /* First position in position-list */
1569 int rc;
1570
1571 UNUSED_PARAMETER(iPhrase);
1572 rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pList);
1573 nTerm = pExpr->pPhrase->nToken;
1574 if( pList ){
1575 fts3GetDeltaPosition(&pList, &iPos);
1576 assert_fts3_nc( iPos>=0 );
1577 }
1578
1579 for(iTerm=0; iTerm<nTerm; iTerm++){
1580 TermOffset *pT = &p->aTerm[p->iTerm++];
1581 pT->iOff = nTerm-iTerm-1;
1582 pT->pList = pList;
1583 pT->iPos = iPos;
1584 }
1585
1586 return rc;
1587 }
1588
1589 /*
1590 ** If expression pExpr is a phrase expression that uses an MSR query,
1591 ** restart it as a regular, non-incremental query. Return SQLITE_OK
1592 ** if successful, or an SQLite error code otherwise.
1593 */
1594 static int fts3ExprRestartIfCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
1595 TermOffsetCtx *p = (TermOffsetCtx*)ctx;
1596 int rc = SQLITE_OK;
1597 UNUSED_PARAMETER(iPhrase);
1598 if( pExpr->pPhrase && pExpr->pPhrase->bIncr ){
1599 rc = sqlite3Fts3MsrCancel(p->pCsr, pExpr);
1600 pExpr->pPhrase->bIncr = 0;
1601 }
1602 return rc;
1603 }
1604
1605 /*
1606 ** Implementation of offsets() function.
1607 */
1608 void sqlite3Fts3Offsets(
1609 sqlite3_context *pCtx, /* SQLite function call context */
1610 Fts3Cursor *pCsr /* Cursor object */
1611 ){
1612 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
1613 sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule;
1614 int rc; /* Return Code */
1615 int nToken; /* Number of tokens in query */
1616 int iCol; /* Column currently being processed */
1617 StrBuffer res = {0, 0, 0}; /* Result string */
1618 TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */
1619
1620 if( !pCsr->pExpr ){
1621 sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC);
1622 return;
1623 }
1624
1625 memset(&sCtx, 0, sizeof(sCtx));
1626 assert( pCsr->isRequireSeek==0 );
1627
1628 /* Count the number of terms in the query */
1629 rc = fts3ExprLoadDoclists(pCsr, 0, &nToken);
1630 if( rc!=SQLITE_OK ) goto offsets_out;
1631
1632 /* Allocate the array of TermOffset iterators. */
1633 sCtx.aTerm = (TermOffset *)sqlite3Fts3MallocZero(sizeof(TermOffset)*nToken);
1634 if( 0==sCtx.aTerm ){
1635 rc = SQLITE_NOMEM;
1636 goto offsets_out;
1637 }
1638 sCtx.iDocid = pCsr->iPrevId;
1639 sCtx.pCsr = pCsr;
1640
1641 /* If a query restart will be required, do it here, rather than later of
1642 ** after pointers to poslist buffers that may be invalidated by a restart
1643 ** have been saved. */
1644 rc = sqlite3Fts3ExprIterate(pCsr->pExpr, fts3ExprRestartIfCb, (void*)&sCtx);
1645 if( rc!=SQLITE_OK ) goto offsets_out;
1646
1647 /* Loop through the table columns, appending offset information to
1648 ** string-buffer res for each column.
1649 */
1650 for(iCol=0; iCol<pTab->nColumn; iCol++){
1651 sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */
1652 const char *ZDUMMY; /* Dummy argument used with xNext() */
1653 int NDUMMY = 0; /* Dummy argument used with xNext() */
1654 int iStart = 0;
1655 int iEnd = 0;
1656 int iCurrent = 0;
1657 const char *zDoc;
1658 int nDoc;
1659
1660 /* Initialize the contents of sCtx.aTerm[] for column iCol. This
1661 ** operation may fail if the database contains corrupt records.
1662 */
1663 sCtx.iCol = iCol;
1664 sCtx.iTerm = 0;
1665 rc = sqlite3Fts3ExprIterate(
1666 pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx
1667 );
1668 if( rc!=SQLITE_OK ) goto offsets_out;
1669
1670 /* Retreive the text stored in column iCol. If an SQL NULL is stored
1671 ** in column iCol, jump immediately to the next iteration of the loop.
1672 ** If an OOM occurs while retrieving the data (this can happen if SQLite
1673 ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM
1674 ** to the caller.
1675 */
1676 zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1);
1677 nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
1678 if( zDoc==0 ){
1679 if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){
1680 continue;
1681 }
1682 rc = SQLITE_NOMEM;
1683 goto offsets_out;
1684 }
1685
1686 /* Initialize a tokenizer iterator to iterate through column iCol. */
1687 rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid,
1688 zDoc, nDoc, &pC
1689 );
1690 if( rc!=SQLITE_OK ) goto offsets_out;
1691
1692 rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
1693 while( rc==SQLITE_OK ){
1694 int i; /* Used to loop through terms */
1695 int iMinPos = 0x7FFFFFFF; /* Position of next token */
1696 TermOffset *pTerm = 0; /* TermOffset associated with next token */
1697
1698 for(i=0; i<nToken; i++){
1699 TermOffset *pT = &sCtx.aTerm[i];
1700 if( pT->pList && (pT->iPos-pT->iOff)<iMinPos ){
1701 iMinPos = pT->iPos-pT->iOff;
1702 pTerm = pT;
1703 }
1704 }
1705
1706 if( !pTerm ){
1707 /* All offsets for this column have been gathered. */
1708 rc = SQLITE_DONE;
1709 }else{
1710 assert_fts3_nc( iCurrent<=iMinPos );
1711 if( 0==(0xFE&*pTerm->pList) ){
1712 pTerm->pList = 0;
1713 }else{
1714 fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos);
1715 }
1716 while( rc==SQLITE_OK && iCurrent<iMinPos ){
1717 rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
1718 }
1719 if( rc==SQLITE_OK ){
1720 char aBuffer[64];
1721 sqlite3_snprintf(sizeof(aBuffer), aBuffer,
1722 "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart
1723 );
1724 rc = fts3StringAppend(&res, aBuffer, -1);
1725 }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){
1726 rc = FTS_CORRUPT_VTAB;
1727 }
1728 }
1729 }
1730 if( rc==SQLITE_DONE ){
1731 rc = SQLITE_OK;
1732 }
1733
1734 pMod->xClose(pC);
1735 if( rc!=SQLITE_OK ) goto offsets_out;
1736 }
1737
1738 offsets_out:
1739 sqlite3_free(sCtx.aTerm);
1740 assert( rc!=SQLITE_DONE );
1741 sqlite3Fts3SegmentsClose(pTab);
1742 if( rc!=SQLITE_OK ){
1743 sqlite3_result_error_code(pCtx, rc);
1744 sqlite3_free(res.z);
1745 }else{
1746 sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free);
1747 }
1748 return;
1749 }
1750
1751 /*
1752 ** Implementation of matchinfo() function.
1753 */
1754 void sqlite3Fts3Matchinfo(
1755 sqlite3_context *pContext, /* Function call context */
1756 Fts3Cursor *pCsr, /* FTS3 table cursor */
1757 const char *zArg /* Second arg to matchinfo() function */
1758 ){
1759 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
1760 const char *zFormat;
1761
1762 if( zArg ){
1763 zFormat = zArg;
1764 }else{
1765 zFormat = FTS3_MATCHINFO_DEFAULT;
1766 }
1767
1768 if( !pCsr->pExpr ){
1769 sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
1770 return;
1771 }else{
1772 /* Retrieve matchinfo() data. */
1773 fts3GetMatchinfo(pContext, pCsr, zFormat);
1774 sqlite3Fts3SegmentsClose(pTab);
1775 }
1776 }
1777
1778 #endif