Debian Codegraph @

ext/fts5/fts5_main.c

raw
1 /*
2 ** 2014 Jun 09
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 **
13 ** This is an SQLite module implementing full-text search.
14 */
15
16
17 #include "fts5Int.h"
18
19 /*
20 ** This variable is set to false when running tests for which the on disk
21 ** structures should not be corrupt. Otherwise, true. If it is false, extra
22 ** assert() conditions in the fts5 code are activated - conditions that are
23 ** only true if it is guaranteed that the fts5 database is not corrupt.
24 */
25 #ifdef SQLITE_DEBUG
26 int sqlite3_fts5_may_be_corrupt = 1;
27 #endif
28
29
30 typedef struct Fts5Auxdata Fts5Auxdata;
31 typedef struct Fts5Auxiliary Fts5Auxiliary;
32 typedef struct Fts5Cursor Fts5Cursor;
33 typedef struct Fts5FullTable Fts5FullTable;
34 typedef struct Fts5Sorter Fts5Sorter;
35 typedef struct Fts5TokenizerModule Fts5TokenizerModule;
36
37 /*
38 ** NOTES ON TRANSACTIONS:
39 **
40 ** SQLite invokes the following virtual table methods as transactions are
41 ** opened and closed by the user:
42 **
43 ** xBegin(): Start of a new transaction.
44 ** xSync(): Initial part of two-phase commit.
45 ** xCommit(): Final part of two-phase commit.
46 ** xRollback(): Rollback the transaction.
47 **
48 ** Anything that is required as part of a commit that may fail is performed
49 ** in the xSync() callback. Current versions of SQLite ignore any errors
50 ** returned by xCommit().
51 **
52 ** And as sub-transactions are opened/closed:
53 **
54 ** xSavepoint(int S): Open savepoint S.
55 ** xRelease(int S): Commit and close savepoint S.
56 ** xRollbackTo(int S): Rollback to start of savepoint S.
57 **
58 ** During a write-transaction the fts5_index.c module may cache some data
59 ** in-memory. It is flushed to disk whenever xSync(), xRelease() or
60 ** xSavepoint() is called. And discarded whenever xRollback() or xRollbackTo()
61 ** is called.
62 **
63 ** Additionally, if SQLITE_DEBUG is defined, an instance of the following
64 ** structure is used to record the current transaction state. This information
65 ** is not required, but it is used in the assert() statements executed by
66 ** function fts5CheckTransactionState() (see below).
67 */
68 struct Fts5TransactionState {
69 int eState; /* 0==closed, 1==open, 2==synced */
70 int iSavepoint; /* Number of open savepoints (0 -> none) */
71 };
72
73 /*
74 ** A single object of this type is allocated when the FTS5 module is
75 ** registered with a database handle. It is used to store pointers to
76 ** all registered FTS5 extensions - tokenizers and auxiliary functions.
77 */
78 struct Fts5Global {
79 fts5_api api; /* User visible part of object (see fts5.h) */
80 sqlite3 *db; /* Associated database connection */
81 i64 iNextId; /* Used to allocate unique cursor ids */
82 Fts5Auxiliary *pAux; /* First in list of all aux. functions */
83 Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */
84 Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */
85 Fts5Cursor *pCsr; /* First in list of all open cursors */
86 u32 aLocaleHdr[4];
87 };
88
89 /*
90 ** Size of header on fts5_locale() values. And macro to access a buffer
91 ** containing a copy of the header from an Fts5Config pointer.
92 */
93 #define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ))
94 #define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr))
95
96 #define FTS5_INSTTOKEN_SUBTYPE 73
97
98 /*
99 ** Each auxiliary function registered with the FTS5 module is represented
100 ** by an object of the following type. All such objects are stored as part
101 ** of the Fts5Global.pAux list.
102 */
103 struct Fts5Auxiliary {
104 Fts5Global *pGlobal; /* Global context for this function */
105 char *zFunc; /* Function name (nul-terminated) */
106 void *pUserData; /* User-data pointer */
107 fts5_extension_function xFunc; /* Callback function */
108 void (*xDestroy)(void*); /* Destructor function */
109 Fts5Auxiliary *pNext; /* Next registered auxiliary function */
110 };
111
112 /*
113 ** Each tokenizer module registered with the FTS5 module is represented
114 ** by an object of the following type. All such objects are stored as part
115 ** of the Fts5Global.pTok list.
116 **
117 ** bV2Native:
118 ** True if the tokenizer was registered using xCreateTokenizer_v2(), false
119 ** for xCreateTokenizer(). If this variable is true, then x2 is populated
120 ** with the routines as supplied by the caller and x1 contains synthesized
121 ** wrapper routines. In this case the user-data pointer passed to
122 ** x1.xCreate should be a pointer to the Fts5TokenizerModule structure,
123 ** not a copy of pUserData.
124 **
125 ** Of course, if bV2Native is false, then x1 contains the real routines and
126 ** x2 the synthesized ones. In this case a pointer to the Fts5TokenizerModule
127 ** object should be passed to x2.xCreate.
128 **
129 ** The synthesized wrapper routines are necessary for xFindTokenizer(_v2)
130 ** calls.
131 */
132 struct Fts5TokenizerModule {
133 char *zName; /* Name of tokenizer */
134 void *pUserData; /* User pointer passed to xCreate() */
135 int bV2Native; /* True if v2 native tokenizer */
136 fts5_tokenizer x1; /* Tokenizer functions */
137 fts5_tokenizer_v2 x2; /* V2 tokenizer functions */
138 void (*xDestroy)(void*); /* Destructor function */
139 Fts5TokenizerModule *pNext; /* Next registered tokenizer module */
140 };
141
142 struct Fts5FullTable {
143 Fts5Table p; /* Public class members from fts5Int.h */
144 Fts5Storage *pStorage; /* Document store */
145 Fts5Global *pGlobal; /* Global (connection wide) data */
146 Fts5Cursor *pSortCsr; /* Sort data from this cursor */
147 int iSavepoint; /* Successful xSavepoint()+1 */
148
149 #ifdef SQLITE_DEBUG
150 struct Fts5TransactionState ts;
151 #endif
152 };
153
154 struct Fts5MatchPhrase {
155 Fts5Buffer *pPoslist; /* Pointer to current poslist */
156 int nTerm; /* Size of phrase in terms */
157 };
158
159 /*
160 ** pStmt:
161 ** SELECT rowid, <fts> FROM <fts> ORDER BY +rank;
162 **
163 ** aIdx[]:
164 ** There is one entry in the aIdx[] array for each phrase in the query,
165 ** the value of which is the offset within aPoslist[] following the last
166 ** byte of the position list for the corresponding phrase.
167 */
168 struct Fts5Sorter {
169 sqlite3_stmt *pStmt;
170 i64 iRowid; /* Current rowid */
171 const u8 *aPoslist; /* Position lists for current row */
172 int nIdx; /* Number of entries in aIdx[] */
173 int aIdx[FLEXARRAY]; /* Offsets into aPoslist for current row */
174 };
175
176 /* Size (int bytes) of an Fts5Sorter object with N indexes */
177 #define SZ_FTS5SORTER(N) (offsetof(Fts5Sorter,nIdx)+((N+2)/2)*sizeof(i64))
178
179 /*
180 ** Virtual-table cursor object.
181 **
182 ** iSpecial:
183 ** If this is a 'special' query (refer to function fts5SpecialMatch()),
184 ** then this variable contains the result of the query.
185 **
186 ** iFirstRowid, iLastRowid:
187 ** These variables are only used for FTS5_PLAN_MATCH cursors. Assuming the
188 ** cursor iterates in ascending order of rowids, iFirstRowid is the lower
189 ** limit of rowids to return, and iLastRowid the upper. In other words, the
190 ** WHERE clause in the user's query might have been:
191 **
192 ** <tbl> MATCH <expr> AND rowid BETWEEN $iFirstRowid AND $iLastRowid
193 **
194 ** If the cursor iterates in descending order of rowid, iFirstRowid
195 ** is the upper limit (i.e. the "first" rowid visited) and iLastRowid
196 ** the lower.
197 */
198 struct Fts5Cursor {
199 sqlite3_vtab_cursor base; /* Base class used by SQLite core */
200 Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */
201 int *aColumnSize; /* Values for xColumnSize() */
202 i64 iCsrId; /* Cursor id */
203
204 /* Zero from this point onwards on cursor reset */
205 int ePlan; /* FTS5_PLAN_XXX value */
206 int bDesc; /* True for "ORDER BY rowid DESC" queries */
207 i64 iFirstRowid; /* Return no rowids earlier than this */
208 i64 iLastRowid; /* Return no rowids later than this */
209 sqlite3_stmt *pStmt; /* Statement used to read %_content */
210 Fts5Expr *pExpr; /* Expression for MATCH queries */
211 Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */
212 int csrflags; /* Mask of cursor flags (see below) */
213 i64 iSpecial; /* Result of special query */
214
215 /* "rank" function. Populated on demand from vtab.xColumn(). */
216 char *zRank; /* Custom rank function */
217 char *zRankArgs; /* Custom rank function args */
218 Fts5Auxiliary *pRank; /* Rank callback (or NULL) */
219 int nRankArg; /* Number of trailing arguments for rank() */
220 sqlite3_value **apRankArg; /* Array of trailing arguments */
221 sqlite3_stmt *pRankArgStmt; /* Origin of objects in apRankArg[] */
222
223 /* Auxiliary data storage */
224 Fts5Auxiliary *pAux; /* Currently executing extension function */
225 Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */
226
227 /* Cache used by auxiliary API functions xInst() and xInstCount() */
228 Fts5PoslistReader *aInstIter; /* One for each phrase */
229 int nInstAlloc; /* Size of aInst[] array (entries / 3) */
230 int nInstCount; /* Number of phrase instances */
231 int *aInst; /* 3 integers per phrase instance */
232 };
233
234 /*
235 ** Bits that make up the "idxNum" parameter passed indirectly by
236 ** xBestIndex() to xFilter().
237 */
238 #define FTS5_BI_MATCH 0x0001 /* <tbl> MATCH ? */
239 #define FTS5_BI_RANK 0x0002 /* rank MATCH ? */
240 #define FTS5_BI_ROWID_EQ 0x0004 /* rowid == ? */
241 #define FTS5_BI_ROWID_LE 0x0008 /* rowid <= ? */
242 #define FTS5_BI_ROWID_GE 0x0010 /* rowid >= ? */
243
244 #define FTS5_BI_ORDER_RANK 0x0020
245 #define FTS5_BI_ORDER_ROWID 0x0040
246 #define FTS5_BI_ORDER_DESC 0x0080
247
248 /*
249 ** Values for Fts5Cursor.csrflags
250 */
251 #define FTS5CSR_EOF 0x01
252 #define FTS5CSR_REQUIRE_CONTENT 0x02
253 #define FTS5CSR_REQUIRE_DOCSIZE 0x04
254 #define FTS5CSR_REQUIRE_INST 0x08
255 #define FTS5CSR_FREE_ZRANK 0x10
256 #define FTS5CSR_REQUIRE_RESEEK 0x20
257 #define FTS5CSR_REQUIRE_POSLIST 0x40
258
259 #define BitFlagAllTest(x,y) (((x) & (y))==(y))
260 #define BitFlagTest(x,y) (((x) & (y))!=0)
261
262
263 /*
264 ** Macros to Set(), Clear() and Test() cursor flags.
265 */
266 #define CsrFlagSet(pCsr, flag) ((pCsr)->csrflags |= (flag))
267 #define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag))
268 #define CsrFlagTest(pCsr, flag) ((pCsr)->csrflags & (flag))
269
270 struct Fts5Auxdata {
271 Fts5Auxiliary *pAux; /* Extension to which this belongs */
272 void *pPtr; /* Pointer value */
273 void(*xDelete)(void*); /* Destructor */
274 Fts5Auxdata *pNext; /* Next object in linked list */
275 };
276
277 #ifdef SQLITE_DEBUG
278 #define FTS5_BEGIN 1
279 #define FTS5_SYNC 2
280 #define FTS5_COMMIT 3
281 #define FTS5_ROLLBACK 4
282 #define FTS5_SAVEPOINT 5
283 #define FTS5_RELEASE 6
284 #define FTS5_ROLLBACKTO 7
285 static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){
286 switch( op ){
287 case FTS5_BEGIN:
288 assert( p->ts.eState==0 );
289 p->ts.eState = 1;
290 p->ts.iSavepoint = -1;
291 break;
292
293 case FTS5_SYNC:
294 assert( p->ts.eState==1 || p->ts.eState==2 );
295 p->ts.eState = 2;
296 break;
297
298 case FTS5_COMMIT:
299 assert( p->ts.eState==2 );
300 p->ts.eState = 0;
301 break;
302
303 case FTS5_ROLLBACK:
304 assert( p->ts.eState==1 || p->ts.eState==2 || p->ts.eState==0 );
305 p->ts.eState = 0;
306 break;
307
308 case FTS5_SAVEPOINT:
309 assert( p->ts.eState>=1 );
310 assert( iSavepoint>=0 );
311 assert( iSavepoint>=p->ts.iSavepoint );
312 p->ts.iSavepoint = iSavepoint;
313 break;
314
315 case FTS5_RELEASE:
316 assert( p->ts.eState>=1 );
317 assert( iSavepoint>=0 );
318 assert( iSavepoint<=p->ts.iSavepoint );
319 p->ts.iSavepoint = iSavepoint-1;
320 break;
321
322 case FTS5_ROLLBACKTO:
323 assert( p->ts.eState>=1 );
324 assert( iSavepoint>=-1 );
325 /* The following assert() can fail if another vtab strikes an error
326 ** within an xSavepoint() call then SQLite calls xRollbackTo() - without
327 ** having called xSavepoint() on this vtab. */
328 /* assert( iSavepoint<=p->ts.iSavepoint ); */
329 p->ts.iSavepoint = iSavepoint;
330 break;
331 }
332 }
333 #else
334 # define fts5CheckTransactionState(x,y,z)
335 #endif
336
337 /*
338 ** Return true if pTab is a contentless table. If parameter bIncludeUnindexed
339 ** is true, this includes contentless tables that store UNINDEXED columns
340 ** only.
341 */
342 static int fts5IsContentless(Fts5FullTable *pTab, int bIncludeUnindexed){
343 int eContent = pTab->p.pConfig->eContent;
344 return (
345 eContent==FTS5_CONTENT_NONE
346 || (bIncludeUnindexed && eContent==FTS5_CONTENT_UNINDEXED)
347 );
348 }
349
350 /*
351 ** Delete a virtual table handle allocated by fts5InitVtab().
352 */
353 static void fts5FreeVtab(Fts5FullTable *pTab){
354 if( pTab ){
355 sqlite3Fts5IndexClose(pTab->p.pIndex);
356 sqlite3Fts5StorageClose(pTab->pStorage);
357 sqlite3Fts5ConfigFree(pTab->p.pConfig);
358 sqlite3_free(pTab);
359 }
360 }
361
362 /*
363 ** The xDisconnect() virtual table method.
364 */
365 static int fts5DisconnectMethod(sqlite3_vtab *pVtab){
366 fts5FreeVtab((Fts5FullTable*)pVtab);
367 return SQLITE_OK;
368 }
369
370 /*
371 ** The xDestroy() virtual table method.
372 */
373 static int fts5DestroyMethod(sqlite3_vtab *pVtab){
374 Fts5Table *pTab = (Fts5Table*)pVtab;
375 int rc = sqlite3Fts5DropAll(pTab->pConfig);
376 if( rc==SQLITE_OK ){
377 fts5FreeVtab((Fts5FullTable*)pVtab);
378 }
379 return rc;
380 }
381
382 /*
383 ** This function is the implementation of both the xConnect and xCreate
384 ** methods of the FTS3 virtual table.
385 **
386 ** The argv[] array contains the following:
387 **
388 ** argv[0] -> module name ("fts5")
389 ** argv[1] -> database name
390 ** argv[2] -> table name
391 ** argv[...] -> "column name" and other module argument fields.
392 */
393 static int fts5InitVtab(
394 int bCreate, /* True for xCreate, false for xConnect */
395 sqlite3 *db, /* The SQLite database connection */
396 void *pAux, /* Hash table containing tokenizers */
397 int argc, /* Number of elements in argv array */
398 const char * const *argv, /* xCreate/xConnect argument array */
399 sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */
400 char **pzErr /* Write any error message here */
401 ){
402 Fts5Global *pGlobal = (Fts5Global*)pAux;
403 const char **azConfig = (const char**)argv;
404 int rc = SQLITE_OK; /* Return code */
405 Fts5Config *pConfig = 0; /* Results of parsing argc/argv */
406 Fts5FullTable *pTab = 0; /* New virtual table object */
407
408 /* Allocate the new vtab object and parse the configuration */
409 pTab = (Fts5FullTable*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5FullTable));
410 if( rc==SQLITE_OK ){
411 rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr);
412 assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
413 }
414 if( rc==SQLITE_OK ){
415 pConfig->pzErrmsg = pzErr;
416 pTab->p.pConfig = pConfig;
417 pTab->pGlobal = pGlobal;
418 if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){
419 rc = sqlite3Fts5LoadTokenizer(pConfig);
420 }
421 }
422
423 /* Open the index sub-system */
424 if( rc==SQLITE_OK ){
425 rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->p.pIndex, pzErr);
426 }
427
428 /* Open the storage sub-system */
429 if( rc==SQLITE_OK ){
430 rc = sqlite3Fts5StorageOpen(
431 pConfig, pTab->p.pIndex, bCreate, &pTab->pStorage, pzErr
432 );
433 }
434
435 /* Call sqlite3_declare_vtab() */
436 if( rc==SQLITE_OK ){
437 rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
438 }
439
440 /* Load the initial configuration */
441 if( rc==SQLITE_OK ){
442 rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie-1);
443 }
444
445 if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
446 rc = sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, (int)1);
447 }
448 if( rc==SQLITE_OK ){
449 rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS);
450 }
451
452 if( pConfig ) pConfig->pzErrmsg = 0;
453 if( rc!=SQLITE_OK ){
454 fts5FreeVtab(pTab);
455 pTab = 0;
456 }else if( bCreate ){
457 fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
458 }
459 *ppVTab = (sqlite3_vtab*)pTab;
460 return rc;
461 }
462
463 /*
464 ** The xConnect() and xCreate() methods for the virtual table. All the
465 ** work is done in function fts5InitVtab().
466 */
467 static int fts5ConnectMethod(
468 sqlite3 *db, /* Database connection */
469 void *pAux, /* Pointer to tokenizer hash table */
470 int argc, /* Number of elements in argv array */
471 const char * const *argv, /* xCreate/xConnect argument array */
472 sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
473 char **pzErr /* OUT: sqlite3_malloc'd error message */
474 ){
475 return fts5InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr);
476 }
477 static int fts5CreateMethod(
478 sqlite3 *db, /* Database connection */
479 void *pAux, /* Pointer to tokenizer hash table */
480 int argc, /* Number of elements in argv array */
481 const char * const *argv, /* xCreate/xConnect argument array */
482 sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
483 char **pzErr /* OUT: sqlite3_malloc'd error message */
484 ){
485 return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
486 }
487
488 /*
489 ** The different query plans.
490 */
491 #define FTS5_PLAN_MATCH 1 /* (<tbl> MATCH ?) */
492 #define FTS5_PLAN_SOURCE 2 /* A source cursor for SORTED_MATCH */
493 #define FTS5_PLAN_SPECIAL 3 /* An internal query */
494 #define FTS5_PLAN_SORTED_MATCH 4 /* (<tbl> MATCH ? ORDER BY rank) */
495 #define FTS5_PLAN_SCAN 5 /* No usable constraint */
496 #define FTS5_PLAN_ROWID 6 /* (rowid = ?) */
497
498 /*
499 ** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this
500 ** extension is currently being used by a version of SQLite too old to
501 ** support index-info flags. In that case this function is a no-op.
502 */
503 static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){
504 #if SQLITE_VERSION_NUMBER>=3008012
505 #ifndef SQLITE_CORE
506 if( sqlite3_libversion_number()>=3008012 )
507 #endif
508 {
509 pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE;
510 }
511 #endif
512 }
513
514 static void fts5SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){
515 #if SQLITE_VERSION_NUMBER>=3008002
516 #ifndef SQLITE_CORE
517 if( sqlite3_libversion_number()>=3008002 )
518 #endif
519 {
520 pIdxInfo->estimatedRows = MAX(1, nRow);
521 }
522 #endif
523 }
524
525 static int fts5UsePatternMatch(
526 Fts5Config *pConfig,
527 struct sqlite3_index_constraint *p
528 ){
529 assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB );
530 assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE );
531 if( pConfig->t.ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){
532 return 1;
533 }
534 if( pConfig->t.ePattern==FTS5_PATTERN_LIKE
535 && (p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB)
536 ){
537 return 1;
538 }
539 return 0;
540 }
541
542 /*
543 ** Implementation of the xBestIndex method for FTS5 tables. Within the
544 ** WHERE constraint, it searches for the following:
545 **
546 ** 1. A MATCH constraint against the table column.
547 ** 2. A MATCH constraint against the "rank" column.
548 ** 3. A MATCH constraint against some other column.
549 ** 4. An == constraint against the rowid column.
550 ** 5. A < or <= constraint against the rowid column.
551 ** 6. A > or >= constraint against the rowid column.
552 **
553 ** Within the ORDER BY, the following are supported:
554 **
555 ** 5. ORDER BY rank [ASC|DESC]
556 ** 6. ORDER BY rowid [ASC|DESC]
557 **
558 ** Information for the xFilter call is passed via both the idxNum and
559 ** idxStr variables. Specifically, idxNum is a bitmask of the following
560 ** flags used to encode the ORDER BY clause:
561 **
562 ** FTS5_BI_ORDER_RANK
563 ** FTS5_BI_ORDER_ROWID
564 ** FTS5_BI_ORDER_DESC
565 **
566 ** idxStr is used to encode data from the WHERE clause. For each argument
567 ** passed to the xFilter method, the following is appended to idxStr:
568 **
569 ** Match against table column: "m"
570 ** Match against rank column: "r"
571 ** Match against other column: "M<column-number>"
572 ** LIKE against other column: "L<column-number>"
573 ** GLOB against other column: "G<column-number>"
574 ** Equality constraint against the rowid: "="
575 ** A < or <= against the rowid: "<"
576 ** A > or >= against the rowid: ">"
577 **
578 ** This function ensures that there is at most one "r" or "=". And that if
579 ** there exists an "=" then there is no "<" or ">".
580 **
581 ** If an unusable MATCH operator is present in the WHERE clause, then
582 ** SQLITE_CONSTRAINT is returned.
583 **
584 ** Costs are assigned as follows:
585 **
586 ** a) If a MATCH operator is present, the cost depends on the other
587 ** constraints also present. As follows:
588 **
589 ** * No other constraints: cost=50000.0
590 ** * One rowid range constraint: cost=37500.0
591 ** * Both rowid range constraints: cost=30000.0
592 ** * An == rowid constraint: cost=25000.0
593 **
594 ** b) Otherwise, if there is no MATCH:
595 **
596 ** * No other constraints: cost=3000000.0
597 ** * One rowid range constraints: cost=2250000.0
598 ** * Both rowid range constraint: cost=750000.0
599 ** * An == rowid constraint: cost=25.0
600 **
601 ** Costs are not modified by the ORDER BY clause.
602 **
603 ** The ratios used in case (a) are based on informal results obtained from
604 ** the tool/fts5cost.tcl script. The "MATCH and ==" combination has the
605 ** cost set quite high because the query may be a prefix query. Unless
606 ** there is a prefix index, prefix queries with rowid constraints are much
607 ** more expensive than non-prefix queries with rowid constraints.
608 **
609 ** The estimated rows returned is set to the cost/40. For simple queries,
610 ** experimental results show that cost/4 might be about right. But for
611 ** more complex queries that use multiple terms the number of rows might
612 ** be far fewer than this. So we compromise and use cost/40.
613 */
614 static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
615 Fts5Table *pTab = (Fts5Table*)pVTab;
616 Fts5Config *pConfig = pTab->pConfig;
617 const int nCol = pConfig->nCol;
618 int idxFlags = 0; /* Parameter passed through to xFilter() */
619 int i;
620
621 char *idxStr;
622 int iIdxStr = 0;
623 int iCons = 0;
624
625 int bSeenEq = 0;
626 int bSeenGt = 0;
627 int bSeenLt = 0;
628 int nSeenMatch = 0;
629 int bSeenRank = 0;
630
631
632 assert( SQLITE_INDEX_CONSTRAINT_EQ<SQLITE_INDEX_CONSTRAINT_MATCH );
633 assert( SQLITE_INDEX_CONSTRAINT_GT<SQLITE_INDEX_CONSTRAINT_MATCH );
634 assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
635 assert( SQLITE_INDEX_CONSTRAINT_GE<SQLITE_INDEX_CONSTRAINT_MATCH );
636 assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
637
638 if( pConfig->bLock ){
639 pTab->base.zErrMsg = sqlite3_mprintf(
640 "recursively defined fts5 content table"
641 );
642 return SQLITE_ERROR;
643 }
644
645 idxStr = (char*)sqlite3_malloc64((i64)pInfo->nConstraint * 8 + 1);
646 if( idxStr==0 ) return SQLITE_NOMEM;
647 pInfo->idxStr = idxStr;
648 pInfo->needToFreeIdxStr = 1;
649
650 for(i=0; i<pInfo->nConstraint; i++){
651 struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
652 int iCol = p->iColumn;
653 if( p->op==SQLITE_INDEX_CONSTRAINT_MATCH
654 || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol>=nCol)
655 ){
656 /* A MATCH operator or equivalent */
657 if( p->usable==0 || iCol<0 ){
658 /* As there exists an unusable MATCH constraint this is an
659 ** unusable plan. Return SQLITE_CONSTRAINT. */
660 idxStr[iIdxStr] = 0;
661 return SQLITE_CONSTRAINT;
662 }else{
663 if( iCol==nCol+1 ){
664 if( bSeenRank ) continue;
665 idxStr[iIdxStr++] = 'r';
666 bSeenRank = 1;
667 }else{
668 nSeenMatch++;
669 idxStr[iIdxStr++] = 'M';
670 sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol);
671 iIdxStr += (int)strlen(&idxStr[iIdxStr]);
672 assert( idxStr[iIdxStr]=='\0' );
673 }
674 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
675 pInfo->aConstraintUsage[i].omit = 1;
676 }
677 }else if( p->usable ){
678 if( iCol>=0 && iCol<nCol && fts5UsePatternMatch(pConfig, p) ){
679 assert( p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB );
680 idxStr[iIdxStr++] = p->op==FTS5_PATTERN_LIKE ? 'L' : 'G';
681 sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol);
682 idxStr += strlen(&idxStr[iIdxStr]);
683 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
684 assert( idxStr[iIdxStr]=='\0' );
685 nSeenMatch++;
686 }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 ){
687 idxStr[iIdxStr++] = '=';
688 bSeenEq = 1;
689 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
690 pInfo->aConstraintUsage[i].omit = 1;
691 }
692 }
693 }
694
695 if( bSeenEq==0 ){
696 for(i=0; i<pInfo->nConstraint; i++){
697 struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
698 if( p->iColumn<0 && p->usable ){
699 int op = p->op;
700 if( op==SQLITE_INDEX_CONSTRAINT_LT || op==SQLITE_INDEX_CONSTRAINT_LE ){
701 if( bSeenLt ) continue;
702 idxStr[iIdxStr++] = '<';
703 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
704 bSeenLt = 1;
705 }else
706 if( op==SQLITE_INDEX_CONSTRAINT_GT || op==SQLITE_INDEX_CONSTRAINT_GE ){
707 if( bSeenGt ) continue;
708 idxStr[iIdxStr++] = '>';
709 pInfo->aConstraintUsage[i].argvIndex = ++iCons;
710 bSeenGt = 1;
711 }
712 }
713 }
714 }
715 idxStr[iIdxStr] = '\0';
716
717 /* Set idxFlags flags for the ORDER BY clause
718 **
719 ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC".
720 */
721 if( pInfo->nOrderBy==1 ){
722 int iSort = pInfo->aOrderBy[0].iColumn;
723 if( iSort==(pConfig->nCol+1) && nSeenMatch>0 ){
724 idxFlags |= FTS5_BI_ORDER_RANK;
725 }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){
726 idxFlags |= FTS5_BI_ORDER_ROWID;
727 }
728 if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){
729 pInfo->orderByConsumed = 1;
730 if( pInfo->aOrderBy[0].desc ){
731 idxFlags |= FTS5_BI_ORDER_DESC;
732 }
733 }
734 }
735
736 /* Calculate the estimated cost based on the flags set in idxFlags. */
737 if( bSeenEq ){
738 pInfo->estimatedCost = nSeenMatch ? 25000.0 : 25.0;
739 fts5SetEstimatedRows(pInfo, 1);
740 fts5SetUniqueFlag(pInfo);
741 }else{
742 i64 nEstRows;
743 if( nSeenMatch ){
744 if( bSeenLt && bSeenGt ){
745 pInfo->estimatedCost = 50000.0;
746 }else if( bSeenLt || bSeenGt ){
747 pInfo->estimatedCost = 37500.0;
748 }else{
749 pInfo->estimatedCost = 50000.0;
750 }
751 nEstRows = (i64)(pInfo->estimatedCost / 40.0);
752 for(i=1; i<nSeenMatch; i++){
753 pInfo->estimatedCost *= 2.5;
754 nEstRows = nEstRows / 2;
755 }
756 }else{
757 if( bSeenLt && bSeenGt ){
758 pInfo->estimatedCost = 750000.0;
759 }else if( bSeenLt || bSeenGt ){
760 pInfo->estimatedCost = 2250000.0;
761 }else{
762 pInfo->estimatedCost = 3000000.0;
763 }
764 nEstRows = (i64)(pInfo->estimatedCost / 4.0);
765 }
766 fts5SetEstimatedRows(pInfo, nEstRows);
767 }
768
769 pInfo->idxNum = idxFlags;
770 return SQLITE_OK;
771 }
772
773 static int fts5NewTransaction(Fts5FullTable *pTab){
774 Fts5Cursor *pCsr;
775 for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
776 if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
777 }
778 return sqlite3Fts5StorageReset(pTab->pStorage);
779 }
780
781 /*
782 ** Implementation of xOpen method.
783 */
784 static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
785 Fts5FullTable *pTab = (Fts5FullTable*)pVTab;
786 Fts5Config *pConfig = pTab->p.pConfig;
787 Fts5Cursor *pCsr = 0; /* New cursor object */
788 sqlite3_int64 nByte; /* Bytes of space to allocate */
789 int rc; /* Return code */
790
791 rc = fts5NewTransaction(pTab);
792 if( rc==SQLITE_OK ){
793 nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
794 pCsr = (Fts5Cursor*)sqlite3_malloc64(nByte);
795 if( pCsr ){
796 Fts5Global *pGlobal = pTab->pGlobal;
797 memset(pCsr, 0, (size_t)nByte);
798 pCsr->aColumnSize = (int*)&pCsr[1];
799 pCsr->pNext = pGlobal->pCsr;
800 pGlobal->pCsr = pCsr;
801 pCsr->iCsrId = ++pGlobal->iNextId;
802 }else{
803 rc = SQLITE_NOMEM;
804 }
805 }
806 *ppCsr = (sqlite3_vtab_cursor*)pCsr;
807 return rc;
808 }
809
810 static int fts5StmtType(Fts5Cursor *pCsr){
811 if( pCsr->ePlan==FTS5_PLAN_SCAN ){
812 return (pCsr->bDesc) ? FTS5_STMT_SCAN_DESC : FTS5_STMT_SCAN_ASC;
813 }
814 return FTS5_STMT_LOOKUP;
815 }
816
817 /*
818 ** This function is called after the cursor passed as the only argument
819 ** is moved to point at a different row. It clears all cached data
820 ** specific to the previous row stored by the cursor object.
821 */
822 static void fts5CsrNewrow(Fts5Cursor *pCsr){
823 CsrFlagSet(pCsr,
824 FTS5CSR_REQUIRE_CONTENT
825 | FTS5CSR_REQUIRE_DOCSIZE
826 | FTS5CSR_REQUIRE_INST
827 | FTS5CSR_REQUIRE_POSLIST
828 );
829 }
830
831 static void fts5FreeCursorComponents(Fts5Cursor *pCsr){
832 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
833 Fts5Auxdata *pData;
834 Fts5Auxdata *pNext;
835
836 sqlite3_free(pCsr->aInstIter);
837 sqlite3_free(pCsr->aInst);
838 if( pCsr->pStmt ){
839 int eStmt = fts5StmtType(pCsr);
840 sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
841 }
842 if( pCsr->pSorter ){
843 Fts5Sorter *pSorter = pCsr->pSorter;
844 sqlite3_finalize(pSorter->pStmt);
845 sqlite3_free(pSorter);
846 }
847
848 if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){
849 sqlite3Fts5ExprFree(pCsr->pExpr);
850 }
851
852 for(pData=pCsr->pAuxdata; pData; pData=pNext){
853 pNext = pData->pNext;
854 if( pData->xDelete ) pData->xDelete(pData->pPtr);
855 sqlite3_free(pData);
856 }
857
858 sqlite3_finalize(pCsr->pRankArgStmt);
859 sqlite3_free(pCsr->apRankArg);
860
861 if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
862 sqlite3_free(pCsr->zRank);
863 sqlite3_free(pCsr->zRankArgs);
864 }
865
866 sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
867 memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr));
868 }
869
870
871 /*
872 ** Close the cursor. For additional information see the documentation
873 ** on the xClose method of the virtual table interface.
874 */
875 static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
876 if( pCursor ){
877 Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab);
878 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
879 Fts5Cursor **pp;
880
881 fts5FreeCursorComponents(pCsr);
882 /* Remove the cursor from the Fts5Global.pCsr list */
883 for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
884 *pp = pCsr->pNext;
885
886 sqlite3_free(pCsr);
887 }
888 return SQLITE_OK;
889 }
890
891 static int fts5SorterNext(Fts5Cursor *pCsr){
892 Fts5Sorter *pSorter = pCsr->pSorter;
893 int rc;
894
895 rc = sqlite3_step(pSorter->pStmt);
896 if( rc==SQLITE_DONE ){
897 rc = SQLITE_OK;
898 CsrFlagSet(pCsr, FTS5CSR_EOF|FTS5CSR_REQUIRE_CONTENT);
899 }else if( rc==SQLITE_ROW ){
900 const u8 *a;
901 const u8 *aBlob;
902 int nBlob;
903 int i;
904 int iOff = 0;
905 rc = SQLITE_OK;
906
907 pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0);
908 nBlob = sqlite3_column_bytes(pSorter->pStmt, 1);
909 aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1);
910
911 /* nBlob==0 in detail=none mode. */
912 if( nBlob>0 ){
913 for(i=0; i<(pSorter->nIdx-1); i++){
914 int iVal;
915 a += fts5GetVarint32(a, iVal);
916 iOff += iVal;
917 pSorter->aIdx[i] = iOff;
918 }
919 pSorter->aIdx[i] = &aBlob[nBlob] - a;
920 pSorter->aPoslist = a;
921 }
922
923 fts5CsrNewrow(pCsr);
924 }
925
926 return rc;
927 }
928
929
930 /*
931 ** Set the FTS5CSR_REQUIRE_RESEEK flag on all FTS5_PLAN_MATCH cursors
932 ** open on table pTab.
933 */
934 static void fts5TripCursors(Fts5FullTable *pTab){
935 Fts5Cursor *pCsr;
936 for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
937 if( pCsr->ePlan==FTS5_PLAN_MATCH
938 && pCsr->base.pVtab==(sqlite3_vtab*)pTab
939 ){
940 CsrFlagSet(pCsr, FTS5CSR_REQUIRE_RESEEK);
941 }
942 }
943 }
944
945 /*
946 ** If the REQUIRE_RESEEK flag is set on the cursor passed as the first
947 ** argument, close and reopen all Fts5IndexIter iterators that the cursor
948 ** is using. Then attempt to move the cursor to a rowid equal to or laster
949 ** (in the cursors sort order - ASC or DESC) than the current rowid.
950 **
951 ** If the new rowid is not equal to the old, set output parameter *pbSkip
952 ** to 1 before returning. Otherwise, leave it unchanged.
953 **
954 ** Return SQLITE_OK if successful or if no reseek was required, or an
955 ** error code if an error occurred.
956 */
957 static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
958 int rc = SQLITE_OK;
959 assert( *pbSkip==0 );
960 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_RESEEK) ){
961 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
962 int bDesc = pCsr->bDesc;
963 i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr);
964
965 rc = sqlite3Fts5ExprFirst(
966 pCsr->pExpr, pTab->p.pIndex, iRowid, pCsr->iLastRowid, bDesc
967 );
968 if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){
969 *pbSkip = 1;
970 }
971
972 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_RESEEK);
973 fts5CsrNewrow(pCsr);
974 if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
975 CsrFlagSet(pCsr, FTS5CSR_EOF);
976 *pbSkip = 1;
977 }
978 }
979 return rc;
980 }
981
982
983 /*
984 ** Advance the cursor to the next row in the table that matches the
985 ** search criteria.
986 **
987 ** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned
988 ** even if we reach end-of-file. The fts5EofMethod() will be called
989 ** subsequently to determine whether or not an EOF was hit.
990 */
991 static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
992 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
993 int rc;
994
995 assert( (pCsr->ePlan<3)==
996 (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE)
997 );
998 assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) );
999
1000 /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table,
1001 ** clear any token mappings accumulated at the fts5_index.c level. In
1002 ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH,
1003 ** we need to retain the mappings for the entire query. */
1004 if( pCsr->ePlan==FTS5_PLAN_MATCH
1005 && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata
1006 ){
1007 sqlite3Fts5ExprClearTokens(pCsr->pExpr);
1008 }
1009
1010 if( pCsr->ePlan<3 ){
1011 int bSkip = 0;
1012 if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
1013 rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
1014 CsrFlagSet(pCsr, sqlite3Fts5ExprEof(pCsr->pExpr));
1015 fts5CsrNewrow(pCsr);
1016 }else{
1017 switch( pCsr->ePlan ){
1018 case FTS5_PLAN_SPECIAL: {
1019 CsrFlagSet(pCsr, FTS5CSR_EOF);
1020 rc = SQLITE_OK;
1021 break;
1022 }
1023
1024 case FTS5_PLAN_SORTED_MATCH: {
1025 rc = fts5SorterNext(pCsr);
1026 break;
1027 }
1028
1029 default: {
1030 Fts5Config *pConfig = ((Fts5Table*)pCursor->pVtab)->pConfig;
1031 pConfig->bLock++;
1032 rc = sqlite3_step(pCsr->pStmt);
1033 pConfig->bLock--;
1034 if( rc!=SQLITE_ROW ){
1035 CsrFlagSet(pCsr, FTS5CSR_EOF);
1036 rc = sqlite3_reset(pCsr->pStmt);
1037 if( rc!=SQLITE_OK ){
1038 pCursor->pVtab->zErrMsg = sqlite3_mprintf(
1039 "%s", sqlite3_errmsg(pConfig->db)
1040 );
1041 }
1042 }else{
1043 rc = SQLITE_OK;
1044 CsrFlagSet(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
1045 }
1046 break;
1047 }
1048 }
1049 }
1050
1051 return rc;
1052 }
1053
1054
1055 static int fts5PrepareStatement(
1056 sqlite3_stmt **ppStmt,
1057 Fts5Config *pConfig,
1058 const char *zFmt,
1059 ...
1060 ){
1061 sqlite3_stmt *pRet = 0;
1062 int rc;
1063 char *zSql;
1064 va_list ap;
1065
1066 va_start(ap, zFmt);
1067 zSql = sqlite3_vmprintf(zFmt, ap);
1068 if( zSql==0 ){
1069 rc = SQLITE_NOMEM;
1070 }else{
1071 rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
1072 SQLITE_PREPARE_PERSISTENT, &pRet, 0);
1073 if( rc!=SQLITE_OK ){
1074 sqlite3Fts5ConfigErrmsg(pConfig, "%s", sqlite3_errmsg(pConfig->db));
1075 }
1076 sqlite3_free(zSql);
1077 }
1078
1079 va_end(ap);
1080 *ppStmt = pRet;
1081 return rc;
1082 }
1083
1084 static int fts5CursorFirstSorted(
1085 Fts5FullTable *pTab,
1086 Fts5Cursor *pCsr,
1087 int bDesc
1088 ){
1089 Fts5Config *pConfig = pTab->p.pConfig;
1090 Fts5Sorter *pSorter;
1091 int nPhrase;
1092 sqlite3_int64 nByte;
1093 int rc;
1094 const char *zRank = pCsr->zRank;
1095 const char *zRankArgs = pCsr->zRankArgs;
1096
1097 nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
1098 nByte = SZ_FTS5SORTER(nPhrase);
1099 pSorter = (Fts5Sorter*)sqlite3_malloc64(nByte);
1100 if( pSorter==0 ) return SQLITE_NOMEM;
1101 memset(pSorter, 0, (size_t)nByte);
1102 pSorter->nIdx = nPhrase;
1103
1104 /* TODO: It would be better to have some system for reusing statement
1105 ** handles here, rather than preparing a new one for each query. But that
1106 ** is not possible as SQLite reference counts the virtual table objects.
1107 ** And since the statement required here reads from this very virtual
1108 ** table, saving it creates a circular reference.
1109 **
1110 ** If SQLite a built-in statement cache, this wouldn't be a problem. */
1111 rc = fts5PrepareStatement(&pSorter->pStmt, pConfig,
1112 "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(\"%w\"%s%s) %s",
1113 pConfig->zDb, pConfig->zName, zRank, pConfig->zName,
1114 (zRankArgs ? ", " : ""),
1115 (zRankArgs ? zRankArgs : ""),
1116 bDesc ? "DESC" : "ASC"
1117 );
1118
1119 pCsr->pSorter = pSorter;
1120 if( rc==SQLITE_OK ){
1121 assert( pTab->pSortCsr==0 );
1122 pTab->pSortCsr = pCsr;
1123 rc = fts5SorterNext(pCsr);
1124 pTab->pSortCsr = 0;
1125 }
1126
1127 if( rc!=SQLITE_OK ){
1128 sqlite3_finalize(pSorter->pStmt);
1129 sqlite3_free(pSorter);
1130 pCsr->pSorter = 0;
1131 }
1132
1133 return rc;
1134 }
1135
1136 static int fts5CursorFirst(Fts5FullTable *pTab, Fts5Cursor *pCsr, int bDesc){
1137 int rc;
1138 Fts5Expr *pExpr = pCsr->pExpr;
1139 rc = sqlite3Fts5ExprFirst(
1140 pExpr, pTab->p.pIndex, pCsr->iFirstRowid, pCsr->iLastRowid, bDesc
1141 );
1142 if( sqlite3Fts5ExprEof(pExpr) ){
1143 CsrFlagSet(pCsr, FTS5CSR_EOF);
1144 }
1145 fts5CsrNewrow(pCsr);
1146 return rc;
1147 }
1148
1149 /*
1150 ** Process a "special" query. A special query is identified as one with a
1151 ** MATCH expression that begins with a '*' character. The remainder of
1152 ** the text passed to the MATCH operator are used as the special query
1153 ** parameters.
1154 */
1155 static int fts5SpecialMatch(
1156 Fts5FullTable *pTab,
1157 Fts5Cursor *pCsr,
1158 const char *zQuery
1159 ){
1160 int rc = SQLITE_OK; /* Return code */
1161 const char *z = zQuery; /* Special query text */
1162 int n; /* Number of bytes in text at z */
1163
1164 while( z[0]==' ' ) z++;
1165 for(n=0; z[n] && z[n]!=' '; n++);
1166
1167 assert( pTab->p.base.zErrMsg==0 );
1168 pCsr->ePlan = FTS5_PLAN_SPECIAL;
1169
1170 if( n==5 && 0==sqlite3_strnicmp("reads", z, n) ){
1171 pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->p.pIndex);
1172 }
1173 else if( n==2 && 0==sqlite3_strnicmp("id", z, n) ){
1174 pCsr->iSpecial = pCsr->iCsrId;
1175 }
1176 else{
1177 /* An unrecognized directive. Return an error message. */
1178 pTab->p.base.zErrMsg = sqlite3_mprintf("unknown special query: %.*s", n, z);
1179 rc = SQLITE_ERROR;
1180 }
1181
1182 return rc;
1183 }
1184
1185 /*
1186 ** Search for an auxiliary function named zName that can be used with table
1187 ** pTab. If one is found, return a pointer to the corresponding Fts5Auxiliary
1188 ** structure. Otherwise, if no such function exists, return NULL.
1189 */
1190 static Fts5Auxiliary *fts5FindAuxiliary(Fts5FullTable *pTab, const char *zName){
1191 Fts5Auxiliary *pAux;
1192
1193 for(pAux=pTab->pGlobal->pAux; pAux; pAux=pAux->pNext){
1194 if( sqlite3_stricmp(zName, pAux->zFunc)==0 ) return pAux;
1195 }
1196
1197 /* No function of the specified name was found. Return 0. */
1198 return 0;
1199 }
1200
1201
1202 static int fts5FindRankFunction(Fts5Cursor *pCsr){
1203 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
1204 Fts5Config *pConfig = pTab->p.pConfig;
1205 int rc = SQLITE_OK;
1206 Fts5Auxiliary *pAux = 0;
1207 const char *zRank = pCsr->zRank;
1208 const char *zRankArgs = pCsr->zRankArgs;
1209
1210 if( zRankArgs ){
1211 char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs);
1212 if( zSql ){
1213 sqlite3_stmt *pStmt = 0;
1214 rc = sqlite3_prepare_v3(pConfig->db, zSql, -1,
1215 SQLITE_PREPARE_PERSISTENT, &pStmt, 0);
1216 sqlite3_free(zSql);
1217 assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 );
1218 if( rc==SQLITE_OK ){
1219 if( SQLITE_ROW==sqlite3_step(pStmt) ){
1220 sqlite3_int64 nByte;
1221 pCsr->nRankArg = sqlite3_column_count(pStmt);
1222 nByte = sizeof(sqlite3_value*)*pCsr->nRankArg;
1223 pCsr->apRankArg = (sqlite3_value**)sqlite3Fts5MallocZero(&rc, nByte);
1224 if( rc==SQLITE_OK ){
1225 int i;
1226 for(i=0; i<pCsr->nRankArg; i++){
1227 pCsr->apRankArg[i] = sqlite3_column_value(pStmt, i);
1228 }
1229 }
1230 pCsr->pRankArgStmt = pStmt;
1231 }else{
1232 rc = sqlite3_finalize(pStmt);
1233 assert( rc!=SQLITE_OK );
1234 }
1235 }
1236 }
1237 }
1238
1239 if( rc==SQLITE_OK ){
1240 pAux = fts5FindAuxiliary(pTab, zRank);
1241 if( pAux==0 ){
1242 assert( pTab->p.base.zErrMsg==0 );
1243 pTab->p.base.zErrMsg = sqlite3_mprintf("no such function: %s", zRank);
1244 rc = SQLITE_ERROR;
1245 }
1246 }
1247
1248 pCsr->pRank = pAux;
1249 return rc;
1250 }
1251
1252
1253 static int fts5CursorParseRank(
1254 Fts5Config *pConfig,
1255 Fts5Cursor *pCsr,
1256 sqlite3_value *pRank
1257 ){
1258 int rc = SQLITE_OK;
1259 if( pRank ){
1260 const char *z = (const char*)sqlite3_value_text(pRank);
1261 char *zRank = 0;
1262 char *zRankArgs = 0;
1263
1264 if( z==0 ){
1265 if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR;
1266 }else{
1267 rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);
1268 }
1269 if( rc==SQLITE_OK ){
1270 pCsr->zRank = zRank;
1271 pCsr->zRankArgs = zRankArgs;
1272 CsrFlagSet(pCsr, FTS5CSR_FREE_ZRANK);
1273 }else if( rc==SQLITE_ERROR ){
1274 pCsr->base.pVtab->zErrMsg = sqlite3_mprintf(
1275 "parse error in rank function: %s", z
1276 );
1277 }
1278 }else{
1279 if( pConfig->zRank ){
1280 pCsr->zRank = (char*)pConfig->zRank;
1281 pCsr->zRankArgs = (char*)pConfig->zRankArgs;
1282 }else{
1283 pCsr->zRank = (char*)FTS5_DEFAULT_RANK;
1284 pCsr->zRankArgs = 0;
1285 }
1286 }
1287 return rc;
1288 }
1289
1290 static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){
1291 if( pVal ){
1292 int eType = sqlite3_value_numeric_type(pVal);
1293 if( eType==SQLITE_INTEGER ){
1294 return sqlite3_value_int64(pVal);
1295 }
1296 }
1297 return iDefault;
1298 }
1299
1300 /*
1301 ** Set the error message on the virtual table passed as the first argument.
1302 */
1303 static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){
1304 va_list ap; /* ... printf arguments */
1305 va_start(ap, zFormat);
1306 sqlite3_free(p->p.base.zErrMsg);
1307 p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap);
1308 va_end(ap);
1309 }
1310
1311 /*
1312 ** Arrange for subsequent calls to sqlite3Fts5Tokenize() to use the locale
1313 ** specified by pLocale/nLocale. The buffer indicated by pLocale must remain
1314 ** valid until after the final call to sqlite3Fts5Tokenize() that will use
1315 ** the locale.
1316 */
1317 static void sqlite3Fts5SetLocale(
1318 Fts5Config *pConfig,
1319 const char *zLocale,
1320 int nLocale
1321 ){
1322 Fts5TokenizerConfig *pT = &pConfig->t;
1323 pT->pLocale = zLocale;
1324 pT->nLocale = nLocale;
1325 }
1326
1327 /*
1328 ** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale().
1329 */
1330 void sqlite3Fts5ClearLocale(Fts5Config *pConfig){
1331 sqlite3Fts5SetLocale(pConfig, 0, 0);
1332 }
1333
1334 /*
1335 ** Return true if the value passed as the only argument is an
1336 ** fts5_locale() value.
1337 */
1338 int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){
1339 int ret = 0;
1340 if( sqlite3_value_type(pVal)==SQLITE_BLOB ){
1341 /* Call sqlite3_value_bytes() after sqlite3_value_blob() in this case.
1342 ** If the blob was created using zeroblob(), then sqlite3_value_blob()
1343 ** may call malloc(). If this malloc() fails, then the values returned
1344 ** by both value_blob() and value_bytes() will be 0. If value_bytes() were
1345 ** called first, then the NULL pointer returned by value_blob() might
1346 ** be dereferenced. */
1347 const u8 *pBlob = sqlite3_value_blob(pVal);
1348 int nBlob = sqlite3_value_bytes(pVal);
1349 if( nBlob>FTS5_LOCALE_HDR_SIZE
1350 && 0==memcmp(pBlob, FTS5_LOCALE_HDR(pConfig), FTS5_LOCALE_HDR_SIZE)
1351 ){
1352 ret = 1;
1353 }
1354 }
1355 return ret;
1356 }
1357
1358 /*
1359 ** Value pVal is guaranteed to be an fts5_locale() value, according to
1360 ** sqlite3Fts5IsLocaleValue(). This function extracts the text and locale
1361 ** from the value and returns them separately.
1362 **
1363 ** If successful, SQLITE_OK is returned and (*ppText) and (*ppLoc) set
1364 ** to point to buffers containing the text and locale, as utf-8,
1365 ** respectively. In this case output parameters (*pnText) and (*pnLoc) are
1366 ** set to the sizes in bytes of these two buffers.
1367 **
1368 ** Or, if an error occurs, then an SQLite error code is returned. The final
1369 ** value of the four output parameters is undefined in this case.
1370 */
1371 int sqlite3Fts5DecodeLocaleValue(
1372 sqlite3_value *pVal,
1373 const char **ppText,
1374 int *pnText,
1375 const char **ppLoc,
1376 int *pnLoc
1377 ){
1378 const char *p = sqlite3_value_blob(pVal);
1379 int n = sqlite3_value_bytes(pVal);
1380 int nLoc = 0;
1381
1382 assert( sqlite3_value_type(pVal)==SQLITE_BLOB );
1383 assert( n>FTS5_LOCALE_HDR_SIZE );
1384
1385 for(nLoc=FTS5_LOCALE_HDR_SIZE; p[nLoc]; nLoc++){
1386 if( nLoc==(n-1) ){
1387 return SQLITE_MISMATCH;
1388 }
1389 }
1390 *ppLoc = &p[FTS5_LOCALE_HDR_SIZE];
1391 *pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE;
1392
1393 *ppText = &p[nLoc+1];
1394 *pnText = n - nLoc - 1;
1395 return SQLITE_OK;
1396 }
1397
1398 /*
1399 ** Argument pVal is the text of a full-text search expression. It may or
1400 ** may not have been wrapped by fts5_locale(). This function extracts
1401 ** the text of the expression, and sets output variable (*pzText) to
1402 ** point to a nul-terminated buffer containing the expression.
1403 **
1404 ** If pVal was an fts5_locale() value, then sqlite3Fts5SetLocale() is called
1405 ** to set the tokenizer to use the specified locale.
1406 **
1407 ** If output variable (*pbFreeAndReset) is set to true, then the caller
1408 ** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer
1409 ** locale, and (b) call sqlite3_free() to free (*pzText).
1410 */
1411 static int fts5ExtractExprText(
1412 Fts5Config *pConfig, /* Fts5 configuration */
1413 sqlite3_value *pVal, /* Value to extract expression text from */
1414 char **pzText, /* OUT: nul-terminated buffer of text */
1415 int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */
1416 ){
1417 int rc = SQLITE_OK;
1418
1419 if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
1420 const char *pText = 0;
1421 int nText = 0;
1422 const char *pLoc = 0;
1423 int nLoc = 0;
1424 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
1425 *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, pText);
1426 if( rc==SQLITE_OK ){
1427 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
1428 }
1429 *pbFreeAndReset = 1;
1430 }else{
1431 *pzText = (char*)sqlite3_value_text(pVal);
1432 *pbFreeAndReset = 0;
1433 }
1434
1435 return rc;
1436 }
1437
1438
1439 /*
1440 ** This is the xFilter interface for the virtual table. See
1441 ** the virtual table xFilter method documentation for additional
1442 ** information.
1443 **
1444 ** There are three possible query strategies:
1445 **
1446 ** 1. Full-text search using a MATCH operator.
1447 ** 2. A by-rowid lookup.
1448 ** 3. A full-table scan.
1449 */
1450 static int fts5FilterMethod(
1451 sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */
1452 int idxNum, /* Strategy index */
1453 const char *idxStr, /* Unused */
1454 int nVal, /* Number of elements in apVal */
1455 sqlite3_value **apVal /* Arguments for the indexing scheme */
1456 ){
1457 Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab);
1458 Fts5Config *pConfig = pTab->p.pConfig;
1459 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
1460 int rc = SQLITE_OK; /* Error code */
1461 int bDesc; /* True if ORDER BY [rank|rowid] DESC */
1462 int bOrderByRank; /* True if ORDER BY rank */
1463 sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */
1464 sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */
1465 sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */
1466 sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */
1467 int iCol; /* Column on LHS of MATCH operator */
1468 char **pzErrmsg = pConfig->pzErrmsg;
1469 int bPrefixInsttoken = pConfig->bPrefixInsttoken;
1470 int i;
1471 int iIdxStr = 0;
1472 Fts5Expr *pExpr = 0;
1473
1474 assert( pConfig->bLock==0 );
1475 if( pCsr->ePlan ){
1476 fts5FreeCursorComponents(pCsr);
1477 memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr));
1478 }
1479
1480 assert( pCsr->pStmt==0 );
1481 assert( pCsr->pExpr==0 );
1482 assert( pCsr->csrflags==0 );
1483 assert( pCsr->pRank==0 );
1484 assert( pCsr->zRank==0 );
1485 assert( pCsr->zRankArgs==0 );
1486 assert( pTab->pSortCsr==0 || nVal==0 );
1487
1488 assert( pzErrmsg==0 || pzErrmsg==&pTab->p.base.zErrMsg );
1489 pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
1490
1491 /* Decode the arguments passed through to this function. */
1492 for(i=0; i<nVal; i++){
1493 switch( idxStr[iIdxStr++] ){
1494 case 'r':
1495 pRank = apVal[i];
1496 break;
1497 case 'M': {
1498 char *zText = 0;
1499 int bFreeAndReset = 0;
1500 int bInternal = 0;
1501
1502 rc = fts5ExtractExprText(pConfig, apVal[i], &zText, &bFreeAndReset);
1503 if( rc!=SQLITE_OK ) goto filter_out;
1504 if( zText==0 ) zText = "";
1505 if( sqlite3_value_subtype(apVal[i])==FTS5_INSTTOKEN_SUBTYPE ){
1506 pConfig->bPrefixInsttoken = 1;
1507 }
1508
1509 iCol = 0;
1510 do{
1511 iCol = iCol*10 + (idxStr[iIdxStr]-'0');
1512 iIdxStr++;
1513 }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' );
1514
1515 if( zText[0]=='*' ){
1516 /* The user has issued a query of the form "MATCH '*...'". This
1517 ** indicates that the MATCH expression is not a full text query,
1518 ** but a request for an internal parameter. */
1519 rc = fts5SpecialMatch(pTab, pCsr, &zText[1]);
1520 bInternal = 1;
1521 }else{
1522 char **pzErr = &pTab->p.base.zErrMsg;
1523 rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr);
1524 if( rc==SQLITE_OK ){
1525 rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr);
1526 pExpr = 0;
1527 }
1528 }
1529
1530 if( bFreeAndReset ){
1531 sqlite3_free(zText);
1532 sqlite3Fts5ClearLocale(pConfig);
1533 }
1534
1535 if( bInternal || rc!=SQLITE_OK ) goto filter_out;
1536
1537 break;
1538 }
1539 case 'L':
1540 case 'G': {
1541 int bGlob = (idxStr[iIdxStr-1]=='G');
1542 const char *zText = (const char*)sqlite3_value_text(apVal[i]);
1543 iCol = 0;
1544 do{
1545 iCol = iCol*10 + (idxStr[iIdxStr]-'0');
1546 iIdxStr++;
1547 }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' );
1548 if( zText ){
1549 rc = sqlite3Fts5ExprPattern(pConfig, bGlob, iCol, zText, &pExpr);
1550 }
1551 if( rc==SQLITE_OK ){
1552 rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr);
1553 pExpr = 0;
1554 }
1555 if( rc!=SQLITE_OK ) goto filter_out;
1556 break;
1557 }
1558 case '=':
1559 pRowidEq = apVal[i];
1560 break;
1561 case '<':
1562 pRowidLe = apVal[i];
1563 break;
1564 default: assert( idxStr[iIdxStr-1]=='>' );
1565 pRowidGe = apVal[i];
1566 break;
1567 }
1568 }
1569 bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0);
1570 pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0);
1571
1572 /* Set the cursor upper and lower rowid limits. Only some strategies
1573 ** actually use them. This is ok, as the xBestIndex() method leaves the
1574 ** sqlite3_index_constraint.omit flag clear for range constraints
1575 ** on the rowid field. */
1576 if( pRowidEq ){
1577 pRowidLe = pRowidGe = pRowidEq;
1578 }
1579 if( bDesc ){
1580 pCsr->iFirstRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64);
1581 pCsr->iLastRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64);
1582 }else{
1583 pCsr->iLastRowid = fts5GetRowidLimit(pRowidLe, LARGEST_INT64);
1584 pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64);
1585 }
1586
1587 rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
1588 if( rc!=SQLITE_OK ) goto filter_out;
1589
1590 if( pTab->pSortCsr ){
1591 /* If pSortCsr is non-NULL, then this call is being made as part of
1592 ** processing for a "... MATCH <expr> ORDER BY rank" query (ePlan is
1593 ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will
1594 ** return results to the user for this query. The current cursor
1595 ** (pCursor) is used to execute the query issued by function
1596 ** fts5CursorFirstSorted() above. */
1597 assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 );
1598 assert( nVal==0 && bOrderByRank==0 && bDesc==0 );
1599 assert( pCsr->iLastRowid==LARGEST_INT64 );
1600 assert( pCsr->iFirstRowid==SMALLEST_INT64 );
1601 if( pTab->pSortCsr->bDesc ){
1602 pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid;
1603 pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid;
1604 }else{
1605 pCsr->iLastRowid = pTab->pSortCsr->iLastRowid;
1606 pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid;
1607 }
1608 pCsr->ePlan = FTS5_PLAN_SOURCE;
1609 pCsr->pExpr = pTab->pSortCsr->pExpr;
1610 rc = fts5CursorFirst(pTab, pCsr, bDesc);
1611 }else if( pCsr->pExpr ){
1612 assert( rc==SQLITE_OK );
1613 rc = fts5CursorParseRank(pConfig, pCsr, pRank);
1614 if( rc==SQLITE_OK ){
1615 if( bOrderByRank ){
1616 pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
1617 rc = fts5CursorFirstSorted(pTab, pCsr, bDesc);
1618 }else{
1619 pCsr->ePlan = FTS5_PLAN_MATCH;
1620 rc = fts5CursorFirst(pTab, pCsr, bDesc);
1621 }
1622 }
1623 }else if( pConfig->zContent==0 ){
1624 fts5SetVtabError(pTab,"%s: table does not support scanning",pConfig->zName);
1625 rc = SQLITE_ERROR;
1626 }else{
1627 /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup
1628 ** by rowid (ePlan==FTS5_PLAN_ROWID). */
1629 pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN);
1630 rc = sqlite3Fts5StorageStmt(
1631 pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->p.base.zErrMsg
1632 );
1633 if( rc==SQLITE_OK ){
1634 if( pRowidEq!=0 ){
1635 assert( pCsr->ePlan==FTS5_PLAN_ROWID );
1636 sqlite3_bind_value(pCsr->pStmt, 1, pRowidEq);
1637 }else{
1638 sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid);
1639 sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid);
1640 }
1641 rc = fts5NextMethod(pCursor);
1642 }
1643 }
1644
1645 filter_out:
1646 sqlite3Fts5ExprFree(pExpr);
1647 pConfig->pzErrmsg = pzErrmsg;
1648 pConfig->bPrefixInsttoken = bPrefixInsttoken;
1649 return rc;
1650 }
1651
1652 /*
1653 ** This is the xEof method of the virtual table. SQLite calls this
1654 ** routine to find out if it has reached the end of a result set.
1655 */
1656 static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){
1657 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
1658 return (CsrFlagTest(pCsr, FTS5CSR_EOF) ? 1 : 0);
1659 }
1660
1661 /*
1662 ** Return the rowid that the cursor currently points to.
1663 */
1664 static i64 fts5CursorRowid(Fts5Cursor *pCsr){
1665 assert( pCsr->ePlan==FTS5_PLAN_MATCH
1666 || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH
1667 || pCsr->ePlan==FTS5_PLAN_SOURCE
1668 || pCsr->ePlan==FTS5_PLAN_SCAN
1669 || pCsr->ePlan==FTS5_PLAN_ROWID
1670 );
1671 if( pCsr->pSorter ){
1672 return pCsr->pSorter->iRowid;
1673 }else if( pCsr->ePlan>=FTS5_PLAN_SCAN ){
1674 return sqlite3_column_int64(pCsr->pStmt, 0);
1675 }else{
1676 return sqlite3Fts5ExprRowid(pCsr->pExpr);
1677 }
1678 }
1679
1680 /*
1681 ** This is the xRowid method. The SQLite core calls this routine to
1682 ** retrieve the rowid for the current row of the result set. fts5
1683 ** exposes %_content.rowid as the rowid for the virtual table. The
1684 ** rowid should be written to *pRowid.
1685 */
1686 static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
1687 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
1688 int ePlan = pCsr->ePlan;
1689
1690 assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
1691 if( ePlan==FTS5_PLAN_SPECIAL ){
1692 *pRowid = 0;
1693 }else{
1694 *pRowid = fts5CursorRowid(pCsr);
1695 }
1696
1697 return SQLITE_OK;
1698 }
1699
1700
1701 /*
1702 ** If the cursor requires seeking (bSeekRequired flag is set), seek it.
1703 ** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise.
1704 **
1705 ** If argument bErrormsg is true and an error occurs, an error message may
1706 ** be left in sqlite3_vtab.zErrMsg.
1707 */
1708 static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){
1709 int rc = SQLITE_OK;
1710
1711 /* If the cursor does not yet have a statement handle, obtain one now. */
1712 if( pCsr->pStmt==0 ){
1713 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
1714 int eStmt = fts5StmtType(pCsr);
1715 rc = sqlite3Fts5StorageStmt(
1716 pTab->pStorage, eStmt, &pCsr->pStmt, (bErrormsg?&pTab->p.base.zErrMsg:0)
1717 );
1718 assert( rc!=SQLITE_OK || pTab->p.base.zErrMsg==0 );
1719 assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) );
1720 }
1721
1722 if( rc==SQLITE_OK && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){
1723 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
1724 assert( pCsr->pExpr );
1725 sqlite3_reset(pCsr->pStmt);
1726 sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr));
1727 pTab->pConfig->bLock++;
1728 rc = sqlite3_step(pCsr->pStmt);
1729 pTab->pConfig->bLock--;
1730 if( rc==SQLITE_ROW ){
1731 rc = SQLITE_OK;
1732 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT);
1733 }else{
1734 rc = sqlite3_reset(pCsr->pStmt);
1735 if( rc==SQLITE_OK ){
1736 rc = FTS5_CORRUPT;
1737 fts5SetVtabError((Fts5FullTable*)pTab,
1738 "fts5: missing row %lld from content table %s",
1739 fts5CursorRowid(pCsr),
1740 pTab->pConfig->zContent
1741 );
1742 }else if( pTab->pConfig->pzErrmsg ){
1743 fts5SetVtabError((Fts5FullTable*)pTab,
1744 "%s", sqlite3_errmsg(pTab->pConfig->db)
1745 );
1746 }
1747 }
1748 }
1749 return rc;
1750 }
1751
1752 /*
1753 ** This function is called to handle an FTS INSERT command. In other words,
1754 ** an INSERT statement of the form:
1755 **
1756 ** INSERT INTO fts(fts) VALUES($pCmd)
1757 ** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal)
1758 **
1759 ** Argument pVal is the value assigned to column "fts" by the INSERT
1760 ** statement. This function returns SQLITE_OK if successful, or an SQLite
1761 ** error code if an error occurs.
1762 **
1763 ** The commands implemented by this function are documented in the "Special
1764 ** INSERT Directives" section of the documentation. It should be updated if
1765 ** more commands are added to this function.
1766 */
1767 static int fts5SpecialInsert(
1768 Fts5FullTable *pTab, /* Fts5 table object */
1769 const char *zCmd, /* Text inserted into table-name column */
1770 sqlite3_value *pVal /* Value inserted into rank column */
1771 ){
1772 Fts5Config *pConfig = pTab->p.pConfig;
1773 int rc = SQLITE_OK;
1774 int bError = 0;
1775 int bLoadConfig = 0;
1776
1777 if( 0==sqlite3_stricmp("delete-all", zCmd) ){
1778 if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
1779 fts5SetVtabError(pTab,
1780 "'delete-all' may only be used with a "
1781 "contentless or external content fts5 table"
1782 );
1783 rc = SQLITE_ERROR;
1784 }else{
1785 rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
1786 }
1787 bLoadConfig = 1;
1788 }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){
1789 if( fts5IsContentless(pTab, 1) ){
1790 fts5SetVtabError(pTab,
1791 "'rebuild' may not be used with a contentless fts5 table"
1792 );
1793 rc = SQLITE_ERROR;
1794 }else{
1795 rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
1796 }
1797 bLoadConfig = 1;
1798 }else if( 0==sqlite3_stricmp("optimize", zCmd) ){
1799 rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
1800 }else if( 0==sqlite3_stricmp("merge", zCmd) ){
1801 int nMerge = sqlite3_value_int(pVal);
1802 rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge);
1803 }else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){
1804 int iArg = sqlite3_value_int(pVal);
1805 rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, iArg);
1806 #ifdef SQLITE_DEBUG
1807 }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){
1808 pConfig->bPrefixIndex = sqlite3_value_int(pVal);
1809 #endif
1810 }else if( 0==sqlite3_stricmp("flush", zCmd) ){
1811 rc = sqlite3Fts5FlushToDisk(&pTab->p);
1812 }else{
1813 rc = sqlite3Fts5FlushToDisk(&pTab->p);
1814 if( rc==SQLITE_OK ){
1815 rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
1816 }
1817 if( rc==SQLITE_OK ){
1818 rc = sqlite3Fts5ConfigSetValue(pTab->p.pConfig, zCmd, pVal, &bError);
1819 }
1820 if( rc==SQLITE_OK ){
1821 if( bError ){
1822 rc = SQLITE_ERROR;
1823 }else{
1824 rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, zCmd, pVal, 0);
1825 }
1826 }
1827 }
1828
1829 if( rc==SQLITE_OK && bLoadConfig ){
1830 pTab->p.pConfig->iCookie--;
1831 rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
1832 }
1833
1834 return rc;
1835 }
1836
1837 static int fts5SpecialDelete(
1838 Fts5FullTable *pTab,
1839 sqlite3_value **apVal
1840 ){
1841 int rc = SQLITE_OK;
1842 int eType1 = sqlite3_value_type(apVal[1]);
1843 if( eType1==SQLITE_INTEGER ){
1844 sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
1845 rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2], 0);
1846 }
1847 return rc;
1848 }
1849
1850 static void fts5StorageInsert(
1851 int *pRc,
1852 Fts5FullTable *pTab,
1853 sqlite3_value **apVal,
1854 i64 *piRowid
1855 ){
1856 int rc = *pRc;
1857 if( rc==SQLITE_OK ){
1858 rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, 0, apVal, piRowid);
1859 }
1860 if( rc==SQLITE_OK ){
1861 rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid);
1862 }
1863 *pRc = rc;
1864 }
1865
1866 /*
1867 **
1868 ** This function is called when the user attempts an UPDATE on a contentless
1869 ** table. Parameter bRowidModified is true if the UPDATE statement modifies
1870 ** the rowid value. Parameter apVal[] contains the new values for each user
1871 ** defined column of the fts5 table. pConfig is the configuration object of the
1872 ** table being updated (guaranteed to be contentless). The contentless_delete=1
1873 ** and contentless_unindexed=1 options may or may not be set.
1874 **
1875 ** This function returns SQLITE_OK if the UPDATE can go ahead, or an SQLite
1876 ** error code if it cannot. In this case an error message is also loaded into
1877 ** pConfig. Output parameter (*pbContent) is set to true if the caller should
1878 ** update the %_content table only - not the FTS index or any other shadow
1879 ** table. This occurs when an UPDATE modifies only UNINDEXED columns of the
1880 ** table.
1881 **
1882 ** An UPDATE may proceed if:
1883 **
1884 ** * The only columns modified are UNINDEXED columns, or
1885 **
1886 ** * The contentless_delete=1 option was specified and all of the indexed
1887 ** columns (not a subset) have been modified.
1888 */
1889 static int fts5ContentlessUpdate(
1890 Fts5Config *pConfig,
1891 sqlite3_value **apVal,
1892 int bRowidModified,
1893 int *pbContent
1894 ){
1895 int ii;
1896 int bSeenIndex = 0; /* Have seen modified indexed column */
1897 int bSeenIndexNC = 0; /* Have seen unmodified indexed column */
1898 int rc = SQLITE_OK;
1899
1900 for(ii=0; ii<pConfig->nCol; ii++){
1901 if( pConfig->abUnindexed[ii]==0 ){
1902 if( sqlite3_value_nochange(apVal[ii]) ){
1903 bSeenIndexNC++;
1904 }else{
1905 bSeenIndex++;
1906 }
1907 }
1908 }
1909
1910 if( bSeenIndex==0 && bRowidModified==0 ){
1911 *pbContent = 1;
1912 }else{
1913 if( bSeenIndexNC || pConfig->bContentlessDelete==0 ){
1914 rc = SQLITE_ERROR;
1915 sqlite3Fts5ConfigErrmsg(pConfig,
1916 (pConfig->bContentlessDelete ?
1917 "%s a subset of columns on fts5 contentless-delete table: %s" :
1918 "%s contentless fts5 table: %s")
1919 , "cannot UPDATE", pConfig->zName
1920 );
1921 }
1922 }
1923
1924 return rc;
1925 }
1926
1927 /*
1928 ** This function is the implementation of the xUpdate callback used by
1929 ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
1930 ** inserted, updated or deleted.
1931 **
1932 ** A delete specifies a single argument - the rowid of the row to remove.
1933 **
1934 ** Update and insert operations pass:
1935 **
1936 ** 1. The "old" rowid, or NULL.
1937 ** 2. The "new" rowid.
1938 ** 3. Values for each of the nCol matchable columns.
1939 ** 4. Values for the two hidden columns (<tablename> and "rank").
1940 */
1941 static int fts5UpdateMethod(
1942 sqlite3_vtab *pVtab, /* Virtual table handle */
1943 int nArg, /* Size of argument array */
1944 sqlite3_value **apVal, /* Array of arguments */
1945 sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
1946 ){
1947 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
1948 Fts5Config *pConfig = pTab->p.pConfig;
1949 int eType0; /* value_type() of apVal[0] */
1950 int rc = SQLITE_OK; /* Return code */
1951
1952 /* A transaction must be open when this is called. */
1953 assert( pTab->ts.eState==1 || pTab->ts.eState==2 );
1954
1955 assert( pVtab->zErrMsg==0 );
1956 assert( nArg==1 || nArg==(2+pConfig->nCol+2) );
1957 assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER
1958 || sqlite3_value_type(apVal[0])==SQLITE_NULL
1959 );
1960 assert( pTab->p.pConfig->pzErrmsg==0 );
1961 if( pConfig->pgsz==0 ){
1962 rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie);
1963 if( rc!=SQLITE_OK ) return rc;
1964 }
1965
1966 pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
1967
1968 /* Put any active cursors into REQUIRE_SEEK state. */
1969 fts5TripCursors(pTab);
1970
1971 eType0 = sqlite3_value_type(apVal[0]);
1972 if( eType0==SQLITE_NULL
1973 && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL
1974 ){
1975 /* A "special" INSERT op. These are handled separately. */
1976 const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]);
1977 if( pConfig->eContent!=FTS5_CONTENT_NORMAL
1978 && 0==sqlite3_stricmp("delete", z)
1979 ){
1980 if( pConfig->bContentlessDelete ){
1981 fts5SetVtabError(pTab,
1982 "'delete' may not be used with a contentless_delete=1 table"
1983 );
1984 rc = SQLITE_ERROR;
1985 }else{
1986 rc = fts5SpecialDelete(pTab, apVal);
1987 }
1988 }else{
1989 rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
1990 }
1991 }else{
1992 /* A regular INSERT, UPDATE or DELETE statement. The trick here is that
1993 ** any conflict on the rowid value must be detected before any
1994 ** modifications are made to the database file. There are 4 cases:
1995 **
1996 ** 1) DELETE
1997 ** 2) UPDATE (rowid not modified)
1998 ** 3) UPDATE (rowid modified)
1999 ** 4) INSERT
2000 **
2001 ** Cases 3 and 4 may violate the rowid constraint.
2002 */
2003 int eConflict = SQLITE_ABORT;
2004 if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){
2005 eConflict = sqlite3_vtab_on_conflict(pConfig->db);
2006 }
2007
2008 assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
2009 assert( nArg!=1 || eType0==SQLITE_INTEGER );
2010
2011 /* DELETE */
2012 if( nArg==1 ){
2013 /* It is only possible to DELETE from a contentless table if the
2014 ** contentless_delete=1 flag is set. */
2015 if( fts5IsContentless(pTab, 1) && pConfig->bContentlessDelete==0 ){
2016 fts5SetVtabError(pTab,
2017 "cannot DELETE from contentless fts5 table: %s", pConfig->zName
2018 );
2019 rc = SQLITE_ERROR;
2020 }else{
2021 i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */
2022 rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0, 0);
2023 }
2024 }
2025
2026 /* INSERT or UPDATE */
2027 else{
2028 int eType1 = sqlite3_value_numeric_type(apVal[1]);
2029
2030 /* It is an error to write an fts5_locale() value to a table without
2031 ** the locale=1 option. */
2032 if( pConfig->bLocale==0 ){
2033 int ii;
2034 for(ii=0; ii<pConfig->nCol; ii++){
2035 sqlite3_value *pVal = apVal[ii+2];
2036 if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
2037 fts5SetVtabError(pTab, "fts5_locale() requires locale=1");
2038 rc = SQLITE_MISMATCH;
2039 goto update_out;
2040 }
2041 }
2042 }
2043
2044 if( eType0!=SQLITE_INTEGER ){
2045 /* An INSERT statement. If the conflict-mode is REPLACE, first remove
2046 ** the current entry (if any). */
2047 if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){
2048 i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */
2049 rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0);
2050 }
2051 fts5StorageInsert(&rc, pTab, apVal, pRowid);
2052 }
2053
2054 /* UPDATE */
2055 else{
2056 Fts5Storage *pStorage = pTab->pStorage;
2057 i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */
2058 i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */
2059 int bContent = 0; /* Content only update */
2060
2061 /* If this is a contentless table (including contentless_unindexed=1
2062 ** tables), check if the UPDATE may proceed. */
2063 if( fts5IsContentless(pTab, 1) ){
2064 rc = fts5ContentlessUpdate(pConfig, &apVal[2], iOld!=iNew, &bContent);
2065 if( rc!=SQLITE_OK ) goto update_out;
2066 }
2067
2068 if( eType1!=SQLITE_INTEGER ){
2069 rc = SQLITE_MISMATCH;
2070 }else if( iOld!=iNew ){
2071 assert( bContent==0 );
2072 if( eConflict==SQLITE_REPLACE ){
2073 rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1);
2074 if( rc==SQLITE_OK ){
2075 rc = sqlite3Fts5StorageDelete(pStorage, iNew, 0, 0);
2076 }
2077 fts5StorageInsert(&rc, pTab, apVal, pRowid);
2078 }else{
2079 rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld);
2080 if( rc==SQLITE_OK ){
2081 rc = sqlite3Fts5StorageContentInsert(pStorage, 0, apVal, pRowid);
2082 }
2083 if( rc==SQLITE_OK ){
2084 rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 0);
2085 }
2086 if( rc==SQLITE_OK ){
2087 rc = sqlite3Fts5StorageIndexInsert(pStorage, apVal, *pRowid);
2088 }
2089 }
2090 }else if( bContent ){
2091 /* This occurs when an UPDATE on a contentless table affects *only*
2092 ** UNINDEXED columns. This is a no-op for contentless_unindexed=0
2093 ** tables, or a write to the %_content table only for =1 tables. */
2094 assert( fts5IsContentless(pTab, 1) );
2095 rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld);
2096 if( rc==SQLITE_OK ){
2097 rc = sqlite3Fts5StorageContentInsert(pStorage, 1, apVal, pRowid);
2098 }
2099 }else{
2100 rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1);
2101 fts5StorageInsert(&rc, pTab, apVal, pRowid);
2102 }
2103 sqlite3Fts5StorageReleaseDeleteRow(pStorage);
2104 }
2105 }
2106 }
2107
2108 update_out:
2109 sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
2110 pTab->p.pConfig->pzErrmsg = 0;
2111 return rc;
2112 }
2113
2114 /*
2115 ** Implementation of xSync() method.
2116 */
2117 static int fts5SyncMethod(sqlite3_vtab *pVtab){
2118 int rc;
2119 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
2120 fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
2121 pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
2122 rc = sqlite3Fts5FlushToDisk(&pTab->p);
2123 pTab->p.pConfig->pzErrmsg = 0;
2124 return rc;
2125 }
2126
2127 /*
2128 ** Implementation of xBegin() method.
2129 */
2130 static int fts5BeginMethod(sqlite3_vtab *pVtab){
2131 int rc = fts5NewTransaction((Fts5FullTable*)pVtab);
2132 if( rc==SQLITE_OK ){
2133 fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0);
2134 }
2135 return rc;
2136 }
2137
2138 /*
2139 ** Implementation of xCommit() method. This is a no-op. The contents of
2140 ** the pending-terms hash-table have already been flushed into the database
2141 ** by fts5SyncMethod().
2142 */
2143 static int fts5CommitMethod(sqlite3_vtab *pVtab){
2144 UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */
2145 fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_COMMIT, 0);
2146 return SQLITE_OK;
2147 }
2148
2149 /*
2150 ** Implementation of xRollback(). Discard the contents of the pending-terms
2151 ** hash-table. Any changes made to the database are reverted by SQLite.
2152 */
2153 static int fts5RollbackMethod(sqlite3_vtab *pVtab){
2154 int rc;
2155 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
2156 fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0);
2157 rc = sqlite3Fts5StorageRollback(pTab->pStorage);
2158 pTab->p.pConfig->pgsz = 0;
2159 return rc;
2160 }
2161
2162 static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*);
2163
2164 static void *fts5ApiUserData(Fts5Context *pCtx){
2165 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2166 return pCsr->pAux->pUserData;
2167 }
2168
2169 static int fts5ApiColumnCount(Fts5Context *pCtx){
2170 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2171 return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol;
2172 }
2173
2174 static int fts5ApiColumnTotalSize(
2175 Fts5Context *pCtx,
2176 int iCol,
2177 sqlite3_int64 *pnToken
2178 ){
2179 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2180 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
2181 return sqlite3Fts5StorageSize(pTab->pStorage, iCol, pnToken);
2182 }
2183
2184 static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){
2185 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2186 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
2187 return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow);
2188 }
2189
2190 /*
2191 ** Implementation of xTokenize_v2() API.
2192 */
2193 static int fts5ApiTokenize_v2(
2194 Fts5Context *pCtx,
2195 const char *pText, int nText,
2196 const char *pLoc, int nLoc,
2197 void *pUserData,
2198 int (*xToken)(void*, int, const char*, int, int, int)
2199 ){
2200 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2201 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
2202 int rc = SQLITE_OK;
2203
2204 sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc);
2205 rc = sqlite3Fts5Tokenize(pTab->pConfig,
2206 FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken
2207 );
2208 sqlite3Fts5SetLocale(pTab->pConfig, 0, 0);
2209
2210 return rc;
2211 }
2212
2213 /*
2214 ** Implementation of xTokenize() API. This is just xTokenize_v2() with NULL/0
2215 ** passed as the locale.
2216 */
2217 static int fts5ApiTokenize(
2218 Fts5Context *pCtx,
2219 const char *pText, int nText,
2220 void *pUserData,
2221 int (*xToken)(void*, int, const char*, int, int, int)
2222 ){
2223 return fts5ApiTokenize_v2(pCtx, pText, nText, 0, 0, pUserData, xToken);
2224 }
2225
2226 static int fts5ApiPhraseCount(Fts5Context *pCtx){
2227 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2228 return sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
2229 }
2230
2231 static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
2232 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2233 return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
2234 }
2235
2236 /*
2237 ** Argument pStmt is an SQL statement of the type used by Fts5Cursor. This
2238 ** function extracts the text value of column iCol of the current row.
2239 ** Additionally, if there is an associated locale, it invokes
2240 ** sqlite3Fts5SetLocale() to configure the tokenizer. In all cases the caller
2241 ** should invoke sqlite3Fts5ClearLocale() to clear the locale at some point
2242 ** after this function returns.
2243 **
2244 ** If successful, (*ppText) is set to point to a buffer containing the text
2245 ** value as utf-8 and SQLITE_OK returned. (*pnText) is set to the size of that
2246 ** buffer in bytes. It is not guaranteed to be nul-terminated. If an error
2247 ** occurs, an SQLite error code is returned. The final values of the two
2248 ** output parameters are undefined in this case.
2249 */
2250 static int fts5TextFromStmt(
2251 Fts5Config *pConfig,
2252 sqlite3_stmt *pStmt,
2253 int iCol,
2254 const char **ppText,
2255 int *pnText
2256 ){
2257 sqlite3_value *pVal = sqlite3_column_value(pStmt, iCol+1);
2258 const char *pLoc = 0;
2259 int nLoc = 0;
2260 int rc = SQLITE_OK;
2261
2262 if( pConfig->bLocale
2263 && pConfig->eContent==FTS5_CONTENT_EXTERNAL
2264 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
2265 ){
2266 rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc);
2267 }else{
2268 *ppText = (const char*)sqlite3_value_text(pVal);
2269 *pnText = sqlite3_value_bytes(pVal);
2270 if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL ){
2271 pLoc = (const char*)sqlite3_column_text(pStmt, iCol+1+pConfig->nCol);
2272 nLoc = sqlite3_column_bytes(pStmt, iCol+1+pConfig->nCol);
2273 }
2274 }
2275 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
2276 return rc;
2277 }
2278
2279 static int fts5ApiColumnText(
2280 Fts5Context *pCtx,
2281 int iCol,
2282 const char **pz,
2283 int *pn
2284 ){
2285 int rc = SQLITE_OK;
2286 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2287 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
2288
2289 assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL );
2290 if( iCol<0 || iCol>=pTab->pConfig->nCol ){
2291 rc = SQLITE_RANGE;
2292 }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab), 0) ){
2293 *pz = 0;
2294 *pn = 0;
2295 }else{
2296 rc = fts5SeekCursor(pCsr, 0);
2297 if( rc==SQLITE_OK ){
2298 rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn);
2299 sqlite3Fts5ClearLocale(pTab->pConfig);
2300 }
2301 }
2302 return rc;
2303 }
2304
2305 /*
2306 ** This is called by various API functions - xInst, xPhraseFirst,
2307 ** xPhraseFirstColumn etc. - to obtain the position list for phrase iPhrase
2308 ** of the current row. This function works for both detail=full tables (in
2309 ** which case the position-list was read from the fts index) or for other
2310 ** detail= modes if the row content is available.
2311 */
2312 static int fts5CsrPoslist(
2313 Fts5Cursor *pCsr, /* Fts5 cursor object */
2314 int iPhrase, /* Phrase to find position list for */
2315 const u8 **pa, /* OUT: Pointer to position list buffer */
2316 int *pn /* OUT: Size of (*pa) in bytes */
2317 ){
2318 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
2319 int rc = SQLITE_OK;
2320 int bLive = (pCsr->pSorter==0);
2321
2322 if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){
2323 rc = SQLITE_RANGE;
2324 }else if( pConfig->eDetail!=FTS5_DETAIL_FULL
2325 && fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1)
2326 ){
2327 *pa = 0;
2328 *pn = 0;
2329 return SQLITE_OK;
2330 }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
2331 if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
2332 Fts5PoslistPopulator *aPopulator;
2333 int i;
2334
2335 aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive);
2336 if( aPopulator==0 ) rc = SQLITE_NOMEM;
2337 if( rc==SQLITE_OK ){
2338 rc = fts5SeekCursor(pCsr, 0);
2339 }
2340 for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
2341 const char *z = 0;
2342 int n = 0;
2343 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
2344 if( rc==SQLITE_OK ){
2345 rc = sqlite3Fts5ExprPopulatePoslists(
2346 pConfig, pCsr->pExpr, aPopulator, i, z, n
2347 );
2348 }
2349 sqlite3Fts5ClearLocale(pConfig);
2350 }
2351 sqlite3_free(aPopulator);
2352
2353 if( pCsr->pSorter ){
2354 sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid);
2355 }
2356 }
2357 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST);
2358 }
2359
2360 if( rc==SQLITE_OK ){
2361 if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){
2362 Fts5Sorter *pSorter = pCsr->pSorter;
2363 int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
2364 *pn = pSorter->aIdx[iPhrase] - i1;
2365 *pa = &pSorter->aPoslist[i1];
2366 }else{
2367 *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
2368 }
2369 }else{
2370 *pa = 0;
2371 *pn = 0;
2372 }
2373
2374 return rc;
2375 }
2376
2377 /*
2378 ** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated
2379 ** correctly for the current view. Return SQLITE_OK if successful, or an
2380 ** SQLite error code otherwise.
2381 */
2382 static int fts5CacheInstArray(Fts5Cursor *pCsr){
2383 int rc = SQLITE_OK;
2384 Fts5PoslistReader *aIter; /* One iterator for each phrase */
2385 int nIter; /* Number of iterators/phrases */
2386 int nCol = ((Fts5Table*)pCsr->base.pVtab)->pConfig->nCol;
2387
2388 nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
2389 if( pCsr->aInstIter==0 ){
2390 sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nIter;
2391 pCsr->aInstIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte);
2392 }
2393 aIter = pCsr->aInstIter;
2394
2395 if( aIter ){
2396 int nInst = 0; /* Number instances seen so far */
2397 int i;
2398
2399 /* Initialize all iterators */
2400 for(i=0; i<nIter && rc==SQLITE_OK; i++){
2401 const u8 *a;
2402 int n;
2403 rc = fts5CsrPoslist(pCsr, i, &a, &n);
2404 if( rc==SQLITE_OK ){
2405 sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
2406 }
2407 }
2408
2409 if( rc==SQLITE_OK ){
2410 while( 1 ){
2411 int *aInst;
2412 int iBest = -1;
2413 for(i=0; i<nIter; i++){
2414 if( (aIter[i].bEof==0)
2415 && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos)
2416 ){
2417 iBest = i;
2418 }
2419 }
2420 if( iBest<0 ) break;
2421
2422 nInst++;
2423 if( nInst>=pCsr->nInstAlloc ){
2424 int nNewSize = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
2425 aInst = (int*)sqlite3_realloc64(
2426 pCsr->aInst, nNewSize*sizeof(int)*3
2427 );
2428 if( aInst ){
2429 pCsr->aInst = aInst;
2430 pCsr->nInstAlloc = nNewSize;
2431 }else{
2432 nInst--;
2433 rc = SQLITE_NOMEM;
2434 break;
2435 }
2436 }
2437
2438 aInst = &pCsr->aInst[3 * (nInst-1)];
2439 aInst[0] = iBest;
2440 aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos);
2441 aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos);
2442 assert( aInst[1]>=0 );
2443 if( aInst[1]>=nCol ){
2444 rc = FTS5_CORRUPT;
2445 break;
2446 }
2447 sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
2448 }
2449 }
2450
2451 pCsr->nInstCount = nInst;
2452 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST);
2453 }
2454 return rc;
2455 }
2456
2457 static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){
2458 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2459 int rc = SQLITE_OK;
2460 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0
2461 || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){
2462 *pnInst = pCsr->nInstCount;
2463 }
2464 return rc;
2465 }
2466
2467 static int fts5ApiInst(
2468 Fts5Context *pCtx,
2469 int iIdx,
2470 int *piPhrase,
2471 int *piCol,
2472 int *piOff
2473 ){
2474 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2475 int rc = SQLITE_OK;
2476 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0
2477 || SQLITE_OK==(rc = fts5CacheInstArray(pCsr))
2478 ){
2479 if( iIdx<0 || iIdx>=pCsr->nInstCount ){
2480 rc = SQLITE_RANGE;
2481 }else{
2482 *piPhrase = pCsr->aInst[iIdx*3];
2483 *piCol = pCsr->aInst[iIdx*3 + 1];
2484 *piOff = pCsr->aInst[iIdx*3 + 2];
2485 }
2486 }
2487 return rc;
2488 }
2489
2490 static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
2491 return fts5CursorRowid((Fts5Cursor*)pCtx);
2492 }
2493
2494 static int fts5ColumnSizeCb(
2495 void *pContext, /* Pointer to int */
2496 int tflags,
2497 const char *pUnused, /* Buffer containing token */
2498 int nUnused, /* Size of token in bytes */
2499 int iUnused1, /* Start offset of token */
2500 int iUnused2 /* End offset of token */
2501 ){
2502 int *pCnt = (int*)pContext;
2503 UNUSED_PARAM2(pUnused, nUnused);
2504 UNUSED_PARAM2(iUnused1, iUnused2);
2505 if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
2506 (*pCnt)++;
2507 }
2508 return SQLITE_OK;
2509 }
2510
2511 static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
2512 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2513 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
2514 Fts5Config *pConfig = pTab->p.pConfig;
2515 int rc = SQLITE_OK;
2516
2517 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
2518 if( pConfig->bColumnsize ){
2519 i64 iRowid = fts5CursorRowid(pCsr);
2520 rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
2521 }else if( !pConfig->zContent || pConfig->eContent==FTS5_CONTENT_UNINDEXED ){
2522 int i;
2523 for(i=0; i<pConfig->nCol; i++){
2524 if( pConfig->abUnindexed[i]==0 ){
2525 pCsr->aColumnSize[i] = -1;
2526 }
2527 }
2528 }else{
2529 int i;
2530 rc = fts5SeekCursor(pCsr, 0);
2531 for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
2532 if( pConfig->abUnindexed[i]==0 ){
2533 const char *z = 0;
2534 int n = 0;
2535 pCsr->aColumnSize[i] = 0;
2536 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n);
2537 if( rc==SQLITE_OK ){
2538 rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX,
2539 z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb
2540 );
2541 }
2542 sqlite3Fts5ClearLocale(pConfig);
2543 }
2544 }
2545 }
2546 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
2547 }
2548 if( iCol<0 ){
2549 int i;
2550 *pnToken = 0;
2551 for(i=0; i<pConfig->nCol; i++){
2552 *pnToken += pCsr->aColumnSize[i];
2553 }
2554 }else if( iCol<pConfig->nCol ){
2555 *pnToken = pCsr->aColumnSize[iCol];
2556 }else{
2557 *pnToken = 0;
2558 rc = SQLITE_RANGE;
2559 }
2560 return rc;
2561 }
2562
2563 /*
2564 ** Implementation of the xSetAuxdata() method.
2565 */
2566 static int fts5ApiSetAuxdata(
2567 Fts5Context *pCtx, /* Fts5 context */
2568 void *pPtr, /* Pointer to save as auxdata */
2569 void(*xDelete)(void*) /* Destructor for pPtr (or NULL) */
2570 ){
2571 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2572 Fts5Auxdata *pData;
2573
2574 /* Search through the cursors list of Fts5Auxdata objects for one that
2575 ** corresponds to the currently executing auxiliary function. */
2576 for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){
2577 if( pData->pAux==pCsr->pAux ) break;
2578 }
2579
2580 if( pData ){
2581 if( pData->xDelete ){
2582 pData->xDelete(pData->pPtr);
2583 }
2584 }else{
2585 int rc = SQLITE_OK;
2586 pData = (Fts5Auxdata*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Auxdata));
2587 if( pData==0 ){
2588 if( xDelete ) xDelete(pPtr);
2589 return rc;
2590 }
2591 pData->pAux = pCsr->pAux;
2592 pData->pNext = pCsr->pAuxdata;
2593 pCsr->pAuxdata = pData;
2594 }
2595
2596 pData->xDelete = xDelete;
2597 pData->pPtr = pPtr;
2598 return SQLITE_OK;
2599 }
2600
2601 static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){
2602 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2603 Fts5Auxdata *pData;
2604 void *pRet = 0;
2605
2606 for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){
2607 if( pData->pAux==pCsr->pAux ) break;
2608 }
2609
2610 if( pData ){
2611 pRet = pData->pPtr;
2612 if( bClear ){
2613 pData->pPtr = 0;
2614 pData->xDelete = 0;
2615 }
2616 }
2617
2618 return pRet;
2619 }
2620
2621 static void fts5ApiPhraseNext(
2622 Fts5Context *pCtx,
2623 Fts5PhraseIter *pIter,
2624 int *piCol, int *piOff
2625 ){
2626 if( pIter->a>=pIter->b ){
2627 *piCol = -1;
2628 *piOff = -1;
2629 }else{
2630 int iVal;
2631 pIter->a += fts5GetVarint32(pIter->a, iVal);
2632 if( iVal==1 ){
2633 /* Avoid returning a (*piCol) value that is too large for the table,
2634 ** even if the position-list is corrupt. The caller might not be
2635 ** expecting it. */
2636 int nCol = ((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab))->pConfig->nCol;
2637 pIter->a += fts5GetVarint32(pIter->a, iVal);
2638 *piCol = (iVal>=nCol ? nCol-1 : iVal);
2639 *piOff = 0;
2640 pIter->a += fts5GetVarint32(pIter->a, iVal);
2641 }
2642 *piOff += (iVal-2);
2643 }
2644 }
2645
2646 static int fts5ApiPhraseFirst(
2647 Fts5Context *pCtx,
2648 int iPhrase,
2649 Fts5PhraseIter *pIter,
2650 int *piCol, int *piOff
2651 ){
2652 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2653 int n;
2654 int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
2655 if( rc==SQLITE_OK ){
2656 assert( pIter->a || n==0 );
2657 pIter->b = (pIter->a ? &pIter->a[n] : 0);
2658 *piCol = 0;
2659 *piOff = 0;
2660 fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
2661 }
2662 return rc;
2663 }
2664
2665 static void fts5ApiPhraseNextColumn(
2666 Fts5Context *pCtx,
2667 Fts5PhraseIter *pIter,
2668 int *piCol
2669 ){
2670 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2671 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
2672
2673 if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
2674 if( pIter->a>=pIter->b ){
2675 *piCol = -1;
2676 }else{
2677 int iIncr;
2678 pIter->a += fts5GetVarint32(&pIter->a[0], iIncr);
2679 *piCol += (iIncr-2);
2680 }
2681 }else{
2682 while( 1 ){
2683 int dummy;
2684 if( pIter->a>=pIter->b ){
2685 *piCol = -1;
2686 return;
2687 }
2688 if( pIter->a[0]==0x01 ) break;
2689 pIter->a += fts5GetVarint32(pIter->a, dummy);
2690 }
2691 pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
2692 }
2693 }
2694
2695 static int fts5ApiPhraseFirstColumn(
2696 Fts5Context *pCtx,
2697 int iPhrase,
2698 Fts5PhraseIter *pIter,
2699 int *piCol
2700 ){
2701 int rc = SQLITE_OK;
2702 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2703 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
2704
2705 if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
2706 Fts5Sorter *pSorter = pCsr->pSorter;
2707 int n;
2708 if( pSorter ){
2709 int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
2710 n = pSorter->aIdx[iPhrase] - i1;
2711 pIter->a = &pSorter->aPoslist[i1];
2712 }else{
2713 rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
2714 }
2715 if( rc==SQLITE_OK ){
2716 assert( pIter->a || n==0 );
2717 pIter->b = (pIter->a ? &pIter->a[n] : 0);
2718 *piCol = 0;
2719 fts5ApiPhraseNextColumn(pCtx, pIter, piCol);
2720 }
2721 }else{
2722 int n;
2723 rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
2724 if( rc==SQLITE_OK ){
2725 assert( pIter->a || n==0 );
2726 pIter->b = (pIter->a ? &pIter->a[n] : 0);
2727 if( n<=0 ){
2728 *piCol = -1;
2729 }else if( pIter->a[0]==0x01 ){
2730 pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
2731 }else{
2732 *piCol = 0;
2733 }
2734 }
2735 }
2736
2737 return rc;
2738 }
2739
2740 /*
2741 ** xQueryToken() API implemenetation.
2742 */
2743 static int fts5ApiQueryToken(
2744 Fts5Context* pCtx,
2745 int iPhrase,
2746 int iToken,
2747 const char **ppOut,
2748 int *pnOut
2749 ){
2750 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2751 return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut);
2752 }
2753
2754 /*
2755 ** xInstToken() API implemenetation.
2756 */
2757 static int fts5ApiInstToken(
2758 Fts5Context *pCtx,
2759 int iIdx,
2760 int iToken,
2761 const char **ppOut, int *pnOut
2762 ){
2763 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2764 int rc = SQLITE_OK;
2765 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0
2766 || SQLITE_OK==(rc = fts5CacheInstArray(pCsr))
2767 ){
2768 if( iIdx<0 || iIdx>=pCsr->nInstCount ){
2769 rc = SQLITE_RANGE;
2770 }else{
2771 int iPhrase = pCsr->aInst[iIdx*3];
2772 int iCol = pCsr->aInst[iIdx*3 + 1];
2773 int iOff = pCsr->aInst[iIdx*3 + 2];
2774 i64 iRowid = fts5CursorRowid(pCsr);
2775 rc = sqlite3Fts5ExprInstToken(
2776 pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut
2777 );
2778 }
2779 }
2780 return rc;
2781 }
2782
2783
2784 static int fts5ApiQueryPhrase(Fts5Context*, int, void*,
2785 int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
2786 );
2787
2788 /*
2789 ** The xColumnLocale() API.
2790 */
2791 static int fts5ApiColumnLocale(
2792 Fts5Context *pCtx,
2793 int iCol,
2794 const char **pzLocale,
2795 int *pnLocale
2796 ){
2797 int rc = SQLITE_OK;
2798 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2799 Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
2800
2801 *pzLocale = 0;
2802 *pnLocale = 0;
2803
2804 assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL );
2805 if( iCol<0 || iCol>=pConfig->nCol ){
2806 rc = SQLITE_RANGE;
2807 }else if(
2808 pConfig->abUnindexed[iCol]==0
2809 && 0==fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1)
2810 && pConfig->bLocale
2811 ){
2812 rc = fts5SeekCursor(pCsr, 0);
2813 if( rc==SQLITE_OK ){
2814 const char *zDummy = 0;
2815 int nDummy = 0;
2816 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &zDummy, &nDummy);
2817 if( rc==SQLITE_OK ){
2818 *pzLocale = pConfig->t.pLocale;
2819 *pnLocale = pConfig->t.nLocale;
2820 }
2821 sqlite3Fts5ClearLocale(pConfig);
2822 }
2823 }
2824
2825 return rc;
2826 }
2827
2828 static const Fts5ExtensionApi sFts5Api = {
2829 4, /* iVersion */
2830 fts5ApiUserData,
2831 fts5ApiColumnCount,
2832 fts5ApiRowCount,
2833 fts5ApiColumnTotalSize,
2834 fts5ApiTokenize,
2835 fts5ApiPhraseCount,
2836 fts5ApiPhraseSize,
2837 fts5ApiInstCount,
2838 fts5ApiInst,
2839 fts5ApiRowid,
2840 fts5ApiColumnText,
2841 fts5ApiColumnSize,
2842 fts5ApiQueryPhrase,
2843 fts5ApiSetAuxdata,
2844 fts5ApiGetAuxdata,
2845 fts5ApiPhraseFirst,
2846 fts5ApiPhraseNext,
2847 fts5ApiPhraseFirstColumn,
2848 fts5ApiPhraseNextColumn,
2849 fts5ApiQueryToken,
2850 fts5ApiInstToken,
2851 fts5ApiColumnLocale,
2852 fts5ApiTokenize_v2
2853 };
2854
2855 /*
2856 ** Implementation of API function xQueryPhrase().
2857 */
2858 static int fts5ApiQueryPhrase(
2859 Fts5Context *pCtx,
2860 int iPhrase,
2861 void *pUserData,
2862 int(*xCallback)(const Fts5ExtensionApi*, Fts5Context*, void*)
2863 ){
2864 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
2865 Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab);
2866 int rc;
2867 Fts5Cursor *pNew = 0;
2868
2869 rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew);
2870 if( rc==SQLITE_OK ){
2871 pNew->ePlan = FTS5_PLAN_MATCH;
2872 pNew->iFirstRowid = SMALLEST_INT64;
2873 pNew->iLastRowid = LARGEST_INT64;
2874 pNew->base.pVtab = (sqlite3_vtab*)pTab;
2875 rc = sqlite3Fts5ExprClonePhrase(pCsr->pExpr, iPhrase, &pNew->pExpr);
2876 }
2877
2878 if( rc==SQLITE_OK ){
2879 for(rc = fts5CursorFirst(pTab, pNew, 0);
2880 rc==SQLITE_OK && CsrFlagTest(pNew, FTS5CSR_EOF)==0;
2881 rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew)
2882 ){
2883 rc = xCallback(&sFts5Api, (Fts5Context*)pNew, pUserData);
2884 if( rc!=SQLITE_OK ){
2885 if( rc==SQLITE_DONE ) rc = SQLITE_OK;
2886 break;
2887 }
2888 }
2889 }
2890
2891 fts5CloseMethod((sqlite3_vtab_cursor*)pNew);
2892 return rc;
2893 }
2894
2895 static void fts5ApiInvoke(
2896 Fts5Auxiliary *pAux,
2897 Fts5Cursor *pCsr,
2898 sqlite3_context *context,
2899 int argc,
2900 sqlite3_value **argv
2901 ){
2902 assert( pCsr->pAux==0 );
2903 assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL );
2904 pCsr->pAux = pAux;
2905 pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv);
2906 pCsr->pAux = 0;
2907 }
2908
2909 static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){
2910 Fts5Cursor *pCsr;
2911 for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
2912 if( pCsr->iCsrId==iCsrId ) break;
2913 }
2914 return pCsr;
2915 }
2916
2917 /*
2918 ** Parameter zFmt is a printf() style formatting string. This function
2919 ** formats it using the trailing arguments and returns the result as
2920 ** an error message to the context passed as the first argument.
2921 */
2922 static void fts5ResultError(sqlite3_context *pCtx, const char *zFmt, ...){
2923 char *zErr = 0;
2924 va_list ap;
2925 va_start(ap, zFmt);
2926 zErr = sqlite3_vmprintf(zFmt, ap);
2927 sqlite3_result_error(pCtx, zErr, -1);
2928 sqlite3_free(zErr);
2929 va_end(ap);
2930 }
2931
2932 static void fts5ApiCallback(
2933 sqlite3_context *context,
2934 int argc,
2935 sqlite3_value **argv
2936 ){
2937
2938 Fts5Auxiliary *pAux;
2939 Fts5Cursor *pCsr;
2940 i64 iCsrId;
2941
2942 assert( argc>=1 );
2943 pAux = (Fts5Auxiliary*)sqlite3_user_data(context);
2944 iCsrId = sqlite3_value_int64(argv[0]);
2945
2946 pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId);
2947 if( pCsr==0 || (pCsr->ePlan==0 || pCsr->ePlan==FTS5_PLAN_SPECIAL) ){
2948 fts5ResultError(context, "no such cursor: %lld", iCsrId);
2949 }else{
2950 sqlite3_vtab *pTab = pCsr->base.pVtab;
2951 fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]);
2952 sqlite3_free(pTab->zErrMsg);
2953 pTab->zErrMsg = 0;
2954 }
2955 }
2956
2957
2958 /*
2959 ** Given cursor id iId, return a pointer to the corresponding Fts5Table
2960 ** object. Or NULL If the cursor id does not exist.
2961 */
2962 Fts5Table *sqlite3Fts5TableFromCsrid(
2963 Fts5Global *pGlobal, /* FTS5 global context for db handle */
2964 i64 iCsrId /* Id of cursor to find */
2965 ){
2966 Fts5Cursor *pCsr;
2967 pCsr = fts5CursorFromCsrid(pGlobal, iCsrId);
2968 if( pCsr ){
2969 return (Fts5Table*)pCsr->base.pVtab;
2970 }
2971 return 0;
2972 }
2973
2974 /*
2975 ** Return a "position-list blob" corresponding to the current position of
2976 ** cursor pCsr via sqlite3_result_blob(). A position-list blob contains
2977 ** the current position-list for each phrase in the query associated with
2978 ** cursor pCsr.
2979 **
2980 ** A position-list blob begins with (nPhrase-1) varints, where nPhrase is
2981 ** the number of phrases in the query. Following the varints are the
2982 ** concatenated position lists for each phrase, in order.
2983 **
2984 ** The first varint (if it exists) contains the size of the position list
2985 ** for phrase 0. The second (same disclaimer) contains the size of position
2986 ** list 1. And so on. There is no size field for the final position list,
2987 ** as it can be derived from the total size of the blob.
2988 */
2989 static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
2990 int i;
2991 int rc = SQLITE_OK;
2992 int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
2993 Fts5Buffer val;
2994
2995 memset(&val, 0, sizeof(Fts5Buffer));
2996 switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){
2997 case FTS5_DETAIL_FULL:
2998
2999 /* Append the varints */
3000 for(i=0; i<(nPhrase-1); i++){
3001 const u8 *dummy;
3002 int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
3003 sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
3004 }
3005
3006 /* Append the position lists */
3007 for(i=0; i<nPhrase; i++){
3008 const u8 *pPoslist;
3009 int nPoslist;
3010 nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
3011 sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
3012 }
3013 break;
3014
3015 case FTS5_DETAIL_COLUMNS:
3016
3017 /* Append the varints */
3018 for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){
3019 const u8 *dummy;
3020 int nByte;
3021 rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte);
3022 sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
3023 }
3024
3025 /* Append the position lists */
3026 for(i=0; rc==SQLITE_OK && i<nPhrase; i++){
3027 const u8 *pPoslist;
3028 int nPoslist;
3029 rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &pPoslist, &nPoslist);
3030 sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
3031 }
3032 break;
3033
3034 default:
3035 break;
3036 }
3037
3038 sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free);
3039 return rc;
3040 }
3041
3042 /*
3043 ** This is the xColumn method, called by SQLite to request a value from
3044 ** the row that the supplied cursor currently points to.
3045 */
3046 static int fts5ColumnMethod(
3047 sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */
3048 sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */
3049 int iCol /* Index of column to read value from */
3050 ){
3051 Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab);
3052 Fts5Config *pConfig = pTab->p.pConfig;
3053 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
3054 int rc = SQLITE_OK;
3055
3056 assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
3057
3058 if( pCsr->ePlan==FTS5_PLAN_SPECIAL ){
3059 if( iCol==pConfig->nCol ){
3060 sqlite3_result_int64(pCtx, pCsr->iSpecial);
3061 }
3062 }else
3063
3064 if( iCol==pConfig->nCol ){
3065 /* User is requesting the value of the special column with the same name
3066 ** as the table. Return the cursor integer id number. This value is only
3067 ** useful in that it may be passed as the first argument to an FTS5
3068 ** auxiliary function. */
3069 sqlite3_result_int64(pCtx, pCsr->iCsrId);
3070 }else if( iCol==pConfig->nCol+1 ){
3071 /* The value of the "rank" column. */
3072
3073 if( pCsr->ePlan==FTS5_PLAN_SOURCE ){
3074 fts5PoslistBlob(pCtx, pCsr);
3075 }else if(
3076 pCsr->ePlan==FTS5_PLAN_MATCH
3077 || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH
3078 ){
3079 if( pCsr->pRank || SQLITE_OK==(rc = fts5FindRankFunction(pCsr)) ){
3080 fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg);
3081 }
3082 }
3083 }else{
3084 if( !sqlite3_vtab_nochange(pCtx) && pConfig->eContent!=FTS5_CONTENT_NONE ){
3085 pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
3086 rc = fts5SeekCursor(pCsr, 1);
3087 if( rc==SQLITE_OK ){
3088 sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1);
3089 if( pConfig->bLocale
3090 && pConfig->eContent==FTS5_CONTENT_EXTERNAL
3091 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
3092 ){
3093 const char *z = 0;
3094 int n = 0;
3095 rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n);
3096 if( rc==SQLITE_OK ){
3097 sqlite3_result_text(pCtx, z, n, SQLITE_TRANSIENT);
3098 }
3099 sqlite3Fts5ClearLocale(pConfig);
3100 }else{
3101 sqlite3_result_value(pCtx, pVal);
3102 }
3103 }
3104
3105 pConfig->pzErrmsg = 0;
3106 }
3107 }
3108
3109 return rc;
3110 }
3111
3112
3113 /*
3114 ** This routine implements the xFindFunction method for the FTS3
3115 ** virtual table.
3116 */
3117 static int fts5FindFunctionMethod(
3118 sqlite3_vtab *pVtab, /* Virtual table handle */
3119 int nUnused, /* Number of SQL function arguments */
3120 const char *zName, /* Name of SQL function */
3121 void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
3122 void **ppArg /* OUT: User data for *pxFunc */
3123 ){
3124 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
3125 Fts5Auxiliary *pAux;
3126
3127 UNUSED_PARAM(nUnused);
3128 pAux = fts5FindAuxiliary(pTab, zName);
3129 if( pAux ){
3130 *pxFunc = fts5ApiCallback;
3131 *ppArg = (void*)pAux;
3132 return 1;
3133 }
3134
3135 /* No function of the specified name was found. Return 0. */
3136 return 0;
3137 }
3138
3139 /*
3140 ** Implementation of FTS5 xRename method. Rename an fts5 table.
3141 */
3142 static int fts5RenameMethod(
3143 sqlite3_vtab *pVtab, /* Virtual table handle */
3144 const char *zName /* New name of table */
3145 ){
3146 int rc;
3147 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
3148 rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
3149 return rc;
3150 }
3151
3152 int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
3153 fts5TripCursors((Fts5FullTable*)pTab);
3154 return sqlite3Fts5StorageSync(((Fts5FullTable*)pTab)->pStorage);
3155 }
3156
3157 /*
3158 ** The xSavepoint() method.
3159 **
3160 ** Flush the contents of the pending-terms table to disk.
3161 */
3162 static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
3163 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
3164 int rc = SQLITE_OK;
3165
3166 fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
3167 rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
3168 if( rc==SQLITE_OK ){
3169 pTab->iSavepoint = iSavepoint+1;
3170 }
3171 return rc;
3172 }
3173
3174 /*
3175 ** The xRelease() method.
3176 **
3177 ** This is a no-op.
3178 */
3179 static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
3180 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
3181 int rc = SQLITE_OK;
3182 fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
3183 if( (iSavepoint+1)<pTab->iSavepoint ){
3184 rc = sqlite3Fts5FlushToDisk(&pTab->p);
3185 if( rc==SQLITE_OK ){
3186 pTab->iSavepoint = iSavepoint;
3187 }
3188 }
3189 return rc;
3190 }
3191
3192 /*
3193 ** The xRollbackTo() method.
3194 **
3195 ** Discard the contents of the pending terms table.
3196 */
3197 static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
3198 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
3199 int rc = SQLITE_OK;
3200 fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
3201 fts5TripCursors(pTab);
3202 if( (iSavepoint+1)<=pTab->iSavepoint ){
3203 pTab->p.pConfig->pgsz = 0;
3204 rc = sqlite3Fts5StorageRollback(pTab->pStorage);
3205 }
3206 return rc;
3207 }
3208
3209 /*
3210 ** Register a new auxiliary function with global context pGlobal.
3211 */
3212 static int fts5CreateAux(
3213 fts5_api *pApi, /* Global context (one per db handle) */
3214 const char *zName, /* Name of new function */
3215 void *pUserData, /* User data for aux. function */
3216 fts5_extension_function xFunc, /* Aux. function implementation */
3217 void(*xDestroy)(void*) /* Destructor for pUserData */
3218 ){
3219 Fts5Global *pGlobal = (Fts5Global*)pApi;
3220 int rc = sqlite3_overload_function(pGlobal->db, zName, -1);
3221 if( rc==SQLITE_OK ){
3222 Fts5Auxiliary *pAux;
3223 sqlite3_int64 nName; /* Size of zName in bytes, including \0 */
3224 sqlite3_int64 nByte; /* Bytes of space to allocate */
3225
3226 nName = strlen(zName) + 1;
3227 nByte = sizeof(Fts5Auxiliary) + nName;
3228 pAux = (Fts5Auxiliary*)sqlite3_malloc64(nByte);
3229 if( pAux ){
3230 memset(pAux, 0, (size_t)nByte);
3231 pAux->zFunc = (char*)&pAux[1];
3232 memcpy(pAux->zFunc, zName, nName);
3233 pAux->pGlobal = pGlobal;
3234 pAux->pUserData = pUserData;
3235 pAux->xFunc = xFunc;
3236 pAux->xDestroy = xDestroy;
3237 pAux->pNext = pGlobal->pAux;
3238 pGlobal->pAux = pAux;
3239 }else{
3240 rc = SQLITE_NOMEM;
3241 }
3242 }
3243
3244 return rc;
3245 }
3246
3247 /*
3248 ** This function is used by xCreateTokenizer_v2() and xCreateTokenizer().
3249 ** It allocates and partially populates a new Fts5TokenizerModule object.
3250 ** The new object is already linked into the Fts5Global context before
3251 ** returning.
3252 **
3253 ** If successful, SQLITE_OK is returned and a pointer to the new
3254 ** Fts5TokenizerModule object returned via output parameter (*ppNew). All
3255 ** that is required is for the caller to fill in the methods in
3256 ** Fts5TokenizerModule.x1 and x2, and to set Fts5TokenizerModule.bV2Native
3257 ** as appropriate.
3258 **
3259 ** If an error occurs, an SQLite error code is returned and the final value
3260 ** of (*ppNew) undefined.
3261 */
3262 static int fts5NewTokenizerModule(
3263 Fts5Global *pGlobal, /* Global context (one per db handle) */
3264 const char *zName, /* Name of new function */
3265 void *pUserData, /* User data for aux. function */
3266 void(*xDestroy)(void*), /* Destructor for pUserData */
3267 Fts5TokenizerModule **ppNew
3268 ){
3269 int rc = SQLITE_OK;
3270 Fts5TokenizerModule *pNew;
3271 sqlite3_int64 nName; /* Size of zName and its \0 terminator */
3272 sqlite3_int64 nByte; /* Bytes of space to allocate */
3273
3274 nName = strlen(zName) + 1;
3275 nByte = sizeof(Fts5TokenizerModule) + nName;
3276 *ppNew = pNew = (Fts5TokenizerModule*)sqlite3Fts5MallocZero(&rc, nByte);
3277 if( pNew ){
3278 pNew->zName = (char*)&pNew[1];
3279 memcpy(pNew->zName, zName, nName);
3280 pNew->pUserData = pUserData;
3281 pNew->xDestroy = xDestroy;
3282 pNew->pNext = pGlobal->pTok;
3283 pGlobal->pTok = pNew;
3284 if( pNew->pNext==0 ){
3285 pGlobal->pDfltTok = pNew;
3286 }
3287 }
3288
3289 return rc;
3290 }
3291
3292 /*
3293 ** An instance of this type is used as the Fts5Tokenizer object for
3294 ** wrapper tokenizers - those that provide access to a v1 tokenizer via
3295 ** the fts5_tokenizer_v2 API, and those that provide access to a v2 tokenizer
3296 ** via the fts5_tokenizer API.
3297 */
3298 typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer;
3299 struct Fts5VtoVTokenizer {
3300 int bV2Native; /* True if v2 native tokenizer */
3301 fts5_tokenizer x1; /* Tokenizer functions */
3302 fts5_tokenizer_v2 x2; /* V2 tokenizer functions */
3303 Fts5Tokenizer *pReal;
3304 };
3305
3306 /*
3307 ** Create a wrapper tokenizer. The context argument pCtx points to the
3308 ** Fts5TokenizerModule object.
3309 */
3310 static int fts5VtoVCreate(
3311 void *pCtx,
3312 const char **azArg,
3313 int nArg,
3314 Fts5Tokenizer **ppOut
3315 ){
3316 Fts5TokenizerModule *pMod = (Fts5TokenizerModule*)pCtx;
3317 Fts5VtoVTokenizer *pNew = 0;
3318 int rc = SQLITE_OK;
3319
3320 pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew));
3321 if( rc==SQLITE_OK ){
3322 pNew->x1 = pMod->x1;
3323 pNew->x2 = pMod->x2;
3324 pNew->bV2Native = pMod->bV2Native;
3325 if( pMod->bV2Native ){
3326 rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal);
3327 }else{
3328 rc = pMod->x1.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal);
3329 }
3330 if( rc!=SQLITE_OK ){
3331 sqlite3_free(pNew);
3332 pNew = 0;
3333 }
3334 }
3335
3336 *ppOut = (Fts5Tokenizer*)pNew;
3337 return rc;
3338 }
3339
3340 /*
3341 ** Delete an Fts5VtoVTokenizer wrapper tokenizer.
3342 */
3343 static void fts5VtoVDelete(Fts5Tokenizer *pTok){
3344 Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
3345 if( p ){
3346 if( p->bV2Native ){
3347 p->x2.xDelete(p->pReal);
3348 }else{
3349 p->x1.xDelete(p->pReal);
3350 }
3351 sqlite3_free(p);
3352 }
3353 }
3354
3355
3356 /*
3357 ** xTokenizer method for a wrapper tokenizer that offers the v1 interface
3358 ** (no support for locales).
3359 */
3360 static int fts5V1toV2Tokenize(
3361 Fts5Tokenizer *pTok,
3362 void *pCtx, int flags,
3363 const char *pText, int nText,
3364 int (*xToken)(void*, int, const char*, int, int, int)
3365 ){
3366 Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
3367 assert( p->bV2Native );
3368 return p->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken);
3369 }
3370
3371 /*
3372 ** xTokenizer method for a wrapper tokenizer that offers the v2 interface
3373 ** (with locale support).
3374 */
3375 static int fts5V2toV1Tokenize(
3376 Fts5Tokenizer *pTok,
3377 void *pCtx, int flags,
3378 const char *pText, int nText,
3379 const char *pLocale, int nLocale,
3380 int (*xToken)(void*, int, const char*, int, int, int)
3381 ){
3382 Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok;
3383 assert( p->bV2Native==0 );
3384 UNUSED_PARAM2(pLocale,nLocale);
3385 return p->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken);
3386 }
3387
3388 /*
3389 ** Register a new tokenizer. This is the implementation of the
3390 ** fts5_api.xCreateTokenizer_v2() method.
3391 */
3392 static int fts5CreateTokenizer_v2(
3393 fts5_api *pApi, /* Global context (one per db handle) */
3394 const char *zName, /* Name of new function */
3395 void *pUserData, /* User data for aux. function */
3396 fts5_tokenizer_v2 *pTokenizer, /* Tokenizer implementation */
3397 void(*xDestroy)(void*) /* Destructor for pUserData */
3398 ){
3399 Fts5Global *pGlobal = (Fts5Global*)pApi;
3400 int rc = SQLITE_OK;
3401
3402 if( pTokenizer->iVersion>2 ){
3403 rc = SQLITE_ERROR;
3404 }else{
3405 Fts5TokenizerModule *pNew = 0;
3406 rc = fts5NewTokenizerModule(pGlobal, zName, pUserData, xDestroy, &pNew);
3407 if( pNew ){
3408 pNew->x2 = *pTokenizer;
3409 pNew->bV2Native = 1;
3410 pNew->x1.xCreate = fts5VtoVCreate;
3411 pNew->x1.xTokenize = fts5V1toV2Tokenize;
3412 pNew->x1.xDelete = fts5VtoVDelete;
3413 }
3414 }
3415
3416 return rc;
3417 }
3418
3419 /*
3420 ** The fts5_api.xCreateTokenizer() method.
3421 */
3422 static int fts5CreateTokenizer(
3423 fts5_api *pApi, /* Global context (one per db handle) */
3424 const char *zName, /* Name of new function */
3425 void *pUserData, /* User data for aux. function */
3426 fts5_tokenizer *pTokenizer, /* Tokenizer implementation */
3427 void(*xDestroy)(void*) /* Destructor for pUserData */
3428 ){
3429 Fts5TokenizerModule *pNew = 0;
3430 int rc = SQLITE_OK;
3431
3432 rc = fts5NewTokenizerModule(
3433 (Fts5Global*)pApi, zName, pUserData, xDestroy, &pNew
3434 );
3435 if( pNew ){
3436 pNew->x1 = *pTokenizer;
3437 pNew->x2.xCreate = fts5VtoVCreate;
3438 pNew->x2.xTokenize = fts5V2toV1Tokenize;
3439 pNew->x2.xDelete = fts5VtoVDelete;
3440 }
3441 return rc;
3442 }
3443
3444 /*
3445 ** Search the global context passed as the first argument for a tokenizer
3446 ** module named zName. If found, return a pointer to the Fts5TokenizerModule
3447 ** object. Otherwise, return NULL.
3448 */
3449 static Fts5TokenizerModule *fts5LocateTokenizer(
3450 Fts5Global *pGlobal, /* Global (one per db handle) object */
3451 const char *zName /* Name of tokenizer module to find */
3452 ){
3453 Fts5TokenizerModule *pMod = 0;
3454
3455 if( zName==0 ){
3456 pMod = pGlobal->pDfltTok;
3457 }else{
3458 for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
3459 if( sqlite3_stricmp(zName, pMod->zName)==0 ) break;
3460 }
3461 }
3462
3463 return pMod;
3464 }
3465
3466 /*
3467 ** Find a tokenizer. This is the implementation of the
3468 ** fts5_api.xFindTokenizer_v2() method.
3469 */
3470 static int fts5FindTokenizer_v2(
3471 fts5_api *pApi, /* Global context (one per db handle) */
3472 const char *zName, /* Name of tokenizer */
3473 void **ppUserData,
3474 fts5_tokenizer_v2 **ppTokenizer /* Populate this object */
3475 ){
3476 int rc = SQLITE_OK;
3477 Fts5TokenizerModule *pMod;
3478
3479 pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName);
3480 if( pMod ){
3481 if( pMod->bV2Native ){
3482 *ppUserData = pMod->pUserData;
3483 }else{
3484 *ppUserData = (void*)pMod;
3485 }
3486 *ppTokenizer = &pMod->x2;
3487 }else{
3488 *ppTokenizer = 0;
3489 *ppUserData = 0;
3490 rc = SQLITE_ERROR;
3491 }
3492
3493 return rc;
3494 }
3495
3496 /*
3497 ** Find a tokenizer. This is the implementation of the
3498 ** fts5_api.xFindTokenizer() method.
3499 */
3500 static int fts5FindTokenizer(
3501 fts5_api *pApi, /* Global context (one per db handle) */
3502 const char *zName, /* Name of new function */
3503 void **ppUserData,
3504 fts5_tokenizer *pTokenizer /* Populate this object */
3505 ){
3506 int rc = SQLITE_OK;
3507 Fts5TokenizerModule *pMod;
3508
3509 pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName);
3510 if( pMod ){
3511 if( pMod->bV2Native==0 ){
3512 *ppUserData = pMod->pUserData;
3513 }else{
3514 *ppUserData = (void*)pMod;
3515 }
3516 *pTokenizer = pMod->x1;
3517 }else{
3518 memset(pTokenizer, 0, sizeof(*pTokenizer));
3519 *ppUserData = 0;
3520 rc = SQLITE_ERROR;
3521 }
3522
3523 return rc;
3524 }
3525
3526 /*
3527 ** Attempt to instantiate the tokenizer.
3528 */
3529 int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){
3530 const char **azArg = pConfig->t.azArg;
3531 const int nArg = pConfig->t.nArg;
3532 Fts5TokenizerModule *pMod = 0;
3533 int rc = SQLITE_OK;
3534
3535 pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]);
3536 if( pMod==0 ){
3537 assert( nArg>0 );
3538 rc = SQLITE_ERROR;
3539 sqlite3Fts5ConfigErrmsg(pConfig, "no such tokenizer: %s", azArg[0]);
3540 }else{
3541 int (*xCreate)(void*, const char**, int, Fts5Tokenizer**) = 0;
3542 if( pMod->bV2Native ){
3543 xCreate = pMod->x2.xCreate;
3544 pConfig->t.pApi2 = &pMod->x2;
3545 }else{
3546 pConfig->t.pApi1 = &pMod->x1;
3547 xCreate = pMod->x1.xCreate;
3548 }
3549
3550 rc = xCreate(pMod->pUserData,
3551 (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok
3552 );
3553
3554 if( rc!=SQLITE_OK ){
3555 if( rc!=SQLITE_NOMEM ){
3556 sqlite3Fts5ConfigErrmsg(pConfig, "error in tokenizer constructor");
3557 }
3558 }else if( pMod->bV2Native==0 ){
3559 pConfig->t.ePattern = sqlite3Fts5TokenizerPattern(
3560 pMod->x1.xCreate, pConfig->t.pTok
3561 );
3562 }
3563 }
3564
3565 if( rc!=SQLITE_OK ){
3566 pConfig->t.pApi1 = 0;
3567 pConfig->t.pApi2 = 0;
3568 pConfig->t.pTok = 0;
3569 }
3570
3571 return rc;
3572 }
3573
3574
3575 /*
3576 ** xDestroy callback passed to sqlite3_create_module(). This is invoked
3577 ** when the db handle is being closed. Free memory associated with
3578 ** tokenizers and aux functions registered with this db handle.
3579 */
3580 static void fts5ModuleDestroy(void *pCtx){
3581 Fts5TokenizerModule *pTok, *pNextTok;
3582 Fts5Auxiliary *pAux, *pNextAux;
3583 Fts5Global *pGlobal = (Fts5Global*)pCtx;
3584
3585 for(pAux=pGlobal->pAux; pAux; pAux=pNextAux){
3586 pNextAux = pAux->pNext;
3587 if( pAux->xDestroy ) pAux->xDestroy(pAux->pUserData);
3588 sqlite3_free(pAux);
3589 }
3590
3591 for(pTok=pGlobal->pTok; pTok; pTok=pNextTok){
3592 pNextTok = pTok->pNext;
3593 if( pTok->xDestroy ) pTok->xDestroy(pTok->pUserData);
3594 sqlite3_free(pTok);
3595 }
3596
3597 sqlite3_free(pGlobal);
3598 }
3599
3600 /*
3601 ** Implementation of the fts5() function used by clients to obtain the
3602 ** API pointer.
3603 */
3604 static void fts5Fts5Func(
3605 sqlite3_context *pCtx, /* Function call context */
3606 int nArg, /* Number of args */
3607 sqlite3_value **apArg /* Function arguments */
3608 ){
3609 Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx);
3610 fts5_api **ppApi;
3611 UNUSED_PARAM(nArg);
3612 assert( nArg==1 );
3613 ppApi = (fts5_api**)sqlite3_value_pointer(apArg[0], "fts5_api_ptr");
3614 if( ppApi ) *ppApi = &pGlobal->api;
3615 }
3616
3617 /*
3618 ** Implementation of fts5_source_id() function.
3619 */
3620 static void fts5SourceIdFunc(
3621 sqlite3_context *pCtx, /* Function call context */
3622 int nArg, /* Number of args */
3623 sqlite3_value **apUnused /* Function arguments */
3624 ){
3625 assert( nArg==0 );
3626 UNUSED_PARAM2(nArg, apUnused);
3627 sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT);
3628 }
3629
3630 /*
3631 ** Implementation of fts5_locale(LOCALE, TEXT) function.
3632 **
3633 ** If parameter LOCALE is NULL, or a zero-length string, then a copy of
3634 ** TEXT is returned. Otherwise, both LOCALE and TEXT are interpreted as
3635 ** text, and the value returned is a blob consisting of:
3636 **
3637 ** * The 4 bytes 0x00, 0xE0, 0xB2, 0xEb (FTS5_LOCALE_HEADER).
3638 ** * The LOCALE, as utf-8 text, followed by
3639 ** * 0x00, followed by
3640 ** * The TEXT, as utf-8 text.
3641 **
3642 ** There is no final nul-terminator following the TEXT value.
3643 */
3644 static void fts5LocaleFunc(
3645 sqlite3_context *pCtx, /* Function call context */
3646 int nArg, /* Number of args */
3647 sqlite3_value **apArg /* Function arguments */
3648 ){
3649 const char *zLocale = 0;
3650 i64 nLocale = 0;
3651 const char *zText = 0;
3652 i64 nText = 0;
3653
3654 assert( nArg==2 );
3655 UNUSED_PARAM(nArg);
3656
3657 zLocale = (const char*)sqlite3_value_text(apArg[0]);
3658 nLocale = sqlite3_value_bytes(apArg[0]);
3659
3660 zText = (const char*)sqlite3_value_text(apArg[1]);
3661 nText = sqlite3_value_bytes(apArg[1]);
3662
3663 if( zLocale==0 || zLocale[0]=='\0' ){
3664 sqlite3_result_text(pCtx, zText, nText, SQLITE_TRANSIENT);
3665 }else{
3666 Fts5Global *p = (Fts5Global*)sqlite3_user_data(pCtx);
3667 u8 *pBlob = 0;
3668 u8 *pCsr = 0;
3669 i64 nBlob = 0;
3670
3671 nBlob = FTS5_LOCALE_HDR_SIZE + nLocale + 1 + nText;
3672 pBlob = (u8*)sqlite3_malloc64(nBlob);
3673 if( pBlob==0 ){
3674 sqlite3_result_error_nomem(pCtx);
3675 return;
3676 }
3677
3678 pCsr = pBlob;
3679 memcpy(pCsr, (const u8*)p->aLocaleHdr, FTS5_LOCALE_HDR_SIZE);
3680 pCsr += FTS5_LOCALE_HDR_SIZE;
3681 memcpy(pCsr, zLocale, nLocale);
3682 pCsr += nLocale;
3683 (*pCsr++) = 0x00;
3684 if( zText ) memcpy(pCsr, zText, nText);
3685 assert( &pCsr[nText]==&pBlob[nBlob] );
3686
3687 sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free);
3688 }
3689 }
3690
3691 /*
3692 ** Implementation of fts5_insttoken() function.
3693 */
3694 static void fts5InsttokenFunc(
3695 sqlite3_context *pCtx, /* Function call context */
3696 int nArg, /* Number of args */
3697 sqlite3_value **apArg /* Function arguments */
3698 ){
3699 assert( nArg==1 );
3700 (void)nArg;
3701 sqlite3_result_value(pCtx, apArg[0]);
3702 sqlite3_result_subtype(pCtx, FTS5_INSTTOKEN_SUBTYPE);
3703 }
3704
3705 /*
3706 ** Return true if zName is the extension on one of the shadow tables used
3707 ** by this module.
3708 */
3709 static int fts5ShadowName(const char *zName){
3710 static const char *azName[] = {
3711 "config", "content", "data", "docsize", "idx"
3712 };
3713 unsigned int i;
3714 for(i=0; i<sizeof(azName)/sizeof(azName[0]); i++){
3715 if( sqlite3_stricmp(zName, azName[i])==0 ) return 1;
3716 }
3717 return 0;
3718 }
3719
3720 /*
3721 ** Run an integrity check on the FTS5 data structures. Return a string
3722 ** if anything is found amiss. Return a NULL pointer if everything is
3723 ** OK.
3724 */
3725 static int fts5IntegrityMethod(
3726 sqlite3_vtab *pVtab, /* the FTS5 virtual table to check */
3727 const char *zSchema, /* Name of schema in which this table lives */
3728 const char *zTabname, /* Name of the table itself */
3729 int isQuick, /* True if this is a quick-check */
3730 char **pzErr /* Write error message here */
3731 ){
3732 Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
3733 int rc;
3734
3735 assert( pzErr!=0 && *pzErr==0 );
3736 UNUSED_PARAM(isQuick);
3737 assert( pTab->p.pConfig->pzErrmsg==0 );
3738 pTab->p.pConfig->pzErrmsg = pzErr;
3739 rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0);
3740 if( *pzErr==0 && rc!=SQLITE_OK ){
3741 if( (rc&0xff)==SQLITE_CORRUPT ){
3742 *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
3743 zSchema, zTabname);
3744 rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM;
3745 }else{
3746 *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
3747 " FTS5 table %s.%s: %s",
3748 zSchema, zTabname, sqlite3_errstr(rc));
3749 }
3750 }else if( (rc&0xff)==SQLITE_CORRUPT ){
3751 rc = SQLITE_OK;
3752 }
3753 sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
3754 pTab->p.pConfig->pzErrmsg = 0;
3755
3756 return rc;
3757 }
3758
3759 static int fts5Init(sqlite3 *db){
3760 static const sqlite3_module fts5Mod = {
3761 /* iVersion */ 4,
3762 /* xCreate */ fts5CreateMethod,
3763 /* xConnect */ fts5ConnectMethod,
3764 /* xBestIndex */ fts5BestIndexMethod,
3765 /* xDisconnect */ fts5DisconnectMethod,
3766 /* xDestroy */ fts5DestroyMethod,
3767 /* xOpen */ fts5OpenMethod,
3768 /* xClose */ fts5CloseMethod,
3769 /* xFilter */ fts5FilterMethod,
3770 /* xNext */ fts5NextMethod,
3771 /* xEof */ fts5EofMethod,
3772 /* xColumn */ fts5ColumnMethod,
3773 /* xRowid */ fts5RowidMethod,
3774 /* xUpdate */ fts5UpdateMethod,
3775 /* xBegin */ fts5BeginMethod,
3776 /* xSync */ fts5SyncMethod,
3777 /* xCommit */ fts5CommitMethod,
3778 /* xRollback */ fts5RollbackMethod,
3779 /* xFindFunction */ fts5FindFunctionMethod,
3780 /* xRename */ fts5RenameMethod,
3781 /* xSavepoint */ fts5SavepointMethod,
3782 /* xRelease */ fts5ReleaseMethod,
3783 /* xRollbackTo */ fts5RollbackToMethod,
3784 /* xShadowName */ fts5ShadowName,
3785 /* xIntegrity */ fts5IntegrityMethod
3786 };
3787
3788 int rc;
3789 Fts5Global *pGlobal = 0;
3790
3791 pGlobal = (Fts5Global*)sqlite3_malloc64(sizeof(Fts5Global));
3792 if( pGlobal==0 ){
3793 rc = SQLITE_NOMEM;
3794 }else{
3795 void *p = (void*)pGlobal;
3796 memset(pGlobal, 0, sizeof(Fts5Global));
3797 pGlobal->db = db;
3798 pGlobal->api.iVersion = 3;
3799 pGlobal->api.xCreateFunction = fts5CreateAux;
3800 pGlobal->api.xCreateTokenizer = fts5CreateTokenizer;
3801 pGlobal->api.xFindTokenizer = fts5FindTokenizer;
3802 pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2;
3803 pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2;
3804
3805 /* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector.
3806 ** The constants below were generated randomly. */
3807 sqlite3_randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr);
3808 pGlobal->aLocaleHdr[0] ^= 0xF924976D;
3809 pGlobal->aLocaleHdr[1] ^= 0x16596E13;
3810 pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA;
3811 pGlobal->aLocaleHdr[3] ^= 0x9B03A67F;
3812 assert( sizeof(pGlobal->aLocaleHdr)==16 );
3813
3814 rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy);
3815 if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db);
3816 if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db);
3817 if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api);
3818 if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api);
3819 if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db);
3820 if( rc==SQLITE_OK ){
3821 rc = sqlite3_create_function(
3822 db, "fts5", 1, SQLITE_UTF8, p, fts5Fts5Func, 0, 0
3823 );
3824 }
3825 if( rc==SQLITE_OK ){
3826 rc = sqlite3_create_function(
3827 db, "fts5_source_id", 0,
3828 SQLITE_UTF8|SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS,
3829 p, fts5SourceIdFunc, 0, 0
3830 );
3831 }
3832 if( rc==SQLITE_OK ){
3833 rc = sqlite3_create_function(
3834 db, "fts5_locale", 2,
3835 SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE|SQLITE_SUBTYPE,
3836 p, fts5LocaleFunc, 0, 0
3837 );
3838 }
3839 if( rc==SQLITE_OK ){
3840 rc = sqlite3_create_function(
3841 db, "fts5_insttoken", 1,
3842 SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE,
3843 p, fts5InsttokenFunc, 0, 0
3844 );
3845 }
3846 }
3847
3848 /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file
3849 ** fts5_test_mi.c is compiled and linked into the executable. And call
3850 ** its entry point to enable the matchinfo() demo. */
3851 #ifdef SQLITE_FTS5_ENABLE_TEST_MI
3852 if( rc==SQLITE_OK ){
3853 extern int sqlite3Fts5TestRegisterMatchinfoAPI(fts5_api*);
3854 rc = sqlite3Fts5TestRegisterMatchinfoAPI(&pGlobal->api);
3855 }
3856 #endif
3857
3858 return rc;
3859 }
3860
3861 /*
3862 ** The following functions are used to register the module with SQLite. If
3863 ** this module is being built as part of the SQLite core (SQLITE_CORE is
3864 ** defined), then sqlite3_open() will call sqlite3Fts5Init() directly.
3865 **
3866 ** Or, if this module is being built as a loadable extension,
3867 ** sqlite3Fts5Init() is omitted and the two standard entry points
3868 ** sqlite3_fts_init() and sqlite3_fts5_init() defined instead.
3869 */
3870 #ifndef SQLITE_CORE
3871 #ifdef _WIN32
3872 __declspec(dllexport)
3873 #endif
3874 int sqlite3_fts_init(
3875 sqlite3 *db,
3876 char **pzErrMsg,
3877 const sqlite3_api_routines *pApi
3878 ){
3879 SQLITE_EXTENSION_INIT2(pApi);
3880 (void)pzErrMsg; /* Unused parameter */
3881 return fts5Init(db);
3882 }
3883
3884 #ifdef _WIN32
3885 __declspec(dllexport)
3886 #endif
3887 int sqlite3_fts5_init(
3888 sqlite3 *db,
3889 char **pzErrMsg,
3890 const sqlite3_api_routines *pApi
3891 ){
3892 SQLITE_EXTENSION_INIT2(pApi);
3893 (void)pzErrMsg; /* Unused parameter */
3894 return fts5Init(db);
3895 }
3896 #else
3897 int sqlite3Fts5Init(sqlite3 *db){
3898 return fts5Init(db);
3899 }
3900 #endif