Debian Codegraph @

ext/fts5/fts5_storage.c

raw
1 /*
2 ** 2014 May 31
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 **
13 */
14
15
16
17 #include "fts5Int.h"
18
19 /*
20 ** pSavedRow:
21 ** SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it
22 ** does a by-rowid lookup to retrieve a single row from the %_content
23 ** table or equivalent external-content table/view.
24 **
25 ** However, FTS5_STMT_LOOKUP2 is only used when retrieving the original
26 ** values for a row being UPDATEd. In that case, the SQL statement is
27 ** not reset and pSavedRow is set to point at it. This is so that the
28 ** insert operation that follows the delete may access the original
29 ** row values for any new values for which sqlite3_value_nochange() returns
30 ** true. i.e. if the user executes:
31 **
32 ** CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1);
33 ** ...
34 ** UPDATE fts SET a=?, b=? WHERE rowid=?;
35 **
36 ** then the value passed to the xUpdate() method of this table as the
37 ** new.c value is an sqlite3_value_nochange() value. So in this case it
38 ** must be read from the saved row stored in Fts5Storage.pSavedRow.
39 **
40 ** This is necessary - using sqlite3_value_nochange() instead of just having
41 ** SQLite pass the original value back via xUpdate() - so as not to discard
42 ** any locale information associated with such values.
43 **
44 */
45 struct Fts5Storage {
46 Fts5Config *pConfig;
47 Fts5Index *pIndex;
48 int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */
49 i64 nTotalRow; /* Total number of rows in FTS table */
50 i64 *aTotalSize; /* Total sizes of each column */
51 sqlite3_stmt *pSavedRow;
52 sqlite3_stmt *aStmt[12];
53 };
54
55
56 #if FTS5_STMT_SCAN_ASC!=0
57 # error "FTS5_STMT_SCAN_ASC mismatch"
58 #endif
59 #if FTS5_STMT_SCAN_DESC!=1
60 # error "FTS5_STMT_SCAN_DESC mismatch"
61 #endif
62 #if FTS5_STMT_LOOKUP!=2
63 # error "FTS5_STMT_LOOKUP mismatch"
64 #endif
65
66 #define FTS5_STMT_LOOKUP2 3
67 #define FTS5_STMT_INSERT_CONTENT 4
68 #define FTS5_STMT_REPLACE_CONTENT 5
69 #define FTS5_STMT_DELETE_CONTENT 6
70 #define FTS5_STMT_REPLACE_DOCSIZE 7
71 #define FTS5_STMT_DELETE_DOCSIZE 8
72 #define FTS5_STMT_LOOKUP_DOCSIZE 9
73 #define FTS5_STMT_REPLACE_CONFIG 10
74 #define FTS5_STMT_SCAN 11
75
76 /*
77 ** Prepare the two insert statements - Fts5Storage.pInsertContent and
78 ** Fts5Storage.pInsertDocsize - if they have not already been prepared.
79 ** Return SQLITE_OK if successful, or an SQLite error code if an error
80 ** occurs.
81 */
82 static int fts5StorageGetStmt(
83 Fts5Storage *p, /* Storage handle */
84 int eStmt, /* FTS5_STMT_XXX constant */
85 sqlite3_stmt **ppStmt, /* OUT: Prepared statement handle */
86 char **pzErrMsg /* OUT: Error message (if any) */
87 ){
88 int rc = SQLITE_OK;
89
90 /* If there is no %_docsize table, there should be no requests for
91 ** statements to operate on it. */
92 assert( p->pConfig->bColumnsize || (
93 eStmt!=FTS5_STMT_REPLACE_DOCSIZE
94 && eStmt!=FTS5_STMT_DELETE_DOCSIZE
95 && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE
96 ));
97
98 assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
99 if( p->aStmt[eStmt]==0 ){
100 const char *azStmt[] = {
101 "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
102 "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
103 "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */
104 "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP2 */
105
106 "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */
107 "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */
108 "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */
109 "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */
110 "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */
111
112 "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */
113
114 "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */
115 "SELECT %s FROM %s AS T", /* SCAN */
116 };
117 Fts5Config *pC = p->pConfig;
118 char *zSql = 0;
119
120 assert( ArraySize(azStmt)==ArraySize(p->aStmt) );
121
122 switch( eStmt ){
123 case FTS5_STMT_SCAN:
124 zSql = sqlite3_mprintf(azStmt[eStmt],
125 pC->zContentExprlist, pC->zContent
126 );
127 break;
128
129 case FTS5_STMT_SCAN_ASC:
130 case FTS5_STMT_SCAN_DESC:
131 zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist,
132 pC->zContent, pC->zContentRowid, pC->zContentRowid,
133 pC->zContentRowid
134 );
135 break;
136
137 case FTS5_STMT_LOOKUP:
138 case FTS5_STMT_LOOKUP2:
139 zSql = sqlite3_mprintf(azStmt[eStmt],
140 pC->zContentExprlist, pC->zContent, pC->zContentRowid
141 );
142 break;
143
144 case FTS5_STMT_INSERT_CONTENT:
145 case FTS5_STMT_REPLACE_CONTENT: {
146 char *zBind = 0;
147 int i;
148
149 assert( pC->eContent==FTS5_CONTENT_NORMAL
150 || pC->eContent==FTS5_CONTENT_UNINDEXED
151 );
152
153 /* Add bindings for the "c*" columns - those that store the actual
154 ** table content. If eContent==NORMAL, then there is one binding
155 ** for each column. Or, if eContent==UNINDEXED, then there are only
156 ** bindings for the UNINDEXED columns. */
157 for(i=0; rc==SQLITE_OK && i<(pC->nCol+1); i++){
158 if( !i || pC->eContent==FTS5_CONTENT_NORMAL || pC->abUnindexed[i-1] ){
159 zBind = sqlite3Fts5Mprintf(&rc, "%z%s?%d", zBind, zBind?",":"",i+1);
160 }
161 }
162
163 /* Add bindings for any "l*" columns. Only non-UNINDEXED columns
164 ** require these. */
165 if( pC->bLocale && pC->eContent==FTS5_CONTENT_NORMAL ){
166 for(i=0; rc==SQLITE_OK && i<pC->nCol; i++){
167 if( pC->abUnindexed[i]==0 ){
168 zBind = sqlite3Fts5Mprintf(&rc, "%z,?%d", zBind, pC->nCol+i+2);
169 }
170 }
171 }
172
173 zSql = sqlite3Fts5Mprintf(&rc, azStmt[eStmt], pC->zDb, pC->zName,zBind);
174 sqlite3_free(zBind);
175 break;
176 }
177
178 case FTS5_STMT_REPLACE_DOCSIZE:
179 zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName,
180 (pC->bContentlessDelete ? ",?" : "")
181 );
182 break;
183
184 case FTS5_STMT_LOOKUP_DOCSIZE:
185 zSql = sqlite3_mprintf(azStmt[eStmt],
186 (pC->bContentlessDelete ? ",origin" : ""),
187 pC->zDb, pC->zName
188 );
189 break;
190
191 default:
192 zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
193 break;
194 }
195
196 if( zSql==0 ){
197 rc = SQLITE_NOMEM;
198 }else{
199 int f = SQLITE_PREPARE_PERSISTENT;
200 if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB;
201 p->pConfig->bLock++;
202 rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0);
203 p->pConfig->bLock--;
204 sqlite3_free(zSql);
205 if( rc!=SQLITE_OK && pzErrMsg ){
206 *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db));
207 }
208 if( rc==SQLITE_ERROR && eStmt>FTS5_STMT_LOOKUP2 && eStmt<FTS5_STMT_SCAN ){
209 /* One of the internal tables - not the %_content table - is missing.
210 ** This counts as a corrupted table. */
211 rc = SQLITE_CORRUPT;
212 }
213 }
214 }
215
216 *ppStmt = p->aStmt[eStmt];
217 sqlite3_reset(*ppStmt);
218 return rc;
219 }
220
221
222 static int fts5ExecPrintf(
223 sqlite3 *db,
224 char **pzErr,
225 const char *zFormat,
226 ...
227 ){
228 int rc;
229 va_list ap; /* ... printf arguments */
230 char *zSql;
231
232 va_start(ap, zFormat);
233 zSql = sqlite3_vmprintf(zFormat, ap);
234
235 if( zSql==0 ){
236 rc = SQLITE_NOMEM;
237 }else{
238 rc = sqlite3_exec(db, zSql, 0, 0, pzErr);
239 sqlite3_free(zSql);
240 }
241
242 va_end(ap);
243 return rc;
244 }
245
246 /*
247 ** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error
248 ** code otherwise.
249 */
250 int sqlite3Fts5DropAll(Fts5Config *pConfig){
251 int rc = fts5ExecPrintf(pConfig->db, 0,
252 "DROP TABLE IF EXISTS %Q.'%q_data';"
253 "DROP TABLE IF EXISTS %Q.'%q_idx';"
254 "DROP TABLE IF EXISTS %Q.'%q_config';",
255 pConfig->zDb, pConfig->zName,
256 pConfig->zDb, pConfig->zName,
257 pConfig->zDb, pConfig->zName
258 );
259 if( rc==SQLITE_OK && pConfig->bColumnsize ){
260 rc = fts5ExecPrintf(pConfig->db, 0,
261 "DROP TABLE IF EXISTS %Q.'%q_docsize';",
262 pConfig->zDb, pConfig->zName
263 );
264 }
265 if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
266 rc = fts5ExecPrintf(pConfig->db, 0,
267 "DROP TABLE IF EXISTS %Q.'%q_content';",
268 pConfig->zDb, pConfig->zName
269 );
270 }
271 return rc;
272 }
273
274 static void fts5StorageRenameOne(
275 Fts5Config *pConfig, /* Current FTS5 configuration */
276 int *pRc, /* IN/OUT: Error code */
277 const char *zTail, /* Tail of table name e.g. "data", "config" */
278 const char *zName /* New name of FTS5 table */
279 ){
280 if( *pRc==SQLITE_OK ){
281 *pRc = fts5ExecPrintf(pConfig->db, 0,
282 "ALTER TABLE %Q.'%q_%s' RENAME TO '%q_%s';",
283 pConfig->zDb, pConfig->zName, zTail, zName, zTail
284 );
285 }
286 }
287
288 int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){
289 Fts5Config *pConfig = pStorage->pConfig;
290 int rc = sqlite3Fts5StorageSync(pStorage);
291
292 fts5StorageRenameOne(pConfig, &rc, "data", zName);
293 fts5StorageRenameOne(pConfig, &rc, "idx", zName);
294 fts5StorageRenameOne(pConfig, &rc, "config", zName);
295 if( pConfig->bColumnsize ){
296 fts5StorageRenameOne(pConfig, &rc, "docsize", zName);
297 }
298 if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
299 fts5StorageRenameOne(pConfig, &rc, "content", zName);
300 }
301 return rc;
302 }
303
304 /*
305 ** Create the shadow table named zPost, with definition zDefn. Return
306 ** SQLITE_OK if successful, or an SQLite error code otherwise.
307 */
308 int sqlite3Fts5CreateTable(
309 Fts5Config *pConfig, /* FTS5 configuration */
310 const char *zPost, /* Shadow table to create (e.g. "content") */
311 const char *zDefn, /* Columns etc. for shadow table */
312 int bWithout, /* True for without rowid */
313 char **pzErr /* OUT: Error message */
314 ){
315 int rc;
316 char *zErr = 0;
317
318 rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s",
319 pConfig->zDb, pConfig->zName, zPost, zDefn,
320 #ifndef SQLITE_FTS5_NO_WITHOUT_ROWID
321 bWithout?" WITHOUT ROWID":
322 #endif
323 ""
324 );
325 if( zErr ){
326 *pzErr = sqlite3_mprintf(
327 "fts5: error creating shadow table %q_%s: %s",
328 pConfig->zName, zPost, zErr
329 );
330 sqlite3_free(zErr);
331 }
332
333 return rc;
334 }
335
336 /*
337 ** Open a new Fts5Index handle. If the bCreate argument is true, create
338 ** and initialize the underlying tables
339 **
340 ** If successful, set *pp to point to the new object and return SQLITE_OK.
341 ** Otherwise, set *pp to NULL and return an SQLite error code.
342 */
343 int sqlite3Fts5StorageOpen(
344 Fts5Config *pConfig,
345 Fts5Index *pIndex,
346 int bCreate,
347 Fts5Storage **pp,
348 char **pzErr /* OUT: Error message */
349 ){
350 int rc = SQLITE_OK;
351 Fts5Storage *p; /* New object */
352 sqlite3_int64 nByte; /* Bytes of space to allocate */
353
354 nByte = sizeof(Fts5Storage) /* Fts5Storage object */
355 + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */
356 *pp = p = (Fts5Storage*)sqlite3_malloc64(nByte);
357 if( !p ) return SQLITE_NOMEM;
358
359 memset(p, 0, (size_t)nByte);
360 p->aTotalSize = (i64*)&p[1];
361 p->pConfig = pConfig;
362 p->pIndex = pIndex;
363
364 if( bCreate ){
365 if( pConfig->eContent==FTS5_CONTENT_NORMAL
366 || pConfig->eContent==FTS5_CONTENT_UNINDEXED
367 ){
368 int nDefn = 32 + pConfig->nCol*10;
369 char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20);
370 if( zDefn==0 ){
371 rc = SQLITE_NOMEM;
372 }else{
373 int i;
374 int iOff;
375 sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY");
376 iOff = (int)strlen(zDefn);
377 for(i=0; i<pConfig->nCol; i++){
378 if( pConfig->eContent==FTS5_CONTENT_NORMAL
379 || pConfig->abUnindexed[i]
380 ){
381 sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
382 iOff += (int)strlen(&zDefn[iOff]);
383 }
384 }
385 if( pConfig->bLocale ){
386 for(i=0; i<pConfig->nCol; i++){
387 if( pConfig->abUnindexed[i]==0 ){
388 sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i);
389 iOff += (int)strlen(&zDefn[iOff]);
390 }
391 }
392 }
393 rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
394 }
395 sqlite3_free(zDefn);
396 }
397
398 if( rc==SQLITE_OK && pConfig->bColumnsize ){
399 const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
400 if( pConfig->bContentlessDelete ){
401 zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER";
402 }
403 rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
404 }
405 if( rc==SQLITE_OK ){
406 rc = sqlite3Fts5CreateTable(
407 pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
408 );
409 }
410 if( rc==SQLITE_OK ){
411 rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION);
412 }
413 }
414
415 if( rc ){
416 sqlite3Fts5StorageClose(p);
417 *pp = 0;
418 }
419 return rc;
420 }
421
422 /*
423 ** Close a handle opened by an earlier call to sqlite3Fts5StorageOpen().
424 */
425 int sqlite3Fts5StorageClose(Fts5Storage *p){
426 int rc = SQLITE_OK;
427 if( p ){
428 int i;
429
430 /* Finalize all SQL statements */
431 for(i=0; i<ArraySize(p->aStmt); i++){
432 sqlite3_finalize(p->aStmt[i]);
433 }
434
435 sqlite3_free(p);
436 }
437 return rc;
438 }
439
440 typedef struct Fts5InsertCtx Fts5InsertCtx;
441 struct Fts5InsertCtx {
442 Fts5Storage *pStorage;
443 int iCol;
444 int szCol; /* Size of column value in tokens */
445 };
446
447 /*
448 ** Tokenization callback used when inserting tokens into the FTS index.
449 */
450 static int fts5StorageInsertCallback(
451 void *pContext, /* Pointer to Fts5InsertCtx object */
452 int tflags,
453 const char *pToken, /* Buffer containing token */
454 int nToken, /* Size of token in bytes */
455 int iUnused1, /* Start offset of token */
456 int iUnused2 /* End offset of token */
457 ){
458 Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
459 Fts5Index *pIdx = pCtx->pStorage->pIndex;
460 UNUSED_PARAM2(iUnused1, iUnused2);
461 if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
462 if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
463 pCtx->szCol++;
464 }
465 return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
466 }
467
468 /*
469 ** This function is used as part of an UPDATE statement that modifies the
470 ** rowid of a row. In that case, this function is called first to set
471 ** Fts5Storage.pSavedRow to point to a statement that may be used to
472 ** access the original values of the row being deleted - iDel.
473 **
474 ** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
475 ** It is not considered an error if row iDel does not exist. In this case
476 ** pSavedRow is not set and SQLITE_OK returned.
477 */
478 int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){
479 int rc = SQLITE_OK;
480 sqlite3_stmt *pSeek = 0;
481
482 assert( p->pSavedRow==0 );
483 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0);
484 if( rc==SQLITE_OK ){
485 sqlite3_bind_int64(pSeek, 1, iDel);
486 if( sqlite3_step(pSeek)!=SQLITE_ROW ){
487 rc = sqlite3_reset(pSeek);
488 }else{
489 p->pSavedRow = pSeek;
490 }
491 }
492
493 return rc;
494 }
495
496 /*
497 ** If a row with rowid iDel is present in the %_content table, add the
498 ** delete-markers to the FTS index necessary to delete it. Do not actually
499 ** remove the %_content row at this time though.
500 **
501 ** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left
502 ** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access
503 ** the original values of the row being deleted. This is used by UPDATE
504 ** statements.
505 */
506 static int fts5StorageDeleteFromIndex(
507 Fts5Storage *p,
508 i64 iDel,
509 sqlite3_value **apVal,
510 int bSaveRow /* True to set pSavedRow */
511 ){
512 Fts5Config *pConfig = p->pConfig;
513 sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */
514 int rc = SQLITE_OK; /* Return code */
515 int rc2; /* sqlite3_reset() return code */
516 int iCol;
517 Fts5InsertCtx ctx;
518
519 assert( bSaveRow==0 || apVal==0 );
520 assert( bSaveRow==0 || bSaveRow==1 );
521 assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 );
522
523 if( apVal==0 ){
524 if( p->pSavedRow && bSaveRow ){
525 pSeek = p->pSavedRow;
526 p->pSavedRow = 0;
527 }else{
528 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0);
529 if( rc!=SQLITE_OK ) return rc;
530 sqlite3_bind_int64(pSeek, 1, iDel);
531 if( sqlite3_step(pSeek)!=SQLITE_ROW ){
532 return sqlite3_reset(pSeek);
533 }
534 }
535 }
536
537 ctx.pStorage = p;
538 ctx.iCol = -1;
539 for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
540 if( pConfig->abUnindexed[iCol-1]==0 ){
541 sqlite3_value *pVal = 0;
542 sqlite3_value *pFree = 0;
543 const char *pText = 0;
544 int nText = 0;
545 const char *pLoc = 0;
546 int nLoc = 0;
547
548 assert( pSeek==0 || apVal==0 );
549 assert( pSeek!=0 || apVal!=0 );
550 if( pSeek ){
551 pVal = sqlite3_column_value(pSeek, iCol);
552 }else{
553 pVal = apVal[iCol-1];
554 }
555
556 if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
557 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
558 }else{
559 if( sqlite3_value_type(pVal)!=SQLITE_TEXT ){
560 /* Make a copy of the value to work with. This is because the call
561 ** to sqlite3_value_text() below forces the type of the value to
562 ** SQLITE_TEXT, and we may need to use it again later. */
563 pFree = pVal = sqlite3_value_dup(pVal);
564 if( pVal==0 ){
565 rc = SQLITE_NOMEM;
566 }
567 }
568 if( rc==SQLITE_OK ){
569 pText = (const char*)sqlite3_value_text(pVal);
570 nText = sqlite3_value_bytes(pVal);
571 if( pConfig->bLocale && pSeek ){
572 pLoc = (const char*)sqlite3_column_text(pSeek, iCol+pConfig->nCol);
573 nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol);
574 }
575 }
576 }
577
578 if( rc==SQLITE_OK ){
579 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
580 ctx.szCol = 0;
581 rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT,
582 pText, nText, (void*)&ctx, fts5StorageInsertCallback
583 );
584 p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
585 if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){
586 rc = FTS5_CORRUPT;
587 }
588 sqlite3Fts5ClearLocale(pConfig);
589 }
590 sqlite3_value_free(pFree);
591 }
592 }
593 if( rc==SQLITE_OK && p->nTotalRow<1 ){
594 rc = FTS5_CORRUPT;
595 }else{
596 p->nTotalRow--;
597 }
598
599 if( rc==SQLITE_OK && bSaveRow ){
600 assert( p->pSavedRow==0 );
601 p->pSavedRow = pSeek;
602 }else{
603 rc2 = sqlite3_reset(pSeek);
604 if( rc==SQLITE_OK ) rc = rc2;
605 }
606 return rc;
607 }
608
609 /*
610 ** Reset any saved statement pSavedRow. Zero pSavedRow as well. This
611 ** should be called by the xUpdate() method of the fts5 table before
612 ** returning from any operation that may have set Fts5Storage.pSavedRow.
613 */
614 void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){
615 assert( pStorage->pSavedRow==0
616 || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2]
617 );
618 sqlite3_reset(pStorage->pSavedRow);
619 pStorage->pSavedRow = 0;
620 }
621
622 /*
623 ** This function is called to process a DELETE on a contentless_delete=1
624 ** table. It adds the tombstone required to delete the entry with rowid
625 ** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs,
626 ** an SQLite error code.
627 */
628 static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
629 i64 iOrigin = 0;
630 sqlite3_stmt *pLookup = 0;
631 int rc = SQLITE_OK;
632
633 assert( p->pConfig->bContentlessDelete );
634 assert( p->pConfig->eContent==FTS5_CONTENT_NONE
635 || p->pConfig->eContent==FTS5_CONTENT_UNINDEXED
636 );
637
638 /* Look up the origin of the document in the %_docsize table. Store
639 ** this in stack variable iOrigin. */
640 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
641 if( rc==SQLITE_OK ){
642 sqlite3_bind_int64(pLookup, 1, iDel);
643 if( SQLITE_ROW==sqlite3_step(pLookup) ){
644 iOrigin = sqlite3_column_int64(pLookup, 1);
645 }
646 rc = sqlite3_reset(pLookup);
647 }
648
649 if( rc==SQLITE_OK && iOrigin!=0 ){
650 rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel);
651 }
652
653 return rc;
654 }
655
656 /*
657 ** Insert a record into the %_docsize table. Specifically, do:
658 **
659 ** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
660 **
661 ** If there is no %_docsize table (as happens if the columnsize=0 option
662 ** is specified when the FTS5 table is created), this function is a no-op.
663 */
664 static int fts5StorageInsertDocsize(
665 Fts5Storage *p, /* Storage module to write to */
666 i64 iRowid, /* id value */
667 Fts5Buffer *pBuf /* sz value */
668 ){
669 int rc = SQLITE_OK;
670 if( p->pConfig->bColumnsize ){
671 sqlite3_stmt *pReplace = 0;
672 rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
673 if( rc==SQLITE_OK ){
674 sqlite3_bind_int64(pReplace, 1, iRowid);
675 if( p->pConfig->bContentlessDelete ){
676 i64 iOrigin = 0;
677 rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
678 sqlite3_bind_int64(pReplace, 3, iOrigin);
679 }
680 }
681 if( rc==SQLITE_OK ){
682 sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
683 sqlite3_step(pReplace);
684 rc = sqlite3_reset(pReplace);
685 sqlite3_bind_null(pReplace, 2);
686 }
687 }
688 return rc;
689 }
690
691 /*
692 ** Load the contents of the "averages" record from disk into the
693 ** p->nTotalRow and p->aTotalSize[] variables. If successful, and if
694 ** argument bCache is true, set the p->bTotalsValid flag to indicate
695 ** that the contents of aTotalSize[] and nTotalRow are valid until
696 ** further notice.
697 **
698 ** Return SQLITE_OK if successful, or an SQLite error code if an error
699 ** occurs.
700 */
701 static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){
702 int rc = SQLITE_OK;
703 if( p->bTotalsValid==0 ){
704 rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize);
705 p->bTotalsValid = bCache;
706 }
707 return rc;
708 }
709
710 /*
711 ** Store the current contents of the p->nTotalRow and p->aTotalSize[]
712 ** variables in the "averages" record on disk.
713 **
714 ** Return SQLITE_OK if successful, or an SQLite error code if an error
715 ** occurs.
716 */
717 static int fts5StorageSaveTotals(Fts5Storage *p){
718 int nCol = p->pConfig->nCol;
719 int i;
720 Fts5Buffer buf;
721 int rc = SQLITE_OK;
722 memset(&buf, 0, sizeof(buf));
723
724 sqlite3Fts5BufferAppendVarint(&rc, &buf, p->nTotalRow);
725 for(i=0; i<nCol; i++){
726 sqlite3Fts5BufferAppendVarint(&rc, &buf, p->aTotalSize[i]);
727 }
728 if( rc==SQLITE_OK ){
729 rc = sqlite3Fts5IndexSetAverages(p->pIndex, buf.p, buf.n);
730 }
731 sqlite3_free(buf.p);
732
733 return rc;
734 }
735
736 /*
737 ** Remove a row from the FTS table.
738 */
739 int sqlite3Fts5StorageDelete(
740 Fts5Storage *p, /* Storage object */
741 i64 iDel, /* Rowid to delete from table */
742 sqlite3_value **apVal, /* Optional - values to remove from index */
743 int bSaveRow /* If true, set pSavedRow for deleted row */
744 ){
745 Fts5Config *pConfig = p->pConfig;
746 int rc;
747 sqlite3_stmt *pDel = 0;
748
749 assert( pConfig->eContent!=FTS5_CONTENT_NORMAL || apVal==0 );
750 rc = fts5StorageLoadTotals(p, 1);
751
752 /* Delete the index records */
753 if( rc==SQLITE_OK ){
754 rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
755 }
756
757 if( rc==SQLITE_OK ){
758 if( p->pConfig->bContentlessDelete ){
759 rc = fts5StorageContentlessDelete(p, iDel);
760 if( rc==SQLITE_OK
761 && bSaveRow
762 && p->pConfig->eContent==FTS5_CONTENT_UNINDEXED
763 ){
764 rc = sqlite3Fts5StorageFindDeleteRow(p, iDel);
765 }
766 }else{
767 rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow);
768 }
769 }
770
771 /* Delete the %_docsize record */
772 if( rc==SQLITE_OK && pConfig->bColumnsize ){
773 rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
774 if( rc==SQLITE_OK ){
775 sqlite3_bind_int64(pDel, 1, iDel);
776 sqlite3_step(pDel);
777 rc = sqlite3_reset(pDel);
778 }
779 }
780
781 /* Delete the %_content record */
782 if( pConfig->eContent==FTS5_CONTENT_NORMAL
783 || pConfig->eContent==FTS5_CONTENT_UNINDEXED
784 ){
785 if( rc==SQLITE_OK ){
786 rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
787 }
788 if( rc==SQLITE_OK ){
789 sqlite3_bind_int64(pDel, 1, iDel);
790 sqlite3_step(pDel);
791 rc = sqlite3_reset(pDel);
792 }
793 }
794
795 return rc;
796 }
797
798 /*
799 ** Delete all entries in the FTS5 index.
800 */
801 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
802 Fts5Config *pConfig = p->pConfig;
803 int rc;
804
805 p->bTotalsValid = 0;
806
807 /* Delete the contents of the %_data and %_docsize tables. */
808 rc = fts5ExecPrintf(pConfig->db, 0,
809 "DELETE FROM %Q.'%q_data';"
810 "DELETE FROM %Q.'%q_idx';",
811 pConfig->zDb, pConfig->zName,
812 pConfig->zDb, pConfig->zName
813 );
814 if( rc==SQLITE_OK && pConfig->bColumnsize ){
815 rc = fts5ExecPrintf(pConfig->db, 0,
816 "DELETE FROM %Q.'%q_docsize';", pConfig->zDb, pConfig->zName
817 );
818 }
819
820 if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_UNINDEXED ){
821 rc = fts5ExecPrintf(pConfig->db, 0,
822 "DELETE FROM %Q.'%q_content';", pConfig->zDb, pConfig->zName
823 );
824 }
825
826 /* Reinitialize the %_data table. This call creates the initial structure
827 ** and averages records. */
828 if( rc==SQLITE_OK ){
829 rc = sqlite3Fts5IndexReinit(p->pIndex);
830 }
831 if( rc==SQLITE_OK ){
832 rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION);
833 }
834 return rc;
835 }
836
837 int sqlite3Fts5StorageRebuild(Fts5Storage *p){
838 Fts5Buffer buf = {0,0,0};
839 Fts5Config *pConfig = p->pConfig;
840 sqlite3_stmt *pScan = 0;
841 Fts5InsertCtx ctx;
842 int rc, rc2;
843
844 memset(&ctx, 0, sizeof(Fts5InsertCtx));
845 ctx.pStorage = p;
846 rc = sqlite3Fts5StorageDeleteAll(p);
847 if( rc==SQLITE_OK ){
848 rc = fts5StorageLoadTotals(p, 1);
849 }
850
851 if( rc==SQLITE_OK ){
852 rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, pConfig->pzErrmsg);
853 }
854
855 while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
856 i64 iRowid = sqlite3_column_int64(pScan, 0);
857
858 sqlite3Fts5BufferZero(&buf);
859 rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
860 for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
861 ctx.szCol = 0;
862 if( pConfig->abUnindexed[ctx.iCol]==0 ){
863 int nText = 0; /* Size of pText in bytes */
864 const char *pText = 0; /* Pointer to buffer containing text value */
865 int nLoc = 0; /* Size of pLoc in bytes */
866 const char *pLoc = 0; /* Pointer to buffer containing text value */
867
868 sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1);
869 if( pConfig->eContent==FTS5_CONTENT_EXTERNAL
870 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
871 ){
872 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
873 }else{
874 pText = (const char*)sqlite3_value_text(pVal);
875 nText = sqlite3_value_bytes(pVal);
876 if( pConfig->bLocale ){
877 int iCol = ctx.iCol + 1 + pConfig->nCol;
878 pLoc = (const char*)sqlite3_column_text(pScan, iCol);
879 nLoc = sqlite3_column_bytes(pScan, iCol);
880 }
881 }
882
883 if( rc==SQLITE_OK ){
884 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
885 rc = sqlite3Fts5Tokenize(pConfig,
886 FTS5_TOKENIZE_DOCUMENT,
887 pText, nText,
888 (void*)&ctx,
889 fts5StorageInsertCallback
890 );
891 sqlite3Fts5ClearLocale(pConfig);
892 }
893 }
894 sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
895 p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
896 }
897 p->nTotalRow++;
898
899 if( rc==SQLITE_OK ){
900 rc = fts5StorageInsertDocsize(p, iRowid, &buf);
901 }
902 }
903 sqlite3_free(buf.p);
904 rc2 = sqlite3_reset(pScan);
905 if( rc==SQLITE_OK ) rc = rc2;
906
907 /* Write the averages record */
908 if( rc==SQLITE_OK ){
909 rc = fts5StorageSaveTotals(p);
910 }
911 return rc;
912 }
913
914 int sqlite3Fts5StorageOptimize(Fts5Storage *p){
915 return sqlite3Fts5IndexOptimize(p->pIndex);
916 }
917
918 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
919 return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
920 }
921
922 int sqlite3Fts5StorageReset(Fts5Storage *p){
923 return sqlite3Fts5IndexReset(p->pIndex);
924 }
925
926 /*
927 ** Allocate a new rowid. This is used for "external content" tables when
928 ** a NULL value is inserted into the rowid column. The new rowid is allocated
929 ** by inserting a dummy row into the %_docsize table. The dummy will be
930 ** overwritten later.
931 **
932 ** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In
933 ** this case the user is required to provide a rowid explicitly.
934 */
935 static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
936 int rc = SQLITE_MISMATCH;
937 if( p->pConfig->bColumnsize ){
938 sqlite3_stmt *pReplace = 0;
939 rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
940 if( rc==SQLITE_OK ){
941 sqlite3_bind_null(pReplace, 1);
942 sqlite3_bind_null(pReplace, 2);
943 sqlite3_step(pReplace);
944 rc = sqlite3_reset(pReplace);
945 }
946 if( rc==SQLITE_OK ){
947 *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
948 }
949 }
950 return rc;
951 }
952
953 /*
954 ** Insert a new row into the FTS content table.
955 */
956 int sqlite3Fts5StorageContentInsert(
957 Fts5Storage *p,
958 int bReplace, /* True to use REPLACE instead of INSERT */
959 sqlite3_value **apVal,
960 i64 *piRowid
961 ){
962 Fts5Config *pConfig = p->pConfig;
963 int rc = SQLITE_OK;
964
965 /* Insert the new row into the %_content table. */
966 if( pConfig->eContent!=FTS5_CONTENT_NORMAL
967 && pConfig->eContent!=FTS5_CONTENT_UNINDEXED
968 ){
969 if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
970 *piRowid = sqlite3_value_int64(apVal[1]);
971 }else{
972 rc = fts5StorageNewRowid(p, piRowid);
973 }
974 }else{
975 sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */
976 int i; /* Counter variable */
977
978 assert( FTS5_STMT_INSERT_CONTENT+1==FTS5_STMT_REPLACE_CONTENT );
979 assert( bReplace==0 || bReplace==1 );
980 rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT+bReplace, &pInsert, 0);
981 if( pInsert ) sqlite3_clear_bindings(pInsert);
982
983 /* Bind the rowid value */
984 sqlite3_bind_value(pInsert, 1, apVal[1]);
985
986 /* Loop through values for user-defined columns. i=2 is the leftmost
987 ** user-defined column. As is column 1 of pSavedRow. */
988 for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
989 int bUnindexed = pConfig->abUnindexed[i-2];
990 if( pConfig->eContent==FTS5_CONTENT_NORMAL || bUnindexed ){
991 sqlite3_value *pVal = apVal[i];
992
993 if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
994 /* This is an UPDATE statement, and user-defined column (i-2) was not
995 ** modified. Retrieve the value from Fts5Storage.pSavedRow. */
996 pVal = sqlite3_column_value(p->pSavedRow, i-1);
997 if( pConfig->bLocale && bUnindexed==0 ){
998 sqlite3_bind_value(pInsert, pConfig->nCol + i,
999 sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1)
1000 );
1001 }
1002 }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
1003 const char *pText = 0;
1004 const char *pLoc = 0;
1005 int nText = 0;
1006 int nLoc = 0;
1007 assert( pConfig->bLocale );
1008
1009 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
1010 if( rc==SQLITE_OK ){
1011 sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
1012 if( bUnindexed==0 ){
1013 int iLoc = pConfig->nCol + i;
1014 sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT);
1015 }
1016 }
1017
1018 continue;
1019 }
1020
1021 rc = sqlite3_bind_value(pInsert, i, pVal);
1022 }
1023 }
1024 if( rc==SQLITE_OK ){
1025 sqlite3_step(pInsert);
1026 rc = sqlite3_reset(pInsert);
1027 }
1028 *piRowid = sqlite3_last_insert_rowid(pConfig->db);
1029 }
1030
1031 return rc;
1032 }
1033
1034 /*
1035 ** Insert new entries into the FTS index and %_docsize table.
1036 */
1037 int sqlite3Fts5StorageIndexInsert(
1038 Fts5Storage *p,
1039 sqlite3_value **apVal,
1040 i64 iRowid
1041 ){
1042 Fts5Config *pConfig = p->pConfig;
1043 int rc = SQLITE_OK; /* Return code */
1044 Fts5InsertCtx ctx; /* Tokenization callback context object */
1045 Fts5Buffer buf; /* Buffer used to build up %_docsize blob */
1046
1047 memset(&buf, 0, sizeof(Fts5Buffer));
1048 ctx.pStorage = p;
1049 rc = fts5StorageLoadTotals(p, 1);
1050
1051 if( rc==SQLITE_OK ){
1052 rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid);
1053 }
1054 for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
1055 ctx.szCol = 0;
1056 if( pConfig->abUnindexed[ctx.iCol]==0 ){
1057 int nText = 0; /* Size of pText in bytes */
1058 const char *pText = 0; /* Pointer to buffer containing text value */
1059 int nLoc = 0; /* Size of pText in bytes */
1060 const char *pLoc = 0; /* Pointer to buffer containing text value */
1061
1062 sqlite3_value *pVal = apVal[ctx.iCol+2];
1063 if( p->pSavedRow && sqlite3_value_nochange(pVal) ){
1064 pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1);
1065 if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
1066 int iCol = ctx.iCol + 1 + pConfig->nCol;
1067 pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol);
1068 nLoc = sqlite3_column_bytes(p->pSavedRow, iCol);
1069 }
1070 }else{
1071 pVal = apVal[ctx.iCol+2];
1072 }
1073
1074 if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
1075 rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
1076 }else{
1077 pText = (const char*)sqlite3_value_text(pVal);
1078 nText = sqlite3_value_bytes(pVal);
1079 }
1080
1081 if( rc==SQLITE_OK ){
1082 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
1083 rc = sqlite3Fts5Tokenize(pConfig,
1084 FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx,
1085 fts5StorageInsertCallback
1086 );
1087 sqlite3Fts5ClearLocale(pConfig);
1088 }
1089 }
1090 sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
1091 p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
1092 }
1093 p->nTotalRow++;
1094
1095 /* Write the %_docsize record */
1096 if( rc==SQLITE_OK ){
1097 rc = fts5StorageInsertDocsize(p, iRowid, &buf);
1098 }
1099 sqlite3_free(buf.p);
1100
1101 return rc;
1102 }
1103
1104 static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){
1105 Fts5Config *pConfig = p->pConfig;
1106 char *zSql;
1107 int rc;
1108
1109 zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'",
1110 pConfig->zDb, pConfig->zName, zSuffix
1111 );
1112 if( zSql==0 ){
1113 rc = SQLITE_NOMEM;
1114 }else{
1115 sqlite3_stmt *pCnt = 0;
1116 rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pCnt, 0);
1117 if( rc==SQLITE_OK ){
1118 if( SQLITE_ROW==sqlite3_step(pCnt) ){
1119 *pnRow = sqlite3_column_int64(pCnt, 0);
1120 }
1121 rc = sqlite3_finalize(pCnt);
1122 }
1123 }
1124
1125 sqlite3_free(zSql);
1126 return rc;
1127 }
1128
1129 /*
1130 ** Context object used by sqlite3Fts5StorageIntegrity().
1131 */
1132 typedef struct Fts5IntegrityCtx Fts5IntegrityCtx;
1133 struct Fts5IntegrityCtx {
1134 i64 iRowid;
1135 int iCol;
1136 int szCol;
1137 u64 cksum;
1138 Fts5Termset *pTermset;
1139 Fts5Config *pConfig;
1140 };
1141
1142
1143 /*
1144 ** Tokenization callback used by integrity check.
1145 */
1146 static int fts5StorageIntegrityCallback(
1147 void *pContext, /* Pointer to Fts5IntegrityCtx object */
1148 int tflags,
1149 const char *pToken, /* Buffer containing token */
1150 int nToken, /* Size of token in bytes */
1151 int iUnused1, /* Start offset of token */
1152 int iUnused2 /* End offset of token */
1153 ){
1154 Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
1155 Fts5Termset *pTermset = pCtx->pTermset;
1156 int bPresent;
1157 int ii;
1158 int rc = SQLITE_OK;
1159 int iPos;
1160 int iCol;
1161
1162 UNUSED_PARAM2(iUnused1, iUnused2);
1163 if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
1164
1165 if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
1166 pCtx->szCol++;
1167 }
1168
1169 switch( pCtx->pConfig->eDetail ){
1170 case FTS5_DETAIL_FULL:
1171 iPos = pCtx->szCol-1;
1172 iCol = pCtx->iCol;
1173 break;
1174
1175 case FTS5_DETAIL_COLUMNS:
1176 iPos = pCtx->iCol;
1177 iCol = 0;
1178 break;
1179
1180 default:
1181 assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE );
1182 iPos = 0;
1183 iCol = 0;
1184 break;
1185 }
1186
1187 rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
1188 if( rc==SQLITE_OK && bPresent==0 ){
1189 pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
1190 pCtx->iRowid, iCol, iPos, 0, pToken, nToken
1191 );
1192 }
1193
1194 for(ii=0; rc==SQLITE_OK && ii<pCtx->pConfig->nPrefix; ii++){
1195 const int nChar = pCtx->pConfig->aPrefix[ii];
1196 int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
1197 if( nByte ){
1198 rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent);
1199 if( bPresent==0 ){
1200 pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
1201 pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte
1202 );
1203 }
1204 }
1205 }
1206
1207 return rc;
1208 }
1209
1210 /*
1211 ** Check that the contents of the FTS index match that of the %_content
1212 ** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return
1213 ** some other SQLite error code if an error occurs while attempting to
1214 ** determine this.
1215 */
1216 int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
1217 Fts5Config *pConfig = p->pConfig;
1218 int rc = SQLITE_OK; /* Return code */
1219 int *aColSize; /* Array of size pConfig->nCol */
1220 i64 *aTotalSize; /* Array of size pConfig->nCol */
1221 Fts5IntegrityCtx ctx;
1222 sqlite3_stmt *pScan;
1223 int bUseCksum;
1224
1225 memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
1226 ctx.pConfig = p->pConfig;
1227 aTotalSize = (i64*)sqlite3_malloc64(pConfig->nCol*(sizeof(int)+sizeof(i64)));
1228 if( !aTotalSize ) return SQLITE_NOMEM;
1229 aColSize = (int*)&aTotalSize[pConfig->nCol];
1230 memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
1231
1232 bUseCksum = (pConfig->eContent==FTS5_CONTENT_NORMAL
1233 || (pConfig->eContent==FTS5_CONTENT_EXTERNAL && iArg)
1234 );
1235 if( bUseCksum ){
1236 /* Generate the expected index checksum based on the contents of the
1237 ** %_content table. This block stores the checksum in ctx.cksum. */
1238 rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
1239 if( rc==SQLITE_OK ){
1240 int rc2;
1241 while( SQLITE_ROW==sqlite3_step(pScan) ){
1242 int i;
1243 ctx.iRowid = sqlite3_column_int64(pScan, 0);
1244 ctx.szCol = 0;
1245 if( pConfig->bColumnsize ){
1246 rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
1247 }
1248 if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){
1249 rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
1250 }
1251 for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
1252 if( pConfig->abUnindexed[i]==0 ){
1253 const char *pText = 0;
1254 int nText = 0;
1255 const char *pLoc = 0;
1256 int nLoc = 0;
1257 sqlite3_value *pVal = sqlite3_column_value(pScan, i+1);
1258
1259 if( pConfig->eContent==FTS5_CONTENT_EXTERNAL
1260 && sqlite3Fts5IsLocaleValue(pConfig, pVal)
1261 ){
1262 rc = sqlite3Fts5DecodeLocaleValue(
1263 pVal, &pText, &nText, &pLoc, &nLoc
1264 );
1265 }else{
1266 if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
1267 int iCol = i + 1 + pConfig->nCol;
1268 pLoc = (const char*)sqlite3_column_text(pScan, iCol);
1269 nLoc = sqlite3_column_bytes(pScan, iCol);
1270 }
1271 pText = (const char*)sqlite3_value_text(pVal);
1272 nText = sqlite3_value_bytes(pVal);
1273 }
1274
1275 ctx.iCol = i;
1276 ctx.szCol = 0;
1277
1278 if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
1279 rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
1280 }
1281
1282 if( rc==SQLITE_OK ){
1283 sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
1284 rc = sqlite3Fts5Tokenize(pConfig,
1285 FTS5_TOKENIZE_DOCUMENT,
1286 pText, nText,
1287 (void*)&ctx,
1288 fts5StorageIntegrityCallback
1289 );
1290 sqlite3Fts5ClearLocale(pConfig);
1291 }
1292
1293 /* If this is not a columnsize=0 database, check that the number
1294 ** of tokens in the value matches the aColSize[] value read from
1295 ** the %_docsize table. */
1296 if( rc==SQLITE_OK
1297 && pConfig->bColumnsize
1298 && ctx.szCol!=aColSize[i]
1299 ){
1300 rc = FTS5_CORRUPT;
1301 }
1302 aTotalSize[i] += ctx.szCol;
1303 if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
1304 sqlite3Fts5TermsetFree(ctx.pTermset);
1305 ctx.pTermset = 0;
1306 }
1307 }
1308 }
1309 sqlite3Fts5TermsetFree(ctx.pTermset);
1310 ctx.pTermset = 0;
1311
1312 if( rc!=SQLITE_OK ) break;
1313 }
1314 rc2 = sqlite3_reset(pScan);
1315 if( rc==SQLITE_OK ) rc = rc2;
1316 }
1317
1318 /* Test that the "totals" (sometimes called "averages") record looks Ok */
1319 if( rc==SQLITE_OK ){
1320 int i;
1321 rc = fts5StorageLoadTotals(p, 0);
1322 for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
1323 if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT;
1324 }
1325 }
1326
1327 /* Check that the %_docsize and %_content tables contain the expected
1328 ** number of rows. */
1329 if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
1330 i64 nRow = 0;
1331 rc = fts5StorageCount(p, "content", &nRow);
1332 if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
1333 }
1334 if( rc==SQLITE_OK && pConfig->bColumnsize ){
1335 i64 nRow = 0;
1336 rc = fts5StorageCount(p, "docsize", &nRow);
1337 if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
1338 }
1339 }
1340
1341 /* Pass the expected checksum down to the FTS index module. It will
1342 ** verify, amongst other things, that it matches the checksum generated by
1343 ** inspecting the index itself. */
1344 if( rc==SQLITE_OK ){
1345 rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum, bUseCksum);
1346 }
1347
1348 sqlite3_free(aTotalSize);
1349 return rc;
1350 }
1351
1352 /*
1353 ** Obtain an SQLite statement handle that may be used to read data from the
1354 ** %_content table.
1355 */
1356 int sqlite3Fts5StorageStmt(
1357 Fts5Storage *p,
1358 int eStmt,
1359 sqlite3_stmt **pp,
1360 char **pzErrMsg
1361 ){
1362 int rc;
1363 assert( eStmt==FTS5_STMT_SCAN_ASC
1364 || eStmt==FTS5_STMT_SCAN_DESC
1365 || eStmt==FTS5_STMT_LOOKUP
1366 );
1367 rc = fts5StorageGetStmt(p, eStmt, pp, pzErrMsg);
1368 if( rc==SQLITE_OK ){
1369 assert( p->aStmt[eStmt]==*pp );
1370 p->aStmt[eStmt] = 0;
1371 }
1372 return rc;
1373 }
1374
1375 /*
1376 ** Release an SQLite statement handle obtained via an earlier call to
1377 ** sqlite3Fts5StorageStmt(). The eStmt parameter passed to this function
1378 ** must match that passed to the sqlite3Fts5StorageStmt() call.
1379 */
1380 void sqlite3Fts5StorageStmtRelease(
1381 Fts5Storage *p,
1382 int eStmt,
1383 sqlite3_stmt *pStmt
1384 ){
1385 assert( eStmt==FTS5_STMT_SCAN_ASC
1386 || eStmt==FTS5_STMT_SCAN_DESC
1387 || eStmt==FTS5_STMT_LOOKUP
1388 );
1389 if( p->aStmt[eStmt]==0 ){
1390 sqlite3_reset(pStmt);
1391 p->aStmt[eStmt] = pStmt;
1392 }else{
1393 sqlite3_finalize(pStmt);
1394 }
1395 }
1396
1397 static int fts5StorageDecodeSizeArray(
1398 int *aCol, int nCol, /* Array to populate */
1399 const u8 *aBlob, int nBlob /* Record to read varints from */
1400 ){
1401 int i;
1402 int iOff = 0;
1403 for(i=0; i<nCol; i++){
1404 if( iOff>=nBlob ) return 1;
1405 iOff += fts5GetVarint32(&aBlob[iOff], aCol[i]);
1406 }
1407 return (iOff!=nBlob);
1408 }
1409
1410 /*
1411 ** Argument aCol points to an array of integers containing one entry for
1412 ** each table column. This function reads the %_docsize record for the
1413 ** specified rowid and populates aCol[] with the results.
1414 **
1415 ** An SQLite error code is returned if an error occurs, or SQLITE_OK
1416 ** otherwise.
1417 */
1418 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
1419 int nCol = p->pConfig->nCol; /* Number of user columns in table */
1420 sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */
1421 int rc; /* Return Code */
1422
1423 assert( p->pConfig->bColumnsize );
1424 rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
1425 if( pLookup ){
1426 int bCorrupt = 1;
1427 assert( rc==SQLITE_OK );
1428 sqlite3_bind_int64(pLookup, 1, iRowid);
1429 if( SQLITE_ROW==sqlite3_step(pLookup) ){
1430 const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
1431 int nBlob = sqlite3_column_bytes(pLookup, 0);
1432 if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
1433 bCorrupt = 0;
1434 }
1435 }
1436 rc = sqlite3_reset(pLookup);
1437 if( bCorrupt && rc==SQLITE_OK ){
1438 rc = FTS5_CORRUPT;
1439 }
1440 }else{
1441 assert( rc!=SQLITE_OK );
1442 }
1443
1444 return rc;
1445 }
1446
1447 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
1448 int rc = fts5StorageLoadTotals(p, 0);
1449 if( rc==SQLITE_OK ){
1450 *pnToken = 0;
1451 if( iCol<0 ){
1452 int i;
1453 for(i=0; i<p->pConfig->nCol; i++){
1454 *pnToken += p->aTotalSize[i];
1455 }
1456 }else if( iCol<p->pConfig->nCol ){
1457 *pnToken = p->aTotalSize[iCol];
1458 }else{
1459 rc = SQLITE_RANGE;
1460 }
1461 }
1462 return rc;
1463 }
1464
1465 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
1466 int rc = fts5StorageLoadTotals(p, 0);
1467 if( rc==SQLITE_OK ){
1468 /* nTotalRow being zero does not necessarily indicate a corrupt
1469 ** database - it might be that the FTS5 table really does contain zero
1470 ** rows. However this function is only called from the xRowCount() API,
1471 ** and there is no way for that API to be invoked if the table contains
1472 ** no rows. Hence the FTS5_CORRUPT return. */
1473 *pnRow = p->nTotalRow;
1474 if( p->nTotalRow<=0 ) rc = FTS5_CORRUPT;
1475 }
1476 return rc;
1477 }
1478
1479 /*
1480 ** Flush any data currently held in-memory to disk.
1481 */
1482 int sqlite3Fts5StorageSync(Fts5Storage *p){
1483 int rc = SQLITE_OK;
1484 i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
1485 if( p->bTotalsValid ){
1486 rc = fts5StorageSaveTotals(p);
1487 if( rc==SQLITE_OK ){
1488 p->bTotalsValid = 0;
1489 }
1490 }
1491 if( rc==SQLITE_OK ){
1492 rc = sqlite3Fts5IndexSync(p->pIndex);
1493 }
1494 sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid);
1495 return rc;
1496 }
1497
1498 int sqlite3Fts5StorageRollback(Fts5Storage *p){
1499 p->bTotalsValid = 0;
1500 return sqlite3Fts5IndexRollback(p->pIndex);
1501 }
1502
1503 int sqlite3Fts5StorageConfigValue(
1504 Fts5Storage *p,
1505 const char *z,
1506 sqlite3_value *pVal,
1507 int iVal
1508 ){
1509 sqlite3_stmt *pReplace = 0;
1510 int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0);
1511 if( rc==SQLITE_OK ){
1512 sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC);
1513 if( pVal ){
1514 sqlite3_bind_value(pReplace, 2, pVal);
1515 }else{
1516 sqlite3_bind_int(pReplace, 2, iVal);
1517 }
1518 sqlite3_step(pReplace);
1519 rc = sqlite3_reset(pReplace);
1520 sqlite3_bind_null(pReplace, 1);
1521 }
1522 if( rc==SQLITE_OK && pVal ){
1523 int iNew = p->pConfig->iCookie + 1;
1524 rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
1525 if( rc==SQLITE_OK ){
1526 p->pConfig->iCookie = iNew;
1527 }
1528 }
1529 return rc;
1530 }