Debian Codegraph @

ext/fts5/fts5_config.c

raw
1 /*
2 ** 2014 Jun 09
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 **
13 ** This is an SQLite module implementing full-text search.
14 */
15
16
17 #include "fts5Int.h"
18
19 #define FTS5_DEFAULT_PAGE_SIZE 4050
20 #define FTS5_DEFAULT_AUTOMERGE 4
21 #define FTS5_DEFAULT_USERMERGE 4
22 #define FTS5_DEFAULT_CRISISMERGE 16
23 #define FTS5_DEFAULT_HASHSIZE (1024*1024)
24
25 #define FTS5_DEFAULT_DELETE_AUTOMERGE 10 /* default 10% */
26
27 /* Maximum allowed page size */
28 #define FTS5_MAX_PAGE_SIZE (64*1024)
29
30 static int fts5_iswhitespace(char x){
31 return (x==' ');
32 }
33
34 static int fts5_isopenquote(char x){
35 return (x=='"' || x=='\'' || x=='[' || x=='`');
36 }
37
38 /*
39 ** Argument pIn points to a character that is part of a nul-terminated
40 ** string. Return a pointer to the first character following *pIn in
41 ** the string that is not a white-space character.
42 */
43 static const char *fts5ConfigSkipWhitespace(const char *pIn){
44 const char *p = pIn;
45 if( p ){
46 while( fts5_iswhitespace(*p) ){ p++; }
47 }
48 return p;
49 }
50
51 /*
52 ** Argument pIn points to a character that is part of a nul-terminated
53 ** string. Return a pointer to the first character following *pIn in
54 ** the string that is not a "bareword" character.
55 */
56 static const char *fts5ConfigSkipBareword(const char *pIn){
57 const char *p = pIn;
58 while ( sqlite3Fts5IsBareword(*p) ) p++;
59 if( p==pIn ) p = 0;
60 return p;
61 }
62
63 static int fts5_isdigit(char a){
64 return (a>='0' && a<='9');
65 }
66
67
68
69 static const char *fts5ConfigSkipLiteral(const char *pIn){
70 const char *p = pIn;
71 switch( *p ){
72 case 'n': case 'N':
73 if( sqlite3_strnicmp("null", p, 4)==0 ){
74 p = &p[4];
75 }else{
76 p = 0;
77 }
78 break;
79
80 case 'x': case 'X':
81 p++;
82 if( *p=='\'' ){
83 p++;
84 while( (*p>='a' && *p<='f')
85 || (*p>='A' && *p<='F')
86 || (*p>='0' && *p<='9')
87 ){
88 p++;
89 }
90 if( *p=='\'' && 0==((p-pIn)%2) ){
91 p++;
92 }else{
93 p = 0;
94 }
95 }else{
96 p = 0;
97 }
98 break;
99
100 case '\'':
101 p++;
102 while( p ){
103 if( *p=='\'' ){
104 p++;
105 if( *p!='\'' ) break;
106 }
107 p++;
108 if( *p==0 ) p = 0;
109 }
110 break;
111
112 default:
113 /* maybe a number */
114 if( *p=='+' || *p=='-' ) p++;
115 while( fts5_isdigit(*p) ) p++;
116
117 /* At this point, if the literal was an integer, the parse is
118 ** finished. Or, if it is a floating point value, it may continue
119 ** with either a decimal point or an 'E' character. */
120 if( *p=='.' && fts5_isdigit(p[1]) ){
121 p += 2;
122 while( fts5_isdigit(*p) ) p++;
123 }
124 if( p==pIn ) p = 0;
125
126 break;
127 }
128
129 return p;
130 }
131
132 /*
133 ** The first character of the string pointed to by argument z is guaranteed
134 ** to be an open-quote character (see function fts5_isopenquote()).
135 **
136 ** This function searches for the corresponding close-quote character within
137 ** the string and, if found, dequotes the string in place and adds a new
138 ** nul-terminator byte.
139 **
140 ** If the close-quote is found, the value returned is the byte offset of
141 ** the character immediately following it. Or, if the close-quote is not
142 ** found, -1 is returned. If -1 is returned, the buffer is left in an
143 ** undefined state.
144 */
145 static int fts5Dequote(char *z){
146 char q;
147 int iIn = 1;
148 int iOut = 0;
149 q = z[0];
150
151 /* Set stack variable q to the close-quote character */
152 assert( q=='[' || q=='\'' || q=='"' || q=='`' );
153 if( q=='[' ) q = ']';
154
155 while( z[iIn] ){
156 if( z[iIn]==q ){
157 if( z[iIn+1]!=q ){
158 /* Character iIn was the close quote. */
159 iIn++;
160 break;
161 }else{
162 /* Character iIn and iIn+1 form an escaped quote character. Skip
163 ** the input cursor past both and copy a single quote character
164 ** to the output buffer. */
165 iIn += 2;
166 z[iOut++] = q;
167 }
168 }else{
169 z[iOut++] = z[iIn++];
170 }
171 }
172
173 z[iOut] = '\0';
174 return iIn;
175 }
176
177 /*
178 ** Convert an SQL-style quoted string into a normal string by removing
179 ** the quote characters. The conversion is done in-place. If the
180 ** input does not begin with a quote character, then this routine
181 ** is a no-op.
182 **
183 ** Examples:
184 **
185 ** "abc" becomes abc
186 ** 'xyz' becomes xyz
187 ** [pqr] becomes pqr
188 ** `mno` becomes mno
189 */
190 void sqlite3Fts5Dequote(char *z){
191 char quote; /* Quote character (if any ) */
192
193 assert( 0==fts5_iswhitespace(z[0]) );
194 quote = z[0];
195 if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
196 fts5Dequote(z);
197 }
198 }
199
200
201 struct Fts5Enum {
202 const char *zName;
203 int eVal;
204 };
205 typedef struct Fts5Enum Fts5Enum;
206
207 static int fts5ConfigSetEnum(
208 const Fts5Enum *aEnum,
209 const char *zEnum,
210 int *peVal
211 ){
212 int nEnum = (int)strlen(zEnum);
213 int i;
214 int iVal = -1;
215
216 for(i=0; aEnum[i].zName; i++){
217 if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
218 if( iVal>=0 ) return SQLITE_ERROR;
219 iVal = aEnum[i].eVal;
220 }
221 }
222
223 *peVal = iVal;
224 return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
225 }
226
227 /*
228 ** Parse a "special" CREATE VIRTUAL TABLE directive and update
229 ** configuration object pConfig as appropriate.
230 **
231 ** If successful, object pConfig is updated and SQLITE_OK returned. If
232 ** an error occurs, an SQLite error code is returned and an error message
233 ** may be left in *pzErr. It is the responsibility of the caller to
234 ** eventually free any such error message using sqlite3_free().
235 */
236 static int fts5ConfigParseSpecial(
237 Fts5Config *pConfig, /* Configuration object to update */
238 const char *zCmd, /* Special command to parse */
239 const char *zArg, /* Argument to parse */
240 char **pzErr /* OUT: Error message */
241 ){
242 int rc = SQLITE_OK;
243 int nCmd = (int)strlen(zCmd);
244
245 if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
246 const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
247 const char *p;
248 int bFirst = 1;
249 if( pConfig->aPrefix==0 ){
250 pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
251 if( rc ) return rc;
252 }
253
254 p = zArg;
255 while( 1 ){
256 int nPre = 0;
257
258 while( p[0]==' ' ) p++;
259 if( bFirst==0 && p[0]==',' ){
260 p++;
261 while( p[0]==' ' ) p++;
262 }else if( p[0]=='\0' ){
263 break;
264 }
265 if( p[0]<'0' || p[0]>'9' ){
266 *pzErr = sqlite3_mprintf("malformed prefix=... directive");
267 rc = SQLITE_ERROR;
268 break;
269 }
270
271 if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES ){
272 *pzErr = sqlite3_mprintf(
273 "too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES
274 );
275 rc = SQLITE_ERROR;
276 break;
277 }
278
279 while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
280 nPre = nPre*10 + (p[0] - '0');
281 p++;
282 }
283
284 if( nPre<=0 || nPre>=1000 ){
285 *pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
286 rc = SQLITE_ERROR;
287 break;
288 }
289
290 pConfig->aPrefix[pConfig->nPrefix] = nPre;
291 pConfig->nPrefix++;
292 bFirst = 0;
293 }
294 assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES );
295 return rc;
296 }
297
298 if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
299 const char *p = (const char*)zArg;
300 sqlite3_int64 nArg = strlen(zArg) + 1;
301 char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg);
302
303 if( azArg ){
304 char *pSpace = (char*)&azArg[nArg];
305 if( pConfig->t.azArg ){
306 *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
307 rc = SQLITE_ERROR;
308 }else{
309 for(nArg=0; p && *p; nArg++){
310 const char *p2 = fts5ConfigSkipWhitespace(p);
311 if( *p2=='\'' ){
312 p = fts5ConfigSkipLiteral(p2);
313 }else{
314 p = fts5ConfigSkipBareword(p2);
315 }
316 if( p ){
317 memcpy(pSpace, p2, p-p2);
318 azArg[nArg] = pSpace;
319 sqlite3Fts5Dequote(pSpace);
320 pSpace += (p - p2) + 1;
321 p = fts5ConfigSkipWhitespace(p);
322 }
323 }
324 if( p==0 ){
325 *pzErr = sqlite3_mprintf("parse error in tokenize directive");
326 rc = SQLITE_ERROR;
327 }else{
328 pConfig->t.azArg = (const char**)azArg;
329 pConfig->t.nArg = nArg;
330 azArg = 0;
331 }
332 }
333 }
334 sqlite3_free(azArg);
335
336 return rc;
337 }
338
339 if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
340 if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
341 *pzErr = sqlite3_mprintf("multiple content=... directives");
342 rc = SQLITE_ERROR;
343 }else{
344 if( zArg[0] ){
345 pConfig->eContent = FTS5_CONTENT_EXTERNAL;
346 pConfig->zContent = sqlite3Fts5Mprintf(&rc, "%Q.%Q", pConfig->zDb,zArg);
347 }else{
348 pConfig->eContent = FTS5_CONTENT_NONE;
349 }
350 }
351 return rc;
352 }
353
354 if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){
355 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
356 *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive");
357 rc = SQLITE_ERROR;
358 }else{
359 pConfig->bContentlessDelete = (zArg[0]=='1');
360 }
361 return rc;
362 }
363
364 if( sqlite3_strnicmp("contentless_unindexed", zCmd, nCmd)==0 ){
365 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
366 *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive");
367 rc = SQLITE_ERROR;
368 }else{
369 pConfig->bContentlessUnindexed = (zArg[0]=='1');
370 }
371 return rc;
372 }
373
374 if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
375 if( pConfig->zContentRowid ){
376 *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
377 rc = SQLITE_ERROR;
378 }else{
379 pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1);
380 }
381 return rc;
382 }
383
384 if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){
385 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
386 *pzErr = sqlite3_mprintf("malformed columnsize=... directive");
387 rc = SQLITE_ERROR;
388 }else{
389 pConfig->bColumnsize = (zArg[0]=='1');
390 }
391 return rc;
392 }
393
394 if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){
395 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
396 *pzErr = sqlite3_mprintf("malformed locale=... directive");
397 rc = SQLITE_ERROR;
398 }else{
399 pConfig->bLocale = (zArg[0]=='1');
400 }
401 return rc;
402 }
403
404 if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
405 const Fts5Enum aDetail[] = {
406 { "none", FTS5_DETAIL_NONE },
407 { "full", FTS5_DETAIL_FULL },
408 { "columns", FTS5_DETAIL_COLUMNS },
409 { 0, 0 }
410 };
411
412 if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){
413 *pzErr = sqlite3_mprintf("malformed detail=... directive");
414 }
415 return rc;
416 }
417
418 if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){
419 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
420 *pzErr = sqlite3_mprintf("malformed tokendata=... directive");
421 rc = SQLITE_ERROR;
422 }else{
423 pConfig->bTokendata = (zArg[0]=='1');
424 }
425 return rc;
426 }
427
428 *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
429 return SQLITE_ERROR;
430 }
431
432 /*
433 ** Gobble up the first bareword or quoted word from the input buffer zIn.
434 ** Return a pointer to the character immediately following the last in
435 ** the gobbled word if successful, or a NULL pointer otherwise (failed
436 ** to find close-quote character).
437 **
438 ** Before returning, set pzOut to point to a new buffer containing a
439 ** nul-terminated, dequoted copy of the gobbled word. If the word was
440 ** quoted, *pbQuoted is also set to 1 before returning.
441 **
442 ** If *pRc is other than SQLITE_OK when this function is called, it is
443 ** a no-op (NULL is returned). Otherwise, if an OOM occurs within this
444 ** function, *pRc is set to SQLITE_NOMEM before returning. *pRc is *not*
445 ** set if a parse error (failed to find close quote) occurs.
446 */
447 static const char *fts5ConfigGobbleWord(
448 int *pRc, /* IN/OUT: Error code */
449 const char *zIn, /* Buffer to gobble string/bareword from */
450 char **pzOut, /* OUT: malloc'd buffer containing str/bw */
451 int *pbQuoted /* OUT: Set to true if dequoting required */
452 ){
453 const char *zRet = 0;
454
455 sqlite3_int64 nIn = strlen(zIn);
456 char *zOut = sqlite3_malloc64(nIn+1);
457
458 assert( *pRc==SQLITE_OK );
459 *pbQuoted = 0;
460 *pzOut = 0;
461
462 if( zOut==0 ){
463 *pRc = SQLITE_NOMEM;
464 }else{
465 memcpy(zOut, zIn, (size_t)(nIn+1));
466 if( fts5_isopenquote(zOut[0]) ){
467 int ii = fts5Dequote(zOut);
468 zRet = &zIn[ii];
469 *pbQuoted = 1;
470 }else{
471 zRet = fts5ConfigSkipBareword(zIn);
472 if( zRet ){
473 zOut[zRet-zIn] = '\0';
474 }
475 }
476 }
477
478 if( zRet==0 ){
479 sqlite3_free(zOut);
480 }else{
481 *pzOut = zOut;
482 }
483
484 return zRet;
485 }
486
487 static int fts5ConfigParseColumn(
488 Fts5Config *p,
489 char *zCol,
490 char *zArg,
491 char **pzErr,
492 int *pbUnindexed
493 ){
494 int rc = SQLITE_OK;
495 if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME)
496 || 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME)
497 ){
498 *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol);
499 rc = SQLITE_ERROR;
500 }else if( zArg ){
501 if( 0==sqlite3_stricmp(zArg, "unindexed") ){
502 p->abUnindexed[p->nCol] = 1;
503 *pbUnindexed = 1;
504 }else{
505 *pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg);
506 rc = SQLITE_ERROR;
507 }
508 }
509
510 p->azCol[p->nCol++] = zCol;
511 return rc;
512 }
513
514 /*
515 ** Populate the Fts5Config.zContentExprlist string.
516 */
517 static int fts5ConfigMakeExprlist(Fts5Config *p){
518 int i;
519 int rc = SQLITE_OK;
520 Fts5Buffer buf = {0, 0, 0};
521
522 sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid);
523 if( p->eContent!=FTS5_CONTENT_NONE ){
524 assert( p->eContent==FTS5_CONTENT_EXTERNAL
525 || p->eContent==FTS5_CONTENT_NORMAL
526 || p->eContent==FTS5_CONTENT_UNINDEXED
527 );
528 for(i=0; i<p->nCol; i++){
529 if( p->eContent==FTS5_CONTENT_EXTERNAL ){
530 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]);
531 }else if( p->eContent==FTS5_CONTENT_NORMAL || p->abUnindexed[i] ){
532 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i);
533 }else{
534 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
535 }
536 }
537 }
538 if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){
539 for(i=0; i<p->nCol; i++){
540 if( p->abUnindexed[i]==0 ){
541 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i);
542 }else{
543 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
544 }
545 }
546 }
547
548 assert( p->zContentExprlist==0 );
549 p->zContentExprlist = (char*)buf.p;
550 return rc;
551 }
552
553 /*
554 ** Arguments nArg/azArg contain the string arguments passed to the xCreate
555 ** or xConnect method of the virtual table. This function attempts to
556 ** allocate an instance of Fts5Config containing the results of parsing
557 ** those arguments.
558 **
559 ** If successful, SQLITE_OK is returned and *ppOut is set to point to the
560 ** new Fts5Config object. If an error occurs, an SQLite error code is
561 ** returned, *ppOut is set to NULL and an error message may be left in
562 ** *pzErr. It is the responsibility of the caller to eventually free any
563 ** such error message using sqlite3_free().
564 */
565 int sqlite3Fts5ConfigParse(
566 Fts5Global *pGlobal,
567 sqlite3 *db,
568 int nArg, /* Number of arguments */
569 const char **azArg, /* Array of nArg CREATE VIRTUAL TABLE args */
570 Fts5Config **ppOut, /* OUT: Results of parse */
571 char **pzErr /* OUT: Error message */
572 ){
573 int rc = SQLITE_OK; /* Return code */
574 Fts5Config *pRet; /* New object to return */
575 int i;
576 sqlite3_int64 nByte;
577 int bUnindexed = 0; /* True if there are one or more UNINDEXED */
578
579 *ppOut = pRet = (Fts5Config*)sqlite3_malloc64(sizeof(Fts5Config));
580 if( pRet==0 ) return SQLITE_NOMEM;
581 memset(pRet, 0, sizeof(Fts5Config));
582 pRet->pGlobal = pGlobal;
583 pRet->db = db;
584 pRet->iCookie = -1;
585
586 nByte = nArg * (sizeof(char*) + sizeof(u8));
587 pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
588 pRet->abUnindexed = pRet->azCol ? (u8*)&pRet->azCol[nArg] : 0;
589 pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
590 pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
591 pRet->bColumnsize = 1;
592 pRet->eDetail = FTS5_DETAIL_FULL;
593 #ifdef SQLITE_DEBUG
594 pRet->bPrefixIndex = 1;
595 #endif
596 if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
597 *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
598 rc = SQLITE_ERROR;
599 }
600
601 assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK );
602 for(i=3; rc==SQLITE_OK && i<nArg; i++){
603 const char *zOrig = azArg[i];
604 const char *z;
605 char *zOne = 0;
606 char *zTwo = 0;
607 int bOption = 0;
608 int bMustBeCol = 0;
609
610 z = fts5ConfigGobbleWord(&rc, zOrig, &zOne, &bMustBeCol);
611 z = fts5ConfigSkipWhitespace(z);
612 if( z && *z=='=' ){
613 bOption = 1;
614 assert( zOne!=0 );
615 z++;
616 if( bMustBeCol ) z = 0;
617 }
618 z = fts5ConfigSkipWhitespace(z);
619 if( z && z[0] ){
620 int bDummy;
621 z = fts5ConfigGobbleWord(&rc, z, &zTwo, &bDummy);
622 if( z && z[0] ) z = 0;
623 }
624
625 if( rc==SQLITE_OK ){
626 if( z==0 ){
627 *pzErr = sqlite3_mprintf("parse error in \"%s\"", zOrig);
628 rc = SQLITE_ERROR;
629 }else{
630 if( bOption ){
631 rc = fts5ConfigParseSpecial(pRet,
632 ALWAYS(zOne)?zOne:"",
633 zTwo?zTwo:"",
634 pzErr
635 );
636 }else{
637 rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr, &bUnindexed);
638 zOne = 0;
639 }
640 }
641 }
642
643 sqlite3_free(zOne);
644 sqlite3_free(zTwo);
645 }
646
647 /* We only allow contentless_delete=1 if the table is indeed contentless. */
648 if( rc==SQLITE_OK
649 && pRet->bContentlessDelete
650 && pRet->eContent!=FTS5_CONTENT_NONE
651 ){
652 *pzErr = sqlite3_mprintf(
653 "contentless_delete=1 requires a contentless table"
654 );
655 rc = SQLITE_ERROR;
656 }
657
658 /* We only allow contentless_delete=1 if columnsize=0 is not present.
659 **
660 ** This restriction may be removed at some point.
661 */
662 if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){
663 *pzErr = sqlite3_mprintf(
664 "contentless_delete=1 is incompatible with columnsize=0"
665 );
666 rc = SQLITE_ERROR;
667 }
668
669 /* We only allow contentless_unindexed=1 if the table is actually a
670 ** contentless one.
671 */
672 if( rc==SQLITE_OK
673 && pRet->bContentlessUnindexed
674 && pRet->eContent!=FTS5_CONTENT_NONE
675 ){
676 *pzErr = sqlite3_mprintf(
677 "contentless_unindexed=1 requires a contentless table"
678 );
679 rc = SQLITE_ERROR;
680 }
681
682 /* If no zContent option was specified, fill in the default values. */
683 if( rc==SQLITE_OK && pRet->zContent==0 ){
684 const char *zTail = 0;
685 assert( pRet->eContent==FTS5_CONTENT_NORMAL
686 || pRet->eContent==FTS5_CONTENT_NONE
687 );
688 if( pRet->eContent==FTS5_CONTENT_NORMAL ){
689 zTail = "content";
690 }else if( bUnindexed && pRet->bContentlessUnindexed ){
691 pRet->eContent = FTS5_CONTENT_UNINDEXED;
692 zTail = "content";
693 }else if( pRet->bColumnsize ){
694 zTail = "docsize";
695 }
696
697 if( zTail ){
698 pRet->zContent = sqlite3Fts5Mprintf(
699 &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail
700 );
701 }
702 }
703
704 if( rc==SQLITE_OK && pRet->zContentRowid==0 ){
705 pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1);
706 }
707
708 /* Formulate the zContentExprlist text */
709 if( rc==SQLITE_OK ){
710 rc = fts5ConfigMakeExprlist(pRet);
711 }
712
713 if( rc!=SQLITE_OK ){
714 sqlite3Fts5ConfigFree(pRet);
715 *ppOut = 0;
716 }
717 return rc;
718 }
719
720 /*
721 ** Free the configuration object passed as the only argument.
722 */
723 void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
724 if( pConfig ){
725 int i;
726 if( pConfig->t.pTok ){
727 if( pConfig->t.pApi1 ){
728 pConfig->t.pApi1->xDelete(pConfig->t.pTok);
729 }else{
730 pConfig->t.pApi2->xDelete(pConfig->t.pTok);
731 }
732 }
733 sqlite3_free((char*)pConfig->t.azArg);
734 sqlite3_free(pConfig->zDb);
735 sqlite3_free(pConfig->zName);
736 for(i=0; i<pConfig->nCol; i++){
737 sqlite3_free(pConfig->azCol[i]);
738 }
739 sqlite3_free(pConfig->azCol);
740 sqlite3_free(pConfig->aPrefix);
741 sqlite3_free(pConfig->zRank);
742 sqlite3_free(pConfig->zRankArgs);
743 sqlite3_free(pConfig->zContent);
744 sqlite3_free(pConfig->zContentRowid);
745 sqlite3_free(pConfig->zContentExprlist);
746 sqlite3_free(pConfig);
747 }
748 }
749
750 /*
751 ** Call sqlite3_declare_vtab() based on the contents of the configuration
752 ** object passed as the only argument. Return SQLITE_OK if successful, or
753 ** an SQLite error code if an error occurs.
754 */
755 int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){
756 int i;
757 int rc = SQLITE_OK;
758 char *zSql;
759
760 zSql = sqlite3Fts5Mprintf(&rc, "CREATE TABLE x(");
761 for(i=0; zSql && i<pConfig->nCol; i++){
762 const char *zSep = (i==0?"":", ");
763 zSql = sqlite3Fts5Mprintf(&rc, "%z%s%Q", zSql, zSep, pConfig->azCol[i]);
764 }
765 zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)",
766 zSql, pConfig->zName, FTS5_RANK_NAME
767 );
768
769 assert( zSql || rc==SQLITE_NOMEM );
770 if( zSql ){
771 rc = sqlite3_declare_vtab(pConfig->db, zSql);
772 sqlite3_free(zSql);
773 }
774
775 return rc;
776 }
777
778 /*
779 ** Tokenize the text passed via the second and third arguments.
780 **
781 ** The callback is invoked once for each token in the input text. The
782 ** arguments passed to it are, in order:
783 **
784 ** void *pCtx // Copy of 4th argument to sqlite3Fts5Tokenize()
785 ** const char *pToken // Pointer to buffer containing token
786 ** int nToken // Size of token in bytes
787 ** int iStart // Byte offset of start of token within input text
788 ** int iEnd // Byte offset of end of token within input text
789 ** int iPos // Position of token in input (first token is 0)
790 **
791 ** If the callback returns a non-zero value the tokenization is abandoned
792 ** and no further callbacks are issued.
793 **
794 ** This function returns SQLITE_OK if successful or an SQLite error code
795 ** if an error occurs. If the tokenization was abandoned early because
796 ** the callback returned SQLITE_DONE, this is not an error and this function
797 ** still returns SQLITE_OK. Or, if the tokenization was abandoned early
798 ** because the callback returned another non-zero value, it is assumed
799 ** to be an SQLite error code and returned to the caller.
800 */
801 int sqlite3Fts5Tokenize(
802 Fts5Config *pConfig, /* FTS5 Configuration object */
803 int flags, /* FTS5_TOKENIZE_* flags */
804 const char *pText, int nText, /* Text to tokenize */
805 void *pCtx, /* Context passed to xToken() */
806 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
807 ){
808 int rc = SQLITE_OK;
809 if( pText ){
810 if( pConfig->t.pTok==0 ){
811 rc = sqlite3Fts5LoadTokenizer(pConfig);
812 }
813 if( rc==SQLITE_OK ){
814 if( pConfig->t.pApi1 ){
815 rc = pConfig->t.pApi1->xTokenize(
816 pConfig->t.pTok, pCtx, flags, pText, nText, xToken
817 );
818 }else{
819 rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags,
820 pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken
821 );
822 }
823 }
824 }
825 return rc;
826 }
827
828 /*
829 ** Argument pIn points to the first character in what is expected to be
830 ** a comma-separated list of SQL literals followed by a ')' character.
831 ** If it actually is this, return a pointer to the ')'. Otherwise, return
832 ** NULL to indicate a parse error.
833 */
834 static const char *fts5ConfigSkipArgs(const char *pIn){
835 const char *p = pIn;
836
837 while( 1 ){
838 p = fts5ConfigSkipWhitespace(p);
839 p = fts5ConfigSkipLiteral(p);
840 p = fts5ConfigSkipWhitespace(p);
841 if( p==0 || *p==')' ) break;
842 if( *p!=',' ){
843 p = 0;
844 break;
845 }
846 p++;
847 }
848
849 return p;
850 }
851
852 /*
853 ** Parameter zIn contains a rank() function specification. The format of
854 ** this is:
855 **
856 ** + Bareword (function name)
857 ** + Open parenthesis - "("
858 ** + Zero or more SQL literals in a comma separated list
859 ** + Close parenthesis - ")"
860 */
861 int sqlite3Fts5ConfigParseRank(
862 const char *zIn, /* Input string */
863 char **pzRank, /* OUT: Rank function name */
864 char **pzRankArgs /* OUT: Rank function arguments */
865 ){
866 const char *p = zIn;
867 const char *pRank;
868 char *zRank = 0;
869 char *zRankArgs = 0;
870 int rc = SQLITE_OK;
871
872 *pzRank = 0;
873 *pzRankArgs = 0;
874
875 if( p==0 ){
876 rc = SQLITE_ERROR;
877 }else{
878 p = fts5ConfigSkipWhitespace(p);
879 pRank = p;
880 p = fts5ConfigSkipBareword(p);
881
882 if( p ){
883 zRank = sqlite3Fts5MallocZero(&rc, 1 + p - pRank);
884 if( zRank ) memcpy(zRank, pRank, p-pRank);
885 }else{
886 rc = SQLITE_ERROR;
887 }
888
889 if( rc==SQLITE_OK ){
890 p = fts5ConfigSkipWhitespace(p);
891 if( *p!='(' ) rc = SQLITE_ERROR;
892 p++;
893 }
894 if( rc==SQLITE_OK ){
895 const char *pArgs;
896 p = fts5ConfigSkipWhitespace(p);
897 pArgs = p;
898 if( *p!=')' ){
899 p = fts5ConfigSkipArgs(p);
900 if( p==0 ){
901 rc = SQLITE_ERROR;
902 }else{
903 zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
904 if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
905 }
906 }
907 }
908 }
909
910 if( rc!=SQLITE_OK ){
911 sqlite3_free(zRank);
912 assert( zRankArgs==0 );
913 }else{
914 *pzRank = zRank;
915 *pzRankArgs = zRankArgs;
916 }
917 return rc;
918 }
919
920 int sqlite3Fts5ConfigSetValue(
921 Fts5Config *pConfig,
922 const char *zKey,
923 sqlite3_value *pVal,
924 int *pbBadkey
925 ){
926 int rc = SQLITE_OK;
927
928 if( 0==sqlite3_stricmp(zKey, "pgsz") ){
929 int pgsz = 0;
930 if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
931 pgsz = sqlite3_value_int(pVal);
932 }
933 if( pgsz<32 || pgsz>FTS5_MAX_PAGE_SIZE ){
934 *pbBadkey = 1;
935 }else{
936 pConfig->pgsz = pgsz;
937 }
938 }
939
940 else if( 0==sqlite3_stricmp(zKey, "hashsize") ){
941 int nHashSize = -1;
942 if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
943 nHashSize = sqlite3_value_int(pVal);
944 }
945 if( nHashSize<=0 ){
946 *pbBadkey = 1;
947 }else{
948 pConfig->nHashSize = nHashSize;
949 }
950 }
951
952 else if( 0==sqlite3_stricmp(zKey, "automerge") ){
953 int nAutomerge = -1;
954 if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
955 nAutomerge = sqlite3_value_int(pVal);
956 }
957 if( nAutomerge<0 || nAutomerge>64 ){
958 *pbBadkey = 1;
959 }else{
960 if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE;
961 pConfig->nAutomerge = nAutomerge;
962 }
963 }
964
965 else if( 0==sqlite3_stricmp(zKey, "usermerge") ){
966 int nUsermerge = -1;
967 if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
968 nUsermerge = sqlite3_value_int(pVal);
969 }
970 if( nUsermerge<2 || nUsermerge>16 ){
971 *pbBadkey = 1;
972 }else{
973 pConfig->nUsermerge = nUsermerge;
974 }
975 }
976
977 else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){
978 int nCrisisMerge = -1;
979 if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
980 nCrisisMerge = sqlite3_value_int(pVal);
981 }
982 if( nCrisisMerge<0 ){
983 *pbBadkey = 1;
984 }else{
985 if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
986 if( nCrisisMerge>=FTS5_MAX_SEGMENT ) nCrisisMerge = FTS5_MAX_SEGMENT-1;
987 pConfig->nCrisisMerge = nCrisisMerge;
988 }
989 }
990
991 else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){
992 int nVal = -1;
993 if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
994 nVal = sqlite3_value_int(pVal);
995 }else{
996 *pbBadkey = 1;
997 }
998 if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE;
999 if( nVal>100 ) nVal = 0;
1000 pConfig->nDeleteMerge = nVal;
1001 }
1002
1003 else if( 0==sqlite3_stricmp(zKey, "rank") ){
1004 const char *zIn = (const char*)sqlite3_value_text(pVal);
1005 char *zRank;
1006 char *zRankArgs;
1007 rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs);
1008 if( rc==SQLITE_OK ){
1009 sqlite3_free(pConfig->zRank);
1010 sqlite3_free(pConfig->zRankArgs);
1011 pConfig->zRank = zRank;
1012 pConfig->zRankArgs = zRankArgs;
1013 }else if( rc==SQLITE_ERROR ){
1014 rc = SQLITE_OK;
1015 *pbBadkey = 1;
1016 }
1017 }
1018
1019 else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){
1020 int bVal = -1;
1021 if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
1022 bVal = sqlite3_value_int(pVal);
1023 }
1024 if( bVal<0 ){
1025 *pbBadkey = 1;
1026 }else{
1027 pConfig->bSecureDelete = (bVal ? 1 : 0);
1028 }
1029 }
1030
1031 else if( 0==sqlite3_stricmp(zKey, "insttoken") ){
1032 int bVal = -1;
1033 if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
1034 bVal = sqlite3_value_int(pVal);
1035 }
1036 if( bVal<0 ){
1037 *pbBadkey = 1;
1038 }else{
1039 pConfig->bPrefixInsttoken = (bVal ? 1 : 0);
1040 }
1041
1042 }else{
1043 *pbBadkey = 1;
1044 }
1045 return rc;
1046 }
1047
1048 /*
1049 ** Load the contents of the %_config table into memory.
1050 */
1051 int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
1052 const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
1053 char *zSql;
1054 sqlite3_stmt *p = 0;
1055 int rc = SQLITE_OK;
1056 int iVersion = 0;
1057
1058 /* Set default values */
1059 pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
1060 pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
1061 pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
1062 pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
1063 pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
1064 pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE;
1065
1066 zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
1067 if( zSql ){
1068 rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
1069 sqlite3_free(zSql);
1070 }
1071
1072 assert( rc==SQLITE_OK || p==0 );
1073 if( rc==SQLITE_OK ){
1074 while( SQLITE_ROW==sqlite3_step(p) ){
1075 const char *zK = (const char*)sqlite3_column_text(p, 0);
1076 sqlite3_value *pVal = sqlite3_column_value(p, 1);
1077 if( 0==sqlite3_stricmp(zK, "version") ){
1078 iVersion = sqlite3_value_int(pVal);
1079 }else{
1080 int bDummy = 0;
1081 sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, &bDummy);
1082 }
1083 }
1084 rc = sqlite3_finalize(p);
1085 }
1086
1087 if( rc==SQLITE_OK
1088 && iVersion!=FTS5_CURRENT_VERSION
1089 && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE
1090 ){
1091 rc = SQLITE_ERROR;
1092 sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format "
1093 "(found %d, expected %d or %d) - run 'rebuild'",
1094 iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
1095 );
1096 }else{
1097 pConfig->iVersion = iVersion;
1098 }
1099
1100 if( rc==SQLITE_OK ){
1101 pConfig->iCookie = iCookie;
1102 }
1103 return rc;
1104 }
1105
1106 /*
1107 ** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer
1108 ** containing the error message created using printf() style formatting
1109 ** string zFmt and its trailing arguments.
1110 */
1111 void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){
1112 va_list ap; /* ... printf arguments */
1113 char *zMsg = 0;
1114
1115 va_start(ap, zFmt);
1116 zMsg = sqlite3_vmprintf(zFmt, ap);
1117 if( pConfig->pzErrmsg ){
1118 assert( *pConfig->pzErrmsg==0 );
1119 *pConfig->pzErrmsg = zMsg;
1120 }else{
1121 sqlite3_free(zMsg);
1122 }
1123
1124 va_end(ap);
1125 }