| 1 |
/* |
| 2 |
** 2011 Jun 13 |
| 3 |
** |
| 4 |
** The author disclaims copyright to this source code. In place of |
| 5 |
** a legal notice, here is a blessing: |
| 6 |
** |
| 7 |
** May you do good and not evil. |
| 8 |
** May you find forgiveness for yourself and forgive others. |
| 9 |
** May you share freely, never taking more than you give. |
| 10 |
** |
| 11 |
****************************************************************************** |
| 12 |
** |
| 13 |
** This file is not part of the production FTS code. It is only used for |
| 14 |
** testing. It contains a Tcl command that can be used to test if a document |
| 15 |
** matches an FTS NEAR expression. |
| 16 |
** |
| 17 |
** As of March 2012, it also contains a version 1 tokenizer used for testing |
| 18 |
** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly. |
| 19 |
*/ |
| 20 |
|
| 21 |
#include "tclsqlite.h" |
| 22 |
#include <string.h> |
| 23 |
#include <assert.h> |
| 24 |
|
| 25 |
#if defined(SQLITE_TEST) |
| 26 |
#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) |
| 27 |
|
| 28 |
/* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */ |
| 29 |
#include "fts3Int.h" |
| 30 |
|
| 31 |
#define NM_MAX_TOKEN 12 |
| 32 |
|
| 33 |
typedef struct NearPhrase NearPhrase; |
| 34 |
typedef struct NearDocument NearDocument; |
| 35 |
typedef struct NearToken NearToken; |
| 36 |
|
| 37 |
struct NearDocument { |
| 38 |
int nToken; /* Length of token in bytes */ |
| 39 |
NearToken *aToken; /* Token array */ |
| 40 |
}; |
| 41 |
|
| 42 |
struct NearToken { |
| 43 |
int n; /* Length of token in bytes */ |
| 44 |
const char *z; /* Pointer to token string */ |
| 45 |
}; |
| 46 |
|
| 47 |
struct NearPhrase { |
| 48 |
int nNear; /* Preceding NEAR value */ |
| 49 |
int nToken; /* Number of tokens in this phrase */ |
| 50 |
NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */ |
| 51 |
}; |
| 52 |
|
| 53 |
static int nm_phrase_match( |
| 54 |
NearPhrase *p, |
| 55 |
NearToken *aToken |
| 56 |
){ |
| 57 |
int ii; |
| 58 |
|
| 59 |
for(ii=0; ii<p->nToken; ii++){ |
| 60 |
NearToken *pToken = &p->aToken[ii]; |
| 61 |
if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){ |
| 62 |
if( aToken[ii].n<(pToken->n-1) ) return 0; |
| 63 |
if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0; |
| 64 |
}else{ |
| 65 |
if( aToken[ii].n!=pToken->n ) return 0; |
| 66 |
if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0; |
| 67 |
} |
| 68 |
} |
| 69 |
|
| 70 |
return 1; |
| 71 |
} |
| 72 |
|
| 73 |
static int nm_near_chain( |
| 74 |
int iDir, /* Direction to iterate through aPhrase[] */ |
| 75 |
NearDocument *pDoc, /* Document to match against */ |
| 76 |
int iPos, /* Position at which iPhrase was found */ |
| 77 |
int nPhrase, /* Size of phrase array */ |
| 78 |
NearPhrase *aPhrase, /* Phrase array */ |
| 79 |
int iPhrase /* Index of phrase found */ |
| 80 |
){ |
| 81 |
int iStart; |
| 82 |
int iStop; |
| 83 |
int ii; |
| 84 |
int nNear; |
| 85 |
int iPhrase2; |
| 86 |
NearPhrase *p; |
| 87 |
NearPhrase *pPrev; |
| 88 |
|
| 89 |
assert( iDir==1 || iDir==-1 ); |
| 90 |
|
| 91 |
if( iDir==1 ){ |
| 92 |
if( (iPhrase+1)==nPhrase ) return 1; |
| 93 |
nNear = aPhrase[iPhrase+1].nNear; |
| 94 |
}else{ |
| 95 |
if( iPhrase==0 ) return 1; |
| 96 |
nNear = aPhrase[iPhrase].nNear; |
| 97 |
} |
| 98 |
pPrev = &aPhrase[iPhrase]; |
| 99 |
iPhrase2 = iPhrase+iDir; |
| 100 |
p = &aPhrase[iPhrase2]; |
| 101 |
|
| 102 |
iStart = iPos - nNear - p->nToken; |
| 103 |
iStop = iPos + nNear + pPrev->nToken; |
| 104 |
|
| 105 |
if( iStart<0 ) iStart = 0; |
| 106 |
if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken; |
| 107 |
|
| 108 |
for(ii=iStart; ii<=iStop; ii++){ |
| 109 |
if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ |
| 110 |
if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1; |
| 111 |
} |
| 112 |
} |
| 113 |
|
| 114 |
return 0; |
| 115 |
} |
| 116 |
|
| 117 |
static int nm_match_count( |
| 118 |
NearDocument *pDoc, /* Document to match against */ |
| 119 |
int nPhrase, /* Size of phrase array */ |
| 120 |
NearPhrase *aPhrase, /* Phrase array */ |
| 121 |
int iPhrase /* Index of phrase to count matches for */ |
| 122 |
){ |
| 123 |
int nOcc = 0; |
| 124 |
int ii; |
| 125 |
NearPhrase *p = &aPhrase[iPhrase]; |
| 126 |
|
| 127 |
for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){ |
| 128 |
if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ |
| 129 |
/* Test forward NEAR chain (i>iPhrase) */ |
| 130 |
if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue; |
| 131 |
|
| 132 |
/* Test reverse NEAR chain (i<iPhrase) */ |
| 133 |
if( 0==nm_near_chain(-1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue; |
| 134 |
|
| 135 |
/* This is a real match. Increment the counter. */ |
| 136 |
nOcc++; |
| 137 |
} |
| 138 |
} |
| 139 |
|
| 140 |
return nOcc; |
| 141 |
} |
| 142 |
|
| 143 |
/* |
| 144 |
** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS? |
| 145 |
*/ |
| 146 |
static int SQLITE_TCLAPI fts3_near_match_cmd( |
| 147 |
ClientData clientData, |
| 148 |
Tcl_Interp *interp, |
| 149 |
int objc, |
| 150 |
Tcl_Obj *CONST objv[] |
| 151 |
){ |
| 152 |
int nTotal = 0; |
| 153 |
int rc; |
| 154 |
int ii; |
| 155 |
int nPhrase; |
| 156 |
NearPhrase *aPhrase = 0; |
| 157 |
NearDocument doc = {0, 0}; |
| 158 |
Tcl_Obj **apDocToken; |
| 159 |
Tcl_Obj *pRet; |
| 160 |
Tcl_Obj *pPhrasecount = 0; |
| 161 |
|
| 162 |
Tcl_Obj **apExprToken; |
| 163 |
Tcl_Size nExprToken; |
| 164 |
Tcl_Size nn; |
| 165 |
|
| 166 |
UNUSED_PARAMETER(clientData); |
| 167 |
|
| 168 |
/* Must have 3 or more arguments. */ |
| 169 |
if( objc<3 || (objc%2)==0 ){ |
| 170 |
Tcl_WrongNumArgs(interp, 1, objv, "DOCUMENT EXPR ?OPTION VALUE?..."); |
| 171 |
rc = TCL_ERROR; |
| 172 |
goto near_match_out; |
| 173 |
} |
| 174 |
|
| 175 |
for(ii=3; ii<objc; ii+=2){ |
| 176 |
enum NM_enum { NM_PHRASECOUNTS }; |
| 177 |
struct TestnmSubcmd { |
| 178 |
char *zName; |
| 179 |
enum NM_enum eOpt; |
| 180 |
} aOpt[] = { |
| 181 |
{ "-phrasecountvar", NM_PHRASECOUNTS }, |
| 182 |
{ 0, 0 } |
| 183 |
}; |
| 184 |
int iOpt; |
| 185 |
if( Tcl_GetIndexFromObjStruct( |
| 186 |
interp, objv[ii], aOpt, sizeof(aOpt[0]), "option", 0, &iOpt) |
| 187 |
){ |
| 188 |
return TCL_ERROR; |
| 189 |
} |
| 190 |
|
| 191 |
switch( aOpt[iOpt].eOpt ){ |
| 192 |
case NM_PHRASECOUNTS: |
| 193 |
pPhrasecount = objv[ii+1]; |
| 194 |
break; |
| 195 |
} |
| 196 |
} |
| 197 |
|
| 198 |
rc = Tcl_ListObjGetElements(interp, objv[1], &nn, &apDocToken); |
| 199 |
doc.nToken = (int)nn; |
| 200 |
if( rc!=TCL_OK ) goto near_match_out; |
| 201 |
doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken)); |
| 202 |
for(ii=0; ii<doc.nToken; ii++){ |
| 203 |
doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &nn); |
| 204 |
doc.aToken[ii].n = (int)nn; |
| 205 |
} |
| 206 |
|
| 207 |
rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken); |
| 208 |
if( rc!=TCL_OK ) goto near_match_out; |
| 209 |
|
| 210 |
nPhrase = (int)(nExprToken + 1) / 2; |
| 211 |
aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase)); |
| 212 |
memset(aPhrase, 0, nPhrase * sizeof(NearPhrase)); |
| 213 |
for(ii=0; ii<nPhrase; ii++){ |
| 214 |
Tcl_Obj *pPhrase = apExprToken[ii*2]; |
| 215 |
Tcl_Obj **apToken; |
| 216 |
Tcl_Size nToken; |
| 217 |
int jj; |
| 218 |
|
| 219 |
rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken); |
| 220 |
if( rc!=TCL_OK ) goto near_match_out; |
| 221 |
if( nToken>NM_MAX_TOKEN ){ |
| 222 |
Tcl_AppendResult(interp, "Too many tokens in phrase", NULL); |
| 223 |
rc = TCL_ERROR; |
| 224 |
goto near_match_out; |
| 225 |
} |
| 226 |
for(jj=0; jj<(int)nToken; jj++){ |
| 227 |
NearToken *pT = &aPhrase[ii].aToken[jj]; |
| 228 |
pT->z = Tcl_GetStringFromObj(apToken[jj], &nn); |
| 229 |
pT->n = (int)nn; |
| 230 |
} |
| 231 |
aPhrase[ii].nToken = (int)nToken; |
| 232 |
} |
| 233 |
for(ii=1; ii<nPhrase; ii++){ |
| 234 |
Tcl_Obj *pNear = apExprToken[2*ii-1]; |
| 235 |
int nNear; |
| 236 |
rc = Tcl_GetIntFromObj(interp, pNear, &nNear); |
| 237 |
if( rc!=TCL_OK ) goto near_match_out; |
| 238 |
aPhrase[ii].nNear = nNear; |
| 239 |
} |
| 240 |
|
| 241 |
pRet = Tcl_NewObj(); |
| 242 |
Tcl_IncrRefCount(pRet); |
| 243 |
for(ii=0; ii<nPhrase; ii++){ |
| 244 |
int nOcc = nm_match_count(&doc, nPhrase, aPhrase, ii); |
| 245 |
Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(nOcc)); |
| 246 |
nTotal += nOcc; |
| 247 |
} |
| 248 |
if( pPhrasecount ){ |
| 249 |
Tcl_ObjSetVar2(interp, pPhrasecount, 0, pRet, 0); |
| 250 |
} |
| 251 |
Tcl_DecrRefCount(pRet); |
| 252 |
Tcl_SetObjResult(interp, Tcl_NewBooleanObj(nTotal>0)); |
| 253 |
|
| 254 |
near_match_out: |
| 255 |
ckfree((char *)aPhrase); |
| 256 |
ckfree((char *)doc.aToken); |
| 257 |
return rc; |
| 258 |
} |
| 259 |
|
| 260 |
/* |
| 261 |
** Tclcmd: fts3_configure_incr_load ?CHUNKSIZE THRESHOLD? |
| 262 |
** |
| 263 |
** Normally, FTS uses hard-coded values to determine the minimum doclist |
| 264 |
** size eligible for incremental loading, and the size of the chunks loaded |
| 265 |
** when a doclist is incrementally loaded. This command allows the built-in |
| 266 |
** values to be overridden for testing purposes. |
| 267 |
** |
| 268 |
** If present, the first argument is the chunksize in bytes to load doclists |
| 269 |
** in. The second argument is the minimum doclist size in bytes to use |
| 270 |
** incremental loading with. |
| 271 |
** |
| 272 |
** Whether or not the arguments are present, this command returns a list of |
| 273 |
** two integers - the initial chunksize and threshold when the command is |
| 274 |
** invoked. This can be used to restore the default behavior after running |
| 275 |
** tests. For example: |
| 276 |
** |
| 277 |
** # Override incr-load settings for testing: |
| 278 |
** set cfg [fts3_configure_incr_load $new_chunksize $new_threshold] |
| 279 |
** |
| 280 |
** .... run tests .... |
| 281 |
** |
| 282 |
** # Restore initial incr-load settings: |
| 283 |
** eval fts3_configure_incr_load $cfg |
| 284 |
*/ |
| 285 |
static int SQLITE_TCLAPI fts3_configure_incr_load_cmd( |
| 286 |
ClientData clientData, |
| 287 |
Tcl_Interp *interp, |
| 288 |
int objc, |
| 289 |
Tcl_Obj *CONST objv[] |
| 290 |
){ |
| 291 |
#ifdef SQLITE_ENABLE_FTS3 |
| 292 |
extern int test_fts3_node_chunksize; |
| 293 |
extern int test_fts3_node_chunk_threshold; |
| 294 |
Tcl_Obj *pRet; |
| 295 |
|
| 296 |
if( objc!=1 && objc!=3 ){ |
| 297 |
Tcl_WrongNumArgs(interp, 1, objv, "?CHUNKSIZE THRESHOLD?"); |
| 298 |
return TCL_ERROR; |
| 299 |
} |
| 300 |
|
| 301 |
pRet = Tcl_NewObj(); |
| 302 |
Tcl_IncrRefCount(pRet); |
| 303 |
Tcl_ListObjAppendElement( |
| 304 |
interp, pRet, Tcl_NewIntObj(test_fts3_node_chunksize)); |
| 305 |
Tcl_ListObjAppendElement( |
| 306 |
interp, pRet, Tcl_NewIntObj(test_fts3_node_chunk_threshold)); |
| 307 |
|
| 308 |
if( objc==3 ){ |
| 309 |
int iArg1; |
| 310 |
int iArg2; |
| 311 |
if( Tcl_GetIntFromObj(interp, objv[1], &iArg1) |
| 312 |
|| Tcl_GetIntFromObj(interp, objv[2], &iArg2) |
| 313 |
){ |
| 314 |
Tcl_DecrRefCount(pRet); |
| 315 |
return TCL_ERROR; |
| 316 |
} |
| 317 |
test_fts3_node_chunksize = iArg1; |
| 318 |
test_fts3_node_chunk_threshold = iArg2; |
| 319 |
} |
| 320 |
|
| 321 |
Tcl_SetObjResult(interp, pRet); |
| 322 |
Tcl_DecrRefCount(pRet); |
| 323 |
#endif |
| 324 |
UNUSED_PARAMETER(clientData); |
| 325 |
return TCL_OK; |
| 326 |
} |
| 327 |
|
| 328 |
#ifdef SQLITE_ENABLE_FTS3 |
| 329 |
/************************************************************************** |
| 330 |
** Beginning of test tokenizer code. |
| 331 |
** |
| 332 |
** For language 0, this tokenizer is similar to the default 'simple' |
| 333 |
** tokenizer. For other languages L, the following: |
| 334 |
** |
| 335 |
** * Odd numbered languages are case-sensitive. Even numbered |
| 336 |
** languages are not. |
| 337 |
** |
| 338 |
** * Language ids 100 or greater are considered an error. |
| 339 |
** |
| 340 |
** The implementation assumes that the input contains only ASCII characters |
| 341 |
** (i.e. those that may be encoded in UTF-8 using a single byte). |
| 342 |
*/ |
| 343 |
typedef struct test_tokenizer { |
| 344 |
sqlite3_tokenizer base; |
| 345 |
} test_tokenizer; |
| 346 |
|
| 347 |
typedef struct test_tokenizer_cursor { |
| 348 |
sqlite3_tokenizer_cursor base; |
| 349 |
const char *aInput; /* Input being tokenized */ |
| 350 |
int nInput; /* Size of the input in bytes */ |
| 351 |
int iInput; /* Current offset in aInput */ |
| 352 |
int iToken; /* Index of next token to be returned */ |
| 353 |
char *aBuffer; /* Buffer containing current token */ |
| 354 |
int nBuffer; /* Number of bytes allocated at pToken */ |
| 355 |
int iLangid; /* Configured language id */ |
| 356 |
} test_tokenizer_cursor; |
| 357 |
|
| 358 |
static int testTokenizerCreate( |
| 359 |
int argc, const char * const *argv, |
| 360 |
sqlite3_tokenizer **ppTokenizer |
| 361 |
){ |
| 362 |
test_tokenizer *pNew; |
| 363 |
UNUSED_PARAMETER(argc); |
| 364 |
UNUSED_PARAMETER(argv); |
| 365 |
|
| 366 |
pNew = sqlite3_malloc(sizeof(test_tokenizer)); |
| 367 |
if( !pNew ) return SQLITE_NOMEM; |
| 368 |
memset(pNew, 0, sizeof(test_tokenizer)); |
| 369 |
|
| 370 |
*ppTokenizer = (sqlite3_tokenizer *)pNew; |
| 371 |
return SQLITE_OK; |
| 372 |
} |
| 373 |
|
| 374 |
static int testTokenizerDestroy(sqlite3_tokenizer *pTokenizer){ |
| 375 |
test_tokenizer *p = (test_tokenizer *)pTokenizer; |
| 376 |
sqlite3_free(p); |
| 377 |
return SQLITE_OK; |
| 378 |
} |
| 379 |
|
| 380 |
static int testTokenizerOpen( |
| 381 |
sqlite3_tokenizer *pTokenizer, /* The tokenizer */ |
| 382 |
const char *pInput, int nBytes, /* String to be tokenized */ |
| 383 |
sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ |
| 384 |
){ |
| 385 |
int rc = SQLITE_OK; /* Return code */ |
| 386 |
test_tokenizer_cursor *pCsr; /* New cursor object */ |
| 387 |
|
| 388 |
UNUSED_PARAMETER(pTokenizer); |
| 389 |
|
| 390 |
pCsr = (test_tokenizer_cursor *)sqlite3_malloc(sizeof(test_tokenizer_cursor)); |
| 391 |
if( pCsr==0 ){ |
| 392 |
rc = SQLITE_NOMEM; |
| 393 |
}else{ |
| 394 |
memset(pCsr, 0, sizeof(test_tokenizer_cursor)); |
| 395 |
pCsr->aInput = pInput; |
| 396 |
if( nBytes<0 ){ |
| 397 |
pCsr->nInput = (int)strlen(pInput); |
| 398 |
}else{ |
| 399 |
pCsr->nInput = nBytes; |
| 400 |
} |
| 401 |
} |
| 402 |
|
| 403 |
*ppCursor = (sqlite3_tokenizer_cursor *)pCsr; |
| 404 |
return rc; |
| 405 |
} |
| 406 |
|
| 407 |
static int testTokenizerClose(sqlite3_tokenizer_cursor *pCursor){ |
| 408 |
test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; |
| 409 |
sqlite3_free(pCsr->aBuffer); |
| 410 |
sqlite3_free(pCsr); |
| 411 |
return SQLITE_OK; |
| 412 |
} |
| 413 |
|
| 414 |
static int testIsTokenChar(char c){ |
| 415 |
return (c>='a' && c<='z') || (c>='A' && c<='Z'); |
| 416 |
} |
| 417 |
static int testTolower(char c){ |
| 418 |
char ret = c; |
| 419 |
if( ret>='A' && ret<='Z') ret = ret - ('A'-'a'); |
| 420 |
return ret; |
| 421 |
} |
| 422 |
|
| 423 |
static int testTokenizerNext( |
| 424 |
sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by testTokenizerOpen */ |
| 425 |
const char **ppToken, /* OUT: *ppToken is the token text */ |
| 426 |
int *pnBytes, /* OUT: Number of bytes in token */ |
| 427 |
int *piStartOffset, /* OUT: Starting offset of token */ |
| 428 |
int *piEndOffset, /* OUT: Ending offset of token */ |
| 429 |
int *piPosition /* OUT: Position integer of token */ |
| 430 |
){ |
| 431 |
test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; |
| 432 |
int rc = SQLITE_OK; |
| 433 |
const char *p; |
| 434 |
const char *pEnd; |
| 435 |
|
| 436 |
p = &pCsr->aInput[pCsr->iInput]; |
| 437 |
pEnd = &pCsr->aInput[pCsr->nInput]; |
| 438 |
|
| 439 |
/* Skip past any white-space */ |
| 440 |
assert( p<=pEnd ); |
| 441 |
while( p<pEnd && testIsTokenChar(*p)==0 ) p++; |
| 442 |
|
| 443 |
if( p==pEnd ){ |
| 444 |
rc = SQLITE_DONE; |
| 445 |
}else{ |
| 446 |
/* Advance to the end of the token */ |
| 447 |
const char *pToken = p; |
| 448 |
sqlite3_int64 nToken; |
| 449 |
while( p<pEnd && testIsTokenChar(*p) ) p++; |
| 450 |
nToken = (sqlite3_int64)(p-pToken); |
| 451 |
|
| 452 |
/* Copy the token into the buffer */ |
| 453 |
if( nToken>pCsr->nBuffer ){ |
| 454 |
sqlite3_free(pCsr->aBuffer); |
| 455 |
pCsr->aBuffer = sqlite3_malloc64(nToken); |
| 456 |
} |
| 457 |
if( pCsr->aBuffer==0 ){ |
| 458 |
rc = SQLITE_NOMEM; |
| 459 |
}else{ |
| 460 |
int i; |
| 461 |
|
| 462 |
if( pCsr->iLangid & 0x00000001 ){ |
| 463 |
for(i=0; i<nToken; i++) pCsr->aBuffer[i] = pToken[i]; |
| 464 |
}else{ |
| 465 |
for(i=0; i<nToken; i++) pCsr->aBuffer[i] = (char)testTolower(pToken[i]); |
| 466 |
} |
| 467 |
pCsr->iToken++; |
| 468 |
pCsr->iInput = (int)(p - pCsr->aInput); |
| 469 |
|
| 470 |
*ppToken = pCsr->aBuffer; |
| 471 |
*pnBytes = (int)nToken; |
| 472 |
*piStartOffset = (int)(pToken - pCsr->aInput); |
| 473 |
*piEndOffset = (int)(p - pCsr->aInput); |
| 474 |
*piPosition = pCsr->iToken; |
| 475 |
} |
| 476 |
} |
| 477 |
|
| 478 |
return rc; |
| 479 |
} |
| 480 |
|
| 481 |
static int testTokenizerLanguage( |
| 482 |
sqlite3_tokenizer_cursor *pCursor, |
| 483 |
int iLangid |
| 484 |
){ |
| 485 |
int rc = SQLITE_OK; |
| 486 |
test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor; |
| 487 |
pCsr->iLangid = iLangid; |
| 488 |
if( pCsr->iLangid>=100 ){ |
| 489 |
rc = SQLITE_ERROR; |
| 490 |
} |
| 491 |
return rc; |
| 492 |
} |
| 493 |
#endif |
| 494 |
|
| 495 |
static int SQLITE_TCLAPI fts3_test_tokenizer_cmd( |
| 496 |
ClientData clientData, |
| 497 |
Tcl_Interp *interp, |
| 498 |
int objc, |
| 499 |
Tcl_Obj *CONST objv[] |
| 500 |
){ |
| 501 |
#ifdef SQLITE_ENABLE_FTS3 |
| 502 |
static const sqlite3_tokenizer_module testTokenizerModule = { |
| 503 |
1, |
| 504 |
testTokenizerCreate, |
| 505 |
testTokenizerDestroy, |
| 506 |
testTokenizerOpen, |
| 507 |
testTokenizerClose, |
| 508 |
testTokenizerNext, |
| 509 |
testTokenizerLanguage |
| 510 |
}; |
| 511 |
const sqlite3_tokenizer_module *pPtr = &testTokenizerModule; |
| 512 |
if( objc!=1 ){ |
| 513 |
Tcl_WrongNumArgs(interp, 1, objv, ""); |
| 514 |
return TCL_ERROR; |
| 515 |
} |
| 516 |
Tcl_SetObjResult(interp, Tcl_NewByteArrayObj( |
| 517 |
(const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *) |
| 518 |
)); |
| 519 |
#endif |
| 520 |
UNUSED_PARAMETER(clientData); |
| 521 |
return TCL_OK; |
| 522 |
} |
| 523 |
|
| 524 |
static int SQLITE_TCLAPI fts3_test_varint_cmd( |
| 525 |
ClientData clientData, |
| 526 |
Tcl_Interp *interp, |
| 527 |
int objc, |
| 528 |
Tcl_Obj *CONST objv[] |
| 529 |
){ |
| 530 |
#ifdef SQLITE_ENABLE_FTS3 |
| 531 |
char aBuf[24]; |
| 532 |
int rc; |
| 533 |
Tcl_WideInt w; |
| 534 |
sqlite3_int64 w2; |
| 535 |
int nByte, nByte2; |
| 536 |
|
| 537 |
if( objc!=2 ){ |
| 538 |
Tcl_WrongNumArgs(interp, 1, objv, "INTEGER"); |
| 539 |
return TCL_ERROR; |
| 540 |
} |
| 541 |
|
| 542 |
rc = Tcl_GetWideIntFromObj(interp, objv[1], &w); |
| 543 |
if( rc!=TCL_OK ) return rc; |
| 544 |
|
| 545 |
nByte = sqlite3Fts3PutVarint(aBuf, w); |
| 546 |
nByte2 = sqlite3Fts3GetVarint(aBuf, &w2); |
| 547 |
if( w!=w2 || nByte!=nByte2 ){ |
| 548 |
char *zErr = sqlite3_mprintf("error testing %lld", w); |
| 549 |
Tcl_ResetResult(interp); |
| 550 |
Tcl_AppendResult(interp, zErr, NULL); |
| 551 |
return TCL_ERROR; |
| 552 |
} |
| 553 |
|
| 554 |
if( w<=2147483647 && w>=0 ){ |
| 555 |
int i; |
| 556 |
nByte2 = fts3GetVarint32(aBuf, &i); |
| 557 |
if( (int)w!=i || nByte!=nByte2 ){ |
| 558 |
char *zErr = sqlite3_mprintf("error testing %lld (32-bit)", w); |
| 559 |
Tcl_ResetResult(interp); |
| 560 |
Tcl_AppendResult(interp, zErr, NULL); |
| 561 |
return TCL_ERROR; |
| 562 |
} |
| 563 |
} |
| 564 |
|
| 565 |
#endif |
| 566 |
UNUSED_PARAMETER(clientData); |
| 567 |
return TCL_OK; |
| 568 |
} |
| 569 |
|
| 570 |
/* |
| 571 |
** End of tokenizer code. |
| 572 |
**************************************************************************/ |
| 573 |
|
| 574 |
/* |
| 575 |
** sqlite3_fts3_may_be_corrupt BOOLEAN |
| 576 |
** |
| 577 |
** Set or clear the global "may-be-corrupt" flag. Return the old value. |
| 578 |
*/ |
| 579 |
static int SQLITE_TCLAPI fts3_may_be_corrupt( |
| 580 |
void * clientData, |
| 581 |
Tcl_Interp *interp, |
| 582 |
int objc, |
| 583 |
Tcl_Obj *CONST objv[] |
| 584 |
){ |
| 585 |
#ifdef SQLITE_DEBUG |
| 586 |
int bOld = sqlite3_fts3_may_be_corrupt; |
| 587 |
|
| 588 |
if( objc!=2 && objc!=1 ){ |
| 589 |
Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?"); |
| 590 |
return TCL_ERROR; |
| 591 |
} |
| 592 |
if( objc==2 ){ |
| 593 |
int bNew; |
| 594 |
if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR; |
| 595 |
sqlite3_fts3_may_be_corrupt = bNew; |
| 596 |
} |
| 597 |
|
| 598 |
Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld)); |
| 599 |
#endif |
| 600 |
return TCL_OK; |
| 601 |
} |
| 602 |
|
| 603 |
int Sqlitetestfts3_Init(Tcl_Interp *interp){ |
| 604 |
Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); |
| 605 |
Tcl_CreateObjCommand(interp, |
| 606 |
"fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0 |
| 607 |
); |
| 608 |
Tcl_CreateObjCommand( |
| 609 |
interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0 |
| 610 |
); |
| 611 |
Tcl_CreateObjCommand( |
| 612 |
interp, "fts3_test_varint", fts3_test_varint_cmd, 0, 0 |
| 613 |
); |
| 614 |
Tcl_CreateObjCommand( |
| 615 |
interp, "sqlite3_fts3_may_be_corrupt", fts3_may_be_corrupt, 0, 0 |
| 616 |
); |
| 617 |
return TCL_OK; |
| 618 |
} |
| 619 |
#endif /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */ |
| 620 |
#endif /* ifdef SQLITE_TEST */ |