| 1 |
/* |
| 2 |
** 2014 Dec 01 |
| 3 |
** |
| 4 |
** The author disclaims copyright to this source code. In place of |
| 5 |
** a legal notice, here is a blessing: |
| 6 |
** |
| 7 |
** May you do good and not evil. |
| 8 |
** May you find forgiveness for yourself and forgive others. |
| 9 |
** May you share freely, never taking more than you give. |
| 10 |
** |
| 11 |
****************************************************************************** |
| 12 |
** |
| 13 |
*/ |
| 14 |
|
| 15 |
|
| 16 |
#ifdef SQLITE_TEST |
| 17 |
#include "tclsqlite.h" |
| 18 |
|
| 19 |
#ifdef SQLITE_ENABLE_FTS5 |
| 20 |
|
| 21 |
#include "fts5.h" |
| 22 |
#include <string.h> |
| 23 |
#include <assert.h> |
| 24 |
#include <stdlib.h> |
| 25 |
|
| 26 |
#ifdef SQLITE_DEBUG |
| 27 |
extern int sqlite3_fts5_may_be_corrupt; |
| 28 |
#endif |
| 29 |
extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*); |
| 30 |
extern int sqlite3Fts5TestRegisterTok(sqlite3*, fts5_api*); |
| 31 |
|
| 32 |
/************************************************************************* |
| 33 |
** This is a copy of the first part of the SqliteDb structure in |
| 34 |
** tclsqlite.c. We need it here so that the get_sqlite_pointer routine |
| 35 |
** can extract the sqlite3* pointer from an existing Tcl SQLite |
| 36 |
** connection. |
| 37 |
*/ |
| 38 |
|
| 39 |
extern const char *sqlite3ErrName(int); |
| 40 |
|
| 41 |
struct SqliteDb { |
| 42 |
sqlite3 *db; |
| 43 |
}; |
| 44 |
|
| 45 |
/* |
| 46 |
** Decode a pointer to an sqlite3 object. |
| 47 |
*/ |
| 48 |
static int f5tDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **ppDb){ |
| 49 |
struct SqliteDb *p; |
| 50 |
Tcl_CmdInfo cmdInfo; |
| 51 |
char *z = Tcl_GetString(pObj); |
| 52 |
if( Tcl_GetCommandInfo(interp, z, &cmdInfo) ){ |
| 53 |
p = (struct SqliteDb*)cmdInfo.objClientData; |
| 54 |
*ppDb = p->db; |
| 55 |
return TCL_OK; |
| 56 |
} |
| 57 |
return TCL_ERROR; |
| 58 |
} |
| 59 |
|
| 60 |
/* End of code that accesses the SqliteDb struct. |
| 61 |
**************************************************************************/ |
| 62 |
|
| 63 |
static int f5tResultToErrorCode(const char *zRes){ |
| 64 |
struct ErrorCode { |
| 65 |
int rc; |
| 66 |
const char *zError; |
| 67 |
} aErr[] = { |
| 68 |
{ SQLITE_DONE, "SQLITE_DONE" }, |
| 69 |
{ SQLITE_ERROR, "SQLITE_ERROR" }, |
| 70 |
{ SQLITE_OK, "SQLITE_OK" }, |
| 71 |
{ SQLITE_OK, "" }, |
| 72 |
}; |
| 73 |
int i; |
| 74 |
|
| 75 |
for(i=0; i<sizeof(aErr)/sizeof(aErr[0]); i++){ |
| 76 |
if( 0==sqlite3_stricmp(zRes, aErr[i].zError) ){ |
| 77 |
return aErr[i].rc; |
| 78 |
} |
| 79 |
} |
| 80 |
|
| 81 |
return SQLITE_ERROR; |
| 82 |
} |
| 83 |
|
| 84 |
static int SQLITE_TCLAPI f5tDbAndApi( |
| 85 |
Tcl_Interp *interp, |
| 86 |
Tcl_Obj *pObj, |
| 87 |
sqlite3 **ppDb, |
| 88 |
fts5_api **ppApi |
| 89 |
){ |
| 90 |
sqlite3 *db = 0; |
| 91 |
int rc = f5tDbPointer(interp, pObj, &db); |
| 92 |
if( rc!=TCL_OK ){ |
| 93 |
return TCL_ERROR; |
| 94 |
}else{ |
| 95 |
sqlite3_stmt *pStmt = 0; |
| 96 |
fts5_api *pApi = 0; |
| 97 |
|
| 98 |
rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0); |
| 99 |
if( rc!=SQLITE_OK ){ |
| 100 |
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0); |
| 101 |
return TCL_ERROR; |
| 102 |
} |
| 103 |
sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0); |
| 104 |
sqlite3_step(pStmt); |
| 105 |
|
| 106 |
if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ |
| 107 |
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0); |
| 108 |
return TCL_ERROR; |
| 109 |
} |
| 110 |
|
| 111 |
*ppDb = db; |
| 112 |
*ppApi = pApi; |
| 113 |
} |
| 114 |
|
| 115 |
return TCL_OK; |
| 116 |
} |
| 117 |
|
| 118 |
typedef struct F5tFunction F5tFunction; |
| 119 |
struct F5tFunction { |
| 120 |
Tcl_Interp *interp; |
| 121 |
Tcl_Obj *pScript; |
| 122 |
}; |
| 123 |
|
| 124 |
typedef struct F5tApi F5tApi; |
| 125 |
struct F5tApi { |
| 126 |
const Fts5ExtensionApi *pApi; |
| 127 |
Fts5Context *pFts; |
| 128 |
}; |
| 129 |
|
| 130 |
/* |
| 131 |
** An object of this type is used with the xSetAuxdata() and xGetAuxdata() |
| 132 |
** API test wrappers. The tcl interface allows a single tcl value to be |
| 133 |
** saved using xSetAuxdata(). Instead of simply storing a pointer to the |
| 134 |
** tcl object, the code in this file wraps it in an sqlite3_malloc'd |
| 135 |
** instance of the following struct so that if the destructor is not |
| 136 |
** correctly invoked it will be reported as an SQLite memory leak. |
| 137 |
*/ |
| 138 |
typedef struct F5tAuxData F5tAuxData; |
| 139 |
struct F5tAuxData { |
| 140 |
Tcl_Obj *pObj; |
| 141 |
}; |
| 142 |
|
| 143 |
static int xTokenizeCb( |
| 144 |
void *pCtx, |
| 145 |
int tflags, |
| 146 |
const char *zToken, int nToken, |
| 147 |
int iStart, int iEnd |
| 148 |
){ |
| 149 |
F5tFunction *p = (F5tFunction*)pCtx; |
| 150 |
Tcl_Obj *pEval = Tcl_DuplicateObj(p->pScript); |
| 151 |
int rc; |
| 152 |
|
| 153 |
Tcl_IncrRefCount(pEval); |
| 154 |
Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken)); |
| 155 |
Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iStart)); |
| 156 |
Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iEnd)); |
| 157 |
|
| 158 |
rc = Tcl_EvalObjEx(p->interp, pEval, 0); |
| 159 |
Tcl_DecrRefCount(pEval); |
| 160 |
if( rc==TCL_OK ){ |
| 161 |
rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp)); |
| 162 |
} |
| 163 |
|
| 164 |
return rc; |
| 165 |
} |
| 166 |
|
| 167 |
static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []); |
| 168 |
|
| 169 |
static int xQueryPhraseCb( |
| 170 |
const Fts5ExtensionApi *pApi, |
| 171 |
Fts5Context *pFts, |
| 172 |
void *pCtx |
| 173 |
){ |
| 174 |
F5tFunction *p = (F5tFunction*)pCtx; |
| 175 |
static sqlite3_int64 iCmd = 0; |
| 176 |
Tcl_Obj *pEval; |
| 177 |
int rc; |
| 178 |
|
| 179 |
char zCmd[64]; |
| 180 |
F5tApi sApi; |
| 181 |
|
| 182 |
sApi.pApi = pApi; |
| 183 |
sApi.pFts = pFts; |
| 184 |
sprintf(zCmd, "f5t_2_%lld", iCmd++); |
| 185 |
Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0); |
| 186 |
|
| 187 |
pEval = Tcl_DuplicateObj(p->pScript); |
| 188 |
Tcl_IncrRefCount(pEval); |
| 189 |
Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1)); |
| 190 |
rc = Tcl_EvalObjEx(p->interp, pEval, 0); |
| 191 |
Tcl_DecrRefCount(pEval); |
| 192 |
Tcl_DeleteCommand(p->interp, zCmd); |
| 193 |
|
| 194 |
if( rc==TCL_OK ){ |
| 195 |
rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp)); |
| 196 |
} |
| 197 |
|
| 198 |
return rc; |
| 199 |
} |
| 200 |
|
| 201 |
static void xSetAuxdataDestructor(void *p){ |
| 202 |
F5tAuxData *pData = (F5tAuxData*)p; |
| 203 |
Tcl_DecrRefCount(pData->pObj); |
| 204 |
sqlite3_free(pData); |
| 205 |
} |
| 206 |
|
| 207 |
/* |
| 208 |
** api sub-command... |
| 209 |
** |
| 210 |
** Description... |
| 211 |
*/ |
| 212 |
static int SQLITE_TCLAPI xF5tApi( |
| 213 |
void * clientData, |
| 214 |
Tcl_Interp *interp, |
| 215 |
int objc, |
| 216 |
Tcl_Obj *CONST objv[] |
| 217 |
){ |
| 218 |
struct Sub { |
| 219 |
const char *zName; |
| 220 |
int nArg; |
| 221 |
const char *zMsg; |
| 222 |
} aSub[] = { |
| 223 |
{ "xColumnCount", 0, "" }, /* 0 */ |
| 224 |
{ "xRowCount", 0, "" }, /* 1 */ |
| 225 |
{ "xColumnTotalSize", 1, "COL" }, /* 2 */ |
| 226 |
{ "xTokenize", 2, "TEXT SCRIPT" }, /* 3 */ |
| 227 |
{ "xPhraseCount", 0, "" }, /* 4 */ |
| 228 |
{ "xPhraseSize", 1, "PHRASE" }, /* 5 */ |
| 229 |
{ "xInstCount", 0, "" }, /* 6 */ |
| 230 |
{ "xInst", 1, "IDX" }, /* 7 */ |
| 231 |
{ "xRowid", 0, "" }, /* 8 */ |
| 232 |
{ "xColumnText", 1, "COL" }, /* 9 */ |
| 233 |
{ "xColumnSize", 1, "COL" }, /* 10 */ |
| 234 |
{ "xQueryPhrase", 2, "PHRASE SCRIPT" }, /* 11 */ |
| 235 |
{ "xSetAuxdata", 1, "VALUE" }, /* 12 */ |
| 236 |
{ "xGetAuxdata", 1, "CLEAR" }, /* 13 */ |
| 237 |
{ "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */ |
| 238 |
{ "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ |
| 239 |
{ "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ |
| 240 |
{ "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */ |
| 241 |
|
| 242 |
{ "xQueryToken", 2, "IPHRASE ITERM" }, /* 18 */ |
| 243 |
{ "xInstToken", 2, "IDX ITERM" }, /* 19 */ |
| 244 |
{ "xColumnLocale", 1, "COL" }, /* 20 */ |
| 245 |
{ 0, 0, 0} |
| 246 |
}; |
| 247 |
|
| 248 |
int rc; |
| 249 |
int iSub = 0; |
| 250 |
F5tApi *p = (F5tApi*)clientData; |
| 251 |
|
| 252 |
if( objc<2 ){ |
| 253 |
Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND"); |
| 254 |
return TCL_ERROR; |
| 255 |
} |
| 256 |
|
| 257 |
rc = Tcl_GetIndexFromObjStruct( |
| 258 |
interp, objv[1], aSub, sizeof(aSub[0]), "SUB-COMMAND", 0, &iSub |
| 259 |
); |
| 260 |
if( rc!=TCL_OK ) return rc; |
| 261 |
if( aSub[iSub].nArg!=objc-2 ){ |
| 262 |
Tcl_WrongNumArgs(interp, 1, objv, aSub[iSub].zMsg); |
| 263 |
return TCL_ERROR; |
| 264 |
} |
| 265 |
|
| 266 |
#define CASE(i,str) case i: assert( strcmp(aSub[i].zName, str)==0 ); |
| 267 |
switch( iSub ){ |
| 268 |
CASE(0, "xColumnCount") { |
| 269 |
int nCol; |
| 270 |
nCol = p->pApi->xColumnCount(p->pFts); |
| 271 |
if( rc==SQLITE_OK ){ |
| 272 |
Tcl_SetObjResult(interp, Tcl_NewIntObj(nCol)); |
| 273 |
} |
| 274 |
break; |
| 275 |
} |
| 276 |
CASE(1, "xRowCount") { |
| 277 |
sqlite3_int64 nRow; |
| 278 |
rc = p->pApi->xRowCount(p->pFts, &nRow); |
| 279 |
if( rc==SQLITE_OK ){ |
| 280 |
Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nRow)); |
| 281 |
} |
| 282 |
break; |
| 283 |
} |
| 284 |
CASE(2, "xColumnTotalSize") { |
| 285 |
int iCol; |
| 286 |
sqlite3_int64 nSize; |
| 287 |
if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ) return TCL_ERROR; |
| 288 |
rc = p->pApi->xColumnTotalSize(p->pFts, iCol, &nSize); |
| 289 |
if( rc==SQLITE_OK ){ |
| 290 |
Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize)); |
| 291 |
} |
| 292 |
break; |
| 293 |
} |
| 294 |
CASE(3, "xTokenize") { |
| 295 |
Tcl_Size nText; |
| 296 |
char *zText = Tcl_GetStringFromObj(objv[2], &nText); |
| 297 |
F5tFunction ctx; |
| 298 |
ctx.interp = interp; |
| 299 |
ctx.pScript = objv[3]; |
| 300 |
rc = p->pApi->xTokenize(p->pFts, zText, (int)nText, &ctx, xTokenizeCb); |
| 301 |
if( rc==SQLITE_OK ){ |
| 302 |
Tcl_ResetResult(interp); |
| 303 |
} |
| 304 |
return rc; |
| 305 |
} |
| 306 |
CASE(4, "xPhraseCount") { |
| 307 |
int nPhrase; |
| 308 |
nPhrase = p->pApi->xPhraseCount(p->pFts); |
| 309 |
if( rc==SQLITE_OK ){ |
| 310 |
Tcl_SetObjResult(interp, Tcl_NewIntObj(nPhrase)); |
| 311 |
} |
| 312 |
break; |
| 313 |
} |
| 314 |
CASE(5, "xPhraseSize") { |
| 315 |
int iPhrase; |
| 316 |
int sz; |
| 317 |
if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){ |
| 318 |
return TCL_ERROR; |
| 319 |
} |
| 320 |
sz = p->pApi->xPhraseSize(p->pFts, iPhrase); |
| 321 |
if( rc==SQLITE_OK ){ |
| 322 |
Tcl_SetObjResult(interp, Tcl_NewIntObj(sz)); |
| 323 |
} |
| 324 |
break; |
| 325 |
} |
| 326 |
CASE(6, "xInstCount") { |
| 327 |
int nInst; |
| 328 |
rc = p->pApi->xInstCount(p->pFts, &nInst); |
| 329 |
if( rc==SQLITE_OK ){ |
| 330 |
Tcl_SetObjResult(interp, Tcl_NewIntObj(nInst)); |
| 331 |
} |
| 332 |
break; |
| 333 |
} |
| 334 |
CASE(7, "xInst") { |
| 335 |
int iIdx, ip, ic, io; |
| 336 |
if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ){ |
| 337 |
return TCL_ERROR; |
| 338 |
} |
| 339 |
rc = p->pApi->xInst(p->pFts, iIdx, &ip, &ic, &io); |
| 340 |
if( rc==SQLITE_OK ){ |
| 341 |
Tcl_Obj *pList = Tcl_NewObj(); |
| 342 |
Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ip)); |
| 343 |
Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ic)); |
| 344 |
Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(io)); |
| 345 |
Tcl_SetObjResult(interp, pList); |
| 346 |
} |
| 347 |
break; |
| 348 |
} |
| 349 |
CASE(8, "xRowid") { |
| 350 |
sqlite3_int64 iRowid = p->pApi->xRowid(p->pFts); |
| 351 |
Tcl_SetObjResult(interp, Tcl_NewWideIntObj(iRowid)); |
| 352 |
break; |
| 353 |
} |
| 354 |
CASE(9, "xColumnText") { |
| 355 |
const char *z = 0; |
| 356 |
int n = 0; |
| 357 |
int iCol; |
| 358 |
if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){ |
| 359 |
return TCL_ERROR; |
| 360 |
} |
| 361 |
rc = p->pApi->xColumnText(p->pFts, iCol, &z, &n); |
| 362 |
if( rc==SQLITE_OK ){ |
| 363 |
Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n)); |
| 364 |
} |
| 365 |
break; |
| 366 |
} |
| 367 |
CASE(10, "xColumnSize") { |
| 368 |
int n = 0; |
| 369 |
int iCol; |
| 370 |
if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){ |
| 371 |
return TCL_ERROR; |
| 372 |
} |
| 373 |
rc = p->pApi->xColumnSize(p->pFts, iCol, &n); |
| 374 |
if( rc==SQLITE_OK ){ |
| 375 |
Tcl_SetObjResult(interp, Tcl_NewIntObj(n)); |
| 376 |
} |
| 377 |
break; |
| 378 |
} |
| 379 |
CASE(11, "xQueryPhrase") { |
| 380 |
int iPhrase; |
| 381 |
F5tFunction ctx; |
| 382 |
if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){ |
| 383 |
return TCL_ERROR; |
| 384 |
} |
| 385 |
ctx.interp = interp; |
| 386 |
ctx.pScript = objv[3]; |
| 387 |
rc = p->pApi->xQueryPhrase(p->pFts, iPhrase, &ctx, xQueryPhraseCb); |
| 388 |
if( rc==SQLITE_OK ){ |
| 389 |
Tcl_ResetResult(interp); |
| 390 |
} |
| 391 |
break; |
| 392 |
} |
| 393 |
CASE(12, "xSetAuxdata") { |
| 394 |
F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc64(sizeof(F5tAuxData)); |
| 395 |
if( pData==0 ){ |
| 396 |
Tcl_AppendResult(interp, "out of memory", (char*)0); |
| 397 |
return TCL_ERROR; |
| 398 |
} |
| 399 |
pData->pObj = objv[2]; |
| 400 |
Tcl_IncrRefCount(pData->pObj); |
| 401 |
rc = p->pApi->xSetAuxdata(p->pFts, pData, xSetAuxdataDestructor); |
| 402 |
break; |
| 403 |
} |
| 404 |
CASE(13, "xGetAuxdata") { |
| 405 |
F5tAuxData *pData; |
| 406 |
int bClear; |
| 407 |
if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ){ |
| 408 |
return TCL_ERROR; |
| 409 |
} |
| 410 |
pData = (F5tAuxData*)p->pApi->xGetAuxdata(p->pFts, bClear); |
| 411 |
if( pData==0 ){ |
| 412 |
Tcl_ResetResult(interp); |
| 413 |
}else{ |
| 414 |
Tcl_SetObjResult(interp, pData->pObj); |
| 415 |
if( bClear ){ |
| 416 |
xSetAuxdataDestructor((void*)pData); |
| 417 |
} |
| 418 |
} |
| 419 |
break; |
| 420 |
} |
| 421 |
|
| 422 |
/* These two - xSetAuxdataInt and xGetAuxdataInt - are similar to the |
| 423 |
** xSetAuxdata and xGetAuxdata methods implemented above. The difference |
| 424 |
** is that they may only save an integer value as auxiliary data, and |
| 425 |
** do not specify a destructor function. */ |
| 426 |
CASE(14, "xSetAuxdataInt") { |
| 427 |
int iVal; |
| 428 |
if( Tcl_GetIntFromObj(interp, objv[2], &iVal) ) return TCL_ERROR; |
| 429 |
rc = p->pApi->xSetAuxdata(p->pFts, (void*)((char*)0 + iVal), 0); |
| 430 |
break; |
| 431 |
} |
| 432 |
CASE(15, "xGetAuxdataInt") { |
| 433 |
int iVal; |
| 434 |
int bClear; |
| 435 |
if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR; |
| 436 |
iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0); |
| 437 |
Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); |
| 438 |
break; |
| 439 |
} |
| 440 |
|
| 441 |
CASE(16, "xPhraseForeach") { |
| 442 |
int iPhrase; |
| 443 |
int iCol; |
| 444 |
int iOff; |
| 445 |
const char *zColvar; |
| 446 |
const char *zOffvar; |
| 447 |
Tcl_Obj *pScript = objv[5]; |
| 448 |
Fts5PhraseIter iter; |
| 449 |
|
| 450 |
if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; |
| 451 |
zColvar = Tcl_GetString(objv[3]); |
| 452 |
zOffvar = Tcl_GetString(objv[4]); |
| 453 |
|
| 454 |
rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff); |
| 455 |
if( rc!=SQLITE_OK ){ |
| 456 |
Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0); |
| 457 |
return TCL_ERROR; |
| 458 |
} |
| 459 |
for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){ |
| 460 |
Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); |
| 461 |
Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0); |
| 462 |
rc = Tcl_EvalObjEx(interp, pScript, 0); |
| 463 |
if( rc==TCL_CONTINUE ) rc = TCL_OK; |
| 464 |
if( rc!=TCL_OK ){ |
| 465 |
if( rc==TCL_BREAK ) rc = TCL_OK; |
| 466 |
break; |
| 467 |
} |
| 468 |
} |
| 469 |
|
| 470 |
break; |
| 471 |
} |
| 472 |
|
| 473 |
CASE(17, "xPhraseColumnForeach") { |
| 474 |
int iPhrase; |
| 475 |
int iCol; |
| 476 |
const char *zColvar; |
| 477 |
Tcl_Obj *pScript = objv[4]; |
| 478 |
Fts5PhraseIter iter; |
| 479 |
|
| 480 |
if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; |
| 481 |
zColvar = Tcl_GetString(objv[3]); |
| 482 |
|
| 483 |
rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol); |
| 484 |
if( rc!=SQLITE_OK ){ |
| 485 |
Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); |
| 486 |
return TCL_ERROR; |
| 487 |
} |
| 488 |
for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){ |
| 489 |
Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); |
| 490 |
rc = Tcl_EvalObjEx(interp, pScript, 0); |
| 491 |
if( rc==TCL_CONTINUE ) rc = TCL_OK; |
| 492 |
if( rc!=TCL_OK ){ |
| 493 |
if( rc==TCL_BREAK ) rc = TCL_OK; |
| 494 |
break; |
| 495 |
} |
| 496 |
} |
| 497 |
|
| 498 |
break; |
| 499 |
} |
| 500 |
|
| 501 |
CASE(18, "xQueryToken") { |
| 502 |
const char *pTerm = 0; |
| 503 |
int nTerm = 0; |
| 504 |
int iPhrase = 0; |
| 505 |
int iTerm = 0; |
| 506 |
|
| 507 |
if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; |
| 508 |
if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR; |
| 509 |
rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm); |
| 510 |
if( rc==SQLITE_OK ){ |
| 511 |
Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm)); |
| 512 |
} |
| 513 |
|
| 514 |
break; |
| 515 |
} |
| 516 |
|
| 517 |
CASE(19, "xInstToken") { |
| 518 |
const char *pTerm = 0; |
| 519 |
int nTerm = 0; |
| 520 |
int iIdx = 0; |
| 521 |
int iTerm = 0; |
| 522 |
|
| 523 |
if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR; |
| 524 |
if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR; |
| 525 |
rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm); |
| 526 |
if( rc==SQLITE_OK ){ |
| 527 |
Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm)); |
| 528 |
} |
| 529 |
|
| 530 |
break; |
| 531 |
} |
| 532 |
|
| 533 |
CASE(20, "xColumnLocale") { |
| 534 |
const char *z = 0; |
| 535 |
int n = 0; |
| 536 |
int iCol; |
| 537 |
if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){ |
| 538 |
return TCL_ERROR; |
| 539 |
} |
| 540 |
rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n); |
| 541 |
if( rc==SQLITE_OK && z ){ |
| 542 |
Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n)); |
| 543 |
} |
| 544 |
break; |
| 545 |
} |
| 546 |
|
| 547 |
default: |
| 548 |
assert( 0 ); |
| 549 |
break; |
| 550 |
} |
| 551 |
#undef CASE |
| 552 |
|
| 553 |
if( rc!=SQLITE_OK ){ |
| 554 |
Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); |
| 555 |
return TCL_ERROR; |
| 556 |
} |
| 557 |
|
| 558 |
return TCL_OK; |
| 559 |
} |
| 560 |
|
| 561 |
static void xF5tFunction( |
| 562 |
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ |
| 563 |
Fts5Context *pFts, /* First arg to pass to pApi functions */ |
| 564 |
sqlite3_context *pCtx, /* Context for returning result/error */ |
| 565 |
int nVal, /* Number of values in apVal[] array */ |
| 566 |
sqlite3_value **apVal /* Array of trailing arguments */ |
| 567 |
){ |
| 568 |
F5tFunction *p = (F5tFunction*)pApi->xUserData(pFts); |
| 569 |
Tcl_Obj *pEval; /* Script to evaluate */ |
| 570 |
int i; |
| 571 |
int rc; |
| 572 |
|
| 573 |
static sqlite3_int64 iCmd = 0; |
| 574 |
char zCmd[64]; |
| 575 |
F5tApi sApi; |
| 576 |
sApi.pApi = pApi; |
| 577 |
sApi.pFts = pFts; |
| 578 |
|
| 579 |
sprintf(zCmd, "f5t_%lld", iCmd++); |
| 580 |
Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0); |
| 581 |
pEval = Tcl_DuplicateObj(p->pScript); |
| 582 |
Tcl_IncrRefCount(pEval); |
| 583 |
Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1)); |
| 584 |
|
| 585 |
for(i=0; i<nVal; i++){ |
| 586 |
Tcl_Obj *pObj = 0; |
| 587 |
switch( sqlite3_value_type(apVal[i]) ){ |
| 588 |
case SQLITE_TEXT: |
| 589 |
pObj = Tcl_NewStringObj((const char*)sqlite3_value_text(apVal[i]), -1); |
| 590 |
break; |
| 591 |
case SQLITE_BLOB: |
| 592 |
pObj = Tcl_NewByteArrayObj( |
| 593 |
sqlite3_value_blob(apVal[i]), sqlite3_value_bytes(apVal[i]) |
| 594 |
); |
| 595 |
break; |
| 596 |
case SQLITE_INTEGER: |
| 597 |
pObj = Tcl_NewWideIntObj(sqlite3_value_int64(apVal[i])); |
| 598 |
break; |
| 599 |
case SQLITE_FLOAT: |
| 600 |
pObj = Tcl_NewDoubleObj(sqlite3_value_double(apVal[i])); |
| 601 |
break; |
| 602 |
default: |
| 603 |
pObj = Tcl_NewObj(); |
| 604 |
break; |
| 605 |
} |
| 606 |
Tcl_ListObjAppendElement(p->interp, pEval, pObj); |
| 607 |
} |
| 608 |
|
| 609 |
rc = Tcl_EvalObjEx(p->interp, pEval, TCL_GLOBAL_ONLY); |
| 610 |
Tcl_DecrRefCount(pEval); |
| 611 |
Tcl_DeleteCommand(p->interp, zCmd); |
| 612 |
|
| 613 |
if( rc!=TCL_OK ){ |
| 614 |
sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1); |
| 615 |
}else{ |
| 616 |
Tcl_Obj *pVar = Tcl_GetObjResult(p->interp); |
| 617 |
const char *zType = (pVar->typePtr ? pVar->typePtr->name : ""); |
| 618 |
char c = zType[0]; |
| 619 |
if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){ |
| 620 |
/* Only return a BLOB type if the Tcl variable is a bytearray and |
| 621 |
** has no string representation. */ |
| 622 |
Tcl_Size nn; |
| 623 |
unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &nn); |
| 624 |
sqlite3_result_blob(pCtx, data, (int)nn, SQLITE_TRANSIENT); |
| 625 |
}else if( c=='b' && strcmp(zType,"boolean")==0 ){ |
| 626 |
int n; |
| 627 |
Tcl_GetIntFromObj(0, pVar, &n); |
| 628 |
sqlite3_result_int(pCtx, n); |
| 629 |
}else if( c=='d' && strcmp(zType,"double")==0 ){ |
| 630 |
double r; |
| 631 |
Tcl_GetDoubleFromObj(0, pVar, &r); |
| 632 |
sqlite3_result_double(pCtx, r); |
| 633 |
}else if( (c=='w' && strcmp(zType,"wideInt")==0) || |
| 634 |
(c=='i' && strcmp(zType,"int")==0) ){ |
| 635 |
Tcl_WideInt v; |
| 636 |
Tcl_GetWideIntFromObj(0, pVar, &v); |
| 637 |
sqlite3_result_int64(pCtx, v); |
| 638 |
}else{ |
| 639 |
Tcl_Size nn; |
| 640 |
unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &nn); |
| 641 |
sqlite3_result_text(pCtx, (char*)data, (int)nn, SQLITE_TRANSIENT); |
| 642 |
} |
| 643 |
} |
| 644 |
} |
| 645 |
|
| 646 |
static void xF5tDestroy(void *pCtx){ |
| 647 |
F5tFunction *p = (F5tFunction*)pCtx; |
| 648 |
Tcl_DecrRefCount(p->pScript); |
| 649 |
ckfree((char *)p); |
| 650 |
} |
| 651 |
|
| 652 |
/* |
| 653 |
** sqlite3_fts5_create_function DB NAME SCRIPT |
| 654 |
** |
| 655 |
** Description... |
| 656 |
*/ |
| 657 |
static int SQLITE_TCLAPI f5tCreateFunction( |
| 658 |
void * clientData, |
| 659 |
Tcl_Interp *interp, |
| 660 |
int objc, |
| 661 |
Tcl_Obj *CONST objv[] |
| 662 |
){ |
| 663 |
char *zName; |
| 664 |
Tcl_Obj *pScript; |
| 665 |
sqlite3 *db = 0; |
| 666 |
fts5_api *pApi = 0; |
| 667 |
F5tFunction *pCtx = 0; |
| 668 |
int rc; |
| 669 |
|
| 670 |
if( objc!=4 ){ |
| 671 |
Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT"); |
| 672 |
return TCL_ERROR; |
| 673 |
} |
| 674 |
if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR; |
| 675 |
|
| 676 |
zName = Tcl_GetString(objv[2]); |
| 677 |
pScript = objv[3]; |
| 678 |
pCtx = (F5tFunction*)ckalloc(sizeof(F5tFunction)); |
| 679 |
pCtx->interp = interp; |
| 680 |
pCtx->pScript = pScript; |
| 681 |
Tcl_IncrRefCount(pScript); |
| 682 |
|
| 683 |
rc = pApi->xCreateFunction( |
| 684 |
pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy |
| 685 |
); |
| 686 |
if( rc!=SQLITE_OK ){ |
| 687 |
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0); |
| 688 |
return TCL_ERROR; |
| 689 |
} |
| 690 |
|
| 691 |
return TCL_OK; |
| 692 |
} |
| 693 |
|
| 694 |
typedef struct F5tTokenizeCtx F5tTokenizeCtx; |
| 695 |
struct F5tTokenizeCtx { |
| 696 |
Tcl_Obj *pRet; |
| 697 |
int bSubst; |
| 698 |
const char *zInput; |
| 699 |
}; |
| 700 |
|
| 701 |
static int xTokenizeCb2( |
| 702 |
void *pCtx, |
| 703 |
int tflags, |
| 704 |
const char *zToken, int nToken, |
| 705 |
int iStart, int iEnd |
| 706 |
){ |
| 707 |
F5tTokenizeCtx *p = (F5tTokenizeCtx*)pCtx; |
| 708 |
if( p->bSubst ){ |
| 709 |
Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken)); |
| 710 |
Tcl_ListObjAppendElement( |
| 711 |
0, p->pRet, Tcl_NewStringObj(&p->zInput[iStart], iEnd-iStart) |
| 712 |
); |
| 713 |
}else{ |
| 714 |
Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken)); |
| 715 |
Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewIntObj(iStart)); |
| 716 |
Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewIntObj(iEnd)); |
| 717 |
} |
| 718 |
return SQLITE_OK; |
| 719 |
} |
| 720 |
|
| 721 |
|
| 722 |
/* |
| 723 |
** sqlite3_fts5_tokenize DB TOKENIZER TEXT |
| 724 |
** |
| 725 |
** Description... |
| 726 |
*/ |
| 727 |
static int SQLITE_TCLAPI f5tTokenize( |
| 728 |
void * clientData, |
| 729 |
Tcl_Interp *interp, |
| 730 |
int objc, |
| 731 |
Tcl_Obj *CONST objv[] |
| 732 |
){ |
| 733 |
char *pCopy = 0; |
| 734 |
char *zText = 0; |
| 735 |
Tcl_Size nText = 0; |
| 736 |
sqlite3 *db = 0; |
| 737 |
fts5_api *pApi = 0; |
| 738 |
Fts5Tokenizer *pTok = 0; |
| 739 |
fts5_tokenizer tokenizer; |
| 740 |
Tcl_Obj *pRet = 0; |
| 741 |
void *pUserdata; |
| 742 |
int rc; |
| 743 |
|
| 744 |
Tcl_Size nArg; |
| 745 |
const char **azArg; |
| 746 |
F5tTokenizeCtx ctx; |
| 747 |
|
| 748 |
if( objc!=4 && objc!=5 ){ |
| 749 |
Tcl_WrongNumArgs(interp, 1, objv, "?-subst? DB NAME TEXT"); |
| 750 |
return TCL_ERROR; |
| 751 |
} |
| 752 |
if( objc==5 ){ |
| 753 |
char *zOpt = Tcl_GetString(objv[1]); |
| 754 |
if( strcmp("-subst", zOpt) ){ |
| 755 |
Tcl_AppendResult(interp, "unrecognized option: ", zOpt, (char*)0); |
| 756 |
return TCL_ERROR; |
| 757 |
} |
| 758 |
} |
| 759 |
if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ) return TCL_ERROR; |
| 760 |
if( Tcl_SplitList(interp, Tcl_GetString(objv[objc-2]), &nArg, &azArg) ){ |
| 761 |
return TCL_ERROR; |
| 762 |
} |
| 763 |
if( nArg==0 ){ |
| 764 |
Tcl_AppendResult(interp, "no such tokenizer: ", (char*)0); |
| 765 |
Tcl_Free((void*)azArg); |
| 766 |
return TCL_ERROR; |
| 767 |
} |
| 768 |
zText = Tcl_GetStringFromObj(objv[objc-1], &nText); |
| 769 |
|
| 770 |
rc = pApi->xFindTokenizer(pApi, azArg[0], &pUserdata, &tokenizer); |
| 771 |
if( rc!=SQLITE_OK ){ |
| 772 |
Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], (char*)0); |
| 773 |
return TCL_ERROR; |
| 774 |
} |
| 775 |
|
| 776 |
rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok); |
| 777 |
if( rc!=SQLITE_OK ){ |
| 778 |
Tcl_AppendResult(interp, "error in tokenizer.xCreate()", (char*)0); |
| 779 |
return TCL_ERROR; |
| 780 |
} |
| 781 |
|
| 782 |
if( nText>0 ){ |
| 783 |
pCopy = sqlite3_malloc64(nText); |
| 784 |
if( pCopy==0 ){ |
| 785 |
tokenizer.xDelete(pTok); |
| 786 |
Tcl_AppendResult(interp, "error in sqlite3_malloc()", (char*)0); |
| 787 |
return TCL_ERROR; |
| 788 |
}else{ |
| 789 |
memcpy(pCopy, zText, nText); |
| 790 |
} |
| 791 |
} |
| 792 |
|
| 793 |
pRet = Tcl_NewObj(); |
| 794 |
Tcl_IncrRefCount(pRet); |
| 795 |
ctx.bSubst = (objc==5); |
| 796 |
ctx.pRet = pRet; |
| 797 |
ctx.zInput = pCopy; |
| 798 |
rc = tokenizer.xTokenize( |
| 799 |
pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, pCopy,(int)nText, xTokenizeCb2 |
| 800 |
); |
| 801 |
tokenizer.xDelete(pTok); |
| 802 |
sqlite3_free(pCopy); |
| 803 |
if( rc!=SQLITE_OK ){ |
| 804 |
Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", (char*)0); |
| 805 |
Tcl_DecrRefCount(pRet); |
| 806 |
return TCL_ERROR; |
| 807 |
} |
| 808 |
|
| 809 |
Tcl_Free((void*)azArg); |
| 810 |
Tcl_SetObjResult(interp, pRet); |
| 811 |
Tcl_DecrRefCount(pRet); |
| 812 |
return TCL_OK; |
| 813 |
} |
| 814 |
|
| 815 |
/************************************************************************* |
| 816 |
** Start of tokenizer wrapper. |
| 817 |
*/ |
| 818 |
|
| 819 |
typedef struct F5tTokenizerContext F5tTokenizerContext; |
| 820 |
typedef struct F5tTokenizerCb F5tTokenizerCb; |
| 821 |
typedef struct F5tTokenizerModule F5tTokenizerModule; |
| 822 |
typedef struct F5tTokenizerInstance F5tTokenizerInstance; |
| 823 |
|
| 824 |
struct F5tTokenizerContext { |
| 825 |
void *pCtx; |
| 826 |
int (*xToken)(void*, int, const char*, int, int, int); |
| 827 |
F5tTokenizerInstance *pInst; |
| 828 |
}; |
| 829 |
|
| 830 |
struct F5tTokenizerModule { |
| 831 |
Tcl_Interp *interp; |
| 832 |
Tcl_Obj *pScript; |
| 833 |
void *pParentCtx; |
| 834 |
fts5_tokenizer_v2 parent_v2; |
| 835 |
fts5_tokenizer parent; |
| 836 |
F5tTokenizerContext *pContext; |
| 837 |
}; |
| 838 |
|
| 839 |
/* |
| 840 |
** zLocale: |
| 841 |
** Within a call to xTokenize_v2(), pLocale/nLocale store the locale |
| 842 |
** passed to the call by fts5. This can be retrieved by a Tcl tokenize |
| 843 |
** script using [sqlite3_fts5_locale]. |
| 844 |
*/ |
| 845 |
struct F5tTokenizerInstance { |
| 846 |
Tcl_Interp *interp; |
| 847 |
Tcl_Obj *pScript; |
| 848 |
F5tTokenizerModule *pModule; |
| 849 |
Fts5Tokenizer *pParent; |
| 850 |
F5tTokenizerContext *pContext; |
| 851 |
const char *pLocale; |
| 852 |
int nLocale; |
| 853 |
}; |
| 854 |
|
| 855 |
static int f5tTokenizerCreate( |
| 856 |
void *pCtx, |
| 857 |
const char **azArg, |
| 858 |
int nArg, |
| 859 |
Fts5Tokenizer **ppOut |
| 860 |
){ |
| 861 |
Fts5Tokenizer *pParent = 0; |
| 862 |
F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx; |
| 863 |
Tcl_Obj *pEval; |
| 864 |
int rc = TCL_OK; |
| 865 |
int i; |
| 866 |
|
| 867 |
assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 ); |
| 868 |
if( pMod->parent_v2.xCreate ){ |
| 869 |
rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent); |
| 870 |
} |
| 871 |
if( pMod->parent.xCreate ){ |
| 872 |
rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent); |
| 873 |
} |
| 874 |
|
| 875 |
pEval = Tcl_DuplicateObj(pMod->pScript); |
| 876 |
Tcl_IncrRefCount(pEval); |
| 877 |
for(i=0; rc==TCL_OK && i<nArg; i++){ |
| 878 |
Tcl_Obj *pObj = Tcl_NewStringObj(azArg[i], -1); |
| 879 |
rc = Tcl_ListObjAppendElement(pMod->interp, pEval, pObj); |
| 880 |
} |
| 881 |
|
| 882 |
if( rc==TCL_OK ){ |
| 883 |
rc = Tcl_EvalObjEx(pMod->interp, pEval, TCL_GLOBAL_ONLY); |
| 884 |
} |
| 885 |
Tcl_DecrRefCount(pEval); |
| 886 |
|
| 887 |
if( rc==TCL_OK ){ |
| 888 |
F5tTokenizerInstance *pInst; |
| 889 |
pInst = (F5tTokenizerInstance*)ckalloc(sizeof(F5tTokenizerInstance)); |
| 890 |
memset(pInst, 0, sizeof(F5tTokenizerInstance)); |
| 891 |
pInst->interp = pMod->interp; |
| 892 |
pInst->pScript = Tcl_GetObjResult(pMod->interp); |
| 893 |
pInst->pContext = pMod->pContext; |
| 894 |
pInst->pParent = pParent; |
| 895 |
pInst->pModule = pMod; |
| 896 |
Tcl_IncrRefCount(pInst->pScript); |
| 897 |
*ppOut = (Fts5Tokenizer*)pInst; |
| 898 |
} |
| 899 |
|
| 900 |
return rc; |
| 901 |
} |
| 902 |
|
| 903 |
|
| 904 |
static void f5tTokenizerDelete(Fts5Tokenizer *p){ |
| 905 |
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; |
| 906 |
if( pInst ){ |
| 907 |
if( pInst->pParent ){ |
| 908 |
if( pInst->pModule->parent_v2.xDelete ){ |
| 909 |
pInst->pModule->parent_v2.xDelete(pInst->pParent); |
| 910 |
}else{ |
| 911 |
pInst->pModule->parent.xDelete(pInst->pParent); |
| 912 |
} |
| 913 |
} |
| 914 |
Tcl_DecrRefCount(pInst->pScript); |
| 915 |
ckfree((char *)pInst); |
| 916 |
} |
| 917 |
} |
| 918 |
|
| 919 |
|
| 920 |
static int f5tTokenizerReallyTokenize( |
| 921 |
Fts5Tokenizer *p, |
| 922 |
void *pCtx, |
| 923 |
int flags, |
| 924 |
const char *pText, int nText, |
| 925 |
int (*xToken)(void*, int, const char*, int, int, int) |
| 926 |
){ |
| 927 |
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; |
| 928 |
F5tTokenizerInstance *pOldInst = 0; |
| 929 |
void *pOldCtx; |
| 930 |
int (*xOldToken)(void*, int, const char*, int, int, int); |
| 931 |
Tcl_Obj *pEval; |
| 932 |
int rc; |
| 933 |
const char *zFlags; |
| 934 |
|
| 935 |
pOldCtx = pInst->pContext->pCtx; |
| 936 |
xOldToken = pInst->pContext->xToken; |
| 937 |
pOldInst = pInst->pContext->pInst; |
| 938 |
|
| 939 |
pInst->pContext->pCtx = pCtx; |
| 940 |
pInst->pContext->xToken = xToken; |
| 941 |
pInst->pContext->pInst = pInst; |
| 942 |
|
| 943 |
assert( |
| 944 |
flags==FTS5_TOKENIZE_DOCUMENT |
| 945 |
|| flags==FTS5_TOKENIZE_AUX |
| 946 |
|| flags==FTS5_TOKENIZE_QUERY |
| 947 |
|| flags==(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) |
| 948 |
); |
| 949 |
pEval = Tcl_DuplicateObj(pInst->pScript); |
| 950 |
Tcl_IncrRefCount(pEval); |
| 951 |
switch( flags ){ |
| 952 |
case FTS5_TOKENIZE_DOCUMENT: |
| 953 |
zFlags = "document"; |
| 954 |
break; |
| 955 |
case FTS5_TOKENIZE_AUX: |
| 956 |
zFlags = "aux"; |
| 957 |
break; |
| 958 |
case FTS5_TOKENIZE_QUERY: |
| 959 |
zFlags = "query"; |
| 960 |
break; |
| 961 |
case (FTS5_TOKENIZE_PREFIX | FTS5_TOKENIZE_QUERY): |
| 962 |
zFlags = "prefixquery"; |
| 963 |
break; |
| 964 |
default: |
| 965 |
assert( 0 ); |
| 966 |
zFlags = "invalid"; |
| 967 |
break; |
| 968 |
} |
| 969 |
|
| 970 |
Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(zFlags, -1)); |
| 971 |
Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(pText,nText)); |
| 972 |
rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY); |
| 973 |
Tcl_DecrRefCount(pEval); |
| 974 |
|
| 975 |
pInst->pContext->pCtx = pOldCtx; |
| 976 |
pInst->pContext->xToken = xOldToken; |
| 977 |
pInst->pContext->pInst = pOldInst; |
| 978 |
return rc; |
| 979 |
} |
| 980 |
|
| 981 |
typedef struct CallbackCtx CallbackCtx; |
| 982 |
struct CallbackCtx { |
| 983 |
Fts5Tokenizer *p; |
| 984 |
void *pCtx; |
| 985 |
int flags; |
| 986 |
int (*xToken)(void*, int, const char*, int, int, int); |
| 987 |
}; |
| 988 |
|
| 989 |
static int f5tTokenizeCallback( |
| 990 |
void *pCtx, |
| 991 |
int tflags, |
| 992 |
const char *z, int n, |
| 993 |
int iStart, int iEnd |
| 994 |
){ |
| 995 |
CallbackCtx *p = (CallbackCtx*)pCtx; |
| 996 |
return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken); |
| 997 |
} |
| 998 |
|
| 999 |
static int f5tTokenizerTokenize_v2( |
| 1000 |
Fts5Tokenizer *p, |
| 1001 |
void *pCtx, |
| 1002 |
int flags, |
| 1003 |
const char *pText, int nText, |
| 1004 |
const char *pLoc, int nLoc, |
| 1005 |
int (*xToken)(void*, int, const char*, int, int, int) |
| 1006 |
){ |
| 1007 |
int rc = SQLITE_OK; |
| 1008 |
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p; |
| 1009 |
|
| 1010 |
pInst->pLocale = pLoc; |
| 1011 |
pInst->nLocale = nLoc; |
| 1012 |
|
| 1013 |
if( pInst->pParent ){ |
| 1014 |
CallbackCtx ctx; |
| 1015 |
ctx.p = p; |
| 1016 |
ctx.pCtx = pCtx; |
| 1017 |
ctx.flags = flags; |
| 1018 |
ctx.xToken = xToken; |
| 1019 |
if( pInst->pModule->parent_v2.xTokenize ){ |
| 1020 |
rc = pInst->pModule->parent_v2.xTokenize( |
| 1021 |
pInst->pParent, (void*)&ctx, flags, pText, nText, |
| 1022 |
pLoc, nLoc, f5tTokenizeCallback |
| 1023 |
); |
| 1024 |
}else{ |
| 1025 |
rc = pInst->pModule->parent.xTokenize( |
| 1026 |
pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback |
| 1027 |
); |
| 1028 |
} |
| 1029 |
}else{ |
| 1030 |
rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken); |
| 1031 |
} |
| 1032 |
|
| 1033 |
pInst->pLocale = 0; |
| 1034 |
pInst->nLocale = 0; |
| 1035 |
return rc; |
| 1036 |
} |
| 1037 |
static int f5tTokenizerTokenize( |
| 1038 |
Fts5Tokenizer *p, |
| 1039 |
void *pCtx, |
| 1040 |
int flags, |
| 1041 |
const char *pText, int nText, |
| 1042 |
int (*xToken)(void*, int, const char*, int, int, int) |
| 1043 |
){ |
| 1044 |
return f5tTokenizerTokenize_v2(p, pCtx, flags, pText, nText, 0, 0, xToken); |
| 1045 |
} |
| 1046 |
|
| 1047 |
/* |
| 1048 |
** sqlite3_fts5_locale |
| 1049 |
*/ |
| 1050 |
static int SQLITE_TCLAPI f5tTokenizerLocale( |
| 1051 |
void * clientData, |
| 1052 |
Tcl_Interp *interp, |
| 1053 |
int objc, |
| 1054 |
Tcl_Obj *CONST objv[] |
| 1055 |
){ |
| 1056 |
F5tTokenizerContext *p = (F5tTokenizerContext*)clientData; |
| 1057 |
|
| 1058 |
if( objc!=1 ){ |
| 1059 |
Tcl_WrongNumArgs(interp, 1, objv, ""); |
| 1060 |
return TCL_ERROR; |
| 1061 |
} |
| 1062 |
|
| 1063 |
if( p->xToken==0 ){ |
| 1064 |
Tcl_AppendResult(interp, |
| 1065 |
"sqlite3_fts5_locale may only be used by tokenizer callback", (char*)0 |
| 1066 |
); |
| 1067 |
return TCL_ERROR; |
| 1068 |
} |
| 1069 |
|
| 1070 |
Tcl_SetObjResult(interp, |
| 1071 |
Tcl_NewStringObj(p->pInst->pLocale, p->pInst->nLocale) |
| 1072 |
); |
| 1073 |
return TCL_OK; |
| 1074 |
} |
| 1075 |
|
| 1076 |
/* |
| 1077 |
** sqlite3_fts5_token ?-colocated? TEXT START END |
| 1078 |
*/ |
| 1079 |
static int SQLITE_TCLAPI f5tTokenizerReturn( |
| 1080 |
void * clientData, |
| 1081 |
Tcl_Interp *interp, |
| 1082 |
int objc, |
| 1083 |
Tcl_Obj *CONST objv[] |
| 1084 |
){ |
| 1085 |
F5tTokenizerContext *p = (F5tTokenizerContext*)clientData; |
| 1086 |
int iStart; |
| 1087 |
int iEnd; |
| 1088 |
Tcl_Size nToken; |
| 1089 |
int tflags = 0; |
| 1090 |
char *zToken; |
| 1091 |
int rc; |
| 1092 |
|
| 1093 |
if( objc==5 ){ |
| 1094 |
Tcl_Size nArg; |
| 1095 |
char *zArg = Tcl_GetStringFromObj(objv[1], &nArg); |
| 1096 |
if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){ |
| 1097 |
tflags |= FTS5_TOKEN_COLOCATED; |
| 1098 |
}else{ |
| 1099 |
goto usage; |
| 1100 |
} |
| 1101 |
}else if( objc!=4 ){ |
| 1102 |
goto usage; |
| 1103 |
} |
| 1104 |
|
| 1105 |
zToken = Tcl_GetStringFromObj(objv[objc-3], &nToken); |
| 1106 |
if( Tcl_GetIntFromObj(interp, objv[objc-2], &iStart) |
| 1107 |
|| Tcl_GetIntFromObj(interp, objv[objc-1], &iEnd) |
| 1108 |
){ |
| 1109 |
return TCL_ERROR; |
| 1110 |
} |
| 1111 |
|
| 1112 |
if( p->xToken==0 ){ |
| 1113 |
Tcl_AppendResult(interp, |
| 1114 |
"sqlite3_fts5_token may only be used by tokenizer callback", (char*)0 |
| 1115 |
); |
| 1116 |
return TCL_ERROR; |
| 1117 |
} |
| 1118 |
|
| 1119 |
rc = p->xToken(p->pCtx, tflags, zToken, (int)nToken, iStart, iEnd); |
| 1120 |
Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); |
| 1121 |
return rc==SQLITE_OK ? TCL_OK : TCL_ERROR; |
| 1122 |
|
| 1123 |
usage: |
| 1124 |
Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END"); |
| 1125 |
return TCL_ERROR; |
| 1126 |
} |
| 1127 |
|
| 1128 |
static void f5tDelTokenizer(void *pCtx){ |
| 1129 |
F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx; |
| 1130 |
Tcl_DecrRefCount(pMod->pScript); |
| 1131 |
ckfree((char *)pMod); |
| 1132 |
} |
| 1133 |
|
| 1134 |
/* |
| 1135 |
** sqlite3_fts5_create_tokenizer DB NAME SCRIPT |
| 1136 |
** |
| 1137 |
** Register a tokenizer named NAME implemented by script SCRIPT. When |
| 1138 |
** a tokenizer instance is created (fts5_tokenizer.xCreate), any tokenizer |
| 1139 |
** arguments are appended to SCRIPT and the result executed. |
| 1140 |
** |
| 1141 |
** The value returned by (SCRIPT + args) is itself a tcl script. This |
| 1142 |
** script - call it SCRIPT2 - is executed to tokenize text using the |
| 1143 |
** tokenizer instance "returned" by SCRIPT. Specifically, to tokenize |
| 1144 |
** text SCRIPT2 is invoked with a single argument appended to it - the |
| 1145 |
** text to tokenize. |
| 1146 |
** |
| 1147 |
** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each |
| 1148 |
** token within the tokenized text. |
| 1149 |
*/ |
| 1150 |
static int SQLITE_TCLAPI f5tCreateTokenizer( |
| 1151 |
ClientData clientData, |
| 1152 |
Tcl_Interp *interp, |
| 1153 |
int objc, |
| 1154 |
Tcl_Obj *CONST objv[] |
| 1155 |
){ |
| 1156 |
F5tTokenizerContext *pContext = (F5tTokenizerContext*)clientData; |
| 1157 |
sqlite3 *db; |
| 1158 |
fts5_api *pApi; |
| 1159 |
char *zName; |
| 1160 |
Tcl_Obj *pScript; |
| 1161 |
F5tTokenizerModule *pMod; |
| 1162 |
int rc = SQLITE_OK; |
| 1163 |
int bV2 = 0; /* True to use _v2 API */ |
| 1164 |
int iVersion = 2; /* Value for _v2.iVersion */ |
| 1165 |
const char *zParent = 0; /* Name of parent tokenizer, if any */ |
| 1166 |
int ii = 0; |
| 1167 |
|
| 1168 |
if( objc<4 ){ |
| 1169 |
Tcl_WrongNumArgs(interp, 1, objv, "?OPTIONS? DB NAME SCRIPT"); |
| 1170 |
return TCL_ERROR; |
| 1171 |
} |
| 1172 |
|
| 1173 |
/* Parse any options. Set stack variables bV2 and zParent. */ |
| 1174 |
for(ii=1; ii<objc-3; ii++){ |
| 1175 |
int iOpt = 0; |
| 1176 |
const char *azOpt[] = { "-v2", "-parent", "-version", 0 }; |
| 1177 |
if( Tcl_GetIndexFromObj(interp, objv[ii], azOpt, "OPTION", 0, &iOpt) ){ |
| 1178 |
return TCL_ERROR; |
| 1179 |
} |
| 1180 |
switch( iOpt ){ |
| 1181 |
case 0: /* -v2 */ { |
| 1182 |
bV2 = 1; |
| 1183 |
break; |
| 1184 |
} |
| 1185 |
case 1: /* -parent */ { |
| 1186 |
ii++; |
| 1187 |
if( ii==objc-3 ){ |
| 1188 |
Tcl_AppendResult( |
| 1189 |
interp, "option requires an argument: -parent", (char*)0 |
| 1190 |
); |
| 1191 |
return TCL_ERROR; |
| 1192 |
} |
| 1193 |
zParent = Tcl_GetString(objv[ii]); |
| 1194 |
break; |
| 1195 |
} |
| 1196 |
case 2: /* -version */ { |
| 1197 |
ii++; |
| 1198 |
if( ii==objc-3 ){ |
| 1199 |
Tcl_AppendResult( |
| 1200 |
interp, "option requires an argument: -version", (char*)0 |
| 1201 |
); |
| 1202 |
return TCL_ERROR; |
| 1203 |
} |
| 1204 |
if( Tcl_GetIntFromObj(interp, objv[ii], &iVersion) ){ |
| 1205 |
return TCL_ERROR; |
| 1206 |
} |
| 1207 |
break; |
| 1208 |
} |
| 1209 |
default: |
| 1210 |
assert( 0 ); |
| 1211 |
break; |
| 1212 |
} |
| 1213 |
} |
| 1214 |
|
| 1215 |
if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ){ |
| 1216 |
return TCL_ERROR; |
| 1217 |
} |
| 1218 |
zName = Tcl_GetString(objv[objc-2]); |
| 1219 |
pScript = objv[objc-1]; |
| 1220 |
|
| 1221 |
pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule)); |
| 1222 |
memset(pMod, 0, sizeof(F5tTokenizerModule)); |
| 1223 |
pMod->interp = interp; |
| 1224 |
pMod->pScript = pScript; |
| 1225 |
Tcl_IncrRefCount(pScript); |
| 1226 |
pMod->pContext = pContext; |
| 1227 |
if( zParent ){ |
| 1228 |
if( bV2 ){ |
| 1229 |
fts5_tokenizer_v2 *pParent = 0; |
| 1230 |
rc = pApi->xFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent); |
| 1231 |
if( rc==SQLITE_OK ){ |
| 1232 |
memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2)); |
| 1233 |
pMod->parent_v2.xDelete(0); |
| 1234 |
} |
| 1235 |
}else{ |
| 1236 |
rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent); |
| 1237 |
if( rc==SQLITE_OK ){ |
| 1238 |
pMod->parent.xDelete(0); |
| 1239 |
} |
| 1240 |
} |
| 1241 |
} |
| 1242 |
|
| 1243 |
if( rc==SQLITE_OK ){ |
| 1244 |
void *pModCtx = (void*)pMod; |
| 1245 |
if( bV2==0 ){ |
| 1246 |
fts5_tokenizer t; |
| 1247 |
t.xCreate = f5tTokenizerCreate; |
| 1248 |
t.xTokenize = f5tTokenizerTokenize; |
| 1249 |
t.xDelete = f5tTokenizerDelete; |
| 1250 |
rc = pApi->xCreateTokenizer(pApi, zName, pModCtx, &t, f5tDelTokenizer); |
| 1251 |
}else{ |
| 1252 |
fts5_tokenizer_v2 t2; |
| 1253 |
memset(&t2, 0, sizeof(t2)); |
| 1254 |
t2.iVersion = iVersion; |
| 1255 |
t2.xCreate = f5tTokenizerCreate; |
| 1256 |
t2.xTokenize = f5tTokenizerTokenize_v2; |
| 1257 |
t2.xDelete = f5tTokenizerDelete; |
| 1258 |
rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer); |
| 1259 |
} |
| 1260 |
} |
| 1261 |
|
| 1262 |
if( rc!=SQLITE_OK ){ |
| 1263 |
Tcl_AppendResult(interp, ( |
| 1264 |
bV2 ? "error in fts5_api.xCreateTokenizer_v2()" |
| 1265 |
: "error in fts5_api.xCreateTokenizer()" |
| 1266 |
), (char*)0); |
| 1267 |
return TCL_ERROR; |
| 1268 |
} |
| 1269 |
|
| 1270 |
return TCL_OK; |
| 1271 |
} |
| 1272 |
|
| 1273 |
static void SQLITE_TCLAPI xF5tFree(ClientData clientData){ |
| 1274 |
ckfree(clientData); |
| 1275 |
} |
| 1276 |
|
| 1277 |
/* |
| 1278 |
** sqlite3_fts5_may_be_corrupt BOOLEAN |
| 1279 |
** |
| 1280 |
** Set or clear the global "may-be-corrupt" flag. Return the old value. |
| 1281 |
*/ |
| 1282 |
static int SQLITE_TCLAPI f5tMayBeCorrupt( |
| 1283 |
void * clientData, |
| 1284 |
Tcl_Interp *interp, |
| 1285 |
int objc, |
| 1286 |
Tcl_Obj *CONST objv[] |
| 1287 |
){ |
| 1288 |
#ifdef SQLITE_DEBUG |
| 1289 |
int bOld = sqlite3_fts5_may_be_corrupt; |
| 1290 |
|
| 1291 |
if( objc!=2 && objc!=1 ){ |
| 1292 |
Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?"); |
| 1293 |
return TCL_ERROR; |
| 1294 |
} |
| 1295 |
if( objc==2 ){ |
| 1296 |
int bNew; |
| 1297 |
if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR; |
| 1298 |
sqlite3_fts5_may_be_corrupt = bNew; |
| 1299 |
} |
| 1300 |
|
| 1301 |
Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld)); |
| 1302 |
#endif |
| 1303 |
return TCL_OK; |
| 1304 |
} |
| 1305 |
|
| 1306 |
|
| 1307 |
static unsigned int f5t_fts5HashKey(int nSlot, const char *p, int n){ |
| 1308 |
int i; |
| 1309 |
unsigned int h = 13; |
| 1310 |
for(i=n-1; i>=0; i--){ |
| 1311 |
h = (h << 3) ^ h ^ p[i]; |
| 1312 |
} |
| 1313 |
return (h % nSlot); |
| 1314 |
} |
| 1315 |
|
| 1316 |
static int SQLITE_TCLAPI f5tTokenHash( |
| 1317 |
void * clientData, |
| 1318 |
Tcl_Interp *interp, |
| 1319 |
int objc, |
| 1320 |
Tcl_Obj *CONST objv[] |
| 1321 |
){ |
| 1322 |
char *z; |
| 1323 |
Tcl_Size n; |
| 1324 |
unsigned int iVal; |
| 1325 |
int nSlot; |
| 1326 |
|
| 1327 |
if( objc!=3 ){ |
| 1328 |
Tcl_WrongNumArgs(interp, 1, objv, "NSLOT TOKEN"); |
| 1329 |
return TCL_ERROR; |
| 1330 |
} |
| 1331 |
if( Tcl_GetIntFromObj(interp, objv[1], &nSlot) ){ |
| 1332 |
return TCL_ERROR; |
| 1333 |
} |
| 1334 |
z = Tcl_GetStringFromObj(objv[2], &n); |
| 1335 |
|
| 1336 |
iVal = f5t_fts5HashKey(nSlot, z, (int)n); |
| 1337 |
Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); |
| 1338 |
return TCL_OK; |
| 1339 |
} |
| 1340 |
|
| 1341 |
static int SQLITE_TCLAPI f5tRegisterMatchinfo( |
| 1342 |
void * clientData, |
| 1343 |
Tcl_Interp *interp, |
| 1344 |
int objc, |
| 1345 |
Tcl_Obj *CONST objv[] |
| 1346 |
){ |
| 1347 |
int rc; |
| 1348 |
sqlite3 *db = 0; |
| 1349 |
|
| 1350 |
if( objc!=2 ){ |
| 1351 |
Tcl_WrongNumArgs(interp, 1, objv, "DB"); |
| 1352 |
return TCL_ERROR; |
| 1353 |
} |
| 1354 |
if( f5tDbPointer(interp, objv[1], &db) ){ |
| 1355 |
return TCL_ERROR; |
| 1356 |
} |
| 1357 |
|
| 1358 |
rc = sqlite3Fts5TestRegisterMatchinfo(db); |
| 1359 |
if( rc!=SQLITE_OK ){ |
| 1360 |
Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); |
| 1361 |
return TCL_ERROR; |
| 1362 |
} |
| 1363 |
return TCL_OK; |
| 1364 |
} |
| 1365 |
|
| 1366 |
static int SQLITE_TCLAPI f5tRegisterTok( |
| 1367 |
void * clientData, |
| 1368 |
Tcl_Interp *interp, |
| 1369 |
int objc, |
| 1370 |
Tcl_Obj *CONST objv[] |
| 1371 |
){ |
| 1372 |
int rc; |
| 1373 |
sqlite3 *db = 0; |
| 1374 |
fts5_api *pApi = 0; |
| 1375 |
|
| 1376 |
if( objc!=2 ){ |
| 1377 |
Tcl_WrongNumArgs(interp, 1, objv, "DB"); |
| 1378 |
return TCL_ERROR; |
| 1379 |
} |
| 1380 |
if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){ |
| 1381 |
return TCL_ERROR; |
| 1382 |
} |
| 1383 |
|
| 1384 |
rc = sqlite3Fts5TestRegisterTok(db, pApi); |
| 1385 |
if( rc!=SQLITE_OK ){ |
| 1386 |
Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); |
| 1387 |
return TCL_ERROR; |
| 1388 |
} |
| 1389 |
return TCL_OK; |
| 1390 |
} |
| 1391 |
|
| 1392 |
typedef struct OriginTextCtx OriginTextCtx; |
| 1393 |
struct OriginTextCtx { |
| 1394 |
sqlite3 *db; |
| 1395 |
fts5_api *pApi; |
| 1396 |
}; |
| 1397 |
|
| 1398 |
typedef struct OriginTextTokenizer OriginTextTokenizer; |
| 1399 |
struct OriginTextTokenizer { |
| 1400 |
Fts5Tokenizer *pTok; /* Underlying tokenizer object */ |
| 1401 |
fts5_tokenizer tokapi; /* API implementation for pTok */ |
| 1402 |
}; |
| 1403 |
|
| 1404 |
/* |
| 1405 |
** Delete the OriginTextCtx object indicated by the only argument. |
| 1406 |
*/ |
| 1407 |
static void f5tOrigintextTokenizerDelete(void *pCtx){ |
| 1408 |
OriginTextCtx *p = (OriginTextCtx*)pCtx; |
| 1409 |
ckfree((char*)p); |
| 1410 |
} |
| 1411 |
|
| 1412 |
static int f5tOrigintextCreate( |
| 1413 |
void *pCtx, |
| 1414 |
const char **azArg, |
| 1415 |
int nArg, |
| 1416 |
Fts5Tokenizer **ppOut |
| 1417 |
){ |
| 1418 |
OriginTextCtx *p = (OriginTextCtx*)pCtx; |
| 1419 |
OriginTextTokenizer *pTok = 0; |
| 1420 |
void *pTokCtx = 0; |
| 1421 |
int rc = SQLITE_OK; |
| 1422 |
|
| 1423 |
pTok = (OriginTextTokenizer*)sqlite3_malloc64(sizeof(OriginTextTokenizer)); |
| 1424 |
if( pTok==0 ){ |
| 1425 |
rc = SQLITE_NOMEM; |
| 1426 |
}else if( nArg<1 ){ |
| 1427 |
rc = SQLITE_ERROR; |
| 1428 |
}else{ |
| 1429 |
/* Locate the underlying tokenizer */ |
| 1430 |
rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi); |
| 1431 |
} |
| 1432 |
|
| 1433 |
/* Create the new tokenizer instance */ |
| 1434 |
if( rc==SQLITE_OK ){ |
| 1435 |
rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok); |
| 1436 |
} |
| 1437 |
|
| 1438 |
if( rc!=SQLITE_OK ){ |
| 1439 |
sqlite3_free(pTok); |
| 1440 |
pTok = 0; |
| 1441 |
} |
| 1442 |
*ppOut = (Fts5Tokenizer*)pTok; |
| 1443 |
return rc; |
| 1444 |
} |
| 1445 |
|
| 1446 |
static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){ |
| 1447 |
OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer; |
| 1448 |
if( p->pTok ){ |
| 1449 |
p->tokapi.xDelete(p->pTok); |
| 1450 |
} |
| 1451 |
sqlite3_free(p); |
| 1452 |
} |
| 1453 |
|
| 1454 |
typedef struct OriginTextCb OriginTextCb; |
| 1455 |
struct OriginTextCb { |
| 1456 |
void *pCtx; |
| 1457 |
const char *pText; |
| 1458 |
int nText; |
| 1459 |
int (*xToken)(void *, int, const char *, int, int, int); |
| 1460 |
|
| 1461 |
char *aBuf; /* Buffer to use */ |
| 1462 |
int nBuf; /* Allocated size of aBuf[] */ |
| 1463 |
}; |
| 1464 |
|
| 1465 |
static int xOriginToken( |
| 1466 |
void *pCtx, /* Copy of 2nd argument to xTokenize() */ |
| 1467 |
int tflags, /* Mask of FTS5_TOKEN_* flags */ |
| 1468 |
const char *pToken, /* Pointer to buffer containing token */ |
| 1469 |
int nToken, /* Size of token in bytes */ |
| 1470 |
int iStart, /* Byte offset of token within input text */ |
| 1471 |
int iEnd /* Byte offset of end of token within input */ |
| 1472 |
){ |
| 1473 |
OriginTextCb *p = (OriginTextCb*)pCtx; |
| 1474 |
int ret = 0; |
| 1475 |
|
| 1476 |
if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){ |
| 1477 |
/* Token exactly matches document text. Pass it through as is. */ |
| 1478 |
ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd); |
| 1479 |
}else{ |
| 1480 |
int nReq = nToken + 1 + (iEnd-iStart); |
| 1481 |
if( nReq>p->nBuf ){ |
| 1482 |
sqlite3_free(p->aBuf); |
| 1483 |
p->aBuf = sqlite3_malloc64(nReq*2); |
| 1484 |
if( p->aBuf==0 ) return SQLITE_NOMEM; |
| 1485 |
p->nBuf = nReq*2; |
| 1486 |
} |
| 1487 |
|
| 1488 |
memcpy(p->aBuf, pToken, nToken); |
| 1489 |
p->aBuf[nToken] = '\0'; |
| 1490 |
memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart); |
| 1491 |
ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd); |
| 1492 |
} |
| 1493 |
|
| 1494 |
return ret; |
| 1495 |
} |
| 1496 |
|
| 1497 |
|
| 1498 |
static int f5tOrigintextTokenize( |
| 1499 |
Fts5Tokenizer *pTokenizer, |
| 1500 |
void *pCtx, |
| 1501 |
int flags, /* Mask of FTS5_TOKENIZE_* flags */ |
| 1502 |
const char *pText, int nText, |
| 1503 |
int (*xToken)(void *, int, const char *, int, int, int) |
| 1504 |
){ |
| 1505 |
OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer; |
| 1506 |
OriginTextCb cb; |
| 1507 |
int ret; |
| 1508 |
|
| 1509 |
memset(&cb, 0, sizeof(cb)); |
| 1510 |
cb.pCtx = pCtx; |
| 1511 |
cb.pText = pText; |
| 1512 |
cb.nText = nText; |
| 1513 |
cb.xToken = xToken; |
| 1514 |
|
| 1515 |
ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken); |
| 1516 |
sqlite3_free(cb.aBuf); |
| 1517 |
return ret; |
| 1518 |
} |
| 1519 |
|
| 1520 |
/* |
| 1521 |
** sqlite3_fts5_register_origintext DB |
| 1522 |
** |
| 1523 |
** Description... |
| 1524 |
*/ |
| 1525 |
static int SQLITE_TCLAPI f5tRegisterOriginText( |
| 1526 |
void * clientData, |
| 1527 |
Tcl_Interp *interp, |
| 1528 |
int objc, |
| 1529 |
Tcl_Obj *CONST objv[] |
| 1530 |
){ |
| 1531 |
sqlite3 *db = 0; |
| 1532 |
fts5_api *pApi = 0; |
| 1533 |
int rc; |
| 1534 |
fts5_tokenizer tok = {0, 0, 0}; |
| 1535 |
OriginTextCtx *pCtx = 0; |
| 1536 |
|
| 1537 |
if( objc!=2 ){ |
| 1538 |
Tcl_WrongNumArgs(interp, 1, objv, "DB"); |
| 1539 |
return TCL_ERROR; |
| 1540 |
} |
| 1541 |
if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR; |
| 1542 |
|
| 1543 |
pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx)); |
| 1544 |
pCtx->db = db; |
| 1545 |
pCtx->pApi = pApi; |
| 1546 |
|
| 1547 |
tok.xCreate = f5tOrigintextCreate; |
| 1548 |
tok.xDelete = f5tOrigintextDelete; |
| 1549 |
tok.xTokenize = f5tOrigintextTokenize; |
| 1550 |
rc = pApi->xCreateTokenizer( |
| 1551 |
pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete |
| 1552 |
); |
| 1553 |
|
| 1554 |
Tcl_ResetResult(interp); |
| 1555 |
if( rc!=SQLITE_OK ){ |
| 1556 |
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0); |
| 1557 |
return TCL_ERROR; |
| 1558 |
} |
| 1559 |
return TCL_OK; |
| 1560 |
} |
| 1561 |
|
| 1562 |
/* |
| 1563 |
** This function is used to DROP an fts5 table. It works even if the data |
| 1564 |
** structures fts5 stores within the database are corrupt, which sometimes |
| 1565 |
** prevents a straight "DROP TABLE" command from succeeding. |
| 1566 |
** |
| 1567 |
** The first parameter is the database handle to use for the DROP TABLE |
| 1568 |
** operation. The second is the name of the database to drop the fts5 table |
| 1569 |
** from (i.e. "main", "temp" or the name of an attached database). The |
| 1570 |
** third parameter is the name of the fts5 table to drop. |
| 1571 |
** |
| 1572 |
** SQLITE_OK is returned if the table is successfully dropped. Or, if an |
| 1573 |
** error occurs, an SQLite error code. |
| 1574 |
*/ |
| 1575 |
static int sqlite3_fts5_drop_corrupt_table( |
| 1576 |
sqlite3 *db, /* Database handle */ |
| 1577 |
const char *zDb, /* Database name ("main", "temp" etc.) */ |
| 1578 |
const char *zTab /* Name of fts5 table to drop */ |
| 1579 |
){ |
| 1580 |
int rc = SQLITE_OK; |
| 1581 |
int bDef = 0; |
| 1582 |
|
| 1583 |
rc = sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDef); |
| 1584 |
if( rc==SQLITE_OK ){ |
| 1585 |
char *zScript = sqlite3_mprintf( |
| 1586 |
"DELETE FROM %Q.'%q_data';" |
| 1587 |
"DELETE FROM %Q.'%q_config';" |
| 1588 |
"INSERT INTO %Q.'%q_data' VALUES(10, X'0000000000');" |
| 1589 |
"INSERT INTO %Q.'%q_config' VALUES('version', 4);" |
| 1590 |
"DROP TABLE %Q.'%q';", |
| 1591 |
zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab |
| 1592 |
); |
| 1593 |
|
| 1594 |
if( zScript==0 ){ |
| 1595 |
rc = SQLITE_NOMEM; |
| 1596 |
}else{ |
| 1597 |
if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0); |
| 1598 |
rc = sqlite3_exec(db, zScript, 0, 0, 0); |
| 1599 |
if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0); |
| 1600 |
sqlite3_free(zScript); |
| 1601 |
} |
| 1602 |
} |
| 1603 |
|
| 1604 |
return rc; |
| 1605 |
} |
| 1606 |
|
| 1607 |
/* |
| 1608 |
** sqlite3_fts5_drop_corrupt_table DB DATABASE TABLE |
| 1609 |
** |
| 1610 |
** Description... |
| 1611 |
*/ |
| 1612 |
static int SQLITE_TCLAPI f5tDropCorruptTable( |
| 1613 |
void * clientData, |
| 1614 |
Tcl_Interp *interp, |
| 1615 |
int objc, |
| 1616 |
Tcl_Obj *CONST objv[] |
| 1617 |
){ |
| 1618 |
sqlite3 *db = 0; |
| 1619 |
const char *zDb = 0; |
| 1620 |
const char *zTab = 0; |
| 1621 |
int rc = SQLITE_OK; |
| 1622 |
|
| 1623 |
if( objc!=4 ){ |
| 1624 |
Tcl_WrongNumArgs(interp, 1, objv, "DB DATABASE TABLE"); |
| 1625 |
return TCL_ERROR; |
| 1626 |
} |
| 1627 |
if( f5tDbPointer(interp, objv[1], &db) ){ |
| 1628 |
return TCL_ERROR; |
| 1629 |
} |
| 1630 |
zDb = Tcl_GetString(objv[2]); |
| 1631 |
zTab = Tcl_GetString(objv[3]); |
| 1632 |
|
| 1633 |
rc = sqlite3_fts5_drop_corrupt_table(db, zDb, zTab); |
| 1634 |
if( rc!=SQLITE_OK ){ |
| 1635 |
Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0); |
| 1636 |
return TCL_ERROR; |
| 1637 |
} |
| 1638 |
|
| 1639 |
return TCL_OK; |
| 1640 |
} |
| 1641 |
|
| 1642 |
/* |
| 1643 |
** Free a buffer returned to SQLite by the str() function. |
| 1644 |
*/ |
| 1645 |
void f5tFree(void *p){ |
| 1646 |
char *x = (char *)p; |
| 1647 |
ckfree(&x[-8]); |
| 1648 |
} |
| 1649 |
|
| 1650 |
/* |
| 1651 |
** Implementation of str(). |
| 1652 |
*/ |
| 1653 |
void f5tStrFunc(sqlite3_context *pCtx, int nArg, sqlite3_value **apArg){ |
| 1654 |
const char *zText = 0; |
| 1655 |
assert( nArg==1 ); |
| 1656 |
|
| 1657 |
zText = (const char*)sqlite3_value_text(apArg[0]); |
| 1658 |
if( zText ){ |
| 1659 |
sqlite3_int64 nText = strlen(zText); |
| 1660 |
char *zCopy = (char*)ckalloc(nText+8); |
| 1661 |
if( zCopy==0 ){ |
| 1662 |
sqlite3_result_error_nomem(pCtx); |
| 1663 |
}else{ |
| 1664 |
zCopy += 8; |
| 1665 |
memcpy(zCopy, zText, nText); |
| 1666 |
sqlite3_result_text64(pCtx, zCopy, nText, f5tFree, SQLITE_UTF8); |
| 1667 |
} |
| 1668 |
} |
| 1669 |
} |
| 1670 |
|
| 1671 |
/* |
| 1672 |
** sqlite3_fts5_register_str DB |
| 1673 |
** |
| 1674 |
** Register the str() function with database handle DB. str() interprets |
| 1675 |
** its only argument as text and returns a copy of the value in a |
| 1676 |
** non-nul-terminated buffer. |
| 1677 |
*/ |
| 1678 |
static int SQLITE_TCLAPI f5tRegisterStr( |
| 1679 |
void * clientData, |
| 1680 |
Tcl_Interp *interp, |
| 1681 |
int objc, |
| 1682 |
Tcl_Obj *CONST objv[] |
| 1683 |
){ |
| 1684 |
sqlite3 *db = 0; |
| 1685 |
|
| 1686 |
if( objc!=2 ){ |
| 1687 |
Tcl_WrongNumArgs(interp, 1, objv, "DB"); |
| 1688 |
return TCL_ERROR; |
| 1689 |
} |
| 1690 |
if( f5tDbPointer(interp, objv[1], &db) ){ |
| 1691 |
return TCL_ERROR; |
| 1692 |
} |
| 1693 |
|
| 1694 |
sqlite3_create_function(db, "str", 1, SQLITE_UTF8, 0, f5tStrFunc, 0, 0); |
| 1695 |
|
| 1696 |
return TCL_OK; |
| 1697 |
} |
| 1698 |
|
| 1699 |
/* |
| 1700 |
** Entry point. |
| 1701 |
*/ |
| 1702 |
int Fts5tcl_Init(Tcl_Interp *interp){ |
| 1703 |
static struct Cmd { |
| 1704 |
char *zName; |
| 1705 |
Tcl_ObjCmdProc *xProc; |
| 1706 |
int bTokenizeCtx; |
| 1707 |
} aCmd[] = { |
| 1708 |
{ "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 }, |
| 1709 |
{ "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, |
| 1710 |
{ "sqlite3_fts5_locale", f5tTokenizerLocale, 1 }, |
| 1711 |
{ "sqlite3_fts5_tokenize", f5tTokenize, 0 }, |
| 1712 |
{ "sqlite3_fts5_create_function", f5tCreateFunction, 0 }, |
| 1713 |
{ "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 }, |
| 1714 |
{ "sqlite3_fts5_token_hash", f5tTokenHash, 0 }, |
| 1715 |
{ "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 }, |
| 1716 |
{ "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }, |
| 1717 |
{ "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }, |
| 1718 |
{ "sqlite3_fts5_drop_corrupt_table", f5tDropCorruptTable, 0 }, |
| 1719 |
{ "sqlite3_fts5_register_str", f5tRegisterStr, 0 } |
| 1720 |
}; |
| 1721 |
int i; |
| 1722 |
F5tTokenizerContext *pContext; |
| 1723 |
|
| 1724 |
pContext = (F5tTokenizerContext*)ckalloc(sizeof(F5tTokenizerContext)); |
| 1725 |
memset(pContext, 0, sizeof(*pContext)); |
| 1726 |
|
| 1727 |
for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){ |
| 1728 |
struct Cmd *p = &aCmd[i]; |
| 1729 |
void *pCtx = 0; |
| 1730 |
if( p->bTokenizeCtx ) pCtx = (void*)pContext; |
| 1731 |
Tcl_CreateObjCommand(interp, p->zName, p->xProc, pCtx, (i ? 0 : xF5tFree)); |
| 1732 |
} |
| 1733 |
|
| 1734 |
return TCL_OK; |
| 1735 |
} |
| 1736 |
#else /* SQLITE_ENABLE_FTS5 */ |
| 1737 |
int Fts5tcl_Init(Tcl_Interp *interp){ |
| 1738 |
return TCL_OK; |
| 1739 |
} |
| 1740 |
#endif /* SQLITE_ENABLE_FTS5 */ |
| 1741 |
#endif /* SQLITE_TEST */ |