Debian Codegraph @

ext/fts5/fts5_tcl.c

raw
1 /*
2 ** 2014 Dec 01
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 **
13 */
14
15
16 #ifdef SQLITE_TEST
17 #include "tclsqlite.h"
18
19 #ifdef SQLITE_ENABLE_FTS5
20
21 #include "fts5.h"
22 #include <string.h>
23 #include <assert.h>
24 #include <stdlib.h>
25
26 #ifdef SQLITE_DEBUG
27 extern int sqlite3_fts5_may_be_corrupt;
28 #endif
29 extern int sqlite3Fts5TestRegisterMatchinfo(sqlite3*);
30 extern int sqlite3Fts5TestRegisterTok(sqlite3*, fts5_api*);
31
32 /*************************************************************************
33 ** This is a copy of the first part of the SqliteDb structure in
34 ** tclsqlite.c. We need it here so that the get_sqlite_pointer routine
35 ** can extract the sqlite3* pointer from an existing Tcl SQLite
36 ** connection.
37 */
38
39 extern const char *sqlite3ErrName(int);
40
41 struct SqliteDb {
42 sqlite3 *db;
43 };
44
45 /*
46 ** Decode a pointer to an sqlite3 object.
47 */
48 static int f5tDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **ppDb){
49 struct SqliteDb *p;
50 Tcl_CmdInfo cmdInfo;
51 char *z = Tcl_GetString(pObj);
52 if( Tcl_GetCommandInfo(interp, z, &cmdInfo) ){
53 p = (struct SqliteDb*)cmdInfo.objClientData;
54 *ppDb = p->db;
55 return TCL_OK;
56 }
57 return TCL_ERROR;
58 }
59
60 /* End of code that accesses the SqliteDb struct.
61 **************************************************************************/
62
63 static int f5tResultToErrorCode(const char *zRes){
64 struct ErrorCode {
65 int rc;
66 const char *zError;
67 } aErr[] = {
68 { SQLITE_DONE, "SQLITE_DONE" },
69 { SQLITE_ERROR, "SQLITE_ERROR" },
70 { SQLITE_OK, "SQLITE_OK" },
71 { SQLITE_OK, "" },
72 };
73 int i;
74
75 for(i=0; i<sizeof(aErr)/sizeof(aErr[0]); i++){
76 if( 0==sqlite3_stricmp(zRes, aErr[i].zError) ){
77 return aErr[i].rc;
78 }
79 }
80
81 return SQLITE_ERROR;
82 }
83
84 static int SQLITE_TCLAPI f5tDbAndApi(
85 Tcl_Interp *interp,
86 Tcl_Obj *pObj,
87 sqlite3 **ppDb,
88 fts5_api **ppApi
89 ){
90 sqlite3 *db = 0;
91 int rc = f5tDbPointer(interp, pObj, &db);
92 if( rc!=TCL_OK ){
93 return TCL_ERROR;
94 }else{
95 sqlite3_stmt *pStmt = 0;
96 fts5_api *pApi = 0;
97
98 rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0);
99 if( rc!=SQLITE_OK ){
100 Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
101 return TCL_ERROR;
102 }
103 sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0);
104 sqlite3_step(pStmt);
105
106 if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
107 Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
108 return TCL_ERROR;
109 }
110
111 *ppDb = db;
112 *ppApi = pApi;
113 }
114
115 return TCL_OK;
116 }
117
118 typedef struct F5tFunction F5tFunction;
119 struct F5tFunction {
120 Tcl_Interp *interp;
121 Tcl_Obj *pScript;
122 };
123
124 typedef struct F5tApi F5tApi;
125 struct F5tApi {
126 const Fts5ExtensionApi *pApi;
127 Fts5Context *pFts;
128 };
129
130 /*
131 ** An object of this type is used with the xSetAuxdata() and xGetAuxdata()
132 ** API test wrappers. The tcl interface allows a single tcl value to be
133 ** saved using xSetAuxdata(). Instead of simply storing a pointer to the
134 ** tcl object, the code in this file wraps it in an sqlite3_malloc'd
135 ** instance of the following struct so that if the destructor is not
136 ** correctly invoked it will be reported as an SQLite memory leak.
137 */
138 typedef struct F5tAuxData F5tAuxData;
139 struct F5tAuxData {
140 Tcl_Obj *pObj;
141 };
142
143 static int xTokenizeCb(
144 void *pCtx,
145 int tflags,
146 const char *zToken, int nToken,
147 int iStart, int iEnd
148 ){
149 F5tFunction *p = (F5tFunction*)pCtx;
150 Tcl_Obj *pEval = Tcl_DuplicateObj(p->pScript);
151 int rc;
152
153 Tcl_IncrRefCount(pEval);
154 Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken));
155 Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iStart));
156 Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iEnd));
157
158 rc = Tcl_EvalObjEx(p->interp, pEval, 0);
159 Tcl_DecrRefCount(pEval);
160 if( rc==TCL_OK ){
161 rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp));
162 }
163
164 return rc;
165 }
166
167 static int SQLITE_TCLAPI xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);
168
169 static int xQueryPhraseCb(
170 const Fts5ExtensionApi *pApi,
171 Fts5Context *pFts,
172 void *pCtx
173 ){
174 F5tFunction *p = (F5tFunction*)pCtx;
175 static sqlite3_int64 iCmd = 0;
176 Tcl_Obj *pEval;
177 int rc;
178
179 char zCmd[64];
180 F5tApi sApi;
181
182 sApi.pApi = pApi;
183 sApi.pFts = pFts;
184 sprintf(zCmd, "f5t_2_%lld", iCmd++);
185 Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0);
186
187 pEval = Tcl_DuplicateObj(p->pScript);
188 Tcl_IncrRefCount(pEval);
189 Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1));
190 rc = Tcl_EvalObjEx(p->interp, pEval, 0);
191 Tcl_DecrRefCount(pEval);
192 Tcl_DeleteCommand(p->interp, zCmd);
193
194 if( rc==TCL_OK ){
195 rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp));
196 }
197
198 return rc;
199 }
200
201 static void xSetAuxdataDestructor(void *p){
202 F5tAuxData *pData = (F5tAuxData*)p;
203 Tcl_DecrRefCount(pData->pObj);
204 sqlite3_free(pData);
205 }
206
207 /*
208 ** api sub-command...
209 **
210 ** Description...
211 */
212 static int SQLITE_TCLAPI xF5tApi(
213 void * clientData,
214 Tcl_Interp *interp,
215 int objc,
216 Tcl_Obj *CONST objv[]
217 ){
218 struct Sub {
219 const char *zName;
220 int nArg;
221 const char *zMsg;
222 } aSub[] = {
223 { "xColumnCount", 0, "" }, /* 0 */
224 { "xRowCount", 0, "" }, /* 1 */
225 { "xColumnTotalSize", 1, "COL" }, /* 2 */
226 { "xTokenize", 2, "TEXT SCRIPT" }, /* 3 */
227 { "xPhraseCount", 0, "" }, /* 4 */
228 { "xPhraseSize", 1, "PHRASE" }, /* 5 */
229 { "xInstCount", 0, "" }, /* 6 */
230 { "xInst", 1, "IDX" }, /* 7 */
231 { "xRowid", 0, "" }, /* 8 */
232 { "xColumnText", 1, "COL" }, /* 9 */
233 { "xColumnSize", 1, "COL" }, /* 10 */
234 { "xQueryPhrase", 2, "PHRASE SCRIPT" }, /* 11 */
235 { "xSetAuxdata", 1, "VALUE" }, /* 12 */
236 { "xGetAuxdata", 1, "CLEAR" }, /* 13 */
237 { "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */
238 { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */
239 { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
240 { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
241
242 { "xQueryToken", 2, "IPHRASE ITERM" }, /* 18 */
243 { "xInstToken", 2, "IDX ITERM" }, /* 19 */
244 { "xColumnLocale", 1, "COL" }, /* 20 */
245 { 0, 0, 0}
246 };
247
248 int rc;
249 int iSub = 0;
250 F5tApi *p = (F5tApi*)clientData;
251
252 if( objc<2 ){
253 Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND");
254 return TCL_ERROR;
255 }
256
257 rc = Tcl_GetIndexFromObjStruct(
258 interp, objv[1], aSub, sizeof(aSub[0]), "SUB-COMMAND", 0, &iSub
259 );
260 if( rc!=TCL_OK ) return rc;
261 if( aSub[iSub].nArg!=objc-2 ){
262 Tcl_WrongNumArgs(interp, 1, objv, aSub[iSub].zMsg);
263 return TCL_ERROR;
264 }
265
266 #define CASE(i,str) case i: assert( strcmp(aSub[i].zName, str)==0 );
267 switch( iSub ){
268 CASE(0, "xColumnCount") {
269 int nCol;
270 nCol = p->pApi->xColumnCount(p->pFts);
271 if( rc==SQLITE_OK ){
272 Tcl_SetObjResult(interp, Tcl_NewIntObj(nCol));
273 }
274 break;
275 }
276 CASE(1, "xRowCount") {
277 sqlite3_int64 nRow;
278 rc = p->pApi->xRowCount(p->pFts, &nRow);
279 if( rc==SQLITE_OK ){
280 Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nRow));
281 }
282 break;
283 }
284 CASE(2, "xColumnTotalSize") {
285 int iCol;
286 sqlite3_int64 nSize;
287 if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ) return TCL_ERROR;
288 rc = p->pApi->xColumnTotalSize(p->pFts, iCol, &nSize);
289 if( rc==SQLITE_OK ){
290 Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize));
291 }
292 break;
293 }
294 CASE(3, "xTokenize") {
295 Tcl_Size nText;
296 char *zText = Tcl_GetStringFromObj(objv[2], &nText);
297 F5tFunction ctx;
298 ctx.interp = interp;
299 ctx.pScript = objv[3];
300 rc = p->pApi->xTokenize(p->pFts, zText, (int)nText, &ctx, xTokenizeCb);
301 if( rc==SQLITE_OK ){
302 Tcl_ResetResult(interp);
303 }
304 return rc;
305 }
306 CASE(4, "xPhraseCount") {
307 int nPhrase;
308 nPhrase = p->pApi->xPhraseCount(p->pFts);
309 if( rc==SQLITE_OK ){
310 Tcl_SetObjResult(interp, Tcl_NewIntObj(nPhrase));
311 }
312 break;
313 }
314 CASE(5, "xPhraseSize") {
315 int iPhrase;
316 int sz;
317 if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){
318 return TCL_ERROR;
319 }
320 sz = p->pApi->xPhraseSize(p->pFts, iPhrase);
321 if( rc==SQLITE_OK ){
322 Tcl_SetObjResult(interp, Tcl_NewIntObj(sz));
323 }
324 break;
325 }
326 CASE(6, "xInstCount") {
327 int nInst;
328 rc = p->pApi->xInstCount(p->pFts, &nInst);
329 if( rc==SQLITE_OK ){
330 Tcl_SetObjResult(interp, Tcl_NewIntObj(nInst));
331 }
332 break;
333 }
334 CASE(7, "xInst") {
335 int iIdx, ip, ic, io;
336 if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ){
337 return TCL_ERROR;
338 }
339 rc = p->pApi->xInst(p->pFts, iIdx, &ip, &ic, &io);
340 if( rc==SQLITE_OK ){
341 Tcl_Obj *pList = Tcl_NewObj();
342 Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ip));
343 Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ic));
344 Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(io));
345 Tcl_SetObjResult(interp, pList);
346 }
347 break;
348 }
349 CASE(8, "xRowid") {
350 sqlite3_int64 iRowid = p->pApi->xRowid(p->pFts);
351 Tcl_SetObjResult(interp, Tcl_NewWideIntObj(iRowid));
352 break;
353 }
354 CASE(9, "xColumnText") {
355 const char *z = 0;
356 int n = 0;
357 int iCol;
358 if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
359 return TCL_ERROR;
360 }
361 rc = p->pApi->xColumnText(p->pFts, iCol, &z, &n);
362 if( rc==SQLITE_OK ){
363 Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));
364 }
365 break;
366 }
367 CASE(10, "xColumnSize") {
368 int n = 0;
369 int iCol;
370 if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
371 return TCL_ERROR;
372 }
373 rc = p->pApi->xColumnSize(p->pFts, iCol, &n);
374 if( rc==SQLITE_OK ){
375 Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
376 }
377 break;
378 }
379 CASE(11, "xQueryPhrase") {
380 int iPhrase;
381 F5tFunction ctx;
382 if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){
383 return TCL_ERROR;
384 }
385 ctx.interp = interp;
386 ctx.pScript = objv[3];
387 rc = p->pApi->xQueryPhrase(p->pFts, iPhrase, &ctx, xQueryPhraseCb);
388 if( rc==SQLITE_OK ){
389 Tcl_ResetResult(interp);
390 }
391 break;
392 }
393 CASE(12, "xSetAuxdata") {
394 F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc64(sizeof(F5tAuxData));
395 if( pData==0 ){
396 Tcl_AppendResult(interp, "out of memory", (char*)0);
397 return TCL_ERROR;
398 }
399 pData->pObj = objv[2];
400 Tcl_IncrRefCount(pData->pObj);
401 rc = p->pApi->xSetAuxdata(p->pFts, pData, xSetAuxdataDestructor);
402 break;
403 }
404 CASE(13, "xGetAuxdata") {
405 F5tAuxData *pData;
406 int bClear;
407 if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ){
408 return TCL_ERROR;
409 }
410 pData = (F5tAuxData*)p->pApi->xGetAuxdata(p->pFts, bClear);
411 if( pData==0 ){
412 Tcl_ResetResult(interp);
413 }else{
414 Tcl_SetObjResult(interp, pData->pObj);
415 if( bClear ){
416 xSetAuxdataDestructor((void*)pData);
417 }
418 }
419 break;
420 }
421
422 /* These two - xSetAuxdataInt and xGetAuxdataInt - are similar to the
423 ** xSetAuxdata and xGetAuxdata methods implemented above. The difference
424 ** is that they may only save an integer value as auxiliary data, and
425 ** do not specify a destructor function. */
426 CASE(14, "xSetAuxdataInt") {
427 int iVal;
428 if( Tcl_GetIntFromObj(interp, objv[2], &iVal) ) return TCL_ERROR;
429 rc = p->pApi->xSetAuxdata(p->pFts, (void*)((char*)0 + iVal), 0);
430 break;
431 }
432 CASE(15, "xGetAuxdataInt") {
433 int iVal;
434 int bClear;
435 if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
436 iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
437 Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
438 break;
439 }
440
441 CASE(16, "xPhraseForeach") {
442 int iPhrase;
443 int iCol;
444 int iOff;
445 const char *zColvar;
446 const char *zOffvar;
447 Tcl_Obj *pScript = objv[5];
448 Fts5PhraseIter iter;
449
450 if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
451 zColvar = Tcl_GetString(objv[3]);
452 zOffvar = Tcl_GetString(objv[4]);
453
454 rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
455 if( rc!=SQLITE_OK ){
456 Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0);
457 return TCL_ERROR;
458 }
459 for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){
460 Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
461 Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0);
462 rc = Tcl_EvalObjEx(interp, pScript, 0);
463 if( rc==TCL_CONTINUE ) rc = TCL_OK;
464 if( rc!=TCL_OK ){
465 if( rc==TCL_BREAK ) rc = TCL_OK;
466 break;
467 }
468 }
469
470 break;
471 }
472
473 CASE(17, "xPhraseColumnForeach") {
474 int iPhrase;
475 int iCol;
476 const char *zColvar;
477 Tcl_Obj *pScript = objv[4];
478 Fts5PhraseIter iter;
479
480 if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
481 zColvar = Tcl_GetString(objv[3]);
482
483 rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
484 if( rc!=SQLITE_OK ){
485 Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
486 return TCL_ERROR;
487 }
488 for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
489 Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
490 rc = Tcl_EvalObjEx(interp, pScript, 0);
491 if( rc==TCL_CONTINUE ) rc = TCL_OK;
492 if( rc!=TCL_OK ){
493 if( rc==TCL_BREAK ) rc = TCL_OK;
494 break;
495 }
496 }
497
498 break;
499 }
500
501 CASE(18, "xQueryToken") {
502 const char *pTerm = 0;
503 int nTerm = 0;
504 int iPhrase = 0;
505 int iTerm = 0;
506
507 if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
508 if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
509 rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm);
510 if( rc==SQLITE_OK ){
511 Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
512 }
513
514 break;
515 }
516
517 CASE(19, "xInstToken") {
518 const char *pTerm = 0;
519 int nTerm = 0;
520 int iIdx = 0;
521 int iTerm = 0;
522
523 if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR;
524 if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
525 rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm);
526 if( rc==SQLITE_OK ){
527 Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
528 }
529
530 break;
531 }
532
533 CASE(20, "xColumnLocale") {
534 const char *z = 0;
535 int n = 0;
536 int iCol;
537 if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
538 return TCL_ERROR;
539 }
540 rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n);
541 if( rc==SQLITE_OK && z ){
542 Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));
543 }
544 break;
545 }
546
547 default:
548 assert( 0 );
549 break;
550 }
551 #undef CASE
552
553 if( rc!=SQLITE_OK ){
554 Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
555 return TCL_ERROR;
556 }
557
558 return TCL_OK;
559 }
560
561 static void xF5tFunction(
562 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
563 Fts5Context *pFts, /* First arg to pass to pApi functions */
564 sqlite3_context *pCtx, /* Context for returning result/error */
565 int nVal, /* Number of values in apVal[] array */
566 sqlite3_value **apVal /* Array of trailing arguments */
567 ){
568 F5tFunction *p = (F5tFunction*)pApi->xUserData(pFts);
569 Tcl_Obj *pEval; /* Script to evaluate */
570 int i;
571 int rc;
572
573 static sqlite3_int64 iCmd = 0;
574 char zCmd[64];
575 F5tApi sApi;
576 sApi.pApi = pApi;
577 sApi.pFts = pFts;
578
579 sprintf(zCmd, "f5t_%lld", iCmd++);
580 Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0);
581 pEval = Tcl_DuplicateObj(p->pScript);
582 Tcl_IncrRefCount(pEval);
583 Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1));
584
585 for(i=0; i<nVal; i++){
586 Tcl_Obj *pObj = 0;
587 switch( sqlite3_value_type(apVal[i]) ){
588 case SQLITE_TEXT:
589 pObj = Tcl_NewStringObj((const char*)sqlite3_value_text(apVal[i]), -1);
590 break;
591 case SQLITE_BLOB:
592 pObj = Tcl_NewByteArrayObj(
593 sqlite3_value_blob(apVal[i]), sqlite3_value_bytes(apVal[i])
594 );
595 break;
596 case SQLITE_INTEGER:
597 pObj = Tcl_NewWideIntObj(sqlite3_value_int64(apVal[i]));
598 break;
599 case SQLITE_FLOAT:
600 pObj = Tcl_NewDoubleObj(sqlite3_value_double(apVal[i]));
601 break;
602 default:
603 pObj = Tcl_NewObj();
604 break;
605 }
606 Tcl_ListObjAppendElement(p->interp, pEval, pObj);
607 }
608
609 rc = Tcl_EvalObjEx(p->interp, pEval, TCL_GLOBAL_ONLY);
610 Tcl_DecrRefCount(pEval);
611 Tcl_DeleteCommand(p->interp, zCmd);
612
613 if( rc!=TCL_OK ){
614 sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1);
615 }else{
616 Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);
617 const char *zType = (pVar->typePtr ? pVar->typePtr->name : "");
618 char c = zType[0];
619 if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){
620 /* Only return a BLOB type if the Tcl variable is a bytearray and
621 ** has no string representation. */
622 Tcl_Size nn;
623 unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &nn);
624 sqlite3_result_blob(pCtx, data, (int)nn, SQLITE_TRANSIENT);
625 }else if( c=='b' && strcmp(zType,"boolean")==0 ){
626 int n;
627 Tcl_GetIntFromObj(0, pVar, &n);
628 sqlite3_result_int(pCtx, n);
629 }else if( c=='d' && strcmp(zType,"double")==0 ){
630 double r;
631 Tcl_GetDoubleFromObj(0, pVar, &r);
632 sqlite3_result_double(pCtx, r);
633 }else if( (c=='w' && strcmp(zType,"wideInt")==0) ||
634 (c=='i' && strcmp(zType,"int")==0) ){
635 Tcl_WideInt v;
636 Tcl_GetWideIntFromObj(0, pVar, &v);
637 sqlite3_result_int64(pCtx, v);
638 }else{
639 Tcl_Size nn;
640 unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &nn);
641 sqlite3_result_text(pCtx, (char*)data, (int)nn, SQLITE_TRANSIENT);
642 }
643 }
644 }
645
646 static void xF5tDestroy(void *pCtx){
647 F5tFunction *p = (F5tFunction*)pCtx;
648 Tcl_DecrRefCount(p->pScript);
649 ckfree((char *)p);
650 }
651
652 /*
653 ** sqlite3_fts5_create_function DB NAME SCRIPT
654 **
655 ** Description...
656 */
657 static int SQLITE_TCLAPI f5tCreateFunction(
658 void * clientData,
659 Tcl_Interp *interp,
660 int objc,
661 Tcl_Obj *CONST objv[]
662 ){
663 char *zName;
664 Tcl_Obj *pScript;
665 sqlite3 *db = 0;
666 fts5_api *pApi = 0;
667 F5tFunction *pCtx = 0;
668 int rc;
669
670 if( objc!=4 ){
671 Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
672 return TCL_ERROR;
673 }
674 if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
675
676 zName = Tcl_GetString(objv[2]);
677 pScript = objv[3];
678 pCtx = (F5tFunction*)ckalloc(sizeof(F5tFunction));
679 pCtx->interp = interp;
680 pCtx->pScript = pScript;
681 Tcl_IncrRefCount(pScript);
682
683 rc = pApi->xCreateFunction(
684 pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy
685 );
686 if( rc!=SQLITE_OK ){
687 Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
688 return TCL_ERROR;
689 }
690
691 return TCL_OK;
692 }
693
694 typedef struct F5tTokenizeCtx F5tTokenizeCtx;
695 struct F5tTokenizeCtx {
696 Tcl_Obj *pRet;
697 int bSubst;
698 const char *zInput;
699 };
700
701 static int xTokenizeCb2(
702 void *pCtx,
703 int tflags,
704 const char *zToken, int nToken,
705 int iStart, int iEnd
706 ){
707 F5tTokenizeCtx *p = (F5tTokenizeCtx*)pCtx;
708 if( p->bSubst ){
709 Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken));
710 Tcl_ListObjAppendElement(
711 0, p->pRet, Tcl_NewStringObj(&p->zInput[iStart], iEnd-iStart)
712 );
713 }else{
714 Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewStringObj(zToken, nToken));
715 Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewIntObj(iStart));
716 Tcl_ListObjAppendElement(0, p->pRet, Tcl_NewIntObj(iEnd));
717 }
718 return SQLITE_OK;
719 }
720
721
722 /*
723 ** sqlite3_fts5_tokenize DB TOKENIZER TEXT
724 **
725 ** Description...
726 */
727 static int SQLITE_TCLAPI f5tTokenize(
728 void * clientData,
729 Tcl_Interp *interp,
730 int objc,
731 Tcl_Obj *CONST objv[]
732 ){
733 char *pCopy = 0;
734 char *zText = 0;
735 Tcl_Size nText = 0;
736 sqlite3 *db = 0;
737 fts5_api *pApi = 0;
738 Fts5Tokenizer *pTok = 0;
739 fts5_tokenizer tokenizer;
740 Tcl_Obj *pRet = 0;
741 void *pUserdata;
742 int rc;
743
744 Tcl_Size nArg;
745 const char **azArg;
746 F5tTokenizeCtx ctx;
747
748 if( objc!=4 && objc!=5 ){
749 Tcl_WrongNumArgs(interp, 1, objv, "?-subst? DB NAME TEXT");
750 return TCL_ERROR;
751 }
752 if( objc==5 ){
753 char *zOpt = Tcl_GetString(objv[1]);
754 if( strcmp("-subst", zOpt) ){
755 Tcl_AppendResult(interp, "unrecognized option: ", zOpt, (char*)0);
756 return TCL_ERROR;
757 }
758 }
759 if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ) return TCL_ERROR;
760 if( Tcl_SplitList(interp, Tcl_GetString(objv[objc-2]), &nArg, &azArg) ){
761 return TCL_ERROR;
762 }
763 if( nArg==0 ){
764 Tcl_AppendResult(interp, "no such tokenizer: ", (char*)0);
765 Tcl_Free((void*)azArg);
766 return TCL_ERROR;
767 }
768 zText = Tcl_GetStringFromObj(objv[objc-1], &nText);
769
770 rc = pApi->xFindTokenizer(pApi, azArg[0], &pUserdata, &tokenizer);
771 if( rc!=SQLITE_OK ){
772 Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], (char*)0);
773 return TCL_ERROR;
774 }
775
776 rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok);
777 if( rc!=SQLITE_OK ){
778 Tcl_AppendResult(interp, "error in tokenizer.xCreate()", (char*)0);
779 return TCL_ERROR;
780 }
781
782 if( nText>0 ){
783 pCopy = sqlite3_malloc64(nText);
784 if( pCopy==0 ){
785 tokenizer.xDelete(pTok);
786 Tcl_AppendResult(interp, "error in sqlite3_malloc()", (char*)0);
787 return TCL_ERROR;
788 }else{
789 memcpy(pCopy, zText, nText);
790 }
791 }
792
793 pRet = Tcl_NewObj();
794 Tcl_IncrRefCount(pRet);
795 ctx.bSubst = (objc==5);
796 ctx.pRet = pRet;
797 ctx.zInput = pCopy;
798 rc = tokenizer.xTokenize(
799 pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, pCopy,(int)nText, xTokenizeCb2
800 );
801 tokenizer.xDelete(pTok);
802 sqlite3_free(pCopy);
803 if( rc!=SQLITE_OK ){
804 Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", (char*)0);
805 Tcl_DecrRefCount(pRet);
806 return TCL_ERROR;
807 }
808
809 Tcl_Free((void*)azArg);
810 Tcl_SetObjResult(interp, pRet);
811 Tcl_DecrRefCount(pRet);
812 return TCL_OK;
813 }
814
815 /*************************************************************************
816 ** Start of tokenizer wrapper.
817 */
818
819 typedef struct F5tTokenizerContext F5tTokenizerContext;
820 typedef struct F5tTokenizerCb F5tTokenizerCb;
821 typedef struct F5tTokenizerModule F5tTokenizerModule;
822 typedef struct F5tTokenizerInstance F5tTokenizerInstance;
823
824 struct F5tTokenizerContext {
825 void *pCtx;
826 int (*xToken)(void*, int, const char*, int, int, int);
827 F5tTokenizerInstance *pInst;
828 };
829
830 struct F5tTokenizerModule {
831 Tcl_Interp *interp;
832 Tcl_Obj *pScript;
833 void *pParentCtx;
834 fts5_tokenizer_v2 parent_v2;
835 fts5_tokenizer parent;
836 F5tTokenizerContext *pContext;
837 };
838
839 /*
840 ** zLocale:
841 ** Within a call to xTokenize_v2(), pLocale/nLocale store the locale
842 ** passed to the call by fts5. This can be retrieved by a Tcl tokenize
843 ** script using [sqlite3_fts5_locale].
844 */
845 struct F5tTokenizerInstance {
846 Tcl_Interp *interp;
847 Tcl_Obj *pScript;
848 F5tTokenizerModule *pModule;
849 Fts5Tokenizer *pParent;
850 F5tTokenizerContext *pContext;
851 const char *pLocale;
852 int nLocale;
853 };
854
855 static int f5tTokenizerCreate(
856 void *pCtx,
857 const char **azArg,
858 int nArg,
859 Fts5Tokenizer **ppOut
860 ){
861 Fts5Tokenizer *pParent = 0;
862 F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
863 Tcl_Obj *pEval;
864 int rc = TCL_OK;
865 int i;
866
867 assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 );
868 if( pMod->parent_v2.xCreate ){
869 rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent);
870 }
871 if( pMod->parent.xCreate ){
872 rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent);
873 }
874
875 pEval = Tcl_DuplicateObj(pMod->pScript);
876 Tcl_IncrRefCount(pEval);
877 for(i=0; rc==TCL_OK && i<nArg; i++){
878 Tcl_Obj *pObj = Tcl_NewStringObj(azArg[i], -1);
879 rc = Tcl_ListObjAppendElement(pMod->interp, pEval, pObj);
880 }
881
882 if( rc==TCL_OK ){
883 rc = Tcl_EvalObjEx(pMod->interp, pEval, TCL_GLOBAL_ONLY);
884 }
885 Tcl_DecrRefCount(pEval);
886
887 if( rc==TCL_OK ){
888 F5tTokenizerInstance *pInst;
889 pInst = (F5tTokenizerInstance*)ckalloc(sizeof(F5tTokenizerInstance));
890 memset(pInst, 0, sizeof(F5tTokenizerInstance));
891 pInst->interp = pMod->interp;
892 pInst->pScript = Tcl_GetObjResult(pMod->interp);
893 pInst->pContext = pMod->pContext;
894 pInst->pParent = pParent;
895 pInst->pModule = pMod;
896 Tcl_IncrRefCount(pInst->pScript);
897 *ppOut = (Fts5Tokenizer*)pInst;
898 }
899
900 return rc;
901 }
902
903
904 static void f5tTokenizerDelete(Fts5Tokenizer *p){
905 F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
906 if( pInst ){
907 if( pInst->pParent ){
908 if( pInst->pModule->parent_v2.xDelete ){
909 pInst->pModule->parent_v2.xDelete(pInst->pParent);
910 }else{
911 pInst->pModule->parent.xDelete(pInst->pParent);
912 }
913 }
914 Tcl_DecrRefCount(pInst->pScript);
915 ckfree((char *)pInst);
916 }
917 }
918
919
920 static int f5tTokenizerReallyTokenize(
921 Fts5Tokenizer *p,
922 void *pCtx,
923 int flags,
924 const char *pText, int nText,
925 int (*xToken)(void*, int, const char*, int, int, int)
926 ){
927 F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
928 F5tTokenizerInstance *pOldInst = 0;
929 void *pOldCtx;
930 int (*xOldToken)(void*, int, const char*, int, int, int);
931 Tcl_Obj *pEval;
932 int rc;
933 const char *zFlags;
934
935 pOldCtx = pInst->pContext->pCtx;
936 xOldToken = pInst->pContext->xToken;
937 pOldInst = pInst->pContext->pInst;
938
939 pInst->pContext->pCtx = pCtx;
940 pInst->pContext->xToken = xToken;
941 pInst->pContext->pInst = pInst;
942
943 assert(
944 flags==FTS5_TOKENIZE_DOCUMENT
945 || flags==FTS5_TOKENIZE_AUX
946 || flags==FTS5_TOKENIZE_QUERY
947 || flags==(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)
948 );
949 pEval = Tcl_DuplicateObj(pInst->pScript);
950 Tcl_IncrRefCount(pEval);
951 switch( flags ){
952 case FTS5_TOKENIZE_DOCUMENT:
953 zFlags = "document";
954 break;
955 case FTS5_TOKENIZE_AUX:
956 zFlags = "aux";
957 break;
958 case FTS5_TOKENIZE_QUERY:
959 zFlags = "query";
960 break;
961 case (FTS5_TOKENIZE_PREFIX | FTS5_TOKENIZE_QUERY):
962 zFlags = "prefixquery";
963 break;
964 default:
965 assert( 0 );
966 zFlags = "invalid";
967 break;
968 }
969
970 Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(zFlags, -1));
971 Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(pText,nText));
972 rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY);
973 Tcl_DecrRefCount(pEval);
974
975 pInst->pContext->pCtx = pOldCtx;
976 pInst->pContext->xToken = xOldToken;
977 pInst->pContext->pInst = pOldInst;
978 return rc;
979 }
980
981 typedef struct CallbackCtx CallbackCtx;
982 struct CallbackCtx {
983 Fts5Tokenizer *p;
984 void *pCtx;
985 int flags;
986 int (*xToken)(void*, int, const char*, int, int, int);
987 };
988
989 static int f5tTokenizeCallback(
990 void *pCtx,
991 int tflags,
992 const char *z, int n,
993 int iStart, int iEnd
994 ){
995 CallbackCtx *p = (CallbackCtx*)pCtx;
996 return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken);
997 }
998
999 static int f5tTokenizerTokenize_v2(
1000 Fts5Tokenizer *p,
1001 void *pCtx,
1002 int flags,
1003 const char *pText, int nText,
1004 const char *pLoc, int nLoc,
1005 int (*xToken)(void*, int, const char*, int, int, int)
1006 ){
1007 int rc = SQLITE_OK;
1008 F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
1009
1010 pInst->pLocale = pLoc;
1011 pInst->nLocale = nLoc;
1012
1013 if( pInst->pParent ){
1014 CallbackCtx ctx;
1015 ctx.p = p;
1016 ctx.pCtx = pCtx;
1017 ctx.flags = flags;
1018 ctx.xToken = xToken;
1019 if( pInst->pModule->parent_v2.xTokenize ){
1020 rc = pInst->pModule->parent_v2.xTokenize(
1021 pInst->pParent, (void*)&ctx, flags, pText, nText,
1022 pLoc, nLoc, f5tTokenizeCallback
1023 );
1024 }else{
1025 rc = pInst->pModule->parent.xTokenize(
1026 pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback
1027 );
1028 }
1029 }else{
1030 rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken);
1031 }
1032
1033 pInst->pLocale = 0;
1034 pInst->nLocale = 0;
1035 return rc;
1036 }
1037 static int f5tTokenizerTokenize(
1038 Fts5Tokenizer *p,
1039 void *pCtx,
1040 int flags,
1041 const char *pText, int nText,
1042 int (*xToken)(void*, int, const char*, int, int, int)
1043 ){
1044 return f5tTokenizerTokenize_v2(p, pCtx, flags, pText, nText, 0, 0, xToken);
1045 }
1046
1047 /*
1048 ** sqlite3_fts5_locale
1049 */
1050 static int SQLITE_TCLAPI f5tTokenizerLocale(
1051 void * clientData,
1052 Tcl_Interp *interp,
1053 int objc,
1054 Tcl_Obj *CONST objv[]
1055 ){
1056 F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
1057
1058 if( objc!=1 ){
1059 Tcl_WrongNumArgs(interp, 1, objv, "");
1060 return TCL_ERROR;
1061 }
1062
1063 if( p->xToken==0 ){
1064 Tcl_AppendResult(interp,
1065 "sqlite3_fts5_locale may only be used by tokenizer callback", (char*)0
1066 );
1067 return TCL_ERROR;
1068 }
1069
1070 Tcl_SetObjResult(interp,
1071 Tcl_NewStringObj(p->pInst->pLocale, p->pInst->nLocale)
1072 );
1073 return TCL_OK;
1074 }
1075
1076 /*
1077 ** sqlite3_fts5_token ?-colocated? TEXT START END
1078 */
1079 static int SQLITE_TCLAPI f5tTokenizerReturn(
1080 void * clientData,
1081 Tcl_Interp *interp,
1082 int objc,
1083 Tcl_Obj *CONST objv[]
1084 ){
1085 F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
1086 int iStart;
1087 int iEnd;
1088 Tcl_Size nToken;
1089 int tflags = 0;
1090 char *zToken;
1091 int rc;
1092
1093 if( objc==5 ){
1094 Tcl_Size nArg;
1095 char *zArg = Tcl_GetStringFromObj(objv[1], &nArg);
1096 if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){
1097 tflags |= FTS5_TOKEN_COLOCATED;
1098 }else{
1099 goto usage;
1100 }
1101 }else if( objc!=4 ){
1102 goto usage;
1103 }
1104
1105 zToken = Tcl_GetStringFromObj(objv[objc-3], &nToken);
1106 if( Tcl_GetIntFromObj(interp, objv[objc-2], &iStart)
1107 || Tcl_GetIntFromObj(interp, objv[objc-1], &iEnd)
1108 ){
1109 return TCL_ERROR;
1110 }
1111
1112 if( p->xToken==0 ){
1113 Tcl_AppendResult(interp,
1114 "sqlite3_fts5_token may only be used by tokenizer callback", (char*)0
1115 );
1116 return TCL_ERROR;
1117 }
1118
1119 rc = p->xToken(p->pCtx, tflags, zToken, (int)nToken, iStart, iEnd);
1120 Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
1121 return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
1122
1123 usage:
1124 Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END");
1125 return TCL_ERROR;
1126 }
1127
1128 static void f5tDelTokenizer(void *pCtx){
1129 F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
1130 Tcl_DecrRefCount(pMod->pScript);
1131 ckfree((char *)pMod);
1132 }
1133
1134 /*
1135 ** sqlite3_fts5_create_tokenizer DB NAME SCRIPT
1136 **
1137 ** Register a tokenizer named NAME implemented by script SCRIPT. When
1138 ** a tokenizer instance is created (fts5_tokenizer.xCreate), any tokenizer
1139 ** arguments are appended to SCRIPT and the result executed.
1140 **
1141 ** The value returned by (SCRIPT + args) is itself a tcl script. This
1142 ** script - call it SCRIPT2 - is executed to tokenize text using the
1143 ** tokenizer instance "returned" by SCRIPT. Specifically, to tokenize
1144 ** text SCRIPT2 is invoked with a single argument appended to it - the
1145 ** text to tokenize.
1146 **
1147 ** SCRIPT2 should invoke the [sqlite3_fts5_token] command once for each
1148 ** token within the tokenized text.
1149 */
1150 static int SQLITE_TCLAPI f5tCreateTokenizer(
1151 ClientData clientData,
1152 Tcl_Interp *interp,
1153 int objc,
1154 Tcl_Obj *CONST objv[]
1155 ){
1156 F5tTokenizerContext *pContext = (F5tTokenizerContext*)clientData;
1157 sqlite3 *db;
1158 fts5_api *pApi;
1159 char *zName;
1160 Tcl_Obj *pScript;
1161 F5tTokenizerModule *pMod;
1162 int rc = SQLITE_OK;
1163 int bV2 = 0; /* True to use _v2 API */
1164 int iVersion = 2; /* Value for _v2.iVersion */
1165 const char *zParent = 0; /* Name of parent tokenizer, if any */
1166 int ii = 0;
1167
1168 if( objc<4 ){
1169 Tcl_WrongNumArgs(interp, 1, objv, "?OPTIONS? DB NAME SCRIPT");
1170 return TCL_ERROR;
1171 }
1172
1173 /* Parse any options. Set stack variables bV2 and zParent. */
1174 for(ii=1; ii<objc-3; ii++){
1175 int iOpt = 0;
1176 const char *azOpt[] = { "-v2", "-parent", "-version", 0 };
1177 if( Tcl_GetIndexFromObj(interp, objv[ii], azOpt, "OPTION", 0, &iOpt) ){
1178 return TCL_ERROR;
1179 }
1180 switch( iOpt ){
1181 case 0: /* -v2 */ {
1182 bV2 = 1;
1183 break;
1184 }
1185 case 1: /* -parent */ {
1186 ii++;
1187 if( ii==objc-3 ){
1188 Tcl_AppendResult(
1189 interp, "option requires an argument: -parent", (char*)0
1190 );
1191 return TCL_ERROR;
1192 }
1193 zParent = Tcl_GetString(objv[ii]);
1194 break;
1195 }
1196 case 2: /* -version */ {
1197 ii++;
1198 if( ii==objc-3 ){
1199 Tcl_AppendResult(
1200 interp, "option requires an argument: -version", (char*)0
1201 );
1202 return TCL_ERROR;
1203 }
1204 if( Tcl_GetIntFromObj(interp, objv[ii], &iVersion) ){
1205 return TCL_ERROR;
1206 }
1207 break;
1208 }
1209 default:
1210 assert( 0 );
1211 break;
1212 }
1213 }
1214
1215 if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ){
1216 return TCL_ERROR;
1217 }
1218 zName = Tcl_GetString(objv[objc-2]);
1219 pScript = objv[objc-1];
1220
1221 pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule));
1222 memset(pMod, 0, sizeof(F5tTokenizerModule));
1223 pMod->interp = interp;
1224 pMod->pScript = pScript;
1225 Tcl_IncrRefCount(pScript);
1226 pMod->pContext = pContext;
1227 if( zParent ){
1228 if( bV2 ){
1229 fts5_tokenizer_v2 *pParent = 0;
1230 rc = pApi->xFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent);
1231 if( rc==SQLITE_OK ){
1232 memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2));
1233 pMod->parent_v2.xDelete(0);
1234 }
1235 }else{
1236 rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent);
1237 if( rc==SQLITE_OK ){
1238 pMod->parent.xDelete(0);
1239 }
1240 }
1241 }
1242
1243 if( rc==SQLITE_OK ){
1244 void *pModCtx = (void*)pMod;
1245 if( bV2==0 ){
1246 fts5_tokenizer t;
1247 t.xCreate = f5tTokenizerCreate;
1248 t.xTokenize = f5tTokenizerTokenize;
1249 t.xDelete = f5tTokenizerDelete;
1250 rc = pApi->xCreateTokenizer(pApi, zName, pModCtx, &t, f5tDelTokenizer);
1251 }else{
1252 fts5_tokenizer_v2 t2;
1253 memset(&t2, 0, sizeof(t2));
1254 t2.iVersion = iVersion;
1255 t2.xCreate = f5tTokenizerCreate;
1256 t2.xTokenize = f5tTokenizerTokenize_v2;
1257 t2.xDelete = f5tTokenizerDelete;
1258 rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer);
1259 }
1260 }
1261
1262 if( rc!=SQLITE_OK ){
1263 Tcl_AppendResult(interp, (
1264 bV2 ? "error in fts5_api.xCreateTokenizer_v2()"
1265 : "error in fts5_api.xCreateTokenizer()"
1266 ), (char*)0);
1267 return TCL_ERROR;
1268 }
1269
1270 return TCL_OK;
1271 }
1272
1273 static void SQLITE_TCLAPI xF5tFree(ClientData clientData){
1274 ckfree(clientData);
1275 }
1276
1277 /*
1278 ** sqlite3_fts5_may_be_corrupt BOOLEAN
1279 **
1280 ** Set or clear the global "may-be-corrupt" flag. Return the old value.
1281 */
1282 static int SQLITE_TCLAPI f5tMayBeCorrupt(
1283 void * clientData,
1284 Tcl_Interp *interp,
1285 int objc,
1286 Tcl_Obj *CONST objv[]
1287 ){
1288 #ifdef SQLITE_DEBUG
1289 int bOld = sqlite3_fts5_may_be_corrupt;
1290
1291 if( objc!=2 && objc!=1 ){
1292 Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?");
1293 return TCL_ERROR;
1294 }
1295 if( objc==2 ){
1296 int bNew;
1297 if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR;
1298 sqlite3_fts5_may_be_corrupt = bNew;
1299 }
1300
1301 Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld));
1302 #endif
1303 return TCL_OK;
1304 }
1305
1306
1307 static unsigned int f5t_fts5HashKey(int nSlot, const char *p, int n){
1308 int i;
1309 unsigned int h = 13;
1310 for(i=n-1; i>=0; i--){
1311 h = (h << 3) ^ h ^ p[i];
1312 }
1313 return (h % nSlot);
1314 }
1315
1316 static int SQLITE_TCLAPI f5tTokenHash(
1317 void * clientData,
1318 Tcl_Interp *interp,
1319 int objc,
1320 Tcl_Obj *CONST objv[]
1321 ){
1322 char *z;
1323 Tcl_Size n;
1324 unsigned int iVal;
1325 int nSlot;
1326
1327 if( objc!=3 ){
1328 Tcl_WrongNumArgs(interp, 1, objv, "NSLOT TOKEN");
1329 return TCL_ERROR;
1330 }
1331 if( Tcl_GetIntFromObj(interp, objv[1], &nSlot) ){
1332 return TCL_ERROR;
1333 }
1334 z = Tcl_GetStringFromObj(objv[2], &n);
1335
1336 iVal = f5t_fts5HashKey(nSlot, z, (int)n);
1337 Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
1338 return TCL_OK;
1339 }
1340
1341 static int SQLITE_TCLAPI f5tRegisterMatchinfo(
1342 void * clientData,
1343 Tcl_Interp *interp,
1344 int objc,
1345 Tcl_Obj *CONST objv[]
1346 ){
1347 int rc;
1348 sqlite3 *db = 0;
1349
1350 if( objc!=2 ){
1351 Tcl_WrongNumArgs(interp, 1, objv, "DB");
1352 return TCL_ERROR;
1353 }
1354 if( f5tDbPointer(interp, objv[1], &db) ){
1355 return TCL_ERROR;
1356 }
1357
1358 rc = sqlite3Fts5TestRegisterMatchinfo(db);
1359 if( rc!=SQLITE_OK ){
1360 Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
1361 return TCL_ERROR;
1362 }
1363 return TCL_OK;
1364 }
1365
1366 static int SQLITE_TCLAPI f5tRegisterTok(
1367 void * clientData,
1368 Tcl_Interp *interp,
1369 int objc,
1370 Tcl_Obj *CONST objv[]
1371 ){
1372 int rc;
1373 sqlite3 *db = 0;
1374 fts5_api *pApi = 0;
1375
1376 if( objc!=2 ){
1377 Tcl_WrongNumArgs(interp, 1, objv, "DB");
1378 return TCL_ERROR;
1379 }
1380 if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
1381 return TCL_ERROR;
1382 }
1383
1384 rc = sqlite3Fts5TestRegisterTok(db, pApi);
1385 if( rc!=SQLITE_OK ){
1386 Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
1387 return TCL_ERROR;
1388 }
1389 return TCL_OK;
1390 }
1391
1392 typedef struct OriginTextCtx OriginTextCtx;
1393 struct OriginTextCtx {
1394 sqlite3 *db;
1395 fts5_api *pApi;
1396 };
1397
1398 typedef struct OriginTextTokenizer OriginTextTokenizer;
1399 struct OriginTextTokenizer {
1400 Fts5Tokenizer *pTok; /* Underlying tokenizer object */
1401 fts5_tokenizer tokapi; /* API implementation for pTok */
1402 };
1403
1404 /*
1405 ** Delete the OriginTextCtx object indicated by the only argument.
1406 */
1407 static void f5tOrigintextTokenizerDelete(void *pCtx){
1408 OriginTextCtx *p = (OriginTextCtx*)pCtx;
1409 ckfree((char*)p);
1410 }
1411
1412 static int f5tOrigintextCreate(
1413 void *pCtx,
1414 const char **azArg,
1415 int nArg,
1416 Fts5Tokenizer **ppOut
1417 ){
1418 OriginTextCtx *p = (OriginTextCtx*)pCtx;
1419 OriginTextTokenizer *pTok = 0;
1420 void *pTokCtx = 0;
1421 int rc = SQLITE_OK;
1422
1423 pTok = (OriginTextTokenizer*)sqlite3_malloc64(sizeof(OriginTextTokenizer));
1424 if( pTok==0 ){
1425 rc = SQLITE_NOMEM;
1426 }else if( nArg<1 ){
1427 rc = SQLITE_ERROR;
1428 }else{
1429 /* Locate the underlying tokenizer */
1430 rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi);
1431 }
1432
1433 /* Create the new tokenizer instance */
1434 if( rc==SQLITE_OK ){
1435 rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok);
1436 }
1437
1438 if( rc!=SQLITE_OK ){
1439 sqlite3_free(pTok);
1440 pTok = 0;
1441 }
1442 *ppOut = (Fts5Tokenizer*)pTok;
1443 return rc;
1444 }
1445
1446 static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){
1447 OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
1448 if( p->pTok ){
1449 p->tokapi.xDelete(p->pTok);
1450 }
1451 sqlite3_free(p);
1452 }
1453
1454 typedef struct OriginTextCb OriginTextCb;
1455 struct OriginTextCb {
1456 void *pCtx;
1457 const char *pText;
1458 int nText;
1459 int (*xToken)(void *, int, const char *, int, int, int);
1460
1461 char *aBuf; /* Buffer to use */
1462 int nBuf; /* Allocated size of aBuf[] */
1463 };
1464
1465 static int xOriginToken(
1466 void *pCtx, /* Copy of 2nd argument to xTokenize() */
1467 int tflags, /* Mask of FTS5_TOKEN_* flags */
1468 const char *pToken, /* Pointer to buffer containing token */
1469 int nToken, /* Size of token in bytes */
1470 int iStart, /* Byte offset of token within input text */
1471 int iEnd /* Byte offset of end of token within input */
1472 ){
1473 OriginTextCb *p = (OriginTextCb*)pCtx;
1474 int ret = 0;
1475
1476 if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){
1477 /* Token exactly matches document text. Pass it through as is. */
1478 ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
1479 }else{
1480 int nReq = nToken + 1 + (iEnd-iStart);
1481 if( nReq>p->nBuf ){
1482 sqlite3_free(p->aBuf);
1483 p->aBuf = sqlite3_malloc64(nReq*2);
1484 if( p->aBuf==0 ) return SQLITE_NOMEM;
1485 p->nBuf = nReq*2;
1486 }
1487
1488 memcpy(p->aBuf, pToken, nToken);
1489 p->aBuf[nToken] = '\0';
1490 memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart);
1491 ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd);
1492 }
1493
1494 return ret;
1495 }
1496
1497
1498 static int f5tOrigintextTokenize(
1499 Fts5Tokenizer *pTokenizer,
1500 void *pCtx,
1501 int flags, /* Mask of FTS5_TOKENIZE_* flags */
1502 const char *pText, int nText,
1503 int (*xToken)(void *, int, const char *, int, int, int)
1504 ){
1505 OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
1506 OriginTextCb cb;
1507 int ret;
1508
1509 memset(&cb, 0, sizeof(cb));
1510 cb.pCtx = pCtx;
1511 cb.pText = pText;
1512 cb.nText = nText;
1513 cb.xToken = xToken;
1514
1515 ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken);
1516 sqlite3_free(cb.aBuf);
1517 return ret;
1518 }
1519
1520 /*
1521 ** sqlite3_fts5_register_origintext DB
1522 **
1523 ** Description...
1524 */
1525 static int SQLITE_TCLAPI f5tRegisterOriginText(
1526 void * clientData,
1527 Tcl_Interp *interp,
1528 int objc,
1529 Tcl_Obj *CONST objv[]
1530 ){
1531 sqlite3 *db = 0;
1532 fts5_api *pApi = 0;
1533 int rc;
1534 fts5_tokenizer tok = {0, 0, 0};
1535 OriginTextCtx *pCtx = 0;
1536
1537 if( objc!=2 ){
1538 Tcl_WrongNumArgs(interp, 1, objv, "DB");
1539 return TCL_ERROR;
1540 }
1541 if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
1542
1543 pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx));
1544 pCtx->db = db;
1545 pCtx->pApi = pApi;
1546
1547 tok.xCreate = f5tOrigintextCreate;
1548 tok.xDelete = f5tOrigintextDelete;
1549 tok.xTokenize = f5tOrigintextTokenize;
1550 rc = pApi->xCreateTokenizer(
1551 pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete
1552 );
1553
1554 Tcl_ResetResult(interp);
1555 if( rc!=SQLITE_OK ){
1556 Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0);
1557 return TCL_ERROR;
1558 }
1559 return TCL_OK;
1560 }
1561
1562 /*
1563 ** This function is used to DROP an fts5 table. It works even if the data
1564 ** structures fts5 stores within the database are corrupt, which sometimes
1565 ** prevents a straight "DROP TABLE" command from succeeding.
1566 **
1567 ** The first parameter is the database handle to use for the DROP TABLE
1568 ** operation. The second is the name of the database to drop the fts5 table
1569 ** from (i.e. "main", "temp" or the name of an attached database). The
1570 ** third parameter is the name of the fts5 table to drop.
1571 **
1572 ** SQLITE_OK is returned if the table is successfully dropped. Or, if an
1573 ** error occurs, an SQLite error code.
1574 */
1575 static int sqlite3_fts5_drop_corrupt_table(
1576 sqlite3 *db, /* Database handle */
1577 const char *zDb, /* Database name ("main", "temp" etc.) */
1578 const char *zTab /* Name of fts5 table to drop */
1579 ){
1580 int rc = SQLITE_OK;
1581 int bDef = 0;
1582
1583 rc = sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDef);
1584 if( rc==SQLITE_OK ){
1585 char *zScript = sqlite3_mprintf(
1586 "DELETE FROM %Q.'%q_data';"
1587 "DELETE FROM %Q.'%q_config';"
1588 "INSERT INTO %Q.'%q_data' VALUES(10, X'0000000000');"
1589 "INSERT INTO %Q.'%q_config' VALUES('version', 4);"
1590 "DROP TABLE %Q.'%q';",
1591 zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab
1592 );
1593
1594 if( zScript==0 ){
1595 rc = SQLITE_NOMEM;
1596 }else{
1597 if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0);
1598 rc = sqlite3_exec(db, zScript, 0, 0, 0);
1599 if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0);
1600 sqlite3_free(zScript);
1601 }
1602 }
1603
1604 return rc;
1605 }
1606
1607 /*
1608 ** sqlite3_fts5_drop_corrupt_table DB DATABASE TABLE
1609 **
1610 ** Description...
1611 */
1612 static int SQLITE_TCLAPI f5tDropCorruptTable(
1613 void * clientData,
1614 Tcl_Interp *interp,
1615 int objc,
1616 Tcl_Obj *CONST objv[]
1617 ){
1618 sqlite3 *db = 0;
1619 const char *zDb = 0;
1620 const char *zTab = 0;
1621 int rc = SQLITE_OK;
1622
1623 if( objc!=4 ){
1624 Tcl_WrongNumArgs(interp, 1, objv, "DB DATABASE TABLE");
1625 return TCL_ERROR;
1626 }
1627 if( f5tDbPointer(interp, objv[1], &db) ){
1628 return TCL_ERROR;
1629 }
1630 zDb = Tcl_GetString(objv[2]);
1631 zTab = Tcl_GetString(objv[3]);
1632
1633 rc = sqlite3_fts5_drop_corrupt_table(db, zDb, zTab);
1634 if( rc!=SQLITE_OK ){
1635 Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0);
1636 return TCL_ERROR;
1637 }
1638
1639 return TCL_OK;
1640 }
1641
1642 /*
1643 ** Free a buffer returned to SQLite by the str() function.
1644 */
1645 void f5tFree(void *p){
1646 char *x = (char *)p;
1647 ckfree(&x[-8]);
1648 }
1649
1650 /*
1651 ** Implementation of str().
1652 */
1653 void f5tStrFunc(sqlite3_context *pCtx, int nArg, sqlite3_value **apArg){
1654 const char *zText = 0;
1655 assert( nArg==1 );
1656
1657 zText = (const char*)sqlite3_value_text(apArg[0]);
1658 if( zText ){
1659 sqlite3_int64 nText = strlen(zText);
1660 char *zCopy = (char*)ckalloc(nText+8);
1661 if( zCopy==0 ){
1662 sqlite3_result_error_nomem(pCtx);
1663 }else{
1664 zCopy += 8;
1665 memcpy(zCopy, zText, nText);
1666 sqlite3_result_text64(pCtx, zCopy, nText, f5tFree, SQLITE_UTF8);
1667 }
1668 }
1669 }
1670
1671 /*
1672 ** sqlite3_fts5_register_str DB
1673 **
1674 ** Register the str() function with database handle DB. str() interprets
1675 ** its only argument as text and returns a copy of the value in a
1676 ** non-nul-terminated buffer.
1677 */
1678 static int SQLITE_TCLAPI f5tRegisterStr(
1679 void * clientData,
1680 Tcl_Interp *interp,
1681 int objc,
1682 Tcl_Obj *CONST objv[]
1683 ){
1684 sqlite3 *db = 0;
1685
1686 if( objc!=2 ){
1687 Tcl_WrongNumArgs(interp, 1, objv, "DB");
1688 return TCL_ERROR;
1689 }
1690 if( f5tDbPointer(interp, objv[1], &db) ){
1691 return TCL_ERROR;
1692 }
1693
1694 sqlite3_create_function(db, "str", 1, SQLITE_UTF8, 0, f5tStrFunc, 0, 0);
1695
1696 return TCL_OK;
1697 }
1698
1699 /*
1700 ** Entry point.
1701 */
1702 int Fts5tcl_Init(Tcl_Interp *interp){
1703 static struct Cmd {
1704 char *zName;
1705 Tcl_ObjCmdProc *xProc;
1706 int bTokenizeCtx;
1707 } aCmd[] = {
1708 { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 },
1709 { "sqlite3_fts5_token", f5tTokenizerReturn, 1 },
1710 { "sqlite3_fts5_locale", f5tTokenizerLocale, 1 },
1711 { "sqlite3_fts5_tokenize", f5tTokenize, 0 },
1712 { "sqlite3_fts5_create_function", f5tCreateFunction, 0 },
1713 { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 },
1714 { "sqlite3_fts5_token_hash", f5tTokenHash, 0 },
1715 { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
1716 { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
1717 { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 },
1718 { "sqlite3_fts5_drop_corrupt_table", f5tDropCorruptTable, 0 },
1719 { "sqlite3_fts5_register_str", f5tRegisterStr, 0 }
1720 };
1721 int i;
1722 F5tTokenizerContext *pContext;
1723
1724 pContext = (F5tTokenizerContext*)ckalloc(sizeof(F5tTokenizerContext));
1725 memset(pContext, 0, sizeof(*pContext));
1726
1727 for(i=0; i<sizeof(aCmd)/sizeof(aCmd[0]); i++){
1728 struct Cmd *p = &aCmd[i];
1729 void *pCtx = 0;
1730 if( p->bTokenizeCtx ) pCtx = (void*)pContext;
1731 Tcl_CreateObjCommand(interp, p->zName, p->xProc, pCtx, (i ? 0 : xF5tFree));
1732 }
1733
1734 return TCL_OK;
1735 }
1736 #else /* SQLITE_ENABLE_FTS5 */
1737 int Fts5tcl_Init(Tcl_Interp *interp){
1738 return TCL_OK;
1739 }
1740 #endif /* SQLITE_ENABLE_FTS5 */
1741 #endif /* SQLITE_TEST */