Debian Codegraph @

ext/fts3/fts3_test.c

raw
1 /*
2 ** 2011 Jun 13
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 **
13 ** This file is not part of the production FTS code. It is only used for
14 ** testing. It contains a Tcl command that can be used to test if a document
15 ** matches an FTS NEAR expression.
16 **
17 ** As of March 2012, it also contains a version 1 tokenizer used for testing
18 ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
19 */
20
21 #include "tclsqlite.h"
22 #include <string.h>
23 #include <assert.h>
24
25 #if defined(SQLITE_TEST)
26 #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)
27
28 /* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */
29 #include "fts3Int.h"
30
31 #define NM_MAX_TOKEN 12
32
33 typedef struct NearPhrase NearPhrase;
34 typedef struct NearDocument NearDocument;
35 typedef struct NearToken NearToken;
36
37 struct NearDocument {
38 int nToken; /* Length of token in bytes */
39 NearToken *aToken; /* Token array */
40 };
41
42 struct NearToken {
43 int n; /* Length of token in bytes */
44 const char *z; /* Pointer to token string */
45 };
46
47 struct NearPhrase {
48 int nNear; /* Preceding NEAR value */
49 int nToken; /* Number of tokens in this phrase */
50 NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */
51 };
52
53 static int nm_phrase_match(
54 NearPhrase *p,
55 NearToken *aToken
56 ){
57 int ii;
58
59 for(ii=0; ii<p->nToken; ii++){
60 NearToken *pToken = &p->aToken[ii];
61 if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){
62 if( aToken[ii].n<(pToken->n-1) ) return 0;
63 if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0;
64 }else{
65 if( aToken[ii].n!=pToken->n ) return 0;
66 if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0;
67 }
68 }
69
70 return 1;
71 }
72
73 static int nm_near_chain(
74 int iDir, /* Direction to iterate through aPhrase[] */
75 NearDocument *pDoc, /* Document to match against */
76 int iPos, /* Position at which iPhrase was found */
77 int nPhrase, /* Size of phrase array */
78 NearPhrase *aPhrase, /* Phrase array */
79 int iPhrase /* Index of phrase found */
80 ){
81 int iStart;
82 int iStop;
83 int ii;
84 int nNear;
85 int iPhrase2;
86 NearPhrase *p;
87 NearPhrase *pPrev;
88
89 assert( iDir==1 || iDir==-1 );
90
91 if( iDir==1 ){
92 if( (iPhrase+1)==nPhrase ) return 1;
93 nNear = aPhrase[iPhrase+1].nNear;
94 }else{
95 if( iPhrase==0 ) return 1;
96 nNear = aPhrase[iPhrase].nNear;
97 }
98 pPrev = &aPhrase[iPhrase];
99 iPhrase2 = iPhrase+iDir;
100 p = &aPhrase[iPhrase2];
101
102 iStart = iPos - nNear - p->nToken;
103 iStop = iPos + nNear + pPrev->nToken;
104
105 if( iStart<0 ) iStart = 0;
106 if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken;
107
108 for(ii=iStart; ii<=iStop; ii++){
109 if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
110 if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1;
111 }
112 }
113
114 return 0;
115 }
116
117 static int nm_match_count(
118 NearDocument *pDoc, /* Document to match against */
119 int nPhrase, /* Size of phrase array */
120 NearPhrase *aPhrase, /* Phrase array */
121 int iPhrase /* Index of phrase to count matches for */
122 ){
123 int nOcc = 0;
124 int ii;
125 NearPhrase *p = &aPhrase[iPhrase];
126
127 for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){
128 if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
129 /* Test forward NEAR chain (i>iPhrase) */
130 if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;
131
132 /* Test reverse NEAR chain (i<iPhrase) */
133 if( 0==nm_near_chain(-1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;
134
135 /* This is a real match. Increment the counter. */
136 nOcc++;
137 }
138 }
139
140 return nOcc;
141 }
142
143 /*
144 ** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS?
145 */
146 static int SQLITE_TCLAPI fts3_near_match_cmd(
147 ClientData clientData,
148 Tcl_Interp *interp,
149 int objc,
150 Tcl_Obj *CONST objv[]
151 ){
152 int nTotal = 0;
153 int rc;
154 int ii;
155 int nPhrase;
156 NearPhrase *aPhrase = 0;
157 NearDocument doc = {0, 0};
158 Tcl_Obj **apDocToken;
159 Tcl_Obj *pRet;
160 Tcl_Obj *pPhrasecount = 0;
161
162 Tcl_Obj **apExprToken;
163 Tcl_Size nExprToken;
164 Tcl_Size nn;
165
166 UNUSED_PARAMETER(clientData);
167
168 /* Must have 3 or more arguments. */
169 if( objc<3 || (objc%2)==0 ){
170 Tcl_WrongNumArgs(interp, 1, objv, "DOCUMENT EXPR ?OPTION VALUE?...");
171 rc = TCL_ERROR;
172 goto near_match_out;
173 }
174
175 for(ii=3; ii<objc; ii+=2){
176 enum NM_enum { NM_PHRASECOUNTS };
177 struct TestnmSubcmd {
178 char *zName;
179 enum NM_enum eOpt;
180 } aOpt[] = {
181 { "-phrasecountvar", NM_PHRASECOUNTS },
182 { 0, 0 }
183 };
184 int iOpt;
185 if( Tcl_GetIndexFromObjStruct(
186 interp, objv[ii], aOpt, sizeof(aOpt[0]), "option", 0, &iOpt)
187 ){
188 return TCL_ERROR;
189 }
190
191 switch( aOpt[iOpt].eOpt ){
192 case NM_PHRASECOUNTS:
193 pPhrasecount = objv[ii+1];
194 break;
195 }
196 }
197
198 rc = Tcl_ListObjGetElements(interp, objv[1], &nn, &apDocToken);
199 doc.nToken = (int)nn;
200 if( rc!=TCL_OK ) goto near_match_out;
201 doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken));
202 for(ii=0; ii<doc.nToken; ii++){
203 doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &nn);
204 doc.aToken[ii].n = (int)nn;
205 }
206
207 rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken);
208 if( rc!=TCL_OK ) goto near_match_out;
209
210 nPhrase = (int)(nExprToken + 1) / 2;
211 aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase));
212 memset(aPhrase, 0, nPhrase * sizeof(NearPhrase));
213 for(ii=0; ii<nPhrase; ii++){
214 Tcl_Obj *pPhrase = apExprToken[ii*2];
215 Tcl_Obj **apToken;
216 Tcl_Size nToken;
217 int jj;
218
219 rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken);
220 if( rc!=TCL_OK ) goto near_match_out;
221 if( nToken>NM_MAX_TOKEN ){
222 Tcl_AppendResult(interp, "Too many tokens in phrase", NULL);
223 rc = TCL_ERROR;
224 goto near_match_out;
225 }
226 for(jj=0; jj<(int)nToken; jj++){
227 NearToken *pT = &aPhrase[ii].aToken[jj];
228 pT->z = Tcl_GetStringFromObj(apToken[jj], &nn);
229 pT->n = (int)nn;
230 }
231 aPhrase[ii].nToken = (int)nToken;
232 }
233 for(ii=1; ii<nPhrase; ii++){
234 Tcl_Obj *pNear = apExprToken[2*ii-1];
235 int nNear;
236 rc = Tcl_GetIntFromObj(interp, pNear, &nNear);
237 if( rc!=TCL_OK ) goto near_match_out;
238 aPhrase[ii].nNear = nNear;
239 }
240
241 pRet = Tcl_NewObj();
242 Tcl_IncrRefCount(pRet);
243 for(ii=0; ii<nPhrase; ii++){
244 int nOcc = nm_match_count(&doc, nPhrase, aPhrase, ii);
245 Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(nOcc));
246 nTotal += nOcc;
247 }
248 if( pPhrasecount ){
249 Tcl_ObjSetVar2(interp, pPhrasecount, 0, pRet, 0);
250 }
251 Tcl_DecrRefCount(pRet);
252 Tcl_SetObjResult(interp, Tcl_NewBooleanObj(nTotal>0));
253
254 near_match_out:
255 ckfree((char *)aPhrase);
256 ckfree((char *)doc.aToken);
257 return rc;
258 }
259
260 /*
261 ** Tclcmd: fts3_configure_incr_load ?CHUNKSIZE THRESHOLD?
262 **
263 ** Normally, FTS uses hard-coded values to determine the minimum doclist
264 ** size eligible for incremental loading, and the size of the chunks loaded
265 ** when a doclist is incrementally loaded. This command allows the built-in
266 ** values to be overridden for testing purposes.
267 **
268 ** If present, the first argument is the chunksize in bytes to load doclists
269 ** in. The second argument is the minimum doclist size in bytes to use
270 ** incremental loading with.
271 **
272 ** Whether or not the arguments are present, this command returns a list of
273 ** two integers - the initial chunksize and threshold when the command is
274 ** invoked. This can be used to restore the default behavior after running
275 ** tests. For example:
276 **
277 ** # Override incr-load settings for testing:
278 ** set cfg [fts3_configure_incr_load $new_chunksize $new_threshold]
279 **
280 ** .... run tests ....
281 **
282 ** # Restore initial incr-load settings:
283 ** eval fts3_configure_incr_load $cfg
284 */
285 static int SQLITE_TCLAPI fts3_configure_incr_load_cmd(
286 ClientData clientData,
287 Tcl_Interp *interp,
288 int objc,
289 Tcl_Obj *CONST objv[]
290 ){
291 #ifdef SQLITE_ENABLE_FTS3
292 extern int test_fts3_node_chunksize;
293 extern int test_fts3_node_chunk_threshold;
294 Tcl_Obj *pRet;
295
296 if( objc!=1 && objc!=3 ){
297 Tcl_WrongNumArgs(interp, 1, objv, "?CHUNKSIZE THRESHOLD?");
298 return TCL_ERROR;
299 }
300
301 pRet = Tcl_NewObj();
302 Tcl_IncrRefCount(pRet);
303 Tcl_ListObjAppendElement(
304 interp, pRet, Tcl_NewIntObj(test_fts3_node_chunksize));
305 Tcl_ListObjAppendElement(
306 interp, pRet, Tcl_NewIntObj(test_fts3_node_chunk_threshold));
307
308 if( objc==3 ){
309 int iArg1;
310 int iArg2;
311 if( Tcl_GetIntFromObj(interp, objv[1], &iArg1)
312 || Tcl_GetIntFromObj(interp, objv[2], &iArg2)
313 ){
314 Tcl_DecrRefCount(pRet);
315 return TCL_ERROR;
316 }
317 test_fts3_node_chunksize = iArg1;
318 test_fts3_node_chunk_threshold = iArg2;
319 }
320
321 Tcl_SetObjResult(interp, pRet);
322 Tcl_DecrRefCount(pRet);
323 #endif
324 UNUSED_PARAMETER(clientData);
325 return TCL_OK;
326 }
327
328 #ifdef SQLITE_ENABLE_FTS3
329 /**************************************************************************
330 ** Beginning of test tokenizer code.
331 **
332 ** For language 0, this tokenizer is similar to the default 'simple'
333 ** tokenizer. For other languages L, the following:
334 **
335 ** * Odd numbered languages are case-sensitive. Even numbered
336 ** languages are not.
337 **
338 ** * Language ids 100 or greater are considered an error.
339 **
340 ** The implementation assumes that the input contains only ASCII characters
341 ** (i.e. those that may be encoded in UTF-8 using a single byte).
342 */
343 typedef struct test_tokenizer {
344 sqlite3_tokenizer base;
345 } test_tokenizer;
346
347 typedef struct test_tokenizer_cursor {
348 sqlite3_tokenizer_cursor base;
349 const char *aInput; /* Input being tokenized */
350 int nInput; /* Size of the input in bytes */
351 int iInput; /* Current offset in aInput */
352 int iToken; /* Index of next token to be returned */
353 char *aBuffer; /* Buffer containing current token */
354 int nBuffer; /* Number of bytes allocated at pToken */
355 int iLangid; /* Configured language id */
356 } test_tokenizer_cursor;
357
358 static int testTokenizerCreate(
359 int argc, const char * const *argv,
360 sqlite3_tokenizer **ppTokenizer
361 ){
362 test_tokenizer *pNew;
363 UNUSED_PARAMETER(argc);
364 UNUSED_PARAMETER(argv);
365
366 pNew = sqlite3_malloc(sizeof(test_tokenizer));
367 if( !pNew ) return SQLITE_NOMEM;
368 memset(pNew, 0, sizeof(test_tokenizer));
369
370 *ppTokenizer = (sqlite3_tokenizer *)pNew;
371 return SQLITE_OK;
372 }
373
374 static int testTokenizerDestroy(sqlite3_tokenizer *pTokenizer){
375 test_tokenizer *p = (test_tokenizer *)pTokenizer;
376 sqlite3_free(p);
377 return SQLITE_OK;
378 }
379
380 static int testTokenizerOpen(
381 sqlite3_tokenizer *pTokenizer, /* The tokenizer */
382 const char *pInput, int nBytes, /* String to be tokenized */
383 sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
384 ){
385 int rc = SQLITE_OK; /* Return code */
386 test_tokenizer_cursor *pCsr; /* New cursor object */
387
388 UNUSED_PARAMETER(pTokenizer);
389
390 pCsr = (test_tokenizer_cursor *)sqlite3_malloc(sizeof(test_tokenizer_cursor));
391 if( pCsr==0 ){
392 rc = SQLITE_NOMEM;
393 }else{
394 memset(pCsr, 0, sizeof(test_tokenizer_cursor));
395 pCsr->aInput = pInput;
396 if( nBytes<0 ){
397 pCsr->nInput = (int)strlen(pInput);
398 }else{
399 pCsr->nInput = nBytes;
400 }
401 }
402
403 *ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
404 return rc;
405 }
406
407 static int testTokenizerClose(sqlite3_tokenizer_cursor *pCursor){
408 test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor;
409 sqlite3_free(pCsr->aBuffer);
410 sqlite3_free(pCsr);
411 return SQLITE_OK;
412 }
413
414 static int testIsTokenChar(char c){
415 return (c>='a' && c<='z') || (c>='A' && c<='Z');
416 }
417 static int testTolower(char c){
418 char ret = c;
419 if( ret>='A' && ret<='Z') ret = ret - ('A'-'a');
420 return ret;
421 }
422
423 static int testTokenizerNext(
424 sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by testTokenizerOpen */
425 const char **ppToken, /* OUT: *ppToken is the token text */
426 int *pnBytes, /* OUT: Number of bytes in token */
427 int *piStartOffset, /* OUT: Starting offset of token */
428 int *piEndOffset, /* OUT: Ending offset of token */
429 int *piPosition /* OUT: Position integer of token */
430 ){
431 test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor;
432 int rc = SQLITE_OK;
433 const char *p;
434 const char *pEnd;
435
436 p = &pCsr->aInput[pCsr->iInput];
437 pEnd = &pCsr->aInput[pCsr->nInput];
438
439 /* Skip past any white-space */
440 assert( p<=pEnd );
441 while( p<pEnd && testIsTokenChar(*p)==0 ) p++;
442
443 if( p==pEnd ){
444 rc = SQLITE_DONE;
445 }else{
446 /* Advance to the end of the token */
447 const char *pToken = p;
448 sqlite3_int64 nToken;
449 while( p<pEnd && testIsTokenChar(*p) ) p++;
450 nToken = (sqlite3_int64)(p-pToken);
451
452 /* Copy the token into the buffer */
453 if( nToken>pCsr->nBuffer ){
454 sqlite3_free(pCsr->aBuffer);
455 pCsr->aBuffer = sqlite3_malloc64(nToken);
456 }
457 if( pCsr->aBuffer==0 ){
458 rc = SQLITE_NOMEM;
459 }else{
460 int i;
461
462 if( pCsr->iLangid & 0x00000001 ){
463 for(i=0; i<nToken; i++) pCsr->aBuffer[i] = pToken[i];
464 }else{
465 for(i=0; i<nToken; i++) pCsr->aBuffer[i] = (char)testTolower(pToken[i]);
466 }
467 pCsr->iToken++;
468 pCsr->iInput = (int)(p - pCsr->aInput);
469
470 *ppToken = pCsr->aBuffer;
471 *pnBytes = (int)nToken;
472 *piStartOffset = (int)(pToken - pCsr->aInput);
473 *piEndOffset = (int)(p - pCsr->aInput);
474 *piPosition = pCsr->iToken;
475 }
476 }
477
478 return rc;
479 }
480
481 static int testTokenizerLanguage(
482 sqlite3_tokenizer_cursor *pCursor,
483 int iLangid
484 ){
485 int rc = SQLITE_OK;
486 test_tokenizer_cursor *pCsr = (test_tokenizer_cursor *)pCursor;
487 pCsr->iLangid = iLangid;
488 if( pCsr->iLangid>=100 ){
489 rc = SQLITE_ERROR;
490 }
491 return rc;
492 }
493 #endif
494
495 static int SQLITE_TCLAPI fts3_test_tokenizer_cmd(
496 ClientData clientData,
497 Tcl_Interp *interp,
498 int objc,
499 Tcl_Obj *CONST objv[]
500 ){
501 #ifdef SQLITE_ENABLE_FTS3
502 static const sqlite3_tokenizer_module testTokenizerModule = {
503 1,
504 testTokenizerCreate,
505 testTokenizerDestroy,
506 testTokenizerOpen,
507 testTokenizerClose,
508 testTokenizerNext,
509 testTokenizerLanguage
510 };
511 const sqlite3_tokenizer_module *pPtr = &testTokenizerModule;
512 if( objc!=1 ){
513 Tcl_WrongNumArgs(interp, 1, objv, "");
514 return TCL_ERROR;
515 }
516 Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(
517 (const unsigned char *)&pPtr, sizeof(sqlite3_tokenizer_module *)
518 ));
519 #endif
520 UNUSED_PARAMETER(clientData);
521 return TCL_OK;
522 }
523
524 static int SQLITE_TCLAPI fts3_test_varint_cmd(
525 ClientData clientData,
526 Tcl_Interp *interp,
527 int objc,
528 Tcl_Obj *CONST objv[]
529 ){
530 #ifdef SQLITE_ENABLE_FTS3
531 char aBuf[24];
532 int rc;
533 Tcl_WideInt w;
534 sqlite3_int64 w2;
535 int nByte, nByte2;
536
537 if( objc!=2 ){
538 Tcl_WrongNumArgs(interp, 1, objv, "INTEGER");
539 return TCL_ERROR;
540 }
541
542 rc = Tcl_GetWideIntFromObj(interp, objv[1], &w);
543 if( rc!=TCL_OK ) return rc;
544
545 nByte = sqlite3Fts3PutVarint(aBuf, w);
546 nByte2 = sqlite3Fts3GetVarint(aBuf, &w2);
547 if( w!=w2 || nByte!=nByte2 ){
548 char *zErr = sqlite3_mprintf("error testing %lld", w);
549 Tcl_ResetResult(interp);
550 Tcl_AppendResult(interp, zErr, NULL);
551 return TCL_ERROR;
552 }
553
554 if( w<=2147483647 && w>=0 ){
555 int i;
556 nByte2 = fts3GetVarint32(aBuf, &i);
557 if( (int)w!=i || nByte!=nByte2 ){
558 char *zErr = sqlite3_mprintf("error testing %lld (32-bit)", w);
559 Tcl_ResetResult(interp);
560 Tcl_AppendResult(interp, zErr, NULL);
561 return TCL_ERROR;
562 }
563 }
564
565 #endif
566 UNUSED_PARAMETER(clientData);
567 return TCL_OK;
568 }
569
570 /*
571 ** End of tokenizer code.
572 **************************************************************************/
573
574 /*
575 ** sqlite3_fts3_may_be_corrupt BOOLEAN
576 **
577 ** Set or clear the global "may-be-corrupt" flag. Return the old value.
578 */
579 static int SQLITE_TCLAPI fts3_may_be_corrupt(
580 void * clientData,
581 Tcl_Interp *interp,
582 int objc,
583 Tcl_Obj *CONST objv[]
584 ){
585 #ifdef SQLITE_DEBUG
586 int bOld = sqlite3_fts3_may_be_corrupt;
587
588 if( objc!=2 && objc!=1 ){
589 Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?");
590 return TCL_ERROR;
591 }
592 if( objc==2 ){
593 int bNew;
594 if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR;
595 sqlite3_fts3_may_be_corrupt = bNew;
596 }
597
598 Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld));
599 #endif
600 return TCL_OK;
601 }
602
603 int Sqlitetestfts3_Init(Tcl_Interp *interp){
604 Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
605 Tcl_CreateObjCommand(interp,
606 "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0
607 );
608 Tcl_CreateObjCommand(
609 interp, "fts3_test_tokenizer", fts3_test_tokenizer_cmd, 0, 0
610 );
611 Tcl_CreateObjCommand(
612 interp, "fts3_test_varint", fts3_test_varint_cmd, 0, 0
613 );
614 Tcl_CreateObjCommand(
615 interp, "sqlite3_fts3_may_be_corrupt", fts3_may_be_corrupt, 0, 0
616 );
617 return TCL_OK;
618 }
619 #endif /* SQLITE_ENABLE_FTS3 || SQLITE_ENABLE_FTS4 */
620 #endif /* ifdef SQLITE_TEST */