Debian Codegraph @

ext/fts5/fts5_tokenize.c

raw
1 /*
2 ** 2014 May 31
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 */
13
14
15 #include "fts5Int.h"
16
17 /**************************************************************************
18 ** Start of ascii tokenizer implementation.
19 */
20
21 /*
22 ** For tokenizers with no "unicode" modifier, the set of token characters
23 ** is the same as the set of ASCII range alphanumeric characters.
24 */
25 static unsigned char aAsciiTokenChar[128] = {
26 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */
27 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10..0x1F */
28 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20..0x2F */
29 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30..0x3F */
30 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40..0x4F */
31 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x50..0x5F */
32 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60..0x6F */
33 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 0x70..0x7F */
34 };
35
36 typedef struct AsciiTokenizer AsciiTokenizer;
37 struct AsciiTokenizer {
38 unsigned char aTokenChar[128];
39 };
40
41 static void fts5AsciiAddExceptions(
42 AsciiTokenizer *p,
43 const char *zArg,
44 int bTokenChars
45 ){
46 int i;
47 for(i=0; zArg[i]; i++){
48 if( (zArg[i] & 0x80)==0 ){
49 p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars;
50 }
51 }
52 }
53
54 /*
55 ** Delete a "ascii" tokenizer.
56 */
57 static void fts5AsciiDelete(Fts5Tokenizer *p){
58 sqlite3_free(p);
59 }
60
61 /*
62 ** Create an "ascii" tokenizer.
63 */
64 static int fts5AsciiCreate(
65 void *pUnused,
66 const char **azArg, int nArg,
67 Fts5Tokenizer **ppOut
68 ){
69 int rc = SQLITE_OK;
70 AsciiTokenizer *p = 0;
71 UNUSED_PARAM(pUnused);
72 if( nArg%2 ){
73 rc = SQLITE_ERROR;
74 }else{
75 p = sqlite3_malloc64(sizeof(AsciiTokenizer));
76 if( p==0 ){
77 rc = SQLITE_NOMEM;
78 }else{
79 int i;
80 memset(p, 0, sizeof(AsciiTokenizer));
81 memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
82 for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
83 const char *zArg = azArg[i+1];
84 if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
85 fts5AsciiAddExceptions(p, zArg, 1);
86 }else
87 if( 0==sqlite3_stricmp(azArg[i], "separators") ){
88 fts5AsciiAddExceptions(p, zArg, 0);
89 }else{
90 rc = SQLITE_ERROR;
91 }
92 }
93 if( rc!=SQLITE_OK ){
94 fts5AsciiDelete((Fts5Tokenizer*)p);
95 p = 0;
96 }
97 }
98 }
99
100 *ppOut = (Fts5Tokenizer*)p;
101 return rc;
102 }
103
104
105 static void asciiFold(char *aOut, const char *aIn, int nByte){
106 int i;
107 for(i=0; i<nByte; i++){
108 char c = aIn[i];
109 if( c>='A' && c<='Z' ) c += 32;
110 aOut[i] = c;
111 }
112 }
113
114 /*
115 ** Tokenize some text using the ascii tokenizer.
116 */
117 static int fts5AsciiTokenize(
118 Fts5Tokenizer *pTokenizer,
119 void *pCtx,
120 int iUnused,
121 const char *pText, int nText,
122 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
123 ){
124 AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer;
125 int rc = SQLITE_OK;
126 int ie;
127 int is = 0;
128
129 char aFold[64];
130 int nFold = sizeof(aFold);
131 char *pFold = aFold;
132 unsigned char *a = p->aTokenChar;
133
134 UNUSED_PARAM(iUnused);
135
136 while( is<nText && rc==SQLITE_OK ){
137 int nByte;
138
139 /* Skip any leading divider characters. */
140 while( is<nText && ((pText[is]&0x80)==0 && a[(int)pText[is]]==0) ){
141 is++;
142 }
143 if( is==nText ) break;
144
145 /* Count the token characters */
146 ie = is+1;
147 while( ie<nText && ((pText[ie]&0x80) || a[(int)pText[ie]] ) ){
148 ie++;
149 }
150
151 /* Fold to lower case */
152 nByte = ie-is;
153 if( nByte>nFold ){
154 if( pFold!=aFold ) sqlite3_free(pFold);
155 pFold = sqlite3_malloc64((sqlite3_int64)nByte*2);
156 if( pFold==0 ){
157 rc = SQLITE_NOMEM;
158 break;
159 }
160 nFold = nByte*2;
161 }
162 asciiFold(pFold, &pText[is], nByte);
163
164 /* Invoke the token callback */
165 rc = xToken(pCtx, 0, pFold, nByte, is, ie);
166 is = ie+1;
167 }
168
169 if( pFold!=aFold ) sqlite3_free(pFold);
170 if( rc==SQLITE_DONE ) rc = SQLITE_OK;
171 return rc;
172 }
173
174 /**************************************************************************
175 ** Start of unicode61 tokenizer implementation.
176 */
177
178
179 /*
180 ** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied
181 ** from the sqlite3 source file utf.c. If this file is compiled as part
182 ** of the amalgamation, they are not required.
183 */
184 #ifndef SQLITE_AMALGAMATION
185
186 static const unsigned char sqlite3Utf8Trans1[] = {
187 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
188 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
189 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
190 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
191 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
192 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
193 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
194 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00,
195 };
196
197 #define READ_UTF8(zIn, zTerm, c) \
198 c = *(zIn++); \
199 if( c>=0xc0 ){ \
200 c = sqlite3Utf8Trans1[c-0xc0]; \
201 while( zIn<zTerm && (*zIn & 0xc0)==0x80 ){ \
202 c = (c<<6) + (0x3f & *(zIn++)); \
203 } \
204 if( c<0x80 \
205 || (c&0xFFFFF800)==0xD800 \
206 || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \
207 }
208
209
210 #define WRITE_UTF8(zOut, c) { \
211 if( c<0x00080 ){ \
212 *zOut++ = (unsigned char)(c&0xFF); \
213 } \
214 else if( c<0x00800 ){ \
215 *zOut++ = 0xC0 + (unsigned char)((c>>6)&0x1F); \
216 *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
217 } \
218 else if( c<0x10000 ){ \
219 *zOut++ = 0xE0 + (unsigned char)((c>>12)&0x0F); \
220 *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
221 *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
222 }else{ \
223 *zOut++ = 0xF0 + (unsigned char)((c>>18) & 0x07); \
224 *zOut++ = 0x80 + (unsigned char)((c>>12) & 0x3F); \
225 *zOut++ = 0x80 + (unsigned char)((c>>6) & 0x3F); \
226 *zOut++ = 0x80 + (unsigned char)(c & 0x3F); \
227 } \
228 }
229
230 #endif /* ifndef SQLITE_AMALGAMATION */
231
232 #define FTS5_SKIP_UTF8(zIn) { \
233 if( ((unsigned char)(*(zIn++)))>=0xc0 ){ \
234 while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; } \
235 } \
236 }
237
238 typedef struct Unicode61Tokenizer Unicode61Tokenizer;
239 struct Unicode61Tokenizer {
240 unsigned char aTokenChar[128]; /* ASCII range token characters */
241 char *aFold; /* Buffer to fold text into */
242 int nFold; /* Size of aFold[] in bytes */
243 int eRemoveDiacritic; /* True if remove_diacritics=1 is set */
244 int nException;
245 int *aiException;
246
247 unsigned char aCategory[32]; /* True for token char categories */
248 };
249
250 /* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */
251 #define FTS5_REMOVE_DIACRITICS_NONE 0
252 #define FTS5_REMOVE_DIACRITICS_SIMPLE 1
253 #define FTS5_REMOVE_DIACRITICS_COMPLEX 2
254
255 static int fts5UnicodeAddExceptions(
256 Unicode61Tokenizer *p, /* Tokenizer object */
257 const char *z, /* Characters to treat as exceptions */
258 int bTokenChars /* 1 for 'tokenchars', 0 for 'separators' */
259 ){
260 int rc = SQLITE_OK;
261 int n = (int)strlen(z);
262 int *aNew;
263
264 if( n>0 ){
265 aNew = (int*)sqlite3_realloc64(p->aiException,
266 (n+p->nException)*sizeof(int));
267 if( aNew ){
268 int nNew = p->nException;
269 const unsigned char *zCsr = (const unsigned char*)z;
270 const unsigned char *zTerm = (const unsigned char*)&z[n];
271 while( zCsr<zTerm ){
272 u32 iCode;
273 int bToken;
274 READ_UTF8(zCsr, zTerm, iCode);
275 if( iCode<128 ){
276 p->aTokenChar[iCode] = (unsigned char)bTokenChars;
277 }else{
278 bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
279 assert( (bToken==0 || bToken==1) );
280 assert( (bTokenChars==0 || bTokenChars==1) );
281 if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
282 int i;
283 for(i=0; i<nNew; i++){
284 if( (u32)aNew[i]>iCode ) break;
285 }
286 memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int));
287 aNew[i] = iCode;
288 nNew++;
289 }
290 }
291 }
292 p->aiException = aNew;
293 p->nException = nNew;
294 }else{
295 rc = SQLITE_NOMEM;
296 }
297 }
298
299 return rc;
300 }
301
302 /*
303 ** Return true if the p->aiException[] array contains the value iCode.
304 */
305 static int fts5UnicodeIsException(Unicode61Tokenizer *p, int iCode){
306 if( p->nException>0 ){
307 int *a = p->aiException;
308 int iLo = 0;
309 int iHi = p->nException-1;
310
311 while( iHi>=iLo ){
312 int iTest = (iHi + iLo) / 2;
313 if( iCode==a[iTest] ){
314 return 1;
315 }else if( iCode>a[iTest] ){
316 iLo = iTest+1;
317 }else{
318 iHi = iTest-1;
319 }
320 }
321 }
322
323 return 0;
324 }
325
326 /*
327 ** Delete a "unicode61" tokenizer.
328 */
329 static void fts5UnicodeDelete(Fts5Tokenizer *pTok){
330 if( pTok ){
331 Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTok;
332 sqlite3_free(p->aiException);
333 sqlite3_free(p->aFold);
334 sqlite3_free(p);
335 }
336 return;
337 }
338
339 static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){
340 const char *z = zCat;
341
342 while( *z ){
343 while( *z==' ' || *z=='\t' ) z++;
344 if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){
345 return SQLITE_ERROR;
346 }
347 while( *z!=' ' && *z!='\t' && *z!='\0' ) z++;
348 }
349
350 sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar);
351 return SQLITE_OK;
352 }
353
354 /*
355 ** Create a "unicode61" tokenizer.
356 */
357 static int fts5UnicodeCreate(
358 void *pUnused,
359 const char **azArg, int nArg,
360 Fts5Tokenizer **ppOut
361 ){
362 int rc = SQLITE_OK; /* Return code */
363 Unicode61Tokenizer *p = 0; /* New tokenizer object */
364
365 UNUSED_PARAM(pUnused);
366
367 if( nArg%2 ){
368 rc = SQLITE_ERROR;
369 }else{
370 p = (Unicode61Tokenizer*)sqlite3_malloc64(sizeof(Unicode61Tokenizer));
371 if( p ){
372 const char *zCat = "L* N* Co";
373 int i;
374 memset(p, 0, sizeof(Unicode61Tokenizer));
375
376 p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE;
377 p->nFold = 64;
378 p->aFold = sqlite3_malloc64(p->nFold * sizeof(char));
379 if( p->aFold==0 ){
380 rc = SQLITE_NOMEM;
381 }
382
383 /* Search for a "categories" argument */
384 for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
385 if( 0==sqlite3_stricmp(azArg[i], "categories") ){
386 zCat = azArg[i+1];
387 }
388 }
389 if( rc==SQLITE_OK ){
390 rc = unicodeSetCategories(p, zCat);
391 }
392
393 for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
394 const char *zArg = azArg[i+1];
395 if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
396 if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
397 rc = SQLITE_ERROR;
398 }else{
399 p->eRemoveDiacritic = (zArg[0] - '0');
400 assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE
401 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE
402 || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX
403 );
404 }
405 }else
406 if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
407 rc = fts5UnicodeAddExceptions(p, zArg, 1);
408 }else
409 if( 0==sqlite3_stricmp(azArg[i], "separators") ){
410 rc = fts5UnicodeAddExceptions(p, zArg, 0);
411 }else
412 if( 0==sqlite3_stricmp(azArg[i], "categories") ){
413 /* no-op */
414 }else{
415 rc = SQLITE_ERROR;
416 }
417 }
418 }else{
419 rc = SQLITE_NOMEM;
420 }
421 if( rc!=SQLITE_OK ){
422 fts5UnicodeDelete((Fts5Tokenizer*)p);
423 p = 0;
424 }
425 *ppOut = (Fts5Tokenizer*)p;
426 }
427 return rc;
428 }
429
430 /*
431 ** Return true if, for the purposes of tokenizing with the tokenizer
432 ** passed as the first argument, codepoint iCode is considered a token
433 ** character (not a separator).
434 */
435 static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
436 return (
437 p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)]
438 ^ fts5UnicodeIsException(p, iCode)
439 );
440 }
441
442 static int fts5UnicodeTokenize(
443 Fts5Tokenizer *pTokenizer,
444 void *pCtx,
445 int iUnused,
446 const char *pText, int nText,
447 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
448 ){
449 Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer;
450 int rc = SQLITE_OK;
451 unsigned char *a = p->aTokenChar;
452
453 unsigned char *zTerm = (unsigned char*)&pText[nText];
454 unsigned char *zCsr = (unsigned char *)pText;
455
456 /* Output buffer */
457 char *aFold = p->aFold;
458 int nFold = p->nFold;
459 const char *pEnd = &aFold[nFold-6];
460
461 UNUSED_PARAM(iUnused);
462
463 /* Each iteration of this loop gobbles up a contiguous run of separators,
464 ** then the next token. */
465 while( rc==SQLITE_OK ){
466 u32 iCode; /* non-ASCII codepoint read from input */
467 char *zOut = aFold;
468 int is;
469 int ie;
470
471 /* Skip any separator characters. */
472 while( 1 ){
473 if( zCsr>=zTerm ) goto tokenize_done;
474 if( *zCsr & 0x80 ) {
475 /* A character outside of the ascii range. Skip past it if it is
476 ** a separator character. Or break out of the loop if it is not. */
477 is = zCsr - (unsigned char*)pText;
478 READ_UTF8(zCsr, zTerm, iCode);
479 if( fts5UnicodeIsAlnum(p, iCode) ){
480 goto non_ascii_tokenchar;
481 }
482 }else{
483 if( a[*zCsr] ){
484 is = zCsr - (unsigned char*)pText;
485 goto ascii_tokenchar;
486 }
487 zCsr++;
488 }
489 }
490
491 /* Run through the tokenchars. Fold them into the output buffer along
492 ** the way. */
493 while( zCsr<zTerm ){
494
495 /* Grow the output buffer so that there is sufficient space to fit the
496 ** largest possible utf-8 character. */
497 if( zOut>pEnd ){
498 aFold = sqlite3_malloc64((sqlite3_int64)nFold*2);
499 if( aFold==0 ){
500 rc = SQLITE_NOMEM;
501 goto tokenize_done;
502 }
503 zOut = &aFold[zOut - p->aFold];
504 memcpy(aFold, p->aFold, nFold);
505 sqlite3_free(p->aFold);
506 p->aFold = aFold;
507 p->nFold = nFold = nFold*2;
508 pEnd = &aFold[nFold-6];
509 }
510
511 if( *zCsr & 0x80 ){
512 /* An non-ascii-range character. Fold it into the output buffer if
513 ** it is a token character, or break out of the loop if it is not. */
514 READ_UTF8(zCsr, zTerm, iCode);
515 if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){
516 non_ascii_tokenchar:
517 iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic);
518 if( iCode ) WRITE_UTF8(zOut, iCode);
519 }else{
520 break;
521 }
522 }else if( a[*zCsr]==0 ){
523 /* An ascii-range separator character. End of token. */
524 break;
525 }else{
526 ascii_tokenchar:
527 if( *zCsr>='A' && *zCsr<='Z' ){
528 *zOut++ = *zCsr + 32;
529 }else{
530 *zOut++ = *zCsr;
531 }
532 zCsr++;
533 }
534 ie = zCsr - (unsigned char*)pText;
535 }
536
537 /* Invoke the token callback */
538 rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie);
539 }
540
541 tokenize_done:
542 if( rc==SQLITE_DONE ) rc = SQLITE_OK;
543 return rc;
544 }
545
546 /**************************************************************************
547 ** Start of porter stemmer implementation.
548 */
549
550 /* Any tokens larger than this (in bytes) are passed through without
551 ** stemming. */
552 #define FTS5_PORTER_MAX_TOKEN 64
553
554 typedef struct PorterTokenizer PorterTokenizer;
555 struct PorterTokenizer {
556 fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */
557 Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */
558 char aBuf[FTS5_PORTER_MAX_TOKEN + 64];
559 };
560
561 /*
562 ** Delete a "porter" tokenizer.
563 */
564 static void fts5PorterDelete(Fts5Tokenizer *pTok){
565 if( pTok ){
566 PorterTokenizer *p = (PorterTokenizer*)pTok;
567 if( p->pTokenizer ){
568 p->tokenizer_v2.xDelete(p->pTokenizer);
569 }
570 sqlite3_free(p);
571 }
572 }
573
574 /*
575 ** Create a "porter" tokenizer.
576 */
577 static int fts5PorterCreate(
578 void *pCtx,
579 const char **azArg, int nArg,
580 Fts5Tokenizer **ppOut
581 ){
582 fts5_api *pApi = (fts5_api*)pCtx;
583 int rc = SQLITE_OK;
584 PorterTokenizer *pRet;
585 void *pUserdata = 0;
586 const char *zBase = "unicode61";
587 fts5_tokenizer_v2 *pV2 = 0;
588
589 while( nArg>0 ){
590 if( sqlite3_stricmp(azArg[0],"porter")==0 ){
591 nArg--;
592 azArg++;
593 }else{
594 zBase = azArg[0];
595 break;
596 }
597 }
598
599 pRet = (PorterTokenizer*)sqlite3_malloc64(sizeof(PorterTokenizer));
600 if( pRet ){
601 memset(pRet, 0, sizeof(PorterTokenizer));
602 rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2);
603 }else{
604 rc = SQLITE_NOMEM;
605 }
606 if( rc==SQLITE_OK ){
607 int nArg2 = (nArg>0 ? nArg-1 : 0);
608 const char **az2 = (nArg2 ? &azArg[1] : 0);
609 memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2));
610 rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer);
611 }
612
613 if( rc!=SQLITE_OK ){
614 fts5PorterDelete((Fts5Tokenizer*)pRet);
615 pRet = 0;
616 }
617 *ppOut = (Fts5Tokenizer*)pRet;
618 return rc;
619 }
620
621 typedef struct PorterContext PorterContext;
622 struct PorterContext {
623 void *pCtx;
624 int (*xToken)(void*, int, const char*, int, int, int);
625 char *aBuf;
626 };
627
628 typedef struct PorterRule PorterRule;
629 struct PorterRule {
630 const char *zSuffix;
631 int nSuffix;
632 int (*xCond)(char *zStem, int nStem);
633 const char *zOutput;
634 int nOutput;
635 };
636
637 #if 0
638 static int fts5PorterApply(char *aBuf, int *pnBuf, PorterRule *aRule){
639 int ret = -1;
640 int nBuf = *pnBuf;
641 PorterRule *p;
642
643 for(p=aRule; p->zSuffix; p++){
644 assert( strlen(p->zSuffix)==p->nSuffix );
645 assert( strlen(p->zOutput)==p->nOutput );
646 if( nBuf<p->nSuffix ) continue;
647 if( 0==memcmp(&aBuf[nBuf - p->nSuffix], p->zSuffix, p->nSuffix) ) break;
648 }
649
650 if( p->zSuffix ){
651 int nStem = nBuf - p->nSuffix;
652 if( p->xCond==0 || p->xCond(aBuf, nStem) ){
653 memcpy(&aBuf[nStem], p->zOutput, p->nOutput);
654 *pnBuf = nStem + p->nOutput;
655 ret = p - aRule;
656 }
657 }
658
659 return ret;
660 }
661 #endif
662
663 static int fts5PorterIsVowel(char c, int bYIsVowel){
664 return (
665 c=='a' || c=='e' || c=='i' || c=='o' || c=='u' || (bYIsVowel && c=='y')
666 );
667 }
668
669 static int fts5PorterGobbleVC(char *zStem, int nStem, int bPrevCons){
670 int i;
671 int bCons = bPrevCons;
672
673 /* Scan for a vowel */
674 for(i=0; i<nStem; i++){
675 if( 0==(bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) break;
676 }
677
678 /* Scan for a consonent */
679 for(i++; i<nStem; i++){
680 if( (bCons = !fts5PorterIsVowel(zStem[i], bCons)) ) return i+1;
681 }
682 return 0;
683 }
684
685 /* porter rule condition: (m > 0) */
686 static int fts5Porter_MGt0(char *zStem, int nStem){
687 return !!fts5PorterGobbleVC(zStem, nStem, 0);
688 }
689
690 /* porter rule condition: (m > 1) */
691 static int fts5Porter_MGt1(char *zStem, int nStem){
692 int n;
693 n = fts5PorterGobbleVC(zStem, nStem, 0);
694 if( n && fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
695 return 1;
696 }
697 return 0;
698 }
699
700 /* porter rule condition: (m = 1) */
701 static int fts5Porter_MEq1(char *zStem, int nStem){
702 int n;
703 n = fts5PorterGobbleVC(zStem, nStem, 0);
704 if( n && 0==fts5PorterGobbleVC(&zStem[n], nStem-n, 1) ){
705 return 1;
706 }
707 return 0;
708 }
709
710 /* porter rule condition: (*o) */
711 static int fts5Porter_Ostar(char *zStem, int nStem){
712 if( zStem[nStem-1]=='w' || zStem[nStem-1]=='x' || zStem[nStem-1]=='y' ){
713 return 0;
714 }else{
715 int i;
716 int mask = 0;
717 int bCons = 0;
718 for(i=0; i<nStem; i++){
719 bCons = !fts5PorterIsVowel(zStem[i], bCons);
720 assert( bCons==0 || bCons==1 );
721 mask = (mask << 1) + bCons;
722 }
723 return ((mask & 0x0007)==0x0005);
724 }
725 }
726
727 /* porter rule condition: (m > 1 and (*S or *T)) */
728 static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){
729 assert( nStem>0 );
730 return (zStem[nStem-1]=='s' || zStem[nStem-1]=='t')
731 && fts5Porter_MGt1(zStem, nStem);
732 }
733
734 /* porter rule condition: (*v*) */
735 static int fts5Porter_Vowel(char *zStem, int nStem){
736 int i;
737 for(i=0; i<nStem; i++){
738 if( fts5PorterIsVowel(zStem[i], i>0) ){
739 return 1;
740 }
741 }
742 return 0;
743 }
744
745
746 /**************************************************************************
747 ***************************************************************************
748 ** GENERATED CODE STARTS HERE (mkportersteps.tcl)
749 */
750
751 static int fts5PorterStep4(char *aBuf, int *pnBuf){
752 int ret = 0;
753 int nBuf = *pnBuf;
754 switch( aBuf[nBuf-2] ){
755
756 case 'a':
757 if( nBuf>2 && 0==memcmp("al", &aBuf[nBuf-2], 2) ){
758 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
759 *pnBuf = nBuf - 2;
760 }
761 }
762 break;
763
764 case 'c':
765 if( nBuf>4 && 0==memcmp("ance", &aBuf[nBuf-4], 4) ){
766 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
767 *pnBuf = nBuf - 4;
768 }
769 }else if( nBuf>4 && 0==memcmp("ence", &aBuf[nBuf-4], 4) ){
770 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
771 *pnBuf = nBuf - 4;
772 }
773 }
774 break;
775
776 case 'e':
777 if( nBuf>2 && 0==memcmp("er", &aBuf[nBuf-2], 2) ){
778 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
779 *pnBuf = nBuf - 2;
780 }
781 }
782 break;
783
784 case 'i':
785 if( nBuf>2 && 0==memcmp("ic", &aBuf[nBuf-2], 2) ){
786 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
787 *pnBuf = nBuf - 2;
788 }
789 }
790 break;
791
792 case 'l':
793 if( nBuf>4 && 0==memcmp("able", &aBuf[nBuf-4], 4) ){
794 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
795 *pnBuf = nBuf - 4;
796 }
797 }else if( nBuf>4 && 0==memcmp("ible", &aBuf[nBuf-4], 4) ){
798 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
799 *pnBuf = nBuf - 4;
800 }
801 }
802 break;
803
804 case 'n':
805 if( nBuf>3 && 0==memcmp("ant", &aBuf[nBuf-3], 3) ){
806 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
807 *pnBuf = nBuf - 3;
808 }
809 }else if( nBuf>5 && 0==memcmp("ement", &aBuf[nBuf-5], 5) ){
810 if( fts5Porter_MGt1(aBuf, nBuf-5) ){
811 *pnBuf = nBuf - 5;
812 }
813 }else if( nBuf>4 && 0==memcmp("ment", &aBuf[nBuf-4], 4) ){
814 if( fts5Porter_MGt1(aBuf, nBuf-4) ){
815 *pnBuf = nBuf - 4;
816 }
817 }else if( nBuf>3 && 0==memcmp("ent", &aBuf[nBuf-3], 3) ){
818 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
819 *pnBuf = nBuf - 3;
820 }
821 }
822 break;
823
824 case 'o':
825 if( nBuf>3 && 0==memcmp("ion", &aBuf[nBuf-3], 3) ){
826 if( fts5Porter_MGt1_and_S_or_T(aBuf, nBuf-3) ){
827 *pnBuf = nBuf - 3;
828 }
829 }else if( nBuf>2 && 0==memcmp("ou", &aBuf[nBuf-2], 2) ){
830 if( fts5Porter_MGt1(aBuf, nBuf-2) ){
831 *pnBuf = nBuf - 2;
832 }
833 }
834 break;
835
836 case 's':
837 if( nBuf>3 && 0==memcmp("ism", &aBuf[nBuf-3], 3) ){
838 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
839 *pnBuf = nBuf - 3;
840 }
841 }
842 break;
843
844 case 't':
845 if( nBuf>3 && 0==memcmp("ate", &aBuf[nBuf-3], 3) ){
846 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
847 *pnBuf = nBuf - 3;
848 }
849 }else if( nBuf>3 && 0==memcmp("iti", &aBuf[nBuf-3], 3) ){
850 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
851 *pnBuf = nBuf - 3;
852 }
853 }
854 break;
855
856 case 'u':
857 if( nBuf>3 && 0==memcmp("ous", &aBuf[nBuf-3], 3) ){
858 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
859 *pnBuf = nBuf - 3;
860 }
861 }
862 break;
863
864 case 'v':
865 if( nBuf>3 && 0==memcmp("ive", &aBuf[nBuf-3], 3) ){
866 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
867 *pnBuf = nBuf - 3;
868 }
869 }
870 break;
871
872 case 'z':
873 if( nBuf>3 && 0==memcmp("ize", &aBuf[nBuf-3], 3) ){
874 if( fts5Porter_MGt1(aBuf, nBuf-3) ){
875 *pnBuf = nBuf - 3;
876 }
877 }
878 break;
879
880 }
881 return ret;
882 }
883
884
885 static int fts5PorterStep1B2(char *aBuf, int *pnBuf){
886 int ret = 0;
887 int nBuf = *pnBuf;
888 switch( aBuf[nBuf-2] ){
889
890 case 'a':
891 if( nBuf>2 && 0==memcmp("at", &aBuf[nBuf-2], 2) ){
892 memcpy(&aBuf[nBuf-2], "ate", 3);
893 *pnBuf = nBuf - 2 + 3;
894 ret = 1;
895 }
896 break;
897
898 case 'b':
899 if( nBuf>2 && 0==memcmp("bl", &aBuf[nBuf-2], 2) ){
900 memcpy(&aBuf[nBuf-2], "ble", 3);
901 *pnBuf = nBuf - 2 + 3;
902 ret = 1;
903 }
904 break;
905
906 case 'i':
907 if( nBuf>2 && 0==memcmp("iz", &aBuf[nBuf-2], 2) ){
908 memcpy(&aBuf[nBuf-2], "ize", 3);
909 *pnBuf = nBuf - 2 + 3;
910 ret = 1;
911 }
912 break;
913
914 }
915 return ret;
916 }
917
918
919 static int fts5PorterStep2(char *aBuf, int *pnBuf){
920 int ret = 0;
921 int nBuf = *pnBuf;
922 switch( aBuf[nBuf-2] ){
923
924 case 'a':
925 if( nBuf>7 && 0==memcmp("ational", &aBuf[nBuf-7], 7) ){
926 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
927 memcpy(&aBuf[nBuf-7], "ate", 3);
928 *pnBuf = nBuf - 7 + 3;
929 }
930 }else if( nBuf>6 && 0==memcmp("tional", &aBuf[nBuf-6], 6) ){
931 if( fts5Porter_MGt0(aBuf, nBuf-6) ){
932 memcpy(&aBuf[nBuf-6], "tion", 4);
933 *pnBuf = nBuf - 6 + 4;
934 }
935 }
936 break;
937
938 case 'c':
939 if( nBuf>4 && 0==memcmp("enci", &aBuf[nBuf-4], 4) ){
940 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
941 memcpy(&aBuf[nBuf-4], "ence", 4);
942 *pnBuf = nBuf - 4 + 4;
943 }
944 }else if( nBuf>4 && 0==memcmp("anci", &aBuf[nBuf-4], 4) ){
945 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
946 memcpy(&aBuf[nBuf-4], "ance", 4);
947 *pnBuf = nBuf - 4 + 4;
948 }
949 }
950 break;
951
952 case 'e':
953 if( nBuf>4 && 0==memcmp("izer", &aBuf[nBuf-4], 4) ){
954 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
955 memcpy(&aBuf[nBuf-4], "ize", 3);
956 *pnBuf = nBuf - 4 + 3;
957 }
958 }
959 break;
960
961 case 'g':
962 if( nBuf>4 && 0==memcmp("logi", &aBuf[nBuf-4], 4) ){
963 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
964 memcpy(&aBuf[nBuf-4], "log", 3);
965 *pnBuf = nBuf - 4 + 3;
966 }
967 }
968 break;
969
970 case 'l':
971 if( nBuf>3 && 0==memcmp("bli", &aBuf[nBuf-3], 3) ){
972 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
973 memcpy(&aBuf[nBuf-3], "ble", 3);
974 *pnBuf = nBuf - 3 + 3;
975 }
976 }else if( nBuf>4 && 0==memcmp("alli", &aBuf[nBuf-4], 4) ){
977 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
978 memcpy(&aBuf[nBuf-4], "al", 2);
979 *pnBuf = nBuf - 4 + 2;
980 }
981 }else if( nBuf>5 && 0==memcmp("entli", &aBuf[nBuf-5], 5) ){
982 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
983 memcpy(&aBuf[nBuf-5], "ent", 3);
984 *pnBuf = nBuf - 5 + 3;
985 }
986 }else if( nBuf>3 && 0==memcmp("eli", &aBuf[nBuf-3], 3) ){
987 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
988 memcpy(&aBuf[nBuf-3], "e", 1);
989 *pnBuf = nBuf - 3 + 1;
990 }
991 }else if( nBuf>5 && 0==memcmp("ousli", &aBuf[nBuf-5], 5) ){
992 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
993 memcpy(&aBuf[nBuf-5], "ous", 3);
994 *pnBuf = nBuf - 5 + 3;
995 }
996 }
997 break;
998
999 case 'o':
1000 if( nBuf>7 && 0==memcmp("ization", &aBuf[nBuf-7], 7) ){
1001 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
1002 memcpy(&aBuf[nBuf-7], "ize", 3);
1003 *pnBuf = nBuf - 7 + 3;
1004 }
1005 }else if( nBuf>5 && 0==memcmp("ation", &aBuf[nBuf-5], 5) ){
1006 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
1007 memcpy(&aBuf[nBuf-5], "ate", 3);
1008 *pnBuf = nBuf - 5 + 3;
1009 }
1010 }else if( nBuf>4 && 0==memcmp("ator", &aBuf[nBuf-4], 4) ){
1011 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
1012 memcpy(&aBuf[nBuf-4], "ate", 3);
1013 *pnBuf = nBuf - 4 + 3;
1014 }
1015 }
1016 break;
1017
1018 case 's':
1019 if( nBuf>5 && 0==memcmp("alism", &aBuf[nBuf-5], 5) ){
1020 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
1021 memcpy(&aBuf[nBuf-5], "al", 2);
1022 *pnBuf = nBuf - 5 + 2;
1023 }
1024 }else if( nBuf>7 && 0==memcmp("iveness", &aBuf[nBuf-7], 7) ){
1025 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
1026 memcpy(&aBuf[nBuf-7], "ive", 3);
1027 *pnBuf = nBuf - 7 + 3;
1028 }
1029 }else if( nBuf>7 && 0==memcmp("fulness", &aBuf[nBuf-7], 7) ){
1030 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
1031 memcpy(&aBuf[nBuf-7], "ful", 3);
1032 *pnBuf = nBuf - 7 + 3;
1033 }
1034 }else if( nBuf>7 && 0==memcmp("ousness", &aBuf[nBuf-7], 7) ){
1035 if( fts5Porter_MGt0(aBuf, nBuf-7) ){
1036 memcpy(&aBuf[nBuf-7], "ous", 3);
1037 *pnBuf = nBuf - 7 + 3;
1038 }
1039 }
1040 break;
1041
1042 case 't':
1043 if( nBuf>5 && 0==memcmp("aliti", &aBuf[nBuf-5], 5) ){
1044 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
1045 memcpy(&aBuf[nBuf-5], "al", 2);
1046 *pnBuf = nBuf - 5 + 2;
1047 }
1048 }else if( nBuf>5 && 0==memcmp("iviti", &aBuf[nBuf-5], 5) ){
1049 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
1050 memcpy(&aBuf[nBuf-5], "ive", 3);
1051 *pnBuf = nBuf - 5 + 3;
1052 }
1053 }else if( nBuf>6 && 0==memcmp("biliti", &aBuf[nBuf-6], 6) ){
1054 if( fts5Porter_MGt0(aBuf, nBuf-6) ){
1055 memcpy(&aBuf[nBuf-6], "ble", 3);
1056 *pnBuf = nBuf - 6 + 3;
1057 }
1058 }
1059 break;
1060
1061 }
1062 return ret;
1063 }
1064
1065
1066 static int fts5PorterStep3(char *aBuf, int *pnBuf){
1067 int ret = 0;
1068 int nBuf = *pnBuf;
1069 switch( aBuf[nBuf-2] ){
1070
1071 case 'a':
1072 if( nBuf>4 && 0==memcmp("ical", &aBuf[nBuf-4], 4) ){
1073 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
1074 memcpy(&aBuf[nBuf-4], "ic", 2);
1075 *pnBuf = nBuf - 4 + 2;
1076 }
1077 }
1078 break;
1079
1080 case 's':
1081 if( nBuf>4 && 0==memcmp("ness", &aBuf[nBuf-4], 4) ){
1082 if( fts5Porter_MGt0(aBuf, nBuf-4) ){
1083 *pnBuf = nBuf - 4;
1084 }
1085 }
1086 break;
1087
1088 case 't':
1089 if( nBuf>5 && 0==memcmp("icate", &aBuf[nBuf-5], 5) ){
1090 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
1091 memcpy(&aBuf[nBuf-5], "ic", 2);
1092 *pnBuf = nBuf - 5 + 2;
1093 }
1094 }else if( nBuf>5 && 0==memcmp("iciti", &aBuf[nBuf-5], 5) ){
1095 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
1096 memcpy(&aBuf[nBuf-5], "ic", 2);
1097 *pnBuf = nBuf - 5 + 2;
1098 }
1099 }
1100 break;
1101
1102 case 'u':
1103 if( nBuf>3 && 0==memcmp("ful", &aBuf[nBuf-3], 3) ){
1104 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
1105 *pnBuf = nBuf - 3;
1106 }
1107 }
1108 break;
1109
1110 case 'v':
1111 if( nBuf>5 && 0==memcmp("ative", &aBuf[nBuf-5], 5) ){
1112 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
1113 *pnBuf = nBuf - 5;
1114 }
1115 }
1116 break;
1117
1118 case 'z':
1119 if( nBuf>5 && 0==memcmp("alize", &aBuf[nBuf-5], 5) ){
1120 if( fts5Porter_MGt0(aBuf, nBuf-5) ){
1121 memcpy(&aBuf[nBuf-5], "al", 2);
1122 *pnBuf = nBuf - 5 + 2;
1123 }
1124 }
1125 break;
1126
1127 }
1128 return ret;
1129 }
1130
1131
1132 static int fts5PorterStep1B(char *aBuf, int *pnBuf){
1133 int ret = 0;
1134 int nBuf = *pnBuf;
1135 switch( aBuf[nBuf-2] ){
1136
1137 case 'e':
1138 if( nBuf>3 && 0==memcmp("eed", &aBuf[nBuf-3], 3) ){
1139 if( fts5Porter_MGt0(aBuf, nBuf-3) ){
1140 memcpy(&aBuf[nBuf-3], "ee", 2);
1141 *pnBuf = nBuf - 3 + 2;
1142 }
1143 }else if( nBuf>2 && 0==memcmp("ed", &aBuf[nBuf-2], 2) ){
1144 if( fts5Porter_Vowel(aBuf, nBuf-2) ){
1145 *pnBuf = nBuf - 2;
1146 ret = 1;
1147 }
1148 }
1149 break;
1150
1151 case 'n':
1152 if( nBuf>3 && 0==memcmp("ing", &aBuf[nBuf-3], 3) ){
1153 if( fts5Porter_Vowel(aBuf, nBuf-3) ){
1154 *pnBuf = nBuf - 3;
1155 ret = 1;
1156 }
1157 }
1158 break;
1159
1160 }
1161 return ret;
1162 }
1163
1164 /*
1165 ** GENERATED CODE ENDS HERE (mkportersteps.tcl)
1166 ***************************************************************************
1167 **************************************************************************/
1168
1169 static void fts5PorterStep1A(char *aBuf, int *pnBuf){
1170 int nBuf = *pnBuf;
1171 if( aBuf[nBuf-1]=='s' ){
1172 if( aBuf[nBuf-2]=='e' ){
1173 if( (nBuf>4 && aBuf[nBuf-4]=='s' && aBuf[nBuf-3]=='s')
1174 || (nBuf>3 && aBuf[nBuf-3]=='i' )
1175 ){
1176 *pnBuf = nBuf-2;
1177 }else{
1178 *pnBuf = nBuf-1;
1179 }
1180 }
1181 else if( aBuf[nBuf-2]!='s' ){
1182 *pnBuf = nBuf-1;
1183 }
1184 }
1185 }
1186
1187 static int fts5PorterCb(
1188 void *pCtx,
1189 int tflags,
1190 const char *pToken,
1191 int nToken,
1192 int iStart,
1193 int iEnd
1194 ){
1195 PorterContext *p = (PorterContext*)pCtx;
1196
1197 char *aBuf;
1198 int nBuf;
1199
1200 if( nToken>FTS5_PORTER_MAX_TOKEN || nToken<3 ) goto pass_through;
1201 aBuf = p->aBuf;
1202 nBuf = nToken;
1203 memcpy(aBuf, pToken, nBuf);
1204
1205 /* Step 1. */
1206 fts5PorterStep1A(aBuf, &nBuf);
1207 if( fts5PorterStep1B(aBuf, &nBuf) ){
1208 if( fts5PorterStep1B2(aBuf, &nBuf)==0 ){
1209 char c = aBuf[nBuf-1];
1210 if( fts5PorterIsVowel(c, 0)==0
1211 && c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2]
1212 ){
1213 nBuf--;
1214 }else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){
1215 aBuf[nBuf++] = 'e';
1216 }
1217 }
1218 }
1219
1220 /* Step 1C. */
1221 if( aBuf[nBuf-1]=='y' && fts5Porter_Vowel(aBuf, nBuf-1) ){
1222 aBuf[nBuf-1] = 'i';
1223 }
1224
1225 /* Steps 2 through 4. */
1226 fts5PorterStep2(aBuf, &nBuf);
1227 fts5PorterStep3(aBuf, &nBuf);
1228 fts5PorterStep4(aBuf, &nBuf);
1229
1230 /* Step 5a. */
1231 assert( nBuf>0 );
1232 if( aBuf[nBuf-1]=='e' ){
1233 if( fts5Porter_MGt1(aBuf, nBuf-1)
1234 || (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1))
1235 ){
1236 nBuf--;
1237 }
1238 }
1239
1240 /* Step 5b. */
1241 if( nBuf>1 && aBuf[nBuf-1]=='l'
1242 && aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1)
1243 ){
1244 nBuf--;
1245 }
1246
1247 return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd);
1248
1249 pass_through:
1250 return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
1251 }
1252
1253 /*
1254 ** Tokenize using the porter tokenizer.
1255 */
1256 static int fts5PorterTokenize(
1257 Fts5Tokenizer *pTokenizer,
1258 void *pCtx,
1259 int flags,
1260 const char *pText, int nText,
1261 const char *pLoc, int nLoc,
1262 int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
1263 ){
1264 PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
1265 PorterContext sCtx;
1266 sCtx.xToken = xToken;
1267 sCtx.pCtx = pCtx;
1268 sCtx.aBuf = p->aBuf;
1269 return p->tokenizer_v2.xTokenize(
1270 p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb
1271 );
1272 }
1273
1274 /**************************************************************************
1275 ** Start of trigram implementation.
1276 */
1277 typedef struct TrigramTokenizer TrigramTokenizer;
1278 struct TrigramTokenizer {
1279 int bFold; /* True to fold to lower-case */
1280 int iFoldParam; /* Parameter to pass to Fts5UnicodeFold() */
1281 };
1282
1283 /*
1284 ** Free a trigram tokenizer.
1285 */
1286 static void fts5TriDelete(Fts5Tokenizer *p){
1287 sqlite3_free(p);
1288 }
1289
1290 /*
1291 ** Allocate a trigram tokenizer.
1292 */
1293 static int fts5TriCreate(
1294 void *pUnused,
1295 const char **azArg,
1296 int nArg,
1297 Fts5Tokenizer **ppOut
1298 ){
1299 int rc = SQLITE_OK;
1300 TrigramTokenizer *pNew = 0;
1301 UNUSED_PARAM(pUnused);
1302 if( nArg%2 ){
1303 rc = SQLITE_ERROR;
1304 }else{
1305 int i;
1306 pNew = (TrigramTokenizer*)sqlite3_malloc64(sizeof(*pNew));
1307 if( pNew==0 ){
1308 rc = SQLITE_NOMEM;
1309 }else{
1310 pNew->bFold = 1;
1311 pNew->iFoldParam = 0;
1312
1313 for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
1314 const char *zArg = azArg[i+1];
1315 if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
1316 if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
1317 rc = SQLITE_ERROR;
1318 }else{
1319 pNew->bFold = (zArg[0]=='0');
1320 }
1321 }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
1322 if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
1323 rc = SQLITE_ERROR;
1324 }else{
1325 pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
1326 }
1327 }else{
1328 rc = SQLITE_ERROR;
1329 }
1330 }
1331
1332 if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
1333 rc = SQLITE_ERROR;
1334 }
1335
1336 if( rc!=SQLITE_OK ){
1337 fts5TriDelete((Fts5Tokenizer*)pNew);
1338 pNew = 0;
1339 }
1340 }
1341 }
1342 *ppOut = (Fts5Tokenizer*)pNew;
1343 return rc;
1344 }
1345
1346 /*
1347 ** Trigram tokenizer tokenize routine.
1348 */
1349 static int fts5TriTokenize(
1350 Fts5Tokenizer *pTok,
1351 void *pCtx,
1352 int unusedFlags,
1353 const char *pText, int nText,
1354 int (*xToken)(void*, int, const char*, int, int, int)
1355 ){
1356 TrigramTokenizer *p = (TrigramTokenizer*)pTok;
1357 int rc = SQLITE_OK;
1358 char aBuf[32];
1359 char *zOut = aBuf;
1360 int ii;
1361 const unsigned char *zIn = (const unsigned char*)pText;
1362 const unsigned char *zEof = (zIn ? &zIn[nText] : 0);
1363 u32 iCode = 0;
1364 int aStart[3]; /* Input offset of each character in aBuf[] */
1365
1366 UNUSED_PARAM(unusedFlags);
1367
1368 /* Populate aBuf[] with the characters for the first trigram. */
1369 for(ii=0; ii<3; ii++){
1370 do {
1371 aStart[ii] = zIn - (const unsigned char*)pText;
1372 if( zIn>=zEof ) return SQLITE_OK;
1373 READ_UTF8(zIn, zEof, iCode);
1374 if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
1375 }while( iCode==0 );
1376 WRITE_UTF8(zOut, iCode);
1377 }
1378
1379 /* At the start of each iteration of this loop:
1380 **
1381 ** aBuf: Contains 3 characters. The 3 characters of the next trigram.
1382 ** zOut: Points to the byte following the last character in aBuf.
1383 ** aStart[3]: Contains the byte offset in the input text corresponding
1384 ** to the start of each of the three characters in the buffer.
1385 */
1386 assert( zIn<=zEof );
1387 while( 1 ){
1388 int iNext; /* Start of character following current tri */
1389 const char *z1;
1390
1391 /* Read characters from the input up until the first non-diacritic */
1392 do {
1393 iNext = zIn - (const unsigned char*)pText;
1394 if( zIn>=zEof ){
1395 iCode = 0;
1396 break;
1397 }
1398 READ_UTF8(zIn, zEof, iCode);
1399 if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
1400 }while( iCode==0 );
1401
1402 /* Pass the current trigram back to fts5 */
1403 rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext);
1404 if( iCode==0 || rc!=SQLITE_OK ) break;
1405
1406 /* Remove the first character from buffer aBuf[]. Append the character
1407 ** with codepoint iCode. */
1408 z1 = aBuf;
1409 FTS5_SKIP_UTF8(z1);
1410 memmove(aBuf, z1, zOut - z1);
1411 zOut -= (z1 - aBuf);
1412 WRITE_UTF8(zOut, iCode);
1413
1414 /* Update the aStart[] array */
1415 aStart[0] = aStart[1];
1416 aStart[1] = aStart[2];
1417 aStart[2] = iNext;
1418 }
1419
1420 return rc;
1421 }
1422
1423 /*
1424 ** Argument xCreate is a pointer to a constructor function for a tokenizer.
1425 ** pTok is a tokenizer previously created using the same method. This function
1426 ** returns one of FTS5_PATTERN_NONE, FTS5_PATTERN_LIKE or FTS5_PATTERN_GLOB
1427 ** indicating the style of pattern matching that the tokenizer can support.
1428 ** In practice, this is:
1429 **
1430 ** "trigram" tokenizer, case_sensitive=1 - FTS5_PATTERN_GLOB
1431 ** "trigram" tokenizer, case_sensitive=0 (the default) - FTS5_PATTERN_LIKE
1432 ** all other tokenizers - FTS5_PATTERN_NONE
1433 */
1434 int sqlite3Fts5TokenizerPattern(
1435 int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
1436 Fts5Tokenizer *pTok
1437 ){
1438 if( xCreate==fts5TriCreate ){
1439 TrigramTokenizer *p = (TrigramTokenizer*)pTok;
1440 if( p->iFoldParam==0 ){
1441 return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
1442 }
1443 }
1444 return FTS5_PATTERN_NONE;
1445 }
1446
1447 /*
1448 ** Return true if the tokenizer described by p->azArg[] is the trigram
1449 ** tokenizer. This tokenizer needs to be loaded before xBestIndex is
1450 ** called for the first time in order to correctly handle LIKE/GLOB.
1451 */
1452 int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){
1453 return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram"));
1454 }
1455
1456
1457 /*
1458 ** Register all built-in tokenizers with FTS5.
1459 */
1460 int sqlite3Fts5TokenizerInit(fts5_api *pApi){
1461 struct BuiltinTokenizer {
1462 const char *zName;
1463 fts5_tokenizer x;
1464 } aBuiltin[] = {
1465 { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
1466 { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
1467 { "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}},
1468 };
1469
1470 int rc = SQLITE_OK; /* Return code */
1471 int i; /* To iterate through builtin functions */
1472
1473 for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
1474 rc = pApi->xCreateTokenizer(pApi,
1475 aBuiltin[i].zName,
1476 (void*)pApi,
1477 &aBuiltin[i].x,
1478 0
1479 );
1480 }
1481 if( rc==SQLITE_OK ){
1482 fts5_tokenizer_v2 sPorter = {
1483 2,
1484 fts5PorterCreate,
1485 fts5PorterDelete,
1486 fts5PorterTokenize
1487 };
1488 rc = pApi->xCreateTokenizer_v2(pApi,
1489 "porter",
1490 (void*)pApi,
1491 &sPorter,
1492 0
1493 );
1494 }
1495 return rc;
1496 }