Debian Codegraph @

ext/fts5/fts5.h

raw
1 /*
2 ** 2014 May 31
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 **
13 ** Interfaces to extend FTS5. Using the interfaces defined in this file,
14 ** FTS5 may be extended with:
15 **
16 ** * custom tokenizers, and
17 ** * custom auxiliary functions.
18 */
19
20
21 #ifndef _FTS5_H
22 #define _FTS5_H
23
24 #include "sqlite3.h"
25
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29
30 /*************************************************************************
31 ** CUSTOM AUXILIARY FUNCTIONS
32 **
33 ** Virtual table implementations may overload SQL functions by implementing
34 ** the sqlite3_module.xFindFunction() method.
35 */
36
37 typedef struct Fts5ExtensionApi Fts5ExtensionApi;
38 typedef struct Fts5Context Fts5Context;
39 typedef struct Fts5PhraseIter Fts5PhraseIter;
40
41 typedef void (*fts5_extension_function)(
42 const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
43 Fts5Context *pFts, /* First arg to pass to pApi functions */
44 sqlite3_context *pCtx, /* Context for returning result/error */
45 int nVal, /* Number of values in apVal[] array */
46 sqlite3_value **apVal /* Array of trailing arguments */
47 );
48
49 struct Fts5PhraseIter {
50 const unsigned char *a;
51 const unsigned char *b;
52 };
53
54 /*
55 ** EXTENSION API FUNCTIONS
56 **
57 ** xUserData(pFts):
58 ** Return a copy of the pUserData pointer passed to the xCreateFunction()
59 ** API when the extension function was registered.
60 **
61 ** xColumnTotalSize(pFts, iCol, pnToken):
62 ** If parameter iCol is less than zero, set output variable *pnToken
63 ** to the total number of tokens in the FTS5 table. Or, if iCol is
64 ** non-negative but less than the number of columns in the table, return
65 ** the total number of tokens in column iCol, considering all rows in
66 ** the FTS5 table.
67 **
68 ** If parameter iCol is greater than or equal to the number of columns
69 ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
70 ** an OOM condition or IO error), an appropriate SQLite error code is
71 ** returned.
72 **
73 ** xColumnCount(pFts):
74 ** Return the number of columns in the table.
75 **
76 ** xColumnSize(pFts, iCol, pnToken):
77 ** If parameter iCol is less than zero, set output variable *pnToken
78 ** to the total number of tokens in the current row. Or, if iCol is
79 ** non-negative but less than the number of columns in the table, set
80 ** *pnToken to the number of tokens in column iCol of the current row.
81 **
82 ** If parameter iCol is greater than or equal to the number of columns
83 ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g.
84 ** an OOM condition or IO error), an appropriate SQLite error code is
85 ** returned.
86 **
87 ** This function may be quite inefficient if used with an FTS5 table
88 ** created with the "columnsize=0" option.
89 **
90 ** xColumnText:
91 ** If parameter iCol is less than zero, or greater than or equal to the
92 ** number of columns in the table, SQLITE_RANGE is returned.
93 **
94 ** Otherwise, this function attempts to retrieve the text of column iCol of
95 ** the current document. If successful, (*pz) is set to point to a buffer
96 ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes
97 ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
98 ** if an error occurs, an SQLite error code is returned and the final values
99 ** of (*pz) and (*pn) are undefined.
100 **
101 ** xPhraseCount:
102 ** Returns the number of phrases in the current query expression.
103 **
104 ** xPhraseSize:
105 ** If parameter iCol is less than zero, or greater than or equal to the
106 ** number of phrases in the current query, as returned by xPhraseCount,
107 ** 0 is returned. Otherwise, this function returns the number of tokens in
108 ** phrase iPhrase of the query. Phrases are numbered starting from zero.
109 **
110 ** xInstCount:
111 ** Set *pnInst to the total number of occurrences of all phrases within
112 ** the query within the current row. Return SQLITE_OK if successful, or
113 ** an error code (i.e. SQLITE_NOMEM) if an error occurs.
114 **
115 ** This API can be quite slow if used with an FTS5 table created with the
116 ** "detail=none" or "detail=column" option. If the FTS5 table is created
117 ** with either "detail=none" or "detail=column" and "content=" option
118 ** (i.e. if it is a contentless table), then this API always returns 0.
119 **
120 ** xInst:
121 ** Query for the details of phrase match iIdx within the current row.
122 ** Phrase matches are numbered starting from zero, so the iIdx argument
123 ** should be greater than or equal to zero and smaller than the value
124 ** output by xInstCount(). If iIdx is less than zero or greater than
125 ** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.
126 **
127 ** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol
128 ** to the column in which it occurs and *piOff the token offset of the
129 ** first token of the phrase. SQLITE_OK is returned if successful, or an
130 ** error code (i.e. SQLITE_NOMEM) if an error occurs.
131 **
132 ** This API can be quite slow if used with an FTS5 table created with the
133 ** "detail=none" or "detail=column" option.
134 **
135 ** xRowid:
136 ** Returns the rowid of the current row.
137 **
138 ** xTokenize:
139 ** Tokenize text using the tokenizer belonging to the FTS5 table.
140 **
141 ** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
142 ** This API function is used to query the FTS table for phrase iPhrase
143 ** of the current query. Specifically, a query equivalent to:
144 **
145 ** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid
146 **
147 ** with $p set to a phrase equivalent to the phrase iPhrase of the
148 ** current query is executed. Any column filter that applies to
149 ** phrase iPhrase of the current query is included in $p. For each
150 ** row visited, the callback function passed as the fourth argument
151 ** is invoked. The context and API objects passed to the callback
152 ** function may be used to access the properties of each matched row.
153 ** Invoking Api.xUserData() returns a copy of the pointer passed as
154 ** the third argument to pUserData.
155 **
156 ** If parameter iPhrase is less than zero, or greater than or equal to
157 ** the number of phrases in the query, as returned by xPhraseCount(),
158 ** this function returns SQLITE_RANGE.
159 **
160 ** If the callback function returns any value other than SQLITE_OK, the
161 ** query is abandoned and the xQueryPhrase function returns immediately.
162 ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
163 ** Otherwise, the error code is propagated upwards.
164 **
165 ** If the query runs to completion without incident, SQLITE_OK is returned.
166 ** Or, if some error occurs before the query completes or is aborted by
167 ** the callback, an SQLite error code is returned.
168 **
169 **
170 ** xSetAuxdata(pFts5, pAux, xDelete)
171 **
172 ** Save the pointer passed as the second argument as the extension function's
173 ** "auxiliary data". The pointer may then be retrieved by the current or any
174 ** future invocation of the same fts5 extension function made as part of
175 ** the same MATCH query using the xGetAuxdata() API.
176 **
177 ** Each extension function is allocated a single auxiliary data slot for
178 ** each FTS query (MATCH expression). If the extension function is invoked
179 ** more than once for a single FTS query, then all invocations share a
180 ** single auxiliary data context.
181 **
182 ** If there is already an auxiliary data pointer when this function is
183 ** invoked, then it is replaced by the new pointer. If an xDelete callback
184 ** was specified along with the original pointer, it is invoked at this
185 ** point.
186 **
187 ** The xDelete callback, if one is specified, is also invoked on the
188 ** auxiliary data pointer after the FTS5 query has finished.
189 **
190 ** If an error (e.g. an OOM condition) occurs within this function,
191 ** the auxiliary data is set to NULL and an error code returned. If the
192 ** xDelete parameter was not NULL, it is invoked on the auxiliary data
193 ** pointer before returning.
194 **
195 **
196 ** xGetAuxdata(pFts5, bClear)
197 **
198 ** Returns the current auxiliary data pointer for the fts5 extension
199 ** function. See the xSetAuxdata() method for details.
200 **
201 ** If the bClear argument is non-zero, then the auxiliary data is cleared
202 ** (set to NULL) before this function returns. In this case the xDelete,
203 ** if any, is not invoked.
204 **
205 **
206 ** xRowCount(pFts5, pnRow)
207 **
208 ** This function is used to retrieve the total number of rows in the table.
209 ** In other words, the same value that would be returned by:
210 **
211 ** SELECT count(*) FROM ftstable;
212 **
213 ** xPhraseFirst()
214 ** This function is used, along with type Fts5PhraseIter and the xPhraseNext
215 ** method, to iterate through all instances of a single query phrase within
216 ** the current row. This is the same information as is accessible via the
217 ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient
218 ** to use, this API may be faster under some circumstances. To iterate
219 ** through instances of phrase iPhrase, use the following code:
220 **
221 ** Fts5PhraseIter iter;
222 ** int iCol, iOff;
223 ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
224 ** iCol>=0;
225 ** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
226 ** ){
227 ** // An instance of phrase iPhrase at offset iOff of column iCol
228 ** }
229 **
230 ** The Fts5PhraseIter structure is defined above. Applications should not
231 ** modify this structure directly - it should only be used as shown above
232 ** with the xPhraseFirst() and xPhraseNext() API methods (and by
233 ** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below).
234 **
235 ** This API can be quite slow if used with an FTS5 table created with the
236 ** "detail=none" or "detail=column" option. If the FTS5 table is created
237 ** with either "detail=none" or "detail=column" and "content=" option
238 ** (i.e. if it is a contentless table), then this API always iterates
239 ** through an empty set (all calls to xPhraseFirst() set iCol to -1).
240 **
241 ** In all cases, matches are visited in (column ASC, offset ASC) order.
242 ** i.e. all those in column 0, sorted by offset, followed by those in
243 ** column 1, etc.
244 **
245 ** xPhraseNext()
246 ** See xPhraseFirst above.
247 **
248 ** xPhraseFirstColumn()
249 ** This function and xPhraseNextColumn() are similar to the xPhraseFirst()
250 ** and xPhraseNext() APIs described above. The difference is that instead
251 ** of iterating through all instances of a phrase in the current row, these
252 ** APIs are used to iterate through the set of columns in the current row
253 ** that contain one or more instances of a specified phrase. For example:
254 **
255 ** Fts5PhraseIter iter;
256 ** int iCol;
257 ** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
258 ** iCol>=0;
259 ** pApi->xPhraseNextColumn(pFts, &iter, &iCol)
260 ** ){
261 ** // Column iCol contains at least one instance of phrase iPhrase
262 ** }
263 **
264 ** This API can be quite slow if used with an FTS5 table created with the
265 ** "detail=none" option. If the FTS5 table is created with either
266 ** "detail=none" "content=" option (i.e. if it is a contentless table),
267 ** then this API always iterates through an empty set (all calls to
268 ** xPhraseFirstColumn() set iCol to -1).
269 **
270 ** The information accessed using this API and its companion
271 ** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext
272 ** (or xInst/xInstCount). The chief advantage of this API is that it is
273 ** significantly more efficient than those alternatives when used with
274 ** "detail=column" tables.
275 **
276 ** xPhraseNextColumn()
277 ** See xPhraseFirstColumn above.
278 **
279 ** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
280 ** This is used to access token iToken of phrase iPhrase of the current
281 ** query. Before returning, output parameter *ppToken is set to point
282 ** to a buffer containing the requested token, and *pnToken to the
283 ** size of this buffer in bytes.
284 **
285 ** If iPhrase or iToken are less than zero, or if iPhrase is greater than
286 ** or equal to the number of phrases in the query as reported by
287 ** xPhraseCount(), or if iToken is equal to or greater than the number of
288 ** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken
289 are both zeroed.
290 **
291 ** The output text is not a copy of the query text that specified the
292 ** token. It is the output of the tokenizer module. For tokendata=1
293 ** tables, this includes any embedded 0x00 and trailing data.
294 **
295 ** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
296 ** This is used to access token iToken of phrase hit iIdx within the
297 ** current row. If iIdx is less than zero or greater than or equal to the
298 ** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise,
299 ** output variable (*ppToken) is set to point to a buffer containing the
300 ** matching document token, and (*pnToken) to the size of that buffer in
301 ** bytes.
302 **
303 ** The output text is not a copy of the document text that was tokenized.
304 ** It is the output of the tokenizer module. For tokendata=1 tables, this
305 ** includes any embedded 0x00 and trailing data.
306 **
307 ** This API may be slow in some cases if the token identified by parameters
308 ** iIdx and iToken matched a prefix token in the query. In most cases, the
309 ** first call to this API for each prefix token in the query is forced
310 ** to scan the portion of the full-text index that matches the prefix
311 ** token to collect the extra data required by this API. If the prefix
312 ** token matches a large number of token instances in the document set,
313 ** this may be a performance problem.
314 **
315 ** If the user knows in advance that a query may use this API for a
316 ** prefix token, FTS5 may be configured to collect all required data as part
317 ** of the initial querying of the full-text index, avoiding the second scan
318 ** entirely. This also causes prefix queries that do not use this API to
319 ** run more slowly and use more memory. FTS5 may be configured in this way
320 ** either on a per-table basis using the [FTS5 insttoken | 'insttoken']
321 ** option, or on a per-query basis using the
322 ** [fts5_insttoken | fts5_insttoken()] user function.
323 **
324 ** This API can be quite slow if used with an FTS5 table created with the
325 ** "detail=none" or "detail=column" option.
326 **
327 ** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale)
328 ** If parameter iCol is less than zero, or greater than or equal to the
329 ** number of columns in the table, SQLITE_RANGE is returned.
330 **
331 ** Otherwise, this function attempts to retrieve the locale associated
332 ** with column iCol of the current row. Usually, there is no associated
333 ** locale, and output parameters (*pzLocale) and (*pnLocale) are set
334 ** to NULL and 0, respectively. However, if the fts5_locale() function
335 ** was used to associate a locale with the value when it was inserted
336 ** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated
337 ** buffer containing the name of the locale in utf-8 encoding. (*pnLocale)
338 ** is set to the size in bytes of the buffer, not including the
339 ** nul-terminator.
340 **
341 ** If successful, SQLITE_OK is returned. Or, if an error occurs, an
342 ** SQLite error code is returned. The final value of the output parameters
343 ** is undefined in this case.
344 **
345 ** xTokenize_v2:
346 ** Tokenize text using the tokenizer belonging to the FTS5 table. This
347 ** API is the same as the xTokenize() API, except that it allows a tokenizer
348 ** locale to be specified.
349 */
350 struct Fts5ExtensionApi {
351 int iVersion; /* Currently always set to 4 */
352
353 void *(*xUserData)(Fts5Context*);
354
355 int (*xColumnCount)(Fts5Context*);
356 int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
357 int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);
358
359 int (*xTokenize)(Fts5Context*,
360 const char *pText, int nText, /* Text to tokenize */
361 void *pCtx, /* Context passed to xToken() */
362 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
363 );
364
365 int (*xPhraseCount)(Fts5Context*);
366 int (*xPhraseSize)(Fts5Context*, int iPhrase);
367
368 int (*xInstCount)(Fts5Context*, int *pnInst);
369 int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff);
370
371 sqlite3_int64 (*xRowid)(Fts5Context*);
372 int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
373 int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
374
375 int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
376 int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
377 );
378 int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
379 void *(*xGetAuxdata)(Fts5Context*, int bClear);
380
381 int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
382 void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);
383
384 int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
385 void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
386
387 /* Below this point are iVersion>=3 only */
388 int (*xQueryToken)(Fts5Context*,
389 int iPhrase, int iToken,
390 const char **ppToken, int *pnToken
391 );
392 int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
393
394 /* Below this point are iVersion>=4 only */
395 int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn);
396 int (*xTokenize_v2)(Fts5Context*,
397 const char *pText, int nText, /* Text to tokenize */
398 const char *pLocale, int nLocale, /* Locale to pass to tokenizer */
399 void *pCtx, /* Context passed to xToken() */
400 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
401 );
402 };
403
404 /*
405 ** CUSTOM AUXILIARY FUNCTIONS
406 *************************************************************************/
407
408 /*************************************************************************
409 ** CUSTOM TOKENIZERS
410 **
411 ** Applications may also register custom tokenizer types. A tokenizer
412 ** is registered by providing fts5 with a populated instance of the
413 ** following structure. All structure methods must be defined, setting
414 ** any member of the fts5_tokenizer struct to NULL leads to undefined
415 ** behaviour. The structure methods are expected to function as follows:
416 **
417 ** xCreate:
418 ** This function is used to allocate and initialize a tokenizer instance.
419 ** A tokenizer instance is required to actually tokenize text.
420 **
421 ** The first argument passed to this function is a copy of the (void*)
422 ** pointer provided by the application when the fts5_tokenizer_v2 object
423 ** was registered with FTS5 (the third argument to xCreateTokenizer()).
424 ** The second and third arguments are an array of nul-terminated strings
425 ** containing the tokenizer arguments, if any, specified following the
426 ** tokenizer name as part of the CREATE VIRTUAL TABLE statement used
427 ** to create the FTS5 table.
428 **
429 ** The final argument is an output variable. If successful, (*ppOut)
430 ** should be set to point to the new tokenizer handle and SQLITE_OK
431 ** returned. If an error occurs, some value other than SQLITE_OK should
432 ** be returned. In this case, fts5 assumes that the final value of *ppOut
433 ** is undefined.
434 **
435 ** xDelete:
436 ** This function is invoked to delete a tokenizer handle previously
437 ** allocated using xCreate(). Fts5 guarantees that this function will
438 ** be invoked exactly once for each successful call to xCreate().
439 **
440 ** xTokenize:
441 ** This function is expected to tokenize the nText byte string indicated
442 ** by argument pText. pText may or may not be nul-terminated. The first
443 ** argument passed to this function is a pointer to an Fts5Tokenizer object
444 ** returned by an earlier call to xCreate().
445 **
446 ** The third argument indicates the reason that FTS5 is requesting
447 ** tokenization of the supplied text. This is always one of the following
448 ** four values:
449 **
450 ** <ul><li> <b>FTS5_TOKENIZE_DOCUMENT</b> - A document is being inserted into
451 ** or removed from the FTS table. The tokenizer is being invoked to
452 ** determine the set of tokens to add to (or delete from) the
453 ** FTS index.
454 **
455 ** <li> <b>FTS5_TOKENIZE_QUERY</b> - A MATCH query is being executed
456 ** against the FTS index. The tokenizer is being called to tokenize
457 ** a bareword or quoted string specified as part of the query.
458 **
459 ** <li> <b>(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)</b> - Same as
460 ** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is
461 ** followed by a "*" character, indicating that the last token
462 ** returned by the tokenizer will be treated as a token prefix.
463 **
464 ** <li> <b>FTS5_TOKENIZE_AUX</b> - The tokenizer is being invoked to
465 ** satisfy an fts5_api.xTokenize() request made by an auxiliary
466 ** function. Or an fts5_api.xColumnSize() request made by the same
467 ** on a columnsize=0 database.
468 ** </ul>
469 **
470 ** The sixth and seventh arguments passed to xTokenize() - pLocale and
471 ** nLocale - are a pointer to a buffer containing the locale to use for
472 ** tokenization (e.g. "en_US") and its size in bytes, respectively. The
473 ** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in
474 ** which case nLocale is always 0) to indicate that the tokenizer should
475 ** use its default locale.
476 **
477 ** For each token in the input string, the supplied callback xToken() must
478 ** be invoked. The first argument to it should be a copy of the pointer
479 ** passed as the second argument to xTokenize(). The third and fourth
480 ** arguments are a pointer to a buffer containing the token text, and the
481 ** size of the token in bytes. The 4th and 5th arguments are the byte offsets
482 ** of the first byte of and first byte immediately following the text from
483 ** which the token is derived within the input.
484 **
485 ** The second argument passed to the xToken() callback ("tflags") should
486 ** normally be set to 0. The exception is if the tokenizer supports
487 ** synonyms. In this case see the discussion below for details.
488 **
489 ** FTS5 assumes the xToken() callback is invoked for each token in the
490 ** order that they occur within the input text.
491 **
492 ** If an xToken() callback returns any value other than SQLITE_OK, then
493 ** the tokenization should be abandoned and the xTokenize() method should
494 ** immediately return a copy of the xToken() return value. Or, if the
495 ** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally,
496 ** if an error occurs with the xTokenize() implementation itself, it
497 ** may abandon the tokenization and return any error code other than
498 ** SQLITE_OK or SQLITE_DONE.
499 **
500 ** If the tokenizer is registered using an fts5_tokenizer_v2 object,
501 ** then the xTokenize() method has two additional arguments - pLocale
502 ** and nLocale. These specify the locale that the tokenizer should use
503 ** for the current request. If pLocale and nLocale are both 0, then the
504 ** tokenizer should use its default locale. Otherwise, pLocale points to
505 ** an nLocale byte buffer containing the name of the locale to use as utf-8
506 ** text. pLocale is not nul-terminated.
507 **
508 ** FTS5_TOKENIZER
509 **
510 ** There is also an fts5_tokenizer object. This is an older, deprecated,
511 ** version of fts5_tokenizer_v2. It is similar except that:
512 **
513 ** <ul>
514 ** <li> There is no "iVersion" field, and
515 ** <li> The xTokenize() method does not take a locale argument.
516 ** </ul>
517 **
518 ** Legacy fts5_tokenizer tokenizers must be registered using the
519 ** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2().
520 **
521 ** Tokenizer implementations registered using either API may be retrieved
522 ** using both xFindTokenizer() and xFindTokenizer_v2().
523 **
524 ** SYNONYM SUPPORT
525 **
526 ** Custom tokenizers may also support synonyms. Consider a case in which a
527 ** user wishes to query for a phrase such as "first place". Using the
528 ** built-in tokenizers, the FTS5 query 'first + place' will match instances
529 ** of "first place" within the document set, but not alternative forms
530 ** such as "1st place". In some applications, it would be better to match
531 ** all instances of "first place" or "1st place" regardless of which form
532 ** the user specified in the MATCH query text.
533 **
534 ** There are several ways to approach this in FTS5:
535 **
536 ** <ol><li> By mapping all synonyms to a single token. In this case, using
537 ** the above example, this means that the tokenizer returns the
538 ** same token for inputs "first" and "1st". Say that token is in
539 ** fact "first", so that when the user inserts the document "I won
540 ** 1st place" entries are added to the index for tokens "i", "won",
541 ** "first" and "place". If the user then queries for '1st + place',
542 ** the tokenizer substitutes "first" for "1st" and the query works
543 ** as expected.
544 **
545 ** <li> By querying the index for all synonyms of each query term
546 ** separately. In this case, when tokenizing query text, the
547 ** tokenizer may provide multiple synonyms for a single term
548 ** within the document. FTS5 then queries the index for each
549 ** synonym individually. For example, faced with the query:
550 **
551 ** <codeblock>
552 ** ... MATCH 'first place'</codeblock>
553 **
554 ** the tokenizer offers both "1st" and "first" as synonyms for the
555 ** first token in the MATCH query and FTS5 effectively runs a query
556 ** similar to:
557 **
558 ** <codeblock>
559 ** ... MATCH '(first OR 1st) place'</codeblock>
560 **
561 ** except that, for the purposes of auxiliary functions, the query
562 ** still appears to contain just two phrases - "(first OR 1st)"
563 ** being treated as a single phrase.
564 **
565 ** <li> By adding multiple synonyms for a single term to the FTS index.
566 ** Using this method, when tokenizing document text, the tokenizer
567 ** provides multiple synonyms for each token. So that when a
568 ** document such as "I won first place" is tokenized, entries are
569 ** added to the FTS index for "i", "won", "first", "1st" and
570 ** "place".
571 **
572 ** This way, even if the tokenizer does not provide synonyms
573 ** when tokenizing query text (it should not - to do so would be
574 ** inefficient), it doesn't matter if the user queries for
575 ** 'first + place' or '1st + place', as there are entries in the
576 ** FTS index corresponding to both forms of the first token.
577 ** </ol>
578 **
579 ** Whether it is parsing document or query text, any call to xToken that
580 ** specifies a <i>tflags</i> argument with the FTS5_TOKEN_COLOCATED bit
581 ** is considered to supply a synonym for the previous token. For example,
582 ** when parsing the document "I won first place", a tokenizer that supports
583 ** synonyms would call xToken() 5 times, as follows:
584 **
585 ** <codeblock>
586 ** xToken(pCtx, 0, "i", 1, 0, 1);
587 ** xToken(pCtx, 0, "won", 3, 2, 5);
588 ** xToken(pCtx, 0, "first", 5, 6, 11);
589 ** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11);
590 ** xToken(pCtx, 0, "place", 5, 12, 17);
591 **</codeblock>
592 **
593 ** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time
594 ** xToken() is called. Multiple synonyms may be specified for a single token
595 ** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence.
596 ** There is no limit to the number of synonyms that may be provided for a
597 ** single token.
598 **
599 ** In many cases, method (1) above is the best approach. It does not add
600 ** extra data to the FTS index or require FTS5 to query for multiple terms,
601 ** so it is efficient in terms of disk space and query speed. However, it
602 ** does not support prefix queries very well. If, as suggested above, the
603 ** token "first" is substituted for "1st" by the tokenizer, then the query:
604 **
605 ** <codeblock>
606 ** ... MATCH '1s*'</codeblock>
607 **
608 ** will not match documents that contain the token "1st" (as the tokenizer
609 ** will probably not map "1s" to any prefix of "first").
610 **
611 ** For full prefix support, method (3) may be preferred. In this case,
612 ** because the index contains entries for both "first" and "1st", prefix
613 ** queries such as 'fi*' or '1s*' will match correctly. However, because
614 ** extra entries are added to the FTS index, this method uses more space
615 ** within the database.
616 **
617 ** Method (2) offers a midpoint between (1) and (3). Using this method,
618 ** a query such as '1s*' will match documents that contain the literal
619 ** token "1st", but not "first" (assuming the tokenizer is not able to
620 ** provide synonyms for prefixes). However, a non-prefix query like '1st'
621 ** will match against "1st" and "first". This method does not require
622 ** extra disk space, as no extra entries are added to the FTS index.
623 ** On the other hand, it may require more CPU cycles to run MATCH queries,
624 ** as separate queries of the FTS index are required for each synonym.
625 **
626 ** When using methods (2) or (3), it is important that the tokenizer only
627 ** provide synonyms when tokenizing document text (method (3)) or query
628 ** text (method (2)), not both. Doing so will not cause any errors, but is
629 ** inefficient.
630 */
631 typedef struct Fts5Tokenizer Fts5Tokenizer;
632 typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2;
633 struct fts5_tokenizer_v2 {
634 int iVersion; /* Currently always 2 */
635
636 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
637 void (*xDelete)(Fts5Tokenizer*);
638 int (*xTokenize)(Fts5Tokenizer*,
639 void *pCtx,
640 int flags, /* Mask of FTS5_TOKENIZE_* flags */
641 const char *pText, int nText,
642 const char *pLocale, int nLocale,
643 int (*xToken)(
644 void *pCtx, /* Copy of 2nd argument to xTokenize() */
645 int tflags, /* Mask of FTS5_TOKEN_* flags */
646 const char *pToken, /* Pointer to buffer containing token */
647 int nToken, /* Size of token in bytes */
648 int iStart, /* Byte offset of token within input text */
649 int iEnd /* Byte offset of end of token within input text */
650 )
651 );
652 };
653
654 /*
655 ** New code should use the fts5_tokenizer_v2 type to define tokenizer
656 ** implementations. The following type is included for legacy applications
657 ** that still use it.
658 */
659 typedef struct fts5_tokenizer fts5_tokenizer;
660 struct fts5_tokenizer {
661 int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
662 void (*xDelete)(Fts5Tokenizer*);
663 int (*xTokenize)(Fts5Tokenizer*,
664 void *pCtx,
665 int flags, /* Mask of FTS5_TOKENIZE_* flags */
666 const char *pText, int nText,
667 int (*xToken)(
668 void *pCtx, /* Copy of 2nd argument to xTokenize() */
669 int tflags, /* Mask of FTS5_TOKEN_* flags */
670 const char *pToken, /* Pointer to buffer containing token */
671 int nToken, /* Size of token in bytes */
672 int iStart, /* Byte offset of token within input text */
673 int iEnd /* Byte offset of end of token within input text */
674 )
675 );
676 };
677
678
679 /* Flags that may be passed as the third argument to xTokenize() */
680 #define FTS5_TOKENIZE_QUERY 0x0001
681 #define FTS5_TOKENIZE_PREFIX 0x0002
682 #define FTS5_TOKENIZE_DOCUMENT 0x0004
683 #define FTS5_TOKENIZE_AUX 0x0008
684
685 /* Flags that may be passed by the tokenizer implementation back to FTS5
686 ** as the third argument to the supplied xToken callback. */
687 #define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */
688
689 /*
690 ** END OF CUSTOM TOKENIZERS
691 *************************************************************************/
692
693 /*************************************************************************
694 ** FTS5 EXTENSION REGISTRATION API
695 */
696 typedef struct fts5_api fts5_api;
697 struct fts5_api {
698 int iVersion; /* Currently always set to 3 */
699
700 /* Create a new tokenizer */
701 int (*xCreateTokenizer)(
702 fts5_api *pApi,
703 const char *zName,
704 void *pUserData,
705 fts5_tokenizer *pTokenizer,
706 void (*xDestroy)(void*)
707 );
708
709 /* Find an existing tokenizer */
710 int (*xFindTokenizer)(
711 fts5_api *pApi,
712 const char *zName,
713 void **ppUserData,
714 fts5_tokenizer *pTokenizer
715 );
716
717 /* Create a new auxiliary function */
718 int (*xCreateFunction)(
719 fts5_api *pApi,
720 const char *zName,
721 void *pUserData,
722 fts5_extension_function xFunction,
723 void (*xDestroy)(void*)
724 );
725
726 /* APIs below this point are only available if iVersion>=3 */
727
728 /* Create a new tokenizer */
729 int (*xCreateTokenizer_v2)(
730 fts5_api *pApi,
731 const char *zName,
732 void *pUserData,
733 fts5_tokenizer_v2 *pTokenizer,
734 void (*xDestroy)(void*)
735 );
736
737 /* Find an existing tokenizer */
738 int (*xFindTokenizer_v2)(
739 fts5_api *pApi,
740 const char *zName,
741 void **ppUserData,
742 fts5_tokenizer_v2 **ppTokenizer
743 );
744 };
745
746 /*
747 ** END OF REGISTRATION API
748 *************************************************************************/
749
750 #ifdef __cplusplus
751 } /* end of the 'extern "C"' block */
752 #endif
753
754 #endif /* _FTS5_H */