Debian Codegraph @

ext/fts5/fts5Int.h

raw
1 /*
2 ** 2014 May 31
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 **
13 */
14 #ifndef _FTS5INT_H
15 #define _FTS5INT_H
16
17 #include "fts5.h"
18 #include "sqlite3ext.h"
19 SQLITE_EXTENSION_INIT1
20
21 #include <string.h>
22 #include <assert.h>
23 #include <stddef.h>
24
25 #ifndef SQLITE_AMALGAMATION
26
27 typedef unsigned char u8;
28 typedef unsigned int u32;
29 typedef unsigned short u16;
30 typedef short i16;
31 typedef sqlite3_int64 i64;
32 typedef sqlite3_uint64 u64;
33
34 #ifndef ArraySize
35 # define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
36 #endif
37
38 #define testcase(x)
39
40 #if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
41 # define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1
42 #endif
43 #if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS)
44 # define ALWAYS(X) (1)
45 # define NEVER(X) (0)
46 #elif !defined(NDEBUG)
47 # define ALWAYS(X) ((X)?1:(assert(0),0))
48 # define NEVER(X) ((X)?(assert(0),1):0)
49 #else
50 # define ALWAYS(X) (X)
51 # define NEVER(X) (X)
52 #endif
53
54 #define MIN(x,y) (((x) < (y)) ? (x) : (y))
55 #define MAX(x,y) (((x) > (y)) ? (x) : (y))
56
57 /*
58 ** Constants for the largest and smallest possible 64-bit signed integers.
59 */
60 # define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
61 # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
62
63 /*
64 ** This macro is used in a single assert() within fts5 to check that an
65 ** allocation is aligned to an 8-byte boundary. But it is a complicated
66 ** macro to get right for multiple platforms without generating warnings.
67 ** So instead of reproducing the entire definition from sqliteInt.h, we
68 ** just do without this assert() for the rare non-amalgamation builds.
69 */
70 #define EIGHT_BYTE_ALIGNMENT(x) 1
71
72 /*
73 ** Macros needed to provide flexible arrays in a portable way
74 */
75 #ifndef offsetof
76 # define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0))
77 #endif
78 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
79 # define FLEXARRAY
80 #else
81 # define FLEXARRAY 1
82 #endif
83
84 #endif /* SQLITE_AMALGAMATION */
85
86 /*
87 ** Constants for the largest and smallest possible 32-bit signed integers.
88 */
89 # define LARGEST_INT32 ((int)(0x7fffffff))
90 # define SMALLEST_INT32 ((int)((-1) - LARGEST_INT32))
91
92 /* Truncate very long tokens to this many bytes. Hard limit is
93 ** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset
94 ** field that occurs at the start of each leaf page (see fts5_index.c). */
95 #define FTS5_MAX_TOKEN_SIZE 32768
96
97 /*
98 ** Maximum number of prefix indexes on single FTS5 table. This must be
99 ** less than 32. If it is set to anything large than that, an #error
100 ** directive in fts5_index.c will cause the build to fail.
101 */
102 #define FTS5_MAX_PREFIX_INDEXES 31
103
104 /*
105 ** Maximum segments permitted in a single index
106 */
107 #define FTS5_MAX_SEGMENT 2000
108
109 #define FTS5_DEFAULT_NEARDIST 10
110 #define FTS5_DEFAULT_RANK "bm25"
111
112 /* Name of rank and rowid columns */
113 #define FTS5_RANK_NAME "rank"
114 #define FTS5_ROWID_NAME "rowid"
115
116 #ifdef SQLITE_DEBUG
117 # define FTS5_CORRUPT sqlite3Fts5Corrupt()
118 int sqlite3Fts5Corrupt(void);
119 #else
120 # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB
121 #endif
122
123 /*
124 ** The assert_nc() macro is similar to the assert() macro, except that it
125 ** is used for assert() conditions that are true only if it can be
126 ** guranteed that the database is not corrupt.
127 */
128 #ifdef SQLITE_DEBUG
129 extern int sqlite3_fts5_may_be_corrupt;
130 # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x))
131 #else
132 # define assert_nc(x) assert(x)
133 #endif
134
135 /*
136 ** A version of memcmp() that does not cause asan errors if one of the pointer
137 ** parameters is NULL and the number of bytes to compare is zero.
138 */
139 #define fts5Memcmp(s1, s2, n) ((n)<=0 ? 0 : memcmp((s1), (s2), (n)))
140
141 /* Mark a function parameter as unused, to suppress nuisance compiler
142 ** warnings. */
143 #ifndef UNUSED_PARAM
144 # define UNUSED_PARAM(X) (void)(X)
145 #endif
146
147 #ifndef UNUSED_PARAM2
148 # define UNUSED_PARAM2(X, Y) (void)(X), (void)(Y)
149 #endif
150
151 typedef struct Fts5Global Fts5Global;
152 typedef struct Fts5Colset Fts5Colset;
153
154 /* If a NEAR() clump or phrase may only match a specific set of columns,
155 ** then an object of the following type is used to record the set of columns.
156 ** Each entry in the aiCol[] array is a column that may be matched.
157 **
158 ** This object is used by fts5_expr.c and fts5_index.c.
159 */
160 struct Fts5Colset {
161 int nCol;
162 int aiCol[FLEXARRAY];
163 };
164
165 /* Size (int bytes) of a complete Fts5Colset object with N columns. */
166 #define SZ_FTS5COLSET(N) (sizeof(i64)*((N+2)/2))
167
168 /**************************************************************************
169 ** Interface to code in fts5_config.c. fts5_config.c contains contains code
170 ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement.
171 */
172
173 typedef struct Fts5Config Fts5Config;
174 typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;
175
176 struct Fts5TokenizerConfig {
177 Fts5Tokenizer *pTok;
178 fts5_tokenizer_v2 *pApi2;
179 fts5_tokenizer *pApi1;
180 const char **azArg;
181 int nArg;
182 int ePattern; /* FTS_PATTERN_XXX constant */
183 const char *pLocale; /* Current locale to use */
184 int nLocale; /* Size of pLocale in bytes */
185 };
186
187 /*
188 ** An instance of the following structure encodes all information that can
189 ** be gleaned from the CREATE VIRTUAL TABLE statement.
190 **
191 ** And all information loaded from the %_config table.
192 **
193 ** nAutomerge:
194 ** The minimum number of segments that an auto-merge operation should
195 ** attempt to merge together. A value of 1 sets the object to use the
196 ** compile time default. Zero disables auto-merge altogether.
197 **
198 ** bContentlessDelete:
199 ** True if the contentless_delete option was present in the CREATE
200 ** VIRTUAL TABLE statement.
201 **
202 ** zContent:
203 **
204 ** zContentRowid:
205 ** The value of the content_rowid= option, if one was specified. Or
206 ** the string "rowid" otherwise. This text is not quoted - if it is
207 ** used as part of an SQL statement it needs to be quoted appropriately.
208 **
209 ** zContentExprlist:
210 **
211 ** pzErrmsg:
212 ** This exists in order to allow the fts5_index.c module to return a
213 ** decent error message if it encounters a file-format version it does
214 ** not understand.
215 **
216 ** bColumnsize:
217 ** True if the %_docsize table is created.
218 **
219 ** bPrefixIndex:
220 ** This is only used for debugging. If set to false, any prefix indexes
221 ** are ignored. This value is configured using:
222 **
223 ** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
224 **
225 ** bLocale:
226 ** Set to true if locale=1 was specified when the table was created.
227 */
228 struct Fts5Config {
229 sqlite3 *db; /* Database handle */
230 Fts5Global *pGlobal; /* Global fts5 object for handle db */
231 char *zDb; /* Database holding FTS index (e.g. "main") */
232 char *zName; /* Name of FTS index */
233 int nCol; /* Number of columns */
234 char **azCol; /* Column names */
235 u8 *abUnindexed; /* True for unindexed columns */
236 int nPrefix; /* Number of prefix indexes */
237 int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */
238 int eContent; /* An FTS5_CONTENT value */
239 int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */
240 int bContentlessUnindexed; /* "contentless_unindexed=" option (dflt=0) */
241 char *zContent; /* content table */
242 char *zContentRowid; /* "content_rowid=" option value */
243 int bColumnsize; /* "columnsize=" option value (dflt==1) */
244 int bTokendata; /* "tokendata=" option value (dflt==0) */
245 int bLocale; /* "locale=" option value (dflt==0) */
246 int eDetail; /* FTS5_DETAIL_XXX value */
247 char *zContentExprlist;
248 Fts5TokenizerConfig t;
249 int bLock; /* True when table is preparing statement */
250
251
252 /* Values loaded from the %_config table */
253 int iVersion; /* fts5 file format 'version' */
254 int iCookie; /* Incremented when %_config is modified */
255 int pgsz; /* Approximate page size used in %_data */
256 int nAutomerge; /* 'automerge' setting */
257 int nCrisisMerge; /* Maximum allowed segments per level */
258 int nUsermerge; /* 'usermerge' setting */
259 int nHashSize; /* Bytes of memory for in-memory hash */
260 char *zRank; /* Name of rank function */
261 char *zRankArgs; /* Arguments to rank function */
262 int bSecureDelete; /* 'secure-delete' */
263 int nDeleteMerge; /* 'deletemerge' */
264 int bPrefixInsttoken; /* 'prefix-insttoken' */
265
266 /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
267 char **pzErrmsg;
268
269 #ifdef SQLITE_DEBUG
270 int bPrefixIndex; /* True to use prefix-indexes */
271 #endif
272 };
273
274 /* Current expected value of %_config table 'version' field. And
275 ** the expected version if the 'secure-delete' option has ever been
276 ** set on the table. */
277 #define FTS5_CURRENT_VERSION 4
278 #define FTS5_CURRENT_VERSION_SECUREDELETE 5
279
280 #define FTS5_CONTENT_NORMAL 0
281 #define FTS5_CONTENT_NONE 1
282 #define FTS5_CONTENT_EXTERNAL 2
283 #define FTS5_CONTENT_UNINDEXED 3
284
285 #define FTS5_DETAIL_FULL 0
286 #define FTS5_DETAIL_NONE 1
287 #define FTS5_DETAIL_COLUMNS 2
288
289 #define FTS5_PATTERN_NONE 0
290 #define FTS5_PATTERN_LIKE 65 /* matches SQLITE_INDEX_CONSTRAINT_LIKE */
291 #define FTS5_PATTERN_GLOB 66 /* matches SQLITE_INDEX_CONSTRAINT_GLOB */
292
293 int sqlite3Fts5ConfigParse(
294 Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
295 );
296 void sqlite3Fts5ConfigFree(Fts5Config*);
297
298 int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
299
300 int sqlite3Fts5Tokenize(
301 Fts5Config *pConfig, /* FTS5 Configuration object */
302 int flags, /* FTS5_TOKENIZE_* flags */
303 const char *pText, int nText, /* Text to tokenize */
304 void *pCtx, /* Context passed to xToken() */
305 int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
306 );
307
308 void sqlite3Fts5Dequote(char *z);
309
310 /* Load the contents of the %_config table */
311 int sqlite3Fts5ConfigLoad(Fts5Config*, int);
312
313 /* Set the value of a single config attribute */
314 int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
315
316 int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
317
318 void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...);
319
320 /*
321 ** End of interface to code in fts5_config.c.
322 **************************************************************************/
323
324 /**************************************************************************
325 ** Interface to code in fts5_buffer.c.
326 */
327
328 /*
329 ** Buffer object for the incremental building of string data.
330 */
331 typedef struct Fts5Buffer Fts5Buffer;
332 struct Fts5Buffer {
333 u8 *p;
334 int n;
335 int nSpace;
336 };
337
338 int sqlite3Fts5BufferSize(int*, Fts5Buffer*, u32);
339 void sqlite3Fts5BufferAppendVarint(int*, Fts5Buffer*, i64);
340 void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, u32, const u8*);
341 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
342 void sqlite3Fts5BufferFree(Fts5Buffer*);
343 void sqlite3Fts5BufferZero(Fts5Buffer*);
344 void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
345 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
346
347 char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
348
349 #define fts5BufferZero(x) sqlite3Fts5BufferZero(x)
350 #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c)
351 #define fts5BufferFree(a) sqlite3Fts5BufferFree(a)
352 #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
353 #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d)
354
355 #define fts5BufferGrow(pRc,pBuf,nn) ( \
356 (u32)((pBuf)->n) + (u32)(nn) <= (u32)((pBuf)->nSpace) ? 0 : \
357 sqlite3Fts5BufferSize((pRc),(pBuf),(nn)+(pBuf)->n) \
358 )
359
360 /* Write and decode big-endian 32-bit integer values */
361 void sqlite3Fts5Put32(u8*, int);
362 int sqlite3Fts5Get32(const u8*);
363
364 #define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF)
365 #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF)
366
367 typedef struct Fts5PoslistReader Fts5PoslistReader;
368 struct Fts5PoslistReader {
369 /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */
370 const u8 *a; /* Position list to iterate through */
371 int n; /* Size of buffer at a[] in bytes */
372 int i; /* Current offset in a[] */
373
374 u8 bFlag; /* For client use (any custom purpose) */
375
376 /* Output variables */
377 u8 bEof; /* Set to true at EOF */
378 i64 iPos; /* (iCol<<32) + iPos */
379 };
380 int sqlite3Fts5PoslistReaderInit(
381 const u8 *a, int n, /* Poslist buffer to iterate through */
382 Fts5PoslistReader *pIter /* Iterator object to initialize */
383 );
384 int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*);
385
386 typedef struct Fts5PoslistWriter Fts5PoslistWriter;
387 struct Fts5PoslistWriter {
388 i64 iPrev;
389 };
390 int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
391 void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
392
393 int sqlite3Fts5PoslistNext64(
394 const u8 *a, int n, /* Buffer containing poslist */
395 int *pi, /* IN/OUT: Offset within a[] */
396 i64 *piOff /* IN/OUT: Current offset */
397 );
398
399 /* Malloc utility */
400 void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte);
401 char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
402
403 /* Character set tests (like isspace(), isalpha() etc.) */
404 int sqlite3Fts5IsBareword(char t);
405
406
407 /* Bucket of terms object used by the integrity-check in offsets=0 mode. */
408 typedef struct Fts5Termset Fts5Termset;
409 int sqlite3Fts5TermsetNew(Fts5Termset**);
410 int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
411 void sqlite3Fts5TermsetFree(Fts5Termset*);
412
413 /*
414 ** End of interface to code in fts5_buffer.c.
415 **************************************************************************/
416
417 /**************************************************************************
418 ** Interface to code in fts5_index.c. fts5_index.c contains contains code
419 ** to access the data stored in the %_data table.
420 */
421
422 typedef struct Fts5Index Fts5Index;
423 typedef struct Fts5IndexIter Fts5IndexIter;
424
425 struct Fts5IndexIter {
426 i64 iRowid;
427 const u8 *pData;
428 int nData;
429 u8 bEof;
430 };
431
432 #define sqlite3Fts5IterEof(x) ((x)->bEof)
433
434 /*
435 ** Values used as part of the flags argument passed to IndexQuery().
436 */
437 #define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */
438 #define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */
439 #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */
440 #define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */
441
442 /* The following are used internally by the fts5_index.c module. They are
443 ** defined here only to make it easier to avoid clashes with the flags
444 ** above. */
445 #define FTS5INDEX_QUERY_SKIPEMPTY 0x0010
446 #define FTS5INDEX_QUERY_NOOUTPUT 0x0020
447 #define FTS5INDEX_QUERY_SKIPHASH 0x0040
448 #define FTS5INDEX_QUERY_NOTOKENDATA 0x0080
449 #define FTS5INDEX_QUERY_SCANONETERM 0x0100
450
451 /*
452 ** Create/destroy an Fts5Index object.
453 */
454 int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
455 int sqlite3Fts5IndexClose(Fts5Index *p);
456
457 /*
458 ** Return a simple checksum value based on the arguments.
459 */
460 u64 sqlite3Fts5IndexEntryCksum(
461 i64 iRowid,
462 int iCol,
463 int iPos,
464 int iIdx,
465 const char *pTerm,
466 int nTerm
467 );
468
469 /*
470 ** Argument p points to a buffer containing utf-8 text that is n bytes in
471 ** size. Return the number of bytes in the nChar character prefix of the
472 ** buffer, or 0 if there are less than nChar characters in total.
473 */
474 int sqlite3Fts5IndexCharlenToBytelen(
475 const char *p,
476 int nByte,
477 int nChar
478 );
479
480 /*
481 ** Open a new iterator to iterate though all rowids that match the
482 ** specified token or token prefix.
483 */
484 int sqlite3Fts5IndexQuery(
485 Fts5Index *p, /* FTS index to query */
486 const char *pToken, int nToken, /* Token (or prefix) to query for */
487 int flags, /* Mask of FTS5INDEX_QUERY_X flags */
488 Fts5Colset *pColset, /* Match these columns only */
489 Fts5IndexIter **ppIter /* OUT: New iterator object */
490 );
491
492 /*
493 ** The various operations on open token or token prefix iterators opened
494 ** using sqlite3Fts5IndexQuery().
495 */
496 int sqlite3Fts5IterNext(Fts5IndexIter*);
497 int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
498
499 /*
500 ** Close an iterator opened by sqlite3Fts5IndexQuery().
501 */
502 void sqlite3Fts5IterClose(Fts5IndexIter*);
503
504 /*
505 ** Close the reader blob handle, if it is open.
506 */
507 void sqlite3Fts5IndexCloseReader(Fts5Index*);
508
509 /*
510 ** This interface is used by the fts5vocab module.
511 */
512 const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*);
513 int sqlite3Fts5IterNextScan(Fts5IndexIter*);
514 void *sqlite3Fts5StructureRef(Fts5Index*);
515 void sqlite3Fts5StructureRelease(void*);
516 int sqlite3Fts5StructureTest(Fts5Index*, void*);
517
518 /*
519 ** Used by xInstToken():
520 */
521 int sqlite3Fts5IterToken(
522 Fts5IndexIter *pIndexIter,
523 const char *pToken, int nToken,
524 i64 iRowid,
525 int iCol,
526 int iOff,
527 const char **ppOut, int *pnOut
528 );
529
530 /*
531 ** Insert or remove data to or from the index. Each time a document is
532 ** added to or removed from the index, this function is called one or more
533 ** times.
534 **
535 ** For an insert, it must be called once for each token in the new document.
536 ** If the operation is a delete, it must be called (at least) once for each
537 ** unique token in the document with an iCol value less than zero. The iPos
538 ** argument is ignored for a delete.
539 */
540 int sqlite3Fts5IndexWrite(
541 Fts5Index *p, /* Index to write to */
542 int iCol, /* Column token appears in (-ve -> delete) */
543 int iPos, /* Position of token within column */
544 const char *pToken, int nToken /* Token to add or remove to or from index */
545 );
546
547 /*
548 ** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to
549 ** document iDocid.
550 */
551 int sqlite3Fts5IndexBeginWrite(
552 Fts5Index *p, /* Index to write to */
553 int bDelete, /* True if current operation is a delete */
554 i64 iDocid /* Docid to add or remove data from */
555 );
556
557 /*
558 ** Flush any data stored in the in-memory hash tables to the database.
559 ** Also close any open blob handles.
560 */
561 int sqlite3Fts5IndexSync(Fts5Index *p);
562
563 /*
564 ** Discard any data stored in the in-memory hash tables. Do not write it
565 ** to the database. Additionally, assume that the contents of the %_data
566 ** table may have changed on disk. So any in-memory caches of %_data
567 ** records must be invalidated.
568 */
569 int sqlite3Fts5IndexRollback(Fts5Index *p);
570
571 /*
572 ** Get or set the "averages" values.
573 */
574 int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
575 int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
576
577 /*
578 ** Functions called by the storage module as part of integrity-check.
579 */
580 int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum, int bUseCksum);
581
582 /*
583 ** Called during virtual module initialization to register UDF
584 ** fts5_decode() with SQLite
585 */
586 int sqlite3Fts5IndexInit(sqlite3*);
587
588 int sqlite3Fts5IndexSetCookie(Fts5Index*, int);
589
590 /*
591 ** Return the total number of entries read from the %_data table by
592 ** this connection since it was created.
593 */
594 int sqlite3Fts5IndexReads(Fts5Index *p);
595
596 int sqlite3Fts5IndexReinit(Fts5Index *p);
597 int sqlite3Fts5IndexOptimize(Fts5Index *p);
598 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
599 int sqlite3Fts5IndexReset(Fts5Index *p);
600
601 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
602
603 int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
604 int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
605
606 void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
607
608 /* Used to populate hash tables for xInstToken in detail=none/column mode. */
609 int sqlite3Fts5IndexIterWriteTokendata(
610 Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff
611 );
612
613 /*
614 ** End of interface to code in fts5_index.c.
615 **************************************************************************/
616
617 /**************************************************************************
618 ** Interface to code in fts5_varint.c.
619 */
620 int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
621 int sqlite3Fts5GetVarintLen(u32 iVal);
622 u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
623 int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
624
625 #define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&(b))
626 #define fts5GetVarint sqlite3Fts5GetVarint
627
628 #define fts5FastGetVarint32(a, iOff, nVal) { \
629 nVal = (a)[iOff++]; \
630 if( nVal & 0x80 ){ \
631 iOff--; \
632 iOff += fts5GetVarint32(&(a)[iOff], nVal); \
633 } \
634 }
635
636
637 /*
638 ** End of interface to code in fts5_varint.c.
639 **************************************************************************/
640
641
642 /**************************************************************************
643 ** Interface to code in fts5_main.c.
644 */
645
646 /*
647 ** Virtual-table object.
648 */
649 typedef struct Fts5Table Fts5Table;
650 struct Fts5Table {
651 sqlite3_vtab base; /* Base class used by SQLite core */
652 Fts5Config *pConfig; /* Virtual table configuration */
653 Fts5Index *pIndex; /* Full-text index */
654 };
655
656 int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig);
657
658 Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
659
660 int sqlite3Fts5FlushToDisk(Fts5Table*);
661
662 void sqlite3Fts5ClearLocale(Fts5Config *pConfig);
663 void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc);
664
665 int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal);
666 int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal,
667 const char **ppText, int *pnText, const char **ppLoc, int *pnLoc
668 );
669
670 /*
671 ** End of interface to code in fts5.c.
672 **************************************************************************/
673
674 /**************************************************************************
675 ** Interface to code in fts5_hash.c.
676 */
677 typedef struct Fts5Hash Fts5Hash;
678
679 /*
680 ** Create a hash table, free a hash table.
681 */
682 int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
683 void sqlite3Fts5HashFree(Fts5Hash*);
684
685 int sqlite3Fts5HashWrite(
686 Fts5Hash*,
687 i64 iRowid, /* Rowid for this entry */
688 int iCol, /* Column token appears in (-ve -> delete) */
689 int iPos, /* Position of token within column */
690 char bByte,
691 const char *pToken, int nToken /* Token to add or remove to or from index */
692 );
693
694 /*
695 ** Empty (but do not delete) a hash table.
696 */
697 void sqlite3Fts5HashClear(Fts5Hash*);
698
699 /*
700 ** Return true if the hash is empty, false otherwise.
701 */
702 int sqlite3Fts5HashIsEmpty(Fts5Hash*);
703
704 int sqlite3Fts5HashQuery(
705 Fts5Hash*, /* Hash table to query */
706 int nPre,
707 const char *pTerm, int nTerm, /* Query term */
708 void **ppObj, /* OUT: Pointer to doclist for pTerm */
709 int *pnDoclist /* OUT: Size of doclist in bytes */
710 );
711
712 int sqlite3Fts5HashScanInit(
713 Fts5Hash*, /* Hash table to query */
714 const char *pTerm, int nTerm /* Query prefix */
715 );
716 void sqlite3Fts5HashScanNext(Fts5Hash*);
717 int sqlite3Fts5HashScanEof(Fts5Hash*);
718 void sqlite3Fts5HashScanEntry(Fts5Hash *,
719 const char **pzTerm, /* OUT: term (nul-terminated) */
720 int *pnTerm, /* OUT: Size of term in bytes */
721 const u8 **ppDoclist, /* OUT: pointer to doclist */
722 int *pnDoclist /* OUT: size of doclist in bytes */
723 );
724
725
726
727 /*
728 ** End of interface to code in fts5_hash.c.
729 **************************************************************************/
730
731 /**************************************************************************
732 ** Interface to code in fts5_storage.c. fts5_storage.c contains contains
733 ** code to access the data stored in the %_content and %_docsize tables.
734 */
735
736 #define FTS5_STMT_SCAN_ASC 0 /* SELECT rowid, * FROM ... ORDER BY 1 ASC */
737 #define FTS5_STMT_SCAN_DESC 1 /* SELECT rowid, * FROM ... ORDER BY 1 DESC */
738 #define FTS5_STMT_LOOKUP 2 /* SELECT rowid, * FROM ... WHERE rowid=? */
739
740 typedef struct Fts5Storage Fts5Storage;
741
742 int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
743 int sqlite3Fts5StorageClose(Fts5Storage *p);
744 int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
745
746 int sqlite3Fts5DropAll(Fts5Config*);
747 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
748
749 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int);
750 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, int, sqlite3_value**, i64*);
751 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
752
753 int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg);
754
755 int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
756 void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);
757
758 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
759 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
760 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
761
762 int sqlite3Fts5StorageSync(Fts5Storage *p);
763 int sqlite3Fts5StorageRollback(Fts5Storage *p);
764
765 int sqlite3Fts5StorageConfigValue(
766 Fts5Storage *p, const char*, sqlite3_value*, int
767 );
768
769 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
770 int sqlite3Fts5StorageRebuild(Fts5Storage *p);
771 int sqlite3Fts5StorageOptimize(Fts5Storage *p);
772 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
773 int sqlite3Fts5StorageReset(Fts5Storage *p);
774
775 void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*);
776 int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel);
777
778 /*
779 ** End of interface to code in fts5_storage.c.
780 **************************************************************************/
781
782
783 /**************************************************************************
784 ** Interface to code in fts5_expr.c.
785 */
786 typedef struct Fts5Expr Fts5Expr;
787 typedef struct Fts5ExprNode Fts5ExprNode;
788 typedef struct Fts5Parse Fts5Parse;
789 typedef struct Fts5Token Fts5Token;
790 typedef struct Fts5ExprPhrase Fts5ExprPhrase;
791 typedef struct Fts5ExprNearset Fts5ExprNearset;
792
793 struct Fts5Token {
794 const char *p; /* Token text (not NULL terminated) */
795 int n; /* Size of buffer p in bytes */
796 };
797
798 /* Parse a MATCH expression. */
799 int sqlite3Fts5ExprNew(
800 Fts5Config *pConfig,
801 int bPhraseToAnd,
802 int iCol, /* Column on LHS of MATCH operator */
803 const char *zExpr,
804 Fts5Expr **ppNew,
805 char **pzErr
806 );
807 int sqlite3Fts5ExprPattern(
808 Fts5Config *pConfig,
809 int bGlob,
810 int iCol,
811 const char *zText,
812 Fts5Expr **pp
813 );
814
815 /*
816 ** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc);
817 ** rc==SQLITE_OK && 0==sqlite3Fts5ExprEof(pExpr);
818 ** rc = sqlite3Fts5ExprNext(pExpr)
819 ** ){
820 ** // The document with rowid iRowid matches the expression!
821 ** i64 iRowid = sqlite3Fts5ExprRowid(pExpr);
822 ** }
823 */
824 int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, i64, int bDesc);
825 int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
826 int sqlite3Fts5ExprEof(Fts5Expr*);
827 i64 sqlite3Fts5ExprRowid(Fts5Expr*);
828
829 void sqlite3Fts5ExprFree(Fts5Expr*);
830 int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2);
831
832 /* Called during startup to register a UDF with SQLite */
833 int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
834
835 int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
836 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
837 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
838
839 typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
840 Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
841 int sqlite3Fts5ExprPopulatePoslists(
842 Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
843 );
844 void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
845
846 int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
847
848 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
849
850 int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
851 int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
852 void sqlite3Fts5ExprClearTokens(Fts5Expr*);
853
854 /*******************************************
855 ** The fts5_expr.c API above this point is used by the other hand-written
856 ** C code in this module. The interfaces below this point are called by
857 ** the parser code in fts5parse.y. */
858
859 void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
860
861 Fts5ExprNode *sqlite3Fts5ParseNode(
862 Fts5Parse *pParse,
863 int eType,
864 Fts5ExprNode *pLeft,
865 Fts5ExprNode *pRight,
866 Fts5ExprNearset *pNear
867 );
868
869 Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
870 Fts5Parse *pParse,
871 Fts5ExprNode *pLeft,
872 Fts5ExprNode *pRight
873 );
874
875 Fts5ExprPhrase *sqlite3Fts5ParseTerm(
876 Fts5Parse *pParse,
877 Fts5ExprPhrase *pPhrase,
878 Fts5Token *pToken,
879 int bPrefix
880 );
881
882 void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
883
884 Fts5ExprNearset *sqlite3Fts5ParseNearset(
885 Fts5Parse*,
886 Fts5ExprNearset*,
887 Fts5ExprPhrase*
888 );
889
890 Fts5Colset *sqlite3Fts5ParseColset(
891 Fts5Parse*,
892 Fts5Colset*,
893 Fts5Token *
894 );
895
896 void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
897 void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
898 void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
899
900 void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
901 void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
902 Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
903 void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
904 void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
905
906 /*
907 ** End of interface to code in fts5_expr.c.
908 **************************************************************************/
909
910
911
912 /**************************************************************************
913 ** Interface to code in fts5_aux.c.
914 */
915
916 int sqlite3Fts5AuxInit(fts5_api*);
917 /*
918 ** End of interface to code in fts5_aux.c.
919 **************************************************************************/
920
921 /**************************************************************************
922 ** Interface to code in fts5_tokenizer.c.
923 */
924
925 int sqlite3Fts5TokenizerInit(fts5_api*);
926 int sqlite3Fts5TokenizerPattern(
927 int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
928 Fts5Tokenizer *pTok
929 );
930 int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*);
931 /*
932 ** End of interface to code in fts5_tokenizer.c.
933 **************************************************************************/
934
935 /**************************************************************************
936 ** Interface to code in fts5_vocab.c.
937 */
938
939 int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*);
940
941 /*
942 ** End of interface to code in fts5_vocab.c.
943 **************************************************************************/
944
945
946 /**************************************************************************
947 ** Interface to automatically generated code in fts5_unicode2.c.
948 */
949 int sqlite3Fts5UnicodeIsdiacritic(int c);
950 int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
951
952 int sqlite3Fts5UnicodeCatParse(const char*, u8*);
953 int sqlite3Fts5UnicodeCategory(u32 iCode);
954 void sqlite3Fts5UnicodeAscii(u8*, u8*);
955 /*
956 ** End of interface to code in fts5_unicode2.c.
957 **************************************************************************/
958
959 #endif