Debian Codegraph @

ext/fts3/fts3Int.h

raw
1 /*
2 ** 2009 Nov 12
3 **
4 ** The author disclaims copyright to this source code. In place of
5 ** a legal notice, here is a blessing:
6 **
7 ** May you do good and not evil.
8 ** May you find forgiveness for yourself and forgive others.
9 ** May you share freely, never taking more than you give.
10 **
11 ******************************************************************************
12 **
13 */
14 #ifndef _FTSINT_H
15 #define _FTSINT_H
16
17 /*
18 ** Activate assert() only if SQLITE_TEST is enabled.
19 */
20 #if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
21 # define NDEBUG 1
22 #endif
23
24 #include <assert.h>
25 #include <stdlib.h>
26 #include <stddef.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <stdarg.h>
30
31 /* FTS3/FTS4 require virtual tables */
32 #ifdef SQLITE_OMIT_VIRTUALTABLE
33 # undef SQLITE_ENABLE_FTS3
34 # undef SQLITE_ENABLE_FTS4
35 #endif
36
37 /*
38 ** FTS4 is really an extension for FTS3. It is enabled using the
39 ** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all
40 ** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3.
41 */
42 #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3)
43 # define SQLITE_ENABLE_FTS3
44 #endif
45
46 #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)
47
48 /* If not building as part of the core, include sqlite3ext.h. */
49 #ifndef SQLITE_CORE
50 # include "sqlite3ext.h"
51 SQLITE_EXTENSION_INIT3
52 #endif
53
54 #include "sqlite3.h"
55 #include "fts3_tokenizer.h"
56 #include "fts3_hash.h"
57
58 /*
59 ** This constant determines the maximum depth of an FTS expression tree
60 ** that the library will create and use. FTS uses recursion to perform
61 ** various operations on the query tree, so the disadvantage of a large
62 ** limit is that it may allow very large queries to use large amounts
63 ** of stack space (perhaps causing a stack overflow).
64 */
65 #ifndef SQLITE_FTS3_MAX_EXPR_DEPTH
66 # define SQLITE_FTS3_MAX_EXPR_DEPTH 12
67 #endif
68
69
70 /*
71 ** This constant controls how often segments are merged. Once there are
72 ** FTS3_MERGE_COUNT segments of level N, they are merged into a single
73 ** segment of level N+1.
74 */
75 #define FTS3_MERGE_COUNT 16
76
77 /*
78 ** This is the maximum amount of data (in bytes) to store in the
79 ** Fts3Table.pendingTerms hash table. Normally, the hash table is
80 ** populated as documents are inserted/updated/deleted in a transaction
81 ** and used to create a new segment when the transaction is committed.
82 ** However if this limit is reached midway through a transaction, a new
83 ** segment is created and the hash table cleared immediately.
84 */
85 #define FTS3_MAX_PENDING_DATA (1*1024*1024)
86
87 /*
88 ** Macro to return the number of elements in an array. SQLite has a
89 ** similar macro called ArraySize(). Use a different name to avoid
90 ** a collision when building an amalgamation with built-in FTS3.
91 */
92 #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
93
94
95 #ifndef MIN
96 # define MIN(x,y) ((x)<(y)?(x):(y))
97 #endif
98 #ifndef MAX
99 # define MAX(x,y) ((x)>(y)?(x):(y))
100 #endif
101
102 /*
103 ** Maximum length of a varint encoded integer. The varint format is different
104 ** from that used by SQLite, so the maximum length is 10, not 9.
105 */
106 #define FTS3_VARINT_MAX 10
107
108 #define FTS3_BUFFER_PADDING 8
109
110 /*
111 ** FTS4 virtual tables may maintain multiple indexes - one index of all terms
112 ** in the document set and zero or more prefix indexes. All indexes are stored
113 ** as one or more b+-trees in the %_segments and %_segdir tables.
114 **
115 ** It is possible to determine which index a b+-tree belongs to based on the
116 ** value stored in the "%_segdir.level" column. Given this value L, the index
117 ** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with
118 ** level values between 0 and 1023 (inclusive) belong to index 0, all levels
119 ** between 1024 and 2047 to index 1, and so on.
120 **
121 ** It is considered impossible for an index to use more than 1024 levels. In
122 ** theory though this may happen, but only after at least
123 ** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables.
124 */
125 #define FTS3_SEGDIR_MAXLEVEL 1024
126 #define FTS3_SEGDIR_MAXLEVEL_STR "1024"
127
128 /*
129 ** The testcase() macro is only used by the amalgamation. If undefined,
130 ** make it a no-op.
131 */
132 #ifndef testcase
133 # define testcase(X)
134 #endif
135
136 /*
137 ** Terminator values for position-lists and column-lists.
138 */
139 #define POS_COLUMN (1) /* Column-list terminator */
140 #define POS_END (0) /* Position-list terminator */
141
142 /*
143 ** The assert_fts3_nc() macro is similar to the assert() macro, except that it
144 ** is used for assert() conditions that are true only if it can be
145 ** guranteed that the database is not corrupt.
146 */
147 #ifdef SQLITE_DEBUG
148 extern int sqlite3_fts3_may_be_corrupt;
149 # define assert_fts3_nc(x) assert(sqlite3_fts3_may_be_corrupt || (x))
150 #else
151 # define assert_fts3_nc(x) assert(x)
152 #endif
153
154 /*
155 ** This section provides definitions to allow the
156 ** FTS3 extension to be compiled outside of the
157 ** amalgamation.
158 */
159 #ifndef SQLITE_AMALGAMATION
160 /*
161 ** Macros indicating that conditional expressions are always true or
162 ** false.
163 */
164 #if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
165 # define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1
166 #endif
167 #if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS)
168 # define ALWAYS(X) (1)
169 # define NEVER(X) (0)
170 #elif !defined(NDEBUG)
171 # define ALWAYS(X) ((X)?1:(assert(0),0))
172 # define NEVER(X) ((X)?(assert(0),1):0)
173 #else
174 # define ALWAYS(X) (X)
175 # define NEVER(X) (X)
176 #endif
177
178 /*
179 ** Internal types used by SQLite.
180 */
181 typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */
182 typedef short int i16; /* 2-byte (or larger) signed integer */
183 typedef unsigned int u32; /* 4-byte unsigned integer */
184 typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */
185 typedef sqlite3_int64 i64; /* 8-byte signed integer */
186
187 /*
188 ** Macro used to suppress compiler warnings for unused parameters.
189 */
190 #define UNUSED_PARAMETER(x) (void)(x)
191
192 /*
193 ** The TESTONLY macro is used to enclose variable declarations or
194 ** other bits of code that are needed to support the arguments
195 ** within testcase() and assert() macros.
196 */
197 #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
198 # define TESTONLY(X) X
199 #else
200 # define TESTONLY(X)
201 #endif
202
203 #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
204 #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
205
206 #if !defined(deliberate_fall_through)
207 # if defined(__has_attribute)
208 # if __has_attribute(fallthrough)
209 # define deliberate_fall_through __attribute__((fallthrough));
210 # endif
211 # endif
212 #endif
213 #if !defined(deliberate_fall_through)
214 # define deliberate_fall_through
215 #endif
216
217 /*
218 ** Macros needed to provide flexible arrays in a portable way
219 */
220 #ifndef offsetof
221 # define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0))
222 #endif
223 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
224 # define FLEXARRAY
225 #else
226 # define FLEXARRAY 1
227 #endif
228
229
230 #endif /* SQLITE_AMALGAMATION */
231
232 #ifdef SQLITE_DEBUG
233 int sqlite3Fts3Corrupt(void);
234 # define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt()
235 #else
236 # define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB
237 #endif
238
239 typedef struct Fts3Table Fts3Table;
240 typedef struct Fts3Cursor Fts3Cursor;
241 typedef struct Fts3Expr Fts3Expr;
242 typedef struct Fts3Phrase Fts3Phrase;
243 typedef struct Fts3PhraseToken Fts3PhraseToken;
244
245 typedef struct Fts3Doclist Fts3Doclist;
246 typedef struct Fts3SegFilter Fts3SegFilter;
247 typedef struct Fts3DeferredToken Fts3DeferredToken;
248 typedef struct Fts3SegReader Fts3SegReader;
249 typedef struct Fts3MultiSegReader Fts3MultiSegReader;
250
251 typedef struct MatchinfoBuffer MatchinfoBuffer;
252
253 /*
254 ** A connection to a fulltext index is an instance of the following
255 ** structure. The xCreate and xConnect methods create an instance
256 ** of this structure and xDestroy and xDisconnect free that instance.
257 ** All other methods receive a pointer to the structure as one of their
258 ** arguments.
259 */
260 struct Fts3Table {
261 sqlite3_vtab base; /* Base class used by SQLite core */
262 sqlite3 *db; /* The database connection */
263 const char *zDb; /* logical database name */
264 const char *zName; /* virtual table name */
265 int nColumn; /* number of named columns in virtual table */
266 char **azColumn; /* column names. malloced */
267 u8 *abNotindexed; /* True for 'notindexed' columns */
268 sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */
269 char *zContentTbl; /* content=xxx option, or NULL */
270 char *zLanguageid; /* languageid=xxx option, or NULL */
271 int nAutoincrmerge; /* Value configured by 'automerge' */
272 u32 nLeafAdd; /* Number of leaf blocks added this trans */
273 int bLock; /* Used to prevent recursive content= tbls */
274
275 /* Precompiled statements used by the implementation. Each of these
276 ** statements is run and reset within a single virtual table API call.
277 */
278 sqlite3_stmt *aStmt[40];
279 sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */
280
281 char *zReadExprlist;
282 char *zWriteExprlist;
283
284 int nNodeSize; /* Soft limit for node size */
285 u8 bFts4; /* True for FTS4, false for FTS3 */
286 u8 bHasStat; /* True if %_stat table exists (2==unknown) */
287 u8 bHasDocsize; /* True if %_docsize table exists */
288 u8 bDescIdx; /* True if doclists are in reverse order */
289 u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */
290 int nPgsz; /* Page size for host database */
291 char *zSegmentsTbl; /* Name of %_segments table */
292 sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
293 int iSavepoint;
294
295 /*
296 ** The following array of hash tables is used to buffer pending index
297 ** updates during transactions. All pending updates buffered at any one
298 ** time must share a common language-id (see the FTS4 langid= feature).
299 ** The current language id is stored in variable iPrevLangid.
300 **
301 ** A single FTS4 table may have multiple full-text indexes. For each index
302 ** there is an entry in the aIndex[] array. Index 0 is an index of all the
303 ** terms that appear in the document set. Each subsequent index in aIndex[]
304 ** is an index of prefixes of a specific length.
305 **
306 ** Variable nPendingData contains an estimate the memory consumed by the
307 ** pending data structures, including hash table overhead, but not including
308 ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash
309 ** tables are flushed to disk. Variable iPrevDocid is the docid of the most
310 ** recently inserted record.
311 */
312 int nIndex; /* Size of aIndex[] */
313 struct Fts3Index {
314 int nPrefix; /* Prefix length (0 for main terms index) */
315 Fts3Hash hPending; /* Pending terms table for this index */
316 } *aIndex;
317 int nMaxPendingData; /* Max pending data before flush to disk */
318 int nPendingData; /* Current bytes of pending data */
319 sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */
320 int iPrevLangid; /* Langid of recently inserted document */
321 int bPrevDelete; /* True if last operation was a delete */
322
323 #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
324 /* State variables used for validating that the transaction control
325 ** methods of the virtual table are called at appropriate times. These
326 ** values do not contribute to FTS functionality; they are used for
327 ** verifying the operation of the SQLite core.
328 */
329 int inTransaction; /* True after xBegin but before xCommit/xRollback */
330 int mxSavepoint; /* Largest valid xSavepoint integer */
331 #endif
332
333 #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
334 /* True to disable the incremental doclist optimization. This is controlled
335 ** by special insert command 'test-no-incr-doclist'. */
336 int bNoIncrDoclist;
337
338 /* Number of segments in a level */
339 int nMergeCount;
340 #endif
341 };
342
343 /* Macro to find the number of segments to merge */
344 #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
345 # define MergeCount(P) ((P)->nMergeCount)
346 #else
347 # define MergeCount(P) FTS3_MERGE_COUNT
348 #endif
349
350 /*
351 ** When the core wants to read from the virtual table, it creates a
352 ** virtual table cursor (an instance of the following structure) using
353 ** the xOpen method. Cursors are destroyed using the xClose method.
354 */
355 struct Fts3Cursor {
356 sqlite3_vtab_cursor base; /* Base class used by SQLite core */
357 i16 eSearch; /* Search strategy (see below) */
358 u8 isEof; /* True if at End Of Results */
359 u8 isRequireSeek; /* True if must seek pStmt to %_content row */
360 u8 bSeekStmt; /* True if pStmt is a seek */
361 sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */
362 Fts3Expr *pExpr; /* Parsed MATCH query string */
363 int iLangid; /* Language being queried for */
364 int nPhrase; /* Number of matchable phrases in query */
365 Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */
366 sqlite3_int64 iPrevId; /* Previous id read from aDoclist */
367 char *pNextId; /* Pointer into the body of aDoclist */
368 char *aDoclist; /* List of docids for full-text queries */
369 int nDoclist; /* Size of buffer at aDoclist */
370 u8 bDesc; /* True to sort in descending order */
371 int eEvalmode; /* An FTS3_EVAL_XX constant */
372 int nRowAvg; /* Average size of database rows, in pages */
373 sqlite3_int64 nDoc; /* Documents in table */
374 i64 iMinDocid; /* Minimum docid to return */
375 i64 iMaxDocid; /* Maximum docid to return */
376 int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
377 MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */
378 };
379
380 #define FTS3_EVAL_FILTER 0
381 #define FTS3_EVAL_NEXT 1
382 #define FTS3_EVAL_MATCHINFO 2
383
384 /*
385 ** The Fts3Cursor.eSearch member is always set to one of the following.
386 ** Actually, Fts3Cursor.eSearch can be greater than or equal to
387 ** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index
388 ** of the column to be searched. For example, in
389 **
390 ** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d);
391 ** SELECT docid FROM ex1 WHERE b MATCH 'one two three';
392 **
393 ** Because the LHS of the MATCH operator is 2nd column "b",
394 ** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a,
395 ** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1"
396 ** indicating that all columns should be searched,
397 ** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4.
398 */
399 #define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */
400 #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
401 #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
402
403 /*
404 ** The lower 16-bits of the sqlite3_index_info.idxNum value set by
405 ** the xBestIndex() method contains the Fts3Cursor.eSearch value described
406 ** above. The upper 16-bits contain a combination of the following
407 ** bits, used to describe extra constraints on full-text searches.
408 */
409 #define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */
410 #define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */
411 #define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */
412
413 struct Fts3Doclist {
414 char *aAll; /* Array containing doclist (or NULL) */
415 int nAll; /* Size of a[] in bytes */
416 char *pNextDocid; /* Pointer to next docid */
417
418 sqlite3_int64 iDocid; /* Current docid (if pList!=0) */
419 int bFreeList; /* True if pList should be sqlite3_free()d */
420 char *pList; /* Pointer to position list following iDocid */
421 int nList; /* Length of position list */
422 };
423
424 /*
425 ** A "phrase" is a sequence of one or more tokens that must match in
426 ** sequence. A single token is the base case and the most common case.
427 ** For a sequence of tokens contained in double-quotes (i.e. "one two three")
428 ** nToken will be the number of tokens in the string.
429 */
430 struct Fts3PhraseToken {
431 char *z; /* Text of the token */
432 int n; /* Number of bytes in buffer z */
433 int isPrefix; /* True if token ends with a "*" character */
434 int bFirst; /* True if token must appear at position 0 */
435
436 /* Variables above this point are populated when the expression is
437 ** parsed (by code in fts3_expr.c). Below this point the variables are
438 ** used when evaluating the expression. */
439 Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
440 Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
441 };
442
443 struct Fts3Phrase {
444 /* Cache of doclist for this phrase. */
445 Fts3Doclist doclist;
446 int bIncr; /* True if doclist is loaded incrementally */
447 int iDoclistToken;
448
449 /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an
450 ** OR condition. */
451 char *pOrPoslist;
452 i64 iOrDocid;
453
454 /* Variables below this point are populated by fts3_expr.c when parsing
455 ** a MATCH expression. Everything above is part of the evaluation phase.
456 */
457 int nToken; /* Number of tokens in the phrase */
458 int iColumn; /* Index of column this phrase must match */
459 Fts3PhraseToken aToken[FLEXARRAY]; /* One for each token in the phrase */
460 };
461
462 /* Size (in bytes) of an Fts3Phrase object large enough to hold N tokens */
463 #define SZ_FTS3PHRASE(N) \
464 (offsetof(Fts3Phrase,aToken)+(N)*sizeof(Fts3PhraseToken))
465
466 /*
467 ** A tree of these objects forms the RHS of a MATCH operator.
468 **
469 ** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist
470 ** points to a malloced buffer, size nDoclist bytes, containing the results
471 ** of this phrase query in FTS3 doclist format. As usual, the initial
472 ** "Length" field found in doclists stored on disk is omitted from this
473 ** buffer.
474 **
475 ** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global
476 ** matchinfo data. If it is not NULL, it points to an array of size nCol*3,
477 ** where nCol is the number of columns in the queried FTS table. The array
478 ** is populated as follows:
479 **
480 ** aMI[iCol*3 + 0] = Undefined
481 ** aMI[iCol*3 + 1] = Number of occurrences
482 ** aMI[iCol*3 + 2] = Number of rows containing at least one instance
483 **
484 ** The aMI array is allocated using sqlite3_malloc(). It should be freed
485 ** when the expression node is.
486 */
487 struct Fts3Expr {
488 int eType; /* One of the FTSQUERY_XXX values defined below */
489 int nNear; /* Valid if eType==FTSQUERY_NEAR */
490 Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */
491 Fts3Expr *pLeft; /* Left operand */
492 Fts3Expr *pRight; /* Right operand */
493 Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
494
495 /* The following are used by the fts3_eval.c module. */
496 sqlite3_int64 iDocid; /* Current docid */
497 u8 bEof; /* True this expression is at EOF already */
498 u8 bStart; /* True if iDocid is valid */
499 u8 bDeferred; /* True if this expression is entirely deferred */
500
501 /* The following are used by the fts3_snippet.c module. */
502 int iPhrase; /* Index of this phrase in matchinfo() results */
503 u32 *aMI; /* See above */
504 };
505
506 /*
507 ** Candidate values for Fts3Query.eType. Note that the order of the first
508 ** four values is in order of precedence when parsing expressions. For
509 ** example, the following:
510 **
511 ** "a OR b AND c NOT d NEAR e"
512 **
513 ** is equivalent to:
514 **
515 ** "a OR (b AND (c NOT (d NEAR e)))"
516 */
517 #define FTSQUERY_NEAR 1
518 #define FTSQUERY_NOT 2
519 #define FTSQUERY_AND 3
520 #define FTSQUERY_OR 4
521 #define FTSQUERY_PHRASE 5
522
523
524 /* fts3_write.c */
525 int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
526 int sqlite3Fts3PendingTermsFlush(Fts3Table *);
527 void sqlite3Fts3PendingTermsClear(Fts3Table *);
528 int sqlite3Fts3Optimize(Fts3Table *);
529 int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64,
530 sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
531 int sqlite3Fts3SegReaderPending(
532 Fts3Table*,int,const char*,int,int,Fts3SegReader**);
533 void sqlite3Fts3SegReaderFree(Fts3SegReader *);
534 int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **);
535 int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
536
537 int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
538 int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
539
540 #ifndef SQLITE_DISABLE_FTS4_DEFERRED
541 void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
542 int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
543 int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
544 void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
545 int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
546 #else
547 # define sqlite3Fts3FreeDeferredTokens(x)
548 # define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK
549 # define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK
550 # define sqlite3Fts3FreeDeferredDoclists(x)
551 # define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK
552 #endif
553
554 void sqlite3Fts3SegmentsClose(Fts3Table *);
555 int sqlite3Fts3MaxLevel(Fts3Table *, int *);
556
557 /* Special values interpreted by sqlite3SegReaderCursor() */
558 #define FTS3_SEGCURSOR_PENDING -1
559 #define FTS3_SEGCURSOR_ALL -2
560
561 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
562 int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
563 void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
564
565 int sqlite3Fts3SegReaderCursor(Fts3Table *,
566 int, int, int, const char *, int, int, int, Fts3MultiSegReader *);
567
568 /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
569 #define FTS3_SEGMENT_REQUIRE_POS 0x00000001
570 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002
571 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
572 #define FTS3_SEGMENT_PREFIX 0x00000008
573 #define FTS3_SEGMENT_SCAN 0x00000010
574 #define FTS3_SEGMENT_FIRST 0x00000020
575
576 /* Type passed as 4th argument to SegmentReaderIterate() */
577 struct Fts3SegFilter {
578 const char *zTerm;
579 int nTerm;
580 int iCol;
581 int flags;
582 };
583
584 struct Fts3MultiSegReader {
585 /* Used internally by sqlite3Fts3SegReaderXXX() calls */
586 Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */
587 int nSegment; /* Size of apSegment array */
588 int nAdvance; /* How many seg-readers to advance */
589 Fts3SegFilter *pFilter; /* Pointer to filter object */
590 char *aBuffer; /* Buffer to merge doclists in */
591 i64 nBuffer; /* Allocated size of aBuffer[] in bytes */
592
593 int iColFilter; /* If >=0, filter for this column */
594 int bRestart;
595
596 /* Used by fts3.c only. */
597 int nCost; /* Cost of running iterator */
598 int bLookup; /* True if a lookup of a single entry. */
599
600 /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
601 char *zTerm; /* Pointer to term buffer */
602 int nTerm; /* Size of zTerm in bytes */
603 char *aDoclist; /* Pointer to doclist buffer */
604 int nDoclist; /* Size of aDoclist[] in bytes */
605 };
606
607 int sqlite3Fts3Incrmerge(Fts3Table*,int,int);
608
609 #define fts3GetVarint32(p, piVal) ( \
610 (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \
611 )
612
613 int sqlite3Fts3PrepareStmt(
614 Fts3Table *p, /* Prepare for this connection */
615 const char *zSql, /* SQL to prepare */
616 int bPersist, /* True to set SQLITE_PREPARE_PERSISTENT */
617 int bAllowVtab, /* True to omit SQLITE_PREPARE_NO_VTAB */
618 sqlite3_stmt **pp /* OUT: Prepared statement */
619 );
620
621
622 /* fts3.c */
623 void sqlite3Fts3ErrMsg(char**,const char*,...);
624 int sqlite3Fts3PutVarint(char *, sqlite3_int64);
625 int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
626 int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *);
627 int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*);
628 int sqlite3Fts3GetVarint32(const char *, int *);
629 int sqlite3Fts3VarintLen(sqlite3_uint64);
630 void sqlite3Fts3Dequote(char *);
631 void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
632 int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
633 int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *);
634 void sqlite3Fts3CreateStatTable(int*, Fts3Table*);
635 int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc);
636 int sqlite3Fts3ReadInt(const char *z, int *pnOut);
637
638 /* fts3_tokenizer.c */
639 const char *sqlite3Fts3NextToken(const char *, int *);
640 int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);
641 int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *,
642 sqlite3_tokenizer **, char **
643 );
644 int sqlite3Fts3IsIdChar(char);
645
646 /* fts3_snippet.c */
647 void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
648 void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
649 const char *, const char *, int, int
650 );
651 void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
652 void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p);
653
654 /* fts3_expr.c */
655 int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
656 char **, int, int, int, const char *, int, Fts3Expr **, char **
657 );
658 void sqlite3Fts3ExprFree(Fts3Expr *);
659 #ifdef SQLITE_TEST
660 int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*);
661 int sqlite3Fts3InitTerm(sqlite3 *db);
662 #endif
663 void *sqlite3Fts3MallocZero(i64 nByte);
664
665 int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int,
666 sqlite3_tokenizer_cursor **
667 );
668
669 /* fts3_aux.c */
670 int sqlite3Fts3InitAux(sqlite3 *db);
671
672 void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
673
674 int sqlite3Fts3MsrIncrStart(
675 Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
676 int sqlite3Fts3MsrIncrNext(
677 Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
678 int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **);
679 int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
680 int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr);
681 int sqlite3Fts3MsrCancel(Fts3Cursor*, Fts3Expr*);
682
683 /* fts3_tokenize_vtab.c */
684 int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *, void(*xDestroy)(void*));
685
686 /* fts3_unicode2.c (functions generated by parsing unicode text files) */
687 #ifndef SQLITE_DISABLE_FTS3_UNICODE
688 int sqlite3FtsUnicodeFold(int, int);
689 int sqlite3FtsUnicodeIsalnum(int);
690 int sqlite3FtsUnicodeIsdiacritic(int);
691 #endif
692
693 int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*);
694
695 int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk);
696
697 #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
698 #endif /* _FTSINT_H */