| 1 |
/* |
| 2 |
** 2009 Nov 12 |
| 3 |
** |
| 4 |
** The author disclaims copyright to this source code. In place of |
| 5 |
** a legal notice, here is a blessing: |
| 6 |
** |
| 7 |
** May you do good and not evil. |
| 8 |
** May you find forgiveness for yourself and forgive others. |
| 9 |
** May you share freely, never taking more than you give. |
| 10 |
** |
| 11 |
****************************************************************************** |
| 12 |
** |
| 13 |
*/ |
| 14 |
#ifndef _FTSINT_H |
| 15 |
#define _FTSINT_H |
| 16 |
|
| 17 |
/* |
| 18 |
** Activate assert() only if SQLITE_TEST is enabled. |
| 19 |
*/ |
| 20 |
#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) |
| 21 |
# define NDEBUG 1 |
| 22 |
#endif |
| 23 |
|
| 24 |
#include <assert.h> |
| 25 |
#include <stdlib.h> |
| 26 |
#include <stddef.h> |
| 27 |
#include <stdio.h> |
| 28 |
#include <string.h> |
| 29 |
#include <stdarg.h> |
| 30 |
|
| 31 |
/* FTS3/FTS4 require virtual tables */ |
| 32 |
#ifdef SQLITE_OMIT_VIRTUALTABLE |
| 33 |
# undef SQLITE_ENABLE_FTS3 |
| 34 |
# undef SQLITE_ENABLE_FTS4 |
| 35 |
#endif |
| 36 |
|
| 37 |
/* |
| 38 |
** FTS4 is really an extension for FTS3. It is enabled using the |
| 39 |
** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all |
| 40 |
** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. |
| 41 |
*/ |
| 42 |
#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) |
| 43 |
# define SQLITE_ENABLE_FTS3 |
| 44 |
#endif |
| 45 |
|
| 46 |
#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) |
| 47 |
|
| 48 |
/* If not building as part of the core, include sqlite3ext.h. */ |
| 49 |
#ifndef SQLITE_CORE |
| 50 |
# include "sqlite3ext.h" |
| 51 |
SQLITE_EXTENSION_INIT3 |
| 52 |
#endif |
| 53 |
|
| 54 |
#include "sqlite3.h" |
| 55 |
#include "fts3_tokenizer.h" |
| 56 |
#include "fts3_hash.h" |
| 57 |
|
| 58 |
/* |
| 59 |
** This constant determines the maximum depth of an FTS expression tree |
| 60 |
** that the library will create and use. FTS uses recursion to perform |
| 61 |
** various operations on the query tree, so the disadvantage of a large |
| 62 |
** limit is that it may allow very large queries to use large amounts |
| 63 |
** of stack space (perhaps causing a stack overflow). |
| 64 |
*/ |
| 65 |
#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH |
| 66 |
# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 |
| 67 |
#endif |
| 68 |
|
| 69 |
|
| 70 |
/* |
| 71 |
** This constant controls how often segments are merged. Once there are |
| 72 |
** FTS3_MERGE_COUNT segments of level N, they are merged into a single |
| 73 |
** segment of level N+1. |
| 74 |
*/ |
| 75 |
#define FTS3_MERGE_COUNT 16 |
| 76 |
|
| 77 |
/* |
| 78 |
** This is the maximum amount of data (in bytes) to store in the |
| 79 |
** Fts3Table.pendingTerms hash table. Normally, the hash table is |
| 80 |
** populated as documents are inserted/updated/deleted in a transaction |
| 81 |
** and used to create a new segment when the transaction is committed. |
| 82 |
** However if this limit is reached midway through a transaction, a new |
| 83 |
** segment is created and the hash table cleared immediately. |
| 84 |
*/ |
| 85 |
#define FTS3_MAX_PENDING_DATA (1*1024*1024) |
| 86 |
|
| 87 |
/* |
| 88 |
** Macro to return the number of elements in an array. SQLite has a |
| 89 |
** similar macro called ArraySize(). Use a different name to avoid |
| 90 |
** a collision when building an amalgamation with built-in FTS3. |
| 91 |
*/ |
| 92 |
#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) |
| 93 |
|
| 94 |
|
| 95 |
#ifndef MIN |
| 96 |
# define MIN(x,y) ((x)<(y)?(x):(y)) |
| 97 |
#endif |
| 98 |
#ifndef MAX |
| 99 |
# define MAX(x,y) ((x)>(y)?(x):(y)) |
| 100 |
#endif |
| 101 |
|
| 102 |
/* |
| 103 |
** Maximum length of a varint encoded integer. The varint format is different |
| 104 |
** from that used by SQLite, so the maximum length is 10, not 9. |
| 105 |
*/ |
| 106 |
#define FTS3_VARINT_MAX 10 |
| 107 |
|
| 108 |
#define FTS3_BUFFER_PADDING 8 |
| 109 |
|
| 110 |
/* |
| 111 |
** FTS4 virtual tables may maintain multiple indexes - one index of all terms |
| 112 |
** in the document set and zero or more prefix indexes. All indexes are stored |
| 113 |
** as one or more b+-trees in the %_segments and %_segdir tables. |
| 114 |
** |
| 115 |
** It is possible to determine which index a b+-tree belongs to based on the |
| 116 |
** value stored in the "%_segdir.level" column. Given this value L, the index |
| 117 |
** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with |
| 118 |
** level values between 0 and 1023 (inclusive) belong to index 0, all levels |
| 119 |
** between 1024 and 2047 to index 1, and so on. |
| 120 |
** |
| 121 |
** It is considered impossible for an index to use more than 1024 levels. In |
| 122 |
** theory though this may happen, but only after at least |
| 123 |
** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. |
| 124 |
*/ |
| 125 |
#define FTS3_SEGDIR_MAXLEVEL 1024 |
| 126 |
#define FTS3_SEGDIR_MAXLEVEL_STR "1024" |
| 127 |
|
| 128 |
/* |
| 129 |
** The testcase() macro is only used by the amalgamation. If undefined, |
| 130 |
** make it a no-op. |
| 131 |
*/ |
| 132 |
#ifndef testcase |
| 133 |
# define testcase(X) |
| 134 |
#endif |
| 135 |
|
| 136 |
/* |
| 137 |
** Terminator values for position-lists and column-lists. |
| 138 |
*/ |
| 139 |
#define POS_COLUMN (1) /* Column-list terminator */ |
| 140 |
#define POS_END (0) /* Position-list terminator */ |
| 141 |
|
| 142 |
/* |
| 143 |
** The assert_fts3_nc() macro is similar to the assert() macro, except that it |
| 144 |
** is used for assert() conditions that are true only if it can be |
| 145 |
** guranteed that the database is not corrupt. |
| 146 |
*/ |
| 147 |
#ifdef SQLITE_DEBUG |
| 148 |
extern int sqlite3_fts3_may_be_corrupt; |
| 149 |
# define assert_fts3_nc(x) assert(sqlite3_fts3_may_be_corrupt || (x)) |
| 150 |
#else |
| 151 |
# define assert_fts3_nc(x) assert(x) |
| 152 |
#endif |
| 153 |
|
| 154 |
/* |
| 155 |
** This section provides definitions to allow the |
| 156 |
** FTS3 extension to be compiled outside of the |
| 157 |
** amalgamation. |
| 158 |
*/ |
| 159 |
#ifndef SQLITE_AMALGAMATION |
| 160 |
/* |
| 161 |
** Macros indicating that conditional expressions are always true or |
| 162 |
** false. |
| 163 |
*/ |
| 164 |
#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) |
| 165 |
# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 |
| 166 |
#endif |
| 167 |
#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) |
| 168 |
# define ALWAYS(X) (1) |
| 169 |
# define NEVER(X) (0) |
| 170 |
#elif !defined(NDEBUG) |
| 171 |
# define ALWAYS(X) ((X)?1:(assert(0),0)) |
| 172 |
# define NEVER(X) ((X)?(assert(0),1):0) |
| 173 |
#else |
| 174 |
# define ALWAYS(X) (X) |
| 175 |
# define NEVER(X) (X) |
| 176 |
#endif |
| 177 |
|
| 178 |
/* |
| 179 |
** Internal types used by SQLite. |
| 180 |
*/ |
| 181 |
typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ |
| 182 |
typedef short int i16; /* 2-byte (or larger) signed integer */ |
| 183 |
typedef unsigned int u32; /* 4-byte unsigned integer */ |
| 184 |
typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ |
| 185 |
typedef sqlite3_int64 i64; /* 8-byte signed integer */ |
| 186 |
|
| 187 |
/* |
| 188 |
** Macro used to suppress compiler warnings for unused parameters. |
| 189 |
*/ |
| 190 |
#define UNUSED_PARAMETER(x) (void)(x) |
| 191 |
|
| 192 |
/* |
| 193 |
** The TESTONLY macro is used to enclose variable declarations or |
| 194 |
** other bits of code that are needed to support the arguments |
| 195 |
** within testcase() and assert() macros. |
| 196 |
*/ |
| 197 |
#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) |
| 198 |
# define TESTONLY(X) X |
| 199 |
#else |
| 200 |
# define TESTONLY(X) |
| 201 |
#endif |
| 202 |
|
| 203 |
#define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) |
| 204 |
#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) |
| 205 |
|
| 206 |
#if !defined(deliberate_fall_through) |
| 207 |
# if defined(__has_attribute) |
| 208 |
# if __has_attribute(fallthrough) |
| 209 |
# define deliberate_fall_through __attribute__((fallthrough)); |
| 210 |
# endif |
| 211 |
# endif |
| 212 |
#endif |
| 213 |
#if !defined(deliberate_fall_through) |
| 214 |
# define deliberate_fall_through |
| 215 |
#endif |
| 216 |
|
| 217 |
/* |
| 218 |
** Macros needed to provide flexible arrays in a portable way |
| 219 |
*/ |
| 220 |
#ifndef offsetof |
| 221 |
# define offsetof(ST,M) ((size_t)((char*)&((ST*)0)->M - (char*)0)) |
| 222 |
#endif |
| 223 |
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) |
| 224 |
# define FLEXARRAY |
| 225 |
#else |
| 226 |
# define FLEXARRAY 1 |
| 227 |
#endif |
| 228 |
|
| 229 |
|
| 230 |
#endif /* SQLITE_AMALGAMATION */ |
| 231 |
|
| 232 |
#ifdef SQLITE_DEBUG |
| 233 |
int sqlite3Fts3Corrupt(void); |
| 234 |
# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() |
| 235 |
#else |
| 236 |
# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB |
| 237 |
#endif |
| 238 |
|
| 239 |
typedef struct Fts3Table Fts3Table; |
| 240 |
typedef struct Fts3Cursor Fts3Cursor; |
| 241 |
typedef struct Fts3Expr Fts3Expr; |
| 242 |
typedef struct Fts3Phrase Fts3Phrase; |
| 243 |
typedef struct Fts3PhraseToken Fts3PhraseToken; |
| 244 |
|
| 245 |
typedef struct Fts3Doclist Fts3Doclist; |
| 246 |
typedef struct Fts3SegFilter Fts3SegFilter; |
| 247 |
typedef struct Fts3DeferredToken Fts3DeferredToken; |
| 248 |
typedef struct Fts3SegReader Fts3SegReader; |
| 249 |
typedef struct Fts3MultiSegReader Fts3MultiSegReader; |
| 250 |
|
| 251 |
typedef struct MatchinfoBuffer MatchinfoBuffer; |
| 252 |
|
| 253 |
/* |
| 254 |
** A connection to a fulltext index is an instance of the following |
| 255 |
** structure. The xCreate and xConnect methods create an instance |
| 256 |
** of this structure and xDestroy and xDisconnect free that instance. |
| 257 |
** All other methods receive a pointer to the structure as one of their |
| 258 |
** arguments. |
| 259 |
*/ |
| 260 |
struct Fts3Table { |
| 261 |
sqlite3_vtab base; /* Base class used by SQLite core */ |
| 262 |
sqlite3 *db; /* The database connection */ |
| 263 |
const char *zDb; /* logical database name */ |
| 264 |
const char *zName; /* virtual table name */ |
| 265 |
int nColumn; /* number of named columns in virtual table */ |
| 266 |
char **azColumn; /* column names. malloced */ |
| 267 |
u8 *abNotindexed; /* True for 'notindexed' columns */ |
| 268 |
sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ |
| 269 |
char *zContentTbl; /* content=xxx option, or NULL */ |
| 270 |
char *zLanguageid; /* languageid=xxx option, or NULL */ |
| 271 |
int nAutoincrmerge; /* Value configured by 'automerge' */ |
| 272 |
u32 nLeafAdd; /* Number of leaf blocks added this trans */ |
| 273 |
int bLock; /* Used to prevent recursive content= tbls */ |
| 274 |
|
| 275 |
/* Precompiled statements used by the implementation. Each of these |
| 276 |
** statements is run and reset within a single virtual table API call. |
| 277 |
*/ |
| 278 |
sqlite3_stmt *aStmt[40]; |
| 279 |
sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ |
| 280 |
|
| 281 |
char *zReadExprlist; |
| 282 |
char *zWriteExprlist; |
| 283 |
|
| 284 |
int nNodeSize; /* Soft limit for node size */ |
| 285 |
u8 bFts4; /* True for FTS4, false for FTS3 */ |
| 286 |
u8 bHasStat; /* True if %_stat table exists (2==unknown) */ |
| 287 |
u8 bHasDocsize; /* True if %_docsize table exists */ |
| 288 |
u8 bDescIdx; /* True if doclists are in reverse order */ |
| 289 |
u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ |
| 290 |
int nPgsz; /* Page size for host database */ |
| 291 |
char *zSegmentsTbl; /* Name of %_segments table */ |
| 292 |
sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ |
| 293 |
int iSavepoint; |
| 294 |
|
| 295 |
/* |
| 296 |
** The following array of hash tables is used to buffer pending index |
| 297 |
** updates during transactions. All pending updates buffered at any one |
| 298 |
** time must share a common language-id (see the FTS4 langid= feature). |
| 299 |
** The current language id is stored in variable iPrevLangid. |
| 300 |
** |
| 301 |
** A single FTS4 table may have multiple full-text indexes. For each index |
| 302 |
** there is an entry in the aIndex[] array. Index 0 is an index of all the |
| 303 |
** terms that appear in the document set. Each subsequent index in aIndex[] |
| 304 |
** is an index of prefixes of a specific length. |
| 305 |
** |
| 306 |
** Variable nPendingData contains an estimate the memory consumed by the |
| 307 |
** pending data structures, including hash table overhead, but not including |
| 308 |
** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash |
| 309 |
** tables are flushed to disk. Variable iPrevDocid is the docid of the most |
| 310 |
** recently inserted record. |
| 311 |
*/ |
| 312 |
int nIndex; /* Size of aIndex[] */ |
| 313 |
struct Fts3Index { |
| 314 |
int nPrefix; /* Prefix length (0 for main terms index) */ |
| 315 |
Fts3Hash hPending; /* Pending terms table for this index */ |
| 316 |
} *aIndex; |
| 317 |
int nMaxPendingData; /* Max pending data before flush to disk */ |
| 318 |
int nPendingData; /* Current bytes of pending data */ |
| 319 |
sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ |
| 320 |
int iPrevLangid; /* Langid of recently inserted document */ |
| 321 |
int bPrevDelete; /* True if last operation was a delete */ |
| 322 |
|
| 323 |
#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) |
| 324 |
/* State variables used for validating that the transaction control |
| 325 |
** methods of the virtual table are called at appropriate times. These |
| 326 |
** values do not contribute to FTS functionality; they are used for |
| 327 |
** verifying the operation of the SQLite core. |
| 328 |
*/ |
| 329 |
int inTransaction; /* True after xBegin but before xCommit/xRollback */ |
| 330 |
int mxSavepoint; /* Largest valid xSavepoint integer */ |
| 331 |
#endif |
| 332 |
|
| 333 |
#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) |
| 334 |
/* True to disable the incremental doclist optimization. This is controlled |
| 335 |
** by special insert command 'test-no-incr-doclist'. */ |
| 336 |
int bNoIncrDoclist; |
| 337 |
|
| 338 |
/* Number of segments in a level */ |
| 339 |
int nMergeCount; |
| 340 |
#endif |
| 341 |
}; |
| 342 |
|
| 343 |
/* Macro to find the number of segments to merge */ |
| 344 |
#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) |
| 345 |
# define MergeCount(P) ((P)->nMergeCount) |
| 346 |
#else |
| 347 |
# define MergeCount(P) FTS3_MERGE_COUNT |
| 348 |
#endif |
| 349 |
|
| 350 |
/* |
| 351 |
** When the core wants to read from the virtual table, it creates a |
| 352 |
** virtual table cursor (an instance of the following structure) using |
| 353 |
** the xOpen method. Cursors are destroyed using the xClose method. |
| 354 |
*/ |
| 355 |
struct Fts3Cursor { |
| 356 |
sqlite3_vtab_cursor base; /* Base class used by SQLite core */ |
| 357 |
i16 eSearch; /* Search strategy (see below) */ |
| 358 |
u8 isEof; /* True if at End Of Results */ |
| 359 |
u8 isRequireSeek; /* True if must seek pStmt to %_content row */ |
| 360 |
u8 bSeekStmt; /* True if pStmt is a seek */ |
| 361 |
sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ |
| 362 |
Fts3Expr *pExpr; /* Parsed MATCH query string */ |
| 363 |
int iLangid; /* Language being queried for */ |
| 364 |
int nPhrase; /* Number of matchable phrases in query */ |
| 365 |
Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ |
| 366 |
sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ |
| 367 |
char *pNextId; /* Pointer into the body of aDoclist */ |
| 368 |
char *aDoclist; /* List of docids for full-text queries */ |
| 369 |
int nDoclist; /* Size of buffer at aDoclist */ |
| 370 |
u8 bDesc; /* True to sort in descending order */ |
| 371 |
int eEvalmode; /* An FTS3_EVAL_XX constant */ |
| 372 |
int nRowAvg; /* Average size of database rows, in pages */ |
| 373 |
sqlite3_int64 nDoc; /* Documents in table */ |
| 374 |
i64 iMinDocid; /* Minimum docid to return */ |
| 375 |
i64 iMaxDocid; /* Maximum docid to return */ |
| 376 |
int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ |
| 377 |
MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ |
| 378 |
}; |
| 379 |
|
| 380 |
#define FTS3_EVAL_FILTER 0 |
| 381 |
#define FTS3_EVAL_NEXT 1 |
| 382 |
#define FTS3_EVAL_MATCHINFO 2 |
| 383 |
|
| 384 |
/* |
| 385 |
** The Fts3Cursor.eSearch member is always set to one of the following. |
| 386 |
** Actually, Fts3Cursor.eSearch can be greater than or equal to |
| 387 |
** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index |
| 388 |
** of the column to be searched. For example, in |
| 389 |
** |
| 390 |
** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); |
| 391 |
** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; |
| 392 |
** |
| 393 |
** Because the LHS of the MATCH operator is 2nd column "b", |
| 394 |
** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, |
| 395 |
** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" |
| 396 |
** indicating that all columns should be searched, |
| 397 |
** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. |
| 398 |
*/ |
| 399 |
#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ |
| 400 |
#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ |
| 401 |
#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ |
| 402 |
|
| 403 |
/* |
| 404 |
** The lower 16-bits of the sqlite3_index_info.idxNum value set by |
| 405 |
** the xBestIndex() method contains the Fts3Cursor.eSearch value described |
| 406 |
** above. The upper 16-bits contain a combination of the following |
| 407 |
** bits, used to describe extra constraints on full-text searches. |
| 408 |
*/ |
| 409 |
#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ |
| 410 |
#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ |
| 411 |
#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ |
| 412 |
|
| 413 |
struct Fts3Doclist { |
| 414 |
char *aAll; /* Array containing doclist (or NULL) */ |
| 415 |
int nAll; /* Size of a[] in bytes */ |
| 416 |
char *pNextDocid; /* Pointer to next docid */ |
| 417 |
|
| 418 |
sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ |
| 419 |
int bFreeList; /* True if pList should be sqlite3_free()d */ |
| 420 |
char *pList; /* Pointer to position list following iDocid */ |
| 421 |
int nList; /* Length of position list */ |
| 422 |
}; |
| 423 |
|
| 424 |
/* |
| 425 |
** A "phrase" is a sequence of one or more tokens that must match in |
| 426 |
** sequence. A single token is the base case and the most common case. |
| 427 |
** For a sequence of tokens contained in double-quotes (i.e. "one two three") |
| 428 |
** nToken will be the number of tokens in the string. |
| 429 |
*/ |
| 430 |
struct Fts3PhraseToken { |
| 431 |
char *z; /* Text of the token */ |
| 432 |
int n; /* Number of bytes in buffer z */ |
| 433 |
int isPrefix; /* True if token ends with a "*" character */ |
| 434 |
int bFirst; /* True if token must appear at position 0 */ |
| 435 |
|
| 436 |
/* Variables above this point are populated when the expression is |
| 437 |
** parsed (by code in fts3_expr.c). Below this point the variables are |
| 438 |
** used when evaluating the expression. */ |
| 439 |
Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ |
| 440 |
Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ |
| 441 |
}; |
| 442 |
|
| 443 |
struct Fts3Phrase { |
| 444 |
/* Cache of doclist for this phrase. */ |
| 445 |
Fts3Doclist doclist; |
| 446 |
int bIncr; /* True if doclist is loaded incrementally */ |
| 447 |
int iDoclistToken; |
| 448 |
|
| 449 |
/* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an |
| 450 |
** OR condition. */ |
| 451 |
char *pOrPoslist; |
| 452 |
i64 iOrDocid; |
| 453 |
|
| 454 |
/* Variables below this point are populated by fts3_expr.c when parsing |
| 455 |
** a MATCH expression. Everything above is part of the evaluation phase. |
| 456 |
*/ |
| 457 |
int nToken; /* Number of tokens in the phrase */ |
| 458 |
int iColumn; /* Index of column this phrase must match */ |
| 459 |
Fts3PhraseToken aToken[FLEXARRAY]; /* One for each token in the phrase */ |
| 460 |
}; |
| 461 |
|
| 462 |
/* Size (in bytes) of an Fts3Phrase object large enough to hold N tokens */ |
| 463 |
#define SZ_FTS3PHRASE(N) \ |
| 464 |
(offsetof(Fts3Phrase,aToken)+(N)*sizeof(Fts3PhraseToken)) |
| 465 |
|
| 466 |
/* |
| 467 |
** A tree of these objects forms the RHS of a MATCH operator. |
| 468 |
** |
| 469 |
** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist |
| 470 |
** points to a malloced buffer, size nDoclist bytes, containing the results |
| 471 |
** of this phrase query in FTS3 doclist format. As usual, the initial |
| 472 |
** "Length" field found in doclists stored on disk is omitted from this |
| 473 |
** buffer. |
| 474 |
** |
| 475 |
** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global |
| 476 |
** matchinfo data. If it is not NULL, it points to an array of size nCol*3, |
| 477 |
** where nCol is the number of columns in the queried FTS table. The array |
| 478 |
** is populated as follows: |
| 479 |
** |
| 480 |
** aMI[iCol*3 + 0] = Undefined |
| 481 |
** aMI[iCol*3 + 1] = Number of occurrences |
| 482 |
** aMI[iCol*3 + 2] = Number of rows containing at least one instance |
| 483 |
** |
| 484 |
** The aMI array is allocated using sqlite3_malloc(). It should be freed |
| 485 |
** when the expression node is. |
| 486 |
*/ |
| 487 |
struct Fts3Expr { |
| 488 |
int eType; /* One of the FTSQUERY_XXX values defined below */ |
| 489 |
int nNear; /* Valid if eType==FTSQUERY_NEAR */ |
| 490 |
Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ |
| 491 |
Fts3Expr *pLeft; /* Left operand */ |
| 492 |
Fts3Expr *pRight; /* Right operand */ |
| 493 |
Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ |
| 494 |
|
| 495 |
/* The following are used by the fts3_eval.c module. */ |
| 496 |
sqlite3_int64 iDocid; /* Current docid */ |
| 497 |
u8 bEof; /* True this expression is at EOF already */ |
| 498 |
u8 bStart; /* True if iDocid is valid */ |
| 499 |
u8 bDeferred; /* True if this expression is entirely deferred */ |
| 500 |
|
| 501 |
/* The following are used by the fts3_snippet.c module. */ |
| 502 |
int iPhrase; /* Index of this phrase in matchinfo() results */ |
| 503 |
u32 *aMI; /* See above */ |
| 504 |
}; |
| 505 |
|
| 506 |
/* |
| 507 |
** Candidate values for Fts3Query.eType. Note that the order of the first |
| 508 |
** four values is in order of precedence when parsing expressions. For |
| 509 |
** example, the following: |
| 510 |
** |
| 511 |
** "a OR b AND c NOT d NEAR e" |
| 512 |
** |
| 513 |
** is equivalent to: |
| 514 |
** |
| 515 |
** "a OR (b AND (c NOT (d NEAR e)))" |
| 516 |
*/ |
| 517 |
#define FTSQUERY_NEAR 1 |
| 518 |
#define FTSQUERY_NOT 2 |
| 519 |
#define FTSQUERY_AND 3 |
| 520 |
#define FTSQUERY_OR 4 |
| 521 |
#define FTSQUERY_PHRASE 5 |
| 522 |
|
| 523 |
|
| 524 |
/* fts3_write.c */ |
| 525 |
int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); |
| 526 |
int sqlite3Fts3PendingTermsFlush(Fts3Table *); |
| 527 |
void sqlite3Fts3PendingTermsClear(Fts3Table *); |
| 528 |
int sqlite3Fts3Optimize(Fts3Table *); |
| 529 |
int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, |
| 530 |
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); |
| 531 |
int sqlite3Fts3SegReaderPending( |
| 532 |
Fts3Table*,int,const char*,int,int,Fts3SegReader**); |
| 533 |
void sqlite3Fts3SegReaderFree(Fts3SegReader *); |
| 534 |
int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); |
| 535 |
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); |
| 536 |
|
| 537 |
int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); |
| 538 |
int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); |
| 539 |
|
| 540 |
#ifndef SQLITE_DISABLE_FTS4_DEFERRED |
| 541 |
void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); |
| 542 |
int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); |
| 543 |
int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); |
| 544 |
void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); |
| 545 |
int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); |
| 546 |
#else |
| 547 |
# define sqlite3Fts3FreeDeferredTokens(x) |
| 548 |
# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK |
| 549 |
# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK |
| 550 |
# define sqlite3Fts3FreeDeferredDoclists(x) |
| 551 |
# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK |
| 552 |
#endif |
| 553 |
|
| 554 |
void sqlite3Fts3SegmentsClose(Fts3Table *); |
| 555 |
int sqlite3Fts3MaxLevel(Fts3Table *, int *); |
| 556 |
|
| 557 |
/* Special values interpreted by sqlite3SegReaderCursor() */ |
| 558 |
#define FTS3_SEGCURSOR_PENDING -1 |
| 559 |
#define FTS3_SEGCURSOR_ALL -2 |
| 560 |
|
| 561 |
int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); |
| 562 |
int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); |
| 563 |
void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); |
| 564 |
|
| 565 |
int sqlite3Fts3SegReaderCursor(Fts3Table *, |
| 566 |
int, int, int, const char *, int, int, int, Fts3MultiSegReader *); |
| 567 |
|
| 568 |
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ |
| 569 |
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 |
| 570 |
#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 |
| 571 |
#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 |
| 572 |
#define FTS3_SEGMENT_PREFIX 0x00000008 |
| 573 |
#define FTS3_SEGMENT_SCAN 0x00000010 |
| 574 |
#define FTS3_SEGMENT_FIRST 0x00000020 |
| 575 |
|
| 576 |
/* Type passed as 4th argument to SegmentReaderIterate() */ |
| 577 |
struct Fts3SegFilter { |
| 578 |
const char *zTerm; |
| 579 |
int nTerm; |
| 580 |
int iCol; |
| 581 |
int flags; |
| 582 |
}; |
| 583 |
|
| 584 |
struct Fts3MultiSegReader { |
| 585 |
/* Used internally by sqlite3Fts3SegReaderXXX() calls */ |
| 586 |
Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ |
| 587 |
int nSegment; /* Size of apSegment array */ |
| 588 |
int nAdvance; /* How many seg-readers to advance */ |
| 589 |
Fts3SegFilter *pFilter; /* Pointer to filter object */ |
| 590 |
char *aBuffer; /* Buffer to merge doclists in */ |
| 591 |
i64 nBuffer; /* Allocated size of aBuffer[] in bytes */ |
| 592 |
|
| 593 |
int iColFilter; /* If >=0, filter for this column */ |
| 594 |
int bRestart; |
| 595 |
|
| 596 |
/* Used by fts3.c only. */ |
| 597 |
int nCost; /* Cost of running iterator */ |
| 598 |
int bLookup; /* True if a lookup of a single entry. */ |
| 599 |
|
| 600 |
/* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ |
| 601 |
char *zTerm; /* Pointer to term buffer */ |
| 602 |
int nTerm; /* Size of zTerm in bytes */ |
| 603 |
char *aDoclist; /* Pointer to doclist buffer */ |
| 604 |
int nDoclist; /* Size of aDoclist[] in bytes */ |
| 605 |
}; |
| 606 |
|
| 607 |
int sqlite3Fts3Incrmerge(Fts3Table*,int,int); |
| 608 |
|
| 609 |
#define fts3GetVarint32(p, piVal) ( \ |
| 610 |
(*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ |
| 611 |
) |
| 612 |
|
| 613 |
int sqlite3Fts3PrepareStmt( |
| 614 |
Fts3Table *p, /* Prepare for this connection */ |
| 615 |
const char *zSql, /* SQL to prepare */ |
| 616 |
int bPersist, /* True to set SQLITE_PREPARE_PERSISTENT */ |
| 617 |
int bAllowVtab, /* True to omit SQLITE_PREPARE_NO_VTAB */ |
| 618 |
sqlite3_stmt **pp /* OUT: Prepared statement */ |
| 619 |
); |
| 620 |
|
| 621 |
|
| 622 |
/* fts3.c */ |
| 623 |
void sqlite3Fts3ErrMsg(char**,const char*,...); |
| 624 |
int sqlite3Fts3PutVarint(char *, sqlite3_int64); |
| 625 |
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); |
| 626 |
int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *); |
| 627 |
int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*); |
| 628 |
int sqlite3Fts3GetVarint32(const char *, int *); |
| 629 |
int sqlite3Fts3VarintLen(sqlite3_uint64); |
| 630 |
void sqlite3Fts3Dequote(char *); |
| 631 |
void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); |
| 632 |
int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); |
| 633 |
int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); |
| 634 |
void sqlite3Fts3CreateStatTable(int*, Fts3Table*); |
| 635 |
int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); |
| 636 |
int sqlite3Fts3ReadInt(const char *z, int *pnOut); |
| 637 |
|
| 638 |
/* fts3_tokenizer.c */ |
| 639 |
const char *sqlite3Fts3NextToken(const char *, int *); |
| 640 |
int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); |
| 641 |
int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, |
| 642 |
sqlite3_tokenizer **, char ** |
| 643 |
); |
| 644 |
int sqlite3Fts3IsIdChar(char); |
| 645 |
|
| 646 |
/* fts3_snippet.c */ |
| 647 |
void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); |
| 648 |
void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, |
| 649 |
const char *, const char *, int, int |
| 650 |
); |
| 651 |
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); |
| 652 |
void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); |
| 653 |
|
| 654 |
/* fts3_expr.c */ |
| 655 |
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, |
| 656 |
char **, int, int, int, const char *, int, Fts3Expr **, char ** |
| 657 |
); |
| 658 |
void sqlite3Fts3ExprFree(Fts3Expr *); |
| 659 |
#ifdef SQLITE_TEST |
| 660 |
int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*); |
| 661 |
int sqlite3Fts3InitTerm(sqlite3 *db); |
| 662 |
#endif |
| 663 |
void *sqlite3Fts3MallocZero(i64 nByte); |
| 664 |
|
| 665 |
int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, |
| 666 |
sqlite3_tokenizer_cursor ** |
| 667 |
); |
| 668 |
|
| 669 |
/* fts3_aux.c */ |
| 670 |
int sqlite3Fts3InitAux(sqlite3 *db); |
| 671 |
|
| 672 |
void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); |
| 673 |
|
| 674 |
int sqlite3Fts3MsrIncrStart( |
| 675 |
Fts3Table*, Fts3MultiSegReader*, int, const char*, int); |
| 676 |
int sqlite3Fts3MsrIncrNext( |
| 677 |
Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); |
| 678 |
int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); |
| 679 |
int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); |
| 680 |
int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); |
| 681 |
int sqlite3Fts3MsrCancel(Fts3Cursor*, Fts3Expr*); |
| 682 |
|
| 683 |
/* fts3_tokenize_vtab.c */ |
| 684 |
int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *, void(*xDestroy)(void*)); |
| 685 |
|
| 686 |
/* fts3_unicode2.c (functions generated by parsing unicode text files) */ |
| 687 |
#ifndef SQLITE_DISABLE_FTS3_UNICODE |
| 688 |
int sqlite3FtsUnicodeFold(int, int); |
| 689 |
int sqlite3FtsUnicodeIsalnum(int); |
| 690 |
int sqlite3FtsUnicodeIsdiacritic(int); |
| 691 |
#endif |
| 692 |
|
| 693 |
int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*); |
| 694 |
|
| 695 |
int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk); |
| 696 |
|
| 697 |
#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ |
| 698 |
#endif /* _FTSINT_H */ |