| 1 |
# 2014 Dec 20 |
| 2 |
# |
| 3 |
# The author disclaims copyright to this source code. In place of |
| 4 |
# a legal notice, here is a blessing: |
| 5 |
# |
| 6 |
# May you do good and not evil. |
| 7 |
# May you find forgiveness for yourself and forgive others. |
| 8 |
# May you share freely, never taking more than you give. |
| 9 |
# |
| 10 |
#*********************************************************************** |
| 11 |
# |
| 12 |
# Tests focusing on the built-in fts5 tokenizers. |
| 13 |
# |
| 14 |
|
| 15 |
source [file join [file dirname [info script]] fts5_common.tcl] |
| 16 |
set testprefix fts5locale |
| 17 |
|
| 18 |
# If SQLITE_ENABLE_FTS5 is not defined, omit this file. |
| 19 |
ifcapable !fts5 { |
| 20 |
finish_test |
| 21 |
return |
| 22 |
} |
| 23 |
|
| 24 |
proc transform_token {locale token} { |
| 25 |
switch -- $locale { |
| 26 |
reverse { |
| 27 |
set ret "" |
| 28 |
foreach c [split $token ""] { |
| 29 |
set ret "$c$ret" |
| 30 |
} |
| 31 |
set token $ret |
| 32 |
} |
| 33 |
|
| 34 |
default { |
| 35 |
# no-op |
| 36 |
} |
| 37 |
} |
| 38 |
|
| 39 |
set token |
| 40 |
} |
| 41 |
|
| 42 |
proc tcl_create {args} { return "tcl_tokenize" } |
| 43 |
proc tcl_tokenize {tflags text} { |
| 44 |
set iToken 1 |
| 45 |
set bSkip 0 |
| 46 |
if {[sqlite3_fts5_locale]=="second"} { set bSkip 1 } |
| 47 |
foreach {w iStart iEnd} [fts5_tokenize_split $text] { |
| 48 |
incr iToken |
| 49 |
if {(($iToken) % ($bSkip + 1))} continue |
| 50 |
|
| 51 |
set w [transform_token [sqlite3_fts5_locale] $w] |
| 52 |
sqlite3_fts5_token $w $iStart $iEnd |
| 53 |
} |
| 54 |
} |
| 55 |
|
| 56 |
#------------------------------------------------------------------------- |
| 57 |
# Check that queries can have a locale attached to them. |
| 58 |
# |
| 59 |
reset_db |
| 60 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 61 |
|
| 62 |
do_execsql_test 1.0 { |
| 63 |
CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl); |
| 64 |
INSERT INTO t1 VALUES('abc'); |
| 65 |
INSERT INTO t1 VALUES('cba'); |
| 66 |
} {} |
| 67 |
|
| 68 |
do_execsql_test 1.1 { |
| 69 |
SELECT rowid, a FROM t1( fts5_locale('en_US', 'abc') ); |
| 70 |
} {1 abc} |
| 71 |
|
| 72 |
do_execsql_test 1.2 { |
| 73 |
SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') ); |
| 74 |
} {2 cba} |
| 75 |
|
| 76 |
|
| 77 |
#------------------------------------------------------------------------- |
| 78 |
# Test that the locale= option exists and seems to accept values. And |
| 79 |
# that fts5_locale() values may only be inserted into an internal-content |
| 80 |
# table if the locale=1 option was specified. |
| 81 |
# |
| 82 |
reset_db |
| 83 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 84 |
|
| 85 |
do_execsql_test 2.1 { |
| 86 |
CREATE VIRTUAL TABLE b1 USING fts5(x, y, locale=1, tokenize=tcl); |
| 87 |
CREATE VIRTUAL TABLE b2 USING fts5(x, y, locale=0, tokenize=tcl); |
| 88 |
|
| 89 |
CREATE VIRTUAL TABLE ttt USING fts5vocab('b1', instance); |
| 90 |
} |
| 91 |
|
| 92 |
do_catchsql_test 2.2.1 { |
| 93 |
CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=2); |
| 94 |
} {1 {malformed locale=... directive}} |
| 95 |
do_catchsql_test 2.2.2 { |
| 96 |
CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=111); |
| 97 |
} {1 {malformed locale=... directive}} |
| 98 |
|
| 99 |
do_catchsql_test 2.3 { |
| 100 |
INSERT INTO b1(b1, rank) VALUES('locale', 0); |
| 101 |
} {1 {SQL logic error}} |
| 102 |
|
| 103 |
do_execsql_test 2.4.1 { |
| 104 |
INSERT INTO b1 VALUES('abc', 'one two three'); |
| 105 |
} |
| 106 |
|
| 107 |
do_execsql_test 2.4.2 { |
| 108 |
INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six')); |
| 109 |
} |
| 110 |
|
| 111 |
do_execsql_test 2.5 { |
| 112 |
INSERT INTO b2 VALUES('abc', 'one two three'); |
| 113 |
} |
| 114 |
|
| 115 |
do_catchsql_test 2.6 { |
| 116 |
INSERT INTO b2 VALUES('def', fts5_locale('reverse', 'four five six')); |
| 117 |
} {1 {fts5_locale() requires locale=1}} |
| 118 |
|
| 119 |
do_execsql_test 2.7 { SELECT rowid FROM b1('one') } {1} |
| 120 |
do_execsql_test 2.8 { SELECT rowid FROM b1('four') } {} |
| 121 |
do_execsql_test 2.9 { SELECT rowid FROM b1('ruof') } 2 |
| 122 |
do_execsql_test 2.10 { SELECT rowid FROM b1(fts5_locale('reverse', 'five'))} 2 |
| 123 |
|
| 124 |
do_execsql_test 2.11 { |
| 125 |
SELECT x, quote(y) FROM b1 |
| 126 |
} { |
| 127 |
abc {'one two three'} |
| 128 |
def {'four five six'} |
| 129 |
} |
| 130 |
|
| 131 |
do_execsql_test 2.12 { SELECT quote(y) FROM b1('ruof') } { |
| 132 |
{'four five six'} |
| 133 |
} |
| 134 |
|
| 135 |
do_execsql_test 2.13 { |
| 136 |
INSERT INTO b1(b1) VALUES('integrity-check'); |
| 137 |
} |
| 138 |
|
| 139 |
do_execsql_test 2.14 { |
| 140 |
INSERT INTO b1(b1) VALUES('rebuild'); |
| 141 |
} |
| 142 |
do_execsql_test 2.15 { |
| 143 |
INSERT INTO b1(b1) VALUES('integrity-check'); |
| 144 |
} |
| 145 |
|
| 146 |
do_execsql_test 2.16 { |
| 147 |
DELETE FROM b1 WHERE rowid=2 |
| 148 |
} |
| 149 |
do_execsql_test 2.17 { |
| 150 |
INSERT INTO b1(b1) VALUES('integrity-check'); |
| 151 |
} |
| 152 |
|
| 153 |
do_execsql_test 2.18 { |
| 154 |
INSERT INTO b1(rowid, x, y) VALUES( |
| 155 |
test_setsubtype(45, 76), 'abc def', 'def abc' |
| 156 |
); |
| 157 |
} |
| 158 |
|
| 159 |
#------------------------------------------------------------------------- |
| 160 |
# Test the 'delete' command with contentless tables. |
| 161 |
# |
| 162 |
reset_db |
| 163 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 164 |
|
| 165 |
do_execsql_test 3.1 { |
| 166 |
CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl, locale=1); |
| 167 |
CREATE VIRTUAL TABLE c2 USING fts5vocab('c1', instance); |
| 168 |
|
| 169 |
INSERT INTO c1 VALUES('hello world'); |
| 170 |
INSERT INTO c1 VALUES( fts5_locale('reverse', 'one two three') ); |
| 171 |
} |
| 172 |
|
| 173 |
do_execsql_test 3.2 { |
| 174 |
SELECT DISTINCT term FROM c2 ORDER BY 1 |
| 175 |
} { |
| 176 |
eerht eno hello owt world |
| 177 |
} |
| 178 |
|
| 179 |
do_execsql_test 3.3 { |
| 180 |
INSERT INTO c1(c1, rowid, x) |
| 181 |
VALUES('delete', 2, fts5_locale('reverse', 'one two three') ); |
| 182 |
} |
| 183 |
|
| 184 |
do_execsql_test 3.4 { |
| 185 |
SELECT DISTINCT term FROM c2 ORDER BY 1 |
| 186 |
} { |
| 187 |
hello world |
| 188 |
} |
| 189 |
|
| 190 |
#------------------------------------------------------------------------- |
| 191 |
# Test that an UPDATE that updates a subset of the columns does not |
| 192 |
# magically discard the locale from those columns not updated. |
| 193 |
# |
| 194 |
reset_db |
| 195 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 196 |
|
| 197 |
do_execsql_test 4.1 { |
| 198 |
CREATE VIRTUAL TABLE d1 USING fts5(x, y, locale=1, tokenize=tcl); |
| 199 |
CREATE VIRTUAL TABLE d2 USING fts5vocab('d1', instance); |
| 200 |
|
| 201 |
INSERT INTO d1(rowid, x, y) VALUES(1, 'abc', 'def'); |
| 202 |
INSERT INTO d1(rowid, x, y) VALUES(2, 'ghi', fts5_locale('reverse', 'hello')); |
| 203 |
} |
| 204 |
|
| 205 |
do_execsql_test 4.2 { |
| 206 |
SELECT DISTINCT term FROM d2 ORDER BY 1 |
| 207 |
} { |
| 208 |
abc def ghi olleh |
| 209 |
} |
| 210 |
|
| 211 |
do_execsql_test 4.3 { |
| 212 |
UPDATE d1 SET x='jkl' WHERE rowid=2; |
| 213 |
} |
| 214 |
|
| 215 |
do_execsql_test 4.4 { |
| 216 |
SELECT DISTINCT term FROM d2 ORDER BY 1 |
| 217 |
} { |
| 218 |
abc def jkl olleh |
| 219 |
} |
| 220 |
|
| 221 |
do_execsql_test 4.5 { |
| 222 |
SELECT rowid, * FROM d1 |
| 223 |
} { |
| 224 |
1 abc def |
| 225 |
2 jkl hello |
| 226 |
} |
| 227 |
|
| 228 |
do_execsql_test 4.6 { |
| 229 |
UPDATE d1 SET rowid=4 WHERE rowid=2 |
| 230 |
} |
| 231 |
|
| 232 |
do_execsql_test 4.7 { |
| 233 |
SELECT rowid, * FROM d1 |
| 234 |
} { |
| 235 |
1 abc def |
| 236 |
4 jkl hello |
| 237 |
} |
| 238 |
|
| 239 |
fts5_aux_test_functions db |
| 240 |
|
| 241 |
do_execsql_test 4.8.1 { |
| 242 |
SELECT fts5_test_columntext(d1) FROM d1('jkl') |
| 243 |
} {{jkl hello}} |
| 244 |
do_execsql_test 4.8.2 { |
| 245 |
SELECT fts5_test_columntext(d1) FROM d1(fts5_locale('reverse', 'hello')) |
| 246 |
} {{jkl hello}} |
| 247 |
|
| 248 |
do_execsql_test 4.9 { |
| 249 |
SELECT fts5_test_columnlocale(d1) FROM d1(fts5_locale('reverse', 'hello')) |
| 250 |
} {{{} reverse}} |
| 251 |
|
| 252 |
do_execsql_test 4.10 { |
| 253 |
SELECT fts5_test_columnlocale(d1) FROM d1 |
| 254 |
} { |
| 255 |
{{} {}} |
| 256 |
{{} reverse} |
| 257 |
} |
| 258 |
|
| 259 |
#------------------------------------------------------------------------- |
| 260 |
# Test that if an fts5_locale() value is written to an UNINDEXED |
| 261 |
# column it is stored as text. This is so that blobs and other values |
| 262 |
# can also be stored as is. |
| 263 |
# |
| 264 |
reset_db |
| 265 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 266 |
|
| 267 |
do_execsql_test 5.1 { |
| 268 |
CREATE VIRTUAL TABLE t1 USING fts5( |
| 269 |
x, y UNINDEXED, locale=1, tokenize=tcl |
| 270 |
); |
| 271 |
|
| 272 |
INSERT INTO t1(rowid, x, y) VALUES(111, |
| 273 |
fts5_locale('reverse', 'one two three'), |
| 274 |
fts5_locale('reverse', 'four five six') |
| 275 |
); |
| 276 |
} |
| 277 |
|
| 278 |
do_execsql_test 5.2 { |
| 279 |
SELECT rowid, x, y FROM t1 |
| 280 |
} { |
| 281 |
111 {one two three} {four five six} |
| 282 |
} |
| 283 |
|
| 284 |
do_execsql_test 5.3 { |
| 285 |
SELECT typeof(c0), typeof(c1), typeof(l0) FROM t1_content |
| 286 |
} { |
| 287 |
text text text |
| 288 |
} |
| 289 |
|
| 290 |
#------------------------------------------------------------------------- |
| 291 |
|
| 292 |
foreach {tn opt} { |
| 293 |
1 {} |
| 294 |
2 {, columnsize=0} |
| 295 |
} { |
| 296 |
reset_db |
| 297 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 298 |
do_execsql_test 6.$tn.1 " |
| 299 |
CREATE VIRTUAL TABLE y1 USING fts5(t, locale=1, tokenize=tcl $opt); |
| 300 |
" |
| 301 |
|
| 302 |
do_execsql_test 6.$tn.2 { |
| 303 |
INSERT INTO y1(rowid, t) VALUES |
| 304 |
(1, fts5_locale('second', 'the city of London')), |
| 305 |
(2, fts5_locale('second', 'shall have all the old')), |
| 306 |
(3, fts5_locale('second', 'Liberties and Customs')), |
| 307 |
(4, fts5_locale('second', 'which it hath been used to have')); |
| 308 |
} |
| 309 |
|
| 310 |
fts5_aux_test_functions db |
| 311 |
|
| 312 |
do_execsql_test 6.$tn.3 { |
| 313 |
SELECT fts5_test_columnsize(y1) FROM y1 |
| 314 |
} { |
| 315 |
2 3 2 4 |
| 316 |
} |
| 317 |
|
| 318 |
do_execsql_test 6.$tn.4 { |
| 319 |
SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); |
| 320 |
} { |
| 321 |
2 3 |
| 322 |
} |
| 323 |
|
| 324 |
do_execsql_test 6.$tn.5 { |
| 325 |
SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall'); |
| 326 |
} { |
| 327 |
2 3 |
| 328 |
} |
| 329 |
|
| 330 |
do_execsql_test 6.$tn.6 { |
| 331 |
SELECT rowid, fts5_test_columnsize(y1) FROM y1('have'); |
| 332 |
} { |
| 333 |
4 4 |
| 334 |
} |
| 335 |
|
| 336 |
do_execsql_test 6.$tn.7 { |
| 337 |
SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have'); |
| 338 |
} { |
| 339 |
4 {which it hath been used to [have]} |
| 340 |
} |
| 341 |
|
| 342 |
do_execsql_test 6.$tn.8 { |
| 343 |
SELECT rowid, |
| 344 |
highlight(y1, 0, '[', ']'), |
| 345 |
snippet(y1, 0, '[', ']', '...', 10) |
| 346 |
FROM y1('Liberties + Customs'); |
| 347 |
} { |
| 348 |
3 {[Liberties and Customs]} |
| 349 |
{[Liberties and Customs]} |
| 350 |
} |
| 351 |
} |
| 352 |
|
| 353 |
#------------------------------------------------------------------------- |
| 354 |
reset_db |
| 355 |
do_execsql_test 6.0 { |
| 356 |
CREATE VIRTUAL TABLE x1 USING fts5(x); |
| 357 |
} |
| 358 |
do_catchsql_test 6.1 { |
| 359 |
INSERT INTO x1(rowid, x) VALUES(123, fts5_locale('en_AU', 'hello world')); |
| 360 |
} {1 {fts5_locale() requires locale=1}} |
| 361 |
|
| 362 |
do_execsql_test 6.2 { |
| 363 |
SELECT typeof( fts5_locale(NULL, 'xyz') ), typeof( fts5_locale('', 'abc') ); |
| 364 |
} {text text} |
| 365 |
|
| 366 |
#-------------------------------------------------------------------------- |
| 367 |
# Test that fts5_locale() works with external-content tables. |
| 368 |
# |
| 369 |
reset_db |
| 370 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 371 |
|
| 372 |
do_execsql_test 7.1 { |
| 373 |
CREATE TABLE t1(ii INTEGER PRIMARY KEY, bb BLOB, tt TEXT, locale TEXT); |
| 374 |
CREATE VIEW v1 AS |
| 375 |
SELECT ii AS rowid, bb, fts5_locale(locale, tt) AS tt FROM t1; |
| 376 |
|
| 377 |
CREATE VIRTUAL TABLE ft USING fts5( |
| 378 |
bb, tt, locale=1, tokenize=tcl, content=v1 |
| 379 |
); |
| 380 |
|
| 381 |
INSERT INTO t1 VALUES(1, NULL, 'one two three', NULL); |
| 382 |
INSERT INTO t1 VALUES(2, '7800616263', 'four five six', 'reverse'); |
| 383 |
INSERT INTO t1 VALUES(3, '000000007800616263', 'seven eight nine', 'second'); |
| 384 |
} |
| 385 |
|
| 386 |
do_execsql_test 7.2 { |
| 387 |
INSERT INTO ft(ft) VALUES('rebuild'); |
| 388 |
INSERT INTO ft(ft) VALUES('integrity-check'); |
| 389 |
} |
| 390 |
|
| 391 |
do_execsql_test 7.3 { |
| 392 |
SELECT rowid, quote(bb), quote(tt) FROM ft |
| 393 |
} { |
| 394 |
1 NULL {'one two three'} |
| 395 |
2 '7800616263' {'four five six'} |
| 396 |
3 '000000007800616263' {'seven eight nine'} |
| 397 |
} |
| 398 |
|
| 399 |
do_execsql_test 7.4 { SELECT rowid FROM ft('six'); } |
| 400 |
do_execsql_test 7.5 { SELECT rowid FROM ft(fts5_locale('reverse','six')); } 2 |
| 401 |
|
| 402 |
fts5_aux_test_functions db |
| 403 |
|
| 404 |
do_execsql_test 7.6 { |
| 405 |
SELECT fts5_test_columnlocale(ft) FROM ft; |
| 406 |
} { |
| 407 |
{{} {}} {{} reverse} {{} second} |
| 408 |
} |
| 409 |
|
| 410 |
#------------------------------------------------------------------------- |
| 411 |
# Test that the porter tokenizer works with locales. |
| 412 |
# |
| 413 |
reset_db |
| 414 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 415 |
|
| 416 |
do_execsql_test 8.1 { |
| 417 |
CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize="porter tcl"); |
| 418 |
CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance); |
| 419 |
|
| 420 |
INSERT INTO ft(rowid, tt) VALUES |
| 421 |
(111, fts5_locale('second', 'the porter tokenizer is a wrapper tokenizer')), |
| 422 |
(222, fts5_locale('reverse', 'This value may also be set')); |
| 423 |
} |
| 424 |
|
| 425 |
do_execsql_test 8.1 { |
| 426 |
SELECT DISTINCT term FROM vocab ORDER BY 1 |
| 427 |
} { |
| 428 |
a eb eulav osla sihT te the token yam |
| 429 |
} |
| 430 |
|
| 431 |
#------------------------------------------------------------------------- |
| 432 |
# Test that position-lists (used by xInst, xPhraseFirst etc.) work with |
| 433 |
# locales and modes other than detail=full. |
| 434 |
# |
| 435 |
foreach {tn detail} { |
| 436 |
1 detail=full |
| 437 |
2 detail=none |
| 438 |
3 detail=column |
| 439 |
} { |
| 440 |
reset_db |
| 441 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 442 |
do_execsql_test 9.$tn.0 " |
| 443 |
CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize=tcl, $detail); |
| 444 |
" |
| 445 |
do_execsql_test 9.$tn.1 { |
| 446 |
CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance); |
| 447 |
INSERT INTO ft(rowid, tt) VALUES |
| 448 |
(-1, fts5_locale('second', 'it is an ancient mariner')); |
| 449 |
} |
| 450 |
|
| 451 |
do_execsql_test 9.$tn.2 { |
| 452 |
SELECT DISTINCT term FROM vocab |
| 453 |
} {an it mariner} |
| 454 |
|
| 455 |
do_execsql_test 9.$tn.3 { |
| 456 |
SELECT highlight(ft, 0, '[', ']') FROM ft('mariner') |
| 457 |
} {{it is an ancient [mariner]}} |
| 458 |
} |
| 459 |
|
| 460 |
#------------------------------------------------------------------------- |
| 461 |
# Check some corrupt fts5_locale() blob formats are detected. |
| 462 |
# |
| 463 |
foreach_detail_mode $::testprefix { |
| 464 |
|
| 465 |
reset_db |
| 466 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 467 |
fts5_aux_test_functions db |
| 468 |
do_execsql_test 10.1 { |
| 469 |
CREATE TABLE x1(ii INTEGER PRIMARY KEY, x); |
| 470 |
CREATE VIRTUAL TABLE ft USING fts5(x, |
| 471 |
content=x1, content_rowid=ii, locale=1, detail=%DETAIL%, columnsize=0 |
| 472 |
); |
| 473 |
|
| 474 |
CREATE VIRTUAL TABLE ft2 USING fts5( |
| 475 |
x, locale=1, detail=%DETAIL%, columnsize=0 |
| 476 |
); |
| 477 |
} |
| 478 |
|
| 479 |
foreach {tn v} { |
| 480 |
1 X'001152' |
| 481 |
2 X'0011223344' |
| 482 |
3 X'00E0B2EB68656c6c6f' |
| 483 |
4 X'00E0B2EB0068656c6c6f' |
| 484 |
} { |
| 485 |
do_execsql_test 10.2.$tn.0 { INSERT INTO ft(ft) VALUES('delete-all') } |
| 486 |
do_execsql_test 10.2.$tn.1 { DELETE FROM x1; } |
| 487 |
do_execsql_test 10.2.$tn.2 " INSERT INTO x1 VALUES(NULL, $v) " |
| 488 |
|
| 489 |
do_catchsql_test 10.2.$tn.3 { |
| 490 |
INSERT INTO ft(ft) VALUES('rebuild'); |
| 491 |
} {0 {}} |
| 492 |
|
| 493 |
do_catchsql_test 10.2.$tn.4 " |
| 494 |
SELECT * FROM ft( test_setsubtype($v, 76) ); |
| 495 |
" {1 {fts5: syntax error near ""}} |
| 496 |
|
| 497 |
do_execsql_test 10.2.$tn.5 { |
| 498 |
INSERT INTO ft(rowid, x) VALUES(1, 'hello world'); |
| 499 |
} |
| 500 |
|
| 501 |
if {"%DETAIL%"=="full"} { |
| 502 |
do_execsql_test 10.2.$tn.6 { |
| 503 |
SELECT fts5_test_poslist(ft) FROM ft('world'); |
| 504 |
} {0.0.1} |
| 505 |
|
| 506 |
do_execsql_test 10.2.$tn.7.1 { |
| 507 |
SELECT fts5_test_columnsize(ft) FROM ft('world'); |
| 508 |
} {1} |
| 509 |
|
| 510 |
do_execsql_test 10.2.$tn.7.2 { |
| 511 |
SELECT fts5_test_columnlocale(ft) FROM ft('world'); |
| 512 |
} {{{}}} |
| 513 |
} |
| 514 |
|
| 515 |
do_catchsql_test 10.2.$tn.8 { |
| 516 |
SELECT count(*) FROM ft('hello') |
| 517 |
} {0 1} |
| 518 |
|
| 519 |
do_catchsql_test 10.2.$tn.9 { |
| 520 |
PRAGMA integrity_check; |
| 521 |
} {0 ok} |
| 522 |
|
| 523 |
do_execsql_test 10.2.$tn.10 { |
| 524 |
DELETE FROM x1; |
| 525 |
INSERT INTO x1(ii, x) VALUES(1, 'hello world'); |
| 526 |
} |
| 527 |
|
| 528 |
do_catchsql_test 10.2.$tn.11 " |
| 529 |
INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) ) |
| 530 |
" {0 {}} |
| 531 |
|
| 532 |
do_catchsql_test 10.2.$tn.12 " |
| 533 |
INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) ) |
| 534 |
" {0 {}} |
| 535 |
|
| 536 |
do_execsql_test 10.2.$tn.13 { |
| 537 |
INSERT INTO ft2(rowid, x) VALUES(1, 'hello world'); |
| 538 |
} |
| 539 |
do_execsql_test 10.2.$tn.14 "UPDATE ft2_content SET c0=$v" |
| 540 |
|
| 541 |
do_catchsql_test 10.2.$tn.15 { |
| 542 |
PRAGMA integrity_check; |
| 543 |
} {0 {{malformed inverted index for FTS5 table main.ft2}}} |
| 544 |
|
| 545 |
do_execsql_test 10.2.$tn.16 { |
| 546 |
DELETE FROM ft2_content; |
| 547 |
INSERT INTO ft2(ft2) VALUES('rebuild'); |
| 548 |
} |
| 549 |
} |
| 550 |
|
| 551 |
} |
| 552 |
|
| 553 |
#------------------------------------------------------------------------- |
| 554 |
# |
| 555 |
reset_db |
| 556 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 557 |
fts5_aux_test_functions db |
| 558 |
do_execsql_test 11.0 { |
| 559 |
CREATE VIRTUAL TABLE x1 USING fts5(abc, locale=1); |
| 560 |
INSERT INTO x1(rowid, abc) VALUES(123, fts5_locale('en_US', 'one two three')); |
| 561 |
} |
| 562 |
|
| 563 |
do_catchsql_test 11.1 { |
| 564 |
SELECT fts5_columnlocale(x1, -1) FROM x1('two'); |
| 565 |
} {1 SQLITE_RANGE} |
| 566 |
do_catchsql_test 11.2 { |
| 567 |
SELECT fts5_columnlocale(x1, 1) FROM x1('two'); |
| 568 |
} {1 SQLITE_RANGE} |
| 569 |
|
| 570 |
#------------------------------------------------------------------------- |
| 571 |
# |
| 572 |
reset_db |
| 573 |
do_test 12.0 { |
| 574 |
list [catch { |
| 575 |
sqlite3_fts5_create_tokenizer -v2 -version 3 db tcl tcl_create |
| 576 |
} msg] $msg |
| 577 |
} {1 {error in fts5_api.xCreateTokenizer_v2()}} |
| 578 |
|
| 579 |
#------------------------------------------------------------------------- |
| 580 |
# Tests for auxiliary function fts5_get_locale(). |
| 581 |
# |
| 582 |
reset_db |
| 583 |
|
| 584 |
# Check that if the table does not support locale=1, fts5_get_locale() |
| 585 |
# always returns NULL. |
| 586 |
do_execsql_test 13.1.0 { |
| 587 |
CREATE VIRTUAL TABLE nolocale USING fts5(a, b); |
| 588 |
INSERT INTO nolocale VALUES('one two three', 'four five six'); |
| 589 |
INSERT INTO nolocale VALUES('three two one', 'seven eight nine'); |
| 590 |
} |
| 591 |
do_execsql_test 13.1.1 { |
| 592 |
SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale; |
| 593 |
} {1 1} |
| 594 |
do_execsql_test 13.1.2 { |
| 595 |
SELECT fts5_get_locale(nolocale, 1) IS NULL FROM nolocale('one + two'); |
| 596 |
} {1} |
| 597 |
do_execsql_test 13.1.3 { |
| 598 |
SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale('one AND two'); |
| 599 |
} {1 1} |
| 600 |
do_execsql_test 13.1.4 { |
| 601 |
SELECT |
| 602 |
fts5_get_locale(nolocale, 1) IS NULL |
| 603 |
FROM nolocale('three AND two') ORDER BY rank |
| 604 |
} {1 1} |
| 605 |
do_catchsql_test 13.1.5 { |
| 606 |
SELECT fts5_get_locale(nolocale, 2) IS NULL FROM nolocale('three AND two'); |
| 607 |
} {1 {column index out of range}} |
| 608 |
do_catchsql_test 13.1.6 { |
| 609 |
SELECT fts5_get_locale(nolocale, -1) IS NULL FROM nolocale('three AND two'); |
| 610 |
} {1 {column index out of range}} |
| 611 |
do_catchsql_test 13.1.7 { |
| 612 |
SELECT fts5_get_locale(nolocale) IS NULL FROM nolocale('three AND two'); |
| 613 |
} {1 {wrong number of arguments to function fts5_get_locale()}} |
| 614 |
do_catchsql_test 13.1.8 { |
| 615 |
SELECT fts5_get_locale(nolocale, 0, 0) IS NULL FROM nolocale('three AND two'); |
| 616 |
} {1 {wrong number of arguments to function fts5_get_locale()}} |
| 617 |
do_catchsql_test 13.1.9 { |
| 618 |
SELECT fts5_get_locale(nolocale, 'text') FROM nolocale('three AND two'); |
| 619 |
} {1 {non-integer argument passed to function fts5_get_locale()}} |
| 620 |
|
| 621 |
|
| 622 |
# Check that if the table does support locale=1, fts5_get_locale() |
| 623 |
# returns the locale of the identified row/column. |
| 624 |
do_execsql_test 13.2.0 { |
| 625 |
CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1); |
| 626 |
INSERT INTO ft VALUES( |
| 627 |
fts5_locale('th_TH', 'one two three'), 'four five six seven' |
| 628 |
); |
| 629 |
INSERT INTO ft VALUES( |
| 630 |
'three two one', fts5_locale('en_AU', 'seven eight nine') |
| 631 |
); |
| 632 |
} |
| 633 |
|
| 634 |
do_execsql_test 13.2.1 { |
| 635 |
SELECT quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) FROM ft |
| 636 |
} { 'th_TH' NULL NULL 'en_AU' } |
| 637 |
do_execsql_test 13.2.2 { |
| 638 |
SELECT |
| 639 |
quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) |
| 640 |
FROM ft('one AND three') |
| 641 |
} { 'th_TH' NULL NULL 'en_AU' } |
| 642 |
do_execsql_test 13.2.3 { |
| 643 |
SELECT |
| 644 |
quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) |
| 645 |
FROM ft('one AND three') ORDER BY rank |
| 646 |
} { NULL 'en_AU' 'th_TH' NULL } |
| 647 |
do_execsql_test 13.2.4 { |
| 648 |
SELECT |
| 649 |
quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) |
| 650 |
FROM ft('one AND three') ORDER BY rowid |
| 651 |
} { 'th_TH' NULL NULL 'en_AU' } |
| 652 |
|
| 653 |
do_execsql_test 13.2.5 { |
| 654 |
SELECT |
| 655 |
quote(fts5_get_locale(ft, '0')), quote(fts5_get_locale(ft, 1)) |
| 656 |
FROM ft('one AND three') ORDER BY rowid |
| 657 |
} { 'th_TH' NULL NULL 'en_AU' } |
| 658 |
|
| 659 |
do_catchsql_test 13.2.6 { |
| 660 |
SELECT |
| 661 |
quote(fts5_get_locale(ft, '0.0')), quote(fts5_get_locale(ft, 1)) |
| 662 |
FROM ft('one AND three') ORDER BY rowid |
| 663 |
} {1 {non-integer argument passed to function fts5_get_locale()}} |
| 664 |
do_catchsql_test 13.2.7 { |
| 665 |
SELECT |
| 666 |
quote(fts5_get_locale(ft, 0.0)), quote(fts5_get_locale(ft, 1)) |
| 667 |
FROM ft('one AND three') ORDER BY rowid |
| 668 |
} {1 {non-integer argument passed to function fts5_get_locale()}} |
| 669 |
|
| 670 |
#------------------------------------------------------------------------- |
| 671 |
# Check that UPDATE statements that may affect more than one row work. |
| 672 |
# |
| 673 |
reset_db |
| 674 |
do_execsql_test 14.1 { |
| 675 |
CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1); |
| 676 |
} |
| 677 |
|
| 678 |
do_execsql_test 14.2 { |
| 679 |
INSERT INTO ft VALUES('hello', 'world'); |
| 680 |
} |
| 681 |
|
| 682 |
do_execsql_test 14.3 { |
| 683 |
UPDATE ft SET b = fts5_locale('en_AU', 'world'); |
| 684 |
} |
| 685 |
|
| 686 |
do_execsql_test 14.4 { |
| 687 |
INSERT INTO ft VALUES(X'abcd', X'1234'); |
| 688 |
} {} |
| 689 |
|
| 690 |
do_execsql_test 14.5 { |
| 691 |
SELECT quote(a), quote(b) FROM ft |
| 692 |
} {'hello' 'world' X'ABCD' X'1234'} |
| 693 |
|
| 694 |
do_execsql_test 14.6 { |
| 695 |
DELETE FROM ft; |
| 696 |
INSERT INTO ft VALUES(NULL, 'null'); |
| 697 |
INSERT INTO ft VALUES(123, 'int'); |
| 698 |
INSERT INTO ft VALUES(345.0, 'real'); |
| 699 |
INSERT INTO ft VALUES('abc', 'text'); |
| 700 |
INSERT INTO ft VALUES(fts5_locale('abc', 'def'), 'text'); |
| 701 |
|
| 702 |
SELECT a, typeof(a), b FROM ft |
| 703 |
} { |
| 704 |
{} null null |
| 705 |
123 integer int |
| 706 |
345.0 real real |
| 707 |
abc text text |
| 708 |
def text text |
| 709 |
} |
| 710 |
|
| 711 |
do_execsql_test 14.7 { |
| 712 |
SELECT quote(c0), typeof(c0) FROM ft_content |
| 713 |
} { |
| 714 |
NULL null |
| 715 |
123 integer |
| 716 |
345.0 real |
| 717 |
'abc' text |
| 718 |
'def' text |
| 719 |
} |
| 720 |
|
| 721 |
#------------------------------------------------------------------------- |
| 722 |
# Check that inserting UNINDEXED columns between indexed columns of a |
| 723 |
# locale=1 table does not cause a problem. |
| 724 |
# |
| 725 |
reset_db |
| 726 |
sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create |
| 727 |
fts5_aux_test_functions db |
| 728 |
|
| 729 |
do_execsql_test 15.1 { |
| 730 |
CREATE VIRTUAL TABLE ft USING fts5(a, b UNINDEXED, c, locale=1, tokenize=tcl); |
| 731 |
} |
| 732 |
|
| 733 |
do_execsql_test 15.2 { |
| 734 |
INSERT INTO ft VALUES('one', 'two', 'three'); |
| 735 |
INSERT INTO ft VALUES('one', 'two', fts5_locale('loc', 'three')); |
| 736 |
} |
| 737 |
|
| 738 |
do_execsql_test 15.3 { |
| 739 |
SELECT c2, l2 FROM ft_content |
| 740 |
} {three {} three loc} |
| 741 |
|
| 742 |
do_execsql_test 15.4 { |
| 743 |
SELECT c, fts5_columnlocale(ft, 2) FROM ft |
| 744 |
} {three {} three loc} |
| 745 |
|
| 746 |
|
| 747 |
finish_test |
| 748 |
|