| 1 |
# 2016 Jan 15 |
| 2 |
# |
| 3 |
# The author disclaims copyright to this source code. In place of |
| 4 |
# a legal notice, here is a blessing: |
| 5 |
# |
| 6 |
# May you do good and not evil. |
| 7 |
# May you find forgiveness for yourself and forgive others. |
| 8 |
# May you share freely, never taking more than you give. |
| 9 |
# |
| 10 |
#************************************************************************* |
| 11 |
# |
| 12 |
|
| 13 |
source [file join [file dirname [info script]] fts5_common.tcl] |
| 14 |
ifcapable !fts5 { finish_test ; return } |
| 15 |
set ::testprefix fts5tok1 |
| 16 |
|
| 17 |
|
| 18 |
sqlite3_fts5_register_fts5tokenize db |
| 19 |
|
| 20 |
#------------------------------------------------------------------------- |
| 21 |
# Simple test cases. Using the default (ascii) tokenizer. |
| 22 |
# |
| 23 |
do_execsql_test 1.0 { |
| 24 |
CREATE VIRTUAL TABLE t1 USING fts5tokenize(ascii); |
| 25 |
CREATE VIRTUAL TABLE t2 USING fts5tokenize(); |
| 26 |
CREATE VIRTUAL TABLE t3 USING fts5tokenize( |
| 27 |
ascii, 'separators', 'xyz', tokenchars, '''' |
| 28 |
); |
| 29 |
} |
| 30 |
|
| 31 |
foreach {tn tbl} {1 t1 2 t2 3 t3} { |
| 32 |
do_execsql_test 1.$tn.1 "SELECT input, * FROM $tbl ('one two three')" { |
| 33 |
{one two three} one 0 3 0 |
| 34 |
{one two three} two 4 7 1 |
| 35 |
{one two three} three 8 13 2 |
| 36 |
} |
| 37 |
|
| 38 |
do_execsql_test 1.$tn.2 " |
| 39 |
SELECT token FROM $tbl WHERE input = 'OnE tWo tHrEe' |
| 40 |
" { |
| 41 |
one two three |
| 42 |
} |
| 43 |
} |
| 44 |
|
| 45 |
do_execsql_test 1.4 { |
| 46 |
SELECT token FROM t3 WHERE input = '1x2x3x' |
| 47 |
} {1 2 3} |
| 48 |
|
| 49 |
do_execsql_test 1.5 { |
| 50 |
SELECT token FROM t1 WHERE input = '1x2x3x' |
| 51 |
} {1x2x3x} |
| 52 |
|
| 53 |
do_execsql_test 1.6 { |
| 54 |
SELECT token FROM t3 WHERE input = '1''2x3x' |
| 55 |
} {1'2 3} |
| 56 |
|
| 57 |
do_execsql_test 1.7 { |
| 58 |
SELECT token FROM t3 WHERE input = '' |
| 59 |
} {} |
| 60 |
|
| 61 |
do_execsql_test 1.8 { |
| 62 |
SELECT token FROM t3 WHERE input = NULL |
| 63 |
} {} |
| 64 |
|
| 65 |
do_execsql_test 1.9 { |
| 66 |
SELECT input, * FROM t3 WHERE input = 123 |
| 67 |
} {123 123 0 3 0} |
| 68 |
|
| 69 |
do_execsql_test 1.10 { |
| 70 |
SELECT input, * FROM t1 WHERE input = 'a b c' AND token = 'b'; |
| 71 |
} { |
| 72 |
{a b c} b 2 3 1 |
| 73 |
} |
| 74 |
|
| 75 |
do_execsql_test 1.11 { |
| 76 |
SELECT input, * FROM t1 WHERE token = 'b' AND input = 'a b c'; |
| 77 |
} { |
| 78 |
{a b c} b 2 3 1 |
| 79 |
} |
| 80 |
|
| 81 |
do_execsql_test 1.12 { |
| 82 |
SELECT input, * FROM t1 WHERE input < 'b' AND input = 'a b c'; |
| 83 |
} { |
| 84 |
{a b c} a 0 1 0 |
| 85 |
{a b c} b 2 3 1 |
| 86 |
{a b c} c 4 5 2 |
| 87 |
} |
| 88 |
|
| 89 |
do_execsql_test 1.13.1 { |
| 90 |
CREATE TABLE c1(x); |
| 91 |
INSERT INTO c1(x) VALUES('a b c'); |
| 92 |
INSERT INTO c1(x) VALUES('d e f'); |
| 93 |
} |
| 94 |
do_execsql_test 1.13.2 { |
| 95 |
SELECT c1.*, input, t1.* FROM c1, t1 WHERE input = x AND c1.rowid=t1.rowid; |
| 96 |
} { |
| 97 |
{a b c} {a b c} a 0 1 0 |
| 98 |
{d e f} {d e f} e 2 3 1 |
| 99 |
} |
| 100 |
|
| 101 |
|
| 102 |
#------------------------------------------------------------------------- |
| 103 |
# Error cases. |
| 104 |
# |
| 105 |
do_catchsql_test 2.0 { |
| 106 |
CREATE VIRTUAL TABLE tX USING fts5tokenize(nosuchtokenizer); |
| 107 |
} {1 {vtable constructor failed: tX}} |
| 108 |
|
| 109 |
do_catchsql_test 2.1 { |
| 110 |
CREATE VIRTUAL TABLE t4 USING fts5tokenize; |
| 111 |
SELECT * FROM t4; |
| 112 |
} {1 {SQL logic error}} |
| 113 |
|
| 114 |
#------------------------------------------------------------------------- |
| 115 |
# Embedded 0x00 characters. |
| 116 |
# |
| 117 |
reset_db |
| 118 |
do_execsql_test 3.1.0 { |
| 119 |
CREATE VIRTUAL TABLE t1 USING fts5(z); |
| 120 |
CREATE VIRTUAL TABLE tt USING fts5vocab(t1, 'instance'); |
| 121 |
INSERT INTO t1 VALUES('abc' || char(0) || 'def'); |
| 122 |
SELECT * FROM tt; |
| 123 |
} { abc 1 z 0 def 1 z 1 } |
| 124 |
do_execsql_test 3.1.1 { |
| 125 |
SELECT hex(z) FROM t1; |
| 126 |
} {61626300646566} |
| 127 |
do_execsql_test 3.1.2 { |
| 128 |
INSERT INTO t1(t1) VALUES('integrity-check'); |
| 129 |
} {} |
| 130 |
|
| 131 |
do_execsql_test 3.2.0 { |
| 132 |
CREATE VIRTUAL TABLE t2 USING fts5(z, |
| 133 |
tokenize="unicode61 categories 'L* N* Co Cc'" |
| 134 |
); |
| 135 |
CREATE VIRTUAL TABLE tu USING fts5vocab(t2, 'instance'); |
| 136 |
|
| 137 |
INSERT INTO t2 VALUES('abc' || char(0) || 'def'); |
| 138 |
SELECT * FROM tu; |
| 139 |
} { abc 1 z 0 def 1 z 1 } |
| 140 |
|
| 141 |
do_execsql_test 3.2.1 { |
| 142 |
SELECT hex(z) FROM t1; |
| 143 |
} {61626300646566} |
| 144 |
|
| 145 |
do_execsql_test 3.2.2 { |
| 146 |
INSERT INTO t1(t1) VALUES('integrity-check'); |
| 147 |
} {} |
| 148 |
|
| 149 |
|
| 150 |
finish_test |