Debian Codegraph @

ext/fts5/test/fts5tok1.test

raw
1 # 2016 Jan 15
2 #
3 # The author disclaims copyright to this source code. In place of
4 # a legal notice, here is a blessing:
5 #
6 # May you do good and not evil.
7 # May you find forgiveness for yourself and forgive others.
8 # May you share freely, never taking more than you give.
9 #
10 #*************************************************************************
11 #
12
13 source [file join [file dirname [info script]] fts5_common.tcl]
14 ifcapable !fts5 { finish_test ; return }
15 set ::testprefix fts5tok1
16
17
18 sqlite3_fts5_register_fts5tokenize db
19
20 #-------------------------------------------------------------------------
21 # Simple test cases. Using the default (ascii) tokenizer.
22 #
23 do_execsql_test 1.0 {
24 CREATE VIRTUAL TABLE t1 USING fts5tokenize(ascii);
25 CREATE VIRTUAL TABLE t2 USING fts5tokenize();
26 CREATE VIRTUAL TABLE t3 USING fts5tokenize(
27 ascii, 'separators', 'xyz', tokenchars, ''''
28 );
29 }
30
31 foreach {tn tbl} {1 t1 2 t2 3 t3} {
32 do_execsql_test 1.$tn.1 "SELECT input, * FROM $tbl ('one two three')" {
33 {one two three} one 0 3 0
34 {one two three} two 4 7 1
35 {one two three} three 8 13 2
36 }
37
38 do_execsql_test 1.$tn.2 "
39 SELECT token FROM $tbl WHERE input = 'OnE tWo tHrEe'
40 " {
41 one two three
42 }
43 }
44
45 do_execsql_test 1.4 {
46 SELECT token FROM t3 WHERE input = '1x2x3x'
47 } {1 2 3}
48
49 do_execsql_test 1.5 {
50 SELECT token FROM t1 WHERE input = '1x2x3x'
51 } {1x2x3x}
52
53 do_execsql_test 1.6 {
54 SELECT token FROM t3 WHERE input = '1''2x3x'
55 } {1'2 3}
56
57 do_execsql_test 1.7 {
58 SELECT token FROM t3 WHERE input = ''
59 } {}
60
61 do_execsql_test 1.8 {
62 SELECT token FROM t3 WHERE input = NULL
63 } {}
64
65 do_execsql_test 1.9 {
66 SELECT input, * FROM t3 WHERE input = 123
67 } {123 123 0 3 0}
68
69 do_execsql_test 1.10 {
70 SELECT input, * FROM t1 WHERE input = 'a b c' AND token = 'b';
71 } {
72 {a b c} b 2 3 1
73 }
74
75 do_execsql_test 1.11 {
76 SELECT input, * FROM t1 WHERE token = 'b' AND input = 'a b c';
77 } {
78 {a b c} b 2 3 1
79 }
80
81 do_execsql_test 1.12 {
82 SELECT input, * FROM t1 WHERE input < 'b' AND input = 'a b c';
83 } {
84 {a b c} a 0 1 0
85 {a b c} b 2 3 1
86 {a b c} c 4 5 2
87 }
88
89 do_execsql_test 1.13.1 {
90 CREATE TABLE c1(x);
91 INSERT INTO c1(x) VALUES('a b c');
92 INSERT INTO c1(x) VALUES('d e f');
93 }
94 do_execsql_test 1.13.2 {
95 SELECT c1.*, input, t1.* FROM c1, t1 WHERE input = x AND c1.rowid=t1.rowid;
96 } {
97 {a b c} {a b c} a 0 1 0
98 {d e f} {d e f} e 2 3 1
99 }
100
101
102 #-------------------------------------------------------------------------
103 # Error cases.
104 #
105 do_catchsql_test 2.0 {
106 CREATE VIRTUAL TABLE tX USING fts5tokenize(nosuchtokenizer);
107 } {1 {vtable constructor failed: tX}}
108
109 do_catchsql_test 2.1 {
110 CREATE VIRTUAL TABLE t4 USING fts5tokenize;
111 SELECT * FROM t4;
112 } {1 {SQL logic error}}
113
114 #-------------------------------------------------------------------------
115 # Embedded 0x00 characters.
116 #
117 reset_db
118 do_execsql_test 3.1.0 {
119 CREATE VIRTUAL TABLE t1 USING fts5(z);
120 CREATE VIRTUAL TABLE tt USING fts5vocab(t1, 'instance');
121 INSERT INTO t1 VALUES('abc' || char(0) || 'def');
122 SELECT * FROM tt;
123 } { abc 1 z 0 def 1 z 1 }
124 do_execsql_test 3.1.1 {
125 SELECT hex(z) FROM t1;
126 } {61626300646566}
127 do_execsql_test 3.1.2 {
128 INSERT INTO t1(t1) VALUES('integrity-check');
129 } {}
130
131 do_execsql_test 3.2.0 {
132 CREATE VIRTUAL TABLE t2 USING fts5(z,
133 tokenize="unicode61 categories 'L* N* Co Cc'"
134 );
135 CREATE VIRTUAL TABLE tu USING fts5vocab(t2, 'instance');
136
137 INSERT INTO t2 VALUES('abc' || char(0) || 'def');
138 SELECT * FROM tu;
139 } { abc 1 z 0 def 1 z 1 }
140
141 do_execsql_test 3.2.1 {
142 SELECT hex(z) FROM t1;
143 } {61626300646566}
144
145 do_execsql_test 3.2.2 {
146 INSERT INTO t1(t1) VALUES('integrity-check');
147 } {}
148
149
150 finish_test