Debian Codegraph @

ext/fts5/test/fts5origintext.test

raw
1 # 2014 Jan 08
2 #
3 # The author disclaims copyright to this source code. In place of
4 # a legal notice, here is a blessing:
5 #
6 # May you do good and not evil.
7 # May you find forgiveness for yourself and forgive others.
8 # May you share freely, never taking more than you give.
9 #
10 #***********************************************************************
11 #
12 # Tests focused on phrase queries.
13 #
14
15 source [file join [file dirname [info script]] fts5_common.tcl]
16 set testprefix fts5origintext
17
18 # If SQLITE_ENABLE_FTS5 is not defined, omit this file.
19 ifcapable !fts5 {
20 finish_test
21 return
22 }
23
24 foreach_detail_mode $testprefix {
25 foreach {tn insttoken} {
26 1 0
27 2 1
28 } {
29 reset_db
30
31 sqlite3_fts5_register_origintext db
32 do_execsql_test $tn.1.0 {
33 CREATE VIRTUAL TABLE ft USING fts5(
34 x, tokenize="origintext unicode61", detail=%DETAIL%
35 );
36 INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
37 CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
38 }
39
40 do_execsql_test $tn.1.1 {
41 INSERT INTO ft VALUES('Hello world');
42 }
43
44 do_execsql_test $tn.1.2 {
45 INSERT INTO ft(ft) VALUES('integrity-check');
46 }
47
48 proc b {x} { string map [list "\0" "."] $x }
49 db func b b
50
51 do_execsql_test $tn.1.3 {
52 select b(term) from vocab;
53 } {
54 hello.Hello
55 world
56 }
57
58 do_execsql_test $tn.1.4 {
59 SELECT rowid FROM ft('Hello');
60 } {1}
61
62 #-------------------------------------------------------------------------
63 reset_db
64
65 # Return a random integer between 0 and n-1.
66 #
67 proc random {n} {
68 expr {abs(int(rand()*$n))}
69 }
70
71 proc select_one {list} {
72 set n [llength $list]
73 lindex $list [random $n]
74 }
75
76 proc term {} {
77 set first_letter {
78 a b c d e f g h i j k l m n o p q r s t u v w x y z
79 A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
80 }
81
82 set term [select_one $first_letter]
83 append term [random 100]
84 }
85
86 proc document {} {
87 set nTerm [expr [random 5] + 5]
88 set doc ""
89 for {set ii 0} {$ii < $nTerm} {incr ii} {
90 lappend doc [term]
91 }
92 set doc
93 }
94 db func document document
95
96 sqlite3_fts5_register_origintext db
97 do_execsql_test $tn.2.0 {
98 CREATE VIRTUAL TABLE ft USING fts5(
99 x, tokenize="origintext unicode61", detail=%DETAIL%
100 );
101 INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
102 INSERT INTO ft(ft, rank) VALUES('pgsz', 128);
103 CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
104 }
105
106 do_test $tn.2.1 {
107 for {set ii 0} {$ii < 500} {incr ii} {
108 execsql { INSERT INTO ft VALUES( document() ) }
109 }
110 } {}
111
112 do_execsql_test $tn.2.2 {
113 INSERT INTO ft(ft) VALUES('integrity-check');
114 }
115
116 do_execsql_test $tn.2.3 {
117 INSERT INTO ft(ft, rank) VALUES('merge', 16);
118 }
119
120 do_execsql_test $tn.2.4 {
121 INSERT INTO ft(ft) VALUES('integrity-check');
122 }
123
124 do_execsql_test $tn.2.5 {
125 INSERT INTO ft(ft) VALUES('optimize');
126 }
127
128 #-------------------------------------------------------------------------
129 reset_db
130
131 sqlite3_fts5_register_origintext db
132 do_execsql_test $tn.3.0 {
133 CREATE VIRTUAL TABLE ft USING fts5(
134 x, tokenize="origintext unicode61", detail=%DETAIL%
135 );
136 INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
137 CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
138
139 INSERT INTO ft(rowid, x) VALUES(1, 'hello');
140 INSERT INTO ft(rowid, x) VALUES(2, 'Hello');
141 INSERT INTO ft(rowid, x) VALUES(3, 'HELLO');
142 }
143
144 #proc b {x} { string map [list "\0" "."] $x }
145 #db func b b
146 #execsql_pp { SELECT b(term) FROM vocab }
147
148 do_execsql_test $tn.3.1.1 { SELECT rowid FROM ft('hello') } 1
149 do_execsql_test $tn.3.1.2 { SELECT rowid FROM ft('Hello') } 2
150 do_execsql_test $tn.3.1.3 { SELECT rowid FROM ft('HELLO') } 3
151
152 do_execsql_test $tn.3.2 {
153 CREATE VIRTUAL TABLE ft2 USING fts5(x,
154 tokenize="origintext unicode61",
155 tokendata=1,
156 detail=%DETAIL%
157 );
158 INSERT INTO ft2(ft2, rank) VALUES('insttoken', $insttoken);
159 CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance);
160
161 INSERT INTO ft2(rowid, x) VALUES(1, 'hello');
162 INSERT INTO ft2(rowid, x) VALUES(2, 'Hello');
163 INSERT INTO ft2(rowid, x) VALUES(3, 'HELLO');
164
165 INSERT INTO ft2(rowid, x) VALUES(10, 'helloooo');
166 }
167
168 #proc b {x} { string map [list "\0" "."] $x }
169 #db func b b
170 #execsql_pp { SELECT b(term) FROM vocab }
171
172 do_execsql_test $tn.3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3}
173 do_execsql_test $tn.3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3}
174 do_execsql_test $tn.3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}
175
176 do_execsql_test $tn.3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}
177
178 do_execsql_test $tn.3.3.5.1 { SELECT rowid FROM ft2('HELLO') ORDER BY rowid DESC} {
179 3 2 1
180 }
181 do_execsql_test $tn.3.3.5.2 { SELECT rowid FROM ft2('HELLO') ORDER BY +rowid DESC} {
182 3 2 1
183 }
184
185 #-------------------------------------------------------------------------
186 #
187 reset_db
188 sqlite3_fts5_register_origintext db
189 proc querytoken {cmd iPhrase iToken} {
190 set txt [$cmd xQueryToken $iPhrase $iToken]
191 string map [list "\0" "."] $txt
192 }
193 sqlite3_fts5_create_function db querytoken querytoken
194
195 do_execsql_test $tn.4.0 {
196 CREATE VIRTUAL TABLE ft USING fts5(
197 x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
198 );
199 INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
200 INSERT INTO ft VALUES('one two three four');
201 }
202
203 do_execsql_test $tn.4.1 {
204 SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO')
205 } {1 two.TwO}
206 do_execsql_test $tn.4.2 {
207 SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE')
208 } {1 one}
209 do_execsql_test $tn.4.3 {
210 SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
211 } {1 two.TWO}
212
213 if {"%DETAIL%"=="full"} {
214 # Phrase queries are only supported for detail=full.
215 #
216 do_execsql_test $tn.4.4 {
217 SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
218 } {1 three.ThreE}
219 do_catchsql_test $tn.4.5 {
220 SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
221 } {1 SQLITE_RANGE}
222 do_catchsql_test $tn.4.6 {
223 SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
224 } {1 SQLITE_RANGE}
225 do_catchsql_test $tn.4.7 {
226 SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
227 } {1 SQLITE_RANGE}
228 }
229
230 #-------------------------------------------------------------------------
231 #
232 reset_db
233 sqlite3_fts5_register_origintext db
234 proc insttoken {cmd iIdx iToken} {
235 set txt [$cmd xInstToken $iIdx $iToken]
236 string map [list "\0" "."] $txt
237 }
238 sqlite3_fts5_create_function db insttoken insttoken
239 fts5_aux_test_functions db
240
241 do_execsql_test $tn.5.0 {
242 CREATE VIRTUAL TABLE ft USING fts5(
243 x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
244 );
245 INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
246 INSERT INTO ft VALUES('one ONE One oNe oNE one');
247 }
248
249 do_execsql_test $tn.5.1 {
250 SELECT insttoken(ft, 0, 0),
251 insttoken(ft, 1, 0),
252 insttoken(ft, 2, 0),
253 insttoken(ft, 3, 0),
254 insttoken(ft, 4, 0),
255 insttoken(ft, 5, 0)
256 FROM ft('one');
257 } {
258 one one.ONE one.One one.oNe one.oNE one
259 }
260
261 do_execsql_test $tn.5.2 {
262 SELECT insttoken(ft, 0, 0),
263 insttoken(ft, 1, 0),
264 insttoken(ft, 2, 0),
265 insttoken(ft, 3, 0),
266 insttoken(ft, 4, 0),
267 insttoken(ft, 5, 0)
268 FROM ft('on*');
269 } {
270 one one.ONE one.One one.oNe one.oNE one
271 }
272
273 do_execsql_test $tn.5.3 {
274 SELECT insttoken(ft, 0, 0),
275 insttoken(ft, 1, 0),
276 insttoken(ft, 2, 0),
277 insttoken(ft, 3, 0),
278 insttoken(ft, 4, 0),
279 insttoken(ft, 5, 0)
280 FROM ft(fts5_insttoken('on*'));
281 } {
282 one one.ONE one.One one.oNe one.oNE one
283 }
284
285 do_execsql_test $tn.5.4 {
286 SELECT insttoken(ft, 1, 0) FROM ft('one');
287 } {
288 one.ONE
289 }
290
291 do_execsql_test $tn.5.5 {
292 SELECT fts5_test_poslist(ft) FROM ft('one');
293 } {
294 {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
295 }
296
297 #-------------------------------------------------------------------------
298 # Test the xInstToken() API with:
299 #
300 # * a non tokendata=1 table.
301 # * prefix queries.
302 #
303 reset_db
304 sqlite3_fts5_register_origintext db
305 do_execsql_test $tn.6.0 {
306 CREATE VIRTUAL TABLE ft USING fts5(
307 x, y, tokenize='origintext unicode61', detail=%DETAIL%, tokendata=0
308 );
309 INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
310
311 INSERT INTO ft VALUES('One Two', 'Three two');
312 INSERT INTO ft VALUES('three Three', 'one One');
313 }
314 proc tokens {cmd} {
315 set ret [list]
316 for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
317 set txt [$cmd xInstToken $iTok 0]
318 set txt [string map [list "\0" "."] $txt]
319 lappend ret $txt
320 }
321 set ret
322 }
323 sqlite3_fts5_create_function db tokens tokens
324
325 do_execsql_test $tn.6.1 {
326 SELECT rowid, tokens(ft) FROM ft('One');
327 } {1 one.One 2 one.One}
328
329 do_execsql_test $tn.6.2 {
330 SELECT rowid, tokens(ft) FROM ft('on*');
331 } {1 one.One 2 {one one.One}}
332
333 do_execsql_test $tn.6.3 {
334 SELECT rowid, tokens(ft) FROM ft('Three*');
335 } {1 three.Three 2 three.Three}
336
337 fts5_aux_test_functions db
338 do_catchsql_test $tn.6.4 {
339 SELECT fts5_test_insttoken(ft, -1, 0) FROM ft('one');
340 } {1 SQLITE_RANGE}
341
342 do_catchsql_test $tn.6.5 {
343 SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one');
344 } {1 SQLITE_RANGE}
345
346 do_catchsql_test $tn.6.6 {
347 CREATE VIRTUAL TABLE ft2 USING fts5(x, tokendata=2);
348 } {1 {malformed tokendata=... directive}}
349 do_catchsql_test $tn.6.7 {
350 CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', tokendata=11);
351 } {1 {malformed tokendata=... directive}}
352
353 }
354 }
355
356 finish_test
357