Debian Codegraph @

ext/fts5/test/fts5trigram.test

raw
1 # 2020 September 30
2 #
3 # The author disclaims copyright to this source code. In place of
4 # a legal notice, here is a blessing:
5 #
6 # May you do good and not evil.
7 # May you find forgiveness for yourself and forgive others.
8 # May you share freely, never taking more than you give.
9 #
10 #*************************************************************************
11 #
12 # Tests for the fts5 "trigram" tokenizer.
13 #
14
15 source [file join [file dirname [info script]] fts5_common.tcl]
16 ifcapable !fts5 { finish_test ; return }
17 set ::testprefix fts5trigram
18
19 do_execsql_test 1.0 {
20 CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize=trigram);
21 INSERT INTO t1 VALUES('abcdefghijklm');
22 INSERT INTO t1 VALUES('กรุงเทพมหานคร');
23 }
24
25 foreach {tn s res} {
26 1 abc "(abc)defghijklm"
27 2 defgh "abc(defgh)ijklm"
28 3 abcdefghijklm "(abcdefghijklm)"
29 4 กรุ "(กรุ)งเทพมหานคร"
30 5 งเทพมห "กรุ(งเทพมห)านคร"
31 6 กรุงเทพมหานคร "(กรุงเทพมหานคร)"
32 7 Abc "(abc)defghijklm"
33 8 deFgh "abc(defgh)ijklm"
34 9 aBcdefGhijKlm "(abcdefghijklm)"
35 } {
36 do_execsql_test 1.1.$tn {
37 SELECT highlight(t1, 0, '(', ')') FROM t1($s)
38 } $res
39 }
40
41 do_execsql_test 1.2.0 {
42 SELECT fts5_expr('ABCD', 'tokenize=trigram')
43 } {{"abc" + "bcd"}}
44
45 do_execsql_test 1.2.1 {
46 SELECT * FROM t1 WHERE y LIKE ? ESCAPE 'a'
47 }
48
49 foreach {tn like res} {
50 1 {%cDef%} 1
51 2 {cDef%} {}
52 3 {%f%} 1
53 4 {%f_h%} 1
54 5 {%f_g%} {}
55 6 {abc%klm} 1
56 7 {ABCDEFG%} 1
57 8 {%รุงเ%} 2
58 9 {%งเ%} 2
59 10 {%"งเ"%} {}
60 } {
61 do_execsql_test 1.3.$tn {
62 SELECT rowid FROM t1 WHERE y LIKE $like
63 } $res
64 }
65
66 #-------------------------------------------------------------------------
67 reset_db
68 do_execsql_test 2.0 {
69 CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize="trigram case_sensitive 1");
70 INSERT INTO t1 VALUES('abcdefghijklm');
71 INSERT INTO t1 VALUES('กรุงเทพมหานคร');
72 }
73 do_catchsql_test 2.0.1 {
74 CREATE VIRTUAL TABLE t2 USING fts5(z, tokenize='trigram case_sensitive');
75 } {1 {error in tokenizer constructor}}
76
77 foreach {tn s res} {
78 1 abc "(abc)defghijklm"
79 2 defgh "abc(defgh)ijklm"
80 3 abcdefghijklm "(abcdefghijklm)"
81 4 กรุ "(กรุ)งเทพมหานคร"
82 5 งเทพมห "กรุ(งเทพมห)านคร"
83 6 กรุงเทพมหานคร "(กรุงเทพมหานคร)"
84 7 Abc ""
85 8 deFgh ""
86 9 aBcdefGhijKlm ""
87 } {
88 do_execsql_test 2.1.$tn {
89 SELECT highlight(t1, 0, '(', ')') FROM t1($s)
90 } $res
91 }
92 foreach {tn like res} {
93 1 {%cDef%} 1
94 2 {cDef%} {}
95 3 {%f%} 1
96 4 {%f_h%} 1
97 5 {%f_g%} {}
98 6 {abc%klm} 1
99 7 {ABCDEFG%} 1
100 8 {%รุงเ%} 2
101 } {
102 do_execsql_test 2.2.$tn {
103 SELECT rowid FROM t1 WHERE y LIKE $like
104 } $res
105 }
106 foreach {tn like res} {
107 1 {*cdef*} 1
108 2 {cdef*} {}
109 3 {*f*} 1
110 4 {*f?h*} 1
111 5 {*f?g*} {}
112 6 {abc*klm} 1
113 7 {abcdefg*} 1
114 8 {*รุงเ*} 2
115 9 {abc[d]efg*} 1
116 10 {abc[]d]efg*} 1
117 11 {abc[^]d]efg*} {}
118 12 {abc[^]XYZ]efg*} 1
119 } {
120 do_execsql_test 2.3.$tn {
121 SELECT rowid FROM t1 WHERE y GLOB $like
122 } $res
123 }
124
125 do_execsql_test 2.3.null.1 {
126 SELECT rowid FROM t1 WHERE y LIKE NULL
127 }
128
129 #-------------------------------------------------------------------------
130 reset_db
131 do_catchsql_test 3.1 {
132 CREATE VIRTUAL TABLE ttt USING fts5(c, tokenize="trigram case_sensitive 2");
133 } {1 {error in tokenizer constructor}}
134 do_catchsql_test 3.2 {
135 CREATE VIRTUAL TABLE ttt USING fts5(c, tokenize="trigram case_sensitive 11");
136 } {1 {error in tokenizer constructor}}
137 do_catchsql_test 3.3 {
138 CREATE VIRTUAL TABLE ttt USING fts5(c, "tokenize=trigram case_sensitive 1");
139 } {0 {}}
140
141 #-------------------------------------------------------------------------
142 reset_db
143 do_execsql_test 4.0 {
144 CREATE VIRTUAL TABLE t0 USING fts5(b, tokenize = "trigram");
145 }
146 do_execsql_test 4.1 {
147 INSERT INTO t0 VALUES (x'000b01');
148 }
149 do_execsql_test 4.2 {
150 INSERT INTO t0(t0) VALUES('integrity-check');
151 }
152
153 #-------------------------------------------------------------------------
154 reset_db
155 foreach_detail_mode $::testprefix {
156 foreach {ci} {0 1} {
157 reset_db
158 do_execsql_test 5.cs=$ci.0.1 "
159 CREATE VIRTUAL TABLE t1 USING fts5(
160 y, tokenize=\"trigram case_sensitive $ci\", detail=%DETAIL%
161 );
162 "
163 do_execsql_test 5.cs=$ci.0.2 {
164 INSERT INTO t1 VALUES('abcdefghijklm');
165 INSERT INTO t1 VALUES('กรุงเทพมหานคร');
166 }
167
168 foreach {tn like res} {
169 1 {%cDef%} 1
170 2 {cDef%} {}
171 3 {%f%} 1
172 4 {%f_h%} 1
173 5 {%f_g%} {}
174 6 {abc%klm} 1
175 7 {ABCDEFG%} 1
176 8 {%รุงเ%} 2
177 } {
178 do_execsql_test 5.cs=$ci.1.$tn {
179 SELECT rowid FROM t1 WHERE y LIKE $like
180 } $res
181 }
182 }
183 }
184
185 do_execsql_test 6.0 {
186 CREATE VIRTUAL TABLE ci0 USING fts5(x, tokenize="trigram");
187 CREATE VIRTUAL TABLE ci1 USING fts5(x, tokenize="trigram case_sensitive 1");
188 }
189
190 # LIKE and GLOB both work with case-insensitive tokenizers. Only GLOB works
191 # with case-sensitive.
192 do_eqp_test 6.1 {
193 SELECT * FROM ci0 WHERE x LIKE ?
194 } {VIRTUAL TABLE INDEX 0:L0}
195 do_eqp_test 6.2 {
196 SELECT * FROM ci0 WHERE x GLOB ?
197 } {VIRTUAL TABLE INDEX 0:G0}
198 do_eqp_test 6.3 {
199 SELECT * FROM ci1 WHERE x LIKE ?
200 } {{SCAN ci1 VIRTUAL TABLE INDEX 0:}}
201 do_eqp_test 6.4 {
202 SELECT * FROM ci1 WHERE x GLOB ?
203 } {VIRTUAL TABLE INDEX 0:G0}
204 do_eqp_test 6.5 {
205 SELECT * FROM ci1 WHERE x < ?
206 } {{SCAN ci1 VIRTUAL TABLE INDEX 0:}}
207 do_eqp_test 6.6 {
208 SELECT * FROM ci0 WHERE x < ?
209 } {{SCAN ci0 VIRTUAL TABLE INDEX 0:}}
210
211 reset_db
212 do_execsql_test 7.0 {
213 CREATE VIRTUAL TABLE f USING FTS5(filename, tokenize="trigram");
214 INSERT INTO f (rowid, filename) VALUES
215 (10, "giraffe.png"),
216 (20, "жираф.png"),
217 (30, "cat.png"),
218 (40, "кот.png"),
219 (50, "misic-🎵-.mp3");
220 }
221 do_execsql_test 7.1 {
222 SELECT rowid FROM f WHERE +filename GLOB '*ир*';
223 } {20}
224 do_execsql_test 7.2 {
225 SELECT rowid FROM f WHERE filename GLOB '*ир*';
226 } {20}
227
228
229 #-------------------------------------------------------------------------
230 reset_db
231 do_execsql_test 8.0 {
232 CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize=trigram);
233 INSERT INTO t1 VALUES('abcdefghijklm');
234 }
235
236 foreach {tn match res} {
237 1 "abc ghi" "(abc)def(ghi)jklm"
238 2 "def ghi" "abc(defghi)jklm"
239 3 "efg ghi" "abcd(efghi)jklm"
240 4 "efghi" "abcd(efghi)jklm"
241 5 "abcd jklm" "(abcd)efghi(jklm)"
242 6 "ijkl jklm" "abcdefgh(ijklm)"
243 7 "ijk ijkl hijk" "abcdefg(hijkl)m"
244
245 } {
246 do_execsql_test 8.1.$tn {
247 SELECT highlight(t1, 0, '(', ')') FROM t1($match)
248 } $res
249 }
250
251 do_execsql_test 8.2 {
252 CREATE VIRTUAL TABLE ft2 USING fts5(a, tokenize="trigram");
253 INSERT INTO ft2 VALUES('abc x cde');
254 INSERT INTO ft2 VALUES('abc cde');
255 INSERT INTO ft2 VALUES('abcde');
256 }
257
258 do_execsql_test 8.3 {
259 SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'abc AND cde';
260 } {
261 {[abc] x [cde]}
262 {[abc] [cde]}
263 {[abcde]}
264 }
265
266 #-------------------------------------------------------------------------
267 reset_db
268 do_execsql_test 9.0 {
269 CREATE VIRTUAL TABLE t1 USING fts5(
270 a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12,
271 tokenize=trigram
272 );
273
274 INSERT INTO t1(rowid, a12) VALUES(111, 'thats a tricky case though');
275 INSERT INTO t1(rowid, a12) VALUES(222, 'the query planner cannot do');
276 }
277
278 do_execsql_test 9.1 {
279 SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%'
280 } {111}
281
282 do_execsql_test 9.2 {
283 SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%' AND a12 LIKE '%case%'
284 } {111}
285
286 do_execsql_test 9.3 {
287 SELECT rowid FROM t1 WHERE a12 LIKE NULL
288 } {}
289
290 #-------------------------------------------------------------------------
291 reset_db
292 do_execsql_test 10.0 {
293 CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=trigram);
294 }
295
296 do_test 10.1 {
297 foreach {val} {
298 "abc \UFFjkl\UFF"
299 "abc \UFFFjkl\UFFF"
300 "abc \UFFFFjkl\UFFFF"
301 "abc \UFFFFFjkl\UFFFFF"
302 "\UFFjkl\UFF abc"
303 "\UFFFjkl\UFFF abc"
304 "\UFFFFjkl\UFFFF abc"
305 "\UFFFFFjkl\UFFFFF abc"
306 "\U10001jkl\U10001 abc"
307 } {
308 execsql { INSERT INTO t1 VALUES( $val ) }
309 }
310 } {}
311
312 do_test 10.2 {
313 foreach {val} {
314 X'E18000626320646566'
315 X'61EDA0806320646566'
316 X'61EDA0806320646566'
317 X'61EFBFBE6320646566'
318 X'76686920E18000626320646566'
319 X'7668692061EDA0806320646566'
320 X'7668692061EDA0806320646566'
321 X'7668692061EFBFBE6320646566'
322 } {
323 execsql " INSERT INTO t1 VALUES( $val ) "
324 }
325 } {}
326
327 do_test 10.3 {
328 set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}]
329 set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}]
330 set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
331 set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
332 execsql {
333 INSERT INTO t1 VALUES($a);
334 INSERT INTO t1 VALUES($b);
335 INSERT INTO t1 VALUES($c);
336 INSERT INTO t1 VALUES($d);
337
338 INSERT INTO t1 VALUES('abcd' || $a);
339 INSERT INTO t1 VALUES('abcd' || $b);
340 INSERT INTO t1 VALUES('abcd' || $c);
341 INSERT INTO t1 VALUES('abcd' || $d);
342 }
343 } {}
344
345 do_execsql_test 11.0 {
346 CREATE VIRTUAL TABLE t4 USING fts5(y, tokenize=trigram);
347 }
348 sqlite3_fts5_register_str db
349 do_execsql_test 11.1 {
350 INSERT INTO t4 VALUES( str('') );
351 }
352
353 do_test 12.0 {
354 sqlite3_fts5_tokenize db trigram "abcd"
355 } {abc 0 3 bcd 1 4}
356
357 do_test 12.1 {
358 sqlite3_fts5_tokenize db trigram "a"
359 } {}
360
361 do_test 12.2 {
362 sqlite3_fts5_tokenize db trigram ""
363 } {}
364
365 finish_test
366