Debian Codegraph @

ext/fts5/test/fts5origintext5.test

raw
1 # 2023 Dec 04
2 #
3 # The author disclaims copyright to this source code. In place of
4 # a legal notice, here is a blessing:
5 #
6 # May you do good and not evil.
7 # May you find forgiveness for yourself and forgive others.
8 # May you share freely, never taking more than you give.
9 #
10 #***********************************************************************
11 #
12 # Tests for tables that use both tokendata=1 and contentless_delete=1.
13 #
14
15 source [file join [file dirname [info script]] fts5_common.tcl]
16 set testprefix fts5origintext
17
18 # If SQLITE_ENABLE_FTS5 is not defined, omit this file.
19 ifcapable !fts5 {
20 finish_test
21 return
22 }
23
24 # Return a random integer between 0 and n-1.
25 #
26 proc random {n} { expr {abs(int(rand()*$n))} }
27
28 # Select an element of the list passed as the only argument at random and
29 # return it.
30 #
31 proc select_one {list} {
32 set n [llength $list]
33 lindex $list [random $n]
34 }
35
36 # Given a term that consists entirely of alphabet characters, return all
37 # permutations of the term using upper and lower case characters. e.g.
38 #
39 # "abc" -> {CBA cBA CbA cbA CBa cBa Cba cba}
40 #
41 proc casify {term {lRet {{}}}} {
42 if {$term==""} { return $lRet }
43 set t [string range $term 1 end]
44 set f1 [string toupper [string range $term 0 0]]
45 set f2 [string tolower [string range $term 0 0]]
46 set ret [list]
47 foreach x $lRet {
48 lappend ret "$x$f1"
49 lappend ret "$x$f2"
50 }
51 return [casify $t $ret]
52 }
53
54 proc vocab {} {
55 list abc def ghi jkl mno pqr stu vwx yza
56 }
57
58 # Return a random 3 letter term.
59 #
60 proc term {} {
61 if {[info exists ::expanded_vocab]==0} {
62 foreach v [vocab] { lappend ::expanded_vocab {*}[casify $v] }
63 }
64
65 select_one $::expanded_vocab
66 }
67
68 # Return a document - between 3 and 10 terms.
69 #
70 proc document {} {
71 set nTerm [expr [random 3] + 7]
72 set doc ""
73 for {set ii 0} {$ii < $nTerm} {incr ii} {
74 lappend doc [term]
75 }
76 set doc
77 }
78 db func document document
79
80 #-------------------------------------------------------------------------
81
82 expr srand(6)
83
84 set NDOC 200
85 set NLOOP 50
86
87 sqlite3_fts5_register_origintext db
88
89 proc tokens {cmd} {
90 set ret [list]
91 for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
92 set txt [$cmd xInstToken $iTok 0]
93 set txt [string map [list "\0" "."] $txt]
94 lappend ret $txt
95 }
96 set ret
97 }
98 sqlite3_fts5_create_function db tokens tokens
99
100 proc rankfunc {cmd} {
101 $cmd xRowid
102 }
103 sqlite3_fts5_create_function db rankfunc rankfunc
104
105 proc ctrl_tokens {term args} {
106 set ret [list]
107 set term [string tolower $term]
108 foreach doc $args {
109 foreach a $doc {
110 if {[string tolower $a]==$term} {
111 if {$a==$term} {
112 lappend ret $a
113 } else {
114 lappend ret [string tolower $a].$a
115 }
116 }
117 }
118 }
119 set ret
120 }
121 db func ctrl_tokens ctrl_tokens
122
123 proc do_all_vocab_test {tn} {
124 foreach ::v [concat [vocab] nnn] {
125 set answer [execsql {
126 SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%'
127 }]
128 do_execsql_test $tn.$::v.1 {
129 SELECT rowid, tokens(ft) FROM ft($::v)
130 } $answer
131 do_execsql_test $tn.$::v.2 {
132 SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank
133 } $answer
134 }
135 }
136
137 do_execsql_test 1.0 {
138 CREATE VIRTUAL TABLE ft USING fts5(
139 x, tokenize="origintext unicode61", content=, contentless_delete=1,
140 tokendata=1
141 );
142
143 CREATE TABLE ctrl(id INTEGER PRIMARY KEY, x TEXT);
144 INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
145 INSERT INTO ft(ft, rank) VALUES('rank', 'rankfunc()');
146 }
147 do_test 1.1 {
148 for {set ii 0} {$ii < $NDOC} {incr ii} {
149 set doc [document]
150 execsql {
151 INSERT INTO ft(rowid, x) VALUES($ii, $doc);
152 INSERT INTO ctrl(id, x) VALUES($ii, $doc);
153 }
154 }
155 } {}
156
157 #execsql_pp { SELECT * FROM ctrl }
158 #execsql_pp { SELECT * FROM ft }
159 #fts5_aux_test_functions db
160 #execsql_pp { SELECT rowid, tokens(ft), fts5_test_poslist(ft) FROM ft('ghi'); }
161
162 do_all_vocab_test 1.2
163
164 for {set ii 0} {$ii < $NLOOP} {incr ii} {
165 set lRowid [execsql { SELECT id FROM ctrl WHERE random() % 2 }]
166 foreach r $lRowid {
167 execsql { DELETE FROM ft WHERE rowid = $r }
168 execsql { DELETE FROM ctrl WHERE rowid = $r }
169
170 set doc [document]
171 execsql { INSERT INTO ft(rowid, x) VALUES($r, $doc) }
172 execsql { INSERT INTO ctrl(id, x) VALUES($r, $doc) }
173 }
174 do_all_vocab_test 1.3.$ii
175 }
176
177 #-------------------------------------------------------------------------
178
179 do_execsql_test 2.0 {
180 CREATE VIRTUAL TABLE ft2 USING fts5(
181 x, y, tokenize="origintext unicode61", content=, contentless_delete=1,
182 tokendata=1
183 );
184
185 CREATE TABLE ctrl2(id INTEGER PRIMARY KEY, x TEXT, y TEXT);
186 INSERT INTO ft2(ft2, rank) VALUES('pgsz', 64);
187 INSERT INTO ft2(ft2, rank) VALUES('rank', 'rankfunc()');
188 }
189 do_test 2.1 {
190 for {set ii 0} {$ii < $NDOC} {incr ii} {
191 set doc1 [document]
192 set doc2 [document]
193 execsql {
194 INSERT INTO ft2(rowid, x, y) VALUES($ii, $doc, $doc2);
195 INSERT INTO ctrl2(id, x, y) VALUES($ii, $doc, $doc2);
196 }
197 }
198 } {}
199
200 proc do_all_vocab_test2 {tn} {
201 foreach ::v [vocab] {
202 set answer [execsql {
203 SELECT id, ctrl_tokens($::v, x, y) FROM ctrl2
204 WHERE x LIKE '%' || $::v || '%' OR y LIKE '%' || $::v || '%';
205 }]
206 do_execsql_test $tn.$::v.1 {
207 SELECT rowid, tokens(ft2) FROM ft2($::v)
208 } $answer
209 do_execsql_test $tn.$::v.2 {
210 SELECT rowid, tokens(ft2) FROM ft2($::v) ORDER BY rank
211 } $answer
212 }
213 }
214
215 do_all_vocab_test2 2.2
216
217 for {set ii 0} {$ii < $NLOOP} {incr ii} {
218 set lRowid [execsql { SELECT id FROM ctrl2 WHERE random() % 2 }]
219 foreach r $lRowid {
220 execsql { DELETE FROM ft2 WHERE rowid = $r }
221 execsql { DELETE FROM ctrl2 WHERE rowid = $r }
222
223 set doc1 [document]
224 set doc2 [document]
225 execsql { INSERT INTO ft2(rowid, x, y) VALUES($r, $doc, $doc1) }
226 execsql { INSERT INTO ctrl2(id, x, y) VALUES($r, $doc, $doc2) }
227 }
228 do_all_vocab_test 2.3.$ii
229 }
230
231 #-------------------------------------------------------------------------
232
233 unset -nocomplain ::expanded_vocab
234 proc vocab {} {
235 list abcde fghij klmno
236 }
237
238 proc do_all_vocab_test3 {tn} {
239 foreach ::v [concat [vocab] nnn] {
240 set answer [execsql {
241 SELECT rowid, ctrl_tokens($::v, w) FROM ctrl3 WHERE w LIKE '%' || $::v || '%'
242 }]
243 do_execsql_test $tn.$::v.1 {
244 SELECT rowid, tokens(ft3) FROM ft3($::v)
245 } $answer
246 do_execsql_test $tn.$::v.2 {
247 SELECT rowid, tokens(ft3) FROM ft3($::v) ORDER BY rank
248 } $answer
249 }
250 }
251
252 do_execsql_test 3.0 {
253 CREATE VIRTUAL TABLE ft3 USING fts5(
254 w, tokenize="origintext unicode61", content=, contentless_delete=1,
255 tokendata=1
256 );
257 INSERT INTO ft3(ft3, rank) VALUES('rank', 'rankfunc()');
258 CREATE TABLE ctrl3(w);
259 }
260
261 do_execsql_test 3.1 {
262 WITH s(i) AS (
263 SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2
264 )
265 INSERT INTO ctrl3 SELECT document() FROM s;
266 INSERT INTO ft3(rowid, w) SELECT rowid, w FROM ctrl3;
267 }
268
269 do_all_vocab_test3 3.2
270
271
272 finish_test
273