Debian Codegraph @

ext/fts5/test/fts5_common.tcl

raw
1 # 2014 Dec 19
2 #
3 # The author disclaims copyright to this source code. In place of
4 # a legal notice, here is a blessing:
5 #
6 # May you do good and not evil.
7 # May you find forgiveness for yourself and forgive others.
8 # May you share freely, never taking more than you give.
9 #
10 #***********************************************************************
11 #
12
13 if {![info exists testdir]} {
14 set testdir [file join [file dirname [info script]] .. .. .. test]
15 }
16 source $testdir/tester.tcl
17
18 ifcapable !fts5 {
19 proc return_if_no_fts5 {} {
20 finish_test
21 return -code return
22 }
23 return
24 } else {
25 proc return_if_no_fts5 {} {}
26 }
27
28 catch {
29 sqlite3_fts5_may_be_corrupt 0
30 reset_db
31 }
32
33 proc fts5_test_poslist {cmd} {
34 set res [list]
35 for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
36 lappend res [string map {{ } .} [$cmd xInst $i]]
37 }
38 set res
39 }
40
41 proc fts5_test_poslist2 {cmd} {
42 set res [list]
43
44 for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
45 $cmd xPhraseForeach $i c o {
46 lappend res $i.$c.$o
47 }
48 }
49
50 #set res
51 sort_poslist $res
52 }
53
54 proc fts5_test_insttoken {cmd iInst iToken} {
55 $cmd xInstToken $iInst $iToken
56 }
57
58 proc fts5_test_collist {cmd} {
59 set res [list]
60
61 for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
62 $cmd xPhraseColumnForeach $i c { lappend res $i.$c }
63 }
64
65 set res
66 }
67
68 proc fts5_collist {cmd iPhrase} {
69 set res [list]
70 $cmd xPhraseColumnForeach $iPhrase c { lappend res $c }
71 set res
72 }
73
74 proc fts5_test_columnsize {cmd} {
75 set res [list]
76 for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
77 lappend res [$cmd xColumnSize $i]
78 }
79 set res
80 }
81
82 proc fts5_columntext {cmd iCol} {
83 $cmd xColumnText $iCol
84 }
85 proc fts5_columnlocale {cmd iCol} {
86 $cmd xColumnLocale $iCol
87 }
88
89 proc fts5_test_columntext {cmd} {
90 set res [list]
91 for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
92 lappend res [$cmd xColumnText $i]
93 }
94 set res
95 }
96
97 proc fts5_test_columnlocale {cmd} {
98 set res [list]
99 for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
100 lappend res [$cmd xColumnLocale $i]
101 }
102 set res
103 }
104
105 proc fts5_test_columntotalsize {cmd} {
106 set res [list]
107 for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
108 lappend res [$cmd xColumnTotalSize $i]
109 }
110 set res
111 }
112
113 proc test_append_token {varname token iStart iEnd} {
114 upvar $varname var
115 lappend var $token
116 return "SQLITE_OK"
117 }
118 proc fts5_test_tokenize {cmd} {
119 set res [list]
120 for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
121 set tokens [list]
122 $cmd xTokenize [$cmd xColumnText $i] [list test_append_token tokens]
123 lappend res $tokens
124 }
125 set res
126 }
127
128 proc fts5_test_rowcount {cmd} {
129 $cmd xRowCount
130 }
131
132 proc fts5_test_rowid {cmd} {
133 $cmd xRowid
134 }
135
136 proc test_queryphrase_cb {cnt cmd} {
137 upvar $cnt L
138 for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
139 foreach {ip ic io} [$cmd xInst $i] break
140 set A($ic) 1
141 }
142 foreach ic [array names A] {
143 lset L $ic [expr {[lindex $L $ic] + 1}]
144 }
145 }
146 proc fts5_test_queryphrase {cmd} {
147 set res [list]
148 for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
149 set cnt [list]
150 for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
151 $cmd xQueryPhrase $i [list test_queryphrase_cb cnt]
152 lappend res $cnt
153 }
154 set res
155 }
156
157 proc fts5_queryphrase {cmd iPhrase} {
158 set cnt [list]
159 for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
160 $cmd xQueryPhrase $iPhrase [list test_queryphrase_cb cnt]
161 set cnt
162 }
163
164 proc fts5_test_phrasecount {cmd} {
165 $cmd xPhraseCount
166 }
167
168 proc fts5_test_all {cmd} {
169 set res [list]
170 lappend res columnsize [fts5_test_columnsize $cmd]
171 lappend res columntext [fts5_test_columntext $cmd]
172 lappend res columntotalsize [fts5_test_columntotalsize $cmd]
173 lappend res poslist [fts5_test_poslist $cmd]
174 lappend res tokenize [fts5_test_tokenize $cmd]
175 lappend res rowcount [fts5_test_rowcount $cmd]
176 set res
177 }
178
179 proc fts5_aux_test_functions {db} {
180 foreach f {
181 fts5_test_columnsize
182 fts5_test_columntext
183 fts5_test_columnlocale
184 fts5_test_columntotalsize
185 fts5_test_poslist
186 fts5_test_poslist2
187 fts5_test_collist
188 fts5_test_insttoken
189 fts5_test_tokenize
190 fts5_test_rowcount
191 fts5_test_rowid
192 fts5_test_all
193
194 fts5_test_queryphrase
195 fts5_test_phrasecount
196 fts5_columntext
197 fts5_columnlocale
198 fts5_queryphrase
199 fts5_collist
200 } {
201 sqlite3_fts5_create_function $db $f $f
202 }
203 }
204
205 proc fts5_segcount {tbl} {
206 set N 0
207 foreach n [fts5_level_segs $tbl] { incr N $n }
208 set N
209 }
210
211 proc fts5_level_segs {tbl} {
212 set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
213 set ret [list]
214 foreach L [lrange [db one $sql] 1 end] {
215 lappend ret [expr [llength $L] - 3]
216 }
217 set ret
218 }
219
220 proc fts5_level_segids {tbl} {
221 set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
222 set ret [list]
223 foreach L [lrange [db one $sql] 1 end] {
224 set lvl [list]
225 foreach S [lrange $L 3 end] {
226 regexp {id=([1234567890]*)} $S -> segid
227 lappend lvl $segid
228 }
229 lappend ret $lvl
230 }
231 set ret
232 }
233
234 proc fts5_rnddoc {n} {
235 set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j]
236 set doc [list]
237 for {set i 0} {$i < $n} {incr i} {
238 lappend doc "x[string map $map [format %.3d [expr int(rand()*1000)]]]"
239 }
240 set doc
241 }
242
243 #-------------------------------------------------------------------------
244 # Usage:
245 #
246 # nearset aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2...
247 #
248 # This command is used to test if a document (set of column values) matches
249 # the logical equivalent of a single FTS5 NEAR() clump and, if so, return
250 # the equivalent of an FTS5 position list.
251 #
252 # Parameter $aCol is passed a list of the column values for the document
253 # to test. Parameters $phrase1 and so on are the phrases.
254 #
255 # The result is a list of phrase hits. Each phrase hit is formatted as
256 # three integers separated by "." characters, in the following format:
257 #
258 # <phrase number> . <column number> . <token offset>
259 #
260 # Options:
261 #
262 # -near N (NEAR distance. Default 10)
263 # -col C (List of column indexes to match against)
264 # -pc VARNAME (variable in caller frame to use for phrase numbering)
265 # -dict VARNAME (array in caller frame to use for synonyms)
266 #
267 proc nearset {aCol args} {
268
269 # Process the command line options.
270 #
271 set O(-near) 10
272 set O(-col) {}
273 set O(-pc) ""
274 set O(-dict) ""
275
276 set nOpt [lsearch -exact $args --]
277 if {$nOpt<0} { error "no -- option" }
278
279 # Set $lPhrase to be a list of phrases. $nPhrase its length.
280 set lPhrase [lrange $args [expr $nOpt+1] end]
281 set nPhrase [llength $lPhrase]
282
283 foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
284 if {[info exists O($k)]==0} { error "unrecognized option $k" }
285 set O($k) $v
286 }
287
288 if {$O(-pc) == ""} {
289 set counter 0
290 } else {
291 upvar $O(-pc) counter
292 }
293
294 if {$O(-dict)!=""} { upvar $O(-dict) aDict }
295
296 for {set j 0} {$j < [llength $aCol]} {incr j} {
297 for {set i 0} {$i < $nPhrase} {incr i} {
298 set A($j,$i) [list]
299 }
300 }
301
302 # Loop through each column of the current row.
303 for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
304
305 # If there is a column filter, test whether this column is excluded. If
306 # so, skip to the next iteration of this loop. Otherwise, set zCol to the
307 # column value and nToken to the number of tokens that comprise it.
308 if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue
309 set zCol [lindex $aCol $iCol]
310 set nToken [llength $zCol]
311
312 # Each iteration of the following loop searches a substring of the
313 # column value for phrase matches. The last token of the substring
314 # is token $iLast of the column value. The first token is:
315 #
316 # iFirst = ($iLast - $O(-near) - 1)
317 #
318 # where $sz is the length of the phrase being searched for. A phrase
319 # counts as matching the substring if its first token lies on or before
320 # $iLast and its last token on or after $iFirst.
321 #
322 # For example, if the query is "NEAR(a+b c, 2)" and the column value:
323 #
324 # "x x x x A B x x C x"
325 # 0 1 2 3 4 5 6 7 8 9"
326 #
327 # when (iLast==8 && iFirst=5) the range will contain both phrases and
328 # so both instances can be added to the output poslists.
329 #
330 set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
331 for { } {$iLast < $nToken} {incr iLast} {
332
333 catch { array unset B }
334
335 for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
336 set p [lindex $lPhrase $iPhrase]
337 set nPm1 [expr {[llength $p] - 1}]
338 set iFirst [expr $iLast - $O(-near) - [llength $p]]
339
340 for {set i $iFirst} {$i <= $iLast} {incr i} {
341 set lCand [lrange $zCol $i [expr $i+$nPm1]]
342 set bMatch 1
343 foreach tok $p term $lCand {
344 if {[nearset_match aDict $tok $term]==0} { set bMatch 0 ; break }
345 }
346 if {$bMatch} { lappend B($iPhrase) $i }
347 }
348
349 if {![info exists B($iPhrase)]} break
350 }
351
352 if {$iPhrase==$nPhrase} {
353 for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
354 set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)]
355 set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)]
356 }
357 }
358 }
359 }
360
361 set res [list]
362 #puts [array names A]
363
364 for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
365 for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
366 foreach a $A($iCol,$iPhrase) {
367 lappend res "$counter.$iCol.$a"
368 }
369 }
370 incr counter
371 }
372
373 #puts "$aCol -> $res"
374 sort_poslist $res
375 }
376
377 proc nearset_match {aDictVar tok term} {
378 if {[string match $tok $term]} { return 1 }
379
380 upvar $aDictVar aDict
381 if {[info exists aDict($tok)]} {
382 foreach s $aDict($tok) {
383 if {[string match $s $term]} { return 1 }
384 }
385 }
386 return 0;
387 }
388
389 #-------------------------------------------------------------------------
390 # Usage:
391 #
392 # sort_poslist LIST
393 #
394 # Sort a position list of the type returned by command [nearset]
395 #
396 proc sort_poslist {L} {
397 lsort -command instcompare $L
398 }
399 proc instcompare {lhs rhs} {
400 foreach {p1 c1 o1} [split $lhs .] {}
401 foreach {p2 c2 o2} [split $rhs .] {}
402
403 set res [expr $c1 - $c2]
404 if {$res==0} { set res [expr $o1 - $o2] }
405 if {$res==0} { set res [expr $p1 - $p2] }
406
407 return $res
408 }
409
410 #-------------------------------------------------------------------------
411 # Logical operators used by the commands returned by fts5_tcl_expr().
412 #
413 proc AND {args} {
414 foreach a $args {
415 if {[llength $a]==0} { return [list] }
416 }
417 sort_poslist [concat {*}$args]
418 }
419 proc OR {args} {
420 sort_poslist [concat {*}$args]
421 }
422 proc NOT {a b} {
423 if {[llength $b]>0} { return [list] }
424 return $a
425 }
426
427 #-------------------------------------------------------------------------
428 # This command is similar to [split], except that it also provides the
429 # start and end offsets of each token. For example:
430 #
431 # [fts5_tokenize_split "abc d ef"] -> {abc 0 3 d 4 5 ef 6 8}
432 #
433
434 proc gobble_whitespace {textvar} {
435 upvar $textvar t
436 regexp {([ ]*)(.*)} $t -> space t
437 return [string length $space]
438 }
439
440 proc gobble_text {textvar wordvar} {
441 upvar $textvar t
442 upvar $wordvar w
443 regexp {([^ ]*)(.*)} $t -> w t
444 return [string length $w]
445 }
446
447 proc fts5_tokenize_split {text} {
448 set token ""
449 set ret [list]
450 set iOff [gobble_whitespace text]
451 while {[set nToken [gobble_text text word]]} {
452 lappend ret $word $iOff [expr $iOff+$nToken]
453 incr iOff $nToken
454 incr iOff [gobble_whitespace text]
455 }
456
457 set ret
458 }
459
460 #-------------------------------------------------------------------------
461 #
462 proc foreach_detail_mode {prefix script} {
463 set saved $::testprefix
464 foreach d [list full col none] {
465 set s [string map [list %DETAIL% $d] $script]
466 set ::detail $d
467 set ::testprefix "$prefix-$d"
468 reset_db
469 uplevel $s
470 unset ::detail
471 }
472 set ::testprefix $saved
473 }
474
475 proc detail_check {} {
476 if {$::detail != "none" && $::detail!="full" && $::detail!="col"} {
477 error "not in foreach_detail_mode {...} block"
478 }
479 }
480 proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
481 proc detail_is_col {} { detail_check ; expr {$::detail == "col" } }
482 proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
483
484 proc foreach_tokenizer_mode {prefix script} {
485 set saved $::testprefix
486 foreach {d mapping} {
487 "" {}
488 "-origintext" {, tokenize="origintext unicode61", tokendata=1}
489 } {
490 set s [string map [list %TOKENIZER% $mapping] $script]
491 set ::testprefix "$prefix$d"
492 reset_db
493 sqlite3_fts5_register_origintext db
494 uplevel $s
495 }
496 set ::testprefix $saved
497 }
498
499 #-------------------------------------------------------------------------
500 # Convert a poslist of the type returned by fts5_test_poslist() to a
501 # collist as returned by fts5_test_collist().
502 #
503 proc fts5_poslist2collist {poslist} {
504 set res [list]
505 foreach h $poslist {
506 regexp {(.*)\.[1234567890]+} $h -> cand
507 lappend res $cand
508 }
509 set res [lsort -command fts5_collist_elem_compare -unique $res]
510 return $res
511 }
512
513 # Comparison function used by fts5_poslist2collist to sort collist entries.
514 proc fts5_collist_elem_compare {a b} {
515 foreach {a1 a2} [split $a .] {}
516 foreach {b1 b2} [split $b .] {}
517
518 if {$a1==$b1} { return [expr $a2 - $b2] }
519 return [expr $a1 - $b1]
520 }
521
522
523 #--------------------------------------------------------------------------
524 # Construct and return a tcl list equivalent to that returned by the SQL
525 # query executed against database handle [db]:
526 #
527 # SELECT
528 # rowid,
529 # fts5_test_poslist($tbl),
530 # fts5_test_collist($tbl)
531 # FROM $tbl('$expr')
532 # ORDER BY rowid $order;
533 #
534 proc fts5_query_data {expr tbl {order ASC} {aDictVar ""}} {
535
536 # Figure out the set of columns in the FTS5 table. This routine does
537 # not handle tables with UNINDEXED columns, but if it did, it would
538 # have to be here.
539 db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
540
541 set d ""
542 if {$aDictVar != ""} {
543 upvar $aDictVar aDict
544 set d aDict
545 }
546
547 set cols ""
548 foreach e $lCols { append cols ", '$e'" }
549 set tclexpr [db one [subst -novar {
550 SELECT fts5_expr_tcl( $expr, 'nearset $cols -dict $d -pc ::pc' [set cols] )
551 }]]
552
553 set res [list]
554 db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
555 set cols [list]
556 foreach col $lCols { lappend cols $x($col) }
557
558 set ::pc 0
559 set rowdata [eval $tclexpr]
560 if {$rowdata != ""} {
561 lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata]
562 }
563 }
564
565 set res
566 }
567
568 #-------------------------------------------------------------------------
569 # Similar to [fts5_query_data], but omit the collist field.
570 #
571 proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} {
572 set res [list]
573
574 if {$aDictVar!=""} {
575 upvar $aDictVar aDict
576 set dict aDict
577 } else {
578 set dict ""
579 }
580
581 foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
582 lappend res $rowid $poslist
583 }
584 set res
585 }
586
587 proc fts5_collist_data {expr tbl {order ASC} {aDictVar ""}} {
588 set res [list]
589
590 if {$aDictVar!=""} {
591 upvar $aDictVar aDict
592 set dict aDict
593 } else {
594 set dict ""
595 }
596
597 foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
598 lappend res $rowid $collist
599 }
600 set res
601 }
602
603 #-------------------------------------------------------------------------
604 #
605
606 # This command will only work inside a [foreach_detail_mode] block. It tests
607 # whether or not expression $expr run on FTS5 table $tbl is supported by
608 # the current mode. If so, 1 is returned. If not, 0.
609 #
610 # detail=full (all queries supported)
611 # detail=col (all but phrase queries and NEAR queries)
612 # detail=none (all but phrase queries, NEAR queries, and column filters)
613 #
614 proc fts5_expr_ok {expr tbl} {
615
616 if {![detail_is_full]} {
617 set nearset "nearset_rc"
618 if {[detail_is_col]} { set nearset "nearset_rf" }
619
620 set ::expr_not_ok 0
621 db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
622
623 set cols ""
624 foreach e $lCols { append cols ", '$e'" }
625 set ::pc 0
626 set tclexpr [db one [subst -novar {
627 SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] )
628 }]]
629 eval $tclexpr
630 if {$::expr_not_ok} { return 0 }
631 }
632
633 return 1
634 }
635
636 # Helper for [fts5_expr_ok]
637 proc nearset_rf {aCol args} {
638 set idx [lsearch -exact $args --]
639 if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} {
640 set ::expr_not_ok 1
641 }
642 list
643 }
644
645 # Helper for [fts5_expr_ok]
646 proc nearset_rc {aCol args} {
647 nearset_rf $aCol {*}$args
648 if {[lsearch $args -col]>=0} {
649 set ::expr_not_ok 1
650 }
651 list
652 }
653
654 proc dump {tname} {
655 execsql_pp "SELECT * FROM ${tname}_idx"
656 execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data"
657 }
658
659 #-------------------------------------------------------------------------
660 # Code for a simple Tcl tokenizer that supports synonyms at query time.
661 #
662 proc tclnum_tokenize {mode tflags text} {
663 foreach {w iStart iEnd} [fts5_tokenize_split $text] {
664 sqlite3_fts5_token $w $iStart $iEnd
665 if {$tflags == $mode && [info exists ::tclnum_syn($w)]} {
666 foreach s $::tclnum_syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd }
667 }
668 }
669 }
670
671 proc tclnum_create {args} {
672 set mode query
673 if {[llength $args]} {
674 set mode [lindex $args 0]
675 }
676 if {$mode != "query" && $mode != "document"} { error "bad mode: $mode" }
677 return [list tclnum_tokenize $mode]
678 }
679
680 proc fts5_tclnum_register {db} {
681 foreach SYNDICT {
682 {zero 0}
683 {one 1 i}
684 {two 2 ii}
685 {three 3 iii}
686 {four 4 iv}
687 {five 5 v}
688 {six 6 vi}
689 {seven 7 vii}
690 {eight 8 viii}
691 {nine 9 ix}
692
693 {a1 a2 a3 a4 a5 a6 a7 a8 a9}
694 {b1 b2 b3 b4 b5 b6 b7 b8 b9}
695 {c1 c2 c3 c4 c5 c6 c7 c8 c9}
696 } {
697 foreach s $SYNDICT {
698 set o [list]
699 foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
700 set ::tclnum_syn($s) $o
701 }
702 }
703 sqlite3_fts5_create_tokenizer db tclnum tclnum_create
704 }
705 #
706 # End of tokenizer code.
707 #-------------------------------------------------------------------------
708