| 1 |
# 2015 April 21 |
| 2 |
# |
| 3 |
# The author disclaims copyright to this source code. In place of |
| 4 |
# a legal notice, here is a blessing: |
| 5 |
# |
| 6 |
# May you do good and not evil. |
| 7 |
# May you find forgiveness for yourself and forgive others. |
| 8 |
# May you share freely, never taking more than you give. |
| 9 |
# |
| 10 |
#*********************************************************************** |
| 11 |
# |
| 12 |
# The tests in this file are focused on the code in fts5_hash.c. |
| 13 |
# |
| 14 |
|
| 15 |
source [file join [file dirname [info script]] fts5_common.tcl] |
| 16 |
set testprefix fts5hash |
| 17 |
|
| 18 |
# If SQLITE_ENABLE_FTS5 is not defined, omit this file. |
| 19 |
ifcapable !fts5 { |
| 20 |
finish_test |
| 21 |
return |
| 22 |
} |
| 23 |
|
| 24 |
#------------------------------------------------------------------------- |
| 25 |
# Return a list of tokens (a vocabulary) that all share the same hash |
| 26 |
# key value. This can be used to test hash collisions. |
| 27 |
# |
| 28 |
proc build_vocab1 {args} { |
| 29 |
|
| 30 |
set O(-nslot) 1024 |
| 31 |
set O(-nword) 20 |
| 32 |
set O(-hash) 88 |
| 33 |
set O(-prefix) "" |
| 34 |
|
| 35 |
if {[llength $args] % 2} { error "bad args" } |
| 36 |
array set O2 $args |
| 37 |
foreach {k v} $args { |
| 38 |
if {[info exists O($k)]==0} { error "bad option: $k" } |
| 39 |
set O($k) $v |
| 40 |
} |
| 41 |
|
| 42 |
set L [list] |
| 43 |
while {[llength $L] < $O(-nword)} { |
| 44 |
set t "$O(-prefix)[random_token]" |
| 45 |
set h [sqlite3_fts5_token_hash $O(-nslot) $t] |
| 46 |
if {$O(-hash)==$h} { lappend L $t } |
| 47 |
} |
| 48 |
return $L |
| 49 |
} |
| 50 |
|
| 51 |
proc random_token {} { |
| 52 |
set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j] |
| 53 |
set iVal [expr int(rand() * 2000000)] |
| 54 |
return [string map $map $iVal] |
| 55 |
} |
| 56 |
|
| 57 |
proc random_doc {vocab nWord} { |
| 58 |
set doc "" |
| 59 |
set nVocab [llength $vocab] |
| 60 |
for {set i 0} {$i<$nWord} {incr i} { |
| 61 |
set j [expr {int(rand() * $nVocab)}] |
| 62 |
lappend doc [lindex $vocab $j] |
| 63 |
} |
| 64 |
return $doc |
| 65 |
} |
| 66 |
|
| 67 |
foreach_detail_mode $testprefix { |
| 68 |
|
| 69 |
set vocab [build_vocab1] |
| 70 |
db func r random_doc |
| 71 |
|
| 72 |
do_execsql_test 1.0 { |
| 73 |
CREATE VIRTUAL TABLE eee USING fts5(e, ee, detail=%DETAIL%); |
| 74 |
BEGIN; |
| 75 |
WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100) |
| 76 |
INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii; |
| 77 |
INSERT INTO eee(eee) VALUES('integrity-check'); |
| 78 |
COMMIT; |
| 79 |
INSERT INTO eee(eee) VALUES('integrity-check'); |
| 80 |
} |
| 81 |
|
| 82 |
set hash [sqlite3_fts5_token_hash 1024 xyz] |
| 83 |
set vocab [build_vocab1 -prefix xyz -hash $hash] |
| 84 |
lappend vocab xyz |
| 85 |
|
| 86 |
do_execsql_test 1.1 { |
| 87 |
CREATE VIRTUAL TABLE vocab USING fts5vocab(eee, 'row'); |
| 88 |
BEGIN; |
| 89 |
} |
| 90 |
do_test 1.2 { |
| 91 |
for {set i 1} {$i <= 100} {incr i} { |
| 92 |
execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) } |
| 93 |
} |
| 94 |
} {} |
| 95 |
|
| 96 |
do_test 1.3 { |
| 97 |
db eval { SELECT term, doc FROM vocab } { |
| 98 |
set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}] |
| 99 |
if {$nRow != $doc} { |
| 100 |
error "term=$term fts5vocab=$doc cnt=$nRow" |
| 101 |
} |
| 102 |
} |
| 103 |
set {} {} |
| 104 |
} {} |
| 105 |
|
| 106 |
do_execsql_test 1.4 { |
| 107 |
COMMIT; |
| 108 |
INSERT INTO eee(eee) VALUES('integrity-check'); |
| 109 |
} |
| 110 |
|
| 111 |
#----------------------------------------------------------------------- |
| 112 |
# Add a small and very large token with the same hash value to an |
| 113 |
# empty table. At one point this would provoke an asan error. |
| 114 |
# |
| 115 |
do_test 1.5 { |
| 116 |
set big [string repeat 12345 40] |
| 117 |
set hash [sqlite3_fts5_token_hash 1024 $big] |
| 118 |
while {1} { |
| 119 |
set small [random_token] |
| 120 |
if {[sqlite3_fts5_token_hash 1024 $small]==$hash} break |
| 121 |
} |
| 122 |
|
| 123 |
execsql { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%) } |
| 124 |
execsql { |
| 125 |
INSERT INTO t2 VALUES($small || ' ' || $big); |
| 126 |
} |
| 127 |
} {} |
| 128 |
|
| 129 |
} ;# foreach_detail_mode |
| 130 |
|
| 131 |
#------------------------------------------------------------------------- |
| 132 |
reset_db |
| 133 |
do_execsql_test 2.1 { |
| 134 |
CREATE VIRTUAL TABLE t1 USING fts5(x); |
| 135 |
INSERT INTO t1(t1, rank) VALUES('hashsize', 1024); |
| 136 |
INSERT INTO t1(t1, rank) VALUES('automerge', 0); |
| 137 |
INSERT INTO t1(t1, rank) VALUES('crisismerge', 1000); |
| 138 |
} |
| 139 |
|
| 140 |
do_execsql_test 2.2 { |
| 141 |
BEGIN; |
| 142 |
INSERT INTO t1 VALUES('abc def ghi'); |
| 143 |
SELECT count(*) FROM t1_data; |
| 144 |
} {2} |
| 145 |
|
| 146 |
do_execsql_test 2.3 { |
| 147 |
WITH s(i) AS ( |
| 148 |
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1024 |
| 149 |
) |
| 150 |
INSERT INTO t1 SELECT 'abc def ghi' FROM s; |
| 151 |
SELECT (SELECT count(*) FROM t1_data) > 10; |
| 152 |
} {1} |
| 153 |
|
| 154 |
do_execsql_test 2.4 { |
| 155 |
COMMIT; |
| 156 |
DROP TABLE t1; |
| 157 |
CREATE VIRTUAL TABLE t1 USING fts5(x); |
| 158 |
INSERT INTO t1(t1, rank) VALUES('hashsize', 1024); |
| 159 |
INSERT INTO t1(t1, rank) VALUES('automerge', 0); |
| 160 |
INSERT INTO t1(t1, rank) VALUES('crisismerge', 1000); |
| 161 |
WITH s(i) AS ( |
| 162 |
SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1024 |
| 163 |
) |
| 164 |
INSERT INTO t1 SELECT 'abc' || i || ' def' || i || ' ghi' || i FROM s; |
| 165 |
SELECT (SELECT count(*) FROM t1_data) > 100; |
| 166 |
} {1} |
| 167 |
|
| 168 |
finish_test |