| 1 |
# 2007 June 21 |
| 2 |
# |
| 3 |
# The author disclaims copyright to this source code. In place of |
| 4 |
# a legal notice, here is a blessing: |
| 5 |
# |
| 6 |
# May you do good and not evil. |
| 7 |
# May you find forgiveness for yourself and forgive others. |
| 8 |
# May you share freely, never taking more than you give. |
| 9 |
# |
| 10 |
#************************************************************************* |
| 11 |
# This file implements regression tests for SQLite library. The focus |
| 12 |
# of this script is testing the pluggable tokeniser feature of the |
| 13 |
# FTS3 module. |
| 14 |
# |
| 15 |
# $Id: fts3atoken.test,v 1.1 2007/08/20 17:38:42 shess Exp $ |
| 16 |
# |
| 17 |
|
| 18 |
set testdir [file dirname $argv0] |
| 19 |
source $testdir/tester.tcl |
| 20 |
|
| 21 |
# If SQLITE_ENABLE_FTS3 is defined, omit this file. |
| 22 |
ifcapable !fts3 { |
| 23 |
finish_test |
| 24 |
return |
| 25 |
} |
| 26 |
|
| 27 |
set ::testprefix fts3atoken |
| 28 |
|
| 29 |
proc escape_string {str} { |
| 30 |
set out "" |
| 31 |
foreach char [split $str ""] { |
| 32 |
scan $char %c i |
| 33 |
if {$i<=127} { |
| 34 |
append out $char |
| 35 |
} else { |
| 36 |
append out [format {\x%.4x} $i] |
| 37 |
} |
| 38 |
} |
| 39 |
set out |
| 40 |
} |
| 41 |
|
| 42 |
#-------------------------------------------------------------------------- |
| 43 |
# Test cases fts3atoken-1.* are the warm-body test for the SQL scalar |
| 44 |
# function fts3_tokenizer(). The procedure is as follows: |
| 45 |
# |
| 46 |
# 1: Verify that there is no such fts3 tokenizer as 'blah'. |
| 47 |
# |
| 48 |
# 2: Query for the built-in tokenizer 'simple'. Insert a copy of the |
| 49 |
# retrieved value as tokenizer 'blah'. |
| 50 |
# |
| 51 |
# 3: Test that the value returned for tokenizer 'blah' is now the |
| 52 |
# same as that retrieved for 'simple'. |
| 53 |
# |
| 54 |
# 4: Test that it is now possible to create an fts3 table using |
| 55 |
# tokenizer 'blah' (it was not possible in step 1). |
| 56 |
# |
| 57 |
# 5: Test that the table created to use tokenizer 'blah' is usable. |
| 58 |
# |
| 59 |
sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1 |
| 60 |
do_test fts3atoken-1.1 { |
| 61 |
catchsql { |
| 62 |
CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); |
| 63 |
} |
| 64 |
} {1 {unknown tokenizer: blah}} |
| 65 |
do_test fts3atoken-1.2 { |
| 66 |
execsql { |
| 67 |
SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL; |
| 68 |
} |
| 69 |
} {0} |
| 70 |
do_test fts3atoken-1.3 { |
| 71 |
execsql { |
| 72 |
SELECT fts3_tokenizer('blah') == fts3_tokenizer('simple'); |
| 73 |
} |
| 74 |
} {1} |
| 75 |
do_test fts3atoken-1.4 { |
| 76 |
catchsql { |
| 77 |
CREATE VIRTUAL TABLE t1 USING fts3(content, tokenize blah); |
| 78 |
} |
| 79 |
} {0 {}} |
| 80 |
do_test fts3atoken-1.5 { |
| 81 |
execsql { |
| 82 |
INSERT INTO t1(content) VALUES('There was movement at the station'); |
| 83 |
INSERT INTO t1(content) VALUES('For the word has passed around'); |
| 84 |
INSERT INTO t1(content) VALUES('That the colt from ol regret had got'); |
| 85 |
SELECT content FROM t1 WHERE content MATCH 'movement' |
| 86 |
} |
| 87 |
} {{There was movement at the station}} |
| 88 |
|
| 89 |
unset -nocomplain simple blah2name simplename |
| 90 |
set simplename "simple" |
| 91 |
set blah2name "blah2" |
| 92 |
set simple [db one {SELECT fts3_tokenizer('simple')}] |
| 93 |
sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 0 |
| 94 |
do_catchsql_test 1.6 { |
| 95 |
SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL; |
| 96 |
} {1 {fts3tokenize disabled}} |
| 97 |
do_test fts3atoken-1.7 { |
| 98 |
execsql { |
| 99 |
SELECT fts3_tokenizer('blah2', $simple) IS NULL; |
| 100 |
} |
| 101 |
} {1} |
| 102 |
|
| 103 |
# With ENABLE_FTS3_TOKENIZER off, the fts3_tokenzer(1) function |
| 104 |
# returns NULL unless the first parameter is a bound parameter. |
| 105 |
# If the first parameter is a bound parameter, then fts3_tokenizer(1) |
| 106 |
# returns the actual pointer value as a BLOB. |
| 107 |
# |
| 108 |
do_test fts3atoken-1.8 { |
| 109 |
execsql { |
| 110 |
SELECT fts3_tokenizer($blah2name) == fts3_tokenizer($simplename), |
| 111 |
typeof(fts3_tokenizer($blah2name)), |
| 112 |
typeof(fts3_tokenizer('blah2')), |
| 113 |
typeof(fts3_tokenizer($simplename)), |
| 114 |
typeof(fts3_tokenizer('simple')); |
| 115 |
} |
| 116 |
} {1 blob null blob null} |
| 117 |
|
| 118 |
# With ENABLE_FTS3_TOKENIZER on, fts3_tokenizer() always returns |
| 119 |
# the BLOB pointer, regardless the parameter |
| 120 |
# |
| 121 |
sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1 |
| 122 |
do_test fts3atoken-1.9 { |
| 123 |
execsql { |
| 124 |
SELECT fts3_tokenizer('blah2') == fts3_tokenizer('simple'), |
| 125 |
typeof(fts3_tokenizer($blah2name)), |
| 126 |
typeof(fts3_tokenizer('blah2')), |
| 127 |
typeof(fts3_tokenizer($simplename)), |
| 128 |
typeof(fts3_tokenizer('simple')); |
| 129 |
} |
| 130 |
} {1 blob blob blob blob} |
| 131 |
|
| 132 |
# 2019-12-31: The fts3_tokenizer() function can never be invoked from |
| 133 |
# within a trigger or view. |
| 134 |
# |
| 135 |
do_catchsql_test fts3atoken-1.10 { |
| 136 |
CREATE VIEW v110(x) AS |
| 137 |
SELECT fts3_tokenizer('tok110', fts3_tokenizer('simple')) IS NULL; |
| 138 |
} {0 {}} |
| 139 |
do_catchsql_test fts3atoken-1.11 { |
| 140 |
SELECT * FROM v110; |
| 141 |
} {1 {unsafe use of fts3_tokenizer()}} |
| 142 |
do_catchsql_test fts3atoken-1.12 { |
| 143 |
CREATE TABLE t110(a,b); |
| 144 |
CREATE TRIGGER r110 AFTER INSERT ON t110 BEGIN |
| 145 |
SELECT fts3_tokenizer('tok110', fts3_tokenizer('simple')) IS NULL; |
| 146 |
END; |
| 147 |
} {0 {}} |
| 148 |
do_catchsql_test fts3atoken-1.13 { |
| 149 |
INSERT INTO t110(a,b) VALUES(1,2); |
| 150 |
} {1 {unsafe use of fts3_tokenizer()}} |
| 151 |
do_catchsql_test fts3atoken-1.14 { |
| 152 |
SELECT * FROM t110; |
| 153 |
} {0 {}} |
| 154 |
|
| 155 |
#-------------------------------------------------------------------------- |
| 156 |
# Test cases fts3atoken-2.* test error cases in the scalar function based |
| 157 |
# API for getting and setting tokenizers. |
| 158 |
# |
| 159 |
do_test fts3atoken-2.1 { |
| 160 |
catchsql { |
| 161 |
SELECT fts3_tokenizer('nosuchtokenizer'); |
| 162 |
} |
| 163 |
} {1 {unknown tokenizer: nosuchtokenizer}} |
| 164 |
|
| 165 |
#-------------------------------------------------------------------------- |
| 166 |
# Test cases fts3atoken-3.* test the three built-in tokenizers with a |
| 167 |
# simple input string via the built-in test function. This is as much |
| 168 |
# to test the test function as the tokenizer implementations. |
| 169 |
# |
| 170 |
sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1 |
| 171 |
do_test fts3atoken-3.1 { |
| 172 |
execsql { |
| 173 |
SELECT fts3_tokenizer_test('simple', 'I don''t see how'); |
| 174 |
} |
| 175 |
} {{0 i I 1 don don 2 t t 3 see see 4 how how}} |
| 176 |
do_test fts3atoken-3.2 { |
| 177 |
execsql { |
| 178 |
SELECT fts3_tokenizer_test('porter', 'I don''t see how'); |
| 179 |
} |
| 180 |
} {{0 i I 1 don don 2 t t 3 see see 4 how how}} |
| 181 |
ifcapable icu { |
| 182 |
do_test fts3atoken-3.3 { |
| 183 |
execsql { |
| 184 |
SELECT fts3_tokenizer_test('icu', 'I don''t see how'); |
| 185 |
} |
| 186 |
} {{0 i I 1 don't don't 2 see see 3 how how}} |
| 187 |
} |
| 188 |
|
| 189 |
#-------------------------------------------------------------------------- |
| 190 |
# Test cases fts3atoken-4.* test the ICU tokenizer. In practice, this |
| 191 |
# tokenizer only has two modes - "thai" and "everybody else". Some other |
| 192 |
# Asian languages (Lao, Khmer etc.) require the same special treatment as |
| 193 |
# Thai, but ICU doesn't support them yet. |
| 194 |
# |
| 195 |
ifcapable icu { |
| 196 |
|
| 197 |
proc do_icu_test {name locale input output} { |
| 198 |
set ::out [db eval { SELECT fts3_tokenizer_test('icu', $locale, $input) }] |
| 199 |
do_test $name { |
| 200 |
lindex $::out 0 |
| 201 |
} $output |
| 202 |
} |
| 203 |
|
| 204 |
do_icu_test fts3atoken-4.1 en_US {} {} |
| 205 |
do_icu_test fts3atoken-4.2 en_US {Test cases fts3} [list \ |
| 206 |
0 test Test 1 cases cases 2 fts3 fts3 |
| 207 |
] |
| 208 |
|
| 209 |
# The following test shows that ICU is smart enough to recognise |
| 210 |
# Thai chararacters, even when the locale is set to English/United |
| 211 |
# States. |
| 212 |
# |
| 213 |
set input "\u0e2d\u0e30\u0e44\u0e23\u0e19\u0e30\u0e04\u0e23\u0e31\u0e1a" |
| 214 |
set output "0 \u0e2d\u0e30\u0e44\u0e23 \u0e2d\u0e30\u0e44\u0e23 " |
| 215 |
append output "1 \u0e19\u0e30 \u0e19\u0e30 " |
| 216 |
append output "2 \u0e04\u0e23\u0e31\u0e1a \u0e04\u0e23\u0e31\u0e1a" |
| 217 |
|
| 218 |
do_icu_test fts3atoken-4.3 th_TH $input $output |
| 219 |
do_icu_test fts3atoken-4.4 en_US $input $output |
| 220 |
|
| 221 |
# ICU handles an unknown locale by falling back to the default. |
| 222 |
# So this is not an error. |
| 223 |
do_icu_test fts3atoken-4.5 MiddleOfTheOcean $input $output |
| 224 |
|
| 225 |
set longtoken "AReallyReallyLongTokenOneThatWillSurelyRequire" |
| 226 |
append longtoken "AReallocInTheIcuTokenizerCode" |
| 227 |
|
| 228 |
set input "short tokens then " |
| 229 |
append input $longtoken |
| 230 |
set output "0 short short " |
| 231 |
append output "1 tokens tokens " |
| 232 |
append output "2 then then " |
| 233 |
append output "3 [string tolower $longtoken] $longtoken" |
| 234 |
|
| 235 |
do_icu_test fts3atoken-4.6 MiddleOfTheOcean $input $output |
| 236 |
do_icu_test fts3atoken-4.7 th_TH $input $output |
| 237 |
do_icu_test fts3atoken-4.8 en_US $input $output |
| 238 |
|
| 239 |
do_execsql_test 5.1 { |
| 240 |
CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US); |
| 241 |
insert into x1 (name) values (NULL); |
| 242 |
insert into x1 (name) values (NULL); |
| 243 |
delete from x1; |
| 244 |
} |
| 245 |
|
| 246 |
proc cp_to_str {codepoint_list} { |
| 247 |
set fmt [string repeat %c [llength $codepoint_list]] |
| 248 |
eval [list format $fmt] $codepoint_list |
| 249 |
} |
| 250 |
|
| 251 |
do_test 5.2 { |
| 252 |
set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}] |
| 253 |
execsql { INSERT INTO x1 VALUES($str) } |
| 254 |
} {} |
| 255 |
} |
| 256 |
|
| 257 |
do_test fts3atoken-internal { |
| 258 |
execsql { SELECT fts3_tokenizer_internal_test() } |
| 259 |
} {ok} |
| 260 |
|
| 261 |
#------------------------------------------------------------------------- |
| 262 |
# Test empty tokenizer names. |
| 263 |
# |
| 264 |
do_catchsql_test 6.1.1 { |
| 265 |
CREATE VIRTUAL TABLE t3 USING fts4(tokenize=""); |
| 266 |
} {1 {unknown tokenizer: }} |
| 267 |
do_catchsql_test 6.1.2 { |
| 268 |
CREATE VIRTUAL TABLE t3 USING fts4(tokenize=); |
| 269 |
} {1 {unknown tokenizer: }} |
| 270 |
do_catchsql_test 6.1.3 { |
| 271 |
CREATE VIRTUAL TABLE t3 USING fts4(tokenize=" "); |
| 272 |
} {1 {unknown tokenizer: }} |
| 273 |
|
| 274 |
do_catchsql_test 6.2.1 { |
| 275 |
SELECT fts3_tokenizer(NULL); |
| 276 |
} {1 {unknown tokenizer: }} |
| 277 |
|
| 278 |
sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1 |
| 279 |
do_catchsql_test 6.2.2 { |
| 280 |
SELECT fts3_tokenizer(NULL, X'1234567812345678'); |
| 281 |
} {1 {argument type mismatch}} |
| 282 |
do_catchsql_test 6.2.3 { |
| 283 |
SELECT fts3_tokenizer(NULL, X'12345678'); |
| 284 |
} {1 {argument type mismatch}} |
| 285 |
|
| 286 |
|
| 287 |
finish_test |