| 1 |
# 2024 Aug 10 |
| 2 |
# |
| 3 |
# The author disclaims copyright to this source code. In place of |
| 4 |
# a legal notice, here is a blessing: |
| 5 |
# |
| 6 |
# May you do good and not evil. |
| 7 |
# May you find forgiveness for yourself and forgive others. |
| 8 |
# May you share freely, never taking more than you give. |
| 9 |
# |
| 10 |
#*********************************************************************** |
| 11 |
# |
| 12 |
# Tests focusing on the built-in fts5 tokenizers. |
| 13 |
# |
| 14 |
|
| 15 |
source [file join [file dirname [info script]] fts5_common.tcl] |
| 16 |
set testprefix fts5tokenizer3 |
| 17 |
|
| 18 |
# If SQLITE_ENABLE_FTS5 is not defined, omit this file. |
| 19 |
ifcapable !fts5 { |
| 20 |
finish_test |
| 21 |
return |
| 22 |
} |
| 23 |
|
| 24 |
|
| 25 |
proc get_sod {args} { return "split_on_dot" } |
| 26 |
proc get_lowercase {args} { return "lowercase" } |
| 27 |
|
| 28 |
proc lowercase {flags txt} { |
| 29 |
set n [string length $txt] |
| 30 |
sqlite3_fts5_token [string tolower $txt] 0 $n |
| 31 |
return 0 |
| 32 |
} |
| 33 |
|
| 34 |
proc split_on_dot {flags txt} { |
| 35 |
set iOff 0 |
| 36 |
foreach t [split $txt "."] { |
| 37 |
set n [string length $txt] |
| 38 |
sqlite3_fts5_token $t $iOff [expr $iOff+$n] |
| 39 |
incr iOff [expr {$n+1}] |
| 40 |
} |
| 41 |
return "" |
| 42 |
} |
| 43 |
|
| 44 |
foreach {tn script} { |
| 45 |
1 { |
| 46 |
sqlite3_fts5_create_tokenizer db lowercase get_lowercase |
| 47 |
sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod |
| 48 |
} |
| 49 |
2 { |
| 50 |
sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase |
| 51 |
sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod |
| 52 |
} |
| 53 |
3 { |
| 54 |
sqlite3_fts5_create_tokenizer db lowercase get_lowercase |
| 55 |
sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod |
| 56 |
} |
| 57 |
4 { |
| 58 |
sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase |
| 59 |
sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod |
| 60 |
} |
| 61 |
} { |
| 62 |
reset_db |
| 63 |
eval $script |
| 64 |
|
| 65 |
do_execsql_test 1.$tn.0 { |
| 66 |
CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=split_on_dot); |
| 67 |
CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, instance); |
| 68 |
INSERT INTO t1 VALUES('ABC.Def.ghi'); |
| 69 |
} |
| 70 |
|
| 71 |
do_execsql_test 1.$tn.1 { |
| 72 |
SELECT term FROM t1vocab ORDER BY 1 |
| 73 |
} {abc def ghi} |
| 74 |
} |
| 75 |
|
| 76 |
|
| 77 |
finish_test |