Debian Codegraph @

ext/fts5/test/fts5tokenizer3.test

raw
1 # 2024 Aug 10
2 #
3 # The author disclaims copyright to this source code. In place of
4 # a legal notice, here is a blessing:
5 #
6 # May you do good and not evil.
7 # May you find forgiveness for yourself and forgive others.
8 # May you share freely, never taking more than you give.
9 #
10 #***********************************************************************
11 #
12 # Tests focusing on the built-in fts5 tokenizers.
13 #
14
15 source [file join [file dirname [info script]] fts5_common.tcl]
16 set testprefix fts5tokenizer3
17
18 # If SQLITE_ENABLE_FTS5 is not defined, omit this file.
19 ifcapable !fts5 {
20 finish_test
21 return
22 }
23
24
25 proc get_sod {args} { return "split_on_dot" }
26 proc get_lowercase {args} { return "lowercase" }
27
28 proc lowercase {flags txt} {
29 set n [string length $txt]
30 sqlite3_fts5_token [string tolower $txt] 0 $n
31 return 0
32 }
33
34 proc split_on_dot {flags txt} {
35 set iOff 0
36 foreach t [split $txt "."] {
37 set n [string length $txt]
38 sqlite3_fts5_token $t $iOff [expr $iOff+$n]
39 incr iOff [expr {$n+1}]
40 }
41 return ""
42 }
43
44 foreach {tn script} {
45 1 {
46 sqlite3_fts5_create_tokenizer db lowercase get_lowercase
47 sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod
48 }
49 2 {
50 sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase
51 sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod
52 }
53 3 {
54 sqlite3_fts5_create_tokenizer db lowercase get_lowercase
55 sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod
56 }
57 4 {
58 sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase
59 sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod
60 }
61 } {
62 reset_db
63 eval $script
64
65 do_execsql_test 1.$tn.0 {
66 CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=split_on_dot);
67 CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, instance);
68 INSERT INTO t1 VALUES('ABC.Def.ghi');
69 }
70
71 do_execsql_test 1.$tn.1 {
72 SELECT term FROM t1vocab ORDER BY 1
73 } {abc def ghi}
74 }
75
76
77 finish_test