Debian Codegraph @

ext/fts5/test/fts5tokenizer2.test

raw
1 # 2023 Nov 03
2 #
3 # The author disclaims copyright to this source code. In place of
4 # a legal notice, here is a blessing:
5 #
6 # May you do good and not evil.
7 # May you find forgiveness for yourself and forgive others.
8 # May you share freely, never taking more than you give.
9 #
10 #***********************************************************************
11 #
12 # Tests focusing on the built-in fts5 tokenizers.
13 #
14
15 source [file join [file dirname [info script]] fts5_common.tcl]
16 set testprefix fts5tokenizer2
17
18 # If SQLITE_ENABLE_FTS5 is not defined, omit this file.
19 ifcapable !fts5 {
20 finish_test
21 return
22 }
23
24 sqlite3_fts5_create_tokenizer db tst get_tst_tokenizer
25 proc get_tst_tokenizer {args} {
26 return "tst_tokenizer"
27 }
28 proc tst_tokenizer {flags txt} {
29 set token ""
30 set lTok [list]
31
32 foreach c [split $txt {}] {
33 if {$token==""} {
34 append token $c
35 } else {
36 set t1 [string is upper $token]
37 set t2 [string is upper $c]
38
39 if {$t1!=$t2} {
40 lappend lTok $token
41 set token ""
42 }
43 append token $c
44 }
45 }
46 if {$token!=""} { lappend lTok $token }
47
48 set iOff 0
49 foreach t $lTok {
50 set n [string length $t]
51 sqlite3_fts5_token $t $iOff [expr $iOff+$n]
52 incr iOff $n
53 }
54 }
55
56 do_execsql_test 1.0 {
57 CREATE VIRTUAL TABLE t1 USING fts5(t, tokenize=tst);
58 }
59
60 do_execsql_test 1.1 {
61 INSERT INTO t1 VALUES('AAdontBBmess');
62 }
63
64 do_execsql_test 1.2 {
65 SELECT snippet(t1, 0, '>', '<', '...', 4) FROM t1('BB');
66 } {AAdont>BB<mess}
67
68 do_execsql_test 1.3 {
69 SELECT highlight(t1, 0, '>', '<') FROM t1('BB');
70 } {AAdont>BB<mess}
71
72 do_execsql_test 1.4 {
73 SELECT highlight(t1, 0, '>', '<') FROM t1('AA');
74 } {>AA<dontBBmess}
75
76 do_execsql_test 1.5 {
77 SELECT highlight(t1, 0, '>', '<') FROM t1('dont');
78 } {AA>dont<BBmess}
79
80 do_execsql_test 1.6 {
81 SELECT highlight(t1, 0, '>', '<') FROM t1('mess');
82 } {AAdontBB>mess<}
83
84 do_execsql_test 1.7 {
85 SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess');
86 } {AAdont>BBmess<}
87
88 # 2024-08-06 https://sqlite.org/forum/forumpost/171bcc2bcd
89 # Error handling of tokenize= arguments.
90 #
91 foreach {n tkz} {
92 1 {ascii none}
93 2 {unicode61 none}
94 3 {porter none}
95 4 {trigram none}
96 5 {ascii none 0}
97 6 {unicode61 none 0}
98 7 {porter none 0}
99 8 {trigram none 0}
100 } {
101 db eval {DROP TABLE IF EXISTS t2;}
102 do_catchsql_test 2.$n "
103 DROP TABLE IF EXISTS t2;
104 CREATE VIRTUAL TABLE t2 USING fts5(a,b,c,tokenize='$tkz');
105 " {1 {error in tokenizer constructor}}
106 }
107
108
109 finish_test