Debian Codegraph @

ext/fts5/test/fts5unicode.test

raw
1 # 2014 Dec 20
2 #
3 # The author disclaims copyright to this source code. In place of
4 # a legal notice, here is a blessing:
5 #
6 # May you do good and not evil.
7 # May you find forgiveness for yourself and forgive others.
8 # May you share freely, never taking more than you give.
9 #
10 #***********************************************************************
11 #
12 # Tests focusing on the fts5 tokenizers
13 #
14
15 source [file join [file dirname [info script]] fts5_common.tcl]
16 set testprefix fts5unicode
17
18 # If SQLITE_ENABLE_FTS5 is not defined, omit this file.
19 ifcapable !fts5 {
20 finish_test
21 return
22 }
23
24 proc tokenize_test {tn tokenizer input output} {
25 uplevel [list do_test $tn [subst -nocommands {
26 set ret {}
27 foreach {z s e} [sqlite3_fts5_tokenize db {$tokenizer} {$input}] {
28 lappend ret [set z]
29 }
30 set ret
31 }] [list {*}$output]]
32 }
33
34 foreach {tn t} {1 ascii 2 unicode61} {
35 tokenize_test 1.$tn.0 $t {A B C D} {a b c d}
36 tokenize_test 1.$tn.1 $t {May you share freely,} {may you share freely}
37 tokenize_test 1.$tn.2 $t {..May...you.shAre.freely} {may you share freely}
38 tokenize_test 1.$tn.3 $t {} {}
39 }
40
41 #-------------------------------------------------------------------------
42 # Check that "unicode61" really is the default tokenizer.
43 #
44 do_execsql_test 2.0 "
45 CREATE VIRTUAL TABLE t1 USING fts5(x);
46 CREATE VIRTUAL TABLE t2 USING fts5(x, tokenize = unicode61);
47 CREATE VIRTUAL TABLE t3 USING fts5(x, tokenize = ascii);
48 INSERT INTO t1 VALUES('\xC0\xC8\xCC');
49 INSERT INTO t2 VALUES('\xC0\xC8\xCC');
50 INSERT INTO t3 VALUES('\xC0\xC8\xCC');
51 "
52 do_execsql_test 2.1 "
53 SELECT 't1' FROM t1 WHERE t1 MATCH '\xE0\xE8\xEC';
54 SELECT 't2' FROM t2 WHERE t2 MATCH '\xE0\xE8\xEC';
55 SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC';
56 " {t1 t2}
57
58 #-------------------------------------------------------------------------
59 # Check that codepoints that require 4 bytes to store in utf-8 (those that
60 # require 17 or more bits to store).
61 #
62
63 unset -nocomplain A B C D
64 set A [db one {SELECT char(0x1F75E)}] ;# Type So
65 set B [db one {SELECT char(0x1F5FD)}] ;# Type So
66 set C [db one {SELECT char(0x2F802)}] ;# Type Lo
67 set D [db one {SELECT char(0x2F808)}] ;# Type Lo
68
69 do_execsql_test 3.0 "
70 CREATE VIRTUAL TABLE xyz USING fts5(x,
71 tokenize = \"unicode61 separators '$C' tokenchars '$A'\"
72 );
73 CREATE VIRTUAL TABLE xyz_v USING fts5vocab(xyz, row);
74
75 INSERT INTO xyz VALUES('$A$B$C$D');
76 "
77
78 do_execsql_test 3.1 {
79 SELECT * FROM xyz_v;
80 } [list $A 1 1 $D 1 1]
81
82
83
84
85
86 finish_test