| 1 |
# 2016 Jan 15 |
| 2 |
# |
| 3 |
# The author disclaims copyright to this source code. In place of |
| 4 |
# a legal notice, here is a blessing: |
| 5 |
# |
| 6 |
# May you do good and not evil. |
| 7 |
# May you find forgiveness for yourself and forgive others. |
| 8 |
# May you share freely, never taking more than you give. |
| 9 |
# |
| 10 |
#************************************************************************* |
| 11 |
# |
| 12 |
|
| 13 |
source [file join [file dirname [info script]] fts5_common.tcl] |
| 14 |
ifcapable !fts5||!fts3 { finish_test ; return } |
| 15 |
set ::testprefix fts5tok2 |
| 16 |
|
| 17 |
sqlite3_fts5_register_fts5tokenize db |
| 18 |
|
| 19 |
#------------------------------------------------------------------------- |
| 20 |
# Simple test cases. Using the default (ascii) tokenizer. |
| 21 |
# |
| 22 |
do_execsql_test 1.0 { |
| 23 |
CREATE VIRTUAL TABLE t5 USING fts5tokenize(unicode61); |
| 24 |
CREATE VIRTUAL TABLE t3 USING fts3tokenize(unicode61); |
| 25 |
} |
| 26 |
|
| 27 |
do_test 1.1 { |
| 28 |
array unset -nocomplain A |
| 29 |
|
| 30 |
for {set i 1} {$i < 65536} {incr i} { |
| 31 |
set input [format "abc%cxyz" $i] |
| 32 |
set expect [execsql { |
| 33 |
SELECT input, token, start, end FROM t3 WHERE input=$input |
| 34 |
}] |
| 35 |
|
| 36 |
incr A([llength $expect]) |
| 37 |
|
| 38 |
set res [execsql { |
| 39 |
SELECT input, token, start, end FROM t5($input) |
| 40 |
}] |
| 41 |
if {$res != $expect} {error "failed at i=$i"} |
| 42 |
} |
| 43 |
} {} |
| 44 |
|
| 45 |
do_test 1.1.nTokenChars=$A(4).nSeparators=$A(8) {} {} |
| 46 |
|
| 47 |
finish_test |