Debian Codegraph @

mysql-test/suite/innodb_fts/include/mecab.inc

raw
1 #
2 #-----------------------------------------------------------------------------
3 # wl6607 : InnoDB FULLTEXT SEARCH: CJK support (mecab parser)
4 # Adding FTS check with mecab parser
5 # - Table with mecab parser
6 # - Basic FTS query ( mainly BOOLEAN MODE check)
7 # - FTS index with single and two columns
8 # - external parser index using alter table
9 #
10 #------------------------------------------------------------------------------
11
12
13 call mtr.add_suppression("unknown variable 'loose_mecab_rc_file");
14 call mtr.add_suppression("Fulltext index charset 'big5' doesn't match mecab charset");
15 call mtr.add_suppression("Mecab: createModel.* failed: param.cpp");
16 call mtr.add_suppression("Plugin 'mecab' init function returned error");
17
18 -- disable_query_log
19
20 -- error 0, ER_CANT_INITIALIZE_UDF
21
22 # Restart with package dictionary
23 let $MYSQL_DATADIR=`select @@datadir`;
24 let MYSQL_BASEDIR=`select @@basedir`;
25 let MECABRC=$MYSQL_DATADIR/mecabrc;
26 let DICDIR=ipadic_$ipadic_charset;
27
28 # In standard layout libdir is
29 # $basedir/lib
30 # while RPMS (el and sles) uses
31 # $basedir/lib/mysql
32 # $basedir/lib64/mysql
33 #
34 # Fedora, Debian and Ubuntu uses system mecab
35 #
36 # Fedora has
37 # UTF-8 in %{_libdir}/mecab/dic/ipadic
38 # EUCJP in %{_libdir}/mecab/dic/ipadic-EUCJP
39 #
40 # Debian and Ubuntu have
41 # UTF-8 in /var/lib/mecab/dic/ipadic-utf8
42 # EUCJP in /var/lib/mecab/dic/ipadic
43 #
44 --perl
45 use strict;
46 use lib "lib/";
47 use My::Find;
48 my $dir = "$ENV{'DICDIR'}";
49 my $basedir = $ENV{'MYSQL_BASEDIR'};
50
51 my $dicdir = my_find_dir("$basedir",
52 ['lib/mecab/dic', # Bundled mecab
53 'lib/mysql/mecab/dic', # 32 bits RPMS
54 'lib64/mysql/mecab/dic'] , # 64 bits RPMS
55 ["$dir"], 1);
56
57 if ($dicdir eq '' && index($dir, 'utf-8') != -1) {
58 # Fedora
59 $dicdir = my_find_dir("/usr",
60 ['lib/mecab/dic', # System mecab 32 bits
61 'lib64/mecab/dic'], # System mecab 64 bits
62 ["ipadic"], 1);
63 # Debian/Ubuntu
64 if ($dicdir eq '') {
65 $dicdir = my_find_dir("/var",
66 ['lib/mecab/dic'], # System mecab
67 ["ipadic-utf8"], 1);
68 }
69 } elsif ($dicdir eq '' && index($dir, 'euc-jp') != -1) {
70 # Fedora
71 $dicdir = my_find_dir("/usr",
72 ['lib/mecab/dic/', # System mecab 32 bits
73 'lib64/mecab/dic/', ], # System mecab 64 bits
74 ["ipadic-EUCJP"], 1);
75 # Debian/Ubuntu
76 if ($dicdir eq '') {
77 $dicdir = my_find_dir("/var",
78 ['lib/mecab/dic'], # System mecab
79 ["ipadic"], 1);
80 }
81 }
82 open(my $rc, '>', "$ENV{'MECABRC'}") || die "Could not open $ENV{'MECABRC'}: $!";
83 print $rc "dicdir = ${dicdir}\n";
84 close $rc;
85 EOF
86
87 -- source include/shutdown_mysqld.inc
88 -- exec echo "restart: --loose_mecab_rc_file=$mecabrc $MECAB_OPT --innodb_ft_min_token_size=2" >$MYSQLTEST_VARDIR/tmp/mysqld.1.expect
89 -- enable_reconnect
90 -- source include/wait_until_connected_again.inc
91 -- disable_reconnect
92
93 -- disable_query_log
94
95 -- error 0, ER_CANT_INITIALIZE_UDF
96 eval INSTALL PLUGIN mecab SONAME '$MECAB';
97
98 let $mecab_charset=`SELECT variable_value FROM performance_schema.global_status WHERE VARIABLE_NAME='mecab_charset'`;
99
100 if ($mecab_charset == '') {
101 --remove_file $MYSQL_DATADIR/mecabrc
102 -- skip Test fail to load mecab parser, IPADIC files for $ipadic_charset might be missing.
103 }
104
105 --enable_query_log
106
107 SHOW STATUS LIKE 'mecab_charset';
108
109 SET NAMES utf8mb3;
110 eval CREATE TABLE page (
111 page_id int(8) unsigned NOT NULL AUTO_INCREMENT,
112 page_title varchar(255) CHARACTER SET $mysql_charset NOT NULL DEFAULT '',
113 PRIMARY KEY (page_id),
114 FULLTEXT KEY page_title (page_title) WITH PARSER mecab
115 ) ENGINE=InnoDB;
116
117 INSERT INTO page VALUES
118 (1,'レモナ'),
119 (2,'SDレモナ'),
120 (3,'レモナ'),
121 (4,'データベース管理システム'),
122 (5,'キム'),
123 (6,'鉄道100電車'),
124 (7,'100'),
125 (8,'ラフィタ・カスティージョ'),
126 (9,'ツル科_(Sibley)'),
127 (10,'ツル'),
128 (11,'鉄道');
129
130 SET GLOBAL innodb_ft_aux_table="test/page";
131 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE
132 ORDER BY WORD, DOC_ID, POSITION;
133 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE;
134
135 SHOW CREATE TABLE page;
136
137 SELECT * FROM page WHERE MATCH(page_title) AGAINST("レモナ" IN BOOLEAN MODE);
138 SELECT * FROM page WHERE MATCH(page_title) AGAINST("キムショウカン" IN BOOLEAN MODE);
139 SELECT * FROM page WHERE MATCH(page_title) AGAINST("100" IN BOOLEAN MODE);
140 SELECT * FROM page WHERE MATCH(page_title) AGAINST("ツル*" IN BOOLEAN MODE);
141 SELECT * FROM page WHERE MATCH(page_title) AGAINST("ラフィタ・カスティージョ");
142 SELECT * FROM page WHERE MATCH(page_title) AGAINST("ラフィタ・カスティージョ" IN BOOLEAN MODE);
143 SELECT * FROM page WHERE MATCH(page_title) AGAINST("+ラフィタ・カスティージョ +データベース" IN BOOLEAN MODE);
144 SELECT * FROM page WHERE MATCH(page_title) AGAINST("+ラフィタ・カスティージョ +(データベース)" IN BOOLEAN MODE);
145 SELECT * FROM page WHERE MATCH(page_title) AGAINST("-ラフィタ・カスティージョ +(データベース)" IN BOOLEAN MODE);
146 SELECT * FROM page WHERE MATCH(page_title) AGAINST("ラフィタ・カスティージョ 鉄道" IN BOOLEAN MODE);
147 SELECT * FROM page WHERE MATCH(page_title) AGAINST("<ラフィタ・カスティージョ >鉄道" IN BOOLEAN MODE);
148 SELECT * FROM page WHERE MATCH(page_title) AGAINST('鉄道');
149 SELECT * FROM page WHERE MATCH(page_title) AGAINST('鉄道' WITH QUERY EXPANSION);
150
151 eval ALTER TABLE page add column c1 varchar(100) CHARACTER SET $mysql_charset DEFAULT 'レモナ';
152 ALTER TABLE page ADD FULLTEXT INDEX idx1 (page_title,c1) WITH PARSER mecab;
153 SELECT * FROM page WHERE MATCH(page_title,c1) AGAINST('レモナ');
154 SELECT * FROM page WHERE MATCH(page_title,c1) AGAINST('-レモナ' IN BOOLEAN MODE);
155
156 DROP TABLE page;
157
158 # Test charset mismatch.
159 CREATE TABLE page (
160 page_id int(8) unsigned NOT NULL AUTO_INCREMENT,
161 page_title varchar(255) CHARACTER SET big5 NOT NULL DEFAULT '',
162 PRIMARY KEY (page_id)
163 )ENGINE=InnoDB;
164
165 INSERT INTO page VALUES (1,'日本');
166
167 --error ER_ERROR_ON_WRITE
168 CREATE FULLTEXT INDEX ft_idx ON page(page_title) WITH PARSER mecab;
169
170 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE
171 ORDER BY WORD, DOC_ID, POSITION;
172 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE;
173
174 INSERT INTO page VALUES (2,'日本');
175
176 --error ER_FT_MATCHING_KEY_NOT_FOUND
177 SELECT * FROM page WHERE MATCH(page_title) AGAINST('日本');
178
179 DROP TABLE page;
180
181 # Test utf8mb4
182 if ($mecab_charset == 'utf8mb4')
183 {
184 SET NAMES utf8mb4;
185
186 CREATE TABLE page (
187 page_id int(8) unsigned NOT NULL AUTO_INCREMENT,
188 page_title varchar(255) CHARACTER SET utf8mb4 NOT NULL DEFAULT '',
189 PRIMARY KEY (page_id),
190 FULLTEXT KEY page_title (page_title) WITH PARSER mecab
191 ) ENGINE=InnoDB;
192
193 # insert some emoji chars with 4 bytes.
194 INSERT INTO page VALUES
195 (1,'😃レモナ'),
196 (2,'鉄道😄'),
197 (3,'鉄道😊100電車');
198
199 SET GLOBAL innodb_ft_aux_table="test/page";
200 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE
201 ORDER BY WORD, DOC_ID, POSITION;
202 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE;
203
204 SHOW CREATE TABLE page;
205
206 SELECT * FROM page WHERE MATCH(page_title) AGAINST("レモナ" IN BOOLEAN MODE);
207 SELECT * FROM page WHERE MATCH(page_title) AGAINST('鉄道');
208 SELECT * FROM page WHERE MATCH(page_title) AGAINST('鉄道😄');
209 SELECT * FROM page WHERE MATCH(page_title) AGAINST('鉄道' WITH QUERY EXPANSION);
210
211 DROP TABLE page;
212 }
213
214 # Test eucjpms & cp932
215 if ($mysql_win_charset)
216 {
217 eval CREATE TABLE page (
218 page_id int(8) unsigned NOT NULL AUTO_INCREMENT,
219 page_title varchar(255) CHARACTER SET $mysql_win_charset NOT NULL DEFAULT '',
220 PRIMARY KEY (page_id),
221 FULLTEXT KEY page_title (page_title) WITH PARSER mecab
222 ) ENGINE=InnoDB;
223
224 # insert some eucjpms & cp932 chars.
225 INSERT INTO page VALUES
226 (1,'ⅰレモナ'),
227 (2,'鉄道ⅹ'),
228 (3,'鉄道ⅴ100電車');
229
230 SET GLOBAL innodb_ft_aux_table="test/page";
231 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE
232 ORDER BY WORD, DOC_ID, POSITION;
233 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE;
234
235 SHOW CREATE TABLE page;
236
237 SELECT * FROM page WHERE MATCH(page_title) AGAINST("レモナ" IN BOOLEAN MODE);
238 SELECT * FROM page WHERE MATCH(page_title) AGAINST('鉄道');
239 SELECT * FROM page WHERE MATCH(page_title) AGAINST('鉄道ⅴ');
240 SELECT * FROM page WHERE MATCH(page_title) AGAINST('鉄道' WITH QUERY EXPANSION);
241
242 DROP TABLE page;
243 }
244
245 SET GLOBAL innodb_ft_aux_table=default;
246
247 UNINSTALL PLUGIN mecab;
248 --remove_file $MYSQL_DATADIR/mecabrc