Debian Codegraph @

mysql-test/suite/group_replication/t/gr_crash_safe_master.test

raw
1 ################################################################################
2 # Test to check the behavior of Group Replication members when the crash
3 # happens on a member followed by a recovery on the same.
4 #
5 # Test:
6 # 0. The test requires two servers: M1 and M2.
7 # 1. With both the members are ONLINE, crash a member M1 after a transaction but
8 # before the commit stage using DEBUG POINT.
9 # 2. Restart the server M1. Data should ROLLBACK post restart.
10 # 3. Check that no data is replicated on M2.
11 # 4. Bring M1 ONLINE and check if group is working fine now.
12 # 5. Crash a member M1 before the data is committed to make crash for
13 # transaction using DEBUG POINT.
14 # 6. Restart the server M1. Data should be recovered post restart.
15 # 7. Check that data is replicated on M2.
16 # 8. Bring M1 ONLINE and check if group is working fine now.
17 # 9. Set DEBUG POINT after binlogging half of the transaction and then crash
18 # the member M1.
19 # 10. Restart the server M1. Data will not be recovered until GR start.
20 # 11. Bring M1 ONLINE and check data is recovered from M2.
21 # 12. Check if all members are running and group is working fine.
22 ################################################################################
23
24 --source include/not_crashrep.inc
25 # Don't test this under valgrind, memory leaks will occur
26 --source include/not_valgrind.inc
27 --source include/big_test.inc
28 --source include/have_debug.inc
29 --source include/force_restart.inc
30 --source include/have_group_replication_plugin.inc
31 --source include/group_replication.inc
32
33 SET sql_log_bin=0;
34 call mtr.add_suppression("Attempting backtrace");
35 call mtr.add_suppression("allocated tablespace *., old maximum was 0");
36 call mtr.add_suppression("Error in Log_event::read_log_event()");
37 call mtr.add_suppression("Buffered warning: Performance schema disabled");
38 SET sql_log_bin=1;
39
40 --let $_group_replication_local_address_server1= `SELECT @@GLOBAL.group_replication_local_address`
41 --let $_group_replication_group_seeds_server1= `SELECT @@GLOBAL.group_replication_group_seeds`
42 --let $member1_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
43 --let $old_debug = `select @@global.debug`
44
45
46 --echo # Test case1: Set DEBUG POINT before binlog to make
47 --echo # the first member crash for transaction
48 CREATE TABLE t1(i INT, a LONGBLOB, PRIMARY KEY(i)) ENGINE=INNODB;
49
50 BEGIN;
51 --let $rows= 3
52 WHILE($rows)
53 {
54 --eval INSERT INTO t1 VALUES ($rows, REPEAT('a',2));
55 dec $rows;
56 }
57
58 # Write file to make mysql-test-run.pl expect crash and restart
59 --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
60 --disable_query_log
61 --eval SET SESSION debug=CONCAT(IF(LENGTH('$old_debug') > 0, "$old_debug:", ""), "d,debug,info,enter,return,crash_commit_after_prepare")
62 --enable_query_log
63
64 # Run the crashing query
65 --error 2013
66 COMMIT;
67
68 --enable_reconnect
69 --echo # Restart the crashed member
70 --exec echo "restart:--group_replication_local_address=$_group_replication_local_address_server1 --group_replication_group_seeds=$_group_replication_group_seeds_server1" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
71 --source include/wait_until_connected_again.inc
72 --disable_reconnect
73 --let $rpl_server_number= 1
74 --source include/rpl_reconnect.inc
75
76 --let $assert_cond= `SELECT COUNT(*) = 0 FROM t1`
77 --let $assert_text= On server1, test the data will be rolled back after restart.
78 --source include/assert.inc
79
80 # Sync the second member with the first one.
81 --source include/rpl_sync.inc
82
83 --echo # On the second member, test replication will work fine, and the data
84 --echo # is not replicated
85 --let $diff_tables= server1:test.t1, server2:test.t1
86 --source include/diff_tables.inc
87
88 # Wait until group has only 1 member, that is, server1 crash is detected by
89 # the group.
90 --let $rpl_connection_name= server2
91 --source include/rpl_connection.inc
92
93 --let $group_replication_member_state= UNREACHABLE
94 --let $group_replication_member_id= $member1_uuid
95 --source include/gr_wait_for_member_state.inc
96
97 # unblock the group, since server1 has gone away and
98 # there were only two in the group - majority lost
99 --let $local_address_server2= `SELECT @@GLOBAL.group_replication_local_address`
100 --disable_query_log
101 # Reseting the group to server2 only
102 --eval SET GLOBAL group_replication_force_members= "$local_address_server2"
103 --enable_query_log
104
105 --let $group_replication_number_of_members= 1
106 --source include/gr_wait_for_number_of_members.inc
107
108 --let $rpl_connection_name= server1
109 --source include/rpl_connection.inc
110 --source include/start_group_replication.inc
111
112 # Check if the group is still working
113 CREATE TABLE temp (i INT PRIMARY KEY) engine=INNODB;
114 INSERT INTO temp VALUES (1);
115 --source include/rpl_sync.inc
116
117 --connection server2
118 INSERT INTO temp VALUES (2);
119 --source include/rpl_sync.inc
120
121 --let $diff_tables= server1:test.temp, server2:test.temp
122 --source include/diff_tables.inc
123
124 DROP TABLE t1;
125 DROP TABLE temp;
126 --source include/rpl_sync.inc
127
128
129 --echo # Test case2: Set DEBUG POINT after binlog, and before the data
130 --echo # is committed to make crash for transaction
131
132 --connection server1
133 CREATE TABLE t1(i INT, a LONGBLOB, PRIMARY KEY(i)) ENGINE=INNODB;
134
135 BEGIN;
136 --let $rows= 3
137 WHILE($rows)
138 {
139 --eval INSERT INTO t1 VALUES ($rows, REPEAT('a',2));
140 --dec $rows
141 }
142
143 # Write file to make mysql-test-run.pl expect crash and restart
144 --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
145 --disable_query_log
146 --eval SET SESSION debug=CONCAT(IF(LENGTH('$old_debug') > 0, "$old_debug:", ""), "d,debug,info,enter,return,crash_commit_after_log")
147 --enable_query_log
148
149 # Run the crashing query
150 --error 2013
151 COMMIT;
152
153 --source include/wait_until_disconnected.inc
154 --enable_reconnect
155 --echo # Restart the crashed member
156 --exec echo "restart:--group_replication_local_address=$_group_replication_local_address_server1 --group_replication_group_seeds=$_group_replication_group_seeds_server1" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
157 --source include/wait_until_connected_again.inc
158 --disable_reconnect
159 --let $rpl_server_number= 1
160 --source include/rpl_reconnect.inc
161
162 --let $assert_cond= `SELECT COUNT(*) = 3 FROM t1`
163 --let $assert_text= On server1, test the data will be recovered after the server1 restart.
164 --source include/assert.inc
165
166 --echo # On the second member, test that the replication is working fine, and
167 --echo # data gets replicated here.
168 --let $diff_tables= server1:test.t1, server2:test.t1
169 --source include/diff_tables.inc
170
171 # Wait until group has only 1 member, that is, server1 crash is detected by
172 # the group.
173 --let $rpl_connection_name= server2
174 --source include/rpl_connection.inc
175
176 --let $group_replication_member_state= UNREACHABLE
177 --let $group_replication_member_id= $member1_uuid
178 --source include/gr_wait_for_member_state.inc
179
180 # unblock the group, since server2 has gone away and
181 # there were only two in the group - majority lost
182 --let $local_address_server2= `SELECT @@GLOBAL.group_replication_local_address`
183 --disable_query_log
184 # Reseting the group to server2 only
185 --eval SET GLOBAL group_replication_force_members= "$local_address_server2"
186 --enable_query_log
187
188 --let $group_replication_number_of_members= 1
189 --source include/gr_wait_for_number_of_members.inc
190
191 --let $rpl_connection_name= server1
192 --source include/rpl_connection.inc
193 --source include/start_group_replication.inc
194
195 # Check if the group is still working
196 CREATE TABLE temp (i INT PRIMARY KEY) engine=INNODB;
197 INSERT INTO temp VALUES (1);
198 --source include/rpl_sync.inc
199
200 --connection server2
201 INSERT INTO temp VALUES (2);
202 --source include/rpl_sync.inc
203
204 --let $diff_tables= server1:test.temp, server2:test.temp
205 --source include/diff_tables.inc
206
207 DROP TABLE t1;
208 DROP TABLE temp;
209 --source include/rpl_sync.inc
210
211
212 # Test transaction with large data inserted
213 --connection server1
214 CREATE TABLE t1(i INT AUTO_INCREMENT PRIMARY KEY, a LONGBLOB) ENGINE=INNODB;
215
216 --echo # Test case3: Set DEBUG POINT after binlogging half of the transaction and
217 --echo # then crash the member and followed by restarting the member.
218 set AUTOCOMMIT=0;
219 BEGIN;
220 --let $rows= 24
221 WHILE($rows)
222 {
223 INSERT INTO t1 (a) VALUES ( REPEAT('a',6144));
224 --dec $rows
225 }
226 # Write file to make mysql-test-run.pl expect crash and restart
227 --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
228 --disable_query_log
229 --eval SET SESSION debug=CONCAT(IF(LENGTH('$old_debug') > 0, "$old_debug:", ""), "d,debug,info,enter,return,half_binlogged_transaction")
230 --enable_query_log
231 # Run the crashing query
232 --error 2013
233 COMMIT;
234
235 --source include/wait_until_disconnected.inc
236 --enable_reconnect
237 --echo # Restart the crashed member
238 --exec echo "restart:--group_replication_local_address=$_group_replication_local_address_server1 --group_replication_group_seeds=$_group_replication_group_seeds_server1" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
239 --source include/wait_until_connected_again.inc
240 --disable_reconnect
241 --let $rpl_server_number= 1
242 --source include/rpl_reconnect.inc
243
244 --echo # Test the data will not be recovered successfully
245 --echo # after the crashed first member restarts.
246
247 --let $assert_cond= `SELECT COUNT(*) = 0 FROM t1`
248 --let $assert_text= On server1, test the data will be recovered after the server1 restart.
249 --source include/assert.inc
250
251 # Wait until group has only 1 member, that is, server1 crash is detected by
252 # the group.
253 --let $rpl_connection_name= server2
254 --source include/rpl_connection.inc
255
256 --let $group_replication_member_state= UNREACHABLE
257 --let $group_replication_member_id= $member1_uuid
258 --source include/gr_wait_for_member_state.inc
259
260 # unblock the group, since server2 has gone away and
261 # there were only two in the group - majority lost
262 --let $local_address_server2= `SELECT @@GLOBAL.group_replication_local_address`
263 --disable_query_log
264 # Reseting the group to server2 only
265 --eval SET GLOBAL group_replication_force_members= "$local_address_server2"
266 --enable_query_log
267
268 --let $group_replication_number_of_members= 1
269 --source include/gr_wait_for_number_of_members.inc
270
271 --let $rpl_connection_name= server1
272 --source include/rpl_connection.inc
273 --source include/start_group_replication.inc
274
275 # On restarting group replication on the first member it will be getting the transactions
276 # from a donor and try to get in sync with the other members in the group.
277
278 --let $diff_tables= server1:test.t1, server2:test.t1
279 --source include/diff_tables.inc
280
281 --connection server2
282 --let $assert_cond= `SELECT COUNT(*) = 24 FROM t1`
283 --let $assert_text= The table has 24 rows after the restart.
284 --source include/assert.inc
285
286 # Check if the group is still running
287 --connection server1
288
289 CREATE TABLE temp (i INT PRIMARY KEY) engine=INNODB;
290 INSERT INTO temp VALUES (1);
291 --source include/rpl_sync.inc
292
293 --connection server2
294 INSERT INTO temp VALUES (2);
295 --source include/rpl_sync.inc
296
297 --let $diff_tables= server1:test.temp, server2:test.temp
298 --source include/diff_tables.inc
299
300 DROP TABLE t1;
301 DROP TABLE temp;
302 --source include/group_replication_end.inc