Debian Codegraph @

mysql-test/suite/group_replication/t/gr_crash_recovery_server.test

raw
1 ################################################################################
2 # This test verifies whether the group_replication works fine if the
3 # server is killed during its recovery phase.
4 #
5 # Test:
6 # 0. The test requires three servers: M1,M2 and M3.
7 # 1. Start GR on servers M1 and M2.
8 # 2. Execute some operations on the group through server M1
9 # using procedure dml_operations.
10 # 3. Join M3 to the group and wait until the server is in Recovery.
11 # 4. start executing DML operations on the group through server1 to
12 # ensure that the cached transactions are present at the recovery
13 # server when it gets killed.
14 # 5. Kill and restart the server M3 during phase-1 of recovery.
15 # 6. Again Join the restarted member to the group
16 # 7. Wait until the recovery phase-1 of the server M3 ends.
17 # 8. Start executing the DML transactions on the group through server1
18 # so that the cached transaction will be added on the server M3.
19 # 9. Kill and restart the server M3 during phase-2 of recovery.
20 # 10.Wait until all the servers are online.
21 # 11.Verify that all the members have same data.
22 # 12.Cleanup
23 ################################################################################
24
25 # This test does crashes servers, thence we skip it on valgrind.
26 --source include/not_valgrind.inc
27 --source include/big_test.inc
28 --source include/force_restart.inc
29
30 --source include/set_privilege_checks_user_as_system_user.inc
31 --source include/have_group_replication_plugin.inc
32 --let $rpl_server_count= 3
33 --let $rpl_skip_group_replication_start= 1
34 --source include/group_replication.inc
35
36 # START GR on Two servers M1 and M2
37 --let $rpl_connection_name= server1
38 --source include/rpl_connection.inc
39
40 SET sql_log_bin=0;
41 CREATE TABLE t1(a int primary key);
42 SET sql_log_bin=1;
43
44 --source include/start_and_bootstrap_group_replication.inc
45
46 --let $rpl_connection_name= server2
47 --source include/rpl_connection.inc
48
49 SET sql_log_bin=0;
50 CREATE TABLE t1(a int primary key);
51 SET sql_log_bin=1;
52
53 --source include/start_group_replication.inc
54
55 # Execute the operations on the group through server1 so
56 # that when M3 joins it will be in recovery for some time.
57
58 --let $rpl_connection_name= server1
59 --source include/rpl_connection.inc
60
61 # Create the procedure to perform dml operations
62 delimiter $$;
63 CREATE PROCEDURE dml_operations(IN p INT,IN q INT)
64 BEGIN
65 declare x INT;
66 set x=p;
67 while x<q do
68 insert into t1 values (x);
69 update t1 set a=x+400 where a=x;
70 delete from t1 where a<420;
71 set x=x+1;
72 end while;
73 end$$
74 delimiter ;$$
75 --echo
76
77 --echo ----call procedure----
78 call dml_operations(1,100);
79 --echo
80
81 # JOIN M3 to the group
82 --let $rpl_connection_name= server3
83 --source include/rpl_connection.inc
84
85 set sql_log_bin=0;
86 --disable_query_log
87 call mtr.add_suppression(".*Replica SQL for channel 'group_replication_applier': ... The replica coordinator and worker threads are stopped, possibly leaving data in inconsistent state*");
88 call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] Database page corruption or a failed file read of page");
89 call mtr.add_suppression("There was an error when connecting to the donor server. Please check that group_replication_recovery channel credentials and all MEMBER_HOST column.*");
90 call mtr.add_suppression("Replica I/O for channel 'group_replication_recovery': Fatal error: Invalid (empty) username .*Connection attempt terminated. Error_code: MY-013117");
91 call mtr.add_suppression(".*For details please check performance_schema.replication_connection_status table and error log messages of Replica I/O for channel group_replication_recovery.");
92 --enable_query_log
93 CREATE TABLE t1(a int primary key);
94 SET sql_log_bin=1;
95 LOCK TABLES t1 READ;
96
97 --let $rpl_connection_name= server_3
98 --source include/rpl_connection.inc
99 --let $group_replication_start_member_state= RECOVERING
100 --source include/start_group_replication.inc
101
102 --let $rpl_connection_name= server1
103 --source include/rpl_connection.inc
104
105 --echo ----call procedure----
106 send call dml_operations(100,200);
107 --echo
108
109 --let $rpl_connection_name= server_1
110 --source include/rpl_connection.inc
111
112 # Wait to ensure that some cached transactions will be present at the time
113 # of kill of the recovery server.
114 --let $wait_timeout= 200
115 --let $wait_condition= SELECT MAX(a) > 520 FROM t1
116 --source include/wait_condition.inc
117
118 --let $rpl_connection_name= server3
119 --source include/rpl_connection.inc
120 UNLOCK TABLES;
121
122 # sleep for sometime so that some transactions executes on server3.
123 sleep 2;
124
125 # Kill server3
126 --echo # killing
127 --let $group_replication_local_address= `SELECT @@GLOBAL.group_replication_local_address`
128 --let $group_replication_group_seeds= `SELECT @@GLOBAL.group_replication_group_seeds`
129 --let $group_replication_comm_stack= `SELECT @@GLOBAL.group_replication_communication_stack`
130 --let $restart_parameters=restart:--group_replication_local_address=$group_replication_local_address --group_replication_group_seeds=$group_replication_group_seeds --group_replication_group_name=$group_replication_group_name --group_replication_communication_stack=$group_replication_comm_stack
131 --replace_result $group_replication_local_address GROUP_REPLICATION_LOCAL_ADDRESS $group_replication_group_seeds GROUP_REPLICATION_GROUP_SEEDS $group_replication_group_name GROUP_REPLICATION_GROUP_NAME $group_replication_comm_stack GROUP_REPLICATION_COMMUNICATION_STACK
132
133 --source include/kill_and_restart_mysqld.inc
134 --echo # restarting
135
136 # Needed as we are not using rpl_restart_server.inc
137 --let $rpl_server_number= 3
138 --source include/rpl_reconnect.inc
139
140 --let $rpl_connection_name= server2
141 --source include/rpl_connection.inc
142
143 # Wait until group has only 2 members, that is, server3's death is detected by
144 # the group.
145 --echo # check that there are 2 mebers in the group
146 --let $group_replication_number_of_members= 2
147 --source include/gr_wait_for_number_of_members.inc
148
149 --let $rpl_connection_name= server1
150 --source include/rpl_connection.inc
151 reap;
152
153 --echo ----call procedure----
154 call dml_operations(200,300);
155 --echo
156
157 # Start GR on server M3
158 --let $rpl_connection_name= server3
159 --source include/rpl_connection.inc
160 LOCK TABLES t1 READ;
161
162 --let $rpl_connection_name= server_3
163 --source include/rpl_connection.inc
164 start group_replication;
165
166 --let $group_replication_member_state= RECOVERING
167 --source include/gr_wait_for_member_state.inc
168
169 --let $rpl_connection_name= server1
170 --source include/rpl_connection.inc
171
172 --echo ----call procedure----
173 send call dml_operations(300,400);
174 --echo
175
176 --let $rpl_connection_name= server3
177 --source include/rpl_connection.inc
178 UNLOCK TABLES;
179
180 # Wait until Recovery phase1 ends.(until the service_state of group_replication_recovery goes to OFF state.)
181 --let $rpl_connection_name= server3
182 --source include/rpl_connection.inc
183
184 --let $wait_timeout=300
185 --let $wait_condition= SELECT COUNT(*)=1 FROM performance_schema.replication_connection_status WHERE channel_name='group_replication_recovery' AND service_state='OFF'
186 --source include/wait_condition.inc
187
188 sleep 1;
189
190 # Kill the recovery server
191 # kill server M3
192 --echo # killing
193 --let $restart_parameters=restart:--group_replication_local_address=$group_replication_local_address --group_replication_group_seeds=$group_replication_group_seeds --group_replication_group_name=$group_replication_group_name --group_replication_communication_stack=$group_replication_comm_stack
194 --let $group_replication_comm_stack= `SELECT @@GLOBAL.group_replication_communication_stack`
195 --replace_result $group_replication_local_address GROUP_REPLICATION_LOCAL_ADDRESS $group_replication_group_seeds GROUP_REPLICATION_GROUP_SEEDS $group_replication_group_name GROUP_REPLICATION_GROUP_NAME $group_replication_comm_stack GROUP_REPLICATION_COMMUNICATION_STACK
196 --source include/kill_and_restart_mysqld.inc
197 --echo # restarting
198
199 # Needed as we are not using rpl_restart_server.inc
200 --let $rpl_server_number= 3
201 --source include/rpl_reconnect.inc
202
203 --let $rpl_connection_name= server2
204 --source include/rpl_connection.inc
205
206 # Wait until group has only 2 members, that is, recovery server's death is detected by
207 # the group.
208 --echo # check that there are 2 mebers in a group
209 --let $group_replication_number_of_members= 2
210 --source include/gr_wait_for_number_of_members.inc
211
212 --let $rpl_connection_name= server1
213 --source include/rpl_connection.inc
214 reap;
215
216 # Start GR on killed and restarted recovery server.
217 --let $rpl_connection_name= server3
218 --source include/rpl_connection.inc
219
220 # Setting recovery user to avoid sporadic failures due to empty user.
221 --disable_warnings
222 RESET SLAVE FOR CHANNEL 'group_replication_recovery';
223 CHANGE REPLICATION SOURCE TO SOURCE_USER= 'root' FOR CHANNEL 'group_replication_recovery';
224 --enable_warnings
225 --let $wait_timeout= 300
226 --source include/start_group_replication.inc
227
228 --source include/rpl_sync.inc
229
230 # Verify that table t1 has same data on all the servers.
231 --let $diff_tables=server1:t1, server2:t1, server3:t1
232 --source include/diff_tables.inc
233
234 # Clean up.
235 drop table t1;
236 drop procedure dml_operations;
237 --let $skip_restore_connection= 0
238 --source include/group_replication_end.inc