Debian Codegraph @

mysql-test/suite/group_replication/t/gr_recovery_no_more_donors.test

raw
1 ################################################################################
2 # This test evaluates the limit case where no more donors remain in the group.
3 # It is divided into 2 separated but similar phases.
4 # Phase 1:
5 # A node joins but all the members/donors leave.
6 # Recovery fails and the member leaves
7 # Phase 2:
8 # A node joins and the only donor leaves remaining another member on recovery.
9 # Recovery should fail on both members.
10 #
11 # Test:
12 # 0. The test requires three servers: M1, M2 and M3.
13 # 1. Setup a new 2 member group. Bootstrap start GR on M1. Insert some data
14 # for recovery on M1. Start GR on M2. Block the applier on both servers to
15 # block recovery
16 # Phase 1:
17 # 2. Start recovery on a new member M3. Identify the donor, stop GR on donor.
18 # Identify the failover donor, stop GR on the last donor.
19 # 3. Recovery should fail on the joiner M3. Check M3 goes in ERROR state.
20 # Phase 2:
21 # 4. Bootstrap start GR on M1. Block the applier on M1. Start GR on M2. Check
22 # M2 is in recovering state.
23 # 5. To avoid issues with simultaneous exits from the group set DEBUG in M3.
24 # START GR on M3. Check M3 is in recovering state.
25 # 6. Stop GR on M1. Recovery should fail on both joiners i.e. check M2 goes
26 # in ERROR state, send SIGNAL on M3, check M3 goes in ERROR state.
27 # 7. Clean up.
28 ################################################################################
29
30 --source include/big_test.inc
31 --source include/have_debug.inc
32 --source include/have_debug_sync.inc
33 --let $group_replication_group_name=0645c380-262c-11e5-867f-0800200c9a66
34 --source include/have_group_replication_plugin.inc
35 --let $rpl_skip_group_replication_start= 1
36 --let $rpl_server_count= 3
37 --source include/group_replication.inc
38
39 --echo #
40 --echo # Setup a new 2 member group
41 --echo #
42
43 --connection server1
44
45 SET SESSION sql_log_bin= 0;
46 call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': Relay log read failure: Could not parse relay log event entry.*");
47 call mtr.add_suppression("The applier thread execution was aborted. Unable to process more transactions, this member will now leave the group.");
48 call mtr.add_suppression("Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.");
49 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
50 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
51 SET SESSION sql_log_bin= 1;
52
53 #extract server uuid for donor calculations below
54 --let $server1_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
55 --source include/start_and_bootstrap_group_replication.inc
56
57 #insert some data
58 CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB;
59 INSERT INTO t1 VALUES (1);
60
61 --connection server2
62
63 SET SESSION sql_log_bin= 0;
64 call mtr.add_suppression("The member is leaving a group without being on one");
65 call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': Relay log read failure: Could not parse relay log event entry.*");
66 call mtr.add_suppression("The applier thread execution was aborted. Unable to process more transactions, this member will now leave the group.");
67 call mtr.add_suppression("Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.");
68 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
69 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
70 SET SESSION sql_log_bin= 1;
71
72 #extract server uuid for donor calculations below
73 --let $server2_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
74 --source include/start_group_replication.inc
75
76 #Block the applier on both servers to block recovery
77 SET @@GLOBAL.DEBUG='+d,block_applier_updates';
78
79 --connection server1
80
81 SET @@GLOBAL.DEBUG='+d,block_applier_updates';
82
83 --echo #
84 --echo # Phase 1:
85 --echo # Start recovery on a new member and kill both donors
86 --echo #
87
88 --connection server3
89 SET SESSION sql_log_bin= 0;
90 call mtr.add_suppression("No suitable donor found, group replication recovery aborting.");
91 call mtr.add_suppression("Fatal error during the incremental recovery process of Group Replication. The server will leave the group.");
92 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
93 call mtr.add_suppression("All donors left. Aborting group replication incremental recovery.");
94 call mtr.add_suppression("Maximum number of retries when trying to connect to a donor reached. Aborting group replication incremental recovery.");
95 call mtr.add_suppression("The member is leaving a group without being on one");
96 call mtr.add_suppression("The member is already leaving or joining a group.");
97 call mtr.add_suppression("Error leaving the group");
98 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
99 SET SESSION sql_log_bin= 1;
100
101 --let $group_replication_start_member_state= RECOVERING
102 --source include/start_group_replication.inc
103
104 --echo # Find if server 1 or 2 is the donor
105
106 #give it time to connect
107 --sleep 5
108
109 #Find what member is the donor
110 --let $donor_id= 1
111 --let $failover= 2
112 --let $failover_uuid= $server2_uuid
113 if (`SELECT COUNT(*)=1 FROM performance_schema.replication_connection_status WHERE channel_name='group_replication_recovery' AND service_state='ON' AND source_uuid='$server2_uuid'`)
114 {
115 --let $donor_id= 2
116 --let $failover= 1
117 --let $failover_uuid= $server1_uuid
118 }
119
120 --echo # Stop group replication on the donor making it leave the group
121 --connection server$donor_id
122 SET DEBUG_SYNC = "now WAIT_FOR applier_read_blocked";
123 SET @@GLOBAL.DEBUG='-d,block_applier_updates';
124 SET @@GLOBAL.DEBUG='+d,force_sql_thread_error';
125 SET DEBUG_SYNC = "now SIGNAL resume_applier_read";
126 --let $group_replication_member_state= ERROR
127 --source include/gr_wait_for_member_state.inc
128 SET @@GLOBAL.DEBUG='-d,force_sql_thread_error';
129 --source include/stop_group_replication.inc
130
131 # Wait for the joiner to failover to the other member
132 --connection server3
133 --let $wait_condition= SELECT COUNT(*)=1 FROM performance_schema.replication_connection_status WHERE channel_name='group_replication_recovery' AND service_state='ON' AND source_uuid="$failover_uuid";
134 --source include/wait_condition.inc
135
136 --echo # Stop group replication on the failover donor making it leave the group
137 --connection server$failover
138 SET DEBUG_SYNC = "now WAIT_FOR applier_read_blocked";
139 SET @@GLOBAL.DEBUG='-d,block_applier_updates';
140 SET @@GLOBAL.DEBUG='+d,force_sql_thread_error';
141 SET DEBUG_SYNC = "now SIGNAL resume_applier_read";
142 --let $group_replication_member_state= ERROR
143 --source include/gr_wait_for_member_state.inc
144 SET @@GLOBAL.DEBUG='-d,force_sql_thread_error';
145 --source include/stop_group_replication.inc
146
147 --echo # Recovery should fail on the joiner
148 --connection server3
149
150 --let $group_replication_member_state= ERROR
151 --source include/gr_wait_for_member_state.inc
152
153 --source include/stop_group_replication.inc
154
155 --echo #
156 --echo # Phase 2
157 --echo # Have 2 members on recovery and kill the donor
158 --echo #
159
160 #sleep so a different view id is generated
161 --sleep 1
162
163 --connection server1
164 --source include/start_and_bootstrap_group_replication.inc
165
166 # Block the applier on server 1 so view change event wont be written and recovery blocks on other servers
167 SET @@GLOBAL.DEBUG='+d,block_applier_updates';
168
169 --echo # Start recovery on the other 2 members
170
171 --connection server2
172
173 SET SESSION sql_log_bin= 0;
174 call mtr.add_suppression("No suitable donor found, group replication recovery aborting.");
175 call mtr.add_suppression("Fatal error during the incremental recovery process of Group Replication. The server will leave the group.");
176 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
177 call mtr.add_suppression("Maximum number of retries when trying to connect to a donor reached. Aborting group replication incremental recovery.");
178 call mtr.add_suppression("The member is leaving a group without being on one");
179 call mtr.add_suppression("The member is already leaving or joining a group.");
180 call mtr.add_suppression("Error leaving the group");
181 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
182 SET SESSION sql_log_bin= 1;
183
184 --let $server2_reconnect_interval= `SELECT @@GLOBAL.group_replication_recovery_reconnect_interval;`
185 --let $server2_retry_count= `SELECT @@GLOBAL.group_replication_recovery_retry_count;`
186
187 SET GLOBAL group_replication_recovery_reconnect_interval= 1;
188 SET GLOBAL group_replication_recovery_retry_count= 2;
189
190 --let $group_replication_start_member_state= RECOVERING
191 --source include/start_group_replication.inc
192
193 --connection server3
194
195 # Here to avoid issues with simultaneous exits from the group (serve 2 and 3)
196 SET @debug_save= @@GLOBAL.DEBUG;
197 SET @@GLOBAL.DEBUG='d,recovery_thread_wait_before_finish';
198
199 --let $server3_reconnect_interval= `SELECT @@GLOBAL.group_replication_recovery_reconnect_interval;`
200 --let $server3_retry_count= `SELECT @@GLOBAL.group_replication_recovery_retry_count;`
201
202 SET GLOBAL group_replication_recovery_reconnect_interval= 1;
203 SET GLOBAL group_replication_recovery_retry_count= 2;
204
205 --let $group_replication_start_member_state= RECOVERING
206 --source include/start_group_replication.inc
207
208 --echo # Stop group replication on the donor making it leave the group
209 --connection server1
210 SET DEBUG_SYNC = "now WAIT_FOR applier_read_blocked";
211 SET @@GLOBAL.DEBUG='-d,block_applier_updates';
212 SET @@GLOBAL.DEBUG='+d,force_sql_thread_error';
213 SET DEBUG_SYNC = "now SIGNAL resume_applier_read";
214 --let $group_replication_member_state= ERROR
215 --source include/gr_wait_for_member_state.inc
216 SET @@GLOBAL.DEBUG='-d,force_sql_thread_error';
217 --source include/stop_group_replication.inc
218
219 --echo # Recovery should fail on both joiners
220
221 --connection server2
222
223 --let $group_replication_member_state= ERROR
224 --source include/gr_wait_for_member_state.inc
225
226 --source include/stop_group_replication.inc
227
228 --connection server3
229
230 SET DEBUG_SYNC= "now SIGNAL signal.recovery_end";
231
232 --let $group_replication_member_state= ERROR
233 --source include/gr_wait_for_member_state.inc
234
235 --source include/stop_group_replication.inc
236
237 --echo #
238 --echo # Cleaning up
239 --echo #
240
241 #sleep so a different view id is generated
242 --sleep 1
243
244 --connection server1
245 SET DEBUG_SYNC= 'RESET';
246 --source include/start_and_bootstrap_group_replication.inc
247
248 --connection server2
249 SET DEBUG_SYNC= 'RESET';
250
251 --disable_query_log
252 --eval SET GLOBAL group_replication_recovery_reconnect_interval= $server2_reconnect_interval
253 --eval SET GLOBAL group_replication_recovery_retry_count= $server2_retry_count
254 --enable_query_log
255
256 --source include/start_group_replication.inc
257
258 --let $assert_text= On the recovered member, the table should exist and have 1 elements
259 --let $assert_cond= [SELECT COUNT(*) FROM t1] = 1;
260 --source include/assert.inc
261
262 --connection server3
263
264 SET @@GLOBAL.DEBUG= @debug_save;
265
266 --disable_query_log
267 --eval SET GLOBAL group_replication_recovery_reconnect_interval= $server3_reconnect_interval
268 --eval SET GLOBAL group_replication_recovery_retry_count= $server3_retry_count
269 --enable_query_log
270
271 --source include/start_group_replication.inc
272
273 --let $assert_text= On the recovered member, the table should exist and have 1 elements
274 --let $assert_cond= [SELECT COUNT(*) FROM t1] = 1;
275 --source include/assert.inc
276
277 --connection server1
278
279 DROP TABLE t1;
280
281 --source include/group_replication_end.inc