Debian Codegraph @

mysql-test/suite/group_replication/t/gr_clone_applier_stop.test

raw
1 ################################################################################
2 # This test confirms that:
3 # 1: On applier failure clone does not start.
4 # 2: During clone group_replication_applier SQL thread is OFF.
5 # 3: If clone fails group_replication_applier is restarted.
6 #
7 # Test:
8 # 0. The test requires two servers: M1 and M2
9 # 1. Bootstrap M1 and install clone plugin.
10 # 2. Setup clone on M2.
11 # 3. Force clone. Force applier failure on M2.
12 # Start GR on M2 fails. Applier fails before clone stops applier.
13 # Assert clone never starts.
14 # 4. Reset debug points for applier failures.
15 # Restart GR on M2.
16 # Assert clone starts and group_replication_applier SQL thread is OFF.
17 # 5. Block applier on M2.
18 # Uninstall clone on M1 so that clone fails.
19 # Create some transactions on M1 to create applier backlog on M2.
20 # 6. Stop GR on M2 without committing the received transactions.
21 # Create transactions on M1 for M2 to clone.
22 # 7. Start GR on M2.
23 # Clone will fail and incremental recovery will start.
24 # Applier will be OFF till clone failure is detected.
25 # 8. Cleanup.
26 ################################################################################
27
28 --source include/big_test.inc
29 --source include/have_debug_sync.inc
30 --source include/have_mysqld_monitoring_process.inc
31 --source include/have_clone_plugin.inc
32 --source include/force_restart.inc
33 --source include/have_group_replication_plugin.inc
34 --let $rpl_skip_group_replication_start= 1
35 --source include/group_replication.inc
36
37 # Validate plugins
38 --let plugins = CLONE_PLUGIN,GROUP_REPLICATION
39 --source include/check_plugin_dir.inc
40
41 --echo
42 --echo # 1. Bootstrap M1 and install clone plugin.
43 --echo
44 --source include/start_and_bootstrap_group_replication.inc
45 --replace_result $CLONE_PLUGIN CLONE_PLUGIN
46 --eval INSTALL PLUGIN clone SONAME '$CLONE_PLUGIN'
47
48 CREATE TABLE t1 (a INT PRIMARY KEY);
49 INSERT INTO t1 values (1);
50 INSERT INTO t1 values (2);
51
52 --echo
53 --echo # 2. Setup clone on M2.
54 --echo
55
56 --let $rpl_connection_name= server2
57 --source include/rpl_connection.inc
58
59 --let $member2_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
60 --let $_group_replication_local_address= `SELECT @@GLOBAL.group_replication_local_address`
61 --let $_group_replication_group_seeds= `SELECT @@GLOBAL.group_replication_group_seeds`
62 --let $_group_replication_comm_stack= `SELECT @@GLOBAL.group_replication_communication_stack`
63
64 --let $plugin_list= $GROUP_REPLICATION
65 --source include/spawn_monitoring_process.inc
66
67 --disable_query_log
68 --eval SET PERSIST group_replication_group_name= "$group_replication_group_name"
69 --eval SET PERSIST group_replication_local_address= "$_group_replication_local_address"
70 --eval SET PERSIST group_replication_group_seeds= "$_group_replication_group_seeds"
71 --eval SET PERSIST group_replication_communication_stack= "$_group_replication_comm_stack"
72 SET PERSIST group_replication_start_on_boot= ON;
73 --enable_query_log
74 --replace_result $CLONE_PLUGIN CLONE_PLUGIN
75 --eval INSTALL PLUGIN clone SONAME '$CLONE_PLUGIN'
76
77 --echo
78 --echo # 3. Force clone. Force applier failure on M2.
79 --echo # Start GR on M2 fails. Applier fails before clone stops applier.
80 --echo # Assert clone never starts.
81 --echo
82
83 --let $rpl_connection_name= server2
84 --source include/rpl_connection.inc
85 --let $_group_replication_threshold_save= `SELECT @@GLOBAL.group_replication_clone_threshold`
86 SET GLOBAL group_replication_clone_threshold= 1;
87 --replace_result $group_replication_group_name GROUP_NAME
88 --eval SET GLOBAL group_replication_group_name= "$group_replication_group_name"
89 SET @@GLOBAL.DEBUG= '+d,force_sql_thread_error';
90 SET @@GLOBAL.DEBUG='+d,gr_clone_before_applier_stop';
91
92 --send START GROUP_REPLICATION;
93
94 --let $rpl_connection_name= server_2
95 --source include/rpl_connection.inc
96
97 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_applier_status WHERE CHANNEL_NAME="group_replication_applier" AND SERVICE_STATE = "OFF"
98 --source include/wait_condition_or_abort.inc
99
100 SET DEBUG_SYNC = 'now SIGNAL applier_stopped';
101 SET @@GLOBAL.DEBUG='-d,gr_clone_before_applier_stop';
102
103 --let $rpl_connection_name= server2
104 --source include/rpl_connection.inc
105
106 # One of three things can happen:
107 # A. The applier fails after the join and the member leaves the group
108 # but the START GROUP_REPLICATION command does not fail
109 # B. The applier fails after initialization and sets the state to ERROR,
110 # making the group join fail that makes the START GROUP_REPLICATION
111 # command fail with a generic ER_GROUP_REPLICATION_CONFIGURATION.
112 # C. The applier fails and its initialization processes catches the error.
113 # During the start, it is known that the applier failed so the START
114 # GROUP_REPLICATION fails with: ER_GROUP_REPLICATION_APPLIER_INIT_ERROR
115
116 --error 0, ER_GROUP_REPLICATION_CONFIGURATION, ER_GROUP_REPLICATION_APPLIER_INIT_ERROR
117 --reap
118
119 SET @@GLOBAL.DEBUG= '-d,force_sql_thread_error';
120 SET DEBUG_SYNC= 'RESET';
121
122 --let $assert_text= Clone must not start.
123 --let $assert_cond= [SELECT COUNT(*) FROM performance_schema.clone_status] = 0;
124 --source include/assert.inc
125
126 --echo
127 --echo # 4. Reset debug points for applier failures.
128 --echo # Restart GR on M2.
129 --echo # Assert clone starts and group_replication_applier SQL thread is OFF.
130 --echo
131
132 SET @@GLOBAL.DEBUG='+d,gr_clone_wait';
133
134 START GROUP_REPLICATION;
135
136 SET DEBUG_SYNC = 'now WAIT_FOR gr_clone_paused';
137 SET @@GLOBAL.DEBUG='-d,gr_clone_wait';
138
139 --let $assert_text= group_replication_applier SQL Thread will be off.
140 --let $assert_cond= [SELECT COUNT(*) as count FROM performance_schema.replication_applier_status WHERE CHANNEL_NAME="group_replication_applier" AND SERVICE_STATE = "OFF",count, 1] = 1
141 --source include/assert.inc
142
143 SET DEBUG_SYNC = 'now SIGNAL gr_clone_continue';
144
145 --source include/wait_until_disconnected.inc
146
147 --let $rpl_server_number= 2
148 --source include/rpl_reconnect.inc
149
150 --let $group_replication_member_state=ONLINE
151 --source include/gr_wait_for_member_state.inc
152
153 --let $assert_text= Clone must be completed
154 --let $assert_cond= [SELECT state="Completed" FROM performance_schema.clone_status] = 1;
155 --source include/assert.inc
156
157 --echo
158 --echo # 5. Block applier on M2.
159 --echo # Uninstall clone on M1 so that clone fails.
160 --echo # Create some transactions on M1 to create applier backlog on M2.
161 --echo
162
163 SET @@GLOBAL.DEBUG='+d,block_applier_updates';
164
165 --let $rpl_connection_name= server1
166 --source include/rpl_connection.inc
167 UNINSTALL PLUGIN clone;
168 INSERT INTO t1 values (3);
169 INSERT INTO t1 values (4);
170
171 --let $rpl_connection_name= server2
172 --source include/rpl_connection.inc
173
174 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_group_member_stats WHERE member_id='$member2_uuid' AND count_transactions_remote_in_applier_queue = 2
175 --source include/wait_condition_or_abort.inc
176
177 --echo
178 --echo # 6. Stop GR on M2 without committing the received transactions.
179 --echo # Create transactions on M1 for M2 to clone.
180 --echo
181
182 SET DEBUG_SYNC = "now WAIT_FOR applier_read_blocked";
183 SET @@GLOBAL.DEBUG='-d,block_applier_updates';
184 SET @@GLOBAL.DEBUG='+d,force_sql_thread_error';
185 SET DEBUG_SYNC = "now SIGNAL resume_applier_read";
186 --let $group_replication_member_state= ERROR
187 --source include/gr_wait_for_member_state.inc
188 SET @@GLOBAL.DEBUG='-d,force_sql_thread_error';
189
190 --source include/stop_group_replication.inc
191
192 --let $rpl_connection_name= server1
193 --source include/rpl_connection.inc
194 # Force clone.
195 INSERT INTO t1 values (5);
196 INSERT INTO t1 values (6);
197
198 --echo
199 --echo # 7. Start GR on M2.
200 --echo # Clone will fail and incremental recovery will start.
201 --echo # Applier will be OFF till clone failure is detected.
202 --echo
203
204 --let $rpl_connection_name= server1
205 --source include/rpl_connection.inc
206 SET @@GLOBAL.DEBUG='+d,block_applier_updates';
207
208 --let $rpl_connection_name= server2
209 --source include/rpl_connection.inc
210
211 SET @@GLOBAL.DEBUG='+d,gr_run_clone_query_fail_once';
212 SET GLOBAL group_replication_clone_threshold= 1;
213 START GROUP_REPLICATION;
214
215 SET DEBUG_SYNC = 'now WAIT_FOR signal.run_clone_query_waiting';
216 SET @@GLOBAL.DEBUG='-d,gr_run_clone_query_fail_once';
217
218 # Clone is executing
219 --let $assert_text= "Clone is executing"
220 --let $assert_cond= [SELECT COUNT(*) FROM performance_schema.events_stages_current WHERE event_name LIKE "%stage/group_rpl/Group Replication Cloning%"] = 1
221 --source include/assert.inc
222
223 --let $assert_text= group_replication_applier SQL Thread will be OFF.
224 --let $assert_cond= [SELECT COUNT(*) as count FROM performance_schema.replication_applier_status WHERE CHANNEL_NAME="group_replication_applier" AND SERVICE_STATE = "OFF",count, 1] = 1
225 --source include/assert.inc
226
227 SET DEBUG_SYNC = 'now SIGNAL signal.run_clone_query_continue';
228
229 # Clone will fail and will start channel group_replication_recovery
230 --let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.replication_connection_status WHERE CHANNEL_NAME="group_replication_recovery" AND SERVICE_STATE='ON'
231 --source include/wait_condition.inc
232
233 --let $assert_text= group_replication_applier SQL Thread will be ON.
234 --let $assert_cond= [SELECT COUNT(*) as count FROM performance_schema.replication_applier_status WHERE CHANNEL_NAME="group_replication_applier" AND SERVICE_STATE = "ON",count, 1] = 1
235 --source include/assert.inc
236
237 # Allow the recovery to continue
238 --let $rpl_connection_name= server1
239 --source include/rpl_connection.inc
240 SET DEBUG_SYNC = "now WAIT_FOR applier_read_blocked";
241 SET @@GLOBAL.DEBUG='-d,block_applier_updates';
242 SET DEBUG_SYNC = "now SIGNAL resume_applier_read";
243
244 --let $rpl_connection_name= server2
245 --source include/rpl_connection.inc
246
247 --let $group_replication_member_state=ONLINE
248 --source include/gr_wait_for_member_state.inc
249
250 --let $diff_tables= server1:test.t1, server2:test.t1
251 --source include/diff_tables.inc
252
253
254 --echo
255 --echo # 8. Cleanup.
256 --echo
257
258 --let $rpl_connection_name= server1
259 --source include/rpl_connection.inc
260
261 DROP TABLE t1;
262
263 set session sql_log_bin=0;
264 call mtr.add_suppression("Timeout while waiting for the group communication engine to exit!");
265 call mtr.add_suppression("The member has failed to gracefully leave the group.");
266 set session sql_log_bin=1;
267
268 --let $rpl_connection_name= server2
269 --source include/rpl_connection.inc
270 set session sql_log_bin=0;
271 call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': Relay log read failure: *.*");
272 call mtr.add_suppression("The applier thread execution was aborted. *.*");
273 call mtr.add_suppression("Error running query, replica SQL thread aborted. *");
274 call mtr.add_suppression("Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.");
275 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
276 call mtr.add_suppression("Unable to confirm whether the server has left the group or not. Check performance_schema.replication_group_members to check group membership information.");
277 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
278 call mtr.add_suppression("Unable to initialize the Group Replication applier module.");
279 call mtr.add_suppression("There was a previous plugin error while the member joined the group. The member will now exit the group.");
280 call mtr.add_suppression("This member will start distributed recovery using clone. It is due to the number of missing transactions being higher than the configured threshold of 1.");
281 call mtr.add_suppression("Clone removing all user data for provisioning: *");
282 call mtr.add_suppression("Internal query: CLONE INSTANCE FROM *");
283 call mtr.add_suppression("There was an issue when cloning from another server: *");
284 call mtr.add_suppression("Due to some issue on the previous step distributed recovery is now executing: Incremental Recovery.");
285 call mtr.add_suppression("Timeout while waiting for the group communication engine to be ready!");
286 call mtr.add_suppression("The group communication engine is not ready for the member to join. .*");
287 call mtr.add_suppression("The member was unable to join the group.*");
288 call mtr.add_suppression("There was an issue when configuring the remote cloning process: The plugin was not able to stop the group_replication_applier channel.");
289 set session sql_log_bin=1;
290
291 RESET PERSIST group_replication_group_name;
292 RESET PERSIST group_replication_local_address;
293 RESET PERSIST group_replication_group_seeds;
294 RESET PERSIST group_replication_start_on_boot;
295 RESET PERSIST IF EXISTS group_replication_communication_stack;
296
297 --source include/clean_monitoring_process.inc
298 --source include/group_replication_end.inc