Debian Codegraph @

mysql-test/suite/group_replication/t/gr_message_service_server_status.test

raw
1 ########################################################################
2 #
3 # This test validate services send and recv only delivery messages when
4 # server status is ONLINE or RECOVERING.
5 #
6 # Test:
7 # 0. This test requires 2 servers with plugin GR and replication
8 # observers example with a table t1
9 #
10 # 1. Server can't use send service when server status is OFFLINE
11 # 1.1. Service send isn't registered when status is OFFLINE
12 # 1.2. Send message on server2 that won't be received by server1
13 #
14 # 2. Server can use send service when server status is RECOVERING
15 # 2.1. Force server1 to stay in RECOVERING status
16 # 2.2. Start GR on server 1 and wait status is RECOVERING
17 # 2.3. Server1 shall be able to send messages when status is RECOVERING
18 # 2.4. Send message on server2 that shall be received by server1
19 #
20 # 3. Server can use send service when server status is ONLINE
21 # 3.1. Allow server1 to finish recovery
22 # 3.2. Wait server1 change is status to ONLINE
23 # 3.3. Server1 shall be able to send messages when status is ONLINE
24 # 3.4. Send message on server2 that shall be received by server1
25 #
26 # 4. Server can't use send service when server status is ERROR
27 # 4.1. Insert data on server1 that won't be transmitted to the group
28 # 4.2. On server2 insert same data of server1 to create a conflict
29 # 4.3. Server1 set its status to ERROR server due to a data conflict
30 # with group
31 # 4.4. Server1 return error when trying to use service send
32 # 4.5. Send message by server2 that won't be received by server1
33 #
34 # 5. When group looses quorum server can send messages but they only be
35 # deliver if quorum is reestablished
36 # 5.1. Stop server1 and remove table t1 and reset master to allow
37 # joining back to the group
38 # 5.2. Start server1 and wait to be ONLINE
39 # 5.3. Kill server2
40 # 5.4. Server1 will update server2 status to UNREACHABLE
41 # 5.5. Send message on server1, it will be delivered if quorum is achieved
42 #
43 # 6. Assert messages received on group delivery message service
44 # 6.1. Stop GR on server1, which will make it wait for the delivery of
45 # new view, and that will happen only after delivery of service
46 # message. This will give some time to server to receive and write
47 # it to error log.
48 # 6.2. When OFFLINE server1 didn't send or receive any message
49 # 6.3. When RECOVERING server1 shall send and receive all messages
50 # 6.4. When ONLINE server1 shall send and receive all messages
51 # 6.5. When ERROR server1 didn't send or receive any message
52 # 6.6. No messages are received when loss of quorum occur
53 #
54 # 7. Cleanup
55 #
56 ########################################################################
57
58 --source include/have_debug_sync.inc
59 --source include/have_replication_observers_example_plugin.inc
60 --source include/have_group_replication_plugin.inc
61 --let $rpl_skip_group_replication_start= 1
62 --source include/group_replication.inc
63
64 --echo
65 --let $rpl_connection_name= server2
66 --source include/rpl_connection.inc
67
68 --source include/install_replication_observers_example.inc
69
70 --source include/start_and_bootstrap_group_replication.inc
71
72 CREATE TABLE t1 (a INT NOT NULL AUTO_INCREMENT PRIMARY KEY , b INT);
73
74 --let $rpl_connection_name= server1
75 --source include/rpl_connection.inc
76
77 --source include/install_replication_observers_example.inc
78
79 --echo
80 --echo # 1. Server can't use send service when server status is OFFLINE
81
82 --echo
83 --echo # 1.1. Service send isn't registered when status is OFFLINE
84
85
86 --error ER_UDF_ERROR
87 SELECT group_replication_service_message_send("tag", "Server1 sent a message when OFFLINE");
88
89 --echo
90 --echo # 1.2. Send message on server2 that won't be received by server1
91
92 --let $rpl_connection_name= server2
93 --source include/rpl_connection.inc
94
95 SELECT group_replication_service_message_send("tag", "Server2 sent a message when server1 was OFFLINE");
96
97 --echo
98 --echo # 2. Server can use send service when server status is RECOVERING
99
100 --echo
101 --echo # 2.1. Force server1 to stay in RECOVERING status
102
103 --let $rpl_connection_name= server1
104 --source include/rpl_connection.inc
105 SET @@GLOBAL.DEBUG='+d,recovery_thread_wait_before_finish';
106
107 --echo
108 --echo # 2.2. Start GR on server 1 and wait status is RECOVERING
109
110 --let $rpl_connection_name= server1
111 --source include/rpl_connection.inc
112
113 --let $group_replication_start_member_state= RECOVERING
114 --source include/start_group_replication.inc
115
116 --echo
117 --echo # 2.3. Server1 shall be able to send messages when status is RECOVERING
118
119 SELECT group_replication_service_message_send("tag", "Server1 sent a message when RECOVERING");
120
121 --echo
122 --echo # 2.4. Send message on server2 that shall be received by server1
123
124 --let $rpl_connection_name= server2
125 --source include/rpl_connection.inc
126
127 SELECT group_replication_service_message_send("tag", "Server2 sent a message when server1 was RECOVERING");
128
129 --echo
130 --echo # 3. Server can use send service when server status is ONLINE
131
132 --echo
133 --echo # 3.1. Allow server1 to finish recovery
134
135 --let $rpl_connection_name= server1
136 --source include/rpl_connection.inc
137 SET DEBUG_SYNC= "now WAIT_FOR signal.recovery_thread_wait_before_finish_reached";
138 SET @@GLOBAL.DEBUG='-d,recovery_thread_wait_before_finish';
139 SET DEBUG_SYNC= "now SIGNAL signal.recovery_end";
140
141 --echo
142 --echo # 3.2. Wait server1 change is status to ONLINE
143
144 --let $rpl_connection_name= server1
145 --source include/rpl_connection.inc
146
147 --let $group_replication_member_state= ONLINE
148 --source include/gr_wait_for_member_state.inc
149
150 SET DEBUG_SYNC= 'RESET';
151
152 --echo
153 --echo # 3.3. Server1 shall be able to send messages when status is ONLINE
154
155 SELECT group_replication_service_message_send("tag", "Server1 sent a message when ONLINE");
156
157 --echo
158 --echo # 3.4. Send message on server2 that shall be received by server1
159
160 --let $rpl_connection_name= server2
161 --source include/rpl_connection.inc
162
163 SELECT group_replication_service_message_send("tag", "Server2 sent a message when server1 was ONLINE");
164
165 --echo
166 --echo # 4. Server can't use send service when server status is ERROR
167
168 --echo
169 --echo # 4.1. Insert data on server1 that won't be transmitted to the group
170
171 --let $rpl_connection_name= server1
172 --source include/rpl_connection.inc
173
174 set session sql_log_bin=0;
175 INSERT INTO test.t1 (a, b) VALUES (1, 1);
176 set session sql_log_bin=1;
177
178 --echo
179 --echo # 4.2. On server2 insert same data of server1 to create a conflict
180
181 --let $rpl_connection_name= server2
182 --source include/rpl_connection.inc
183
184 INSERT INTO test.t1 (a, b) VALUES (1, 1);
185
186 --echo
187 --echo # 4.3. Server1 set its status to ERROR server due to a data conflict
188 --echo # with group
189
190 --let $rpl_connection_name= server1
191 --source include/rpl_connection.inc
192
193 --let $group_replication_member_state= ERROR
194 --source include/gr_wait_for_member_state.inc
195
196 --echo
197 --echo # 4.4. Server1 return error when trying to use service send
198
199 --error ER_UDF_ERROR
200 SELECT group_replication_service_message_send("tag", "Server1 sent a message when ERROR");
201
202 --echo
203 --echo # 4.5. Send message by server2 that won't be received by server1
204
205 --let $rpl_connection_name= server2
206 --source include/rpl_connection.inc
207
208 SELECT group_replication_service_message_send("tag", "Server2 sent a message when server1 was ERROR");
209
210 --echo
211 --echo # 5. When group looses quorum server can send messages but they only be
212 --echo # deliver if quorum is reestablished
213
214 --echo
215 --echo # 5.1. Stop server1 and remove table t1 and reset master to allow
216 --echo # joining back to the group
217
218 --let $rpl_connection_name= server1
219 --source include/rpl_connection.inc
220
221 --source include/stop_group_replication.inc
222
223 DROP TABLE t1;
224 RESET MASTER;
225
226 --echo
227 --echo # 5.2. Start server1 and wait to be ONLINE
228
229 --let $group_replication_start_member_state= ONLINE
230 --source include/start_group_replication.inc
231
232 --echo
233 --echo # 5.3. Kill server2
234
235 --let $rpl_connection_name= server2
236 --source include/rpl_connection.inc
237
238 --let $group_replication_local_address= `SELECT @@GLOBAL.group_replication_local_address`
239 --let $group_replication_group_seeds= `SELECT @@GLOBAL.group_replication_group_seeds`
240 --let $restart_parameters=restart:--group_replication_local_address=$group_replication_local_address --group_replication_group_seeds=$group_replication_group_seeds --group_replication_group_name=$group_replication_group_name
241 --replace_result $group_replication_local_address GROUP_REPLICATION_LOCAL_ADDRESS $group_replication_group_seeds GROUP_REPLICATION_GROUP_SEEDS $group_replication_group_name GROUP_REPLICATION_GROUP_NAME
242
243 --source include/kill_and_restart_mysqld.inc
244
245 --let $rpl_server_number= 2
246 --source include/rpl_reconnect.inc
247
248 --echo
249 --echo # 5.4. Server 1 will update server2 status to UNREACHABLE
250
251 --let $rpl_connection_name= server1
252 --source include/rpl_connection.inc
253
254 let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.replication_group_members where MEMBER_STATE="UNREACHABLE";
255 --source include/wait_condition.inc
256
257 --echo
258 --echo # 5.5. Send message on server1, it will be delivered if quorum is achieved
259
260 SELECT group_replication_service_message_send("tag", "Server1 sent a message without QUORUM");
261
262 # quorum will not be achieved, we will stop server1
263
264 --echo
265 --echo # 6. Assert messages received on group delivery message service
266
267 --echo
268 --echo # 6.1. To decrease failure rate on assert_grep we do a transaction with
269 --echo # AFTER consistency to add some rounds on GCS communication and
270 --echo # stop GR on server1, which will make it wait for the delivery of
271 --echo # new view, and that will happen only after delivery of service
272 --echo # message. This will give some time to server to receive and write
273 --echo # it to error log.
274
275 --let $rpl_connection_name= server2
276 --source include/rpl_connection.inc
277
278 SET @@SESSION.group_replication_consistency= 'AFTER';
279 INSERT INTO test.t1 (a, b) VALUES (3, 3);
280
281 --let $rpl_connection_name= server1
282 --source include/rpl_connection.inc
283 --source include/stop_group_replication.inc
284
285 --echo
286 --echo # 6.2. When OFFLINE server1 didn't send or receive any message
287
288 --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err
289 --let $assert_text = Server1 did not receive any message when OFFLINE.
290 --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*OFFLINE.*.'
291 --let $assert_count = 0
292 --source include/assert_grep.inc
293
294 --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.err
295 --let $assert_text = Server2 receive a message from itself when server1 was OFFLINE.
296 --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*OFFLINE.*.'
297 --let $assert_count = 1
298 --source include/assert_grep.inc
299
300 --echo
301 --echo # 6.3. When RECOVERING server1 shall send and receive all messages
302
303 --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err
304 --let $assert_text = Server1 receive a message from itself and one from server2 when RECOVERING.
305 --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*RECOVERING.*.'
306 --let $assert_count = 2
307 --source include/assert_grep.inc
308
309 --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.err
310 --let $assert_text = Server2 receive a message from itself and from server1.
311 --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*RECOVERING.*.'
312 --let $assert_count = 2
313 --source include/assert_grep.inc
314
315 --echo
316 --echo # 6.4. When ONLINE server1 shall send and receive all messages
317
318 --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err
319 --let $assert_text = Server1 receive a message from itself and one from server2 when ONLINE.
320 --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*ONLINE.*.'
321 --let $assert_count = 2
322 --source include/assert_grep.inc
323
324 --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.err
325 --let $assert_text = Server2 receive a message from itself and from server1.
326 --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*ONLINE.*.'
327 --let $assert_count = 2
328 --source include/assert_grep.inc
329
330 --echo
331 --echo # 6.5. When ERROR server1 didn't send or receive any message
332
333 --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err
334 --let $assert_text = Server1 did not receive any message when ERROR.
335 --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*ERROR.*.'
336 --let $assert_count = 0
337 --source include/assert_grep.inc
338
339 --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.err
340 --let $assert_text = Server2 receive a message from itself when server1 was ERROR.
341 --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*ERROR.*.'
342 --let $assert_count = 1
343 --source include/assert_grep.inc
344
345 --echo
346 --echo # 6.6. No messages are received when loss of quorum occur
347
348 --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err
349 --let $assert_text = Server1 did not receive any message when group loss quorum.
350 --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*QUORUM.*.'
351 --let $assert_count = 0
352 --source include/assert_grep.inc
353
354 --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.err
355 --let $assert_text = Server2 did not receive any message when group loss quorum.
356 --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*QUORUM.*.'
357 --let $assert_count = 0
358 --source include/assert_grep.inc
359
360 --echo
361 --echo # 7. Cleanup
362
363 --let $rpl_connection_name= server2
364 --source include/rpl_connection.inc
365
366 DROP TABLE t1;
367
368 --source include/uninstall_replication_observers_example.inc
369
370 --let $rpl_connection_name= server1
371 --source include/rpl_connection.inc
372
373 --source include/uninstall_replication_observers_example.inc
374
375 set session sql_log_bin=0;
376 call mtr.add_suppression("Could not execute Write_rows event on table test.t1; Duplicate entry '1'.*");
377 call mtr.add_suppression("Plugin group_replication reported: 'The applier thread execution was aborted. Unable to process more transactions, this member will now leave the group.'");
378 call mtr.add_suppression("Plugin group_replication reported: 'Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.'");
379 call mtr.add_suppression("Plugin group_replication reported: 'The server was automatically set into read only mode after an error was detected.'");
380 call mtr.add_suppression("Plugin group_replication reported: 'Skipping leave operation: concurrent attempt to leave the group is on-going.'");
381 call mtr.add_suppression("Plugin group_replication reported: '\\[GCS\\] Timeout while waiting for the group communication engine to exit!'");
382 call mtr.add_suppression("Plugin group_replication reported: '\\[GCS\\] The member has failed to gracefully leave the group.'");
383 call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': ... The replica coordinator and worker threads are stopped,*");
384 set session sql_log_bin=1;
385
386 set session sql_log_bin=0;
387 DROP TABLE t1;
388 set session sql_log_bin=1;
389 --source include/group_replication_end.inc