######################################################################## # # This test validate services send and recv only delivery messages when # server status is ONLINE or RECOVERING. # # Test: # 0. This test requires 2 servers with plugin GR and replication # observers example with a table t1 # # 1. Server can't use send service when server status is OFFLINE # 1.1. Service send isn't registered when status is OFFLINE # 1.2. Send message on server2 that won't be received by server1 # # 2. Server can use send service when server status is RECOVERING # 2.1. Force server1 to stay in RECOVERING status # 2.2. Start GR on server 1 and wait status is RECOVERING # 2.3. Server1 shall be able to send messages when status is RECOVERING # 2.4. Send message on server2 that shall be received by server1 # # 3. Server can use send service when server status is ONLINE # 3.1. Allow server1 to finish recovery # 3.2. Wait server1 change is status to ONLINE # 3.3. Server1 shall be able to send messages when status is ONLINE # 3.4. Send message on server2 that shall be received by server1 # # 4. Server can't use send service when server status is ERROR # 4.1. Insert data on server1 that won't be transmitted to the group # 4.2. On server2 insert same data of server1 to create a conflict # 4.3. Server1 set its status to ERROR server due to a data conflict # with group # 4.4. Server1 return error when trying to use service send # 4.5. Send message by server2 that won't be received by server1 # # 5. When group looses quorum server can send messages but they only be # deliver if quorum is reestablished # 5.1. Stop server1 and remove table t1 and reset master to allow # joining back to the group # 5.2. Start server1 and wait to be ONLINE # 5.3. Kill server2 # 5.4. Server1 will update server2 status to UNREACHABLE # 5.5. Send message on server1, it will be delivered if quorum is achieved # # 6. Assert messages received on group delivery message service # 6.1. Stop GR on server1, which will make it wait for the delivery of # new view, and that will happen only after delivery of service # message. This will give some time to server to receive and write # it to error log. # 6.2. When OFFLINE server1 didn't send or receive any message # 6.3. When RECOVERING server1 shall send and receive all messages # 6.4. When ONLINE server1 shall send and receive all messages # 6.5. When ERROR server1 didn't send or receive any message # 6.6. No messages are received when loss of quorum occur # # 7. Cleanup # ######################################################################## --source include/have_debug_sync.inc --source include/have_replication_observers_example_plugin.inc --source include/have_group_replication_plugin.inc --let $rpl_skip_group_replication_start= 1 --source include/group_replication.inc --echo --let $rpl_connection_name= server2 --source include/rpl_connection.inc --source include/install_replication_observers_example.inc --source include/start_and_bootstrap_group_replication.inc CREATE TABLE t1 (a INT NOT NULL AUTO_INCREMENT PRIMARY KEY , b INT); --let $rpl_connection_name= server1 --source include/rpl_connection.inc --source include/install_replication_observers_example.inc --echo --echo # 1. Server can't use send service when server status is OFFLINE --echo --echo # 1.1. Service send isn't registered when status is OFFLINE --error ER_UDF_ERROR SELECT group_replication_service_message_send("tag", "Server1 sent a message when OFFLINE"); --echo --echo # 1.2. Send message on server2 that won't be received by server1 --let $rpl_connection_name= server2 --source include/rpl_connection.inc SELECT group_replication_service_message_send("tag", "Server2 sent a message when server1 was OFFLINE"); --echo --echo # 2. Server can use send service when server status is RECOVERING --echo --echo # 2.1. Force server1 to stay in RECOVERING status --let $rpl_connection_name= server1 --source include/rpl_connection.inc SET @@GLOBAL.DEBUG='+d,recovery_thread_wait_before_finish'; --echo --echo # 2.2. Start GR on server 1 and wait status is RECOVERING --let $rpl_connection_name= server1 --source include/rpl_connection.inc --let $group_replication_start_member_state= RECOVERING --source include/start_group_replication.inc --echo --echo # 2.3. Server1 shall be able to send messages when status is RECOVERING SELECT group_replication_service_message_send("tag", "Server1 sent a message when RECOVERING"); --echo --echo # 2.4. Send message on server2 that shall be received by server1 --let $rpl_connection_name= server2 --source include/rpl_connection.inc SELECT group_replication_service_message_send("tag", "Server2 sent a message when server1 was RECOVERING"); --echo --echo # 3. Server can use send service when server status is ONLINE --echo --echo # 3.1. Allow server1 to finish recovery --let $rpl_connection_name= server1 --source include/rpl_connection.inc SET DEBUG_SYNC= "now WAIT_FOR signal.recovery_thread_wait_before_finish_reached"; SET @@GLOBAL.DEBUG='-d,recovery_thread_wait_before_finish'; SET DEBUG_SYNC= "now SIGNAL signal.recovery_end"; --echo --echo # 3.2. Wait server1 change is status to ONLINE --let $rpl_connection_name= server1 --source include/rpl_connection.inc --let $group_replication_member_state= ONLINE --source include/gr_wait_for_member_state.inc SET DEBUG_SYNC= 'RESET'; --echo --echo # 3.3. Server1 shall be able to send messages when status is ONLINE SELECT group_replication_service_message_send("tag", "Server1 sent a message when ONLINE"); --echo --echo # 3.4. Send message on server2 that shall be received by server1 --let $rpl_connection_name= server2 --source include/rpl_connection.inc SELECT group_replication_service_message_send("tag", "Server2 sent a message when server1 was ONLINE"); --echo --echo # 4. Server can't use send service when server status is ERROR --echo --echo # 4.1. Insert data on server1 that won't be transmitted to the group --let $rpl_connection_name= server1 --source include/rpl_connection.inc set session sql_log_bin=0; INSERT INTO test.t1 (a, b) VALUES (1, 1); set session sql_log_bin=1; --echo --echo # 4.2. On server2 insert same data of server1 to create a conflict --let $rpl_connection_name= server2 --source include/rpl_connection.inc INSERT INTO test.t1 (a, b) VALUES (1, 1); --echo --echo # 4.3. Server1 set its status to ERROR server due to a data conflict --echo # with group --let $rpl_connection_name= server1 --source include/rpl_connection.inc --let $group_replication_member_state= ERROR --source include/gr_wait_for_member_state.inc --echo --echo # 4.4. Server1 return error when trying to use service send --error ER_UDF_ERROR SELECT group_replication_service_message_send("tag", "Server1 sent a message when ERROR"); --echo --echo # 4.5. Send message by server2 that won't be received by server1 --let $rpl_connection_name= server2 --source include/rpl_connection.inc SELECT group_replication_service_message_send("tag", "Server2 sent a message when server1 was ERROR"); --echo --echo # 5. When group looses quorum server can send messages but they only be --echo # deliver if quorum is reestablished --echo --echo # 5.1. Stop server1 and remove table t1 and reset master to allow --echo # joining back to the group --let $rpl_connection_name= server1 --source include/rpl_connection.inc --source include/stop_group_replication.inc DROP TABLE t1; RESET MASTER; --echo --echo # 5.2. Start server1 and wait to be ONLINE --let $group_replication_start_member_state= ONLINE --source include/start_group_replication.inc --echo --echo # 5.3. Kill server2 --let $rpl_connection_name= server2 --source include/rpl_connection.inc --let $group_replication_local_address= `SELECT @@GLOBAL.group_replication_local_address` --let $group_replication_group_seeds= `SELECT @@GLOBAL.group_replication_group_seeds` --let $restart_parameters=restart:--group_replication_local_address=$group_replication_local_address --group_replication_group_seeds=$group_replication_group_seeds --group_replication_group_name=$group_replication_group_name --replace_result $group_replication_local_address GROUP_REPLICATION_LOCAL_ADDRESS $group_replication_group_seeds GROUP_REPLICATION_GROUP_SEEDS $group_replication_group_name GROUP_REPLICATION_GROUP_NAME --source include/kill_and_restart_mysqld.inc --let $rpl_server_number= 2 --source include/rpl_reconnect.inc --echo --echo # 5.4. Server 1 will update server2 status to UNREACHABLE --let $rpl_connection_name= server1 --source include/rpl_connection.inc let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.replication_group_members where MEMBER_STATE="UNREACHABLE"; --source include/wait_condition.inc --echo --echo # 5.5. Send message on server1, it will be delivered if quorum is achieved SELECT group_replication_service_message_send("tag", "Server1 sent a message without QUORUM"); # quorum will not be achieved, we will stop server1 --echo --echo # 6. Assert messages received on group delivery message service --echo --echo # 6.1. To decrease failure rate on assert_grep we do a transaction with --echo # AFTER consistency to add some rounds on GCS communication and --echo # stop GR on server1, which will make it wait for the delivery of --echo # new view, and that will happen only after delivery of service --echo # message. This will give some time to server to receive and write --echo # it to error log. --let $rpl_connection_name= server2 --source include/rpl_connection.inc SET @@SESSION.group_replication_consistency= 'AFTER'; INSERT INTO test.t1 (a, b) VALUES (3, 3); --let $rpl_connection_name= server1 --source include/rpl_connection.inc --source include/stop_group_replication.inc --echo --echo # 6.2. When OFFLINE server1 didn't send or receive any message --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err --let $assert_text = Server1 did not receive any message when OFFLINE. --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*OFFLINE.*.' --let $assert_count = 0 --source include/assert_grep.inc --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.err --let $assert_text = Server2 receive a message from itself when server1 was OFFLINE. --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*OFFLINE.*.' --let $assert_count = 1 --source include/assert_grep.inc --echo --echo # 6.3. When RECOVERING server1 shall send and receive all messages --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err --let $assert_text = Server1 receive a message from itself and one from server2 when RECOVERING. --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*RECOVERING.*.' --let $assert_count = 2 --source include/assert_grep.inc --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.err --let $assert_text = Server2 receive a message from itself and from server1. --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*RECOVERING.*.' --let $assert_count = 2 --source include/assert_grep.inc --echo --echo # 6.4. When ONLINE server1 shall send and receive all messages --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err --let $assert_text = Server1 receive a message from itself and one from server2 when ONLINE. --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*ONLINE.*.' --let $assert_count = 2 --source include/assert_grep.inc --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.err --let $assert_text = Server2 receive a message from itself and from server1. --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*ONLINE.*.' --let $assert_count = 2 --source include/assert_grep.inc --echo --echo # 6.5. When ERROR server1 didn't send or receive any message --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err --let $assert_text = Server1 did not receive any message when ERROR. --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*ERROR.*.' --let $assert_count = 0 --source include/assert_grep.inc --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.err --let $assert_text = Server2 receive a message from itself when server1 was ERROR. --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*ERROR.*.' --let $assert_count = 1 --source include/assert_grep.inc --echo --echo # 6.6. No messages are received when loss of quorum occur --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err --let $assert_text = Server1 did not receive any message when group loss quorum. --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*QUORUM.*.' --let $assert_count = 0 --source include/assert_grep.inc --let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.2.err --let $assert_text = Server2 did not receive any message when group loss quorum. --let $assert_select = Plugin replication_observers_example reported: 'Service message recv .*QUORUM.*.' --let $assert_count = 0 --source include/assert_grep.inc --echo --echo # 7. Cleanup --let $rpl_connection_name= server2 --source include/rpl_connection.inc DROP TABLE t1; --source include/uninstall_replication_observers_example.inc --let $rpl_connection_name= server1 --source include/rpl_connection.inc --source include/uninstall_replication_observers_example.inc set session sql_log_bin=0; call mtr.add_suppression("Could not execute Write_rows event on table test.t1; Duplicate entry '1'.*"); call mtr.add_suppression("Plugin group_replication reported: 'The applier thread execution was aborted. Unable to process more transactions, this member will now leave the group.'"); call mtr.add_suppression("Plugin group_replication reported: 'Fatal error during execution on the Applier process of Group Replication. The server will now leave the group.'"); call mtr.add_suppression("Plugin group_replication reported: 'The server was automatically set into read only mode after an error was detected.'"); call mtr.add_suppression("Plugin group_replication reported: 'Skipping leave operation: concurrent attempt to leave the group is on-going.'"); call mtr.add_suppression("Plugin group_replication reported: '\\[GCS\\] Timeout while waiting for the group communication engine to exit!'"); call mtr.add_suppression("Plugin group_replication reported: '\\[GCS\\] The member has failed to gracefully leave the group.'"); call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': ... The replica coordinator and worker threads are stopped,*"); set session sql_log_bin=1; set session sql_log_bin=0; DROP TABLE t1; set session sql_log_bin=1; --source include/group_replication_end.inc