################################################################################ # This test evaluates that whenever the donor has conflicting data that causes # errors in the joiner, a failover to another donor will happen. # # The test steps are: # 0) The test requires three servers: M1, M2 and M3. # 1) Bootstrap start M1 with no replication user for recovery. Insert some # data. # 2) Insert some extra data on M2 before starting. Create a replication user # recovery. # 3) Add conflicting data on M3 and start it with the configured replication # user and watch it fail as: # 1. M1 does not has the correct replication user. # 2. M2 has conflicting data. # 4) Create the recovery user on M1. The joiner M3 should now connect to M1 # when failing over. # 5) The joiner M3 should become ONLINE. Validate data on M3. # 6) Clean up. ################################################################################ --source include/big_test.inc --let $group_replication_group_name=65b51c20-262d-11e5-867f-0800200c9a66 --source include/have_group_replication_plugin.inc --let $rpl_skip_group_replication_start= 1 --let $rpl_server_count= 3 --source include/group_replication.inc --let $recovery_user= recovery_user --let $recovery_password= recovery_password --let $fake_transaction_uuid= aaaaaaaa-bbbb-aaaa-bbbb-aaaaaaaaaaaa --echo # --echo # Start a new member with some group data but no replication user --echo # --connection server1 --source include/start_and_bootstrap_group_replication.inc #insert some data CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); # Log fake transactions to allow server2 to join the group with # extra data. --eval SET GTID_NEXT= "$fake_transaction_uuid:1" BEGIN; COMMIT; SET GTID_NEXT= "AUTOMATIC"; --eval SET GTID_NEXT= "$fake_transaction_uuid:2" BEGIN; COMMIT; SET GTID_NEXT= "AUTOMATIC"; --echo # --echo # Add some extra data on server 2 and then start group replication --echo # Create a replication user so recovery can connect --echo # --connection server2 --eval SET GTID_NEXT= "$fake_transaction_uuid:1" CREATE TABLE lost_test_table (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB; SET GTID_NEXT= "AUTOMATIC"; --eval SET GTID_NEXT= "$fake_transaction_uuid:2" INSERT INTO lost_test_table VALUES (1); SET GTID_NEXT= "AUTOMATIC"; SET SESSION sql_log_bin= 0; --eval CREATE USER "$recovery_user" IDENTIFIED BY "$recovery_password" --eval GRANT REPLICATION SLAVE ON *.* TO "$recovery_user" --eval GRANT GROUP_REPLICATION_STREAM ON *.* TO "$recovery_user"@'%'; FLUSH PRIVILEGES; SET SESSION sql_log_bin= 1; --source include/start_group_replication.inc --echo # --echo # Start recovery on member 3 and watch it fail for a bit as: --echo # 1) Server 1 does not has the correct replication user --echo # 2) Server 2 has conflicting data --echo # --connection server3 SET SESSION sql_log_bin= 0; --eval CREATE USER "$recovery_user" IDENTIFIED BY "$recovery_password" --eval GRANT GROUP_REPLICATION_STREAM ON *.* TO "$recovery_user"@'%'; FLUSH PRIVILEGES; SET SESSION sql_log_bin= 1; SET SESSION sql_log_bin= 0; call mtr.add_suppression("There was an error when connecting to the donor*"); call mtr.add_suppression("For details please check performance_schema.replication_connection_status table and error log messages of Replica I/O for channel group_replication_recovery."); call mtr.add_suppression("Replica SQL for channel 'group_replication_recovery': Error 'Table 'lost_test_table'*"); call mtr.add_suppression("Replica: Table 'lost_test_table' already exists Error_code:*"); call mtr.add_suppression("Replica SQL for channel 'group_replication_recovery': Worker .* failed executing transaction .*; Error 'Table 'lost_test_table' already exists' on query."); call mtr.add_suppression("Replica SQL for channel 'group_replication_recovery': ... The replica coordinator and worker threads are stopped.*"); call mtr.add_suppression("Error while starting the group replication incremental recovery receiver/applier threads"); call mtr.add_suppression("Replica I/O for channel 'group_replication_recovery': Source command COM_REGISTER_REPLICA failed: Access denied for user 'recovery_user'.*"); call mtr.add_suppression("Replica I/O for channel 'group_replication_recovery': Source command COM_REGISTER_REPLICA failed: failed registering on source, reconnecting to try again.*"); call mtr.add_suppression("Replica I/O thread couldn't register on source"); call mtr.add_suppression(".*This member has more executed transactions *.*"); call mtr.add_suppression("The member contains transactions not present in the group. It is only allowed to *.*"); SET SESSION sql_log_bin= 1; SET @debug_save_rec_int= @@GLOBAL.group_replication_recovery_reconnect_interval; --disable_warnings --eval CHANGE REPLICATION SOURCE TO SOURCE_USER='$recovery_user', SOURCE_PASSWORD='$recovery_password' FOR CHANNEL 'group_replication_recovery' --enable_warnings SET GLOBAL group_replication_recovery_reconnect_interval= 1; # 1 second SET SESSION sql_log_bin= 0; CREATE TABLE lost_test_table (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB; INSERT INTO lost_test_table VALUES (1); SET SESSION sql_log_bin= 1; --eval SET GLOBAL group_replication_group_name= "$group_replication_group_name" --source include/start_group_replication_command.inc #give it time to fail several times --sleep 5 --let $group_replication_member_state= RECOVERING --source include/gr_wait_for_member_state.inc --echo # --echo # Create the recovery user on server 1 --echo # The joiner should now connect to server 1 when failing over. --echo # --connection server1 SET SESSION sql_log_bin= 0; --eval CREATE USER "$recovery_user" IDENTIFIED BY "$recovery_password" --eval GRANT REPLICATION SLAVE ON *.* TO "$recovery_user" --eval GRANT GROUP_REPLICATION_STREAM ON *.* TO "$recovery_user"@'%'; FLUSH PRIVILEGES; SET SESSION sql_log_bin= 1; --connection server3 --let $group_replication_member_state= ONLINE --source include/gr_wait_for_member_state.inc --let $assert_text= On the recovered member, the table should exist and have 1 elements --let $assert_cond= [select count(*) from t1] = 1; --source include/assert.inc --echo # --echo # Cleaning up --echo # We use RESET MASTER here due to deviations on all members GTID sets --echo # SET @@GLOBAL.group_replication_recovery_reconnect_interval= @debug_save_rec_int; DROP TABLE t1; --source include/stop_group_replication.inc DROP TABLE lost_test_table; RESET MASTER; SET SESSION sql_log_bin= 0; --eval DROP USER "$recovery_user" SET SESSION sql_log_bin= 1; --connection server2 --source include/stop_group_replication.inc SET SESSION sql_log_bin= 0; --eval DROP USER "$recovery_user" SET SESSION sql_log_bin= 1; DROP TABLE lost_test_table; RESET MASTER; --connection server1 --source include/stop_group_replication.inc SET SESSION sql_log_bin= 0; --eval DROP USER "$recovery_user" SET SESSION sql_log_bin= 1; RESET MASTER; --source include/group_replication_end.inc