| 1 |
################################################################################ |
| 2 |
# Recovery donor failover test |
| 3 |
# This test focus on the failover mechanism where a joiner changes it's donor |
| 4 |
# whenever the one in use leaves the group while recovery is happening. |
| 5 |
# |
| 6 |
# To test this, a new member is started but recovery blocked. While at this state |
| 7 |
# the donor is killed, forcing the joiner to change to a new one. As there is |
| 8 |
# no donor identification as of now, we just unblock and check that recovery |
| 9 |
# was successful |
| 10 |
# |
| 11 |
# Test: |
| 12 |
# 0. The test requires three servers: M1, M2 and M3. |
| 13 |
# 1. Bootstrap group on M1. Insert some data. Start GR on M2. |
| 14 |
# 2. Block the applier on both servers to block recovery. |
| 15 |
# 3. Start GR on M3. It should go into RECOVERING state. Identify the DONOR. |
| 16 |
# 4. Stop GR on the donor making it leave the group. |
| 17 |
# 5. Unblock recovery i.e. Resume the applier on failover member and watch M3 go |
| 18 |
# ONLINE despite the donor exit. Vaidate data on M3. |
| 19 |
# 6. Clean up. |
| 20 |
################################################################################ |
| 21 |
|
| 22 |
--source include/big_test.inc |
| 23 |
--source include/have_debug_sync.inc |
| 24 |
--let $group_replication_group_name= 55d07150-9a4d-11e3-a5e2-0800200c9a66 |
| 25 |
--source include/have_group_replication_plugin.inc |
| 26 |
--let $rpl_skip_group_replication_start= 1 |
| 27 |
--let $rpl_server_count= 3 |
| 28 |
--source include/group_replication.inc |
| 29 |
|
| 30 |
--echo # |
| 31 |
--echo # Setup a new 2 member group |
| 32 |
--echo # |
| 33 |
|
| 34 |
--connection server1 |
| 35 |
SET SESSION sql_log_bin = 0; |
| 36 |
call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': Relay log read failure: Could not parse relay log event entry.*"); |
| 37 |
call mtr.add_suppression("The applier thread execution was aborted. Unable to process more transactions, this member will now leave the group."); |
| 38 |
call mtr.add_suppression("Fatal error during execution on the Applier process of Group Replication. The server will now leave the group."); |
| 39 |
call mtr.add_suppression("The server was automatically set into read only mode after an error was detected."); |
| 40 |
call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going."); |
| 41 |
SET SESSION sql_log_bin = 1; |
| 42 |
|
| 43 |
--source include/start_and_bootstrap_group_replication.inc |
| 44 |
|
| 45 |
--let $group_replication_number_of_members= 1 |
| 46 |
--source include/gr_wait_for_number_of_members.inc |
| 47 |
|
| 48 |
#insert some data |
| 49 |
CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB; |
| 50 |
INSERT INTO t1 VALUES (1); |
| 51 |
|
| 52 |
--connection server2 |
| 53 |
SET SESSION sql_log_bin = 0; |
| 54 |
call mtr.add_suppression("Replica SQL for channel 'group_replication_applier': Relay log read failure: Could not parse relay log event entry.*"); |
| 55 |
call mtr.add_suppression("The applier thread execution was aborted. Unable to process more transactions, this member will now leave the group."); |
| 56 |
call mtr.add_suppression("Fatal error during execution on the Applier process of Group Replication. The server will now leave the group."); |
| 57 |
call mtr.add_suppression("The server was automatically set into read only mode after an error was detected."); |
| 58 |
call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going."); |
| 59 |
SET SESSION sql_log_bin = 1; |
| 60 |
|
| 61 |
#extract server uuid for calculations below |
| 62 |
--let $server2_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) |
| 63 |
|
| 64 |
--source include/start_group_replication.inc |
| 65 |
|
| 66 |
--let $group_replication_number_of_members= 2 |
| 67 |
--source include/gr_wait_for_number_of_members.inc |
| 68 |
|
| 69 |
#Block the applier on both servers to block recovery |
| 70 |
SET @@GLOBAL.DEBUG='+d,block_applier_updates'; |
| 71 |
|
| 72 |
--connection server1 |
| 73 |
SET @@GLOBAL.DEBUG='+d,block_applier_updates'; |
| 74 |
|
| 75 |
--echo # |
| 76 |
--echo # Start recovery on a new member and kill the donor |
| 77 |
--echo # |
| 78 |
|
| 79 |
--connection server3 |
| 80 |
SET SESSION sql_log_bin= 0; |
| 81 |
call mtr.add_suppression(".*Error when configuring the asynchronous recovery channel connection *.*"); |
| 82 |
SET SESSION sql_log_bin= 1; |
| 83 |
|
| 84 |
--let $group_replication_start_member_state= RECOVERING |
| 85 |
--source include/start_group_replication.inc |
| 86 |
|
| 87 |
--echo # Find if server 1 or 2 is the donor |
| 88 |
|
| 89 |
#give it time to connect |
| 90 |
--sleep 5 |
| 91 |
|
| 92 |
#Find what member is the donor |
| 93 |
--let $donor_id= 1 |
| 94 |
--let $failover= 2 |
| 95 |
if (`SELECT COUNT(*)=1 FROM performance_schema.replication_connection_status WHERE channel_name='group_replication_recovery' AND service_state='ON' AND source_uuid='$server2_uuid'`) |
| 96 |
{ |
| 97 |
--let $donor_id= 2 |
| 98 |
--let $failover= 1 |
| 99 |
} |
| 100 |
|
| 101 |
--echo # Stop group replication on the donor making it leave the group |
| 102 |
--connection server$donor_id |
| 103 |
SET DEBUG_SYNC = "now WAIT_FOR applier_read_blocked"; |
| 104 |
SET @@GLOBAL.DEBUG='-d,block_applier_updates'; |
| 105 |
SET @@GLOBAL.DEBUG='+d,force_sql_thread_error'; |
| 106 |
SET DEBUG_SYNC = "now SIGNAL resume_applier_read"; |
| 107 |
--let $group_replication_member_state= ERROR |
| 108 |
--source include/gr_wait_for_member_state.inc |
| 109 |
SET @@GLOBAL.DEBUG='-d,force_sql_thread_error'; |
| 110 |
SET DEBUG_SYNC= 'RESET'; |
| 111 |
--source include/stop_group_replication.inc |
| 112 |
|
| 113 |
--echo # Unblock recovery and watch the member go online despite the donor exit |
| 114 |
--connection server$failover |
| 115 |
SET DEBUG_SYNC = "now WAIT_FOR applier_read_blocked"; |
| 116 |
SET @@GLOBAL.DEBUG='-d,block_applier_updates'; |
| 117 |
SET DEBUG_SYNC = "now SIGNAL resume_applier_read"; |
| 118 |
|
| 119 |
--connection server3 |
| 120 |
|
| 121 |
--let $group_replication_member_state= ONLINE |
| 122 |
--source include/gr_wait_for_member_state.inc |
| 123 |
|
| 124 |
--let $assert_text= On the recovered member, the table should exist and have 1 elements; |
| 125 |
--let $assert_cond= [select count(*) from t1] = 1; |
| 126 |
--source include/assert.inc |
| 127 |
|
| 128 |
--let $assert_text= On the recovered member, the executed GTID should be the same as on the donor plus the view changes |
| 129 |
--let $assert_cond= "[SELECT @@GLOBAL.GTID_EXECUTED]" = "55d07150-9a4d-11e3-a5e2-0800200c9a66:1-5"; |
| 130 |
--source include/assert.inc |
| 131 |
|
| 132 |
--echo # |
| 133 |
--echo # Cleaning up |
| 134 |
--echo # |
| 135 |
|
| 136 |
--connection server$failover |
| 137 |
SET DEBUG_SYNC= 'RESET'; |
| 138 |
|
| 139 |
--connection server$donor_id |
| 140 |
--source include/start_group_replication.inc |
| 141 |
|
| 142 |
DROP TABLE t1; |
| 143 |
|
| 144 |
--source include/group_replication_end.inc |