| 1 |
################################################################################ |
| 2 |
# This test causes to consecutive group updates on recovery in a failure |
| 3 |
# situation. First the only donor leaves (first view change) and then as a |
| 4 |
# consequence the joiner also leaves as he cannot recover. |
| 5 |
# |
| 6 |
# Test: |
| 7 |
# 0) The test requires two servers. |
| 8 |
# 1) Start group replication on server1. Add some data for recovery. |
| 9 |
# 2) Set a bad password on server2 so it cannot recover. |
| 10 |
# Also set only 2 recovery attempts with a sleep time of 10 seconds |
| 11 |
# 3) Start group replication on server2 and wait for it to fail once. |
| 12 |
# 4) Make the donor (server1) leave. |
| 13 |
# 5) Wait for the joiner (server2) to be in ERROR state. |
| 14 |
# 6) Check all is fine. |
| 15 |
# 7) Clean up. |
| 16 |
################################################################################ |
| 17 |
|
| 18 |
--source include/big_test.inc |
| 19 |
--source include/have_debug.inc |
| 20 |
--source include/have_debug_sync.inc |
| 21 |
--source include/have_group_replication_xcom_communication_stack.inc |
| 22 |
--let $group_replication_group_name= 6749cab0-93ae-11e5-a837-0800200c9a66 |
| 23 |
--source include/have_group_replication_plugin.inc |
| 24 |
--let $rpl_skip_group_replication_start= 1 |
| 25 |
--source include/group_replication.inc |
| 26 |
|
| 27 |
|
| 28 |
--echo # |
| 29 |
--echo # Start group replication on server 1 |
| 30 |
--echo # |
| 31 |
|
| 32 |
--connection server1 |
| 33 |
--echo server1 |
| 34 |
--source include/start_and_bootstrap_group_replication.inc |
| 35 |
|
| 36 |
# Add some data for recovery |
| 37 |
CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB; |
| 38 |
|
| 39 |
--echo # |
| 40 |
--echo # Set server 2 options to fail on recovery |
| 41 |
--echo # |
| 42 |
|
| 43 |
--connection server2 |
| 44 |
--echo server2 |
| 45 |
|
| 46 |
SET @debug_save_rec_int= @@GLOBAL.group_replication_recovery_reconnect_interval; |
| 47 |
SET @debug_save_ret_count= @@GLOBAL.group_replication_recovery_retry_count; |
| 48 |
|
| 49 |
--disable_warnings |
| 50 |
CHANGE REPLICATION SOURCE TO SOURCE_PASSWORD='bad_password' FOR CHANNEL 'group_replication_recovery'; |
| 51 |
--enable_warnings |
| 52 |
|
| 53 |
--eval SET GLOBAL group_replication_recovery_reconnect_interval= 10 # seconds |
| 54 |
--eval SET GLOBAL group_replication_recovery_retry_count= 2 |
| 55 |
|
| 56 |
--echo # |
| 57 |
--echo # Set server 2 making sure 2 views are received when recovery fails |
| 58 |
--echo # |
| 59 |
|
| 60 |
set session sql_log_bin=0; |
| 61 |
call mtr.add_suppression("There was an error when connecting to the donor*"); |
| 62 |
call mtr.add_suppression("For details please check performance_schema.replication_connection_status table and error log messages of Replica I/O for channel group_replication_recovery."); |
| 63 |
call mtr.add_suppression("Maximum number of retries when*"); |
| 64 |
call mtr.add_suppression("Fatal error during the incremental recovery process of Group Replication.*"); |
| 65 |
call mtr.add_suppression("The member is leaving a group without being on one"); |
| 66 |
call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going."); |
| 67 |
call mtr.add_suppression("The server was automatically set into read only mode after an error was detected."); |
| 68 |
# on slow runs (valgrind) it can happen because the connection |
| 69 |
# of the recovery thread may be slower than 5 seconds and the |
| 70 |
# server1 may actually be stopped before the reconnection attempt |
| 71 |
# TODO: the sleeps below should be replaced by debug sync points |
| 72 |
# For now ensure that we get coverage in most of the runs and |
| 73 |
# do not fail on valgrind |
| 74 |
call mtr.add_suppression("All donors left. Aborting group replication incremental recovery.*"); |
| 75 |
set session sql_log_bin=1; |
| 76 |
|
| 77 |
SET @debug_save= @@GLOBAL.DEBUG; |
| 78 |
SET @@GLOBAL.DEBUG='d,recovery_thread_wait_before_cleanup'; |
| 79 |
|
| 80 |
--eval SET GLOBAL group_replication_group_name= "$group_replication_group_name" |
| 81 |
--source include/start_group_replication_command.inc |
| 82 |
|
| 83 |
--echo # |
| 84 |
--echo # Give time for server 2 to fail the connection once |
| 85 |
--echo # Stop recovery on the donor (server1) |
| 86 |
--echo # |
| 87 |
|
| 88 |
# give it time to fail |
| 89 |
--sleep 5 |
| 90 |
|
| 91 |
--connection server1 |
| 92 |
--echo server1 |
| 93 |
--source include/stop_group_replication.inc |
| 94 |
|
| 95 |
#give it time to fail and leave as there are no donors. |
| 96 |
--sleep 10 |
| 97 |
|
| 98 |
--echo # |
| 99 |
--echo # Watch server 2 enter an error state. |
| 100 |
--echo # |
| 101 |
|
| 102 |
--connection server2 |
| 103 |
--echo server2 |
| 104 |
|
| 105 |
SET DEBUG_SYNC= "now SIGNAL signal.recovery_end_end"; |
| 106 |
SET @@GLOBAL.DEBUG= @debug_save; |
| 107 |
|
| 108 |
--let $group_replication_member_state= ERROR |
| 109 |
--source include/gr_wait_for_member_state.inc |
| 110 |
|
| 111 |
--echo # |
| 112 |
--echo # Check all is fine |
| 113 |
--echo # |
| 114 |
|
| 115 |
--connection server1 |
| 116 |
--echo server1 |
| 117 |
--source include/start_and_bootstrap_group_replication.inc |
| 118 |
|
| 119 |
--connection server2 |
| 120 |
--echo server2 |
| 121 |
|
| 122 |
--source include/stop_group_replication.inc |
| 123 |
--disable_warnings |
| 124 |
CHANGE REPLICATION SOURCE TO SOURCE_USER='root', SOURCE_PASSWORD='' FOR CHANNEL 'group_replication_recovery'; |
| 125 |
--enable_warnings |
| 126 |
--source include/start_group_replication.inc |
| 127 |
|
| 128 |
INSERT INTO t1 VALUES (1); |
| 129 |
|
| 130 |
--source include/rpl_sync.inc |
| 131 |
|
| 132 |
--let $assert_text= The table should contain 1 elements |
| 133 |
--let $assert_cond= [SELECT COUNT(*) FROM t1] = 1; |
| 134 |
--source include/assert.inc |
| 135 |
|
| 136 |
DROP TABLE t1; |
| 137 |
|
| 138 |
--echo # |
| 139 |
--echo # Cleanup |
| 140 |
--echo # |
| 141 |
|
| 142 |
SET @@GLOBAL.group_replication_recovery_reconnect_interval= @debug_save_rec_int; |
| 143 |
SET @@GLOBAL.group_replication_recovery_retry_count= @debug_save_ret_count; |
| 144 |
|
| 145 |
--source include/group_replication_end.inc |