| 1 |
# ==== Purpose ==== |
| 2 |
# |
| 3 |
# This test case will test if a member server will not fail to know that the |
| 4 |
# replication threads of the group replication recovery have stopped. |
| 5 |
# |
| 6 |
# The applier thread test: |
| 7 |
# |
| 8 |
# During the startup of the applier thread of the group replication recovery |
| 9 |
# on server2, debug points will ensure that the hook for getting the stop |
| 10 |
# signal will only be set up after the applier thread has passed the hook |
| 11 |
# notification point. This will ensure that, even missing the stop |
| 12 |
# notification, the group replication recovery will still detect that the |
| 13 |
# applier thread has stopped. |
| 14 |
# |
| 15 |
# The receiver thread test: |
| 16 |
# |
| 17 |
# The test case will turn the only possible donor into an improper |
| 18 |
# one by purging its binary log. In this way, the server trying to recover |
| 19 |
# will not succeed, restating the receiver thread for some time. |
| 20 |
# |
| 21 |
# During the restart of the receiver thread of the group replication recovery |
| 22 |
# on server3, debug points will ensure that the hook for getting the stop |
| 23 |
# signal will only be set up after the receiver thread has passed the hook |
| 24 |
# notification point. This will ensure that, even missing the stop |
| 25 |
# notification, the group replication recovery will still detect that the |
| 26 |
# receiver thread has stopped. |
| 27 |
# |
| 28 |
# ==== Related Bugs and Worklogs ==== |
| 29 |
# |
| 30 |
# BUG#23502642 TEST GROUP_REPLICATION_RECOVERY_PURGED_DONOR_FAILOVER FAILING |
| 31 |
# |
| 32 |
--source include/big_test.inc |
| 33 |
--source include/have_debug.inc |
| 34 |
--source include/have_debug_sync.inc |
| 35 |
--source include/have_group_replication_plugin.inc |
| 36 |
--let $rpl_skip_group_replication_start= 1 |
| 37 |
--let $rpl_server_count= 3 |
| 38 |
--source include/group_replication.inc |
| 39 |
|
| 40 |
--echo |
| 41 |
--echo ##################################################################### |
| 42 |
--echo # 1. Create a table on server1 and generate workload to be recovered |
| 43 |
|
| 44 |
--source include/start_and_bootstrap_group_replication.inc |
| 45 |
|
| 46 |
# Insert some data |
| 47 |
CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB; |
| 48 |
INSERT INTO t1 VALUES (1); |
| 49 |
|
| 50 |
--echo |
| 51 |
--echo ######################## |
| 52 |
--echo # 2. SQL thread testing |
| 53 |
|
| 54 |
--echo |
| 55 |
--echo ####################################################### |
| 56 |
--echo # 2.1 Try to make server2 to join the group for 3 times |
| 57 |
--echo # forcing the SQL thread to stop |
| 58 |
|
| 59 |
--let $rpl_connection_name= server2 |
| 60 |
--source include/rpl_connection.inc |
| 61 |
SET SESSION sql_log_bin= 0; |
| 62 |
CALL mtr.add_suppression("Error while creating the group replication recovery channel with donor"); |
| 63 |
CALL mtr.add_suppression("Maximum number of retries when trying to connect to a donor reached"); |
| 64 |
CALL mtr.add_suppression("Fatal error during the incremental recovery process of Group Replication. The server will leave the group."); |
| 65 |
CALL mtr.add_suppression("Already leaving or joining a group"); |
| 66 |
CALL mtr.add_suppression("Error leaving the group"); |
| 67 |
CALL mtr.add_suppression("Table 't1' already exists"); |
| 68 |
CALL mtr.add_suppression("Error while starting the group replication incremental recovery receiver/applier threads"); |
| 69 |
call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going."); |
| 70 |
call mtr.add_suppression("The server was automatically set into read only mode after an error was detected."); |
| 71 |
SET SESSION sql_log_bin= 1; |
| 72 |
|
| 73 |
SET @saved_reconnect_interval= @@GLOBAL.group_replication_recovery_reconnect_interval; |
| 74 |
--eval SET GLOBAL group_replication_recovery_reconnect_interval= 1 # second |
| 75 |
SET @saved_retry_count= @@GLOBAL.group_replication_recovery_retry_count; |
| 76 |
--eval SET GLOBAL group_replication_recovery_retry_count= 3 |
| 77 |
|
| 78 |
# This is to make the SQL thread to fail |
| 79 |
SET SESSION sql_log_bin= 0; |
| 80 |
CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB; |
| 81 |
SET SESSION sql_log_bin= 1; |
| 82 |
--let $debug_point=pause_after_sql_thread_stop_hook |
| 83 |
--source include/add_debug_point.inc |
| 84 |
|
| 85 |
--echo |
| 86 |
--echo ##################################################### |
| 87 |
--echo # 2.2 Wait until server2 gave up on joining the group |
| 88 |
|
| 89 |
--let $group_replication_start_member_state= ERROR |
| 90 |
--source include/start_group_replication.inc |
| 91 |
|
| 92 |
--source include/remove_debug_point.inc |
| 93 |
|
| 94 |
SET @@GLOBAL.group_replication_recovery_reconnect_interval= @saved_reconnect_interval; |
| 95 |
SET @@GLOBAL.group_replication_recovery_retry_count= @saved_retry_count; |
| 96 |
|
| 97 |
# Wait until server2 has stopped Group Replication |
| 98 |
--source include/stop_group_replication.inc |
| 99 |
|
| 100 |
SET SESSION sql_log_bin= 0; |
| 101 |
DROP TABLE t1; |
| 102 |
SET SESSION sql_log_bin= 1; |
| 103 |
|
| 104 |
--echo |
| 105 |
--echo ##################################################################### |
| 106 |
--echo # 2.3 Wait until server2 had left the group to move to the next phase |
| 107 |
|
| 108 |
--connection server1 |
| 109 |
--let $group_replication_number_of_members= 1 |
| 110 |
--source include/gr_wait_for_number_of_members.inc |
| 111 |
|
| 112 |
--echo |
| 113 |
--echo ####################### |
| 114 |
--echo # 3. I/O thread testing |
| 115 |
|
| 116 |
--echo |
| 117 |
--echo ############################################################################ |
| 118 |
--echo # 3.1 Rotate and purge the binary log on server1 to make recovery unfeasible |
| 119 |
|
| 120 |
FLUSH BINARY LOGS; |
| 121 |
--let $server_binlog_file_cur= query_get_value(SHOW MASTER STATUS, File, 1) |
| 122 |
|
| 123 |
# Need to kill the dump thread explicitly: stop slave does not do |
| 124 |
# that, and if the dump thread is slow, it will still be reading from |
| 125 |
# the old binlog at the time we do PURGE BINARY LOGS below. That would |
| 126 |
# prevent PURGE BINARY LOGS from purging the log. |
| 127 |
--source include/stop_dump_threads.inc |
| 128 |
|
| 129 |
--replace_result $server_binlog_file_cur CURRENT_BINLOG_FILE |
| 130 |
--eval PURGE BINARY LOGS TO '$server_binlog_file_cur' |
| 131 |
|
| 132 |
SET SESSION sql_log_bin= 0; |
| 133 |
CALL mtr.add_suppression("Cannot replicate to server with server_uuid*"); |
| 134 |
SET SESSION sql_log_bin= 1; |
| 135 |
|
| 136 |
--echo |
| 137 |
--echo ############################################################# |
| 138 |
--echo # 3.2 Try to make server3 to join the group for up to 3 times |
| 139 |
|
| 140 |
--connection server3 |
| 141 |
SET SESSION sql_log_bin= 0; |
| 142 |
CALL mtr.add_suppression("Replica I/O for channel 'group_replication_recovery': Got fatal error 1236*"); |
| 143 |
CALL mtr.add_suppression("Maximum number of retries when trying to connect to a donor reached"); |
| 144 |
CALL mtr.add_suppression("Fatal error during the incremental recovery process of Group Replication. The server will leave the group."); |
| 145 |
CALL mtr.add_suppression("Already leaving or joining a group"); |
| 146 |
CALL mtr.add_suppression("Error leaving the group"); |
| 147 |
CALL mtr.add_suppression("Error while starting the group replication incremental recovery"); |
| 148 |
call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going."); |
| 149 |
call mtr.add_suppression("This member will start distributed recovery using clone. It is due to no ONLINE member has the missing data for recovering in its binary logs."); |
| 150 |
call mtr.add_suppression("There was an issue when configuring the remote cloning process: The clone plugin is not present or active in this server."); |
| 151 |
call mtr.add_suppression("No valid or ONLINE members exist to get the missing data from the group. For cloning check if donors of the same version and with clone plugin installed exist. For incremental recovery check.*"); |
| 152 |
call mtr.add_suppression("The server was automatically set into read only mode after an error was detected."); |
| 153 |
SET SESSION sql_log_bin= 1; |
| 154 |
|
| 155 |
SET @saved_reconnect_interval= @@GLOBAL.group_replication_recovery_reconnect_interval; |
| 156 |
--eval SET GLOBAL group_replication_recovery_reconnect_interval= 1 # second |
| 157 |
SET @saved_retry_count= @@GLOBAL.group_replication_recovery_retry_count; |
| 158 |
--eval SET GLOBAL group_replication_recovery_retry_count= 3 |
| 159 |
|
| 160 |
--let $debug_point=pause_after_io_thread_stop_hook |
| 161 |
--source include/add_debug_point.inc |
| 162 |
|
| 163 |
--echo |
| 164 |
--echo ##################################################### |
| 165 |
--echo # 3.3 Wait until server3 gave up on joining the group |
| 166 |
|
| 167 |
--let $group_replication_start_member_state= ERROR |
| 168 |
--source include/start_group_replication.inc |
| 169 |
|
| 170 |
SET @@GLOBAL.group_replication_recovery_reconnect_interval= @saved_reconnect_interval; |
| 171 |
SET @@GLOBAL.group_replication_recovery_retry_count= @saved_retry_count; |
| 172 |
|
| 173 |
--source include/remove_debug_point.inc |
| 174 |
|
| 175 |
--echo |
| 176 |
--echo ################################################################ |
| 177 |
--echo # 4. Wait until server1 be alone in the group to do the clean up |
| 178 |
|
| 179 |
--connection server1 |
| 180 |
--let $group_replication_number_of_members= 1 |
| 181 |
--source include/gr_wait_for_number_of_members.inc |
| 182 |
|
| 183 |
DROP TABLE t1; |
| 184 |
|
| 185 |
--source include/group_replication_end.inc |