| 1 |
--source include/no_valgrind_without_big.inc |
| 2 |
|
| 3 |
# |
| 4 |
# Purpose: |
| 5 |
# This test validates that data is consistent between a primary and replica |
| 6 |
# in semi-sync mode when the primary is issued `SHUTDOWN WAIT FOR SLAVES` |
| 7 |
# during an active communication. More specifically, the primary should not |
| 8 |
# kill the Ack_thread until it is sure a replica has received all binlog |
| 9 |
# data, i.e. once the primary receives the ACK. If a primary is issued a |
| 10 |
# shutdown before receiving an ACK, it should wait until either 1) the ACK is |
| 11 |
# received, or 2) the configured timeout (rpl_semi_sync_master_timeout) is |
| 12 |
# reached. |
| 13 |
# |
| 14 |
# Methodology: |
| 15 |
# Using a topology consisting of one primary with two replicas, all in |
| 16 |
# semi-sync mode, we use DEBUG_DBUG to simulate an error or delay on the |
| 17 |
# replicas during an active communication while the primary is issued |
| 18 |
# `SHUTDOWN WAIT FOR SLAVES`. We create four test cases to ensure the |
| 19 |
# Ack_thread is not prematurely killed due to the shutdown. |
| 20 |
# Test Case 1) If both replicas simulate a delay that is within the allowed |
| 21 |
# timeout, the primary should delay killing the Ack_thread |
| 22 |
# until an ACK is received. |
| 23 |
# Test Case 2) If both replicas simulate an error before sending an ACK, the |
| 24 |
# primary should delay killing the Ack_thread until the |
| 25 |
# the timeout is reached. |
| 26 |
# Test Case 3) If one replica simulates a delay within the allowed timeout |
| 27 |
# and the other simulates an error before sending an ACK, the |
| 28 |
# primary should delay killing the Ack_thread until it |
| 29 |
# receives an ACK from the delayed slave. |
| 30 |
# Test Case 4) If a replica errors before sending an ACK, it will cause the |
| 31 |
# IO thread to stop and handle the error. During error handling, |
| 32 |
# if semi-sync is active, the replica will form a new connection |
| 33 |
# with the primary to kill the active connection. However, if |
| 34 |
# the primary is shutting down, it may kill the new connection, |
| 35 |
# thereby leaving the active semi-sync connection in-tact. The |
| 36 |
# slave should notice this, and not issue a `QUIT` command to |
| 37 |
# the primary, which would otherwise be sent to kill an active |
| 38 |
# connection. This test case validates that the slave does not |
| 39 |
# send a `QUIT` in this case. |
| 40 |
# Test Case 5) If a waiting-for-ACK user thread is killed (disconnected) |
| 41 |
# during SHUTDOWN WAIT FOR ALL SLAVES, ensure the primary will |
| 42 |
# still await the ACK from the replica before killing the |
| 43 |
# Ack_thread. |
| 44 |
# |
| 45 |
# References: |
| 46 |
# MDEV-11853: semisync thread can be killed after sync binlog but before ACK |
| 47 |
# in the sync state |
| 48 |
# MDEV-28114: Semi-sync Master ACK Receiver Thread Can Error on COM_QUIT |
| 49 |
# |
| 50 |
|
| 51 |
--echo ############################# |
| 52 |
--echo # Common setup for all tests |
| 53 |
--echo ############################# |
| 54 |
|
| 55 |
--echo # Note: Simulated slave delay is hardcoded to 800 milliseconds |
| 56 |
--echo # Note: Simulated master shutdown delay is hardcoded to 500 milliseconds |
| 57 |
|
| 58 |
--source include/have_innodb.inc |
| 59 |
--source include/have_debug.inc |
| 60 |
--let $rpl_topology=1->2, 1->3 |
| 61 |
--source include/rpl_init.inc |
| 62 |
|
| 63 |
--connection server_1 |
| 64 |
|
| 65 |
--echo # Slaves which simulate an error will produce a timeout on the primary |
| 66 |
call mtr.add_suppression("Timeout waiting"); |
| 67 |
call mtr.add_suppression("did not exit"); |
| 68 |
call mtr.add_suppression("Got an error reading communication packets"); |
| 69 |
|
| 70 |
--let $sav_master_timeout= `SELECT @@GLOBAL.rpl_semi_sync_master_timeout` |
| 71 |
--let $sav_enabled_master= `SELECT @@GLOBAL.rpl_semi_sync_master_enabled` |
| 72 |
|
| 73 |
--echo # Suppress slave errors related to the simulated error |
| 74 |
--connection server_2 |
| 75 |
call mtr.add_suppression("reply failed"); |
| 76 |
call mtr.add_suppression("Replication event checksum verification"); |
| 77 |
call mtr.add_suppression("Relay log write failure"); |
| 78 |
call mtr.add_suppression("Failed to kill the active semi-sync connection"); |
| 79 |
set @sav_enabled_server_2= @@GLOBAL.rpl_semi_sync_slave_enabled; |
| 80 |
set @sav_server_2_dbug= @@GLOBAL.debug_dbug; |
| 81 |
|
| 82 |
--connection server_3 |
| 83 |
call mtr.add_suppression("reply failed"); |
| 84 |
call mtr.add_suppression("Replication event checksum verification"); |
| 85 |
call mtr.add_suppression("Relay log write failure"); |
| 86 |
call mtr.add_suppression("Failed to kill the active semi-sync connection"); |
| 87 |
set @sav_enabled_server_3= @@GLOBAL.rpl_semi_sync_slave_enabled; |
| 88 |
set @sav_server_3_dbug= @@GLOBAL.debug_dbug; |
| 89 |
|
| 90 |
--connection server_1 |
| 91 |
CREATE TABLE t1 (a int) engine=innodb; |
| 92 |
--save_master_pos |
| 93 |
|
| 94 |
--let i= 2 |
| 95 |
--let slave_last= 3 |
| 96 |
while (`SELECT $i <= $slave_last`) |
| 97 |
{ |
| 98 |
--connection server_$i |
| 99 |
--sync_with_master |
| 100 |
--inc $i |
| 101 |
} |
| 102 |
|
| 103 |
# Set up the connection used to issue the shutdown |
| 104 |
--connect(server_1_con2, localhost, root,,) |
| 105 |
|
| 106 |
|
| 107 |
--echo ############################# |
| 108 |
--echo # Test cases |
| 109 |
--echo ############################# |
| 110 |
|
| 111 |
--echo # |
| 112 |
--echo # Test Case 1) If both replicas simulate a delay that is within the |
| 113 |
--echo # allowed timeout, the primary should delay killing the Ack_thread |
| 114 |
--echo # until an ACK is received. |
| 115 |
--echo # |
| 116 |
--let server_2_sync_slave_reply=1 |
| 117 |
--let server_2_dbug= "+d,simulate_delay_semisync_slave_reply" |
| 118 |
--let server_3_sync_slave_reply=1 |
| 119 |
--let server_3_dbug= "+d,simulate_delay_semisync_slave_reply" |
| 120 |
--let semisync_timeout= 1600 |
| 121 |
--let server_2_expect_row_count= 1 |
| 122 |
--let server_3_expect_row_count= 1 |
| 123 |
--source rpl_semi_sync_shutdown_await_ack.inc |
| 124 |
|
| 125 |
--echo # |
| 126 |
--echo # Test Case 2) If both replicas simulate an error before sending an ACK, |
| 127 |
--echo # the primary should delay killing the Ack_thread until the |
| 128 |
--echo # timeout is reached. |
| 129 |
--echo # |
| 130 |
--let server_2_sync_slave_reply=0 |
| 131 |
--let server_2_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141" |
| 132 |
--let server_3_sync_slave_reply=0 |
| 133 |
--let server_3_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141" |
| 134 |
--let semisync_timeout= 500 |
| 135 |
--let server_2_expect_row_count= 0 |
| 136 |
--let server_3_expect_row_count= 0 |
| 137 |
--source rpl_semi_sync_shutdown_await_ack.inc |
| 138 |
|
| 139 |
--echo # |
| 140 |
--echo # Test Case 3) If one replica simulates a delay within the allowed |
| 141 |
--echo # timeout and the other simulates an error before sending an ACK, the |
| 142 |
--echo # primary should delay killing the Ack_thread until it receives an |
| 143 |
--echo # ACK from the delayed slave. |
| 144 |
--echo # |
| 145 |
--let server_2_sync_slave_reply=0 |
| 146 |
--let server_2_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141" |
| 147 |
--let server_3_sync_slave_reply=1 |
| 148 |
--let server_3_dbug= "+d,simulate_delay_semisync_slave_reply" |
| 149 |
--let semisync_timeout= 1600 |
| 150 |
--let server_2_expect_row_count= 0 |
| 151 |
--let server_3_expect_row_count= 1 |
| 152 |
--source rpl_semi_sync_shutdown_await_ack.inc |
| 153 |
|
| 154 |
--echo # |
| 155 |
--echo # Test Case 4) If a replica errors before sending an ACK, it will cause |
| 156 |
--echo # the IO thread to stop and handle the error. During error handling, if |
| 157 |
--echo # semi-sync is active, the replica will form a new connection with the |
| 158 |
--echo # primary to kill the active connection. However, if the primary is |
| 159 |
--echo # shutting down, it may kill the new connection, thereby leaving the |
| 160 |
--echo # active semi-sync connection in-tact. The slave should notice this, and |
| 161 |
--echo # not issue a `QUIT` command to the primary, which would otherwise be |
| 162 |
--echo # sent to kill an active connection. This test case validates that the |
| 163 |
--echo # slave does not send a `QUIT` in this case. |
| 164 |
--echo # |
| 165 |
--let server_2_sync_slave_reply=0 |
| 166 |
--let server_2_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141" |
| 167 |
--let server_3_sync_slave_reply=1 |
| 168 |
--let server_3_dbug= "+d,simulate_delay_semisync_slave_reply" |
| 169 |
--let semisync_timeout= 1600 |
| 170 |
--let server_2_expect_row_count= 0 |
| 171 |
--let server_3_expect_row_count= 1 |
| 172 |
--source rpl_semi_sync_shutdown_await_ack.inc |
| 173 |
|
| 174 |
# |
| 175 |
# Added with MDEV-33551 |
| 176 |
# |
| 177 |
--echo # |
| 178 |
--echo # Test Case 5) If a waiting-for-ACK user thread is killed (disconnected) |
| 179 |
--echo # during SHUTDOWN WAIT FOR ALL SLAVES, ensure the primary will still |
| 180 |
--echo # await the ACK from the replica before killing the Ack_receiver thread |
| 181 |
--echo # |
| 182 |
--connection server_1 |
| 183 |
insert into t1 values (1); |
| 184 |
--source include/save_master_gtid.inc |
| 185 |
|
| 186 |
--connection server_2 |
| 187 |
--source include/sync_with_master_gtid.inc |
| 188 |
--source include/stop_slave.inc |
| 189 |
SET GLOBAL rpl_semi_sync_slave_enabled= 1; |
| 190 |
--source include/start_slave.inc |
| 191 |
|
| 192 |
--connection server_1 |
| 193 |
SET GLOBAL rpl_semi_sync_master_enabled= 1; |
| 194 |
SET GLOBAL rpl_semi_sync_master_timeout= 2000; |
| 195 |
|
| 196 |
--let $status_var= Rpl_semi_sync_master_clients |
| 197 |
--let $status_var_value= 1 |
| 198 |
source include/wait_for_status_var.inc; |
| 199 |
|
| 200 |
show status like 'Rpl_semi_sync_master_status'; |
| 201 |
show status like 'Rpl_semi_sync_master_clients'; |
| 202 |
|
| 203 |
--connection server_2 |
| 204 |
SET @old_dbug= @@GLOBAL.debug_dbug; |
| 205 |
SET GLOBAL debug_dbug="+d,simulate_delay_semisync_slave_reply"; |
| 206 |
|
| 207 |
--connect(con1, localhost, root,,) |
| 208 |
--connect(con2, localhost, root,,) |
| 209 |
|
| 210 |
--connection con1 |
| 211 |
--send insert into t1 values (2) |
| 212 |
|
| 213 |
--connection server_1 |
| 214 |
--echo # Wait for thd to begin semi-sync wait.. |
| 215 |
--let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = 'Waiting for semi-sync ACK from slave' |
| 216 |
--source include/wait_condition.inc |
| 217 |
--echo # ..done |
| 218 |
|
| 219 |
--connection server_2 |
| 220 |
set debug_sync= "now wait_for io_thd_at_slave_reply"; |
| 221 |
|
| 222 |
--dirty_close con1 |
| 223 |
|
| 224 |
--connection default |
| 225 |
--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect |
| 226 |
wait |
| 227 |
EOF |
| 228 |
|
| 229 |
--connection con2 |
| 230 |
SHUTDOWN WAIT FOR ALL SLAVES; |
| 231 |
|
| 232 |
--echo # Waitng for shutdown to be delayed.. |
| 233 |
--connection server_2 |
| 234 |
--inc $n_logged_delayed_shutdown_notes |
| 235 |
let SEARCH_FILE= $log_error_file; |
| 236 |
let SEARCH_PATTERN=Delaying shutdown to await semi-sync ACK; |
| 237 |
let SEARCH_WAIT=FOUND $n_logged_delayed_shutdown_notes; |
| 238 |
source include/search_pattern_in_file.inc; |
| 239 |
|
| 240 |
--connection default |
| 241 |
--source include/wait_until_disconnected.inc |
| 242 |
|
| 243 |
--connection server_1 |
| 244 |
--source include/wait_until_disconnected.inc |
| 245 |
|
| 246 |
--connection con2 |
| 247 |
--source include/wait_until_disconnected.inc |
| 248 |
|
| 249 |
--connection server_2 |
| 250 |
set debug_sync= "now signal io_thd_do_reply"; |
| 251 |
--let $rpl_allow_error= 1 |
| 252 |
source include/stop_slave.inc; |
| 253 |
--connection server_3 |
| 254 |
source include/stop_slave.inc; |
| 255 |
--let $rpl_allow_error= |
| 256 |
|
| 257 |
--connection default |
| 258 |
--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect |
| 259 |
restart |
| 260 |
EOF |
| 261 |
--enable_reconnect |
| 262 |
--source include/wait_until_connected_again.inc |
| 263 |
|
| 264 |
--connection server_1 |
| 265 |
--enable_reconnect |
| 266 |
--source include/wait_until_connected_again.inc |
| 267 |
|
| 268 |
|
| 269 |
--echo ############################# |
| 270 |
--echo # Cleanup |
| 271 |
--echo ############################# |
| 272 |
|
| 273 |
--connection server_2 |
| 274 |
SET @@GLOBAL.rpl_semi_sync_slave_enabled = @sav_enabled_server_2; |
| 275 |
SET @@GLOBAL.debug_dbug= @sav_server_2_dbug; |
| 276 |
source include/start_slave.inc; |
| 277 |
|
| 278 |
--connection server_3 |
| 279 |
SET @@GLOBAL.rpl_semi_sync_slave_enabled = @sav_enabled_server_3; |
| 280 |
SET @@GLOBAL.debug_dbug= @sav_server_3_dbug; |
| 281 |
source include/start_slave.inc; |
| 282 |
|
| 283 |
--connection server_1 |
| 284 |
let $status_var= Rpl_semi_sync_master_clients; |
| 285 |
let $status_var_value= 0; |
| 286 |
source include/wait_for_status_var.inc; |
| 287 |
|
| 288 |
--disable_query_log |
| 289 |
--eval SET @@GLOBAL.rpl_semi_sync_master_timeout= $sav_master_timeout; |
| 290 |
--eval SET @@GLOBAL.rpl_semi_sync_master_enabled= $sav_enabled_master; |
| 291 |
--enable_query_log |
| 292 |
|
| 293 |
drop table t1; |
| 294 |
|
| 295 |
--source include/rpl_end.inc |