Debian Codegraph @

mysql-test/suite/rpl/t/rpl_semi_sync_shutdown_await_ack.test

raw
1 --source include/no_valgrind_without_big.inc
2
3 #
4 # Purpose:
5 # This test validates that data is consistent between a primary and replica
6 # in semi-sync mode when the primary is issued `SHUTDOWN WAIT FOR SLAVES`
7 # during an active communication. More specifically, the primary should not
8 # kill the Ack_thread until it is sure a replica has received all binlog
9 # data, i.e. once the primary receives the ACK. If a primary is issued a
10 # shutdown before receiving an ACK, it should wait until either 1) the ACK is
11 # received, or 2) the configured timeout (rpl_semi_sync_master_timeout) is
12 # reached.
13 #
14 # Methodology:
15 # Using a topology consisting of one primary with two replicas, all in
16 # semi-sync mode, we use DEBUG_DBUG to simulate an error or delay on the
17 # replicas during an active communication while the primary is issued
18 # `SHUTDOWN WAIT FOR SLAVES`. We create four test cases to ensure the
19 # Ack_thread is not prematurely killed due to the shutdown.
20 # Test Case 1) If both replicas simulate a delay that is within the allowed
21 # timeout, the primary should delay killing the Ack_thread
22 # until an ACK is received.
23 # Test Case 2) If both replicas simulate an error before sending an ACK, the
24 # primary should delay killing the Ack_thread until the
25 # the timeout is reached.
26 # Test Case 3) If one replica simulates a delay within the allowed timeout
27 # and the other simulates an error before sending an ACK, the
28 # primary should delay killing the Ack_thread until it
29 # receives an ACK from the delayed slave.
30 # Test Case 4) If a replica errors before sending an ACK, it will cause the
31 # IO thread to stop and handle the error. During error handling,
32 # if semi-sync is active, the replica will form a new connection
33 # with the primary to kill the active connection. However, if
34 # the primary is shutting down, it may kill the new connection,
35 # thereby leaving the active semi-sync connection in-tact. The
36 # slave should notice this, and not issue a `QUIT` command to
37 # the primary, which would otherwise be sent to kill an active
38 # connection. This test case validates that the slave does not
39 # send a `QUIT` in this case.
40 # Test Case 5) If a waiting-for-ACK user thread is killed (disconnected)
41 # during SHUTDOWN WAIT FOR ALL SLAVES, ensure the primary will
42 # still await the ACK from the replica before killing the
43 # Ack_thread.
44 #
45 # References:
46 # MDEV-11853: semisync thread can be killed after sync binlog but before ACK
47 # in the sync state
48 # MDEV-28114: Semi-sync Master ACK Receiver Thread Can Error on COM_QUIT
49 #
50
51 --echo #############################
52 --echo # Common setup for all tests
53 --echo #############################
54
55 --echo # Note: Simulated slave delay is hardcoded to 800 milliseconds
56 --echo # Note: Simulated master shutdown delay is hardcoded to 500 milliseconds
57
58 --source include/have_innodb.inc
59 --source include/have_debug.inc
60 --let $rpl_topology=1->2, 1->3
61 --source include/rpl_init.inc
62
63 --connection server_1
64
65 --echo # Slaves which simulate an error will produce a timeout on the primary
66 call mtr.add_suppression("Timeout waiting");
67 call mtr.add_suppression("did not exit");
68 call mtr.add_suppression("Got an error reading communication packets");
69
70 --let $sav_master_timeout= `SELECT @@GLOBAL.rpl_semi_sync_master_timeout`
71 --let $sav_enabled_master= `SELECT @@GLOBAL.rpl_semi_sync_master_enabled`
72
73 --echo # Suppress slave errors related to the simulated error
74 --connection server_2
75 call mtr.add_suppression("reply failed");
76 call mtr.add_suppression("Replication event checksum verification");
77 call mtr.add_suppression("Relay log write failure");
78 call mtr.add_suppression("Failed to kill the active semi-sync connection");
79 set @sav_enabled_server_2= @@GLOBAL.rpl_semi_sync_slave_enabled;
80 set @sav_server_2_dbug= @@GLOBAL.debug_dbug;
81
82 --connection server_3
83 call mtr.add_suppression("reply failed");
84 call mtr.add_suppression("Replication event checksum verification");
85 call mtr.add_suppression("Relay log write failure");
86 call mtr.add_suppression("Failed to kill the active semi-sync connection");
87 set @sav_enabled_server_3= @@GLOBAL.rpl_semi_sync_slave_enabled;
88 set @sav_server_3_dbug= @@GLOBAL.debug_dbug;
89
90 --connection server_1
91 CREATE TABLE t1 (a int) engine=innodb;
92 --save_master_pos
93
94 --let i= 2
95 --let slave_last= 3
96 while (`SELECT $i <= $slave_last`)
97 {
98 --connection server_$i
99 --sync_with_master
100 --inc $i
101 }
102
103 # Set up the connection used to issue the shutdown
104 --connect(server_1_con2, localhost, root,,)
105
106
107 --echo #############################
108 --echo # Test cases
109 --echo #############################
110
111 --echo #
112 --echo # Test Case 1) If both replicas simulate a delay that is within the
113 --echo # allowed timeout, the primary should delay killing the Ack_thread
114 --echo # until an ACK is received.
115 --echo #
116 --let server_2_sync_slave_reply=1
117 --let server_2_dbug= "+d,simulate_delay_semisync_slave_reply"
118 --let server_3_sync_slave_reply=1
119 --let server_3_dbug= "+d,simulate_delay_semisync_slave_reply"
120 --let semisync_timeout= 1600
121 --let server_2_expect_row_count= 1
122 --let server_3_expect_row_count= 1
123 --source rpl_semi_sync_shutdown_await_ack.inc
124
125 --echo #
126 --echo # Test Case 2) If both replicas simulate an error before sending an ACK,
127 --echo # the primary should delay killing the Ack_thread until the
128 --echo # timeout is reached.
129 --echo #
130 --let server_2_sync_slave_reply=0
131 --let server_2_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141"
132 --let server_3_sync_slave_reply=0
133 --let server_3_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141"
134 --let semisync_timeout= 500
135 --let server_2_expect_row_count= 0
136 --let server_3_expect_row_count= 0
137 --source rpl_semi_sync_shutdown_await_ack.inc
138
139 --echo #
140 --echo # Test Case 3) If one replica simulates a delay within the allowed
141 --echo # timeout and the other simulates an error before sending an ACK, the
142 --echo # primary should delay killing the Ack_thread until it receives an
143 --echo # ACK from the delayed slave.
144 --echo #
145 --let server_2_sync_slave_reply=0
146 --let server_2_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141"
147 --let server_3_sync_slave_reply=1
148 --let server_3_dbug= "+d,simulate_delay_semisync_slave_reply"
149 --let semisync_timeout= 1600
150 --let server_2_expect_row_count= 0
151 --let server_3_expect_row_count= 1
152 --source rpl_semi_sync_shutdown_await_ack.inc
153
154 --echo #
155 --echo # Test Case 4) If a replica errors before sending an ACK, it will cause
156 --echo # the IO thread to stop and handle the error. During error handling, if
157 --echo # semi-sync is active, the replica will form a new connection with the
158 --echo # primary to kill the active connection. However, if the primary is
159 --echo # shutting down, it may kill the new connection, thereby leaving the
160 --echo # active semi-sync connection in-tact. The slave should notice this, and
161 --echo # not issue a `QUIT` command to the primary, which would otherwise be
162 --echo # sent to kill an active connection. This test case validates that the
163 --echo # slave does not send a `QUIT` in this case.
164 --echo #
165 --let server_2_sync_slave_reply=0
166 --let server_2_dbug= "+d,corrupt_queue_event,delay_semisync_kill_connection_for_mdev_28141"
167 --let server_3_sync_slave_reply=1
168 --let server_3_dbug= "+d,simulate_delay_semisync_slave_reply"
169 --let semisync_timeout= 1600
170 --let server_2_expect_row_count= 0
171 --let server_3_expect_row_count= 1
172 --source rpl_semi_sync_shutdown_await_ack.inc
173
174 #
175 # Added with MDEV-33551
176 #
177 --echo #
178 --echo # Test Case 5) If a waiting-for-ACK user thread is killed (disconnected)
179 --echo # during SHUTDOWN WAIT FOR ALL SLAVES, ensure the primary will still
180 --echo # await the ACK from the replica before killing the Ack_receiver thread
181 --echo #
182 --connection server_1
183 insert into t1 values (1);
184 --source include/save_master_gtid.inc
185
186 --connection server_2
187 --source include/sync_with_master_gtid.inc
188 --source include/stop_slave.inc
189 SET GLOBAL rpl_semi_sync_slave_enabled= 1;
190 --source include/start_slave.inc
191
192 --connection server_1
193 SET GLOBAL rpl_semi_sync_master_enabled= 1;
194 SET GLOBAL rpl_semi_sync_master_timeout= 2000;
195
196 --let $status_var= Rpl_semi_sync_master_clients
197 --let $status_var_value= 1
198 source include/wait_for_status_var.inc;
199
200 show status like 'Rpl_semi_sync_master_status';
201 show status like 'Rpl_semi_sync_master_clients';
202
203 --connection server_2
204 SET @old_dbug= @@GLOBAL.debug_dbug;
205 SET GLOBAL debug_dbug="+d,simulate_delay_semisync_slave_reply";
206
207 --connect(con1, localhost, root,,)
208 --connect(con2, localhost, root,,)
209
210 --connection con1
211 --send insert into t1 values (2)
212
213 --connection server_1
214 --echo # Wait for thd to begin semi-sync wait..
215 --let $wait_condition= SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE state = 'Waiting for semi-sync ACK from slave'
216 --source include/wait_condition.inc
217 --echo # ..done
218
219 --connection server_2
220 set debug_sync= "now wait_for io_thd_at_slave_reply";
221
222 --dirty_close con1
223
224 --connection default
225 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
226 wait
227 EOF
228
229 --connection con2
230 SHUTDOWN WAIT FOR ALL SLAVES;
231
232 --echo # Waitng for shutdown to be delayed..
233 --connection server_2
234 --inc $n_logged_delayed_shutdown_notes
235 let SEARCH_FILE= $log_error_file;
236 let SEARCH_PATTERN=Delaying shutdown to await semi-sync ACK;
237 let SEARCH_WAIT=FOUND $n_logged_delayed_shutdown_notes;
238 source include/search_pattern_in_file.inc;
239
240 --connection default
241 --source include/wait_until_disconnected.inc
242
243 --connection server_1
244 --source include/wait_until_disconnected.inc
245
246 --connection con2
247 --source include/wait_until_disconnected.inc
248
249 --connection server_2
250 set debug_sync= "now signal io_thd_do_reply";
251 --let $rpl_allow_error= 1
252 source include/stop_slave.inc;
253 --connection server_3
254 source include/stop_slave.inc;
255 --let $rpl_allow_error=
256
257 --connection default
258 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
259 restart
260 EOF
261 --enable_reconnect
262 --source include/wait_until_connected_again.inc
263
264 --connection server_1
265 --enable_reconnect
266 --source include/wait_until_connected_again.inc
267
268
269 --echo #############################
270 --echo # Cleanup
271 --echo #############################
272
273 --connection server_2
274 SET @@GLOBAL.rpl_semi_sync_slave_enabled = @sav_enabled_server_2;
275 SET @@GLOBAL.debug_dbug= @sav_server_2_dbug;
276 source include/start_slave.inc;
277
278 --connection server_3
279 SET @@GLOBAL.rpl_semi_sync_slave_enabled = @sav_enabled_server_3;
280 SET @@GLOBAL.debug_dbug= @sav_server_3_dbug;
281 source include/start_slave.inc;
282
283 --connection server_1
284 let $status_var= Rpl_semi_sync_master_clients;
285 let $status_var_value= 0;
286 source include/wait_for_status_var.inc;
287
288 --disable_query_log
289 --eval SET @@GLOBAL.rpl_semi_sync_master_timeout= $sav_master_timeout;
290 --eval SET @@GLOBAL.rpl_semi_sync_master_enabled= $sav_enabled_master;
291 --enable_query_log
292
293 drop table t1;
294
295 --source include/rpl_end.inc