Debian Codegraph @

mysql-test/suite/rpl/t/rpl_semi_sync_shutdown_await_ack.inc

raw
1 #
2 # Helper file to ensure that a primary waits for all ACKS (or timeout) from its
3 # replicas before shutting down.
4 #
5 # Parameters:
6 # server_2_dbug (string) Debug setting to simulate delay or error on
7 # the first replica (server 2)
8 # server_3_dbug (string) Debug setting to simulate delay or error on
9 # the second replica (server 3)
10 # semisync_timeout (int) Rpl_semi_sync_master_timeout to use
11 # server_2_expect_row_count (int) The number of rows expected on the first
12 # replica after the shutdown
13 # server_3_expect_row_count (int) The number of rows expected on the second
14 # replica after the shutdown
15 # server_2_sync_slave_reply (bool) True if server_2_dbug is configured to use
16 # debug_sync to synchronize the slave reply
17 # (i.e. to pause before the reply)
18 # server_3_sync_slave_reply (bool) True if server_3_dbug is configured to use
19 # debug_sync to synchronize the slave reply
20 # (i.e. to pause before the reply)
21 #
22
23 --connection server_1
24 let $log_error_file= `SELECT @@GLOBAL.log_error`;
25 if (!$n_logged_delayed_shutdown_notes)
26 {
27 let $n_logged_delayed_shutdown_notes= 0;
28 }
29
30 --echo #--
31 --echo #-- Semi-sync Setup
32
33 --connection server_1
34 --save_master_pos
35
36 echo #-- Enable semi-sync on slaves
37 let slave_last= 3;
38 --let i= 2
39 while (`SELECT $i <= $slave_last`)
40 {
41 --connection server_$i
42 --sync_with_master
43
44 source include/stop_slave.inc;
45 set global rpl_semi_sync_slave_enabled = 1;
46 source include/start_slave.inc;
47 show status like 'Rpl_semi_sync_slave_status';
48
49 --inc $i
50 }
51
52 --echo #-- Enable semi-sync on master
53 --connection server_1
54 SET @@GLOBAL.rpl_semi_sync_master_enabled = 1;
55 --eval set @@global.rpl_semi_sync_master_timeout= $semisync_timeout
56
57 --echo #-- Wait for master to recognize semi-sync slaves
58 --connection server_1
59 let $status_var= Rpl_semi_sync_master_clients;
60 let $status_var_value= 2;
61 source include/wait_for_status_var.inc;
62
63 --echo #-- Master should have semi-sync enabled with 2 connections
64 show status like 'Rpl_semi_sync_master_status';
65 show status like 'Rpl_semi_sync_master_clients';
66
67 --echo #-- Prepare servers to simulate delay or error
68 --connection server_2
69 --eval SET @@GLOBAL.debug_dbug= $server_2_dbug
70 --connection server_3
71 --eval SET @@GLOBAL.debug_dbug= $server_3_dbug
72
73 --echo #--
74 --echo #-- Test begins
75
76 --connection server_1_con2
77 --write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
78 wait
79 EOF
80
81 --connection server_1
82 --echo #-- Begin semi-sync transaction
83 --send INSERT INTO t1 VALUES (1)
84
85 --connection server_1_con2
86 --echo #-- Wait until master recognizes a connection is awaiting semi-sync ACK
87 let $status_var= Rpl_semi_sync_master_wait_sessions;
88 let $status_var_value= 1;
89 source include/wait_for_status_var.inc;
90
91 if ($server_2_sync_slave_reply)
92 {
93 --connection server_2
94 set debug_sync= "now wait_for io_thd_at_slave_reply";
95 }
96 if ($server_3_sync_slave_reply)
97 {
98 --connection server_3
99 set debug_sync= "now wait_for io_thd_at_slave_reply";
100 }
101
102 --connection server_1_con2
103 --echo #-- Begin master shutdown
104 SHUTDOWN WAIT FOR ALL SLAVES;
105
106 # Use server_2 to search error log because 1 is down
107 --connection server_2
108
109 --echo # Waitng for shutdown to be delayed..
110 # Increment the number of notes to find each test case to ensure the pattern is
111 # ours
112 --inc $n_logged_delayed_shutdown_notes
113 let SEARCH_FILE= $log_error_file;
114 let SEARCH_PATTERN=Delaying shutdown to await semi-sync ACK;
115 let SEARCH_WAIT=FOUND $n_logged_delayed_shutdown_notes;
116 source include/search_pattern_in_file.inc;
117
118 if (`SELECT $server_2_sync_slave_reply AND $server_2_expect_row_count`)
119 {
120 --connection server_2
121 set debug_sync= "now signal io_thd_do_reply";
122 }
123 if (`SELECT $server_3_sync_slave_reply AND $server_3_expect_row_count`)
124 {
125 --connection server_3
126 set debug_sync= "now signal io_thd_do_reply";
127 }
128
129 --echo # Reaping transaction..
130 --connection server_1
131 --error 2013
132 --reap
133 --source include/wait_until_disconnected.inc
134
135 # Timeout should be hit from prior reap
136 --connection server_2
137 if (`SELECT $server_2_sync_slave_reply AND NOT $server_2_expect_row_count`)
138 {
139 --connection server_2
140 set debug_sync= "now signal io_thd_do_reply";
141 }
142 if (`SELECT $server_3_sync_slave_reply AND NOT $server_3_expect_row_count`)
143 {
144 --connection server_3
145 set debug_sync= "now signal io_thd_do_reply";
146 }
147
148 --echo # Reaping shutdown..
149 --connection server_1_con2
150 --source include/wait_until_disconnected.inc
151
152 --echo # Validate slave data is in correct state
153 --connection server_2
154 --eval select count(*)=$server_2_expect_row_count from t1
155 --connection server_3
156 --eval select count(*)=$server_3_expect_row_count from t1
157
158 --echo #
159 --echo #-- Re-synchronize slaves with master and disable semi-sync
160
161 if (`SELECT ($server_2_expect_row_count + $server_3_expect_row_count) < 2`)
162 {
163 --echo #-- FIXME: workaround for MDEV-28141, preventing errored replicas from
164 --echo # killing their semi-sync connections
165 # I.e. we can't create a new kill connection to the primary if we know that the
166 # primary is shutting down for risk of Packets out of order error. So we wait
167 # to hit a debug_sync point before the creation of the new kill_connection, and
168 # don't progress until the primary has been shutdown, so no new connection can
169 # be formed.
170 # Note this is only needed in the error case (using corrupt_queue_event), as
171 # the running io_thread will otherwise automatically detect that the primary
172 # has shutdown before progressing to the cleanup of the io thread.
173 }
174
175 if (!$server_2_expect_row_count)
176 {
177 --connection server_2
178 set debug_sync= "now wait_for at_semisync_kill_connection";
179 set debug_sync= "now signal continue_semisync_kill_connection";
180 --echo # Wait for debug_sync signal to have been received before issuing RESET
181 let $wait_condition= select count(*)=0 from information_schema.processlist where state like "debug sync point%";
182 source include/wait_condition.inc;
183 set debug_sync= "reset";
184 }
185 if (!$server_3_expect_row_count)
186 {
187 --connection server_3
188 set debug_sync= "now wait_for at_semisync_kill_connection";
189 set debug_sync= "now signal continue_semisync_kill_connection";
190 --echo # Wait for debug_sync signal to have been received before issuing RESET
191 let $wait_condition= select count(*)=0 from information_schema.processlist where state like "debug sync point%";
192 source include/wait_condition.inc;
193 set debug_sync= "reset";
194 }
195
196 --echo #-- Stop slaves
197
198 --connection server_2
199 # If server_2_expect_row_count is 0, we are simulating an error on the replica
200 # and the IO thread will end with errno 1595.
201 # Otherwise, we still expect error, because the master has shutdown at this
202 # point, and the IO thread may or may not have realized the shutdown, and
203 # started to try to automatically reconnect. This may result in the IO thread
204 # giving a 2003 error if the slave tries to reconnect to a shutdown master.
205 # Additionally disable warnings because the slave may have stopped in err
206 # automatically, and we don't want a sporadic "Slave is already stopped"
207 # warning.
208 --disable_warnings
209 --let $rpl_allow_error= 1
210 --source include/stop_slave_io.inc
211 --enable_warnings
212 --let $rpl_allow_error=
213 --source include/stop_slave_sql.inc
214 SET @@GLOBAL.debug_dbug= @sav_server_2_dbug;
215 SET @@GLOBAL.rpl_semi_sync_slave_enabled= @sav_enabled_server_2;
216
217 --connection server_3
218 # Expect error for IO thread, see above comment for stopping server_2
219 --disable_warnings
220 --let $rpl_allow_error= 1
221 --source include/stop_slave_io.inc
222 --enable_warnings
223 --let $rpl_allow_error=
224 --source include/stop_slave_sql.inc
225 SET @@GLOBAL.debug_dbug= @sav_server_3_dbug;
226 SET @@GLOBAL.rpl_semi_sync_slave_enabled= @sav_enabled_server_3;
227
228 --echo #-- Bring the master back up
229 --connection server_1_con2
230 --append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
231 restart
232 EOF
233
234 --enable_reconnect
235 --source include/wait_until_connected_again.inc
236
237 --connection default
238 --enable_reconnect
239 --source include/wait_until_connected_again.inc
240
241 --connection server_1
242 --enable_reconnect
243 --source include/wait_until_connected_again.inc
244
245 TRUNCATE TABLE t1;
246 --save_master_pos
247
248 --echo #-- Bring slaves back up
249 --let i= 2
250 while (`SELECT $i <= $slave_last`)
251 {
252 --connection server_$i
253 source include/start_slave.inc;
254 show status like 'Rpl_semi_sync_slave_status';
255 --sync_with_master
256 SELECT COUNT(*)=0 from t1;
257 --inc $i
258 }