Debian Codegraph @

mysql-test/suite/rpl_gtid/t/rpl_async_conn_failover.test

raw
1 # ==== Purpose ====
2 #
3 # WL#12649: Automatic reconnection for Async replication between InnoDB Clusters
4 #
5 # It tests that asynchronous replication connection is re-established to new
6 # sender in case the current asynchronous replication connection fails.
7 #
8 # ==== Implementation ====
9 # 1. Create a chain replication between server 1, 2, 3 and 4.
10 # 2. Insert data on server1 and verify that its synced to the end of
11 # chain i.e. server4
12 # 3. Verify Asynchronous Replication Connection Failover is initially
13 # disabled. Also verify Asynchronous Replication is initially
14 # connected to server 3.
15 # 4. Enable Asynchronous Replication Connection Failover and set retry
16 # count and retry time to 2 and verify its enabled in
17 # performance_schema.replication_connection_configuration.
18 # 5. Stop server 3.
19 # 6. Verify that asynchronous replication fails with error
20 # ER_RPL_ASYNC_RECONNECT_FAIL_NO_SOURCE as there is no source added to
21 # table performance_schema.replication_asynchronous_connection_failover to
22 # connect.
23 # 7. Verify that asynchronous replication tries connecting to server 3
24 # two (retry_count=2) times before it fails.
25 # 8. Add three asynchronous connection failover servers and
26 # verify that they are inserted successfully in
27 # performance_schema.replication_asynchronous_connection_failover.
28 # 9. Restart replica IO so asynchronous replication connection failover
29 # can execute and connect to new sender/source.
30 # 10. Verify that asynchronous replication channel gets connected to
31 # server 2 and also verify that asynchronous replication tries
32 # connecting to server 3 two (retry_count=2) times.
33 # 11. Insert more data on server1 and verify that its synced to the end
34 # of chain i.e. server 4.
35 # 12. Stop server 2.
36 # 13. Verify that asynchronous replication channel gets connected to
37 # server 1 and also verify that asynchronous replication tries
38 # connecting to server 2 two (retry_count=2) times.
39 # 14. Insert more data on server1 and verify that its synced to the
40 # server 4.
41 # 15. Stop server 1.
42 # 16. Even after all source are stopped the replica IO thread keeps
43 # on re-trying to connect with all logged source in
44 # performance_schema.replication_asynchronous_connection_failover
45 # table. It tries connecting to server 3, 2 and again server 3.
46 # 17. Stop replica IO
47 # 18. Clean up
48 #
49 --source include/big_test.inc
50 --source include/have_debug.inc
51 --source include/have_debug_sync.inc
52
53 --echo #
54 --echo # 1. Create a chain replication between server 1, 2, 3 and 4.
55 --echo #.
56 --let $rpl_server_count= 4
57 --let $rpl_topology= 1->2->3->4
58 --source include/rpl_init.inc
59
60
61 --echo #
62 --echo # 2. Insert data on server1 and verify that its synced to the end of
63 --echo # chain i.e. server4
64 --echo #
65 --let $rpl_connection_name= server_1
66 --source include/rpl_connection.inc
67 CREATE TABLE t1 (a INT) ENGINE = InnoDB;
68 INSERT INTO t1 VALUES (1);
69
70 --source include/rpl_sync.inc
71
72 --let $rpl_connection_name= server_4
73 --source include/rpl_connection.inc
74
75 SET @old_debug= @@GLOBAL.DEBUG;
76 SET GLOBAL debug = '+d,async_conn_failover_disable_weight_check';
77
78 --let $assert_text= 'There is one row on replica'
79 --let $assert_cond= [SELECT COUNT(*) count FROM t1, count, 1] = 1
80 --source include/assert.inc
81
82
83 --echo #
84 --echo # 3. Verify Asynchronous Replication Connection Failover is initially
85 --echo # disabled. Also verify Asynchronous Replication is initially
86 --echo # connected to server 3.
87 --echo #
88 --let $assert_text= 'Asynchronous Replication Connection Failover is disabled'
89 --let $assert_cond= [SELECT SOURCE_CONNECTION_AUTO_FAILOVER FROM performance_schema.replication_connection_configuration, SOURCE_CONNECTION_AUTO_FAILOVER, 1] = 0
90 --source include/assert.inc
91
92 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_connection_configuration WHERE CHANNEL_NAME='' AND PORT=$SERVER_MYPORT_3
93 --source include/wait_condition.inc
94
95
96 --echo #
97 --echo # 4. Enable Asynchronous Replication Connection Failover and set retry
98 --echo # count and retry time to 2 and verify its enabled in
99 --echo # performance_schema.replication_connection_configuration.
100 --echo #
101 --source include/stop_slave.inc
102 CHANGE REPLICATION SOURCE TO SOURCE_CONNECT_RETRY=1, SOURCE_RETRY_COUNT=2, SOURCE_CONNECTION_AUTO_FAILOVER=1;
103 --source include/start_slave.inc
104
105 # Verify Asynchronous Replication Connection Failover is enabled.
106 --let $assert_text= 'Asynchronous Replication Connection Failover is enabled'
107 --let $assert_cond= [SELECT SOURCE_CONNECTION_AUTO_FAILOVER FROM performance_schema.replication_connection_configuration, SOURCE_CONNECTION_AUTO_FAILOVER, 1] = 1
108 --source include/assert.inc
109
110
111 --echo #
112 --echo # 5. Stop server 3.
113 --echo #
114 --let $rpl_server_number= 3
115 --source include/rpl_stop_server.inc
116
117
118 --echo #
119 --echo # 6. Verify that asynchronous replication fails with error
120 --echo # ER_RPL_ASYNC_RECONNECT_FAIL_NO_SOURCE as there is no source
121 --echo # added to table
122 --echo # performance_schema.replication_asynchronous_connection_failover to
123 --echo # connect.
124 --echo #
125 --let $rpl_connection_name= server_4
126 --source include/rpl_connection.inc
127
128 --let $slave_io_errno=2003
129 --source include/wait_for_slave_io_error.inc
130
131
132 --echo #
133 --echo # 7. Verify that asynchronous replication tries connecting to server 3
134 --echo # two (retry_count=2) times before it fails.
135 --echo #
136
137 --let $assert_text= Assert that the IO thread tried two times connecting to server_3
138 --let $assert_file=$MYSQLTEST_VARDIR/log/mysqld.4.err
139 --let $assert_select= .*Error reconnecting to source '[email protected]:$SERVER_MYPORT_3'. This was attempt .*, with a delay of 1 seconds between attempts.*
140 --let $assert_only_after= CURRENT_TEST: rpl_gtid.rpl_async_conn_failover
141 --let $assert_count= 2
142 --source include/assert_grep.inc
143
144
145 --echo #
146 --echo # 8. Add three asynchronous connection failover servers and
147 --echo # verify that they are inserted successfully in
148 --echo # performance_schema.replication_asynchronous_connection_failover.
149 --echo #
150
151 --replace_result $SERVER_MYPORT_3 SERVER_MYPORT_3
152 --eval SELECT asynchronous_connection_failover_add_source('', '127.0.0.1', $SERVER_MYPORT_3, '', 60);
153 --replace_result $SERVER_MYPORT_2 SERVER_MYPORT_2
154 --eval SELECT asynchronous_connection_failover_add_source('', '127.0.0.1', $SERVER_MYPORT_2, '', 50);
155 --replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1
156 --eval SELECT asynchronous_connection_failover_add_source('', '127.0.0.1', $SERVER_MYPORT_1, '', 40);
157
158 --let $assert_text= 'There are three rows in performance_schema.replication_asynchronous_connection_failover'
159 --let $assert_cond= [SELECT COUNT(*) count FROM performance_schema.replication_asynchronous_connection_failover, count, 1] = 3
160 --source include/assert.inc
161
162
163 --echo #
164 --echo # 9. Restart replica IO so asynchronous replication connection failover
165 --echo # can execute and connect to new sender/source.
166 --echo #
167 --source include/start_slave_io.inc
168
169
170 --echo #
171 --echo # 10. Verify that asynchronous replication channel gets connected to
172 --echo # server 2 and also verify that asynchronous replication tries
173 --echo # connecting to server 3 two (retry_count=2) times.
174 --echo #
175 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_connection_configuration WHERE CHANNEL_NAME='' AND PORT=$SERVER_MYPORT_2
176 --source include/wait_condition.inc
177
178 # Verify that it tried connecting to server_3 two (retry_count=2) times.
179 --let $assert_text= Assert that the IO thread tried two times connecting to server_3
180 --let $assert_file=$MYSQLTEST_VARDIR/log/mysqld.4.err
181 --let $assert_select= .*Error connecting to source '[email protected]:$SERVER_MYPORT_3'. This was attempt .*, with a delay of 1 seconds between attempts.*
182 --let $assert_only_after= CURRENT_TEST: rpl_gtid.rpl_async_conn_failover
183 --let $assert_count= 4
184 --source include/assert_grep.inc
185
186
187 --echo #
188 --echo # 11. Insert more data on server1 and verify that its synced to the end
189 --echo # of chain i.e. server 4.
190 --echo #
191 --let $rpl_connection_name= server_1
192 --source include/rpl_connection.inc
193
194 INSERT INTO t1 VALUES (2);
195
196
197 # Check data is synced to server 4
198 --let $rpl_connection_name= server_4
199 --source include/rpl_connection.inc
200
201 --let $wait_condition= SELECT COUNT(*) = 2 FROM t1
202 --source include/wait_condition.inc
203
204
205 --echo #
206 --echo # 12. Stop server 2.
207 --echo #
208 --let $rpl_server_number= 2
209 --source include/rpl_stop_server.inc
210
211
212 --echo #
213 --echo # 13. Verify that asynchronous replication channel gets connected to
214 --echo # server 1 and also verify that asynchronous replication tries
215 --echo # connecting to server 2 two (retry_count=2) times.
216 --echo #
217
218 --let $rpl_connection_name= server_4
219 --source include/rpl_connection.inc
220
221 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_connection_configuration WHERE CHANNEL_NAME='' AND PORT=$SERVER_MYPORT_1
222 --source include/wait_condition.inc
223
224 # Verify that it tried connecting to server_2 two (retry_count=2) times.
225 --let $assert_text= Assert that the IO thread tried two times connecting to server_2
226 --let $assert_file=$MYSQLTEST_VARDIR/log/mysqld.4.err
227 --let $assert_select= .*Error reconnecting to source '[email protected]:$SERVER_MYPORT_2'. This was attempt .*, with a delay of 1 seconds between attempts.*
228 --let $assert_only_after= CURRENT_TEST: rpl_gtid.rpl_async_conn_failover
229 --let $assert_count= 2
230 --source include/assert_grep.inc
231
232
233 --echo #
234 --echo # 14. Insert more data on server1 and verify that its synced to the
235 --echo # server 4.
236 --echo #
237
238 --let $rpl_connection_name= server_1
239 --source include/rpl_connection.inc
240
241 INSERT INTO t1 VALUES (3);
242
243
244 # Check data is synced to server 4
245 --let $rpl_connection_name= server_4
246 --source include/rpl_connection.inc
247
248 --let $wait_condition= SELECT COUNT(*) = 3 FROM t1
249 --source include/wait_condition.inc
250
251
252 --echo #
253 --echo # 15. Stop server 1.
254 --echo #
255 --let $rpl_connection_name= server_4
256 --source include/rpl_connection.inc
257
258 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_connection_configuration WHERE CHANNEL_NAME='' AND PORT=$SERVER_MYPORT_1
259 --source include/wait_condition.inc
260
261 SET GLOBAL debug = '+d,replica_retry_count_exceed';
262
263 --let $rpl_server_number= 1
264 --source include/rpl_stop_server.inc
265
266
267 --echo #
268 --echo # 16. Even after all source are stopped the replica IO thread keeps
269 --echo # on re-trying to connect with all logged source in
270 --echo # performance_schema.replication_asynchronous_connection_failover
271 --echo # table. It tries connecting to server 3, 2, 1, and again 3, 2 1.
272 --echo #
273 --let $rpl_connection_name= server_4
274 --source include/rpl_connection.inc
275
276 SET DEBUG_SYNC= 'now WAIT_FOR wait_for_retry_count_exceed';
277 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_connection_configuration WHERE CHANNEL_NAME='' AND PORT=$SERVER_MYPORT_3
278 --source include/wait_condition.inc
279 SET DEBUG_SYNC= 'now SIGNAL continue_retry_count_exceed';
280
281 SET DEBUG_SYNC= 'now WAIT_FOR wait_for_retry_count_exceed';
282 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_connection_configuration WHERE CHANNEL_NAME='' AND PORT=$SERVER_MYPORT_2
283 --source include/wait_condition.inc
284 SET DEBUG_SYNC= 'now SIGNAL continue_retry_count_exceed';
285
286 SET DEBUG_SYNC= 'now WAIT_FOR wait_for_retry_count_exceed';
287 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_connection_configuration WHERE CHANNEL_NAME='' AND PORT=$SERVER_MYPORT_1
288 --source include/wait_condition.inc
289 SET DEBUG_SYNC= 'now SIGNAL continue_retry_count_exceed';
290
291 SET DEBUG_SYNC= 'now WAIT_FOR wait_for_retry_count_exceed';
292 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_connection_configuration WHERE CHANNEL_NAME='' AND PORT=$SERVER_MYPORT_3
293 --source include/wait_condition.inc
294 SET DEBUG_SYNC= 'now SIGNAL continue_retry_count_exceed';
295
296 SET DEBUG_SYNC= 'now WAIT_FOR wait_for_retry_count_exceed';
297 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_connection_configuration WHERE CHANNEL_NAME='' AND PORT=$SERVER_MYPORT_2
298 --source include/wait_condition.inc
299 SET DEBUG_SYNC= 'now SIGNAL continue_retry_count_exceed';
300
301 SET DEBUG_SYNC= 'now WAIT_FOR wait_for_retry_count_exceed';
302 --let $wait_condition= SELECT COUNT(*) = 1 FROM performance_schema.replication_connection_configuration WHERE CHANNEL_NAME='' AND PORT=$SERVER_MYPORT_1
303 --source include/wait_condition.inc
304 SET @@GLOBAL.DEBUG= @old_debug;
305 SET DEBUG_SYNC= 'now SIGNAL continue_retry_count_exceed';
306 SET DEBUG_SYNC= 'RESET';
307
308
309 --echo #
310 --echo # 17. Stop replica IO
311 --echo #
312 --source include/stop_slave.inc
313
314
315 --echo #
316 --echo # 18. Clean up
317 --echo #
318
319 CHANGE REPLICATION SOURCE TO SOURCE_CONNECTION_AUTO_FAILOVER=0;
320
321 --replace_result $SERVER_MYPORT_3 SERVER_MYPORT_3
322 --eval SELECT asynchronous_connection_failover_delete_source('', '127.0.0.1', $SERVER_MYPORT_3, '');
323 --replace_result $SERVER_MYPORT_2 SERVER_MYPORT_2
324 --eval SELECT asynchronous_connection_failover_delete_source('', '127.0.0.1', $SERVER_MYPORT_2, '');
325 --replace_result $SERVER_MYPORT_1 SERVER_MYPORT_1
326 --eval SELECT asynchronous_connection_failover_delete_source('', '127.0.0.1', $SERVER_MYPORT_1, '');
327
328 --let $rpl_server_number=1
329 --source include/rpl_start_server.inc
330
331 --let $_rpl_server= 2
332 while ($_rpl_server < $rpl_server_count)
333 {
334 --let $rpl_connection_name= server_$_rpl_server
335 --source include/rpl_connection.inc
336
337 --let $rpl_server_number= $_rpl_server
338 --source include/rpl_start_server.inc
339
340 --disable_warnings
341 --source include/start_slave.inc
342 --enable_warnings
343
344 --inc $_rpl_server
345 }
346
347 --let $rpl_connection_name= server_4
348 --source include/rpl_connection.inc
349 --source include/start_slave.inc
350
351 --let $rpl_connection_name= server_1
352 --source include/rpl_connection.inc
353 DROP TABLE t1;
354
355 --source include/rpl_sync.inc
356
357 --let $rpl_skip_sync= 1
358 --let $rpl_only_running_threads= 1
359 --source include/rpl_end.inc