Debian Codegraph @

mysql-test/suite/ndb_rpl/t/ndb_rpl_slave_replay.test

raw
1 --source include/have_ndb.inc
2 --source include/have_binlog_format_mixed_or_row.inc
3 --source suite/ndb_rpl/ndb_master-slave.inc
4 --source include/have_debug.inc
5
6
7 # Test how the slave behaves when Binlog is replayed intentionally
8 # and unintentionally
9 #
10 --connection master
11 use test;
12 create table test.t1 (a int primary key, content varchar(1000)) engine=ndb;
13
14 --sync_slave_with_master
15 --connection master
16 reset master;
17
18 --connection slave
19 # Add a suppression for the warning that will appear in the slave's .err file
20 --disable_query_log
21 call mtr.add_suppression("Replica: At SQL thread start applying epoch .*");
22 --enable_query_log
23
24 STOP REPLICA;
25 CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS=4;
26 START REPLICA;
27
28 --echo Capture current count of epoch decline warnings
29 --let $mysqld_name=mysqld.1.slave
30 --source ndb_rpl_get_server_error_log.inc
31 --disable_query_log
32 --disable_result_log
33 select @init_lower_warn_count:=count(1) from test.server_error_log where line like '%Replica%lower than previously applied%Check replica positioning%';
34 select @init_lower_err_count:=count(1) from test.server_error_log where line like '%Replica%SQL thread stopped%lower than previously applied%';
35 select @init_repeat_err_count:=count(1) from test.server_error_log where line like '%Replica%SQL thread stopped%reapply already committed epoch%';
36 select @init_skipped_err_count:=count(1) from test.server_error_log where line like '%Replica%SQL thread stopped%has not been committed%';
37 --enable_query_log
38 --enable_result_log
39
40 --connection master
41
42 # Create some distinct epochs by writing data, and
43 # synchronising on binlog...
44 --let $i=6
45 while($i)
46 {
47 eval insert into test.t1 values ($i, repeat('B', 1000));
48 --disable_query_log
49 --disable_result_log
50 show binlog events;
51 --enable_result_log
52 --enable_query_log
53 --dec $i
54 }
55
56 --echo Expect that we have at least 2 epochs logged.
57 select count(1) > 1 from
58 (select distinct epoch from mysql.ndb_binlog_index) as logged_epochs;
59
60 --sync_slave_with_master
61 --connection slave
62
63 --echo Any slave positioning warnings?
64 --echo Should be none
65 --source ndb_rpl_get_server_error_log.inc
66 --disable_query_log
67 select count(1) - @init_lower_warn_count as lower_warn_count from test.server_error_log where line like '%Replica%lower than previously applied%Check replica positioning%';
68 select count(1) - @init_lower_err_count as lower_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%lower than previously applied%';
69 select count(1) - @init_repeat_err_count as repeat_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%reapply already committed epoch%';
70 select count(1) - @init_skipped_err_count as skipped_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%has not been committed%';
71 --enable_query_log
72 --echo Now stop and manually replay old binlog.
73 STOP REPLICA;
74 CHANGE REPLICATION SOURCE to SOURCE_LOG_POS=4;
75 START REPLICA;
76
77 --connection master
78 --sync_slave_with_master
79 --connection slave
80
81 --echo Any slave positioning warnings?
82 --echo Should be one
83 --source ndb_rpl_get_server_error_log.inc
84 --disable_query_log
85 select count(1) - @init_lower_warn_count as lower_warn_count from test.server_error_log where line like '%Replica%lower than previously applied%Check replica positioning%';
86 select count(1) - @init_lower_err_count as lower_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%lower than previously applied%';
87 select count(1) - @init_repeat_err_count as repeat_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%reapply already committed epoch%';
88 select count(1) - @init_skipped_err_count as skipped_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%has not been committed%';
89 --enable_query_log
90
91 --echo Repeat
92 STOP REPLICA;
93 CHANGE REPLICATION SOURCE to SOURCE_LOG_POS=4;
94 START REPLICA;
95
96 --connection master
97 --sync_slave_with_master
98 --connection slave
99
100 --echo Any slave positioning warnings?
101 --echo Should be two
102 --source ndb_rpl_get_server_error_log.inc
103 --disable_query_log
104 select count(1) - @init_lower_warn_count as lower_warn_count from test.server_error_log where line like '%Replica%lower than previously applied%Check replica positioning%';
105 select count(1) - @init_lower_err_count as lower_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%lower than previously applied%';
106 select count(1) - @init_repeat_err_count as repeat_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%reapply already committed epoch%';
107 select count(1) - @init_skipped_err_count as skipped_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%has not been committed%';
108 --enable_query_log
109
110
111 --echo Now try re-positioning after RESET SLAVE
112 STOP REPLICA;
113 RESET REPLICA;
114
115 --source include/rpl_start_slaves.inc
116 --connection master
117 --sync_slave_with_master
118 --connection slave
119
120 --echo Any slave positioning warnings?
121 --echo Should be two, e.g. no extras after the RESET SLAVE
122 --echo This avoids warnings when slave context has been explicitly reset
123
124 --source ndb_rpl_get_server_error_log.inc
125 --disable_query_log
126 select count(1) - @init_lower_warn_count as lower_warn_count from test.server_error_log where line like '%Replica%lower than previously applied%Check replica positioning%';
127 select count(1) - @init_lower_err_count as lower_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%lower than previously applied%';
128 select count(1) - @init_repeat_err_count as repeat_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%reapply already committed epoch%';
129 select count(1) - @init_skipped_err_count as skipped_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%has not been committed%';
130 --enable_query_log
131
132
133 # TODO : Test replay exactly on last-applied epoch, as this is probably a
134 # common case for users who position on last epoch rather than just-after it.
135
136 --connection master
137 --echo Now get unexpected epoch back-step using fault injection
138 --echo on the Master.
139 call mtr.add_suppression("Binlog injector cycling gcis .*");
140
141 set @old_debug = @@global.debug;
142 set global debug="";
143 set global debug="+d,ndb_binlog_injector_cycle_gcis";
144 show variables like 'debug';
145
146 --echo Let's make some epochs
147 --let $i=6
148 while($i)
149 {
150 eval insert into test.t1 values ($i + 6, repeat('E', 1000));
151 --disable_query_log
152 --disable_result_log
153 show binlog events;
154 --enable_result_log
155 --enable_query_log
156 --dec $i
157 }
158
159 --connection slave
160 --disable_query_log
161 call mtr.add_suppression("Replica: SQL thread stopped as applying epoch .*");
162 call mtr.add_suppression("Replica SQL for channel '': Could not execute .* HA_ERR_ROWS_EVENT_APPLY; .*");
163 call mtr.add_suppression("Replica: Got error 173 .*");
164 --enable_query_log
165
166 --echo Check that slave stopped with an error
167 --let $slave_sql_errno=1296
168 --source include/wait_for_slave_sql_error.inc
169
170 --echo Let's look at the warnings and errors now...
171
172 --source ndb_rpl_get_server_error_log.inc
173 --disable_query_log
174 select count(1) - @init_lower_warn_count as lower_warn_count from test.server_error_log where line like '%Replica%lower than previously applied%Check replica positioning%';
175 select count(1) - @init_lower_err_count as lower_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%lower than previously applied%';
176 select count(1) - @init_repeat_err_count as repeat_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%reapply already committed epoch%';
177 select count(1) - @init_skipped_err_count as skipped_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%has not been committed%';
178 --enable_query_log
179
180
181 --echo Now check that we can manually restart the slave
182
183 --connection master
184 set global debug = @old_debug;
185
186 reset master;
187
188 --connection slave
189 START REPLICA SQL_THREAD UNTIL SQL_AFTER_MTS_GAPS;
190 sleep 1.5;
191 STOP REPLICA;
192 CHANGE REPLICATION SOURCE to SOURCE_LOG_POS=4;
193 START REPLICA;
194
195 --connection master
196 insert into test.t1 values (13, repeat('A', 1000));
197
198 --sync_slave_with_master
199 --connection slave
200
201 --echo Check slave warnings now, should be no extras
202 --echo e.g. 2 warnings, 1 error
203
204 --source ndb_rpl_get_server_error_log.inc
205 --disable_query_log
206 select count(1) - @init_lower_warn_count as lower_warn_count from test.server_error_log where line like '%Replica%lower than previously applied%Check replica positioning%';
207 select count(1) - @init_lower_err_count as lower_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%lower than previously applied%';
208 select count(1) - @init_repeat_err_count as repeat_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%reapply already committed epoch%';
209 select count(1) - @init_skipped_err_count as skipped_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%has not been committed%';
210 --enable_query_log
211
212 --connection master
213
214 delete from test.t1;
215 --sync_slave_with_master
216
217
218 --connection master
219 --echo Now get unexpected epoch repeat using fault-injection on the master
220 call mtr.add_suppression("Binlog injector repeating gcis .*");
221
222 set @old_debug = @@global.debug;
223 set global debug="";
224 set global debug="+d,ndb_binlog_injector_repeat_gcis";
225 show variables like 'debug';
226
227 --echo Let's make some epochs
228 --let $i=6
229 while($i)
230 {
231 eval insert into test.t1 values ($i, repeat('E', 1000));
232 --disable_query_log
233 --disable_result_log
234 show binlog events;
235 --enable_result_log
236 --enable_query_log
237 --dec $i
238 }
239
240 --connection slave
241 --disable_query_log
242 call mtr.add_suppression("Replica: SQL thread stopped as attempted to reapply already committed epoch .*");
243 call mtr.add_suppression("Replica SQL for channel '': Worker 1 failed executing .* HA_ERR_ROWS_EVENT_APPLY; .*");
244 call mtr.add_suppression(".*The replica coordinator and worker threads are stopped.*MY-001756");
245 --enable_query_log
246
247 --echo Check that slave stopped with an error
248 --let $slave_sql_errno=1296
249 --source include/wait_for_slave_sql_error.inc
250
251 --echo Let's look at the warnings and errors now...
252 --echo Should be increase in repeat error count.
253
254 --source ndb_rpl_get_server_error_log.inc
255 --disable_query_log
256 select count(1) - @init_lower_warn_count as lower_warn_count from test.server_error_log where line like '%Replica%lower than previously applied%Check replica positioning%';
257 select count(1) - @init_lower_err_count as lower_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%lower than previously applied%';
258 select count(1) - @init_repeat_err_count as repeat_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%reapply already committed epoch%';
259 select count(1) - @init_skipped_err_count as skipped_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%has not been committed%';
260 --enable_query_log
261
262 --echo Now check that we can manually restart the slave
263
264 --connection master
265 set global debug = @old_debug;
266
267 reset master;
268
269 --connection slave
270 START REPLICA SQL_THREAD UNTIL SQL_AFTER_MTS_GAPS;
271 sleep 1.5;
272 STOP REPLICA;
273 CHANGE REPLICATION SOURCE to SOURCE_LOG_POS=4;
274 START REPLICA;
275
276 --connection master
277 insert into test.t1 values (13, repeat('A', 1000));
278
279 --sync_slave_with_master
280 --connection slave
281
282 --echo Check slave warnings now, should be no extras
283
284 --source ndb_rpl_get_server_error_log.inc
285 --disable_query_log
286 select count(1) - @init_lower_warn_count as lower_warn_count from test.server_error_log where line like '%Replica%lower than previously applied%Check replica positioning%';
287 select count(1) - @init_lower_err_count as lower_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%lower than previously applied%';
288 select count(1) - @init_repeat_err_count as repeat_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%reapply already committed epoch%';
289 select count(1) - @init_skipped_err_count as skipped_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%has not been committed%';
290 --enable_query_log
291
292 --connection master
293
294 delete from test.t1;
295 --sync_slave_with_master
296
297
298 --connection slave
299 --echo Now get skipped epoch using fault injection on the slave
300
301 set @old_debug = @@global.debug;
302 set global debug="";
303 set global debug="+d,ndb_replica_fail_marking_epoch_committed";
304 show variables like 'debug';
305
306 --connection master
307
308 --echo Let's make some epochs
309 --let $i=6
310 while($i)
311 {
312 eval insert into test.t1 values ($i, repeat('E', 1000));
313 --disable_query_log
314 --disable_result_log
315 show binlog events;
316 --enable_result_log
317 --enable_query_log
318 --dec $i
319 }
320
321 --connection slave
322 --disable_query_log
323 call mtr.add_suppression("Replica: SQL thread stopped as attempting to apply new epoch .*");
324 call mtr.add_suppression("Replica SQL for channel '': Could not execute .* HA_ERR_ROWS_EVENT_APPLY; .*");
325 call mtr.add_suppression("Replica: Got error 173 .*");
326 --enable_query_log
327
328 --echo Check that slave stopped with an error
329 --let $slave_sql_errno=1296
330 --source include/wait_for_slave_sql_error.inc
331
332 --echo Let's look at the warnings and errors now...
333 --echo Should be increase in repeat error count.
334
335 --source ndb_rpl_get_server_error_log.inc
336 --disable_query_log
337 select count(1) - @init_lower_warn_count as lower_warn_count from test.server_error_log where line like '%Replica%lower than previously applied%Check replica positioning%';
338 select count(1) - @init_lower_err_count as lower_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%lower than previously applied%';
339 select count(1) - @init_repeat_err_count as repeat_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%reapply already committed epoch%';
340 select count(1) - @init_skipped_err_count as skipped_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%has not been committed%';
341 --enable_query_log
342
343
344 --echo Now check that we can manually restart the slave
345
346 --connection slave
347 set global debug = @old_debug;
348
349 --connection master
350
351 reset master;
352
353 --connection slave
354 START REPLICA SQL_THREAD UNTIL SQL_AFTER_MTS_GAPS;
355 sleep 1.5;
356 STOP REPLICA;
357 CHANGE REPLICATION SOURCE to SOURCE_LOG_POS=4;
358 START REPLICA;
359
360 --connection master
361 insert into test.t1 values (13, repeat('A', 1000));
362
363 --sync_slave_with_master
364 --connection slave
365
366 --echo Check slave warnings now, should be no extras
367
368 --source ndb_rpl_get_server_error_log.inc
369 --disable_query_log
370 select count(1) - @init_lower_warn_count as lower_warn_count from test.server_error_log where line like '%Replica%lower than previously applied%Check replica positioning%';
371 select count(1) - @init_lower_err_count as lower_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%lower than previously applied%';
372 select count(1) - @init_repeat_err_count as repeat_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%reapply already committed epoch%';
373 select count(1) - @init_skipped_err_count as skipped_err_count from test.server_error_log where line like '%Replica%SQL thread stopped%has not been committed%';
374 --enable_query_log
375
376
377 drop table test.server_error_log;
378
379 --connection master
380 drop table test.t1;
381 --sync_slave_with_master
382
383 --source include/rpl_end.inc