Debian Codegraph @

mysql-test/suite/ndb/t/ndb_restore_debug.test

raw
1 --source include/have_ndb.inc
2 --source include/have_ndb_debug.inc
3 --source suite/ndb/include/backup_restore_setup.inc
4
5 --echo ********************************************************
6 --echo ndb_restore threads exit cleanly on tuple restore error
7 --echo ********************************************************
8 --echo Initial tests using ndb_restore error insert to trigger
9 --echo error on all parts.
10
11 create table t1(id int primary key, val int) engine=ndb;
12 --disable_query_log
13 let $i = 0;
14 while($i < 1000)
15 {
16 eval insert into t1 values ($i, $i);
17 inc $i;
18 }
19 --enable_query_log
20
21 --echo Cause backup stall
22 --exec $NDB_MGM -e "all error 10039" >> $NDB_TOOLS_OUTPUT
23
24 --source suite/ndb/t/ndb_backup_nowait_start.inc
25
26 update t1 set val=12345 where id between 125 and 850;
27
28 --exec $NDB_MGM -e "all error 0" >> $NDB_TOOLS_OUTPUT
29 --source suite/ndb/t/ndb_backup_nowait_wait.inc
30
31 --disable_query_log
32 --echo Test restore failure handling for log-replay failure
33 --let dump_file = $MYSQLTEST_VARDIR/tmp/restore.out
34 --error 1
35 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 1 -r --error-insert=3 $NDB_BACKUPS-$the_backup_id >> $dump_file 2>&1
36
37 # Load the file into a temporary table
38 CREATE TEMPORARY TABLE ndb_restore_output (a varchar(512));
39
40 eval LOAD DATA INFILE '$dump_file' INTO TABLE ndb_restore_output
41 FIELDS TERMINATED BY '\n';
42 --remove_file $dump_file
43
44 # Remove Windows line feeds
45 UPDATE ndb_restore_output SET a = TRIM(TRAILING "\r" FROM a);
46 --enable_query_log
47
48 select count(*) into @msgcount from ndb_restore_output where a like "%Thread % failed, exiting" order by a;
49 select @msgcount as 'Count of thread exit messages';
50 select * from ndb_restore_output where @msgcount != 4;
51
52 truncate table t1;
53
54 --disable_query_log
55 --echo Test restore failure handling for tuple-restore failure
56 --error 1
57 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 1 -r --error-insert=4 $NDB_BACKUPS-$the_backup_id >> $dump_file 2>&1
58
59 # Load the file into a temporary table
60 truncate table ndb_restore_output;
61
62 eval LOAD DATA INFILE '$dump_file' INTO TABLE ndb_restore_output
63 FIELDS TERMINATED BY '\n';
64 --remove_file $dump_file
65
66 # Remove Windows line feeds
67 UPDATE ndb_restore_output SET a = TRIM(TRAILING "\r" FROM a);
68
69 --enable_query_log
70 select count(*) into @msgcount from ndb_restore_output where a like "%Thread % failed, exiting" order by a;
71 select @msgcount as 'Count of thread exit messages';
72 select * from ndb_restore_output where @msgcount != 4;
73 drop table t1;
74
75
76 --echo Test single part restore failures, should cause all parts to stop promptly
77 --echo Constraint violation on DATA restore
78 # Testing single restore thread errors
79 create table test.t1 (a int primary key, b int, c int) engine=ndb;
80 --disable_query_log
81 let $i = 0;
82 while($i < 1000)
83 {
84 eval insert into t1 values ($i, $i, $i);
85 inc $i;
86 }
87 --enable_query_log
88
89 # Have rows 0,0,0 ... 999,999,999
90 # Now insert :
91 insert into test.t1 values (1000,1000,999);
92
93 # Backup
94 --source suite/ndb/t/ndb_backup_nowait_start.inc
95 --source suite/ndb/t/ndb_backup_nowait_wait.inc
96
97 truncate test.t1;
98 alter table test.t1 add unique(c);
99
100 # Restore backup - should fail on duplicate key
101 # during data restore
102 # Check process exit code show failure
103 --disable_query_log
104 --echo Test restore failure handling for data apply failure
105 --let dump_file = $MYSQLTEST_VARDIR/tmp/restore.out
106 let $error_result = 0;
107 --error 0,1
108 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 1 -r $NDB_BACKUPS-$the_backup_id >> $dump_file 2>&1
109 if ($__error)
110 {
111 let $error_result = 1;
112 }
113 --error 0,1
114 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 2 -r $NDB_BACKUPS-$the_backup_id >> $dump_file 2>&1
115 if ($__error)
116 {
117 let $error_result = 1;
118 }
119
120 if (!$error_result)
121 {
122 --die One ndb_restore invocation should have failed
123 }
124
125
126 # Load the file into a temporary table
127 truncate table ndb_restore_output;
128
129 eval LOAD DATA INFILE '$dump_file' INTO TABLE ndb_restore_output
130 FIELDS TERMINATED BY '\n';
131 --remove_file $dump_file
132
133 # Remove Windows line feeds
134 UPDATE ndb_restore_output SET a = TRIM(TRAILING "\r" FROM a);
135 --enable_query_log
136
137 # Check output contains useful messages
138 --echo Expect 1 error type message
139 select count(*) into @msgcount from ndb_restore_output where a like "%Permanent% duplicate value in unique index%";
140 select @msgcount as 'Count of error message 1';
141 select * from ndb_restore_output where @msgcount != 1;
142 --echo Expect 1 error location message
143 select count(*) into @msgcount from ndb_restore_output where a like "%Failed restoring DATA to table%";
144 select @msgcount as 'Count of error message 2';
145 select * from ndb_restore_output where @msgcount != 1;
146 --echo Expect 1 error detail message
147 select count(*) into @msgcount from ndb_restore_output where a like "%Failing tuple :%";
148 select @msgcount as 'Count of error message 3';
149 select * from ndb_restore_output where @msgcount != 1;
150
151 drop table test.t1;
152
153 --echo Constraint violation on LOG restore
154 # Schema uses composite primary key + partitioning to allow 2 rows fighting
155 # over secondary key value to be placed in the same fragment + so restored
156 # by the same ndb_restore process + thread, so that the restore proceeds
157 # in DATA, LOG order, so that we get a deterministic order + testcase.
158
159 create table test.t1 (a int, b int, c int, primary key (a,b)) engine=ndb partition by key(a);
160 --disable_query_log
161 let $i = 0;
162 while($i < 1000)
163 {
164 eval insert into t1 values ($i, $i, $i);
165 inc $i;
166 }
167 --enable_query_log
168
169 --echo Cause backup stall while log open
170 --exec $NDB_MGM -e "all error 10040" >> $NDB_TOOLS_OUTPUT
171
172 # Backup
173 --source suite/ndb/t/ndb_backup_nowait_start.inc
174
175 # Give some time for data without extra row
176 # to be captured
177 --sleep 4
178
179 # Insert extra row, should be placed in log only
180 insert into t1 values(999,1000,999);
181
182 --exec $NDB_MGM -e "all error 0" >> $NDB_TOOLS_OUTPUT
183 --source suite/ndb/t/ndb_backup_nowait_wait.inc
184
185 truncate test.t1;
186 alter table test.t1 add unique(c);
187
188 # Restore backup - should fail on duplicate key
189 # during log restore
190 # Check process exit code shows failure
191 --disable_query_log
192 --echo Test restore failure handling for log apply failure
193 --let dump_file = $MYSQLTEST_VARDIR/tmp/restore.out
194 let $error_result = 0;
195 --error 0,1
196 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 1 -r $NDB_BACKUPS-$the_backup_id --error-insert=5 >> $dump_file 2>&1
197 if ($__error)
198 {
199 let $error_result = 1;
200 }
201 --error 0,1
202 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 2 -r $NDB_BACKUPS-$the_backup_id --error-insert=5 >> $dump_file 2>&1
203 if ($__error)
204 {
205 let $error_result = 1;
206 }
207
208 if (!$error_result)
209 {
210 --die One ndb_restore invocation should have failed
211 }
212
213 # Load the file into a temporary table
214 truncate table ndb_restore_output;
215
216 eval LOAD DATA INFILE '$dump_file' INTO TABLE ndb_restore_output
217 FIELDS TERMINATED BY '\n';
218 --remove_file $dump_file
219
220 # Remove Windows line feeds
221 UPDATE ndb_restore_output SET a = TRIM(TRAILING "\r" FROM a);
222 --enable_query_log
223
224 # Check output contains useful messages
225 --echo Expect 1 error type message
226 select count(*) into @msgcount from ndb_restore_output where a like "%Permanent% duplicate value in unique index%";
227 select @msgcount as 'Count of error message 1';
228 select * from ndb_restore_output where @msgcount != 1;
229 --echo Expect 1 error location message
230 select count(*) into @msgcount from ndb_restore_output where a like "%Failed restoring LOG entry type INSERT to table%";
231 select @msgcount as 'Count of error message 2';
232 select * from ndb_restore_output where @msgcount != 1;
233 --echo Expect 1 error detail message
234 select count(*) into @msgcount from ndb_restore_output where a like "%Failing log entry :%";
235 select @msgcount as 'Count of error message 3';
236 select * from ndb_restore_output where @msgcount != 1;
237
238 #select * from ndb_restore_output;
239
240 drop table test.t1;
241
242
243 --echo Test that a temporary error applying
244 --echo log eventually results in failure
245
246 create table test.t1 (a int primary key, b int) engine=ndb;
247
248 --echo Cause backup stall while log open
249 --exec $NDB_MGM -e "all error 10040" >> $NDB_TOOLS_OUTPUT
250
251 # Backup
252 --source suite/ndb/t/ndb_backup_nowait_start.inc
253
254 # Give some time for data without extra row
255 # to be captured
256 --sleep 4
257
258 # Insert extra row, should be placed in log only
259 insert into t1 values(1,1);
260
261 --exec $NDB_MGM -e "all error 0" >> $NDB_TOOLS_OUTPUT
262 --source suite/ndb/t/ndb_backup_nowait_wait.inc
263
264
265 --echo Take a row lock to block restore
266 begin;
267 select * from test.t1 where a=1 for update;
268
269
270 # Restore backup - should fail on temporary error
271 # during log restore
272 # Check process exit code shows failure
273 --disable_query_log
274 --echo Test restore failure handling for log apply failure
275 --echo after retries
276 --let dump_file = $MYSQLTEST_VARDIR/tmp/restore.out
277 let $error_result = 0;
278 --error 0,1
279 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 1 -r $NDB_BACKUPS-$the_backup_id >> $dump_file 2>&1
280 if ($__error)
281 {
282 let $error_result = 1;
283 }
284 --error 0,1
285 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 2 -r $NDB_BACKUPS-$the_backup_id >> $dump_file 2>&1
286 if ($__error)
287 {
288 let $error_result = 1;
289 }
290
291 if (!$error_result)
292 {
293 --die One ndb_restore invocation should have failed
294 }
295
296 rollback;
297
298 # Load the file into a temporary table
299 truncate table ndb_restore_output;
300
301 eval LOAD DATA INFILE '$dump_file' INTO TABLE ndb_restore_output
302 FIELDS TERMINATED BY '\n';
303 --remove_file $dump_file
304
305 # Remove Windows line feeds
306 UPDATE ndb_restore_output SET a = TRIM(TRAILING "\r" FROM a);
307 --enable_query_log
308
309 # Check output contains useful messages
310 --echo Expect error message
311 select count(*) into @msgcount from ndb_restore_output where a like "%Error restoring the data log%" or a like "%Error reading the data log%";
312 select @msgcount as 'Count of error message 1';
313 select * from ndb_restore_output where @msgcount != 1;
314
315 #select * from ndb_restore_output;
316
317 drop table test.t1;
318
319 --echo ----------------
320 --echo Test hint option
321 --echo ----------------
322
323 --echo Create backup with good amount of DATA + LOG content
324 create table t1(id int primary key, val int) engine=ndb;
325 --disable_query_log
326 let $i = 0;
327 while($i < 1000)
328 {
329 eval insert into t1 values ($i, $i);
330 inc $i;
331 }
332 --enable_query_log
333
334 --echo Cause backup stall
335 --exec $NDB_MGM -e "all error 10039" >> $NDB_TOOLS_OUTPUT
336
337 --source suite/ndb/t/ndb_backup_nowait_start.inc
338
339 update t1 set val=val+1;
340
341 --exec $NDB_MGM -e "all error 0" >> $NDB_TOOLS_OUTPUT
342 --source suite/ndb/t/ndb_backup_nowait_wait.inc
343
344 --disable_query_log
345 --echo Show transactions balanced across nodes by default
346 --echo (based on optimized node selection on localhost)
347
348 #
349 # Intention is to show rough 'balance' of where transactions
350 # are hinted when restoring DATA and LOG parts
351 # We are examining the transaction balance when restoring the
352 # backup of one (of two) data nodes
353 # The balance of rows between the data node's backup files
354 # should be roughly 50:50, based on MD5 hash etc.
355 #
356 # During restore there are two data nodes, so the transaction
357 # balance options are basically :
358 # 50% : 50% (Round robin)
359 # 100% : 0% (All hinted to node 1)
360 # 0% : 100% (All hinted to node 2)
361 #
362 # The BACKUP should overall roughly contain :
363 # - Some system DATA rows to restore
364 # - 1000 DATA rows to restore
365 # - 1000 LOG rows to restore
366 #
367 # One node's backup files should roughly contain :
368 # - Some system DATA rows to restore
369 # - ~1000/2 DATA rows to restore (depending on MD5)
370 # - ~1000/2 LOG rows to restore (depending on MD5)
371 #
372 # When restoring, we may expect the DATA + LOG transactions
373 # to be restored :
374 # - No hinting : 50:50 to each node
375 # ~50% of ~1000/2 + ~1000/2 = ~500
376 # - Hinting : 100:0 to one node
377 # ~100% of ~1000/2 + ~1000/2 = ~1000
378 #
379 # To accomodate uncertainties (#sys rows, MD5 balance, exact RR)
380 # this is validated by rounding to the number of '45% of expected' units
381 # e.g.
382 # - Expect restore of one data node backup to generate
383 # ~1000 transactions (~500 DATA, ~500 LOG)
384 # - Look at how many transactions each data node processes
385 # 0..449 : 0
386 # 450..899 : 1
387 # 900..1349 : 2
388 # - Therefore expect to see
389 # Unhinted case (50%:50%)
390 # Node 1 : 1 Node 2 : 1
391 # Hinted case : (100%:0%)
392 # Node 1 : 2 Node 2 : 0
393 #
394
395 # Get baseline
396 select sum(val) from ndbinfo.counters where counter_name="COMMITS" and node_id=1 into @node1_trans;
397 select sum(val) from ndbinfo.counters where counter_name="COMMITS" and node_id=2 into @node2_trans;
398
399 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 1 -r $NDB_BACKUPS-$the_backup_id >> $NDB_TOOLS_OUTPUT
400 --echo Transaction balance of node 1 restore
401
402 # Calculate transaction diff per node, in units of 45% of expected transactions
403 --echo Expect balance ~ 50% : 50% (1 : 1)
404 select FLOOR((sum(val)-@node1_trans)/450) as n1_trans from ndbinfo.counters where counter_name="COMMITS" and node_id=1;
405 select FLOOR((sum(val)-@node2_trans)/450) as n2_trans from ndbinfo.counters where counter_name="COMMITS" and node_id=2;
406
407 # Get baseline
408 select sum(val) from ndbinfo.counters where counter_name="COMMITS" and node_id=1 into @node1_trans;
409 select sum(val) from ndbinfo.counters where counter_name="COMMITS" and node_id=2 into @node2_trans;
410
411 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 2 -r $NDB_BACKUPS-$the_backup_id >> $NDB_TOOLS_OUTPUT
412 --echo Transaction balance of node 2 restore
413
414 # Calculate transaction diff per node, in units of 45% of expected transactions
415 --echo Expect balance ~ 50% : 50% (1 : 1)
416 select FLOOR((sum(val)-@node1_trans)/450) as n1_trans from ndbinfo.counters where counter_name="COMMITS" and node_id=1;
417 select FLOOR((sum(val)-@node2_trans)/450) as n2_trans from ndbinfo.counters where counter_name="COMMITS" and node_id=2;
418
419 --echo Show that --hint causes each restore to use transactions
420 --echo hinted towards the specific node
421
422 # Get baseline
423 select sum(val) from ndbinfo.counters where counter_name="COMMITS" and node_id=1 into @node1_trans;
424 select sum(val) from ndbinfo.counters where counter_name="COMMITS" and node_id=2 into @node2_trans;
425
426 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 1 -r --hint $NDB_BACKUPS-$the_backup_id >> $NDB_TOOLS_OUTPUT
427 --echo Transaction balance of node 1 hinted restore
428
429 # Calculate transaction diff per node, in units of 45% of expected transactions
430 --echo Expect balance ~ 100% : 0% (2 : 0)
431 select FLOOR((sum(val)-@node1_trans)/450) as n1_trans from ndbinfo.counters where counter_name="COMMITS" and node_id=1;
432 select FLOOR((sum(val)-@node2_trans)/450) as n2_trans from ndbinfo.counters where counter_name="COMMITS" and node_id=2;
433
434 # Get baseline
435 select sum(val) from ndbinfo.counters where counter_name="COMMITS" and node_id=1 into @node1_trans;
436 select sum(val) from ndbinfo.counters where counter_name="COMMITS" and node_id=2 into @node2_trans;
437
438 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 2 -r --hint $NDB_BACKUPS-$the_backup_id >> $NDB_TOOLS_OUTPUT
439 --echo Transaction balance of node 2 hinted restore
440
441 # Calculate transaction diff per node, in units of 45% of expected transactions
442 --echo Expect balance ~ 0% : 100% (0 : 2)
443 select FLOOR((sum(val)-@node1_trans)/450) as n1_trans from ndbinfo.counters where counter_name="COMMITS" and node_id=1;
444 select FLOOR((sum(val)-@node2_trans)/450) as n2_trans from ndbinfo.counters where counter_name="COMMITS" and node_id=2;
445 --enable_query_log
446
447 drop table t1;
448 --echo Test concurrent log apply outcomes
449 # Basic table
450 create table t1 (a int primary key, b int) engine=ndb;
451 # Table with case insensitive collation on pk
452 # Testing that concurrent log apply serialises correctly
453 # with different binary representations of same pk
454 #
455 create table t2 (a varchar(100) CHARSET utf8mb4 COLLATE utf8mb4_0900_ai_ci primary key, b int) engine=ndb;
456 create table t3 (a int primary key, b int) engine=ndb;
457 --let $rows=100
458 --echo Insert rows
459 --disable_query_log
460 --let $r=0
461 while ($r < $rows)
462 {
463 --eval insert into t1 values ($r, 0);
464 --eval insert into t2 values (concat('A', $r), 0);
465 --inc $r
466 }
467 --enable_query_log
468 --echo Start backup with stall-while-logging
469 --echo Cause backup stall while log open
470 --exec $NDB_MGM -e "all error 10040" >> $NDB_TOOLS_OUTPUT
471 # Backup
472 --source suite/ndb/t/ndb_backup_nowait_start.inc
473 --echo Modify rows so that backup log contains
474 --echo changes closely spaced
475 --echo This should maximise the chance of an out-of-order
476 --echo event apply leading to an incorrect final state
477 --disable_query_log
478 --let $r=0
479 while ($r < $rows)
480 {
481 --eval update t1 set b=1 where a=$r;
482 --eval update t1 set b=2 where a=$r;
483
484 --eval update t2 set a=concat('a', $r), b=1 where a = concat('A', $r);
485 --eval update t2 set a=concat('A', $r), b=2 where a = concat('a', $r);
486
487 --inc $r
488 }
489 --echo Generate single long sequence of log ops on one key
490 insert into t3 values (1, -1);
491 --let $updates=3000
492 --let $u=0
493 while ($u < $updates)
494 {
495 --eval update t3 set b=$u where a=1;
496 --inc $u
497 }
498 --enable_query_log
499 --echo Complete backup
500 --exec $NDB_MGM -e "all error 0" >> $NDB_TOOLS_OUTPUT
501 --source suite/ndb/t/ndb_backup_nowait_wait.inc
502 --echo Expect all values in t1 to be 2
503 select count(1) from t1 where b = 2;
504 select count(1) from t1 where b != 2;
505 --echo Expect all values in t2 to be 2
506 select count(1) from t2 where b = 2;
507 select count(1) from t2 where b != 2;
508 --echo Expect all values in t3 to be 2999
509 select * from t3 order by a;
510 --echo Now restore data and check content
511 truncate t1;
512 truncate t2;
513 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 1 -r $NDB_BACKUPS-$the_backup_id >> $NDB_TOOLS_OUTPUT 2>&1
514 --exec $NDB_RESTORE --show-part-id -b $the_backup_id -n 2 -r $NDB_BACKUPS-$the_backup_id >> $NDB_TOOLS_OUTPUT 2>&1
515 --echo Expect all values in t1 to be 2
516 select count(1) from t1 where b = 2;
517 select count(1) from t1 where b != 2;
518 --echo Expect all values in t2 to be 2
519 select count(1) from t2 where b = 2;
520 select count(1) from t2 where b != 2;
521 --echo Expect all values in t3 to be 2999
522 select * from t3 order by a;
523 drop table t1;
524 drop table t2;
525 drop table t3;
526
527 --echo With a big log (>128K) check that ndb_restore will not fail close
528 --echo partially read log file when restoring epochs.
529
530 create table t1 (pk int primary key, data varbinary(20000)) engine=ndb;
531 --echo Stall backup completion
532 --exec $NDB_MGM -e "all error 10039"
533 --echo Run backup
534 --source suite/ndb/t/ndb_backup_nowait_start.inc
535
536 let $pk = 100;
537 --disable_query_log
538 while ($pk) {
539 dec $pk;
540 eval insert into t1 values ($pk, REPEAT(HEX(pk), 10000));
541 }
542 --enable_query_log
543
544 --echo Allow backup to complete...
545 --exec $NDB_MGM -e "all error 0"
546 --source suite/ndb/t/ndb_backup_nowait_wait.inc
547
548 # Mess up control file to cause file read error
549 --remove_file $NDB_BACKUPS-$the_backup_id/BACKUP-$the_backup_id-PART-1-OF-4/BACKUP-$the_backup_id.1.log
550 --write_file $NDB_BACKUPS-$the_backup_id/BACKUP-$the_backup_id-PART-1-OF-4/BACKUP-$the_backup_id.1.log
551 KNAK
552 EOF
553 --error 1
554 --exec $NDB_RESTORE --error-insert=6 --show-part-id --restore-epoch -b $the_backup_id -n 1 $NDB_BACKUPS-$the_backup_id >> $NDB_TOOLS_OUTPUT 2>&1
555
556 # Do not restore any data, only epochs. Only first part of log file is read.
557 --exec $NDB_RESTORE --error-insert=6 --show-part-id --restore-epoch -b $the_backup_id -n 2 $NDB_BACKUPS-$the_backup_id >> $NDB_TOOLS_OUTPUT 2>&1
558
559 drop table t1;
560
561 --source suite/ndb/include/backup_restore_cleanup.inc
562 --remove_file $NDB_TOOLS_OUTPUT