******************************************************** ndb_restore threads exit cleanly on tuple restore error ******************************************************** Initial tests using ndb_restore error insert to trigger error on all parts. create table t1(id int primary key, val int) engine=ndb; Cause backup stall Starting backup Backup started update t1 set val=12345 where id between 125 and 850; Waiting for backup to complete Backup completed Test restore failure handling for log-replay failure select count(*) into @msgcount from ndb_restore_output where a like "%Thread % failed, exiting" order by a; select @msgcount as 'Count of thread exit messages'; Count of thread exit messages 4 select * from ndb_restore_output where @msgcount != 4; a truncate table t1; Test restore failure handling for tuple-restore failure select count(*) into @msgcount from ndb_restore_output where a like "%Thread % failed, exiting" order by a; select @msgcount as 'Count of thread exit messages'; Count of thread exit messages 4 select * from ndb_restore_output where @msgcount != 4; a drop table t1; Test single part restore failures, should cause all parts to stop promptly Constraint violation on DATA restore create table test.t1 (a int primary key, b int, c int) engine=ndb; insert into test.t1 values (1000,1000,999); Starting backup Backup started Waiting for backup to complete Backup completed truncate test.t1; alter table test.t1 add unique(c); Test restore failure handling for data apply failure Expect 1 error type message select count(*) into @msgcount from ndb_restore_output where a like "%Permanent% duplicate value in unique index%"; select @msgcount as 'Count of error message 1'; Count of error message 1 1 select * from ndb_restore_output where @msgcount != 1; a Expect 1 error location message select count(*) into @msgcount from ndb_restore_output where a like "%Failed restoring DATA to table%"; select @msgcount as 'Count of error message 2'; Count of error message 2 1 select * from ndb_restore_output where @msgcount != 1; a Expect 1 error detail message select count(*) into @msgcount from ndb_restore_output where a like "%Failing tuple :%"; select @msgcount as 'Count of error message 3'; Count of error message 3 1 select * from ndb_restore_output where @msgcount != 1; a drop table test.t1; Constraint violation on LOG restore create table test.t1 (a int, b int, c int, primary key (a,b)) engine=ndb partition by key(a); Cause backup stall while log open Starting backup Backup started insert into t1 values(999,1000,999); Waiting for backup to complete Backup completed truncate test.t1; alter table test.t1 add unique(c); Test restore failure handling for log apply failure Expect 1 error type message select count(*) into @msgcount from ndb_restore_output where a like "%Permanent% duplicate value in unique index%"; select @msgcount as 'Count of error message 1'; Count of error message 1 1 select * from ndb_restore_output where @msgcount != 1; a Expect 1 error location message select count(*) into @msgcount from ndb_restore_output where a like "%Failed restoring LOG entry type INSERT to table%"; select @msgcount as 'Count of error message 2'; Count of error message 2 1 select * from ndb_restore_output where @msgcount != 1; a Expect 1 error detail message select count(*) into @msgcount from ndb_restore_output where a like "%Failing log entry :%"; select @msgcount as 'Count of error message 3'; Count of error message 3 1 select * from ndb_restore_output where @msgcount != 1; a drop table test.t1; Test that a temporary error applying log eventually results in failure create table test.t1 (a int primary key, b int) engine=ndb; Cause backup stall while log open Starting backup Backup started insert into t1 values(1,1); Waiting for backup to complete Backup completed Take a row lock to block restore begin; select * from test.t1 where a=1 for update; a b 1 1 Test restore failure handling for log apply failure after retries Expect error message select count(*) into @msgcount from ndb_restore_output where a like "%Error restoring the data log%" or a like "%Error reading the data log%"; select @msgcount as 'Count of error message 1'; Count of error message 1 1 select * from ndb_restore_output where @msgcount != 1; a drop table test.t1; ---------------- Test hint option ---------------- Create backup with good amount of DATA + LOG content create table t1(id int primary key, val int) engine=ndb; Cause backup stall Starting backup Backup started update t1 set val=val+1; Waiting for backup to complete Backup completed Show transactions balanced across nodes by default (based on optimized node selection on localhost) Transaction balance of node 1 restore Expect balance ~ 50% : 50% (1 : 1) n1_trans 1 n2_trans 1 Transaction balance of node 2 restore Expect balance ~ 50% : 50% (1 : 1) n1_trans 1 n2_trans 1 Show that --hint causes each restore to use transactions hinted towards the specific node Transaction balance of node 1 hinted restore Expect balance ~ 100% : 0% (2 : 0) n1_trans 2 n2_trans 0 Transaction balance of node 2 hinted restore Expect balance ~ 0% : 100% (0 : 2) n1_trans 0 n2_trans 2 drop table t1; Test concurrent log apply outcomes create table t1 (a int primary key, b int) engine=ndb; create table t2 (a varchar(100) CHARSET utf8mb4 COLLATE utf8mb4_0900_ai_ci primary key, b int) engine=ndb; create table t3 (a int primary key, b int) engine=ndb; Insert rows Start backup with stall-while-logging Cause backup stall while log open Starting backup Backup started Modify rows so that backup log contains changes closely spaced This should maximise the chance of an out-of-order event apply leading to an incorrect final state Generate single long sequence of log ops on one key Complete backup Waiting for backup to complete Backup completed Expect all values in t1 to be 2 select count(1) from t1 where b = 2; count(1) 100 select count(1) from t1 where b != 2; count(1) 0 Expect all values in t2 to be 2 select count(1) from t2 where b = 2; count(1) 100 select count(1) from t2 where b != 2; count(1) 0 Expect all values in t3 to be 2999 select * from t3 order by a; a b 1 2999 Now restore data and check content truncate t1; truncate t2; Expect all values in t1 to be 2 select count(1) from t1 where b = 2; count(1) 100 select count(1) from t1 where b != 2; count(1) 0 Expect all values in t2 to be 2 select count(1) from t2 where b = 2; count(1) 100 select count(1) from t2 where b != 2; count(1) 0 Expect all values in t3 to be 2999 select * from t3 order by a; a b 1 2999 drop table t1; drop table t2; drop table t3; With a big log (>128K) check that ndb_restore will not fail close partially read log file when restoring epochs. create table t1 (pk int primary key, data varbinary(20000)) engine=ndb; Stall backup completion Run backup Starting backup Backup started Allow backup to complete... Waiting for backup to complete Backup completed drop table t1;