| 1 |
# |
| 2 |
# Test a case where a joiner encounters an error during IST |
| 3 |
# Instead of voting it should assume error and bail out. |
| 4 |
# |
| 5 |
|
| 6 |
--source include/galera_cluster.inc |
| 7 |
--source include/big_test.inc |
| 8 |
--source include/have_debug.inc |
| 9 |
--source include/have_debug_sync.inc |
| 10 |
|
| 11 |
# Make sure that the test is operating on the right version of galera library. |
| 12 |
--let $galera_version=26.4.19 |
| 13 |
source ../wsrep/include/check_galera_version.inc; |
| 14 |
|
| 15 |
--let $node_1=node_1 |
| 16 |
--let $node_2=node_2 |
| 17 |
--let $node_3=node_3 |
| 18 |
--let $node_4=node_4 |
| 19 |
--source ../include/auto_increment_offset_save.inc |
| 20 |
|
| 21 |
# create table t1 and procedure p1 to generate wirtesets |
| 22 |
--connection node_1 |
| 23 |
|
| 24 |
--let $members = 4 |
| 25 |
--source include/wsrep_wait_membership.inc |
| 26 |
|
| 27 |
CREATE TABLE t1(pk INT AUTO_INCREMENT PRIMARY KEY); |
| 28 |
|
| 29 |
DELIMITER |; |
| 30 |
CREATE PROCEDURE p1(IN max INT) |
| 31 |
BEGIN |
| 32 |
DECLARE i INT; |
| 33 |
DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END; |
| 34 |
|
| 35 |
SET i = 0; |
| 36 |
WHILE i < max DO |
| 37 |
INSERT IGNORE INTO t1 VALUES (DEFAULT); |
| 38 |
SET i = i + 1; |
| 39 |
END WHILE; |
| 40 |
END| |
| 41 |
DELIMITER ;| |
| 42 |
|
| 43 |
CALL p1(130); |
| 44 |
|
| 45 |
--connection node_4 |
| 46 |
--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'; |
| 47 |
--source include/wait_condition.inc |
| 48 |
--let $wait_condition = SELECT COUNT(*) = 130 FROM t1; |
| 49 |
--source include/wait_condition.inc |
| 50 |
|
| 51 |
--echo Shutting down server 4... |
| 52 |
--let $node_4_server_id= `SELECT @@server_id` |
| 53 |
--let $node_4_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$node_4_server_id.expect |
| 54 |
--let $node_4_pid_file= `SELECT @@pid_file` |
| 55 |
--source include/shutdown_mysqld.inc |
| 56 |
--source include/wait_until_disconnected.inc |
| 57 |
|
| 58 |
# Wait for node #4 to leave cluster |
| 59 |
--let $members = 3 |
| 60 |
--connection node_1 |
| 61 |
--source include/wsrep_wait_membership.inc |
| 62 |
--connection node_2 |
| 63 |
--source include/wsrep_wait_membership.inc |
| 64 |
--connection node_3 |
| 65 |
--source include/wsrep_wait_membership.inc |
| 66 |
--echo Server 4 left the cluster |
| 67 |
|
| 68 |
# Create some writesets for IST |
| 69 |
--connection node_1 |
| 70 |
CALL p1(130); |
| 71 |
|
| 72 |
# Create a writeset that node 4 won't be able to apply by creating a table |
| 73 |
# that won't be present in the replication stream |
| 74 |
--connection node_1 |
| 75 |
SET SESSION wsrep_on = OFF; |
| 76 |
CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); |
| 77 |
SET SESSION wsrep_on = ON; |
| 78 |
--source include/wait_until_ready.inc |
| 79 |
|
| 80 |
--connection node_2 |
| 81 |
--let $wait_condition = SELECT COUNT(*) = 260 FROM t1; |
| 82 |
--source include/wait_condition.inc |
| 83 |
SET SESSION wsrep_on = OFF; |
| 84 |
CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); |
| 85 |
SET SESSION wsrep_on = ON; |
| 86 |
--source include/wait_until_ready.inc |
| 87 |
|
| 88 |
--connection node_3 |
| 89 |
--let $wait_condition = SELECT COUNT(*) = 260 FROM t1; |
| 90 |
--source include/wait_condition.inc |
| 91 |
SET SESSION wsrep_on = OFF; |
| 92 |
CREATE TABLE t2(pk INT AUTO_INCREMENT PRIMARY KEY); |
| 93 |
SET SESSION wsrep_on = ON; |
| 94 |
--source include/wait_until_ready.inc |
| 95 |
|
| 96 |
# This should cause error during IST |
| 97 |
INSERT INTO t2 VALUES (DEFAULT); |
| 98 |
|
| 99 |
# make sure nodes 1,2,3 progress far enough for commit cut update |
| 100 |
CALL p1(130); |
| 101 |
|
| 102 |
--connection node_2 |
| 103 |
--let $wait_condition = SELECT COUNT(*) = 390 FROM t1; |
| 104 |
--source include/wait_condition.inc |
| 105 |
--connection node_3 |
| 106 |
--let $wait_condition = SELECT COUNT(*) = 390 FROM t1; |
| 107 |
--source include/wait_condition.inc |
| 108 |
|
| 109 |
--connection node_1 |
| 110 |
# prepare to stop SST donor thread when it receives a request from starting node #4 |
| 111 |
SET GLOBAL debug = "+d,sync.wsrep_sst_donor_after_donation"; |
| 112 |
|
| 113 |
--echo Restarting server 4 |
| 114 |
# Need to use this form instead of start_mysqld.inc because the latter is blocking |
| 115 |
--exec echo "restart:$start_mysqld_params" > $node_4_expect_file_name |
| 116 |
|
| 117 |
--echo Wait for server 1 to become a donor |
| 118 |
SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_sst_donor_after_donation_reached"; |
| 119 |
--echo Server 1 got SST request from server 4 |
| 120 |
SET SESSION DEBUG_SYNC = "now SIGNAL signal.wsrep_sst_donor_after_donation_continue"; |
| 121 |
SET GLOBAL debug = ""; |
| 122 |
SET DEBUG_SYNC='RESET'; |
| 123 |
# |
| 124 |
# After this point node #4 shall proceed to IST and bail out |
| 125 |
# |
| 126 |
|
| 127 |
--echo Waiting for server 4 to leave the cluster |
| 128 |
--let $members = 3 |
| 129 |
--connection node_2 |
| 130 |
--source include/wsrep_wait_membership.inc |
| 131 |
--connection node_3 |
| 132 |
--source include/wsrep_wait_membership.inc |
| 133 |
|
| 134 |
--connection node_4 |
| 135 |
--echo Server 4 left the cluster, killing it... |
| 136 |
|
| 137 |
# Kill the connected server |
| 138 |
--exec echo "wait" > $node_4_expect_file_name |
| 139 |
--let KILL_NODE_PIDFILE = $node_4_pid_file |
| 140 |
--perl |
| 141 |
my $pid_filename = $ENV{'KILL_NODE_PIDFILE'}; |
| 142 |
my $mysqld_pid = `cat $pid_filename`; |
| 143 |
chomp($mysqld_pid); |
| 144 |
system("kill -9 $mysqld_pid"); |
| 145 |
exit(0); |
| 146 |
EOF |
| 147 |
--echo Killed server 4... |
| 148 |
--source include/wait_until_disconnected.inc |
| 149 |
|
| 150 |
--connection node_1 |
| 151 |
--source include/wait_until_ready.inc |
| 152 |
--let $wait_condition = SELECT VARIABLE_VALUE = 'Synced' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'; |
| 153 |
--let $wait_condition_on_error_output = SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_state_comment'; |
| 154 |
--source include/wait_condition_with_debug.inc |
| 155 |
|
| 156 |
--connection node_4 |
| 157 |
--echo Restarting server 4... |
| 158 |
--source include/start_mysqld.inc |
| 159 |
--source include/galera_wait_ready.inc |
| 160 |
|
| 161 |
|
| 162 |
# Confirm node #4 has rejoined |
| 163 |
--connection node_1 |
| 164 |
--let $members = 4 |
| 165 |
--source include/wsrep_wait_membership.inc |
| 166 |
--connection node_2 |
| 167 |
--source include/wsrep_wait_membership.inc |
| 168 |
--connection node_3 |
| 169 |
--source include/wsrep_wait_membership.inc |
| 170 |
|
| 171 |
# Confirm that all is good and all nodes have identical data |
| 172 |
|
| 173 |
--connection node_1 |
| 174 |
SELECT count(*) AS expect1_390 FROM t1; |
| 175 |
SELECT count(*) AS expect1_1 FROM t2; |
| 176 |
|
| 177 |
--connection node_2 |
| 178 |
SELECT count(*) AS expect2_390 FROM t1; |
| 179 |
SELECT count(*) AS expect2_1 FROM t2; |
| 180 |
|
| 181 |
--connection node_3 |
| 182 |
SELECT count(*) AS expect3_390 FROM t1; |
| 183 |
SELECT count(*) AS expect3_1 FROM t2; |
| 184 |
|
| 185 |
--connection node_4 |
| 186 |
--let $wait_condition = SELECT COUNT(*) = 390 FROM t1; |
| 187 |
--source include/wait_condition.inc |
| 188 |
--let $wait_condition = SELECT COUNT(*) = 1 FROM t2; |
| 189 |
--source include/wait_condition.inc |
| 190 |
SELECT count(*) AS expect4_390 FROM t1; |
| 191 |
SELECT count(*) AS expect4_1 FROM t2; |
| 192 |
|
| 193 |
DROP TABLE t1; |
| 194 |
DROP TABLE t2; |
| 195 |
DROP PROCEDURE p1; |
| 196 |
|
| 197 |
CALL mtr.add_suppression("BF applier thread=.+ failed to open_and_lock_tables for Table "); |
| 198 |
CALL mtr.add_suppression("Event 3 Write_rows_v1 apply failed: 1146"); |
| 199 |
CALL mtr.add_suppression("Inconsistency detected: Failed on preordered"); |
| 200 |
CALL mtr.add_suppression("Failed to apply write set"); |
| 201 |
CALL mtr.add_suppression("Sending JOIN failed: -103"); |
| 202 |
CALL mtr.add_suppression("Failed to JOIN the cluster after SST"); |
| 203 |
|
| 204 |
--source ../include/auto_increment_offset_restore.inc |