| 1 |
# |
| 2 |
# This include file is used by more than one test suite |
| 3 |
# (currently rpl and binlog_encryption). |
| 4 |
# Please check all dependent tests after modifying it |
| 5 |
# |
| 6 |
|
| 7 |
############################################################ |
| 8 |
# Purpose: WL#5064 Testing with corrupted events. |
| 9 |
# The test emulates the corruption at the vary stages |
| 10 |
# of replication: |
| 11 |
# - in binlog file |
| 12 |
# - in network |
| 13 |
# - in relay log |
| 14 |
############################################################ |
| 15 |
|
| 16 |
# |
| 17 |
# The tests intensively utilize @@global.debug_dbug. Note, |
| 18 |
# Bug#11765758 - 58754, |
| 19 |
# @@global.debug_dbug is read by the slave threads through dbug-interface. |
| 20 |
# Hence, before a client thread set @@global.debug_dbug we have to ensure that: |
| 21 |
# (a) the slave threads are stopped, or (b) the slave threads are in |
| 22 |
# sync and waiting. |
| 23 |
|
| 24 |
--source include/have_debug.inc |
| 25 |
--source include/master-slave.inc |
| 26 |
|
| 27 |
--connection slave |
| 28 |
--source include/stop_slave.inc |
| 29 |
CHANGE MASTER TO MASTER_USE_GTID=NO; |
| 30 |
--source include/start_slave.inc |
| 31 |
--connection master |
| 32 |
|
| 33 |
# Block legal errors for MTR |
| 34 |
call mtr.add_suppression('Found invalid event in binary log'); |
| 35 |
call mtr.add_suppression('Slave I/O: Relay log write failure: could not queue event from master'); |
| 36 |
call mtr.add_suppression('event read from binlog did not pass crc check'); |
| 37 |
call mtr.add_suppression('Replication event checksum verification failed'); |
| 38 |
call mtr.add_suppression('Event crc check failed! Most likely there is event corruption'); |
| 39 |
call mtr.add_suppression('Slave SQL: Error initializing relay log position: I/O error reading event at position .*, error.* 1593'); |
| 40 |
|
| 41 |
SET @old_master_verify_checksum = @@master_verify_checksum; |
| 42 |
|
| 43 |
# Creating test table/data and set corruption position for testing |
| 44 |
--echo # 1. Creating test table/data and set corruption position for testing |
| 45 |
--connection master |
| 46 |
--echo * insert/update/delete rows in table t1 * |
| 47 |
# Corruption algorithm modifies only the first event and |
| 48 |
# then will be reset. To avoid checking always the first event |
| 49 |
# from binlog (usually it is FD) we randomly execute different |
| 50 |
# statements and set position for corruption inside events. |
| 51 |
|
| 52 |
CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY, b VARCHAR(10), c VARCHAR(100)); |
| 53 |
--disable_query_log |
| 54 |
let $i=`SELECT 3+CEILING(10*RAND())`; |
| 55 |
let $j=1; |
| 56 |
let $pos=0; |
| 57 |
while ($i) { |
| 58 |
eval INSERT INTO t1 VALUES ($j, 'a', NULL); |
| 59 |
if (`SELECT RAND() > 0.7`) |
| 60 |
{ |
| 61 |
eval UPDATE t1 SET c = REPEAT('a', 20) WHERE a = $j; |
| 62 |
} |
| 63 |
if (`SELECT RAND() > 0.8`) |
| 64 |
{ |
| 65 |
eval DELETE FROM t1 WHERE a = $j; |
| 66 |
} |
| 67 |
if (!$pos) { |
| 68 |
let $pos= query_get_value(SHOW MASTER STATUS, Position, 1); |
| 69 |
--sync_slave_with_master |
| 70 |
--source include/stop_slave.inc |
| 71 |
--disable_query_log |
| 72 |
--connection master |
| 73 |
} |
| 74 |
dec $i; |
| 75 |
inc $j; |
| 76 |
} |
| 77 |
--enable_query_log |
| 78 |
|
| 79 |
|
| 80 |
# Emulate corruption in binlog file when SHOW BINLOG EVENTS is executing |
| 81 |
--echo # 2. Corruption in master binlog and SHOW BINLOG EVENTS |
| 82 |
SET @saved_dbug = @@global.debug_dbug; |
| 83 |
SET @@global.debug_dbug="d,corrupt_read_log_event_char"; |
| 84 |
--echo SHOW BINLOG EVENTS; |
| 85 |
--disable_query_log |
| 86 |
send_eval SHOW BINLOG EVENTS FROM $pos; |
| 87 |
--enable_query_log |
| 88 |
--error ER_ERROR_WHEN_EXECUTING_COMMAND |
| 89 |
reap; |
| 90 |
|
| 91 |
SET @@global.debug_dbug=@saved_dbug; |
| 92 |
|
| 93 |
# Emulate corruption on master with crc checking on master |
| 94 |
--echo # 3. Master read a corrupted event from binlog and send the error to slave |
| 95 |
|
| 96 |
# We have a rare but nasty potential race here: if the dump thread on |
| 97 |
# the master for the _old_ slave connection has not yet discovered |
| 98 |
# that the slave has disconnected, we will inject the corrupt event on |
| 99 |
# the wrong connection, and the test will fail |
| 100 |
# (+d,corrupt_read_log_event2 corrupts only one event). |
| 101 |
# So kill any lingering dump thread (we need to kill; otherwise dump thread |
| 102 |
# could manage to send all events down the socket before seeing it close, and |
| 103 |
# hang forever waiting for new binlog events to be created). |
| 104 |
let $id= `select id from information_schema.processlist where command = "Binlog Dump"`; |
| 105 |
if ($id) |
| 106 |
{ |
| 107 |
--disable_query_log |
| 108 |
--error 0,1094 |
| 109 |
eval kill $id; |
| 110 |
--enable_query_log |
| 111 |
} |
| 112 |
let $wait_condition= |
| 113 |
SELECT COUNT(*)=0 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE command = 'Binlog Dump'; |
| 114 |
--source include/wait_condition.inc |
| 115 |
|
| 116 |
SET @@global.debug_dbug="d,corrupt_read_log_event2_set"; |
| 117 |
--connection slave |
| 118 |
START SLAVE IO_THREAD; |
| 119 |
let $slave_io_errno= 1236; |
| 120 |
--let $slave_timeout= 10 |
| 121 |
--source include/wait_for_slave_io_error.inc |
| 122 |
--connection master |
| 123 |
SET @@global.debug_dbug=@saved_dbug; |
| 124 |
|
| 125 |
# Emulate corruption on master without crc checking on master |
| 126 |
--echo # 4. Master read a corrupted event from binlog and send it to slave |
| 127 |
--connection master |
| 128 |
SET GLOBAL master_verify_checksum=0; |
| 129 |
SET @@global.debug_dbug="d,corrupt_read_log_event2_set"; |
| 130 |
--connection slave |
| 131 |
START SLAVE IO_THREAD; |
| 132 |
# When the checksum error is detected, the slave sets error code 1743 |
| 133 |
# (ER_NETWORK_READ_EVENT_CHECKSUM_FAILURE) in queue_event(), then immediately |
| 134 |
# sets error 1595 (ER_SLAVE_RELAY_LOG_WRITE_FAILURE) in handle_slave_io(). |
| 135 |
# So we usually get 1595, but it is occasionally possible to get 1743. |
| 136 |
let $slave_io_errno= 1595,1743; # ER_SLAVE_RELAY_LOG_WRITE_FAILURE, ER_NETWORK_READ_EVENT_CHECKSUM_FAILURE |
| 137 |
--source include/wait_for_slave_io_error.inc |
| 138 |
--connection master |
| 139 |
SET @@global.debug_dbug=@saved_dbug; |
| 140 |
SET GLOBAL master_verify_checksum=1; |
| 141 |
|
| 142 |
# Emulate corruption in network |
| 143 |
--echo # 5. Slave. Corruption in network |
| 144 |
--connection slave |
| 145 |
SET @saved_dbug_slave = @@GLOBAL.debug_dbug; |
| 146 |
SET @@global.debug_dbug="d,corrupt_queue_event"; |
| 147 |
START SLAVE IO_THREAD; |
| 148 |
let $slave_io_errno= 1595,1743; # ER_SLAVE_RELAY_LOG_WRITE_FAILURE, ER_NETWORK_READ_EVENT_CHECKSUM_FAILURE |
| 149 |
--source include/wait_for_slave_io_error.inc |
| 150 |
SET @@global.debug_dbug=@saved_dbug_slave; |
| 151 |
|
| 152 |
# Emulate corruption in relay log |
| 153 |
--echo # 6. Slave. Corruption in relay log |
| 154 |
|
| 155 |
SET @@global.debug_dbug="d,corrupt_read_log_event_char"; |
| 156 |
|
| 157 |
START SLAVE SQL_THREAD; |
| 158 |
let $slave_sql_errno= 1593; |
| 159 |
--source include/wait_for_slave_sql_error.inc |
| 160 |
|
| 161 |
SET @@global.debug_dbug=@saved_dbug_slave; |
| 162 |
|
| 163 |
# Start normal replication and compare same table on master |
| 164 |
# and slave |
| 165 |
--echo # 7. Seek diff for tables on master and slave |
| 166 |
--connection slave |
| 167 |
--source include/start_slave.inc |
| 168 |
--connection master |
| 169 |
--sync_slave_with_master |
| 170 |
let $diff_tables= master:test.t1, slave:test.t1; |
| 171 |
--source include/diff_tables.inc |
| 172 |
|
| 173 |
# Clean up |
| 174 |
--echo # 8. Clean up |
| 175 |
--connection master |
| 176 |
set @@global.debug_dbug = @saved_dbug; |
| 177 |
SET GLOBAL master_verify_checksum = @old_master_verify_checksum; |
| 178 |
DROP TABLE t1; |
| 179 |
--sync_slave_with_master |
| 180 |
|
| 181 |
--source include/rpl_end.inc |