| 1 |
# Populate a table with 1000 records. Allow the replica to sync with the master. |
| 2 |
# Run concurrent threads that run OLTP transactions on master. |
| 3 |
# Kill the master database server at random points. |
| 4 |
# Check the table against the replica. |
| 5 |
# Reinvoke the threads. |
| 6 |
|
| 7 |
|
| 8 |
# Set python interpreter to use (On Windows executable is python.exe) |
| 9 |
--let $PYTHON = python3 |
| 10 |
if (`SELECT CONVERT(@@VERSION_COMPILE_OS USING latin1) IN ('Win32', 'Win64', 'Windows')`) |
| 11 |
{ |
| 12 |
--let $PYTHON = python |
| 13 |
} |
| 14 |
|
| 15 |
# create the directory for temporary log files. |
| 16 |
--mkdir $MYSQL_TMP_DIR/load_generator |
| 17 |
|
| 18 |
if ($fake_changes) |
| 19 |
{ |
| 20 |
--mkdir $MYSQL_TMP_DIR/load_generator_slave |
| 21 |
} |
| 22 |
|
| 23 |
--connection master |
| 24 |
|
| 25 |
CREATE TABLE errors (msg TEXT); |
| 26 |
|
| 27 |
--sync_slave_with_master |
| 28 |
|
| 29 |
--connection master |
| 30 |
--let $pid_file = `SELECT @@pid_file` |
| 31 |
--let $crash_num = 0 |
| 32 |
--let $master_host = 127.0.0.1 |
| 33 |
--let $table = test |
| 34 |
--let $user = root |
| 35 |
--let $checksum = 0 |
| 36 |
--let $secondary_index_checks = 0 |
| 37 |
|
| 38 |
if ($do_checksum) |
| 39 |
{ |
| 40 |
# populate the table and store its checksum before any load. |
| 41 |
let $exec = |
| 42 |
$PYTHON $MYSQL_TEST_DIR/suite/innodb_stress/t/load_generator.py $pid_file $kill_db_after |
| 43 |
$num_records 0 0 $user $master_host $MASTER_MYPORT |
| 44 |
$table 0 $max_rows $MYSQL_TMP_DIR/load_generator 0 0 0; |
| 45 |
exec $exec >> $MYSQL_TMP_DIR/load_generator/errors.log; |
| 46 |
let $checksum=query_get_value(CHECKSUM TABLE t1, Checksum, 1); |
| 47 |
# Master needs to be restarted to start with an empty buffer pool so |
| 48 |
# that logical read ahead gets used. |
| 49 |
let rpl_server_number = 1; |
| 50 |
source include/rpl_restart_server.inc; |
| 51 |
connection slave; |
| 52 |
# Start slave to avoid I/O thread retry errors |
| 53 |
disable_warnings; |
| 54 |
source include/start_slave.inc; |
| 55 |
enable_warnings; |
| 56 |
let $num_records = 0; |
| 57 |
} |
| 58 |
|
| 59 |
while ($num_crashes) |
| 60 |
{ |
| 61 |
connection master; |
| 62 |
exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect; |
| 63 |
if ($crash_num) |
| 64 |
{ |
| 65 |
let $num_records = 0; # do not populate the table except for the first run. |
| 66 |
} |
| 67 |
|
| 68 |
if ($use_blob) |
| 69 |
{ |
| 70 |
let $exec = |
| 71 |
$PYTHON $MYSQL_TEST_DIR/suite/innodb_stress/t/load_generator.py $pid_file $kill_db_after |
| 72 |
$num_records $num_workers $num_transactions $user $master_host $MASTER_MYPORT |
| 73 |
$table 1 $max_rows $MYSQL_TMP_DIR/load_generator 0 $checksum $secondary_index_checks; |
| 74 |
} |
| 75 |
if (!$use_blob) |
| 76 |
{ |
| 77 |
let $exec = |
| 78 |
$PYTHON $MYSQL_TEST_DIR/suite/innodb_stress/t/load_generator.py $pid_file $kill_db_after |
| 79 |
$num_records $num_workers $num_transactions $user $master_host $MASTER_MYPORT |
| 80 |
$table 0 $max_rows $MYSQL_TMP_DIR/load_generator 0 $checksum $secondary_index_checks; |
| 81 |
} |
| 82 |
|
| 83 |
exec $exec >> $MYSQL_TMP_DIR/load_generator/errors.log; |
| 84 |
|
| 85 |
if ($do_crash) |
| 86 |
{ |
| 87 |
--echo Wait for reconnect |
| 88 |
enable_reconnect; |
| 89 |
# Call script that will poll the server waiting for it to be back online again |
| 90 |
source include/wait_until_connected_again.inc; |
| 91 |
connection slave; |
| 92 |
source include/wait_until_connected_again.inc; |
| 93 |
connection master; |
| 94 |
} |
| 95 |
|
| 96 |
--echo Checksum master |
| 97 |
let $master_checksum = query_get_value(CHECKSUM TABLE t1, Checksum, 1); |
| 98 |
|
| 99 |
# if sync_slave_with_master had a configurable timeout this would not be needed |
| 100 |
let $slave_sync_timeout = 7200; |
| 101 |
--source include/wait_for_slave_to_sync_with_master.inc |
| 102 |
|
| 103 |
connection slave; |
| 104 |
if ($fake_changes) |
| 105 |
{ |
| 106 |
--echo applying fake updates to the slave |
| 107 |
let $slave_pid_file = `SELECT @@pid_file`; |
| 108 |
let $slave_exec = |
| 109 |
$PYTHON $MYSQL_TEST_DIR/suite/innodb_stress/t/load_generator.py $slave_pid_file $kill_db_after |
| 110 |
0 $num_workers $num_transactions $user $master_host $SLAVE_MYPORT |
| 111 |
$table 0 $max_rows $MYSQL_TMP_DIR/load_generator_slave 1 $checksum $secondary_index_checks; |
| 112 |
exec $slave_exec >> $MYSQL_TMP_DIR/load_generator/errors.log; |
| 113 |
} |
| 114 |
--echo Checksum slave |
| 115 |
let $slave_checksum=query_get_value(CHECKSUM TABLE t1, Checksum, 1); |
| 116 |
let $not_same = `SELECT $master_checksum-$slave_checksum`; |
| 117 |
if ($not_same) |
| 118 |
{ |
| 119 |
let $msg = |
| 120 |
The checksums of table t1 for master and slave do not match for $crash_num th |
| 121 |
crash. This may happen if there is a corrupt recovery log or a bug in crash |
| 122 |
recovery. You can take a look at the logs in $MYSQL_TMP_DIR/load_generator to see the |
| 123 |
queries issued before the crash.; |
| 124 |
echo $msg; |
| 125 |
|
| 126 |
connection master; |
| 127 |
eval select * into outfile '$MYSQLTEST_VARDIR/tmp/master_all' from t1 order by id; |
| 128 |
eval select id into outfile '$MYSQLTEST_VARDIR/tmp/master_id' from t1 order by id; |
| 129 |
show master status; |
| 130 |
|
| 131 |
connection slave; |
| 132 |
eval select * into outfile '$MYSQLTEST_VARDIR/tmp/slave_all' from t1 order by id; |
| 133 |
eval select id into outfile '$MYSQLTEST_VARDIR/tmp/slave_id' from t1 order by id; |
| 134 |
show slave status; |
| 135 |
|
| 136 |
die; |
| 137 |
} |
| 138 |
dec $num_crashes; |
| 139 |
inc $crash_num; |
| 140 |
} |
| 141 |
|
| 142 |
# final cleanup |
| 143 |
--connection master |
| 144 |
|
| 145 |
let $primary=`select count(*) from t1 use index (primary)`; |
| 146 |
let $secondary=`select count(*) from t1 use index (msg_i)`; |
| 147 |
if ($primary != $secondary) |
| 148 |
{ |
| 149 |
--echo Secondary index inconsistent! $primary != $secondary |
| 150 |
--die |
| 151 |
} |
| 152 |
|
| 153 |
if (!$do_crash) |
| 154 |
{ |
| 155 |
--cat_file $MYSQL_TMP_DIR/load_generator/errors.log |
| 156 |
SELECT * FROM errors; |
| 157 |
} |
| 158 |
|
| 159 |
DROP TABLE t1; |
| 160 |
DROP TABLE errors; |
| 161 |
|
| 162 |
# if sync_slave_with_master had a configurable timeout this would not be needed |
| 163 |
let $slave_sync_timeout = 7200; |
| 164 |
--source include/wait_for_slave_to_sync_with_master.inc |
| 165 |
|
| 166 |
--connection slave |
| 167 |
--source include/stop_slave.inc |
| 168 |
# For stress tests sometimes the replication thread can not connect to master |
| 169 |
# temporarily. This is either because the master crashed and it is recovering |
| 170 |
# or the master is too busy and could not service the slave's requests. |
| 171 |
# mtr's internal check requires that there be no errors in slave status. |
| 172 |
# restarting replication clears the errors. |
| 173 |
--source include/start_slave.inc |
| 174 |
--source include/stop_slave.inc |
| 175 |
|
| 176 |
connection master; |
| 177 |
|
| 178 |
--force-rmdir $MYSQL_TMP_DIR/load_generator |
| 179 |
if ($fake_changes) |
| 180 |
{ |
| 181 |
--force-rmdir $MYSQL_TMP_DIR/load_generator_slave |
| 182 |
} |
| 183 |
|
| 184 |
# Reconnect client in case master had been killed |
| 185 |
if ($kill_db_after) |
| 186 |
{ |
| 187 |
--let $rpl_server_number = 1 |
| 188 |
--source include/rpl_reconnect.inc |
| 189 |
} |
| 190 |
|
| 191 |
--let $rpl_only_running_threads= 1 |
| 192 |
--source include/rpl_end.inc |