| 1 |
############################################################# |
| 2 |
# Test behaviout of the binlog injector during restart of itself. |
| 3 |
# A restart should be triggered when the mysqld disconnects from |
| 4 |
# the datanodes. |
| 5 |
# |
| 6 |
############################################################# |
| 7 |
|
| 8 |
-- source include/have_binlog_format_mixed_or_row.inc |
| 9 |
-- source include/have_multi_ndb.inc |
| 10 |
|
| 11 |
# Ignore the warning generated by ndbcluster's binlog thread |
| 12 |
# when mysqld is disconnected from cluster |
| 13 |
--disable_query_log ONCE |
| 14 |
call mtr.add_suppression("cluster disconnect An incident event has been written"); |
| 15 |
|
| 16 |
################################################## |
| 17 |
# Check whether we have a debug compiled binary: |
| 18 |
# @@global.debug can't be referred without this. |
| 19 |
# However, instead of skipping entire test without debug, |
| 20 |
# we run it without setting the specific debug flags. |
| 21 |
################################################### |
| 22 |
--let $have_debug= `select (version() like '%debug%')` |
| 23 |
|
| 24 |
connection server1; |
| 25 |
# Find NodeId of the mysqld we are connected to: |
| 26 |
--let $node_id= `SHOW STATUS LIKE 'Ndb_cluster_node_id'` |
| 27 |
--let $node_id= `SELECT SUBSTRING('$node_id', 20)+0` |
| 28 |
|
| 29 |
connection server2; |
| 30 |
# Find NodeId2 of 'server2' we are connected to: |
| 31 |
--let $node_id2= `SHOW STATUS LIKE 'Ndb_cluster_node_id'` |
| 32 |
--let $node_id2= `SELECT SUBSTRING('$node_id2', 20)+0` |
| 33 |
|
| 34 |
# Ignore the warning generated by ndbcluster's binlog thread |
| 35 |
# when mysqld is disconnected from cluster also on second Server |
| 36 |
--disable_query_log ONCE |
| 37 |
call mtr.add_suppression("cluster disconnect An incident event has been written"); |
| 38 |
|
| 39 |
connection server1; |
| 40 |
reset master; |
| 41 |
|
| 42 |
############################################################# |
| 43 |
# A TE_FAILURE event will be received by the binlog thread |
| 44 |
# due to the DUMP disconnecting mysqld from datanodes. |
| 45 |
# When binlog handles the FAILURE event, mysqld is put into |
| 46 |
# readonly mode. A restart of the Binlog thread is required |
| 47 |
# to bring it out of RO-mode. |
| 48 |
############################################################# |
| 49 |
--echo Disconnect mysqld from all datanodes |
| 50 |
--exec $NDB_MGM -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT |
| 51 |
|
| 52 |
############################################################# |
| 53 |
# The disconnect will create a GAP event in the binlog. |
| 54 |
# To avoid 'disconnect failures' while probing for 'not_readonly', |
| 55 |
# we use the binlog GAP as an indicator that disconnect has |
| 56 |
# already completed. |
| 57 |
############################################################# |
| 58 |
--echo Wait for binlog rotate to indicate disconnect |
| 59 |
--let $wait_binlog_event= binlog.000002 |
| 60 |
--source include/wait_for_binlog_event.inc |
| 61 |
|
| 62 |
############################################################# |
| 63 |
# Until the binlog injector has completed the restart all tables |
| 64 |
# will temporarily be in a read-only mode. |
| 65 |
# (As the binlog will not be able to log any updates) |
| 66 |
############################################################# |
| 67 |
--echo Wait to get out of read-only mode |
| 68 |
--source include/ndb_not_readonly.inc |
| 69 |
|
| 70 |
############################################################# |
| 71 |
# During the restart a 'Gap event' should be written to the |
| 72 |
# binlog and the log rotated to 'binlog.000002'. |
| 73 |
############################################################# |
| 74 |
--echo |
| 75 |
--echo Check that the binlog was rotated: |
| 76 |
--echo There should be a 'LOST EVENT', then 'Rotate ... binlog.000002' |
| 77 |
--source include/show_binlog_events.inc |
| 78 |
|
| 79 |
|
| 80 |
############################################################# |
| 81 |
# Bug#22204186: NDB REPLICATION : READ ONLY TABLES AND NO BINLOGGING |
| 82 |
# |
| 83 |
# Tables could get stuck in read only mode due to slow handling |
| 84 |
# of cluster failure in the binlog. |
| 85 |
# |
| 86 |
# Testcase is identical to the one above, except that |
| 87 |
# we use debug inserts to force the binlog injector to take |
| 88 |
# a break when handling failures. When not using debug compiled |
| 89 |
# binaries, there will be no such breaks inserted, but the |
| 90 |
# test should still succeed. |
| 91 |
# |
| 92 |
# This previously caused a race condition where the Util-thread |
| 93 |
# managed to recreate the schema distr. tables before |
| 94 |
# the binlog thread had completed the failure handling. Thus, |
| 95 |
# the binlog never reached its restart condition of all |
| 96 |
# eventOps have been dropped. |
| 97 |
############################################################# |
| 98 |
|
| 99 |
--echo |
| 100 |
--echo Testing for Bug#22204186: Stuck in Readonly if slow failure handling |
| 101 |
--echo ==================================================================== |
| 102 |
|
| 103 |
disable_query_log; |
| 104 |
if ($have_debug) |
| 105 |
{ |
| 106 |
set @save_debug= @@global.debug; |
| 107 |
set global debug='+d,ndb_binlog_slow_failure_handling'; |
| 108 |
} |
| 109 |
enable_query_log; |
| 110 |
|
| 111 |
reset master; |
| 112 |
|
| 113 |
--echo Disconnect mysqld from all datanodes |
| 114 |
--exec $NDB_MGM -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT |
| 115 |
|
| 116 |
--echo Wait for binlog rotate to indicate disconnect |
| 117 |
--let $wait_binlog_event= binlog.000002 |
| 118 |
--source include/wait_for_binlog_event.inc |
| 119 |
|
| 120 |
--echo Wait to get out of read-only mode |
| 121 |
--source include/ndb_not_readonly.inc |
| 122 |
|
| 123 |
--echo |
| 124 |
--echo Check that the binlog was rotated: |
| 125 |
--echo There should be a 'LOST EVENT', then 'Rotate ... binlog.000002' |
| 126 |
--source include/show_binlog_events.inc |
| 127 |
|
| 128 |
disable_query_log; |
| 129 |
if ($have_debug) |
| 130 |
{ |
| 131 |
set global debug= @save_debug; |
| 132 |
} |
| 133 |
enable_query_log; |
| 134 |
|
| 135 |
############################################################# |
| 136 |
# Bug#22494024 |
| 137 |
# SQL NODE FAILS TO RECONNECT AFTER NETWORK OUTAGE; BLOCKS OTHER NODES |
| 138 |
# |
| 139 |
# If a mysqld completes ndb_binlog_setup() before the others, |
| 140 |
# and executes a schema change operation which has to be distributed, |
| 141 |
# it can grab the MDL lock, which is also required by the other |
| 142 |
# mysqlds to complete ndb_binlog_setup(). Thus we effectively |
| 143 |
# deadlocks on the MDL. |
| 144 |
# |
| 145 |
# This could happen if one of the mysqlds opens the special table |
| 146 |
# 'ndb_schema' *without* also completing the ndb_binlog_setup() |
| 147 |
# in the same 'atomic' (MDL locked) operation. Opening of this table |
| 148 |
# will cause a SUBSCRIBE-event to be sent to all other mysqlds, |
| 149 |
# which is the signal used to enroll this node into the |
| 150 |
# schema distribution. However, as it is not yet 'setup', it |
| 151 |
# will not yet act on schema distr. events, thus blocking other |
| 152 |
# mysqlds waiting on its response. |
| 153 |
# |
| 154 |
############################################################# |
| 155 |
|
| 156 |
--echo |
| 157 |
--echo Testcase1 for Bug#22494024 Slow binlog-setup -> MDL deadlock (Meta Data Lock) |
| 158 |
--echo ndb_binlog_setup() fails to fully complete and does not clean up |
| 159 |
--echo its partial setup binlog subscriptions on exit |
| 160 |
--echo ============================================================================= |
| 161 |
|
| 162 |
--echo Prepare mysqld 'server2' to not entirely complete binlog_setup |
| 163 |
disable_query_log; |
| 164 |
if ($have_debug) |
| 165 |
{ |
| 166 |
connection server2; |
| 167 |
set @save_debug= @@global.debug; |
| 168 |
set global debug='+d,ndb_binlog_setup_incomplete'; |
| 169 |
connection server1; |
| 170 |
} |
| 171 |
enable_query_log; |
| 172 |
|
| 173 |
--echo Disconnect both mysqld servers from all datanodes -> binlog restarts |
| 174 |
--exec $NDB_MGM -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT |
| 175 |
--exec $NDB_MGM -e "ALL DUMP 900 $node_id2" > $NDB_TOOLS_OUTPUT |
| 176 |
|
| 177 |
connection server1; |
| 178 |
--echo Wait for 'server1' binlog rotate to indicate disconnect |
| 179 |
--let $wait_binlog_event= binlog.000002 |
| 180 |
--source include/wait_for_binlog_event.inc |
| 181 |
|
| 182 |
connection server2; |
| 183 |
--echo Wait for 'server2' binlog rotate to indicate disconnect |
| 184 |
--let $wait_binlog_event= binlog.000002 |
| 185 |
--source include/wait_for_binlog_event.inc |
| 186 |
|
| 187 |
connection server1; |
| 188 |
--echo Wait for 'server1' to complete setup and get out of read-only mode |
| 189 |
--source include/ndb_not_readonly.inc |
| 190 |
|
| 191 |
--echo Give 'server2' some time to start, and fail, a binlog_setup |
| 192 |
sleep 2; |
| 193 |
|
| 194 |
connection server1; |
| 195 |
--echo Create table operation on 'server1' grab MDL lock while waiting for |
| 196 |
--echo schema change distribution to all subscribing mysqld. |
| 197 |
create table t1 (a int primary key) engine=ndb; |
| 198 |
drop table t1; |
| 199 |
|
| 200 |
disable_query_log; |
| 201 |
if ($have_debug) |
| 202 |
{ |
| 203 |
connection server2; |
| 204 |
set global debug= @save_debug; |
| 205 |
connection server1; |
| 206 |
} |
| 207 |
enable_query_log; |
| 208 |
|
| 209 |
connection server2; |
| 210 |
--echo Wait for 'server2' to complete setup and get out of read-only mode |
| 211 |
--source include/ndb_not_readonly.inc |
| 212 |
|
| 213 |
|
| 214 |
--echo |
| 215 |
--echo Testcase2 for Bug#22494024 Slow binlog-setup -> MDL deadlock (Meta Data Lock) |
| 216 |
--echo Table 'ndb_schema' is accessed from a SQL query. Causing it to be opened |
| 217 |
--echo and a SUBSCRIBE-event sent to other mysqld. |
| 218 |
--echo ============================================================================= |
| 219 |
|
| 220 |
--echo Prepare mysqld 'server2' to do slow binlog_setup upon restart |
| 221 |
disable_query_log; |
| 222 |
if ($have_debug) |
| 223 |
{ |
| 224 |
connection server2; |
| 225 |
set @save_debug= @@global.debug; |
| 226 |
set global debug='+d,ndb_binlog_setup_slow'; |
| 227 |
connection server1; |
| 228 |
} |
| 229 |
enable_query_log; |
| 230 |
|
| 231 |
--echo Disconnect both mysqld servers from all datanodes -> binlog restarts |
| 232 |
--exec $NDB_MGM -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT |
| 233 |
--exec $NDB_MGM -e "ALL DUMP 900 $node_id2" > $NDB_TOOLS_OUTPUT |
| 234 |
|
| 235 |
connection server1; |
| 236 |
--echo Wait for 'server1' binlog rotate to indicate disconnect |
| 237 |
--let $wait_binlog_event= binlog.000002 |
| 238 |
--source include/wait_for_binlog_event.inc |
| 239 |
|
| 240 |
connection server2; |
| 241 |
--echo Wait for 'server2' binlog rotate to indicate disconnect |
| 242 |
--let $wait_binlog_event= binlog.000002 |
| 243 |
--source include/wait_for_binlog_event.inc |
| 244 |
|
| 245 |
connection server1; |
| 246 |
--echo Wait for (fast) 'server1' to complete setup and get out of read-only mode |
| 247 |
--source include/ndb_not_readonly.inc |
| 248 |
|
| 249 |
connection server1; |
| 250 |
--echo Create table operation on 'server1' grab MDL lock while waiting for |
| 251 |
--echo schema distribution to all subscribing mysqld -> stuck on 'server2' |
| 252 |
create table t1 (a int primary key) engine=ndb; |
| 253 |
drop table t1; |
| 254 |
|
| 255 |
disable_query_log; |
| 256 |
if ($have_debug) |
| 257 |
{ |
| 258 |
connection server2; |
| 259 |
set global debug= @save_debug; |
| 260 |
connection server1; |
| 261 |
} |
| 262 |
enable_query_log; |
| 263 |
|
| 264 |
connection server2; |
| 265 |
--echo Wait for 'server2' to complete setup and get out of read-only mode |
| 266 |
--source include/ndb_not_readonly.inc |
| 267 |
--remove_file $NDB_TOOLS_OUTPUT |