Debian Codegraph @

mysql-test/suite/ndb_binlog/t/ndb_binlog_restart.test

raw
1 #############################################################
2 # Test behaviout of the binlog injector during restart of itself.
3 # A restart should be triggered when the mysqld disconnects from
4 # the datanodes.
5 #
6 #############################################################
7
8 -- source include/have_binlog_format_mixed_or_row.inc
9 -- source include/have_multi_ndb.inc
10
11 # Ignore the warning generated by ndbcluster's binlog thread
12 # when mysqld is disconnected from cluster
13 --disable_query_log ONCE
14 call mtr.add_suppression("cluster disconnect An incident event has been written");
15
16 ##################################################
17 # Check whether we have a debug compiled binary:
18 # @@global.debug can't be referred without this.
19 # However, instead of skipping entire test without debug,
20 # we run it without setting the specific debug flags.
21 ###################################################
22 --let $have_debug= `select (version() like '%debug%')`
23
24 connection server1;
25 # Find NodeId of the mysqld we are connected to:
26 --let $node_id= `SHOW STATUS LIKE 'Ndb_cluster_node_id'`
27 --let $node_id= `SELECT SUBSTRING('$node_id', 20)+0`
28
29 connection server2;
30 # Find NodeId2 of 'server2' we are connected to:
31 --let $node_id2= `SHOW STATUS LIKE 'Ndb_cluster_node_id'`
32 --let $node_id2= `SELECT SUBSTRING('$node_id2', 20)+0`
33
34 # Ignore the warning generated by ndbcluster's binlog thread
35 # when mysqld is disconnected from cluster also on second Server
36 --disable_query_log ONCE
37 call mtr.add_suppression("cluster disconnect An incident event has been written");
38
39 connection server1;
40 reset master;
41
42 #############################################################
43 # A TE_FAILURE event will be received by the binlog thread
44 # due to the DUMP disconnecting mysqld from datanodes.
45 # When binlog handles the FAILURE event, mysqld is put into
46 # readonly mode. A restart of the Binlog thread is required
47 # to bring it out of RO-mode.
48 #############################################################
49 --echo Disconnect mysqld from all datanodes
50 --exec $NDB_MGM -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT
51
52 #############################################################
53 # The disconnect will create a GAP event in the binlog.
54 # To avoid 'disconnect failures' while probing for 'not_readonly',
55 # we use the binlog GAP as an indicator that disconnect has
56 # already completed.
57 #############################################################
58 --echo Wait for binlog rotate to indicate disconnect
59 --let $wait_binlog_event= binlog.000002
60 --source include/wait_for_binlog_event.inc
61
62 #############################################################
63 # Until the binlog injector has completed the restart all tables
64 # will temporarily be in a read-only mode.
65 # (As the binlog will not be able to log any updates)
66 #############################################################
67 --echo Wait to get out of read-only mode
68 --source include/ndb_not_readonly.inc
69
70 #############################################################
71 # During the restart a 'Gap event' should be written to the
72 # binlog and the log rotated to 'binlog.000002'.
73 #############################################################
74 --echo
75 --echo Check that the binlog was rotated:
76 --echo There should be a 'LOST EVENT', then 'Rotate ... binlog.000002'
77 --source include/show_binlog_events.inc
78
79
80 #############################################################
81 # Bug#22204186: NDB REPLICATION : READ ONLY TABLES AND NO BINLOGGING
82 #
83 # Tables could get stuck in read only mode due to slow handling
84 # of cluster failure in the binlog.
85 #
86 # Testcase is identical to the one above, except that
87 # we use debug inserts to force the binlog injector to take
88 # a break when handling failures. When not using debug compiled
89 # binaries, there will be no such breaks inserted, but the
90 # test should still succeed.
91 #
92 # This previously caused a race condition where the Util-thread
93 # managed to recreate the schema distr. tables before
94 # the binlog thread had completed the failure handling. Thus,
95 # the binlog never reached its restart condition of all
96 # eventOps have been dropped.
97 #############################################################
98
99 --echo
100 --echo Testing for Bug#22204186: Stuck in Readonly if slow failure handling
101 --echo ====================================================================
102
103 disable_query_log;
104 if ($have_debug)
105 {
106 set @save_debug= @@global.debug;
107 set global debug='+d,ndb_binlog_slow_failure_handling';
108 }
109 enable_query_log;
110
111 reset master;
112
113 --echo Disconnect mysqld from all datanodes
114 --exec $NDB_MGM -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT
115
116 --echo Wait for binlog rotate to indicate disconnect
117 --let $wait_binlog_event= binlog.000002
118 --source include/wait_for_binlog_event.inc
119
120 --echo Wait to get out of read-only mode
121 --source include/ndb_not_readonly.inc
122
123 --echo
124 --echo Check that the binlog was rotated:
125 --echo There should be a 'LOST EVENT', then 'Rotate ... binlog.000002'
126 --source include/show_binlog_events.inc
127
128 disable_query_log;
129 if ($have_debug)
130 {
131 set global debug= @save_debug;
132 }
133 enable_query_log;
134
135 #############################################################
136 # Bug#22494024
137 # SQL NODE FAILS TO RECONNECT AFTER NETWORK OUTAGE; BLOCKS OTHER NODES
138 #
139 # If a mysqld completes ndb_binlog_setup() before the others,
140 # and executes a schema change operation which has to be distributed,
141 # it can grab the MDL lock, which is also required by the other
142 # mysqlds to complete ndb_binlog_setup(). Thus we effectively
143 # deadlocks on the MDL.
144 #
145 # This could happen if one of the mysqlds opens the special table
146 # 'ndb_schema' *without* also completing the ndb_binlog_setup()
147 # in the same 'atomic' (MDL locked) operation. Opening of this table
148 # will cause a SUBSCRIBE-event to be sent to all other mysqlds,
149 # which is the signal used to enroll this node into the
150 # schema distribution. However, as it is not yet 'setup', it
151 # will not yet act on schema distr. events, thus blocking other
152 # mysqlds waiting on its response.
153 #
154 #############################################################
155
156 --echo
157 --echo Testcase1 for Bug#22494024 Slow binlog-setup -> MDL deadlock (Meta Data Lock)
158 --echo ndb_binlog_setup() fails to fully complete and does not clean up
159 --echo its partial setup binlog subscriptions on exit
160 --echo =============================================================================
161
162 --echo Prepare mysqld 'server2' to not entirely complete binlog_setup
163 disable_query_log;
164 if ($have_debug)
165 {
166 connection server2;
167 set @save_debug= @@global.debug;
168 set global debug='+d,ndb_binlog_setup_incomplete';
169 connection server1;
170 }
171 enable_query_log;
172
173 --echo Disconnect both mysqld servers from all datanodes -> binlog restarts
174 --exec $NDB_MGM -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT
175 --exec $NDB_MGM -e "ALL DUMP 900 $node_id2" > $NDB_TOOLS_OUTPUT
176
177 connection server1;
178 --echo Wait for 'server1' binlog rotate to indicate disconnect
179 --let $wait_binlog_event= binlog.000002
180 --source include/wait_for_binlog_event.inc
181
182 connection server2;
183 --echo Wait for 'server2' binlog rotate to indicate disconnect
184 --let $wait_binlog_event= binlog.000002
185 --source include/wait_for_binlog_event.inc
186
187 connection server1;
188 --echo Wait for 'server1' to complete setup and get out of read-only mode
189 --source include/ndb_not_readonly.inc
190
191 --echo Give 'server2' some time to start, and fail, a binlog_setup
192 sleep 2;
193
194 connection server1;
195 --echo Create table operation on 'server1' grab MDL lock while waiting for
196 --echo schema change distribution to all subscribing mysqld.
197 create table t1 (a int primary key) engine=ndb;
198 drop table t1;
199
200 disable_query_log;
201 if ($have_debug)
202 {
203 connection server2;
204 set global debug= @save_debug;
205 connection server1;
206 }
207 enable_query_log;
208
209 connection server2;
210 --echo Wait for 'server2' to complete setup and get out of read-only mode
211 --source include/ndb_not_readonly.inc
212
213
214 --echo
215 --echo Testcase2 for Bug#22494024 Slow binlog-setup -> MDL deadlock (Meta Data Lock)
216 --echo Table 'ndb_schema' is accessed from a SQL query. Causing it to be opened
217 --echo and a SUBSCRIBE-event sent to other mysqld.
218 --echo =============================================================================
219
220 --echo Prepare mysqld 'server2' to do slow binlog_setup upon restart
221 disable_query_log;
222 if ($have_debug)
223 {
224 connection server2;
225 set @save_debug= @@global.debug;
226 set global debug='+d,ndb_binlog_setup_slow';
227 connection server1;
228 }
229 enable_query_log;
230
231 --echo Disconnect both mysqld servers from all datanodes -> binlog restarts
232 --exec $NDB_MGM -e "ALL DUMP 900 $node_id" > $NDB_TOOLS_OUTPUT
233 --exec $NDB_MGM -e "ALL DUMP 900 $node_id2" > $NDB_TOOLS_OUTPUT
234
235 connection server1;
236 --echo Wait for 'server1' binlog rotate to indicate disconnect
237 --let $wait_binlog_event= binlog.000002
238 --source include/wait_for_binlog_event.inc
239
240 connection server2;
241 --echo Wait for 'server2' binlog rotate to indicate disconnect
242 --let $wait_binlog_event= binlog.000002
243 --source include/wait_for_binlog_event.inc
244
245 connection server1;
246 --echo Wait for (fast) 'server1' to complete setup and get out of read-only mode
247 --source include/ndb_not_readonly.inc
248
249 connection server1;
250 --echo Create table operation on 'server1' grab MDL lock while waiting for
251 --echo schema distribution to all subscribing mysqld -> stuck on 'server2'
252 create table t1 (a int primary key) engine=ndb;
253 drop table t1;
254
255 disable_query_log;
256 if ($have_debug)
257 {
258 connection server2;
259 set global debug= @save_debug;
260 connection server1;
261 }
262 enable_query_log;
263
264 connection server2;
265 --echo Wait for 'server2' to complete setup and get out of read-only mode
266 --source include/ndb_not_readonly.inc
267 --remove_file $NDB_TOOLS_OUTPUT