| 1 |
--echo *** MDEV-7326 Server deadlock in connection with parallel replication *** |
| 2 |
--source include/have_innodb.inc |
| 3 |
--source include/have_debug.inc |
| 4 |
--source include/have_debug_sync.inc |
| 5 |
--source include/master-slave.inc |
| 6 |
|
| 7 |
--connection server_2 |
| 8 |
SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads; |
| 9 |
SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode; |
| 10 |
--source include/stop_slave.inc |
| 11 |
# Test assumes that 'conservative' mode is in effect. i.e |
| 12 |
# Do not start parallel execution of this event group until all prior groups |
| 13 |
# have reached the commit phase. Refer 'rpl_parallel_start_waiting_for_prior' |
| 14 |
# debug simumation. |
| 15 |
SET GLOBAL slave_parallel_mode='conservative'; |
| 16 |
SET GLOBAL slave_parallel_threads=10; |
| 17 |
CHANGE MASTER TO master_use_gtid=slave_pos; |
| 18 |
--source include/start_slave.inc |
| 19 |
|
| 20 |
--connection server_1 |
| 21 |
ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; |
| 22 |
CREATE TABLE t1 (a int PRIMARY KEY) ENGINE=MyISAM; |
| 23 |
CREATE TABLE t2 (a int PRIMARY KEY) ENGINE=InnoDB; |
| 24 |
CREATE TABLE t6 (a INT) ENGINE=MyISAM; |
| 25 |
--save_master_pos |
| 26 |
|
| 27 |
--connection server_2 |
| 28 |
--sync_with_master |
| 29 |
|
| 30 |
--connection server_1 |
| 31 |
# Use a stored function to inject a debug_sync into the appropriate THD. |
| 32 |
# The function does nothing on the master, and on the slave it injects the |
| 33 |
# desired debug_sync action(s). |
| 34 |
SET sql_log_bin=0; |
| 35 |
--delimiter || |
| 36 |
CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) |
| 37 |
RETURNS INT DETERMINISTIC |
| 38 |
BEGIN |
| 39 |
RETURN x; |
| 40 |
END |
| 41 |
|| |
| 42 |
--delimiter ; |
| 43 |
SET sql_log_bin=1; |
| 44 |
|
| 45 |
--connection server_2 |
| 46 |
SET sql_log_bin=0; |
| 47 |
--delimiter || |
| 48 |
CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500)) |
| 49 |
RETURNS INT DETERMINISTIC |
| 50 |
BEGIN |
| 51 |
IF d1 != '' THEN |
| 52 |
SET debug_sync = d1; |
| 53 |
END IF; |
| 54 |
IF d2 != '' THEN |
| 55 |
SET debug_sync = d2; |
| 56 |
END IF; |
| 57 |
RETURN x; |
| 58 |
END |
| 59 |
|| |
| 60 |
--delimiter ; |
| 61 |
SET sql_log_bin=1; |
| 62 |
|
| 63 |
# We use three transactions, each in a separate group commit. |
| 64 |
# T1 does mark_start_commit(), then gets a deadlock error. |
| 65 |
# T2 wakes up and starts running |
| 66 |
# T1 does unmark_start_commit() |
| 67 |
# T3 goes to wait for T2 to start its commit |
| 68 |
# T2 does mark_start_commit() |
| 69 |
# The bug was that at this point, T3 got deadlocked. Because T1 has unmarked(), |
| 70 |
# T3 did not yet see the count_committing_event_groups reach its target value |
| 71 |
# yet. But when T1 later re-did mark_start_commit(), it failed to send a wakeup |
| 72 |
# to T3. |
| 73 |
|
| 74 |
--connection server_2 |
| 75 |
--source include/stop_slave.inc |
| 76 |
SET GLOBAL slave_parallel_threads=0; |
| 77 |
SET GLOBAL slave_parallel_threads=3; |
| 78 |
SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_xid"; |
| 79 |
--source include/start_slave.inc |
| 80 |
|
| 81 |
--connection server_1 |
| 82 |
SET @old_format= @@SESSION.binlog_format; |
| 83 |
SET binlog_format= STATEMENT; |
| 84 |
# This debug_sync will linger on and be used to control T3 later. |
| 85 |
INSERT INTO t1 VALUES (foo(50, |
| 86 |
"rpl_parallel_start_waiting_for_prior SIGNAL t3_ready", |
| 87 |
"rpl_parallel_end_of_group SIGNAL prep_ready WAIT_FOR prep_cont")); |
| 88 |
--save_master_pos |
| 89 |
--connection server_2 |
| 90 |
# Wait for the debug_sync point for T3 to be set. But let the preparation |
| 91 |
# transaction remain hanging, so that T1 and T2 will be scheduled for the |
| 92 |
# remaining two worker threads. |
| 93 |
SET DEBUG_SYNC= "now WAIT_FOR prep_ready"; |
| 94 |
|
| 95 |
--connection server_1 |
| 96 |
INSERT INTO t2 VALUES (foo(50, |
| 97 |
"rpl_parallel_simulate_temp_err_xid SIGNAL t1_ready1 WAIT_FOR t1_cont1", |
| 98 |
"rpl_parallel_retry_after_unmark SIGNAL t1_ready2 WAIT_FOR t1_cont2")); |
| 99 |
--save_master_pos |
| 100 |
|
| 101 |
--connection server_2 |
| 102 |
SET DEBUG_SYNC= "now WAIT_FOR t1_ready1"; |
| 103 |
# T1 has now done mark_start_commit(). It will later do a rollback and retry. |
| 104 |
|
| 105 |
--connection server_1 |
| 106 |
# Use a MyISAM table for T2 and T3, so they do not trigger the |
| 107 |
# rpl_parallel_simulate_temp_err_xid DBUG insertion on XID event. |
| 108 |
INSERT INTO t1 VALUES (foo(51, |
| 109 |
"rpl_parallel_before_mark_start_commit SIGNAL t2_ready1 WAIT_FOR t2_cont1", |
| 110 |
"rpl_parallel_after_mark_start_commit SIGNAL t2_ready2")); |
| 111 |
|
| 112 |
--connection server_2 |
| 113 |
SET DEBUG_SYNC= "now WAIT_FOR t2_ready1"; |
| 114 |
# T2 has now started running, but has not yet done mark_start_commit() |
| 115 |
SET DEBUG_SYNC= "now SIGNAL t1_cont1"; |
| 116 |
SET DEBUG_SYNC= "now WAIT_FOR t1_ready2"; |
| 117 |
# T1 has now done unmark_start_commit() in preparation for its retry. |
| 118 |
|
| 119 |
--connection server_1 |
| 120 |
INSERT INTO t1 VALUES (52); |
| 121 |
SET BINLOG_FORMAT= @old_format; |
| 122 |
SELECT * FROM t2 WHERE a>=50 ORDER BY a; |
| 123 |
SELECT * FROM t1 WHERE a>=50 ORDER BY a; |
| 124 |
|
| 125 |
--connection server_2 |
| 126 |
# Let the preparation transaction complete, so that the same worker thread |
| 127 |
# can continue with the transaction T3. |
| 128 |
SET DEBUG_SYNC= "now SIGNAL prep_cont"; |
| 129 |
SET DEBUG_SYNC= "now WAIT_FOR t3_ready"; |
| 130 |
# T3 has now gone to wait for T2 to start committing |
| 131 |
SET DEBUG_SYNC= "now SIGNAL t2_cont1"; |
| 132 |
SET DEBUG_SYNC= "now WAIT_FOR t2_ready2"; |
| 133 |
# T2 has now done mark_start_commit(). |
| 134 |
# Let things run, and check that T3 does not get deadlocked. |
| 135 |
SET DEBUG_SYNC= "now SIGNAL t1_cont2"; |
| 136 |
--sync_with_master |
| 137 |
|
| 138 |
--connection server_1 |
| 139 |
--save_master_pos |
| 140 |
--connection server_2 |
| 141 |
--sync_with_master |
| 142 |
SELECT * FROM t2 WHERE a>=50 ORDER BY a; |
| 143 |
SELECT * FROM t1 WHERE a>=50 ORDER BY a; |
| 144 |
SET DEBUG_SYNC="reset"; |
| 145 |
|
| 146 |
# Re-spawn the worker threads to remove any DBUG injections or DEBUG_SYNC. |
| 147 |
--source include/stop_slave.inc |
| 148 |
SET GLOBAL debug_dbug=@old_dbug; |
| 149 |
SET GLOBAL slave_parallel_mode=@old_parallel_mode; |
| 150 |
SET GLOBAL slave_parallel_threads=0; |
| 151 |
SET GLOBAL slave_parallel_threads=10; |
| 152 |
--source include/start_slave.inc |
| 153 |
|
| 154 |
|
| 155 |
--echo *** MDEV-7326 Server deadlock in connection with parallel replication *** |
| 156 |
# Similar to the previous test, but with T2 and T3 in the same GCO. |
| 157 |
# We use three transactions, T1 in one group commit and T2/T3 in another. |
| 158 |
# T1 does mark_start_commit(), then gets a deadlock error. |
| 159 |
# T2 wakes up and starts running |
| 160 |
# T1 does unmark_start_commit() |
| 161 |
# T3 goes to wait for T1 to start its commit |
| 162 |
# T2 does mark_start_commit() |
| 163 |
# The bug was that at this point, T3 got deadlocked. T2 increments the |
| 164 |
# count_committing_event_groups but does not signal T3, as they are in |
| 165 |
# the same GCO. Then later when T1 increments, it would also not signal |
| 166 |
# T3, because now the count_committing_event_groups is not equal to the |
| 167 |
# wait_count of T3 (it is one larger). |
| 168 |
|
| 169 |
--connect (con_temp3,127.0.0.1,root,,test,$SERVER_MYPORT_1,) |
| 170 |
--connect (con_temp4,127.0.0.1,root,,test,$SERVER_MYPORT_1,) |
| 171 |
|
| 172 |
--connection server_2 |
| 173 |
--source include/stop_slave.inc |
| 174 |
SET @old_parallel_mode= @@GLOBAL.slave_parallel_mode; |
| 175 |
SET GLOBAL slave_parallel_mode='conservative'; |
| 176 |
SET GLOBAL slave_parallel_threads=0; |
| 177 |
SET GLOBAL slave_parallel_threads=3; |
| 178 |
SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_xid"; |
| 179 |
--source include/start_slave.inc |
| 180 |
|
| 181 |
--connection server_1 |
| 182 |
SET @old_format= @@SESSION.binlog_format; |
| 183 |
SET binlog_format= STATEMENT; |
| 184 |
# This debug_sync will linger on and be used to control T3 later. |
| 185 |
INSERT INTO t1 VALUES (foo(60, |
| 186 |
"rpl_parallel_start_waiting_for_prior SIGNAL t3_ready", |
| 187 |
"rpl_parallel_end_of_group SIGNAL prep_ready WAIT_FOR prep_cont")); |
| 188 |
--save_master_pos |
| 189 |
--connection server_2 |
| 190 |
# Wait for the debug_sync point for T3 to be set. But let the preparation |
| 191 |
# transaction remain hanging, so that T1 and T2 will be scheduled for the |
| 192 |
# remaining two worker threads. |
| 193 |
SET DEBUG_SYNC= "now WAIT_FOR prep_ready"; |
| 194 |
|
| 195 |
--connection server_1 |
| 196 |
INSERT INTO t2 VALUES (foo(60, |
| 197 |
"rpl_parallel_simulate_temp_err_xid SIGNAL t1_ready1 WAIT_FOR t1_cont1", |
| 198 |
"rpl_parallel_retry_after_unmark SIGNAL t1_ready2 WAIT_FOR t1_cont2")); |
| 199 |
--save_master_pos |
| 200 |
|
| 201 |
--connection server_2 |
| 202 |
SET DEBUG_SYNC= "now WAIT_FOR t1_ready1"; |
| 203 |
# T1 has now done mark_start_commit(). It will later do a rollback and retry. |
| 204 |
|
| 205 |
# Do T2 and T3 in a single group commit. |
| 206 |
# Use a MyISAM table for T2 and T3, so they do not trigger the |
| 207 |
# rpl_parallel_simulate_temp_err_xid DBUG insertion on XID event. |
| 208 |
--connection con_temp3 |
| 209 |
SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1'; |
| 210 |
SET binlog_format=statement; |
| 211 |
send INSERT INTO t1 VALUES (foo(61, |
| 212 |
"rpl_parallel_before_mark_start_commit SIGNAL t2_ready1 WAIT_FOR t2_cont1", |
| 213 |
"rpl_parallel_after_mark_start_commit SIGNAL t2_ready2")); |
| 214 |
|
| 215 |
--connection server_1 |
| 216 |
SET debug_sync='now WAIT_FOR master_queued1'; |
| 217 |
|
| 218 |
--connection con_temp4 |
| 219 |
SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2'; |
| 220 |
send INSERT INTO t6 VALUES (62); |
| 221 |
|
| 222 |
--connection server_1 |
| 223 |
SET debug_sync='now WAIT_FOR master_queued2'; |
| 224 |
SET debug_sync='now SIGNAL master_cont1'; |
| 225 |
|
| 226 |
--connection con_temp3 |
| 227 |
REAP; |
| 228 |
--connection con_temp4 |
| 229 |
REAP; |
| 230 |
|
| 231 |
--connection server_1 |
| 232 |
SET debug_sync='RESET'; |
| 233 |
SET BINLOG_FORMAT= @old_format; |
| 234 |
SELECT * FROM t2 WHERE a>=60 ORDER BY a; |
| 235 |
SELECT * FROM t1 WHERE a>=60 ORDER BY a; |
| 236 |
SELECT * FROM t6 WHERE a>=60 ORDER BY a; |
| 237 |
|
| 238 |
--connection server_2 |
| 239 |
SET DEBUG_SYNC= "now WAIT_FOR t2_ready1"; |
| 240 |
# T2 has now started running, but has not yet done mark_start_commit() |
| 241 |
SET DEBUG_SYNC= "now SIGNAL t1_cont1"; |
| 242 |
SET DEBUG_SYNC= "now WAIT_FOR t1_ready2"; |
| 243 |
# T1 has now done unmark_start_commit() in preparation for its retry. |
| 244 |
|
| 245 |
--connection server_2 |
| 246 |
# Let the preparation transaction complete, so that the same worker thread |
| 247 |
# can continue with the transaction T3. |
| 248 |
SET DEBUG_SYNC= "now SIGNAL prep_cont"; |
| 249 |
SET DEBUG_SYNC= "now WAIT_FOR t3_ready"; |
| 250 |
# T3 has now gone to wait for T2 to start committing |
| 251 |
SET DEBUG_SYNC= "now SIGNAL t2_cont1"; |
| 252 |
SET DEBUG_SYNC= "now WAIT_FOR t2_ready2"; |
| 253 |
# T2 has now done mark_start_commit(). |
| 254 |
# Let things run, and check that T3 does not get deadlocked. |
| 255 |
SET DEBUG_SYNC= "now SIGNAL t1_cont2"; |
| 256 |
--sync_with_master |
| 257 |
|
| 258 |
--connection server_1 |
| 259 |
--save_master_pos |
| 260 |
--connection server_2 |
| 261 |
--sync_with_master |
| 262 |
SELECT * FROM t2 WHERE a>=60 ORDER BY a; |
| 263 |
SELECT * FROM t1 WHERE a>=60 ORDER BY a; |
| 264 |
SELECT * FROM t6 WHERE a>=60 ORDER BY a; |
| 265 |
SET DEBUG_SYNC="reset"; |
| 266 |
|
| 267 |
# Clean up. |
| 268 |
--source include/stop_slave.inc |
| 269 |
SET GLOBAL debug_dbug=@old_dbug; |
| 270 |
SET GLOBAL slave_parallel_mode=@old_parallel_mode; |
| 271 |
SET GLOBAL slave_parallel_threads=@old_parallel_threads; |
| 272 |
--source include/start_slave.inc |
| 273 |
|
| 274 |
--connection server_1 |
| 275 |
DROP function foo; |
| 276 |
DROP TABLE t1,t2,t6; |
| 277 |
--disconnect con_temp3 |
| 278 |
--disconnect con_temp4 |
| 279 |
SET DEBUG_SYNC= 'RESET'; |
| 280 |
|
| 281 |
--source include/rpl_end.inc |