Debian Codegraph @

mysql-test/suite/rpl/t/rpl_parallel_retry_deadlock.test

raw
1 --echo *** MDEV-7326 Server deadlock in connection with parallel replication ***
2 --source include/have_innodb.inc
3 --source include/have_debug.inc
4 --source include/have_debug_sync.inc
5 --source include/master-slave.inc
6
7 --connection server_2
8 SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads;
9 SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode;
10 --source include/stop_slave.inc
11 # Test assumes that 'conservative' mode is in effect. i.e
12 # Do not start parallel execution of this event group until all prior groups
13 # have reached the commit phase. Refer 'rpl_parallel_start_waiting_for_prior'
14 # debug simumation.
15 SET GLOBAL slave_parallel_mode='conservative';
16 SET GLOBAL slave_parallel_threads=10;
17 CHANGE MASTER TO master_use_gtid=slave_pos;
18 --source include/start_slave.inc
19
20 --connection server_1
21 ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
22 CREATE TABLE t1 (a int PRIMARY KEY) ENGINE=MyISAM;
23 CREATE TABLE t2 (a int PRIMARY KEY) ENGINE=InnoDB;
24 CREATE TABLE t6 (a INT) ENGINE=MyISAM;
25 --save_master_pos
26
27 --connection server_2
28 --sync_with_master
29
30 --connection server_1
31 # Use a stored function to inject a debug_sync into the appropriate THD.
32 # The function does nothing on the master, and on the slave it injects the
33 # desired debug_sync action(s).
34 SET sql_log_bin=0;
35 --delimiter ||
36 CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500))
37 RETURNS INT DETERMINISTIC
38 BEGIN
39 RETURN x;
40 END
41 ||
42 --delimiter ;
43 SET sql_log_bin=1;
44
45 --connection server_2
46 SET sql_log_bin=0;
47 --delimiter ||
48 CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500))
49 RETURNS INT DETERMINISTIC
50 BEGIN
51 IF d1 != '' THEN
52 SET debug_sync = d1;
53 END IF;
54 IF d2 != '' THEN
55 SET debug_sync = d2;
56 END IF;
57 RETURN x;
58 END
59 ||
60 --delimiter ;
61 SET sql_log_bin=1;
62
63 # We use three transactions, each in a separate group commit.
64 # T1 does mark_start_commit(), then gets a deadlock error.
65 # T2 wakes up and starts running
66 # T1 does unmark_start_commit()
67 # T3 goes to wait for T2 to start its commit
68 # T2 does mark_start_commit()
69 # The bug was that at this point, T3 got deadlocked. Because T1 has unmarked(),
70 # T3 did not yet see the count_committing_event_groups reach its target value
71 # yet. But when T1 later re-did mark_start_commit(), it failed to send a wakeup
72 # to T3.
73
74 --connection server_2
75 --source include/stop_slave.inc
76 SET GLOBAL slave_parallel_threads=0;
77 SET GLOBAL slave_parallel_threads=3;
78 SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_xid";
79 --source include/start_slave.inc
80
81 --connection server_1
82 SET @old_format= @@SESSION.binlog_format;
83 SET binlog_format= STATEMENT;
84 # This debug_sync will linger on and be used to control T3 later.
85 INSERT INTO t1 VALUES (foo(50,
86 "rpl_parallel_start_waiting_for_prior SIGNAL t3_ready",
87 "rpl_parallel_end_of_group SIGNAL prep_ready WAIT_FOR prep_cont"));
88 --save_master_pos
89 --connection server_2
90 # Wait for the debug_sync point for T3 to be set. But let the preparation
91 # transaction remain hanging, so that T1 and T2 will be scheduled for the
92 # remaining two worker threads.
93 SET DEBUG_SYNC= "now WAIT_FOR prep_ready";
94
95 --connection server_1
96 INSERT INTO t2 VALUES (foo(50,
97 "rpl_parallel_simulate_temp_err_xid SIGNAL t1_ready1 WAIT_FOR t1_cont1",
98 "rpl_parallel_retry_after_unmark SIGNAL t1_ready2 WAIT_FOR t1_cont2"));
99 --save_master_pos
100
101 --connection server_2
102 SET DEBUG_SYNC= "now WAIT_FOR t1_ready1";
103 # T1 has now done mark_start_commit(). It will later do a rollback and retry.
104
105 --connection server_1
106 # Use a MyISAM table for T2 and T3, so they do not trigger the
107 # rpl_parallel_simulate_temp_err_xid DBUG insertion on XID event.
108 INSERT INTO t1 VALUES (foo(51,
109 "rpl_parallel_before_mark_start_commit SIGNAL t2_ready1 WAIT_FOR t2_cont1",
110 "rpl_parallel_after_mark_start_commit SIGNAL t2_ready2"));
111
112 --connection server_2
113 SET DEBUG_SYNC= "now WAIT_FOR t2_ready1";
114 # T2 has now started running, but has not yet done mark_start_commit()
115 SET DEBUG_SYNC= "now SIGNAL t1_cont1";
116 SET DEBUG_SYNC= "now WAIT_FOR t1_ready2";
117 # T1 has now done unmark_start_commit() in preparation for its retry.
118
119 --connection server_1
120 INSERT INTO t1 VALUES (52);
121 SET BINLOG_FORMAT= @old_format;
122 SELECT * FROM t2 WHERE a>=50 ORDER BY a;
123 SELECT * FROM t1 WHERE a>=50 ORDER BY a;
124
125 --connection server_2
126 # Let the preparation transaction complete, so that the same worker thread
127 # can continue with the transaction T3.
128 SET DEBUG_SYNC= "now SIGNAL prep_cont";
129 SET DEBUG_SYNC= "now WAIT_FOR t3_ready";
130 # T3 has now gone to wait for T2 to start committing
131 SET DEBUG_SYNC= "now SIGNAL t2_cont1";
132 SET DEBUG_SYNC= "now WAIT_FOR t2_ready2";
133 # T2 has now done mark_start_commit().
134 # Let things run, and check that T3 does not get deadlocked.
135 SET DEBUG_SYNC= "now SIGNAL t1_cont2";
136 --sync_with_master
137
138 --connection server_1
139 --save_master_pos
140 --connection server_2
141 --sync_with_master
142 SELECT * FROM t2 WHERE a>=50 ORDER BY a;
143 SELECT * FROM t1 WHERE a>=50 ORDER BY a;
144 SET DEBUG_SYNC="reset";
145
146 # Re-spawn the worker threads to remove any DBUG injections or DEBUG_SYNC.
147 --source include/stop_slave.inc
148 SET GLOBAL debug_dbug=@old_dbug;
149 SET GLOBAL slave_parallel_mode=@old_parallel_mode;
150 SET GLOBAL slave_parallel_threads=0;
151 SET GLOBAL slave_parallel_threads=10;
152 --source include/start_slave.inc
153
154
155 --echo *** MDEV-7326 Server deadlock in connection with parallel replication ***
156 # Similar to the previous test, but with T2 and T3 in the same GCO.
157 # We use three transactions, T1 in one group commit and T2/T3 in another.
158 # T1 does mark_start_commit(), then gets a deadlock error.
159 # T2 wakes up and starts running
160 # T1 does unmark_start_commit()
161 # T3 goes to wait for T1 to start its commit
162 # T2 does mark_start_commit()
163 # The bug was that at this point, T3 got deadlocked. T2 increments the
164 # count_committing_event_groups but does not signal T3, as they are in
165 # the same GCO. Then later when T1 increments, it would also not signal
166 # T3, because now the count_committing_event_groups is not equal to the
167 # wait_count of T3 (it is one larger).
168
169 --connect (con_temp3,127.0.0.1,root,,test,$SERVER_MYPORT_1,)
170 --connect (con_temp4,127.0.0.1,root,,test,$SERVER_MYPORT_1,)
171
172 --connection server_2
173 --source include/stop_slave.inc
174 SET @old_parallel_mode= @@GLOBAL.slave_parallel_mode;
175 SET GLOBAL slave_parallel_mode='conservative';
176 SET GLOBAL slave_parallel_threads=0;
177 SET GLOBAL slave_parallel_threads=3;
178 SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_xid";
179 --source include/start_slave.inc
180
181 --connection server_1
182 SET @old_format= @@SESSION.binlog_format;
183 SET binlog_format= STATEMENT;
184 # This debug_sync will linger on and be used to control T3 later.
185 INSERT INTO t1 VALUES (foo(60,
186 "rpl_parallel_start_waiting_for_prior SIGNAL t3_ready",
187 "rpl_parallel_end_of_group SIGNAL prep_ready WAIT_FOR prep_cont"));
188 --save_master_pos
189 --connection server_2
190 # Wait for the debug_sync point for T3 to be set. But let the preparation
191 # transaction remain hanging, so that T1 and T2 will be scheduled for the
192 # remaining two worker threads.
193 SET DEBUG_SYNC= "now WAIT_FOR prep_ready";
194
195 --connection server_1
196 INSERT INTO t2 VALUES (foo(60,
197 "rpl_parallel_simulate_temp_err_xid SIGNAL t1_ready1 WAIT_FOR t1_cont1",
198 "rpl_parallel_retry_after_unmark SIGNAL t1_ready2 WAIT_FOR t1_cont2"));
199 --save_master_pos
200
201 --connection server_2
202 SET DEBUG_SYNC= "now WAIT_FOR t1_ready1";
203 # T1 has now done mark_start_commit(). It will later do a rollback and retry.
204
205 # Do T2 and T3 in a single group commit.
206 # Use a MyISAM table for T2 and T3, so they do not trigger the
207 # rpl_parallel_simulate_temp_err_xid DBUG insertion on XID event.
208 --connection con_temp3
209 SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1';
210 SET binlog_format=statement;
211 send INSERT INTO t1 VALUES (foo(61,
212 "rpl_parallel_before_mark_start_commit SIGNAL t2_ready1 WAIT_FOR t2_cont1",
213 "rpl_parallel_after_mark_start_commit SIGNAL t2_ready2"));
214
215 --connection server_1
216 SET debug_sync='now WAIT_FOR master_queued1';
217
218 --connection con_temp4
219 SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2';
220 send INSERT INTO t6 VALUES (62);
221
222 --connection server_1
223 SET debug_sync='now WAIT_FOR master_queued2';
224 SET debug_sync='now SIGNAL master_cont1';
225
226 --connection con_temp3
227 REAP;
228 --connection con_temp4
229 REAP;
230
231 --connection server_1
232 SET debug_sync='RESET';
233 SET BINLOG_FORMAT= @old_format;
234 SELECT * FROM t2 WHERE a>=60 ORDER BY a;
235 SELECT * FROM t1 WHERE a>=60 ORDER BY a;
236 SELECT * FROM t6 WHERE a>=60 ORDER BY a;
237
238 --connection server_2
239 SET DEBUG_SYNC= "now WAIT_FOR t2_ready1";
240 # T2 has now started running, but has not yet done mark_start_commit()
241 SET DEBUG_SYNC= "now SIGNAL t1_cont1";
242 SET DEBUG_SYNC= "now WAIT_FOR t1_ready2";
243 # T1 has now done unmark_start_commit() in preparation for its retry.
244
245 --connection server_2
246 # Let the preparation transaction complete, so that the same worker thread
247 # can continue with the transaction T3.
248 SET DEBUG_SYNC= "now SIGNAL prep_cont";
249 SET DEBUG_SYNC= "now WAIT_FOR t3_ready";
250 # T3 has now gone to wait for T2 to start committing
251 SET DEBUG_SYNC= "now SIGNAL t2_cont1";
252 SET DEBUG_SYNC= "now WAIT_FOR t2_ready2";
253 # T2 has now done mark_start_commit().
254 # Let things run, and check that T3 does not get deadlocked.
255 SET DEBUG_SYNC= "now SIGNAL t1_cont2";
256 --sync_with_master
257
258 --connection server_1
259 --save_master_pos
260 --connection server_2
261 --sync_with_master
262 SELECT * FROM t2 WHERE a>=60 ORDER BY a;
263 SELECT * FROM t1 WHERE a>=60 ORDER BY a;
264 SELECT * FROM t6 WHERE a>=60 ORDER BY a;
265 SET DEBUG_SYNC="reset";
266
267 # Clean up.
268 --source include/stop_slave.inc
269 SET GLOBAL debug_dbug=@old_dbug;
270 SET GLOBAL slave_parallel_mode=@old_parallel_mode;
271 SET GLOBAL slave_parallel_threads=@old_parallel_threads;
272 --source include/start_slave.inc
273
274 --connection server_1
275 DROP function foo;
276 DROP TABLE t1,t2,t6;
277 --disconnect con_temp3
278 --disconnect con_temp4
279 SET DEBUG_SYNC= 'RESET';
280
281 --source include/rpl_end.inc