Debian Codegraph @

mysql-test/suite/rpl/t/rpl_semi_sync_after_sync_coord_consistency.test

raw
1 #
2 # This test ensures that a race condition (reported in MDEV-35109) which
3 # could cause semi-sync transactions binlogging through MYSQL_BIN_LOG::write()
4 # to hang until the semi-sync timeout has been hit, despite the ACK actually
5 # having been received. The problem was that, for a primary configured with
6 # wait_point=AFTER_SYNC, if two threads T1 (binlogging through
7 # MYSQL_BIN_LOG::write()) and T2 were binlogging at the same time, T1 could
8 # accidentally wait for the binlog coordinates of T2. Prior to MDEV-33551,
9 # this only resulted in delayed transactions, because all transactions shared
10 # the same condition variable for ACK signaling. However, with the MDEV-33551
11 # changes, each thread has its own condition variable to signal. So T1 could
12 # wait indefinitely when either:
13 # 1) T1's ACK is received but not T2's when T1 goes into wait_after_sync(),
14 # because the ACK receiver thread has already notified about the T1 ACK,
15 # but T1 was _actually_ waiting on T2's ACK, and therefore tries to wait
16 # (in vain).
17 #
18 # 2) T1 goes to wait_after_sync() before any ACKs have arrived. When T1's ACK
19 # comes in, T1 is woken up; however, sees it needs to wait more (because
20 # it was actually waiting on T2's ACK), and goes to wait again (this time,
21 # in vain).
22 #
23 # Note that the actual cause of T1 waiting on T2's binlog coordinates is when
24 # MYSQL_BIN_LOG::write() would call Repl_semisync_master::wait_after_sync(),
25 # the binlog offset parameter was read as the end of MYSQL_BIN_LOG::log_file,
26 # which is shared among transactions. So if T2 had updated the binary log
27 # _after_ T1 had released LOCK_log, but not yet invoked wait_after_sync(), it
28 # would use the end of the binary log file as the binlog offset, which was that
29 # of T2 (or any future transaction). The patch itself fixes this issue, and
30 # ensures consistency between the binary log coordinates a transaction uses
31 # between report_binlog_update() and wait_after_sync().
32 #
33 # This test reproduces the above race condition, to ensure that T1 can no
34 # longer use T2's binlog coordinates when wait_point=AFTER_SYNC (AFTER_COMMIT
35 # was never affected). That is, because T1 in the above scenario must binlog
36 # through MYSQL_BIN_LOG::write(), it is a DDL. T2 is an MDL that commits
37 # through binlog group commit. We use debug_sync to stop both T1 and T2
38 # after binlogging, reporting their coordinates to the semi-sync cache (i.e.
39 # after report_binlog_update()), and after releasing LOCK_log. Debug_sync is
40 # used on the replica to ensure no ACKs are sent until after both T1 and T2
41 # are paused. Then, T1 and T2 are signaled for wakeup, and both ACKs are sent.
42 # Prior to MDEV-35109 fixes, this results in T1 timing out after the configured
43 # 2 second timeout.
44 #
45 # References:
46 # MDEV-33551: Semi-sync Wait Point AFTER_COMMIT Slow on Workloads with Heavy
47 # Concurrency
48 # MDEV-35109: Semi-sync Replication stalling Primary using
49 # wait_point=AFTER_SYNC
50 #
51
52 --source include/have_innodb.inc
53 --source include/have_debug_sync.inc
54 # Test is format independent
55 --source include/have_binlog_format_row.inc
56 --source include/master-slave.inc
57
58 --connection slave
59 --source include/stop_slave.inc
60 set @old_enabled= @@global.rpl_semi_sync_slave_enabled;
61 set @old_dbug= @@global.debug_dbug;
62 set global rpl_semi_sync_slave_enabled= 1;
63
64 --connection master
65 set @old_enabled= @@global.rpl_semi_sync_master_enabled;
66 set @old_timeout= @@global.rpl_semi_sync_master_timeout;
67 set @old_wait_point= @@global.rpl_semi_sync_master_wait_point;
68 set global rpl_semi_sync_master_enabled= 1;
69 set global rpl_semi_sync_master_timeout= 2000; # 2s
70 set global rpl_semi_sync_master_wait_point= AFTER_SYNC;
71
72 --connection slave
73 --source include/start_slave.inc
74
75 --echo # Ensure slave connection is semi-simulate_delay_semisync_slave_reply
76 --connection master
77 --let $status_var_value= ON
78 --let $status_var= Rpl_semi_sync_master_status
79 --source include/wait_for_status_var.inc
80 --connection slave
81 --let $status_var_value= ON
82 --let $status_var= Rpl_semi_sync_slave_status
83 --source include/wait_for_status_var.inc
84
85 --echo #
86 --echo # Initialize test data
87 --connection master
88
89 # It is simpler to create t1 before using debug_sync on the slave_reply
90 create table t1 (a int) engine=innodb;
91 --source include/save_master_gtid.inc
92 --connection slave
93 --source include/sync_with_master_gtid.inc
94
95 --source include/stop_slave.inc
96 set @@global.debug_dbug="+d,simulate_delay_semisync_slave_reply";
97 --source include/start_slave.inc
98
99 --echo # Ensure connection is semi-sync
100 --connection master
101 --let $status_var_value= ON
102 --let $status_var= Rpl_semi_sync_master_status
103 --source include/wait_for_status_var.inc
104 --connection slave
105 --let $status_var_value= ON
106 --let $status_var= Rpl_semi_sync_slave_status
107 --source include/wait_for_status_var.inc
108
109 --connection master
110 set @old_bgc_count= @@global.binlog_commit_wait_count;
111 set @old_bgc_usec= @@global.binlog_commit_wait_usec;
112 set @@global.binlog_commit_wait_count=1;
113 set @@global.binlog_commit_wait_usec=100000;
114
115 --connection server_1
116 set debug_sync= "commit_after_release_LOCK_log SIGNAL ddl_binlogged WAIT_FOR ddl_cont";
117 --echo # T1 (DDL)
118 --send create table t2 (a int)
119
120 --connect(server_1_sync, localhost, root,,)
121 --connection server_1_sync
122 set debug_sync= "now WAIT_FOR ddl_binlogged";
123 --source include/save_master_gtid.inc
124
125 --connection server_2
126 set debug_sync= "now WAIT_FOR io_thd_at_slave_reply";
127
128 --connection master
129 set debug_sync= "commit_before_get_LOCK_after_binlog_sync SIGNAL mdl_binlogged WAIT_FOR mdl_cont";
130 --echo # T2 (DML)
131 --send insert into t1 values (100);
132
133 --connection server_1_sync
134 set debug_sync= "now WAIT_FOR mdl_binlogged";
135
136 --echo # Both transactions binlogged and released LOCK_log, and are just before
137 --echo # wait_after_sync()
138 set debug_sync= "now SIGNAL ddl_cont";
139 # Ensure ddl_cont is seen before next SIGNAL
140 --let $wait_condition= SELECT count(*)=0 FROM information_schema.processlist WHERE state LIKE "%commit_after_release_LOCK_log%"
141 --source include/wait_condition.inc
142
143 set debug_sync= "now SIGNAL mdl_cont";
144
145 --connection server_2
146 --echo # slave_reply for DDL
147 set debug_sync= "now SIGNAL io_thd_do_reply";
148 --echo # slave_reply for MDL
149 set debug_sync= "now WAIT_FOR io_thd_at_slave_reply";
150 set debug_sync= "now SIGNAL io_thd_do_reply";
151
152 --echo # Reaping MDL..
153 --connection master
154 --reap
155 --echo # ..done
156
157 --echo # Reaping DDL..
158 --connection server_1
159 --reap
160 --echo # ..done
161
162
163 --echo #
164 --echo # Cleanup
165 --connection slave
166 --source include/stop_slave.inc
167 set @@global.rpl_semi_sync_slave_enabled= @old_enabled;
168 set @@global.debug_dbug= @old_dbug;
169 --source include/start_slave.inc
170
171 --connection master
172 set @@global.binlog_commit_wait_count= @old_bgc_count;
173 set @@global.binlog_commit_wait_usec= @old_bgc_usec;
174 set @@global.rpl_semi_sync_master_enabled= @old_enabled;
175 set @@global.rpl_semi_sync_master_timeout= @old_timeout;
176 set @@global.rpl_semi_sync_master_wait_point= @old_wait_point;
177 drop table t1, t2;
178 --source include/save_master_gtid.inc
179
180 --connection slave
181 --source include/sync_with_master_gtid.inc
182
183 --source include/rpl_end.inc
184 --echo # End of rpl_semi_sync_after_sync_coord_consistency.test