| 1 |
# |
| 2 |
# This test ensures that a race condition (reported in MDEV-35109) which |
| 3 |
# could cause semi-sync transactions binlogging through MYSQL_BIN_LOG::write() |
| 4 |
# to hang until the semi-sync timeout has been hit, despite the ACK actually |
| 5 |
# having been received. The problem was that, for a primary configured with |
| 6 |
# wait_point=AFTER_SYNC, if two threads T1 (binlogging through |
| 7 |
# MYSQL_BIN_LOG::write()) and T2 were binlogging at the same time, T1 could |
| 8 |
# accidentally wait for the binlog coordinates of T2. Prior to MDEV-33551, |
| 9 |
# this only resulted in delayed transactions, because all transactions shared |
| 10 |
# the same condition variable for ACK signaling. However, with the MDEV-33551 |
| 11 |
# changes, each thread has its own condition variable to signal. So T1 could |
| 12 |
# wait indefinitely when either: |
| 13 |
# 1) T1's ACK is received but not T2's when T1 goes into wait_after_sync(), |
| 14 |
# because the ACK receiver thread has already notified about the T1 ACK, |
| 15 |
# but T1 was _actually_ waiting on T2's ACK, and therefore tries to wait |
| 16 |
# (in vain). |
| 17 |
# |
| 18 |
# 2) T1 goes to wait_after_sync() before any ACKs have arrived. When T1's ACK |
| 19 |
# comes in, T1 is woken up; however, sees it needs to wait more (because |
| 20 |
# it was actually waiting on T2's ACK), and goes to wait again (this time, |
| 21 |
# in vain). |
| 22 |
# |
| 23 |
# Note that the actual cause of T1 waiting on T2's binlog coordinates is when |
| 24 |
# MYSQL_BIN_LOG::write() would call Repl_semisync_master::wait_after_sync(), |
| 25 |
# the binlog offset parameter was read as the end of MYSQL_BIN_LOG::log_file, |
| 26 |
# which is shared among transactions. So if T2 had updated the binary log |
| 27 |
# _after_ T1 had released LOCK_log, but not yet invoked wait_after_sync(), it |
| 28 |
# would use the end of the binary log file as the binlog offset, which was that |
| 29 |
# of T2 (or any future transaction). The patch itself fixes this issue, and |
| 30 |
# ensures consistency between the binary log coordinates a transaction uses |
| 31 |
# between report_binlog_update() and wait_after_sync(). |
| 32 |
# |
| 33 |
# This test reproduces the above race condition, to ensure that T1 can no |
| 34 |
# longer use T2's binlog coordinates when wait_point=AFTER_SYNC (AFTER_COMMIT |
| 35 |
# was never affected). That is, because T1 in the above scenario must binlog |
| 36 |
# through MYSQL_BIN_LOG::write(), it is a DDL. T2 is an MDL that commits |
| 37 |
# through binlog group commit. We use debug_sync to stop both T1 and T2 |
| 38 |
# after binlogging, reporting their coordinates to the semi-sync cache (i.e. |
| 39 |
# after report_binlog_update()), and after releasing LOCK_log. Debug_sync is |
| 40 |
# used on the replica to ensure no ACKs are sent until after both T1 and T2 |
| 41 |
# are paused. Then, T1 and T2 are signaled for wakeup, and both ACKs are sent. |
| 42 |
# Prior to MDEV-35109 fixes, this results in T1 timing out after the configured |
| 43 |
# 2 second timeout. |
| 44 |
# |
| 45 |
# References: |
| 46 |
# MDEV-33551: Semi-sync Wait Point AFTER_COMMIT Slow on Workloads with Heavy |
| 47 |
# Concurrency |
| 48 |
# MDEV-35109: Semi-sync Replication stalling Primary using |
| 49 |
# wait_point=AFTER_SYNC |
| 50 |
# |
| 51 |
|
| 52 |
--source include/have_innodb.inc |
| 53 |
--source include/have_debug_sync.inc |
| 54 |
# Test is format independent |
| 55 |
--source include/have_binlog_format_row.inc |
| 56 |
--source include/master-slave.inc |
| 57 |
|
| 58 |
--connection slave |
| 59 |
--source include/stop_slave.inc |
| 60 |
set @old_enabled= @@global.rpl_semi_sync_slave_enabled; |
| 61 |
set @old_dbug= @@global.debug_dbug; |
| 62 |
set global rpl_semi_sync_slave_enabled= 1; |
| 63 |
|
| 64 |
--connection master |
| 65 |
set @old_enabled= @@global.rpl_semi_sync_master_enabled; |
| 66 |
set @old_timeout= @@global.rpl_semi_sync_master_timeout; |
| 67 |
set @old_wait_point= @@global.rpl_semi_sync_master_wait_point; |
| 68 |
set global rpl_semi_sync_master_enabled= 1; |
| 69 |
set global rpl_semi_sync_master_timeout= 2000; # 2s |
| 70 |
set global rpl_semi_sync_master_wait_point= AFTER_SYNC; |
| 71 |
|
| 72 |
--connection slave |
| 73 |
--source include/start_slave.inc |
| 74 |
|
| 75 |
--echo # Ensure slave connection is semi-simulate_delay_semisync_slave_reply |
| 76 |
--connection master |
| 77 |
--let $status_var_value= ON |
| 78 |
--let $status_var= Rpl_semi_sync_master_status |
| 79 |
--source include/wait_for_status_var.inc |
| 80 |
--connection slave |
| 81 |
--let $status_var_value= ON |
| 82 |
--let $status_var= Rpl_semi_sync_slave_status |
| 83 |
--source include/wait_for_status_var.inc |
| 84 |
|
| 85 |
--echo # |
| 86 |
--echo # Initialize test data |
| 87 |
--connection master |
| 88 |
|
| 89 |
# It is simpler to create t1 before using debug_sync on the slave_reply |
| 90 |
create table t1 (a int) engine=innodb; |
| 91 |
--source include/save_master_gtid.inc |
| 92 |
--connection slave |
| 93 |
--source include/sync_with_master_gtid.inc |
| 94 |
|
| 95 |
--source include/stop_slave.inc |
| 96 |
set @@global.debug_dbug="+d,simulate_delay_semisync_slave_reply"; |
| 97 |
--source include/start_slave.inc |
| 98 |
|
| 99 |
--echo # Ensure connection is semi-sync |
| 100 |
--connection master |
| 101 |
--let $status_var_value= ON |
| 102 |
--let $status_var= Rpl_semi_sync_master_status |
| 103 |
--source include/wait_for_status_var.inc |
| 104 |
--connection slave |
| 105 |
--let $status_var_value= ON |
| 106 |
--let $status_var= Rpl_semi_sync_slave_status |
| 107 |
--source include/wait_for_status_var.inc |
| 108 |
|
| 109 |
--connection master |
| 110 |
set @old_bgc_count= @@global.binlog_commit_wait_count; |
| 111 |
set @old_bgc_usec= @@global.binlog_commit_wait_usec; |
| 112 |
set @@global.binlog_commit_wait_count=1; |
| 113 |
set @@global.binlog_commit_wait_usec=100000; |
| 114 |
|
| 115 |
--connection server_1 |
| 116 |
set debug_sync= "commit_after_release_LOCK_log SIGNAL ddl_binlogged WAIT_FOR ddl_cont"; |
| 117 |
--echo # T1 (DDL) |
| 118 |
--send create table t2 (a int) |
| 119 |
|
| 120 |
--connect(server_1_sync, localhost, root,,) |
| 121 |
--connection server_1_sync |
| 122 |
set debug_sync= "now WAIT_FOR ddl_binlogged"; |
| 123 |
--source include/save_master_gtid.inc |
| 124 |
|
| 125 |
--connection server_2 |
| 126 |
set debug_sync= "now WAIT_FOR io_thd_at_slave_reply"; |
| 127 |
|
| 128 |
--connection master |
| 129 |
set debug_sync= "commit_before_get_LOCK_after_binlog_sync SIGNAL mdl_binlogged WAIT_FOR mdl_cont"; |
| 130 |
--echo # T2 (DML) |
| 131 |
--send insert into t1 values (100); |
| 132 |
|
| 133 |
--connection server_1_sync |
| 134 |
set debug_sync= "now WAIT_FOR mdl_binlogged"; |
| 135 |
|
| 136 |
--echo # Both transactions binlogged and released LOCK_log, and are just before |
| 137 |
--echo # wait_after_sync() |
| 138 |
set debug_sync= "now SIGNAL ddl_cont"; |
| 139 |
# Ensure ddl_cont is seen before next SIGNAL |
| 140 |
--let $wait_condition= SELECT count(*)=0 FROM information_schema.processlist WHERE state LIKE "%commit_after_release_LOCK_log%" |
| 141 |
--source include/wait_condition.inc |
| 142 |
|
| 143 |
set debug_sync= "now SIGNAL mdl_cont"; |
| 144 |
|
| 145 |
--connection server_2 |
| 146 |
--echo # slave_reply for DDL |
| 147 |
set debug_sync= "now SIGNAL io_thd_do_reply"; |
| 148 |
--echo # slave_reply for MDL |
| 149 |
set debug_sync= "now WAIT_FOR io_thd_at_slave_reply"; |
| 150 |
set debug_sync= "now SIGNAL io_thd_do_reply"; |
| 151 |
|
| 152 |
--echo # Reaping MDL.. |
| 153 |
--connection master |
| 154 |
--reap |
| 155 |
--echo # ..done |
| 156 |
|
| 157 |
--echo # Reaping DDL.. |
| 158 |
--connection server_1 |
| 159 |
--reap |
| 160 |
--echo # ..done |
| 161 |
|
| 162 |
|
| 163 |
--echo # |
| 164 |
--echo # Cleanup |
| 165 |
--connection slave |
| 166 |
--source include/stop_slave.inc |
| 167 |
set @@global.rpl_semi_sync_slave_enabled= @old_enabled; |
| 168 |
set @@global.debug_dbug= @old_dbug; |
| 169 |
--source include/start_slave.inc |
| 170 |
|
| 171 |
--connection master |
| 172 |
set @@global.binlog_commit_wait_count= @old_bgc_count; |
| 173 |
set @@global.binlog_commit_wait_usec= @old_bgc_usec; |
| 174 |
set @@global.rpl_semi_sync_master_enabled= @old_enabled; |
| 175 |
set @@global.rpl_semi_sync_master_timeout= @old_timeout; |
| 176 |
set @@global.rpl_semi_sync_master_wait_point= @old_wait_point; |
| 177 |
drop table t1, t2; |
| 178 |
--source include/save_master_gtid.inc |
| 179 |
|
| 180 |
--connection slave |
| 181 |
--source include/sync_with_master_gtid.inc |
| 182 |
|
| 183 |
--source include/rpl_end.inc |
| 184 |
--echo # End of rpl_semi_sync_after_sync_coord_consistency.test |