| 1 |
# |
| 2 |
# This test ensures that, when using semi-sync with the wait_point |
| 3 |
# AFTER_COMMIT, each thread awaiting an ACK is only woken up when its ACK (or |
| 4 |
# an ACK for a later commit in binlog) has been received from the slave. |
| 5 |
# |
| 6 |
# Prior to MDEV-33551, all threads would be woken up for each ACK received, |
| 7 |
# leading to large slowdowns, as each thread would check if the ACK was for it |
| 8 |
# in mutual exclusion from the others. |
| 9 |
# |
| 10 |
# To ensure this, a debug-build-only log warning is added into |
| 11 |
# Repl_semi_sync_master::commit_trx() at wakeup time, which will complain if |
| 12 |
# the awoken thread's binlog wait coordinates are after the coordinate of the |
| 13 |
# last ACK coordinates. Then, we use binlog group commit to commit a series of |
| 14 |
# transactions, such that each will await an ACK concurrently. After all |
| 15 |
# transactions have been finished (i.e. ACKed and committed), we check the log |
| 16 |
# for the expected absence of the added debug warning message. |
| 17 |
# |
| 18 |
# |
| 19 |
# References: |
| 20 |
# MDEV-33551: Semi-sync Wait Point AFTER_COMMIT Slow on Workloads with Heavy |
| 21 |
# Concurrency |
| 22 |
# MDEV-34122: Assertion `entry' failed in Active_tranx::assert_thd_is_waiter |
| 23 |
# |
| 24 |
--source include/have_binlog_format_row.inc |
| 25 |
--source include/have_innodb.inc |
| 26 |
--source include/have_debug.inc |
| 27 |
--source include/have_debug_sync.inc |
| 28 |
--source include/master-slave.inc |
| 29 |
|
| 30 |
--connection master |
| 31 |
call mtr.add_suppression("Got an error writing communication packets"); |
| 32 |
call mtr.add_suppression("Got an error reading communication packets"); |
| 33 |
call mtr.add_suppression("Could not read packet:.* vio_errno: 1158"); |
| 34 |
call mtr.add_suppression("Could not write packet:.* vio_errno: 1160"); |
| 35 |
set @save_semi_sync_master_enabled= @@global.rpl_semi_sync_master_enabled; |
| 36 |
set @save_semi_sync_wp= @@global.rpl_semi_sync_master_wait_point; |
| 37 |
set @save_bgc_count= @@global.binlog_commit_wait_count; |
| 38 |
set @save_bgc_usec= @@global.binlog_commit_wait_usec; |
| 39 |
set @old_master_wait_point= @@global.rpl_semi_sync_master_wait_point; |
| 40 |
set @old_master_wait_no_slave= @@global.rpl_semi_sync_master_wait_no_slave; |
| 41 |
set @save_debug_dbug= @@global.debug_dbug; |
| 42 |
set @@global.binlog_commit_wait_count=3; |
| 43 |
set @@global.binlog_commit_wait_usec=10000000; |
| 44 |
set @@global.debug_dbug="+d,testing_cond_var_per_thd"; |
| 45 |
set @@global.rpl_semi_sync_master_enabled= 1; |
| 46 |
set @@global.rpl_semi_sync_master_wait_point= AFTER_COMMIT; |
| 47 |
|
| 48 |
--echo # Ensure semi-sync is on |
| 49 |
--connection slave |
| 50 |
set @save_semi_sync_slave_enabled= @@global.rpl_semi_sync_master_enabled; |
| 51 |
--source include/stop_slave.inc |
| 52 |
set @@global.rpl_semi_sync_slave_enabled=1; |
| 53 |
--source include/start_slave.inc |
| 54 |
|
| 55 |
let $status_var= rpl_semi_sync_slave_status; |
| 56 |
let $status_var_value= ON; |
| 57 |
source include/wait_for_status_var.inc; |
| 58 |
|
| 59 |
--connection master |
| 60 |
let $status_var= rpl_semi_sync_master_status; |
| 61 |
let $status_var_value= ON; |
| 62 |
source include/wait_for_status_var.inc; |
| 63 |
|
| 64 |
--echo # Create three transactions to binlog group commit together |
| 65 |
--connection master |
| 66 |
--send create table t1 (a int) |
| 67 |
--connection server_1 |
| 68 |
--send create table t2 (a int) |
| 69 |
--connection default |
| 70 |
--send create table t3 (a int) |
| 71 |
|
| 72 |
--connection master |
| 73 |
--reap |
| 74 |
--connection server_1 |
| 75 |
--reap |
| 76 |
--connection default |
| 77 |
--reap |
| 78 |
|
| 79 |
--let $assert_text= Check that there is no 'Thread awaiting semi-sync ACK was awoken before its ACK' warning in error log. |
| 80 |
--let $assert_select=Thread awaiting semi-sync ACK was awoken before its ACK |
| 81 |
--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err |
| 82 |
--let $assert_count= 0 |
| 83 |
--let $assert_only_after=CURRENT_TEST |
| 84 |
--source include/assert_grep.inc |
| 85 |
|
| 86 |
--sync_slave_with_master |
| 87 |
|
| 88 |
|
| 89 |
--echo # |
| 90 |
--echo # Cleanup |
| 91 |
--connection master |
| 92 |
set @@global.binlog_commit_wait_count=@save_bgc_count; |
| 93 |
set @@global.binlog_commit_wait_usec=@save_bgc_usec; |
| 94 |
set @@global.debug_dbug=@save_debug_dbug; |
| 95 |
drop table t1, t2, t3; |
| 96 |
--sync_slave_with_master |
| 97 |
|
| 98 |
|
| 99 |
--echo # |
| 100 |
--echo # MDEV-34122 |
| 101 |
--echo # If semi-sync is switched off then on while a transaction is |
| 102 |
--echo # in-between binlogging and waiting for an ACK, ensure that the |
| 103 |
--echo # transaction skips the wait altogether (otherwise it would time-out). |
| 104 |
--echo # Note that prior to MDEV-34122, there was a debug assertion that would |
| 105 |
--echo # trigger if the transaction tried to wait but could not receive an ACK |
| 106 |
--echo # signal. |
| 107 |
|
| 108 |
--echo # |
| 109 |
--echo # MDEV-34122.a: Test wait_point = AFTER_SYNC |
| 110 |
--echo # Here, debug_sync is used to pause the leader thread between reporting |
| 111 |
--echo # the binlogging to semi-sync, and starting the wait for ACK; and during |
| 112 |
--echo # this pause, semi-sync is manually switched off and on. |
| 113 |
|
| 114 |
|
| 115 |
--connection master |
| 116 |
set @@global.rpl_semi_sync_master_wait_point= AFTER_SYNC; |
| 117 |
set @@global.rpl_semi_sync_master_wait_no_slave= 1; |
| 118 |
--eval set @@global.debug_dbug="+d,semisync_log_skip_trx_wait" |
| 119 |
|
| 120 |
--echo # Waiting for semi-sync to turn on.. |
| 121 |
let $status_var= rpl_semi_sync_master_status; |
| 122 |
let $status_var_value= ON; |
| 123 |
source include/wait_for_status_var.inc; |
| 124 |
--connection slave |
| 125 |
let $status_var= rpl_semi_sync_slave_status; |
| 126 |
let $status_var_value= ON; |
| 127 |
source include/wait_for_status_var.inc; |
| 128 |
|
| 129 |
--connection master |
| 130 |
create table t (a int) engine=innodb; |
| 131 |
--sync_slave_with_master |
| 132 |
|
| 133 |
--echo # Wait no slave is on, so we don't need to slave on to wait for the |
| 134 |
--echo # ACK, and this way we guarantee the ACK doesn't come, so we can verify |
| 135 |
--echo # the informational message is displayed |
| 136 |
--connection slave |
| 137 |
--source include/stop_slave.inc |
| 138 |
# Also kill the master-side dump thread so it doesn't try and send an event |
| 139 |
# over a dead connection (potentially resulting in an error in the log) |
| 140 |
--connection master |
| 141 |
--source include/kill_binlog_dump_threads.inc |
| 142 |
|
| 143 |
--connection server_1 |
| 144 |
start transaction; |
| 145 |
insert into t values (0); |
| 146 |
set debug_sync= "commit_after_release_LOCK_log SIGNAL trx_binlogged WAIT_FOR continue_commit"; |
| 147 |
--send commit |
| 148 |
|
| 149 |
--connection master |
| 150 |
set debug_sync= "now WAIT_FOR trx_binlogged"; |
| 151 |
|
| 152 |
--echo # Switching semi-sync off/on |
| 153 |
set @@global.rpl_semi_sync_master_enabled= 0; |
| 154 |
set @@global.rpl_semi_sync_master_enabled= 1; |
| 155 |
let $status_var= rpl_semi_sync_master_status; |
| 156 |
let $status_var_value= ON; |
| 157 |
source include/wait_for_status_var.inc; |
| 158 |
|
| 159 |
--echo # Resuming transaction to await ACK |
| 160 |
set debug_sync= "now SIGNAL continue_commit"; |
| 161 |
|
| 162 |
--connection server_1 |
| 163 |
--reap |
| 164 |
|
| 165 |
--let $assert_text= Ensure error log shows that transaction is skipping its semi-sync wait |
| 166 |
--let $assert_select=Skipping semi-sync wait for transaction |
| 167 |
--let $assert_file= $MYSQLTEST_VARDIR/log/mysqld.1.err |
| 168 |
--let $assert_count= 1 |
| 169 |
--let $assert_only_after=CURRENT_TEST |
| 170 |
--source include/assert_grep.inc |
| 171 |
|
| 172 |
--echo # Cleanup |
| 173 |
--connection slave |
| 174 |
--source include/start_slave.inc |
| 175 |
|
| 176 |
--connection master |
| 177 |
drop table t; |
| 178 |
--sync_slave_with_master |
| 179 |
|
| 180 |
|
| 181 |
--echo # |
| 182 |
--echo # MDEV-34122.b: Test wait_point = AFTER_COMMIT |
| 183 |
--echo # Here, use a transaction with a non-transactional statement to write to |
| 184 |
--echo # the binlog directly, and turn off/on semi-sync before committing the |
| 185 |
--echo # transaction. |
| 186 |
|
| 187 |
--connection master |
| 188 |
set @@global.rpl_semi_sync_master_wait_point= AFTER_COMMIT; |
| 189 |
|
| 190 |
--echo # Waiting for semi-sync to turn on.. |
| 191 |
let $status_var= rpl_semi_sync_master_status; |
| 192 |
let $status_var_value= ON; |
| 193 |
source include/wait_for_status_var.inc; |
| 194 |
--connection slave |
| 195 |
let $status_var= rpl_semi_sync_slave_status; |
| 196 |
let $status_var_value= ON; |
| 197 |
source include/wait_for_status_var.inc; |
| 198 |
|
| 199 |
--connection master |
| 200 |
create table tn (a int) engine=Aria; |
| 201 |
start transaction; |
| 202 |
insert into tn values (NULL); |
| 203 |
set @@global.rpl_semi_sync_master_enabled= 0; |
| 204 |
set @@global.rpl_semi_sync_master_enabled= 1; |
| 205 |
commit; |
| 206 |
|
| 207 |
--echo # Cleanup |
| 208 |
--connection master |
| 209 |
drop table tn; |
| 210 |
set @@global.debug_dbug=@save_debug_dbug; |
| 211 |
|
| 212 |
|
| 213 |
--echo # |
| 214 |
--echo # MDEV-36934 |
| 215 |
--echo # The server could indefinitely hang due to a memory leak which tried to |
| 216 |
--echo # pthread signal on a destroyed condition variable. In effect, no |
| 217 |
--echo # connections could commit transactions because there would be a thread |
| 218 |
--echo # stuck on a never-returning call to pthread_cond_signal() while |
| 219 |
--echo # holding Repl_semi_sync_master::LOCK_log. |
| 220 |
|
| 221 |
--connection master |
| 222 |
set @@global.rpl_semi_sync_master_wait_point= AFTER_COMMIT; |
| 223 |
set @@global.rpl_semi_sync_master_wait_no_slave= 0; |
| 224 |
|
| 225 |
--echo # Ensure servers are in proper state |
| 226 |
--connection master |
| 227 |
let $status_var= rpl_semi_sync_master_status; |
| 228 |
let $status_var_value= ON; |
| 229 |
source include/wait_for_status_var.inc; |
| 230 |
--connection slave |
| 231 |
let $status_var= rpl_semi_sync_slave_status; |
| 232 |
let $status_var_value= ON; |
| 233 |
source include/wait_for_status_var.inc; |
| 234 |
|
| 235 |
--echo # Test case initial set-up |
| 236 |
--connection master |
| 237 |
create table t_36934 (a int) engine=innodb; |
| 238 |
--source include/save_master_gtid.inc |
| 239 |
--connection slave |
| 240 |
--source include/sync_with_master_gtid.inc |
| 241 |
|
| 242 |
--echo # Pause the user transaction before inserting into Active_tranx |
| 243 |
--connect(user_con,localhost,root,,) |
| 244 |
SET debug_sync= 'semisync_at_write_tranx_in_binlog SIGNAL at_write_tranx_in_binlog WAIT_FOR resume_write_tranx_in_binlog'; |
| 245 |
--send insert into t_36934 values (1) |
| 246 |
|
| 247 |
--connection server_1 |
| 248 |
set debug_sync="now wait_for at_write_tranx_in_binlog"; |
| 249 |
|
| 250 |
--echo # Disconnect the slave (note that the binlog dump thread won't yet be |
| 251 |
--echo # notified of a binlog update from the last transaction, so the slave |
| 252 |
--echo # should neither receiver nor ACK the transaction). |
| 253 |
--connection slave |
| 254 |
--source include/stop_slave.inc |
| 255 |
|
| 256 |
--echo # Waiting for master to realize the slave has disconnected.. |
| 257 |
--connection server_1 |
| 258 |
let $status_var= rpl_semi_sync_master_clients; |
| 259 |
let $status_var_value= 0; |
| 260 |
source include/wait_for_status_var.inc; |
| 261 |
--echo # ..done |
| 262 |
|
| 263 |
--echo # Resuming transaction (it will exit commit_trx early without waiting) |
| 264 |
set debug_sync="now signal resume_write_tranx_in_binlog"; |
| 265 |
|
| 266 |
--connection user_con |
| 267 |
--reap |
| 268 |
--let $user_con_tid= `SELECT connection_id()` |
| 269 |
--disconnect user_con |
| 270 |
--source include/wait_until_disconnected.inc |
| 271 |
|
| 272 |
--echo # Force delete the user thread (FLUSH THREADS ensures the thread won't |
| 273 |
--echo # stay in the thread cache) |
| 274 |
--connection master |
| 275 |
FLUSH THREADS; |
| 276 |
|
| 277 |
--echo # BUG: Re-connect slave. MDEV-36934 reports that the master would hang |
| 278 |
--echo # when the slave would re-connect and try to ACK the last transaction |
| 279 |
--echo # who's thread has been deleted |
| 280 |
--connection slave |
| 281 |
--source include/start_slave.inc |
| 282 |
|
| 283 |
--echo # Try to commit another transaction (prior to MDEV-36934 fixes, this |
| 284 |
--echo # would hang indefinitely) |
| 285 |
--connection master |
| 286 |
set debug_sync="RESET"; |
| 287 |
--send insert into t_36934 values (2) |
| 288 |
|
| 289 |
--connection server_1 |
| 290 |
--echo # Waiting 30s for last query to complete.. |
| 291 |
--let $wait_timeout= 30 |
| 292 |
--let $wait_condition= SELECT count(*)=0 FROM information_schema.processlist WHERE info LIKE 'insert into t_36934%'; |
| 293 |
--source include/wait_condition.inc |
| 294 |
|
| 295 |
# Variable `success` is set by wait_condition.inc |
| 296 |
if (!$success) |
| 297 |
{ |
| 298 |
--echo # ..error |
| 299 |
--die Query is hung |
| 300 |
} |
| 301 |
|
| 302 |
--connection master |
| 303 |
--reap |
| 304 |
--echo # ..done |
| 305 |
|
| 306 |
|
| 307 |
--echo # Cleanup |
| 308 |
--connection master |
| 309 |
set @@global.rpl_semi_sync_master_wait_point= @old_master_wait_point; |
| 310 |
set @@global.rpl_semi_sync_master_wait_no_slave= @old_master_wait_no_slave; |
| 311 |
set @@global.debug_dbug=@save_debug_dbug; |
| 312 |
set @@global.rpl_semi_sync_master_enabled= @save_semi_sync_master_enabled; |
| 313 |
set @@global.rpl_semi_sync_master_wait_point= @save_semi_sync_wp; |
| 314 |
drop table t_36934; |
| 315 |
|
| 316 |
--connection slave |
| 317 |
--source include/stop_slave.inc |
| 318 |
set @@global.rpl_semi_sync_slave_enabled= @save_semi_sync_slave_enabled; |
| 319 |
--source include/start_slave.inc |
| 320 |
|
| 321 |
--source include/rpl_end.inc |