| 1 |
# This test forces InnoDB to behave in a wrong way by missing a lot of calls |
| 2 |
# to the log_free_check(). The page flushing is disabled, so the log writer |
| 3 |
# thread quickly enters its private so-called extra_margin. This results in |
| 4 |
# a warning being emitted (verified by the test). Soon the log_writer runs |
| 5 |
# out of space in the extra_margin. Then error is emitted (verified). |
| 6 |
|
| 7 |
# Since then an attempt to update a row ends up in a hanging UPDATE query. |
| 8 |
|
| 9 |
# This is a moment, which simulates the situation that could potentially |
| 10 |
# have happened if some code-path was missing calls to the log_free_check() |
| 11 |
# and the oldest dirty page became re-modified and latched preventing the |
| 12 |
# last_checkpoint_lsn from being advanced (simulated by the disabled page |
| 13 |
# cleaners for sake of simplicity). If that happened during ordinary workload, |
| 14 |
# the only help can come from DBA, who needs to observe the warning / error |
| 15 |
# that were emitted and resize the redo log, increasing its capacity. |
| 16 |
|
| 17 |
# Therefore this is what is done in this test. The redo log becomes increased |
| 18 |
# by 1MB. However page cleaners are still disabled so the redo log becomes |
| 19 |
# filled again (verified). That happens without exiting the extra_margin |
| 20 |
# meanwhile, because the extra_margin is slightly bigger than 1MB. |
| 21 |
|
| 22 |
# Then the page cleaners are enabled and the hanging UPDATE becomes finished |
| 23 |
# (verified). Then the original redo log capacity is restored and it might |
| 24 |
# take some time but eventually the redo log should become resized down. |
| 25 |
|
| 26 |
# A next UPDATE should be finished easily (verified). |
| 27 |
|
| 28 |
--source include/have_debug.inc |
| 29 |
--source include/have_debug_sync.inc |
| 30 |
|
| 31 |
# Required to avoid full log buffer flushes and unexpected waits on mutex |
| 32 |
# inside binlog code when some threads are waiting for free space in redo. |
| 33 |
--source include/not_log_bin.inc |
| 34 |
|
| 35 |
# Save the initial number of concurrent sessions. |
| 36 |
--source include/count_sessions.inc |
| 37 |
|
| 38 |
SELECT @@innodb_redo_log_capacity INTO @orig_redo_log_capacity; |
| 39 |
SELECT @@innodb_flush_log_at_trx_commit INTO @orig_flush_log_at_trx_commit; |
| 40 |
|
| 41 |
SET GLOBAL innodb_flush_log_at_trx_commit = 0; |
| 42 |
|
| 43 |
--echo # Prepare schema used in the tests. |
| 44 |
--source include/ib_log_spammer_init.inc |
| 45 |
CREATE TABLE t (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; |
| 46 |
INSERT INTO t(a,b) VALUES (1,0); |
| 47 |
|
| 48 |
--echo # Disable page cleaners and prevent page flushing/checkpointing. |
| 49 |
--source ../include/log_disable_page_cleaners.inc |
| 50 |
|
| 51 |
--echo # Skip log free checks to fill redo log up to its very end. |
| 52 |
SET GLOBAL DEBUG = '+d,log_free_check_skip'; |
| 53 |
|
| 54 |
--echo # Request the log writer to generate a signal when it enters the extra_margin. |
| 55 |
SET GLOBAL DEBUG = '+d,syncpoint_log_writer_entered_extra_margin'; |
| 56 |
|
| 57 |
--echo # Fill redo log until the log writer enters the extra_margin... |
| 58 |
--connect(spam_worker_connection,localhost,root,,test) |
| 59 |
--send CALL log_spammer() |
| 60 |
--connection default |
| 61 |
|
| 62 |
--echo # Redo log data is being generated... |
| 63 |
|
| 64 |
--echo # Wait until the log_writer signaled that it entered the extra margin. |
| 65 |
SET DEBUG_SYNC = 'now WAIT_FOR reached_log_writer_entered_extra_margin'; |
| 66 |
|
| 67 |
--echo # Log writer entered the extra margin ! |
| 68 |
let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err; |
| 69 |
let SEARCH_PATTERN=\[Warning\] \[[^]]*\] \[[^]]*\] Redo log is running out of free space, pausing user threads; |
| 70 |
--source include/search_pattern.inc |
| 71 |
|
| 72 |
SET GLOBAL innodb_flush_log_at_trx_commit = 2; |
| 73 |
|
| 74 |
--echo # Request the log_writer to generate a signal when it runs out of free space in the extra_margin. |
| 75 |
SET GLOBAL DEBUG = '+d,syncpoint_log_writer_ran_out_space'; |
| 76 |
|
| 77 |
--echo # Let the log writer continue, and wait until it runs out of free space in the extra_margin. |
| 78 |
SET GLOBAL DEBUG = '-d,syncpoint_log_writer_entered_extra_margin'; |
| 79 |
SET DEBUG_SYNC = 'now SIGNAL continue_log_writer_entered_extra_margin WAIT_FOR reached_log_writer_ran_out_space'; |
| 80 |
|
| 81 |
--echo # Log writer ran out of free space ! |
| 82 |
let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err; |
| 83 |
let SEARCH_PATTERN=\[ERROR\] \[[^]]*\] \[[^]]*\] Out of space in the redo log. Checkpoint LSN:; |
| 84 |
--source include/search_pattern.inc |
| 85 |
|
| 86 |
--echo # Let log writer continue to try... It cannot succeed without checkpoint being advanced. |
| 87 |
SET GLOBAL DEBUG = '-d,syncpoint_log_writer_ran_out_space'; |
| 88 |
SET DEBUG_SYNC = 'now SIGNAL continue_log_writer_ran_out_space'; |
| 89 |
|
| 90 |
SET GLOBAL innodb_flush_log_at_trx_commit = 1; |
| 91 |
|
| 92 |
--echo # Start a DML query which should be hanging on trx_commit_complete_for_mysql... |
| 93 |
--connect(dml_connection,localhost,root,,test) |
| 94 |
--send UPDATE t SET b=1 WHERE a=1 |
| 95 |
--connection default |
| 96 |
|
| 97 |
--echo # Wait until UPDATE becomes visible in information_schema.processlist... |
| 98 |
--let $wait_condition=SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE info = 'UPDATE t SET b=1 WHERE a=1' |
| 99 |
--let $wait_timeout=60 |
| 100 |
--source include/wait_condition.inc |
| 101 |
|
| 102 |
--echo # Sleep for 1s to validate it is still there (not yet completed). |
| 103 |
--sleep 1 |
| 104 |
|
| 105 |
--echo # Ensure the DML has not been completed (no redo space)... |
| 106 |
--let $assert_text='The UPDATE is still not finished' |
| 107 |
--let $assert_cond= "[SELECT COUNT(*) FROM information_schema.processlist WHERE info = \'UPDATE t SET b=1 WHERE a=1\']" > 0 |
| 108 |
--source include/assert.inc |
| 109 |
|
| 110 |
--echo # Assume that after resizing up we will be more lucky, not to hit the same issue again |
| 111 |
--echo # but at higher redo log size, so let's resume log_free_check() (else: we could quickly |
| 112 |
--echo # eat the whole extended space and remain in deadlock) |
| 113 |
|
| 114 |
SET GLOBAL DEBUG = '-d,log_free_check_skip'; |
| 115 |
|
| 116 |
--echo # Resize-up redo log a little bit to ensure it will be filled again. |
| 117 |
SET GLOBAL innodb_redo_log_capacity = @@innodb_redo_log_capacity + 1048576; |
| 118 |
|
| 119 |
--echo # Ensure the log writer has still NOT exited the extra margin. |
| 120 |
let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err; |
| 121 |
let SEARCH_PATTERN=\[Note\] \[[^]]*\] \[[^]]*\] Redo log reclaimed some free space, resuming user threads; |
| 122 |
--source include/search_pattern.inc |
| 123 |
|
| 124 |
--echo # Request the log writer to generate a signal when it exits the extra_margin. |
| 125 |
SET GLOBAL DEBUG = '+d,syncpoint_log_writer_exited_extra_margin'; |
| 126 |
|
| 127 |
--echo ############################### |
| 128 |
--echo # Enable page cleaners. |
| 129 |
--echo ############################### |
| 130 |
|
| 131 |
--source ../include/log_enable_page_cleaners.inc |
| 132 |
|
| 133 |
--echo # Ensure the started DML becomes completed eventually. |
| 134 |
--connection dml_connection |
| 135 |
--reap |
| 136 |
--connection default |
| 137 |
--disconnect dml_connection |
| 138 |
|
| 139 |
--let $assert_text='The UPDATE has been completed' |
| 140 |
--let $assert_cond="[SELECT COUNT(*) FROM information_schema.processlist WHERE info = \'UPDATE t SET b=1 WHERE a=1\']" = 0 |
| 141 |
--source include/assert.inc |
| 142 |
|
| 143 |
--echo # Request the spammer to stop, but do not wait because earlier the log_writer/log_checkpointer |
| 144 |
--echo # can be trapped at log_writer_exited_extra_margin |
| 145 |
|
| 146 |
--connect(spam_controller_connection,localhost,root,,test) |
| 147 |
--send UPDATE log_spam_controller SET status = 'Stopping' WHERE id = 1 |
| 148 |
--connection default |
| 149 |
|
| 150 |
--echo # Wait until the log_writer signaled that it exited the extra margin. |
| 151 |
|
| 152 |
SET DEBUG_SYNC = 'now WAIT_FOR reached_log_writer_exited_extra_margin'; |
| 153 |
SET GLOBAL DEBUG = '-d,syncpoint_log_writer_exited_extra_margin'; |
| 154 |
SET DEBUG_SYNC = 'now SIGNAL continue_log_writer_exited_extra_margin'; |
| 155 |
|
| 156 |
--echo # Ensure the log writer has exited the extra margin. |
| 157 |
let SEARCH_FILE=$MYSQLTEST_VARDIR/log/mysqld.1.err; |
| 158 |
let SEARCH_PATTERN=\[Note\] \[[^]]*\] \[[^]]*\] Redo log reclaimed some free space, resuming user threads; |
| 159 |
--source include/search_pattern.inc |
| 160 |
|
| 161 |
--echo # Wait until the request to stop spammer is completed. |
| 162 |
|
| 163 |
--connection spam_controller_connection |
| 164 |
--reap |
| 165 |
--connection default |
| 166 |
--disconnect spam_controller_connection |
| 167 |
|
| 168 |
--echo # Wait until the spammer is stopped. |
| 169 |
|
| 170 |
--connection spam_worker_connection |
| 171 |
--reap |
| 172 |
--connection default |
| 173 |
|
| 174 |
--disconnect spam_worker_connection |
| 175 |
|
| 176 |
--let $assert_text='The log spammer is stopped' |
| 177 |
--let $assert_cond="[SELECT status FROM log_spam_controller WHERE id = 1]" = "Stopped" |
| 178 |
--source include/assert.inc |
| 179 |
|
| 180 |
--echo # Ensure new changes can be done quickly too. |
| 181 |
UPDATE t SET b=2 WHERE a=1; |
| 182 |
SELECT * FROM t; |
| 183 |
|
| 184 |
--echo # Resize the redo log to its original capacity. |
| 185 |
|
| 186 |
SET GLOBAL innodb_redo_log_capacity = @orig_redo_log_capacity; |
| 187 |
|
| 188 |
--echo # Waiting for status = OK.... |
| 189 |
let $wait_timeout= 600; |
| 190 |
let $wait_condition = |
| 191 |
SELECT variable_value = 'OK' |
| 192 |
FROM performance_schema.global_status |
| 193 |
WHERE variable_name = 'Innodb_redo_log_resize_status'; |
| 194 |
--source include/wait_condition.inc |
| 195 |
|
| 196 |
--echo # Cleanup |
| 197 |
DROP TABLE t; |
| 198 |
--source include/ib_log_spammer_cleanup.inc |
| 199 |
|
| 200 |
SET GLOBAL innodb_flush_log_at_trx_commit = @orig_flush_log_at_trx_commit; |
| 201 |
|
| 202 |
SET DEBUG_SYNC = 'RESET'; |
| 203 |
|
| 204 |
--source include/wait_until_count_sessions.inc |