Debian Codegraph @

mysql-test/suite/rpl/t/rpl_parallel_sbm.test

raw
1 #
2 # Ensure that Seconds_Behind_Master works correctly on the parallel replica.
3 #
4 --source include/have_log_bin.inc
5 --source include/have_debug.inc
6 --source include/have_debug_sync.inc
7 --source include/master-slave.inc
8
9 --echo #
10 --echo # MDEV-29639: Seconds_Behind_Master is incorrect for Delayed, Parallel Replicas
11 --echo #
12
13 # This test ensures that after a delayed parallel slave has idled, i.e.
14 # executed everything in its relay log, the next event group that the SQL
15 # thread reads from the relay log will immediately be used in the
16 # Seconds_Behind_Master. In particular, it ensures that the calculation for
17 # Seconds_Behind_Master is based on the timestamp of the new transaction,
18 # rather than the last committed transaction.
19 #
20 # Note that the test doesn't actually use the value of MASTER_DELAY, but
21 # rather uses debug_sync to coordinate the end of the delay. This is to ensure
22 # that on slow machines, a delay doesn't pass before the test gets a chance to
23 # validate results. Additionally, it lets us continue testing after validation
24 # so we don't have to wait out a full delay unnecessarily. The debug_sync point
25 # is enabled via sql_delay_by_debug_sync, which will delay transactions based
26 # only on GTID events, so only one synchronization is needed per transaction.
27 #
28
29 --connection slave
30 --source include/stop_slave.inc
31 --let $save_dbug= `SELECT @@GLOBAL.debug_dbug`
32 --let $save_parallel_mode= `SELECT @@GLOBAL.slave_parallel_mode`
33 set @@GLOBAL.slave_parallel_mode= CONSERVATIVE;
34 set @@GLOBAL.debug_dbug= "d,negate_clock_diff_with_master,sql_delay_by_debug_sync";
35
36 --let $master_delay= 1
37 --eval change master to master_delay=$master_delay, master_use_gtid=Slave_Pos
38 --source include/start_slave.inc
39
40 --connection master
41 --let insert_ctr= 0
42 create table t1 (a int);
43 create table t2 (a int);
44 --source include/save_master_gtid.inc
45 --connection slave
46 --let $slave_sync_method= gtid
47 --let $num_event_groups= 2
48 --source include/sync_with_master_sql_delay_debug_sync.inc
49
50 --echo #
51 --echo # Pt 1) Ensure SBM is updated immediately upon arrival of the next event
52
53 --connection master
54 --echo # Sleep 2 to allow a buffer between events for SBM check
55 sleep 2;
56
57 --let $ts_trx_before_ins= `SELECT UNIX_TIMESTAMP()`
58 --eval insert into t1 values ($insert_ctr)
59 --inc $insert_ctr
60 --source include/save_master_gtid.inc
61
62 --connection slave
63
64 --echo # Waiting for transaction to arrive on slave and begin SQL Delay..
65 set debug_sync= "now WAIT_FOR at_sql_delay";
66
67 --echo # Validating SBM is updated on event arrival..
68 --let $sbm_trx1_arrive= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1)
69 --let $seconds_since_idling= `SELECT UNIX_TIMESTAMP() - $ts_trx_before_ins`
70 if (`SELECT $sbm_trx1_arrive > ($seconds_since_idling + 1)`)
71 {
72 --echo # SBM was $sbm_trx1_arrive yet shouldn't have been larger than $seconds_since_idling + 1 (for possible negative clock_diff_with_master)
73 --die Seconds_Behind_Master should reset after idling
74 }
75 --echo # ..done
76
77 --echo # MDEV-33856: New definition for Seconds_Behind_Master
78 --echo # Validating Master_last_event_time is updated on event arrival..
79 --let $mle_time_trx1_arrive= query_get_value(SHOW ALL SLAVES STATUS, Master_last_event_time, 1)
80 --let $mle_time_trx1_arrive_unix= `SELECT truncate(unix_timestamp("$mle_time_trx1_arrive"), 0)`
81 if (`SELECT $mle_time_trx1_arrive_unix < ($ts_trx_before_ins - 1)`)
82 {
83 --echo # Master_last_event_time: $mle_time_trx1_arrive_unix ($mle_time_trx1_arrive)
84 --die Master_last_event_time was not updated for delayed replica at event arrival time
85 }
86 --echo # ..done
87
88 --echo # Validating Slave_last_event_time is still from the last transaction..
89 # Note we infer Slave_last_event_time via Master_Slave_time_diff
90 --let $time_diff_trx1_arrive= query_get_value(SHOW ALL SLAVES STATUS, Master_Slave_time_diff, 1)
91 if ($time_diff_trx1_arrive < 2)
92 {
93 --let $slave_time_trx1_arrive= query_get_value(SHOW ALL SLAVES STATUS, Slave_last_event_time, 1)
94 --let $slave_time_trx1_arrive_unix= `SELECT truncate(unix_timestamp("$slave_time_trx1_arrive"), 0)`
95 --echo # Slave_last_event_time: $slave_time_trx1_arrive_unix ($slave_time_trx1_arrive)
96 --echo # Master_Slave_time_diff: $time_diff_trx1_arrive
97 --die Slave_last_event_time is too recent, should not be less than 2 seconds, ie. from sleeping
98 }
99 --echo # ..done
100
101
102 --echo # MDEV-32265. At time of STOP SLAVE, if the SQL Thread is currently
103 --echo # delaying a transaction; then when the reciprocal START SLAVE occurs,
104 --echo # if the event is still to be delayed, SBM should resume accordingly
105
106 --connection server_2
107 --echo # Ensure the kill from STOP SLAVE will be received before continuing the
108 --echo # SQL thread
109 set debug_sync="after_thd_awake_kill SIGNAL slave_notified_of_kill";
110 --send STOP SLAVE
111
112 --connection slave
113 set debug_sync= "now WAIT_FOR slave_notified_of_kill";
114 set debug_sync= "now SIGNAL continue_sql_thread";
115
116 --connection server_2
117 --reap
118 --source include/wait_for_slave_to_stop.inc
119 set debug_sync="RESET";
120
121 --echo # Lock t1 on slave to ensure the event can't finish (and thereby update
122 --echo # Seconds_Behind_Master) so slow running servers don't accidentally
123 --echo # catch up to the master before checking SBM.
124 --connection server_2
125 LOCK TABLES t1 WRITE;
126
127 --source include/start_slave.inc
128
129 --connection slave
130
131 --echo # SQL delay has no impact for the rest of the test case, so ignore it
132 --let $slave_sync_method= none
133 --source include/sync_with_master_sql_delay_debug_sync.inc
134
135 --echo # Waiting for replica to get blocked by the table lock
136 --let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Waiting for table metadata lock';
137 --source include/wait_condition.inc
138
139 --echo # Sleeping 1s to increment SBM
140 sleep 1;
141
142 --echo # Ensuring Seconds_Behind_Master increases after sleeping..
143 --let $sbm_trx1_after_1s_sleep= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1)
144 if (`SELECT $sbm_trx1_after_1s_sleep <= $sbm_trx1_arrive`)
145 {
146 --echo # ..failed
147 --die Seconds_Behind_Master did not increase after sleeping, but should have
148 }
149 --echo # ..done
150
151 --connection server_2
152 UNLOCK TABLES;
153
154 --source include/sync_with_master_gtid.inc
155
156 --echo # MDEV-33856: New definition for Seconds_Behind_Master
157
158 # The update of slave timing statistics (Seconds_Behind_Master,
159 # Slave_last_event_time, etc) lags slightly behind the update of
160 # gtid_slave_pos (we just called sync_with_master_gtid), so wait for SBM==0 to
161 # indicate the statistics have been updated
162 --echo # Waiting for slave timing statistics to update ..
163 --let $wait_condition= SELECT count(*) FROM information_schema.slave_status WHERE Seconds_Behind_Master=0
164 --source include/wait_condition.inc
165
166 --echo # Ensuring Slave_last_event_time is now up-to-date once event is executed
167 --let $slave_time_trx1_commit= query_get_value(SHOW ALL SLAVES STATUS, Slave_last_event_time, 1)
168 --let $slave_time_trx1_commit_unix= `SELECT truncate(unix_timestamp("$slave_time_trx1_commit"),0)`
169 if ($slave_time_trx1_commit_unix != $mle_time_trx1_arrive_unix)
170 {
171 --echo # Slave_last_event_time: $slave_time_trx1_commit_unix ($slave_time_trx1_commit)
172 --echo # Master_last_event_time: $mle_time_trx1_arrive_unix ($mle_time_trx1_arrive)
173 --die Slave_last_event_time is not equal to Master_last_event_time despite being up-to-date
174 }
175 --let $time_diff_trx1_commit= query_get_value(SHOW ALL SLAVES STATUS, Master_Slave_time_diff, 1)
176 if ($time_diff_trx1_commit != 0)
177 {
178 --echo # Master_Slave_time_diff: $time_diff_trx1_commit
179 --die Master_Slave_time_diff should be 0, as slave is up-to-date
180 }
181 --echo # ..done
182
183 --echo #
184 --echo # Pt 2) If the worker threads have not entered an idle state, ensure
185 --echo # following events do not update SBM
186
187 --connection slave
188 LOCK TABLES t1 WRITE;
189
190 --connection master
191 --echo # Sleep 2 to allow a buffer between events for SBM check
192 sleep 2;
193 --let $ts_trxpt2_before_ins= `SELECT UNIX_TIMESTAMP()`
194 --eval insert into t1 values ($insert_ctr)
195 --inc $insert_ctr
196 --echo # Sleep 3 to create gap between events
197 sleep 3;
198 --eval insert into t1 values ($insert_ctr)
199 --inc $insert_ctr
200 --let $ts_trx_after_ins= `SELECT UNIX_TIMESTAMP()`
201 --source include/save_master_pos.inc
202
203 --connection slave
204 --echo # Wait for first transaction to complete SQL delay and begin execution..
205 --let $slave_sync_method= none
206 --source include/sync_with_master_sql_delay_debug_sync.inc
207 --let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Waiting for table metadata lock%' AND command LIKE 'Slave_Worker';
208 --source include/wait_condition.inc
209
210 --echo # Wait for second transaction to complete SQL delay..
211 --source include/sync_with_master_sql_delay_debug_sync.inc
212
213 --echo # Validate SBM calculation doesn't use the second transaction because worker threads shouldn't have gone idle..
214 --let $sbm_after_trx_no_idle= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1)
215 --let $timestamp_trxpt2_arrive= `SELECT UNIX_TIMESTAMP()`
216 if (`SELECT $sbm_after_trx_no_idle < $timestamp_trxpt2_arrive - $ts_trx_after_ins - 1`)
217 {
218 --let $cmpv= `SELECT $timestamp_trxpt2_arrive - $ts_trx_after_ins`
219 --echo # SBM $sbm_after_trx_no_idle was more recent than time since last transaction ($cmpv seconds)
220 --die Seconds_Behind_Master should not have used second transaction timestamp
221 }
222 --let $seconds_since_idling= `SELECT ($timestamp_trxpt2_arrive - $ts_trxpt2_before_ins)`
223 --echo # ..and that SBM wasn't calculated using prior committed transactions
224 if (`SELECT $sbm_after_trx_no_idle > ($seconds_since_idling + 1)`)
225 {
226 --echo # SBM was $sbm_after_trx_no_idle yet shouldn't have been larger than $seconds_since_idling + 1 (for possible negative clock_diff_with_master)
227 --die Seconds_Behind_Master calculation should not have used prior committed transaction
228 }
229 --echo # ..done
230
231 --connection slave
232 UNLOCK TABLES;
233 --source include/sync_with_master.inc
234
235 --echo # Cleanup
236 --source include/stop_slave.inc
237 set debug_sync= "RESET";
238 set @@GLOBAL.debug_dbug= "-d,sql_delay_by_debug_sync";
239 --eval CHANGE MASTER TO master_delay=0
240 --source include/start_slave.inc
241
242
243 --echo #
244 --echo # MDEV-30619: Parallel Slave SQL Thread Can Update Seconds_Behind_Master with Active Workers
245 --echo #
246
247 # This test ensures that a parallel slave will not update
248 # Seconds_Behind_Master after the SQL Thread has idled if the worker threads
249 # are still executing events. To test this, two events are executed on the
250 # primary with $sleep seconds in-between them. Once the second event begins
251 # execution on the replica, Seconds_Behind_Master is queried to ensure it
252 # reflects the value of the first transaction, rather than the second.
253
254 --connection slave
255 --echo # Ensure the replica is fully idle before starting transactions
256 --let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Slave has read all relay log%';
257 --source include/wait_condition.inc
258 --let $wait_condition= SELECT count(*)=2 FROM information_schema.processlist WHERE state LIKE 'Waiting for work from SQL thread';
259 --source include/wait_condition.inc
260
261 --echo # Lock t1 on slave so the first received transaction does not complete/commit
262 LOCK TABLES t1 WRITE;
263
264 --connection master
265 --let $ts_t1_before_master_ins= `SELECT UNIX_TIMESTAMP()`
266 --eval insert into t1 values ($insert_ctr)
267 --inc $insert_ctr
268 --source include/save_master_gtid.inc
269
270 --connection slave
271 --echo # Waiting for first transaction to begin..
272 --let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Waiting for table metadata lock';
273 --source include/wait_condition.inc
274
275 --let $sbm_1= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1)
276
277 --connection master
278 --let $sleep = 2
279 --echo # Sleep $sleep sec to create a gap between events
280 sleep $sleep;
281 INSERT INTO t2 VALUES (1);
282 --source include/save_master_gtid.inc
283
284 --connection slave
285 --echo # Waiting for second transaction to begin..
286 --let $wait_condition= SELECT count(*) FROM information_schema.processlist WHERE state LIKE 'Waiting for prior transaction to start commit%';
287 --source include/wait_condition.inc
288
289 --let $sbm_2= query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1)
290
291 if (`SELECT $sbm_1 + $sleep > $sbm_2`)
292 {
293 --echo # Seconds_Behind_Masters: $sbm_1 $sbm_2_0
294 --die Two successive Seconds_Behind_Master timestamps must be separated by the sleep parameter value or greater
295 }
296
297 --connection slave
298 UNLOCK TABLES;
299 --source include/sync_with_master_gtid.inc
300
301
302 --echo #
303 --echo # Cleanup
304
305 --connection master
306 DROP TABLE t1, t2;
307 --source include/save_master_gtid.inc
308
309 --connection slave
310 --source include/sync_with_master_gtid.inc
311 --source include/stop_slave.inc
312 --eval set @@GLOBAL.debug_dbug= "$save_dbug"
313 --evalp set @@GLOBAL.slave_parallel_mode= "$save_parallel_mode"
314 --source include/start_slave.inc
315
316 --source include/rpl_end.inc
317 --echo # End of rpl_parallel_sbm.test