Debian Codegraph @

mysql-test/suite/rpl/t/rpl_atomic_ddl.test

raw
1 # ==== Purpose ====
2 # Prove successful slave service recovery after a simulated failure
3 # at handling recoverable DDL queries.
4 # The recoverable DDL:s are those that are binary-logged with xid info
5 # with enables the server to decide on their commit or rollback at server
6 # recovery.
7 #
8 # ==== Implementation ====
9 # In the test the slave applier processes a sequence of recoverable DDL:s
10 # whose applying is interruped by a few simulated failures.
11 # When a failure is an ordinary error slave stops, the failed DDL
12 # group is retried to success, provided the failure condition was
13 # withdrawn.
14 # The crash error is sorted out depending on its timing.
15 # When it happens at time before the slave applier has logged the DDL
16 # group into its binary log, first the server recovery rolls back and then
17 # the slave service restarts at the beginning of the crashed DDL's group.
18 # When the applier has logged and yet not committed the DDL statement,
19 # the slave server recovery completes with commit and the slave service
20 # resumes at the next group of events following the crashed DDL's group.
21 #
22 # The test requires the TABLE slave info repository type through
23 # the opt file and --log-bin through a have macro
24 # because of crash-restart actions on the slave side.
25 #
26 # ==== References ====
27 # WL#9175 Correct recovery of DDL statements/transactions by binary log
28 #
29 # Params:
30 # --let $rpl_atomic_ddl_print_verbose # get out more info
31 #
32
33 --let $rpl_atomic_ddl_print_verbose=1
34
35 --source include/big_test.inc
36 --source include/not_crashrep.inc
37 --source include/not_valgrind.inc
38 --source include/have_log_bin.inc
39 --source include/have_debug.inc
40 --source include/have_binlog_format_mixed.inc
41 --source include/have_udf.inc
42 --let $rpl_gtid_utils= 1
43 --let $rpl_privilege_checks_user_grant_option = 1
44 --source include/set_privilege_checks_user_as_system_user.inc
45 --source include/master-slave.inc
46
47 --source include/rpl_connection_slave.inc
48
49 # Simulated data inconsistency on slave causes few errors
50 call mtr.add_suppression("Operation CREATE USER failed");
51 call mtr.add_suppression("Operation RENAME USER failed");
52 call mtr.add_suppression("The replica coordinator and worker threads are stopped");
53 call mtr.add_suppression("Replica worker thread has failed to apply an event");
54 call mtr.add_suppression("Error writing relay log configuration");
55 call mtr.add_suppression("Error 'Unknown error' on query");
56 --disable_query_log
57 call mtr.add_suppression("You need to use --log-bin to make.*work");
58 --enable_query_log
59
60 --source include/rpl_connection_master.inc
61
62 # instructing gtid_step_assert that is called by a sourced file
63 # to satisfy to gtid-mode OFF as well.
64 --let $gtid_mode_on= `SELECT @@GLOBAL.GTID_MODE = 'ON'`
65 --let $gtid_mode= `SELECT @@GLOBAL.GTID_MODE`
66 --let $gtid_step_gtid_mode_agnostic=`SELECT '$gtid_mode' != 'ON'`
67
68 if (!$crash_commit)
69 {
70 --let $commit_message=Crash right after the query has been binary-logged before committed in the engine
71 --let $crash_commit="+d,crash_commit_after_log"
72 }
73 if (!$crash_rollback)
74 {
75 --let $rollback_message=Crash right after the query has been prepared in the engine before being logged
76 --let $crash_rollback="+d,crash_commit_before_log"
77 }
78
79 #
80 # The overall logics of the following part of the tests boils down to
81 # this pseudo-code block:
82 #
83 # Run load generator on master to prepare binlog events;
84 # For a number of simulated of failures
85 # turn ON simulator,
86 # start the slave,
87 # wait for an expected slave error,
88 # turn OFF the current simulator
89 # end for.
90 #
91 # Errors due to Interruptive actions described below are meaningful in
92 # the context of the sourced load generator.
93 # In each loop the server crashes with roll-back and roll-forward following recovery
94 # which is asserted.
95
96 # Master load
97 #
98 # Start off with the master side logger table (see the master opt
99 # file) that record DDL queries that will be subject to crash
100 # simulation on the slave side. The table will hold prescribed
101 # post-recovery checks to be performed on the slave side.
102 #
103 --let $master_log_table=t_checks
104 --let $master_log_db=master_db
105 --eval CREATE DATABASE $master_log_db
106
107 --let $save_curr_db=`SELECT database()`
108 --eval USE $master_log_db
109 --eval CREATE TABLE IF NOT EXISTS $master_log_table (id INT AUTO_INCREMENT NOT NULL PRIMARY KEY, ddl_query TEXT, pre_binlog_check TEXT, post_binlog_check TEXT);
110
111 --eval USE $save_curr_db
112
113 # required for the extra section of the being included binlog_ddl.inc
114 --let $table=t_1
115 --eval CREATE TABLE $table (a int)
116 --source include/sync_slave_sql_with_master.inc
117
118 --source include/stop_slave_sql.inc
119
120 --source include/rpl_connection_master.inc
121
122 --let $do_post_prepare=0
123 --let $do_post_binlog=0
124 --let $do_only_regular_logging=1
125 --let $do_show_binlog_events= 0
126 --let $do_count_queries=1
127 --let $count_ddl_queries=0
128 --source extra/binlog_tests/binlog_ddl.inc
129
130 --source include/rpl_connection_slave.inc
131 --source include/start_slave_io.inc
132
133 --source include/rpl_connection_master.inc
134 --source include/sync_slave_io_with_master.inc
135
136 --source include/rpl_connection_slave.inc
137 # First, a regular error
138 CREATE USER user1;
139 START REPLICA SQL_THREAD;
140 --let $slave_sql_errno=convert_error(ER_CANNOT_USER)
141 --source include/wait_for_slave_sql_error.inc
142 DROP USER user1;
143
144 #
145 # Simulated error in pre_commit hook stops the slave applier.
146 #
147 SET @@GLOBAL.debug="+d,rli_pre_commit_error";
148 START REPLICA SQL_THREAD;
149 --let $slave_sql_errno=convert_error(ER_UNKNOWN_ERROR)
150 --source include/wait_for_slave_sql_error.inc
151 #
152 # Like in above, the fact that there has been no advance of the last
153 # executed coordinates is proved later by successful resuming/recovery
154 # from the error.
155 # The error simulation is lifted:
156 SET @@GLOBAL.debug="-d,rli_pre_commit_error";
157
158 # The rest is crash-restart in loop of number of DDL queries
159 # in the master load.
160 --let $loops=$count_ddl_queries
161 --let $id=0
162 while ($loops)
163 {
164 --inc $id
165
166 # Find the loop's query and it post-recovery checks upon the first recovery
167 --source include/rpl_connection_master.inc
168
169 --let $current_query=`SELECT ddl_query FROM $master_log_db.$master_log_table WHERE id = $id `
170 --let $pre_binlog_check=`SELECT pre_binlog_check FROM $master_log_db.$master_log_table WHERE id = $id`
171 --let $post_binlog_check=`SELECT post_binlog_check FROM $master_log_db.$master_log_table WHERE id = $id`
172 if ($rpl_atomic_ddl_print_verbose)
173 {
174 #
175 --let gen_current_query = `SELECT REPLACE("$current_query",'.so','_LIB')`
176 --let gen_current_query = `SELECT REPLACE("$gen_current_query",'.dll','_LIB')`
177 --echo Loop: $id out of $count_ddl_queries; current query: '$gen_current_query'; pre-binlog check: '$pre_binlog_check'; pre-commit check: '$post_binlog_check';
178 }
179 --source include/rpl_connection_slave.inc
180
181 #
182 # The rollback recovery upon the pre-binlog crash
183 #
184 --source include/expect_crash.inc
185 --echo *** Query: '$gen_current_query'. $rollback_message. ***
186 --eval SET @@GLOBAL.debug=$crash_rollback
187
188 # Memorizing execution status right before "scheduled" failure
189 --source include/gtid_step_reset.inc
190 --error 0,2013
191 START REPLICA SQL_THREAD;
192
193 # Server wait for stop and restart
194 --enable_reconnect
195 --source include/wait_until_disconnected.inc
196 --let $rpl_server_number= 2
197 --source include/rpl_start_server.inc
198 --disable_reconnect
199
200 # Proof of the rollback recovery is no exec progress
201 --let $gtid_step_count=0
202 --source include/gtid_step_assert.inc
203
204 # Proof by the master side post-recovery checks
205 if ($pre_binlog_check)
206 {
207 if (!`$pre_binlog_check`)
208 {
209 --echo *** State check upon recovery after the pre-binlog crash fails ***
210 --die
211 }
212 }
213
214 #
215 # Commit recovery upon the pre-commit crash.
216 #
217 --source include/expect_crash.inc
218 --echo *** Query: '$gen_current_query'. $commit_message. ***
219 --eval SET @@GLOBAL.debug=$crash_commit
220 --source include/gtid_step_reset.inc
221 --error 0,2013
222 START REPLICA SQL_THREAD;
223
224 # Server wait for stop and restart
225 --enable_reconnect
226 --source include/wait_until_disconnected.inc
227 --let $rpl_server_number= 2
228 --source include/rpl_start_server.inc
229 --disable_reconnect
230
231 --source include/rpl_connection_slave.inc
232 # Proof of the commit recovery is one committed gtid
233 --let $gtid_step_only_count=1
234 --let $gtid_step_count=1
235 --source include/gtid_step_assert.inc
236
237 # Proof by the master side post-recovery checks
238 if ($post_binlog_check)
239 {
240 if (!`$post_binlog_check`)
241 {
242 --echo *** State check upon recovery after the pre-commit crash fails ***
243 --die
244 }
245 }
246
247 --dec $loops
248 } # end of while
249
250 # Eventually:
251
252 --source include/rpl_connection_slave.inc
253 --source include/start_slave.inc
254
255 --source include/rpl_connection_master.inc
256 --eval DROP table $table
257 --eval DROP DATABASE $master_log_db
258
259 --source include/sync_slave_sql_with_master.inc
260
261
262 --source include/rpl_end.inc