Debian Codegraph @

mysql-test/suite/group_replication/t/gr_majority_loss_5_to_2.test

raw
1 ################################################################################
2 # Verify that a blocked group, a group with 5 members on which 3 do crash
3 # (without executing STOP GROUP_REPLICATION), can be unblocked by DBA.
4 #
5 # Test:
6 # 0) The test requires five servers: M1, M2, M3, M4 and M5.
7 # 1) Create a group with 5 members. Bring all ONLINE and create a table.
8 # 2) Crash 3 members: M3, M4, M5.
9 # - A transaction is executed on one blocked server (M2),
10 # which will be committed when group is unblocked.
11 # 3) Unblock the group with the alive member (M1).
12 # 4) Assert check that group has 2 members on M1 and M2.
13 # 5) Execute a transaction on M1 in order to prove liveness. Assert check that
14 # the blocked transaction from M2 is replicated in the new configuration.
15 # 6) Clean up.
16 ################################################################################
17 # This test does crashes servers, thence we skip it on valgrind.
18 --source include/not_valgrind.inc
19 --source include/big_test.inc
20 --source include/force_restart.inc
21 --source include/have_group_replication_plugin.inc
22
23 --let $rpl_connection_name= server1
24 --source include/rpl_connection.inc
25 --let $member1_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
26
27 --let $rpl_connection_name= server2
28 --source include/rpl_connection.inc
29 --let $member2_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
30
31 --let $rpl_connection_name= server3
32 --source include/rpl_connection.inc
33 --let $member3_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
34
35 --let $rpl_connection_name= server4
36 --source include/rpl_connection.inc
37 --let $member4_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
38
39 --let $rpl_connection_name= server5
40 --source include/rpl_connection.inc
41 --let $member5_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
42
43 --let $rpl_connection_name= server1
44 --source include/rpl_connection.inc
45
46 --echo
47 --echo ############################################################
48 --echo # 1. Create a group with 5 members and a table on it.
49 --let $rpl_server_count= 5
50 --source include/group_replication.inc
51 CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB;
52 --source include/rpl_sync.inc
53
54 --echo
55 --echo ############################################################
56 --echo # 2. Crash three members.
57 # We do kill the servers, using shutdown_server 0, and then MTR will
58 # follow the expect file and restart the server, but we do not wait
59 # for the server restart.
60 # Since the servers loose its GCS configuration they will not rejoin
61 # the group.
62 --let $rpl_connection_name= server3
63 --source include/rpl_connection.inc
64 --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.3.expect
65 --shutdown_server 0
66 --source include/wait_until_disconnected.inc
67
68 --let $rpl_connection_name= server4
69 --source include/rpl_connection.inc
70 --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.4.expect
71 --shutdown_server 0
72 --source include/wait_until_disconnected.inc
73
74 --let $rpl_connection_name= server5
75 --source include/rpl_connection.inc
76 --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.5.expect
77 --shutdown_server 0
78 --source include/wait_until_disconnected.inc
79
80 #
81 # Wait until all members are marked unreachable in
82 # the remaining servers
83 #
84
85 # server1
86
87 --let $rpl_connection_name= server1
88 --source include/rpl_connection.inc
89
90 --let $i=5
91 while($i>2)
92 {
93 if ($i == 5)
94 {
95 --let $member_uuid= $member5_uuid
96 }
97
98 if ($i == 4)
99 {
100 --let $member_uuid= $member4_uuid
101 }
102
103 if ($i == 3)
104 {
105 --let $member_uuid= $member3_uuid
106 }
107
108 --let $group_replication_member_state= UNREACHABLE
109 --let $group_replication_member_id= $member_uuid
110 --source include/gr_wait_for_member_state.inc
111 --echo member$i @ server1 is marked as UNREACHABLE
112 --dec $i
113 }
114 --let $group_replication_member_id= $member1_uuid
115
116 # server2
117
118 --let $rpl_connection_name= server2
119 --source include/rpl_connection.inc
120
121 --let $i=5
122 while($i>2)
123 {
124 if ($i == 5)
125 {
126 --let $member_uuid= $member5_uuid
127 }
128
129 if ($i == 4)
130 {
131 --let $member_uuid= $member4_uuid
132 }
133
134 if ($i == 3)
135 {
136 --let $member_uuid= $member3_uuid
137 }
138
139 --let $group_replication_member_state= UNREACHABLE
140 --let $group_replication_member_id= $member_uuid
141 --source include/gr_wait_for_member_state.inc
142 --echo member$i @ server2 is marked as UNREACHABLE
143 --dec $i
144 }
145 --let $group_replication_member_id= $member2_uuid
146
147 # Group is blocked and so cannot change its configuration.
148 --let $local_address_server2= `SELECT @@GLOBAL.group_replication_local_address`
149
150 # This transaction will be blocked since group lost its majority.
151 # Though it needs to be delivered to the group when unblocked.
152 --let $rpl_connection_name= server_2
153 --source include/rpl_connection.inc
154
155 --send INSERT INTO t1 VALUES (2)
156
157
158 --echo
159 --echo ############################################################
160 --echo # 3. Unblock the group with the alive members: server 1 and 2.
161 --let $rpl_connection_name= server1
162 --source include/rpl_connection.inc
163 --let $local_address_server1= `SELECT @@GLOBAL.group_replication_local_address`
164
165 --disable_query_log
166 --eval SET GLOBAL group_replication_force_members= "$local_address_server1,$local_address_server2"
167 --enable_query_log
168
169 --let $assert_text= group_replication_force_members is correct
170 --let $assert_cond= "[SELECT @@GLOBAL.group_replication_force_members]" = "$local_address_server1,$local_address_server2"
171 --source include/assert.inc
172
173 --echo
174 --echo ############################################################
175 --echo # 4. Assert that group has 2 members: server 1 and 2.
176 # Server 1
177 --let $rpl_connection_name= server1
178 --source include/rpl_connection.inc
179
180 --let $group_replication_number_of_members= 2
181 --source include/gr_wait_for_number_of_members.inc
182
183 --let $assert_text= Server 1 must be present on group members
184 --let $assert_cond= [SELECT COUNT(*) FROM performance_schema.replication_group_members WHERE member_id="$member1_uuid"] = 1
185 --source include/assert.inc
186 --let $assert_text= Server 2 must be present on group members
187 --let $assert_cond= [SELECT COUNT(*) FROM performance_schema.replication_group_members WHERE member_id="$member2_uuid"] = 1
188 --source include/assert.inc
189
190 --let $raw_view_id= query_get_value(SELECT view_id FROM performance_schema.replication_group_member_stats where member_id in (SELECT @@server_uuid), view_id, 1)
191 --let $view_id= `SELECT RIGHT('$raw_view_id', 1)`
192 --let $assert_text= The value of view_id must be random:6
193 --let $assert_cond= $view_id >= 6
194 --source include/assert.inc
195
196 # Server 2
197
198 --let $rpl_connection_name= server2
199 --source include/rpl_connection.inc
200
201 --let $group_replication_number_of_members= 2
202 --source include/gr_wait_for_number_of_members.inc
203
204 --let $assert_text= Server 1 must be present on group members
205 --let $assert_cond= [SELECT COUNT(*) FROM performance_schema.replication_group_members WHERE member_id="$member1_uuid"] = 1
206 --source include/assert.inc
207 --let $assert_text= Server 2 must be present on group members
208 --let $assert_cond= [SELECT COUNT(*) FROM performance_schema.replication_group_members WHERE member_id="$member2_uuid"] = 1
209 --source include/assert.inc
210
211 --let $raw_view_id= query_get_value(SELECT view_id FROM performance_schema.replication_group_member_stats where member_id in (SELECT @@server_uuid), view_id, 1)
212 --let $view_id= `SELECT RIGHT('$raw_view_id', 1)`
213 --let $assert_text= The value of view_id must be random:6
214 --let $assert_cond= $view_id >= 6
215 --source include/assert.inc
216
217
218 --echo
219 --echo ############################################################
220 --echo # 5. Execute a transaction in order to prove liveness and
221 --echo # assert that the blocked transaction was replicated in
222 --echo # the new configuration.
223 --let $rpl_connection_name= server_2
224 --source include/rpl_connection.inc
225 --reap
226
227 --let $rpl_connection_name= server1
228 --source include/rpl_connection.inc
229 INSERT INTO t1 VALUES (1);
230 # We cannot use rpl_sync.inc because it will include servers 3, 4 and 5
231 # which are dead and will make rpl_sync.inc to fail.
232 --let $sync_slave_connection= server2
233 --source include/sync_slave_sql_with_master.inc
234
235 --let $assert_text= 'There are two values in table t1'
236 --let $assert_cond= [SELECT COUNT(*) AS count FROM t1, count, 1] = 2
237 --source include/assert.inc
238
239 --let $diff_tables= server1:test.t1, server2:test.t1
240 --source include/diff_tables.inc
241
242
243 --echo
244 --echo ############################################################
245 --echo # 6. Clean up.
246 --let $rpl_connection_name= server_3
247 --source include/rpl_connection.inc
248 --let $rpl_server_number= 3
249 --source include/rpl_reconnect.inc
250 --let $assert_text= 'There are no values in table t1'
251 --let $assert_cond= [SELECT COUNT(*) AS count FROM t1, count, 1] = 0
252 --source include/assert.inc
253 DROP TABLE t1;
254
255 --let $rpl_connection_name= server_4
256 --source include/rpl_connection.inc
257 --let $rpl_server_number= 4
258 --source include/rpl_reconnect.inc
259 --let $assert_text= 'There are no values in table t1'
260 --let $assert_cond= [SELECT COUNT(*) AS count FROM t1, count, 1] = 0
261 --source include/assert.inc
262 DROP TABLE t1;
263
264 --let $rpl_connection_name= server_5
265 --source include/rpl_connection.inc
266 --let $rpl_server_number= 5
267 --source include/rpl_reconnect.inc
268 --let $assert_text= 'There are no values in table t1'
269 --let $assert_cond= [SELECT COUNT(*) AS count FROM t1, count, 1] = 0
270 --source include/assert.inc
271 DROP TABLE t1;
272
273 --let $rpl_connection_name= server1
274 --source include/rpl_connection.inc
275 DROP TABLE t1;
276
277 set session sql_log_bin=0;
278 call mtr.add_suppression("The member lost contact with a majority of the members in the group. Until the network is restored.*");
279 call mtr.add_suppression("The member resumed contact with a majority of the members in the group.*");
280 set session sql_log_bin=1;
281
282 --let $rpl_connection_name= server2
283 --source include/rpl_connection.inc
284
285 set session sql_log_bin=0;
286 call mtr.add_suppression("The member lost contact with a majority of the members in the group. Until the network is restored.*");
287 call mtr.add_suppression("The member resumed contact with a majority of the members in the group.*");
288 set session sql_log_bin=1;
289
290 --let $rpl_connection_name= server4
291 --source include/rpl_connection.inc
292
293 --let $rpl_connection_name= server5
294 --source include/rpl_connection.inc
295
296 --source include/group_replication_end.inc