Debian Codegraph @

mysql-test/suite/group_replication/t/gr_notifications_quorum.test

raw
1 ################################################################################
2 #
3 # WL#10412: GR: notify listener services on relevant group events
4 #
5 # This test case checks that the correct notifications are
6 # triggered to the listeners registered in the service registry.
7 #
8 # For that it relies on a table that is updated by a listener
9 # through the SQL service API.
10 #
11 # This file covers the scenarios where QUORUM is lost.
12 #
13 # |------------+----+---------------------------------------------+------------+--------|
14 # | Tested | | Scenario\Where | | |
15 # | in this | | | Server | Others |
16 # | file | | | triggering | |
17 # | | | | event | |
18 # |------------+----+---------------------------------------------+------------+--------|
19 # | | 1 | SERVER BOOTSTRAPS GROUP | VC,2xSC | N/A |
20 # | | 2 | SERVER JOINS and STARTS RECOVERY | VC,SC | VC,SC |
21 # | | 3 | SERVER RECOVERED | SC | SC |
22 # | | 4 | SERVER LEAVES(ERROR), SERVER LEAVES(OFFLINE)| VC,SC | VC |
23 # |------------+----+---------------------------------------------+------------+--------|
24 # | | 5 | SERVER BOOTSTRAPS+RECOVERS+PRIMARY ELECTION | VC,RC,2xSC | |
25 # | | 6 | PRIMARY LEAVES | VC,SC,RC | VC,RC |
26 # |------------+----+---------------------------------------------+------------+--------|
27 # | x | 7 | A SERVER BECOMES UNREACHABLE | SC, VC | N/A |
28 # | x | 8 | MAJORITY UNREACHABLE | QL, SC | N/A |
29 # | x | 9 | MAJORITY UNREACHABLE+FORCE MEMBERS | VC | N/A |
30 # | x | 10 | MAJORITY UNREACHABLE+STOP | VC, SC | N/A |
31 # |------------+----+---------------------------------------------+------------+--------|
32 #
33 # Legend:
34 # - QL - Quorum Lost event
35 # - VC - View Changed event
36 # - RC - Role Changed event
37 # - SC - State Changed event
38 #
39 # To test the cases above, the test sets up a group of
40 # 3 servers in multi-master mode and proceeds to testing
41 # the different scenarios. These are highlighted in the
42 # test file.
43 #
44 ################################################################################
45
46 --source include/have_debug.inc
47 --source include/not_valgrind.inc
48 --source include/force_restart.inc
49 --source include/big_test.inc
50 --source include/not_have_privilege_checks_user.inc
51 --source include/have_group_replication_plugin.inc
52
53 ###
54 ### Sets up the group with three servers: server1, server2 and server3
55 ###
56
57 --let $rpl_server_count= 3
58 --let $rpl_skip_group_replication_start= 1
59 --source include/group_replication.inc
60
61 # server 1
62
63 --let $rpl_connection_name= server1
64 --source include/rpl_connection.inc
65
66 --let $debug_point= register_gms_listener_example
67 --source include/add_debug_point.inc
68
69 --let $member1_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
70 --let $local_address_server1= `SELECT @@GLOBAL.group_replication_local_address`
71 --let $group_seeds_server1= `SELECT @@global.group_replication_group_seeds`
72 --disable_query_log
73 SET SQL_LOG_BIN=0;
74 CREATE TABLE test.gms_listener_example(log_message TEXT);
75 call mtr.add_suppression("The member has failed to gracefully leave the group.");
76 call mtr.add_suppression("Timeout while waiting for the group communication");
77 call mtr.add_suppression("read failed");
78 SET SQL_LOG_BIN=1;
79 --enable_query_log
80
81 # server 2
82
83 --let $rpl_connection_name= server2
84 --source include/rpl_connection.inc
85
86 --let $debug_point= register_gms_listener_example
87 --source include/add_debug_point.inc
88
89 --let $member2_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
90 --let $local_address_server2= `SELECT @@GLOBAL.group_replication_local_address`
91 --let $group_seeds_server2= `SELECT @@global.group_replication_group_seeds`
92 SET SQL_LOG_BIN=0;
93 CREATE TABLE test.gms_listener_example(log_message TEXT);
94 SET SQL_LOG_BIN=1;
95
96 # server 3
97
98 --let $rpl_connection_name= server3
99 --source include/rpl_connection.inc
100
101 --let $debug_point= register_gms_listener_example
102 --source include/add_debug_point.inc
103
104 --let $member3_uuid= query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1)
105 --let $local_address_server3= `SELECT @@GLOBAL.group_replication_local_address`
106 --let $group_seeds_server3= `SELECT @@global.group_replication_group_seeds`
107 SET SQL_LOG_BIN=0;
108 CREATE TABLE test.gms_listener_example(log_message TEXT);
109 SET SQL_LOG_BIN=1;
110
111 --echo ### Scenario 7: SERVER UNREACHABLE
112 --echo ### Expected:
113 --echo ### - Correct Servers: 1 STATE CHANGED (and eventually a VIEW CHANGED)
114 --echo ### - Failed server: N/A
115 --echo ##########################################################
116
117 # server1
118
119 --let $rpl_connection_name= server1
120 --source include/rpl_connection.inc
121 --source include/start_and_bootstrap_group_replication.inc
122
123 # server2
124
125 --let $rpl_connection_name= server2
126 --source include/rpl_connection.inc
127 --source include/start_group_replication.inc
128
129 # server3
130
131 --let $rpl_connection_name= server3
132 --source include/rpl_connection.inc
133 --source include/start_group_replication.inc
134
135 # wait for the new view to be installed
136 --let $group_replication_number_of_members= 3
137 --source include/gr_wait_for_number_of_members.inc
138
139 # server2
140
141 --let $rpl_connection_name= server2
142 --source include/rpl_connection.inc
143
144 # wait for the new view to be installed
145 --let $group_replication_number_of_members= 3
146 --source include/gr_wait_for_number_of_members.inc
147
148 # server1
149
150 --let $rpl_connection_name= server1
151 --source include/rpl_connection.inc
152
153 # wait for the new view to be installed
154 --let $group_replication_number_of_members= 3
155 --source include/gr_wait_for_number_of_members.inc
156
157 # clear notification table everywhere
158 TRUNCATE gms_listener_example;
159 CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB;
160 --source include/rpl_sync.inc
161
162 --let $rpl_connection_name= server_3
163 --source include/rpl_connection.inc
164 --enable_reconnect
165
166 --let $rpl_connection_name= server3
167 --source include/rpl_connection.inc
168
169 --source include/kill_and_restart_mysqld.inc
170
171 --let $rpl_connection_name= server_3
172 --source include/rpl_connection.inc
173 --source include/wait_until_connected_again.inc
174 --disable_reconnect
175
176 # server2
177
178 # There is no point in asserting for server2 that the member
179 # state has changed, since the local view may not be delivered
180 # before the view change is received. I.e., server2 may not
181 # have a chance to check locally for possibly detected members
182 # before it gets the view change triggered by server1 (the
183 # killer node, the one that expels)
184
185 # server1
186
187 #
188 # server1 will always suspect (deliver a local view) and trigger
189 # a view change by sending the global view. Therefore we can assert
190 # that an UNREACHABLE notification is performed. The assumption here
191 # is that server1 is the "leader" node on xcom that sends the global
192 # view.
193 #
194 --let $rpl_connection_name= server1
195 --source include/rpl_connection.inc
196
197 # wait for the new view to be installed
198 --let $group_replication_number_of_members= 2
199 --source include/gr_wait_for_number_of_members.inc
200
201 --let $expected_notifications= 1,STATE %:3|1,VIEW %:4
202 --source ../include/assert_notifications.inc
203
204 --echo ### Scenario 8: MAJORITY UNREACHABLE
205 --echo ### Expected:
206 --echo ### - Correct Servers: 1 QUORUM LOST, 1 STATE CHANGED
207 --echo ### - Failed Servers: N/A
208 --echo ##########################################################
209
210 # server2
211
212 --let $rpl_connection_name= server_2
213 --source include/rpl_connection.inc
214 --enable_reconnect
215
216 --let $rpl_connection_name= server2
217 --source include/rpl_connection.inc
218
219 --source include/kill_and_restart_mysqld.inc
220
221 --let $rpl_connection_name= server_2
222 --source include/rpl_connection.inc
223 --source include/wait_until_connected_again.inc
224 --disable_reconnect
225
226 --let $rpl_connection_name= server1
227 --source include/rpl_connection.inc
228
229 # wait for server2 to become unreachable
230 # at this point we have lost quorum
231 --let $group_replication_member_state= UNREACHABLE
232 --let $group_replication_member_id= $member2_uuid
233 --source include/gr_wait_for_member_state.inc
234
235 # assert that notification is triggered
236 --let $expected_notifications= 1,QUORUM LOST: %:4|1,STATE %:4
237 --source ../include/assert_notifications.inc
238
239 --echo ### Scenario 9: MAJORITY UNREACHABLE + FORCE MEMBERS
240 --echo ### Expected:
241 --echo ### - Correct Servers: 1 VIEW CHANGED
242 --echo ### - Failed Servers: N/A
243 --echo ##########################################################
244
245 --let $rpl_connection_name= server_1
246 --source include/rpl_connection.inc
247
248 # Unblock the group by reconfiguring it forcefully to a
249 # single server.
250
251 --disable_query_log
252 --eval SET GLOBAL group_replication_force_members= "$local_address_server1"
253 --enable_query_log
254
255 --let $assert_text= group_replication_force_members is correct
256 --let $assert_cond= @@GLOBAL.group_replication_force_members = "$local_address_server1"
257 --source include/assert.inc
258
259 # verify that the table has now only 1 member
260 --let $group_replication_number_of_members= 1
261 --source include/gr_wait_for_number_of_members.inc
262
263 --disable_query_log
264 SET GLOBAL group_replication_force_members= "";
265 --enable_query_log
266
267 # no more unreachable entries in the table
268 --let $res= `SELECT COUNT(*)=0 FROM performance_schema.replication_group_members WHERE member_state="UNREACHABLE"`
269 --let $assert_text= No more unreachable servers in the P_S table.
270 --let $assert_cond= $res = 1
271 --source include/assert.inc
272
273 # assert that notification is triggered
274 --let $expected_notifications= 1,VIEW %:5
275 --source ../include/assert_notifications.inc
276
277 --source include/stop_group_replication.inc
278
279 --echo ### Scenario 10: MAJORITY UNREACHABLE + STOP
280 --echo ### Expected:
281 --echo ### - Correct Servers: 1 VIEW CHANGED, 1 STATE CHANGED
282 --echo ### - Failed Servers: N/A
283 --echo ##########################################################
284
285 # start1: start group
286 --let $rpl_connection_name= server1
287 --source include/rpl_connection.inc
288
289 --disable_query_log
290 RESET MASTER;
291 --eval SET @@global.group_replication_group_seeds="$group_seeds_server1"
292 --eval SET @@global.group_replication_local_address="$local_address_server1"
293 --eval SET @@global.group_replication_group_name="$group_replication_group_name"
294 --enable_query_log
295
296 --source include/start_and_bootstrap_group_replication.inc
297
298 # server2
299
300 --let $rpl_connection_name= server2
301 --source include/rpl_connection.inc
302
303 --disable_query_log
304 RESET MASTER;
305 --eval SET @@global.group_replication_group_seeds="$group_seeds_server2"
306 --eval SET @@global.group_replication_local_address="$local_address_server2"
307 --eval SET @@global.group_replication_group_name="$group_replication_group_name"
308 --enable_query_log
309
310 --source include/start_group_replication.inc
311
312 # server3: Do not start server3. We don't need it to create the quorum loss.
313
314 # server2
315
316 --let $rpl_connection_name= server_2
317 --source include/rpl_connection.inc
318 --enable_reconnect
319
320 --let $rpl_connection_name= server2
321 --source include/rpl_connection.inc
322 --source include/kill_and_restart_mysqld.inc
323
324 --let $rpl_connection_name= server_2
325 --source include/rpl_connection.inc
326 --source include/wait_until_connected_again.inc
327 --disable_reconnect
328
329 # server1
330
331 --let $rpl_connection_name= server1
332 --source include/rpl_connection.inc
333
334 # wait for server2 to become unreachable.
335 # at this point we have lost quorum
336 --let $group_replication_member_state= UNREACHABLE
337 --let $group_replication_member_id= $member2_uuid
338 --source include/gr_wait_for_member_state.inc
339
340 # wait for 4 group change broadcast.
341 # 1. View_id change(member2 join).
342 # 2. Recovery state change.
343 # 3. Unreachable state change.
344 # 4. Quorum lost.
345 --let $wait_condition= SELECT COUNT(*)=4 FROM test.gms_listener_example WHERE log_message LIKE 'QUORUM LOST: %:2' OR log_message LIKE 'STATE CHANGED: %:2';
346 --source include/wait_condition.inc
347
348 --disable_query_log
349 SET SQL_LOG_BIN=0;
350 TRUNCATE gms_listener_example;
351 SET SQL_LOG_BIN=1;
352 --enable_query_log
353
354 --source include/stop_group_replication.inc
355
356 # assert that notification is triggered
357 --let $expected_notifications= 1,VIEW %: |1,STATE %:%
358 --source ../include/assert_notifications.inc
359
360 ###
361 ### Clean up and bail out
362 ###
363
364 # server1
365
366 --let $rpl_connection_name= server1
367 --source include/rpl_connection.inc
368
369 --let $debug_point= register_gms_listener_example
370 --source include/remove_debug_point.inc
371
372 --disable_query_log
373 RESET MASTER;
374 --eval SET @@global.group_replication_group_seeds="$group_seeds_server1"
375 --eval SET @@global.group_replication_local_address="$local_address_server1"
376 --eval SET @@global.group_replication_group_name="$group_replication_group_name"
377 --enable_query_log
378
379 --source include/start_and_bootstrap_group_replication.inc
380
381 # server2
382
383 --let $rpl_connection_name= server2
384 --source include/rpl_connection.inc
385
386 --disable_query_log
387 RESET MASTER;
388 --eval SET @@global.group_replication_group_seeds="$group_seeds_server2"
389 --eval SET @@global.group_replication_local_address="$local_address_server2"
390 --eval SET @@global.group_replication_group_name="$group_replication_group_name"
391 --enable_query_log
392
393 --source include/start_group_replication.inc
394
395 # server3
396
397 --let $rpl_connection_name= server3
398 --source include/rpl_connection.inc
399
400 --disable_query_log
401 RESET MASTER;
402 --eval SET @@global.group_replication_group_seeds="$group_seeds_server3"
403 --eval SET @@global.group_replication_local_address="$local_address_server3"
404 --eval SET @@global.group_replication_group_name="$group_replication_group_name"
405 --enable_query_log
406
407 --source include/start_group_replication.inc
408
409 --let $rpl_connection_name= server1
410 --source include/rpl_connection.inc
411
412 DROP TABLE gms_listener_example;
413 --source include/rpl_sync.inc
414
415 --source include/group_replication_end.inc