Debian Codegraph @

mysql-test/suite/group_replication/t/gr_autorejoin_failure_during_mid_join.test

raw
1 ################################################################################
2 #
3 # The auto-rejoin process is a mechanism that allows a server that left the
4 # group due to flaky network (either on his side or on the other members) to
5 # try to join again up to group_replication_autorejoin_tries number of times.
6 #
7 # This test verifies that if view changes modification takes time then state
8 # of the member is reset to ERROR state.
9 # Description : Auto rejoin starts a new view and member moves to RECOVERING
10 # and then ONLINE state.
11 # It is possible that on a slow machine member will move to RECOVERING/ONLINE
12 # and view change time will expire. In such scenario auto rejoin threads
13 # initiate member leave, which is in parallel/post to join.
14 # When member leaves old state of RECOVERING/ONLINE should be reset to ERROR
15 # since member is leaving the group.
16 #
17 # Test:
18 # 0. The test requires three servers.
19 # 1. Start 3 servers.
20 # 2. Enable auto-rejoin and expel one of the members.
21 # 3. Make sure view-change initiated by auto-rejoin has changed member
22 # state out of ERROR.
23 # Unblock wait_for_view_modification to fail during auto rejoin.
24 # 4. Verify that the expelled member will try to rejoin the group.
25 # 5. Assert member state is set to ERROR on view change failure.
26 # 6. Cleanup.
27 ################################################################################
28 --source include/have_debug.inc
29 --source include/big_test.inc
30 --source include/linux.inc
31 --source include/have_group_replication_plugin.inc
32 --let $rpl_skip_group_replication_start= 1
33 --let $rpl_server_count = 3
34 --source include/group_replication.inc
35
36 --echo
37 --echo # 1. Start 3 servers.
38
39 SET sql_log_bin = 0;
40 call mtr.add_suppression("Member was expelled from the group due to network failures, changing member status to ERROR.");
41 call mtr.add_suppression("Started auto-rejoin procedure attempt*");
42 call mtr.add_suppression("Auto-rejoin procedure attempt*");
43 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
44 call mtr.add_suppression("Unable to confirm whether the server has left the group or not. Check performance_schema.replication_group_members to check group membership information.");
45 call mtr.add_suppression("Timeout while waiting for a view change event during the leave step before a auto-rejoin attempt.");
46 call mtr.add_suppression("Timeout while waiting for a view change event during the auto-rejoin procedure");
47 call mtr.add_suppression("Error while sending message in the group replication incremental recovery process.");
48 SET sql_log_bin = 1;
49
50 --source include/gr_autorejoin_monitoring.inc
51 --let $server1_uuid= `SELECT @@GLOBAL.SERVER_UUID`
52 SET @debug_saved = @@GLOBAL.DEBUG;
53 SET @group_replication_autorejoin_tries_saved = @@GLOBAL.group_replication_autorejoin_tries;
54
55 --source include/start_and_bootstrap_group_replication.inc
56
57 --let $rpl_connection_name = server2
58 --source include/rpl_connection.inc
59
60 --source include/start_group_replication.inc
61
62 --let $rpl_connection_name = server3
63 --source include/rpl_connection.inc
64
65 --source include/start_group_replication.inc
66
67 --echo
68 --echo # 2. Enable auto-rejoin and expel one of the members.
69
70 --let $rpl_connection_name = server1
71 --source include/rpl_connection.inc
72
73 SET GLOBAL group_replication_autorejoin_tries = 1;
74 SET @@GLOBAL.DEBUG='+d,group_replication_skip_wait_for_view_modification';
75 SET @@GLOBAL.DEBUG='+d,group_replication_force_view_modification_timeout';
76 SET @@GLOBAL.DEBUG='+d,group_replication_autorejoin_allow_join_to_change_state';
77
78 # Force expel on member 1
79 --let $member_id = `SELECT @@GLOBAL.server_uuid`
80 --source include/gr_expel_member_from_group.inc
81
82 --echo
83 --echo # 3. Make sure view-change initiated by auto-rejoin has changed member
84 --echo # state out of ERROR.
85 --echo # Unblock wait_for_view_modification to fail during auto rejoin.
86
87 --let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.replication_group_members WHERE member_id='$server1_uuid' AND MEMBER_STATE!="ERROR";
88 --source include/wait_condition_or_abort.inc
89
90 SET DEBUG_SYNC= "now SIGNAL signal.group_replication_autorejoin_allow_join_to_change_state_resume";
91
92 --echo
93 --echo # 4. Verify that the expelled member will try to rejoin the group.
94
95 --let $wait_condition = SELECT IS_AUTOREJOIN_RUNNING() = FALSE
96 --source include/wait_condition_or_abort.inc
97
98 --let $wait_condition= SELECT COUNT(*)>=1 FROM performance_schema.error_log WHERE error_code='MY-013374' AND data LIKE '%Timeout while waiting for a view change event during the auto-rejoin procedure%'
99 --source include/wait_condition_or_abort.inc
100
101 --let $wait_condition= SELECT COUNT(*)>=1 FROM performance_schema.error_log WHERE error_code='MY-013375' AND data LIKE '%Auto-rejoin procedure attempt 1 of 1 finished. Member was not able to join the group.%'
102 --source include/wait_condition_or_abort.inc
103
104 --echo
105 --echo # 5. Assert member state is set to ERROR on view change failure.
106
107 # Should be in ERROR state
108 --let $group_replication_member_state= ERROR
109 --let $group_replication_member_id= $member_id
110 --source include/gr_wait_for_member_state.inc
111
112 # Verify that super_read_only is enabled
113 --source include/assert_and_disable_read_only.inc
114
115 --echo
116 --echo # 6. Cleanup.
117
118 --let $rpl_connection_name = server1
119 --source include/rpl_connection.inc
120
121 SET @@GLOBAL.DEBUG = @debug_saved;
122
123 --source include/stop_group_replication.inc
124 --source include/start_group_replication.inc
125
126 SET @@GLOBAL.group_replication_autorejoin_tries = @group_replication_autorejoin_tries_saved;
127
128 --source include/gr_end_autorejoin_monitoring.inc
129 --source include/group_replication_end.inc