Debian Codegraph @

mysql-test/suite/group_replication/include/gr_suspect_member_resumes_after_crash.inc

raw
1 #######################################################################################
2 # WL11570 - GR: options to defer member eviction after a suspicion
3 #
4 # In a group of 3 servers, we crash one of them, restart it and make it join the group
5 # again, which fails. When it finally is expelled from the group, as its suspicion
6 # on other members times out, it is finally able to rejoin the group.
7 # Test :
8 # 0. This test requires a group with 3 servers.
9 # 1. Set the group_replication_member_expel_timeout parameter to 3600 seconds.
10 # 2. Create table t1 and insert some data, checking that it is replicated to all
11 # servers.
12 # 3. Crash server 3 and it will restart.
13 # This will make server 3 not answer to "I am alive" GCS messages and it will
14 # eventually be considered faulty.
15 # 4. Check that all members are still in the group on servers 1 and 2, which should
16 # both be ONLINE.
17 # Server 3 should still be in the group but UNREACHABLE.
18 # 5. Insert data into table t1 on server 2 and check that it is replicated to server 1.
19 # 6. Restart GR on server 3 and it should fail since its
20 # suspicion still hasn't timed out on other members.
21 # 7. Wait until server 3 is expelled from the group.
22 # 8. Server 3 was expelled! Start GR on server 3.
23 # 9. Check that server 3 retrieves the data inserted while
24 # it was suspended. Then, use it to insert new data into
25 # table t1 which should be replicated to servers 1 and 2.
26 # 10. Clean up.
27 #######################################################################################
28
29 # Don't test this under valgrind, memory leaks will occur
30 --source include/not_valgrind.inc
31 --source include/big_test.inc
32 --source include/force_restart.inc
33 --source include/have_group_replication_plugin.inc
34 --let $rpl_server_count= 3
35 --source include/group_replication.inc
36
37
38 --echo
39 --echo ############################################################
40 --echo # 1. Set group_replication_member_expel_timeout to
41 --echo # 3600 seconds.
42 --let $rpl_connection_name= server1
43 --source include/rpl_connection.inc
44 SET GLOBAL group_replication_member_expel_timeout = 3600;
45 --let $server1_local_address= `SELECT @@GLOBAL.group_replication_local_address`
46
47 set session sql_log_bin=0;
48 call mtr.add_suppression("Old incarnation found while trying to add node*.*");
49 set session sql_log_bin=1;
50
51 --let $rpl_connection_name= server2
52 --source include/rpl_connection.inc
53 SET GLOBAL group_replication_member_expel_timeout = 3600;
54
55 --let $rpl_connection_name= server3
56 --source include/rpl_connection.inc
57 SET GLOBAL group_replication_member_expel_timeout = 3600;
58 set session sql_log_bin=0;
59 call mtr.add_suppression("read failed");
60 call mtr.add_suppression("The member was unable to join the group. Local port: *.*");
61 call mtr.add_suppression("Timeout on wait for view after joining group");
62 call mtr.add_suppression("Error connecting to all peers. Member join failed. Local port: *.*");
63 call mtr.add_suppression("Unable to start MySQL Network Provider*.*");
64 call mtr.add_suppression("Timeout while waiting for the group communication engine to be ready*.*");
65 call mtr.add_suppression("The group communication engine is not ready for the member to join*.*");
66
67 set session sql_log_bin=1;
68
69
70 --echo
71 --echo ############################################################
72 --echo # 2. Create a table and insert some data. Check that
73 --echo # it is replicated to all servers.
74 --let $rpl_connection_name= server1
75 --source include/rpl_connection.inc
76 CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY);
77 INSERT INTO t1 VALUES (1);
78 --source include/rpl_sync.inc
79
80
81 --echo
82 --echo ############################################################
83 --echo # 3. Crash server 3 and it will restart.
84 --let $rpl_connection_name= server3
85 --source include/rpl_connection.inc
86 --let $group_replication_local_address= `SELECT @@GLOBAL.group_replication_local_address`
87 --let $group_replication_group_seeds= `SELECT @@GLOBAL.group_replication_group_seeds`
88 --let $restart_parameters=restart:--group_replication_local_address=$group_replication_local_address --group_replication_group_seeds=$group_replication_group_seeds --group_replication_group_name=$group_replication_group_name
89 --replace_result $group_replication_local_address GROUP_REPLICATION_LOCAL_ADDRESS $group_replication_group_seeds GROUP_REPLICATION_GROUP_SEEDS $group_replication_group_name GROUP_REPLICATION_GROUP_NAME
90 --source include/kill_and_restart_mysqld.inc
91
92 --echo #
93 --echo # Restarting server 3
94 # Needed as we are not using rpl_restart_server.inc
95 --let $rpl_server_number= 3
96 --source include/rpl_reconnect.inc
97
98
99 --echo
100 --echo ############################################################
101 --echo # 4. Check that all members are still in the group on
102 --echo # servers 1 and 2, which should both be ONLINE.
103 --echo # Server 3 should still be in the group but UNREACHABLE.
104 --let $rpl_connection_name= server1
105 --source include/rpl_connection.inc
106 let $wait_condition=SELECT COUNT(*)=2 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
107 --source include/wait_condition.inc
108 let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.replication_group_members where MEMBER_STATE="UNREACHABLE";
109 --source include/wait_condition.inc
110
111 --let $rpl_connection_name= server2
112 --source include/rpl_connection.inc
113 let $wait_condition=SELECT COUNT(*)=2 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
114 --source include/wait_condition.inc
115 let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.replication_group_members where MEMBER_STATE="UNREACHABLE";
116 --source include/wait_condition.inc
117
118
119 --echo
120 --echo ############################################################
121 --echo # 5. Insert data into table t1 on server 2 and check that
122 --echo # it is replicated to server 1.
123 INSERT INTO t1 VALUES (2);
124 let $wait_condition=SELECT COUNT(*)=2 FROM t1;
125 --source include/wait_condition.inc
126
127 --let $rpl_connection_name= server1
128 --source include/rpl_connection.inc
129 let $wait_condition=SELECT COUNT(*)=2 FROM t1;
130 --source include/wait_condition.inc
131
132
133 --echo
134 --echo ############################################################
135 --echo # 6. Restart GR on server 3 and it should fail since its
136 --echo # suspicion still hasn't timed out on other members.
137 --let $rpl_connection_name= server3
138 --source include/rpl_connection.inc
139 --replace_result $group_replication_group_name GROUP_REPLICATION_GROUP_NAME
140 --eval SET GLOBAL group_replication_group_name= "$group_replication_group_name"
141 --replace_result $server1_local_address SERVER1_LOCAL_ADDRESS
142 --eval SET GLOBAL group_replication_group_seeds= "$server1_local_address"
143 --error ER_GROUP_REPLICATION_CONFIGURATION
144 START GROUP_REPLICATION;
145
146
147 --echo
148 --echo ############################################################
149 --echo # 7. Wait until server 3 is expelled from the group.
150 --let $rpl_connection_name= server1
151 --source include/rpl_connection.inc
152 SET GLOBAL group_replication_member_expel_timeout = DEFAULT;
153 --let $rpl_connection_name= server2
154 --source include/rpl_connection.inc
155 SET GLOBAL group_replication_member_expel_timeout = DEFAULT;
156
157 --let $rpl_connection_name= server1
158 --source include/rpl_connection.inc
159 let $wait_condition=SELECT COUNT(*)=2 FROM performance_schema.replication_group_members;
160 --source include/wait_condition.inc
161
162 --let $rpl_connection_name= server2
163 --source include/rpl_connection.inc
164 let $wait_condition=SELECT COUNT(*)=2 FROM performance_schema.replication_group_members;
165 --source include/wait_condition.inc
166
167
168 --echo
169 --echo ############################################################
170 --echo # 8. Server 3 was expelled! Start GR on server 3.
171 --let $rpl_connection_name= server3
172 --source include/rpl_connection.inc
173 --source include/start_group_replication.inc
174 SET GLOBAL group_replication_member_expel_timeout = DEFAULT;
175
176
177 --let $rpl_connection_name= server1
178 --source include/rpl_connection.inc
179 let $wait_condition=SELECT COUNT(*)=3 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
180 --source include/wait_condition.inc
181
182 --let $rpl_connection_name= server2
183 --source include/rpl_connection.inc
184 let $wait_condition=SELECT COUNT(*)=3 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
185 --source include/wait_condition.inc
186
187 --let $rpl_connection_name= server3
188 --source include/rpl_connection.inc
189 let $wait_condition=SELECT COUNT(*)=3 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
190 --source include/wait_condition.inc
191
192
193 --echo
194 --echo ############################################################
195 --echo # 9. Check that server 3 retrieves the data inserted while
196 --echo # it was suspended. Then, use it to insert new data into
197 --echo # table t1 which should be replicated to servers 1 and 2.
198 --let $rpl_connection_name= server3
199 --source include/rpl_connection.inc
200 let $wait_condition=SELECT COUNT(*)=2 FROM t1;
201 --source include/wait_condition.inc
202 INSERT INTO t1 VALUES (3);
203 --source include/rpl_sync.inc
204 --let $diff_tables=server1:t1, server2:t1, server3:t1
205 --source include/diff_tables.inc
206
207
208 --echo
209 --echo ############################################################
210 --echo # 10. Clean up.
211 DROP TABLE t1;
212
213 --source include/group_replication_end.inc