Debian Codegraph @

mysql-test/suite/group_replication/t/gr_suspect_member_resumes_after_expel.test

raw
1 #######################################################################################
2 # WL11570 - GR: options to defer member eviction after a suspicion
3 #
4 # In a group of 3 servers, we set the member_expel_timeout to 5 seconds and suspend
5 # one of them until it is expelled. We then resume the suspended server and verify
6 # that it enters an error state.
7 # Test :
8 # 0. This test requires a group with 3 servers
9 # 1. Set the group_replication_member_expel_timeout parameter to 5 seconds
10 # 2. Create table t1 and insert some data, checking that it is replicated to all
11 # servers.
12 # 3. Suspend server 3 by sending a signal SIGSTOP to it.
13 # This will make server 3 not answer to "I am alive" GCS messages and it will
14 # eventually be considered faulty.
15 # 4. Check that all members are still in the group on servers 1 and 2, which should
16 # both be ONLINE.
17 # Server 3 should still be in the group but UNREACHABLE.
18 # 5. After some time, server 3 should have been expelled from the group.
19 # 6. Insert data into table t1 on server 2 and check that it is replicated to server 1.
20 # 7. Resume server 3 by sending a signal SIGCONT to it.
21 # Server 3 should go into ERROR state.
22 # 8. Restart GR on server 3
23 # 9. Server 3 should be back in the group
24 # 10.Check that server 3 retrieves the data inserted by server 2 while it was
25 # suspended.
26 # Then, insert new data into table t1 which should be replicated to servers 1 and 2.
27 # 11.Clean up.
28 #######################################################################################
29 # Don't test this under valgrind, memory leaks will occur
30 --source include/not_valgrind.inc
31 --source include/linux.inc
32 --source include/big_test.inc
33 --source include/force_restart.inc
34 --source include/have_group_replication_plugin.inc
35 --let $rpl_server_count= 3
36 --source include/group_replication.inc
37
38 --echo
39 --echo ############################################################
40 --echo # 1. Set group_replication_member_expel_timeout to
41 --echo # 5 seconds.
42 --let $rpl_connection_name= server1
43 --source include/rpl_connection.inc
44 SET GLOBAL group_replication_member_expel_timeout = 5;
45 SELECT @@GLOBAL.group_replication_member_expel_timeout;
46
47 --let $rpl_connection_name= server2
48 --source include/rpl_connection.inc
49 SET GLOBAL group_replication_member_expel_timeout = 5;
50 SELECT @@GLOBAL.group_replication_member_expel_timeout;
51
52 --let $rpl_connection_name= server3
53 --source include/rpl_connection.inc
54 SET GLOBAL group_replication_member_expel_timeout = 5;
55 SELECT @@GLOBAL.group_replication_member_expel_timeout;
56 # Supress log errors since they are expected.
57 SET SESSION sql_log_bin = 0;
58 call mtr.add_suppression("Member was expelled from the group due to network failures, changing member status to ERROR.");
59 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
60 call mtr.add_suppression("\\[GCS\\] read failed");
61 call mtr.add_suppression("\\[GCS\\] Unable to resolve peer address");
62 SET SESSION sql_log_bin = 1;
63
64
65 --echo
66 --echo ############################################################
67 --echo # 2. Create a table and insert some data. Check that
68 --echo # it is replicated to all servers.
69 --let $rpl_connection_name= server1
70 --source include/rpl_connection.inc
71 CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY);
72 INSERT INTO t1 VALUES (1);
73 --source include/rpl_sync.inc
74
75
76 --echo
77 --echo ############################################################
78 --echo # 3. Suspend server 3 by sending signal SIGSTOP to it.
79 --echo # This will make server 3 not answer to "I am alive"
80 --echo # GCS messages and it will eventually be considered
81 --echo # faulty.
82 --let $rpl_connection_name= server3
83 --source include/rpl_connection.inc
84 --echo # Get server 3 pid.
85 SET SESSION sql_log_bin= 0;
86 CREATE TABLE pid_table(pid_no INT);
87 --let $pid_file= `SELECT @@GLOBAL.pid_file`
88 --replace_result $pid_file pid_file
89 --eval LOAD DATA LOCAL INFILE '$pid_file' INTO TABLE pid_table
90 --let $server_pid=`SELECT pid_no FROM pid_table`
91 DROP TABLE pid_table;
92 SET SESSION sql_log_bin= 1;
93
94 --echo # Suspending server 3...
95 --exec kill -19 $server_pid
96
97
98 --echo
99 --echo ############################################################
100 --echo # 4. Check that all members are still in the group on
101 --echo # servers 1 and 2, which should both be ONLINE.
102 --echo # Server 3 should still be in the group but UNREACHABLE.
103 --let $rpl_connection_name= server1
104 --source include/rpl_connection.inc
105 let $wait_condition=SELECT COUNT(*)=2 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
106 --source include/wait_condition.inc
107 let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.replication_group_members where MEMBER_STATE="UNREACHABLE";
108 --source include/wait_condition.inc
109
110
111 --let $rpl_connection_name= server2
112 --source include/rpl_connection.inc
113 let $wait_condition=SELECT COUNT(*)=2 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
114 --source include/wait_condition.inc
115 let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.replication_group_members where MEMBER_STATE="UNREACHABLE";
116 --source include/wait_condition.inc
117
118
119 --echo
120 --echo ############################################################
121 --echo # 5. After some time, server 3 should have been expelled
122 --echo # from the group.
123 --let $rpl_connection_name= server1
124 --source include/rpl_connection.inc
125 let $wait_condition=SELECT COUNT(*)=2 FROM performance_schema.replication_group_members;
126 --source include/wait_condition.inc
127
128 --let $rpl_connection_name= server2
129 --source include/rpl_connection.inc
130 let $wait_condition=SELECT COUNT(*)=2 FROM performance_schema.replication_group_members;
131 --source include/wait_condition.inc
132
133
134 --echo
135 --echo ############################################################
136 --echo # 6. Insert data into table t1 on server 2 and check that
137 --echo # it is replicated to server 1.
138 --let $rpl_connection_name= server2
139 --source include/rpl_connection.inc
140 INSERT INTO t1 VALUES (2);
141 let $wait_condition=SELECT COUNT(*)=2 FROM t1;
142 --source include/wait_condition.inc
143
144 --let $rpl_connection_name= server1
145 --source include/rpl_connection.inc
146 let $wait_condition=SELECT COUNT(*)=2 FROM t1;
147 --source include/wait_condition.inc
148
149 --echo
150 --echo ############################################################
151 --echo # 7. Resume server 3 by sending a signal SIGCONT to it.
152 --echo # Server 3 should change to ERROR state.
153 --let $rpl_connection_name= server3
154 --source include/rpl_connection.inc
155 --echo # Send signal SIGCONT to server 3.
156 --exec kill -18 $server_pid
157
158 --echo #
159 --echo # Check that the returning member enters the error state since it was
160 --echo # expelled from the group.
161 --let $rpl_connection_name= server3
162 --source include/rpl_connection.inc
163 let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.replication_group_members where MEMBER_STATE="ERROR";
164 --source include/wait_condition.inc
165
166 --echo #
167 --echo # Check that the 2 remaining members are alive and online in the group
168 --let $rpl_connection_name= server1
169 --source include/rpl_connection.inc
170 let $wait_condition=SELECT COUNT(*)=2 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
171 --source include/wait_condition.inc
172
173 --let $rpl_connection_name= server2
174 --source include/rpl_connection.inc
175 let $wait_condition=SELECT COUNT(*)=2 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
176 --source include/wait_condition.inc
177
178
179 --echo
180 --echo ############################################################
181 --echo # 8. Restart GR on server 3
182 --let $rpl_connection_name= server3
183 --source include/rpl_connection.inc
184 --source include/stop_group_replication.inc
185 --source include/start_group_replication.inc
186
187 --echo
188 --echo ############################################################
189 --echo # 9. Server 3 should be back in the group
190 --let $rpl_connection_name= server1
191 --source include/rpl_connection.inc
192 let $wait_condition=SELECT COUNT(*)=3 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
193 --source include/wait_condition.inc
194
195 --let $rpl_connection_name= server2
196 --source include/rpl_connection.inc
197 let $wait_condition=SELECT COUNT(*)=3 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
198 --source include/wait_condition.inc
199
200 --let $rpl_connection_name= server3
201 --source include/rpl_connection.inc
202 let $wait_condition=SELECT COUNT(*)=3 FROM performance_schema.replication_group_members where MEMBER_STATE="ONLINE";
203 --source include/wait_condition.inc
204
205
206 --echo
207 --echo ############################################################
208 --echo # 10.Check that server 3 retrieves the data inserted by
209 --echo # server 2 while it was suspended. Then, insert new
210 --echo # data into table t1 which should be replicated to
211 --echo # servers 1 and 2.
212 --let $rpl_connection_name= server3
213 --source include/rpl_connection.inc
214 let $wait_condition=SELECT COUNT(*)=2 FROM t1;
215 --source include/wait_condition.inc
216 INSERT INTO t1 VALUES (3);
217 --source include/rpl_sync.inc
218 --let $diff_tables=server1:t1, server2:t1, server3:t1
219 --source include/diff_tables.inc
220
221
222 --echo
223 --echo ############################################################
224 --echo # 11.Clean up.
225 DROP TABLE t1;
226
227 --source include/group_replication_end.inc