Debian Codegraph @

mysql-test/suite/group_replication/t/gr_autorejoin_while_recovery_locked.test

raw
1 ###############################################################################
2 #
3 # When a member has recovery blocked it should not cause problems on rejoin,
4 # it shall stop (or try too) recovery module and do another rejoin attempt.
5 #
6 # Test:
7 # 0. The test requires three servers
8 # 1. Create table t1 on server1 with binlog disabled and bootstrap the
9 # group
10 # 2. Create table t1 on server2 with binlog disabled and join to the
11 # group
12 # 3. Setup Server3.
13 # Set the attempts of rejoin to 3 so we validate the process wont
14 # block.
15 # Reduce stop_component timeout so recovery module stops faster.
16 # 4. Server3 disable binlog and supreess warnings, create table t1 and
17 # get pid. Pid will be used to expel member from group
18 # 5. Enable super read only, to avoid problem set it after FLUSH
19 # TABLES WITH READ LOCK. Table locked will prevent server from
20 # finishing recovery
21 # 6. Insert data on group
22 # 7. Join server3 to group, it will stay on recovery due transactions
23 # on group that it can't apply as the table is locked
24 # 8. Wait until group_replication_recovery channel waits on table t1
25 # commit lock
26 # 9. Expel server3 from the group
27 # 10. Server3 should change is status to ERROR
28 # 11. Waiting for two auto-rejoin attempts to happen, they will fail
29 # due to recovery failing to stop
30 # 12. After the beginning of the second attempt server3 will stay in
31 # ERROR state
32 # 13. group_replication_recovery channel still waits on table t1
33 # commit lock
34 # 14. UNLOCK tables on server3 and the join shall succeed
35 # 15. Server3 shall be ONLINE
36 # 16. Cleanup
37 #
38 ###############################################################################
39
40 --source include/linux.inc
41 --source include/big_test.inc
42 --source include/not_have_privilege_checks_user.inc
43 --source include/have_group_replication_plugin.inc
44 --let $rpl_server_count= 3
45 --let $rpl_skip_group_replication_start= 1
46 --source include/group_replication.inc
47
48 --echo
49 --echo # 1. Create table t1 on server1 with binlog disabled and bootstrap the
50 --echo # group
51
52 SET sql_log_bin=0;
53 CREATE TABLE t1 (c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY, c2 INT);
54 SET sql_log_bin=1;
55
56 --source include/start_and_bootstrap_group_replication.inc
57
58 --echo
59 --echo # 2. Create table t1 on server2 with binlog disabled and join to the
60 --echo # group
61
62 --let $rpl_connection_name= server2
63 --source include/rpl_connection.inc
64
65 SET sql_log_bin=0;
66 CREATE TABLE t1 (c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY, c2 INT);
67 SET sql_log_bin=1;
68
69 --source include/start_group_replication.inc
70
71 --echo
72 --echo # 3. Setup Server3.
73 --echo # Set the attempts of rejoin to 3 so we validate the process wont
74 --echo # block.
75 --echo # Reduce stop_component timeout so recovery module stops faster.
76
77 --let $rpl_connection_name= server3
78 --source include/rpl_connection.inc
79
80 SET @group_replication_autorejoin_tries_save = @@GLOBAL.group_replication_autorejoin_tries;
81 SET @group_replication_components_stop_timeout_save = @@GLOBAL.group_replication_components_stop_timeout;
82
83 SET GLOBAL group_replication_autorejoin_tries = 3;
84 SET GLOBAL group_replication_components_stop_timeout = 30;
85
86 --echo
87 --echo # 4. Server3 disable binlog and supreess warnings, create table t1 and
88 --echo # get pid. Pid will be used to expel member from group
89
90 SET SESSION sql_log_bin= 0;
91
92 call mtr.add_suppression('On shutdown there was a timeout on the Group Replication recovery module termination. Check the log for more details');
93 call mtr.add_suppression('Member was expelled from the group due to network failures, changing member status to ERROR.');
94 call mtr.add_suppression('The server was automatically set into read only mode after an error was detected.');
95 call mtr.add_suppression("Error when stopping the group replication incremental recovery's donor connection");
96 call mtr.add_suppression('Unable to confirm whether the server has left the group or not. Check performance_schema.replication_group_members to check group membership information.');
97
98 CREATE TABLE pid_table(pid_no INT PRIMARY KEY);
99 --let $pid_file=`SELECT @@pid_file`
100 --replace_result $pid_file pid_file
101 --eval LOAD DATA LOCAL INFILE '$pid_file' INTO TABLE pid_table
102 --let $server_pid=`SELECT pid_no FROM pid_table`
103 DROP TABLE pid_table;
104
105 CREATE TABLE t1 (c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY, c2 INT);
106 SET SESSION sql_log_bin= 1;
107
108 --let $member3_uuid = `SELECT @@GLOBAL.server_uuid`
109
110 --echo
111 --echo # 5. Enable super read only, to avoid problem set it after FLUSH
112 --echo # TABLES WITH READ LOCK. Table locked will prevent server from
113 --echo # finishing recovery
114
115 --let $rpl_connection_name= server_3
116 --source include/rpl_connection.inc
117 SET GLOBAL super_read_only= 1;
118 FLUSH TABLES WITH READ LOCK;
119
120 --echo
121 --echo # 6. Insert data on group
122
123 --let $rpl_connection_name= server1
124 --source include/rpl_connection.inc
125
126 INSERT INTO test.t1 VALUES (1, 1);
127 INSERT INTO test.t1 VALUES (2, 1);
128
129 --echo
130 --echo # 7. Join server3 to group, it will stay on recovery due transactions
131 --echo # on group that it can't apply as the table is locked
132
133 --let $rpl_connection_name= server3
134 --source include/rpl_connection.inc
135
136 --let $group_replication_start_member_state= RECOVERING
137 --source include/start_group_replication.inc
138
139 --echo
140 --echo # 8. Wait until group_replication_recovery channel waits on table t1
141 --echo # commit lock
142
143 --let $wait_condition=SELECT COUNT(*)=1 FROM information_schema.processlist WHERE user='system user' AND state='Waiting for commit lock'
144 --source include/wait_condition.inc
145
146 --echo
147 --echo # 9. Expel server3 from the group
148
149 # Send signal SIGSTOP to server 3.
150 --exec kill -19 $server_pid
151
152 # Wait until the group settles on 2 members
153 --let $rpl_connection_name= server1
154 --source include/rpl_connection.inc
155 --let $group_replication_number_of_members= 2
156 --source include/gr_wait_for_number_of_members.inc
157
158 --let $rpl_connection_name= server2
159 --source include/rpl_connection.inc
160 --let $group_replication_number_of_members= 2
161 --source include/gr_wait_for_number_of_members.inc
162
163 # Send signal SIGCONT to server 3.
164 --let $rpl_connection_name= server3
165 --source include/rpl_connection.inc
166 --exec kill -18 $server_pid
167
168 --echo
169 --echo # 10. Server3 should change is status to ERROR
170
171 --let $group_replication_member_state= ERROR
172 --let $group_replication_member_id= $member3_uuid
173 --source include/gr_wait_for_member_state.inc
174
175 --echo
176 --echo # 11. Waiting for two auto-rejoin attempts to happen, they will fail
177 --echo # due to recovery failing to stop
178
179 --let $wait_timeout= 600
180 --let $wait_condition= SELECT WORK_COMPLETED = 2 FROM performance_schema.events_stages_current WHERE EVENT_NAME LIKE '%auto-rejoin%'
181 --source include/wait_condition.inc
182
183 --echo
184 --echo # 12. After the beginning of the second attempt server3 will stay in
185 --echo # ERROR state
186
187 --let $group_replication_member_state= ERROR
188 --let $group_replication_member_id= $member3_uuid
189 --source include/gr_wait_for_member_state.inc
190
191 --echo
192 --echo # 13. group_replication_recovery channel still waits on table t1
193 --echo # commit lock
194
195 --let $wait_condition=SELECT COUNT(*)=1 FROM information_schema.processlist WHERE user='system user' AND state='Waiting for commit lock'
196 --source include/wait_condition.inc
197
198 --echo
199 --echo # 14. UNLOCK tables on server3 and the join shall succeed
200
201 --let $rpl_connection_name= server_3
202 --source include/rpl_connection.inc
203 UNLOCK TABLES;
204
205 --echo
206 --echo # 15. Server3 shall be ONLINE
207
208 --let $rpl_connection_name= server3
209 --source include/rpl_connection.inc
210 # Due to concurrency between the completeness of second auto-rejoin attempt
211 # and the UNLOCK TABLES, member can successfully rejoin on the second or third
212 # attempt, thence the big wait_timeout.
213 --let $wait_timeout= 600
214 --let $group_replication_member_state= ONLINE
215 --let $group_replication_member_id= $member3_uuid
216 --source include/gr_wait_for_member_state.inc
217
218 --echo
219 --echo # 16. Cleanup
220
221 DROP TABLE t1;
222
223 SET GLOBAl group_replication_autorejoin_tries = @group_replication_autorejoin_tries_save;
224 SET GLOBAl group_replication_components_stop_timeout = @group_replication_components_stop_timeout_save;
225
226 --source include/group_replication_end.inc