Debian Codegraph @

mysql-test/suite/group_replication/t/gr_autorejoin_while_applier_locked.test

raw
1 ###############################################################################
2 #
3 # When a member has applier blocked it should not cause problems on rejoin,
4 # it shall fail on applier initialization.
5 #
6 # Test:
7 # 0. The test requires three servers
8 # 1. Create table t1 on server1 with binlog disabled and bootstrap the
9 # group
10 # 2. Create table t1 on server2 with binlog disabled and join to the
11 # group
12 # 3. Server3 will do three attempts of rejoin to validate that not block
13 # and reduce stop_component timeout to recovery module stop faster
14 # 4. Server3 disable binlog and suppress warnings, create table t1 and
15 # get pid. Pid will be used to expel member from group. Join group.
16 # 5. Enable super read only not to avoid problem set it after LOCK
17 # tables READ. Table locked will prevent server to finish recovery
18 # 6. Insert data on group
19 # 7. Wait until group_replication_applier channel waits on table t1
20 # read lock
21 # 8. Expel server3 from the group
22 # 9. Server3 should change is status to ERROR
23 # 10. Waiting for two auto-rejoin to happen, only the third attempt
24 # will be successful. First will fail due recovery fail to stop.
25 # Second fail to start applier module due channel being active
26 # 11. Wait that message about applier being blocked was logged
27 # 12. UNLOCK tables on server3
28 # 13. Confirm that group_replication_applier channel is stopped
29 # 14. Wait for member to be ONLINE after autorejoin third attempt succeed
30 # 15. Cleanup
31 #
32 ###############################################################################
33
34 --source include/linux.inc
35 --source include/big_test.inc
36 --source include/not_have_privilege_checks_user.inc
37 --source include/have_group_replication_plugin.inc
38 --let $rpl_server_count= 3
39 --let $rpl_skip_group_replication_start= 1
40 --source include/group_replication.inc
41
42 --echo
43 --echo # 1. Create table t1 on server1 with binlog disabled and bootstrap the
44 --echo # group
45
46 SET sql_log_bin=0;
47 CREATE TABLE t1 (c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY, c2 INT);
48 SET sql_log_bin=1;
49
50 --source include/start_and_bootstrap_group_replication.inc
51
52 --echo
53 --echo # 2. Create table t1 on server2 with binlog disabled and join to the
54 --echo # group
55
56 --let $rpl_connection_name= server2
57 --source include/rpl_connection.inc
58
59 SET sql_log_bin=0;
60 CREATE TABLE t1 (c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY, c2 INT);
61 SET sql_log_bin=1;
62
63 --source include/start_group_replication.inc
64
65 --echo
66 --echo # 3. Server3 will do three attempts of rejoin to validate that not block
67 --echo # and reduce stop_component timeout to recovery module stop faster
68
69 --let $rpl_connection_name= server3
70 --source include/rpl_connection.inc
71
72 SET @group_replication_autorejoin_tries_save = @@GLOBAL.group_replication_autorejoin_tries;
73 SET @group_replication_components_stop_timeout_save = @@GLOBAL.group_replication_components_stop_timeout;
74
75 SET GLOBAL group_replication_autorejoin_tries = 3;
76 SET GLOBAL group_replication_components_stop_timeout = 30;
77
78 --echo
79 --echo # 4. Server3 disable binlog and suppress warnings, create table t1 and
80 --echo # get pid. Pid will be used to expel member from group. Join group.
81
82 SET SESSION sql_log_bin= 0;
83
84 call mtr.add_suppression('On shutdown there was a timeout on the Group Replication applier termination.');
85 call mtr.add_suppression('Failed to stop the group replication applier thread.');
86 call mtr.add_suppression('Member was expelled from the group due to network failures, changing member status to ERROR.');
87 call mtr.add_suppression('The server was automatically set into read only mode after an error was detected.');
88 call mtr.add_suppression('The group_replication_applier channel is still running, most likely it is waiting for a database/table lock*');
89 call mtr.add_suppression('Unable to confirm whether the server has left the group or not.*');
90
91 CREATE TABLE pid_table(pid_no INT PRIMARY KEY);
92 --let $pid_file=`SELECT @@pid_file`
93 --replace_result $pid_file pid_file
94 --eval LOAD DATA LOCAL INFILE '$pid_file' INTO TABLE pid_table
95 --let $server_pid=`SELECT pid_no FROM pid_table`
96 DROP TABLE pid_table;
97
98 CREATE TABLE t1 (c1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY, c2 INT);
99 SET SESSION sql_log_bin= 1;
100
101 --let $member3_uuid = `SELECT @@GLOBAL.server_uuid`
102
103
104 --let $rpl_connection_name= server3
105 --source include/rpl_connection.inc
106
107 --source include/start_group_replication.inc
108
109 --echo
110 --echo # 5. Enable super read only not to avoid problem set it after LOCK
111 --echo # tables READ. Table locked will prevent server to finish recovery
112
113 --let $rpl_connection_name= server_3
114 --source include/rpl_connection.inc
115
116 SET GLOBAL super_read_only= 1;
117 FLUSH TABLES WITH READ LOCK;
118
119
120 --echo
121 --echo # 6. Insert data on group
122
123 --let $rpl_connection_name= server1
124 --source include/rpl_connection.inc
125
126 INSERT INTO test.t1 VALUES (1, 1);
127 INSERT INTO test.t1 VALUES (2, 1);
128
129
130 --echo
131 --echo # 7. Wait until group_replication_applier channel waits on table t1
132 --echo # read lock
133
134 --let $rpl_connection_name= server3
135 --source include/rpl_connection.inc
136 --let $wait_condition=SELECT COUNT(*)=1 FROM information_schema.processlist WHERE user='system user' AND state='Waiting for global read lock'
137 --source include/wait_condition.inc
138
139
140 --echo
141 --echo # 8. Expel server3 from the group
142
143 # Send signal SIGSTOP to server 3.
144 --exec kill -19 $server_pid
145
146 # Wait until the group settles on 2 members
147 --let $rpl_connection_name= server1
148 --source include/rpl_connection.inc
149 --let $group_replication_number_of_members= 2
150 --source include/gr_wait_for_number_of_members.inc
151
152 --let $rpl_connection_name= server2
153 --source include/rpl_connection.inc
154 --let $group_replication_number_of_members= 2
155 --source include/gr_wait_for_number_of_members.inc
156
157 # Send signal SIGCONT to server 3.
158 --let $rpl_connection_name= server3
159 --source include/rpl_connection.inc
160 --exec kill -18 $server_pid
161
162 --echo
163 --echo # 9. Server3 should change is status to ERROR
164
165 --let $group_replication_member_state= ERROR
166 --let $group_replication_member_id= $member3_uuid
167 --source include/gr_wait_for_member_state.inc
168
169 --echo
170 --echo # 10. Waiting for two auto-rejoin to happen, only the third attempt
171 --echo # will be successful. First will fail due recovery fail to stop.
172 --echo # Second fail to start applier module due channel being active
173
174 --let $wait_timeout= 600
175 --let $wait_condition= SELECT WORK_COMPLETED = 2 FROM performance_schema.events_stages_current WHERE EVENT_NAME LIKE '%auto-rejoin%'
176 --source include/wait_condition.inc
177
178 --let $wait_condition=SELECT COUNT(*)=1 FROM information_schema.processlist WHERE user='system user' AND state='Waiting for global read lock'
179 --source include/wait_condition.inc
180
181 --let $group_replication_member_state= ERROR
182 --let $group_replication_member_id= $member3_uuid
183 --source include/gr_wait_for_member_state.inc
184
185
186 --echo
187 --echo # 11. Wait that message about applier being blocked was logged
188
189 --let $wait_condition= SELECT COUNT(*)=1 FROM performance_schema.error_log WHERE error_code='MY-013701' AND data LIKE "%The group_replication_applier channel is still running, most likely it is waiting for a database/table lock, which is preventing the channel from stopping. Please check database/table locks, including the ones created by backup tools.%"
190 --source include/wait_condition.inc
191
192
193 --echo
194 --echo # 12. UNLOCK tables on server3
195
196 --let $rpl_connection_name= server_3
197 --source include/rpl_connection.inc
198 UNLOCK TABLES;
199
200 --echo
201 --echo # 13. Confirm that group_replication_applier channel is stopped
202
203 --let $wait_condition=SELECT COUNT(*)=1 FROM performance_schema.replication_applier_status WHERE channel_name='group_replication_applier' AND service_state='OFF'
204 --source include/wait_condition.inc
205
206 --echo
207 --echo # 14. Wait for member to be ONLINE after autorejoin third attempt succeed
208
209 --let $rpl_connection_name= server3
210 --source include/rpl_connection.inc
211 --let $wait_timeout= 600
212 --let $group_replication_member_state= ONLINE
213 --let $group_replication_member_id= $member3_uuid
214 --source include/gr_wait_for_member_state.inc
215
216 --echo
217 --echo # 15. Cleanup
218
219 DROP TABLE t1;
220 SET GLOBAl group_replication_autorejoin_tries = @group_replication_autorejoin_tries_save;
221 SET GLOBAl group_replication_components_stop_timeout = @group_replication_components_stop_timeout_save;
222
223 --source include/group_replication_end.inc