Debian Codegraph @

mysql-test/suite/group_replication/t/gr_recovery_fails_to_detect_thread_stop.test

raw
1 # ==== Purpose ====
2 #
3 # This test case will test if a member server will not fail to know that the
4 # replication threads of the group replication recovery have stopped.
5 #
6 # The applier thread test:
7 #
8 # During the startup of the applier thread of the group replication recovery
9 # on server2, debug points will ensure that the hook for getting the stop
10 # signal will only be set up after the applier thread has passed the hook
11 # notification point. This will ensure that, even missing the stop
12 # notification, the group replication recovery will still detect that the
13 # applier thread has stopped.
14 #
15 # The receiver thread test:
16 #
17 # The test case will turn the only possible donor into an improper
18 # one by purging its binary log. In this way, the server trying to recover
19 # will not succeed, restating the receiver thread for some time.
20 #
21 # During the restart of the receiver thread of the group replication recovery
22 # on server3, debug points will ensure that the hook for getting the stop
23 # signal will only be set up after the receiver thread has passed the hook
24 # notification point. This will ensure that, even missing the stop
25 # notification, the group replication recovery will still detect that the
26 # receiver thread has stopped.
27 #
28 # ==== Related Bugs and Worklogs ====
29 #
30 # BUG#23502642 TEST GROUP_REPLICATION_RECOVERY_PURGED_DONOR_FAILOVER FAILING
31 #
32 --source include/big_test.inc
33 --source include/have_debug.inc
34 --source include/have_debug_sync.inc
35 --source include/have_group_replication_plugin.inc
36 --let $rpl_skip_group_replication_start= 1
37 --let $rpl_server_count= 3
38 --source include/group_replication.inc
39
40 --echo
41 --echo #####################################################################
42 --echo # 1. Create a table on server1 and generate workload to be recovered
43
44 --source include/start_and_bootstrap_group_replication.inc
45
46 # Insert some data
47 CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB;
48 INSERT INTO t1 VALUES (1);
49
50 --echo
51 --echo ########################
52 --echo # 2. SQL thread testing
53
54 --echo
55 --echo #######################################################
56 --echo # 2.1 Try to make server2 to join the group for 3 times
57 --echo # forcing the SQL thread to stop
58
59 --let $rpl_connection_name= server2
60 --source include/rpl_connection.inc
61 SET SESSION sql_log_bin= 0;
62 CALL mtr.add_suppression("Error while creating the group replication recovery channel with donor");
63 CALL mtr.add_suppression("Maximum number of retries when trying to connect to a donor reached");
64 CALL mtr.add_suppression("Fatal error during the incremental recovery process of Group Replication. The server will leave the group.");
65 CALL mtr.add_suppression("Already leaving or joining a group");
66 CALL mtr.add_suppression("Error leaving the group");
67 CALL mtr.add_suppression("Table 't1' already exists");
68 CALL mtr.add_suppression("Error while starting the group replication incremental recovery receiver/applier threads");
69 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
70 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
71 SET SESSION sql_log_bin= 1;
72
73 SET @saved_reconnect_interval= @@GLOBAL.group_replication_recovery_reconnect_interval;
74 --eval SET GLOBAL group_replication_recovery_reconnect_interval= 1 # second
75 SET @saved_retry_count= @@GLOBAL.group_replication_recovery_retry_count;
76 --eval SET GLOBAL group_replication_recovery_retry_count= 3
77
78 # This is to make the SQL thread to fail
79 SET SESSION sql_log_bin= 0;
80 CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB;
81 SET SESSION sql_log_bin= 1;
82 --let $debug_point=pause_after_sql_thread_stop_hook
83 --source include/add_debug_point.inc
84
85 --echo
86 --echo #####################################################
87 --echo # 2.2 Wait until server2 gave up on joining the group
88
89 --let $group_replication_start_member_state= ERROR
90 --source include/start_group_replication.inc
91
92 --source include/remove_debug_point.inc
93
94 SET @@GLOBAL.group_replication_recovery_reconnect_interval= @saved_reconnect_interval;
95 SET @@GLOBAL.group_replication_recovery_retry_count= @saved_retry_count;
96
97 # Wait until server2 has stopped Group Replication
98 --source include/stop_group_replication.inc
99
100 SET SESSION sql_log_bin= 0;
101 DROP TABLE t1;
102 SET SESSION sql_log_bin= 1;
103
104 --echo
105 --echo #####################################################################
106 --echo # 2.3 Wait until server2 had left the group to move to the next phase
107
108 --connection server1
109 --let $group_replication_number_of_members= 1
110 --source include/gr_wait_for_number_of_members.inc
111
112 --echo
113 --echo #######################
114 --echo # 3. I/O thread testing
115
116 --echo
117 --echo ############################################################################
118 --echo # 3.1 Rotate and purge the binary log on server1 to make recovery unfeasible
119
120 FLUSH BINARY LOGS;
121 --let $server_binlog_file_cur= query_get_value(SHOW MASTER STATUS, File, 1)
122
123 # Need to kill the dump thread explicitly: stop slave does not do
124 # that, and if the dump thread is slow, it will still be reading from
125 # the old binlog at the time we do PURGE BINARY LOGS below. That would
126 # prevent PURGE BINARY LOGS from purging the log.
127 --source include/stop_dump_threads.inc
128
129 --replace_result $server_binlog_file_cur CURRENT_BINLOG_FILE
130 --eval PURGE BINARY LOGS TO '$server_binlog_file_cur'
131
132 SET SESSION sql_log_bin= 0;
133 CALL mtr.add_suppression("Cannot replicate to server with server_uuid*");
134 SET SESSION sql_log_bin= 1;
135
136 --echo
137 --echo #############################################################
138 --echo # 3.2 Try to make server3 to join the group for up to 3 times
139
140 --connection server3
141 SET SESSION sql_log_bin= 0;
142 CALL mtr.add_suppression("Replica I/O for channel 'group_replication_recovery': Got fatal error 1236*");
143 CALL mtr.add_suppression("Maximum number of retries when trying to connect to a donor reached");
144 CALL mtr.add_suppression("Fatal error during the incremental recovery process of Group Replication. The server will leave the group.");
145 CALL mtr.add_suppression("Already leaving or joining a group");
146 CALL mtr.add_suppression("Error leaving the group");
147 CALL mtr.add_suppression("Error while starting the group replication incremental recovery");
148 call mtr.add_suppression("Skipping leave operation: concurrent attempt to leave the group is on-going.");
149 call mtr.add_suppression("This member will start distributed recovery using clone. It is due to no ONLINE member has the missing data for recovering in its binary logs.");
150 call mtr.add_suppression("There was an issue when configuring the remote cloning process: The clone plugin is not present or active in this server.");
151 call mtr.add_suppression("No valid or ONLINE members exist to get the missing data from the group. For cloning check if donors of the same version and with clone plugin installed exist. For incremental recovery check.*");
152 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
153 SET SESSION sql_log_bin= 1;
154
155 SET @saved_reconnect_interval= @@GLOBAL.group_replication_recovery_reconnect_interval;
156 --eval SET GLOBAL group_replication_recovery_reconnect_interval= 1 # second
157 SET @saved_retry_count= @@GLOBAL.group_replication_recovery_retry_count;
158 --eval SET GLOBAL group_replication_recovery_retry_count= 3
159
160 --let $debug_point=pause_after_io_thread_stop_hook
161 --source include/add_debug_point.inc
162
163 --echo
164 --echo #####################################################
165 --echo # 3.3 Wait until server3 gave up on joining the group
166
167 --let $group_replication_start_member_state= ERROR
168 --source include/start_group_replication.inc
169
170 SET @@GLOBAL.group_replication_recovery_reconnect_interval= @saved_reconnect_interval;
171 SET @@GLOBAL.group_replication_recovery_retry_count= @saved_retry_count;
172
173 --source include/remove_debug_point.inc
174
175 --echo
176 --echo ################################################################
177 --echo # 4. Wait until server1 be alone in the group to do the clean up
178
179 --connection server1
180 --let $group_replication_number_of_members= 1
181 --source include/gr_wait_for_number_of_members.inc
182
183 DROP TABLE t1;
184
185 --source include/group_replication_end.inc