Debian Codegraph @

mysql-test/suite/group_replication/t/gr_autorejoin_monitoring_test.test

raw
1 ################################################################################
2 #
3 # The auto-rejoin process is a mechanism that allows a server that left the
4 # group due to flaky network (either on his side or on the other members) to
5 # try to join again up to group_replication_autorejoin_tries number of times.
6 #
7 # This test shall verify that the auto-rejoin process is monitorable through
8 # thread stage events, i.e. it will verify that it is possible to:
9 # * Observe that an auto-rejoin is running or not
10 # * See the number of retries being attempted
11 # * Infer the time remaining for the next retry
12 # * See the timestamp of the last retry
13 # * See the number of times the auto-rejoin process was triggered
14 #
15 # Test:
16 # 0) The test requires three servers.
17 # 1) Provoke a member expel.
18 # 2) Verify that the auto-rejoin process is ongoing.
19 # 2.1) Verify that we can see the number of retries being incremented until
20 # depleted.
21 # 2.2) Verify that we can infer the time remaining until the next retry.
22 # 3) Verify that we see the timestamp of the last auto-rejoin process.
23 # 4) Verify that we can see the number of times the auto-rejoin process was
24 # run.
25 # 5) Cleanup.
26 #
27 ################################################################################
28 --source include/have_debug_sync.inc
29 --source include/big_test.inc
30 --source include/linux.inc
31 #
32 # We'll need to verify the remaining time between each retry and, with valgrind,
33 # we may easily go over our estimated time, so we have to disable this test
34 # from running in valgrind jobs.
35 #
36 --source include/not_valgrind.inc
37 #
38 # We manually start the group because we need to set the rejoin timeout before
39 # the START GROUP_REPLICATION command.
40 #
41 --source include/have_group_replication_plugin.inc
42 --let $rpl_skip_group_replication_start= 1
43 --let $rpl_server_count = 3
44 --source include/group_replication.inc
45
46 --echo
47 --echo ####
48 --echo # 0) The test requires three servers.
49 --echo ####
50 --echo
51 --let $rpl_connection_name = server1
52 --source include/rpl_connection.inc
53
54 SET sql_log_bin = 0;
55 call mtr.add_suppression("Member was expelled from the group due to network failures, changing member status to ERROR.");
56 call mtr.add_suppression("The server was automatically set into read only mode after an error was detected.");
57 call mtr.add_suppression("\\[GCS\\] read failed");
58 call mtr.add_suppression("Started auto-rejoin procedure attempt*");
59 call mtr.add_suppression("Auto-rejoin procedure attempt*");
60 call mtr.add_suppression("Unable to confirm whether the server has left the group or not. Check performance_schema.replication_group_members to check group membership information.");
61 SET sql_log_bin = 1;
62
63 --source include/gr_autorejoin_monitoring.inc
64
65 SET @debug_saved = @@GLOBAL.DEBUG;
66 SET @@GLOBAL.DEBUG='+d,group_replication_rejoin_long_retry';
67 SET @@GLOBAL.DEBUG='+d,group_replication_fail_rejoin';
68 SET @@GLOBAL.DEBUG='+d,group_replication_stop_before_rejoin_loop';
69 SET @@GLOBAL.DEBUG='+d,group_replication_stop_before_rejoin';
70
71 --source include/start_and_bootstrap_group_replication.inc
72
73 --let $rpl_connection_name = server2
74 --source include/rpl_connection.inc
75
76 --source include/start_group_replication.inc
77
78 --let $rpl_connection_name = server3
79 --source include/rpl_connection.inc
80
81 --source include/start_group_replication.inc
82
83 --echo
84 --echo ####
85 --echo # 1) Provoke a member expel.
86 --echo ####
87 --echo
88 --let $rpl_connection_name = server1
89 --source include/rpl_connection.inc
90
91 # First, enable auto-rejoin
92 SET GLOBAL group_replication_autorejoin_tries = 3;
93
94 #
95 # Save timestamp before we start the auto-rejoin process, so we can validate the
96 # time it took.
97 #
98 --let $timestamp_before_autorejoin = `SELECT UNIX_TIMESTAMP()`
99
100 # Force expel on member 1
101 --let $member_id = `SELECT @@GLOBAL.server_uuid`
102 --source include/gr_expel_member_from_group.inc
103
104 --echo
105 --echo ####
106 --echo # 2) Verify that the auto-rejoin process is ongoing.
107 --echo ####
108 --echo
109 SET DEBUG_SYNC = "now WAIT_FOR signal.autorejoin_entering_loop";
110
111 # Verify that it is currently running
112 --let $assert_text = Auto-rejoin should be running
113 --let $assert_cond = [SELECT IS_AUTOREJOIN_RUNNING()] = TRUE
114 --source include/assert.inc
115
116 SET DEBUG_SYNC = "now SIGNAL signal.autorejoin_enter_loop";
117
118 #
119 # We go through each retry to observe the retry counter and remaining time being
120 # updated
121 #
122 --let $num_retries = 1
123 while ($num_retries <= 3)
124 {
125 SET DEBUG_SYNC = "now WAIT_FOR signal.autorejoin_waiting";
126
127 # Verify that it is currently running
128 --let $assert_text = Auto-rejoin should be running
129 --let $assert_cond = [SELECT IS_AUTOREJOIN_RUNNING()] = TRUE
130 --source include/assert.inc
131
132
133 --let $assert_text= super_read_only should be enabled
134 --let $assert_cond= [SELECT @@GLOBAL.super_read_only] = 1;
135 --source include/assert.inc
136
137 --echo
138 --echo ####
139 --echo # 2.1) Verify that we can see the number of retries being incremented
140 --echo # until depleted.
141 --echo ####
142 --echo
143 --let $assert_text = We should have attempted $num_retries rejoins
144 --let $assert_cond = [SELECT GET_NUMBER_RETRIES()] = $num_retries
145 --source include/assert.inc
146
147 --echo
148 --echo ####
149 --echo # 2.2) Verify that we can infer the time remaining until the next
150 --echo # retry.
151 --echo ####
152 --echo
153 # Basically the remaining time for the next retry should be under 5 minutes
154 # Note that even though in production we sleep for 5 minutes, for testing
155 # purposes we lower that number to 1 minute.
156 --let $assert_text = Time remaining should be less than 5 mins
157 --let $assert_cond = [SELECT GET_TIME_UNTIL_NEXT_RETRY(60)] <= 60
158 --source include/assert.inc
159
160 # Unblock rejoin loop
161 SET DEBUG_SYNC = "now SIGNAL signal.autorejoin_continue";
162
163 --inc $num_retries
164 }
165
166 # Verify that the auto-rejoin process has terminated
167 --let $wait_condition = SELECT IS_AUTOREJOIN_RUNNING() = FALSE
168 --source include/wait_condition_or_abort.inc
169
170 --echo
171 --echo ####
172 --echo # 3) Verify that we see the timestamp of the last auto-rejoin process.
173 --echo ####
174 --echo
175 --let $last_autorejoin_time = `SELECT GET_LAST_AUTOREJOIN()`
176
177 #
178 # We set an upper bound for execution time comprising of the number of retries
179 # times the sleep interval and add a small overhead factor
180 #
181 --let $num_retries = 3
182 --let $sleep_interval = 60
183 --let $overhead = $sleep_interval
184
185 #
186 # Then we calculate the time we are expecting the auto-rejoin process to run, so
187 # that we can validate if the auto-rejoin process execution time deviates from
188 # this expected interval.
189 #
190 --expr $expected_spent_time = $num_retries * $sleep_interval
191 --expr $expected_time_delta = $timestamp_before_autorejoin + $expected_spent_time
192 --expr $actual_time_delta = $last_autorejoin_time - $timestamp_before_autorejoin
193 --expr $max_time = $expected_time_delta + $overhead
194
195 # If it deviates too much (i.e. more than the overhead), we fail the test.
196 if ($actual_time_delta > $max_time)
197 {
198 --echo The auto-rejoin process took longer than expected or its timestamp is wrong
199 --die
200 }
201
202 --echo
203 --echo ####
204 --echo # 4) Verify that we can see the number of times the auto-rejoin process
205 --echo # was run.
206 --echo ####
207 --echo
208 # We should have already one auto-rejoin there
209 --let $assert_text = There should only be one auto-rejoin process triggered
210 --let $assert_cond = [SELECT GET_COUNT_AUTOREJOIN()] = 1
211 --source include/assert.inc
212
213 SET @@GLOBAL.DEBUG='-d,group_replication_stop_before_rejoin';
214 SET @@GLOBAL.DEBUG='-d,group_replication_stop_before_rejoin_loop';
215 SET @@GLOBAL.DEBUG='-d,group_replication_stop_rejoin_long_retry';
216
217 #
218 # Reset GR so that the member returns to a clean slate and disables
219 # super_read_only mode
220 #
221 --source include/stop_group_replication.inc
222 --source include/start_group_replication.inc
223
224 --echo
225 --echo ####
226 --echo # 5) Cleanup.
227 --echo ####
228 --echo
229 SET @@GLOBAL.DEBUG = @debug_saved;
230 SET GLOBAL group_replication_autorejoin_tries = default;
231 --source include/gr_end_autorejoin_monitoring.inc
232 --source include/group_replication_end.inc