Debian Codegraph @

mysql-test/suite/galera_3nodes/t/inconsistency_shutdown.test

raw
1 #
2 # Check that the node can cleanly shutdown in case of inconsistency
3 # (no locked up threads)
4 #
5
6 --source include/galera_cluster.inc
7 --source include/have_innodb.inc
8 --source include/have_debug.inc
9 --source include/have_debug_sync.inc
10 --source include/galera_have_debug_sync.inc
11
12 # Save original auto_increment_offset values.
13 --let $node_1=node_1
14 --let $node_2=node_2
15 --let $node_3=node_3
16 --source ../galera/include/auto_increment_offset_save.inc
17
18 --connection node_2
19 SELECT @@wsrep_slave_threads = 8;
20
21 #
22 # 1. Inconsistency on slave
23 #
24 --connection node_1
25 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 INT);
26 INSERT INTO t1 VALUES (1, 0),(2, 0),(3, 0),(4, 0),(5, 0),(6, 0),(7, 0),(8, 0);
27
28 --connection node_2
29 # Allow 1K slave queue woithout flow control
30 SET GLOBAL wsrep_provider_options='gcs.fc_limit=1K';
31 # Introduce 2 inconsistencies
32 SET wsrep_on=OFF;
33 DELETE FROM t1 WHERE f1 = 2;
34 DELETE FROM t1 WHERE f1 = 4;
35 SET wsrep_on=ON;
36 --source include/galera_wait_ready.inc
37
38 # Build up slave queue:
39 # - first 8 events will be picked by slave threads
40 # - one moreevent will be waiting in slave queue
41 LOCK TABLES t1 WRITE;
42 --connection node_1
43 UPDATE t1 SET f2 = 1 WHERE f1 = 1;
44 UPDATE t1 SET f2 = 1 WHERE f1 = 2;
45 UPDATE t1 SET f2 = 1 WHERE f1 = 3;
46 UPDATE t1 SET f2 = 1 WHERE f1 = 4;
47 UPDATE t1 SET f2 = 2 WHERE f1 = 4; /* dependent applier */
48 UPDATE t1 SET f2 = 3 WHERE f1 = 4; /* dependent applier */
49 UPDATE t1 SET f2 = 1 WHERE f1 = 5;
50 UPDATE t1 SET f2 = 1 WHERE f1 = 6;
51 UPDATE t1 SET f2 = 1 WHERE f1 = 7;
52 UPDATE t1 SET f2 = 1 WHERE f1 = 8;
53
54 --connection node_2
55 --echo # make sure all events landed to slave queue
56 set wsrep_sync_wait=0;
57 --let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_recv_queue';
58 --source include/wait_condition.inc
59
60 UNLOCK TABLES;
61 --source include/wsrep_wait_disconnect.inc
62 # Wait for the node to shutdown replication
63 --let $members=0
64 --source include/wsrep_wait_membership.inc
65
66 --connection node_1
67 --let $members=2
68 --source include/wsrep_wait_membership.inc
69 --source include/wait_until_ready.inc
70 SHOW STATUS LIKE 'wsrep_cluster_size';
71 SELECT * FROM t1;
72
73 --connection node_2
74 --echo # Gracefully restart the node
75 set wsrep_on=OFF;
76 --source include/shutdown_mysqld.inc
77 --source include/start_mysqld.inc
78 --source include/galera_wait_ready.inc
79
80 --let $wait_condition = SELECT COUNT(*)=1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
81 --source include/wait_condition.inc
82 DROP TABLE t1;
83
84 #
85 # 2. Inconsistency on master
86 #
87 --connection node_1
88 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 INT);
89 START TRANSACTION;
90 INSERT INTO t1 VALUES (1, 0);
91 INSERT INTO t1 VALUES (2, 0);
92 INSERT INTO t1 VALUES (3, 0);
93 INSERT INTO t1 VALUES (4, 0);
94 INSERT INTO t1 VALUES (5, 0);
95 INSERT INTO t1 VALUES (6, 0);
96 INSERT INTO t1 VALUES (7, 0);
97 INSERT INTO t1 VALUES (8, 0);
98 COMMIT;
99 CREATE TABLE t2 (f1 INTEGER PRIMARY KEY, f2 INT);
100
101 --connection node_2
102 --echo # Allow 1K slave queue without flow control
103 SET GLOBAL wsrep_provider_options='gcs.fc_limit=1K';
104 --let $wait_condition = SELECT COUNT(*)=1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't2';
105 --source include/wait_condition.inc
106 --echo # Introduce inconsistency
107 SET wsrep_on=OFF;
108 DROP TABLE t2;
109 SET wsrep_on=ON;
110 --source include/galera_wait_ready.inc
111
112 --echo # set up sync point to ensure DROP TABLE replication order below
113 --let galera_sync_point = after_replicate_sync
114 --source include/galera_set_sync_point.inc
115
116 --echo # Build up slave queue:
117 # - first 8 events will be picked by slave threads
118 # - one more event will be waiting in slave queue
119 LOCK TABLES t1 READ;
120
121 --connection node_1
122 UPDATE t1 SET f2 = 1 WHERE f1 = 1;
123 UPDATE t1 SET f2 = 1 WHERE f1 = 2;
124 UPDATE t1 SET f2 = 1 WHERE f1 = 3;
125 UPDATE t1 SET f2 = 1 WHERE f1 = 4;
126 UPDATE t1 SET f2 = 2 WHERE f1 = 4; /* dependent applier */;
127
128 --echo # interleave a failing statement
129 --connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
130 --connection node_2a
131 --send DROP TABLE t2;
132
133 --echo # make sure DROP TABLE from above has replicated
134 --connection node_2
135 set wsrep_sync_wait=0;
136 --echo "Wait for DROP TABLE to replicate"
137 --source include/galera_wait_sync_point.inc
138 --source include/galera_signal_sync_point.inc
139 --source include/galera_clear_sync_point.inc
140 --echo "DROP TABLE replicated"
141
142 --connection node_1
143 UPDATE t1 SET f2 = 3 WHERE f1 = 4; /* dependent applier */
144 UPDATE t1 SET f2 = 1 WHERE f1 = 5;
145 UPDATE t1 SET f2 = 1 WHERE f1 = 6;
146 UPDATE t1 SET f2 = 1 WHERE f1 = 7;
147 UPDATE t1 SET f2 = 1 WHERE f1 = 8;
148
149 --connection node_2
150 --echo # make sure all events landed to slave queue
151 --let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_recv_queue';
152 --source include/wait_condition.inc
153
154 UNLOCK TABLES;
155
156 --connection node_2a
157 --error ER_BAD_TABLE_ERROR
158 --reap
159
160 --connection node_1
161 --let $members=2
162 --source include/wsrep_wait_membership.inc
163 --source include/wait_until_ready.inc
164 SHOW STATUS LIKE 'wsrep_cluster_size';
165 SELECT * FROM t1;
166
167 --connection node_2
168 set wsrep_on=OFF;
169 --source include/wsrep_wait_disconnect.inc
170 --echo # Wait for the node to shutdown replication
171 --let $members=0
172 --source include/wsrep_wait_membership.inc
173 --echo # Gracefully restart the node
174
175 --source include/shutdown_mysqld.inc
176 --source include/start_mysqld.inc
177 --source include/galera_wait_ready.inc
178
179 --let $wait_condition = SELECT COUNT(*)=1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
180 --source include/wait_condition.inc
181
182 DROP TABLE t1;
183
184 CALL mtr.add_suppression("Can't find record in 't1'");
185 CALL mtr.add_suppression("Update_rows_v1 apply failed");
186 CALL mtr.add_suppression("Inconsistency detected: Inconsistent by consensus on");
187 CALL mtr.add_suppression("last left .* greater than drain seqno");
188 CALL mtr.add_suppression("WSREP: Failed to apply write set: ");
189
190 # Restore original auto_increment_offset values.
191 --source ../galera/include/auto_increment_offset_restore.inc