Debian Codegraph @

mysql-test/suite/rpl/t/rpl_connection_compression.test

raw
1 # ==== Requirements ====
2 #
3 # R1. If the intersection of slave's SOURCE_COMPRESSION_ALGORITHMS with
4 # master's @@global.protocol_compression_algorithms is nonempty, then
5 # the connection in START SLAVE shall succeed and use zlib if
6 # possible; otherwise zstd; otherwise uncompressed.
7 #
8 # R2. If the intersection of slave's SOURCE_COMPRESSION_ALGORITHMS with
9 # master's @@global.protocol_compression_algorithms is empty, then
10 # connection shall fail with an error.
11 #
12 # R3. If slave connects with SOURCE_COMPRESSION_ALGORITHMS = 'zlib' or
13 # 'zstd', the connection should be compressed.
14 #
15 # R4. If slave connects with SOURCE_COMPRESSION_ALGORITHMS = NULL, the
16 # connection should be uncompressed.
17 #
18 # R5. When using the zstd algorithm, SOURCE_ZSTD_COMPRESSION_LEVEL=N
19 # shall provide better compression ratio than M, if N > M.
20 #
21 # ==== References ====
22 #
23 # WL#12475 - Protocol Changes to specify compression configuration for
24 # connections
25 #
26
27 # Test does not depend on binlog_format, so should only run once.
28 --source include/have_binlog_format_row.inc
29 --let $rpl_skip_start_slave = 1
30 --source include/master-slave.inc
31 # do not run with binlog compression on, otherwise it will make
32 # this test fail, since it needs large transactions
33 --source include/not_binlog_transaction_compression_on.inc
34
35
36 --echo #### Initialize ####
37
38 # Don't replicate the auxiliary tables.
39 SET sql_log_bin = 0;
40
41 # List of possible values for SOURCE_COMPRESSION_ALGORITHMS
42 CREATE TABLE algorithms (
43 id INT AUTO_INCREMENT PRIMARY KEY,
44 name TEXT NOT NULL);
45 INSERT INTO algorithms(name) VALUES
46 ('default'),
47 ('zlib'),
48 ('zstd'),
49 ('uncompressed'),
50 ('zlib,zstd'),
51 ('zlib,uncompressed'),
52 ('zstd,uncompressed'),
53 ('zlib,zstd,uncompressed');
54
55 # The options above includes all real possibilities. The options
56 # below are only variants of the syntax, where algorithms appear in
57 # different orders. This test is a bit slow, so we skip the options
58 # below, to save time.
59
60 #('zstd,zlib'),
61 #('uncompressed,zlib'),
62 #('uncompressed,zstd'),
63 #('zlib,uncompressed,zstd'),
64 #('zstd,zlib,uncompressed'),
65 #('zstd,uncompressed,zlib'),
66 #('uncompressed,zlib,zstd'),
67 #('uncompressed,zstd,zlib');
68
69 --let $algorithm_count = `SELECT COUNT(*) FROM algorithms`
70
71 # List of tested values for SOURCE_ZSTD_COMPRESSION_LEVEL
72 CREATE TABLE levels (
73 id INT AUTO_INCREMENT PRIMARY KEY COMMENT 'order in which to set option',
74 level TEXT NOT NULL COMMENT 'compression level',
75 position INT NOT NULL COMMENT 'expected ordinal position of achieved compression'
76 );
77
78 # Exclude some levels, to make the test faster.
79 # Intermediate compression levels are not making much difference, test default and max.
80 INSERT INTO levels(level, position) VALUES
81 # ('1', 1),
82 #('2', 2),
83 ('default', 3),
84 #('3', 3),
85 #('13', 4),
86 ('22', 5);
87 --let $level_count = `SELECT COUNT(*) FROM levels`
88
89 --delimiter |
90
91 # Given two comma-separated lists of strings, returns a
92 # comma-separated list of those strings that appear in both, with
93 # duplicates removed, in the order of set1.
94 CREATE FUNCTION set_intersection(set1 TEXT, set2 TEXT) RETURNS TEXT BEGIN
95 DECLARE ret TEXT DEFAULT '';
96 DECLARE elem TEXT;
97 WHILE set1 != '' DO
98 SET elem = SUBSTRING_INDEX(set1, ',', 1);
99 IF FIND_IN_SET(elem, set2) THEN
100 SET ret = sys.list_add(ret, elem);
101 SET set2 = sys.list_drop(set2, elem);
102 END IF;
103 SET set1 = sys.list_drop(set1, elem);
104 END WHILE;
105 RETURN ret;
106 END|
107 --delimiter ;
108
109 SET sql_log_bin = 1;
110
111 # Statement to reset replication
112 --let $change_master_base = CHANGE REPLICATION SOURCE TO SOURCE_HOST = '127.0.0.1', SOURCE_PORT = $SERVER_MYPORT_1, SOURCE_USER = 'root', SOURCE_RETRY_COUNT=3, SOURCE_CONNECT_RETRY=1;
113
114 --let $script_dir = $MYSQLTEST_VARDIR
115
116 # ---- Purpose ----
117 #
118 # Auxiliary script to set configuration options on slave.
119 #
120 # ---- Usage ----
121 #
122 # --let $change_master_base = <CHANGE REPLICATION SOURCE STATEMENT>
123 # --let $change_master_algorithm = <CHANGE REPLICATION SOURCE STATEMENT>
124 # --let $change_master_level = <CHANGE REPLICATION SOURCE STATEMENT>
125 # --let $replica_compressed_protocol = [0|1]
126 # --source $script_dir/configure_slave.inc
127 #
128 # Parameters:
129 #
130 # $change_master_base
131 # CHANGE REPLICATION SOURCE statement that sets non-compression attributes to
132 # do the base configuration of the channel; for instance
133 # SOURCE_HOST, SOURCE_USER, etc.
134 #
135 # $change_master_algorithm
136 # CHANGE REPLICATION SOURCE statement that sets SOURCE_COMPRESSION_ALGORITHMS.
137 #
138 # $change_master_level
139 # CHANGE REPLICATION SOURCE statement that sets SOURCE_ZSTD_COMPRESSION_LEVEL.
140 #
141 # $replica_compressed_protocol
142 # Set @@global.replica_compressed_protocol to this value.
143 #
144 --write_file $script_dir/configure_slave.inc
145 # Make the 'source ...' files less noisy
146 --let $rpl_connection_silent = 1
147 --let $include_silent = 1
148
149 --source include/rpl_connection_slave.inc
150 --disable_warnings
151 --disable_query_log
152 eval $change_master_base;
153 --enable_query_log
154 if ($change_master_algorithm != '') {
155 eval $change_master_algorithm;
156 }
157 if ($change_master_level != '') {
158 eval $change_master_level;
159 }
160 --enable_warnings
161 eval SET @@global.replica_compressed_protocol = $replica_compressed_protocol;
162 EOF
163
164
165 # ---- Purpose ----
166 #
167 # Test one configuration that is expected to succeed, and verify
168 # requirements R1, R3, and R4.
169 #
170 # ---- Usage ----
171 #
172 # --let $change_master_base = <CHANGE REPLICATION SOURCE STATEMENT>
173 # --let $change_master_algorithm = <CHANGE REPLICATION SOURCE STATEMENT>
174 # --let $change_master_level = <CHANGE REPLICATION SOURCE STATEMENT>
175 # --let $replica_compressed_protocol = [0|1]
176 # --let $data_size = size of data in event
177 # --let $expected_algorithm = [zlib|zstd|uncompressed]
178 # --source $script_dir/test_master_slave_compression.inc
179 #
180 # Parameters:
181 #
182 # $change_master_base, $change_master_algorithm, $change_master_level,
183 # $replica_compressed_protocol
184 # See configure_slave.inc
185 #
186 # $data_size
187 # The size of the data that was inserted on master.
188 # We will expect that it sends more than this if the connection is
189 # uncompressed, and less than this if the connection is compressed.
190 #
191 # $expected_algorithm
192 # The algorithm that should be used in the protocol.
193 #
194 # ---- Implementation ----
195 #
196 # - Configure slave using CHANGE REPLICATION SOURCE
197 # - Start replication threads
198 # - Measure the number of bytes sent on the connection before and after
199 # - If the expected algorithm is uncompressed, assert that it sent
200 # more than $data_size bytes (data plus overhead)
201 # - If the expected algorithm is compressed, assert that it sent less
202 # than $data_size bytes (we assume that compression saves more than
203 # the overhead).
204 --write_file $script_dir/test_master_slave_compression.inc
205
206 --source $script_dir/configure_slave.inc
207
208 # Get initial value for number of bytes sent.
209 --source include/rpl_connection_master.inc
210 --source include/save_master_pos.inc
211
212 --let $bytes_before = `SELECT SUM_NUMBER_OF_BYTES_WRITE FROM performance_schema.socket_summary_by_event_name WHERE EVENT_NAME = 'wait/io/socket/sql/client_connection'`
213
214 --echo START SLAVE IO_THREAD
215 # Verify that slave can connect
216 --source include/rpl_connection_slave.inc
217 --source include/start_slave_io.inc
218
219 # Replicate
220 --source include/sync_slave_io.inc
221 --source include/stop_slave_io.inc
222
223 # Get final value for total number of bytes sent.
224 --source include/rpl_connection_master.inc
225
226 --let $bytes_after = `SELECT SUM_NUMBER_OF_BYTES_WRITE FROM performance_schema.socket_summary_by_event_name WHERE EVENT_NAME = 'wait/io/socket/sql/client_connection'`
227 --let $bytes_sent = `SELECT $bytes_after - $bytes_before`
228
229 # Verify that data was compressed / uncompressed according to option
230 if ($expected_algorithm == uncompressed) {
231 --let $assert_cond = $bytes_sent > $data_size
232 --let $assert_text = More than $data_size bytes should be sent uncompressed
233 }
234 if ($expected_algorithm != uncompressed) {
235 --let $assert_cond = $bytes_sent < $data_size
236 --let $assert_text = Less than $data_size bytes should be sent compressed
237 }
238 --let $include_silent = 0
239 --source include/assert.inc
240
241 --source include/rpl_connection_slave.inc
242 --disable_query_log
243 RESET MASTER;
244 RESET SLAVE ALL;
245 --enable_query_log
246 EOF
247
248
249 # ---- Purpose ----
250 #
251 # Do the same as $script_dir/test_master_slave_compression.inc, and
252 # in addition verify requirement R5.
253 #
254 # ---- Usage ----
255 #
256 # --let $change_master_base = <CHANGE REPLICATION SOURCE STATEMENT>
257 # --let $change_master_algorithm = <CHANGE REPLICATION SOURCE STATEMENT>
258 # --let $change_master_level = <CHANGE REPLICATION SOURCE STATEMENT>
259 # --let $replica_compressed_protocol = [0|1]
260 # --let $data_size = size of data in event
261 # --let $expected_algorithm = [zlib|zstd|uncompressed]
262 # --let $level_it = NUMBER
263 # --source $script_dir/test_master_slave_compression_with_level
264 #
265 # Parameters:
266 #
267 # $change_master_base, $change_master_algorithm, $change_master_level,
268 # $replica_compressed_protocol, $data_size, $expected_algorithm
269 # See test_master_slave_compression.inc, above.
270 #
271 # $level_it
272 # Row number in the 'levels' table which contains the compression
273 # level to test.
274 #
275 # ---- Implementation ----
276 #
277 # - Fetch the compression level from the levels table.
278 # - Add SOURCE_ZSTD_COMPRESSION_LEVEL to the CHANGE REPLICATION SOURCE statement text
279 # - Source test_master_slave_compression
280 # - If level_it != 1 (it's not the first level we try for the
281 # algorithm), compare the compression ratio with the compression
282 # ratio of the last run. If the 'position' column of the 'levels'
283 # table has the same value as the 'position' column for the previous
284 # row in the 'levels' table, then we expect the compression ratios
285 # to be the same. Otherwise (i.e. the 'position' column is greater
286 # than that of the previous row), then we expect the compression
287 # ratio of the current test to be better than that of the last test.
288 --write_file $script_dir/test_master_slave_compression_with_level.inc
289 --source include/rpl_connection_master.inc
290
291 --source $script_dir/test_master_slave_compression.inc
292
293 --source include/rpl_connection_master.inc
294
295 # If this is not the first run with this algorithm, compare with previous
296 # run. If the 'position' value for this row in the 'levels' table equals
297 # the 'position' value for the previous row, then the levels are
298 # the same and we should expect equal compression. Otherwise, this level
299 # is higher than the previous level and we should expect the compression
300 # to be better than the previous run.
301 if ($level_it != 1) {
302 --let $better_than_last = `SELECT position > $last_position FROM levels WHERE id = $level_it`
303 if ($better_than_last) {
304 --let $assert_cond = $bytes_sent <= $last_bytes_sent
305 --let $assert_text = Compression ratio should be better than previous algorithm
306 }
307 if (!$better_than_last) {
308 --let $assert_cond = $bytes_sent == $last_bytes_sent
309 --let $assert_text = Compression ratio should be equal to previous algorithm
310 }
311 --source include/assert.inc
312 }
313
314 --let $last_bytes_sent = $bytes_sent
315
316 --let $last_position = `SELECT position FROM levels WHERE id = $level_it`
317 EOF
318
319
320 # ---- Purpose ----
321 #
322 # Verify that there is an error when master and slave do not have any
323 # compression algorithm in common.
324 #
325 # ---- Usage ----
326 #
327 # --let $change_master_base = <CHANGE REPLICATION SOURCE STATEMENT>
328 # --let $change_master_algorithm = <CHANGE REPLICATION SOURCE STATEMENT>
329 # --source $script_dir/test_master_slave_compression_error.inc
330 #
331 # Parameters: See test_master_slave_compression.inc.
332 #
333 # ---- Implementation ----
334 #
335 # - Execute the CHANGE REPLICATION SOURCE statement
336 # - Execute START SLAVE IO_THREAD
337 # - Wait until error happens.
338 --write_file $script_dir/test_master_slave_compression_error.inc
339
340 --source $script_dir/configure_slave.inc
341
342 START SLAVE IO_THREAD;
343 --let $slave_io_errno = 3922, 2066
344 # 3922 = ER_WRONG_COMPRESSION_ALGORITHM_CLIENT
345 # 2066 = CR_COMPRESSION_WRONGLY_CONFIGURED
346 --source include/wait_for_slave_io_error.inc
347 --echo got error
348 RESET SLAVE ALL;
349 EOF
350
351
352 # Generate some binlog data. This will be more than 10000 bytes
353 # uncompressed, and less than 10000 bytes compressed.
354 --let $data_size = 1000000
355 CREATE TABLE t (a LONGTEXT);
356 eval INSERT INTO t VALUES (REPEAT('a', $data_size));
357
358 --let $saved_protocol_compression_algorithms = `SELECT @@global.protocol_compression_algorithms`
359
360 --echo #### Test ####
361
362 # Iterate over all the algorithms on master
363 --let $master_algorithm_it = 1
364 while ($master_algorithm_it <= $algorithm_count) {
365 --source include/rpl_connection_master.inc
366 --let $master_algorithm_text = `SELECT name FROM algorithms WHERE id = $master_algorithm_it`
367 --let $master_algorithm = $master_algorithm_text
368 if ($master_algorithm == default) {
369 --let $master_algorithm = zlib,zstd,uncompressed
370 }
371
372 --echo ==== master:$master_algorithm_text ====
373
374 --source include/rpl_connection_master.inc
375 if ($master_algorithm_text == default) {
376 eval SET @@global.protocol_compression_algorithms = DEFAULT;
377 }
378 if ($master_algorithm != default) {
379 eval SET @@global.protocol_compression_algorithms = '$master_algorithm';
380 }
381 SELECT @@global.protocol_compression_algorithms;
382
383 # Iterate over all algorithms on slave
384 --let $slave_algorithm_it = 1
385 while ($slave_algorithm_it <= $algorithm_count) {
386 --source include/rpl_connection_master.inc
387 --let $slave_algorithm_text = `SELECT name FROM algorithms WHERE id = $slave_algorithm_it`
388 --let $slave_algorithm = $slave_algorithm_text
389 --let $change_master_algorithm = CHANGE REPLICATION SOURCE TO SOURCE_COMPRESSION_ALGORITHMS = '$slave_algorithm'
390 if ($slave_algorithm == default) {
391 --let $slave_algorithm = uncompressed
392 --let $change_master_algorithm =
393 }
394
395 --let $replica_compressed_protocol = 0
396 --let $replica_compressed_protocol_iterations = 1
397 if ($slave_algorithm == uncompressed) {
398 --let $replica_compressed_protocol_iterations = 2
399 }
400
401 while ($replica_compressed_protocol < $replica_compressed_protocol_iterations) {
402 --source include/rpl_connection_slave.inc
403 if ($replica_compressed_protocol) {
404 --let $slave_algorithm = zlib
405 }
406
407 --source include/rpl_connection_master.inc
408 --let $intersection = `SELECT set_intersection('$master_algorithm', '$slave_algorithm')`
409
410 if ($intersection != '') {
411 let $expected_algorithm = `
412 SELECT IF(FIND_IN_SET('zlib', '$intersection'), 'zlib',
413 IF(FIND_IN_SET('zstd', '$intersection'), 'zstd',
414 'uncompressed'))`;
415
416 if ($expected_algorithm == zstd) {
417 # Iterate over levels.
418 --let $level_it = 1
419 while ($level_it <= $level_count) {
420 # Get the compression level from the table.
421 --let $level = `SELECT level FROM levels WHERE id = $level_it`
422
423 # If $level != 'default', set SOURCE_ZSTD_COMPRESSION_LEVEL.
424 # Otherwise don't.
425 --let $change_master_level =
426 if ($level != 'default') {
427 --let $change_master_level = CHANGE REPLICATION SOURCE TO SOURCE_ZSTD_COMPRESSION_LEVEL = $level
428 }
429
430 --echo ---- master:$master_algorithm slave:$slave_algorithm slave_option:$replica_compressed_protocol expect:$expected_algorithm level:$level ----
431
432 --source $script_dir/test_master_slave_compression_with_level.inc
433
434 --inc $level_it
435 }
436 }
437 if ($expected_algorithm != zstd) {
438 --echo ---- master:$master_algorithm slave:$slave_algorithm_text slave_option:$replica_compressed_protocol expected:$expected_algorithm ----
439 --source $script_dir/test_master_slave_compression.inc
440 }
441 }
442
443 if ($intersection == '') {
444 --echo ---- master:$master_algorithm slave:$slave_algorithm_text slave_option:$replica_compressed_protocol expect:error ----
445 --source $script_dir/test_master_slave_compression_error.inc
446 }
447 --inc $replica_compressed_protocol
448 }
449 --inc $slave_algorithm_it
450 }
451 --inc $master_algorithm_it
452 }
453
454 --echo #### Clean up ####
455
456 --source include/rpl_connection_master.inc
457 --eval SET @@global.protocol_compression_algorithms = "$saved_protocol_compression_algorithms";
458
459 SET sql_log_bin = 0;
460 DROP TABLE algorithms;
461 DROP TABLE levels;
462 DROP TABLE t;
463 DROP FUNCTION set_intersection;
464 SET sql_log_bin = 1;
465
466 --remove_file $script_dir/configure_slave.inc
467 --remove_file $script_dir/test_master_slave_compression.inc
468 --remove_file $script_dir/test_master_slave_compression_with_level.inc
469 --remove_file $script_dir/test_master_slave_compression_error.inc
470
471 --source include/rpl_connection_slave.inc
472 --disable_query_log
473 eval CHANGE REPLICATION SOURCE TO SOURCE_HOST = '127.0.0.1', SOURCE_PORT = $SERVER_MYPORT_1, SOURCE_USER = 'root';
474 --enable_query_log
475
476 # Tell rpl_end to not try to sync
477 --let $rpl_skip_sync = 1
478 --source include/rpl_end.inc