| 1 |
# ==== Requirements ==== |
| 2 |
# |
| 3 |
# The following situations should result in errors: |
| 4 |
# |
| 5 |
# E1. A JSON path exists on the master but not on slave, and is |
| 6 |
# updated/deleted on teh master. |
| 7 |
# |
| 8 |
# E2. A JSON path exists on the slave but not on the master, and is |
| 9 |
# inserted on the master. |
| 10 |
# |
| 11 |
# E3. The slave has an extra column, which is generated and has a uniqueness |
| 12 |
# constaint that gets violated by the update. |
| 13 |
# |
| 14 |
# E4. Both master and slave have a generated column, and the slave has an |
| 15 |
# extra uniquness constraint that gets violated by the update. |
| 16 |
# |
| 17 |
# E5. Out of memory happens while applying a JSON diff. |
| 18 |
# |
| 19 |
# E6. The row event containing partial JSON is corrupted. |
| 20 |
# - This includes a number of different ways to corrupt the event. |
| 21 |
# See implementation of E6 below. |
| 22 |
# |
| 23 |
# The consequence of the error depends on the mode; there are three modes: |
| 24 |
# |
| 25 |
# M1. Normal mode, error causing slave to stop. |
| 26 |
# |
| 27 |
# M2. replica_skip_errors includes the error. |
| 28 |
# |
| 29 |
# M3. replica_exec_mode=IDEMPOTENT |
| 30 |
# |
| 31 |
# In addition, the consequence of corrupted events should be that |
| 32 |
# mysqlbinlog -v prints an appropriate message when failing to decode |
| 33 |
# the row event. |
| 34 |
# |
| 35 |
# The code paths to handle idempotent errors are different for each |
| 36 |
# slave_rows_search_algorithms, hence we must run the test once for each |
| 37 |
# algorithm. |
| 38 |
# |
| 39 |
# ==== Implementation ==== |
| 40 |
# |
| 41 |
# Setup each scenario using |
| 42 |
# extra/rpl_tests/rpl_row_jsondiff_error_scenario.inc |
| 43 |
# |
| 44 |
# Setup scenarios in a loop with three iterations, each iteration testing |
| 45 |
# one of M1-M3. |
| 46 |
# |
| 47 |
# Use include/begin_replace_combination.inc to run with all different |
| 48 |
# slave_rows_search_algorithms. |
| 49 |
# |
| 50 |
# ==== Related Worklog ==== |
| 51 |
# |
| 52 |
# WL#2955 RBR replication of partial JSON updates |
| 53 |
# |
| 54 |
|
| 55 |
--source include/have_debug.inc |
| 56 |
--source include/have_grep.inc |
| 57 |
--source include/master-slave.inc |
| 58 |
|
| 59 |
--echo ######## Configure ######## |
| 60 |
|
| 61 |
--let $dollar_func= CHAR(36) |
| 62 |
--let $dollar= `SELECT $dollar_func` |
| 63 |
|
| 64 |
--echo # Configure slave_rows_search_algorithms on master |
| 65 |
--let $replace_combination_from= BOTH.BINLOG_FORMAT=ROW,MIXED,STATEMENT |
| 66 |
--let $replace_combination_to= GLOBAL.SLAVE_ROWS_SEARCH_ALGORITHMS=INDEX_SCAN,HASH_SCAN,TABLE_SCAN |
| 67 |
--source include/rpl_connection_master.inc |
| 68 |
--source include/begin_replace_combination.inc |
| 69 |
|
| 70 |
--echo # Configure partial JSON on master |
| 71 |
--source include/rpl_connection_master.inc |
| 72 |
SET @old_binlog_row_image= @@SESSION.BINLOG_ROW_IMAGE; |
| 73 |
SET @old_binlog_row_value_options= @@SESSION.BINLOG_ROW_VALUE_OPTIONS; |
| 74 |
SET @@SESSION.BINLOG_ROW_IMAGE = 'MINIMAL'; |
| 75 |
SET @@SESSION.BINLOG_ROW_VALUE_OPTIONS = 'PARTIAL_JSON'; |
| 76 |
|
| 77 |
--echo # Add error suppressions on slave |
| 78 |
--source include/rpl_connection_slave.inc |
| 79 |
call mtr.add_suppression("Could not execute Update_rows_partial event on table test.t"); |
| 80 |
call mtr.add_suppression("Can't find record in 't'"); |
| 81 |
# When mts is enabled |
| 82 |
call mtr.add_suppression("The replica coordinator and worker threads are stopped, possibly leaving data in inconsistent state"); |
| 83 |
call mtr.add_suppression("Replica: Could not apply JSON diff"); |
| 84 |
call mtr.add_suppression("Replica: Corrupted JSON diff"); |
| 85 |
call mtr.add_suppression("Replica: Corrupted replication event was detected"); |
| 86 |
call mtr.add_suppression("Replica: Got error 1610 - "); |
| 87 |
call mtr.add_suppression("Replica: Got error 3648 - "); |
| 88 |
call mtr.add_suppression("Replica: Got error 3649 - "); |
| 89 |
call mtr.add_suppression("Replica: Got error 5 - "); |
| 90 |
|
| 91 |
--echo ######## Test ######## |
| 92 |
|
| 93 |
--let $clean_error= 1 |
| 94 |
|
| 95 |
--let $mode= 0 |
| 96 |
while ($mode < 3) |
| 97 |
{ |
| 98 |
--source include/rpl_connection_slave.inc |
| 99 |
|
| 100 |
if ($mode == 0) |
| 101 |
{ |
| 102 |
--let $error_mode= FAIL |
| 103 |
--source include/stop_slave.inc |
| 104 |
} |
| 105 |
if ($mode == 1) |
| 106 |
{ |
| 107 |
--let $error_mode= SKIP |
| 108 |
|
| 109 |
--let $rpl_server_number= 2 |
| 110 |
# Error ER_KEY_NOT_FOUND |
| 111 |
--let $er_dup_entry= convert_error(ER_DUP_ENTRY) |
| 112 |
--let $er_key_not_found= convert_error(ER_KEY_NOT_FOUND) |
| 113 |
--let $er_could_not_apply_json_diff= convert_error(ER_COULD_NOT_APPLY_JSON_DIFF) |
| 114 |
--let $er_corrupted_json_diff= convert_error(ER_CORRUPTED_JSON_DIFF) |
| 115 |
--let $er_slave_corrupt_event= convert_error(ER_REPLICA_CORRUPT_EVENT) |
| 116 |
--let $rpl_server_parameters= --replica_skip_errors=$er_dup_entry,$er_key_not_found,$er_could_not_apply_json_diff,$er_corrupted_json_diff,$er_slave_corrupt_event,5 |
| 117 |
--source include/rpl_restart_server.inc |
| 118 |
--source include/rpl_connection_slave.inc |
| 119 |
} |
| 120 |
if ($mode == 2) |
| 121 |
{ |
| 122 |
--let $error_mode= IDEMPOTENT |
| 123 |
|
| 124 |
--let $rpl_server_number= 2 |
| 125 |
--let $rpl_server_parameters= --skip-replica-start |
| 126 |
--source include/rpl_restart_server.inc |
| 127 |
|
| 128 |
--source include/rpl_connection_slave.inc |
| 129 |
SET GLOBAL REPLICA_EXEC_MODE = IDEMPOTENT; |
| 130 |
} |
| 131 |
|
| 132 |
--echo ******** ERROR MODE: $error_mode ******** |
| 133 |
|
| 134 |
--echo ---- Configure slave ---- |
| 135 |
# Configure slave options here and not before the loop, since we |
| 136 |
# restart the server above. |
| 137 |
|
| 138 |
--echo # Configure slave_rows_search_algorithms and binlog_format |
| 139 |
--source include/rpl_connection_slave.inc |
| 140 |
--source include/begin_replace_combination.inc |
| 141 |
|
| 142 |
--echo # Configure partial JSON |
| 143 |
SET @old_binlog_row_image= @@GLOBAL.BINLOG_ROW_IMAGE; |
| 144 |
SET @old_binlog_row_value_options= @@GLOBAL.BINLOG_ROW_VALUE_OPTIONS; |
| 145 |
SET @@GLOBAL.BINLOG_ROW_IMAGE = 'MINIMAL'; |
| 146 |
SET @@GLOBAL.BINLOG_ROW_VALUE_OPTIONS = 'PARTIAL_JSON'; |
| 147 |
|
| 148 |
--source include/start_slave.inc |
| 149 |
|
| 150 |
# For show_rpl_debug_info |
| 151 |
--let $extra_debug_table= test.t |
| 152 |
|
| 153 |
--let $table= test.t |
| 154 |
--let $column_def= i INT PRIMARY KEY, j JSON |
| 155 |
--let $stmt_pre= UPDATE t SET j = |
| 156 |
--let $stmt_post= WHERE i = 1 |
| 157 |
--let $insert_columns= (i, j) |
| 158 |
|
| 159 |
--let $desc= E1. Path exists on master but not on slave, JSON_REMOVE |
| 160 |
--let $rows= (1, '[1, {"m" : 1}]') |
| 161 |
--let $rows_slave= (1, '[1, {"s" : 1}]') |
| 162 |
--let $stmt= JSON_REMOVE(j, '$dollar[1].m') |
| 163 |
--let $error= convert_error(ER_COULD_NOT_APPLY_JSON_DIFF) |
| 164 |
--source extra/rpl_tests/rpl_row_jsondiff_error_scenario.inc |
| 165 |
|
| 166 |
--let $desc= E1. Path exists on master but not on slave, JSON_SET |
| 167 |
--let $rows= (1, '[1, {"m" : 1}]') |
| 168 |
--let $rows_slave= (1, '[1, {"s" : 1}]') |
| 169 |
--let $stmt= JSON_SET(j, '$dollar[1].m', 2) |
| 170 |
--let $error= convert_error(ER_COULD_NOT_APPLY_JSON_DIFF) |
| 171 |
--source extra/rpl_tests/rpl_row_jsondiff_error_scenario.inc |
| 172 |
|
| 173 |
--let $desc= E2. Path exists on slave but not on master |
| 174 |
--let $rows= (1, '[1, {"m" : 1}]') |
| 175 |
--let $rows_slave= (1, '[1, {"s" : 1}]') |
| 176 |
--let $stmt= JSON_SET(j, '$dollar[1].s', 2) |
| 177 |
--let $error= convert_error(ER_COULD_NOT_APPLY_JSON_DIFF) |
| 178 |
--source extra/rpl_tests/rpl_row_jsondiff_error_scenario.inc |
| 179 |
|
| 180 |
# Disabled due to BUG#26630497 |
| 181 |
if (0) { |
| 182 |
--let $desc= E3. Violation of uniqueness constraint in slave-only generated column |
| 183 |
--let $column_def= i INT PRIMARY KEY, j JSON |
| 184 |
--let $column_def_slave= $column_def, g INT GENERATED ALWAYS AS (j->"$dollar.id") VIRTUAL UNIQUE KEY |
| 185 |
--let $rows_slave= |
| 186 |
--let $rows= (1, '{"id": 1, "value": "x"}'), (2, '{"id": 2, "value": "y"}') |
| 187 |
--let $stmt= JSON_SET(j, '$dollar.id', 2) |
| 188 |
--let $error= convert_error(ER_DUP_ENTRY) |
| 189 |
--source extra/rpl_tests/rpl_row_jsondiff_error_scenario.inc |
| 190 |
} |
| 191 |
|
| 192 |
--let $desc= E4. Violation of slave-only uniqueness constraint in generated column |
| 193 |
--let $column_def= i INT PRIMARY KEY, j JSON, g INT GENERATED ALWAYS AS (j->"$dollar.id") VIRTUAL |
| 194 |
--let $column_def_slave= $column_def UNIQUE KEY |
| 195 |
--let $rows_slave= |
| 196 |
--let $rows= (1, '{"id": 1, "value": "x"}'), (2, '{"id": 2, "value": "y"}') |
| 197 |
--let $stmt= JSON_SET(j, '$dollar.id', 2) |
| 198 |
--let $error= convert_error(ER_DUP_ENTRY) |
| 199 |
--source extra/rpl_tests/rpl_row_jsondiff_error_scenario.inc |
| 200 |
|
| 201 |
--let $desc= E5. Out of memory applying diff |
| 202 |
--let $column_def= i INT PRIMARY KEY, j JSON |
| 203 |
--let $column_def_slave= |
| 204 |
--let $rows= (1, '{"id": 1, "value": "x"}') |
| 205 |
--let $stmt= JSON_SET(j, '$dollar.id', REPEAT('a', 1024)) |
| 206 |
--let $slave_debug_symbol= simulate_oom_in_apply_json_diffs |
| 207 |
--let $error= 5 |
| 208 |
--source include/rpl_connection_slave.inc |
| 209 |
SET GLOBAL BINLOG_ROW_VALUE_OPTIONS = ''; |
| 210 |
--source include/stop_slave_sql.inc |
| 211 |
--source include/start_slave_sql.inc |
| 212 |
--source extra/rpl_tests/rpl_row_jsondiff_error_scenario.inc |
| 213 |
--source include/rpl_connection_slave.inc |
| 214 |
SET GLOBAL BINLOG_ROW_VALUE_OPTIONS = 'PARTIAL_JSON'; |
| 215 |
--source include/stop_slave_sql.inc |
| 216 |
--source include/start_slave_sql.inc |
| 217 |
--let $slave_debug_symbol= |
| 218 |
|
| 219 |
--let $desc= E6. Corruption in event |
| 220 |
--let $stmt= JSON_SET(j, '$dollar.id', '[1]') |
| 221 |
# Use mysqlbinlog on corrupted events, in order to get coverage for |
| 222 |
# error cases in mysqlbinlog |
| 223 |
--let $use_mysqlbinlog= 1 |
| 224 |
# Make remove_debug_point not complain if the symbol was already removed. |
| 225 |
--let $debug_if_exists= 1 |
| 226 |
|
| 227 |
# 1. Error cases where the format of a json diff is invalid in some |
| 228 |
# way. The cases only differ on the suffix of the debug symbol, |
| 229 |
# so we iterate over them in a loop to reduce copy-paste. |
| 230 |
--let $error= convert_error(ER_CORRUPTED_JSON_DIFF) |
| 231 |
--let $debug_symbol_list= bad_op,truncate_before_path_length,bad_path_length,truncate_before_path,bad_path_char,truncate_before_doc_length,bad_doc_length,truncate_before_doc,bad_doc_char |
| 232 |
--let $debug_symbol_number= 1 |
| 233 |
while ($debug_symbol_number <= 9) |
| 234 |
{ |
| 235 |
# Extract the Nth symbol from comma-separated list |
| 236 |
--let $master_debug_symbol= `SELECT SUBSTRING_INDEX(SUBSTRING_INDEX('$debug_symbol_list', ',', $debug_symbol_number), ',', -1)` |
| 237 |
--let $master_debug_symbol= binlog_corrupt_json_diff_$master_debug_symbol |
| 238 |
--source extra/rpl_tests/rpl_row_jsondiff_error_scenario.inc |
| 239 |
|
| 240 |
--inc $debug_symbol_number |
| 241 |
} |
| 242 |
|
| 243 |
# 2. Other simulated error cases error. |
| 244 |
# This uses a different error and a different debug symbol prefix, |
| 245 |
# so execute outside the loop. |
| 246 |
--let $error= convert_error(ER_REPLICA_CORRUPT_EVENT) |
| 247 |
--let $master_debug_symbol= binlog_omit_last_column_from_table_map_event |
| 248 |
--source extra/rpl_tests/rpl_row_jsondiff_error_scenario.inc |
| 249 |
|
| 250 |
--let $debug_if_exists= 0 |
| 251 |
--let $use_mysqlbinlog= 0 |
| 252 |
--let $master_debug_symbol= |
| 253 |
|
| 254 |
--echo ---- Clean up partial JSON on slave ---- |
| 255 |
--source include/rpl_connection_slave.inc |
| 256 |
SET @@GLOBAL.BINLOG_ROW_VALUE_OPTIONS= @old_binlog_row_value_options; |
| 257 |
SET @@GLOBAL.BINLOG_ROW_IMAGE= @old_binlog_row_image; |
| 258 |
|
| 259 |
--inc $mode |
| 260 |
} |
| 261 |
|
| 262 |
--source include/rpl_connection_master.inc |
| 263 |
SET @@SESSION.BINLOG_ROW_VALUE_OPTIONS= @old_binlog_row_value_options; |
| 264 |
SET @@SESSION.BINLOG_ROW_IMAGE= @old_binlog_row_image; |
| 265 |
--source include/end_replace_combination.inc |
| 266 |
|
| 267 |
--source include/rpl_connection_slave.inc |
| 268 |
# To avoid an error 'ER_RUNNING_APPLIER_PREVENTS_SWITCH_GLOBAL_BINLOG_FORMAT', |
| 269 |
# which will be caused by the following include/end_replace_combination.inc |
| 270 |
--source include/stop_slave_sql.inc |
| 271 |
--source include/end_replace_combination.inc |
| 272 |
|
| 273 |
--connection slave |
| 274 |
SET GLOBAL REPLICA_EXEC_MODE = STRICT; |
| 275 |
--let $rpl_only_running_threads= 1 |
| 276 |
--source include/rpl_end.inc |