Debian Codegraph @

mysql-test/suite/rpl/t/rpl_parallel_retry.test

raw
1 --source include/have_innodb.inc
2 --source include/have_perfschema.inc
3 --source include/have_debug.inc
4 --source include/have_debug_sync.inc
5 --let $rpl_topology=1->2
6 --source include/rpl_init.inc
7
8 --echo *** Test retry of transactions that fail to replicate due to deadlock or similar temporary error. ***
9
10 --connection server_1
11 ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB;
12 CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB;
13 INSERT INTO t1 VALUES (1,1);
14 --save_master_pos
15
16 # Use a stored function to inject a debug_sync into the appropriate THD.
17 # The function does nothing on the master, and on the slave it injects the
18 # desired debug_sync action(s).
19 SET sql_log_bin=0;
20 --delimiter ||
21 CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500))
22 RETURNS INT DETERMINISTIC
23 BEGIN
24 RETURN x;
25 END
26 ||
27 --delimiter ;
28 SET sql_log_bin=1;
29
30 --connection server_2
31 SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads;
32 --source include/stop_slave.inc
33 SET GLOBAL slave_parallel_threads=5;
34 --source include/start_slave.inc
35 --sync_with_master
36 SET sql_log_bin=0;
37 --delimiter ||
38 CREATE FUNCTION foo(x INT, d1 VARCHAR(500), d2 VARCHAR(500))
39 RETURNS INT DETERMINISTIC
40 BEGIN
41 IF d1 != '' THEN
42 SET debug_sync = d1;
43 END IF;
44 IF d2 != '' THEN
45 SET debug_sync = d2;
46 END IF;
47 RETURN x;
48 END
49 ||
50 --delimiter ;
51 SET sql_log_bin=1;
52 --source include/stop_slave.inc
53
54 --connection server_1
55 SET gtid_seq_no = 100;
56 BEGIN;
57 INSERT INTO t1 VALUES (2,1);
58 UPDATE t1 SET b=b+1 WHERE a=1;
59 INSERT INTO t1 VALUES (3,1);
60 COMMIT;
61 SELECT * FROM t1 ORDER BY a;
62 --save_master_pos
63
64 --connection server_2
65 SET @old_dbug= @@GLOBAL.debug_dbug;
66 SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_gtid_0_x_100";
67 let $old_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
68 --source include/start_slave.inc
69 --sync_with_master
70 SET GLOBAL debug_dbug=@old_dbug;
71 let $new_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
72 let $ps_value= query_get_value(select last_trans_retry_count from
73 performance_schema.replication_applier_status_by_worker where
74 last_trans_retry_count > 0, last_trans_retry_count, 1);
75 let $assert_text= Performance Schema retries should match with actual retries;
76 let $assert_cond= "$ps_value" = $new_retry - $old_retry;
77 source include/assert.inc;
78
79
80 SELECT * FROM t1 ORDER BY a;
81
82
83 --echo *** Test that double retry works when the first retry also fails with temp error ***
84 --source include/stop_slave.inc
85
86 --connection server_1
87 SET gtid_seq_no = 100;
88 SET @old_server_id = @@server_id;
89 SET server_id = 10;
90 BEGIN;
91 INSERT INTO t1 VALUES (4,1);
92 UPDATE t1 SET b=b+1 WHERE a=1;
93 INSERT INTO t1 VALUES (5,1);
94 INSERT INTO t1 VALUES (6,1);
95 COMMIT;
96 SET server_id = @old_server_id;
97 SELECT * FROM t1 ORDER BY a;
98 --save_master_pos
99
100 --connection server_2
101 SET @old_dbug= @@GLOBAL.debug_dbug;
102 SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_gtid_0_x_100,rpl_parallel_simulate_double_temp_err_gtid_0_x_100";
103 let $old_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
104 --source include/start_slave.inc
105 --sync_with_master
106 SET GLOBAL debug_dbug=@old_dbug;
107 let $new_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
108 let $ps_value= query_get_value(select last_trans_retry_count from
109 performance_schema.replication_applier_status_by_worker where
110 last_trans_retry_count > 0, last_trans_retry_count, 1);
111 let $assert_text= Performance Schema retries should match with actual retries;
112 let $assert_cond= "$ps_value" = $new_retry - $old_retry;
113 source include/assert.inc;
114
115 SELECT * FROM t1 ORDER BY a;
116
117
118 --echo *** Test too many retries, eventually causing failure. ***
119 --source include/stop_slave.inc
120
121 --connection server_1
122 SET gtid_seq_no = 100;
123 SET @old_server_id = @@server_id;
124 SET server_id = 11;
125 BEGIN;
126 INSERT INTO t1 VALUES (7,1);
127 UPDATE t1 SET b=b+1 WHERE a=1;
128 INSERT INTO t1 VALUES (8,1);
129 INSERT INTO t1 VALUES (9,1);
130 COMMIT;
131 SET server_id = @old_server_id;
132 SELECT * FROM t1 ORDER BY a;
133 --save_master_pos
134
135 --connection server_2
136 SET sql_log_bin=0;
137 CALL mtr.add_suppression("Slave worker thread retried transaction 10 time\\(s\\) in vain, giving up");
138 CALL mtr.add_suppression("Slave: Deadlock found when trying to get lock; try restarting transaction");
139 CALL mtr.add_suppression("Slave worker thread retried transaction .* in vain, giving up");
140 SET sql_log_bin=1;
141
142 SET @old_dbug= @@GLOBAL.debug_dbug;
143 SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_gtid_0_x_100,rpl_parallel_simulate_infinite_temp_err_gtid_0_x_100";
144 let $old_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
145 START SLAVE;
146 --let $slave_sql_errno= 1213
147 --source include/wait_for_slave_sql_error.inc
148 SET GLOBAL debug_dbug=@old_dbug;
149 let $new_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
150 --disable_query_log
151 eval SELECT $new_retry - $old_retry AS retries;
152 --enable_query_log
153 let $ps_value= query_get_value(select last_trans_retry_count from
154 performance_schema.replication_applier_status_by_worker where
155 last_trans_retry_count > 0, last_trans_retry_count, 1);
156 let $assert_text= Performance Schema retries should match with actual retries;
157 let $assert_cond= "$ps_value" = $new_retry - $old_retry;
158 source include/assert.inc;
159
160 SELECT * FROM t1 ORDER BY a;
161 STOP SLAVE IO_THREAD;
162 --source include/start_slave.inc
163 --sync_with_master
164 SELECT * FROM t1 ORDER BY a;
165
166 --echo *** Test retry of event group that spans multiple relay log files. ***
167
168 --connection server_1
169 CREATE TABLE t2 (a int PRIMARY KEY, b BLOB) ENGINE=InnoDB;
170 INSERT INTO t2 VALUES (1,"Hulubullu");
171 --save_master_pos
172
173 --connection server_2
174 --sync_with_master
175 --source include/stop_slave.inc
176 SET @old_max= @@GLOBAL.max_relay_log_size;
177 SET GLOBAL max_relay_log_size=4096;
178
179 --connection server_1
180 --let $big= `SELECT REPEAT("*", 5000)`
181 SET gtid_seq_no = 100;
182 SET @old_server_id = @@server_id;
183 SET server_id = 12;
184 BEGIN;
185 --disable_query_log
186 eval INSERT INTO t2 VALUES (2, CONCAT("Hello ", "$big"));
187 eval INSERT INTO t2 VALUES (3, CONCAT("Long data: ", "$big"));
188 --enable_query_log
189 INSERT INTO t1 VALUES (10, 4);
190 COMMIT;
191 SET server_id = @old_server_id;
192 SELECT * FROM t1 WHERE a >= 10 ORDER BY a;
193 SELECT a, LENGTH(b) FROM t2 ORDER BY a;
194 --save_master_pos
195
196 --connection server_2
197 SET @old_dbug= @@GLOBAL.debug_dbug;
198 SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_gtid_0_x_100";
199 let $old_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
200 --source include/start_slave.inc
201 --sync_with_master
202 SET GLOBAL debug_dbug=@old_dbug;
203 let $new_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
204 let $ps_value= query_get_value(select last_trans_retry_count from
205 performance_schema.replication_applier_status_by_worker where
206 last_trans_retry_count > 0, last_trans_retry_count, 1);
207 let $assert_text= Performance Schema retries should match with actual retries;
208 let $assert_cond= "$ps_value" = $new_retry - $old_retry;
209 source include/assert.inc;
210
211 SELECT * FROM t1 WHERE a >= 10 ORDER BY a;
212 SELECT a, LENGTH(b) FROM t2 ORDER BY a;
213
214 --connection server_1
215 INSERT INTO t1 VALUES (11,11);
216 --disable_query_log
217 eval INSERT INTO t2 VALUES (4, "$big");
218 --enable_query_log
219 --save_master_pos
220
221 --connection server_2
222 --sync_with_master
223 SELECT * FROM t1 WHERE a >= 10 ORDER BY a;
224 SELECT a, LENGTH(b) FROM t2 ORDER BY a;
225 SET GLOBAL max_relay_log_size=@old_max;
226
227
228 --echo *** MDEV-7065: Incorrect relay log position in parallel replication after retry of transaction ***
229
230 --connection server_2
231 --source include/stop_slave.inc
232
233 --connection server_1
234 BEGIN;
235 INSERT INTO t1 VALUES (100, 0);
236 INSERT INTO t1 VALUES (101, 0);
237 INSERT INTO t1 VALUES (102, 0);
238 INSERT INTO t1 VALUES (103, 0);
239 COMMIT;
240 SELECT * FROM t1 WHERE a >= 100 ORDER BY a;
241 --save_master_pos
242
243 --connection server_2
244 # Inject a DBUG error insert to cause the XID event of the single transaction
245 # from the master to fail with a deadlock error and be retried.
246 # The bug was that the retry of the XID would leave the relay log position
247 # incorrect (off by the size of XID event).
248 SET @old_dbug= @@GLOBAL.debug_dbug;
249 SET GLOBAL debug_dbug="+d,rpl_parallel_simulate_temp_err_xid";
250 let $old_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
251 --source include/start_slave.inc
252 --sync_with_master
253 SET GLOBAL debug_dbug=@old_dbug;
254 let $new_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
255 let $ps_value= query_get_value(select last_trans_retry_count from
256 performance_schema.replication_applier_status_by_worker where
257 last_trans_retry_count > 0, last_trans_retry_count, 1);
258 let $assert_text= Performance Schema retries should match with actual retries;
259 let $assert_cond= "$ps_value" = $new_retry - $old_retry;
260 source include/assert.inc;
261
262 SELECT * FROM t1 WHERE a >= 100 ORDER BY a;
263 # Stop the SQL thread. When the bug was there to give the incorrect relay log
264 # position, the restart of the SQL thread would read garbage data from the
265 # middle of an event and fail with relay log IO error.
266 --source include/stop_slave_sql.inc
267
268 --connection server_1
269 INSERT INTO t1 VALUES (104, 1);
270 INSERT INTO t1 VALUES (105, 1);
271 INSERT INTO t1 VALUES (106, 1);
272 INSERT INTO t1 VALUES (107, 1);
273 INSERT INTO t1 VALUES (108, 1);
274 INSERT INTO t1 VALUES (109, 1);
275 --save_master_pos
276
277 --connection server_2
278 --source include/start_slave.inc
279 --sync_with_master
280 SELECT * FROM t1 WHERE a >= 100 ORDER BY a;
281
282
283 --echo *** MDEV-6917: Parallel replication: "Commit failed due to failure of an earlier commit on which this one depends", but no prior failure seen **
284
285 --connection server_1
286 CREATE TABLE t3 (a INT PRIMARY KEY, b INT, KEY b_idx(b)) ENGINE=InnoDB;
287 INSERT INTO t3 VALUES (1,NULL), (2,2), (3,NULL), (4,4), (5, NULL), (6, 6);
288 CREATE TABLE t4 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB;
289 # MDEV-515 takes X-lock on the table for the first insert.
290 # So concurrent insert won't happen on the table
291 INSERT INTO t4 VALUES(100, 100);
292
293 # We need statement binlog format to be able to inject debug_sync statements
294 # on the slave with calls to foo().
295 SET @old_format= @@SESSION.binlog_format;
296 SET binlog_format='statement';
297 --save_master_pos
298
299 --connection server_2
300 --sync_with_master
301 --source include/stop_slave.inc
302 CHANGE MASTER TO master_use_gtid=no;
303
304 --connection server_1
305
306 # Create a group commit with three transactions T1, T2, T3.
307 # T2 will block T1 on the slave where we will make it run first, so it will be
308 # deadlock killed.
309 # The bug was that in this case, T3 was signalled to fail due to T2 failing,
310 # even though the retry of T2 was later successful.
311
312 --connect (con1,127.0.0.1,root,,test,$SERVER_MYPORT_1,)
313 SET @old_format= @@SESSION.binlog_format;
314 SET binlog_format='statement';
315 BEGIN;
316 INSERT INTO t4 VALUES (10, foo(1, 'before_execute_sql_command WAIT_FOR t1_start', ''));
317 UPDATE t3 SET b=NULL WHERE a=6;
318 SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued1 WAIT_FOR master_cont1';
319 send COMMIT;
320 --connection server_1
321 SET debug_sync='now WAIT_FOR master_queued1';
322
323 --connect (con2,127.0.0.1,root,,test,$SERVER_MYPORT_1,)
324 SET @old_format= @@SESSION.binlog_format;
325 SET binlog_format='statement';
326 BEGIN;
327 INSERT INTO t4 VALUES (20, foo(2, 'group_commit_waiting_for_prior SIGNAL t2_waiting', ''));
328 DELETE FROM t3 WHERE b <= 3;
329 SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued2';
330 send COMMIT;
331
332 --connection server_1
333 SET debug_sync='now WAIT_FOR master_queued2';
334
335 --connect (con3,127.0.0.1,root,,test,$SERVER_MYPORT_1,)
336 SET @old_format= @@SESSION.binlog_format;
337 SET binlog_format='statement';
338 BEGIN;
339 INSERT INTO t4 VALUES (30, foo(3, 'before_execute_sql_command WAIT_FOR t3_start', 'group_commit_waiting_for_prior SIGNAL t3_waiting'));
340 INSERT INTO t3 VALUES (7,7);
341 SET debug_sync='commit_after_release_LOCK_prepare_ordered SIGNAL master_queued3';
342 send COMMIT;
343
344 --connection server_1
345 SET debug_sync='now WAIT_FOR master_queued3';
346 SET debug_sync='now SIGNAL master_cont1';
347
348 --connection con1
349 REAP;
350 SET binlog_format=@old_format;
351 --connection con2
352 REAP;
353 SET binlog_format=@old_format;
354 --connection con3
355 REAP;
356 SET debug_sync='RESET';
357 SET binlog_format=@old_format;
358
359 --connection server_1
360 --save_master_pos
361 SELECT * FROM t3 ORDER BY a;
362
363
364 --connection server_2
365 let $old_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
366 SET @old_dbug=@@GLOBAL.debug_dbug;
367 SET GLOBAL debug_dbug="+d,thd_need_ordering_with_force";
368 --source include/start_slave.inc
369 # First, wait for T2 to complete up to where it is waiting for T1 to group
370 # commit for both of them. This will set locks that will block T1, causing
371 # a deadlock kill and retry of T2. T1 and T3 are still blocked at the start
372 # of each their SQL statements.
373 SET debug_sync='now WAIT_FOR t2_waiting';
374 # Now let T3 move on until the point where it is itself ready to commit.
375 SET debug_sync='now SIGNAL t3_start';
376 SET debug_sync='now WAIT_FOR t3_waiting';
377 # Now T2 and T3 are set up, so we can let T1 proceed.
378 SET debug_sync='now SIGNAL t1_start';
379 # Now we can wait for the slave to catch up.
380 # We should see T2 being deadlock killed and retried.
381 # The bug was that T2 deadlock kill would cause T3 to fail due to failure
382 # of an earlier commit. This is wrong as T2 did not fail, it was only
383 # retried.
384 --sync_with_master
385 SET GLOBAL debug_dbug=@old_dbug;
386 SET debug_sync='RESET';
387 let $new_retry= query_get_value(SHOW STATUS LIKE 'Slave_retried_transactions', Value, 1);
388 let $ps_value= query_get_value(select last_trans_retry_count from
389 performance_schema.replication_applier_status_by_worker where
390 last_trans_retry_count > 0, last_trans_retry_count, 1);
391 let $assert_text= Performance Schema retries should match with actual retries;
392 let $assert_cond= "$ps_value" = $new_retry - $old_retry;
393 source include/assert.inc;
394 SELECT * FROM t3 ORDER BY a;
395
396
397 --connection server_1
398 SET binlog_format=@old_format;
399
400
401 # Clean up of the above part.
402 --connection server_2
403 --source include/stop_slave.inc
404 SET GLOBAL slave_parallel_threads=@old_parallel_threads;
405 --source include/start_slave.inc
406
407 --connection server_1
408 DROP TABLE t1, t2, t3, t4;
409 DROP function foo;
410
411 --sync_slave_with_master server_2
412
413 #
414 # MDEV-33303: slave_parallel_mode=optimistic should not report the mode's
415 # specific temporary errors.
416 #
417
418 --connection server_2
419 --source include/stop_slave.inc
420 SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads;
421 SET GLOBAL slave_parallel_threads=4;
422
423 --connection server_1
424 # The problem occurred in the code path for row-based updates in tables
425 # with no primary/unique key, where a scan is needed.
426 CREATE TABLE t1 (a INT, b VARCHAR(123)) ENGINE=InnoDB;
427 INSERT INTO t1 VALUES(1, 'asdf');
428 UPDATE t1 SET b='zxf1' WHERE a=1;
429 UPDATE t1 SET b='\n' WHERE a=1;
430
431 --connection server_2
432 # Inject a small sleep in the code that makes the race easier to hit.
433 SET @old_dbug=@@GLOBAL.debug_dbug;
434 SET GLOBAL debug_dbug="+d,write_row_inject_sleep_before_ha_write_row";
435 --source include/start_slave.inc
436
437 --connection server_1
438 # Here, we would get errors in the slave's error log:
439 # [ERROR] mariadbd: Can't find record in 't1'
440 --sync_slave_with_master server_2
441
442 --connection server_1
443 DROP TABLE t1;
444 --sync_slave_with_master server_2
445 --source include/stop_slave.inc
446 SET GLOBAL debug_dbug=@old_dbug;
447 SET GLOBAL slave_parallel_threads=@old_parallel_threads;
448 --source include/start_slave.inc
449
450
451 #
452 # MDEV-12746 rpl.rpl_parallel_optimistic_nobinlog fails committing out of order at retry
453 #
454
455 --connection server_1
456 CREATE TABLE t1 (a int PRIMARY KEY, b INT) ENGINE=InnoDB;
457 # MDEV-515 takes X-lock on the table for the first insert.
458 # So concurrent insert won't happen on the table
459 INSERT INTO t1 VALUES(100, 100);
460
461 # Replicate create-t1 and prepare to re-start slave in optimistic mode
462 --sync_slave_with_master server_2
463 --source include/stop_slave.inc
464 SET @old_parallel_threads=@@GLOBAL.slave_parallel_threads;
465 SET @@GLOBAL.slave_parallel_threads=5;
466 SET @old_parallel_mode=@@GLOBAL.slave_parallel_mode;
467 SET @@GLOBAL.slave_parallel_mode='aggressive';
468 SET @old_lock_wait_timeout=@@GLOBAL.innodb_lock_wait_timeout;
469 SET @@GLOBAL.innodb_lock_wait_timeout=2;
470 SET @old_slave_transaction_retries=@@GLOBAL.slave_transaction_retries;
471 SET @@GLOBAL.slave_transaction_retries=1;
472
473 --echo # Spoilers on the slave side causing temporary errors
474 --connect (spoiler_21,127.0.0.1,root,,test,$SLAVE_MYPORT)
475 BEGIN;
476 INSERT INTO t1 SET a=1,b=2;
477
478 --connect (spoiler_22,127.0.0.1,root,,test,$SLAVE_MYPORT)
479 BEGIN;
480 INSERT INTO t1 SET a=2,b=2;
481
482 --echo # Master payload
483 --connection server_1
484 SET @@SESSION.GTID_SEQ_NO=1000;
485 INSERT INTO t1 SET a=1,b=1;
486 SET @@SESSION.GTID_SEQ_NO=1001;
487 INSERT INTO t1 SET a=2,b=1;
488
489 --echo # Start slave whose both appliers is destined to being blocked
490 --connection server_2
491 SET @old_dbug= @@GLOBAL.debug_dbug;
492 SET @@GLOBAL.debug_dbug="+d,rpl_parallel_simulate_wait_at_retry";
493 --source include/start_slave.inc
494
495 --echo # Make sure both workers are waiting at their sync points
496 --let $wait_condition= SELECT count(*)=2 FROM information_schema.processlist WHERE state LIKE '%debug sync point%';
497 --source include/wait_condition.inc
498
499
500 --echo # Signal to the 1st to proceed after it has reached termination state
501 SET @@DEBUG_SYNC='now SIGNAL proceed_by_1000';
502 --connection spoiler_21
503 ROLLBACK;
504
505 --let $wait_condition= SELECT count(*)=1 FROM information_schema.processlist WHERE state LIKE '%debug sync point%';
506 --source include/wait_condition.inc
507
508 --echo # Release the 2nd worker to proceed
509 --connection spoiler_22
510 ROLLBACK;
511 --connection server_2
512 SET @@DEBUG_SYNC='now SIGNAL proceed_by_1001';
513
514 --echo # observe how it all ends
515 if (`SELECT count(*) = 1 FROM t1 WHERE a = 1`)
516 {
517 --echo "*** Unexpected commit by the first Worker ***"
518 SELECT * from t1;
519 --die
520 }
521
522 --echo # Wait for the workers to go home and check the result of applying
523 --let $wait_condition=SELECT count(*) = 0 FROM information_schema.processlist WHERE command = 'Slave_worker'
524 --source include/wait_condition.inc
525 if (`SELECT count(*) = 1 FROM t1 WHERE a = 2`)
526 {
527 --echo
528 --echo "*** Error: congrats, you hit MDEV-12746 issue. ***"
529 --echo
530 --die
531 }
532 --echo # which is OK
533
534 #
535 # Clean up
536 #
537 --connection server_2
538 --let $rpl_only_running_threads= 1
539 --source include/stop_slave.inc
540 SET @@GLOBAL.slave_parallel_threads=@old_parallel_threads;
541 SET @@GLOBAL.slave_parallel_mode=@old_parallel_mode;
542 SET @@GLOBAL.innodb_lock_wait_timeout=@old_lock_wait_timeout;
543 SET @@GLOBAL.slave_transaction_retries=@old_slave_transaction_retries;
544 SET @@GLOBAL.debug_dbug=@old_dbug;
545 SET debug_sync='RESET';
546 --source include/start_slave.inc
547
548 --connection server_1
549 DROP TABLE t1;
550
551 --sync_slave_with_master server_2
552
553 --source include/rpl_end.inc