Debian Codegraph @

storage/maria/unittest/ma_test_recovery.pl

raw
1 #!/usr/bin/env perl
2
3 use Getopt::Long;
4 use File::Copy;
5 use File::Compare;
6 use File::Basename;
7 use Digest::MD5;
8
9 $|= 1;
10 $^W = 1; # warnings, because env cannot parse 'perl -w'
11 $VER= "1.2";
12
13 $opt_version= 0;
14 $opt_help= 0;
15 $opt_verbose= 0;
16 $opt_abort_on_error=0;
17
18 my $silent= "-s";
19 my $maria_path; # path to "storage/maria"
20 my $maria_exe_path; # path to executables (ma_test1, aria_chk etc)
21 my $tmp= "./tmp";
22 my $my_progname= $0;
23 my $suffix;
24 my $zerofilled_tables= 0;
25
26 $my_progname=~ s/.*[\/]//;
27 $maria_path= dirname($0) . "/..";
28
29 main();
30
31 ####
32 #### main function
33 ####
34
35 sub main
36 {
37 my ($res, $table);
38
39 if (!GetOptions("abort-on-error", "help", "version", "verbose"))
40 {
41 $flag_exit= 1;
42 }
43 if ($opt_version)
44 {
45 print "$my_progname version $VER\n";
46 exit(0);
47 }
48 usage() if ($opt_help || $flag_exit);
49
50 $suffix= ( $^O =~ /win/i && $^O !~ /darwin/i ) ? ".exe" : "";
51 $maria_exe_path= "$maria_path/release";
52 # we use -f, sometimes -x is unexpectedly false in Cygwin
53 if ( ! -f "$maria_exe_path/ma_test1$suffix" )
54 {
55 $maria_exe_path= "$maria_path/relwithdebinfo";
56 if ( ! -f "$maria_exe_path/ma_test1$suffix" )
57 {
58 $maria_exe_path= "$maria_path/debug";
59 if ( ! -f "$maria_exe_path/ma_test1$suffix" )
60 {
61 $maria_exe_path= $maria_path;
62 if ( ! -f "$maria_exe_path/ma_test1$suffix" )
63 {
64 die("Cannot find ma_test1 executable\n");
65 }
66 }
67 }
68 }
69
70 # test data is always put in the current directory or a tmp subdirectory
71 # of it
72
73 if (! -d "$tmp")
74 {
75 mkdir $tmp;
76 }
77 print "ARIA RECOVERY TESTS\n";
78
79 # To not flood the screen, we redirect all the commands below to a text file
80 # and just give a final error if their output is not as expected
81
82 open (MY_LOG, ">$tmp/ma_test_recovery.output") or die "Can't open log file\n";
83 print MY_LOG "Testing the REDO PHASE ALONE\n";
84
85 # runs a program inserting/deleting rows, then moves the resulting table
86 # elsewhere; applies the log and checks that the data file is
87 # identical to the saved original.
88
89 my @t= ("ma_test1$suffix $silent -M -T -c",
90 "ma_test2$suffix $silent -L -K -W -P -M -T -c -d500",
91 "ma_test2$suffix $silent -M -T -c -b65000",
92 "ma_test2$suffix $silent -M -T -c -b65000 -d800",
93 "ma_test1$suffix $silent -M -T -c -C",
94 "ma_test2$suffix $silent -L -K -W -P -M -T -c -d500 -C",
95 #"ma_rt_test$suffix $silent -M -T -c -C",
96 # @todo: also add to @t2
97 );
98
99 foreach my $prog (@t)
100 {
101 unlink <aria_log.* aria_log_control>;
102 my $prog_no_suffix= $prog;
103 $prog_no_suffix=~ s/$suffix// if ($suffix);
104 print MY_LOG "TEST WITH $prog_no_suffix\n";
105 $res= my_exec("$maria_exe_path/$prog");
106 print MY_LOG $res;
107 # derive table's name from program's name
108 if ($prog =~ m/^ma_(\S+)\s.*/)
109 {
110 $table= $1;
111 }
112 else
113 {
114 die("can't guess table name");
115 }
116 $com= "$maria_exe_path/aria_chk$suffix -dvv $table ";
117 $com.= "| grep -v \"Creation time:\" | grep -v \"recover time:\" | grep -v \"file length\" | grep -v \"LSNs:\" | grep -v \"UUID:\"";
118 $com.= "> $tmp/aria_chk_message.good.txt 2>&1";
119 my_exec($com);
120 my $checksum= my_exec("$maria_exe_path/aria_chk$suffix -dss $table");
121 move("$table.MAD", "$tmp/$table-good.MAD") ||
122 die "Can't move $table.MAD to $tmp/$table-good.MAD\n";
123 move("$table.MAI", "$tmp/$table-good.MAI") ||
124 die "Can't move $table.MAI to $tmp/$table-good.MAI\n";
125 apply_log($table, "shouldnotchangelog");
126 check_table_is_same($table, $checksum);
127 $res= physical_cmp($table, "$tmp/$table-good");
128 print MY_LOG $res;
129 print MY_LOG "testing idempotency\n";
130 apply_log($table, "shouldnotchangelog");
131 check_table_is_same($table, $checksum);
132 $res= physical_cmp($table, "$tmp/$table-good");
133 print MY_LOG $res;
134 }
135
136 print MY_LOG "Testing the REDO AND UNDO PHASE\n";
137 # The test programs look like:
138 # work; commit (time T1); work; exit-without-commit (time T2)
139 # We first run the test program and let it exit after T1's commit.
140 # Then we run it again and let it exit at T2. Then we compare
141 # and expect identity.
142
143 my @take_checkpoints= ("no", "yes");
144 my @blobs= ("", "-b32768");
145 my @test_undo= (1, 2, 3, 4);
146 my @t2= ("ma_test1$suffix $silent -M -T -c -N blob -H1",
147 "--testflag=1",
148 "--testflag=2 --test-undo=",
149 "ma_test1$suffix $silent -M -T -c -N blob -H2",
150 "--testflag=3",
151 "--testflag=4 --test-undo=",
152 "ma_test1$suffix $silent -M -T -c -N blob -H2 --versioning",
153 "--testflag=3",
154 "--testflag=4 --test-undo=",
155 "ma_test1$suffix $silent -M -T -c -N blob -H2",
156 "--testflag=2",
157 "--testflag=3 --test-undo=",
158 "ma_test2$suffix $silent -L -K -W -P -M -T -c blob -H1",
159 "-t1",
160 "-t2 -A",
161 "ma_test2$suffix $silent -L -K -W -P -M -T -c blob -H1",
162 "-t1",
163 "-t6 -A");
164
165 foreach my $take_checkpoint (@take_checkpoints)
166 {
167 my ($i, $j, $k, $commit_run_args, $abort_run_args);
168 # we test table without blobs and then table with blobs
169 for ($i= 0; defined($blobs[$i]); $i++)
170 {
171 for ($j= 0; defined($test_undo[$j]); $j++)
172 {
173 # first iteration tests rollback of insert, second tests rollback of delete
174 # -N (create NULL fields) is needed because --test-undo adds it anyway
175 for ($k= 0; defined($t2[$k]); $k+= 3)
176 {
177 $prog= $t2[$k];
178 $prog=~ s/blob/$blobs[$i]/;
179 if ("$take_checkpoint" eq "no") {
180 $prog=~ s/\s+\-H[0-9]+//;
181 }
182 $commit_run_args= $t2[$k + 1];
183 $abort_run_args= $t2[$k + 2];
184 unlink <aria_log.* aria_log_control>;
185 my $prog_no_suffix= $prog;
186 $prog_no_suffix=~ s/$suffix// if ($suffix);
187 print MY_LOG "TEST WITH $prog_no_suffix $commit_run_args (commit at end)\n";
188 $res= my_exec("$maria_exe_path/$prog $commit_run_args");
189 print MY_LOG $res;
190 # derive table's name from program's name
191 if ($prog =~ m/^ma_(\S+)\s.*/)
192 {
193 $table= $1;
194 }
195 else
196 {
197 die("can't guess table name");
198 }
199 $com= "$maria_exe_path/aria_chk$suffix -dvv $table ";
200 $com.= "| grep -v \"Creation time:\" | grep -v \"recover time:\" | grep -v \"recover time:\" |grep -v \"file length\" | grep -v \"LSNs:\" | grep -v \"UUID:\" ";
201 $com.= "> $tmp/aria_chk_message.good.txt 2>&1";
202 $res= my_exec($com);
203 print MY_LOG $res;
204 $checksum= my_exec("$maria_exe_path/aria_chk$suffix -dss $table");
205 move("$table.MAD", "$tmp/$table-good.MAD") ||
206 die "Can't move $table.MAD to $tmp/$table-good.MAD\n";
207 move("$table.MAI", "$tmp/$table-good.MAI") ||
208 die "Can't move $table.MAI to $tmp/$table-good.MAI\n";
209 unlink <aria_log.* aria_log_control>;
210 print MY_LOG "TEST WITH $prog_no_suffix $abort_run_args$test_undo[$j] (additional aborted work)\n";
211 $res= my_exec("$maria_exe_path/$prog $abort_run_args$test_undo[$j]");
212 print MY_LOG $res;
213 copy("$table.MAD", "$tmp/$table-before_undo.MAD") ||
214 die "Can't copy $table.MAD to $tmp/$table-before_undo.MAD\n";
215 copy("$table.MAI", "$tmp/$table-before_undo.MAI") ||
216 die "Can't copy $table.MAI to $tmp/$table-before_undo.MAI\n";
217
218 # The lines below seem unneeded, will be removed soon
219 # We have to copy and restore logs, as running aria_read_log will
220 # change the aria_control_file
221 # rm -f $tmp/aria_log.* $tmp/aria_log_control
222 # cp $maria_path/aria_log* $tmp
223
224 if ($test_undo[$j] != 3) {
225 apply_log($table, "shouldchangelog"); # should undo aborted work
226 } else {
227 # probably nothing to undo went to log or data file
228 apply_log($table, "dontknow");
229 }
230 copy("$table.MAD", "$tmp/$table-after_undo.MAD") ||
231 die "Can't copy $table.MAD to $tmp/$table-after_undo.MAD\n";
232 copy("$table.MAI", "$tmp/$table-after_undo.MAI") ||
233 die "Can't copy $table.MAI to $tmp/$table-after_undo.MAI\n";
234
235 # It is impossible to do a "cmp" between .good and .after_undo,
236 # because the UNDO phase generated log
237 # records whose LSN tagged pages. Another reason is that rolling back
238 # INSERT only marks the rows free, does not empty them
239 # (optimization), so traces of the INSERT+rollback remain.
240
241 check_table_is_same($table, $checksum);
242 print MY_LOG "testing idempotency\n";
243 apply_log($table, "shouldnotchangelog");
244 check_table_is_same($table, $checksum);
245 $res= physical_cmp($table, "$tmp/$table-after_undo");
246 print MY_LOG $res;
247 print MY_LOG "testing applying of CLRs to recreate table\n";
248 unlink <$table.MA?>;
249 # cp $tmp/aria_log* $maria_path #unneeded
250 apply_log($table, "shouldnotchangelog");
251 check_table_is_same($table, $checksum);
252 $res= physical_cmp($table, "$tmp/$table-after_undo");
253 print MY_LOG $res;
254 }
255 unlink <$table.* $tmp/$table* $tmp/aria_chk_*.txt $tmp/aria_read_log_$table.txt>;
256 }
257 }
258 }
259
260 if ($? >> 8) {
261 print "Some test failed\n";
262 exit(1);
263 }
264
265 close(MY_LOG);
266 # also note that aria_chk -dvv shows differences for ma_test2 in UNDO phase,
267 # this is normal: removing records does not shrink the data/key file,
268 # does not put back the "analyzed,optimized keys"(etc) index state.
269 `diff -b $maria_path/unittest/ma_test_recovery.expected $tmp/ma_test_recovery.output`;
270 if ($? >> 8) {
271 print "UNEXPECTED OUTPUT OF TESTS, FAILED";
272 print " (zerofilled $zerofilled_tables tables)\n";
273 print "For more info, do diff -b $maria_path/unittest/ma_test_recovery.expected ";
274 print "$tmp/ma_test_recovery.output\n";
275 exit(1);
276 }
277 print "ALL RECOVERY TESTS OK (zerofilled $zerofilled_tables tables)\n";
278 }
279
280 ####
281 #### check_table_is_same
282 ####
283
284 sub check_table_is_same
285 {
286 my ($table, $checksum)= @_;
287 my ($com, $checksum2, $res);
288
289 # Computes checksum of new table and compares to checksum of old table
290 # Shows any difference in table's state (info from the index's header)
291 # Data/key file length is random in ma_test2 (as it uses srand() which
292 # may differ between machines).
293
294 if ($opt_verbose)
295 {
296 print "checking if table $table has changed\n";
297 }
298
299 $com= "$maria_exe_path/aria_chk$suffix -dvv $table | grep -v \"Creation time:\" | grep -v \"recover time:\"";
300 $com.= "| grep -v \"file length\" | grep -v \"LSNs:\" | grep -v \"UUID:\" > $tmp/aria_chk_message.txt 2>&1";
301 $res= my_exec2($com);
302 print MY_LOG $res;
303 $res= my_exec2("$maria_exe_path/aria_chk$suffix -ss -e --read-only $table");
304 print MY_LOG $res;
305 $checksum2= my_exec2("$maria_exe_path/aria_chk$suffix -dss $table");
306 if ("$checksum" ne "$checksum2")
307 {
308 print MY_LOG "checksum differs for $table before and after recovery\n";
309 return 1;
310 }
311
312 $com= "diff $tmp/aria_chk_message.good.txt $tmp/aria_chk_message.txt ";
313 $com.= "> $tmp/aria_chk_diff.txt || true";
314 $res= my_exec2($com);
315 print MY_LOG $res;
316
317 if (-s "$tmp/aria_chk_diff.txt")
318 {
319 print MY_LOG "Differences in aria_chk -dvv, recovery not yet perfect !\n";
320 print MY_LOG "========DIFF START=======\n";
321 open(MY_FILE, "<$tmp/aria_chk_diff.txt") || die "Can't open file aria_chk_diff.txt\n";
322 while (<MY_FILE>)
323 {
324 print MY_LOG $_;
325 }
326 close(MY_FILE);
327 print MY_LOG "========DIFF END=======\n";
328 }
329 }
330
331 ####
332 #### apply_log
333 ####
334
335 sub apply_log
336 {
337 my ($table, $shouldchangelog)= @_;
338 my ($log_md5, $log_md5_2);
339
340 # applies log, can verify if applying did write to log or not
341
342 if ("$shouldchangelog" ne "shouldnotchangelog" &&
343 "$shouldchangelog" ne "shouldchangelog" &&
344 "$shouldchangelog" ne "dontknow" )
345 {
346 print MY_LOG "bad argument '$shouldchangelog'\n";
347 return 1;
348 }
349 foreach (<aria_log.*>)
350 {
351 $log_md5.= md5_conv($_);
352 }
353 print MY_LOG "applying log\n";
354 my_exec("$maria_exe_path/aria_read_log$suffix -a > $tmp/aria_read_log_$table.txt");
355 foreach (<aria_log.*>)
356 {
357 $log_md5_2.= md5_conv($_);
358 }
359 if ("$log_md5" ne "$log_md5_2" )
360 {
361 if ("$shouldchangelog" eq "shouldnotchangelog")
362 {
363 print MY_LOG "aria_read_log should not have modified the log\n";
364 return 1;
365 }
366 }
367 elsif ("$shouldchangelog" eq "shouldchangelog")
368 {
369 print MY_LOG "aria_read_log should have modified the log\n";
370 return 1;
371 }
372 }
373
374 ####
375 #### md5_conv
376 ####
377
378 sub md5_conv
379 {
380 my ($file)= @_;
381
382 open(FILE, $file) or die "Can't open '$file': $!\n";
383 binmode(FILE);
384 my $md5= Digest::MD5->new;
385 $md5->addfile(FILE);
386 close (FILE);
387 return $md5->hexdigest . "\n";
388 }
389
390 ####
391 #### physical_cmp: compares two tables (MAI and MAD) physically;
392 #### uses zerofill-keep-lsn to reduce irrelevant differences.
393 ####
394
395 sub physical_cmp
396 {
397 my ($table1, $table2)= @_;
398 my ($zerofilled, $ret_text)= (0, "");
399 #return `cmp $table1.MAD $table2.MAD`.`cmp $table1.MAI $table2.MAI`;
400 foreach my $file_suffix ("MAD", "MAI")
401 {
402 my $file1= "$table1.$file_suffix";
403 my $file2= "$table2.$file_suffix";
404 my $res= File::Compare::compare($file1, $file2);
405 die() if ($res == -1);
406 if ($res == 1 # they differ
407 and !$zerofilled)
408 {
409 # let's try with --zerofill-keep-lsn
410 $zerofilled= 1; # but no need to do it twice
411 $zerofilled_tables= $zerofilled_tables + 1;
412 my $table_no= 1;
413 foreach my $table ($table1, $table2)
414 {
415 # save original tables to restore them later
416 copy("$table.MAD", "$tmp/before_zerofill$table_no.MAD") || die();
417 copy("$table.MAI", "$tmp/before_zerofill$table_no.MAI") || die();
418 $com= "$maria_exe_path/aria_chk$suffix -ss --zerofill-keep-lsn --skip-update-state $table";
419 $res= `$com`;
420 print MY_LOG $res;
421 $table_no= $table_no + 1;
422 }
423 $res= File::Compare::compare($file1, $file2);
424 die() if ($res == -1);
425 }
426 $ret_text.= "$file1 and $file2 differ\n" if ($res != 0);
427 }
428 if ($zerofilled)
429 {
430 my $table_no= 1;
431 foreach my $table ($table1, $table2)
432 {
433 move("$tmp/before_zerofill$table_no.MAD", "$table.MAD") || die();
434 move("$tmp/before_zerofill$table_no.MAI", "$table.MAI") || die();
435 $table_no= $table_no + 1;
436 }
437 }
438 return $ret_text;
439 }
440
441
442 sub my_exec
443 {
444 my($command)= @_;
445 my $res;
446 if ($opt_verbose)
447 {
448 print "$command\n";
449 }
450 $res= `$command`;
451 if ($? != 0 && $opt_abort_on_error)
452 {
453 exit(1);
454 }
455 return $res;
456 }
457
458 sub my_exec2
459 {
460 my($command)= @_;
461 my $res, $err;
462 $res= `$command`;
463 if ($? != 0 && $opt_abort_on_error)
464 {
465 $err= $?;
466 print "$command\n";
467 print "failed with error: $err\n";
468 exit(1);
469 }
470 return $res;
471 }
472
473
474 ####
475 #### usage
476 ####
477
478 sub usage
479 {
480 print <<EOF;
481 $my_progname version $VER
482
483 Description:
484
485 Run various Aria recovery tests and print the results
486
487 Options
488 --help Show this help and exit.
489
490 --abort-on-error Abort at once in case of error.
491 --verbose Show commands while there are executing.
492 --version Show version number and exit.
493
494 EOF
495 exit(0);
496 }