| 1 |
#!/usr/bin/perl |
| 2 |
|
| 3 |
my $html = 0; # set to 1 to output a nicely formatted HTML page |
| 4 |
|
| 5 |
my %toDo=("csv"=>0, "epub"=>0, "html"=>0, "odf"=>0, "raw"=>1, "svg"=>0, "valgrind"=>0 ); |
| 6 |
|
| 7 |
my $pass_colour = "11dd11"; |
| 8 |
my $fail_colour = "dd1111"; |
| 9 |
my $warn_colour = "e59800"; |
| 10 |
my $skip_colour = "9999dd"; |
| 11 |
|
| 12 |
sub DisplayCell |
| 13 |
{ |
| 14 |
my ($bgColor, $text) = @_; |
| 15 |
|
| 16 |
print "<td style='background-color: #$bgColor;'>$text</td>\n"; |
| 17 |
} |
| 18 |
|
| 19 |
sub DiffTest |
| 20 |
{ |
| 21 |
my ($command, $command2, $file, $extension, $condition) = @_; |
| 22 |
my $result = "passed"; |
| 23 |
my $comment = ""; |
| 24 |
|
| 25 |
if (!$condition) { |
| 26 |
DisplayCell($skip_colour, "skipped") if ($html); |
| 27 |
return "skipped"; |
| 28 |
} |
| 29 |
|
| 30 |
my $errPath = $file . ".$extension.err"; |
| 31 |
my $rawPath = $file . ".$extension"; |
| 32 |
my $newRawPath = $file . ".$extension.new"; |
| 33 |
my $diffPath = $file . ".$extension.diff"; |
| 34 |
|
| 35 |
# generate a new raw output to compare with |
| 36 |
`$command $file 1> $newRawPath`; |
| 37 |
if ($command2) { |
| 38 |
`mv $newRawPath $newRawPath.tmp`; |
| 39 |
`$command2 $newRawPath.tmp 1> $newRawPath`; |
| 40 |
`rm $newRawPath.tmp`; |
| 41 |
} |
| 42 |
|
| 43 |
# HACK: check if there is a raw file with _some_ contents. If not, we've had a segfault |
| 44 |
my $err = ""; |
| 45 |
my $diff = ""; |
| 46 |
$newRaw=`cat $newRawPath`; |
| 47 |
if ($newRaw eq "") { |
| 48 |
$err = "Segmentation fault"; |
| 49 |
`echo $err > $errPath`; |
| 50 |
} |
| 51 |
|
| 52 |
if ($err ne "") { |
| 53 |
$result = "fail"; |
| 54 |
} else { |
| 55 |
# remove the generated (empty) error file |
| 56 |
`rm -f $errPath`; |
| 57 |
|
| 58 |
# diff the stored raw data with the newly generated raw data |
| 59 |
`diff -u $rawPath $newRawPath 1>$diffPath 2>$diffPath`; |
| 60 |
$diff=`cat $diffPath | grep -v "No differences encountered"`; |
| 61 |
|
| 62 |
if ($diff ne "") { |
| 63 |
$result = "changed"; |
| 64 |
} else { |
| 65 |
`rm -f $diffPath`; |
| 66 |
} |
| 67 |
} |
| 68 |
|
| 69 |
# remove the generated raw file |
| 70 |
`rm -f $newRawPath`; |
| 71 |
|
| 72 |
# DISPLAYING RESULTS |
| 73 |
if ($html) { |
| 74 |
my $bgColor; |
| 75 |
if ($diff eq "" && $err eq "") { |
| 76 |
$bgColor = $pass_colour; |
| 77 |
} elsif ($err ne "") { |
| 78 |
$bgColor = $fail_colour; |
| 79 |
} else { |
| 80 |
$bgColor = $warn_colour; |
| 81 |
} |
| 82 |
|
| 83 |
if ($err ne "" || $diff ne "") { |
| 84 |
$comment = " <a href='" . ($err ne "" ? $errPath : $diffPath) . "'>" . ($err ne "" ? "error" : "diff") . "<a>"; |
| 85 |
} |
| 86 |
|
| 87 |
DisplayCell($bgColor, $result . $comment); |
| 88 |
} else { |
| 89 |
if ($err ne "" || $diff ne "") { |
| 90 |
$comment = ($err ne "" ? "(error: " : "(diff: ") . ($err ne "" ? $errPath : $diffPath) . ")"; |
| 91 |
} |
| 92 |
print "! $file diff (using $command): $result $comment\n"; |
| 93 |
} |
| 94 |
|
| 95 |
return $result; |
| 96 |
} |
| 97 |
|
| 98 |
sub CgTest |
| 99 |
{ |
| 100 |
my ($command, $file) = @_; |
| 101 |
|
| 102 |
my $callgraph = `sd2raw -c $file`; |
| 103 |
chomp($callgraph); |
| 104 |
|
| 105 |
return $callgraph; |
| 106 |
} |
| 107 |
|
| 108 |
sub CheckVg |
| 109 |
{ |
| 110 |
my ($file,$msg) = @_; |
| 111 |
open VG, "$vgPath"; |
| 112 |
my $result = "passed"; |
| 113 |
my $valgrind = 0; |
| 114 |
my $vgOutput; |
| 115 |
while (<VG>) { |
| 116 |
if (/^\=\=/) { |
| 117 |
$vgOutput .= $_; |
| 118 |
if (/definitely lost: [1-9]/ || |
| 119 |
/ERROR SUMMARY: [1-9]/ || |
| 120 |
/Invalid read of/) { |
| 121 |
$valgrind = 1; |
| 122 |
} |
| 123 |
} |
| 124 |
} |
| 125 |
close VG; |
| 126 |
|
| 127 |
`rm -f $vgPath`; |
| 128 |
if ($valgrind) { |
| 129 |
open VG, ">$vgPath"; |
| 130 |
print VG $vgOutput; |
| 131 |
close VG; |
| 132 |
$result="fail"; |
| 133 |
$valgrind = 1; |
| 134 |
} |
| 135 |
|
| 136 |
if ($html) { |
| 137 |
($valgrind eq 0 ? DisplayCell($pass_colour, "passed") : DisplayCell($fail_colour, "failed <a href='$vgPath'>log<a>")); |
| 138 |
} else { |
| 139 |
print $msg. " " . ($valgrind eq "0" ? "passed" : "failed (see '$vgPath')") . "\n"; |
| 140 |
} |
| 141 |
|
| 142 |
return $result; |
| 143 |
} |
| 144 |
|
| 145 |
sub RegTest |
| 146 |
{ |
| 147 |
my $i; |
| 148 |
# basic raw/html endOfLine |
| 149 |
my $endLine=($html?"<br>\n":"\n"); |
| 150 |
|
| 151 |
# list of directories containing test files |
| 152 |
my @listDirectory = ( "Calc3.1", "Calc4", "Calc5", "Draw3.1", "Draw4", "Draw5", "Pres5", "Text3.1", "Text4", "Text5" ); |
| 153 |
|
| 154 |
# list of commands to test |
| 155 |
# type => [ "name", "extension", "array of accepted types", "command", "filter" ] |
| 156 |
my @commands=("raw", "csv", "epub", "html", "svg", "odf"); |
| 157 |
my %listCommand = ( |
| 158 |
"csv" => [ "CSV", "csv", ["S"], "sdc2csv", 0 ], |
| 159 |
"epub" => [ "Epub", "epub", ["G", "P", "S", "T"], "sd2epub", 0], |
| 160 |
"html" => [ "Html", "html", ["T"], "sdw2html", 0], |
| 161 |
"odf" => ["WriterPerfect", "writerperfect", ["G", "P", "S", "T"], "sd2odf", "xmllint --format"], |
| 162 |
"raw"=>["Raw", "raw", ["G", "P", "S", "T"], "sd2raw", 0], |
| 163 |
"svg" => ["SVG", "svg", ["G", "P"], "sd2svg", 0] |
| 164 |
); |
| 165 |
|
| 166 |
# valgrind command |
| 167 |
my $vgVersionOutput = `valgrind --version`; |
| 168 |
my $vgCommand=( ($vgVersionOutput =~ /\-2.1/ || $vgVersionOutput =~ /\-2.2/) ? "valgrind --tool=memcheck" : "valgrind" ); |
| 169 |
# init failures |
| 170 |
my $callGraphFailures = 0; |
| 171 |
my %diffFailures, %valgrindFailures; |
| 172 |
foreach $i ( @commands ) { |
| 173 |
$diffFailures{$i}=0; |
| 174 |
$valgrindFailures{$i}=0; |
| 175 |
} |
| 176 |
foreach my $directory ( @listDirectory ) { |
| 177 |
if ($html) { |
| 178 |
print "<b>Regression testing the " . $directory . " parser</b><br>\n"; |
| 179 |
print "<table>\n"; |
| 180 |
print "<tr>\n"; |
| 181 |
print "<td style='background-color: rgb(204, 204, 255);'><b>File</b></td>\n"; |
| 182 |
print "<td style='background-color: rgb(204, 204, 255);'><b>Call Graph Test</b></td>\n"; |
| 183 |
foreach $i ( @commands ) { |
| 184 |
print "<td style='background-color: rgb(204, 204, 255);'><b>$listCommand{$i}[0] Diff Test</b></td>\n" if ($toDo{$i}); |
| 185 |
} |
| 186 |
if (!$toDo{"valgrind"}) { |
| 187 |
print "<td style='background-color: rgb(204, 204, 255);'><b>Valgrind Tests</b></td>\n"; |
| 188 |
} |
| 189 |
else { |
| 190 |
foreach $i ( @commands ) { |
| 191 |
print "<td style='background-color: rgb(204, 204, 255);'><b>$listCommand{$i}[0]<br> Valgrind Test</b></td>\n" if ($toDo{$i}); |
| 192 |
} |
| 193 |
} |
| 194 |
print "</tr>\n"; |
| 195 |
} else { |
| 196 |
print "Regression testing the " . $directory . " parser\n"; |
| 197 |
} |
| 198 |
|
| 199 |
my $regrInput = $directory . '/regression.in'; |
| 200 |
|
| 201 |
my @fileList = split(/\n/, `cat $regrInput`); |
| 202 |
foreach $_ ( @fileList ) { |
| 203 |
if (!/^([GPST]):(.*)/) { |
| 204 |
print "Skip ".$_.": unknown type\n"; |
| 205 |
next; |
| 206 |
} |
| 207 |
|
| 208 |
$file=$2; |
| 209 |
$fileType=$1; |
| 210 |
my $filePath = $directory . '/' . $file; |
| 211 |
if ($html) { |
| 212 |
print "<tr>\n"; |
| 213 |
print "<td><a href='" . $filePath . "'>" . $file . "</a></td>\n"; |
| 214 |
} |
| 215 |
|
| 216 |
# ////////////////////////// |
| 217 |
# CALL GRAPH REGRESSION TEST |
| 218 |
# ////////////////////////// |
| 219 |
|
| 220 |
my $cgResult = CgTest("sd2raw --callgraph", $filePath); |
| 221 |
|
| 222 |
$callGraphFailures++ if ($cgResult ne "0"); |
| 223 |
if ($html) { |
| 224 |
($cgResult eq "0" ? DisplayCell($pass_colour, "passed") : |
| 225 |
DisplayCell($fail_colour, "failed")); |
| 226 |
} else { |
| 227 |
print "! $file call graph: " . ($cgResult eq "0" ? "passed" : "failed") . "\n"; |
| 228 |
} |
| 229 |
|
| 230 |
# ///////////////////// |
| 231 |
# DIFF REGRESSION TESTS |
| 232 |
# ///////////////////// |
| 233 |
|
| 234 |
foreach $i ( @commands ) { |
| 235 |
next if (!$toDo{$i}); |
| 236 |
$diffFailures{$i}++ if (DiffTest($listCommand{$i}[3], $listCommand{$i}[4], $filePath, $listCommand{$i}[1], ($fileType ~~ $listCommand{$i}[2]) ) eq "fail"); |
| 237 |
} |
| 238 |
|
| 239 |
# //////////////////////// |
| 240 |
# VALGRIND REGRESSION TEST |
| 241 |
# //////////////////////// |
| 242 |
if (!$toDo{"valgrind"}) { |
| 243 |
if (!$html) { |
| 244 |
print "! $file valgrind: skipped all tests\n"; |
| 245 |
} |
| 246 |
else { |
| 247 |
DisplayCell($skip_colour, "skipped"); |
| 248 |
print "</tr>\n" |
| 249 |
} |
| 250 |
next; |
| 251 |
} |
| 252 |
|
| 253 |
foreach $i ( @commands ) { |
| 254 |
next if (!$toDo{$i}); |
| 255 |
if ($fileType ~~ $listCommand{$i}[2]) { |
| 256 |
$vgPath = $directory . '/' . $file . '.' . $listCommand{$i}[1] . 'vg'; |
| 257 |
`$vgCommand --leak-check=yes $listCommand{$i}[3] $filePath 1> $vgPath 2> $vgPath`; |
| 258 |
$valgrindFailures{$i}++ if (CheckVg($vgPath, "! $file $listCommand{$i}[0] valgrind:") eq "fail"); |
| 259 |
} |
| 260 |
elsif ($html) { |
| 261 |
DisplayCell($skip_colour, "skipped"); |
| 262 |
} |
| 263 |
} |
| 264 |
|
| 265 |
print "</tr>\n" if ($html); |
| 266 |
} |
| 267 |
|
| 268 |
print "</table><br>\n" if ($html); |
| 269 |
} |
| 270 |
|
| 271 |
# |
| 272 |
# Summary |
| 273 |
# |
| 274 |
if ($html) { |
| 275 |
print "<b>Summary</b>".$endLine; |
| 276 |
} else { |
| 277 |
print "\nSummary".$endLine; |
| 278 |
} |
| 279 |
## callgraph errors |
| 280 |
print "Regression test found " . $callGraphFailures . " call graph failure(s)".$endLine; |
| 281 |
# diff errors |
| 282 |
foreach $i ( @commands ) { |
| 283 |
if ($toDo{$i}) { |
| 284 |
print "Regression test found $diffFailures{$i} $listCommand{$i}[0] diff failure(s)".$endLine; |
| 285 |
} else { |
| 286 |
print "$listCommand{$i}[0] test skipped".$endLine; |
| 287 |
} |
| 288 |
} |
| 289 |
## valgrind errors |
| 290 |
if (!$toDo{"valgrind"}) { |
| 291 |
print "Valgrind test skipped".$endLine; |
| 292 |
} |
| 293 |
else { |
| 294 |
foreach $i ( @commands ) { |
| 295 |
if ($toDo{$i}) { |
| 296 |
print "Regression test found $valgrindFailures{$i} $listCommand{$i}[0] valgrind failure(s)".$endLine; |
| 297 |
} else { |
| 298 |
print "$listCommand{$i}[0] valgrind test skipped".$endLine; |
| 299 |
} |
| 300 |
} |
| 301 |
} |
| 302 |
} |
| 303 |
|
| 304 |
my $confused = 0; |
| 305 |
while (scalar(@ARGV) > 0) { |
| 306 |
my $argument = shift @ARGV; |
| 307 |
if ($argument =~ /--output-html/) { |
| 308 |
$html = 1; |
| 309 |
next; |
| 310 |
} |
| 311 |
$find=0; |
| 312 |
foreach $key (keys %toDo) { |
| 313 |
if ($argument =~ /--$key/) { |
| 314 |
$find=1; |
| 315 |
$toDo{$key} = 1; |
| 316 |
break; |
| 317 |
} |
| 318 |
} |
| 319 |
$confused = 1 if (!$find); |
| 320 |
} |
| 321 |
if ($confused) { |
| 322 |
print "Usage: regression.pl [--output-html] [--valgrind] [--csv] [--epub] [--html] [--odf] [--svg]\n"; |
| 323 |
exit; |
| 324 |
} |
| 325 |
|
| 326 |
# Main function |
| 327 |
if ($html) |
| 328 |
{ |
| 329 |
print "<html>\n<head>\n</head>\n<body>\n"; |
| 330 |
print "<h2>libstoff Regression Test Suite</h2>\n"; |
| 331 |
} |
| 332 |
|
| 333 |
&RegTest; |
| 334 |
|
| 335 |
if ($html) |
| 336 |
{ |
| 337 |
print "</body>\n</html>\n"; |
| 338 |
} |
| 339 |
|