Debian Codegraph @

regression/regression.pl

raw
1 #!/usr/bin/perl
2
3 my $html = 0; # set to 1 to output a nicely formatted HTML page
4
5 my %toDo=("csv"=>0, "epub"=>0, "html"=>0, "odf"=>0, "raw"=>1, "svg"=>0, "valgrind"=>0 );
6
7 my $pass_colour = "11dd11";
8 my $fail_colour = "dd1111";
9 my $warn_colour = "e59800";
10 my $skip_colour = "9999dd";
11
12 sub DisplayCell
13 {
14 my ($bgColor, $text) = @_;
15
16 print "<td style='background-color: #$bgColor;'>$text</td>\n";
17 }
18
19 sub DiffTest
20 {
21 my ($command, $command2, $file, $extension, $condition) = @_;
22 my $result = "passed";
23 my $comment = "";
24
25 if (!$condition) {
26 DisplayCell($skip_colour, "skipped") if ($html);
27 return "skipped";
28 }
29
30 my $errPath = $file . ".$extension.err";
31 my $rawPath = $file . ".$extension";
32 my $newRawPath = $file . ".$extension.new";
33 my $diffPath = $file . ".$extension.diff";
34
35 # generate a new raw output to compare with
36 `$command $file 1> $newRawPath`;
37 if ($command2) {
38 `mv $newRawPath $newRawPath.tmp`;
39 `$command2 $newRawPath.tmp 1> $newRawPath`;
40 `rm $newRawPath.tmp`;
41 }
42
43 # HACK: check if there is a raw file with _some_ contents. If not, we've had a segfault
44 my $err = "";
45 my $diff = "";
46 $newRaw=`cat $newRawPath`;
47 if ($newRaw eq "") {
48 $err = "Segmentation fault";
49 `echo $err > $errPath`;
50 }
51
52 if ($err ne "") {
53 $result = "fail";
54 } else {
55 # remove the generated (empty) error file
56 `rm -f $errPath`;
57
58 # diff the stored raw data with the newly generated raw data
59 `diff -u $rawPath $newRawPath 1>$diffPath 2>$diffPath`;
60 $diff=`cat $diffPath | grep -v "No differences encountered"`;
61
62 if ($diff ne "") {
63 $result = "changed";
64 } else {
65 `rm -f $diffPath`;
66 }
67 }
68
69 # remove the generated raw file
70 `rm -f $newRawPath`;
71
72 # DISPLAYING RESULTS
73 if ($html) {
74 my $bgColor;
75 if ($diff eq "" && $err eq "") {
76 $bgColor = $pass_colour;
77 } elsif ($err ne "") {
78 $bgColor = $fail_colour;
79 } else {
80 $bgColor = $warn_colour;
81 }
82
83 if ($err ne "" || $diff ne "") {
84 $comment = " <a href='" . ($err ne "" ? $errPath : $diffPath) . "'>" . ($err ne "" ? "error" : "diff") . "<a>";
85 }
86
87 DisplayCell($bgColor, $result . $comment);
88 } else {
89 if ($err ne "" || $diff ne "") {
90 $comment = ($err ne "" ? "(error: " : "(diff: ") . ($err ne "" ? $errPath : $diffPath) . ")";
91 }
92 print "! $file diff (using $command): $result $comment\n";
93 }
94
95 return $result;
96 }
97
98 sub CgTest
99 {
100 my ($command, $file) = @_;
101
102 my $callgraph = `sd2raw -c $file`;
103 chomp($callgraph);
104
105 return $callgraph;
106 }
107
108 sub CheckVg
109 {
110 my ($file,$msg) = @_;
111 open VG, "$vgPath";
112 my $result = "passed";
113 my $valgrind = 0;
114 my $vgOutput;
115 while (<VG>) {
116 if (/^\=\=/) {
117 $vgOutput .= $_;
118 if (/definitely lost: [1-9]/ ||
119 /ERROR SUMMARY: [1-9]/ ||
120 /Invalid read of/) {
121 $valgrind = 1;
122 }
123 }
124 }
125 close VG;
126
127 `rm -f $vgPath`;
128 if ($valgrind) {
129 open VG, ">$vgPath";
130 print VG $vgOutput;
131 close VG;
132 $result="fail";
133 $valgrind = 1;
134 }
135
136 if ($html) {
137 ($valgrind eq 0 ? DisplayCell($pass_colour, "passed") : DisplayCell($fail_colour, "failed <a href='$vgPath'>log<a>"));
138 } else {
139 print $msg. " " . ($valgrind eq "0" ? "passed" : "failed (see '$vgPath')") . "\n";
140 }
141
142 return $result;
143 }
144
145 sub RegTest
146 {
147 my $i;
148 # basic raw/html endOfLine
149 my $endLine=($html?"<br>\n":"\n");
150
151 # list of directories containing test files
152 my @listDirectory = ( "Calc3.1", "Calc4", "Calc5", "Draw3.1", "Draw4", "Draw5", "Pres5", "Text3.1", "Text4", "Text5" );
153
154 # list of commands to test
155 # type => [ "name", "extension", "array of accepted types", "command", "filter" ]
156 my @commands=("raw", "csv", "epub", "html", "svg", "odf");
157 my %listCommand = (
158 "csv" => [ "CSV", "csv", ["S"], "sdc2csv", 0 ],
159 "epub" => [ "Epub", "epub", ["G", "P", "S", "T"], "sd2epub", 0],
160 "html" => [ "Html", "html", ["T"], "sdw2html", 0],
161 "odf" => ["WriterPerfect", "writerperfect", ["G", "P", "S", "T"], "sd2odf", "xmllint --format"],
162 "raw"=>["Raw", "raw", ["G", "P", "S", "T"], "sd2raw", 0],
163 "svg" => ["SVG", "svg", ["G", "P"], "sd2svg", 0]
164 );
165
166 # valgrind command
167 my $vgVersionOutput = `valgrind --version`;
168 my $vgCommand=( ($vgVersionOutput =~ /\-2.1/ || $vgVersionOutput =~ /\-2.2/) ? "valgrind --tool=memcheck" : "valgrind" );
169 # init failures
170 my $callGraphFailures = 0;
171 my %diffFailures, %valgrindFailures;
172 foreach $i ( @commands ) {
173 $diffFailures{$i}=0;
174 $valgrindFailures{$i}=0;
175 }
176 foreach my $directory ( @listDirectory ) {
177 if ($html) {
178 print "<b>Regression testing the " . $directory . " parser</b><br>\n";
179 print "<table>\n";
180 print "<tr>\n";
181 print "<td style='background-color: rgb(204, 204, 255);'><b>File</b></td>\n";
182 print "<td style='background-color: rgb(204, 204, 255);'><b>Call Graph Test</b></td>\n";
183 foreach $i ( @commands ) {
184 print "<td style='background-color: rgb(204, 204, 255);'><b>$listCommand{$i}[0] Diff Test</b></td>\n" if ($toDo{$i});
185 }
186 if (!$toDo{"valgrind"}) {
187 print "<td style='background-color: rgb(204, 204, 255);'><b>Valgrind Tests</b></td>\n";
188 }
189 else {
190 foreach $i ( @commands ) {
191 print "<td style='background-color: rgb(204, 204, 255);'><b>$listCommand{$i}[0]<br> Valgrind Test</b></td>\n" if ($toDo{$i});
192 }
193 }
194 print "</tr>\n";
195 } else {
196 print "Regression testing the " . $directory . " parser\n";
197 }
198
199 my $regrInput = $directory . '/regression.in';
200
201 my @fileList = split(/\n/, `cat $regrInput`);
202 foreach $_ ( @fileList ) {
203 if (!/^([GPST]):(.*)/) {
204 print "Skip ".$_.": unknown type\n";
205 next;
206 }
207
208 $file=$2;
209 $fileType=$1;
210 my $filePath = $directory . '/' . $file;
211 if ($html) {
212 print "<tr>\n";
213 print "<td><a href='" . $filePath . "'>" . $file . "</a></td>\n";
214 }
215
216 # //////////////////////////
217 # CALL GRAPH REGRESSION TEST
218 # //////////////////////////
219
220 my $cgResult = CgTest("sd2raw --callgraph", $filePath);
221
222 $callGraphFailures++ if ($cgResult ne "0");
223 if ($html) {
224 ($cgResult eq "0" ? DisplayCell($pass_colour, "passed") :
225 DisplayCell($fail_colour, "failed"));
226 } else {
227 print "! $file call graph: " . ($cgResult eq "0" ? "passed" : "failed") . "\n";
228 }
229
230 # /////////////////////
231 # DIFF REGRESSION TESTS
232 # /////////////////////
233
234 foreach $i ( @commands ) {
235 next if (!$toDo{$i});
236 $diffFailures{$i}++ if (DiffTest($listCommand{$i}[3], $listCommand{$i}[4], $filePath, $listCommand{$i}[1], ($fileType ~~ $listCommand{$i}[2]) ) eq "fail");
237 }
238
239 # ////////////////////////
240 # VALGRIND REGRESSION TEST
241 # ////////////////////////
242 if (!$toDo{"valgrind"}) {
243 if (!$html) {
244 print "! $file valgrind: skipped all tests\n";
245 }
246 else {
247 DisplayCell($skip_colour, "skipped");
248 print "</tr>\n"
249 }
250 next;
251 }
252
253 foreach $i ( @commands ) {
254 next if (!$toDo{$i});
255 if ($fileType ~~ $listCommand{$i}[2]) {
256 $vgPath = $directory . '/' . $file . '.' . $listCommand{$i}[1] . 'vg';
257 `$vgCommand --leak-check=yes $listCommand{$i}[3] $filePath 1> $vgPath 2> $vgPath`;
258 $valgrindFailures{$i}++ if (CheckVg($vgPath, "! $file $listCommand{$i}[0] valgrind:") eq "fail");
259 }
260 elsif ($html) {
261 DisplayCell($skip_colour, "skipped");
262 }
263 }
264
265 print "</tr>\n" if ($html);
266 }
267
268 print "</table><br>\n" if ($html);
269 }
270
271 #
272 # Summary
273 #
274 if ($html) {
275 print "<b>Summary</b>".$endLine;
276 } else {
277 print "\nSummary".$endLine;
278 }
279 ## callgraph errors
280 print "Regression test found " . $callGraphFailures . " call graph failure(s)".$endLine;
281 # diff errors
282 foreach $i ( @commands ) {
283 if ($toDo{$i}) {
284 print "Regression test found $diffFailures{$i} $listCommand{$i}[0] diff failure(s)".$endLine;
285 } else {
286 print "$listCommand{$i}[0] test skipped".$endLine;
287 }
288 }
289 ## valgrind errors
290 if (!$toDo{"valgrind"}) {
291 print "Valgrind test skipped".$endLine;
292 }
293 else {
294 foreach $i ( @commands ) {
295 if ($toDo{$i}) {
296 print "Regression test found $valgrindFailures{$i} $listCommand{$i}[0] valgrind failure(s)".$endLine;
297 } else {
298 print "$listCommand{$i}[0] valgrind test skipped".$endLine;
299 }
300 }
301 }
302 }
303
304 my $confused = 0;
305 while (scalar(@ARGV) > 0) {
306 my $argument = shift @ARGV;
307 if ($argument =~ /--output-html/) {
308 $html = 1;
309 next;
310 }
311 $find=0;
312 foreach $key (keys %toDo) {
313 if ($argument =~ /--$key/) {
314 $find=1;
315 $toDo{$key} = 1;
316 break;
317 }
318 }
319 $confused = 1 if (!$find);
320 }
321 if ($confused) {
322 print "Usage: regression.pl [--output-html] [--valgrind] [--csv] [--epub] [--html] [--odf] [--svg]\n";
323 exit;
324 }
325
326 # Main function
327 if ($html)
328 {
329 print "<html>\n<head>\n</head>\n<body>\n";
330 print "<h2>libstoff Regression Test Suite</h2>\n";
331 }
332
333 &RegTest;
334
335 if ($html)
336 {
337 print "</body>\n</html>\n";
338 }
339