| 1 |
#!/usr/bin/env perl |
| 2 |
|
| 3 |
# Copyright (c) 2003, 2005 MySQL AB |
| 4 |
# Use is subject to license terms |
| 5 |
# |
| 6 |
# This program is free software; you can redistribute it and/or modify |
| 7 |
# it under the terms of the GNU General Public License as published by |
| 8 |
# the Free Software Foundation; version 2 of the License. |
| 9 |
# |
| 10 |
# This program is distributed in the hope that it will be useful, |
| 11 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 |
# GNU General Public License for more details. |
| 14 |
# |
| 15 |
# You should have received a copy of the GNU General Public License |
| 16 |
# along with this program; if not, write to the Free Software |
| 17 |
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA |
| 18 |
|
| 19 |
# compares out-files (as created by Ereport.pl) from dir1/*.out and dir2/*.out |
| 20 |
# for each effectiveness column computes the probability of the hypothesis |
| 21 |
# "Both files have the same effectiveness" |
| 22 |
|
| 23 |
# sign test is used to verify that test results are statistically |
| 24 |
# significant to support the hypothesis. Function is computed on the fly. |
| 25 |
|
| 26 |
# basic formula is \sum_{r=0}^R C_N^r 2^{-N} |
| 27 |
# As N can be big, we'll work with logarithms |
| 28 |
$log2=log(2); |
| 29 |
sub probab { |
| 30 |
my $N=shift, $R=shift; |
| 31 |
|
| 32 |
my $r, $sum=0; |
| 33 |
|
| 34 |
for $r (0..$R) { |
| 35 |
$sum+=exp(logfac($N)-logfac($r)-logfac($N-$r)-$N*$log2); |
| 36 |
} |
| 37 |
return $sum; |
| 38 |
} |
| 39 |
|
| 40 |
# log(N!) |
| 41 |
# for N<20 exact value from the table (below) is taken |
| 42 |
# otherwise, Stirling approximation for N! is used |
| 43 |
sub logfac { |
| 44 |
my $n=shift; die "n=$n<0" if $n<0; |
| 45 |
return $logfactab[$n] if $n<=$#logfactab; |
| 46 |
return $n*log($n)-$n+log(2*3.14159265358*$n)/2; |
| 47 |
} |
| 48 |
@logfactab=( |
| 49 |
0, 0, 0.693147180559945, 1.79175946922805, 3.17805383034795, |
| 50 |
4.78749174278205, 6.57925121201010, 8.52516136106541, 10.6046029027453, |
| 51 |
12.8018274800815, 15.1044125730755, 17.5023078458739, 19.9872144956619, |
| 52 |
22.5521638531234, 25.1912211827387, 27.8992713838409, 30.6718601060807, |
| 53 |
33.5050734501369, 36.3954452080331, 39.3398841871995, 42.3356164607535, |
| 54 |
); |
| 55 |
|
| 56 |
############################# main () ############################### |
| 57 |
#$p=shift; $m=shift; $p-=$m; |
| 58 |
#if($p>$m) { |
| 59 |
# print "1 > 2 [+$p-$m]: ", probab($p+$m, $m), "\n"; |
| 60 |
#} elsif($p<$m) { |
| 61 |
# print "1 < 2 [+$p-$m]: ", probab($p+$m, $p), "\n"; |
| 62 |
#} else { |
| 63 |
# print "1 = 2 [+$p-$m]: ", probab($p+$m, $m), "\n"; |
| 64 |
#} |
| 65 |
#exit; |
| 66 |
|
| 67 |
die "Use: $0 dir1 dir2\n" unless @ARGV==2 && |
| 68 |
-d ($dir1=shift) && -d ($dir2=shift); |
| 69 |
$_=`cd $dir1; echo *.out`; |
| 70 |
s/\.out\b//g; |
| 71 |
$total=""; |
| 72 |
|
| 73 |
for $file (split) { |
| 74 |
open(OUT1,$out1="$dir1/$file.out") || die "Cannot open $out1: $!"; |
| 75 |
open(OUT2,$out2="$dir2/$file.out") || die "Cannot open $out2: $!"; |
| 76 |
|
| 77 |
@p=@m=(); |
| 78 |
while(!eof(OUT1) || !eof(OUT2)) { |
| 79 |
$_=<OUT1>; @l1=split; shift @l1; |
| 80 |
$_=<OUT2>; @l2=split; shift @l2; |
| 81 |
|
| 82 |
die "Number of columns differ in line $.\n" unless $#l1 == $#l2; |
| 83 |
|
| 84 |
for (0..$#l1) { |
| 85 |
$p[$_]+= $l1[$_] > $l2[$_]; |
| 86 |
$m[$_]+= $l1[$_] < $l2[$_]; |
| 87 |
} |
| 88 |
} |
| 89 |
|
| 90 |
for (0..$#l1) { |
| 91 |
$pp[$_]+=$p[$_]; $mm[$_]+=$m[$_]; |
| 92 |
$total[$_].=rep($file, ($#l1 ? $_ : undef), $p[$_], $m[$_]); |
| 93 |
} |
| 94 |
close OUT1; |
| 95 |
close OUT2; |
| 96 |
} |
| 97 |
|
| 98 |
for (0..$#l1) { |
| 99 |
rep($total[$_], ($#l1 ? $_ : undef), $pp[$_], $mm[$_]); |
| 100 |
} |
| 101 |
|
| 102 |
sub rep { |
| 103 |
my ($test, $n, $p, $m, $c, $r)=@_; |
| 104 |
|
| 105 |
if ($p>$m) { $c=">"; $r="+"; } |
| 106 |
elsif($p<$m) { $c="<"; $r="-"; } |
| 107 |
else { $c="="; $r="="; } |
| 108 |
$n=" $n: " if defined $n; |
| 109 |
printf "%-8s $n $dir1 $c $dir2 [+%03d-%03d]: %16.15f\n", |
| 110 |
$test, $p, $m, probab($p+$m, ($p>=$m ? $m : $p)); |
| 111 |
$r; |
| 112 |
} |