| 1 |
|
| 2 |
use strict; |
| 3 |
use warnings; |
| 4 |
use diagnostics; |
| 5 |
|
| 6 |
# evaluate.pl - Evaluate CAFA predictions |
| 7 |
|
| 8 |
require GOfunctions; |
| 9 |
|
| 10 |
use Getopt::Long; |
| 11 |
use Pod::Usage; |
| 12 |
|
| 13 |
my ($gotermfile, $goterm2termfile, $sprot2gofile, $identifier2go, $targetcount, $predictioncount, $prefix, $help, $man); |
| 14 |
|
| 15 |
GetOptions( "gotermfile|g=s" => \$gotermfile, |
| 16 |
"term2termfile|t=s" => \$goterm2termfile, |
| 17 |
"sprot2gofile|s=s" => \$sprot2gofile, |
| 18 |
"identifier2go|i" => \$identifier2go, |
| 19 |
"targetcount|c=i" => \$targetcount, |
| 20 |
"predcount|n=i" => \$predictioncount, |
| 21 |
"prefix|p=s" => \$prefix, |
| 22 |
"help|h|?" => \$help, |
| 23 |
"man" => \$man |
| 24 |
) or pod2usage(2); |
| 25 |
|
| 26 |
pod2usage(1) if $help; |
| 27 |
pod2usage(-exitstatus => 0, -verbose => 2) if $man; |
| 28 |
|
| 29 |
if(not defined $gotermfile) { |
| 30 |
print STDERR "Must specify a gotermfile!\n\n"; |
| 31 |
pod2usage(2); |
| 32 |
} |
| 33 |
|
| 34 |
if(not defined $goterm2termfile) { |
| 35 |
print STDERR "Must specify a term2termfile!\n\n"; |
| 36 |
pod2usage(2); |
| 37 |
} |
| 38 |
|
| 39 |
if((not defined $sprot2gofile) && (not $identifier2go)) { |
| 40 |
print STDERR "Must specify a sprot2gofile or use identifier2go mode!!\n\n"; |
| 41 |
pod2usage(2); |
| 42 |
} |
| 43 |
|
| 44 |
if(defined $sprot2gofile && defined $identifier2go) { |
| 45 |
print STDERR "Must specify either sprot2gofile OR identifier2go mode!\n\n"; |
| 46 |
pod2usage(2); |
| 47 |
} |
| 48 |
|
| 49 |
if(not defined $prefix) { |
| 50 |
print STDERR "Must specify a prefix!\n\n"; |
| 51 |
pod2usage(2); |
| 52 |
} |
| 53 |
|
| 54 |
# initialize mark's go expander |
| 55 |
GOfunctions::import($gotermfile, $goterm2termfile); |
| 56 |
|
| 57 |
# load actual go terms from sprot2gofile if we have one |
| 58 |
my %sprot2go = (); |
| 59 |
if(defined $sprot2gofile) { |
| 60 |
open(SPROTGO, "< $sprot2gofile") or die ("Could not open sprot2gofile at $sprot2gofile for reading!"); |
| 61 |
while (my $line = <SPROTGO>) { |
| 62 |
chomp $line; |
| 63 |
my ($sprot, $go) = split(/\t/, $line); |
| 64 |
my @g = split(/,/, $go); |
| 65 |
$sprot2go{$sprot}=\@g; |
| 66 |
} |
| 67 |
} |
| 68 |
|
| 69 |
# do evaluation |
| 70 |
my ($current_target); |
| 71 |
my (@observed_b, @observed_m, @predictions); |
| 72 |
my (@evaluation); |
| 73 |
my ($targets); |
| 74 |
while( my $line = <STDIN> ) { |
| 75 |
chomp $line; |
| 76 |
|
| 77 |
if(substr($line,0,1) eq ">") { |
| 78 |
# start of new target - clear buffers |
| 79 |
@predictions = (); |
| 80 |
|
| 81 |
my @observed; |
| 82 |
if(defined $identifier2go) { |
| 83 |
|
| 84 |
# get observed functions from identifier line |
| 85 |
$current_target = substr($line,1); |
| 86 |
@observed = split(/,/, $current_target); |
| 87 |
|
| 88 |
} else { |
| 89 |
|
| 90 |
# get observed functions from sprot2go file |
| 91 |
if($line =~ m/^>([0-9A-Za-z]+_[0-9A-Za-z]+)-[0-9A-Z]+$/g) { |
| 92 |
$current_target = $1; |
| 93 |
} else { |
| 94 |
die("Could not recognize identifier line: $line"); |
| 95 |
} |
| 96 |
|
| 97 |
if(defined $sprot2go{$current_target}) { |
| 98 |
@observed = @{$sprot2go{$current_target}}; |
| 99 |
} else { |
| 100 |
# if we don't have the swissprot identifier in the sprot2go file, |
| 101 |
# it means that no MFO or BPO terms were associated with it in swissprot. |
| 102 |
@observed = (); |
| 103 |
} |
| 104 |
} |
| 105 |
|
| 106 |
|
| 107 |
print STDERR "Target $current_target"; |
| 108 |
|
| 109 |
# split observed functions by ontologies |
| 110 |
@observed_b = (); |
| 111 |
@observed_m = (); |
| 112 |
foreach my $obs (@observed) { |
| 113 |
my $type = GOfunctions->getType($obs); |
| 114 |
|
| 115 |
die "Could not recognize GO term $obs!" unless defined $type; |
| 116 |
|
| 117 |
if($type eq "B") { |
| 118 |
push @observed_b, $obs; |
| 119 |
} elsif($type eq "M") { |
| 120 |
push @observed_m, $obs; |
| 121 |
} elsif($type eq "C") { |
| 122 |
# skip cellular process |
| 123 |
} else { |
| 124 |
die "unknown type: $type"; |
| 125 |
} |
| 126 |
} |
| 127 |
|
| 128 |
print STDERR "."; |
| 129 |
|
| 130 |
|
| 131 |
} elsif(substr($line,0,2) eq "//") { |
| 132 |
# end of target - evaluate it |
| 133 |
|
| 134 |
print STDERR "."; |
| 135 |
|
| 136 |
# filter, blow up, make set |
| 137 |
my %obs_b = %{{map {$_ => 1} GOfunctions->goAddPath(GOfunctions->goFilter(@observed_b))}}; |
| 138 |
my %obs_m = %{{map {$_ => 1} GOfunctions->goAddPath(GOfunctions->goFilter(@observed_m))}}; |
| 139 |
|
| 140 |
print STDERR "."; |
| 141 |
|
| 142 |
# evaluate all scoring schemes |
| 143 |
for(my $score=0; $score < scalar @evaluation; $score++) { |
| 144 |
|
| 145 |
# sort predictions by current scoring scheme |
| 146 |
my @predicted = sort {$b->[2]->[$score] <=> $a->[2]->[$score]} @predictions; |
| 147 |
|
| 148 |
# split into ontologies |
| 149 |
my @pred_b = map {$_->[0]} grep {$_->[1] eq "B"} @predicted; |
| 150 |
my @pred_m = map {$_->[0]} grep {$_->[1] eq "M"} @predicted; |
| 151 |
|
| 152 |
# only take top-n predictions if n specified |
| 153 |
if(defined $predictioncount) { |
| 154 |
my $max_b = (scalar @pred_b >= $predictioncount) ? ($predictioncount - 1) : ((scalar @pred_b) - 1); |
| 155 |
my $max_m = (scalar @pred_m >= $predictioncount) ? ($predictioncount - 1) : ((scalar @pred_m) - 1); |
| 156 |
@pred_b = @pred_b[0 .. ($predictioncount-1)] if scalar @pred_b; |
| 157 |
@pred_m = @pred_m[0 .. ($predictioncount-1)] if scalar @pred_m; |
| 158 |
} |
| 159 |
|
| 160 |
# blow up, make set |
| 161 |
my %prd_b = scalar @pred_b ? %{{map {$_ => 1} GOfunctions->goAddPath(@pred_b)}} : (); |
| 162 |
my %prd_m = scalar @pred_m ? %{{map {$_ => 1} GOfunctions->goAddPath(@pred_m)}} : (); |
| 163 |
|
| 164 |
# compare sets |
| 165 |
my ($tp_b, $fp_b, $fn_b, $pre_b, $rec_b) = confusion_matrix(\%obs_b, \%prd_b); |
| 166 |
my ($tp_m, $fp_m, $fn_m, $pre_m, $rec_m) = confusion_matrix(\%obs_m, \%prd_m); |
| 167 |
|
| 168 |
# add up cumultative precision, recall for scores and ontologies |
| 169 |
$evaluation[$score]->{b_cum_pre} += $pre_b; |
| 170 |
$evaluation[$score]->{b_cum_rec} += $rec_b; |
| 171 |
$evaluation[$score]->{m_cum_pre} += $pre_m; |
| 172 |
$evaluation[$score]->{m_cum_rec} += $rec_m; |
| 173 |
|
| 174 |
print STDERR "s"; |
| 175 |
} |
| 176 |
|
| 177 |
print STDERR "\tDone.\n"; |
| 178 |
|
| 179 |
$targets++; |
| 180 |
} else { |
| 181 |
# prediction inside target |
| 182 |
# term type score1 score2 ... scoren |
| 183 |
my @pred = split(/\t/, $line, 3); |
| 184 |
my @scores = split(/\t/, $pred[2]); |
| 185 |
|
| 186 |
# do we have an evaluation data structure yet? |
| 187 |
if(not @evaluation) { |
| 188 |
|
| 189 |
# if not, build it up now as we know how many scoring schemes we will have |
| 190 |
@evaluation = (); |
| 191 |
for (my $i=0; $i < scalar @scores; $i++) { |
| 192 |
my %eval = ( |
| 193 |
b_cum_pre => 0, |
| 194 |
b_cum_rec => 0, |
| 195 |
m_cum_pre => 0, |
| 196 |
m_cum_rec => 0 |
| 197 |
); |
| 198 |
|
| 199 |
push @evaluation, \%eval; |
| 200 |
} |
| 201 |
} |
| 202 |
|
| 203 |
# fix comma decimal marks on scores - no idea where they came from |
| 204 |
for(my $i=0; $i < scalar @scores; $i++) { |
| 205 |
$scores[$i] =~ s/,/./g; |
| 206 |
} |
| 207 |
|
| 208 |
$pred[2] = \@scores; |
| 209 |
push @predictions, \@pred; |
| 210 |
|
| 211 |
} |
| 212 |
} |
| 213 |
|
| 214 |
|
| 215 |
# we have gone through all predictions now |
| 216 |
|
| 217 |
# print header |
| 218 |
my @f_steps = (0.25, 0.5, 1, 2, 4); |
| 219 |
print "#profile\tprecision\trecall\tprecision-b\trecall-b\tprecision-m\trecall-m"; |
| 220 |
foreach my $beta (@f_steps) { print "\tf$beta\tf$beta-b\tf$beta-m"; } |
| 221 |
print "\n"; |
| 222 |
|
| 223 |
# If we have specified a number of targets by the command line, use that for division. |
| 224 |
if($targetcount) { $targets = $targetcount; } |
| 225 |
|
| 226 |
# for each scoring scheme, |
| 227 |
for(my $score=0; $score < scalar @evaluation; $score++) { |
| 228 |
|
| 229 |
# calculate precision and recall for the ontologies |
| 230 |
my $precision_b = $evaluation[$score]->{b_cum_pre} / $targets; |
| 231 |
my $recall_b = $evaluation[$score]->{b_cum_rec} / $targets; |
| 232 |
my $precision_m = $evaluation[$score]->{m_cum_pre} / $targets; |
| 233 |
my $recall_m = $evaluation[$score]->{m_cum_rec} / $targets; |
| 234 |
|
| 235 |
# calculate combined precision and recall |
| 236 |
my $precision = ($precision_b + $precision_m) / 2; |
| 237 |
my $recall = ($recall_b + $recall_m) / 2; |
| 238 |
|
| 239 |
print "$prefix-score_$score\t$precision\t$recall\t$precision_b\t$recall_b\t$precision_m\t$recall_m"; |
| 240 |
|
| 241 |
# calculate f-score in steps |
| 242 |
foreach my $beta (@f_steps) { |
| 243 |
my $f_score_b = (1+$beta*$beta) * ($precision_b * $recall_b) / ($beta * $beta * $precision_b + $recall_b); |
| 244 |
my $f_score_m = (1+$beta*$beta) * ($precision_m * $recall_m) / ($beta * $beta * $precision_m + $recall_m); |
| 245 |
my $f_score = ($f_score_b + $f_score_m) / 2; |
| 246 |
print "\t$f_score\t$f_score_b\t$f_score_m"; |
| 247 |
} |
| 248 |
print "\n"; |
| 249 |
} |
| 250 |
|
| 251 |
exit 0; |
| 252 |
|
| 253 |
|
| 254 |
sub confusion_matrix { |
| 255 |
my ($obs, $prd) = @_; |
| 256 |
|
| 257 |
my %o = %$obs; |
| 258 |
my %p = %$prd; |
| 259 |
|
| 260 |
my($tp, $fp, $fn) = (0,0,0); |
| 261 |
|
| 262 |
foreach my $pr (keys %p) { |
| 263 |
if(defined($o{$pr})) { |
| 264 |
$tp++; |
| 265 |
}else{ |
| 266 |
$fp++; |
| 267 |
} |
| 268 |
} |
| 269 |
|
| 270 |
foreach my $ob (keys %o) { |
| 271 |
if(not defined($p{$ob})) { |
| 272 |
$fn++; |
| 273 |
} |
| 274 |
} |
| 275 |
|
| 276 |
# calculate precision and recall for this point |
| 277 |
my $pre = ($tp + $fp) > 0 ? $tp / ($tp + $fp) : 0; |
| 278 |
my $rec = ($tp + $fn) > 0 ? $tp / ($tp + $fn) : 0; |
| 279 |
|
| 280 |
return ($tp, $fp, $fn, $pre, $rec); |
| 281 |
} |
| 282 |
|
| 283 |
=pod |
| 284 |
|
| 285 |
=head1 TITLE |
| 286 |
|
| 287 |
evaluate.pl - Evaluate CAFA predictions |
| 288 |
|
| 289 |
=head1 SYNOPSIS |
| 290 |
|
| 291 |
$ ./evaluate.pl -p "MyEvaluation" -g goterm.txt -t goterm2term.txt -s sprot2go.txt < prediction.out > prediction.eval |
| 292 |
|
| 293 |
Options: |
| 294 |
|
| 295 |
--gotermfile -g File listing GO terms in Mark's format |
| 296 |
--term2termfile -t File listing Terms to terms in Mark's format |
| 297 |
--targetcount -c Specify the number of targets that were initially sent into the predictor. If unspecified, the number of predicted targets will be used. |
| 298 |
--predcount -n Only take top-n ter |