Debian Codegraph @

metastudentPkg/lib/groupB/perl/evaluate.pl

raw
1
2 use strict;
3 use warnings;
4 use diagnostics;
5
6 # evaluate.pl - Evaluate CAFA predictions
7
8 require GOfunctions;
9
10 use Getopt::Long;
11 use Pod::Usage;
12
13 my ($gotermfile, $goterm2termfile, $sprot2gofile, $identifier2go, $targetcount, $predictioncount, $prefix, $help, $man);
14
15 GetOptions( "gotermfile|g=s" => \$gotermfile,
16 "term2termfile|t=s" => \$goterm2termfile,
17 "sprot2gofile|s=s" => \$sprot2gofile,
18 "identifier2go|i" => \$identifier2go,
19 "targetcount|c=i" => \$targetcount,
20 "predcount|n=i" => \$predictioncount,
21 "prefix|p=s" => \$prefix,
22 "help|h|?" => \$help,
23 "man" => \$man
24 ) or pod2usage(2);
25
26 pod2usage(1) if $help;
27 pod2usage(-exitstatus => 0, -verbose => 2) if $man;
28
29 if(not defined $gotermfile) {
30 print STDERR "Must specify a gotermfile!\n\n";
31 pod2usage(2);
32 }
33
34 if(not defined $goterm2termfile) {
35 print STDERR "Must specify a term2termfile!\n\n";
36 pod2usage(2);
37 }
38
39 if((not defined $sprot2gofile) && (not $identifier2go)) {
40 print STDERR "Must specify a sprot2gofile or use identifier2go mode!!\n\n";
41 pod2usage(2);
42 }
43
44 if(defined $sprot2gofile && defined $identifier2go) {
45 print STDERR "Must specify either sprot2gofile OR identifier2go mode!\n\n";
46 pod2usage(2);
47 }
48
49 if(not defined $prefix) {
50 print STDERR "Must specify a prefix!\n\n";
51 pod2usage(2);
52 }
53
54 # initialize mark's go expander
55 GOfunctions::import($gotermfile, $goterm2termfile);
56
57 # load actual go terms from sprot2gofile if we have one
58 my %sprot2go = ();
59 if(defined $sprot2gofile) {
60 open(SPROTGO, "< $sprot2gofile") or die ("Could not open sprot2gofile at $sprot2gofile for reading!");
61 while (my $line = <SPROTGO>) {
62 chomp $line;
63 my ($sprot, $go) = split(/\t/, $line);
64 my @g = split(/,/, $go);
65 $sprot2go{$sprot}=\@g;
66 }
67 }
68
69 # do evaluation
70 my ($current_target);
71 my (@observed_b, @observed_m, @predictions);
72 my (@evaluation);
73 my ($targets);
74 while( my $line = <STDIN> ) {
75 chomp $line;
76
77 if(substr($line,0,1) eq ">") {
78 # start of new target - clear buffers
79 @predictions = ();
80
81 my @observed;
82 if(defined $identifier2go) {
83
84 # get observed functions from identifier line
85 $current_target = substr($line,1);
86 @observed = split(/,/, $current_target);
87
88 } else {
89
90 # get observed functions from sprot2go file
91 if($line =~ m/^>([0-9A-Za-z]+_[0-9A-Za-z]+)-[0-9A-Z]+$/g) {
92 $current_target = $1;
93 } else {
94 die("Could not recognize identifier line: $line");
95 }
96
97 if(defined $sprot2go{$current_target}) {
98 @observed = @{$sprot2go{$current_target}};
99 } else {
100 # if we don't have the swissprot identifier in the sprot2go file,
101 # it means that no MFO or BPO terms were associated with it in swissprot.
102 @observed = ();
103 }
104 }
105
106
107 print STDERR "Target $current_target";
108
109 # split observed functions by ontologies
110 @observed_b = ();
111 @observed_m = ();
112 foreach my $obs (@observed) {
113 my $type = GOfunctions->getType($obs);
114
115 die "Could not recognize GO term $obs!" unless defined $type;
116
117 if($type eq "B") {
118 push @observed_b, $obs;
119 } elsif($type eq "M") {
120 push @observed_m, $obs;
121 } elsif($type eq "C") {
122 # skip cellular process
123 } else {
124 die "unknown type: $type";
125 }
126 }
127
128 print STDERR ".";
129
130
131 } elsif(substr($line,0,2) eq "//") {
132 # end of target - evaluate it
133
134 print STDERR ".";
135
136 # filter, blow up, make set
137 my %obs_b = %{{map {$_ => 1} GOfunctions->goAddPath(GOfunctions->goFilter(@observed_b))}};
138 my %obs_m = %{{map {$_ => 1} GOfunctions->goAddPath(GOfunctions->goFilter(@observed_m))}};
139
140 print STDERR ".";
141
142 # evaluate all scoring schemes
143 for(my $score=0; $score < scalar @evaluation; $score++) {
144
145 # sort predictions by current scoring scheme
146 my @predicted = sort {$b->[2]->[$score] <=> $a->[2]->[$score]} @predictions;
147
148 # split into ontologies
149 my @pred_b = map {$_->[0]} grep {$_->[1] eq "B"} @predicted;
150 my @pred_m = map {$_->[0]} grep {$_->[1] eq "M"} @predicted;
151
152 # only take top-n predictions if n specified
153 if(defined $predictioncount) {
154 my $max_b = (scalar @pred_b >= $predictioncount) ? ($predictioncount - 1) : ((scalar @pred_b) - 1);
155 my $max_m = (scalar @pred_m >= $predictioncount) ? ($predictioncount - 1) : ((scalar @pred_m) - 1);
156 @pred_b = @pred_b[0 .. ($predictioncount-1)] if scalar @pred_b;
157 @pred_m = @pred_m[0 .. ($predictioncount-1)] if scalar @pred_m;
158 }
159
160 # blow up, make set
161 my %prd_b = scalar @pred_b ? %{{map {$_ => 1} GOfunctions->goAddPath(@pred_b)}} : ();
162 my %prd_m = scalar @pred_m ? %{{map {$_ => 1} GOfunctions->goAddPath(@pred_m)}} : ();
163
164 # compare sets
165 my ($tp_b, $fp_b, $fn_b, $pre_b, $rec_b) = confusion_matrix(\%obs_b, \%prd_b);
166 my ($tp_m, $fp_m, $fn_m, $pre_m, $rec_m) = confusion_matrix(\%obs_m, \%prd_m);
167
168 # add up cumultative precision, recall for scores and ontologies
169 $evaluation[$score]->{b_cum_pre} += $pre_b;
170 $evaluation[$score]->{b_cum_rec} += $rec_b;
171 $evaluation[$score]->{m_cum_pre} += $pre_m;
172 $evaluation[$score]->{m_cum_rec} += $rec_m;
173
174 print STDERR "s";
175 }
176
177 print STDERR "\tDone.\n";
178
179 $targets++;
180 } else {
181 # prediction inside target
182 # term type score1 score2 ... scoren
183 my @pred = split(/\t/, $line, 3);
184 my @scores = split(/\t/, $pred[2]);
185
186 # do we have an evaluation data structure yet?
187 if(not @evaluation) {
188
189 # if not, build it up now as we know how many scoring schemes we will have
190 @evaluation = ();
191 for (my $i=0; $i < scalar @scores; $i++) {
192 my %eval = (
193 b_cum_pre => 0,
194 b_cum_rec => 0,
195 m_cum_pre => 0,
196 m_cum_rec => 0
197 );
198
199 push @evaluation, \%eval;
200 }
201 }
202
203 # fix comma decimal marks on scores - no idea where they came from
204 for(my $i=0; $i < scalar @scores; $i++) {
205 $scores[$i] =~ s/,/./g;
206 }
207
208 $pred[2] = \@scores;
209 push @predictions, \@pred;
210
211 }
212 }
213
214
215 # we have gone through all predictions now
216
217 # print header
218 my @f_steps = (0.25, 0.5, 1, 2, 4);
219 print "#profile\tprecision\trecall\tprecision-b\trecall-b\tprecision-m\trecall-m";
220 foreach my $beta (@f_steps) { print "\tf$beta\tf$beta-b\tf$beta-m"; }
221 print "\n";
222
223 # If we have specified a number of targets by the command line, use that for division.
224 if($targetcount) { $targets = $targetcount; }
225
226 # for each scoring scheme,
227 for(my $score=0; $score < scalar @evaluation; $score++) {
228
229 # calculate precision and recall for the ontologies
230 my $precision_b = $evaluation[$score]->{b_cum_pre} / $targets;
231 my $recall_b = $evaluation[$score]->{b_cum_rec} / $targets;
232 my $precision_m = $evaluation[$score]->{m_cum_pre} / $targets;
233 my $recall_m = $evaluation[$score]->{m_cum_rec} / $targets;
234
235 # calculate combined precision and recall
236 my $precision = ($precision_b + $precision_m) / 2;
237 my $recall = ($recall_b + $recall_m) / 2;
238
239 print "$prefix-score_$score\t$precision\t$recall\t$precision_b\t$recall_b\t$precision_m\t$recall_m";
240
241 # calculate f-score in steps
242 foreach my $beta (@f_steps) {
243 my $f_score_b = (1+$beta*$beta) * ($precision_b * $recall_b) / ($beta * $beta * $precision_b + $recall_b);
244 my $f_score_m = (1+$beta*$beta) * ($precision_m * $recall_m) / ($beta * $beta * $precision_m + $recall_m);
245 my $f_score = ($f_score_b + $f_score_m) / 2;
246 print "\t$f_score\t$f_score_b\t$f_score_m";
247 }
248 print "\n";
249 }
250
251 exit 0;
252
253
254 sub confusion_matrix {
255 my ($obs, $prd) = @_;
256
257 my %o = %$obs;
258 my %p = %$prd;
259
260 my($tp, $fp, $fn) = (0,0,0);
261
262 foreach my $pr (keys %p) {
263 if(defined($o{$pr})) {
264 $tp++;
265 }else{
266 $fp++;
267 }
268 }
269
270 foreach my $ob (keys %o) {
271 if(not defined($p{$ob})) {
272 $fn++;
273 }
274 }
275
276 # calculate precision and recall for this point
277 my $pre = ($tp + $fp) > 0 ? $tp / ($tp + $fp) : 0;
278 my $rec = ($tp + $fn) > 0 ? $tp / ($tp + $fn) : 0;
279
280 return ($tp, $fp, $fn, $pre, $rec);
281 }
282
283 =pod
284
285 =head1 TITLE
286
287 evaluate.pl - Evaluate CAFA predictions
288
289 =head1 SYNOPSIS
290
291 $ ./evaluate.pl -p "MyEvaluation" -g goterm.txt -t goterm2term.txt -s sprot2go.txt < prediction.out > prediction.eval
292
293 Options:
294
295 --gotermfile -g File listing GO terms in Mark's format
296 --term2termfile -t File listing Terms to terms in Mark's format
297 --targetcount -c Specify the number of targets that were initially sent into the predictor. If unspecified, the number of predicted targets will be used.
298 --predcount -n Only take top-n ter