Debian Codegraph @

misc/plot-bamstats

raw
1 #!/usr/bin/env perl
2 #
3 # Copyright (C) 2012-2014,2018, 2024 Genome Research Ltd.
4 #
5 # Author: Petr Danecek <[email protected]>
6 #
7 # Permission is hereby granted, free of charge, to any person obtaining a copy
8 # of this software and associated documentation files (the "Software"), to deal
9 # in the Software without restriction, including without limitation the rights
10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 # copies of the Software, and to permit persons to whom the Software is
12 # furnished to do so, subject to the following conditions:
13 #
14 # The above copyright notice and this permission notice shall be included in
15 # all copies or substantial portions of the Software.
16 #
17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
23 # DEALINGS IN THE SOFTWARE.
24
25 use strict;
26 use warnings;
27 use Carp;
28 use POSIX qw/floor ceil/;
29 use URI::Escape qw(uri_escape);
30 use List::Util qw(max);
31
32 my $opts = parse_params();
33 parse_bamcheck($opts);
34 if ( @{$$opts{bamcheck}} > 1 ) { merge_bamcheck($opts); exit; }
35
36 plot_qualities($opts);
37 plot_acgt_cycles($opts);
38 plot_gc($opts);
39 plot_gc_depth($opts);
40 plot_isize($opts);
41 plot_coverage($opts);
42 plot_mismatches_per_cycle($opts);
43 plot_indel_dist($opts);
44 plot_indel_cycles($opts);
45 plot_read_len_hist($opts);
46 create_html($opts);
47
48 exit;
49
50 #--------------------------------
51
52 sub error
53 {
54 my (@msg) = @_;
55 if ( scalar @msg ) { confess @msg; }
56 die
57 "About: Parses output of samtools stats (former bamcheck) and calls gnuplot to create graphs.\n",
58 "Usage: plot-bamstats [OPTIONS] file.bam.bc\n",
59 " plot-bamstats -p outdir/ file.bam.bc\n",
60 "Options:\n",
61 " -k, --keep-files Do not remove temporary files.\n",
62 " -l, --log-y Set the Y axis scale of the Insert Size plot to log 10.\n",
63 " -m, --merge Merge multiple bamstats files and output to STDOUT.\n",
64 " -p, --prefix <path> The output files prefix, add a slash to create new directory.\n",
65 " -r, --ref-stats <file.fa.gc> Optional reference stats file with expected GC content (created with -s).\n",
66 " -s, --do-ref-stats <file.fa> Calculate reference sequence GC for later use with -r\n",
67 " -t, --targets <file.tab> Restrict -s to the listed regions (tab-delimited chr,from,to. 1-based, inclusive)\n",
68 " -q, --alt-qual-avg-method Calculate average base quality using average base error probabilities rather than average quality scores.\n",
69 " -h, -?, --help This help message.\n",
70 "\n";
71 }
72
73
74 sub parse_params
75 {
76 $0 =~ s{^.+/}{};
77 my $opts =
78 {
79 args => join(' ',$0,@ARGV),
80
81 # for merging
82 sections =>
83 [
84 {
85 id=>'SN',
86 header=>
87 "# Summary Numbers. Use `grep ^SN | cut -f 2-` to extract this part.\n",
88 },
89 {
90 id=>'FFQ',
91 header=>
92 "# First Fragment Qualities. Use `grep ^FFQ | cut -f 2-` to extract this part.\n" .
93 "# Columns correspond to qualities and rows to cycles. First column is the cycle number.\n",
94 },
95 {
96 id=>'LFQ',
97 header=>
98 "# Last Fragment Qualities. Use `grep ^LFQ | cut -f 2-` to extract this part.\n" .
99 "# Columns correspond to qualities and rows to cycles. First column is the cycle number.\n",
100 },
101 {
102 id=>'MPC',
103 header=>
104 "# Mismatches per cycle and quality. Use `grep ^MPC | cut -f 2-` to extract this part.\n" .
105 "# Columns correspond to qualities, rows to cycles. First column is the cycle number, second\n" .
106 "# is the number of N's and the rest is the number of mismatches\n",
107 },
108 {
109 id=>'GCF',
110 header=>"# GC Content of first fragments. Use `grep ^GCF | cut -f 2-` to extract this part.\n",
111 },
112 {
113 id=>'GCL',
114 header=>"# GC Content of last fragments. Use `grep ^GCL | cut -f 2-` to extract this part.\n",
115 },
116 {
117 id=>'GCC',
118 header=>"# ACGT content per cycle. Use `grep ^GCC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [\%]; and N and O counts as a percentage of all A/C/G/T bases [\%]\n",
119 },
120 {
121 id => 'FBC',
122 header=>"# ACGT content per cycle for first fragments. Use `grep ^FBC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [\%]; and N and O counts as a percentage of all A/C/G/T bases [\%]\n",
123 },
124 {
125 id => 'LBC',
126 header=>"# ACGT content per cycle for last fragments. Use `grep ^LBC | cut -f 2-` to extract this part. The columns are: cycle; A,C,G,T base counts as a percentage of all A/C/G/T bases [\%]; and N and O counts as a percentage of all A/C/G/T bases [\%]\n",
127 },
128 {
129 id=>'IS',
130 header=>"# Insert sizes. Use `grep ^IS | cut -f 2-` to extract this part. The columns are: insert size, pairs total, inward oriented pairs, outward oriented pairs, other pairs\n",
131 },
132 {
133 id=>'RL',
134 header=>"# Read lengths. Use `grep ^RL | cut -f 2-` to extract this part. The columns are: read length, count\n",
135 },
136 {
137 id=>'FRL',
138 header=>"# Read lengths - first fragments. Use `grep ^FRL | cut -f 2-` to extract this part. The columns are: read length, count\n",
139 },
140 {
141 id=>'LRL',
142 header=>"# Read lengths - last fragments. Use `grep ^LRL | cut -f 2-` to extract this part. The columns are: read length, count\n",
143 },
144 {
145 id=>'ID',
146 header=>"# Indel distribution. Use `grep ^ID | cut -f 2-` to extract this part. The columns are: length, number of insertions, number of deletions\n",
147 },
148 {
149 id=>'IC',
150 header=>"# Indels per cycle. Use `grep ^IC | cut -f 2-` to extract this part. The columns are: cycle, number of insertions (fwd), .. (rev) , number of deletions (fwd), .. (rev)\n",
151 },
152 {
153 id=>'COV',
154 header=>"# Coverage distribution. Use `grep ^COV | cut -f 2-` to extract this part.\n",
155 },
156 # {
157 # id=>'GCD',
158 # header=>"# GC-depth. Use `grep ^GCD | cut -f 2-` to extract this part. The columns are: GC%, unique sequence percentiles, 10th, 25th, 50th, 75th and 90th depth percentile\n",
159 # },
160 ],
161
162 # for merging
163 merge_keys =>
164 {
165 'sum' =>
166 [
167 'raw total sequences:',
168 'filtered sequences:',
169 'sequences:',
170 '1st fragments:',
171 'last fragments:',
172 'reads mapped:',
173 'reads mapped and paired:',
174 'reads unmapped:',
175 'reads properly paired:',
176 'reads paired:',
177 'reads duplicated:',
178 'reads MQ0:',
179 'reads QC failed:',
180 'non-primary alignments:',
181 'total length:',
182 'total first fragment length:',
183 'total last fragment length:',
184 'bases mapped:',
185 'bases mapped (cigar):',
186 'bases trimmed:',
187 'bases duplicated:',
188 'mismatches:',
189 'inward oriented pairs:',
190 'outward oriented pairs:',
191 'pairs with other orientation:',
192 'pairs on different chromosomes:',
193 ],
194 'min' =>
195 [
196 'is sorted:',
197 ],
198 'max' =>
199 [
200 'maximum length:',
201 ],
202 },
203 };
204 for my $key (keys %{$$opts{merge_keys}})
205 {
206 for my $val (@{$$opts{merge_keys}{$key}}) { $$opts{hmerge}{$val} = $key; }
207 }
208 while (defined(my $arg=shift(@ARGV)))
209 {
210 if ( $arg eq '-k' || $arg eq '--keep-files' ) { $$opts{keep_files}=1; next; }
211 if ( $arg eq '-l' || $arg eq '--log-y' ) { $$opts{y_axis_log10}=1; next; }
212 if ( $arg eq '-m' || $arg eq '--merge' ) { $$opts{merge}=1; next; }
213 if ( $arg eq '-r' || $arg eq '--ref-stats' ) { $$opts{ref_stats}=shift(@ARGV); next; }
214 if ( $arg eq '-s' || $arg eq '--do-ref-stats' ) { $$opts{do_ref_stats}=shift(@ARGV); next; }
215 if ( $arg eq '-t' || $arg eq '--targets' ) { $$opts{targets}=shift(@ARGV); next; }
216 if ( $arg eq '-p' || $arg eq '--prefix' ) { $$opts{prefix}=shift(@ARGV); next; }
217 if ( $arg eq '-q' || $arg eq '--alt-qual-avg-method' ) { $$opts{alt_qual_avg_method}=1; next; }
218 if ( $arg eq '-?' || $arg eq '-h' || $arg eq '--help' ) { error(); }
219 if ( $arg eq '-' || -e $arg ) { push @{$$opts{bamcheck}},$arg; next; }
220 error("Unknown parameter or non-existent file \"$arg\". Run -h for help.\n");
221 }
222 if ( exists($$opts{do_ref_stats }) ) { do_ref_stats($opts); exit; }
223 if ( !exists($$opts{bamcheck}) ) { error("No samtools stats file?\n") }
224
225 if ( !exists($$opts{prefix}) )
226 {
227 if ( !$$opts{merge} ) { error("Expected -p parameter.\n") }
228 $$opts{prefix} = './';
229 }
230 elsif ( $$opts{merge} ) { error("Only one of -p or -m should be given.\n"); }
231 if ( $$opts{merge} )
232 {
233 if ( @{$$opts{bamcheck}} < 2 ) { error("Nothing to merge\n") }
234 }
235 else
236 {
237 if ( !exists($$opts{prefix}) ) { error("Expected -p parameter.\n") }
238 if ( $$opts{prefix}=~m{/$} ) { `mkdir -p $$opts{prefix}`; }
239 elsif ( !($$opts{prefix}=~/-$/) ) { $$opts{prefix} .= '-'; }
240 }
241 return $opts;
242 }
243
244
245 # Creates GC stats for either the whole reference or only on target regions for exome QC
246 sub do_ref_stats
247 {
248 my ($opts) = @_;
249
250
251 my %targets = ();
252 if ( exists($$opts{targets}) )
253 {
254 my ($prev_chr,$prev_pos);
255 open(my $fh,'<',$$opts{targets}) or error("$$opts{targets}: $!");
256 while (my $line=<$fh>)
257 {
258 if ( $line=~/^#/ ) { next; }
259 my ($chr,$from,$to) = split(/\s+/,$line);
260 chomp($to);
261 push @{$targets{$chr}}, $from,$to;
262 if ( !defined $prev_chr or $chr ne $prev_chr ) { $prev_chr=$chr; $prev_pos=$from }
263 if ( $prev_pos > $from ) { error("The file must be sorted: $$opts{targets}\n"); }
264 $prev_pos = $from;
265 }
266 close($fh);
267 }
268
269 my $nlen = 0;
270 my %lens = ();
271 my %gc_counts = ();
272 my ($skip_chr,$pos,$ireg,$regions);
273 open(my $fh,'<',$$opts{do_ref_stats}) or error("$$opts{do_ref_stats}: $!");
274 while (my $line=<$fh>)
275 {
276 if ( $line=~/^>/ )
277 {
278 if ( !scalar %targets ) { next; }
279
280 if ( !($line=~/>(\S+)/) ) { error("FIXME: could not determine chromosome name: $line"); }
281 if ( !exists($targets{$1}) ) { $skip_chr=$1; next; }
282 undef $skip_chr;
283 $pos = 0;
284 $ireg = 0;
285 $regions = $targets{$1};
286 next;
287 }
288 if ( defined $skip_chr ) { next; }
289
290 # Only $_len sized lines are considered and no chopping for target regions.
291 chomp($line);
292 my $len = length($line);
293 $lens{$len}++;
294 $nlen++;
295
296 if ( scalar %targets )
297 {
298 while ( $ireg<@$regions && $$regions[$ireg+1]<=$pos ) { $ireg += 2; }
299 $pos += $len;
300 if ( $ireg==@$regions ) { next; }
301 if ( $pos < $$regions[$ireg] ) { next; }
302 }
303
304 my $gc_count = 0;
305 for (my $i=0; $i<$len; $i++)
306 {
307 my $base = substr($line,$i,1);
308 if ( $base eq 'g' || $base eq 'G' || $base eq 'c' || $base eq 'C' ) { $gc_count++; }
309 }
310 $gc_counts{$gc_count}++;
311 }
312
313 # Calculate median length
314 my $n = 0;
315 my $median_len = 0;
316 for my $len (sort { $a<=>$b } keys %lens)
317 {
318 $n += $lens{$len};
319 if ( $n >= $nlen ) { $median_len = $len; last; }
320 }
321 if ( !$median_len ) { error("FIXME: median_len=$median_len??\n"); }
322
323 print "# Generated by $$opts{args}\n";
324 print "# The columns are: GC content bin, normalized frequency\n";
325 my $max;
326 for my $count (values %gc_counts)
327 {
328 if ( !defined $max or $count>$max ) { $max=$count; }
329 }
330 for my $gc (sort {$a<=>$b} keys %gc_counts)
331 {
332 if ( $gc==0 ) { next; }
333 printf "%f\t%f\n", $gc*100./$median_len, $gc_counts{$gc}/$max;
334 }
335 }
336
337 sub plot
338 {
339 my ($cmdfile) = @_;
340 my $cmd = "gnuplot $cmdfile";
341 system($cmd);
342 if ( $? ) { error("The command exited with non-zero status $?:\n\t$cmd\n\n"); }
343 }
344
345 sub open_file
346 {
347 my ($file) = @_;
348 my $fh;
349 if ( $file eq '-' ) { $fh = *STDIN; }
350 elsif ( $file=~/\.gz$/i ) { open($fh, "gunzip -c $file |") or error("gunzip -c $file: $!"); }
351 else { open($fh,'<',$file) or error("$file: $!"); }
352 return $fh;
353 }
354
355 sub parse_bamcheck1
356 {
357 my ($opts, $i) = @_;
358 my $file = $$opts{bamcheck}[$i];
359 print STDERR "Parsing samtools stats output: $file\n" unless !$$opts{verbose};
360 my $fh = open_file($file);
361 my $line = <$fh>;
362 if ( !($line=~/^# This file was produced by (\S+)/) ) { error("Sanity check failed: was this file generated by samtools stats or plot-bamstats?"); }
363 if ( $1 ne 'plot-bamstats' && $1 ne 'samtools' ) { error("Sanity check failed: was this file generated by samtools stats or plot-bamstats?"); }
364 my %dat;
365 my $sn_order = $$opts{sn_order} || [];
366 my %sn_seen = map { $_ => 1 } @$sn_order;
367 while ($line=<$fh>)
368 {
369 if ( $line=~/^#/ ) { next; }
370 my @items = split(/\t/,$line);
371 chomp($items[-1]);
372 if ( $items[0] eq 'SN' )
373 {
374 $dat{SN}{$items[1]} = $items[2];
375 if (!exists($sn_seen{$items[1]})) {
376 push(@$sn_order, $items[1]);
377 $sn_seen{$items[1]} = 1;
378 }
379 next;
380 }
381 push @{$dat{$items[0]}}, [splice(@items,1)];
382 }
383 close($fh) unless $file eq '-';
384
385 $$opts{sn_order} = $sn_order;
386
387 # Check sanity
388 if ( !exists($dat{'SN'}{'sequences:'}) or !$dat{'SN'}{'sequences:'} )
389 {
390 error("Sanity check failed: no sequences found by samtools stats??\n");
391 }
392 my $nseq_new = $dat{'SN'}{'sequences:'};
393 my $nseq_ori = exists($$opts{dat}{'SN'}) ? $$opts{dat}{'SN'}{'sequences:'} : 0;
394
395 my %add = map { $_ => 1 } (qw(FFQ LFQ MPC GCF GCL IS ID IC RL FRL LRL));
396 for my $a (keys %dat)
397 {
398 if ( !exists($$opts{dat}{$a}) ) { $$opts{dat}{$a} = $dat{$a}; next; } # first bamcheck file
399 if ( $a eq 'SN' )
400 {
401 for my $b (keys %{$dat{$a}})
402 {
403 if ( exists($$opts{hmerge}{$b}) )
404 {
405 if ( $$opts{hmerge}{$b} eq 'sum' ) { $$opts{dat}{$a}{$b} += $dat{$a}{$b}; next; }
406 if ( $$opts{hmerge}{$b} eq 'min' ) { $$opts{dat}{$a}{$b} = $$opts{dat}{$a}{$b} < $dat{$a}{$b} ? $$opts{dat}{$a}{$b} : $dat{$a}{$b}; }
407 if ( $$opts{hmerge}{$b} eq 'max' ) { $$opts{dat}{$a}{$b} = $$opts{dat}{$a}{$b} > $dat{$a}{$b} ? $$opts{dat}{$a}{$b} : $dat{$a}{$b}; }
408 }
409 }
410 next;
411 }
412 if ( $add{$a} ) { add_to_matrix($$opts{dat}{$a}, $dat{$a}, 0); next; }
413 if ( $a eq 'COV' ) { merge_coverage($$opts{dat}{$a}, $dat{$a}); next; }
414 if ( $a eq 'GCC' || $a eq 'FBC' || $a eq 'LBC' ) {
415 merge_gcc($nseq_ori, $$opts{dat}{$a}, $nseq_new, $dat{$a});
416 next;
417 }
418 warn("Not processed: $a\n");
419 }
420 if ( $i==0 ) { return; }
421
422 my $bases_mapped_cigar = get_value($opts,'SN','bases mapped (cigar):');
423 $$opts{dat}{SN}{'error rate:'} = $bases_mapped_cigar ? sprintf "%e", get_value($opts,'SN','mismatches:') / $bases_mapped_cigar : 0;
424 update_average_length($opts);
425 update_average_quality($opts);
426 update_average_isize($opts);
427 }
428
429 sub parse_bamcheck
430 {
431 my ($opts) = @_;
432 for (my $i=0; $i<@{$$opts{bamcheck}}; $i++) { parse_bamcheck1($opts,$i); }
433
434 # Determine the default title, for now use the first BAM name:
435 # 5446_6/5446_6.bam.bc.gp -> 5446_6
436 # test.aaa.png -> test.aaa
437 if ( !($$opts{bamcheck}[0]=~m{([^/]+?)(?:\.bam)?(?:\.bc)?$}i) ) { error("FIXME: Could not determine the title from [$$opts{bamcheck}[0]]\n"); }
438 $$opts{title} = $1;
439 }
440
441 sub add_to_matrix
442 {
443 my ($dst,$src,$key) = @_;
444 my $id = 0;
445 my $is = 0;
446 while ($is<@$src)
447 {
448 while ( $id<@$dst && $$src[$is][$key] > $$dst[$id][$key] ) { $id++; }
449 if ( $id<@$dst && $$src[$is][$key] == $$dst[$id][$key] )
450 {
451 for (my $j=0; $j<@{$$src[$is]}; $j++)
452 {
453 if ( $j==$key ) { next; }
454 $$dst[$id][$j] += $$src[$is][$j];
455 }
456 }
457 else { splice(@$dst,$id,0,$$src[$is]); }
458 $is++;
459 }
460 }
461
462 sub rebin_values
463 {
464 my ($vals,$bin_size,$col) = @_;
465 error("pd3\@sanger todo: rebin_values\n");
466 }
467
468
469 sub merge_coverage
470 {
471 my ($dst,$src) = @_;
472 if ( !($$dst[0][0]=~/^\[(\d+)-(\d+)\]$/) ) { error("Could not determine bin size in COV: $$dst[0][0]\n"); }
473 my $dst_bin = $2 - $1 + 1;
474 if ( !($$src[0][0]=~/^\[(\d+)-(\d+)\]$/) ) { error("Could not determine bin size in COV: $$src[0][0]\n"); }
475 my $src_bin = $2 - $1 + 1;
476 my (@dst, @src);
477 for my $row (@$dst) { splice(@$row,0,1); push @dst,$row; }
478 for my $row (@$src) { splice(@$row,0,1); push @src,$row; }
479 my $dst_out = pop(@dst);
480 my $src_out = pop(@src);
481 my $bin_size = $dst_bin;
482 if ( $src_bin > $dst_bin ) { my $vals = rebin_values(\@dst,$src_bin,0); @dst = @$vals; $bin_size = $src_bin; }
483 elsif ( $src_bin < $dst_bin ) { my $vals = rebin_values(\@src, $dst_bin, 0); @src = @$vals; }
484 add_to_matrix(\@dst, \@src, 0);
485 for my $row (@dst) { unshift(@$row, sprintf("[%d-%d]", $$row[0],$$row[0]+$bin_size-1)); }
486 push @dst, [sprintf("[%d<]", $dst[-1][1]), $dst[-1][1], $$dst_out[1]+$$src_out[1] ];
487 @$dst = @dst;
488 }
489
490 sub merge_gcc
491 {
492 my ($ndst,$dst,$nsrc,$src) = @_;
493 if ( @$dst != @$src ) { error("todo: improve me\n"); }
494 for (my $i=0; $i<@$dst; $i++)
495 {
496 if ( $$dst[$i][0] ne $$src[$i][0] ) { error("todo: improve me\n"); }
497 for (my $j=1; $j<@{$$dst[$i]}; $j++)
498 {
499 $$dst[$i][$j] = sprintf("%.2f", ($$dst[$i][$j]*$ndst + $$src[$i][$j]*$nsrc) / ($ndst + $nsrc));
500 }
501 }
502 }
503
504 sub merge_gcd
505 {
506 # todo
507 }
508
509 sub get_value
510 {
511 my ($opts, $id, $key) = @_;
512 if ( !exists($$opts{dat}{$id}) ) { return undef; }
513 if ( !exists($$opts{dat}{$id}{$key}) ) { return undef}
514 return $$opts{dat}{$id}{$key};
515 }
516
517 sub get_values
518 {
519 my ($opts, $id) = @_;
520 if ( !exists($$opts{dat}{$id}) ) { return (); }
521 # todo: add version sanity check here
522 return (@{$$opts{dat}{$id}});
523 }
524
525 sub update_average_length
526 {
527 my ($opts) = @_;
528 my @vals = get_values($opts,'RL');
529 my $sum = 0;
530 my $avg = 0;
531 for my $val (@vals) { $sum += $$val[1]; }
532 for my $val (@vals) { $avg += $$val[0]*$$val[1] / $sum; }
533 $$opts{dat}{SN}{'average length:'} = sprintf "%.1f", $avg;
534 }
535
536 sub update_average_quality
537 {
538 my ($opts) = @_;
539 my @vals;
540 push @vals, get_values($opts, 'FFQ'), get_values($opts,'LFQ');
541
542 my $qsum = 0;
543 for my $row (@vals)
544 {
545 for (my $i=1; $i<@$row; $i++) { $qsum += $$row[$i] }
546 }
547 my $qavg = 0;
548 for my $row (@vals)
549 {
550 for (my $i=1; $i<@$row; $i++) { $qavg += ($i-1)*$$row[$i] / $qsum; }
551 }
552 $$opts{dat}{SN}{'average quality:'} = sprintf "%.1f", $qavg;
553 }
554
555 sub update_average_isize
556 {
557 my ($opts) = @_;
558 my @vals = get_values($opts, 'IS');
559 my $sum = 0;
560 for my $row (@vals) { $sum += $$row[1]; }
561 my $avg = 0;
562 for my $row (@vals) { $avg += $$row[0]*$$row[1] / $sum; }
563 my $dev = 0;
564 for my $row (@vals) { $dev += ($avg - $$row[0])*($avg - $$row[0]) * $$row[1] / $sum; }
565 $$opts{dat}{SN}{'insert size average:'} = sprintf "%.1f", $avg;
566 $$opts{dat}{SN}{'insert size standard deviation:'} = sprintf "%.1f", sqrt($dev);
567 }
568
569 sub get_defaults
570 {
571 my ($opts,$img_fname,%args) = @_;
572
573 if ( !($img_fname=~/\.png$/i) ) { error("FIXME: currently only PNG supported. (Easy to extend.)\n"); }
574
575 # Determine the gnuplot script file name
576 my $gp_file = $img_fname;
577 $gp_file =~ s{\.[^.]+$}{.gp};
578 if ( !($gp_file=~/.gp$/) ) { $gp_file .= '.gp'; }
579
580 my $title = $$opts{title};
581
582 my $dir = $gp_file;
583 $dir =~ s{/[^/]+$}{};
584 if ( $dir && $dir ne $gp_file ) { `mkdir -p $dir`; }
585
586 my $wh = exists($args{wh}) ? $args{wh} : '600,400';
587
588 open(my $fh,'>',$gp_file) or error("$gp_file: $!");
589 return {
590 title => $title,
591 gp => $gp_file,
592 img => $img_fname,
593 fh => $fh,
594 terminal => qq[set terminal png size $wh truecolor],
595 grid => 'set grid xtics ytics y2tics back lc rgb "#cccccc"',
596 };
597 }
598
599 sub percentile
600 {
601 my ($p,@vals) = @_;
602 my $N = 0;
603 for my $val (@vals) { $N += $val; }
604 my $n = $p*($N+1)/100.;
605 my $k = int($n);
606 my $d = $n-$k;
607 if ( $k<=0 ) { return 0; }
608 if ( $k>=$N ) { return scalar @vals-1; }
609 my $cnt;
610 for (my $i=0; $i<@vals; $i++)
611 {
612 $cnt += $vals[$i];
613 if ( $cnt>=$k ) { return $i; }
614 }
615 error("FIXME: this should not happen [percentile]\n");
616 }
617
618 sub plot_qualities
619 {
620 my ($opts) = @_;
621
622 my @ffq = get_values($opts, 'FFQ');
623 if ( !@ffq ) { return; }
624
625 my $yrange = @{$ffq[0]} > 50 ? @{$ffq[0]} : 50;
626 my $is_paired = get_value($opts,'SN','reads paired:');
627
628 # Average quality per cycle, forward and reverse reads in one plot
629 my $args = get_defaults($opts,"$$opts{prefix}quals.png");
630 my $fh = $$args{fh};
631 print $fh qq[
632 $$args{terminal}
633 set output "$$args{img}"
634 $$args{grid}
635 set ylabel "Average Quality"
636 set xlabel "Cycle"
637 set yrange [0:$yrange]
638 set title "$$args{title}" noenhanced
639 plot '-' using 1:2 with lines title 'Forward reads' ] . ($is_paired ? q[, '-' using 1:2 with lines title 'Reverse reads'] : '') . q[
640 ];
641 my (@fp75,@fp50,@fmean);
642 my (@lp75,@lp50,@lmean);
643 my ($fmax,$fmax_qual,$fmax_cycle);
644 my ($lmax,$lmax_qual,$lmax_cycle);
645 for my $cycle (@ffq)
646 {
647 my $sum=0; my $n=0;
648 for (my $iqual=1; $iqual<@$cycle; $iqual++)
649 {
650 if ( exists($$opts{alt_qual_avg_method }) ) {
651 my $prob_base_error = 10 ** (-$iqual/10);
652 $sum += $$cycle[$iqual]*$prob_base_error;
653 } else {
654 $sum += $$cycle[$iqual]*$iqual;
655 }
656 $n += $$cycle[$iqual];
657 if ( !defined $fmax or $fmax<$$cycle[$iqual] ) { $fmax=$$cycle[$iqual]; $fmax_qual=$iqual; $fmax_cycle=$$cycle[0]; }
658 }
659 my $p25 = percentile(25,(@$cycle)[1..$#$cycle]);
660 my $p50 = percentile(50,(@$cycle)[1..$#$cycle]);
661 my $p75 = percentile(75,(@$cycle)[1..$#$cycle]);
662 if ( !$n ) { next; }
663 push @fp75, "$$cycle[0]\t$p25\t$p75\n";
664 push @fp50, "$$cycle[0]\t$p50\n";
665 if ( exists($$opts{alt_qual_avg_method }) ) {
666 my $mean_prob_base_error = $sum/$n;
667 my $mean_error_quality_score = -10 * ( log($mean_prob_base_error) / log(10) );
668 push @fmean, sprintf "%d\t%.2f\n", $$cycle[0],$mean_error_quality_score;
669 } else {
670 push @fmean, sprintf "%d\t%.2f\n", $$cycle[0],$sum/$n;
671 }
672 printf $fh $fmean[-1];
673 }
674 print $fh "end\n";
675 my @lfq = ();
676 if ( $is_paired )
677 {
678 @lfq = get_values($opts, 'LFQ');
679 for my $cycle (@lfq)
680 {
681 my $sum=0; my $n=0;
682 for (my $iqual=1; $iqual<@$cycle; $iqual++)
683 {
684 if ( exists($$opts{alt_qual_avg_method }) ) {
685 my $prob_base_error = 10 ** (-$iqual/10);
686 $sum += $$cycle[$iqual]*$prob_base_error;
687 } else {
688 $sum += $$cycle[$iqual]*$iqual;
689 }
690 $n += $$cycle[$iqual];
691 if ( !defined $lmax or $lmax<$$cycle[$iqual] ) { $lmax=$$cycle[$iqual]; $lmax_qual=$iqual; $lmax_cycle=$$cycle[0]; }
692 }
693 my $p25 = percentile(25,(@$cycle)[1..$#$cycle]);
694 my $p50 = percentile(50,(@$cycle)[1..$#$cycle]);
695 my $p75 = percentile(75,(@$cycle)[1..$#$cycle]);
696 if ( !$n ) { next; }
697 push @lp75, "$$cycle[0]\t$p25\t$p75\n";
698 push @lp50, "$$cycle[0]\t$p50\n";
699 if ( exists($$opts{alt_qual_avg_method }) ) {
700 my $mean_prob_base_error = $sum/$n;
701 my $mean_error_quality_score = -10 * ( log($mean_prob_base_error) / log(10) );
702 push @lmean, sprintf "%d\t%.2f\n", $$cycle[0],$mean_error_quality_score;
703 } else {
704 push @lmean, sprintf "%d\t%.2f\n", $$cycle[0],$sum/$n;
705 }
706 printf $fh $lmean[-1];
707 }
708 print $fh "end\n";
709 }
710 close($fh);
711 plot($$args{gp});
712
713
714
715 # Average, mean and quality percentiles per cycle, forward and reverse reads in separate plots
716 $args = get_defaults($opts,"$$opts{prefix}quals2.png",wh=>$is_paired ? '700,500' : '600,400');
717 $fh = $$args{fh};
718 my $pos_size = " set rmargin 0; set lmargin 0; set tmargin 0; set bmargin 0; set origin 0.1,0.1; set size 0.4,0.8";
719 print $fh qq[
720 $$args{terminal}
721 set output "$$args{img}"
722 $$args{grid}
723 set multiplot
724 $pos_size
725 set yrange [0:$yrange]
726 set ylabel "Quality"
727 set xlabel "Cycle (fwd reads)"
728 plot '-' using 1:2:3 with filledcurve lt 1 lc rgb "#cccccc" t '25-75th percentile' , '-' using 1:2 with lines lc rgb "#000000" t 'Median', '-' using 1:2 with lines lt 1 t 'Mean'
729 ];
730 print $fh join('',@fp75),"end\n";
731 print $fh join('',@fp50),"end\n";
732 print $fh join('',@fmean),"end\n";
733 if ( $is_paired )
734 {
735 print $fh qq[
736 set origin 0.55,0.1
737 set size 0.4,0.8
738 unset ytics
739 set y2tics mirror
740 set y2range [0:$yrange]
741 unset ylabel
742 set xlabel "Cycle (rev reads)"
743 set label "$$args{title}" at screen 0.5,0.95 center noenhanced
744 plot '-' using 1:2:3 with filledcurve lt 1 lc rgb "#cccccc" t '25-75th percentile' , '-' using 1:2 with lines lc rgb "#000000" t 'Median', '-' using 1:2 with lines lt 2 t 'Mean'
745 ];
746 print $fh join('',@lp75),"end\n";
747 print $fh join('',@lp50),"end\n";
748 print $fh join('',@lmean),"end\n";
749 }
750 close($fh);
751 plot($$args{gp});
752
753
754
755 # Quality distribution per cycle, the distribution is for each cycle plotted as a separate curve
756 $args = get_defaults($opts,"$$opts{prefix}quals3.png",wh=>$is_paired ? '600,600' : '600,400');
757 $fh = $$args{fh};
758 my $nquals = @{$ffq[0]}-1;
759 my $ncycles = @ffq;
760 $pos_size = $is_paired ? " set rmargin 0; set lmargin 0; set tmargin 0; set bmargin 0; set origin 0.15,0.52; set size 0.8,0.4" : '';
761 print $fh qq[
762 $$args{terminal}
763 set output "$$args{img}"
764 $$args{grid}
765 set multiplot
766 $pos_size
767 set title "$$args{title}" noenhanced
768 set ylabel "Frequency (fwd reads)"
769 set label "Cycle $fmax_cycle" at $fmax_qual+1,$fmax
770 unset xlabel
771 set xrange [0:$nquals]
772 set format x ""
773 plot '-' using 1:2:3 with lines linecolor variable title ''
774 ];
775 for my $cycle (0..$#ffq)
776 {
777 for (my $iqual=1; $iqual<$nquals; $iqual++) {
778 printf $fh "%d\t%d\t%d\n", $iqual, $ffq[$cycle]->[$iqual], $cycle + 1;
779 }
780 print $fh "\n";
781 }
782 print $fh "end\n";
783 if ( $is_paired )
784 {
785 print $fh qq[
786 set origin 0.15,0.1
787 set size 0.8,0.4
788 unset title
789 unset format
790 set xtics
791 set xlabel "Quality"
792 unset label
793 set label "Cycle $lmax_cycle" at $lmax_qual+1,$lmax
794 set ylabel "Frequency (rev reads)"
795 plot '-' using 1:2:3 with lines linecolor variable title ''
796 ];
797 for my $cycle (0..$#lfq)
798 {
799 for (my $iqual=1; $iqual<$nquals; $iqual++)
800 {
801 printf $fh "%d\t%d\t%d\n", $iqual, $lfq[$cycle]->[$iqual], $cycle + 1;
802 }
803 print $fh "\n";
804 }
805 print $fh "end\n";
806 }
807 close($fh);
808 plot($$args{gp});
809
810 # Heatmap qualitites
811 $args = get_defaults($opts,"$$opts{prefix}quals-hm.png", wh=>'600,500');
812 $fh = $$args{fh};
813 my $max = defined $lmax && $lmax > $fmax ? $lmax : $fmax;
814 my $ytics = generate_ticks([ map { $_->[0] } @ffq ], $is_paired);
815 my $colorbox = "set colorbox vertical user origin first ($nquals+1),0 size screen 0.025,0.812";
816 $pos_size = $is_paired
817 ? " set origin 0.05,0.5\n set size 0.85,0.5" :
818 " set origin 0.05,0.05\n set size 0.85,1.0\n$colorbox" ;
819 my $set_xtics = $is_paired ? "unset" : "set";
820
821 print $fh qq[
822 $$args{terminal}
823 set output "$$args{img}"
824 unset key
825 unset colorbox
826
827 # Perceptually uniform heatmap "plasma" by Nathaniel J. Smith & Stefan van der Walt
828 # used in Python's matplotlib: https://github.com/BIDS/colormap/blob/master/colormaps.py
829 # Values from: https://github.com/Gnuplotting/gnuplot-palettes/blob/master/plasma.pal
830 # Dark end squashed to make low values more distinguishable.
831 set palette model RGB
832 set palette defined ( \\
833 0 '#000000' \\
834 , .7 '#0c0847' \\
835 , 4 '#4b0381' \\
836 , 10 '#7d03a8' \\
837 , 20 '#a82296' \\
838 , 30 '#cb4679' \\
839 , 40 '#e56b5d' \\
840 , 50 '#f89441' \\
841 , 60 '#fdc328' \\
842 , 70 '#f0f921' )
843
844 set cbrange [0:$max]
845 set yrange [0:$ncycles]
846 set xrange [0:$nquals]
847 set view map
848 set multiplot
849 set rmargin 0
850 set lmargin 0
851 set tmargin 0
852 set bmargin 0
853 $pos_size
854 set obj 1 rectangle behind from first 0,0 to first $nquals,$ncycles
855 set obj 1 fillstyle solid 1.0 fillcolor rgbcolor "black"
856 set ylabel "Cycle (fwd reads)" offset character -1,0
857 unset ytics
858 set ytics ($ytics)
859 $set_xtics xtics
860 set title "$$args{title}" noenhanced
861 splot '-' matrix with image
862 ];
863 for my $cycle (@ffq)
864 {
865 for (my $iqual=1; $iqual<@$cycle; $iqual++) { print $fh "\t$$cycle[$iqual]"; }
866 print $fh "\n";
867 }
868 print $fh "\nend\n";
869 if ( $is_paired )
870 {
871 my $ytics = generate_ticks([ map { $_->[0] } @lfq ], $is_paired);
872 print $fh qq[
873 set origin 0.05,0.05
874 set size 0.85,0.5
875 set ylabel "Cycle (rev reads)" offset character -1,0
876 set xlabel "Base Quality"
877 unset title
878 unset ytics
879 set ytics ($ytics)
880 set xrange [0:$nquals]
881 set xtics
882 $colorbox
883 set cblabel "Number of bases"
884 splot '-' matrix with image
885 ];
886 for my $cycle (@lfq)
887 {
888 for (my $iqual=1; $iqual<@$cycle; $iqual++) { print $fh "\t$$cycle[$iqual]"; }
889 print $fh "\n";
890 }
891 print $fh "\nend\n";
892 }
893 close($fh);
894 plot($$args{gp});
895 }
896
897 sub generate_ticks {
898 my ($cycles, $skip_even) = @_;
899
900 my @marks;
901 my $div = 10;
902 while (scalar(@$cycles)/$div > 20) {
903 $div *= 2;
904 }
905 $div = nice_num($div);
906 foreach my $v (@$cycles) {
907 if ($v % $div == 0) {
908 if ($skip_even) {
909 my $label = $v % 20 ? $v : '';
910 push @marks, qq{"$label" $v};
911 }
912 else {
913 push @marks, qq{"$v" $v};
914 }
915 }
916 }
917 return join(', ', @marks);
918 }
919
920 sub plot_acgt_cycles
921 {
922 my ($opts) = @_;
923
924 my @gcc = get_values($opts, 'GCC');
925 if ( !@gcc ) { return; }
926
927 my $args = get_defaults($opts,"$$opts{prefix}acgt-cycles.png");
928 my $fh = $$args{fh};
929 print $fh qq[
930 $$args{terminal}
931 set output "$$args{img}"
932 $$args{grid}
933 set style line 1 linecolor rgb "green"
934 set style line 2 linecolor rgb "red"
935 set style line 3 linecolor rgb "black"
936 set style line 4 linecolor rgb "blue"
937 set style increment user
938 set ylabel "Base content [%]"
939 set xlabel "Read Cycle"
940 set yrange [0:100]
941 set title "$$args{title}" noenhanced
942 plot '-' w l ti 'A', '-' w l ti 'C', '-' w l ti 'G', '-' w l ti 'T'
943 ];
944 for my $base (1..4)
945 {
946 for my $cycle (@gcc)
947 {
948 print $fh $$cycle[0]+1,"\t",$$cycle[$base],"\n";
949 }
950 print $fh "end\n";
951 }
952 close($fh);
953 plot($$args{gp});
954 }
955
956
957 sub plot_gc
958 {
959 my ($opts) = @_;
960
961 my $is_paired = get_value($opts,'SN','reads paired:');
962 my $args = get_defaults($opts,"$$opts{prefix}gc-content.png");
963 my $fh = $$args{fh};
964 my ($gcl_max,$gcf_max,$lmax,$fmax);
965 my @gcf = get_values($opts, 'GCF');
966 my @gcl = get_values($opts, 'GCL');
967 for my $gc (@gcf) { if ( !defined $gcf_max or $gcf_max<$$gc[1] ) { $gcf_max=$$gc[1]; $fmax=$$gc[0]; } }
968 for my $gc (@gcl) { if ( !defined $gcl_max or $gcl_max<$$gc[1] ) { $gcl_max=$$gc[1]; $lmax=$$gc[0]; } }
969 my $gcmax = $is_paired && $gcl_max > $gcf_max ? $lmax : $fmax;
970 print $fh qq[
971 $$args{terminal}
972 set output "$$args{img}"
973 $$args{grid}
974 set title "$$args{title}" noenhanced
975 set ylabel "Normalized Frequency"
976 set xlabel "GC Content [%]"
977 set yrange [0:1.1]
978 set label sprintf("%.1f",$gcmax) at $gcmax,1 front offset 1,0
979 plot ]
980 . (exists($$opts{ref_stats}) ? q['-' smooth csplines with lines lt 0 title 'Reference', ] : '')
981 . q['-' smooth csplines with lines lc 1 title 'First fragments' ]
982 . ($is_paired ? q[, '-' smooth csplines with lines lc 2 title 'Last fragments'] : '')
983 . q[
984 ];
985 if ( exists($$opts{ref_stats}) )
986 {
987 open(my $ref,'<',$$opts{ref_stats}) or error("$$opts{ref_stats}: $!");
988 while (my $line=<$ref>) { print $fh $line }
989 close($ref);
990 print $fh "end\n";
991 }
992 for my $cycle (@gcf) { printf $fh "%d\t%f\n", $$cycle[0],$$cycle[1]/$gcf_max; }
993 print $fh "end\n";
994 if ( $is_paired )
995 {
996 for my $cycle (@gcl) { printf $fh "%d\t%f\n", $$cycle[0],$$cycle[1]/$gcl_max; }
997 print $fh "end\n";
998 }
999 close($fh);
1000 plot($$args{gp});
1001 }
1002
1003
1004 sub plot_gc_depth
1005 {
1006 my ($opts) = @_;
1007
1008 my @gcd = get_values($opts,'GCD');
1009 if ( @gcd <= 1 ) { return; }
1010
1011 # Find unique sequence percentiles for 30,40, and 50% GC content, just to draw x2tics.
1012 my @tics = ( {gc=>30},{gc=>40},{gc=>50} );
1013 for my $gc (@gcd)
1014 {
1015 for my $tic (@tics)
1016 {
1017 my $diff = abs($$gc[0]-$$tic{gc});
1018 if ( !exists($$tic{pr}) or $diff<$$tic{diff} ) { $$tic{pr}=$$gc[1]; $$tic{diff}=$diff; }
1019 }
1020 }
1021
1022 my @x2tics;
1023 for my $tic (@tics) { push @x2tics, qq["$$tic{gc}" $$tic{pr}]; }
1024 my $x2tics = join(',',@x2tics);
1025
1026 my $args = get_defaults($opts,"$$opts{prefix}gc-depth.png", wh=>'600,500');
1027 my $fh = $$args{fh};
1028 print $fh qq[
1029 $$args{terminal}
1030 set output "$$args{img}"
1031 $$args{grid}
1032 set ylabel "Mapped depth"
1033 set xlabel "Percentile of mapped sequence ordered by GC content"
1034 set x2label "GC Content [%]"
1035 set title "$$args{title}" noenhanced
1036 set x2tics ($x2tics)
1037 set xtics nomirror
1038 set xrange [0.1:99.9]
1039
1040 plot '-' using 1:2:3 with filledcurve lt 1 lc rgb "#dedede" t '10-90th percentile' , \\
1041 '-' using 1:2:3 with filledcurve lt 1 lc rgb "#bbdeff" t '25-75th percentile' , \\
1042 '-' using 1:2 with lines lc rgb "#0084ff" t 'Median'
1043 ];
1044 for my $gc (@gcd) { print $fh "$$gc[1]\t$$gc[2]\t$$gc[6]\n"; } print $fh "end\n";
1045 for my $gc (@gcd) { print $fh "$$gc[1]\t$$gc[3]\t$$gc[5]\n"; } print $fh "end\n";
1046 for my $gc (@gcd) { print $fh "$$gc[1]\t$$gc[4]\n"; } print $fh "end\n";
1047 close($fh);
1048 plot($$args{gp});
1049 }
1050
1051
1052 sub plot_isize
1053 {
1054 my ($opts) = @_;
1055
1056 my $is_paired = get_value($opts,'SN','reads paired:');
1057 my @isizes = get_values($opts, 'IS');
1058 if ( !$is_paired or !@isizes ) { return; }
1059
1060 my ($isize_max,$isize_cnt);
1061 for my $isize (@isizes)
1062 {
1063 if ( !defined $isize_max or $isize_cnt<$$isize[1] ) { $isize_cnt=$$isize[1]; $isize_max=$$isize[0]; }
1064 }
1065
1066 my $args = get_defaults($opts,"$$opts{prefix}insert-size.png");
1067 my $fh = $$args{fh};
1068 print $fh qq[
1069 $$args{terminal}
1070 set output "$$args{img}"
1071 $$args{grid}
1072 set rmargin 5
1073 set label sprintf("%d",$isize_max) at $isize_max+10,$isize_cnt
1074 set ylabel "Number of pairs"
1075 set xlabel "Insert Size"
1076 set title "$$args{title}" noenhanced];
1077 print $fh qq[
1078 set logscale y 10] if exists $$opts{y_axis_log10};
1079 print $fh qq[
1080 plot \\
1081 '-' with lines lc rgb 'black' title 'All pairs', \\
1082 '-' with lines title 'Inward', \\
1083 '-' with lines title 'Outward', \\
1084 '-' with lines title 'Other'
1085 ];
1086 for my $isize (@isizes) { print $fh "$$isize[0]\t$$isize[1]\n"; } print $fh "end\n";
1087 for my $isize (@isizes) { print $fh "$$isize[0]\t$$isize[2]\n"; } print $fh "end\n";
1088 for my $isize (@isizes) { print $fh "$$isize[0]\t$$isize[3]\n"; } print $fh "end\n";
1089 for my $isize (@isizes) { print $fh "$$isize[0]\t$$isize[4]\n"; } print $fh "end\n";
1090 close($fh);
1091 plot($$args{gp});
1092 }
1093
1094
1095 sub plot_coverage
1096 {
1097 my ($opts) = @_;
1098
1099 my @cov = get_values($opts, 'COV');
1100 if ( !@cov ) { return; }
1101
1102 my @vals;
1103 for my $cov (@cov) { push @vals,$$cov[2]; }
1104 my $i = percentile(99.8,@vals);
1105 my $p99 = $cov[$i][1];
1106
1107 my $args = get_defaults($opts,"$$opts{prefix}coverage.png");
1108 my $fh = $$args{fh};
1109 print $fh qq[
1110 $$args{terminal}
1111 set output "$$args{img}"
1112 $$args{grid}
1113 set ylabel "Number of mapped bases"
1114 set xlabel "Coverage"
1115 set log y
1116 set style fill solid border -1
1117 set title "$$args{title}" noenhanced
1118 set xrange [:$p99]
1119 plot '-' with lines notitle
1120 ];
1121 for my $cov (@cov)
1122 {
1123 if ( $$cov[2]==0 ) { next; }
1124 print $fh "$$cov[1]\t$$cov[2]\n";
1125 }
1126 print $fh "end\n";
1127 close($fh);
1128 plot($$args{gp});
1129 }
1130
1131
1132 sub plot_mismatches_per_cycle
1133 {
1134 my ($opts) = @_;
1135
1136 my @mpc = get_values($opts, 'MPC');
1137 if ( !@mpc ) { return; }
1138
1139 my $nquals = @{$mpc[0]} - 2;
1140 my $ncycles = @mpc;
1141 my ($style,$with);
1142 if ( $ncycles>100 ) { $style = ''; $with = 'w l'; }
1143 else { $style = 'set style data histogram; set style histogram rowstacked'; $with = ''; }
1144
1145 my $args = get_defaults($opts,"$$opts{prefix}mism-per-cycle.png");
1146 my $fh = $$args{fh};
1147 print $fh qq[
1148 $$args{terminal}
1149 set output "$$args{img}"
1150 $$args{grid}
1151 set style line 1 linecolor rgb "#e40000"
1152 set style line 2 linecolor rgb "#ff9f00"
1153 set style line 3 linecolor rgb "#bbbb00"
1154 set style line 4 linecolor rgb "#4ebd68"
1155 set style line 5 linecolor rgb "#0061ff"
1156 set style increment user
1157 set key left top
1158 $style
1159 set ylabel "Number of mismatches"
1160 set xlabel "Read Cycle"
1161 set style fill solid border -1
1162 set title "$$args{title}" noenhanced
1163 set xrange [-1:$ncycles]
1164 plot '-' $with ti 'Base Quality>30', \\
1165 '-' $with ti '30>=Q>20', \\
1166 '-' $with ti '20>=Q>10', \\
1167 '-' $with ti '10>=Q', \\
1168 '-' $with ti "N's"
1169 ];
1170 for my $cycle (@mpc)
1171 {
1172 my $sum; for my $idx (31..$#$cycle) { $sum += $$cycle[$idx]; }
1173 print $fh "$sum\n";
1174 }
1175 print $fh "end\n";
1176 for my $cycle (@mpc)
1177 {
1178 my $sum; for my $idx (22..31) { $sum += $$cycle[$idx]; }
1179 print $fh "$sum\n";
1180 }
1181 print $fh "end\n";
1182 for my $cycle (@mpc)
1183 {
1184 my $sum; for my $idx (12..21) { $sum += $$cycle[$idx]; }
1185 print $fh "$sum\n";
1186 }
1187 print $fh "end\n";
1188 for my $cycle (@mpc)
1189 {
1190 my $sum; for my $idx (2..11) { $sum += $$cycle[$idx]; }
1191 print $fh "$sum\n";
1192 }
1193 print $fh "end\n";
1194 for my $cycle (@mpc) { print $fh "$$cycle[1]\n"; }
1195 print $fh "end\n";
1196 close($fh);
1197 plot($$args{gp});
1198 }
1199
1200 sub plot_indel_dist
1201 {
1202 my ($opts) = @_;
1203
1204 my @indels = get_values($opts, 'ID');
1205 if ( !@indels ) { return; }
1206
1207 my $args = get_defaults($opts,"$$opts{prefix}indel-dist.png");
1208 my $fh = $$args{fh};
1209 print $fh qq[
1210 $$args{terminal}
1211 set output "$$args{img}"
1212 $$args{grid}
1213 set style line 1 linetype 1 linecolor rgb "red"
1214 set style line 2 linetype 2 linecolor rgb "black"
1215 set style line 3 linetype 3 linecolor rgb "green"
1216 set style increment user
1217 set ylabel "Indel count [log]"
1218 set xlabel "Indel length"
1219 set y2label "Insertions/Deletions ratio"
1220 set log y
1221 set y2tics nomirror
1222 set ytics nomirror
1223 set title "$$args{title}" noenhanced
1224 plot '-' w l ti 'Insertions', '-' w l ti 'Deletions', '-' axes x1y2 w l ti "Ins/Dels ratio"
1225 ];
1226 for my $len (@indels) { print $fh "$$len[0]\t$$len[1]\n"; } print $fh "end\n";
1227 for my $len (@indels) { print $fh "$$len[0]\t$$len[2]\n"; } print $fh "end\n";
1228 for my $len (@indels) { printf $fh "%d\t%f\n", $$len[0],$$len[2]?$$len[1]/$$len[2]:0; } print $fh "end\n";
1229 close($fh);
1230 plot($$args{gp});
1231 }
1232
1233 sub plot_indel_cycles
1234 {
1235 my ($opts) = @_;
1236
1237 my @indels = get_values($opts, 'IC');
1238 if ( !@indels ) { return; }
1239
1240 my $is_paired = get_value($opts,'SN','reads paired:');
1241
1242 my $args = get_defaults($opts,"$$opts{prefix}indel-cycles.png");
1243 my $fh = $$args{fh};
1244 print $fh qq[
1245 $$args{terminal}
1246 set output "$$args{img}"
1247 $$args{grid}
1248 set style line 1 linetype 1 linecolor rgb "red"
1249 set style line 2 linetype 2 linecolor rgb "black"
1250 set style line 3 linetype 3 linecolor rgb "green"
1251 set style line 4 linetype 4 linecolor rgb "blue"
1252 set style increment user
1253 set ylabel "Indel count"
1254 set xlabel "Read Cycle"
1255 set title "$$args{title}" noenhanced
1256 ];
1257 if ( $is_paired )
1258 {
1259 print $fh qq[plot '-' w l ti 'Insertions (fwd)', '' w l ti 'Insertions (rev)', '' w l ti 'Deletions (fwd)', '' w l ti 'Deletions (rev)'\n];
1260 for my $len (@indels) { print $fh "$$len[0]\t$$len[1]\n"; } print $fh "end\n";
1261 for my $len (@indels) { print $fh "$$len[0]\t$$len[2]\n"; } print $fh "end\n";
1262 for my $len (@indels) { print $fh "$$len[0]\t$$len[3]\n"; } print $fh "end\n";
1263 for my $len (@indels) { print $fh "$$len[0]\t$$len[4]\n"; } print $fh "end\n";
1264 }
1265 else
1266 {
1267 print $fh qq[plot '-' w l ti 'Insertions', '' w l ti 'Deletions'\n];
1268 for my $len (@indels) { print $fh "$$len[0]\t$$len[2]\n"; } print $fh "end\n";
1269 for my $len (@indels) { print $fh "$$len[0]\t$$len[4]\n"; } print $fh "end\n";
1270 }
1271 close($fh);
1272 plot($$args{gp});
1273 }
1274
1275
1276 sub nice_num
1277 {
1278 my ($num) = @_;
1279 my $ret;
1280
1281 if ($num < 1) {
1282 $ret = 1;
1283 } else {
1284 my $expon_num = floor(log($num)/log(10));
1285 my $fract_num = ($num / (10.0 ** $expon_num));
1286
1287 my $nf = ceil($fract_num);
1288
1289 $ret = ($nf * (10.0 ** $expon_num));
1290 }
1291
1292 return $ret;
1293 }
1294
1295
1296 sub plot_read_len_hist
1297 {
1298 my ($opts) = @_;
1299
1300 my @rl = get_values($opts, 'RL');
1301 if ( !@rl ) { return; }
1302
1303 # TODO: Add modified version for paired end reads.
1304 # my $is_paired = get_value($opts,'SN','reads paired:');
1305
1306 my $args = get_defaults($opts,"$$opts{prefix}read-length.png");
1307 my $fh = $$args{fh};
1308
1309 # anonymous subroutines for code readibility
1310 my $log_bin = sub {
1311 my $x = $_[0];
1312 return floor((log($x) / log(10)) * 10);
1313 };
1314 my $inv_log_bin = sub {
1315 my $x = $_[0];
1316 return floor(10**($x/10));
1317 };
1318
1319 # Open files to store data: this is more elegant than copying four sets of
1320 # data into a single GNUplot file.
1321 open(my $rd_fh, '>', "$$opts{prefix}read_len_rawdata.txt");
1322 open(my $hd_fh, '>', "$$opts{prefix}read_len_histdata.txt");
1323
1324 # Create gnuplot command file options
1325 my $min_x = $rl[0][0];
1326 my $max_x = $rl[-1][0];
1327 my $min_y = $rl[0][1];
1328 my $max_y = $min_y;
1329
1330
1331 my @logscale_data = (0)x&$log_bin($max_x);
1332 my $log_x = 0;
1333 my $start_i = 1;
1334 for my $line (@rl)
1335 {
1336 my @deref_line = @{$line};
1337 my $curr_x = $deref_line[0];
1338 my $curr_y = $deref_line[1];
1339
1340 if ($min_y > $curr_y) {
1341 $min_y = $curr_y;
1342 }
1343
1344 if ($max_y < $curr_y) {
1345 $max_y = $curr_y;
1346 }
1347
1348 print $rd_fh "$curr_x $curr_y\n";
1349 $log_x = max(&$log_bin($curr_x), 5);
1350 @logscale_data[floor($log_x)] += $curr_y;
1351 $start_i = $curr_x + 1;
1352 }
1353
1354 # Copy binned data into its own file
1355
1356 # This is a special case to deal with the problem that reads of len < 5 do
1357 # not cooperate with the log10(x)*10 transformation and cause small
1358 # fragments to graph badly.
1359 my $start = 1;
1360 my $end = 3;
1361 my $div_binw_val = $logscale_data[5]/($end - $start);
1362 my $start_idx = -1;
1363 for my $i (0..@logscale_data-1) {
1364 if($logscale_data[$i] != 0)
1365 {
1366 $start_idx = $i;
1367 if ($i == 5)
1368 {
1369 print $hd_fh "5 $logscale_data[5] $div_binw_val $start $end\n";
1370 $start_idx += 1;
1371 }
1372 last;
1373 }
1374 }
1375
1376 my $bin_min_x = &$inv_log_bin($start_idx);
1377 my $bin_max_x = &$inv_log_bin($start_idx + 1);
1378 my $log_scale_max = $logscale_data[$start_idx];
1379 my $log_bin_max = $logscale_data[$start_idx] / max(($bin_max_x - $bin_min_x), 1);
1380
1381 for my $i ($start_idx..@logscale_data-1) {
1382 $start = &$inv_log_bin($i);
1383 $end = &$inv_log_bin($i+1);
1384
1385 if ($bin_max_x < $end) {
1386 $bin_max_x = $end;
1387 }
1388
1389 if ($log_scale_max < $logscale_data[$i]) {
1390 $log_scale_max = $logscale_data[$i];
1391 }
1392
1393 $div_binw_val = $logscale_data[$i]/max(($end - $start), 1);
1394
1395 if ($log_bin_max < $div_binw_val) {
1396 $log_bin_max = $div_binw_val;
1397 }
1398
1399 print $hd_fh "$i $logscale_data[$i] $div_binw_val $start $end\n";
1400 }
1401
1402 if ($max_x < $bin_max_x) {
1403 $max_x = $bin_max_x;
1404 }
1405
1406 if ($min_x > $bin_min_x) {
1407 $min_x = $bin_min_x;
1408 }
1409
1410 $max_y = nice_num($max_y);
1411 $log_scale_max = nice_num($log_scale_max);
1412 $log_bin_max = nice_num($log_bin_max);
1413
1414 my $logscale = "";
1415
1416 if (($max_x - $min_x) > 100) {
1417 $logscale = "set logscale x\n";
1418 }
1419
1420 # Gnuplot commands for normalized binned data
1421 # The normalized plot bins reads by read length with bin widths on a
1422 # logarithmic scale (i.e. bins 10-11 bp, 12-14 bp, 15-18 bp, etc.), then
1423 # bin height is normalized by the width of the bins.
1424 # The end goal is to smooth the distribution of reads compared to plotting
1425 # each length without binning. Visualization of unbinned data is included
1426 # in order to make single value spikes more visible, for example in the
1427 # case that spike ins are used as a control.
1428 print $fh qq[
1429 set ytics nomirror
1430 set y2tic nomirror
1431 set terminal png size 800,1000 truecolor
1432 set output "$$args{img}"
1433 $$args{grid}
1434 set grid noy2tics
1435 set multiplot layout 2,1 columnsfirst scale 1,0.9
1436 set yrange[0: $max_y]
1437 set ylabel "Read Count"
1438 $logscale
1439 set xrange[$min_x: $max_x]
1440 set xtics rotate by -45 out scale 1 nomirror
1441 set xlabel "Read Length"
1442 set boxwidth .8 relative
1443 set title "$$args{title}" noenhanced
1444 set title offset -15,-.5
1445 set key at graph 1, 1.15
1446 set y2range[0:$log_bin_max]
1447 plot "$$opts{prefix}read_len_rawdata.txt" using 1:2 lt rgb "orange" with boxes fs solid .7 title 'unbinned', "$$opts{prefix}read_len_histdata.txt" using 4:3 lt rgb "blue" with boxes title 'normalized bins(count/bin width)' axis x1y2
1448 ];
1449 print $fh qq[];
1450
1451 # Gnuplot commands for non-normalized binned data
1452 # The un-normalized plot has reads assigned to bins identically to the
1453 # normalized plot, however, the bin height is not divided by bin width. The
1454 # goal of this visualization is to more accurately reflect the distribution
1455 # of reads by volume, and will generally collect around the mean read
1456 # length.
1457 # Inclusion of unbinned data is to give point of visual comparison to
1458 # normalized histogram.
1459 print $fh qq[
1460 set boxwidth 1 relative
1461 set y2range[0:$log_scale_max]
1462 plot "$$opts{prefix}read_len_rawdata.txt" using 1:2 lt rgb "orange" with boxes fs solid .7 title 'unbinned', "$$opts{prefix}read_len_histdata.txt" using 4:2 lt rgb "blue" with boxes title 'unnormalized bins' axis x1y2
1463 ];
1464
1465 # Close file buffers and clean up workspace
1466 close($fh);
1467 close($hd_fh);
1468 close($rd_fh);
1469 plot($$args{gp});
1470 unlink("$$opts{prefix}read_len_rawdata.txt");
1471 unlink("$$opts{prefix}read_len_histdata.txt");
1472 }
1473
1474 sub merge_bamcheck
1475 {
1476 my ($opts) = @_;
1477 my $fh = *STDOUT;
1478 if ( !$$opts{merge} )
1479 {
1480 open($fh,'>',"$$opts{prefix}merge.bchk") or error("$$opts{prefix}merge.bchk: $!\n");
1481 }
1482
1483 print $fh "# This file was produced by plot-bamstats and can be plotted using plot-bamstats\n# The command line was $$opts{args}\n";
1484
1485 for my $sec (@{$$opts{sections}})
1486 {
1487 my $sid = $$sec{id};
1488 if ( !exists($$opts{dat}{$sid}) ) { next; }
1489
1490 print $fh $$sec{header};
1491 if ($sid eq 'SN') {
1492 for my $key (@{$$opts{sn_order}}) {
1493 if (exists($$opts{dat}{$sid}{$key})) {
1494 print "$sid\t$key\t$$opts{dat}{$sid}{$key}\n";
1495 }
1496 }
1497 next;
1498 }
1499 elsif ( ref($$opts{dat}{$sid}) eq 'HASH' )
1500 {
1501 for my $key (keys %{$$opts{dat}{$sid}})
1502 {
1503 print "$sid\t$key\t$$opts{dat}{$sid}{$key}\n";
1504 }
1505 next;
1506 }
1507 elsif ( ref($$opts{dat}{$sid}) eq 'ARRAY' )
1508 {
1509 for my $rec (@{$$opts{dat}{$sid}})
1510 {
1511 print $fh "$sid\t", join("\t",@$rec), "\n";
1512 }
1513 }
1514
1515 #
1516 # if ( $sid eq 'ID' )
1517 # {
1518 # print $fh "# $$opts{id2sec}{SN}{header}\n$$opts{id2sec}{SN}{exp}";
1519 # # output summary numbers here
1520 # for my $id (keys %{$$opts{dat}})
1521 # {
1522 # if ( exists($$opts{exp}{$id}) ) { next; }
1523 # for my $key (keys %{$$opts{dat}{$id}})
1524 # {
1525 # print $fh "SN\t$id\t$key\t$$opts{dat}{$id}{$key}\n";
1526 # }
1527 # }
1528 # }
1529 }
1530
1531 }
1532
1533 sub bignum
1534 {
1535 my ($num) = @_;
1536 if ( !defined $num ) { return 0; }
1537 my $out = '';
1538 my $slen = length($num);
1539 for (my $i=0; $i<$slen; $i++)
1540 {
1541 $out .= substr($num,$i,1);
1542 if ( $i+1<$slen && ($slen-$i-1)%3==0 ) { $out .= ','; }
1543 }
1544 return $out;
1545 }
1546
1547 sub create_html
1548 {
1549 my ($opts) = @_;
1550 my ($fname,$prefix);
1551 if ( $$opts{prefix}=~m{/$} )
1552 {
1553 $fname = "$$opts{prefix}index.html";
1554 $prefix = '';
1555 }
1556 else
1557 {
1558 $prefix = $$opts{prefix};
1559 $prefix =~ s{^.*/}{};
1560 $fname = $$opts{prefix};
1561 $fname =~ s/-$/.html/;
1562 }
1563 open(my $fh,'>',$fname) or error("$fname: $!");
1564 print $fh q[
1565 <html>
1566 <head>
1567 <style>
1568 .thumbnail
1569 {
1570 text-decoration:none;
1571 color:black;
1572 font-weight:bold;
1573 }
1574 .thumbnail span /*CSS for enlarged image*/
1575 {
1576 visibility: hidden;
1577 position: absolute;
1578 padding: 5px;
1579 border: 1px solid #000;
1580 background-color: #e5e5e5;
1581 }
1582 .thumbnail:hover span /*CSS for enlarged image on hover*/
1583 {
1584 visibility: visible;
1585 left: 550px;
1586 top: 10px;
1587 }
1588 .imgs td
1589 {
1590 vertical-align:middle;
1591 padding: 0.5em;
1592 border: 1px solid black;
1593 }
1594 table.imgs
1595 {
1596 border-collapse:collapse;
1597 margin-left:20px;
1598 }
1599 .nums th
1600 {
1601 text-align: left;
1602 }
1603 table.nums
1604 {
1605 margin-top: 1em;
1606 margin-left:20px;
1607 border: 1px dotted #83A4C3;
1608 background-color: #F5F5F5;
1609 padding: 0.5em;
1610 }
1611 .pad { padding-left:1em; vertical-align:top; }
1612 .right { text-align:right; padding-left:1em; }
1613 </style>
1614 </head>
1615 <body>
1616 <table class="imgs">
1617 ];
1618
1619 my %imgs =
1620 (
1621 'insert-size' => 'Insert size',
1622 'gc-content' => 'GC content',
1623 'acgt-cycles' => 'Per-base sequence content',
1624 'mism-per-cycle' => 'Mismatches per cycle',
1625 'quals' => 'Quality per cycle',
1626 'quals2' => 'Quality per cycle',
1627 'quals3' => 'Quality per cycle',
1628 'quals-hm' => 'Quality per cycle',
1629 'indel-cycles' => 'Indels per cycle',
1630 'indel-dist' => 'Indel lengths',
1631 'gc-depth' => 'Mapped depth vs GC',
1632 'read-length' => 'Read lengths',
1633 # I think this may be broken: 'coverage' => '',
1634 );
1635 my @imgs = (qw(insert-size gc-content acgt-cycles mism-per-cycle quals quals2 quals3 quals-hm indel-cycles indel-dist gc-depth read-length));
1636 for (my $i=0; $i<@imgs; $i++)
1637 {
1638 if ( $i % 3 == 0 )
1639 {
1640 # new row
1641 if ( $i>0 ) { print $fh "</tr>\n"; }
1642 print $fh "<tr>\n";
1643 }
1644 if ( -e "$$opts{prefix}$imgs[$i].png" )
1645 {
1646 my $desc = $imgs{$imgs[$i]};
1647 my $link = uri_escape("$prefix$imgs[$i].png");
1648 print $fh qq[ <td><a class="thumbnail" href="$link"><img src="$link" width="150px"><span>$desc<br><img src="$link"></span></a>\n];
1649 }
1650 else { print $fh " <td />\n"; }
1651 }
1652 print $fh "</tr></table>\n";
1653
1654 my $reads_total = get_value($opts,'SN','raw total sequences:');
1655 my $reads_filtered = get_value($opts,'SN','filtered sequences:');
1656 my $percent_filtered = sprintf "(%.1f%%)", $reads_filtered * 100. / $reads_total;
1657 my $reads_mapped = get_value($opts,'SN','reads mapped:');
1658 my $percent_mapped = sprintf "(%.1f%%)", $reads_mapped * 100. / ($reads_total - $reads_filtered);
1659 my $reads_dup = get_value($opts,'SN','reads duplicated:');
1660 my $percent_dup = sprintf "(%.1f%%)", $reads_dup * 100. / ($reads_total - $reads_filtered);
1661 my $reads_mq0 = get_value($opts,'SN','reads MQ0:');
1662 my $percent_mq0 = sprintf "(%.1f%%)", ($reads_mapped ? $reads_mq0 * 100. / $reads_mapped : 0);
1663 my $reads_nonprim = get_value($opts,'SN','non-primary alignments:');
1664 my $read_avglen = get_value($opts,'SN','average length:');
1665 my $bases_total = get_value($opts,'SN','total length:');
1666 my $bases_mapped = get_value($opts,'SN','bases mapped (cigar):');
1667 my $bpercent_mapped = sprintf "(%.1f%%)", $bases_mapped * 100. / $bases_total;
1668 my $error_rate = sprintf "%.2f%%", 100.*get_value($opts,'SN','error rate:');
1669
1670 $reads_total = bignum($reads_total);
1671 $reads_filtered = bignum($reads_filtered);
1672 $reads_mapped = bignum($reads_mapped);
1673 $reads_dup = bignum($reads_dup);
1674 $reads_mq0 = bignum($reads_mq0);
1675 $reads_nonprim = bignum($reads_nonprim);
1676 $bases_total = bignum($bases_total);
1677 $bases_mapped = bignum($bases_mapped);
1678
1679 print $fh qq[
1680
1681 <table class="nums">
1682 <tr><th>Reads</tr>
1683 <tr>
1684 <td class="pad"><table>
1685 <tr><td>total: <td class="right"> $reads_total <td class="right"></tr>
1686 <tr><td>filtered: <td class="right"> $reads_filtered <td class="right"> $percent_filtered</tr>
1687 <tr><td>non-primary: <td class="right"> $reads_nonprim <td class="right"> </tr>
1688 <tr><td>duplicated: <td class="right"> $reads_dup <td class="right"> $percent_dup</tr>
1689 <tr><td>mapped: <td class="right"> $reads_mapped <td class="right"> $percent_mapped</tr>
1690 <tr><td>zero MQ: <td class="right"> $reads_mq0 <td class="right"> $percent_mq0</tr>
1691 <tr><td>avg read length: <td class="right"> $read_avglen <td class="right"></tr>
1692 </table></tr>
1693
1694 <tr><th>Bases</tr>
1695 <tr>
1696 <td class="pad"><table>
1697 <tr><td>total: <td class="right"> $bases_total <td class="right"></tr>
1698 <tr><td>mapped: <td class="right"> $bases_mapped <td class="right"> $bpercent_mapped</tr>
1699 <tr><td>error rate: <td class="right"> $error_rate <td class="right"></tr>
1700 </table></tr>
1701 </table>
1702
1703 </body>
1704 </html>
1705 ];
1706
1707 close($fh);
1708 }