Debian Codegraph @

bait-fisher.cpp

raw
1 /* BaitFisher (version 1.2.8) a program for designing DNA target enrichment baits
2 * Copyright 2013-2016 by Christoph Mayer
3 *
4 * This source file is part of the BaitFisher-package.
5 *
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with BaitFisher. If not, see <http://www.gnu.org/licenses/>.
18 *
19 *
20 * For any enquiries send an Email to Christoph Mayer
21 * [email protected]
22 *
23 * When publishing work that is based on the results please cite:
24 * Mayer et al. 2016: BaitFisher: A software package for multi-species target DNA enrichment probe design
25 *
26 */
27 #define NODEBUG
28 #include "faststring2.h"
29 #include <iostream>
30 #include "GFF-class.h"
31 #include "GFF-collection.h"
32 #include "mydir-unix.h"
33 #include <list>
34 #include "CFile/CFile2_1.h"
35 #include "CSequences2.h"
36 #include "CSequence_Mol2_1.h"
37 #include "CDnaString2.h"
38 #include <climits>
39 #include <ctime>
40
41 #include "CAligner/CAligner.h"
42 #include "scoring-matrices/CScoreMatrix.h"
43 #include "basic-DNA-RNA-AA-routines.h"
44
45 #include "CDistance_matrix.h"
46 #include "CSeqNameList.h"
47 #include "CTaxonNamesDictionary.h"
48 #include "CRequiredTaxon.h"
49
50 //#include "pipeline-step3.h"
51 #include "Csequence_cluster_and_center_sequence.h"
52 #include "DEBUG_STUFF.h"
53
54
55 #define PROGNAME "BaitFisher"
56 #define VERSION "1.2.8"
57
58
59 using namespace std;
60
61 #include "bait-fisher-helper.h"
62
63 // Verbosity rules:
64 // verbosity 0 -> only error messages that lead to exit() (goes to cerr)
65 // verbosity 1 -> warnings, ALERT (goes to cerr)
66 // verbosity 2 -> more warnings, e.g. skipping features
67 // verbosity 3 -> progress (goes to cout)
68 // verbosity 4 -> more progress (goes to cout)
69 // verbosity 6 -> per alignment/feature success NOTES
70 // verbosity 8 -> per coordinate success NOTES
71 // verbosity >=20: DEGUB
72 // Default: 5
73
74 unsigned global_VERBOSITY = 0; // Will be initialised in parse_parameter_file. This value will never be used.
75
76
77
78
79 // Goal of this program:
80 //
81
82
83 int DEBUG_COUNT_NEW_CALLS_FEATURE_ALIGNMENT = 0;
84 int DEBUG_COUNT_NEW_CALLS_CBAIT_REGION_INFORMATION = 0;
85 int DEBUG_COUNT_NEW_CALLS_CSEQUENCE_LOCI_CLUSTER_COLLECTION = 0;
86 int DEBUG_COUNT_NEW_CALLS_MSA = 0;
87 int DEBUG_COUNT_NEW_ALIGNMENT_FROM_FILE = 0;
88
89 int DEBUG_COUNT_DELETE_CALLS_FEATURE_ALIGNMENT = 0;
90 int DEBUG_COUNT_DELETE_CALLS_CBAIT_REGION_INFORMATION = 0;
91 int DEBUG_COUNT_DELETE_CALLS_CSEQUENCE_LOCI_CLUSTER_COLLECTION = 0;
92 int DEBUG_COUNT_DELETE_CALLS_MSA = 0;
93 int DEBUG_COUNT_DELETE_ALIGNMENT_FROM_FILE = 0;
94
95 void print_DEBUG_COUNT (ostream &os, const char * comment)
96 {
97 os << "DEBUG: NEW/DELETE COUNTs: " << comment << endl;
98 os << "DEBUG: NUMBER OF ALIGNMENTS CREATED/DELETED: " << DEBUG_COUNT_NEW_ALIGNMENT_FROM_FILE << "/" << DEBUG_COUNT_DELETE_ALIGNMENT_FROM_FILE << endl;
99 os << "DEBUG: NUMBER OF FEATUREALIGNMENT C/D: " << DEBUG_COUNT_NEW_CALLS_FEATURE_ALIGNMENT << "/" << DEBUG_COUNT_DELETE_CALLS_FEATURE_ALIGNMENT << endl;
100 os << "DEBUG: MSA CREATED/DELETED " << DEBUG_COUNT_NEW_CALLS_MSA << "/" << DEBUG_COUNT_DELETE_CALLS_MSA << endl;
101 os << "DEBUG: CSEQUENCE_LOCI_CLUSTER_COLLECTION C/D: " << DEBUG_COUNT_NEW_CALLS_CSEQUENCE_LOCI_CLUSTER_COLLECTION << "/" << DEBUG_COUNT_DELETE_CALLS_CSEQUENCE_LOCI_CLUSTER_COLLECTION << endl;
102 os << "DEBUG: CBAIT_REGION_INFORMATION C/D: " << DEBUG_COUNT_NEW_CALLS_CBAIT_REGION_INFORMATION << "/" << DEBUG_COUNT_DELETE_CALLS_CBAIT_REGION_INFORMATION << endl;
103 }
104
105 void get_time_string(faststring &str)
106 {
107 time_t rawtime;
108 struct tm *timeinfo;
109 static char buffer[80];
110
111 std::time (&rawtime);
112 timeinfo = std::localtime(&rawtime);
113 std::strftime (buffer,80,"%F %T",timeinfo);
114 str = faststring(buffer);
115 }
116
117 template<class T>
118 int delete_pointers_in_vector(vector<T> &v)
119 {
120 int i, N, count=0;
121 N = v.size();
122
123 for (i=0; i<N; ++i)
124 {
125 if (v[i])
126 {
127 ++count;
128 delete v[i];
129 }
130 }
131 v.clear();
132 return count;
133 }
134
135
136
137 // Add more checks!! Currently we do not check whether parameters occur twice, etc.
138 void parse_parameter_file(faststring & fname_parameter_file,
139 faststring & fname_gff,
140 faststring & fname_genome,
141 faststring & string_required_taxa,
142 faststring & dname_alignments,
143 faststring & genome_name,
144 unsigned short & bait_length,
145 double & cluster_threshold,
146 faststring & output_directory,
147 unsigned short & bait_offset,
148 unsigned short & bait_number,
149 bool & remove_reference_species_from_alignments,
150 bool & alignment_cutting,
151 faststring & gff_feature_name,
152 faststring & gff_record_attribute_name_for_geneID,
153 char & sequence_name_field_delimiter,
154 unsigned short & sequence_name_taxon_field_number,
155 unsigned short & sequence_name_geneID_field_number,
156 bool & sequence_name_tokenizer_overwritten,
157 char & center_computation_mode,
158 unsigned & verbosity
159 )
160 {
161 fname_gff.clear();
162 fname_genome.clear();
163 string_required_taxa.clear();
164 dname_alignments.clear();
165 genome_name.clear();
166 bait_length = 0;
167 cluster_threshold = -1;
168 output_directory.clear();
169 bait_offset = USHRT_MAX;
170 bait_number = USHRT_MAX;
171 remove_reference_species_from_alignments = false;
172 alignment_cutting = false;
173
174 sequence_name_field_delimiter = '|';
175 sequence_name_taxon_field_number = 2;
176 sequence_name_geneID_field_number = 3;
177 sequence_name_tokenizer_overwritten = false;
178
179 center_computation_mode = 1; // 0:exhaustive, 1:heuristic, 3: special comparison
180 verbosity = 5;
181
182 faststring line, pre, post;
183
184 ifstream is;
185 is.open(fname_parameter_file.c_str());
186 if (is.fail() )
187 {
188 cerr << "Error: Parameter file \"" << fname_parameter_file << "\" could not be opened. I guess it does not exist." << endl;
189 exit(-1);
190 }
191
192 getline(is, line);
193
194 while (!is.fail() )
195 {
196 line.shorten_to_first_occurrence_of('#');
197 line.removeSpacesBack();
198 line.removeSpacesFront();
199
200 if ( line.empty() )
201 {
202 getline(is, line);
203 continue;
204 }
205
206 if(line.divide_at_first_of(" \t",pre, post))
207 {
208 pre.removeSpacesBack();
209 post.removeSpacesFront();
210 }
211 else
212 {
213 pre = line;
214 post.clear();
215 }
216
217 if (pre == "gff-file-reference-genome")
218 {
219 fname_gff = post;
220 }
221 else if (pre == "reference-genome-file")
222 {
223 fname_genome = post;
224 }
225 else if (pre == "required-taxonomic-groups-string")
226 {
227 string_required_taxa = post;
228 }
229 else if (pre == "directory-transcript-files")
230 {
231 dname_alignments = post;
232 }
233 else if (pre == "reference-genome-name")
234 {
235 genome_name = post;
236 }
237 else if (pre == "bait-length")
238 {
239 bait_length = post.ToUnsigned();
240 }
241 else if (pre == "bait-offset")
242 {
243 bait_offset = post.ToUnsigned();
244 }
245 else if (pre == "bait-number")
246 {
247 bait_number = post.ToUnsigned();
248 }
249 else if (pre == "cluster-threshold")
250 {
251 cluster_threshold = post.ToDouble();
252 }
253 else if (pre == "output-directory")
254 {
255 output_directory = post;
256 }
257 else if (pre == "remove-reference-sequence")
258 {
259 post.ToLower();
260 if (post == "yes")
261 remove_reference_species_from_alignments = true;
262 else if (post == "no")
263 remove_reference_species_from_alignments = false;
264 else
265 {
266 cerr << "Error: Unknown option " << post << " for parameter " << pre << endl;
267 cerr << "The " << pre << " parameter will be ignored." << endl;
268 exit(-7);
269 }
270 }
271 else if (pre == "alignment-cutting")
272 {
273 post.ToLower();
274 if (post == "yes")
275 alignment_cutting = true;
276 else if (post == "no")
277 alignment_cutting = false;
278 else
279 {
280 cerr << "Error: Unknown option " << post << " for parameter " << pre << endl;
281 cerr << "The " << pre << " parameter will be ignored." << endl;
282 exit(-7);
283 }
284 }
285 else if (pre == "gff-feature-name")
286 {
287 gff_feature_name = post;
288 }
289 else if (pre == "gff-record-attribute-name-for-geneID")
290 {
291 gff_record_attribute_name_for_geneID = post;
292 }
293 else if (pre == "sequence-name-field-delimiter")
294 {
295 if (post.size()!=1)
296 {
297 cerr << "ERROR when parsing the parameter file: sequence-name-field-delimiter must be a single character, but I found "
298 << sequence_name_field_delimiter << "." << endl;
299 exit(-1);
300 }
301 sequence_name_field_delimiter = post[0];
302 }
303 else if (pre == "sequence-name-taxon-field_number")
304 {
305 sequence_name_taxon_field_number = post.ToUnsigned();
306 }
307 else if (pre == "sequence-name-geneID-field_number")
308 {
309 sequence_name_geneID_field_number = post.ToUnsigned();
310 }
311 else if (pre == "verbosity")
312 {
313 verbosity = post.ToUnsigned();
314 }
315 else if (pre == "Hamming-center-sequence-computation-mode" || pre == "centroid-computation-mode")
316 {
317 if (post == "exhaustive")
318 {
319 center_computation_mode = 0;
320 }
321 else if (post == "heuristic")
322 {
323 center_computation_mode = 1;
324 }
325 else if (post == "comparison")
326 {
327 center_computation_mode = 3;
328 }
329 else
330 {
331 cerr << "Unknown option for parameter \"center-computation-mode\" in parameter file. Allowed options are: \"exhaustive\" and \"heuristic\"." << endl;
332 exit(-1);
333 }
334 }
335 else
336 {
337 cerr << "Error: Unknown parameter in parameter file: \"" << pre << "\"."<< endl;
338 exit(-1);
339 }
340 getline(is, line);
341 } // END while (!is.fail() )
342
343 // Basic consitency checking of parameters.
344 // Are all required parameters supplied?
345
346 if (alignment_cutting) // Then we require some parameters:
347 {
348 if (fname_gff.empty())
349 {
350 cerr << "Error: The parameter \"gff-file\" is required if alignment_cutting is set to yes. This parameter must be specified in the parameter file." << endl;
351 exit(-4);
352 }
353
354 if (fname_genome.empty() )
355 {
356 cerr << "Error: The parameter \"genome-file\" is required if alignment_cutting is set to yes. This parameter must be specified in the parameter file." << endl;
357 exit(-4);
358 }
359
360 if (genome_name.empty() )
361 {
362 cerr << "Error: The parameter \"genome-name\" is required if alignment_cutting is set to yes. This parameter must be specified in the parameter file." << endl;
363 exit(-4);
364 }
365 // Parameters with default values:
366 if (gff_feature_name.empty() )
367 gff_feature_name = "CDS";
368 if (gff_record_attribute_name_for_geneID.empty() )
369 gff_record_attribute_name_for_geneID = "Parent";
370 }
371 else // No alignment cutting
372 {
373 if (!fname_gff.empty())
374 {
375 cerr << "Error: The parameter \"gff-file\" is specified even though alignment_cutting is set to no. Please remove this parameter from the parameter file." << endl;
376 exit(-4);
377 }
378
379 if (!fname_genome.empty() )
380 {
381 cerr << "Error: The parameter \"genome-file\" is specified even though alignment_cutting is set to no. Please remove this parameter from the parameter file." << endl;
382 exit(-4);
383 }
384
385 if (!genome_name.empty() )
386 {
387 cerr << "Error: The parameter \"genome-name\" is specified even though alignment_cutting is set to no. Please remove this parameter from the parameter file." << endl;
388 exit(-4);
389 }
390 if (!gff_feature_name.empty() )
391 {
392 cerr << "Error: The parameter \"gff-feature-name\" is specified even though alignment_cutting is set to no. Please remove this parameter from the parameter file." << endl;
393 exit(-4);
394 }
395 if (!gff_record_attribute_name_for_geneID.empty() )
396 {
397 cerr << "Error: The parameter \"gff-record-attribute-name-for-geneID\" is specified even though alignment_cutting is set to no. Please remove this parameter from the parameter file." << endl;
398 exit(-4);
399 }
400 }
401
402 // if (string_required_taxa.empty() )
403 // {
404 // cerr << "Error: The required parameter \"required-taxonomic-groups-string\" is not specified in the parameter file." << endl;
405 // exit(-4);
406 // }
407 if (dname_alignments.empty() )
408 {
409 cerr << "Error: The required parameter \"directory-transcript-files\" is not specified in the parameter file." << endl;
410 exit(-4);
411 }
412 if (bait_length == 0)
413 {
414 cerr << "Error: Required parameter \"bait-length\" is not specified in the parameter file." << endl;
415 exit(-4);
416 }
417 if (cluster_threshold == -1)
418 {
419 cerr << "Error: Required parameter \"cluster-threshold\" is not specified in the parameter file." << endl;
420 exit(-4);
421 }
422 if ( output_directory.empty() )
423 {
424 cerr << "Error: Required parameter \"output-directory\" is not specified in the parameter file." << endl;
425 exit(-4);
426 }
427 if (bait_number == SHRT_MAX)
428 {
429 cerr << "Error: Required parameter \"bait-number\" is not specified in the parameter file." << endl;
430 exit(-4);
431 }
432 if (bait_number == 0)
433 {
434 cerr << "Error: Parameter \"bait-number\" has been set to 0 in the parameter file, which is not a valid value." << endl;
435 exit(-4);
436 }
437 if (bait_offset == USHRT_MAX)
438 {
439 cerr << "Error: Required parameter \"bait-offset\" is not specified in the parameter file." << endl;
440 exit(-4);
441 }
442 if (bait_offset == 0 && bait_number > 1)
443 {
444 cerr << "Error: The parameter \"bait-offset\" has been set to 0 in the parameter file, while the bait number is larger than 1. This does not make sense." << endl;
445 cerr << "Please specify a valid tiling design with bait number equal to 1 and bait offset equal to 0, or bait number larger than 1 and offset larger than 0." << endl;
446 exit(-4);
447 }
448
449 if (!alignment_cutting && string_required_taxa.empty() ) // In this case, sequence names do not need to contain the taxon name.
450 {
451 sequence_name_field_delimiter = '\0';
452 sequence_name_taxon_field_number = 1;
453 sequence_name_geneID_field_number = 1;
454 sequence_name_tokenizer_overwritten = true;
455 }
456
457 // The field numbers are stored 0-based. In the user interaction, numbers are 1-based.
458 --sequence_name_geneID_field_number;
459 --sequence_name_taxon_field_number;
460 }
461
462
463 void print_parameters(ostream & os,
464 faststring & fname_gff,
465 faststring & fname_genome,
466 faststring & string_required_taxa,
467 faststring & dname_alignments,
468 faststring & genome_name,
469 unsigned short bait_length,
470 double cluster_threshold,
471 faststring & output_directory,
472 unsigned short bait_offset,
473 unsigned short bait_number,
474 bool remove_reference_species_from_alignments,
475 bool alignment_cutting,
476 faststring & gff_feature_name,
477 faststring & gff_record_attribute_name_for_geneID,
478 char sequence_name_field_delimiter,
479 unsigned short sequence_name_taxon_field_number,
480 unsigned short sequence_name_geneID_field_number,
481 bool sequence_name_tokenizer_overwritten,
482 char center_computation_mode,
483 unsigned verbosity
484 )
485 {
486 os << "Welcome to the " << PROGNAME << " program, version " << VERSION << endl << endl
487 << "This software and any of its components are copyright protected by the authors" << endl
488 << "Christoph Mayer and Oliver Niehuis, Zoologisches Forschungsmuseum Alexander" << endl
489 << "Koenig, Bonn, Germany. Copyright 2014." << endl
490 << "This software is distributed as working material within a workshop given by" << endl
491 << "Manuela Sann and Oliver Niehuis at the Australian National University, Canberra." << endl
492 << "As long as it is not published, the software may only be used for the purposes of" << endl
493 << "the workshop or with explicit permission of one of the authors." << endl;
494 os << "Redistribution as source code in part or in whole or in binary form is not permitted." << endl;
495 os << "The software comes without warranty of any kind." << endl << endl;
496
497 os << "PROGRAM has been started with the following parameters:" << endl;
498 os << "Alignment cutting " << (alignment_cutting ? "yes" : "no");
499 os << endl;
500
501 if (alignment_cutting)
502 {
503 os << "Name of gff file: " << fname_gff << endl;
504 os << "Name of genome file: " << fname_genome << endl;
505 os << "Name of genome: " << genome_name << endl;
506 os << "GFF feature to be excised from sequences: " << gff_feature_name << endl;
507 os << "GFF record attribute name for geneID: " << gff_record_attribute_name_for_geneID << endl;
508 }
509
510 if (string_required_taxa.empty())
511 os << "String with required taxonomic groups: " << "no taxa specified" << endl;
512 else
513 os << "String with required taxonomic groups: " << string_required_taxa << endl;
514
515 os << "Name of directory with transcriptome files: " << dname_alignments << endl;
516 os << "Cluster threshold: " << cluster_threshold << endl;
517 os << "Bait length: " << bait_length << endl;
518 os << "Bait offset (for multiple baits in region): " << bait_offset << endl;
519 os << "Bait number (for multiple baits in region): " << bait_number << endl;
520 os << "Output directory: " << output_directory << endl;
521 os << "Remove reference species from alignment: " << (remove_reference_species_from_alignments ? "yes": "no") << endl;
522 if (sequence_name_tokenizer_overwritten)
523 {
524 os << "sequence_name_field_delimiter (default \"|\") " << "Not used since no required taxa specified and no alignment cutting." << endl;
525 // The field numbers are stored 0-based. In the user interaction, numbers are 1-based.
526 os << "Number of taxon field in sequence name: " << "Not used since no required taxa specified and no alignment cutting." << endl;
527 os << "Number of geneID field in sequence name: " << "Not used since no required taxa specified and no alignment cutting." << endl;
528 }
529 else
530 {
531 os << "sequence_name_field_delimiter (default \"|\") " << sequence_name_field_delimiter << endl;
532 // The field numbers are stored 0-based. In the user interaction, numbers are 1-based.
533 os << "Number of taxon field in sequence name: " << sequence_name_taxon_field_number+1 << endl;
534 os << "Number of geneID field in sequence name: " << sequence_name_geneID_field_number+1 << endl;
535 }
536 if (center_computation_mode == 0)
537 os << "Center computation mode: " << "exhaustive" << endl;
538 else if (center_computation_mode == 1)
539 os << "Center computation mode: " << "heuristic" << endl;
540 else
541 os << "Center computation mode: " << "comparison mode" << endl;
542 os << "Verbosity: " << verbosity << endl;
543 os << endl;
544 }
545
546
547
548
549
550 // Computes alignment and determines some characteristics:
551 // - alignment_score, alignment_length,
552 // - s1ins2s: starting pos of s1 in s2. If negative it is outside of s2 by the negative number (0 based)
553 // s1ins2e: position after last symbol of s1 in s2. If negative it is outside of s2 by the negative number (0 based)
554 // s2ins1s: similar to s1ins2s
555 // s2ins1e: similar to s2ins2e
556 inline void respective_sequence_positions(CScoreMatrix &DNAmat, faststring &s1, faststring &s2,
557 int &s1ins2s,
558 int &s1ins2e,
559 int &s2ins1s,
560 int &s2ins1e,
561 int &alignment_score,
562 int &alignment_length,
563 int verbosity)
564 {
565 // CScoreMatrix DNAmat("simpledna.mat"); // It is not efficient to open this file for many million short alignments.
566 short mode = 1; // local - free end gaps
567
568 std::ofstream log;
569 log.open("CAligner.log");
570
571 CAligner CA(s1.c_str(), s1.length(), s2.c_str(), s2.length(), DNAmat, mode, log);
572
573 faststring al;
574 align_coord a, e;
575 faststring l1, l2, l3;
576
577 CA.align();
578 alignment_score = CA.traceback(al, a, e);
579
580 CA.alignment_strings(l1, l2, l3, al);
581
582 if (verbosity >= 50)
583 {
584 cout << "Alignment score: " << alignment_score << endl;
585 cout << "Alignment:" << endl;
586 cout << l1 << endl;
587 cout << l3 << endl;
588 cout << l2 << endl;
589 }
590
591 CA.get_respective_start_end_coordinates(s2ins1s,
592 s2ins1e,
593 s1ins2s,
594 s1ins2e,
595 a,e,
596 s1.length(),
597 s2.length(),
598 al);
599
600 if (verbosity > 50)
601 {
602 al.reverse();
603 std::cout << al << std::endl;
604 std::cout << "Score: " << alignment_score << std::endl;
605 std::cout << "Internal representation of start: "; a.print(std::cout); std::cout << std::endl;
606 std::cout << "Internal representation of end: "; e.print(std::cout); std::cout << std::endl;
607
608 // pretty_print_alignment_base0(stdout, l1, l2, l3, 100);
609 // pretty_print_alignment(stdout, l1, l2, l3, 100);
610
611 // std::cout << l1 << std::endl;
612 // std::cout << l3 << std::endl;
613 // std::cout << l2 << std::endl;
614
615 // std::cout << "Start: "; a.print(std::cout); std::cout << std::endl;
616 // std::cout << "End: "; e.print(std::cout); std::cout << std::endl;
617
618 std::cout << "Respective start/end (internal representation): "
619 << "s2ins1s " << s2ins1s << " "
620 << "s2ins1e " << s2ins1e << " "
621 << "s1ins2s " << s1ins2s << " "
622 << "s1ins2e " << s1ins2e << std::endl;
623
624 std::cout << "len s1 " << s1.length() << std::endl;
625 std::cout << "len s2 " << s2.length() << std::endl;
626 std::cout << "len al " << al.length() << std::endl;
627 }
628
629
630 alignment_length = al.length();
631 }
632
633
634 void find_coords(CScoreMatrix &DNAmat, CDnaString &s1, CDnaString &s2,
635 int &s1ins2s_best,
636 int &s1ins2e_best,
637 int &s2ins1s_best,
638 int &s2ins1e_best,
639 int &alignment_score_best,
640 int &alignment_length_best,
641 int verbosity)
642 {
643 // Coordinates in alignment, in px_y: x is first and second part, y is linker index.
644
645 char better;
646
647 int s1ins2s_tmp1;
648 int s1ins2e_tmp1;
649 int s2ins1s_tmp1;
650 int s2ins1e_tmp1;
651 int alignment_score_tmp1;
652 int alignment_length_tmp1;
653
654 int s1ins2s_tmp2;
655 int s1ins2e_tmp2;
656 int s2ins1s_tmp2;
657 int s2ins1e_tmp2;
658 int alignment_score_tmp2;
659 int alignment_length_tmp2;
660
661 // Try both: feature sequence and its reverse complement
662 respective_sequence_positions(DNAmat, s1, s2,
663 s1ins2s_tmp1, s1ins2e_tmp1, s2ins1s_tmp1, s2ins1e_tmp1, alignment_score_tmp1, alignment_length_tmp1,
664 verbosity);
665
666 CDnaString revcomp_s2;
667
668 revcomp_s2.setToReverseComplementOf(s2);
669
670 respective_sequence_positions(DNAmat, s1, revcomp_s2,
671 s1ins2s_tmp2, s1ins2e_tmp2, s2ins1s_tmp2, s2ins1e_tmp2, alignment_score_tmp2, alignment_length_tmp2,
672 verbosity);
673
674 if (alignment_score_tmp1 > alignment_score_tmp2)
675 {
676 s1ins2s_best = s1ins2s_tmp1;
677 s1ins2e_best = s1ins2e_tmp1;
678 s2ins1s_best = s2ins1s_tmp1;
679 s2ins1e_best = s2ins1e_tmp1;
680 alignment_score_best = alignment_score_tmp1;
681 alignment_length_best = alignment_length_tmp1;
682 better = 1;
683 }
684 else
685 {
686 s1ins2s_best = s1ins2s_tmp2;
687 s1ins2e_best = s1ins2e_tmp2;
688 s2ins1s_best = s2ins1s_tmp2;
689 s2ins1e_best = s2ins1e_tmp2;
690 alignment_score_best = alignment_score_tmp2;
691 alignment_length_best = alignment_length_tmp2;
692 better = 2;
693 }
694
695 if (verbosity >5)
696 {
697 cout << "Best alignment:" << endl;
698 cout << "Score: " << alignment_score_best << endl;
699 cout << "Direction: " << (better==1 ? "Same" : "RevComp") << endl;
700 }
701
702 }
703
704
705 bool check_full_coverage_and_required_taxa_in_range(CSequences2 *seqs, unsigned pos1, unsigned pos2,
706 const vector<unsigned> &unique_taxon_ids,
707 const CRequiredTaxa &required_groups_of_taxa)
708 {
709 unsigned N = seqs->GetTaxaNum();
710 unsigned i;
711
712 CSequence_Mol* seq;
713 set<unsigned> set_of_taxon_ids;
714 faststring taxon_name;
715
716 // Determine set of complete sequences (taxa) of this range.
717
718 // For each sequence in this range we check that it does not contains gaps or Ns.
719 // In the set_of_taxon_ids we store the unique taxon ID, not the sequence number.
720 for (i=0; i<N; ++i)
721 {
722 seq = seqs->get_seq_by_index(i);
723
724 // Has no gaps, Ns in range?
725 if (! seq->range_contains_gaps_or_Ns(pos1, pos2) )
726 {
727 // *seq is a good sequence in this range. (No gaps or Ns.)
728 set_of_taxon_ids.insert(unique_taxon_ids[i]);
729 }
730 }
731
732 // Checks whether there is any sequence left. In case of an empty required_taxa string
733 // this is an important check. Otherwise gap only regions could pass this test.
734 if (set_of_taxon_ids.empty())
735 {
736 if (global_VERBOSITY >= VERBOSITY_COORDINATES)
737 {
738 cout << "NOTE: Not a good bait window. No complete sequences in this range: " << pos1+1 << "-" << pos2
739 << " Num sequences: " << set_of_taxon_ids.size() << endl;
740 }
741 return false;
742 }
743
744 if (required_groups_of_taxa.is_empty() )
745 {
746 if (global_VERBOSITY >= VERBOSITY_COORDINATES)
747 {
748 cout << "NOTE: Good bait window. Valid sequence range (no required taxa specified): " << pos1+1 << "-" << pos2 << " Num sequences: " << set_of_taxon_ids.size() << endl;
749 }
750 return true;
751 }
752
753 // Check that all required taxonomic groups are present in this feature:
754 if (required_groups_of_taxa.check_precense_of_all_groups_of_taxa(set_of_taxon_ids) )
755 {
756 if (global_VERBOSITY >= VERBOSITY_COORDINATES)
757 {
758 cout << "NOTE: Good bait window. All required taxonomic groups present in this range: " << pos1+1 << "-" << pos2 << " Num sequences: " << set_of_taxon_ids.size() << endl;
759 }
760 return true;
761 }
762 else
763 {
764 if (global_VERBOSITY >= VERBOSITY_COORDINATES)
765 {
766 cout << "NOTE: Not a good bait windows. NOT all required taxonomic groups present in this range." << pos1+1 << "-" << pos2 << " Num sequences: " << set_of_taxon_ids.size() << endl;
767 }
768 return false;
769 }
770 } // END check_full_coverage_and_required_taxa_in_range(...)
771
772
773 // Return true if at least one good bait region was found, otherwise it returns false.
774 bool handle_this_alignment(CSequences2 *alignment,
775 CTaxonNamesDictionary &taxonNamesDict,
776 CRequiredTaxa &required_groups_of_taxa,
777 unsigned short bait_length,
778 unsigned short number_of_baits_in_region,
779 unsigned short bait_offset,
780 faststring name, // gene name
781 unsigned short running_number,
782 unsigned short running_number_maximum,
783 double cluster_threshold,
784 ostream &os_result_file,
785 char delim,
786 unsigned short field_num,
787 char center_computation_mode,
788 const faststring &filename)
789 {
790 unsigned alignment_length = alignment->GetPosNum();
791 int region_length = (number_of_baits_in_region-1)*bait_offset+bait_length;
792
793 vector<unsigned> seqNum_to_unique_taxonID_this_alignment; // Used to translate the sequence number in the current file to the unique taxon number.
794 vector<faststring> full_names_this_alignment;
795 vector<faststring> taxon_names_this_alignment;
796 set<faststring> taxon_set_this_alignment;
797
798 //DEBUG BREAK POINT - allows us to catch certain gene names for debugging purposes.
799 // {
800 // // os_result_file << "name: " << name << " " << running_number << endl;
801 //
802 // if (name == "NV18435" && running_number == 4)
803 // {
804 // cerr << "Found name: " << name << " " << running_number << endl;
805 // }
806 //
807 // }
808
809
810 //************************************************
811 // The following four arrays belong together.
812 //************************************************
813 // For each site in this msa is_valid_bait_window stores the information
814 // whether the site represents a good starting point and whether
815 // for the window starting here we need to do the clustering and determine
816 // the center sequence.
817 // For each site in the msa is_valid_bait_region stores the information
818 // whether we have a good bait region starting at this point.
819 // For each site cluster_loci_this_alignment stores the information about
820 // clusters and center sequences for the bait window starting at this point.
821 // If the window does not need to be computed, the pointer in this array is NULL.
822 faststring is_valid_bait_window;
823 faststring is_valid_bait_region;
824 vector<Csequence_loci_cluster_collection *> cluster_loci_this_alignment(alignment_length, NULL);
825 vector<CBait_region_information *> bait_region_info(alignment_length, NULL);
826
827 // TODO: currently, the is_valid_bait_window and is_valid_bait_region are filled up to the end.
828 // Last valid/possible index for is_valid_bait_window: alignment_length - bait_length
829 // Last valid/possible index for is_valid_bait_region: alignment_length - region_length
830
831 // Initialise is_valid_bait_window with fail character '-':
832 is_valid_bait_window.clear();
833 is_valid_bait_window.assign(alignment_length, '-');
834
835 is_valid_bait_region.clear();
836 is_valid_bait_region.assign(alignment_length, '-');
837
838 unsigned i;
839 for (i=alignment_length - bait_length+1; i< alignment_length; ++i)
840 is_valid_bait_window[i] = 'e'; // Indicates that this region extends beyond the end.
841
842 for (i=alignment_length - region_length+1; i< alignment_length; ++i)
843 is_valid_bait_region[i] = 'e'; // Indicates that this region extends beyond the end.
844
845 // cout << is_valid_bait_window.size() << endl;
846 // cout << is_valid_bait_window << endl;
847
848 // cout << is_valid_bait_region.size() << endl;
849 // cout << is_valid_bait_region << endl;
850
851 // Determine translation table: seqNum_to_unique_taxonID_this_alignment
852 // Clear the data for this alignment:
853 seqNum_to_unique_taxonID_this_alignment.clear();
854 full_names_this_alignment.clear();
855 // taxon_names_this_alignment.clear();
856 // taxon_set_this_alignment.clear();
857
858 alignment->get_sequence_names(full_names_this_alignment);
859
860 unsigned j;
861 faststring tmp_name;
862
863 // Determine vector that translates sequence number to unique_taxonIDs
864 for (j=0; j < full_names_this_alignment.size(); ++j)
865 {
866 tmp_name = extract_taxon_name(full_names_this_alignment[j], delim, field_num);
867 // taxon_names_this_alignment.push_back(sequence_to_taxon_name(full_names_this_alignment[j]));
868 // taxon_set_this_alignment.insert(taxon_names_this_alignment[j]);
869 // cout << taxon_names_this_alignment[j] << endl;
870
871 seqNum_to_unique_taxonID_this_alignment.push_back(taxonNamesDict.dictionary_get_index_of_taxonname(tmp_name));
872 // cout << j << "-->" << tmp_name << "->" << taxonNamesDict.dictionary_get_index_of_taxonname(tmp_name) << endl;
873 }
874
875 // Move through alignment sequence alignment and determine those parts which have a sufficient data coverage:
876 {
877 unsigned pos1, pos2;
878 bool is_valid_range;
879
880 pos1 = 0; // 0 based
881 pos2 = pos1 + bait_length;
882
883 // pos2 is the index after the last nucleotide in the bait.
884 // Thus pos2 is allowed to be equal to alignment_length, which specifies the index after
885 // the last nucleotide in the sequence.
886 while (pos2 <= alignment_length)
887 {
888 is_valid_range = check_full_coverage_and_required_taxa_in_range(alignment,
889 pos1, pos2,
890 seqNum_to_unique_taxonID_this_alignment,
891 required_groups_of_taxa);
892 if (is_valid_range)
893 is_valid_bait_window[pos1] = 'x'; // Indicates a valid bait window.
894
895 ++pos1;
896 ++pos2;
897 }
898 // cout << "Valid bait windows in alignment: " << endl << is_valid_bait_window << endl;
899 } // END Move through alignment sequence alignment
900
901 // cout << "Valid bait windows: " << endl;
902 // cout << is_valid_bait_window << endl;
903
904 unsigned num_good_regions=0;
905
906 // Determine good bait regions, i.e. with requested number of baits and starting points separated by an offset of bait_offset
907 {
908 // for all starting point of a bait region:
909 unsigned i, j, pos;
910 bool is_good_region;
911
912 // i is the start index of the current range.
913 for (i=0; i < alignment_length; ++i)
914 {
915 if (is_valid_bait_region[i] != 'e')
916 {
917 // cout << "Starting good: " << i << endl;
918 is_good_region = true;
919 // For all number_of_baits_in_region baits in one region with offset bait_offset
920 for (j=0, pos=i; j < number_of_baits_in_region; ++j, pos += bait_offset)
921 {
922 // Allowed values for is_valid_bait_window[pos] are 'c' and 'x'. Not allowed are: '-' and 'e'
923 // Some checks are redundant here: E.g. pos >= alignment_length and is_valid_bait_window[pos]=='e'
924 // should not be possible, since we check is_valid_bait_region[i] != 'e' before we get here.
925 // Doing these checks is more secure.
926 // TODO: In future versions check whether pos >= alignment_length and is_valid_bait_window[pos]=='e'
927 // are necessary.
928 if (pos >= alignment_length || is_valid_bait_window[pos] == '-' || is_valid_bait_window[pos]=='e')
929 {
930 // cout << "Not a good bait region: " << i << " " << pos << " " << is_valid_bait_window[pos] << endl;
931 is_good_region = false;
932 break; // This region is at best incomplete, so we mark it as bad.
933 }
934 }
935 if (is_good_region)
936 {
937 // cout << "Mark as good: " << i << endl;
938 ++num_good_regions;
939 // Mark good region.
940 is_valid_bait_region[i] = 'x';
941
942 // cout << "Interm. " << i << is_valid_bait_region << endl;
943
944 // For all good bait regions we also allocate memory for the final collection of the data:
945 bait_region_info[i] = new CBait_region_information(name, running_number, running_number_maximum, i, // bait_length, bait_offset,
946 number_of_baits_in_region);
947 // cout << "Intermediate is_valid_bait_region: " << endl << is_valid_bait_region << endl;
948 // For this valid bait region, we mark all windows as "need to be computed".
949
950 ++DEBUG_COUNT_NEW_CALLS_CBAIT_REGION_INFORMATION;
951
952 for (j=0, pos=i; j < number_of_baits_in_region; ++j, pos += bait_offset)
953 {
954 if (pos >= alignment_length)
955 break;
956 is_valid_bait_window[pos] = 'c';
957 }
958 // cout << "Intermediate is_valid_bait_window: " << endl << is_valid_bait_window << endl;
959 if (global_VERBOSITY >= VERBOSITY_COORDINATES)
960 {
961 cout << "NOTE: Good bait region: " << i+1 << " " << i+1+region_length << endl;
962 }
963 }
964 else
965 {
966 if (global_VERBOSITY >= VERBOSITY_COORDINATES)
967 {
968 cout << "NOTE: Not a good bait region: " << i+1 << " " << i+1+region_length << endl;
969 }
970 }
971 } // END if (is_valid_bait_region[i] != 'e')
972 } // END for (i=0; i < alignment_length; ++i)
973 // cout << "Valid bait regions in feature: " << endl << is_valid_bait_region << endl;
974 // cout << "Bait windows that need to be computed" << endl << is_valid_bait_window << endl;
975 } // END Determine baits with requested number_of_baits_in_region good baits separated by an offset of bait_offset
976
977 // If we come here we have the following information:
978 // is_valid_bait_window contains a 'c' in all bait window starting indices that need to be looked at.
979 // is_valid_bait_region contains a 'x' in all bait region starting indices that we will explore,
980 // i.e. for which we have the necessary successive bait windows.
981
982 // If this file has one or more regions that should be looked at:
983 // Cluster loci which need to be clustered.
984 // Compute center sequences.
985
986 if (num_good_regions > 0)
987 {
988 // OK - Where are we?
989 // We have extracted an feature. We determined the loci of possible
990 // bait regions in this feature. This number > 0.
991 // For all bait windows in this feature that belong to good bait regions,
992 // we cluster the sequences and compute an artificial center sequence for
993 // each cluster.
994
995 unsigned i;
996 unsigned num_taxa = alignment->GetTaxaNum();
997
998 // We need a recoded msa in order to compute artificial center sequences.
999 // We could do this for each window, but since the windows overlap, we
1000 // do a lot of redundant work. Doing this per window we would only be on the positive
1001 // side if there would be only very few windows in a long alignment.
1002 // I do not see this as the general case.
1003 // The recoded msa is used for the complete file and only needs to be created once.
1004
1005 // The vector of faststrings allocates the memory for msa:
1006 std::vector<faststring> msa_faststring;
1007 char** msa;
1008 msa = new char* [num_taxa]; // remember to delete this dynamic array
1009 msa_faststring.clear();
1010 ++DEBUG_COUNT_NEW_CALLS_MSA;
1011
1012 // Copy the sequence data to the msa_faststring
1013 // The data is stored in the alignment (type CSequences2) object.
1014 alignment->get_alignment_as_copy(msa_faststring);
1015
1016 for (i=0; i < num_taxa; ++i)
1017 {
1018 // Important: Even though they are initially 0 terminated cstrings,
1019 // this does not provide much advantage since the sequences
1020 // will be recoded anyway. In the recoded form, the bit code 0
1021 // is used. So recoded sequences cannot be interpreted as cstrings.
1022 msa[i] = msa_faststring[i].data();
1023 }
1024
1025 // Recode the msa:
1026 // IMPORTANT: Recoding is possible also for ambiguous characters.
1027 // Gaps are also recoded.
1028 recode_MSA(msa, num_taxa, alignment_length);
1029
1030 unsigned num_valid_bait_windows = 0;
1031 unsigned count_valid_bait_windows = 0;
1032 unsigned report_interval = 0;
1033
1034 for (i=0; i < alignment_length; ++i)
1035 {
1036 if (is_valid_bait_window[i] == 'c')
1037 {
1038 ++num_valid_bait_windows;
1039 }
1040 }
1041
1042 report_interval = num_valid_bait_windows/10;
1043 if (report_interval == 0)
1044 report_interval = 1;
1045
1046 // For all loci:
1047 for (i=0; i < alignment_length; ++i)
1048 {
1049 // Do we have to compute cluster and center sequence for this loci?
1050 if (is_valid_bait_window[i] == 'c')
1051 {
1052 ++count_valid_bait_windows;
1053
1054 if (global_VERBOSITY > VERBOSITY_MOREPROGRESS) // Report progress
1055 {
1056 if (count_valid_bait_windows % report_interval == 0)
1057 {
1058 int p = count_valid_bait_windows*100 / num_valid_bait_windows;
1059 faststring t;
1060 get_time_string(t);
1061 cout << "PROGRESS: " << t << " Percent done this alignment or gene: " << p << "%."
1062 << endl << flush;
1063 }
1064 }
1065
1066 // This constructor again determins the valid sequences and their number in this range
1067 Csequence_loci_cluster_collection* tmp_collection = new Csequence_loci_cluster_collection(i, i+bait_length,
1068 alignment,
1069 // taxonNamesDict,
1070 cluster_threshold, msa );
1071 ++DEBUG_COUNT_NEW_CALLS_CSEQUENCE_LOCI_CLUSTER_COLLECTION;
1072 cluster_loci_this_alignment[i] = tmp_collection;
1073 // Compute what we can compute now for this locus.
1074 tmp_collection->cluster_locus(name, filename);
1075 tmp_collection->compute_center_for_all_clusters(center_computation_mode);
1076
1077 // Debug output:
1078 // cout << "*******************" << endl;
1079 // tmp_collection->print_msa_backrecoded_all(cout, 1);
1080 // cout << "*******************" << endl;
1081 }
1082 } // END for all loci
1083
1084 // Now we can transfer the clustering and center sequence data to the bait_region_info vector.
1085
1086 unsigned short tmp_num_sequences;
1087 unsigned short tmp_num_clusters;
1088
1089 // For all possible starting positions of bait regions.
1090 for (i=0; i < alignment_length; ++i)
1091 {
1092 if (is_valid_bait_region[i] == 'x')
1093 {
1094 // Collect the information for all number_of_baits_in_region baits in this bait region.
1095 for (j=0; j < number_of_baits_in_region; ++j)
1096 {
1097 if (cluster_loci_this_alignment[i+j*bait_offset] != NULL)
1098 {
1099 if (bait_region_info[i] == NULL)
1100 {
1101 cerr << "INTERNAL ERROR: bait_region_info["<< i << "]==NULL" << endl;
1102 exit(-4);
1103 }
1104 cluster_loci_this_alignment[i+j*bait_offset]->get_numbers_of_valid_sequences_number_of_clusters(tmp_num_sequences, tmp_num_clusters);
1105 bait_region_info[i]->add_info__cluster_locus(cluster_loci_this_alignment[i+j*bait_offset],
1106 tmp_num_sequences, tmp_num_clusters);
1107 }
1108 else
1109 {
1110 cerr << "INTERNAL ERROR: The cluster_loci_this_alignment is unexpectedly NULL at position: "
1111 << i+j*bait_offset << endl;
1112 exit(-5);
1113 }
1114 }
1115 }
1116 }
1117
1118 for (i=0; i< alignment_length; ++i)
1119 {
1120 if (is_valid_bait_region[i]=='x')
1121 {
1122 bait_region_info[i]->print_baits_result(os_result_file, filename);
1123 }
1124 }
1125
1126
1127 // The msa array is not needed any more, so we delete it.
1128 // The actual sequences are stored in the msa_faststring
1129 // vector and they will be deleted automatically when this block will be left.
1130 delete [] msa;
1131 ++DEBUG_COUNT_DELETE_CALLS_MSA;
1132 } // END if (num_good_regions > 0) // Compute cluster and center sequence for loci.
1133 else
1134 {
1135 if (global_VERBOSITY >= VERBOSITY_WARNINGS)
1136 {
1137 if (running_number > 0) // In non cutting mode this 0, otherwise > 0
1138 {
1139 cerr << "ALERT: No valid bait regions found for feature: ";
1140 cerr << name << " " << running_number << "/" << running_number_maximum;
1141 cerr << endl;
1142 }
1143 else
1144 {
1145 cerr << "ALERT: No valid bait region found for alignment: " << name << endl;
1146 }
1147 }
1148 }
1149
1150
1151 // Remember to release all memory allocated in this block:
1152
1153 DEBUG_COUNT_DELETE_CALLS_CBAIT_REGION_INFORMATION += delete_pointers_in_vector(bait_region_info);
1154 DEBUG_COUNT_DELETE_CALLS_CSEQUENCE_LOCI_CLUSTER_COLLECTION += delete_pointers_in_vector(cluster_loci_this_alignment);
1155
1156 if (num_good_regions > 0)
1157 return true;
1158 else
1159 return false;
1160
1161 } // END handle_this_alignment()
1162
1163 // Only used in special mode to test gff file integrity:
1164 void gff_file_test(faststring gff_file_name)
1165 {
1166 GFF_collection gff_col;
1167
1168 unsigned good_lines;
1169 unsigned bad_lines;
1170 bool file_error;
1171
1172 if (global_VERBOSITY >= VERBOSITY_MOREPROGRESS)
1173 {
1174 cout << "Parsing the gff file ..." << flush;
1175 }
1176
1177 gff_col.readGFFfile(gff_file_name.c_str(), good_lines, bad_lines, file_error);
1178
1179 if (global_VERBOSITY >= VERBOSITY_DEBUG)
1180 {
1181 cout << " Done." << endl;
1182 cout << "Good lines: " << good_lines << endl;
1183 cout << "Bad lines: " << bad_lines << endl;
1184 cout << "File error: " << file_error << endl;
1185 cout << endl;
1186 cout << endl;
1187 }
1188
1189 if (file_error)
1190 {
1191 cerr << "ERROR: The specified gff-file could not be opened. It seems it does not exist." << endl;
1192 cerr << "Exiting." << endl;
1193 exit(-3);
1194 }
1195
1196 gff_col.print(cout, 1);
1197 }
1198
1199
1200 int main(int argc, char **argv)
1201 {
1202 faststring fname_parameter_file;
1203
1204 faststring fname_gff;
1205 faststring fname_genome;
1206 faststring string_required_taxa;
1207 faststring dname_alignments;
1208 faststring genome_name;
1209
1210 unsigned short bait_length;
1211 double cluster_threshold;
1212 faststring output_directory;
1213
1214 unsigned short bait_offset;
1215 unsigned short number_of_baits_in_region;
1216
1217 bool remove_reference_species_from_alignments;
1218 bool alignment_cutting;
1219
1220 faststring gff_feature_name;
1221 faststring gff_record_attribute_name_for_geneID;
1222
1223 char sequence_name_field_delimiter;
1224 unsigned short sequence_name_taxon_field_number; // Used to find reference species and test for required taxa.
1225 unsigned short sequence_name_geneID_field_number; // Only used if extracting with the aid of a gff file.
1226 bool sequence_name_tokenizer_overwritten;
1227
1228
1229 char center_computation_mode;
1230 int special_mode = 0; // 0: normal, 1: gff-test
1231
1232
1233 if (argc == 2)
1234 {
1235 fname_parameter_file = argv[1];
1236 parse_parameter_file(fname_parameter_file,
1237 fname_gff,
1238 fname_genome,
1239 string_required_taxa,
1240 dname_alignments,
1241 genome_name,
1242 bait_length,
1243 cluster_threshold,
1244 output_directory,
1245 bait_offset,
1246 number_of_baits_in_region,
1247 remove_reference_species_from_alignments,
1248 alignment_cutting,
1249 gff_feature_name,
1250 gff_record_attribute_name_for_geneID,
1251 sequence_name_field_delimiter,
1252 sequence_name_taxon_field_number,
1253 sequence_name_geneID_field_number,
1254 sequence_name_tokenizer_overwritten,
1255 center_computation_mode,
1256 global_VERBOSITY
1257 );
1258 }
1259 else if (argc == 3)
1260 {
1261 if (faststring(argv[2])=="gffFileTest")
1262 {
1263 special_mode = 1;
1264 }
1265 else
1266 {
1267 cerr << "Welcome to the " << PROGNAME << " program, version " << VERSION << endl << endl;
1268 cerr << "Usage: " << argv[0] << " parameter-file [gffFileTest]" << endl;
1269 // cerr << "OR" << endl;
1270 // cerr << "Usage: " << argv[0] << " gff-file genome-file transcriptome-folder required_taxa_file probe-length cluster-threshold output-directory" << endl;
1271 exit(-3);
1272 }
1273
1274 fname_parameter_file = argv[1];
1275 parse_parameter_file(fname_parameter_file,
1276 fname_gff,
1277 fname_genome,
1278 string_required_taxa,
1279 dname_alignments,
1280 genome_name,
1281 bait_length,
1282 cluster_threshold,
1283 output_directory,
1284 bait_offset,
1285 number_of_baits_in_region,
1286 remove_reference_species_from_alignments,
1287 alignment_cutting,
1288 gff_feature_name,
1289 gff_record_attribute_name_for_geneID,
1290 sequence_name_field_delimiter,
1291 sequence_name_taxon_field_number,
1292 sequence_name_geneID_field_number,
1293 sequence_name_tokenizer_overwritten,
1294 center_computation_mode,
1295 global_VERBOSITY
1296 );
1297
1298 }
1299 // else if (argc == 8)
1300 // {
1301 // fname_gff = argv[1];
1302 // fname_genome = argv[2];
1303 // fname_required_taxa = argv[3];
1304 // dname_alignments = argv[4];
1305 // genome_name = "Nasonia_vitripennis";
1306
1307 // bait_length = faststring(argv[5]).ToInt();
1308 // cluster_threshold = faststring(argv[6]).ToDouble();
1309 // output_directory = argv[7];
1310 // }
1311 else
1312 {
1313 cerr << "Welcome to the " << PROGNAME << " program, version " << VERSION << endl << endl;
1314 cerr << "Usage: " << argv[0] << " parameter-file [gffFileTest]" << endl;
1315 // cerr << "OR" << endl;
1316 // cerr << "Usage: " << argv[0] << " gff-file genome-file transcriptome-folder required_taxa_file probe-length cluster-threshold output-directory" << endl;
1317 exit(-3);
1318 }
1319
1320
1321 print_parameters(cout,
1322 fname_gff,
1323 fname_genome,
1324 string_required_taxa,
1325 dname_alignments,
1326 genome_name,
1327 bait_length,
1328 cluster_threshold,
1329 output_directory,
1330 bait_offset,
1331 number_of_baits_in_region,
1332 remove_reference_species_from_alignments,
1333 alignment_cutting,
1334 gff_feature_name,
1335 gff_record_attribute_name_for_geneID,
1336 sequence_name_field_delimiter,
1337 sequence_name_taxon_field_number,
1338 sequence_name_geneID_field_number,
1339 sequence_name_tokenizer_overwritten,
1340 center_computation_mode,
1341 global_VERBOSITY
1342 );
1343
1344
1345 if (special_mode == 1)
1346 {
1347 gff_file_test(fname_gff);
1348 exit(0);
1349 }
1350 // exit(-111);
1351
1352
1353 unsigned short min_alignment_length = bait_length + (number_of_baits_in_region-1)*bait_offset;
1354 CScoreMatrix DNAmat("simpledna_N0.mat");
1355
1356 // Do some initial checks.
1357
1358
1359 // if (!file_exists(fname_required_taxa.c_str()))
1360 // {
1361 // cerr << "I could not open required taxa file: " << fname_required_taxa << endl;
1362 // cerr << "I guess it does not exist. Exiting." << endl;
1363 // exit(-1);
1364 // }
1365
1366
1367 if (cluster_threshold <= 0 || cluster_threshold >= 1)
1368 {
1369 cerr << "The cluster threshold must be greater than 0 and smaller than 1." << endl;
1370 exit(-1);
1371 }
1372
1373 if (bait_length < 1)
1374 {
1375 cerr << "Probe length cannot be 0 or smaller." << endl;
1376 exit(-1);
1377 }
1378
1379 if (!file_or_dir_exists(dname_alignments.c_str()))
1380 {
1381 cerr << "I could not read the transcriptome directory: " << dname_alignments << endl;
1382 cerr << "I guess it does not exist. Exiting." << endl;
1383 exit(-1);
1384 }
1385
1386 if (!file_or_dir_exists(output_directory.c_str()))
1387 {
1388 cerr << "The output-directory " << output_directory << endl;
1389 cerr << "could not be opened. I guess it does not exist. Please make sure this directory exists. Exiting." << endl;
1390 exit(-1);
1391 }
1392
1393 // Open file for output:
1394 faststring result_filename = output_directory + "/loci_baits.txt";
1395 ofstream os_result_file(result_filename.c_str());
1396
1397 if (alignment_cutting)
1398 os_result_file << "# Corresponding sequence name\tGene-name\tfeature-number\tnumber-of-features-of-specified-type-for-this-gene-in-gff-file\tstart-coordinate-of-bait-region-in-alignment\tnumber-of-sequences-in-region-the-baits-are-based-on\tnumber-of-baits-constructed-for-this-bait-region\tList of baits - each bait is followed by CG content and the maximum distance to the sequences in the alignment." << endl;
1399 else
1400 os_result_file << "# Corresponding sequence name\tGene-name\tn.a.\tn.a.\tstart-coordinate-of-bait-region-in-alignment\t number-of-sequences-in-region-the-baits-are-based-on\tnumber-of-baits-constructed-for-this-bait-region\tList of baits - each bait is followed by CG content and the maximum distance to the sequences in the alignment." << endl;
1401
1402 // Read names of alignment files:
1403 list<faststring> fasta_names;
1404 list<faststring> filter_names;
1405
1406 // If we use the Qt library:
1407 // filter_names.push_back("*.fas");
1408 // filter_names.push_back("*.fa");
1409 // filter_names.push_back("*.fasta");
1410
1411 filter_names.push_back("fas");
1412 filter_names.push_back("fa");
1413 filter_names.push_back("fasta");
1414
1415 get_file_names_in_dir(fasta_names, dname_alignments, filter_names);
1416
1417
1418
1419 //****************************************************************
1420 // BEGIN Alignment cutting mode
1421 //****************************************************************
1422 if (alignment_cutting)
1423 {
1424 if (!file_exists(fname_gff.c_str()))
1425 {
1426 cerr << "I could not open the gff file: " << fname_gff << endl;
1427 cerr << "I think it does not exist. Exiting." << endl;
1428 exit(-1);
1429 }
1430
1431 if (!file_exists(fname_genome.c_str()))
1432 {
1433 cerr << "I could not open the genome file: " << fname_genome << endl;
1434 cerr << "I think it does not exist. Exiting." << endl;
1435 exit(-1);
1436 }
1437
1438
1439 // Read gff file:
1440
1441 unsigned good_lines;
1442 unsigned bad_lines;
1443 bool file_error;
1444
1445 GFF_collection gff_col;
1446
1447 if (global_VERBOSITY >= VERBOSITY_PROGRESS)
1448 {
1449 cout << "Parsing the gff file ... " << flush;
1450 }
1451 gff_col.readGFFfile(fname_gff.c_str(), good_lines, bad_lines, file_error);
1452 if (global_VERBOSITY >= VERBOSITY_PROGRESS)
1453 {
1454 cout << "Done." << endl;
1455 }
1456
1457 if (good_lines == 0 || bad_lines > 0 || file_error)
1458 {
1459 cerr << "A problem occurred while reading the first gff file." << endl;
1460 cerr << "Good lines: " << good_lines << endl;
1461 cerr << "Bad lines: " << bad_lines << endl;
1462 cerr << "File error: " << file_error << endl;
1463
1464 if (file_error)
1465 {
1466 cerr << "The specified gff-file could not be opened. It seems it does not exist." << endl;
1467 }
1468 cerr << "Exiting." << endl;
1469 exit(-3);
1470 }
1471
1472 // gff_col.print(cout, 0);
1473
1474 // Read genome file:
1475
1476 if (global_VERBOSITY >= VERBOSITY_PROGRESS)
1477 cout << "Reading the genome file ..." << flush;
1478
1479 CSequences2 genome_seqs(CSequence_Mol::dna);
1480 CSequence_Mol::processing_flag pflag = CSequence_Mol::processing_flag(0);
1481
1482 pflag = CSequence_Mol::processing_flag(pflag | CSequence_Mol::convert_toupper);
1483 // pflag = CSequence_Mol::processing_flag(pflag | CSequence_Mol::convert_ambig2N);
1484
1485 if (global_VERBOSITY >= 50)
1486 {
1487 cout << "with processing flag: " << pflag << "..." << endl;
1488 }
1489
1490 CFile file;
1491 file.ffopen(fname_genome.c_str());
1492 if (genome_seqs.read_from_Fasta_File(file, pflag, 0, -1, false) < 0)
1493 {
1494 cerr << "ERROR while reading the fasta file: " << fname_genome << endl;
1495 cerr << "Please check the line indicated above since this line prevented me" << endl
1496 << "from parsing the fasta file." << endl;
1497 exit(-2);
1498 }
1499 file.ffclose();
1500
1501 if (global_VERBOSITY >= VERBOSITY_PROGRESS)
1502 {
1503 cout << "Done." << endl;
1504
1505 if (global_VERBOSITY >= VERBOSITY_FILEINFO)
1506 {
1507 cout << "NOTE: Found this number of sequences in genome file: " << genome_seqs.GetTaxaNum() << endl;
1508 cout << "NOTE: Found this number of positions in genome file: " << genome_seqs.GetPosNum() << endl;
1509 }
1510 faststring t;
1511 get_time_string(t);
1512 cout << "PROGRESS: " << t << " Genome file has been read." << endl << flush;
1513 }
1514
1515 // Read fasta sequences one by one:
1516 list<faststring>::iterator it, it_end;
1517 it = fasta_names.begin();
1518 it_end = fasta_names.end();
1519
1520 unsigned Num_transcripts = fasta_names.size();
1521 unsigned count_transcripts = 0;
1522 unsigned count_transcripts_no_gff_entries = 0;
1523 unsigned count_transcripts_no_gff_entries_for_feature = 0;
1524
1525 CSequence_Mol *transcript_seq;
1526 unsigned transcript_seq_index;
1527 faststring full_alignment_file_name;
1528 faststring sequence_header_target;
1529 faststring gene_name;
1530 GFF_record_list gene_record_list;
1531 faststring genomic_seq_name;
1532 // unsigned overall_counter = 0;
1533 // double mean_match_score = DNAmat.get_mean_matchscore_of_first_N_symbols(4);
1534
1535 CSequences2 *feature_transcript_alignment = NULL, *transcript_alignment = NULL;
1536 CSequences2 *feature_transcript_alignment_rmgo = NULL;
1537
1538 unsigned feature_number;
1539 unsigned features_this_gene;
1540 bool no_gff_entries_warning;
1541
1542
1543 // cout << "Mean match score: " << DNAmat.get_mean_matchscore_of_first_N_symbols(4) << endl;
1544
1545 // We have to keep a list of all taxon names that we deal with in this analysis.
1546 // We give the constructor the name of the directory that contains all fasta files.
1547 // It will read all fasta files, determine the taxon names from the sequence names and add them
1548 // to the list of taxon names. Furthermore, this object will keep a map that maps taxon names
1549 // to a unique taxon index in this analysis. Internally it is important to have such a unique index.
1550
1551 // The unique taxon indices are used:
1552 // - when checking for all required taxa.
1553 // The required taxa are stored as sets of unique taxon ids.
1554 // To check that there is at least one representative for each required set,
1555 // we determine the unique IDs for the taxa that are present.
1556 // Comparing intersections of sets then tells us whether all required taxonomic groups are present.
1557
1558 CTaxonNamesDictionary taxonNamesDict(dname_alignments, fasta_names, sequence_name_taxon_field_number, sequence_name_field_delimiter);
1559 // Print dictionary:
1560 // taxonNamesDict.print(os_result_file);
1561
1562 // Read required groups of taxa:
1563 int error_required_taxa;
1564 CRequiredTaxa required_groups_of_taxa(string_required_taxa.c_str(), taxonNamesDict, error_required_taxa);
1565
1566 { // Debug output:
1567 if (global_VERBOSITY > 1000)
1568 {
1569 cout << endl;
1570 taxonNamesDict.print_taxon_names_to_index(cout);
1571 cout << "Required groups: From each of the following sets of taxa, we require at least one to be present in the data set." << endl;
1572 required_groups_of_taxa.print(cout, 1);
1573 cout << endl;
1574 }
1575 }
1576
1577 // For all transcript files:
1578 // We extract the gene name and use the gene name and feature name to extract all gff records corresponding to these two.
1579 // For each gff record we cut out the region from the alignment file that corresponds to the genomic region in the genome
1580 // file depicted by the gff-record. In this sub alignment we search for baits.
1581
1582 // For all transcript files:
1583 while (it != it_end)
1584 {
1585 no_gff_entries_warning = false;
1586 ++ count_transcripts;
1587 full_alignment_file_name = dname_alignments + "/" + *it;
1588
1589 if (get_file_size(full_alignment_file_name) > 0)
1590 {
1591 // Here is the main action: We handle this gene:
1592
1593 // For this gene file, read the transcript for the genome we are concerned with
1594
1595 if (global_VERBOSITY >= VERBOSITY_PROGRESS) // Report progress
1596 {
1597 faststring t;
1598 get_time_string(t);
1599 cout << "PROGRESS: " << t << " Reading alignment file ("
1600 << count_transcripts << "/" << Num_transcripts << "): "
1601 << full_alignment_file_name << endl << flush;
1602 }
1603
1604 read_alignment_sequence_of_species2(full_alignment_file_name,
1605 genome_name,
1606 transcript_seq,
1607 transcript_alignment,
1608 transcript_seq_index,
1609 sequence_name_field_delimiter,
1610 sequence_name_taxon_field_number);
1611
1612 if (transcript_alignment == NULL)
1613 {
1614 cerr << "ERROR: Could not read transcript file: "
1615 << full_alignment_file_name << endl;
1616 exit(-4);
1617 }
1618
1619 if (transcript_seq == NULL) // This is rather common. But in this case we have to skip this alignment.
1620 {
1621 if (global_VERBOSITY >= VERBOSITY_WARNINGS)
1622 {
1623 cerr << "WARNING: Requested species " << genome_name << " not found in sequence: "
1624 << full_alignment_file_name << endl;
1625 cerr << " This means that we have to skip this alignment, since it cannot be handled in this mode." << endl;
1626 }
1627 }
1628 else
1629 {
1630 sequence_header_target = transcript_seq->getFullName();
1631 gene_name = extract_gene_name(sequence_header_target, sequence_name_field_delimiter, sequence_name_geneID_field_number);
1632
1633 if (global_VERBOSITY >= VERBOSITY_DEBUG)
1634 {
1635 cout << "INFO: Name of sequence containing the reference species: " << sequence_header_target << endl;
1636 cout << "INFO: Gene name: " << gene_name << endl;
1637
1638 transcript_seq->writeSequence(cout, 100);
1639 }
1640
1641 gff_record_attribute_name_for_geneID.ToLower();
1642
1643 // Determine coordinates of this gene in genome file using the gff file:
1644 gff_col.get_List_of_gff_records_for_attribute(gff_record_attribute_name_for_geneID.c_str(), gene_name.c_str(), gene_record_list);
1645
1646 if (gene_record_list.size() == 0 && global_VERBOSITY >= VERBOSITY_WARNINGS)
1647 {
1648 cerr << "WARNING: For gene \"" << gene_name << "\" no entries where found in the gff file.\n"
1649 " While this is possible, in most cases the reason is either a wrong attribute name for the\n"
1650 " gene-name-field in the gff-file or the gene name specified in the sequence\n"
1651 " name of the transcript alignment is incompatible to the gene names in the gff file.\n"
1652 << " It is recommended to check whether gene name \"" << gene_name << "\" occurs in the\n"
1653 " gff file under the attribute name \"" << gff_record_attribute_name_for_geneID << "\".\n";
1654 cerr << " Note that the gene name as specified in the sequence name and the gene name in\n"
1655 " the gff file have to match exactly. The attribute name specified in the parameter\n"
1656 " file must be an exact (case insensitive) match to the attribute name in the gff file." << endl << endl;
1657 ++count_transcripts_no_gff_entries;
1658 no_gff_entries_warning = true;
1659 }
1660
1661 easystring filter_feature = gff_feature_name.c_str();
1662 gene_record_list.filter_by_feature(filter_feature);
1663
1664 // TODO: Check empty list.
1665 // if (gene_record_list)
1666
1667
1668 if (global_VERBOSITY > VERBOSITY_FILEINFO)
1669 {
1670 // Print gff information:
1671 cout << "Features with this gene name: " << endl;
1672 cout << "----" << endl;
1673 gene_record_list.print(cout);
1674 cout << "----" << endl;
1675 }
1676
1677 // Foreach feature in this gene:
1678 list<GFF_record*>::iterator gl_it, gl_it_end;
1679
1680 gl_it = gene_record_list.begin();
1681 gl_it_end = gene_record_list.end();
1682
1683 // NOTE: We count all genes, not only the once that have a sufficient length.
1684 // TODO: Check that we really want to count all genes, not only the once that are long enough.
1685
1686 features_this_gene = gene_record_list.size();
1687
1688 if (features_this_gene == 0)
1689 {
1690 if (no_gff_entries_warning == false && global_VERBOSITY >= VERBOSITY_WARNINGS)
1691 {
1692 cerr << "WARNING: For gene \"" << gene_name << "\" no entries where found in the gff file that matched the specified feature name.\n"
1693 << " Since entries for this gene name have been found, but with different feature names, the problem could be the feature name\n"
1694 << " or the fact that features exist for this gene." << endl;
1695 }
1696 ++count_transcripts_no_gff_entries_for_feature;
1697 }
1698
1699 // For the gene in the transcript file we are working on we found exons
1700 // in the gff file. I.e. the gene is annotated in the genome.
1701 // The genomic sequences of the exons will be extracted and aligned
1702 // to the sequence in the transcript file. This determines the coordinates
1703 // of the exons in the transcript file.
1704
1705 CDnaString feature_sequence;
1706 CDnaString transcript_sequence;
1707
1708 CSequence_Mol* genome_seq_with_feature;
1709
1710 faststring seqid_feature_in_genome;
1711 unsigned start_of_feature_in_genome;
1712 unsigned end_of_feature_in_genome; // Remember: This is an inclusive end
1713
1714 transcript_sequence = transcript_seq->getSeqStr();
1715
1716 // The transcript sequence can contain gaps. These cause problems when aligning features against the transcripts.
1717 // Removing gaps is not an option, since we need to preserver the original coordinates.
1718 // Converting gaps to Ns will preserve coordinates. It will do no harm, since Ns will nicely align to the features and the sequence will
1719 // not be used for anything except aligning features to it.
1720
1721 transcript_sequence.replace_char('-','N');
1722
1723 /*
1724 vector<unsigned> seqNum_to_unique_taxonID_this_feature; // Used to translate the sequence number in the current file to the unique taxon number.
1725 vector<faststring> full_names_this_feature;
1726 vector<faststring> taxon_names_this_feature;
1727 set<faststring> taxon_set_this_feature;
1728
1729 unsigned feature_transcript_alignment_length;
1730 */
1731
1732 unsigned features_with_at_least_one_bait_regions = 0;
1733 feature_number = 0;
1734 // For each feature of the gene under consideration: #foreach feature
1735 while (gl_it != gl_it_end)
1736 {
1737 ++feature_number;
1738
1739 // The following coordinates are 0 based and the End lies outside of the range.
1740
1741 // Indices are stored 0 based and non inclusive of the end.
1742 seqid_feature_in_genome = (**gl_it).getSeqid().c_str();
1743 start_of_feature_in_genome = (**gl_it).getStartIndex(); // 0-based.
1744 end_of_feature_in_genome = (**gl_it).getEndIndex(); // 0-based, exclusive end
1745
1746 if (global_VERBOSITY >= VERBOSITY_PROGRESS) // Report progress
1747 {
1748 faststring t;
1749 get_time_string(t);
1750 cout << "PROGRESS: "
1751 << " Working with gene: " << gene_name << " "
1752 << feature_number << "/" << features_this_gene
1753 << " found in genome at "
1754 << seqid_feature_in_genome << " "
1755 // This function gives the 0 based index. We want to print the 1 based index.
1756 << start_of_feature_in_genome+1 << " " << end_of_feature_in_genome
1757 << endl << flush;
1758 }
1759
1760 // Find sequence with short_name equal to seqid_feature_in_genome.
1761 genome_seq_with_feature = genome_seqs.get_seq_by_name(seqid_feature_in_genome);
1762 if (genome_seq_with_feature == NULL)
1763 {
1764 cerr << "ERROR: gff and genome file are not compatible: Sequence name: \""
1765 << seqid_feature_in_genome
1766 << "\" does not exist in the genome file." << endl;
1767
1768 cerr << "It could be that some pipeline renamed the sequences in your genome file." << endl;
1769
1770 exit(-2);
1771 }
1772 else
1773 {
1774 // getPartialSeq: Takes 0 based start and length:
1775 feature_sequence = genome_seq_with_feature->getPartialSeq(start_of_feature_in_genome, end_of_feature_in_genome - start_of_feature_in_genome);
1776
1777 if (global_VERBOSITY >= VERBOSITY_DEBUG)
1778 {
1779 cout << "Pair:" << endl;
1780 cout << ">T" << endl;
1781 cout << transcript_sequence << endl;
1782 cout << ">GE" << endl;
1783 cout << feature_sequence << endl;
1784 }
1785 }
1786
1787 if ((int)feature_sequence.length() < min_alignment_length)
1788 {
1789 faststring tmp;
1790
1791 // Notice:
1792 if (alignment_cutting)
1793 tmp = gff_feature_name + " feature alignment";
1794 else
1795 tmp = "alignment";
1796
1797 if (global_VERBOSITY >= VERBOSITY_WARNINGS)
1798 {
1799 cerr << "NOTICE: Skipping " << tmp << " in " << gene_name << " with number " << feature_number << "/" << features_this_gene << " in " << gene_name
1800 << " since it is too short. Alignment length: " << feature_sequence.length() << ". Minimum length: " << min_alignment_length << endl;
1801 }
1802 }
1803 else
1804 {
1805 // Determine feature range in transcriptome MSA:
1806
1807 // transcript_sequence = "AACGTGTCGTAA";
1808 // feature_sequence = "CGTCGTCGT";
1809 // transcript_sequence = "CGTCGTCGT";
1810 // feature_sequence = "AACGTGTCGTAA";
1811
1812 int s1ins2s_best;
1813 int s1ins2e_best;
1814 int s2ins1s_best;
1815 int s2ins1e_best;
1816 int alignment_score_best;
1817 int alignment_length_best;
1818
1819 find_coords(DNAmat, transcript_sequence, feature_sequence,
1820 s1ins2s_best, s1ins2e_best, s2ins1s_best, s2ins1e_best,
1821 alignment_score_best, alignment_length_best, global_VERBOSITY);
1822
1823 //**********************************************************
1824 // Coordinates are 0 based, end is non inclusive.
1825 //**********************************************************
1826
1827 // Start coordinate in transcriptome alignment in which the feature starts (0 based).
1828 // If the number is negative, then the feature starts before the transcript starts.
1829 // Basically, part of the transcript is missing. The effective starting point in our
1830 // transcript is: 0. +++
1831 int in_transcript_start = macromax2(s2ins1s_best, 0);
1832
1833 // End coordinate in transcript alignment in which the feature starts. End is non inclusive.
1834 int in_transcript_end = s2ins1e_best;
1835 if (in_transcript_end <= 0)
1836 {
1837 //
1838 in_transcript_end = (int) transcript_sequence.size();
1839 }
1840
1841 int feature_in_transcript_length = in_transcript_end - in_transcript_start;
1842
1843 if (global_VERBOSITY >= VERBOSITY_DEBUG)
1844 {
1845 cout << "in_transcript_start: " << in_transcript_start+1 << endl;
1846 cout << "in_transcript_end: " << in_transcript_end << endl;
1847 cout << "feature length: " << feature_sequence.size() << endl;
1848 cout << "feature_in_transcript_length: " << feature_in_transcript_length << endl;
1849 cout << "Score: " << alignment_score_best << endl;
1850 cout << "Score/feature-length " << (double)alignment_score_best/feature_sequence.size() << endl;
1851 cout << "Score/feature_in_transcript_length " << (double)alignment_score_best/feature_in_transcript_length << endl;
1852 }
1853
1854 if ( (int)feature_sequence.size() > feature_in_transcript_length)
1855 {
1856 if (global_VERBOSITY >= VERBOSITY_WARNINGS)
1857 cerr << " ALERT: Length of excised genomic feature is longer than the transcript, i.e. it looks as if the transcript is incomplete.\n"
1858 " This can happen. Gene name: " << gene_name
1859 << " Feature number: " << feature_number
1860 << " Lengths are: " << feature_in_transcript_length << "/" << feature_sequence.size()
1861 << endl;
1862 }
1863
1864 if ( feature_in_transcript_length < min_alignment_length)
1865 {
1866 // Notice:
1867 if (global_VERBOSITY >= VERBOSITY_WARNINGS)
1868 {
1869 cerr << "ALERT: Skipping " << gff_feature_name << " feature since its overlap with the transcript is too short. Length of overlap: "
1870 << feature_in_transcript_length << " Minimum: " << min_alignment_length << endl;
1871 }
1872 }
1873 else // We will now deal with this feature here: ####
1874 // This is the block in which a lot of memory is allocated (with new) that needs to be freed when leaving the block.
1875 {
1876 // We extract the sequence alignment in the range: in_transcript_start, in_transcript_end
1877 feature_transcript_alignment = new CSequences2(*transcript_alignment,
1878 in_transcript_start,
1879 in_transcript_end );
1880 if (transcript_seq_index != -1u && remove_reference_species_from_alignments)
1881 feature_transcript_alignment->remove(transcript_seq_index);
1882
1883 // feature_transcript_alignment_length = feature_transcript_alignment->GetPosNum();
1884 ++DEBUG_COUNT_NEW_CALLS_FEATURE_ALIGNMENT;
1885
1886 // Remove gap only positions:
1887 feature_transcript_alignment_rmgo = new CSequences2(CSequence_Mol::dna, feature_transcript_alignment->get_ambiguity_character());
1888 feature_transcript_alignment->alignment_without_gap_only_positions(*feature_transcript_alignment_rmgo);
1889 feature_transcript_alignment_rmgo->remove_gap_or_ambig_only_sequences();
1890
1891 int feature_in_transcript_length_rmgo = feature_transcript_alignment_rmgo->GetPosNum();
1892
1893 if (feature_in_transcript_length_rmgo < min_alignment_length)
1894 {
1895 // Notice:
1896 if (global_VERBOSITY >= VERBOSITY_WARNINGS)
1897 {
1898 cerr << "ALERT: Skipping " << gff_feature_name << " feature since the alignment is tool short after removing gap only positions. Length of overlap: "
1899 << feature_in_transcript_length_rmgo << " Minimum: " << min_alignment_length << endl;
1900 }
1901 }
1902 else
1903 {
1904 ///////////////
1905 // Exporting transcript alignment before removing lines containing gaps and Ns.
1906 ///////////////
1907 if (true) // if export is requested
1908 {
1909 faststring output_file_name = output_directory + "/" + gene_name + "_" + faststring(feature_number) + "_" + faststring(features_this_gene) + "_all.fas";
1910
1911 ofstream os(output_file_name.c_str());
1912 if (os.fail())
1913 {
1914 cerr << "ERROR: Could not open file " << output_file_name << " for writing." << endl;
1915 exit(-9);
1916 }
1917 feature_transcript_alignment_rmgo->ExportSequences(os, 'f', 100000000);
1918 os.close();
1919 }
1920
1921 // Debug output:
1922 // feature_transcript_alignment_rmgo->ExportSequences(cout, 'f', 10000);
1923
1924 /// We have an alignment which we can work with:
1925
1926 bool found_bait_region;
1927
1928 found_bait_region = handle_this_alignment(feature_transcript_alignment_rmgo,
1929 taxonNamesDict,
1930 required_groups_of_taxa,
1931 bait_length,
1932 number_of_baits_in_region,
1933 bait_offset,
1934 gene_name,
1935 feature_number,
1936 features_this_gene,
1937 cluster_threshold,
1938 os_result_file,
1939 sequence_name_field_delimiter,
1940 sequence_name_taxon_field_number,
1941 center_computation_mode, *it);
1942
1943 if (found_bait_region)
1944 ++features_with_at_least_one_bait_regions;
1945
1946 if (feature_transcript_alignment)
1947 {
1948 delete feature_transcript_alignment;
1949 ++DEBUG_COUNT_DELETE_CALLS_FEATURE_ALIGNMENT;
1950 feature_transcript_alignment = NULL;
1951 }
1952 if (feature_transcript_alignment_rmgo)
1953 {
1954 delete feature_transcript_alignment_rmgo;
1955 feature_transcript_alignment_rmgo = NULL;
1956 }
1957 } // END ELSE if (feature_in_transcript_length_rmgo < min_alignment_length)
1958 } // END ELSE if ( feature_in_transcript_length < min_alignment_length)
1959 } // END ELSE: if ((int)feature_sequence.length() < min_alignment_length)
1960 ++gl_it;
1961 } // END while (gl_it != gl_it_end) - For each feature in gene
1962
1963 // Check whether we have found something:
1964 if (features_with_at_least_one_bait_regions)
1965 {
1966 if (global_VERBOSITY >= VERBOSITY_MOREPROGRESS)
1967 {
1968 cout << "NOTE: Alignment contains " << features_this_gene << " features, of which " << features_with_at_least_one_bait_regions
1969 << " have at least one valid bait region." << endl;
1970 }
1971 }
1972 else
1973 {
1974 if (global_VERBOSITY >= VERBOSITY_WARNINGS)
1975 {
1976 cerr << "ALERT: Alignment contains " << features_this_gene << " features but no single valid bait region. (Details: see above.)"
1977 << endl;
1978 }
1979 }
1980
1981 // Do not delete the transcript_seq pointer, since it is part of transcript_alignment.
1982 } // ELSE of if (transcript_seq == NULL)
1983 } // END if (get_file_size(full_alignment_file_name) > 0)
1984 else
1985 {
1986 if (global_VERBOSITY >= VERBOSITY_WARNINGS)
1987 cerr << "WARNING: Skipping file due to zero file size: " << full_alignment_file_name << endl;
1988 }
1989
1990 // Delete alignment file:
1991 delete transcript_alignment;
1992 transcript_alignment = NULL;
1993
1994 ++DEBUG_COUNT_DELETE_ALIGNMENT_FROM_FILE;
1995 if (global_VERBOSITY >= 1000)
1996 {
1997 print_DEBUG_COUNT(cout, "After calling all deletes of this transcript file:");
1998 }
1999 ++it;
2000 } // END while (it != it_end) -- For all transcript files
2001
2002 if (global_VERBOSITY >= VERBOSITY_PROGRESS)
2003 cout << "PROGRESS: Bait-Fisher handled " << count_transcripts << " alignment files." << endl;
2004
2005 if (global_VERBOSITY >= VERBOSITY_WARNINGS)
2006 {
2007 if (count_transcripts_no_gff_entries > 0)
2008 {
2009 cerr << "WARNING: For some alignment files the gene name could not be found in the gff file. These alignments had to be skipped.\n";
2010 cerr << " Number of alignments affected: " << count_transcripts_no_gff_entries << ". See warnings above." << endl;
2011 }
2012 if ((count_transcripts_no_gff_entries_for_feature-count_transcripts_no_gff_entries) > 0)
2013 {
2014 cerr << "WARNING: For some alignment files we found entries for the specified gene name in the gff file. But after filtering for the specified\n"
2015 " feature name no features where left. These alignment files had to be skipped.\n";
2016 cerr << " Number of alignments affected: " << count_transcripts_no_gff_entries_for_feature-count_transcripts_no_gff_entries << ". See warnings above." << endl;
2017 }
2018 }
2019 } // END if (alignment_cutting)
2020
2021 //****************************************************************
2022 // END Alignment cutting mode
2023 //****************************************************************
2024
2025 else // No alignment cutting
2026 {
2027 // Read fasta sequences one by one:
2028 list<faststring>::iterator it, it_end;
2029 it = fasta_names.begin();
2030 it_end = fasta_names.end();
2031
2032 unsigned Num_alignments = fasta_names.size();
2033 unsigned count_alignments = 0;
2034
2035 CSequence_Mol *alignment_seq_dummy; // Only used as parameter. Not used in this case.
2036 unsigned alignment_seq_index_dummy; // Only used as parameter. Not used in this case.
2037 faststring full_alignment_file_name;
2038 faststring sequence_header_target;
2039 // faststring alignment_name;
2040
2041 faststring genomic_seq_name;
2042
2043 // unsigned overall_counter = 0;
2044 // double mean_match_score = DNAmat.get_mean_matchscore_of_first_N_symbols(4);
2045
2046 CSequences2 *alignment_from_file = NULL;
2047
2048 CTaxonNamesDictionary taxonNamesDict(dname_alignments, fasta_names, sequence_name_taxon_field_number, sequence_name_field_delimiter);
2049
2050 int error_required_taxa;
2051 CRequiredTaxa required_groups_of_taxa(string_required_taxa.c_str(), taxonNamesDict, error_required_taxa);
2052
2053 if (error_required_taxa > 0)
2054 {
2055 if (error_required_taxa == 3)
2056 {
2057 cerr << "ERROR: When parsing the required taxa string. At least one of the taxon names did not occur in any of the sequence files that are analysed. Please look for earlier warning messages." << endl;
2058 cerr << "Exiting." << endl;
2059 exit(-3);
2060 }
2061 }
2062
2063 { // Debug output:
2064 if (global_VERBOSITY > 1000)
2065 {
2066 cout << endl;
2067
2068 taxonNamesDict.print_taxon_names_to_index(cout);
2069
2070 cout << "Required groups: From each of the following sets of taxa, we require at least one to be present in the data set." << endl;
2071 required_groups_of_taxa.print(cout, 1);
2072 cout << endl;
2073 }
2074 }
2075
2076 // For all alignment files:
2077 while (it != it_end)
2078 {
2079 ++count_alignments;
2080 full_alignment_file_name = dname_alignments + "/" + *it;
2081
2082 if (get_file_size(full_alignment_file_name) > 0)
2083 {
2084 // Here is the main action: We handle this gene:
2085
2086 // For this gene file, read the alignment for the genome we are concerned with
2087 if (global_VERBOSITY >= VERBOSITY_PROGRESS)
2088 {
2089 faststring t;
2090 get_time_string(t);
2091 cout << "PROGRESS: " << t << " Reading alignment file: ("
2092 << count_alignments << "/" << Num_alignments
2093 << ") " << full_alignment_file_name << endl;
2094 }
2095
2096 read_alignment_sequence_of_species2(full_alignment_file_name,
2097 faststring(),
2098 alignment_seq_dummy,
2099 alignment_from_file,
2100 alignment_seq_index_dummy,
2101 sequence_name_field_delimiter,
2102 sequence_name_taxon_field_number);
2103
2104 if (alignment_from_file == NULL)
2105 {
2106 cerr << "ERROR: Could not read alignment file: "
2107 << full_alignment_file_name << endl;
2108 exit(-4);
2109 }
2110
2111 int alignment_length = alignment_from_file->GetPosNum();
2112
2113 if (alignment_length < min_alignment_length)
2114 {
2115 // Notice:
2116 if (global_VERBOSITY >= VERBOSITY_WARNINGS)
2117 {
2118 cerr << "NOTICE: Skipping alignment since it is too short. Alignment length: "
2119 << alignment_length << " Minimum: " << min_alignment_length << endl;
2120 }
2121 }
2122 else
2123 {
2124 // We will now deal with this alignment here: ####
2125 // This is the block in which a lot of memory is allocated (with new) that needs to be freed when leaving the block.
2126 /// We have an alignment which we can work with:
2127
2128 bool found_bait_region;
2129
2130 found_bait_region = handle_this_alignment(alignment_from_file,
2131 taxonNamesDict,
2132 required_groups_of_taxa,
2133 bait_length,
2134 number_of_baits_in_region,
2135 bait_offset,
2136 *it, // alignment name
2137 0,
2138 0,
2139 cluster_threshold,
2140 os_result_file,
2141 sequence_name_field_delimiter,
2142 sequence_name_taxon_field_number,
2143 center_computation_mode, *it);
2144
2145
2146 } // END ELSE if ( alignment_length < min_alignment_length)
2147
2148 // Do not delete the alignment_seq pointer, since it is part of alignment_from_file.
2149 } // END (get_file_size(full_alignment_file_name) > 0)
2150 else
2151 {
2152 if (global_VERBOSITY >= VERBOSITY_WARNINGS)
2153 cerr << "WARNING: Skipping file due to zero file size: " << full_alignment_file_name << endl;
2154 // cout << "WARNING: Skipping file due to zero file size: " << full_alignment_file_name << endl;
2155 }
2156
2157 // Delete alignment file:
2158 delete alignment_from_file;
2159 alignment_from_file = NULL;
2160
2161 ++DEBUG_COUNT_DELETE_ALIGNMENT_FROM_FILE;
2162
2163 if (global_VERBOSITY > 1000)
2164 {
2165 print_DEBUG_COUNT(cout, "After calling all deletes of this alignment file.:");
2166 }
2167 ++it;
2168 } // END while (it != it_end) -- For all alignment files
2169
2170 if (global_VERBOSITY >= VERBOSITY_PROGRESS)
2171 cout << "Bait-Fisher handled " << count_alignments << " alignment files." << endl;
2172 } // END else // No alignment cutting
2173
2174 os_result_file.close();
2175
2176 cout << "Bait-Fisher finished without errors." << endl;
2177
2178 } // END main(int argc, char **argv)
2179
2180
2181 // Memory leek detection:
2182 //
2183 // The function read_transcript_sequence_of_species2 reserves memory for:
2184 // - transcript alignment. This is variable "transcript_alignment" in the main function.
2185 // This memory is released at the end of the while loop over all transcripts that are read.
2186 //
2187 // In the loop over all features we reserve memory for the
2188 // - "feature_transcript_alignment"
2189 // This memory is released at the the end of the block that handles features that are sufficiently long
2190 // for designing baits.
2191 //
2192 //
2193 //
2194 //
2195 //
2196 //
2197 //
2198 //
2199 //
2200