| 1 |
#!/usr/bin/perl |
| 2 |
use utf8; |
| 3 |
## Copyright (C) 2011-2015 Carnë Draug <[email protected]> |
| 4 |
## |
| 5 |
## This program is free software; you can redistribute it and/or modify |
| 6 |
## it under the terms of the GNU General Public License as published by |
| 7 |
## the Free Software Foundation; either version 3 of the License, or |
| 8 |
## (at your option) any later version. |
| 9 |
## |
| 10 |
## This program is distributed in the hope that it will be useful, |
| 11 |
## but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 |
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 |
## GNU General Public License for more details. |
| 14 |
## |
| 15 |
## You should have received a copy of the GNU General Public License |
| 16 |
## along with this program; if not, see <http://www.gnu.org/licenses/>. |
| 17 |
|
| 18 |
use 5.010; # Use Perl 5.10 |
| 19 |
use warnings; # Replacement for the -w flag, but lexically scoped |
| 20 |
use strict; # Enforce some good programming rules |
| 21 |
use Getopt::Long; # Parse program arguments |
| 22 |
use Cwd; # Determines current working directory |
| 23 |
use File::Spec; # Perform operation on file names |
| 24 |
use Bio::SeqIO; # Handler for SeqIO Formats |
| 25 |
use Bio::DB::EUtilities; # Retrieve entries from Entrez |
| 26 |
use Bio::Root::Version; # Get global bioperl-live version number |
| 27 |
|
| 28 |
# PODNAME: bp_genbank_ref_extractor |
| 29 |
# ABSTRACT: Retrieves all related sequences for a list of searches on Entrez gene |
| 30 |
# AUTHOR: Carnë Draug <[email protected]> |
| 31 |
# OWNER: 2011-2015 Carnë Draug |
| 32 |
# LICENSE: GPL_3 |
| 33 |
|
| 34 |
|
| 35 |
my $assembly_regex = '(primary|reference) assembly'; |
| 36 |
|
| 37 |
my $debug = 0; |
| 38 |
|
| 39 |
my $downstream = 0; |
| 40 |
|
| 41 |
my $email = ""; |
| 42 |
|
| 43 |
my $api_key = ""; |
| 44 |
|
| 45 |
my $format = 'genbank'; |
| 46 |
|
| 47 |
my $genes = ''; |
| 48 |
sub genes_option_parsing { |
| 49 |
for ($_[1]) { |
| 50 |
if (/^(u)?id$/i) { $genes = 'uid'; } |
| 51 |
elsif (/^sym(bol)?$/i) { $genes = 'symbol'; } |
| 52 |
elsif (/^name$/i) { $genes = 'name'; } |
| 53 |
elsif ($_ eq "") { $genes = 'uid' } |
| 54 |
else { die "Invalid identifier '$_[1]' for gene files."; } |
| 55 |
} |
| 56 |
} |
| 57 |
|
| 58 |
my $limit = 200; |
| 59 |
|
| 60 |
my $get_noncoding = 1; |
| 61 |
|
| 62 |
my $proteins = ''; |
| 63 |
sub proteins_option_parsing { |
| 64 |
for ($_[1]) { |
| 65 |
if (/^acc(ession)?$/i) { $proteins = 'accession'; } |
| 66 |
elsif (/^desc(ription)?$/i) { $proteins = 'description'; } |
| 67 |
elsif (/^gene$/i) { $proteins = 'gene'; } |
| 68 |
elsif (/^(transcript|mrna)$/i) { $proteins = 'transcript'; } |
| 69 |
## default is set here, when value is empty |
| 70 |
elsif ($_ eq '') { $proteins = 'accession' } |
| 71 |
else { die "Invalid identifier '$_[1]' for protein files."; } |
| 72 |
} |
| 73 |
} |
| 74 |
|
| 75 |
my $get_pseudo = 1; |
| 76 |
|
| 77 |
my $save = File::Spec->catfile (getcwd, 'extracted sequences'); |
| 78 |
|
| 79 |
my $save_data = ''; |
| 80 |
sub save_data_option_parsing { |
| 81 |
for ($_[1]) { |
| 82 |
if (/^csv$/i) { $save_data = 'csv'; require Text::CSV; } |
| 83 |
elsif ($_ eq '') { $save_data = 'csv'; require Text::CSV; } ## Do nothing. If not set, use default |
| 84 |
else { die "Specified format to save data '$save_data' is not valid."; } |
| 85 |
} |
| 86 |
} |
| 87 |
|
| 88 |
my $transcripts = ''; |
| 89 |
sub transcripts_option_parsing { |
| 90 |
for ($_[1]) { |
| 91 |
if (/^acc(ession)?$/i) { $transcripts = 'accession'; } |
| 92 |
elsif (/^desc(ription)?$/i) { $transcripts = 'description'; } |
| 93 |
elsif (/^gene$/i) { $transcripts = 'gene'; } |
| 94 |
elsif (/^protein$/i) { $transcripts = 'protein'; } |
| 95 |
elsif ($_ eq '') { $transcripts = 'accession' } |
| 96 |
else { die "Invalid identifier '$_[1]' for transcript files."; } |
| 97 |
} |
| 98 |
} |
| 99 |
|
| 100 |
my $upstream = 0; |
| 101 |
|
| 102 |
my $verbose = ''; |
| 103 |
|
| 104 |
my $very_verbose = ''; |
| 105 |
|
| 106 |
|
| 107 |
################################################################################ |
| 108 |
## Parse options, check and create files and directories needed |
| 109 |
################################################################################ |
| 110 |
|
| 111 |
GetOptions( |
| 112 |
'assembly:s' => \$assembly_regex, |
| 113 |
'debug' => \$debug, |
| 114 |
'down|downstream=i' => \$downstream, |
| 115 |
'email=s' => \$email, |
| 116 |
## The 'api_key' is undocumented and only kept for |
| 117 |
## backwards compatibility (it was only during version |
| 118 |
## 1.76). |
| 119 |
'api-key|api_key=s' => \$api_key, |
| 120 |
'format=s' => \$format, |
| 121 |
'genes:s' => \&genes_option_parsing, |
| 122 |
'limit=i' => \$limit, |
| 123 |
'non-coding!' => \$get_noncoding, |
| 124 |
'proteins:s' => \&proteins_option_parsing, |
| 125 |
'pseudo!' => \$get_pseudo, |
| 126 |
'save=s' => \$save, |
| 127 |
'save-data:s' => \&save_data_option_parsing, |
| 128 |
'transcripts|mrna:s' => \&transcripts_option_parsing, |
| 129 |
'up|upstream=i' => \$upstream, |
| 130 |
'verbose|v' => \$verbose, |
| 131 |
'very-verbose|vv' => \$very_verbose, |
| 132 |
'h|help' => sub { exec('perldoc', $0); exit; } |
| 133 |
) or die "Error processing options"; |
| 134 |
## It is necessary to check success of GetOptions since: |
| 135 |
## ''GetOptions returns true to indicate success. It returns false when the function |
| 136 |
## detected one or more errors during option parsing. These errors are signalled |
| 137 |
## using warn() and can be trapped with $SIG{__WARN__}'' |
| 138 |
|
| 139 |
## set verbosity level |
| 140 |
my $verbosity; |
| 141 |
if ($debug) { |
| 142 |
$verbosity = 9; |
| 143 |
$Bio::Root::Root::DEBUG = 1; |
| 144 |
} elsif ($very_verbose) { |
| 145 |
$verbosity = 3; |
| 146 |
} elsif ($verbose) { |
| 147 |
$verbosity = 2; |
| 148 |
} else { |
| 149 |
$verbosity = 1; |
| 150 |
} |
| 151 |
|
| 152 |
my $gene_dir = File::Spec->catfile ($save, 'genes'); |
| 153 |
my $mrna_dir = File::Spec->catfile ($save, 'transcripts'); |
| 154 |
my $prot_dir = File::Spec->catfile ($save, 'proteins'); |
| 155 |
check_dir($_) foreach ($save, $gene_dir, $mrna_dir, $prot_dir); |
| 156 |
my $log_file = File::Spec->catfile ($save, 'extractor.log'); |
| 157 |
open (LOG, ">", $log_file) or die "Couldn't open file $log_file for writing: $!"; |
| 158 |
|
| 159 |
log_it (1, "This is bp_genbank_ref_extractor on Bioperl $Bio::Root::Version::VERSION on ". &get_time); |
| 160 |
|
| 161 |
################################################################################ |
| 162 |
## Everything is ready. Start accessing the database |
| 163 |
################################################################################ |
| 164 |
|
| 165 |
my $data = Structure->new; |
| 166 |
say "Searching on Entrez gene..."; |
| 167 |
my @uids; |
| 168 |
push (@uids, gb_search ($_)) foreach (@ARGV); |
| 169 |
{ |
| 170 |
my $start = scalar(@uids); |
| 171 |
clean_array(\@uids); |
| 172 |
my $diff = $start - scalar(@uids); |
| 173 |
log_it (2, "Entrez gene: removed $diff UIDs from the search results for being repeated.") if $diff > 0; |
| 174 |
log_it (3, "Entrez gene: list of retrieved IDs is: @uids"); |
| 175 |
} |
| 176 |
say "Fetching gene info..."; |
| 177 |
analyze_entrez_genes ($data, \@uids); |
| 178 |
|
| 179 |
if ($genes) { say "Fetching gene sequences..."; get_genes($data); } |
| 180 |
if ($transcripts) { say "Fetching transcript sequences..."; get_products('transcript', $data); } |
| 181 |
if ($proteins) { say "Fetching protein sequences..."; get_products('protein', $data); } |
| 182 |
|
| 183 |
if ($save_data) { save_structure($data); } |
| 184 |
if ($debug) { use Data::Dumper; print Dumper $data; } |
| 185 |
exit; |
| 186 |
|
| 187 |
|
| 188 |
## email and api_key are optional and should removed if empty. In |
| 189 |
## practice, passing an empty email string works fine but the api_key |
| 190 |
## must be valid. See Bio-EUtilities issue #14. |
| 191 |
sub entrez_parameters { |
| 192 |
my @parameters; |
| 193 |
push @parameters, -email => $email if length $email; |
| 194 |
push @parameters, -api_key => $api_key if length $api_key; |
| 195 |
return @parameters; |
| 196 |
} |
| 197 |
|
| 198 |
################################################################################ |
| 199 |
## genbank search subs |
| 200 |
################################################################################ |
| 201 |
|
| 202 |
sub gb_search { |
| 203 |
log_it (2, "Entrez gene: searching with '$_[0]'"); |
| 204 |
my $searcher = Bio::DB::EUtilities->new( |
| 205 |
-eutil => 'esearch', |
| 206 |
-db => 'gene', |
| 207 |
-term => $_[0], |
| 208 |
-retmax => $limit, |
| 209 |
entrez_parameters(), |
| 210 |
); |
| 211 |
log_it (3, "Entrez gene: query $_[0] translated into '" . $searcher->get_query_translation . "'"); |
| 212 |
log_it (2, "Entrez gene: found " . $searcher->get_count . " UIDS"); |
| 213 |
if ($searcher->get_count > $limit) { |
| 214 |
my $w_message = "Entrez gene: search returned more ids than the set limit of $limit. Retrieving only the first $limit genes."; |
| 215 |
log_it (2, $w_message); |
| 216 |
warn $w_message; |
| 217 |
} |
| 218 |
return $searcher->get_ids; |
| 219 |
} |
| 220 |
|
| 221 |
## we are not using esummary because it doesn't tell us the products of the gene thus forcing us |
| 222 |
## to download the sequence and analyze it from that. We are also not using elink because it fails |
| 223 |
## too frequently. Also, entrezgene gives currently the most information so it'll be easier to implement |
| 224 |
## new stuff at a later time |
| 225 |
sub analyze_entrez_genes { |
| 226 |
my $struct = shift; |
| 227 |
my $uid_ref = shift; # a reference for the array containing the list of gene UID |
| 228 |
|
| 229 |
## TODO may be a good idea to limit this and download only a few sequences rather |
| 230 |
## than what can possibly be thousands of them. |
| 231 |
my $fetcher = Bio::DB::EUtilities->new( |
| 232 |
-eutil => 'efetch', |
| 233 |
-db => 'gene', |
| 234 |
-id => $uid_ref, |
| 235 |
-retmode => 'text', |
| 236 |
-rettype => 'asn1', |
| 237 |
entrez_parameters(), |
| 238 |
); |
| 239 |
my $response = $fetcher->get_Response->content; |
| 240 |
open(my $seq_fh, "<", \$response) or die "Could not open sequences string for reading: $!"; |
| 241 |
|
| 242 |
## TODO when this bug is fixed https://redmine.open-bio.org/issues/3261 |
| 243 |
## this should be fixed to use Bio::SeqIO with format=> 'entrezgene' |
| 244 |
## then we could use methods to access the data |
| 245 |
use Bio::ASN1::EntrezGene; |
| 246 |
|
| 247 |
my $parser = Bio::ASN1::EntrezGene->new( |
| 248 |
-fh => $seq_fh, |
| 249 |
); |
| 250 |
|
| 251 |
SEQ: while(my $result = $parser->next_seq){ |
| 252 |
## it's possible that when analyzing genes, if a gene has a RefSeq status of secondary, it will point |
| 253 |
## to another gene, also in the list. To prevent analyzing the same gene twice here, this hash keeps |
| 254 |
## track of the analyzed gene UIDs (even the ones who are pseudo and may not be on $struct) |
| 255 |
state %analyzed_genes; |
| 256 |
$result = $result->[0] if(ref($result) eq 'ARRAY'); |
| 257 |
## Data::Dumper can be used to look into the structure and find where things are |
| 258 |
# use Data::Dumper; |
| 259 |
# print Dumper ($result); |
| 260 |
# exit; |
| 261 |
|
| 262 |
my $uid = find_in_entrezgene ($result, ['track-info', 'geneid']); |
| 263 |
if ($analyzed_genes{$uid}) { |
| 264 |
log_it (9, "DEBUG: skipping analysis of gene with UID='$uid' since it's already been done."); |
| 265 |
next SEQ; |
| 266 |
} |
| 267 |
$analyzed_genes{$uid} = 1; |
| 268 |
|
| 269 |
my ($symbol, $name); |
| 270 |
foreach my $p (@{$result->{'properties'}}){ |
| 271 |
$p = $p->[0] if(ref($p) eq 'ARRAY'); |
| 272 |
next unless ($p->{'label'} && $p->{'label'} eq 'Nomenclature'); |
| 273 |
foreach my $pp (@{$p->{'properties'}}){ |
| 274 |
$pp = $pp->[0] if(ref($pp) eq 'ARRAY'); |
| 275 |
$name = $pp->{'text'} if ($pp->{'label'} && $pp->{'label'} eq 'Official Full Name'); |
| 276 |
$symbol = $pp->{'text'} if ($pp->{'label'} && $pp->{'label'} eq 'Official Symbol'); |
| 277 |
} |
| 278 |
} |
| 279 |
|
| 280 |
## if couldn't find the name and symbol on 'properties', try 'gene' |
| 281 |
## if still couldn't find the name, try 'rna' and then 'prot' |
| 282 |
$symbol //= find_in_entrezgene ($result, ['gene', 'locus']); |
| 283 |
$name //= find_in_entrezgene ( |
| 284 |
$result, |
| 285 |
['gene', 'desc'], |
| 286 |
['rna', 'ext', 'name'], |
| 287 |
['prot', 'name'] |
| 288 |
); |
| 289 |
|
| 290 |
|
| 291 |
my $ensembl = find_in_entrezgene ($result, ['gene', 'db', 'Ensembl', 'tag', 'str']); |
| 292 |
|
| 293 |
## values for the gene-status (different from RefSeq status) |
| 294 |
## live good?? |
| 295 |
## secondary synonym with merged |
| 296 |
## discontinued 'deleted', still index and display to public |
| 297 |
## newentry- for GeneRif submission |
| 298 |
my $status = find_in_entrezgene ($result, ['track-info', 'status']); |
| 299 |
|
| 300 |
if ($status eq 'discontinued') { |
| 301 |
log_it (3, "Discontinued gene: UID='$uid', symbol='$symbol', name='$name'. Forgetting about it..."); |
| 302 |
next SEQ; |
| 303 |
} elsif ($status eq 'secondary') { |
| 304 |
## recursivity! UUUUUUUUUU! |
| 305 |
log_it (3, "Secondary gene: UID='$uid', symbol='$symbol', name='$name'. Attempting to find its current UID..."); |
| 306 |
my $current_id; |
| 307 |
foreach my $c (@{$result->{'track-info'}->[0]->{'current-id'}}) { |
| 308 |
next unless $c->{'db'} eq 'GeneID'; |
| 309 |
$current_id = $c->{'tag'}->[0]->{'id'}; |
| 310 |
next unless $current_id; |
| 311 |
log_it (3, "Update: found current UID '$current_id' of secondary gene with UID='$uid'"); |
| 312 |
analyze_entrez_genes ($struct, [$current_id]); |
| 313 |
} |
| 314 |
log_it (3, "Update: could not find current UID of secondary gene with UID='$uid'") unless $current_id; |
| 315 |
next SEQ; |
| 316 |
} else { |
| 317 |
if (!$status) { |
| 318 |
log_it (1, "WARNING: couldn't find gene status for gene with UID='$uid'. Assuming value of 'live'"); |
| 319 |
$status = 'live-assumed'; |
| 320 |
} |
| 321 |
my @extra_arguments; |
| 322 |
my $ng_message = "New gene: UID='$uid', gene status='$status', symbol='$symbol', name='$name'"; |
| 323 |
if ($ensembl) { |
| 324 |
$ng_message = $ng_message . ", EnsEMBL ID='$ensembl'"; |
| 325 |
push (@extra_arguments, ensembl => $ensembl); |
| 326 |
} |
| 327 |
log_it (3, "$ng_message"); |
| 328 |
$struct->add_gene( |
| 329 |
uid => $uid, |
| 330 |
status => $status, |
| 331 |
name => $name, |
| 332 |
symbol => $symbol, |
| 333 |
@extra_arguments, |
| 334 |
); |
| 335 |
} |
| 336 |
|
| 337 |
## get the gene location (something like 1q21). And if we can't find |
| 338 |
## it in location, we look for it in gene maploc |
| 339 |
my $locus = find_in_entrezgene ( |
| 340 |
$result, |
| 341 |
['location', 'display-str'], |
| 342 |
['gene', 'maploc'] |
| 343 |
); |
| 344 |
if ($locus) { |
| 345 |
$struct->add_gene(uid => $uid, locus => $locus); |
| 346 |
} else { |
| 347 |
log_it (1, "WARNING: couldn't find location for gene with UID='$uid'."); |
| 348 |
} |
| 349 |
|
| 350 |
## get the species names |
| 351 |
my $species = find_in_entrezgene ($result, ['source', 'org', 'taxname']); |
| 352 |
if ($species) { |
| 353 |
$struct->add_gene(uid => $uid, species => $species); |
| 354 |
} else { |
| 355 |
log_it (1, "WARNING: couldn't find species for gene with UID='$uid'."); |
| 356 |
} |
| 357 |
|
| 358 |
## Type can be 'pseudo' or 'protein-coding'. That;s the only type of things |
| 359 |
## we handle. There are also, 'ncRNA' for non coding RNA, and even 'other' |
| 360 |
## for things such as regions |
| 361 |
if ($result->{'type'} eq 'pseudo') { |
| 362 |
log_it (3, "Update: gene with UID='$uid' is '". $result->{'type'} ."' gene. Marking as pseudo..."); |
| 363 |
$struct->add_gene(uid => $uid, pseudo => 1); |
| 364 |
unless ($get_pseudo) { |
| 365 |
log_it (3, "Update: removing gene with UID='$uid' for being pseudo..."); |
| 366 |
$struct->remove_gene($uid); |
| 367 |
next SEQ; |
| 368 |
} |
| 369 |
} elsif ($result->{'type'} eq 'protein-coding') { |
| 370 |
log_it (3, "Update: gene with UID='$uid' is '". $result->{'type'} ."' gene. Marking as protein-coding..."); |
| 371 |
$struct->add_gene(uid => $uid, pseudo => 0); |
| 372 |
} else { |
| 373 |
log_it (1, "WARNING: gene with UID='$uid' is of type '" . $result->{'type'} . "' . Skipping..."); |
| 374 |
$struct->remove_gene($uid); |
| 375 |
next SEQ; |
| 376 |
} |
| 377 |
|
| 378 |
foreach my $l (@{$result->{'locus'}}){ |
| 379 |
$l = $l->[0] if(ref($l) eq 'ARRAY'); |
| 380 |
next unless ($l->{'heading'} && $l->{'heading'} =~ m/$assembly_regex/i); |
| 381 |
my $assembly = $l->{'heading'}; |
| 382 |
my $ChrAccVer = $l->{'accession'}; |
| 383 |
my $ChrStart = find_in_entrezgene ($l, ['seqs', 'int', 'from' ]); |
| 384 |
my $ChrStop = find_in_entrezgene ($l, ['seqs', 'int', 'to' ]); |
| 385 |
my $ChrStrand = find_in_entrezgene ($l, ['seqs', 'int', 'strand']); |
| 386 |
if ($ChrStrand eq 'plus') { |
| 387 |
$ChrStrand = 1; |
| 388 |
$ChrStart += 1 - $upstream; |
| 389 |
$ChrStop += 1 + $downstream; |
| 390 |
} else { |
| 391 |
$ChrStrand = 2; |
| 392 |
$ChrStart += 1 - $downstream; |
| 393 |
$ChrStop += 1 + $upstream; |
| 394 |
} |
| 395 |
log_it (3, "Update: gene with UID='$uid' has Accesion number '$ChrAccVer' between coordinates $ChrStart ... $ChrStop on strand $ChrStrand."); |
| 396 |
$struct->add_gene( |
| 397 |
uid => $uid, |
| 398 |
assembly => $assembly, |
| 399 |
ChrAccVer => $ChrAccVer, |
| 400 |
ChrStart => $ChrStart, |
| 401 |
ChrStop => $ChrStop, |
| 402 |
ChrStrand => $ChrStrand, |
| 403 |
); |
| 404 |
last; # if we got here once, no point in looking on the others |
| 405 |
} |
| 406 |
|
| 407 |
## we will look for products accessions on the comments section instead of the |
| 408 |
## locus section because locus doesn't say the RefSeq status of the products |
| 409 |
foreach my $c (@{$result->{'comments'}}){ |
| 410 |
$c = $c->[0] if(ref($c) eq 'ARRAY'); |
| 411 |
|
| 412 |
## get RefSeq status |
| 413 |
if ($c->{'heading'} && $c->{'heading'} eq 'RefSeq Status') { |
| 414 |
my $refseq_status = $c->{'label'}; |
| 415 |
log_it (3, "Update: gene with UID='$uid' has RefSeq status='$refseq_status'"); |
| 416 |
$struct->add_gene(uid => $uid, 'RefSeq status' => $refseq_status); |
| 417 |
} |
| 418 |
|
| 419 |
## the rest of the block only makes sense if it's not a pseudo gene |
| 420 |
## XXX Also, some genes enconde fancy types of RNA without an actual |
| 421 |
## protein product which we are not skipping. should we check if the |
| 422 |
## gene type has RNA somewhere on its name? |
| 423 |
if ($struct->get_info('gene', $uid, 'pseudo') ) { |
| 424 |
log_it (9, "DEBUG: finished analyzing gene with UID='$uid' earlier since it's pseudo gene " . by_caller_and_location('here') ); |
| 425 |
next SEQ; |
| 426 |
} |
| 427 |
next unless ($c->{'heading'} && $c->{heading} eq 'NCBI Reference Sequences (RefSeq)'); |
| 428 |
foreach my $cc (@{$c->{'comment'}}){ |
| 429 |
$cc = $cc->[0] if(ref($cc) eq 'ARRAY'); |
| 430 |
next unless $cc->{'heading'} eq 'RefSeqs maintained independently of Annotated Genomes'; |
| 431 |
foreach my $ccp (@{$cc->{'products'}}){ |
| 432 |
$ccp = $ccp->[0] if(ref($ccp) eq 'ARRAY'); |
| 433 |
my $mRNA_acc = $ccp->{'accession'}; |
| 434 |
my $prot_acc = $ccp->{'products'}->[0]->{'accession'}; |
| 435 |
## for the RefSeq status needs to be on a loop since there's a list of comments |
| 436 |
## About RefSeq status of products: |
| 437 |
## http://www.ncbi.nlm.nih.gov/entrez/query/static/help/genefaq.html#faq_g7.2 |
| 438 |
## XXX what can we do with this product RefSeq status? |
| 439 |
my $ref_stat; |
| 440 |
foreach my $ccpc (@{$ccp->{'comment'}}){ |
| 441 |
next unless ($ccpc->{'label'} && $ccpc->{'label'} eq 'RefSeq Status'); |
| 442 |
$ref_stat = $ccpc->{'text'}; |
| 443 |
} |
| 444 |
## some transcripts are non-coding. In those cases $prot_acc hence the |
| 445 |
## need for this. For those cases, we want to give no value for 'protein' |
| 446 |
## and not an empty string. The opposite (protein without corresponding |
| 447 |
## transcript) should not be possible |
| 448 |
if (!$prot_acc) { |
| 449 |
log_it (3, "Update: gene with UID='$uid' found to encode transcrip='$mRNA_acc' which is non-coding transcript."); |
| 450 |
## TODO some transcripts, do not encode a protein. This will create errors |
| 451 |
## must fix for genes such as UID 100507436 |
| 452 |
$struct->add_product( |
| 453 |
type => 'transcript', |
| 454 |
accession => $mRNA_acc, |
| 455 |
gene => $uid, |
| 456 |
coding => 0, |
| 457 |
); |
| 458 |
} else { |
| 459 |
log_it (3, "Update: gene with UID='$uid' found to encode transcrip='$mRNA_acc' and protein='$prot_acc' with product RefStatus of '$ref_stat'."); |
| 460 |
## TODO some transcripts, do not encode a protein. This will create errors |
| 461 |
## must fix for genes such as UID 100507436 |
| 462 |
$struct->add_product( |
| 463 |
type => 'transcript', |
| 464 |
accession => $mRNA_acc, |
| 465 |
gene => $uid, |
| 466 |
protein => $prot_acc, |
| 467 |
coding => 1, |
| 468 |
); |
| 469 |
$struct->add_product( |
| 470 |
type => 'protein', |
| 471 |
accession => $prot_acc, |
| 472 |
gene => $uid, |
| 473 |
transcript => $mRNA_acc, |
| 474 |
); |
| 475 |
} |
| 476 |
} |
| 477 |
} |
| 478 |
} |
| 479 |
unless (scalar($struct->get_product_list('transcript', $uid)) >= 1 && |
| 480 |
scalar($struct->get_product_list('protein', $uid)) >= 1) { |
| 481 |
log_it (1, "WARNING: non-pseudo gene with UID='$uid' returned no protein and transcript."); |
| 482 |
} |
| 483 |
|
| 484 |
} |
| 485 |
} |
| 486 |
|
| 487 |
sub create_fetcher { |
| 488 |
my $searched = shift; |
| 489 |
my $rettype; |
| 490 |
if ($format eq 'genbank') {$rettype = 'gb';} |
| 491 |
elsif ($format eq 'fasta') {$rettype = 'fasta';} |
| 492 |
else { |
| 493 |
$rettype = 'native'; |
| 494 |
log_it (1, "WARNING: couldn't convert format '$format' to rettype. Using native as default."); |
| 495 |
} |
| 496 |
|
| 497 |
my $fetcher = Bio::DB::EUtilities->new( |
| 498 |
-eutil => 'efetch', |
| 499 |
-db => $searched, |
| 500 |
-retmode => 'text', |
| 501 |
-rettype => $rettype, |
| 502 |
entrez_parameters(), |
| 503 |
); |
| 504 |
return $fetcher; |
| 505 |
} |
| 506 |
|
| 507 |
sub get_filehandle { |
| 508 |
my $type = shift; |
| 509 |
my $product_key = shift; |
| 510 |
my $name_key = shift; |
| 511 |
my $struct = shift; |
| 512 |
my $base_dir; |
| 513 |
if ($type eq 'gene') { $base_dir = $gene_dir; } |
| 514 |
elsif ($type eq 'transcript') { $base_dir = $mrna_dir; } |
| 515 |
elsif ($type eq 'protein') { $base_dir = $prot_dir; } |
| 516 |
else { |
| 517 |
die "Found a bug. Unknow type provided '$type' to generate filename ". by_caller_and_location('before') ." Please report."; |
| 518 |
} |
| 519 |
my $filename = fix_filename ( $struct->get_info($type, $product_key, $name_key) ); |
| 520 |
my $filepath = File::Spec->catfile ($base_dir, $filename . file_extension_for($format)); |
| 521 |
open (my $filehandle, ">", $filepath) or die "Couldn't open '$filepath' for writing: $!"; |
| 522 |
return $filehandle; |
| 523 |
} |
| 524 |
|
| 525 |
sub get_genes { |
| 526 |
my $struct = shift; |
| 527 |
my @gene_uids = $struct->get_list('gene'); |
| 528 |
my $fetcher = create_fetcher('nucleotide'); |
| 529 |
foreach my $gene_uid (@gene_uids) { |
| 530 |
log_it (2, "Fetching gene: trying to fetch gene with UID='$gene_uid'..."); |
| 531 |
unless ($struct->get_info('gene', $gene_uid, 'ChrAccVer')) { |
| 532 |
log_it (2, "Update: found no genomic info for gene with UID='$gene_uid'..."); |
| 533 |
next; |
| 534 |
} |
| 535 |
my $filehandle = get_filehandle('gene', $gene_uid, $genes, $struct); |
| 536 |
$fetcher->set_parameters ( |
| 537 |
-id => $struct->get_info('gene', $gene_uid, 'ChrAccVer'), |
| 538 |
-seq_start => $struct->get_info('gene', $gene_uid, 'ChrStart'), |
| 539 |
-seq_stop => $struct->get_info('gene', $gene_uid, 'ChrStop'), |
| 540 |
-strand => $struct->get_info('gene', $gene_uid, 'ChrStrand'), |
| 541 |
); |
| 542 |
log_it (3, "Update: fetching gene sequence for gene with UID='$gene_uid'..."); |
| 543 |
print $filehandle $fetcher->get_Response->content; |
| 544 |
close $filehandle or warn $! ? "WARNING: error closing filehandle: $!" : "WARNING: exit status $? from filehandle"; |
| 545 |
} |
| 546 |
} |
| 547 |
|
| 548 |
sub get_products { |
| 549 |
my $product = shift; |
| 550 |
my $struct = shift; |
| 551 |
my @product_acc = $struct->get_list($product); |
| 552 |
|
| 553 |
my ($fetcher, $base_name); |
| 554 |
if ($product eq 'transcript') { |
| 555 |
$fetcher = create_fetcher ('nuccore'); |
| 556 |
$base_name = $transcripts; |
| 557 |
} elsif ($product eq 'protein') { |
| 558 |
$fetcher = create_fetcher ('protein'); |
| 559 |
$base_name = $proteins; |
| 560 |
} else { |
| 561 |
die "Bug found. Invalid product $product argument given ". by_caller_and_location('before') .". Please report it."; |
| 562 |
} |
| 563 |
|
| 564 |
## ideally, there would be no loop, and we'd use $fetcher to get all the sequences in |
| 565 |
## one go. However, that would force us to get the sequences in Bio::Seq files which |
| 566 |
## can be different from the actually downloaded file. Better to not take the chance |
| 567 |
foreach my $product_acc (@product_acc) { |
| 568 |
if ( !$get_noncoding && $product eq 'transcript' && !$struct->get_info($product, $product_acc, 'coding') ) { |
| 569 |
log_it (2, "Fetching $product: skipping $product with accession='$product_acc' since it's non-coding..."); |
| 570 |
} else { |
| 571 |
log_it (2, "Fetching $product: trying to fetch $product with accession='$product_acc'..."); |
| 572 |
} |
| 573 |
$fetcher->set_parameters ( |
| 574 |
-id => $product_acc, |
| 575 |
); |
| 576 |
my $response = $fetcher->get_Response->content; |
| 577 |
open(my $seq_fh, "<", \$response) or die "Could not open sequence string for reading: $!"; |
| 578 |
my $parser = Bio::SeqIO->new( |
| 579 |
-fh => $seq_fh, |
| 580 |
-format => $format, |
| 581 |
); |
| 582 |
while (my $seq = $parser->next_seq) { |
| 583 |
my $product_desc = $seq->description; |
| 584 |
$struct->add_product( |
| 585 |
type => $product, |
| 586 |
accession => $product_acc, |
| 587 |
description => $product_desc, |
| 588 |
); |
| 589 |
} |
| 590 |
my $filehandle = get_filehandle($product, $product_acc, $base_name, $struct); |
| 591 |
print $filehandle $response; |
| 592 |
close $filehandle; |
| 593 |
} |
| 594 |
} |
| 595 |
|
| 596 |
################################################################################ |
| 597 |
## other small useful subs |
| 598 |
################################################################################ |
| 599 |
|
| 600 |
|
| 601 |
## checks if directories exist and creates them as needed |
| 602 |
sub check_dir { |
| 603 |
# TODO must check for permissions as well |
| 604 |
if (!-e $_[0]) { |
| 605 |
mkdir $_[0] or die "Could not mkdir '$_[0]': $!"; |
| 606 |
log_it (9, "DEBUG: directory '$_[0]' created."); |
| 607 |
} elsif (!-d $_[0]) { |
| 608 |
die "Directory '$_[0]' to save output already exists as non-directory."; |
| 609 |
} else { |
| 610 |
log_it (9, "DEBUG: directory '$_[0]' NOT created since it already exists."); |
| 611 |
} |
| 612 |
} |
| 613 |
|
| 614 |
|
| 615 |
sub log_it { |
| 616 |
my $level = shift; |
| 617 |
my $message = shift; |
| 618 |
say STDOUT $message unless $level > $verbosity; |
| 619 |
## debug messages (level 9) will only be logged if on debug mode |
| 620 |
if ($level == 9) { |
| 621 |
say LOG $message if $debug; |
| 622 |
} else { |
| 623 |
say LOG $message; |
| 624 |
} |
| 625 |
} |
| 626 |
|
| 627 |
sub find_in_entrezgene { |
| 628 |
## we have to look into the mess of a structure returned by |
| 629 |
## Bio::ASN1::Entrezgene, causing us to make a bunch of checks. |
| 630 |
## This will return undef if the path does not exist, or the value |
| 631 |
## found. Only hash keys need to be specified, if an array is found |
| 632 |
## it will look into all of its elements. Second argument onwards |
| 633 |
## must be ARRAY refs to the hash keys. They will be looked in order |
| 634 |
## example: |
| 635 |
## find_in_entrezgene ($seq, ['location', 'display-str'], ['gene', 'maploc']); |
| 636 |
|
| 637 |
my $seq = shift; |
| 638 |
my $val; |
| 639 |
foreach my $keys (@_) { |
| 640 |
if (ref($seq) eq 'ARRAY') { |
| 641 |
foreach (@{$seq}) { |
| 642 |
$val = find_in_entrezgene ($_, $keys); |
| 643 |
## value may be false, but will be undefined if just not found |
| 644 |
last if defined $val; |
| 645 |
} |
| 646 |
} elsif (ref($seq) eq 'HASH') { |
| 647 |
my $key = shift (@{$keys}); |
| 648 |
$val = find_in_entrezgene ($seq->{$key}, $keys) if (exists $seq->{$key}); |
| 649 |
} elsif (!ref($seq)) { |
| 650 |
## not a reference, must be the value we are looking for |
| 651 |
$val = $seq; |
| 652 |
} else { |
| 653 |
die "error when transversing entrezgene structure.\n"; |
| 654 |
} |
| 655 |
last if defined $val; |
| 656 |
} |
| 657 |
return $val; |
| 658 |
} |
| 659 |
|
| 660 |
## Removes repeated elements from an array. Does not respect original order |
| 661 |
sub clean_array { |
| 662 |
my %hash; |
| 663 |
foreach (@{$_[0]}) { |
| 664 |
if ($hash{$_}) { |
| 665 |
log_it (9, "DEBUG: value '$_' removed from array " . by_caller_and_location('here') . " called " . by_caller_and_location('before') ); |
| 666 |
} else { |
| 667 |
$hash{$_} = 1; |
| 668 |
} |
| 669 |
} |
| 670 |
@{$_[0]} = keys %hash; |
| 671 |
} |
| 672 |
|
| 673 |
## Returns a pretty string about current time |
| 674 |
sub get_time { |
| 675 |
my ($second, $minute, $hour, $day, $month, $year) = (localtime)[0,1,2,3,4,5]; |
| 676 |
return sprintf ("[%04d-%02d-%02d %02d:%02d:%02d]", $year+1900, $month+1, $day, $hour, $minute, $second); |
| 677 |
} |
| 678 |
|
| 679 |
|
| 680 |
## Tries to sanitize a filename |
| 681 |
sub fix_filename { |
| 682 |
my $file = $_[0]; |
| 683 |
$file =~ s/[^a-z0-9\-\+ \.,\(\){}\[\]']/_/ig; |
| 684 |
log_it (9, "DEBUG: filepath '$_[0]' was converted to '$file' " . by_caller_and_location('here') . " called " . by_caller_and_location('before') ); |
| 685 |
return $file; |
| 686 |
} |
| 687 |
|
| 688 |
sub by_caller_and_location { |
| 689 |
my $level; |
| 690 |
if (!@_ || $_[0] eq 'here') { |
| 691 |
$level = 1; |
| 692 |
} elsif ($_[0] eq 'before'){ |
| 693 |
$level = 2; |
| 694 |
} elsif ($_[0] =~ /^[0-9]+$/){ |
| 695 |
$level = 1 + $_[0]; |
| 696 |
} else { |
| 697 |
die "Bug found when calculating level for caller function. Please report."; |
| 698 |
} |
| 699 |
my $deeper = shift; |
| 700 |
return "by " . (caller($level))[3] . " at line " . (caller($level))[2]; |
| 701 |
} |
| 702 |
|
| 703 |
|
| 704 |
sub file_extension_for { |
| 705 |
## TODO in some cases, extension changes whether it's protein or DNA or whatever |
| 706 |
## and this should be supported |
| 707 |
## XXX there must be a more elegant to handle the formats on this scripts |
| 708 |
|
| 709 |
## to update this list, look in the _guess_format method, inside SeqIO.pm of bioperl |
| 710 |
for ($_[0]) { |
| 711 |
if (/embl/i) {return '.embl';} |
| 712 |
elsif (/entrezgene/i) {return '.asn';} |
| 713 |
elsif (/fasta/i) {return '.fasta';} # fasta|fast|fas|seq|fa|fsa|nt|aa|fna|faa |
| 714 |
elsif (/fastq/i) {return '.fastq';} |
| 715 |
elsif (/gcg/i) {return '.gcg';} |
| 716 |
elsif (/genbank/i) {return '.gb';} # gb|gbank|genbank|gbk|gbs |
| 717 |
elsif (/swiss/i) {return '.swiss';} # swiss|sp |
| 718 |
else { |
| 719 |
log_it (9, "DEBUG: couldn't find the right extension for the requested format. Using '.seq' as default."); |
| 720 |
return ".seq"; |
| 721 |
} |
| 722 |
} |
| 723 |
} |
| 724 |
|
| 725 |
sub save_structure { |
| 726 |
if ($save_data eq 'csv') { create_csv($_[0]); } |
| 727 |
} |
| 728 |
|
| 729 |
sub create_csv { |
| 730 |
my $struct = shift; |
| 731 |
my $csv = Text::CSV->new ({ |
| 732 |
binary => 1, |
| 733 |
eol => $/, |
| 734 |
}) or die "Cannot use Text::CSV: ". Text::CSV->error_diag (); |
| 735 |
|
| 736 |
my $csv_file = File::Spec->catfile ($save, 'data.csv'); |
| 737 |
open (my $fh, ">", $csv_file) or die "Couldn't open file $csv_file for writing: $!"; |
| 738 |
|
| 739 |
$csv->print ($fh, ['gene symbol', 'species', 'gene UID', 'EnsEMBL ID', 'gene name', 'pseudo', 'transcript accession','protein accession', 'locus', 'chromosome accession', 'chromosome start coordinates', 'chromosome stop coordinates', 'assembly'] ); |
| 740 |
|
| 741 |
my @uids = $struct->get_list('gene'); |
| 742 |
foreach my $uid(@uids) { |
| 743 |
my @lines; |
| 744 |
my @mRNA_acc = $struct->get_product_list('transcript', $uid); |
| 745 |
if (!@mRNA_acc) { @mRNA_acc = (''); } # this allows the next loop to run once for pseudo genes |
| 746 |
foreach my $mRNA_acc (@mRNA_acc) { |
| 747 |
push(@lines, [ |
| 748 |
$struct->get_info('gene', $uid, 'symbol'), |
| 749 |
$struct->get_info('gene', $uid, 'species'), |
| 750 |
$uid, |
| 751 |
$struct->get_info('gene', $uid, 'ensembl'), |
| 752 |
$struct->get_info('gene', $uid, 'name'), |
| 753 |
$struct->get_info('gene', $uid, 'pseudo'), |
| 754 |
$mRNA_acc, |
| 755 |
$struct->get_info('transcript', $mRNA_acc, 'protein'), |
| 756 |
$struct->get_info('gene', $uid, 'locus'), |
| 757 |
$struct->get_info('gene', $uid, 'ChrAccVer'), |
| 758 |
$struct->get_info('gene', $uid, 'ChrStart'), |
| 759 |
$struct->get_info('gene', $uid, 'ChrStop'), |
| 760 |
$struct->get_info('gene', $uid, 'assembly'), |
| 761 |
]); |
| 762 |
} |
| 763 |
$csv->print ($fh, $_) for @lines; |
| 764 |
} |
| 765 |
close $fh; |
| 766 |
} |
| 767 |
|
| 768 |
|
| 769 |
################################################################################ |
| 770 |
## Structure methods |
| 771 |
################################################################################ |
| 772 |
package Structure; |
| 773 |
$Structure::VERSION = '1.77'; |
| 774 |
## creates a new instance of the object |
| 775 |
sub new { |
| 776 |
my $class = shift; |
| 777 |
my $self = {}; |
| 778 |
bless ($self, $class); |
| 779 |
return $self; |
| 780 |
} |
| 781 |
|
| 782 |
## adds information to a specific gene and adds the gene to the structure if it doesn't exist |
| 783 |
## $object->add_gene ( |
| 784 |
## uid => gene_uid_of_the_gene, |
| 785 |
## gene_name => gene_name, |
| 786 |
## other_info => corresponding value |
| 787 |
## ); |
| 788 |
sub add_gene { |
| 789 |
my $self = shift; |
| 790 |
my %data = @_; |
| 791 |
## remove the value from the hash so it doesn't show up in the loop |
| 792 |
my $gene_uid = delete $data{'uid'}; |
| 793 |
unless ($gene_uid) { |
| 794 |
log_it (1, "WARNING: no gene UID supplied when adding new gene". main::by_caller_and_location('before') ); |
| 795 |
return; |
| 796 |
} |
| 797 |
## when adding a gene (instead of updating, this goes first. Can't just hope |
| 798 |
## to happen by itself on the loop later because it's possible to add a gene |
| 799 |
## with no info on it |
| 800 |
if ( !exists($self->{'gene'}->{$gene_uid}) ) { |
| 801 |
main::log_it (9, "DEBUG: creating new gene with UID='$gene_uid'."); |
| 802 |
$self->{'gene'}->{$gene_uid} = {}; |
| 803 |
$self->{'gene'}->{$gene_uid}->{'uid'} = $gene_uid; # this is not stupid. Makes it easier to have a general function to create the filename |
| 804 |
$self->{'gene'}->{$gene_uid}->{'transcript'} = []; |
| 805 |
$self->{'gene'}->{$gene_uid}->{'protein'} = []; |
| 806 |
} |
| 807 |
## fill it with all the data |
| 808 |
foreach my $key (keys %data) { |
| 809 |
$self->{'gene'}->{$gene_uid}->{$key} = $data{$key}; |
| 810 |
main::log_it (9, "DEBUG: added $key='$data{$key}' to gene with UID='$gene_uid'."); |
| 811 |
} |
| 812 |
} |
| 813 |
|
| 814 |
## remove genes from the structure given their uid |
| 815 |
## $object->remove_gene($uid) |
| 816 |
## $object->remove_gene(@uids) |
| 817 |
sub remove_gene { |
| 818 |
my $self = shift; |
| 819 |
foreach my $uid (@_) { |
| 820 |
delete $self->{'gene'}->{$uid}; |
| 821 |
main::log_it (9, "DEBUG: removed gene with UID='$uid'"); |
| 822 |
} |
| 823 |
} |
| 824 |
|
| 825 |
sub add_product { |
| 826 |
my $self = shift; |
| 827 |
my %data = @_; |
| 828 |
## remove these values from the hash so they don't show up in the loop later |
| 829 |
my $product = delete $data{'type'}; |
| 830 |
die "Bug found. Please report this. Product requested was $product ". main::by_caller_and_location('before') unless ($product eq 'protein' || $product eq 'transcript'); |
| 831 |
my $product_acc = delete $data{'accession'}; |
| 832 |
unless ($product_acc) { |
| 833 |
main::log_it(1, "WARNING: no $product accession supplied when adding new product ". main::by_caller_and_location('before') . " Adding nothing." ); |
| 834 |
return; |
| 835 |
} |
| 836 |
## Since it's possible that a record for a product be generated automatically when |
| 837 |
## creating it's related product, the only way to know if it's the first time, it's |
| 838 |
## to check it's accession (it's the same as the key. It looks kinda of redundant |
| 839 |
## but allows to have a simpler function to generate filename that uses get_info) |
| 840 |
if ( !exists($self->{$product}->{$product_acc}->{'accession'}) ) { |
| 841 |
main::log_it (9, "DEBUG: creating new $product with accession='$product_acc'."); |
| 842 |
$self->{$product}->{$product_acc}->{'accession'} = $product_acc; |
| 843 |
} |
| 844 |
## fill it with all the data |
| 845 |
foreach my $key (keys %data) { |
| 846 |
$self->{$product}->{$product_acc}->{$key} = $data{$key}; |
| 847 |
## if we're adding info about gene and related products, the array on their |
| 848 |
## part of structure needs to be updated |
| 849 |
if ($key eq 'gene') { |
| 850 |
my $products_on_gene = \@{$self->{'gene'}->{$data{$key}}->{$product}}; |
| 851 |
push (@$products_on_gene, $product_acc); |
| 852 |
main::clean_array($products_on_gene); |
| 853 |
} elsif ($key eq 'transcript' && $product eq 'protein') { |
| 854 |
my $transcript_acc = $data{$key}; |
| 855 |
my $current = $self->{'transcript'}->{$transcript_acc}->{'protein'}; |
| 856 |
if ($current && $current ne $product_acc) { |
| 857 |
warn "WARNING: replacing accession $current with $product_acc as product of $transcript_acc. Please report this bug."; |
| 858 |
$self->{'transcript'}->{'protein'} = $product_acc; |
| 859 |
} |
| 860 |
} elsif ($key eq 'protein' && $product eq 'transcript') { |
| 861 |
my $protein_acc = $data{$key}; |
| 862 |
my $current = $self->{'protein'}->{$protein_acc}->{'transcript'}; |
| 863 |
if ($current && $current ne $product_acc) { |
| 864 |
warn "WARNING: replacing accession $current with $product_acc as ''template'' of $product_acc. Please report this bug."; |
| 865 |
$self->{'protein'}->{'transcript'} = $product_acc; |
| 866 |
} |
| 867 |
} |
| 868 |
} |
| 869 |
} |
| 870 |
## get information from the structure |
| 871 |
## $value = $object->get_info('gene', $gene_id, 'wanted info'); |
| 872 |
## $value = $object->get_info('protein', $protein_acc, 'wanted info'); |
| 873 |
sub get_info { |
| 874 |
my $self = shift; |
| 875 |
my $type = shift; |
| 876 |
my $key = shift; |
| 877 |
my $req = shift; |
| 878 |
## can't check the request here and return the key if the request is accession |
| 879 |
## or id because even though we're using product accessions and gene UID as keys, |
| 880 |
## gene also have accessions and products have id and if we had support for them |
| 881 |
## later, it would create a lot of confusion and bugs |
| 882 |
return $self->{$type}->{$key}->{$req}; |
| 883 |
} |
| 884 |
|
| 885 |
## returns a list of all gene UIDs or product accessions |
| 886 |
## @gene_uids = $structure->get_list('gene') |
| 887 |
## @mRNA_acc = $structure->get_list('transcript') |
| 888 |
sub get_list { |
| 889 |
my $self = shift; |
| 890 |
my $type = shift; |
| 891 |
return keys %{$self->{$type}}; |
| 892 |
} |
| 893 |
|
| 894 |
## for the specified genes UIDs returns list of the requested products accessions |
| 895 |
## If no gene_id is specified, returns a list of all accessions of the product requested |
| 896 |
## @mRNA_acc = structure->get_product_list('transcript', $gene_uid) |
| 897 |
## @prot_acc = structure->get_product_list('protein', @gene_uid) |
| 898 |
## @prot_acc = structure->get_product_list('protein') |
| 899 |
sub get_product_list { |
| 900 |
my $self = shift; |
| 901 |
my $product = shift; |
| 902 |
if (@_ == 1) { return @{$self->{'gene'}->{$_[0]}->{$product}}; } |
| 903 |
elsif (@_ == 0) { return $self->get_list($product); } |
| 904 |
else { return map { $self->get_product_list($product, $_) } @_; } |
| 905 |
} |
| 906 |
|
| 907 |
## this =back closes the last point on the NOTES on usage section |
| 908 |
|
| 909 |
__END__ |
| 910 |
|
| 911 |
=pod |
| 912 |
|
| 913 |
=encoding UTF-8 |
| 914 |
|
| 915 |
=head1 NAME |
| 916 |
|
| 917 |
bp_genbank_ref_extractor - Retrieves all related sequences for a list of searches on Entrez gene |
| 918 |
|
| 919 |
=head1 VERSION |
| 920 |
|
| 921 |
version 1.77 |
| 922 |
|
| 923 |
=head1 SYNOPSIS |
| 924 |
|
| 925 |
B<bp_genbank_ref_extractor> [options] [Entrez Gene Queries] |
| 926 |
|
| 927 |
=head1 DESCRIPTION |
| 928 |
|
| 929 |
This script searches on I<Entrez Gene> database and retrieves not only the gene sequence but |
| 930 |
also the related transcript and protein sequences. |
| 931 |
|
| 932 |
The gene UIDs of multiple searches are collected before attempting to retrieve them so each gene |
| 933 |
will only be analyzed once even if appearing as result on more than one search. |
| 934 |
|
| 935 |
Note that I<by default no sequences are saved> (see options and examples). |
| 936 |
|
| 937 |
=head1 OPTIONS |
| 938 |
|
| 939 |
Several options can be used to fine tune the script behaviour. It is possible to obtain extra |
| 940 |
base pairs upstream and downstream of the gene, control the naming of files and genome assembly to use. |
| 941 |
|
| 942 |
See the section bugs for problems when using default values of options. |
| 943 |
|
| 944 |
=over |
| 945 |
|
| 946 |
=item B<--assembly> |
| 947 |
|
| 948 |
When retrieving the sequence, a specific assemly can be defined. The value expected |
| 949 |
is a regex that will be case-insensitive. If it matches more than one assembly, it will |
| 950 |
use the first match. It defauls to C<(primary|reference) assembly>. |
| 951 |
|
| 952 |
=item B<--debug> |
| 953 |
|
| 954 |
If set, even more output will be printed that may help on debugging. Unlike the messages |
| 955 |
from B<--verbose> and B<--very-verbose>, these will not appear on the log file |
| 956 |
unless this option is selected. This option also sets B<--very-verbose>. |
| 957 |
|
| 958 |
=item B<--downstream>, B<--down> |
| 959 |
|
| 960 |
Specifies the number of extra base pairs to be retrieved downstream of the gene. |
| 961 |
This extra base pairs will only affect the gene sequence, not the transcript or proteins. |
| 962 |
|
| 963 |
=item B<--email> |
| 964 |
|
| 965 |
A valid email used to connect to the NCBI servers. This may be used by |
| 966 |
NCBI to contact users in case of problems and before blocking access in |
| 967 |
case of heavy usage. |
| 968 |
|
| 969 |
=item B <--api-key> |
| 970 |
|
| 971 |
NCBI requires an API key for requests over 10/sec as of December |
| 972 |
2018. You may generate one in the "My NCBI" area. |
| 973 |
|
| 974 |
=item B<--format> |
| 975 |
|
| 976 |
Specifies the format that the sequences will be saved. Defaults to I<genbank> format. |
| 977 |
Valid formats are 'genbank' or 'fasta'. |
| 978 |
|
| 979 |
=item B<--genes> |
| 980 |
|
| 981 |
Specifies the name for gene file. By default, they are not saved. If no value is given |
| 982 |
defaults to its UID. Possible values are 'uid', 'name', 'symbol' (the official symbol or |
| 983 |
nomenclature). |
| 984 |
|
| 985 |
=item B<--help> |
| 986 |
|
| 987 |
Display the documentation (this text). |
| 988 |
|
| 989 |
=item B<--limit> |
| 990 |
|
| 991 |
When making a query, limit the result to these first specific results. This is to |
| 992 |
prevent the use of specially unspecific queries and a warning will be given if a |
| 993 |
query returns more results than the limit. The default value is 200. Note that |
| 994 |
this limit is for I<each> search. |
| 995 |
|
| 996 |
=item B<--non-coding>, B<--nonon-coding> |
| 997 |
|
| 998 |
Some protein coding genes have transcripts that are non-coding. By default, these sequences are |
| 999 |
saved as well. B<--nonon-coding> can be used to ignore those transcripts. |
| 1000 |
|
| 1001 |
=item B<--proteins> |
| 1002 |
|
| 1003 |
Specifies the name for proteins file. By default, they are not saved. If no value is given |
| 1004 |
defaults to its accession. Possible values are 'accession', 'description', 'gene' (the corresponding |
| 1005 |
gene ID) and 'transcript' (the corresponding transcript accesion). |
| 1006 |
|
| 1007 |
Note that if not using 'accession' is possible for files to be overwritten. It is possible for the same gene |
| 1008 |
to encode more than one protein or different proteins to have the same description. |
| 1009 |
|
| 1010 |
=item B<--pseudo>, B<--nopseudo> |
| 1011 |
|
| 1012 |
By default, sequences of pseudo genes will be saved. B<--nopseudo> can be used to ignore those genes. |
| 1013 |
|
| 1014 |
=item B<--save> |
| 1015 |
|
| 1016 |
Specifies the path for the directory where the sequence and log files will be saved. If the |
| 1017 |
directory does not exist it will be created although the path to it must exist. Files on the |
| 1018 |
directory may be rewritten if necessary. If unspecified, a directory named F<extracted sequences> |
| 1019 |
on the current directory will be used. |
| 1020 |
|
| 1021 |
=item B<--save-data> |
| 1022 |
|
| 1023 |
This options saves the data (gene UIDs, description, product accessions, etc) to |
| 1024 |
a file. As an optional value, the file format can be specified. Defaults to CSV. |
| 1025 |
|
| 1026 |
Currently only CSV is supported. |
| 1027 |
|
| 1028 |
Saving the data structure as a CSV file, requires the installation of the Text::CSV module. |
| 1029 |
|
| 1030 |
=item B<--transcripts>, B<--mrna> |
| 1031 |
|
| 1032 |
Specifies the name for transcripts file. By default, they are not saved. If no value is given |
| 1033 |
defaults to its accession. Possible values are 'accession', 'description', 'gene' (the corresponding |
| 1034 |
gene ID) and 'protein' (the protein the transcript encodes). |
| 1035 |
|
| 1036 |
Note that if not using 'accession' is possible for files to be overwritten. It is possible for the same gene |
| 1037 |
to have more than one transcript or different transcripts to have the same description. Also, non-coding |
| 1038 |
transcripts will create problems if using 'protein'. |
| 1039 |
|
| 1040 |
=item B<--upstream>, B<--up> |
| 1041 |
|
| 1042 |
Specifies the number of extra base pairs to be extracted upstream of the gene. |
| 1043 |
This extra base pairs will only affect the gene sequence, not the transcript or proteins. |
| 1044 |
|
| 1045 |
=item B<--verbose>, B<--v> |
| 1046 |
|
| 1047 |
If set, program becomes verbose. For an extremely verbose program, use B<--very-verbose> instead. |
| 1048 |
|
| 1049 |
=item B<--very-verbose>, B<--vv> |
| 1050 |
|
| 1051 |
If set, program becomes extremely verbose. Setting this option, automatically sets B<--verbose> as well. |
| 1052 |
For help in debugging, consider using B<--debug> |
| 1053 |
|
| 1054 |
=back |
| 1055 |
|
| 1056 |
=head1 EXAMPLES |
| 1057 |
|
| 1058 |
bp_genbank_ref_extractor \ |
| 1059 |
--transcripts=accession \ |
| 1060 |
'"homo sapiens"[organism] AND H2B' |
| 1061 |
|
| 1062 |
Search Entrez Gene with the query C<'"homo sapiens"[organism] AND H2B'> |
| 1063 |
and save their transcripts sequences only. Note that default value |
| 1064 |
of B<--limit> may only extract some of the hits. |
| 1065 |
|
| 1066 |
bp_genbank_ref_extractor \ |
| 1067 |
--transcripts=accession --proteins=accession \ |
| 1068 |
--format=fasta \ |
| 1069 |
'"homo sapiens"[organism] AND H2B' \ |
| 1070 |
'"homo sapiens"[organism] AND MCPH1' |
| 1071 |
|
| 1072 |
Save both transcript and protein sequences in the fasta format, for |
| 1073 |
two queries, C<'"homo sapiens"[organism] AND H2B'> and C<'"homo |
| 1074 |
sapiens"[organism] AND MCPH1'>. |
| 1075 |
|
| 1076 |
bp_genbank_ref_extractor \ |
| 1077 |
--genes --down=500 --up=100 \ |
| 1078 |
'"homo sapiens"[organism] AND H2B' |
| 1079 |
|
| 1080 |
Download genomic sequences, including 500 bp downstream and 100 bp |
| 1081 |
upstream of each gene. |
| 1082 |
|
| 1083 |
bp_genbank_ref_extractor \ |
| 1084 |
--genes --asembly='Alternate HuRef' \ |
| 1085 |
'"homo sapiens"[organism] AND H2B' |
| 1086 |
|
| 1087 |
Download genomic sequences from the Alternate HuRef genome assembly. |
| 1088 |
|
| 1089 |
bp_genbank_ref_extractor --save-data=CSV \ |
| 1090 |
'"homo sapiens"[organism] AND H2B' |
| 1091 |
|
| 1092 |
Do not save any sequence, only save the results in a CSV file. |
| 1093 |
|
| 1094 |
bp_genbank_ref_extractor --save='search-results' \ |
| 1095 |
--genes=name downstream=500 --upstream=200 \ |
| 1096 |
--nopseudo --nonnon-coding --transcripts --proteins \ |
| 1097 |
--format=fasta --save-data=CSV \ |
| 1098 |
'"homo sapiens"[organism] AND H2B' \ |
| 1099 |
'"homo sapiens"[organism] AND MCPH1' |
| 1100 |
|
| 1101 |
Ignoring non-coding and pseudo genes, downloads: genomic sequences |
| 1102 |
with 500 and 200 bp downstream and upstream respectively, using the |
| 1103 |
gene name as filename; transcript and proteins sequences using their |
| 1104 |
accession number as filename; everything in fasta format plus a CSV |
| 1105 |
file with search results; saved in a directory named F<search-results> |
| 1106 |
|
| 1107 |
=head1 NON-BUGS |
| 1108 |
|
| 1109 |
=over |
| 1110 |
|
| 1111 |
=item * |
| 1112 |
|
| 1113 |
When supplying options, it's possible to not supply a value and use their default. However, |
| 1114 |
when the expected value is a string, the next argument may be confused as value for the |
| 1115 |
option. For example, when using the following command: |
| 1116 |
|
| 1117 |
bp_genbank_ref_extractor --transcripts \ |
| 1118 |
'H2A AND homo sapiens' |
| 1119 |
|
| 1120 |
we mean to search for 'H2A AND homo sapiens' saving only the transcripts and using the default |
| 1121 |
as base for the filename. However, the search terms will be interpreted as the base for the |
| 1122 |
filenames (but since it's not a valid identifier, it will return an error). To prevent |
| 1123 |
this, you can either specify the values: |
| 1124 |
|
| 1125 |
bp_genbank_ref_extractor --transcripts='accession' \ |
| 1126 |
'H2A AND homo sapiens' |
| 1127 |
|
| 1128 |
or you can use the double hash to stop processing options. Note that this should only be used |
| 1129 |
after the last option. All arguments supplied after the double dash will be interpreted as search terms |
| 1130 |
|
| 1131 |
bp_genbank_ref_extractor --transcripts \ |
| 1132 |
-- 'H2A AND homo sapiens' |
| 1133 |
|
| 1134 |
=back |
| 1135 |
|
| 1136 |
=head1 NOTES ON USAGE |
| 1137 |
|
| 1138 |
=over |
| 1139 |
|
| 1140 |
=item * |
| 1141 |
|
| 1142 |
Genes that are marked as 'live' and 'protein-coding' should have at least one |
| 1143 |
transcript. However, This is not always true due to mistakes on annotation. Such |
| 1144 |
cases will throw a warning. When faced with this, be nice and write to the entrez |
| 1145 |
RefSeq maintainers L<http://www.ncbi.nlm.nih.gov/RefSeq/update.cgi>. |
| 1146 |
|
| 1147 |
=item * |
| 1148 |
When creating the directories to save the files, if the directory already exists it will be used and no error |
| 1149 |
or warning will be issued unless B<--debug> as been set. If a non-directory file already exists with that name |
| 1150 |
bp_genbank_ref_extractor exits with an error. |
| 1151 |
|
| 1152 |
=item * |
| 1153 |
On the subject of verbosity, all messages are saved on the log file. The options |
| 1154 |
B<--verbose> and B<--very-verbose> only affect their printing to standard |
| 1155 |
output. Debug messages are different as they will only show up (and be logged) |
| 1156 |
if requested with B<--debug>. |
| 1157 |
|
| 1158 |
=item * |
| 1159 |
When saving a file, to avoid problems with limited filesystems such as NTFS or FAT, only some |
| 1160 |
characters are allowed. All other characters will be replaced by an underscore. Allowed characters |
| 1161 |
are: |
| 1162 |
|
| 1163 |
B<a-z 0-9 - + . , () {} []'> |
| 1164 |
|
| 1165 |
=item * |
| 1166 |
|
| 1167 |
B<bp_genbank_ref_extractor> tries to use the same file extensions that bioperl |
| 1168 |
would expect when saving the file. If unable it will use the '.seq' extension. |
| 1169 |
|
| 1170 |
=back |
| 1171 |
|
| 1172 |
=head1 FEEDBACK |
| 1173 |
|
| 1174 |
=head2 Mailing lists |
| 1175 |
|
| 1176 |
User feedback is an integral part of the evolution of this and other |
| 1177 |
Bioperl modules. Send your comments and suggestions preferably to |
| 1178 |
the Bioperl mailing list. Your participation is much appreciated. |
| 1179 |
|
| 1180 |
[email protected] - General discussion |
| 1181 |
https://bioperl.org/Support.html - About the mailing lists |
| 1182 |
|
| 1183 |
=head2 Support |
| 1184 |
|
| 1185 |
Please direct usage questions or support issues to the mailing list: |
| 1186 |
I<[email protected]> |
| 1187 |
rather than to the module maintainer directly. Many experienced and |
| 1188 |
reponsive experts will be able look at the problem and quickly |
| 1189 |
address it. Please include a thorough description of the problem |
| 1190 |
with code and data examples if at all possible. |
| 1191 |
|
| 1192 |
=head2 Reporting bugs |
| 1193 |
|
| 1194 |
Report bugs to the Bioperl bug tracking system to help us keep track |
| 1195 |
of the bugs and their resolution. Bug reports can be submitted via the |
| 1196 |
web: |
| 1197 |
|
| 1198 |
https://github.com/bioperl/bio-eutilities/issues |
| 1199 |
|
| 1200 |
=head1 AUTHOR |
| 1201 |
|
| 1202 |
Carnë Draug <[email protected]> |
| 1203 |
|
| 1204 |
=head1 COPYRIGHT |
| 1205 |
|
| 1206 |
This software is copyright (c) 2011-2015 by Carnë Draug. |
| 1207 |
|
| 1208 |
This software is available under the GNU General Public License, Version 3, June 2007. |
| 1209 |
|
| 1210 |
=cut |