| 1 |
# Copyright 2017,2018 Kai Blin |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 |
# you may not use this file except in compliance with the License. |
| 5 |
# You may obtain a copy of the License at |
| 6 |
# |
| 7 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 |
# |
| 9 |
# Unless required by applicable law or agreed to in writing, software |
| 10 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 |
# See the License for the specific language governing permissions and |
| 13 |
# limitations under the License. |
| 14 |
"""Core functions of the ncbi-by-accession downloader.""" |
| 15 |
from __future__ import print_function |
| 16 |
import functools |
| 17 |
from io import StringIO |
| 18 |
import sys |
| 19 |
import time |
| 20 |
from urllib.parse import urlencode |
| 21 |
|
| 22 |
from ncbi_acc_download.download import ( |
| 23 |
build_params, |
| 24 |
get_stream, |
| 25 |
get_url_by_format, |
| 26 |
write_stream, |
| 27 |
) |
| 28 |
from ncbi_acc_download.errors import ( |
| 29 |
TooManyRequests, |
| 30 |
ValidationError, |
| 31 |
) |
| 32 |
from ncbi_acc_download.validate import ( |
| 33 |
HAVE_BIOPYTHON, |
| 34 |
run_extended_validation, |
| 35 |
VALIDATION_LEVELS, |
| 36 |
) |
| 37 |
from ncbi_acc_download.wgs import download_wgs_parts |
| 38 |
|
| 39 |
ENTREZ_URL = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi' |
| 40 |
SVIEWER_URL = 'https://eutils.ncbi.nlm.nih.gov/sviewer/viewer.cgi' |
| 41 |
|
| 42 |
|
| 43 |
class Config(object): |
| 44 |
"""NCBI genome download configuration.""" |
| 45 |
|
| 46 |
__slots__ = ( |
| 47 |
'range', |
| 48 |
'api_key', |
| 49 |
'emit', |
| 50 |
'entrez_url', |
| 51 |
'_extended_validation', |
| 52 |
'format', |
| 53 |
'keep_filename', |
| 54 |
'molecule', |
| 55 |
'recursive', |
| 56 |
'sviewer_url', |
| 57 |
'verbose', |
| 58 |
) |
| 59 |
|
| 60 |
def __init__(self, *, extended_validation="none", molecule="nucleotide", out=None, |
| 61 |
recursive=False, api_key="none", entrez_url=ENTREZ_URL, sviewer_url=SVIEWER_URL, |
| 62 |
format="genbank", verbose=False, **kwargs): |
| 63 |
"""Initialise the config from scratch.""" |
| 64 |
self.extended_validation = extended_validation |
| 65 |
self.molecule = molecule |
| 66 |
self.keep_filename = out is not None |
| 67 |
self.recursive = recursive |
| 68 |
self.api_key = api_key |
| 69 |
self.range = kwargs.get('range', 'none') |
| 70 |
|
| 71 |
self.entrez_url = entrez_url |
| 72 |
self.sviewer_url = sviewer_url |
| 73 |
|
| 74 |
if self.molecule == 'nucleotide': |
| 75 |
self.format = format |
| 76 |
else: |
| 77 |
self.format = 'fasta' |
| 78 |
self.verbose = verbose |
| 79 |
|
| 80 |
def noop(arg): |
| 81 |
"""Don't do anything.""" |
| 82 |
pass |
| 83 |
|
| 84 |
self.emit = noop |
| 85 |
if self.verbose: |
| 86 |
self.emit = functools.partial(print, file=sys.stderr, end='', flush=True) |
| 87 |
|
| 88 |
@property |
| 89 |
def extended_validation(self): |
| 90 |
"""Get the extended validation setting.""" |
| 91 |
return self._extended_validation |
| 92 |
|
| 93 |
@extended_validation.setter |
| 94 |
def extended_validation(self, value): |
| 95 |
if value != 'none' and not HAVE_BIOPYTHON: |
| 96 |
raise ValueError("Asked for extended validation, but Biopython not available") |
| 97 |
if value not in VALIDATION_LEVELS: |
| 98 |
raise ValueError("Invalid validation level {}".format(value)) |
| 99 |
self._extended_validation = value |
| 100 |
|
| 101 |
@classmethod |
| 102 |
def from_args(cls, args): |
| 103 |
"""Initialise from argpase.Namespace object.""" |
| 104 |
config = cls(**args.__dict__) |
| 105 |
return config |
| 106 |
|
| 107 |
|
| 108 |
def download_to_file(dl_id, config, filename=None, append=False): |
| 109 |
"""Download a single ID from NCBI and store it to a file.""" |
| 110 |
# types: string, Config, string, bool -> None |
| 111 |
mode = 'a' if append else 'w' |
| 112 |
|
| 113 |
url = get_url_by_format(config) |
| 114 |
params = build_params(dl_id, config) |
| 115 |
|
| 116 |
try: |
| 117 |
r = get_stream(url, params) |
| 118 |
config.emit("Downloading {}\n".format(r.url)) |
| 119 |
except TooManyRequests as err: |
| 120 |
config.emit("Server requested us to slow down, waiting {} seconds.".format(err.retry_after)) |
| 121 |
time.sleep(int(err.retry_after)) |
| 122 |
r = get_stream(url, params) |
| 123 |
config.emit("Downloading {}\n".format(r.url)) |
| 124 |
|
| 125 |
if config.keep_filename: |
| 126 |
outfile_name = filename |
| 127 |
else: |
| 128 |
outfile_name = _generate_filename(params, filename) |
| 129 |
|
| 130 |
with open(outfile_name, mode) as fh: |
| 131 |
_validate_and_write(r, fh, dl_id, config) |
| 132 |
|
| 133 |
|
| 134 |
def generate_url(dl_id, config): |
| 135 |
"""Generate the Entrez URL to download a file using a separate tool""" |
| 136 |
# types: string, Config -> string |
| 137 |
|
| 138 |
url = get_url_by_format(config) |
| 139 |
params = build_params(dl_id, config) |
| 140 |
|
| 141 |
# remove the tool field, some other tool will do the download |
| 142 |
del params['tool'] |
| 143 |
encoded_params = urlencode(params, doseq=True) |
| 144 |
return "?".join([url, encoded_params]) |
| 145 |
|
| 146 |
|
| 147 |
def _generate_filename(params, filename): |
| 148 |
safe_ids = params['id'][:20].replace(' ', '_') |
| 149 |
file_ending = '.fa' |
| 150 |
|
| 151 |
if params.get('rettype') == 'gbwithparts': |
| 152 |
file_ending = '.gbk' |
| 153 |
elif params.get('rettype') == 'ft': |
| 154 |
file_ending = '.ft' |
| 155 |
elif params.get('report') == 'gff3': |
| 156 |
file_ending = '.gff' |
| 157 |
|
| 158 |
if filename: |
| 159 |
outfile_name = "{filename}{ending}".format(filename=filename, ending=file_ending) |
| 160 |
else: |
| 161 |
outfile_name = "{ncbi_id}{ending}".format(ncbi_id=safe_ids, ending=file_ending) |
| 162 |
|
| 163 |
return outfile_name |
| 164 |
|
| 165 |
|
| 166 |
def _validate_and_write(request, orig_handle, dl_id, config): |
| 167 |
if config.extended_validation != 'none' or config.recursive: |
| 168 |
handle = StringIO() |
| 169 |
else: |
| 170 |
handle = orig_handle |
| 171 |
|
| 172 |
write_stream(request, handle, dl_id, config) |
| 173 |
|
| 174 |
if config.recursive: |
| 175 |
downloaded = download_wgs_parts(handle, config) |
| 176 |
handle = downloaded |
| 177 |
|
| 178 |
if config.extended_validation != 'none': |
| 179 |
if not run_extended_validation(handle, config.format, config.extended_validation): |
| 180 |
raise ValidationError("Sequence(s) downloaded for {} failed to load.".format(dl_id)) |
| 181 |
|
| 182 |
if config.extended_validation != 'none' or config.recursive: |
| 183 |
orig_handle.write(handle.getvalue()) |