| 1 |
# Copyright 2017,2018 Kai Blin |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 |
# you may not use this file except in compliance with the License. |
| 5 |
# You may obtain a copy of the License at |
| 6 |
# |
| 7 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 |
# |
| 9 |
# Unless required by applicable law or agreed to in writing, software |
| 10 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 |
# See the License for the specific language governing permissions and |
| 13 |
# limitations under the License. |
| 14 |
"""The actual download functionality.""" |
| 15 |
|
| 16 |
from __future__ import print_function |
| 17 |
|
| 18 |
from collections import OrderedDict |
| 19 |
from http.client import IncompleteRead |
| 20 |
import requests |
| 21 |
import sys |
| 22 |
|
| 23 |
from ncbi_acc_download.errors import ( |
| 24 |
BadPatternError, |
| 25 |
DownloadError, |
| 26 |
InvalidIdError, |
| 27 |
TooManyRequests, |
| 28 |
) |
| 29 |
|
| 30 |
ERROR_PATTERNS = ( |
| 31 |
u'Error reading from remote server', |
| 32 |
u'Bad gateway', |
| 33 |
u'Bad Gateway', |
| 34 |
u'Cannot process ID list', |
| 35 |
u'server is temporarily unable to service your request', |
| 36 |
u'Service unavailable', |
| 37 |
u'Server Error', |
| 38 |
u'ID list is empty', |
| 39 |
u'Resource temporarily unavailable', |
| 40 |
u'Failed to retrieve sequence', |
| 41 |
u'Failed to understand id', |
| 42 |
) |
| 43 |
|
| 44 |
|
| 45 |
def get_url_by_format(config): |
| 46 |
"""Get URL depending on the format.""" |
| 47 |
# types: Config -> string |
| 48 |
|
| 49 |
if config.format == 'gff3': |
| 50 |
return config.sviewer_url |
| 51 |
|
| 52 |
return config.entrez_url |
| 53 |
|
| 54 |
|
| 55 |
def build_params(dl_id, config): |
| 56 |
"""Build the query parameters for the Entrez query.""" |
| 57 |
params = OrderedDict(tool='ncbi-acc-download', retmode='text') |
| 58 |
|
| 59 |
# delete / characters and as NCBI ignores IDs after #, do the same. |
| 60 |
params['id'] = dl_id |
| 61 |
|
| 62 |
params['db'] = config.molecule |
| 63 |
|
| 64 |
if config.api_key != 'none': |
| 65 |
params['api_key'] = config.api_key |
| 66 |
|
| 67 |
if config.range != 'none': |
| 68 |
rli = config.range.split(":") |
| 69 |
if len(rli)==1: |
| 70 |
rli = config.range.split("..") |
| 71 |
if len(rli)==1: |
| 72 |
rli = config.range.split(".") |
| 73 |
if rli[0] != "": |
| 74 |
params['from'] = int(rli[0]) |
| 75 |
if rli[1] != "": |
| 76 |
params['to'] = int(rli[1]) |
| 77 |
|
| 78 |
if config.molecule == 'nucleotide': |
| 79 |
if config.format == 'genbank': |
| 80 |
params['rettype'] = 'gbwithparts' |
| 81 |
elif config.format == 'featuretable': |
| 82 |
params['rettype'] = 'ft' |
| 83 |
elif config.format == 'gff3': |
| 84 |
params['report'] = 'gff3' |
| 85 |
else: |
| 86 |
params['rettype'] = 'fasta' |
| 87 |
else: |
| 88 |
params['rettype'] = 'fasta' |
| 89 |
|
| 90 |
return params |
| 91 |
|
| 92 |
|
| 93 |
def get_stream(url, params): |
| 94 |
"""Get the actual streamed request from NCBI.""" |
| 95 |
try: |
| 96 |
r = requests.get(url, params=params, stream=True) |
| 97 |
except (requests.exceptions.RequestException, IncompleteRead) as e: |
| 98 |
print("Failed to download {!r} from NCBI".format(params['id']), file=sys.stderr) |
| 99 |
raise DownloadError(str(e)) |
| 100 |
|
| 101 |
if r.status_code != requests.codes.ok: |
| 102 |
if r.status_code == 429: |
| 103 |
retry_after = r.headers.get("retry-after") |
| 104 |
print("Too many requests, please consider using --api-key parameter" |
| 105 |
" (see https://www.ncbi.nlm.nih.gov/books/NBK25497/).") |
| 106 |
raise TooManyRequests("Blocked at NCBI Enterz API for too many requests", retry_after) |
| 107 |
|
| 108 |
print("Failed to download file with id {} from NCBI".format(params['id']), file=sys.stderr) |
| 109 |
raise InvalidIdError("Download failed with return code: {}".format(r.status_code), params["id"], r.status_code) |
| 110 |
|
| 111 |
return r |
| 112 |
|
| 113 |
|
| 114 |
def write_stream(request, handle, dl_id, config): |
| 115 |
"""Write all chunks of the request to the handle.""" |
| 116 |
# use a chunk size of 4k, as that's what most filesystems use these days |
| 117 |
try: |
| 118 |
for chunk in request.iter_content(4096, decode_unicode=True): |
| 119 |
config.emit(u'.') |
| 120 |
for pattern in ERROR_PATTERNS: |
| 121 |
if pattern in chunk: |
| 122 |
raise BadPatternError("Failed to download record(s) with id(s) {} from NCBI: {}".format( |
| 123 |
dl_id, pattern)) |
| 124 |
|
| 125 |
handle.write(chunk) |
| 126 |
except requests.exceptions.ChunkedEncodingError as err: |
| 127 |
print("Download of {!r} aborted: {}".format(dl_id, str(err)), file=sys.stderr) |
| 128 |
raise DownloadError(str(err)) |
| 129 |
config.emit(u'\n') |