Debian Codegraph @

ncbi_acc_download/download.py

raw
1 # Copyright 2017,2018 Kai Blin
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """The actual download functionality."""
15
16 from __future__ import print_function
17
18 from collections import OrderedDict
19 from http.client import IncompleteRead
20 import requests
21 import sys
22
23 from ncbi_acc_download.errors import (
24 BadPatternError,
25 DownloadError,
26 InvalidIdError,
27 TooManyRequests,
28 )
29
30 ERROR_PATTERNS = (
31 u'Error reading from remote server',
32 u'Bad gateway',
33 u'Bad Gateway',
34 u'Cannot process ID list',
35 u'server is temporarily unable to service your request',
36 u'Service unavailable',
37 u'Server Error',
38 u'ID list is empty',
39 u'Resource temporarily unavailable',
40 u'Failed to retrieve sequence',
41 u'Failed to understand id',
42 )
43
44
45 def get_url_by_format(config):
46 """Get URL depending on the format."""
47 # types: Config -> string
48
49 if config.format == 'gff3':
50 return config.sviewer_url
51
52 return config.entrez_url
53
54
55 def build_params(dl_id, config):
56 """Build the query parameters for the Entrez query."""
57 params = OrderedDict(tool='ncbi-acc-download', retmode='text')
58
59 # delete / characters and as NCBI ignores IDs after #, do the same.
60 params['id'] = dl_id
61
62 params['db'] = config.molecule
63
64 if config.api_key != 'none':
65 params['api_key'] = config.api_key
66
67 if config.range != 'none':
68 rli = config.range.split(":")
69 if len(rli)==1:
70 rli = config.range.split("..")
71 if len(rli)==1:
72 rli = config.range.split(".")
73 if rli[0] != "":
74 params['from'] = int(rli[0])
75 if rli[1] != "":
76 params['to'] = int(rli[1])
77
78 if config.molecule == 'nucleotide':
79 if config.format == 'genbank':
80 params['rettype'] = 'gbwithparts'
81 elif config.format == 'featuretable':
82 params['rettype'] = 'ft'
83 elif config.format == 'gff3':
84 params['report'] = 'gff3'
85 else:
86 params['rettype'] = 'fasta'
87 else:
88 params['rettype'] = 'fasta'
89
90 return params
91
92
93 def get_stream(url, params):
94 """Get the actual streamed request from NCBI."""
95 try:
96 r = requests.get(url, params=params, stream=True)
97 except (requests.exceptions.RequestException, IncompleteRead) as e:
98 print("Failed to download {!r} from NCBI".format(params['id']), file=sys.stderr)
99 raise DownloadError(str(e))
100
101 if r.status_code != requests.codes.ok:
102 if r.status_code == 429:
103 retry_after = r.headers.get("retry-after")
104 print("Too many requests, please consider using --api-key parameter"
105 " (see https://www.ncbi.nlm.nih.gov/books/NBK25497/).")
106 raise TooManyRequests("Blocked at NCBI Enterz API for too many requests", retry_after)
107
108 print("Failed to download file with id {} from NCBI".format(params['id']), file=sys.stderr)
109 raise InvalidIdError("Download failed with return code: {}".format(r.status_code), params["id"], r.status_code)
110
111 return r
112
113
114 def write_stream(request, handle, dl_id, config):
115 """Write all chunks of the request to the handle."""
116 # use a chunk size of 4k, as that's what most filesystems use these days
117 try:
118 for chunk in request.iter_content(4096, decode_unicode=True):
119 config.emit(u'.')
120 for pattern in ERROR_PATTERNS:
121 if pattern in chunk:
122 raise BadPatternError("Failed to download record(s) with id(s) {} from NCBI: {}".format(
123 dl_id, pattern))
124
125 handle.write(chunk)
126 except requests.exceptions.ChunkedEncodingError as err:
127 print("Download of {!r} aborted: {}".format(dl_id, str(err)), file=sys.stderr)
128 raise DownloadError(str(err))
129 config.emit(u'\n')