Debian Codegraph @

ncbi_acc_download/core.py

raw
1 # Copyright 2017,2018 Kai Blin
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """Core functions of the ncbi-by-accession downloader."""
15 from __future__ import print_function
16 import functools
17 from io import StringIO
18 import sys
19 import time
20 from urllib.parse import urlencode
21
22 from ncbi_acc_download.download import (
23 build_params,
24 get_stream,
25 get_url_by_format,
26 write_stream,
27 )
28 from ncbi_acc_download.errors import (
29 TooManyRequests,
30 ValidationError,
31 )
32 from ncbi_acc_download.validate import (
33 HAVE_BIOPYTHON,
34 run_extended_validation,
35 VALIDATION_LEVELS,
36 )
37 from ncbi_acc_download.wgs import download_wgs_parts
38
39 ENTREZ_URL = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi'
40 SVIEWER_URL = 'https://eutils.ncbi.nlm.nih.gov/sviewer/viewer.cgi'
41
42
43 class Config(object):
44 """NCBI genome download configuration."""
45
46 __slots__ = (
47 'range',
48 'api_key',
49 'emit',
50 'entrez_url',
51 '_extended_validation',
52 'format',
53 'keep_filename',
54 'molecule',
55 'recursive',
56 'sviewer_url',
57 'verbose',
58 )
59
60 def __init__(self, *, extended_validation="none", molecule="nucleotide", out=None,
61 recursive=False, api_key="none", entrez_url=ENTREZ_URL, sviewer_url=SVIEWER_URL,
62 format="genbank", verbose=False, **kwargs):
63 """Initialise the config from scratch."""
64 self.extended_validation = extended_validation
65 self.molecule = molecule
66 self.keep_filename = out is not None
67 self.recursive = recursive
68 self.api_key = api_key
69 self.range = kwargs.get('range', 'none')
70
71 self.entrez_url = entrez_url
72 self.sviewer_url = sviewer_url
73
74 if self.molecule == 'nucleotide':
75 self.format = format
76 else:
77 self.format = 'fasta'
78 self.verbose = verbose
79
80 def noop(arg):
81 """Don't do anything."""
82 pass
83
84 self.emit = noop
85 if self.verbose:
86 self.emit = functools.partial(print, file=sys.stderr, end='', flush=True)
87
88 @property
89 def extended_validation(self):
90 """Get the extended validation setting."""
91 return self._extended_validation
92
93 @extended_validation.setter
94 def extended_validation(self, value):
95 if value != 'none' and not HAVE_BIOPYTHON:
96 raise ValueError("Asked for extended validation, but Biopython not available")
97 if value not in VALIDATION_LEVELS:
98 raise ValueError("Invalid validation level {}".format(value))
99 self._extended_validation = value
100
101 @classmethod
102 def from_args(cls, args):
103 """Initialise from argpase.Namespace object."""
104 config = cls(**args.__dict__)
105 return config
106
107
108 def download_to_file(dl_id, config, filename=None, append=False):
109 """Download a single ID from NCBI and store it to a file."""
110 # types: string, Config, string, bool -> None
111 mode = 'a' if append else 'w'
112
113 url = get_url_by_format(config)
114 params = build_params(dl_id, config)
115
116 try:
117 r = get_stream(url, params)
118 config.emit("Downloading {}\n".format(r.url))
119 except TooManyRequests as err:
120 config.emit("Server requested us to slow down, waiting {} seconds.".format(err.retry_after))
121 time.sleep(int(err.retry_after))
122 r = get_stream(url, params)
123 config.emit("Downloading {}\n".format(r.url))
124
125 if config.keep_filename:
126 outfile_name = filename
127 else:
128 outfile_name = _generate_filename(params, filename)
129
130 with open(outfile_name, mode) as fh:
131 _validate_and_write(r, fh, dl_id, config)
132
133
134 def generate_url(dl_id, config):
135 """Generate the Entrez URL to download a file using a separate tool"""
136 # types: string, Config -> string
137
138 url = get_url_by_format(config)
139 params = build_params(dl_id, config)
140
141 # remove the tool field, some other tool will do the download
142 del params['tool']
143 encoded_params = urlencode(params, doseq=True)
144 return "?".join([url, encoded_params])
145
146
147 def _generate_filename(params, filename):
148 safe_ids = params['id'][:20].replace(' ', '_')
149 file_ending = '.fa'
150
151 if params.get('rettype') == 'gbwithparts':
152 file_ending = '.gbk'
153 elif params.get('rettype') == 'ft':
154 file_ending = '.ft'
155 elif params.get('report') == 'gff3':
156 file_ending = '.gff'
157
158 if filename:
159 outfile_name = "{filename}{ending}".format(filename=filename, ending=file_ending)
160 else:
161 outfile_name = "{ncbi_id}{ending}".format(ncbi_id=safe_ids, ending=file_ending)
162
163 return outfile_name
164
165
166 def _validate_and_write(request, orig_handle, dl_id, config):
167 if config.extended_validation != 'none' or config.recursive:
168 handle = StringIO()
169 else:
170 handle = orig_handle
171
172 write_stream(request, handle, dl_id, config)
173
174 if config.recursive:
175 downloaded = download_wgs_parts(handle, config)
176 handle = downloaded
177
178 if config.extended_validation != 'none':
179 if not run_extended_validation(handle, config.format, config.extended_validation):
180 raise ValidationError("Sequence(s) downloaded for {} failed to load.".format(dl_id))
181
182 if config.extended_validation != 'none' or config.recursive:
183 orig_handle.write(handle.getvalue())