Debian Codegraph @

ncbi_acc_download/wgs.py

raw
1 # Copyright 2017,2018 Kai Blin
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """Recursively download the actual entries for WGS records."""
15
16 from io import StringIO
17 import time
18
19 from ncbi_acc_download.download import (
20 build_params,
21 get_stream,
22 get_url_by_format,
23 write_stream,
24 )
25
26 from ncbi_acc_download.errors import TooManyRequests
27
28 try:
29 from Bio import SeqIO
30 from Bio.Seq import UndefinedSequenceError
31 HAVE_BIOPYTHON = True
32 except ImportError: # pragma: no cover
33 HAVE_BIOPYTHON = False
34
35 STEP_SIZE = 10
36
37
38 class WgsRange:
39 def __init__(self, identifier, width, start, end):
40 self.identifier = identifier
41 self.width = width
42 self.start = start
43 self.end = end
44
45 def get_ids(self):
46 """Get the list of identifier strings covered by the range."""
47 ids = []
48 for i in range(self.start, self.end+1):
49 ids.append("{s.identifier}{i:0{s.width}}".format(i=i, s=self))
50
51 return ids
52
53 @classmethod
54 def from_string(cls, range_string):
55 if '-' not in range_string:
56 first = last = range_string
57 else:
58 first, last = range_string.split('-', 1)
59
60 if '-' in last:
61 raise ValueError("More than one hyphen in input.")
62
63 if '.' in first:
64 first, _ = first.split('.', 1)
65
66 if '.' in last:
67 last, _ = last.split('.', 1)
68
69 identifier = ""
70 for i, char in enumerate(first):
71 if not char.isdigit():
72 identifier += char
73 else:
74 break
75
76 width = len(first) - len(identifier)
77
78 if width < 1:
79 raise ValueError("String identifier is too large.")
80
81 if not last.startswith(identifier):
82 raise ValueError("Failed to find shared identifier.")
83
84 first_int = int(first[-width:])
85 last_int = int(last[-width:])
86
87 if last_int < first_int:
88 raise ValueError("Last identifier smaller than first.")
89
90 return cls(identifier, width, first_int, last_int)
91
92
93 def download_wgs_parts(handle, config):
94 """Download all parts of all WGS records in a file handle."""
95
96 if not HAVE_BIOPYTHON:
97 return handle
98
99 updated_records = []
100
101 handle.seek(0)
102 records = list(SeqIO.parse(handle, config.format))
103 for record in records:
104 # TODO: If Biopython ever provides a nice check for undefined sequences,
105 # replace the exception-based check here.
106 run_download = False
107 try:
108 bytes(record.seq)
109 except UndefinedSequenceError:
110 run_download = True
111 if run_download and ('wgs_scafld' in record.annotations or
112 'wgs' in record.annotations or
113 'tsa' in record.annotations):
114 updated_records.extend(download_wgs_for_record(record, config))
115 elif run_download and 'contig' in record.annotations:
116 updated_records.extend(fix_supercontigs(record, config))
117 else:
118 updated_records.append(record)
119
120 outhandle = StringIO()
121 SeqIO.write(updated_records, outhandle, config.format)
122 outhandle.seek(0)
123 return outhandle
124
125
126 def download_wgs_for_record(record, config):
127 """Download all WGS records in a record."""
128 if 'wgs_scafld' in record.annotations:
129 # Biopython splits on '-' for us, but doesn't actually calculate the range
130 # Also this is somehow a list of lists
131 wgs_range = WgsRange.from_string('-'.join(record.annotations['wgs_scafld'][0]))
132 elif 'wgs' in record.annotations:
133 # Biopython splits on '-' for us, but doesn't actually calculate the range
134 # Unlike WGS_SCAFLD, this is just a list
135 wgs_range = WgsRange.from_string('-'.join(record.annotations['wgs']))
136 elif 'tsa' in record.annotations:
137 # Biopython splits on '-' for us, but doesn't actually calculate the range
138 # Like WGS, this is just a list
139 wgs_range = WgsRange.from_string('-'.join(record.annotations['tsa']))
140 else:
141 return [record]
142
143 handle = StringIO()
144 id_list = wgs_range.get_ids()
145
146 i = 0
147 while i < len(id_list):
148 dl_id = ",".join(id_list[i:i + STEP_SIZE])
149 i += STEP_SIZE
150
151 url = get_url_by_format(config)
152 params = build_params(dl_id, config)
153
154 try:
155 r = get_stream(url, params)
156 config.emit("Downloading {}\n".format(r.url))
157 except TooManyRequests as err:
158 # Wait, and then retry
159 config.emit("Server requested us to slow down, waiting {} seconds\n".format(err.retry_after))
160 time.sleep(int(err.retry_after))
161 r = get_stream(url, params)
162 config.emit("Downloading {}\n".format(r.url))
163
164 write_stream(r, handle, dl_id, config)
165
166 # Rewind, so Biopython can parse this
167 handle.seek(0)
168
169 return list(SeqIO.parse(handle, config.format))
170
171
172 def fix_supercontigs(record, config):
173 """Fix a record containing a CONTIG entry instead of a seq."""
174
175 handle = StringIO()
176
177 # Let the NCBI assemble the proper record for us by asking for the right format.
178 dl_id = record.id
179 url = get_url_by_format(config)
180 params = build_params(dl_id, config)
181 try:
182 r = get_stream(url, params)
183 config.emit("Downloading {}\n".format(r.url))
184 except TooManyRequests as err:
185 # Wait, and then retry
186 config.emit("Server requested us to slow down, waiting {} seconds\n".format(err.retry_after))
187 time.sleep(int(err.retry_after))
188 r = get_stream(url, params)
189 config.emit("Downloading {}\n".format(r.url))
190
191 write_stream(r, handle, dl_id, config)
192
193 # Rewind, so Biopython can parse this
194 handle.seek(0)
195
196 return list(SeqIO.parse(handle, config.format))