| 1 |
# Copyright 2017,2018 Kai Blin |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 |
# you may not use this file except in compliance with the License. |
| 5 |
# You may obtain a copy of the License at |
| 6 |
# |
| 7 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 |
# |
| 9 |
# Unless required by applicable law or agreed to in writing, software |
| 10 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 |
# See the License for the specific language governing permissions and |
| 13 |
# limitations under the License. |
| 14 |
"""Recursively download the actual entries for WGS records.""" |
| 15 |
|
| 16 |
from io import StringIO |
| 17 |
import time |
| 18 |
|
| 19 |
from ncbi_acc_download.download import ( |
| 20 |
build_params, |
| 21 |
get_stream, |
| 22 |
get_url_by_format, |
| 23 |
write_stream, |
| 24 |
) |
| 25 |
|
| 26 |
from ncbi_acc_download.errors import TooManyRequests |
| 27 |
|
| 28 |
try: |
| 29 |
from Bio import SeqIO |
| 30 |
from Bio.Seq import UndefinedSequenceError |
| 31 |
HAVE_BIOPYTHON = True |
| 32 |
except ImportError: # pragma: no cover |
| 33 |
HAVE_BIOPYTHON = False |
| 34 |
|
| 35 |
STEP_SIZE = 10 |
| 36 |
|
| 37 |
|
| 38 |
class WgsRange: |
| 39 |
def __init__(self, identifier, width, start, end): |
| 40 |
self.identifier = identifier |
| 41 |
self.width = width |
| 42 |
self.start = start |
| 43 |
self.end = end |
| 44 |
|
| 45 |
def get_ids(self): |
| 46 |
"""Get the list of identifier strings covered by the range.""" |
| 47 |
ids = [] |
| 48 |
for i in range(self.start, self.end+1): |
| 49 |
ids.append("{s.identifier}{i:0{s.width}}".format(i=i, s=self)) |
| 50 |
|
| 51 |
return ids |
| 52 |
|
| 53 |
@classmethod |
| 54 |
def from_string(cls, range_string): |
| 55 |
if '-' not in range_string: |
| 56 |
first = last = range_string |
| 57 |
else: |
| 58 |
first, last = range_string.split('-', 1) |
| 59 |
|
| 60 |
if '-' in last: |
| 61 |
raise ValueError("More than one hyphen in input.") |
| 62 |
|
| 63 |
if '.' in first: |
| 64 |
first, _ = first.split('.', 1) |
| 65 |
|
| 66 |
if '.' in last: |
| 67 |
last, _ = last.split('.', 1) |
| 68 |
|
| 69 |
identifier = "" |
| 70 |
for i, char in enumerate(first): |
| 71 |
if not char.isdigit(): |
| 72 |
identifier += char |
| 73 |
else: |
| 74 |
break |
| 75 |
|
| 76 |
width = len(first) - len(identifier) |
| 77 |
|
| 78 |
if width < 1: |
| 79 |
raise ValueError("String identifier is too large.") |
| 80 |
|
| 81 |
if not last.startswith(identifier): |
| 82 |
raise ValueError("Failed to find shared identifier.") |
| 83 |
|
| 84 |
first_int = int(first[-width:]) |
| 85 |
last_int = int(last[-width:]) |
| 86 |
|
| 87 |
if last_int < first_int: |
| 88 |
raise ValueError("Last identifier smaller than first.") |
| 89 |
|
| 90 |
return cls(identifier, width, first_int, last_int) |
| 91 |
|
| 92 |
|
| 93 |
def download_wgs_parts(handle, config): |
| 94 |
"""Download all parts of all WGS records in a file handle.""" |
| 95 |
|
| 96 |
if not HAVE_BIOPYTHON: |
| 97 |
return handle |
| 98 |
|
| 99 |
updated_records = [] |
| 100 |
|
| 101 |
handle.seek(0) |
| 102 |
records = list(SeqIO.parse(handle, config.format)) |
| 103 |
for record in records: |
| 104 |
# TODO: If Biopython ever provides a nice check for undefined sequences, |
| 105 |
# replace the exception-based check here. |
| 106 |
run_download = False |
| 107 |
try: |
| 108 |
bytes(record.seq) |
| 109 |
except UndefinedSequenceError: |
| 110 |
run_download = True |
| 111 |
if run_download and ('wgs_scafld' in record.annotations or |
| 112 |
'wgs' in record.annotations or |
| 113 |
'tsa' in record.annotations): |
| 114 |
updated_records.extend(download_wgs_for_record(record, config)) |
| 115 |
elif run_download and 'contig' in record.annotations: |
| 116 |
updated_records.extend(fix_supercontigs(record, config)) |
| 117 |
else: |
| 118 |
updated_records.append(record) |
| 119 |
|
| 120 |
outhandle = StringIO() |
| 121 |
SeqIO.write(updated_records, outhandle, config.format) |
| 122 |
outhandle.seek(0) |
| 123 |
return outhandle |
| 124 |
|
| 125 |
|
| 126 |
def download_wgs_for_record(record, config): |
| 127 |
"""Download all WGS records in a record.""" |
| 128 |
if 'wgs_scafld' in record.annotations: |
| 129 |
# Biopython splits on '-' for us, but doesn't actually calculate the range |
| 130 |
# Also this is somehow a list of lists |
| 131 |
wgs_range = WgsRange.from_string('-'.join(record.annotations['wgs_scafld'][0])) |
| 132 |
elif 'wgs' in record.annotations: |
| 133 |
# Biopython splits on '-' for us, but doesn't actually calculate the range |
| 134 |
# Unlike WGS_SCAFLD, this is just a list |
| 135 |
wgs_range = WgsRange.from_string('-'.join(record.annotations['wgs'])) |
| 136 |
elif 'tsa' in record.annotations: |
| 137 |
# Biopython splits on '-' for us, but doesn't actually calculate the range |
| 138 |
# Like WGS, this is just a list |
| 139 |
wgs_range = WgsRange.from_string('-'.join(record.annotations['tsa'])) |
| 140 |
else: |
| 141 |
return [record] |
| 142 |
|
| 143 |
handle = StringIO() |
| 144 |
id_list = wgs_range.get_ids() |
| 145 |
|
| 146 |
i = 0 |
| 147 |
while i < len(id_list): |
| 148 |
dl_id = ",".join(id_list[i:i + STEP_SIZE]) |
| 149 |
i += STEP_SIZE |
| 150 |
|
| 151 |
url = get_url_by_format(config) |
| 152 |
params = build_params(dl_id, config) |
| 153 |
|
| 154 |
try: |
| 155 |
r = get_stream(url, params) |
| 156 |
config.emit("Downloading {}\n".format(r.url)) |
| 157 |
except TooManyRequests as err: |
| 158 |
# Wait, and then retry |
| 159 |
config.emit("Server requested us to slow down, waiting {} seconds\n".format(err.retry_after)) |
| 160 |
time.sleep(int(err.retry_after)) |
| 161 |
r = get_stream(url, params) |
| 162 |
config.emit("Downloading {}\n".format(r.url)) |
| 163 |
|
| 164 |
write_stream(r, handle, dl_id, config) |
| 165 |
|
| 166 |
# Rewind, so Biopython can parse this |
| 167 |
handle.seek(0) |
| 168 |
|
| 169 |
return list(SeqIO.parse(handle, config.format)) |
| 170 |
|
| 171 |
|
| 172 |
def fix_supercontigs(record, config): |
| 173 |
"""Fix a record containing a CONTIG entry instead of a seq.""" |
| 174 |
|
| 175 |
handle = StringIO() |
| 176 |
|
| 177 |
# Let the NCBI assemble the proper record for us by asking for the right format. |
| 178 |
dl_id = record.id |
| 179 |
url = get_url_by_format(config) |
| 180 |
params = build_params(dl_id, config) |
| 181 |
try: |
| 182 |
r = get_stream(url, params) |
| 183 |
config.emit("Downloading {}\n".format(r.url)) |
| 184 |
except TooManyRequests as err: |
| 185 |
# Wait, and then retry |
| 186 |
config.emit("Server requested us to slow down, waiting {} seconds\n".format(err.retry_after)) |
| 187 |
time.sleep(int(err.retry_after)) |
| 188 |
r = get_stream(url, params) |
| 189 |
config.emit("Downloading {}\n".format(r.url)) |
| 190 |
|
| 191 |
write_stream(r, handle, dl_id, config) |
| 192 |
|
| 193 |
# Rewind, so Biopython can parse this |
| 194 |
handle.seek(0) |
| 195 |
|
| 196 |
return list(SeqIO.parse(handle, config.format)) |