| 1 |
#!/bin/bash |
| 2 |
|
| 3 |
set -eu -o pipefail |
| 4 |
|
| 5 |
exists() { |
| 6 |
command -v "$1" >/dev/null 2>&1 |
| 7 |
} |
| 8 |
|
| 9 |
cut_after_first_space_or_second_pipe() { |
| 10 |
grep '^>' | sed 's/ .*//' | sed 's/\([^|]*|[^|]*\).*/\1/' |
| 11 |
} |
| 12 |
export -f cut_after_first_space_or_second_pipe |
| 13 |
|
| 14 |
map_headers_to_taxid() { |
| 15 |
grep '^>' | cut_after_first_space_or_second_pipe | sed -e "s/^>//" -e "s/\$/ $1/" |
| 16 |
} |
| 17 |
export -f map_headers_to_taxid |
| 18 |
|
| 19 |
|
| 20 |
|
| 21 |
######################################################### |
| 22 |
## Functions |
| 23 |
|
| 24 |
function download_n_process() { |
| 25 |
IFS=$'\t' read -r TAXID FILEPATH <<< "$1" |
| 26 |
|
| 27 |
NAME=`basename $FILEPATH .gz` |
| 28 |
GZIPPED_FILE="$LIBDIR/$DOMAIN/$NAME.gz" |
| 29 |
UNZIPPED_FILE="$LIBDIR/$DOMAIN/$NAME" |
| 30 |
DUSTMASKED_FILE="$LIBDIR/$DOMAIN/${NAME%.fna}_dustmasked.fna" |
| 31 |
[[ "$DO_DUST" == "1" ]] && RES_FILE=$DUSTMASKED_FILE || RES_FILE=$UNZIPPED_FILE |
| 32 |
|
| 33 |
if [[ ! -s "$RES_FILE" ]]; then |
| 34 |
if [ $DL_MODE = "rsync" ]; then |
| 35 |
FILEPATH=${FILEPATH/ftp/rsync} |
| 36 |
$DL_CMD "$FILEPATH" "$LIBDIR/$DOMAIN/$NAME.gz" || \ |
| 37 |
{ printf "\nError downloading $FILEPATH!\n" >&2 && exit 1; } |
| 38 |
else |
| 39 |
$DL_CMD "$LIBDIR/$DOMAIN/$NAME.gz" "$FILEPATH" || \ |
| 40 |
$DL_CMD "$LIBDIR/$DOMAIN/$NAME.gz" "$FILEPATH" || \ |
| 41 |
$DL_CMD "$LIBDIR/$DOMAIN/$NAME.gz" "$FILEPATH" || \ |
| 42 |
{ printf "\nError downloading $FILEPATH!\n" >&2 && exit 1; } |
| 43 |
fi |
| 44 |
|
| 45 |
[[ -s "$LIBDIR/$DOMAIN/$NAME.gz" ]] || return; |
| 46 |
gunzip -f "$LIBDIR/$DOMAIN/$NAME.gz" ||{ printf "\nError gunzipping $LIBDIR/$DOMAIN/$NAME.gz [ downloaded from $FILEPATH ]!\n" >&2 && exit 255; } |
| 47 |
|
| 48 |
|
| 49 |
if [[ "$CHANGE_HEADER" == "1" ]]; then |
| 50 |
sed -i "s/^>/>kraken:taxid|$TAXID /" $LIBDIR/$DOMAIN/$NAME |
| 51 |
fi |
| 52 |
|
| 53 |
if [[ "$FILTER_UNPLACED" == "1" ]]; then |
| 54 |
echo TODO 2>&1 |
| 55 |
##sed -n '1,/^>.*unplaced/p; /' |
| 56 |
fi |
| 57 |
|
| 58 |
if [[ "$DO_DUST" == "1" ]]; then |
| 59 |
## TODO: Consider hard-masking only low-complexity stretches with 10 or more bps |
| 60 |
dustmasker -infmt fasta -in $LIBDIR/$DOMAIN/$NAME -level 20 -outfmt fasta | sed '/^>/! s/[^AGCT]/N/g' > $DUSTMASKED_FILE |
| 61 |
rm $LIBDIR/$DOMAIN/$NAME |
| 62 |
fi |
| 63 |
fi |
| 64 |
|
| 65 |
## Output sequenceID to taxonomy ID map to STDOUT |
| 66 |
cat $RES_FILE | map_headers_to_taxid $TAXID |
| 67 |
echo done |
| 68 |
} |
| 69 |
export -f download_n_process |
| 70 |
|
| 71 |
|
| 72 |
|
| 73 |
function download_n_process_nofail() { |
| 74 |
download_n_process "$@" || true |
| 75 |
} |
| 76 |
export -f download_n_process_nofail |
| 77 |
|
| 78 |
|
| 79 |
ceol=`tput el || echo -n ""` # terminfo clr_eol |
| 80 |
|
| 81 |
function count { |
| 82 |
typeset C=0 |
| 83 |
while read L; do |
| 84 |
if [[ "$L" == "done" ]]; then |
| 85 |
[[ "$VERBOSE" == 1 ]] && continue; |
| 86 |
C=$(( C + 1 )) |
| 87 |
_progress=$(( (${C}*100/${1}*100)/100 )) |
| 88 |
_done=$(( (${_progress}*4)/10 )) |
| 89 |
_left=$(( 40-$_done )) |
| 90 |
# Build progressbar string lengths |
| 91 |
_done=$(printf "%${_done}s") |
| 92 |
_left=$(printf "%${_left}s") |
| 93 |
|
| 94 |
printf "\rProgress : [${_done// /#}${_left// /-}] ${_progress}%% $C/$1" 1>&2 |
| 95 |
else |
| 96 |
echo "$L" |
| 97 |
fi |
| 98 |
done |
| 99 |
} |
| 100 |
|
| 101 |
function check_or_mkdir_no_fail { |
| 102 |
#echo -n "Creating $1 ... " >&2 |
| 103 |
if [[ -d $1 && ! -n `find $1 -prune -empty -type d` ]]; then |
| 104 |
echo "Directory $1 exists. Continuing" >&2 |
| 105 |
return `true` |
| 106 |
else |
| 107 |
#echo "Done" >&2 |
| 108 |
mkdir -p $1 |
| 109 |
return `true` |
| 110 |
fi |
| 111 |
} |
| 112 |
|
| 113 |
function c_echo() { |
| 114 |
printf "\033[34m$*\033[0m\n" |
| 115 |
} |
| 116 |
|
| 117 |
|
| 118 |
|
| 119 |
## Check if GNU parallel exists |
| 120 |
command -v parallel >/dev/null 2>&1 && PARALLEL=1 || PARALLEL=0 |
| 121 |
|
| 122 |
|
| 123 |
ALL_GENOMES="bacteria viral archaea fungi protozoa invertebrate plant vertebrate_mammalian vertebrate_other" |
| 124 |
ALL_DATABASES="refseq genbank taxonomy contaminants" |
| 125 |
ALL_ASSEMBLY_LEVELS="Complete\ Genome Chromosome Scaffold Contig" |
| 126 |
|
| 127 |
## Option parsing |
| 128 |
DATABASE="refseq" |
| 129 |
ASSEMBLY_LEVEL="Complete Genome" |
| 130 |
REFSEQ_CATEGORY="" |
| 131 |
TAXID="" |
| 132 |
|
| 133 |
|
| 134 |
DL_PROG="NA" |
| 135 |
if hash curl 2>/dev/null; then |
| 136 |
DL_PROG="curl" |
| 137 |
elif hash wget 2>/dev/null; then |
| 138 |
DL_PROG="wget" |
| 139 |
elif hash rsync 2>/dev/null; then |
| 140 |
DL_PROG="rsync" |
| 141 |
fi |
| 142 |
|
| 143 |
|
| 144 |
BASE_DIR="." |
| 145 |
N_PROC=1 |
| 146 |
CHANGE_HEADER=0 |
| 147 |
DOWNLOAD_RNA=0 |
| 148 |
DO_DUST=0 |
| 149 |
FILTER_UNPLACED=0 |
| 150 |
VERBOSE=0 |
| 151 |
|
| 152 |
USAGE=" |
| 153 |
`basename $0` [<options>] <database> |
| 154 |
|
| 155 |
ARGUMENT |
| 156 |
<database> One of refseq, genbank, contaminants or taxonomy: |
| 157 |
- use refseq or genbank for genomic sequences, |
| 158 |
- contaminants gets contaminant sequences from UniVec and EmVec, |
| 159 |
- taxonomy for taxonomy mappings. |
| 160 |
|
| 161 |
COMMON OPTIONS |
| 162 |
-o <directory> Folder to which the files are downloaded. Default: '$BASE_DIR'. |
| 163 |
-P <# of threads> Number of processes when downloading (uses xargs). Default: '$N_PROC' |
| 164 |
|
| 165 |
WHEN USING database refseq OR genbank: |
| 166 |
-d <domain> What domain to download. One or more of ${ALL_GENOMES// /, } (comma separated). |
| 167 |
-a <assembly level> Only download genomes with the specified assembly level. Default: '$ASSEMBLY_LEVEL'. Use 'Any' for any assembly level. |
| 168 |
-c <refseq category> Only download genomes in the specified refseq category. Default: any. |
| 169 |
-t <taxids> Only download the specified taxonomy IDs, comma separated. Default: any. |
| 170 |
-g <program> Download using program. Options: rsync, curl, wget. Default $DL_PROG (auto-detected). |
| 171 |
-r Download RNA sequences, too. |
| 172 |
-u Filter unplaced sequences. |
| 173 |
-m Mask low-complexity regions using dustmasker. Default: off. |
| 174 |
-l Modify header to include taxonomy ID. Default: off. |
| 175 |
-v Verbose mode |
| 176 |
" |
| 177 |
|
| 178 |
# arguments: $OPTFIND (current index), $OPTARG (argument for option), $OPTERR (bash-specific) |
| 179 |
while getopts "o:P:d:a:c:t:g:urlmv" OPT "$@"; do |
| 180 |
case $OPT in |
| 181 |
o) BASE_DIR="$OPTARG" ;; |
| 182 |
P) N_PROC="$OPTARG" ;; |
| 183 |
d) DOMAINS=${OPTARG//,/ } ;; |
| 184 |
a) ASSEMBLY_LEVEL="$OPTARG" ;; |
| 185 |
c) REFSEQ_CATEGORY="$OPTARG" ;; |
| 186 |
g) DL_PROG="$OPTARG" ;; |
| 187 |
t) TAXID="$OPTARG" ;; |
| 188 |
r) DOWNLOAD_RNA=1 ;; |
| 189 |
u) FILTER_UNPLACED=1 ;; |
| 190 |
m) DO_DUST=1 ;; |
| 191 |
v) VERBOSE=1 ;; |
| 192 |
l) CHANGE_HEADER=1 ;; |
| 193 |
\?) echo "Invalid option: -$OPTARG" >&2 |
| 194 |
exit 1 |
| 195 |
;; |
| 196 |
:) echo "Option -$OPTARG requires an argument." >&2 |
| 197 |
exit 1 |
| 198 |
;; |
| 199 |
esac |
| 200 |
done |
| 201 |
shift $((OPTIND-1)) |
| 202 |
|
| 203 |
[[ $# -eq 1 ]] || { printf "$USAGE" >&2 && exit 1; }; |
| 204 |
DATABASE=$1 |
| 205 |
|
| 206 |
|
| 207 |
if [[ "$DL_PROG" == "rsync" ]]; then |
| 208 |
DL_CMD="rsync --no-motd" |
| 209 |
DL_MODE="rsync" |
| 210 |
elif [[ "$DL_PROG" == "wget" ]]; then |
| 211 |
DL_CMD="wget -N --reject=index.html -qO" |
| 212 |
DL_MODE="https" |
| 213 |
elif [[ "$DL_PROG" == "curl" ]]; then |
| 214 |
DL_CMD="curl -s -o" |
| 215 |
DL_MODE="https" |
| 216 |
else |
| 217 |
echo "Unknown download program - please install one of rsync, wget or curl, and specify it with the -g option" >&2 |
| 218 |
exit 1 |
| 219 |
fi |
| 220 |
|
| 221 |
cecho() { |
| 222 |
echo $* 1>&2 |
| 223 |
$* |
| 224 |
} |
| 225 |
|
| 226 |
if [[ "$VERBOSE" == "1" ]]; then |
| 227 |
export -f cecho |
| 228 |
DL_CMD="cecho $DL_CMD" |
| 229 |
fi |
| 230 |
|
| 231 |
export DL_CMD DL_MODE VERBOSE |
| 232 |
|
| 233 |
#### TAXONOMY DOWNLOAD |
| 234 |
FTP="https://ftp.ncbi.nih.gov" |
| 235 |
if [[ "$DATABASE" == "taxonomy" ]]; then |
| 236 |
echo "Downloading NCBI taxonomy ... " >&2 |
| 237 |
if check_or_mkdir_no_fail "$BASE_DIR"; then |
| 238 |
cd "$BASE_DIR" > /dev/null |
| 239 |
if [ $DL_MODE = "rsync" ]; then |
| 240 |
$DL_CMD ${FTP/ftp/rsync}/pub/taxonomy/taxdump.tar.gz taxdump.tar.gz |
| 241 |
else |
| 242 |
$DL_CMD taxdump.tar.gz $FTP/pub/taxonomy/taxdump.tar.gz |
| 243 |
fi |
| 244 |
tar -zxvf taxdump.tar.gz nodes.dmp |
| 245 |
tar -zxvf taxdump.tar.gz names.dmp |
| 246 |
rm taxdump.tar.gz |
| 247 |
cd - > /dev/null |
| 248 |
fi |
| 249 |
exit 0 |
| 250 |
fi |
| 251 |
|
| 252 |
dat_to_fna() { |
| 253 |
grep -E '^DE|^ ' | awk '/^DE/ { sub(/DE */,">"); gsub(/[ |]/,"_") }; { print }' | awk '/^ / { gsub(/ /,""); sub(/[0-9]*$/,"") }; { print }' |
| 254 |
} |
| 255 |
|
| 256 |
#### CONTAMINANT SEQ DOWNLOAD |
| 257 |
if [[ "$DATABASE" == "contaminants" ]]; then |
| 258 |
echo "Downloading contaminant databases ... " >&2 |
| 259 |
CONTAMINANT_TAXID=32630 |
| 260 |
CONTAMINANT_DIR="$BASE_DIR/contaminants" |
| 261 |
if check_or_mkdir_no_fail "$CONTAMINANT_DIR"; then |
| 262 |
cd "$CONTAMINANT_DIR" > /dev/null |
| 263 |
|
| 264 |
# download UniVec and EmVec database |
| 265 |
if [ $DL_MODE = "rsync" ]; then |
| 266 |
$DL_CMD rsync://ftp.ncbi.nlm.nih.gov/pub/UniVec/UniVec UniVec.fna |
| 267 |
$DL_CMD rsync://ftp.ebi.ac.uk/pub/databases/emvec/emvec.dat.gz emvec.dat.gz |
| 268 |
else |
| 269 |
$DL_CMD UniVec.fna https://ftp.ncbi.nlm.nih.gov/pub/UniVec/UniVec |
| 270 |
$DL_CMD emvec.dat.gz https://ftp.ebi.ac.uk/pub/databases/emvec/emvec.dat.gz |
| 271 |
fi |
| 272 |
gunzip -c emvec.dat.gz | dat_to_fna > EmVec.fna |
| 273 |
|
| 274 |
if [[ "$CHANGE_HEADER" == "1" ]]; then |
| 275 |
sed -i "s/^>/>taxid|$CONTAMINANT_TAXID /" UniVec.fna |
| 276 |
sed -i "s/^>/>taxid|$CONTAMINANT_TAXID /" EmVec.fna |
| 277 |
else |
| 278 |
cat UniVec.fna | map_headers_to_taxid $CONTAMINANT_TAXID |
| 279 |
cat EmVec.fna | map_headers_to_taxid $CONTAMINANT_TAXID |
| 280 |
fi |
| 281 |
|
| 282 |
#sed ':a $!N; s/^>.*\n>/>/; P; D' Contaminants/emvec.fa > Contaminants/emvec.fa |
| 283 |
rm emvec.dat.gz |
| 284 |
|
| 285 |
cd - > /dev/null |
| 286 |
exit 0; |
| 287 |
fi |
| 288 |
fi |
| 289 |
|
| 290 |
|
| 291 |
|
| 292 |
#### REFSEQ/GENBANK DOWNLOAD |
| 293 |
|
| 294 |
export LIBDIR="$BASE_DIR" |
| 295 |
export DO_DUST="$DO_DUST" |
| 296 |
export CHANGE_HEADER="$CHANGE_HEADER" |
| 297 |
|
| 298 |
## Fields in the assembly_summary.txt file |
| 299 |
REFSEQ_CAT_FIELD=5 |
| 300 |
TAXID_FIELD=6 |
| 301 |
SPECIES_TAXID_FIELD=7 |
| 302 |
VERSION_STATUS_FIELD=11 |
| 303 |
ASSEMBLY_LEVEL_FIELD=12 |
| 304 |
FTP_PATH_FIELD=20 |
| 305 |
FTP_PATH_FIELD2=21 ## Needed for wrongly formatted virus files - hopefully just a temporary fix |
| 306 |
|
| 307 |
AWK_QUERY="\$$VERSION_STATUS_FIELD==\"latest\"" |
| 308 |
[[ "$ASSEMBLY_LEVEL" != "Any" ]] && AWK_QUERY="$AWK_QUERY && \$$ASSEMBLY_LEVEL_FIELD==\"$ASSEMBLY_LEVEL\"" |
| 309 |
[[ "$REFSEQ_CATEGORY" != "" ]] && AWK_QUERY="$AWK_QUERY && \$$REFSEQ_CAT_FIELD==\"$REFSEQ_CATEGORY\"" |
| 310 |
|
| 311 |
TAXID=${TAXID//,/|} |
| 312 |
[[ "$TAXID" != "" ]] && AWK_QUERY="$AWK_QUERY && match(\$$TAXID_FIELD,\"^($TAXID)\$\")" |
| 313 |
|
| 314 |
#echo "$AWK_QUERY" >&2 |
| 315 |
|
| 316 |
#echo "Downloading genomes for $DOMAINS at assembly level $ASSEMBLY_LEVEL" >&2 |
| 317 |
if exists wget; then |
| 318 |
wget -qO- --no-remove-listing https://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE/ > /dev/null |
| 319 |
else |
| 320 |
curl --disable-epsv -s https://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE/ > .listing |
| 321 |
fi |
| 322 |
|
| 323 |
if [[ "$CHANGE_HEADER" == "1" ]]; then |
| 324 |
echo "Modifying header to include taxonomy ID" >&2 |
| 325 |
fi |
| 326 |
|
| 327 |
|
| 328 |
for DOMAIN in $DOMAINS; do |
| 329 |
if [[ -s .listing ]]; then |
| 330 |
if [[ ! `grep "^d.* $DOMAIN" .listing` ]]; then |
| 331 |
c_echo "$DOMAIN is not a valid domain - use one of the following:" >&2 |
| 332 |
grep '^d' .listing | sed 's/.* //' | sed 's/^/ /' 1>&2 |
| 333 |
exit 1 |
| 334 |
fi |
| 335 |
fi |
| 336 |
|
| 337 |
#if [[ "$CHANGE_HEADER" != "1" ]]; then |
| 338 |
#echo "Writing taxonomy ID to sequence ID map to STDOUT" >&2 |
| 339 |
#[[ -s "$LIBDIR/$DOMAIN.map" ]] && rm "$LIBDIR/$DOMAIN.map" |
| 340 |
#fi |
| 341 |
|
| 342 |
export DOMAIN=$DOMAIN |
| 343 |
check_or_mkdir_no_fail $LIBDIR/$DOMAIN |
| 344 |
|
| 345 |
FULL_ASSEMBLY_SUMMARY_FILE="$LIBDIR/$DOMAIN/assembly_summary.txt" |
| 346 |
ASSEMBLY_SUMMARY_FILE="$LIBDIR/$DOMAIN/assembly_summary_filtered.txt" |
| 347 |
|
| 348 |
echo "Downloading ftp://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE/$DOMAIN/assembly_summary.txt ..." >&2 |
| 349 |
if [ $DL_MODE = "rsync" ]; then |
| 350 |
$DL_CMD rsync://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE/$DOMAIN/assembly_summary.txt "$FULL_ASSEMBLY_SUMMARY_FILE" || { |
| 351 |
c_echo "rsync Download failed! Have a look at valid domains at ftp://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE ." >&2 |
| 352 |
exit 1; |
| 353 |
} |
| 354 |
else |
| 355 |
$DL_CMD "$FULL_ASSEMBLY_SUMMARY_FILE" https://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE/$DOMAIN/assembly_summary.txt > "$FULL_ASSEMBLY_SUMMARY_FILE" || { |
| 356 |
c_echo "Download failed! Have a look at valid domains at ftp://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE ." >&2 |
| 357 |
exit 1; |
| 358 |
} |
| 359 |
fi |
| 360 |
|
| 361 |
awk -F "\t" "BEGIN {OFS=\"\t\"} $AWK_QUERY" "$FULL_ASSEMBLY_SUMMARY_FILE" > "$ASSEMBLY_SUMMARY_FILE" |
| 362 |
|
| 363 |
N_EXPECTED=`cat "$ASSEMBLY_SUMMARY_FILE" | wc -l` |
| 364 |
[[ $N_EXPECTED -gt 0 ]] || { echo "Domain $DOMAIN has no genomes with specified filter." >&2; exit 1; } |
| 365 |
echo "Downloading $N_EXPECTED $DOMAIN genomes at assembly level $ASSEMBLY_LEVEL ... (will take a while)" >&2 |
| 366 |
|
| 367 |
if [[ "$DOMAIN" != "" ]]; then |
| 368 |
## Wrong columns in viral assembly summary files - the path is sometimes in field 20, sometimes 21 |
| 369 |
#cut -f "$TAXID_FIELD,$FTP_PATH_FIELD,$FTP_PATH_FIELD2" "$ASSEMBLY_SUMMARY_FILE" | \ |
| 370 |
# sed 's/^\(.*\)\t\(ftp:.*\)\t.*/\1\t\2/;s/^\(.*\)\t.*\t\(ftp:.*\)/\1\t\2/' | \ |
| 371 |
#sed 's#\([^/]*\)$#\1/\1_genomic.fna.gz#' |\ |
| 372 |
# tr '\n' '\0' | xargs -0 -n1 -P $N_PROC bash -c 'download_n_process_nofail "$@"' _ | count $N_EXPECTED |
| 373 |
cut -f "$TAXID_FIELD,$FTP_PATH_FIELD,$FTP_PATH_FIELD2" "$ASSEMBLY_SUMMARY_FILE" | \ |
| 374 |
awk -F "\t" '{if ($2~/ftp/) print $1"\t"$2; if ($3~/ftp/) print $1"\t"$3}' | \ |
| 375 |
sed 's#\([^/]*\)$#\1/\1_genomic.fna.gz#' |\ |
| 376 |
tr '\n' '\0' | xargs -0 -n1 -P $N_PROC bash -c 'download_n_process_nofail "$@"' _ | count $N_EXPECTED |
| 377 |
|
| 378 |
else |
| 379 |
cut -f "$TAXID_FIELD,$FTP_PATH_FIELD" "$ASSEMBLY_SUMMARY_FILE" | sed 's#\([^/]*\)$#\1/\1_genomic.fna.gz#' |\ |
| 380 |
tr '\n' '\0' | xargs -0 -n1 -P $N_PROC bash -c 'download_n_process_nofail "$@"' _ | count $N_EXPECTED |
| 381 |
fi |
| 382 |
echo >&2 |
| 383 |
|
| 384 |
|
| 385 |
|
| 386 |
if [[ "$DOWNLOAD_RNA" == "1" && ! `echo $DOMAIN | egrep 'bacteria|viral|archaea'` ]]; then |
| 387 |
echo "Downloadinging rna sequence files" >&2 |
| 388 |
cut -f $TAXID_FIELD,$FTP_PATH_FIELD "$ASSEMBLY_SUMMARY_FILE"| sed 's#\([^/]*\)$#\1/\1_rna.fna.gz#' |\ |
| 389 |
tr '\n' '\0' | xargs -0 -n1 -P $N_PROC bash -c 'download_n_process_nofail "$@"' _ | count $N_EXPECTED |
| 390 |
echo >&2 |
| 391 |
fi |
| 392 |
done |