Debian Codegraph @

centrifuge-download

raw
1 #!/bin/bash
2
3 set -eu -o pipefail
4
5 exists() {
6 command -v "$1" >/dev/null 2>&1
7 }
8
9 cut_after_first_space_or_second_pipe() {
10 grep '^>' | sed 's/ .*//' | sed 's/\([^|]*|[^|]*\).*/\1/'
11 }
12 export -f cut_after_first_space_or_second_pipe
13
14 map_headers_to_taxid() {
15 grep '^>' | cut_after_first_space_or_second_pipe | sed -e "s/^>//" -e "s/\$/ $1/"
16 }
17 export -f map_headers_to_taxid
18
19
20
21 #########################################################
22 ## Functions
23
24 function download_n_process() {
25 IFS=$'\t' read -r TAXID FILEPATH <<< "$1"
26
27 NAME=`basename $FILEPATH .gz`
28 GZIPPED_FILE="$LIBDIR/$DOMAIN/$NAME.gz"
29 UNZIPPED_FILE="$LIBDIR/$DOMAIN/$NAME"
30 DUSTMASKED_FILE="$LIBDIR/$DOMAIN/${NAME%.fna}_dustmasked.fna"
31 [[ "$DO_DUST" == "1" ]] && RES_FILE=$DUSTMASKED_FILE || RES_FILE=$UNZIPPED_FILE
32
33 if [[ ! -s "$RES_FILE" ]]; then
34 if [ $DL_MODE = "rsync" ]; then
35 FILEPATH=${FILEPATH/ftp/rsync}
36 $DL_CMD "$FILEPATH" "$LIBDIR/$DOMAIN/$NAME.gz" || \
37 { printf "\nError downloading $FILEPATH!\n" >&2 && exit 1; }
38 else
39 $DL_CMD "$LIBDIR/$DOMAIN/$NAME.gz" "$FILEPATH" || \
40 $DL_CMD "$LIBDIR/$DOMAIN/$NAME.gz" "$FILEPATH" || \
41 $DL_CMD "$LIBDIR/$DOMAIN/$NAME.gz" "$FILEPATH" || \
42 { printf "\nError downloading $FILEPATH!\n" >&2 && exit 1; }
43 fi
44
45 [[ -s "$LIBDIR/$DOMAIN/$NAME.gz" ]] || return;
46 gunzip -f "$LIBDIR/$DOMAIN/$NAME.gz" ||{ printf "\nError gunzipping $LIBDIR/$DOMAIN/$NAME.gz [ downloaded from $FILEPATH ]!\n" >&2 && exit 255; }
47
48
49 if [[ "$CHANGE_HEADER" == "1" ]]; then
50 sed -i "s/^>/>kraken:taxid|$TAXID /" $LIBDIR/$DOMAIN/$NAME
51 fi
52
53 if [[ "$FILTER_UNPLACED" == "1" ]]; then
54 echo TODO 2>&1
55 ##sed -n '1,/^>.*unplaced/p; /'
56 fi
57
58 if [[ "$DO_DUST" == "1" ]]; then
59 ## TODO: Consider hard-masking only low-complexity stretches with 10 or more bps
60 dustmasker -infmt fasta -in $LIBDIR/$DOMAIN/$NAME -level 20 -outfmt fasta | sed '/^>/! s/[^AGCT]/N/g' > $DUSTMASKED_FILE
61 rm $LIBDIR/$DOMAIN/$NAME
62 fi
63 fi
64
65 ## Output sequenceID to taxonomy ID map to STDOUT
66 cat $RES_FILE | map_headers_to_taxid $TAXID
67 echo done
68 }
69 export -f download_n_process
70
71
72
73 function download_n_process_nofail() {
74 download_n_process "$@" || true
75 }
76 export -f download_n_process_nofail
77
78
79 ceol=`tput el || echo -n ""` # terminfo clr_eol
80
81 function count {
82 typeset C=0
83 while read L; do
84 if [[ "$L" == "done" ]]; then
85 [[ "$VERBOSE" == 1 ]] && continue;
86 C=$(( C + 1 ))
87 _progress=$(( (${C}*100/${1}*100)/100 ))
88 _done=$(( (${_progress}*4)/10 ))
89 _left=$(( 40-$_done ))
90 # Build progressbar string lengths
91 _done=$(printf "%${_done}s")
92 _left=$(printf "%${_left}s")
93
94 printf "\rProgress : [${_done// /#}${_left// /-}] ${_progress}%% $C/$1" 1>&2
95 else
96 echo "$L"
97 fi
98 done
99 }
100
101 function check_or_mkdir_no_fail {
102 #echo -n "Creating $1 ... " >&2
103 if [[ -d $1 && ! -n `find $1 -prune -empty -type d` ]]; then
104 echo "Directory $1 exists. Continuing" >&2
105 return `true`
106 else
107 #echo "Done" >&2
108 mkdir -p $1
109 return `true`
110 fi
111 }
112
113 function c_echo() {
114 printf "\033[34m$*\033[0m\n"
115 }
116
117
118
119 ## Check if GNU parallel exists
120 command -v parallel >/dev/null 2>&1 && PARALLEL=1 || PARALLEL=0
121
122
123 ALL_GENOMES="bacteria viral archaea fungi protozoa invertebrate plant vertebrate_mammalian vertebrate_other"
124 ALL_DATABASES="refseq genbank taxonomy contaminants"
125 ALL_ASSEMBLY_LEVELS="Complete\ Genome Chromosome Scaffold Contig"
126
127 ## Option parsing
128 DATABASE="refseq"
129 ASSEMBLY_LEVEL="Complete Genome"
130 REFSEQ_CATEGORY=""
131 TAXID=""
132
133
134 DL_PROG="NA"
135 if hash curl 2>/dev/null; then
136 DL_PROG="curl"
137 elif hash wget 2>/dev/null; then
138 DL_PROG="wget"
139 elif hash rsync 2>/dev/null; then
140 DL_PROG="rsync"
141 fi
142
143
144 BASE_DIR="."
145 N_PROC=1
146 CHANGE_HEADER=0
147 DOWNLOAD_RNA=0
148 DO_DUST=0
149 FILTER_UNPLACED=0
150 VERBOSE=0
151
152 USAGE="
153 `basename $0` [<options>] <database>
154
155 ARGUMENT
156 <database> One of refseq, genbank, contaminants or taxonomy:
157 - use refseq or genbank for genomic sequences,
158 - contaminants gets contaminant sequences from UniVec and EmVec,
159 - taxonomy for taxonomy mappings.
160
161 COMMON OPTIONS
162 -o <directory> Folder to which the files are downloaded. Default: '$BASE_DIR'.
163 -P <# of threads> Number of processes when downloading (uses xargs). Default: '$N_PROC'
164
165 WHEN USING database refseq OR genbank:
166 -d <domain> What domain to download. One or more of ${ALL_GENOMES// /, } (comma separated).
167 -a <assembly level> Only download genomes with the specified assembly level. Default: '$ASSEMBLY_LEVEL'. Use 'Any' for any assembly level.
168 -c <refseq category> Only download genomes in the specified refseq category. Default: any.
169 -t <taxids> Only download the specified taxonomy IDs, comma separated. Default: any.
170 -g <program> Download using program. Options: rsync, curl, wget. Default $DL_PROG (auto-detected).
171 -r Download RNA sequences, too.
172 -u Filter unplaced sequences.
173 -m Mask low-complexity regions using dustmasker. Default: off.
174 -l Modify header to include taxonomy ID. Default: off.
175 -v Verbose mode
176 "
177
178 # arguments: $OPTFIND (current index), $OPTARG (argument for option), $OPTERR (bash-specific)
179 while getopts "o:P:d:a:c:t:g:urlmv" OPT "$@"; do
180 case $OPT in
181 o) BASE_DIR="$OPTARG" ;;
182 P) N_PROC="$OPTARG" ;;
183 d) DOMAINS=${OPTARG//,/ } ;;
184 a) ASSEMBLY_LEVEL="$OPTARG" ;;
185 c) REFSEQ_CATEGORY="$OPTARG" ;;
186 g) DL_PROG="$OPTARG" ;;
187 t) TAXID="$OPTARG" ;;
188 r) DOWNLOAD_RNA=1 ;;
189 u) FILTER_UNPLACED=1 ;;
190 m) DO_DUST=1 ;;
191 v) VERBOSE=1 ;;
192 l) CHANGE_HEADER=1 ;;
193 \?) echo "Invalid option: -$OPTARG" >&2
194 exit 1
195 ;;
196 :) echo "Option -$OPTARG requires an argument." >&2
197 exit 1
198 ;;
199 esac
200 done
201 shift $((OPTIND-1))
202
203 [[ $# -eq 1 ]] || { printf "$USAGE" >&2 && exit 1; };
204 DATABASE=$1
205
206
207 if [[ "$DL_PROG" == "rsync" ]]; then
208 DL_CMD="rsync --no-motd"
209 DL_MODE="rsync"
210 elif [[ "$DL_PROG" == "wget" ]]; then
211 DL_CMD="wget -N --reject=index.html -qO"
212 DL_MODE="https"
213 elif [[ "$DL_PROG" == "curl" ]]; then
214 DL_CMD="curl -s -o"
215 DL_MODE="https"
216 else
217 echo "Unknown download program - please install one of rsync, wget or curl, and specify it with the -g option" >&2
218 exit 1
219 fi
220
221 cecho() {
222 echo $* 1>&2
223 $*
224 }
225
226 if [[ "$VERBOSE" == "1" ]]; then
227 export -f cecho
228 DL_CMD="cecho $DL_CMD"
229 fi
230
231 export DL_CMD DL_MODE VERBOSE
232
233 #### TAXONOMY DOWNLOAD
234 FTP="https://ftp.ncbi.nih.gov"
235 if [[ "$DATABASE" == "taxonomy" ]]; then
236 echo "Downloading NCBI taxonomy ... " >&2
237 if check_or_mkdir_no_fail "$BASE_DIR"; then
238 cd "$BASE_DIR" > /dev/null
239 if [ $DL_MODE = "rsync" ]; then
240 $DL_CMD ${FTP/ftp/rsync}/pub/taxonomy/taxdump.tar.gz taxdump.tar.gz
241 else
242 $DL_CMD taxdump.tar.gz $FTP/pub/taxonomy/taxdump.tar.gz
243 fi
244 tar -zxvf taxdump.tar.gz nodes.dmp
245 tar -zxvf taxdump.tar.gz names.dmp
246 rm taxdump.tar.gz
247 cd - > /dev/null
248 fi
249 exit 0
250 fi
251
252 dat_to_fna() {
253 grep -E '^DE|^ ' | awk '/^DE/ { sub(/DE */,">"); gsub(/[ |]/,"_") }; { print }' | awk '/^ / { gsub(/ /,""); sub(/[0-9]*$/,"") }; { print }'
254 }
255
256 #### CONTAMINANT SEQ DOWNLOAD
257 if [[ "$DATABASE" == "contaminants" ]]; then
258 echo "Downloading contaminant databases ... " >&2
259 CONTAMINANT_TAXID=32630
260 CONTAMINANT_DIR="$BASE_DIR/contaminants"
261 if check_or_mkdir_no_fail "$CONTAMINANT_DIR"; then
262 cd "$CONTAMINANT_DIR" > /dev/null
263
264 # download UniVec and EmVec database
265 if [ $DL_MODE = "rsync" ]; then
266 $DL_CMD rsync://ftp.ncbi.nlm.nih.gov/pub/UniVec/UniVec UniVec.fna
267 $DL_CMD rsync://ftp.ebi.ac.uk/pub/databases/emvec/emvec.dat.gz emvec.dat.gz
268 else
269 $DL_CMD UniVec.fna https://ftp.ncbi.nlm.nih.gov/pub/UniVec/UniVec
270 $DL_CMD emvec.dat.gz https://ftp.ebi.ac.uk/pub/databases/emvec/emvec.dat.gz
271 fi
272 gunzip -c emvec.dat.gz | dat_to_fna > EmVec.fna
273
274 if [[ "$CHANGE_HEADER" == "1" ]]; then
275 sed -i "s/^>/>taxid|$CONTAMINANT_TAXID /" UniVec.fna
276 sed -i "s/^>/>taxid|$CONTAMINANT_TAXID /" EmVec.fna
277 else
278 cat UniVec.fna | map_headers_to_taxid $CONTAMINANT_TAXID
279 cat EmVec.fna | map_headers_to_taxid $CONTAMINANT_TAXID
280 fi
281
282 #sed ':a $!N; s/^>.*\n>/>/; P; D' Contaminants/emvec.fa > Contaminants/emvec.fa
283 rm emvec.dat.gz
284
285 cd - > /dev/null
286 exit 0;
287 fi
288 fi
289
290
291
292 #### REFSEQ/GENBANK DOWNLOAD
293
294 export LIBDIR="$BASE_DIR"
295 export DO_DUST="$DO_DUST"
296 export CHANGE_HEADER="$CHANGE_HEADER"
297
298 ## Fields in the assembly_summary.txt file
299 REFSEQ_CAT_FIELD=5
300 TAXID_FIELD=6
301 SPECIES_TAXID_FIELD=7
302 VERSION_STATUS_FIELD=11
303 ASSEMBLY_LEVEL_FIELD=12
304 FTP_PATH_FIELD=20
305 FTP_PATH_FIELD2=21 ## Needed for wrongly formatted virus files - hopefully just a temporary fix
306
307 AWK_QUERY="\$$VERSION_STATUS_FIELD==\"latest\""
308 [[ "$ASSEMBLY_LEVEL" != "Any" ]] && AWK_QUERY="$AWK_QUERY && \$$ASSEMBLY_LEVEL_FIELD==\"$ASSEMBLY_LEVEL\""
309 [[ "$REFSEQ_CATEGORY" != "" ]] && AWK_QUERY="$AWK_QUERY && \$$REFSEQ_CAT_FIELD==\"$REFSEQ_CATEGORY\""
310
311 TAXID=${TAXID//,/|}
312 [[ "$TAXID" != "" ]] && AWK_QUERY="$AWK_QUERY && match(\$$TAXID_FIELD,\"^($TAXID)\$\")"
313
314 #echo "$AWK_QUERY" >&2
315
316 #echo "Downloading genomes for $DOMAINS at assembly level $ASSEMBLY_LEVEL" >&2
317 if exists wget; then
318 wget -qO- --no-remove-listing https://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE/ > /dev/null
319 else
320 curl --disable-epsv -s https://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE/ > .listing
321 fi
322
323 if [[ "$CHANGE_HEADER" == "1" ]]; then
324 echo "Modifying header to include taxonomy ID" >&2
325 fi
326
327
328 for DOMAIN in $DOMAINS; do
329 if [[ -s .listing ]]; then
330 if [[ ! `grep "^d.* $DOMAIN" .listing` ]]; then
331 c_echo "$DOMAIN is not a valid domain - use one of the following:" >&2
332 grep '^d' .listing | sed 's/.* //' | sed 's/^/ /' 1>&2
333 exit 1
334 fi
335 fi
336
337 #if [[ "$CHANGE_HEADER" != "1" ]]; then
338 #echo "Writing taxonomy ID to sequence ID map to STDOUT" >&2
339 #[[ -s "$LIBDIR/$DOMAIN.map" ]] && rm "$LIBDIR/$DOMAIN.map"
340 #fi
341
342 export DOMAIN=$DOMAIN
343 check_or_mkdir_no_fail $LIBDIR/$DOMAIN
344
345 FULL_ASSEMBLY_SUMMARY_FILE="$LIBDIR/$DOMAIN/assembly_summary.txt"
346 ASSEMBLY_SUMMARY_FILE="$LIBDIR/$DOMAIN/assembly_summary_filtered.txt"
347
348 echo "Downloading ftp://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE/$DOMAIN/assembly_summary.txt ..." >&2
349 if [ $DL_MODE = "rsync" ]; then
350 $DL_CMD rsync://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE/$DOMAIN/assembly_summary.txt "$FULL_ASSEMBLY_SUMMARY_FILE" || {
351 c_echo "rsync Download failed! Have a look at valid domains at ftp://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE ." >&2
352 exit 1;
353 }
354 else
355 $DL_CMD "$FULL_ASSEMBLY_SUMMARY_FILE" https://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE/$DOMAIN/assembly_summary.txt > "$FULL_ASSEMBLY_SUMMARY_FILE" || {
356 c_echo "Download failed! Have a look at valid domains at ftp://ftp.ncbi.nlm.nih.gov/genomes/$DATABASE ." >&2
357 exit 1;
358 }
359 fi
360
361 awk -F "\t" "BEGIN {OFS=\"\t\"} $AWK_QUERY" "$FULL_ASSEMBLY_SUMMARY_FILE" > "$ASSEMBLY_SUMMARY_FILE"
362
363 N_EXPECTED=`cat "$ASSEMBLY_SUMMARY_FILE" | wc -l`
364 [[ $N_EXPECTED -gt 0 ]] || { echo "Domain $DOMAIN has no genomes with specified filter." >&2; exit 1; }
365 echo "Downloading $N_EXPECTED $DOMAIN genomes at assembly level $ASSEMBLY_LEVEL ... (will take a while)" >&2
366
367 if [[ "$DOMAIN" != "" ]]; then
368 ## Wrong columns in viral assembly summary files - the path is sometimes in field 20, sometimes 21
369 #cut -f "$TAXID_FIELD,$FTP_PATH_FIELD,$FTP_PATH_FIELD2" "$ASSEMBLY_SUMMARY_FILE" | \
370 # sed 's/^\(.*\)\t\(ftp:.*\)\t.*/\1\t\2/;s/^\(.*\)\t.*\t\(ftp:.*\)/\1\t\2/' | \
371 #sed 's#\([^/]*\)$#\1/\1_genomic.fna.gz#' |\
372 # tr '\n' '\0' | xargs -0 -n1 -P $N_PROC bash -c 'download_n_process_nofail "$@"' _ | count $N_EXPECTED
373 cut -f "$TAXID_FIELD,$FTP_PATH_FIELD,$FTP_PATH_FIELD2" "$ASSEMBLY_SUMMARY_FILE" | \
374 awk -F "\t" '{if ($2~/ftp/) print $1"\t"$2; if ($3~/ftp/) print $1"\t"$3}' | \
375 sed 's#\([^/]*\)$#\1/\1_genomic.fna.gz#' |\
376 tr '\n' '\0' | xargs -0 -n1 -P $N_PROC bash -c 'download_n_process_nofail "$@"' _ | count $N_EXPECTED
377
378 else
379 cut -f "$TAXID_FIELD,$FTP_PATH_FIELD" "$ASSEMBLY_SUMMARY_FILE" | sed 's#\([^/]*\)$#\1/\1_genomic.fna.gz#' |\
380 tr '\n' '\0' | xargs -0 -n1 -P $N_PROC bash -c 'download_n_process_nofail "$@"' _ | count $N_EXPECTED
381 fi
382 echo >&2
383
384
385
386 if [[ "$DOWNLOAD_RNA" == "1" && ! `echo $DOMAIN | egrep 'bacteria|viral|archaea'` ]]; then
387 echo "Downloadinging rna sequence files" >&2
388 cut -f $TAXID_FIELD,$FTP_PATH_FIELD "$ASSEMBLY_SUMMARY_FILE"| sed 's#\([^/]*\)$#\1/\1_rna.fna.gz#' |\
389 tr '\n' '\0' | xargs -0 -n1 -P $N_PROC bash -c 'download_n_process_nofail "$@"' _ | count $N_EXPECTED
390 echo >&2
391 fi
392 done