Debian Codegraph @

src/main/java/org/broadinstitute/hellbender/utils/bwa/BwaMemIndex.java

raw ◀ ▶
1 Debian patch touch this file
1 package org.broadinstitute.hellbender.utils.bwa;
2
3 import java.io.*;
4 import java.nio.ByteBuffer;
5 import java.nio.ByteOrder;
6 import java.nio.file.Files;
7 import java.nio.file.StandardCopyOption;
8 import java.util.*;
9 import java.util.concurrent.atomic.AtomicInteger;
10 import java.util.stream.Collectors;
11
12 /**
13 * BwaMemIndex manages the mapping of a bwa index image file into (non-Java) memory.
14 * It's typically a huge chunk of memory, so you need to manage it as a precious resource.
15 *
16 * Usage pattern is:
17 * Create a BwaMemIndex for some reference.
18 * Create Aligners as needed to do some aligning -- they're pretty lightweight and thread safe.
19 * (But you may need to manage memory by controlling the number of reads that you align in one chunk.)
20 * Close the BwaMemIndex when you're done aligning.
21 *
22 * This class doesn't know anything about Spark. You can use it in a distributed setting if you distribute the index
23 * file to each node using the Yarn --files mechanism. You might find it convenient to manage a singleton instance
24 * of a BwaMemIndex on each Java VM when you're running distributed: check out BwaMemIndexSingleton in GATK.
25 *
26 * Alternatively, you could use this class directly to run bwa multi-threaded on a single machine,
27 * if that's what you want to do.
28 */
29 public final class BwaMemIndex implements AutoCloseable {
30
31 private static final int MAXIMUM_NUMBER_OF_CHARACTER_BEFORE_FIRST_HEADER_IN_FASTA_FILES = 4092;
32 private static final char FASTA_HEADER_PREFIX_CHAR = '>';
33
34 public static final List<String> INDEX_FILE_EXTENSIONS =
35 Collections.unmodifiableList(Arrays.asList(
36 ".amb", ".ann", ".bwt", ".pac", ".sa" ));
37
38 public static final String IMAGE_FILE_EXTENSION = ".img";
39
40 public static final List<String> FASTA_FILE_EXTENSIONS =
41 Collections.unmodifiableList(Arrays.asList(".fasta", ".fa"));
42
43 /**
44 * Indexing algorithms supported by Bwa.
45 */
46 public enum Algorithm {
47
48 /**
49 * Chooses the most appropriate algorithm based on characteristic
50 * of the reference. For references shorter than 2Gbases it will use {@link #IS}
51 * whereas for larger reference it will employ {@link #RB2}.
52 */
53 AUTO,
54
55 /**
56 * Linear-time algorithm for constructing suffix array.
57 * It requires 5.37N memory where N is the size of the database.
58 * IS is moderately fast, but does not work with database larger than 2GB.
59 * IS is the default algorithm due to its simplicity. The current codes for
60 * IS algorithm are reimplemented by Yuta Mori.
61 *
62 * @see "http://bio-bwa.sourceforge.net/bwa.shtml"
63 */
64 IS,
65
66 /**
67 * The <i>Ropebwt2</i> algorithm.
68 *
69 * @see "https://arxiv.org/pdf/1406.0426.pdf"
70 */
71 RB2;
72
73 /**
74 * The string name use by the Bwa command line to denote that algorithm.
75 * @return never {@code null} and unique across algorithms.
76 */
77 public String toBwaName() {
78 return this.toString().toLowerCase();
79 }
80 }
81
82 private final String indexImageFile; // stash this for error messages
83 private volatile long indexAddress; // address where the index was memory-mapped (for use by C code)
84 private final AtomicInteger refCount; // keep track of how many threads are actively aligning
85 private final List<String> refContigNames; // the reference dictionary from the index
86 private static volatile boolean nativeLibLoaded = false; // whether we've loaded the native library or not
87
88 private static String resolveFastaFileExtension(final String fasta) {
89 final Optional<String> extension = FASTA_FILE_EXTENSIONS.stream()
90 .filter(fasta::endsWith).findFirst();
91 if (!extension.isPresent()) {
92 throw new IllegalArgumentException(
93 String.format("the fasta file provided '%s' does not have any of the standard fasta extensions: %s",
94 fasta, FASTA_FILE_EXTENSIONS.stream().collect(Collectors.joining(", "))));
95 } else {
96 return extension.get();
97 }
98 }
99
100 /**
101 * Create the index image file for a complete set of BWA index files
102 * @param indexPrefix the location of the index files.
103 * @param imageFile the location of the new index image file.
104 *
105 * <p>
106 * <b><i>WARNING!</i></b>: Notice that currently this method is making JNI call that might result in an abrupt process
107 * interruption (e.g. exit or abort system call) and so the control may never be returned.
108 * </p>
109 *
110 * @throws IllegalArgumentException if {@code indexPrefix} is {@code null}
111 * or it does not look like it points to a complete set of index files.
112 * @throws IllegalArgumentException if {@code imageFile} is {@code null}.
113 */
114 public static void createIndexImageFromIndexFiles(final String indexPrefix, final String imageFile) {
115 if (indexPrefix == null) {
116 throw new IllegalArgumentException("the index prefix cannot be null");
117 } else if (imageFile == null) {
118 throw new IllegalArgumentException("the image file cannot be null");
119 }
120 assertLooksLikeIndexPrefix(indexPrefix);
121 loadNativeLibrary();
122 createIndexImageFile(indexPrefix, imageFile);
123 }
124
125 /**
126 * Create the index image file for a fasta file.
127 * <p>The name of the output index file will be determined by the name of the fasta file using
128 * {@link #getDefaultIndexImageNameFromFastaFile(String)}.
129 *
130 * <p>
131 * This is equivalent to calling
132 * {@code {@link #createIndexImageFromFastaFile(String,String) createIndexImageFromFastaFile(X, getDefaultIndexImageNameFromFastaFile(X)))}}
133 * </p>
134 * <p>
135 * <b><i>WARNING!</i></b>: Notice that currently this method is making JNI call that might result in an abrupt process
136 * interruption (e.g. exit or abort system call) and so the control may never be returned.
137 * </p>
138 *
139 * @param fasta the location of the fasta reference file.
140 * @return location of the generated index image file (equivaled to {@link #getDefaultIndexImageNameFromFastaFile(String)}).
141 * @throws IllegalArgumentException if {@code fasta} is {@code null} or
142 * does not finish with out of the standard fasta extension names (listed in {@link #FASTA_FILE_EXTENSIONS}).
143 * @throws InvalidFileFormatException if {@code fasta} does not seem to be a fasta formatted regular readable file.
144 * @throws CouldNotCreateIndexImageException if for some reason we could not create the index file.
145 * @see #getDefaultIndexImageNameFromFastaFile(String)
146 */
147 public static String createIndexImageFromFastaFile(final String fasta) {
148 final String imageFile = getDefaultIndexImageNameFromFastaFile(fasta);
149 createIndexImageFromFastaFile(fasta, imageFile);
150 return imageFile;
151 }
152
153 /**
154 * Gets the default index image name for the provided FASTA.
155 * <p>The default name is substitutes its extension (typically <i>.fasta</i> or <i>.fa</i>)
156 * by {@link #IMAGE_FILE_EXTENSION} (<i>.img</i>).</p>
157 *
158 *
159 * @param fasta the location of the fasta reference file.
160 * @throws IllegalArgumentException if {@code fasta} is {@code null} or
161 * does not finish with out of the standard fasta extension names (listed in {@link #FASTA_FILE_EXTENSIONS}).
162 */
163 public static String getDefaultIndexImageNameFromFastaFile(final String fasta) {
164 if (fasta == null) {
165 throw new IllegalArgumentException("the input fasta file name cannot be null");
166 }
167 final String extension = resolveFastaFileExtension(fasta);
168 final String prefix = fasta.substring(0, fasta.length() - extension.length());
169 return prefix + IMAGE_FILE_EXTENSION;
170 }
171
172 /**
173 * Creates the index image file for a reference fasta file.
174 * <p>
175 * The index will be created using the default algorithm {@link Algorithm#AUTO}.
176 * </p>
177 * <p>
178 * Calling this method is equivalent to calling {@link #createIndexImageFromFastaFile(String, String, Algorithm) createIndexImageFromFastaFile(a, b, Algorithm.AUTO)}
179 * <p>
180 * <b><i>WARNING!</i></b>: Notice that currently this method is making JNI call that might result in an abrupt process
181 * interruption (e.g. exit or abort system call) and so the control may never be returned.
182 * </p>
183 * @param fasta the location of the targeted reference.
184 * @param imageFile the location of the new index image file.
185 * @throws IllegalArgumentException if {@code fasta} is {@code null}
186 * or it does not look like it points to a fasta formatted readable file.
187 * @throws IllegalArgumentException if {@code imageFile} is {@code null}.
188 * @throws InvalidFileFormatException if {@code fasta} does not seem to be
189 * a fasta formatted regular and readable file.
190 * @throws CouldNotCreateIndexImageException if there was a problem creating
191 * the output image.
192 * @throws CouldNotCreateIndexException if there was some problem while creating
193 * the intermediary index file set.
194 */
195 public static void createIndexImageFromFastaFile(final String fasta, final String imageFile) {
196 createIndexImageFromFastaFile(fasta, imageFile, Algorithm.AUTO);
197 }
198
199 /**
200 * Creates the index image file for a reference fasta file.
201 * <p>
202 * <b><i>WARNING!</i></b>: Notice that currently this method is making JNI call that might result in an abrupt process
203 * interruption (e.g. exit or abort system call) and so the control may never be returned.
204 * </p>
205 * @param fasta the location of the targeted reference.
206 * @param imageFile the location of the new index image file.
207 * @param algo the algorithm to use to construct the index (see {@link Algorithm} to see what there is available.).
208 * @throws IllegalArgumentException if {@code fasta} is {@code null}
209 * or it does not look like it points to a fasta formatted readable file.
210 * @throws IllegalArgumentException if {@code imageFile} is {@code null}.
211 * @throws InvalidFileFormatException if {@code fasta} does not seem to be
212 * a fasta formatted regular and readable file.
213 * @throws CouldNotCreateIndexImageException if there was a problem creating
214 * the output image.
215 * @throws CouldNotCreateIndexException if there was some problem while creating
216 * the intermediary index file set.
217 */
218 public static void createIndexImageFromFastaFile( final String fasta, final String imageFile, final Algorithm algo) {
219 assertLooksLikeFastaFile(fasta);
220 assertCanCreateOrOverwriteImageFile(imageFile);
221 if (algo == null) {
222 throw new IllegalArgumentException("the input algorithm must not be null");
223 }
224
225 final File indexPrefix = createTempIndexPrefix(fasta);
226 loadNativeLibrary();
227 createReferenceIndex(fasta, indexPrefix.getPath(), algo.toBwaName());
228 createIndexImageFile(indexPrefix.getPath(), imageFile);
229 deleteIndexFiles(indexPrefix);
230 }
231
232 private static void assertCanCreateOrOverwriteImageFile(final String imageFile) {
233 if (imageFile == null) {
234 throw new IllegalArgumentException("the image file cannot be null");
235 } else {
236 final File file = new File(imageFile);
237 try {
238 if (!file.createNewFile()) {
239 if (!file.isFile() || !file.canWrite()) {
240 throw new CouldNotCreateIndexImageException(imageFile, "already exists as a non-regular or unwritable file");
241 }
242 } else {
243 file.delete();
244 }
245 } catch (final IOException ex) {
246 throw new CouldNotCreateIndexImageException(imageFile, ex.getMessage(), ex);
247 }
248 }
249 }
250
251 /**
252 * Checks whether the input index prefix seems to point to a complete set
253 * of readable index files.
254 *
255 * @param indexPrefix the target index prefix.
256 * @throws IllegalArgumentException if {@code indexPrefix} is {@code null}.
257 * @throws CouldNotReadIndexException if that is not the case.
258 */
259 private static void assertLooksLikeIndexPrefix(final String indexPrefix) {
260 if (indexPrefix == null) {
261 throw new IllegalArgumentException("the input index prefix cannot be null");
262 }
263 INDEX_FILE_EXTENSIONS.stream()
264 .map(ext -> indexPrefix + ext)
265 .forEach(file -> assertNonEmptyReadableIndexFile(indexPrefix, file));
266 }
267
268 private static void deleteIndexFiles(final File indexPrefix) {
269 INDEX_FILE_EXTENSIONS.stream()
270 .map(ext -> new File(indexPrefix + ext))
271 .forEach(File::delete);
272 indexPrefix.delete();
273 }
274
275 private static File createTempIndexPrefix(final String fasta) {
276 final File indexPrefix;
277 try {
278 indexPrefix = File.createTempFile("temporal-index","");
279 } catch (final IOException ex) {
280 throw new CouldNotCreateIndexException(fasta, "no-location","failure to create a temporal file");
281 }
282 indexPrefix.deleteOnExit();
283 INDEX_FILE_EXTENSIONS.stream()
284 .map(ext -> new File(indexPrefix + ext))
285 .forEach(File::deleteOnExit);
286 return indexPrefix;
287 }
288
289 private static void assertLooksLikeFastaFile(final String fasta) {
290 resolveFastaFileExtension(fasta);
291 if (!nonEmptyReadableFile(fasta)) {
292 throw new CouldNotReadReferenceException(fasta, "input file unreachable or not a file");
293 }
294 try (final BufferedReader reader = new BufferedReader(new FileReader(fasta))) {
295 int c;
296 int offset = 0;
297 while (offset++ < MAXIMUM_NUMBER_OF_CHARACTER_BEFORE_FIRST_HEADER_IN_FASTA_FILES && (c = reader.read()) != -1) {
298 if (!Character.isSpaceChar(c)) {
299 if (c == FASTA_HEADER_PREFIX_CHAR) {
300 break;
301 } else {
302 throw new InvalidFileFormatException(fasta, "the file provided does not seem to be a fasta file (first non-space character in the first 4K is not '" + FASTA_HEADER_PREFIX_CHAR + "'");
303 }
304 }
305 }
306
307 } catch (final IOException ex) {
308 throw new InvalidFileFormatException(fasta, "problems reading the content of the reference fasta file'", ex);
309 }
310 }
311
312 /**
313 * Loads an index from an image file.
314 * <p>
315 * You can use other methods to create such
316 * indexes from fasta reference ({@link #createIndexImageFromFastaFile} or their index files
317 * ({@link #createIndexImageFromIndexFiles}).
318 * </p>
319 * <p>
320 * <b><i>WARNING!</i></b>: Notice that currently this method is making JNI call that might result in an abrupt process
321 * interruption (e.g. exit or abort system call) and so the control may never be returned.
322 * </p>
323 *
324 * @throws IllegalArgumentException if {@code indexImageFile} is {@code null}.
325 * @throws CouldNotReadImageException if some problem occurred when loading the
326 * image file.
327 */
328 public BwaMemIndex( final String indexImageFile ) {
329 this.indexImageFile = indexImageFile;
330 loadNativeLibrary();
331 assertNonEmptyReadableImageFile(indexImageFile);
332 refCount = new AtomicInteger();
333 indexAddress = openIndex(indexImageFile);
334 if ( indexAddress == 0L ) {
335 throw new CouldNotReadImageException(indexImageFile, "unable to open bwa-mem index");
336 }
337 ByteBuffer refContigNamesBuf = getRefContigNames(indexAddress);
338 if ( refContigNamesBuf == null ) {
339 throw new CouldNotReadImageException("unable to retrieve reference contig names from bwa-mem index");
340 }
341 refContigNamesBuf.order(ByteOrder.nativeOrder()).position(0).limit(refContigNamesBuf.capacity());
342 int nRefContigNames = refContigNamesBuf.getInt();
343 refContigNames = new ArrayList<>(nRefContigNames);
344 for ( int idx = 0; idx < nRefContigNames; ++idx ) {
345 int nameLen = refContigNamesBuf.getInt();
346 byte[] nameBytes = new byte[nameLen];
347 refContigNamesBuf.get(nameBytes);
348 refContigNames.add(new String(nameBytes));
349 }
350 destroyByteBuffer(refContigNamesBuf);
351 }
352
353 private void assertNonEmptyReadableImageFile(final String image) {
354 if (!nonEmptyReadableFile(image)) {
355 throw new CouldNotReadImageException(image, "is empty or is not readable");
356 }
357 }
358
359 /** true if index has not been closed */
360 public boolean isOpen() { return indexAddress != 0L; }
361
362 /** there's someone using the index -- don't allow it to be closed */
363 public long refIndex() {
364 refCount.incrementAndGet();
365 if ( indexAddress == 0L ) {
366 throw new IllegalStateException("Index image " + indexImageFile + " has been closed");
367 }
368 return indexAddress;
369 }
370
371 /** done using the index -- if ref count has fallen to 0, a call to close can be expected to succeed */
372 public void deRefIndex() { refCount.decrementAndGet(); }
373
374 /**
375 * Close the index and release the (non-Java) memory that's been allocated
376 *
377 * <p>
378 * <b><i>WARNING!</i></b>: Notice that currently this method is making JNI call that might result in an abrupt process
379 * interruption (e.g. exit or abort system call) and so the control may never be returned.
380 * </p>
381 */
382 @Override
383 public void close() {
384 long addr = indexAddress;
385 if ( addr != 0L ) {
386 synchronized (BwaMemIndex.class) {
387 addr = indexAddress;
388 if ( addr != 0L ) {
389 if ( refCount.intValue() != 0 ) {
390 throw new IllegalStateException("Index image "+indexImageFile+" can't be closed: it's in use.");
391 }
392 indexAddress = 0L;
393 destroyIndex(addr);
394 }
395 }
396 }
397 }
398
399 /** retrieve list of contig names in the reference dictionary */
400 public List<String> getReferenceContigNames() {
401 return refContigNames;
402 }
403
404 /** returns github GUID for the version of bwa that has been compiled */
405 public static String getBWAVersion() {
406 loadNativeLibrary();
407 return getVersion();
408 }
409
410 ByteBuffer doAlignment( final ByteBuffer seqs, final ByteBuffer opts ) {
411 final ByteBuffer alignments = createAlignments(seqs, indexAddress, opts);
412 if ( alignments == null ) {
413 throw new IllegalStateException("Unable to get alignments from bwa-mem index "+indexImageFile+": We don't know why.");
414 }
415 return alignments;
416 }
417
418 private static void assertNonEmptyReadableIndexFile(final String index, final String fileName ) {
419 if ( !nonEmptyReadableFile(fileName) )
420 throw new CouldNotReadIndexException(index, "Missing bwa index file: "+ fileName);
421 }
422
423 private static boolean nonEmptyReadableFile( final String fileName ) {
424 if (fileName == null) {
425 throw new IllegalArgumentException("the input file name cannot be null");
426 }
427 try ( final FileInputStream is = new FileInputStream(fileName) ) {
428 return is.read() != -1;
429 } catch ( final IOException ioe ) {
430 return false;
431 }
432 }
433
434 private static void loadNativeLibrary() {
435 if ( !nativeLibLoaded ) {
436 synchronized(BwaMemIndex.class) {
437 if ( !nativeLibLoaded ) {
•438 System.loadLibrary("bwa");
439 nativeLibLoaded = true;
440 }
441 }
442 }
443 }
444
445 private static native boolean createReferenceIndex(String referenceName, String indexPrefix, String algorithmName);
446 private static native boolean createIndexImageFile(String indexPrefix, String imageName );
447 private static native long openIndex( String indexImageFile );
448 private static native int destroyIndex( long indexAddress );
449 static native ByteBuffer createDefaultOptions();
450 private static native ByteBuffer getRefContigNames( long indexAddress );
451 private static native ByteBuffer createAlignments( ByteBuffer seqs, long indexAddress, ByteBuffer opts );
452 static native void destroyByteBuffer( ByteBuffer alignments );
453 private static native String getVersion();
454 }