Debian Codegraph @

src/java/org/broadinstitute/dropseqrna/barnyard/digitalallelecounts/sampleassignment/multisample/DetectDoublets.java

raw
1 /*
2 * MIT License
3 *
4 * Copyright 2017 Broad Institute
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 package org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.sampleassignment.multisample;
25
26 import java.io.File;
27 import java.io.PrintStream;
28 import java.text.DecimalFormat;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.Collections;
32 import java.util.Comparator;
33 import java.util.HashMap;
34 import java.util.HashSet;
35 import java.util.List;
36 import java.util.Map;
37 import java.util.Set;
38 import java.util.stream.Collectors;
39
40 import org.apache.commons.lang3.StringUtils;
41 import org.broadinstitute.barclay.argparser.Argument;
42 import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
43 import org.broadinstitute.dropseqrna.barnyard.GeneFunctionCommandLineBase;
44 import org.broadinstitute.dropseqrna.barnyard.ParseBarcodeFile;
45 import org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.SNPUMIBasePileupIterator;
46 import org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.SortOrder;
47 import org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.sampleassignment.CellAssignmentUtils;
48 import org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.sampleassignment.CellCollectionSampleLikelihoodCollection;
49 import org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.sampleassignment.SampleGenotypeProbabilities;
50 import org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.sampleassignment.SampleGenotypeProbabilitiesIterator;
51 import org.broadinstitute.dropseqrna.cmdline.CustomCommandLineValidationHelper;
52 import org.broadinstitute.dropseqrna.cmdline.DropSeq;
53 import org.broadinstitute.dropseqrna.utils.AssertSequenceDictionaryIntersection;
54 import org.broadinstitute.dropseqrna.utils.FileListParsingUtils;
55 import org.broadinstitute.dropseqrna.utils.GroupingIterator;
56 import org.broadinstitute.dropseqrna.utils.VCFUtils;
57 import org.broadinstitute.dropseqrna.utils.io.ErrorCheckingPrintStream;
58 import org.broadinstitute.dropseqrna.utils.readiterators.IgnoreGeneAnnotationTagger;
59 import org.broadinstitute.dropseqrna.utils.readiterators.PCRDuplicateFilteringIterator;
60 import org.broadinstitute.dropseqrna.utils.readiterators.SamFileMergeUtil;
61 import org.broadinstitute.dropseqrna.utils.readiterators.SamHeaderAndIterator;
62 import org.broadinstitute.dropseqrna.vcftools.SampleAssignmentVCFUtils;
63
64 import htsjdk.samtools.SAMRecord;
65 import htsjdk.samtools.SAMSequenceDictionary;
66 import htsjdk.samtools.SamReaderFactory;
67 import htsjdk.samtools.util.CloseableIterator;
68 import htsjdk.samtools.util.IOUtil;
69 import htsjdk.samtools.util.Interval;
70 import htsjdk.samtools.util.IntervalList;
71 import htsjdk.samtools.util.Log;
72 import htsjdk.samtools.util.PeekableIterator;
73 import htsjdk.variant.variantcontext.VariantContext;
74 import htsjdk.variant.vcf.VCFFileReader;
75 import picard.cmdline.StandardOptionDefinitions;
76
77 @CommandLineProgramProperties(summary = "Detect Doublets in Dropulation Data. Uses the outputs of AssignCellsToSamples to make decisions. It's highly recommended to use the VCF output from AssignCellsToSamples as input, as"
78 + "the memory usage of a full VCF may be prohibitive compared to the AssignCellsToSamples VCF, which contains only the variants that were observed in the data. This also greatly speeds up"
79 + "analysis.", oneLineSummary = "Detect Doublets in Dropulation Data", programGroup = DropSeq.class)
80 public class DetectDoublets extends GeneFunctionCommandLineBase {
81
82 private static final Log log = Log.getInstance(DetectDoublets.class);
83
84 @Argument(shortName = StandardOptionDefinitions.INPUT_SHORT_NAME, doc = "The input SAM or BAM file to analyze. This argument can accept wildcards, or a file with the suffix .bam_list that contains the locations of multiple BAM files", minElements = 1)
85 public List<File> INPUT_BAM;
86
87 @Argument(doc = "The input VCF file to analyze. Use the output VCF from AssignCellsToSamples to save memory.")
88 public File VCF;
89
90 @Argument(doc = "The output likelihood file from AssignCellsToSamples")
91 public File SINGLE_DONOR_LIKELIHOOD_FILE;
92
93 @Argument(shortName = StandardOptionDefinitions.OUTPUT_SHORT_NAME, doc = "Output file of doublet likelihoods. This supports zipped formats like gz and bz2.")
94 public File OUTPUT;
95
96 @Argument(doc = "Output file of per-sample-pair doublet likelihoods. Insted of just the best pair as seen in the OUTPUT file, "
97 + "this outputs all tested pairs for each cell. This supports zipped formats like gz and bz2.", optional = true)
98 public File OUTPUT_ALL_PAIRS = null;
99
100 @Argument(doc = "Output file of per-snp/sample-pair doublet likelihoods. Insted of just the best pair as seen in the OUTPUT file, "
101 + "this outputs all tested pairs for each cell, for each SNP. This file can get pretty obnoxiously huge."
102 + "This supports zipped formats like gz and bz2.", optional = true)
103 public File OUTPUT_PER_SNP = null;
104
105 @Argument(doc = "The cell barcode tag. If there are no reads with this tag, the program will assume that all reads belong to the same cell and process in single sample mode.")
106 public String CELL_BARCODE_TAG = "XC";
107
108 @Argument(doc = "The molecular barcode tag.")
109 public String MOLECULAR_BARCODE_TAG = "XM";
110
111 @Argument(doc = "The edit distance that molecular barcodes should be combined at within a gene/SNP.")
112 public Integer EDIT_DISTANCE = 1;
113
114 @Argument(doc = "The map quality of the read to be included.")
115 public Integer READ_MQ = 10;
116
117 @Argument(doc = "Override NUM_CORE_BARCODES and process reads that have the cell barcodes in this file instead. The file has 1 column with no header.", optional = false)
118 public File CELL_BC_FILE = null;
119
120 @Argument(doc = "The minimum genotype quality for a variant. Set this value to 0 to not filter by GQ scores if they are present, or to -1 to completely "
121 + "ignore GQ values if they are not set in the genotype info field. If the GQ field is not set in the VCF header, this will be set to -1 by default.")
122 public Integer GQ_THRESHOLD = 30;
123
124 @Argument(doc = "A file with a list of samples in the VCF to consider as samples in the doublets. This subsets the VCF into a smaller data set containing only the samples listed. "
125 + "The file has 1 column with no header. If this list contains only one donor and no contaminating donors were found via single donor assignment, "
126 + "doublet detection calculations will not take place. Instead, the program will emit a default output for each cell, and the program "
127 + "will then quit with an exit status.", optional = false)
128 public File SAMPLE_FILE;
129
130 @Argument(doc = "Instead of useing base qualities to determine error rate, use a fixed error rate instead. This is rounded to the nearest phread score internally.", optional = true)
131 public Double FIXED_ERROR_RATE = null;
132
133 @Argument(doc = "Caps the base error rate at a maximum probability so no SNP can be weighed more than this value. For example, if this value was 0.01, "
134 + "then a base quality 30 value (normally an erro rate of 0.001) would become 0.01. With the same threshold, a base with an error rate of 0.1 would be unaffected.", optional = true)
135 public Double MAX_ERROR_RATE = null;
136
137 @Argument(doc = "A file that contains an estimate of how much ambient RNA is in each cell. This is a fractional estimate between 0 and 1. File is tab seperated, with 2 columns:"
138 + "cell_barcode and frac_contamination. When supplied along side the ALLELE_FREQUENCY_ESTIMATE_FILE, this modifies the likelihood error rates to take into account how often"
139 + "the allele observed can be drawn from ambient RNA. We use cellbender remove background [https://github.com/broadinstitute/CellBender] to estimate the "
140 + "number of transcripts before and after ambient cleanup to define the fraction of transcripts that come from ambient RNA.", optional = true)
141 public File CELL_CONTAMINATION_ESTIMATE_FILE = null;
142
143 @Argument(doc = "A file that contains an estimate of the allele frequency expected for each SNP across donors. The best estimate of this will come from the fraction of reference and alternate allele"
144 + "UMIs that are observed at each snp site. This report can be generated via GatherDigitalAlleleCounts. This is a fractional estimate between 0 and 1. File is tab seperated, with at least 3 columns:"
145 + "chromosome, position, maf_umi. When supplied and CELL_CONTAMINATION_ESTIMATE_FILE is provided, this modifies the likelihood error rates to take into account how often "
146 + "the allele observed can be drawn from ambient RNA.", optional = true)
147 public File ALLELE_FREQUENCY_ESTIMATE_FILE = null;
148
149 @Argument(doc = "At least <FRACTION_SAMPLES_PASSING> samples must have genotype scores >= GQ_THRESHOLD for the variant in the VCF to be included in the analysis.")
150 public double FRACTION_SAMPLES_PASSING = 0.5;
151
152 @Argument(doc = "A list of chromosomes to omit from the analysis. The default is to omit the sex chromosomes.")
153 public List<String> IGNORED_CHROMOSOMES = new ArrayList<>(Arrays.asList("X", "Y", "MT"));
154
155 /**
156 * this produces worse results with in-silico testing, generating more misclassifications of singlets as doublets.
157 *
158 * @Argument(doc="For SNPs that don't have a high quality genotype in the VCF, should we infer a global penalty per SNP
159 * and apply it to donors for that SNP that are not confidently called?") public boolean
160 * USE_MISSING_DATA=true;
161 */
162 private boolean USE_MISSING_DATA = false;
163
164 @Argument(doc = "Force evaluation of the doublet at the given mixture ratio. Should be a number between 0 and 1.", optional = true)
165 public Double FORCED_RATIO = 0.8;
166
167 @Argument(doc = "Should cells that were assigned to a donor not on the sample list be tested? When enabled, "
168 + "this forces the program to load genotype information for all donors seen at least once, even when not on the sample list. If there are many off-target assignments,"
169 + "this can use large amounts of memory. Set to false to skip testing cells that you'll probably discard as incorrectly assigned later anyway, set to true to "
170 + "test each cell not on the donor lists against all possible donors on the list.")
171 public Boolean TEST_UNEXPECTED_DONORS = true;
172
173 @Argument(doc = "For each cell, when comparing donor pairs to each other, scale the likelihoods to the number of UMIs. "
174 + "This provides an additional penalty score to donor pairs with more incomplete information that makes donor pairs more comparable.")
175 public Boolean SCALE_LIKELIHOODS = true;
176
177 @Argument(doc = "EXPERIMENTAL!!! Run the program in DNA Mode. In this mode, reads should have a cell barcode, but will be missing gene annotations and UMIs. All reads will be "
178 + "accepted as passing, and each read (or read pair) will be treated as a single UMI If the data is PCR Duplicate marked, duplicate reads will be filtered. ")
179 public Boolean DNA_MODE = false;
180
181 /*
182 * @Argument
183 * (doc="The model tests the best donor against the all the possible second most likely donors to find the pair that best explain the data. Sort the donors by their single donor likelihood score,"
184 * +
185 * "and only test the best <MIN_DONOR_PAIRS_TESTED> donors. This can reduce the number of tests / runtime / memory for large pools while producing approximately the same output."
186 * )
187 */
188
189 private Integer MIN_DONOR_PAIRS_TESTED = Integer.MAX_VALUE;
190
191 // @Argument(doc="See MIN_DONOR_PAIRS_TESTED. This limits the number of donors tested to a fraction of the total number
192 // of donors. The number of donors used is the maximum of FRACTION_DONOR_PAIRS_TESTED and MIN_DONOR_PAIRS_TESTED.")
193
194 private Double FRACTION_DONOR_PAIRS_TESTED = 1.0;
195
196 private final String SNP_TAG = "YS";
197
198 private static DecimalFormat mixtureFormat = new DecimalFormat("#.###");
199
200 @Override
201 protected int doWork() {
202
203 List<String> donorList = ParseBarcodeFile.readCellBarcodeFile(this.SAMPLE_FILE);
204 log.info("Number of donors in donor list [" + donorList.size() + "]");
205 int pairsToTest = getNumberOfDonorsToTest(donorList.size(), this.MIN_DONOR_PAIRS_TESTED, this.FRACTION_DONOR_PAIRS_TESTED);
206
207 PrintStream perDonorWriter = null;
208 if (OUTPUT_ALL_PAIRS != null) {
209 perDonorWriter = new ErrorCheckingPrintStream(IOUtil.openFileForWriting(this.OUTPUT_ALL_PAIRS));
210 writeAssignmentHeader(perDonorWriter, pairsToTest, false, true);
211 }
212
213 PrintStream perSNPWriter = null;
214 if (OUTPUT_PER_SNP != null) {
215 IOUtil.assertFileIsWritable(this.OUTPUT_PER_SNP);
216 perSNPWriter = new ErrorCheckingPrintStream(IOUtil.openFileForWriting(this.OUTPUT_PER_SNP));
217 writePerSNPHeader(perSNPWriter);
218 }
219
220 PrintStream writer = new ErrorCheckingPrintStream(IOUtil.openFileForWriting(this.OUTPUT));
221 writeAssignmentHeader(writer, pairsToTest, true, false);
222
223 final VCFFileReader vcfReader = new VCFFileReader(this.VCF, false);
224 SamHeaderAndIterator headerAndIter = SamFileMergeUtil.mergeInputs(this.INPUT_BAM, false, SamReaderFactory.makeDefault());
225 AssertSequenceDictionaryIntersection.assertIntersectionObjectVcf(headerAndIter.header, "BAM INPUT(S)", this.VCF, log);
226
227 // extract the sequence dictionary to build the interval list.
228 SAMSequenceDictionary vcfDict = vcfReader.getFileHeader().getSequenceDictionary();
229
230 // disable GQ filter if it's not in the header.
231 if (!VCFUtils.GQInHeader(vcfReader)) {
232 this.GQ_THRESHOLD = -1;
233 log.info("Genotype Quality [GQ] not found in header. Disabling GQ_THRESHOLD parameter");
234 }
235
236 CellCollectionSampleLikelihoodCollection cslc = CellCollectionSampleLikelihoodCollection.parseFile(this.SINGLE_DONOR_LIKELIHOOD_FILE);
237
238 // A map where the key is the cell barcode, and the value is the best donor.
239 Map<String, String> bestDonorForCell = getBestDonorForCell(this.SINGLE_DONOR_LIKELIHOOD_FILE);
240
241 // Filter the single donor assignment by the donor list if requested.
242 if (!this.TEST_UNEXPECTED_DONORS)
243 bestDonorForCell = filterDonorMap(bestDonorForCell, donorList);
244
245 // read in the per-cell penalty score and validate against the best donor per cell to make sure you have penalties for
246 // every donor.
247 Map<String, Double> contaminationMap = CellAssignmentUtils.getCellContamination(this.CELL_CONTAMINATION_ESTIMATE_FILE, bestDonorForCell.keySet());
248 Map<Interval, Double> variantMinorAlleleFrequency = CellAssignmentUtils.getMinorAlleleFrequencyMap(this.ALLELE_FREQUENCY_ESTIMATE_FILE);
249
250 // all donors is the input set of donors plus any best donor for a cell.
251 // this list of donors is restricted to the sample list if TEST_UNEXPECTED_DONORS=false.
252 Set<String> allDonors = new HashSet<>();
253 allDonors.addAll(donorList);
254 allDonors.addAll(new HashSet<>(bestDonorForCell.values()));
255 List<String> allDonorsList = new ArrayList<>(allDonors);
256
257 log.info("Number of donors that can be either donor in a donor pair [" + allDonorsList.size() + "]");
258
259 // Keep all the barcodes that are in the barcode list AND in the single donor assignments.
260 List<String> cellBarcodes = getCellBarcodes(this.CELL_BC_FILE, bestDonorForCell, this.TEST_UNEXPECTED_DONORS);
261
262 // If there is only one donor at this point, doublet detection should not continue.
263 if (allDonorsList.size()<2) {
264 singleDonorGracefulExit(bestDonorForCell, writer, perDonorWriter, perSNPWriter);
265 return 0;
266 }
267
268 // Pass a list of all donors requested + best calls if TEST_UNEXPECTED_DONORS is true.
269 // set the %passing to be 0, since the snpIntervals will properly filter on the right set of donors, but you want a
270 // super-set of donors available
271 // so if you call donors A-D, but the single cell assigned E, then E would still be an available donor in the genotype
272 // matrix.
273 PeekableIterator<VariantContext> vcfIterator = SampleAssignmentVCFUtils.getVCFIterator(vcfReader, allDonorsList, false, this.GQ_THRESHOLD,
274 this.FRACTION_SAMPLES_PASSING, IGNORED_CHROMOSOMES, log);
275
276 GenotypeMatrix genotypeMatrix = new GenotypeMatrix(vcfIterator, this.GQ_THRESHOLD, allDonorsList);
277 vcfIterator.close();
278
279 Map<Interval, Double> genotypeQuality = genotypeMatrix.getAverageGenotypeQuality();
280
281 final IntervalList snpIntervals = new IntervalList(vcfDict);
282 snpIntervals.addall(genotypeMatrix.getSNPIntervals());
283 // a requested early exit if there are no SNPs.
284 if (snpIntervals.getIntervals().isEmpty()) {
285 log.error("No SNP intervals detected! Check to see if your VCF filter thresholds are too restrictive!");
286 return 1;
287 }
288
289 PeekableIterator<List<SampleGenotypeProbabilities>> sampleGenotypeIterator = prepareIterator(snpIntervals, cellBarcodes, genotypeQuality);
290
291 int cellCount = 0;
292 int reportInterval = 100;
293 log.info("Calling doublets");
294 if (!sampleGenotypeIterator.hasNext()) {
295 log.warn("No Cells found for analysis.");
296 } else {
297
298 while (sampleGenotypeIterator.hasNext()) {
299 cellCount++;
300 if (cellCount % reportInterval == 0)
301 log.info("Tested cell #" + cellCount);
302 List<SampleGenotypeProbabilities> probs = sampleGenotypeIterator.next();
303
304 String cell = probs.get(0).getCell();
305 String bestDonor = bestDonorForCell.get(cell);
306 if (bestDonor == null)
307 throw new IllegalStateException("Cell [" + cell + "] has no best donor assignment in file.");
308
309 VariantDataFactory f = null;
310 f = new VariantDataFactory(cell, probs, genotypeMatrix, FIXED_ERROR_RATE, USE_MISSING_DATA, MAX_ERROR_RATE, contaminationMap,
311 variantMinorAlleleFrequency);
312
313 FindOptimalDonorMixture fodm = new FindOptimalDonorMixture(f);
314
315 // AllPairedSampleAssignmentsForCell allAssignments = fodm.findBestDonorPair(bestDonor, donorList, FORCED_RATIO);
316 // List<String> donorsThisCell = getExpectedSecondDonorsRankedByLikelihood(cell, cslc, pairsToTest, donorList, bestDonor);
317 AllPairedSampleAssignmentsForCell allAssignments = fodm.findBestDonorPair(bestDonor, donorList, FORCED_RATIO, SCALE_LIKELIHOODS);
318 SamplePairAssignmentForCell best = allAssignments.getBestAssignment();
319
320 // edge case: assignment is null because there's no data for this cell. This only happens
321 // when a user has a cell selection error or similar and attempts to call cell barcodes that aren't cells.
322 if (best==null) {
323 log.warn("No best pair found for cell ["+cell+"] due to no informative UMIs. Cell selection or similar problem?");
324 best= SamplePairAssignmentForCell.constructEmptyResult(cell, bestDonorForCell.get(cell));
325 }
326
327 double bestPairPvalue = allAssignments.getBestPairPvalue();
328 writeAssignment(best, bestPairPvalue, writer, false);
329 if (OUTPUT_ALL_PAIRS != null) {
330 writeAssignment(allAssignments.getBestAssignment(), null, perDonorWriter, true);
331 // apply ordering to other assignments for stability of outputs. Sort by 2nd donor name.
332 Comparator<SamplePairAssignmentForCell> comparator = java.util.Comparator.comparing(SamplePairAssignmentForCell::getSampleTwo,
333 java.util.Comparator.naturalOrder());
334 List<SamplePairAssignmentForCell> allOther = allAssignments.getOtherAssignments();
335 Collections.sort(allOther, comparator);
336 for (SamplePairAssignmentForCell other : allOther)
337 writeAssignment(other, null, perDonorWriter, true);
338 }
339 reportResultsPerSNP(cell, f, bestDonor, donorList, allAssignments, perSNPWriter);
340 }
341 }
342
343 if (OUTPUT_PER_SNP != null)
344 perSNPWriter.close();
345 if (OUTPUT_ALL_PAIRS != null)
346 perDonorWriter.close();
347 writer.close();
348
349 log.info("Finished!");
350 return 0;
351 }
352
353
354 /**
355 * In the strange edge case where there is only a single donor to be tested, write out a default output file instead of going through testing.
356 * This function additionally closes all potentially open writers and runs logging.
357 * @param bestDonorForCell A map containing cell barcodes and the best donor for each cell.
358 * @param writer The file to write to per-cell outputs.
359 * @param perDonorWriter Closes this file if not null.
360 * @param perSNPWriter Closes this file if not null.
361 */
362 void singleDonorGracefulExit(Map<String, String> bestDonorForCell, PrintStream writer,
363 PrintStream perDonorWriter, PrintStream perSNPWriter) {
364 // clean up more detailed file writers.
365 if (OUTPUT_ALL_PAIRS!=null) perDonorWriter.close();
366 if (OUTPUT_PER_SNP != null) perSNPWriter.close();
367 // write a default output per cell close results and quit.
368 log.error("The donor file only contained a single donor, and no additional donors were detected by single donor assignment. Doublet detection will not continue. "
369 + "A default output will be written to perserve downstream pipeline functionality.");
370 writeSingleDonorEdgeCaseOutput(bestDonorForCell, writer);
371 }
372
373 /**
374 * In the strange edge case where there is only a single donor to be tested, write out a default output file instead of going through testing.
375 * @param bestDonorForCell A map containing cell barcodes and the best donor for each cell.
376 * @param writer The file to write to
377 */
378 void writeSingleDonorEdgeCaseOutput(Map<String, String> bestDonorForCell, PrintStream writer) {
379 for (String cell: bestDonorForCell.keySet()) {
380 SamplePairAssignmentForCell best = SamplePairAssignmentForCell.constructEmptyResult(cell, bestDonorForCell.get(cell));
381 writeAssignment(best, 1.0E-101d, writer, false);
382 }
383 writer.close();
384
385 }
386
387 /**
388 * Filter the map of cell barcode -> donor to only retain cell barcodes where the assigned donor is in the donor list.
389 *
390 * @param bestDonorForCell
391 * @param donorList
392 * @return A submap of the input map where all values are contained in the donor list.
393 */
394 Map<String, String> filterDonorMap(Map<String, String> bestDonorForCell, List<String> donorList) {
395 Set<String> dl = new HashSet<String>(donorList);
396 Map<String, String> result = new HashMap<String, String>();
397
398 for (String cellBC : bestDonorForCell.keySet()) {
399 String donor = bestDonorForCell.get(cellBC);
400 if (dl.contains(donor))
401 result.put(cellBC, donor);
402 }
403 return result;
404 }
405
406 private void reportResultsPerSNP(final String cell, final VariantDataFactory variantFactory, final String bestDonor, final List<String> vcfSamples,
407 final AllPairedSampleAssignmentsForCell allAssignments, final PrintStream out) {
408 if (out == null)
409 return;
410 List<String> other = FindOptimalDonorMixture.getNonPrimarySamples(bestDonor, vcfSamples);
411 for (String o : other) {
412 VariantDataCollection vdc = variantFactory.getVariantData(bestDonor, o);
413 SamplePairAssignmentForCell mixtureResult = allAssignments.getAssignmentForDonorPair(bestDonor, o);
414 List<VariantData> vdList = vdc.getVariantData();
415 double mixture = mixtureResult.getMixture();
416 for (VariantData vd : vdList)
417 writePerSNPReport(cell, vd, bestDonor, o, mixture, out);
418 }
419 }
420
421 private void writePerSNPReport(final String cell, final VariantData vd, final String sampleOne, final String sampleTwo, final Double mixture,
422 final PrintStream out) {
423 /*
424 * if (mixture==null) { String [] line = {cell, sampleOne, sampleTwo, vd.getSNPInterval().getContig(),
425 * Integer.toString(vd.getSNPInterval().getStart()), vd.getGenotypeOne().toString(), vd.getGenotypeTwo().toString(),
426 * Integer.toString(vd.getGenotypeCountReference()), Integer.toString(vd.getGenotypeCountAlternate()), "NA",
427 * Double.toString(vd.getLogLikelihood(1)), Double.toString(vd.getLogLikelihood(0))}; String h = StringUtils.join(line,
428 * "\t"); out.println(h); return; }
429 */
430
431 String[] line = { cell, sampleOne, sampleTwo, vd.getSNPInterval().getContig(), Integer.toString(vd.getSNPInterval().getStart()),
432 vd.getGenotypeOne().toString(), vd.getGenotypeTwo().toString(), Integer.toString(vd.getGenotypeCountReference()),
433 Integer.toString(vd.getGenotypeCountAlternate()), Double.toString(vd.getLogLikelihood(mixture)), Double.toString(vd.getLogLikelihood(1)),
434 Double.toString(vd.getLogLikelihood(0)) };
435 String h = StringUtils.join(line, "\t");
436 out.println(h);
437 }
438
439 private void writePerSNPHeader(final PrintStream out) {
440 String[] line = { "cell", "sampleOne", "sampleTwo", "chr", "pos", "genotype_S1", "genotype_S2", "refAlleleCount", "altAlleleCount",
441 "likelihood_mixture", "likelihood_S1", "likelihood_S2" };
442 String h = StringUtils.join(line, "\t");
443 out.println(h);
444 }
445
446 private String convertNullToString(final Double x) {
447 if (x == null)
448 return ("NA");
449 return Double.toString(x);
450 }
451
452 private void writeAssignmentHeader(final PrintStream out, final int pairsToTest, final boolean outputBestPairPvalue, final boolean writeScaledLikelihoods) {
453 List<String> paths = this.INPUT_BAM.stream().map(x -> x.getAbsolutePath()).collect(Collectors.toList());
454 String bamList = StringUtils.join(paths, ",");
455
456 List<String> header = new ArrayList<>(Arrays.asList("#INPUT_BAM=" + bamList, "INPUT_VCF=" + this.VCF.getAbsolutePath(),
457 "DONOR_FILE=" + this.SAMPLE_FILE, "CELL_BC_FILE=" + CELL_BC_FILE, "GQ_THRESHOLD=" + Integer.toString(this.GQ_THRESHOLD),
458 "FRACTION_SAMPLES_PASSING=" + Double.toString(this.FRACTION_SAMPLES_PASSING), "FORCED_RATIO=" + convertNullToString(FORCED_RATIO),
459 "USE_MISSING_DATA=" + USE_MISSING_DATA, "READ_MQ=" + Integer.toString(this.READ_MQ),
460 "FIXED_ERROR_RATE=" + convertNullToString(this.FIXED_ERROR_RATE), "MAX_ERROR_RATE=" + convertNullToString(this.MAX_ERROR_RATE),
461 "LOCUS_FUNCTION=" + this.LOCUS_FUNCTION_LIST.toString(), "PAIRS_TO_TEST=" + Integer.toString(pairsToTest)));
462
463 if (this.CELL_CONTAMINATION_ESTIMATE_FILE != null && this.ALLELE_FREQUENCY_ESTIMATE_FILE != null) {
464 header.add("CELL_CONTAMINATION_ESTIMATE_FILE=" + CELL_CONTAMINATION_ESTIMATE_FILE.getAbsolutePath());
465 header.add("ALLELE_FREQUENCY_ESTIMATE_FILE=" + ALLELE_FREQUENCY_ESTIMATE_FILE.getAbsolutePath());
466 }
467
468 String h = StringUtils.join(header, "\t");
469 out.println(h);
470
471 writeAssignmentColumnNames(out, outputBestPairPvalue, writeScaledLikelihoods);
472 }
473
474 public static void writeAssignmentColumnNames(final PrintStream out, final boolean outputBestPairPvalue, final boolean writeScaledLikelihoods) {
475 List<String> line = new ArrayList<String>(Arrays.asList("cell", "sampleOneMixtureRatio", "sampleOne", "sampleOneLikelihood", "sampleTwo",
476 "sampleTwoLikelihood", "mixedSample", "mixedSampleLikelihood", "num_paired_snps", "num_inform_snps", "num_umi", "num_inform_umis",
477 "lr_test_stat", "sampleOneWrongAlleleCount", "num_homozygous_inform_umis_s1",
478 "sampleTwoWrongAlleleCount", "num_homozygous_inform_umis_s2", "bestLikelihood", "bestSample", "doublet_pval"));
479
480 if (outputBestPairPvalue)
481 line.add("best_pair_pvalue");
482
483 if (writeScaledLikelihoods) {
484 line.add("bestLikelihoodScaled");
485 }
486
487 String header = StringUtils.join(line, "\t");
488 out.println(header);
489 }
490
491 public static void writeAssignment(final SamplePairAssignmentForCell assignment, final Double bestPairPvalue, final PrintStream out,
492 final boolean writeScaledLikelihood) {
493 String mixture = mixtureFormat.format(assignment.getMixture());
494
495 List<String> line = new ArrayList<String>(
496 Arrays.asList(assignment.getCellBarcode(), mixture, assignment.getSampleOne(), Double.toString(assignment.getSampleOneSingleLikelihood()),
497 assignment.getSampleTwo(), Double.toString(assignment.getSampleTwoSingleLikelihood()), assignment.getCombinedDonorName(),
498 Double.toString(assignment.getDoubletLikelihood()), Integer.toString(assignment.getNumSNPs()),
499 Integer.toString(assignment.getNumInformativeSNPs()), Integer.toString(assignment.getNumUMIs()),
500 Integer.toString(assignment.getNumInformativeUMIs()), Double.toString(assignment.getDoubletLikelihoodRatio()),
501 Integer.toString(assignment.getImpossibleAllelesSampleOne()), Integer.toString(assignment.getNumInformativeHomozygousUMIsSampleOne()),
502 Integer.toString(assignment.getImpossibleAllelesSampleTwo()), Integer.toString(assignment.getNumInformativeHomozygousUMIsSampleTwo()),
503 Double.toString(assignment.getBestLikelihood()), assignment.getBestSample(), Double.toString(assignment.getDoubletPvalue())));
504
505 if (bestPairPvalue != null)
506 line.add(bestPairPvalue.toString());
507
508 if (writeScaledLikelihood)
509 line.add(Double.toString(assignment.getScaledBestLikelihood()));
510
511 String h = StringUtils.join(line, "\t");
512 out.println(h);
513 }
514
515 public PeekableIterator<List<SampleGenotypeProbabilities>> prepareIterator(final IntervalList snpIntervals, List<String> cellBarcodes, Map<Interval, Double> genotypeQuality) {
516
517 SamReaderFactory factory = SamReaderFactory.makeDefault().enable(SamReaderFactory.Option.EAGERLY_DECODE);
518 SamHeaderAndIterator headerAndIter = SamFileMergeUtil.mergeInputs(this.INPUT_BAM, false, factory);
519
520 // override the normal gene annotations with new ones before any other operations.
521 // filter out PCR duplicates.
522 if (this.DNA_MODE) {
523 // Replace UMI tags with read names, set strand tag to read tag, set gene function to an accepted on. Overwrites ALL
524 // reads tags.
525 IgnoreGeneAnnotationTagger tagger = new IgnoreGeneAnnotationTagger(headerAndIter.iterator, this.GENE_NAME_TAG, this.GENE_STRAND_TAG,
526 this.GENE_FUNCTION_TAG, this.LOCUS_FUNCTION_LIST, false, this.MOLECULAR_BARCODE_TAG, true);
527 headerAndIter = new SamHeaderAndIterator(headerAndIter.header, (CloseableIterator<SAMRecord>) tagger.iterator());
528 final PCRDuplicateFilteringIterator pcrDuplicateFilteringIterator = new PCRDuplicateFilteringIterator(headerAndIter.iterator);
529 headerAndIter = new SamHeaderAndIterator(headerAndIter.header, pcrDuplicateFilteringIterator);
530 }
531
532 SNPUMIBasePileupIterator sbpi = new SNPUMIBasePileupIterator(headerAndIter, snpIntervals, GENE_NAME_TAG, GENE_STRAND_TAG, GENE_FUNCTION_TAG,
533 LOCUS_FUNCTION_LIST, STRAND_STRATEGY, this.FUNCTIONAL_STRATEGY, this.CELL_BARCODE_TAG, this.MOLECULAR_BARCODE_TAG, this.SNP_TAG,
534 GeneFunctionCommandLineBase.DEFAULT_FUNCTION_TAG, this.READ_MQ, false, cellBarcodes, genotypeQuality, SortOrder.CELL_SNP);
535
536 final SAMSequenceDictionary dict = snpIntervals.getHeader().getSequenceDictionary();
537
538 // gets a SampleGenotypeProbabilities for each cell.
539 SampleGenotypeProbabilitiesIterator result = new SampleGenotypeProbabilitiesIterator(sbpi, dict, this.EDIT_DISTANCE, SortOrder.CELL_SNP);
540
541 // clusters SampleGenotypeProbabilities objects across all cells for a SNP
542 GroupingIterator<SampleGenotypeProbabilities> groupingIterator = new GroupingIterator<>(result, new Comparator<SampleGenotypeProbabilities>() {
543 @Override
544 public int compare(final SampleGenotypeProbabilities o1, final SampleGenotypeProbabilities o2) {
545 int cmp = o1.getCell().compareTo(o2.getCell());
546 return cmp;
547 }
548 });
549
550 PeekableIterator<List<SampleGenotypeProbabilities>> peekableIter = new PeekableIterator<>(groupingIterator);
551 return (peekableIter);
552 }
553
554 List<String> getCellBarcodes(File cellBarcodeFile, Map<String, String> bestDonorForCell, boolean testUnexpectedDonors) {
555 List<String> cellBarcodes = ParseBarcodeFile.readCellBarcodeFile(this.CELL_BC_FILE);
556 int numBarcodes = cellBarcodes.size();
557
558 log.info("Number of cell barcodes in input file [" + numBarcodes + "]");
559
560 if (!testUnexpectedDonors) {
561 cellBarcodes.retainAll(bestDonorForCell.keySet());
562 log.info("Number of cell barcodes after filtering to expected donors [" + cellBarcodes.size() + "]");
563 double fracRemoved = (double) (numBarcodes - cellBarcodes.size()) / (double) numBarcodes;
564 log.info("% cell barcodes not assigned to an expected donor [" + new DecimalFormat("0.##").format(fracRemoved * 100) + "%]");
565 }
566
567 return cellBarcodes;
568 }
569
570 /**
571 * Parse the single donor likelihood file, retrieve the best donor for each cell barcode.
572 *
573 * @param singleDonorLikelihoodFile The OUTPUT file produced by AssignCellsToSamples.
574 * @return A map where the key is the cell barcode, and the value is the best donor.
575 */
576 Map<String, String> getBestDonorForCell(final File singleDonorLikelihoodFile) {
577 Map<String, String> result = new HashMap<>();
578
579 CellCollectionSampleLikelihoodCollection cslc = CellCollectionSampleLikelihoodCollection.parseFile(singleDonorLikelihoodFile);
580
581 for (String cellBarcode : cslc.getCellBarcodes()) {
582 String donor = cslc.getLikelihoodCollection(cellBarcode).getBestSampleAssignment().getSample();
583 result.put(cellBarcode, donor);
584 }
585 return (result);
586 }
587
588 // getNumberOfDonorsToTest(donorList.size(), this.MIN_DONOR_PAIRS_TESTED, this.FRACTION_DONOR_PAIRS_TESTED);
589 private int getNumberOfDonorsToTest(int totalDonorPairs, int minDonorPairs, double fractionDonorPairs) {
590 int numTest = (int) Math.ceil((double) totalDonorPairs * fractionDonorPairs);
591 int result = Math.max(numTest, minDonorPairs);
592 // don't test more than the total number of donors.
593 // this is for when total donors are < minDonorPairs.
594 if (result > totalDonorPairs)
595 result = totalDonorPairs;
596 log.info("Testing " + Integer.toString(result) + " Donor pairs per cell");
597 return (result);
598
599 }
600
601
602 @Override
603 protected String[] customCommandLineValidation() {
604 IOUtil.assertFileIsReadable(this.VCF);
605 this.INPUT_BAM = FileListParsingUtils.expandFileList(INPUT_BAM);
606 IOUtil.assertFileIsWritable(this.OUTPUT);
607 IOUtil.assertFileIsReadable(this.SAMPLE_FILE);
608 IOUtil.assertFileIsReadable(this.SINGLE_DONOR_LIKELIHOOD_FILE);
609 IOUtil.assertFileIsReadable(this.CELL_BC_FILE);
610 final ArrayList<String> list = new ArrayList<>(1);
611 if (OUTPUT_ALL_PAIRS != null)
612 IOUtil.assertFileIsWritable(this.OUTPUT_ALL_PAIRS);
613 if (OUTPUT_PER_SNP != null)
614 IOUtil.assertFileIsWritable(this.OUTPUT_PER_SNP);
615
616 if (!VCFUtils.hasIndex(this.VCF))
617 list.add("VCF is not indexed! Please index and retry.");
618
619 if (this.CELL_CONTAMINATION_ESTIMATE_FILE != null && this.ALLELE_FREQUENCY_ESTIMATE_FILE == null)
620 list.add("If CELL_CONTAMINATION_ESTIMATE_FILE is supplied, must also supply ALLELE_FREQUENCY_ESTIMATE_FILE");
621
622 if (this.CELL_CONTAMINATION_ESTIMATE_FILE == null && this.ALLELE_FREQUENCY_ESTIMATE_FILE != null)
623 list.add("If ALLELE_FREQUENCY_ESTIMATE_FILE is supplied, must also supply CELL_CONTAMINATION_ESTIMATE_FILE");
624
625 if (this.FIXED_ERROR_RATE != null & this.MAX_ERROR_RATE == null)
626 log.info("Running with a fixed error rate of " + this.FIXED_ERROR_RATE);
627
628 if (this.MAX_ERROR_RATE != null && this.CELL_CONTAMINATION_ESTIMATE_FILE == null && this.ALLELE_FREQUENCY_ESTIMATE_FILE == null)
629 log.info("Running with a maximum cap on error rate of " + this.FIXED_ERROR_RATE);
630
631 if (FRACTION_SAMPLES_PASSING < 0 | FRACTION_SAMPLES_PASSING > 1)
632 list.add("FRACTION_SAMPLES_PASSING must be between 0 and 1, value was " + Double.toString(this.FRACTION_SAMPLES_PASSING));
633
634 return CustomCommandLineValidationHelper.makeValue(super.customCommandLineValidation(), list);
635 }
636
637 /** Stock main method. */
638 public static void main(final String[] args) {
639 System.exit(new DetectDoublets().instanceMain(args));
640 }
641
642 /*
643 private List<String> getExpectedSecondDonorsRankedByLikelihood(String cellBarcode, CellCollectionSampleLikelihoodCollection cslc, int pairsToTest,
644 List<String> expectedDonors, String bestDonor) {
645 // short circuit if the requested number of donors is the same as the number of expected donors IE the non-optimized
646 // strategy.
647 if (expectedDonors.size() == pairsToTest)
648 return expectedDonors;
649
650 CellSampleLikelihoodCollection c = cslc.getLikelihoodCollection(cellBarcode);
651 if (c == null)
652 throw new IllegalArgumentException("Could not find single donor likelihoods for cell " + cellBarcode);
653 List<String> rankedDonors = c.getDonorsRankedByAssignmentLikelihood();
654 // exclude the best donor from the ranked list. We don't want to select that.
655 rankedDonors.remove(bestDonor);
656
657 Set<String> expected = new HashSet<String>(expectedDonors);
658 // filter ranked donors by expected.
659 List<String> rankedExpectedDonors = rankedDonors.stream().filter(x -> expected.contains(x)).collect(Collectors.toList());
660
661 // if pairs to test is set too high somehow (the sample lists doesn't match up with the input single donor assignments)
662 // then limit the return.
663 if (rankedExpectedDonors.size() < pairsToTest) {
664 pairsToTest = rankedExpectedDonors.size();
665 }
666 // get the top <X> donors.
667 rankedExpectedDonors = rankedExpectedDonors.subList(0, pairsToTest);
668 return rankedExpectedDonors;
669
670 }
671 */
672
673
674 }