Debian Codegraph @

src/java/org/broadinstitute/dropseqrna/barnyard/digitalallelecounts/sampleassignment/AssignCellsToSamples.java

raw
1 /*
2 * MIT License
3 *
4 * Copyright 2017 Broad Institute
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 package org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.sampleassignment;
25
26 import java.io.File;
27 import java.io.PrintStream;
28 import java.text.DecimalFormat;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.Collection;
32 import java.util.Comparator;
33 import java.util.HashMap;
34 import java.util.HashSet;
35 import java.util.Iterator;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Set;
39 import java.util.stream.Collectors;
40
41 import htsjdk.samtools.util.*;
42 import org.apache.commons.collections4.CollectionUtils;
43 import org.apache.commons.lang3.StringUtils;
44 import org.apache.commons.math3.stat.descriptive.moment.Mean;
45 import org.broadinstitute.barclay.argparser.Argument;
46 import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
47 import org.broadinstitute.dropseqrna.barnyard.BarcodeListRetrieval;
48 import org.broadinstitute.dropseqrna.barnyard.GeneFunctionCommandLineBase;
49 import org.broadinstitute.dropseqrna.barnyard.ParseBarcodeFile;
50 import org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.SNPInfoCollection;
51 import org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.SNPUMIBasePileup;
52 import org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.SNPUMIBasePileupIterator;
53 import org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.SortOrder;
54 import org.broadinstitute.dropseqrna.censusseq.JointIteratorCounter;
55 import org.broadinstitute.dropseqrna.cmdline.CustomCommandLineValidationHelper;
56 import org.broadinstitute.dropseqrna.cmdline.DropSeq;
57 import org.broadinstitute.dropseqrna.utils.AssertSequenceDictionaryIntersection;
58 import org.broadinstitute.dropseqrna.utils.Bases;
59 import org.broadinstitute.dropseqrna.utils.FileListParsingUtils;
60 import org.broadinstitute.dropseqrna.utils.GroupingIterator;
61 import org.broadinstitute.dropseqrna.utils.IntervalTagComparator;
62 import org.broadinstitute.dropseqrna.utils.ObjectCounter;
63 import org.broadinstitute.dropseqrna.utils.SamHeaderUtil;
64 import org.broadinstitute.dropseqrna.utils.SamWriterSink;
65 import org.broadinstitute.dropseqrna.utils.VCFUtils;
66 import org.broadinstitute.dropseqrna.utils.io.ErrorCheckingPrintStream;
67 import org.broadinstitute.dropseqrna.utils.readiterators.IgnoreGeneAnnotationTagger;
68 import org.broadinstitute.dropseqrna.utils.readiterators.PCRDuplicateFilteringIterator;
69 import org.broadinstitute.dropseqrna.utils.readiterators.SamFileMergeUtil;
70 import org.broadinstitute.dropseqrna.utils.readiterators.SamHeaderAndIterator;
71 import org.broadinstitute.dropseqrna.vcftools.SampleAssignmentVCFUtils;
72 import org.broadinstitute.dropseqrna.vcftools.filters.GenotypeGQFilter;
73
74 import htsjdk.samtools.SAMFileHeader;
75 import htsjdk.samtools.SAMFileWriter;
76 import htsjdk.samtools.SAMFileWriterFactory;
77 import htsjdk.samtools.SAMRecord;
78 import htsjdk.samtools.SAMSequenceDictionary;
79 import htsjdk.samtools.SamReaderFactory;
80 import htsjdk.variant.variantcontext.Genotype;
81 import htsjdk.variant.variantcontext.GenotypeType;
82 import htsjdk.variant.variantcontext.GenotypesContext;
83 import htsjdk.variant.variantcontext.VariantContext;
84 import htsjdk.variant.variantcontext.writer.Options;
85 import htsjdk.variant.variantcontext.writer.VariantContextWriter;
86 import htsjdk.variant.variantcontext.writer.VariantContextWriterBuilder;
87 import htsjdk.variant.variantcontext.writer.VariantContextWriterBuilder.OutputType;
88 import htsjdk.variant.vcf.VCFFileReader;
89 import htsjdk.variant.vcf.VCFHeader;
90 import picard.annotation.LocusFunction;
91 import picard.cmdline.StandardOptionDefinitions;
92 import picard.util.BasicInputParser;
93 import picard.util.TabbedTextFileWithHeaderParser;
94 import picard.util.TabbedTextFileWithHeaderParser.Row;
95
96 @CommandLineProgramProperties(summary = "For a set of cells, and a VCF File," + "calculate the most likely sample assignment for each cell. This is done by"
97 + "looking at each SNP/sample genotype, and calculating the likelihood of that result"
98 + "given the pileup of reads at that genomic location for each cell. The log likelihoods"
99 + "across all genomic locations for each cell are calculated and summed. "
100 + "Each cell then has a likelihood across all SNPs for each of the samples in the VCF file. The most likely sample"
101 + "is assigned to the cell, and the likelihood ratio of the sample assignment to the next best sample assignment is "
102 + "calculated to give us a sense of how confident we are in the assignment.", oneLineSummary = "Assign cells in a BAM to the most likely sample in a VCF", programGroup = DropSeq.class)
103
104 public class AssignCellsToSamples extends GeneFunctionCommandLineBase {
105
106 private static final Log log = Log.getInstance(AssignCellsToSamples.class);
107
108 @Argument(shortName = StandardOptionDefinitions.INPUT_SHORT_NAME, doc = "The input SAM or BAM file to analyze. This argument can accept wildcards, or a file with the suffix .bam_list that contains the locations of multiple BAM files", minElements = 1)
109 public List<File> INPUT_BAM;
110
111 @Argument(doc = "The input VCF file to analyze.")
112 public File VCF;
113
114 @Argument(shortName = StandardOptionDefinitions.OUTPUT_SHORT_NAME, doc = "Output file of sample assignments. This supports zipped formats like gz and bz2.")
115 public File OUTPUT;
116
117 @Argument(doc = "Verbose output of every cell/SNP result. This supports zipped formats like gz and bz2.", optional = true)
118 public File VERBOSE_OUTPUT;
119
120 @Argument(doc = "Verbose output of every cell/SNP result for the BEST donor. This supports zipped formats like gz and bz2.", optional = true)
121 public File VERBOSE_BEST_DONOR_OUTPUT = null;
122
123 @Argument(doc = "Output a version of the VCF containing only the variants that have coverage in the input BAM. All samples are retained. This file can be used as the VCF input file on subsequent runs of the same data set to greatly speed up the run, if the same BAM is being analyzed with different conditions. If you plan on calling doublets with DetectDoublets, you'll need this file.", optional = true)
124 public File VCF_OUTPUT;
125
126 @Argument(doc = "Output a version of the BAM containing only the reads that have coverage in the input BAM and VCF.", optional = true)
127 public File BAM_OUTPUT;
128
129 @Argument(doc = "The cell barcode tag. If there are no reads with this tag, the program will assume that all reads belong to the same cell and process in single sample mode.")
130 public String CELL_BARCODE_TAG = "XC";
131
132 @Argument(doc = "The molecular barcode tag.")
133 public String MOLECULAR_BARCODE_TAG = "XM";
134
135 @Argument(doc = "The functional annotation for the read. If set, extracts the functional annotation(s) [CODING/UTR/etc] at each SNP position and outputs in the verbose output.")
136 public String FUNCTION_TAG = "XF";
137
138 @Argument(doc = "The edit distance that molecular barcodes should be combined at within a gene/SNP.")
139 public Integer EDIT_DISTANCE = 1;
140
141 @Argument(doc = "The map quality of the read to be included.")
142 public Integer READ_MQ = 10;
143
144 @Argument(doc = "The minimum genotype quality for a variant. Set this value to 0 to not filter by GQ scores if they are present, or to -1 to completely "
145 + "ignore GQ values if they are not set in the genotype info field. If the GQ field is not set in the VCF header, this will be set to -1 by default.")
146 public Integer GQ_THRESHOLD = 30;
147
148 @Argument(doc = "Number of cells that you want to extract from the BAM. The program will picks the top <NUM_BARCODES> barcodes by read count.", mutex = {
149 "CELL_BC_FILE" })
150 public Integer NUM_BARCODES = null;
151
152 @Argument(doc = "Override NUM_CORE_BARCODES and process reads that have the cell barcodes in this file instead. The file has 1 column with no header.", mutex = {
153 "NUM_BARCODES" })
154 public File CELL_BC_FILE = null;
155
156 @Argument(doc = "A file with a list of samples in the VCF to match up to cells in the BAM. This subsets the VCF into a smaller data set containing only the samples listed. The file has 1 column with no header.", optional = true)
157 public File SAMPLE_FILE = null;
158
159 @Argument(doc = "This file provides an answer key for the input BAM and VCF. This does not change any of the likelihood analysis results, but appends an additional column to the output that contains the known sample assignment for a cell."
160 + "This is useful when assessing how well the method works, but is completely optional. The format of the file is 2 tab seperated columns with a header [cell sample]. The first column contains the cell barcodes from the BAM, "
161 + "the second the sample assignments from the VCF.", optional = true)
162 public File ANSWER_KEY_FILE = null;
163
164 @Argument(doc = "A file that contains an estimate of how much ambient RNA is in each cell. This is a fractional estimate between 0 and 1. File is tab seperated, with 2 columns:"
165 + "cell_barcode and frac_contamination. When supplied along side the ALLELE_FREQUENCY_ESTIMATE_FILE, this modifies the likelihood error rates to take into account how often"
166 + "the allele observed can be drawn from ambient RNA. We use cellbender remove background [https://github.com/broadinstitute/CellBender] to estimate the "
167 + "number of transcripts before and after ambient cleanup to define the fraction of transcripts that come from ambient RNA.", optional = true)
168 public File CELL_CONTAMINATION_ESTIMATE_FILE = null;
169
170 @Argument(doc = "A file that contains an estimate of the allele frequency expected for each SNP across donors. The best estimate of this will come from the fraction of reference and alternate allele"
171 + "UMIs that are observed at each snp site. This report can be generated via GatherDigitalAlleleCounts. This is a fractional estimate between 0 and 1. File is tab seperated, with at least 3 columns:"
172 + "chromosome, position, maf_umi. When supplied and CELL_CONTAMINATION_ESTIMATE_FILE is provided, this modifies the likelihood error rates to take into account how often "
173 + "the allele observed can be drawn from ambient RNA.", optional = true)
174 public File ALLELE_FREQUENCY_ESTIMATE_FILE = null;
175
176 private final String SNP_TAG = "YS";
177
178 @Argument(doc = "Should monomorphic SNPs across the population be retained?")
179 public boolean RETAIN_MONOMORPIC_SNPS = false;
180
181 @Argument(doc = "At least <FRACTION_SAMPLES_PASSING> samples must have genotype scores >= GQ_THRESHOLD for the variant in the VCF to be included in the analysis.")
182 public double FRACTION_SAMPLES_PASSING = 0.5;
183
184 @Argument(doc = "A list of chromosomes to omit from the analysis. The default is to omit the sex chromosomes.")
185 public List<String> IGNORED_CHROMOSOMES = new ArrayList<>(Arrays.asList("X", "Y", "MT"));
186
187 // to mimic the initial R implementation.
188 @Argument(doc = "Instead of useing base qualities to determine error rate, use a fixed error rate instead. This is rounded to the nearest phread score internally.", optional = true)
189 public Double FIXED_ERROR_RATE = null;
190
191 @Argument(doc = "Caps the base error rate at a maximum probability so no SNP can be weighed more than this value. For example, if this value was 0.01, "
192 + "then a base quality 30 value (normally an erro rate of 0.001) would become 0.01. With the same threshold, a base with an error rate of 0.1 would be unaffected.", optional = true)
193 public Double MAX_ERROR_RATE = null;
194
195 // on a per-snp basis, generate a mixture of all the data across known samples to fill in likelihoods for the missing
196 // samples.
197 @Argument(doc = "on a per-snp basis, generate a mxiture of all the data across known samples to fill in likelihoods for the missing samples.")
198 public Boolean ADD_MISSING_VALUES = true;
199
200 private Boolean EMIT_POPULATION_LIKELIHOOD = true;
201
202 @Argument(doc = "EXPERIMENTAL!!! Run the program in DNA Mode. In this mode, reads should have a cell barcode, but will be missing gene annotations and UMIs. All reads will be "
203 + "accepted as passing, and each read (or read pair) will be treated as a single UMI If the data is PCR Duplicate marked, duplicate reads will be filtered. ")
204 public Boolean DNA_MODE = false;
205
206 @Argument(doc="Logging interval for SNP sequence pileup processing")
207 public Integer SNP_LOG_RATE=1000;
208
209 @Argument(doc="(Optional) If set to a value, this program will quit early with an exception if this number of UMIs are observed without encountering a transcribed SNP - "
210 + "that is, a variant from the VCF file that is will be informative during donor assignment. A value of -1 disables this test. As some fraction of UMIs may not"
211 + "contain transcribed SNPs, the value for this should probably be set to a fairly large number, like 10% of the total UMIs in your experiment. Mostly useful for debugging / or "
212 + "testing new data sets.")
213 public Integer TRANSCRIBED_SNP_FAIL_FAST_THRESHOLD=-1;
214
215 // This isn't fully implemented yet so is disabled.
216 // @Argument(doc = "on a per-genotype basis, use the GQ score to weigh how much we believe the data given genotype.
217 // Weights the likelihood (D|G) by the GQ score, which is converted from phred quality to a probability [0-1].")
218 // public Boolean USE_GENOTYPE_QUALITY=false;
219
220 // private Map <String,String> testMap = new HashMap<>();
221
222 @Override
223 protected int doWork() {
224 // get the verbose output writer if not null, or return null.
225 PrintStream verboseWriter = getVerboseWriter(this.VERBOSE_OUTPUT);
226
227 final VCFFileReader vcfReader = new VCFFileReader(this.VCF, true);
228
229 SamHeaderAndIterator headerAndIter = SamFileMergeUtil.mergeInputs(this.INPUT_BAM, false, SamReaderFactory.makeDefault());
230 AssertSequenceDictionaryIntersection.assertIntersectionObjectVcf(headerAndIter.header, "BAM INPUT(S)", this.VCF, log);
231
232 // disable GQ filter if it's not in the header.
233 if (!VCFUtils.GQInHeader(vcfReader)) {
234 this.GQ_THRESHOLD = -1;
235 log.info("Genotype Quality [GQ] not found in header. Disabling GQ_THRESHOLD parameter");
236 }
237
238 // subsetting the VCF by samples isn't generally needed for the standard way to run AssignCellsToSamples, but is super
239 // expensive to run. We need to get around this.
240 List<String> vcfSamples = SampleAssignmentVCFUtils.getVCFSamples(vcfReader, this.SAMPLE_FILE);
241
242 // set up the VCF writer if needed.
243 VariantContextWriter vcfWriter = getVcfWriter(vcfReader, vcfSamples);
244
245 // populate the answer key if appropriate
246 Map<String, String> answerKey = readAnswerKey();
247
248 // set up the verbose output if needed
249 writeVerboseOutputHeader(vcfSamples, answerKey, this.CELL_CONTAMINATION_ESTIMATE_FILE, this.ALLELE_FREQUENCY_ESTIMATE_FILE, verboseWriter);
250
251 // get from the VCF!
252 final SNPInfoCollection snpIntervals = SampleAssignmentVCFUtils.getSNPInfoCollection(this.VCF, vcfSamples, this.RETAIN_MONOMORPIC_SNPS, this.GQ_THRESHOLD,
253 this.FRACTION_SAMPLES_PASSING, IGNORED_CHROMOSOMES, null, false, log);
254
255 // a requested early exit if there are no SNPs.
256 if (snpIntervals.isEmpty()) {
257 log.error("No SNP intervals detected! Check to see if your VCF filter thresholds are too restrictive!");
258 return 1;
259 }
260
261 // this is the same method of filtering used to generate the snpIntervals. Don't ask for a progress logger, we'll use
262 // the JointIteratorCounter below.
263 // final PeekableIterator<VariantContext> vcfIterator = getVCFIterator (vcfReader, vcfSamples,
264 // this.RETAIN_MONOMORPIC_SNPS, this.log);
265 PeekableIterator<VariantContext> vcfIterator = SampleAssignmentVCFUtils.getVCFIterator(vcfReader, vcfSamples, this.RETAIN_MONOMORPIC_SNPS,
266 this.GQ_THRESHOLD, this.FRACTION_SAMPLES_PASSING, IGNORED_CHROMOSOMES, log);
267
268 // and the BAM iterator.
269 List<String> barcodes = getCellBarcodes();
270 PeekableIterator<List<SampleGenotypeProbabilities>> sampleGenotypeIterator = prepareIterator(snpIntervals, barcodes);
271
272 // load up the contamination and MAF if available.
273 Map<String, Double> contaminationMap = CellAssignmentUtils.getCellContamination(this.CELL_CONTAMINATION_ESTIMATE_FILE, barcodes);
274 Map<Interval, Double> variantMinorAlleleFrequency = CellAssignmentUtils.getMinorAlleleFrequencyMap(this.ALLELE_FREQUENCY_ESTIMATE_FILE);
275
276 // prepare the collection of likelihoods and other metrics
277 CellCollectionSampleLikelihoodCollection likelihoodCollection = initializeLikelihoodCollection(vcfSamples, contaminationMap,
278 variantMinorAlleleFrequency);
279
280 // some counters for information on what's being iterated on. Every sample genotype result *should* match up to a VCF
281 // entry
282 // so we expect those numbers to be the same at the end.
283 JointIteratorCounter counter = new JointIteratorCounter();
284
285 // We need the sequence dictionary in the interval list for all sorting of chromosomes.
286 SAMSequenceDictionary dict = snpIntervals.getIntervalList().getHeader().getSequenceDictionary();
287
288 // TODO: should be able to plug VCFPileupJointIterator <T extends SNPIntervalRecordI>
289 // Would have to wrap SampleGenotypeProbabilities in a collection that extends SNPIntervalRecordI for this to work,
290 // yuck.
291
292 // walk through both the VCF and the BAM at the same time.
293 // the sampleGenotypeIterator gives all results for a SNP, so that's the results for every cell that had reads on the
294 // SNP.
295 while (vcfIterator.hasNext() && sampleGenotypeIterator.hasNext()) {
296 List<SampleGenotypeProbabilities> pileUpsSingleSNP = sampleGenotypeIterator.peek();
297 // only have to compare the first record.
298 int cmp = compareRecords(pileUpsSingleSNP.get(0), vcfIterator.peek(), dict);
299 if (cmp < 0) {
300 sampleGenotypeIterator.next();
301 counter.SAMPLE_GENE_ITER++;
302 } else if (cmp > 0) {
303 vcfIterator.next();
304 counter.VCF_ITER++;
305 } else if (cmp == 0) {
306 counter.BOTH++;
307 counter.SAMPLE_GENE_ITER++;
308 counter.VCF_ITER++;
309 if (counter.BOTH % this.SNP_LOG_RATE == 0)
310 log.info("Processed [" + counter.BOTH + "] SNPs in BAM + VCF");
311
312 // grab the next record and process it.
313 VariantContext vc = vcfIterator.next();
314 // write the VCF record out if needed.
315 if (vcfWriter != null)
316 vcfWriter.add(vc);
317 pileUpsSingleSNP = sampleGenotypeIterator.next();
318 likelihoodCollection = processSNP(vc, pileUpsSingleSNP, likelihoodCollection, ADD_MISSING_VALUES);
319 if (verboseWriter != null) {
320 writeVerboseOutput(vc, pileUpsSingleSNP, answerKey, vcfSamples, contaminationMap, variantMinorAlleleFrequency, verboseWriter);
321 }
322
323 if (counter.SAMPLE_GENE_ITER != counter.BOTH) {
324 log.error("Expected counts don't match: SGICounter [" + counter.SAMPLE_GENE_ITER + "] both counter [" + counter.BOTH + "]");
325 log.error(
326 "VCF and BAM iteration have gone out of sync. Iteration relies on the sequence dictonary order of the VCF file - make sure the body of the VCF is sorted in the order dictated by the VCF header.");
327 vcfIterator.close();
328 return 1;
329 }
330 }
331 }
332 log.info("Processed [" + counter.BOTH + "] SNPs in BAM + VCF");
333
334 if (counter.SAMPLE_GENE_ITER != counter.BOTH)
335 log.info("Expected counts don't match: SGICounter [" + counter.SAMPLE_GENE_ITER + "] both counter [" + counter.BOTH + "]");
336
337 vcfIterator.close();
338 vcfReader.close();
339 if (vcfWriter != null)
340 vcfWriter.close();
341 sampleGenotypeIterator.close();
342
343 if (this.VERBOSE_OUTPUT != null)
344 verboseWriter.close();
345
346 // write out results.
347 ErrorCheckingPrintStream out = new ErrorCheckingPrintStream(IOUtil.openFileForWriting(this.OUTPUT));
348 addMetaDataToHeader(out);
349 writeBestLikelihoods(vcfSamples, likelihoodCollection, answerKey, out, this.EMIT_POPULATION_LIKELIHOOD);
350 out.close();
351
352 // filter out the VERBOSE results if required.
353 if (this.VERBOSE_BEST_DONOR_OUTPUT != null) {
354 log.info("Writing Verbose Best Donor Report");
355 filterVerboseToBest(this.VERBOSE_OUTPUT, this.VERBOSE_BEST_DONOR_OUTPUT, likelihoodCollection);
356 }
357 log.info("Finished!");
358 return 0;
359 }
360
361 // helper method to deal with parsing input file if it exists.
362 private CellCollectionSampleLikelihoodCollection initializeLikelihoodCollection(List<String> vcfSamples, final Map<String, Double> contaminationMap,
363 Map<Interval, Double> variantMinorAlleleFrequency) {
364 if (contaminationMap == null)
365 return (new CellCollectionSampleLikelihoodCollection(FIXED_ERROR_RATE, this.MAX_ERROR_RATE, vcfSamples));
366 return new CellCollectionSampleLikelihoodCollection(contaminationMap, variantMinorAlleleFrequency, vcfSamples);
367 }
368
369 /**
370 * Takes the
371 *
372 * @param verboseOutput
373 * @param bestDonorVerboseOutput
374 * @param likelihoodCollection
375 */
376 static void filterVerboseToBest(final File verboseOutput, final File bestDonorVerboseOutput,
377 CellCollectionSampleLikelihoodCollection likelihoodCollection) {
378 Map<String, String> bestDonorPerCell = new HashMap<String, String>();
379 // get the best donor per cell.
380 likelihoodCollection.getCellBarcodes().stream()
381 .forEach(x -> bestDonorPerCell.put(x, likelihoodCollection.getLikelihoodCollection(x).getBestSampleAssignment().getSample()));
382
383 // get the list of all samples
384 // Collection<String> donorList = likelihoodCollection.getSamples();
385
386 TabbedTextFileWithHeaderParser parser = new TabbedTextFileWithHeaderParser(verboseOutput);
387 // Set<String> colNames = parser.getColumnNames();
388
389 CloseableIterator<Row> rowIter = parser.iterator();
390 log.info("Processing verbose donor assignment file to generate best donor output");
391 ProgressLogger pl = new ProgressLogger(log, 1000000, "Processed");
392
393 // prepare the writer.
394 PrintStream writer = new ErrorCheckingPrintStream(IOUtil.openFileForWriting(bestDonorVerboseOutput));
395
396 // fixed fields we want to extract.
397 List<String> fixedFields = new ArrayList<>();
398 fixedFields.addAll(Arrays.asList("chr", "pos", "gene", "function", "cell"));
399 for (Bases b : Bases.values())
400 fixedFields.add(b.toString());
401 fixedFields.addAll(Arrays.asList("num_umi", "REF", "ALT"));
402
403 // the output header
404 List<String> outputHeader = new ArrayList<>(fixedFields);
405
406 boolean outputPopAverage = false;
407 boolean outputMaf = false;
408 if (parser.getColumnNames().contains("population_average_likelihood"))
409 outputPopAverage = true;
410 if (parser.getColumnNames().contains("minor_allele_freq"))
411 outputMaf = true;
412
413 outputHeader.addAll(Arrays.asList("best_donor", "genotype", "log10_likelihood"));
414 if (outputPopAverage)
415 outputHeader.add("population_average_likelihood");
416 if (outputMaf)
417 outputHeader.add("minor_allele_freq");
418
419 writer.println(StringUtils.join(outputHeader, "\t"));
420
421 // write the body.
422 while (rowIter.hasNext()) {
423 Row r = rowIter.next();
424 pl.record(r.getField("chr"), Integer.parseInt(r.getField("pos")));
425
426 List<String> line = new ArrayList<>();
427 // get all the fixed fields into the output.
428 for (String field : fixedFields) {
429 line.add(r.getField(field));
430 }
431 // get the best donor for this cell and output the likelihood for their genotype
432 String cell = r.getField("cell");
433 String bestDonor = bestDonorPerCell.get(cell);
434 String genotype = r.getField(bestDonor);
435 // This is slightly lame, but I'm currently encoding genotypes as "null" in the verbose output, and
436 // that's getting parsed as an actual null, so need to catch it.
437 if (genotype == null)
438 genotype = "NA";
439 line.add(bestDonor);
440 line.add(genotype);
441
442 switch (genotype) {
443 case "ref":
444 line.add(r.getField("ref_loglikelihood"));
445 break;
446 case "het":
447 line.add(r.getField("het_loglikelihood"));
448 break;
449 case "alt":
450 line.add(r.getField("alt_loglikelihood"));
451 break;
452 default:
453 line.add("NA");
454 }
455 if (outputPopAverage)
456 line.add(r.getField("population_average_likelihood"));
457 if (outputMaf)
458 line.add(r.getField("minor_allele_freq"));
459 writer.println(StringUtils.join(line, "\t"));
460 }
461 CloserUtil.close(parser);
462 writer.close();
463 }
464
465 static void writeBestLikelihoods(final List<String> samples, final CellCollectionSampleLikelihoodCollection likelihoodCollection,
466 final Map<String, String> answerKey, final ErrorCheckingPrintStream out, boolean emitPopulationLikelihood) {
467 // write header
468 writeBestLikelihoodsHeader(samples, answerKey, out, emitPopulationLikelihood);
469
470 // write body
471 DecimalFormat likeFormat = new DecimalFormat("0.####E0");
472
473 Map<String, Double> fdr = likelihoodCollection.getFDRCorrectedPvalues();
474
475 // int counter=0;
476 for (String cellBarcode : likelihoodCollection.getCellBarcodes()) {
477
478 CellSampleLikelihoodCollection cslc = likelihoodCollection.getLikelihoodCollection(cellBarcode);
479 List<String> line = new ArrayList<>();
480 line.add(cellBarcode);
481 // the number of snp and umi observations
482 line.add(Integer.toString(cslc.getNumSNPs()));
483 line.add(Integer.toString(cslc.getNumUMIs()));
484
485 // write best sample info.
486 BestSampleAssignmentForCell b = cslc.getBestSampleAssignment();
487 line.add(likeFormat.format(b.getLogLikelihoodRatio()));
488 line.add(likeFormat.format(b.getOneMinusPvalue()));
489 line.add(likeFormat.format(fdr.get(cellBarcode)));
490 line.add(likeFormat.format(b.getBestLoglikelihood()));
491 line.add(b.getSample());
492 line.add(likeFormat.format(cslc.getMedianLikelihood()));
493 if (emitPopulationLikelihood)
494 line.add(likeFormat.format(cslc.getGlobalPenaltyScore()));
495 if (answerKey != null) {
496 String answer = answerKey.get(cellBarcode);
497 if (answer == null)
498 line.add("NA");
499 else
500 line.add(answer);
501 }
502
503 // add each sample's likelihood.
504
505 for (String sample : samples)
506 if (cslc.isSampleInCollection(sample)) {
507 Double l = cslc.getLoglikelihood(sample);
508 line.add(likeFormat.format(l));
509 } else
510 line.add("NA");
511
512 String h = StringUtils.join(line, "\t");
513 out.println(h);
514 }
515 }
516
517 private void addMetaDataToHeader(final PrintStream out) {
518
519 List<String> paths = this.INPUT_BAM.stream().map(x -> x.getAbsolutePath()).collect(Collectors.toList());
520
521 String bamList = StringUtils.join(paths, ",");
522 List<String> header = new ArrayList<>(Arrays.asList("#INPUT_BAM=" + bamList, "INPUT_VCF=" + this.VCF.getAbsolutePath(),
523 "DONOR_FILE=" + this.SAMPLE_FILE, "CELL_BC_FILE=" + CELL_BC_FILE, "GQ_THRESHOLD=" + Integer.toString(this.GQ_THRESHOLD),
524 "FRACTION_SAMPLES_PASSING=" + Double.toString(this.FRACTION_SAMPLES_PASSING), "READ_MQ=" + Integer.toString(this.READ_MQ),
525 "FIXED_ERROR_RATE=" + CellAssignmentUtils.convertNullToString(this.FIXED_ERROR_RATE),
526 "MAX_ERROR_RATE=" + CellAssignmentUtils.convertNullToString(this.MAX_ERROR_RATE), "LOCUS_FUNCTION=" + this.LOCUS_FUNCTION_LIST.toString()));
527
528 if (this.CELL_CONTAMINATION_ESTIMATE_FILE != null && this.ALLELE_FREQUENCY_ESTIMATE_FILE != null) {
529 header.add("CELL_CONTAMINATION_ESTIMATE_FILE=" + CELL_CONTAMINATION_ESTIMATE_FILE.getAbsolutePath());
530 header.add("ALLELE_FREQUENCY_ESTIMATE_FILE=" + ALLELE_FREQUENCY_ESTIMATE_FILE.getAbsolutePath());
531 }
532
533 String h = StringUtils.join(header, "\t");
534 out.println(h);
535 }
536
537 static void writeBestLikelihoodsHeader(final List<String> samples, final Map<String, String> answerKey, final PrintStream out,
538 boolean emitPopulationLikelihood) {
539 List<String> line = new ArrayList<>();
540 line.add("cell");
541 line.add("num_snps");
542 line.add("num_umis");
543 line.add("ratio");
544 line.add("pvalue");
545 line.add("FDR_pvalue");
546 line.add("bestLikelihood");
547 line.add("bestSample");
548 line.add("median_likelihood");
549 if (emitPopulationLikelihood)
550 line.add("population_average_likelihood");
551 if (answerKey != null)
552 line.add("trueSample");
553
554 line.addAll(samples);
555 String h = StringUtils.join(line, "\t");
556 out.println(h);
557 }
558
559 CellCollectionSampleLikelihoodCollection processSNP(final VariantContext vc, final List<SampleGenotypeProbabilities> sgpList,
560 final CellCollectionSampleLikelihoodCollection likelihoodCollection, final boolean addMissingValues) {
561 Set<String> sampleNames = vc.getSampleNames();
562 return processSNP(vc, sgpList, sampleNames, likelihoodCollection, addMissingValues);
563 }
564
565 /**
566 * @param b byte value of A,C, G or T
567 * @return very compact representation of A, C, G or T (2 bits)
568 */
569 private static int getCompactBaseRepresentation(byte b) {
570 switch (b) {
571 case SequenceUtil.A:
572 return 0;
573 case SequenceUtil.C:
574 return 1;
575 case SequenceUtil.G:
576 return 2;
577 case SequenceUtil.T:
578 return 3;
579 default:
580 throw new IllegalArgumentException(String.format("Unexpected base %d", b));
581 }
582 }
583
584 /**
585 *
586 * @param a1 byte value of A,C, G or T
587 * @param a2 byte value of A,C, G or T
588 * @return very compact representation of this ordered pair of alleles (4 bits)
589 */
590 private static int getAllelesToGroupBy(final byte a1, final byte a2) {
591 return (getCompactBaseRepresentation(a1) << 2) | getCompactBaseRepresentation(a2);
592 }
593
594 // Return pair of alleles in compact form for Collectors.groupingBy
595 private static int getAllelesToGroupBy(final Genotype g) {
596 return getAllelesToGroupBy(g.getAllele(0).getBases()[0], g.getAllele(1).getBases()[0]);
597 }
598
599 // We know that T,T is the largest value in the compact rep.
600 private static final int NumCompactAlleleRepresentations =
601 getAllelesToGroupBy(SequenceUtil.T, SequenceUtil.T) + 1;
602
603 // Recycle to avoid GC
604 // The index into these arrays is produced by getAllelesToGroupBy()
605 private final Genotype[] prototypeGenotypeForAllelePair = new Genotype[NumCompactAlleleRepresentations];
606 // Once a list is created, it is not destroyed but rather cleared for the next SNP.
607 // However, if prototypeGenotypeForAllelePair[i] is null, then samplesForAllelePair[i] may contain sample names
608 // from a previous iteration that are no longer relevant.
609 private final List<String>[] samplesForAllelePair = new List[NumCompactAlleleRepresentations];
610
611 /**
612 * This updates the likelihood all cells that have one or more UMIs at this variant.
613 *
614 * @param vc
615 * @param sgpList
616 * @param sampleNames
617 * @param likelihoodCollection
618 * @param addMissingValues
619 * @return
620 */
621 CellCollectionSampleLikelihoodCollection processSNP(final VariantContext vc, final List<SampleGenotypeProbabilities> sgpList, final Set<String> sampleNames,
622 CellCollectionSampleLikelihoodCollection likelihoodCollection, final boolean addMissingValues) {
623 GenotypesContext gc = vc.getGenotypes(sampleNames);
624 Iterator<Genotype> gi = gc.iterator();
625 GenotypeGQFilter gf = new GenotypeGQFilter(gi, this.GQ_THRESHOLD);
626 char refAlleleBase = StringUtil.byteToChar(vc.getReference().getBases()[0]);
627
628 // Primitive implementation of grouping together the samples that have the same pair of alleles for this SNP.
629 // This way has many fewer objects created than groupBy.
630
631 // This array holds the first Genotype object encountered for a pair of alleles. The index into the
632 // array is produced by getAllelesToGroupBy()
633 Arrays.fill(prototypeGenotypeForAllelePair, null);
634 // Don't bother clearing samplesForAllelePair globally. Just clear the entries for allele pairs
635 // encountered for this SNP.
636
637 // Capture one Genotype object for each distinct pair of alleles, and get the list of
638 // sample names for each distinct pair of alleles.
639 for (final Genotype g: gf) {
640 final int compactAlleleRep = getAllelesToGroupBy(g);
641 if (prototypeGenotypeForAllelePair[compactAlleleRep] == null) {
642 prototypeGenotypeForAllelePair[compactAlleleRep] = g;
643 if (samplesForAllelePair[compactAlleleRep] == null) {
644 samplesForAllelePair[compactAlleleRep] = new ArrayList<>(sampleNames.size());
645 } else {
646 samplesForAllelePair[compactAlleleRep].clear();
647 }
648 }
649 samplesForAllelePair[compactAlleleRep].add(g.getSampleName());
650 }
651
652 // batch update all donors that have the same genotype.
653 // map allele pair to list of Genotype.
654 // then batch update likelihoods on the list of sample names.
655 // this would be a bad plan if we used genotype quality, where each sample could have a different genotype quality
656 // score.
657 for (int i = 0; i < NumCompactAlleleRepresentations; ++i) {
658 if (prototypeGenotypeForAllelePair[i] == null) {
659 continue;
660 }
661 final Genotype g = prototypeGenotypeForAllelePair[i];
662 byte[] a1Bases = g.getAllele(0).getBases();
663 byte[] a2Bases = g.getAllele(1).getBases();
664 char a1 = StringUtil.byteToChar(a1Bases[0]);
665 char a2 = StringUtil.byteToChar(a2Bases[0]);
666 final List<String> sampleNamesForAlleles = samplesForAllelePair[i];
667 // this isn't fully implemented yet so is disabled.
668 /*
669 * Double genotypeProbability=null; if (USE_GENOTYPE_QUALITY) { Integer genotypeQuality = g.getGQ();
670 * genotypeProbability=LikelihoodUtils.getInstance().phredScoreToErrorProbability(genotypeQuality.byteValue()); }
671 */
672 // for this sample genotype, update all the cells where we have data.
673 for (SampleGenotypeProbabilities p : sgpList) {
674 likelihoodCollection.updatelikelihoods(p, sampleNamesForAlleles, a1, a2, null, refAlleleBase);
675 }
676
677 }
678
679 CloserUtil.close(gf);
680
681 // update number of SNPs and umis observed once per SNP in the VCF.
682 for (SampleGenotypeProbabilities p : sgpList)
683 likelihoodCollection.updateNumObservations(p);
684
685 // calculate the global penalty score. Optionally apply this score to donor likelihoods for a cell/snp where the donor
686 // does not have a genotype set.
687 likelihoodCollection = calculateMissingLikelihoodOptionallyApply(vc, sgpList, sampleNames, likelihoodCollection, addMissingValues);
688 return likelihoodCollection;
689
690 }
691
692 /**
693 * For a SNP, runs this process against all cells:
694 *
695 * Calculates the population-wide likelihood score, computed as the weighted average likelihood of the hom/het/var
696 * likelihoods weighted by all the donors at this SNP that have valid genotypes This population wide penalty score is
697 * incremented for every cell Optionally, this score is used as the likelihood score for all donors that do not have a
698 * valid genotype at this SNP.
699 *
700 * @param vc The current SNP from the VCF
701 * @param sgpList The list of SNP pileups objects - one per cell.
702 * @param sampleNames The original set of samples to process from the VCF.
703 * @param likelihoodCollection The set of likelihood scores that are iteratively computed. This object will be updated
704 * and returned.
705 * @param applyToMissingDonors Set to true to assign the population wide likelihood score to all donors that don't have
706 * a genotype at this SNP.
707 * @return The updated likelihood scores and other tracking metrics across snps / donors / cells.s
708 */
709 CellCollectionSampleLikelihoodCollection calculateMissingLikelihoodOptionallyApply(final VariantContext vc, final List<SampleGenotypeProbabilities> sgpList,
710 final Set<String> sampleNames, final CellCollectionSampleLikelihoodCollection likelihoodCollection, boolean applyToMissingDonors) {
711
712 // this class simplifies out a lot of code that was here.
713 GenotypeCountMetrics gm = new GenotypeCountMetrics(vc, sampleNames, this.GQ_THRESHOLD);
714 Double genotypeProbability = null;
715
716 /*
717 * if (this.USE_GENOTYPE_QUALITY) { // the mean of the genotype qualities? if (meanGenotypeQuality.getN()==0)
718 * log.info("STOP"); double meanGQ=gm.getMeanGenotypeQuality();
719 * genotypeProbability=LikelihoodUtils.getInstance().phredScoreToErrorProbability(Double.valueOf(meanGQ).byteValue()); }
720 */
721
722 // for each cell, update the samples where we have no genotype assigned for this variant.
723 for (SampleGenotypeProbabilities p : sgpList) {
724 double globalPenaltyLogLikelihood = likelihoodCollection.getMissingLikelihood(p, gm.getRefAllele(), gm.getAltAllele(), gm.getRefCount(),
725 gm.getHetCount(), gm.getAltCount(), genotypeProbability);
726 likelihoodCollection.incrementGlobalPenaltyAllDonors(p, globalPenaltyLogLikelihood);
727 if (applyToMissingDonors)
728 likelihoodCollection.applyMissingLikelihoodToDonors(p, globalPenaltyLogLikelihood, gm.getSamplesNotObserved());
729 }
730 return (likelihoodCollection);
731 }
732
733 private void writeVerboseOutputHeader(final List<String> sampleNames, final Map<String, String> answerKey, File contaminationMap,
734 File variantMinorAlleleFrequency, final PrintStream out) {
735 if (out == null)
736 return;
737 addMetaDataToHeader(out);
738 List<String> line = new ArrayList<>();
739 line.add("chr");
740 line.add("pos");
741 line.add("gene");
742 if (this.FUNCTION_TAG != null)
743 line.add("function");
744 line.add("cell");
745 for (Bases b : Bases.values())
746 line.add(b.toString());
747 line.add("num_umi");
748 line.add("REF");
749 line.add("ALT");
750 line.add("ref_loglikelihood");
751 line.add("het_loglikelihood");
752 line.add("alt_loglikelihood");
753 line.add("population_average_likelihood");
754 line.add("minor_allele_freq");
755 if (contaminationMap != null)
756 line.add("contamination");
757 if (variantMinorAlleleFrequency != null)
758 line.add("umi_maf");
759
760 if (answerKey != null)
761 line.add("knownSample");
762
763 for (String sample : sampleNames)
764 line.add(sample);
765
766 String h = StringUtils.join(line, "\t");
767 out.println(h);
768 }
769
770 private String getLocusFunctionString(final SampleGenotypeProbabilities p) {
771 Set<LocusFunction> lfSet = p.getLocusFunctions();
772 if (lfSet == null || lfSet.size() == 0)
773 return "";
774
775 StringBuilder b = new StringBuilder();
776
777 Iterator<LocusFunction> iter = lfSet.iterator();
778 b.append(iter.next());
779
780 while (iter.hasNext()) {
781 b.append(",");
782 b.append(iter.next());
783 }
784 return b.toString();
785
786 }
787
788 private void writeVerboseOutput(final VariantContext vc, final List<SampleGenotypeProbabilities> sgpList, final Map<String, String> answerKey,
789 final List<String> sampleNames, Map<String, Double> contaminationMap, Map<Interval, Double> variantMinorAlleleFrequency, final PrintStream out) {
790 if (out == null)
791 return;
792 DecimalFormat likeFormat = new DecimalFormat("0.######################");
793 DecimalFormat mafFormat = new DecimalFormat("0.###");
794
795 // This is expensive, but I'm not sure of a better way to get at the global penalty score per variant.
796 // TODO: rework the code so this doesn't need to be computed once during aggregation and once here.
797 CellCollectionSampleLikelihoodCollection likelihoodCollection = new CellCollectionSampleLikelihoodCollection(FIXED_ERROR_RATE, this.MAX_ERROR_RATE,
798 sampleNames);
799 likelihoodCollection = processSNP(vc, sgpList, likelihoodCollection, ADD_MISSING_VALUES);
800
801 // get genotype information
802 GenotypeCountMetrics gm = new GenotypeCountMetrics(vc, sampleNames, this.GQ_THRESHOLD);
803 double maf = gm.getMinorAlleleFrequency();
804
805 // each cell is one line for the same SNP.
806 for (SampleGenotypeProbabilities p : sgpList) {
807 List<String> line = new ArrayList<>();
808
809 Double contamination = CellAssignmentUtils.getNullableValue(contaminationMap, p.getCell());
810 Double umiMaf = CellAssignmentUtils.getNullableValue(variantMinorAlleleFrequency, p.getSNPInterval());
811
812 line.add(p.getSNPInterval().getContig());
813 line.add(Integer.toString(p.getSNPInterval().getStart()));
814
815 /*
816 for (SNPUMIBasePileup pileup: p.getBackingPileups()) {
817 log.info(pileup);
818 }
819 */
820
821 String gene = p.getBackingPileups().iterator().next().getGene();
822 line.add(gene);
823
824 if (this.FUNCTION_TAG != null)
825 line.add(getLocusFunctionString(p));
826
827 line.add(p.getCell());
828
829 // iterate over bases and get #umis
830 ObjectCounter<Character> umiCount = p.getUMIBaseCounts();
831
832 for (Bases b : Bases.values()) {
833 int n = umiCount.getCountForKey(b.getBase());
834 line.add(Integer.toString(n));
835 }
836 line.add(Integer.toString(umiCount.getTotalCount()));
837 // these work only because we only allow single base reference/alternate alleles
838 char refBase = StringUtil.byteToChar(vc.getReference().getBases()[0]);
839 char altBase = CellAssignmentUtils.getAltAllele(vc);
840
841 line.add(String.valueOf(refBase));
842 line.add(String.valueOf(altBase));
843
844 // TODO is there a way to plug in if the genotype quality is used? It's hardcoded off right now, but the feature is
845 // disabled.
846 double likelihoodRef = p.getLogLikelihood(refBase, refBase, this.FIXED_ERROR_RATE, null, this.MAX_ERROR_RATE, refBase, umiMaf, contamination);
847 double likelihoodHet = p.getLogLikelihood(refBase, altBase, this.FIXED_ERROR_RATE, null, this.MAX_ERROR_RATE, refBase, umiMaf, contamination);
848 double likelihoodAlt = p.getLogLikelihood(altBase, altBase, this.FIXED_ERROR_RATE, null, this.MAX_ERROR_RATE, refBase, umiMaf, contamination);
849
850 line.add(likeFormat.format(likelihoodRef));
851 line.add(likeFormat.format(likelihoodHet));
852 line.add(likeFormat.format(likelihoodAlt));
853
854 double populationPenalty = likelihoodCollection.getLikelihoodCollection(p.getCell()).getGlobalPenaltyScore();
855 line.add(likeFormat.format(populationPenalty));
856 line.add(mafFormat.format(maf));
857
858 if (contaminationMap != null)
859 line.add(mafFormat.format(contamination));
860 if (variantMinorAlleleFrequency != null)
861 line.add(mafFormat.format(umiMaf));
862
863 if (answerKey != null) {
864 String sample = answerKey.get(p.getCell());
865 line.add(sample);
866 }
867
868 List<String> genoStrings = sampleNames.stream().map(x -> gm.getGenotypeString(x)).collect(Collectors.toList());
869 line.addAll(genoStrings);
870
871 String h = StringUtils.join(line, "\t");
872 out.println(h);
873 }
874
875 }
876
877 /**
878 * Compare the genotype probabilities location to the variant context probability. return 0 < if the sgp is before than
879 * the VC return 0 if the GP = VC return > 0 if the VC is before the sgp This is a comparison of the interval of the
880 * SampleGenotypeProbabilities to the VariantContext: sgp.compareTo(vc)
881 *
882 * @param sgp
883 * @param vc
884 * @return
885 */
886 int compareRecords(final SampleGenotypeProbabilities sgp, final VariantContext vc, final SAMSequenceDictionary dict) {
887 Interval sgpI = sgp.getSNPInterval();
888 Interval sgpVC = new Interval(vc.getContig(), vc.getStart(), vc.getEnd());
889 // log.info("SNP Interval" + sgpI.toString() + " VCF interval " + sgpVC.toString());
890 int cmp = IntervalTagComparator.compare(sgpI, sgpVC, dict);
891 return cmp;
892 }
893
894 // done before, pretty well tested elsewhere boilerplate.
895 public PeekableIterator<List<SampleGenotypeProbabilities>> prepareIterator(final SNPInfoCollection snpIntervals, List<String> barcodes) {
896 // generate the reader with the desired options so we can enable EAGERLY_DECODE.
897 SamReaderFactory factory = SamReaderFactory.makeDefault().enable(SamReaderFactory.Option.EAGERLY_DECODE);
898 SamHeaderAndIterator headerAndIter = SamFileMergeUtil.mergeInputs(this.INPUT_BAM, false, factory);
899
900 // override the normal gene annotations with new ones before any other operations.
901 // filter out PCR duplicates.
902 if (this.DNA_MODE) {
903 // Replace UMI tags with read names, set strand tag to read tag, set gene function to an accepted on. Overwrites ALL
904 // reads tags.
905 IgnoreGeneAnnotationTagger tagger = new IgnoreGeneAnnotationTagger(headerAndIter.iterator, this.GENE_NAME_TAG, this.GENE_STRAND_TAG,
906 this.GENE_FUNCTION_TAG, this.LOCUS_FUNCTION_LIST, false, this.MOLECULAR_BARCODE_TAG, true);
907 headerAndIter = new SamHeaderAndIterator(headerAndIter.header, (CloseableIterator<SAMRecord>) tagger.iterator());
908 final PCRDuplicateFilteringIterator pcrDuplicateFilteringIterator = new PCRDuplicateFilteringIterator(headerAndIter.iterator);
909 headerAndIter = new SamHeaderAndIterator(headerAndIter.header, pcrDuplicateFilteringIterator);
910 }
911
912 // this explicitly filters to the best SNP seen on a read if there are multiple SNPs touched by a read
913 Map<Interval, Double> genotypeQuality = snpIntervals.getAverageGQ();
914 SNPUMIBasePileupIterator sbpi = new SNPUMIBasePileupIterator(headerAndIter, snpIntervals.getIntervalList(), GENE_NAME_TAG, GENE_STRAND_TAG, GENE_FUNCTION_TAG,
915 LOCUS_FUNCTION_LIST, STRAND_STRATEGY, this.FUNCTIONAL_STRATEGY, this.CELL_BARCODE_TAG, this.MOLECULAR_BARCODE_TAG, this.SNP_TAG, this.FUNCTION_TAG, this.READ_MQ, false,
916 barcodes, genotypeQuality, SortOrder.SNP_CELL, this.TRANSCRIBED_SNP_FAIL_FAST_THRESHOLD);
917
918
919 if (this.BAM_OUTPUT != null) {
920 SAMFileHeader header = headerAndIter.header;
921 SamHeaderUtil.addPgRecord(header, this);
922 header.addComment("Reads that overlap SNPs in VCF [" + this.VCF + "]");
923 SAMFileWriter writer = new SAMFileWriterFactory().makeSAMOrBAMWriter(header, false, this.BAM_OUTPUT);
924 final SamWriterSink informativeRecSink = new SamWriterSink(writer);
925 sbpi.addReadSink(informativeRecSink);
926 }
927
928 final SAMSequenceDictionary dict = snpIntervals.getIntervalList().getHeader().getSequenceDictionary();
929
930 // gets a SampleGenotypeProbabilities for each cell.
931 SampleGenotypeProbabilitiesIterator result = new SampleGenotypeProbabilitiesIterator(sbpi, dict, this.EDIT_DISTANCE, SortOrder.SNP_CELL);
932
933 // clusters SampleGenotypeProbabilities objects across all cells for a SNP
934 GroupingIterator<SampleGenotypeProbabilities> groupingIterator = new GroupingIterator<>(result, new Comparator<SampleGenotypeProbabilities>() {
935 @Override
936 public int compare(final SampleGenotypeProbabilities o1, final SampleGenotypeProbabilities o2) {
937 int cmp = IntervalTagComparator.compare(o1.getSNPInterval(), o2.getSNPInterval(), dict);
938 return cmp;
939 }
940 });
941
942 PeekableIterator<List<SampleGenotypeProbabilities>> peekableIter = new PeekableIterator<>(groupingIterator);
943 return (peekableIter);
944 }
945
946 // done before, pretty well tested elsewhere boilerplate.
947 public List<String> getCellBarcodes() {
948
949 if (this.CELL_BC_FILE != null) {
950 IOUtil.assertFileIsReadable(this.CELL_BC_FILE);
951 List<String> cellBarcodes = ParseBarcodeFile.readCellBarcodeFile(this.CELL_BC_FILE);
952 log.info("Found " + cellBarcodes.size() + " cell barcodes in file");
953 return (cellBarcodes);
954 }
955 log.info("Gathering barcodes for the top [" + this.NUM_BARCODES + "] cells");
956 return new BarcodeListRetrieval().getListCellBarcodesByReadCount(this.INPUT_BAM, this.CELL_BARCODE_TAG, this.READ_MQ, null, this.NUM_BARCODES);
957 }
958
959 public Map<String, String> readAnswerKey() {
960 if (this.ANSWER_KEY_FILE == null)
961 return null;
962 IOUtil.assertFileIsReadable(this.ANSWER_KEY_FILE);
963
964 Map<String, String> result = new HashMap<>();
965
966 BasicInputParser parser = new BasicInputParser(false, 2, this.ANSWER_KEY_FILE);
967 if (parser.hasNext()) {
968 String[] line = parser.next();
969 if (!line[0].equals("cell") || !line[1].equals("sample")) {
970 parser.close();
971 throw new IllegalArgumentException("Wrong header, is this the answer key file you're looking for? " + parser.getCurrentLine());
972 }
973 }
974 while (parser.hasNext()) {
975 String[] line = parser.next();
976 if (line.length != 2) {
977 parser.close();
978 throw new IllegalArgumentException("Expected answer key file to have 2 columns per line, but this line did not " + parser.getCurrentLine());
979 }
980 result.put(line[0], line[1]);
981 }
982 parser.close();
983 return (result);
984 }
985
986 /**
987 * Returns a writer, or null if the out file is null.
988 *
989 * @param vcfReader
990 * @param out
991 * @return
992 */
993 VariantContextWriter getVCFWriter(final VCFFileReader vcfReader, final File out) {
994 if (out == null)
995 return null;
996 VariantContextWriterBuilder builder = new VariantContextWriterBuilder().setReferenceDictionary(vcfReader.getFileHeader().getSequenceDictionary())
997 .setOption(Options.INDEX_ON_THE_FLY).setBuffer(8192).setOutputFile(out);
998
999 if (out.getName().endsWith(".gz"))
1000 builder.setOutputFileType(OutputType.BLOCK_COMPRESSED_VCF);
1001 else
1002 builder.setOutputFileType(OutputType.VCF);
1003
1004 VariantContextWriter writer = builder.build();
1005
1006 return writer;
1007 }
1008
1009 /**
1010 * If there's a file with a list of samples to use, parse that and use that subset of samples Otherwise, get the list of
1011 * samples in the VCF header.
1012 *
1013 * @param vcfReader
1014 * @return
1015 */
1016 public List<String> getVCFSamples(final VCFFileReader vcfReader) {
1017 if (this.SAMPLE_FILE == null)
1018 return vcfReader.getFileHeader().getSampleNamesInOrder();
1019 IOUtil.assertFileIsReadable(this.SAMPLE_FILE);
1020 return ParseBarcodeFile.readCellBarcodeFile(SAMPLE_FILE);
1021 }
1022
1023 /** Stock main method. */
1024 public static void main(final String[] args) {
1025 System.exit(new AssignCellsToSamples().instanceMain(args));
1026 }
1027
1028 /**
1029 * A helper class to gather statistics about one line of a VCF file
1030 *
1031 * @author nemesh
1032 *
1033 */
1034 private class GenotypeCountMetrics {
1035 private final int genotypeQualityThreshold;
1036 private final Set<String> sampleNamesRequested;
1037 private double meanGenotypeQuality;
1038 private char refAllele;
1039 private char altAllele;
1040 private ObjectCounter<GenotypeType> genotypeCounts;
1041 private Map<String, GenotypeType> genotypes;
1042
1043 public GenotypeCountMetrics(final VariantContext vc, final Collection<String> sampleNames, int genotypeQualityThreshold) {
1044 this.genotypeQualityThreshold = genotypeQualityThreshold;
1045 this.sampleNamesRequested = new HashSet<String>(sampleNames);
1046 this.genotypes = new HashMap<>();
1047
1048 GenotypesContext gc = vc.getGenotypes(sampleNamesRequested);
1049 Iterator<Genotype> gi = gc.iterator();
1050 GenotypeGQFilter gf = new GenotypeGQFilter(gi, genotypeQualityThreshold);
1051
1052 Mean meanGenotypeQuality = new Mean();
1053
1054 this.genotypeCounts = new ObjectCounter<>();
1055
1056 for (Genotype g : gf) {
1057 genotypeCounts.increment(g.getType());
1058 genotypes.put(g.getSampleName(), g.getType());
1059 meanGenotypeQuality.increment(g.getGQ());
1060 }
1061
1062 gf.close();
1063 this.meanGenotypeQuality = meanGenotypeQuality.getResult();
1064 this.refAllele = StringUtil.byteToChar(vc.getReference().getBases()[0]);
1065 this.altAllele = CellAssignmentUtils.getAltAllele(vc);
1066 }
1067
1068 public GenotypeType getGenotype(String sample) {
1069 return (this.genotypes.get(sample));
1070 }
1071
1072 public Collection<String> getSamplesNotObserved() {
1073 return (CollectionUtils.subtract(this.sampleNamesRequested, this.getSampleNamesObserved()));
1074 }
1075
1076 public int getRefCount() {
1077 return this.genotypeCounts.getCountForKey(GenotypeType.HOM_REF);
1078 }
1079
1080 public int getHetCount() {
1081 return genotypeCounts.getCountForKey(GenotypeType.HET);
1082 }
1083
1084 public int getAltCount() {
1085 return genotypeCounts.getCountForKey(GenotypeType.HOM_VAR);
1086 }
1087
1088 public double getMinorAlleleFrequency() {
1089 // stupid java int vs double.
1090 double ref = getRefCount();
1091 double het = getHetCount();
1092 double alt = getAltCount();
1093
1094 double minorAlleleFreq = ((alt * 2) + het) / (ref * 2 + het * 2 * alt * 2);
1095 return (minorAlleleFreq);
1096 }
1097
1098 @SuppressWarnings("unused")
1099 public int getGenotypeQualityThreshold() {
1100 return genotypeQualityThreshold;
1101 }
1102
1103 @SuppressWarnings("unused")
1104 public Set<String> getSampleNamesRequested() {
1105 return sampleNamesRequested;
1106 }
1107
1108 public Collection<String> getSampleNamesObserved() {
1109 return this.genotypes.keySet();
1110 }
1111
1112 @SuppressWarnings("unused")
1113 public double getMeanGenotypeQuality() {
1114 return meanGenotypeQuality;
1115 }
1116
1117 public char getRefAllele() {
1118 return refAllele;
1119 }
1120
1121 public char getAltAllele() {
1122 return altAllele;
1123 }
1124
1125 public String getGenotypeString(String sample) {
1126 GenotypeType gt = this.getGenotype(sample);
1127 String result = "null";
1128 if (gt == null) {
1129 return (result);
1130 } else {
1131 switch (gt) {
1132 case HOM_REF:
1133 result = "ref";
1134 break;
1135 case HET:
1136 result = "het";
1137 break;
1138 case HOM_VAR:
1139 result = "alt";
1140 break;
1141 default:
1142 result = "NA";
1143 }
1144 }
1145 return (result);
1146 }
1147 }
1148
1149 private VariantContextWriter getVcfWriter(final VCFFileReader vcfReader, List<String> vcfSamples) {
1150 // set up the VCF writer if needed.
1151 VariantContextWriter vcfWriter = null;
1152 if (this.VCF_OUTPUT != null) {
1153 IOUtil.assertFileIsWritable(this.VCF_OUTPUT);
1154 vcfWriter = SampleAssignmentVCFUtils.getVCFWriter(vcfReader, this.VCF_OUTPUT);
1155 VCFHeader header = vcfReader.getFileHeader();
1156 VCFHeader h2 = new VCFHeader(header.getMetaDataInInputOrder(), vcfSamples);
1157 vcfWriter.writeHeader(h2);
1158 }
1159 return (vcfWriter);
1160 }
1161
1162 private PrintStream getVerboseWriter(File output) {
1163 PrintStream verboseWriter = null;
1164 if (output != null) {
1165 IOUtil.assertFileIsWritable(output);
1166 verboseWriter = new ErrorCheckingPrintStream(IOUtil.openFileForWriting(output));
1167 }
1168 return (verboseWriter);
1169 }
1170
1171 @Override
1172 protected String[] customCommandLineValidation() {
1173
1174 final ArrayList<String> list = new ArrayList<>(1);
1175
1176 IOUtil.assertFileIsReadable(this.VCF);
1177 this.INPUT_BAM = FileListParsingUtils.expandFileList(INPUT_BAM);
1178 IOUtil.assertFileIsWritable(this.OUTPUT);
1179 if (CELL_BC_FILE != null)
1180 IOUtil.assertFileIsReadable(this.CELL_BC_FILE);
1181 if (this.BAM_OUTPUT != null)
1182 IOUtil.assertFileIsWritable(this.BAM_OUTPUT);
1183 if (this.VERBOSE_BEST_DONOR_OUTPUT != null)
1184 IOUtil.assertFileIsWritable(this.VERBOSE_BEST_DONOR_OUTPUT);
1185 // validate VCF is indexed.
1186 if (!VCFUtils.hasIndex(this.VCF))
1187 list.add("VCF is not indexed! Please index and retry.");
1188
1189 if (this.CELL_CONTAMINATION_ESTIMATE_FILE != null && this.ALLELE_FREQUENCY_ESTIMATE_FILE == null)
1190 list.add("If CELL_CONTAMINATION_ESTIMATE_FILE is supplied, must also supply ALLELE_FREQUENCY_ESTIMATE_FILE");
1191
1192 if (this.CELL_CONTAMINATION_ESTIMATE_FILE == null && this.ALLELE_FREQUENCY_ESTIMATE_FILE != null)
1193 list.add("If ALLELE_FREQUENCY_ESTIMATE_FILE is supplied, must also supply CELL_CONTAMINATION_ESTIMATE_FILE");
1194
1195 if (this.CELL_CONTAMINATION_ESTIMATE_FILE != null && this.ALLELE_FREQUENCY_ESTIMATE_FILE != null && this.MAX_ERROR_RATE != null) {
1196 log.info("Running analysis with contamination esimates, MAX_ERROR_RATE argument ignored!");
1197 MAX_ERROR_RATE = null;
1198 }
1199
1200 if (this.CELL_CONTAMINATION_ESTIMATE_FILE != null && this.ALLELE_FREQUENCY_ESTIMATE_FILE != null && this.FIXED_ERROR_RATE != null) {
1201 log.info("Running analysis with contamination esimates, FIXED_ERROR_RATE argument ignored!");
1202 FIXED_ERROR_RATE = null;
1203 }
1204
1205 if (this.FIXED_ERROR_RATE != null & this.MAX_ERROR_RATE == null)
1206 log.info("Running with a fixed error rate of " + this.FIXED_ERROR_RATE);
1207
1208 if (this.MAX_ERROR_RATE != null && this.CELL_CONTAMINATION_ESTIMATE_FILE == null && this.ALLELE_FREQUENCY_ESTIMATE_FILE == null)
1209 log.info("Running with a maximum cap on error rate of " + this.FIXED_ERROR_RATE);
1210
1211 if (VERBOSE_BEST_DONOR_OUTPUT != null & VERBOSE_OUTPUT == null) {
1212 list.add("If VERBOSE_BEST_DONOR_OUTPUT is requested, then VERBOSE_OUTPUT must also be requested");
1213 }
1214
1215 if (FRACTION_SAMPLES_PASSING < 0 | FRACTION_SAMPLES_PASSING > 1)
1216 list.add("FRACTION_SAMPLES_PASSING must be between 0 and 1, value was " + Double.toString(this.FRACTION_SAMPLES_PASSING));
1217
1218 if (DNA_MODE) {
1219 log.info("IN DNA MODE. Edit distance for UMI collapse disabled!");
1220 this.EDIT_DISTANCE=0;
1221 }
1222
1223 return CustomCommandLineValidationHelper.makeValue(super.customCommandLineValidation(), list);
1224 }
1225
1226 }