| 1 |
/* |
| 2 |
* MIT License |
| 3 |
* |
| 4 |
* Copyright 2017 Broad Institute |
| 5 |
* |
| 6 |
* Permission is hereby granted, free of charge, to any person obtaining a copy |
| 7 |
* of this software and associated documentation files (the "Software"), to deal |
| 8 |
* in the Software without restriction, including without limitation the rights |
| 9 |
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 10 |
* copies of the Software, and to permit persons to whom the Software is |
| 11 |
* furnished to do so, subject to the following conditions: |
| 12 |
* |
| 13 |
* The above copyright notice and this permission notice shall be included in all |
| 14 |
* copies or substantial portions of the Software. |
| 15 |
* |
| 16 |
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 17 |
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 18 |
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 19 |
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 20 |
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 21 |
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 22 |
* SOFTWARE. |
| 23 |
*/ |
| 24 |
package org.broadinstitute.dropseqrna.barnyard.digitalallelecounts.sampleassignment; |
| 25 |
|
| 26 |
import java.io.File; |
| 27 |
import java.util.ArrayList; |
| 28 |
import java.util.Arrays; |
| 29 |
import java.util.Collection; |
| 30 |
import java.util.HashMap; |
| 31 |
import java.util.Iterator; |
| 32 |
import java.util.LinkedHashMap; |
| 33 |
import java.util.List; |
| 34 |
import java.util.Map; |
| 35 |
import java.util.Set; |
| 36 |
|
| 37 |
import org.broadinstitute.dropseqrna.utils.statistics.FDR; |
| 38 |
|
| 39 |
import com.google.common.collect.ImmutableMap; |
| 40 |
|
| 41 |
import htsjdk.samtools.util.Interval; |
| 42 |
import htsjdk.variant.variantcontext.GenotypeType; |
| 43 |
import picard.util.TabbedInputParser; |
| 44 |
|
| 45 |
/** |
| 46 |
* A collection of CellSampleLikelihoodCollection objects for a single variant position. |
| 47 |
* @author nemesh |
| 48 |
* |
| 49 |
*/ |
| 50 |
public class CellCollectionSampleLikelihoodCollection { |
| 51 |
// the key is the cell ID, the value is the collection of likelihoods to samples. |
| 52 |
private final Map<String, CellSampleLikelihoodCollection> map; |
| 53 |
|
| 54 |
// this may be null. |
| 55 |
private final Double fixedGenotypeErrorRate; |
| 56 |
|
| 57 |
// this may be null. |
| 58 |
final Double maximumObservationProbability; |
| 59 |
|
| 60 |
// Map of the donor name to the position of that donor name in the list. Shared by all CellSampleListlihoodCollection objects. |
| 61 |
private final ImmutableMap<String, Integer> sampleIndexMap; |
| 62 |
private final Map<String,Double> cellContaminationMap; |
| 63 |
private final Map<Interval, Double> variantMinorAlleleFrequency; |
| 64 |
private static GenotypeType [] genotypeModels = {GenotypeType.HOM_REF, GenotypeType.HET, GenotypeType.HOM_VAR}; |
| 65 |
|
| 66 |
|
| 67 |
public CellCollectionSampleLikelihoodCollection(List<String> sampleList) { |
| 68 |
this((Double)null, (Double) null, sampleList); |
| 69 |
} |
| 70 |
|
| 71 |
/** |
| 72 |
* Build a collection that holds one or more cells, and the likelihood of the cell's genotype data having come from one or more donors. |
| 73 |
* @param fixedGenotypeErrorRate Instead of using the base qualities to compute likelihoods, used a fixed error rate instead. |
| 74 |
* if set to null, ignore this parameter and use the base error rates. |
| 75 |
* @param maximumObservationProbability Cap the maximum error penalty at this number. |
| 76 |
*/ |
| 77 |
public CellCollectionSampleLikelihoodCollection(final Double fixedGenotypeErrorRate, final Double maximumObservationProbability, List<String> sampleList) { |
| 78 |
map = new HashMap<>(); |
| 79 |
this.fixedGenotypeErrorRate=fixedGenotypeErrorRate; |
| 80 |
this.maximumObservationProbability=maximumObservationProbability; |
| 81 |
this.variantMinorAlleleFrequency=null; |
| 82 |
this.cellContaminationMap=null; |
| 83 |
// build the map of the sample list to the positions, retaining order. |
| 84 |
LinkedHashMap<String, Integer> sampleIndexMap = new LinkedHashMap<>(); |
| 85 |
for (int i=0; i<sampleList.size(); i++) { |
| 86 |
sampleIndexMap.put(sampleList.get(i), i); |
| 87 |
} |
| 88 |
this.sampleIndexMap=ImmutableMap.copyOf(sampleIndexMap); |
| 89 |
} |
| 90 |
|
| 91 |
/** |
| 92 |
* Build a collection that holds one or more cells, and the likelihood of the cell's genotype data having come from one or more donors. |
| 93 |
* @param fixedGenotypeErrorRate Instead of using the base qualities to compute likelihoods, used a fixed error rate instead. |
| 94 |
* if set to null, ignore this parameter and use the base error rates. |
| 95 |
* @param maximumObservationProbability Cap the maximum error penalty at this number. |
| 96 |
*/ |
| 97 |
public CellCollectionSampleLikelihoodCollection(final Map<String,Double> cellContaminationMap, Map<Interval, Double> variantMinorAlleleFrequency, List<String> sampleList) { |
| 98 |
map = new HashMap<>(); |
| 99 |
this.fixedGenotypeErrorRate=null; |
| 100 |
this.maximumObservationProbability=null; |
| 101 |
this.variantMinorAlleleFrequency=variantMinorAlleleFrequency; |
| 102 |
this.cellContaminationMap=cellContaminationMap; |
| 103 |
// build the map of the sample list to the positions, retaining order. |
| 104 |
LinkedHashMap<String, Integer> sampleIndexMap = new LinkedHashMap<>(); |
| 105 |
for (int i=0; i<sampleList.size(); i++) { |
| 106 |
sampleIndexMap.put(sampleList.get(i), i); |
| 107 |
} |
| 108 |
this.sampleIndexMap=ImmutableMap.copyOf(sampleIndexMap); |
| 109 |
} |
| 110 |
|
| 111 |
/** |
| 112 |
* If a contamination score was set for this cell, return it, else return null |
| 113 |
* @param cellBarcode The cell barcode to query. |
| 114 |
* @return The contamination score, or null |
| 115 |
*/ |
| 116 |
public Double getContamination (String cellBarcode) { |
| 117 |
if (this.cellContaminationMap==null) return (null); |
| 118 |
return (this.cellContaminationMap.get(cellBarcode)); |
| 119 |
} |
| 120 |
|
| 121 |
public boolean hasContamination () { |
| 122 |
if (this.cellContaminationMap==null) return (false); |
| 123 |
return !this.cellContaminationMap.isEmpty(); |
| 124 |
} |
| 125 |
|
| 126 |
|
| 127 |
// Constructors purely for testing Object. |
| 128 |
CellSampleLikelihoodCollection buildCellSampleLikelihoodCollection (final String cellBarcode) { |
| 129 |
return (new CellSampleLikelihoodCollection(cellBarcode, 0, 0, 0d, this.sampleIndexMap)); |
| 130 |
} |
| 131 |
|
| 132 |
CellSampleLikelihoodCollection buildCellSampleLikelihoodCollection (final String cellBarcode, final int numSNPs, final int numUMIs, final double globalLikelihoodScore) { |
| 133 |
return (new CellSampleLikelihoodCollection(cellBarcode, numSNPs, numUMIs, globalLikelihoodScore, this.sampleIndexMap)); |
| 134 |
} |
| 135 |
|
| 136 |
/** |
| 137 |
* Merges a CellCollectionSampleLikelihoodCollection into this obect. |
| 138 |
* @param other Another CellCollectionSampleLikelihoodCollection object. |
| 139 |
*/ |
| 140 |
/* This functionality is disabled as it is unused. |
| 141 |
public void add (final CellCollectionSampleLikelihoodCollection other) { |
| 142 |
if (this.fixedGenotypeErrorRate!=other.fixedGenotypeErrorRate) |
| 143 |
throw new IllegalArgumentException("Added a collection in with a different fixed error rate."); |
| 144 |
for (String cellBarcode: other.getCellBarcodes()) { |
| 145 |
CellSampleLikelihoodCollection c = this.getLikelihoodCollection(cellBarcode); |
| 146 |
if (c==null) |
| 147 |
c = other.getLikelihoodCollection(cellBarcode); |
| 148 |
else |
| 149 |
c.add(other.getLikelihoodCollection(cellBarcode)); |
| 150 |
this.map.put(cellBarcode, c); |
| 151 |
} |
| 152 |
} |
| 153 |
*/ |
| 154 |
|
| 155 |
/** |
| 156 |
* Add a CellSampleLikelihoodCollection to this collection. |
| 157 |
* @param data |
| 158 |
*/ |
| 159 |
void add (final CellSampleLikelihoodCollection data) { |
| 160 |
this.map.put(data.getCellBarcode(), data); |
| 161 |
} |
| 162 |
|
| 163 |
/** |
| 164 |
* Get the set of samples from the VCF represented anywhere in the data. |
| 165 |
* @return |
| 166 |
*/ |
| 167 |
|
| 168 |
/** |
| 169 |
* Update the number of SNPs and UMIs observed for the cell captured by the pileup. |
| 170 |
* Call this for each cell/snp pairing. |
| 171 |
* @param p |
| 172 |
*/ |
| 173 |
public void updateNumObservations(final SampleGenotypeProbabilities p) { |
| 174 |
String cellBarcode = p.getCell(); |
| 175 |
CellSampleLikelihoodCollection c = this.map.get(cellBarcode); |
| 176 |
if (c==null) |
| 177 |
c=new CellSampleLikelihoodCollection(cellBarcode, 0, 0, 0, this.sampleIndexMap); |
| 178 |
int numUMIs = p.getBackingPileups().size(); |
| 179 |
c.incrementNumSNPs(1); |
| 180 |
c.incrementNumUMIs(numUMIs); |
| 181 |
// this should be part of the null check. |
| 182 |
this.map.put(cellBarcode, c); |
| 183 |
} |
| 184 |
|
| 185 |
|
| 186 |
|
| 187 |
/** |
| 188 |
* For a pileup of reads on a cell/snp and a sample with two allele calls (ie: a genotype) |
| 189 |
* update the likelihood that the cell should be assigned to the sample. |
| 190 |
* |
| 191 |
* |
| 192 |
* @param p A pileup of observations at a SNP for a single cell. |
| 193 |
* @param samples The samples being updated |
| 194 |
* @param alleleOne The first allele of the SNP for the sample |
| 195 |
* @param alleleTwo The second allele of the SNP for the sample |
| 196 |
* @param genotypeProbability The probability of the genotype. Can be set to null to not consider. |
| 197 |
* @param referenceAllele The reference allele for this variant in the population. (Optional, set null to ignore). |
| 198 |
*/ |
| 199 |
public void updatelikelihoods (final SampleGenotypeProbabilities p, final Collection<String> samples, |
| 200 |
final char alleleOne, final char alleleTwo, final Double genotypeProbability, final Character referenceAllele) { |
| 201 |
Double maf = CellAssignmentUtils.getNullableValue(this.variantMinorAlleleFrequency, p.getSNPInterval()); |
| 202 |
Double contamination = CellAssignmentUtils.getNullableValue(this.cellContaminationMap, p.getCell()); |
| 203 |
double logLikelihood = p.getLogLikelihood(alleleOne, alleleTwo, this.fixedGenotypeErrorRate, genotypeProbability, this.maximumObservationProbability, referenceAllele, maf, contamination); |
| 204 |
updatelikelihoods(p, samples, logLikelihood); |
| 205 |
} |
| 206 |
|
| 207 |
private void updatelikelihoods(final SampleGenotypeProbabilities p, final Collection<String> samples, double logLikelihood) { |
| 208 |
String cellBarcode = p.getCell(); |
| 209 |
final CellSampleLikelihoodCollection c = this.map.computeIfAbsent(cellBarcode, s -> |
| 210 |
new CellSampleLikelihoodCollection(cellBarcode, 0, 0, 0, this.sampleIndexMap) |
| 211 |
); |
| 212 |
samples.forEach(sample -> c.addLikelihood(sample, logLikelihood)); |
| 213 |
} |
| 214 |
|
| 215 |
|
| 216 |
/** |
| 217 |
* Update the likelihoods for this SNP for samples that do not have this genotype set. |
| 218 |
* To do this, create a blended model of the population of known genotypes to calculate a new genotype probability. |
| 219 |
* Apply this likelihood to each unknown sample. |
| 220 |
* @param p the pileup of read observations to process. |
| 221 |
* @param refAllele the reference allele for this variant |
| 222 |
* @param altAllele the alternate allele for this variant |
| 223 |
* @param homRefCount how many homozygous reference donors were observed for this variant |
| 224 |
* @param hetCount how many heterozygous donors were observed for this variant |
| 225 |
* @param altCount how many homozygous alternate donors were observed for this variant |
| 226 |
* @param samples the list of missing samples. |
| 227 |
* @param genotypeProbability The probability of the genotype, between 0 and 1 inclusive. Can be set to null to ignore. |
| 228 |
*/ |
| 229 |
// |
| 230 |
// public void updateMissingLikelihoods (final SampleGenotypeProbabilities p, final char refAllele, final char altAllele, final int homRefCount, final int hetCount, final int homVarCount, |
| 231 |
// final Collection<String> samples, final Double genotypeProbability) { |
| 232 |
// double logLikelihood = getMissingLikelihood(p, refAllele, altAllele, homRefCount, hetCount, homVarCount, genotypeProbability); |
| 233 |
// applyMissingLikelihoodToDonors(p, logLikelihood, samples); |
| 234 |
// } |
| 235 |
// |
| 236 |
|
| 237 |
/** |
| 238 |
* Update the likelihoods for this SNP for samples that do not have this genotype set. |
| 239 |
* To do this, create a blended model of the population of known genotypes to calculate a new genotype probability. |
| 240 |
* Apply this likelihood to each unknown sample. |
| 241 |
* @param p the pileup of read observations to process. |
| 242 |
* @param logLikelihood the missing data penalty score to apply to all samples. |
| 243 |
* @param samples the list of missing samples. |
| 244 |
* |
| 245 |
*/ |
| 246 |
public void applyMissingLikelihoodToDonors (final SampleGenotypeProbabilities p, double logLikelihood, final Collection<String> samples) { |
| 247 |
String cellBarcode = p.getCell(); |
| 248 |
CellSampleLikelihoodCollection c = this.map.get(cellBarcode); |
| 249 |
if (c==null) { |
| 250 |
c=new CellSampleLikelihoodCollection(cellBarcode, 0, 0, 0, this.sampleIndexMap); |
| 251 |
this.map.put(cellBarcode, c); |
| 252 |
} |
| 253 |
for (String sample: samples) |
| 254 |
c.addLikelihood(sample, logLikelihood); |
| 255 |
} |
| 256 |
|
| 257 |
/** |
| 258 |
* For each cell/SNP, increase the global likelihood penalty score. |
| 259 |
* @param p the pileup of read observations to process. |
| 260 |
* @param logLikelihood the missing data penalty score |
| 261 |
*/ |
| 262 |
public void incrementGlobalPenaltyAllDonors(final SampleGenotypeProbabilities p, double logLikelihood) { |
| 263 |
String cellBarcode = p.getCell(); |
| 264 |
CellSampleLikelihoodCollection c = this.map.get(cellBarcode); |
| 265 |
if (c==null) { |
| 266 |
c=new CellSampleLikelihoodCollection(cellBarcode, 0, 0, 0, this.sampleIndexMap); |
| 267 |
this.map.put(cellBarcode, c); |
| 268 |
} |
| 269 |
c.addGlobalPenalty(logLikelihood); |
| 270 |
} |
| 271 |
|
| 272 |
/** |
| 273 |
* Create a blended model of the population of known genotypes to calculate a new genotype probability. |
| 274 |
* @param p the pileup of read observations to process. |
| 275 |
* @param alleleOne The first allele of the SNP for the sample |
| 276 |
* @param alleleTwo The second allele of the SNP for the sample |
| 277 |
* @param homRefCount how many homozygous reference donors were observed for this variant |
| 278 |
* @param hetCount how many heterozygous donors were observed for this variant |
| 279 |
* @param altCount how many homozygous alternate donors were observed for this variant |
| 280 |
* @param genotypeProbability The probability of the genotype, between 0 and 1 inclusive. Can be set to null to ignore. |
| 281 |
* @param minorAlleleFrequency The minor allele frequency in the population. Optional. Set null to ignore. |
| 282 |
* @param contamination The contamination rate in the population . Optional. Set null to ignore. |
| 283 |
* @return The log likelihood of the data for the blended model. |
| 284 |
*/ |
| 285 |
public double getMissingLikelihood(final SampleGenotypeProbabilities p, final char refAllele, final char altAllele, final int homRefCount, final int hetCount, final int homVarCount, |
| 286 |
final Double genotypeProbability) { |
| 287 |
|
| 288 |
Double minorAlleleFrequency = CellAssignmentUtils.getNullableValue(this.variantMinorAlleleFrequency, p.getSNPInterval()); |
| 289 |
Double contamination = CellAssignmentUtils.getNullableValue(this.cellContaminationMap, p.getCell()); |
| 290 |
List<GenotypeType> genotypes = Arrays.asList(genotypeModels); |
| 291 |
List<Double> mixture = Arrays.asList(Double.valueOf(homRefCount), Double.valueOf(hetCount), Double.valueOf(homVarCount)); |
| 292 |
double logLikelihood = p.getLogLikelihoodMissingData(refAllele, altAllele, genotypes, mixture, this.fixedGenotypeErrorRate, genotypeProbability, this.maximumObservationProbability, minorAlleleFrequency, contamination); |
| 293 |
return logLikelihood; |
| 294 |
} |
| 295 |
|
| 296 |
/** |
| 297 |
* Get a set of cell barcodes contained in this dataset. |
| 298 |
* @return |
| 299 |
*/ |
| 300 |
public Set<String> getCellBarcodes () { |
| 301 |
return this.map.keySet(); |
| 302 |
} |
| 303 |
|
| 304 |
/** |
| 305 |
* Get sample list contained in this dataset. |
| 306 |
* @return List of samples |
| 307 |
*/ |
| 308 |
public List<String> getSamples() { |
| 309 |
return new ArrayList<String>(this.sampleIndexMap.keySet()); |
| 310 |
} |
| 311 |
|
| 312 |
public CellSampleLikelihoodCollection getLikelihoodCollection (final String cellBarcode) { |
| 313 |
return this.map.get(cellBarcode); |
| 314 |
} |
| 315 |
|
| 316 |
/** |
| 317 |
* After all data has been added (so you have the best donor per cell), correct the collection of donors pvalues by FDR. |
| 318 |
* Return a map of each cell ID to the FDR corrected pvalue. |
| 319 |
*/ |
| 320 |
public Map<String, Double> getFDRCorrectedPvalues () { |
| 321 |
Iterator<String>cellBCs=this.map.keySet().iterator(); |
| 322 |
double [] pvalues = new double [this.map.size()]; |
| 323 |
|
| 324 |
for (int i=0; i<pvalues.length; i++) { |
| 325 |
String cellBC=cellBCs.next(); |
| 326 |
CellSampleLikelihoodCollection c = this.map.get(cellBC); |
| 327 |
pvalues[i] = c.getBestSampleAssignment().getOneMinusPvalue(); |
| 328 |
} |
| 329 |
double [] fdrCorrectedPvalues = FDR.bhAdjustment(pvalues); |
| 330 |
|
| 331 |
cellBCs=this.map.keySet().iterator(); |
| 332 |
Map<String, Double> result = new HashMap<>(); |
| 333 |
for (int i=0; i<pvalues.length; i++) { |
| 334 |
String cellBC=cellBCs.next(); |
| 335 |
result.put(cellBC, fdrCorrectedPvalues[i]); |
| 336 |
} |
| 337 |
return result; |
| 338 |
|
| 339 |
} |
| 340 |
|
| 341 |
public static CellCollectionSampleLikelihoodCollection parseFile(final File data) { |
| 342 |
|
| 343 |
TabbedInputParser parser = new TabbedInputParser(false, data); |
| 344 |
Iterator<String [] > iter = parser.iterator(); |
| 345 |
String [] header =iter.next(); |
| 346 |
// the expectation is that the first sample starts after either the bestSample or the trueSample column. |
| 347 |
int indexSampleStart=getIndexOfFirstSample(header); |
| 348 |
|
| 349 |
// test if there is a global likelihood score in the header. |
| 350 |
int palIdx=Arrays.asList(header).indexOf("population_average_likelihood"); |
| 351 |
|
| 352 |
String [] samples = Arrays.copyOfRange(header, indexSampleStart, header.length); |
| 353 |
CellCollectionSampleLikelihoodCollection result = new CellCollectionSampleLikelihoodCollection(Arrays.asList(samples)); |
| 354 |
// parse each line into a CellSampleLikelihoodCollection object. |
| 355 |
while (iter.hasNext()) { |
| 356 |
String [] line = iter.next(); |
| 357 |
double pal = 0; |
| 358 |
if (palIdx!=-1) pal=Double.parseDouble(line[palIdx]); |
| 359 |
CellSampleLikelihoodCollection o = result.buildCellSampleLikelihoodCollection(line[0],Integer.parseInt(line[1]), Integer.parseInt(line[2]), pal); |
| 360 |
String [] likes = Arrays.copyOfRange(line, indexSampleStart, line.length); |
| 361 |
for (int i=0; i<likes.length; i++) { |
| 362 |
String sample = samples[i]; |
| 363 |
double logLikelihood = Double.parseDouble(likes[i]); |
| 364 |
o.addLikelihood(sample, logLikelihood); |
| 365 |
} |
| 366 |
result.add(o); |
| 367 |
|
| 368 |
} |
| 369 |
|
| 370 |
parser.close(); |
| 371 |
|
| 372 |
return result; |
| 373 |
} |
| 374 |
|
| 375 |
private static int getIndexOfFirstSample (final String [] header) { |
| 376 |
// any one of these can be the last column before the samples, depending on file versioning and if it's positive control data. |
| 377 |
List<String> lastColumn = Arrays.asList("bestSample", "trueSample", "median_likelihood", "population_average_likelihood"); |
| 378 |
|
| 379 |
int index =-1; |
| 380 |
for (int i=0; i<header.length; i++) { |
| 381 |
String column=header[i]; |
| 382 |
if (lastColumn.contains(column)) { |
| 383 |
if (i>index) index=i; |
| 384 |
} |
| 385 |
} |
| 386 |
return index+1; |
| 387 |
} |
| 388 |
|
| 389 |
|
| 390 |
} |