| 1 |
#include <stdio.h> |
| 2 |
#include <stdlib.h> |
| 3 |
#include <string.h> |
| 4 |
#include <errno.h> |
| 5 |
#include "existDB.H" |
| 6 |
#include "libmeryl.H" |
| 7 |
|
| 8 |
|
| 9 |
bool |
| 10 |
existDB::createFromMeryl(char const *prefix, |
| 11 |
uint32 merSize, |
| 12 |
uint32 lo, |
| 13 |
uint32 hi, |
| 14 |
uint32 flags) { |
| 15 |
|
| 16 |
merylStreamReader *M = new merylStreamReader(prefix); |
| 17 |
|
| 18 |
bool beVerbose = false; |
| 19 |
|
| 20 |
_hashTable = 0L; |
| 21 |
_buckets = 0L; |
| 22 |
_counts = 0L; |
| 23 |
|
| 24 |
_merSizeInBases = M->merSize(); |
| 25 |
|
| 26 |
if (merSize != _merSizeInBases) { |
| 27 |
fprintf(stderr, "createFromMeryl()-- ERROR: requested merSize ("uint32FMT") is different than merSize in meryl database ("uint32FMT").\n", |
| 28 |
merSize, _merSizeInBases); |
| 29 |
exit(1); |
| 30 |
} |
| 31 |
|
| 32 |
// We can set this exactly, but not memory optimal (see meryl/estimate.C:optimalNumberOfBuckets()). |
| 33 |
// Instead, we just blindly use whatever meryl used. |
| 34 |
// |
| 35 |
uint32 tblBits = M->prefixSize(); |
| 36 |
|
| 37 |
// But it is faster to reset to this. Might use 2x the memory. |
| 38 |
//uint32 tblBits = logBaseTwo64(M->numberOfDistinctMers() + 1); |
| 39 |
|
| 40 |
_shift1 = 2 * _merSizeInBases - tblBits; |
| 41 |
_shift2 = _shift1 / 2; |
| 42 |
_mask1 = uint64MASK(tblBits); |
| 43 |
_mask2 = uint64MASK(_shift1); |
| 44 |
|
| 45 |
_hshWidth = uint32ZERO; |
| 46 |
_chkWidth = 2 * _merSizeInBases - tblBits; |
| 47 |
_cntWidth = 16; |
| 48 |
|
| 49 |
uint64 tableSizeInEntries = uint64ONE << tblBits; |
| 50 |
uint64 numberOfMers = uint64ZERO; |
| 51 |
uint64 *countingTable = new uint64 [tableSizeInEntries + 1]; |
| 52 |
|
| 53 |
if (beVerbose) { |
| 54 |
fprintf(stderr, "createFromMeryl()-- tableSizeInEntries "uint64FMT"\n", tableSizeInEntries); |
| 55 |
fprintf(stderr, "createFromMeryl()-- count range "uint32FMT"-"uint32FMT"\n", lo, hi); |
| 56 |
} |
| 57 |
|
| 58 |
for (uint64 i=tableSizeInEntries+1; i--; ) |
| 59 |
countingTable[i] = 0; |
| 60 |
|
| 61 |
_isCanonical = flags & existDBcanonical; |
| 62 |
_isForward = flags & existDBforward; |
| 63 |
|
| 64 |
if (beVerbose) { |
| 65 |
fprintf(stderr, "createFromMeryl()-- canonical %c\n", (_isCanonical) ? 'T' : 'F'); |
| 66 |
fprintf(stderr, "createFromMeryl()-- forward %c\n", (_isForward) ? 'T' : 'F'); |
| 67 |
} |
| 68 |
|
| 69 |
assert(_isCanonical + _isForward == 1); |
| 70 |
|
| 71 |
// 1) Count bucket sizes |
| 72 |
// While we don't know the bucket sizes right now, but we do know |
| 73 |
// how many buckets and how many mers. |
| 74 |
// |
| 75 |
// Because we could be inserting both forward and reverse, we can't |
| 76 |
// really move the direction testing outside the loop, unless we |
| 77 |
// want to do two iterations over M. |
| 78 |
// |
| 79 |
speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose); |
| 80 |
|
| 81 |
while (M->nextMer()) { |
| 82 |
if ((lo <= M->theCount()) && (M->theCount() <= hi)) { |
| 83 |
if (_isForward) { |
| 84 |
countingTable[ HASH(M->theFMer()) ]++; |
| 85 |
numberOfMers++; |
| 86 |
} |
| 87 |
|
| 88 |
if (_isCanonical) { |
| 89 |
kMer r = M->theFMer(); |
| 90 |
r.reverseComplement(); |
| 91 |
|
| 92 |
if (M->theFMer() < r) |
| 93 |
countingTable[ HASH(M->theFMer()) ]++; |
| 94 |
else |
| 95 |
countingTable[ HASH(r) ]++; |
| 96 |
numberOfMers++; |
| 97 |
} |
| 98 |
|
| 99 |
C->tick(); |
| 100 |
} |
| 101 |
} |
| 102 |
|
| 103 |
if (beVerbose) |
| 104 |
fprintf(stderr, "createFromMeryl()-- numberOfMers "uint64FMT"\n", numberOfMers); |
| 105 |
|
| 106 |
delete C; |
| 107 |
delete M; |
| 108 |
|
| 109 |
if (_compressedHash) { |
| 110 |
_hshWidth = 1; |
| 111 |
while ((numberOfMers+1) > (uint64ONE << _hshWidth)) |
| 112 |
_hshWidth++; |
| 113 |
} |
| 114 |
|
| 115 |
if (beVerbose) { |
| 116 |
fprintf(stderr, "existDB::createFromMeryl()-- Found "uint64FMT" mers between count of "uint32FMT" and "uint32FMT"\n", |
| 117 |
numberOfMers, lo, hi); |
| 118 |
} |
| 119 |
|
| 120 |
// 2) Allocate hash table, mer storage buckets |
| 121 |
// |
| 122 |
_hashTableWords = tableSizeInEntries + 2; |
| 123 |
if (_compressedHash) |
| 124 |
_hashTableWords = _hashTableWords * _hshWidth / 64 + 1; |
| 125 |
|
| 126 |
_bucketsWords = numberOfMers + 2; |
| 127 |
if (_compressedBucket) |
| 128 |
_bucketsWords = _bucketsWords * _chkWidth / 64 + 1; |
| 129 |
|
| 130 |
_countsWords = numberOfMers + 2; |
| 131 |
if (_compressedCounts) |
| 132 |
_countsWords = _countsWords * _cntWidth / 64 + 1; |
| 133 |
|
| 134 |
if (beVerbose) { |
| 135 |
fprintf(stderr, "existDB::createFromMeryl()-- hashTable is "uint64FMT"MB\n", _hashTableWords >> 17); |
| 136 |
fprintf(stderr, "existDB::createFromMeryl()-- buckets is "uint64FMT"MB\n", _bucketsWords >> 17); |
| 137 |
if (flags & existDBcounts) |
| 138 |
fprintf(stderr, "existDB::createFromMeryl()-- counts is "uint64FMT"MB\n", _countsWords >> 17); |
| 139 |
} |
| 140 |
|
| 141 |
_hashTable = new uint64 [_hashTableWords]; |
| 142 |
_buckets = new uint64 [_bucketsWords]; |
| 143 |
_countsWords = (flags & existDBcounts) ? _countsWords : 0; |
| 144 |
_counts = (flags & existDBcounts) ? new uint64 [_countsWords] : 0L; |
| 145 |
|
| 146 |
// These aren't strictly needed. _buckets is cleared as it is initialied. _hashTable |
| 147 |
// is also cleared as it is initialized, but in the _compressedHash case, the last |
| 148 |
// few words might be uninitialized. They're unused. |
| 149 |
|
| 150 |
//memset(_hashTable, 0, sizeof(uint64) * _hashTableWords); |
| 151 |
//memset(_buckets, 0, sizeof(uint64) * _bucketsWords); // buckets is cleared as it is built |
| 152 |
//memset(_counts, 0, sizeof(uint64) * _countsWords); |
| 153 |
|
| 154 |
_hashTable[_hashTableWords-1] = 0; |
| 155 |
_hashTable[_hashTableWords-2] = 0; |
| 156 |
_hashTable[_hashTableWords-3] = 0; |
| 157 |
_hashTable[_hashTableWords-4] = 0; |
| 158 |
|
| 159 |
//////////////////////////////////////////////////////////////////////////////// |
| 160 |
// |
| 161 |
// Make the hash table point to the start of the bucket, and reset |
| 162 |
// the counting table -- we're going to use it to fill the buckets. |
| 163 |
// |
| 164 |
uint64 tmpPosition = 0; |
| 165 |
uint64 begPosition = 0; |
| 166 |
uint64 ptr = 0; |
| 167 |
|
| 168 |
if (_compressedHash) { |
| 169 |
for (uint64 i=0; i<tableSizeInEntries; i++) { |
| 170 |
tmpPosition = countingTable[i]; |
| 171 |
countingTable[i] = begPosition; |
| 172 |
|
| 173 |
setDecodedValue(_hashTable, ptr, _hshWidth, begPosition); |
| 174 |
ptr += _hshWidth; |
| 175 |
|
| 176 |
begPosition += tmpPosition; |
| 177 |
} |
| 178 |
|
| 179 |
setDecodedValue(_hashTable, ptr, _hshWidth, begPosition); |
| 180 |
} else { |
| 181 |
for (uint64 i=0; i<tableSizeInEntries; i++) { |
| 182 |
tmpPosition = countingTable[i]; |
| 183 |
countingTable[i] = begPosition; |
| 184 |
|
| 185 |
_hashTable[i] = begPosition; |
| 186 |
|
| 187 |
begPosition += tmpPosition; |
| 188 |
} |
| 189 |
|
| 190 |
// Set the last position in the hash, but we don't care about |
| 191 |
// the temporary counting table. |
| 192 |
// |
| 193 |
_hashTable[tableSizeInEntries] = begPosition; |
| 194 |
} |
| 195 |
|
| 196 |
|
| 197 |
/////////////////////////////////////////////////////////////////////////////// |
| 198 |
// |
| 199 |
// 3) Build list of mers, placed into buckets |
| 200 |
// |
| 201 |
M = new merylStreamReader(prefix); |
| 202 |
C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose); |
| 203 |
|
| 204 |
while (M->nextMer()) { |
| 205 |
if ((lo <= M->theCount()) && (M->theCount() <= hi)) { |
| 206 |
if (_isForward) |
| 207 |
insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), M->theCount(), countingTable); |
| 208 |
|
| 209 |
if (_isCanonical) { |
| 210 |
kMer r = M->theFMer(); |
| 211 |
r.reverseComplement(); |
| 212 |
|
| 213 |
if (M->theFMer() < r) |
| 214 |
insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), M->theCount(), countingTable); |
| 215 |
else |
| 216 |
insertMer(HASH(r), CHECK(r), M->theCount(), countingTable); |
| 217 |
numberOfMers++; |
| 218 |
} |
| 219 |
|
| 220 |
|
| 221 |
C->tick(); |
| 222 |
} |
| 223 |
} |
| 224 |
|
| 225 |
delete C; |
| 226 |
delete M; |
| 227 |
delete [] countingTable; |
| 228 |
|
| 229 |
return(true); |
| 230 |
} |