Debian Codegraph @

libkmer/existDB-create-from-meryl.C

raw
1 #include <stdio.h>
2 #include <stdlib.h>
3 #include <string.h>
4 #include <errno.h>
5 #include "existDB.H"
6 #include "libmeryl.H"
7
8
9 bool
10 existDB::createFromMeryl(char const *prefix,
11 uint32 merSize,
12 uint32 lo,
13 uint32 hi,
14 uint32 flags) {
15
16 merylStreamReader *M = new merylStreamReader(prefix);
17
18 bool beVerbose = false;
19
20 _hashTable = 0L;
21 _buckets = 0L;
22 _counts = 0L;
23
24 _merSizeInBases = M->merSize();
25
26 if (merSize != _merSizeInBases) {
27 fprintf(stderr, "createFromMeryl()-- ERROR: requested merSize ("uint32FMT") is different than merSize in meryl database ("uint32FMT").\n",
28 merSize, _merSizeInBases);
29 exit(1);
30 }
31
32 // We can set this exactly, but not memory optimal (see meryl/estimate.C:optimalNumberOfBuckets()).
33 // Instead, we just blindly use whatever meryl used.
34 //
35 uint32 tblBits = M->prefixSize();
36
37 // But it is faster to reset to this. Might use 2x the memory.
38 //uint32 tblBits = logBaseTwo64(M->numberOfDistinctMers() + 1);
39
40 _shift1 = 2 * _merSizeInBases - tblBits;
41 _shift2 = _shift1 / 2;
42 _mask1 = uint64MASK(tblBits);
43 _mask2 = uint64MASK(_shift1);
44
45 _hshWidth = uint32ZERO;
46 _chkWidth = 2 * _merSizeInBases - tblBits;
47 _cntWidth = 16;
48
49 uint64 tableSizeInEntries = uint64ONE << tblBits;
50 uint64 numberOfMers = uint64ZERO;
51 uint64 *countingTable = new uint64 [tableSizeInEntries + 1];
52
53 if (beVerbose) {
54 fprintf(stderr, "createFromMeryl()-- tableSizeInEntries "uint64FMT"\n", tableSizeInEntries);
55 fprintf(stderr, "createFromMeryl()-- count range "uint32FMT"-"uint32FMT"\n", lo, hi);
56 }
57
58 for (uint64 i=tableSizeInEntries+1; i--; )
59 countingTable[i] = 0;
60
61 _isCanonical = flags & existDBcanonical;
62 _isForward = flags & existDBforward;
63
64 if (beVerbose) {
65 fprintf(stderr, "createFromMeryl()-- canonical %c\n", (_isCanonical) ? 'T' : 'F');
66 fprintf(stderr, "createFromMeryl()-- forward %c\n", (_isForward) ? 'T' : 'F');
67 }
68
69 assert(_isCanonical + _isForward == 1);
70
71 // 1) Count bucket sizes
72 // While we don't know the bucket sizes right now, but we do know
73 // how many buckets and how many mers.
74 //
75 // Because we could be inserting both forward and reverse, we can't
76 // really move the direction testing outside the loop, unless we
77 // want to do two iterations over M.
78 //
79 speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
80
81 while (M->nextMer()) {
82 if ((lo <= M->theCount()) && (M->theCount() <= hi)) {
83 if (_isForward) {
84 countingTable[ HASH(M->theFMer()) ]++;
85 numberOfMers++;
86 }
87
88 if (_isCanonical) {
89 kMer r = M->theFMer();
90 r.reverseComplement();
91
92 if (M->theFMer() < r)
93 countingTable[ HASH(M->theFMer()) ]++;
94 else
95 countingTable[ HASH(r) ]++;
96 numberOfMers++;
97 }
98
99 C->tick();
100 }
101 }
102
103 if (beVerbose)
104 fprintf(stderr, "createFromMeryl()-- numberOfMers "uint64FMT"\n", numberOfMers);
105
106 delete C;
107 delete M;
108
109 if (_compressedHash) {
110 _hshWidth = 1;
111 while ((numberOfMers+1) > (uint64ONE << _hshWidth))
112 _hshWidth++;
113 }
114
115 if (beVerbose) {
116 fprintf(stderr, "existDB::createFromMeryl()-- Found "uint64FMT" mers between count of "uint32FMT" and "uint32FMT"\n",
117 numberOfMers, lo, hi);
118 }
119
120 // 2) Allocate hash table, mer storage buckets
121 //
122 _hashTableWords = tableSizeInEntries + 2;
123 if (_compressedHash)
124 _hashTableWords = _hashTableWords * _hshWidth / 64 + 1;
125
126 _bucketsWords = numberOfMers + 2;
127 if (_compressedBucket)
128 _bucketsWords = _bucketsWords * _chkWidth / 64 + 1;
129
130 _countsWords = numberOfMers + 2;
131 if (_compressedCounts)
132 _countsWords = _countsWords * _cntWidth / 64 + 1;
133
134 if (beVerbose) {
135 fprintf(stderr, "existDB::createFromMeryl()-- hashTable is "uint64FMT"MB\n", _hashTableWords >> 17);
136 fprintf(stderr, "existDB::createFromMeryl()-- buckets is "uint64FMT"MB\n", _bucketsWords >> 17);
137 if (flags & existDBcounts)
138 fprintf(stderr, "existDB::createFromMeryl()-- counts is "uint64FMT"MB\n", _countsWords >> 17);
139 }
140
141 _hashTable = new uint64 [_hashTableWords];
142 _buckets = new uint64 [_bucketsWords];
143 _countsWords = (flags & existDBcounts) ? _countsWords : 0;
144 _counts = (flags & existDBcounts) ? new uint64 [_countsWords] : 0L;
145
146 // These aren't strictly needed. _buckets is cleared as it is initialied. _hashTable
147 // is also cleared as it is initialized, but in the _compressedHash case, the last
148 // few words might be uninitialized. They're unused.
149
150 //memset(_hashTable, 0, sizeof(uint64) * _hashTableWords);
151 //memset(_buckets, 0, sizeof(uint64) * _bucketsWords); // buckets is cleared as it is built
152 //memset(_counts, 0, sizeof(uint64) * _countsWords);
153
154 _hashTable[_hashTableWords-1] = 0;
155 _hashTable[_hashTableWords-2] = 0;
156 _hashTable[_hashTableWords-3] = 0;
157 _hashTable[_hashTableWords-4] = 0;
158
159 ////////////////////////////////////////////////////////////////////////////////
160 //
161 // Make the hash table point to the start of the bucket, and reset
162 // the counting table -- we're going to use it to fill the buckets.
163 //
164 uint64 tmpPosition = 0;
165 uint64 begPosition = 0;
166 uint64 ptr = 0;
167
168 if (_compressedHash) {
169 for (uint64 i=0; i<tableSizeInEntries; i++) {
170 tmpPosition = countingTable[i];
171 countingTable[i] = begPosition;
172
173 setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
174 ptr += _hshWidth;
175
176 begPosition += tmpPosition;
177 }
178
179 setDecodedValue(_hashTable, ptr, _hshWidth, begPosition);
180 } else {
181 for (uint64 i=0; i<tableSizeInEntries; i++) {
182 tmpPosition = countingTable[i];
183 countingTable[i] = begPosition;
184
185 _hashTable[i] = begPosition;
186
187 begPosition += tmpPosition;
188 }
189
190 // Set the last position in the hash, but we don't care about
191 // the temporary counting table.
192 //
193 _hashTable[tableSizeInEntries] = begPosition;
194 }
195
196
197 ///////////////////////////////////////////////////////////////////////////////
198 //
199 // 3) Build list of mers, placed into buckets
200 //
201 M = new merylStreamReader(prefix);
202 C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, beVerbose);
203
204 while (M->nextMer()) {
205 if ((lo <= M->theCount()) && (M->theCount() <= hi)) {
206 if (_isForward)
207 insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), M->theCount(), countingTable);
208
209 if (_isCanonical) {
210 kMer r = M->theFMer();
211 r.reverseComplement();
212
213 if (M->theFMer() < r)
214 insertMer(HASH(M->theFMer()), CHECK(M->theFMer()), M->theCount(), countingTable);
215 else
216 insertMer(HASH(r), CHECK(r), M->theCount(), countingTable);
217 numberOfMers++;
218 }
219
220
221 C->tick();
222 }
223 }
224
225 delete C;
226 delete M;
227 delete [] countingTable;
228
229 return(true);
230 }