| 1 |
#include <stdio.h> |
| 2 |
#include <stdlib.h> |
| 3 |
#include <string.h> |
| 4 |
#include <errno.h> |
| 5 |
|
| 6 |
#include "bio++.H" |
| 7 |
#include "existDB.H" |
| 8 |
#include "libmeryl.H" |
| 9 |
|
| 10 |
#include "seqCache.H" |
| 11 |
#include "seqStream.H" |
| 12 |
#include "merStream.H" |
| 13 |
|
| 14 |
// Driver for the existDB creation. Reads a sequence.fasta, builds |
| 15 |
// an existDB for the mers in the file, and then writes the internal |
| 16 |
// structures to disk. |
| 17 |
// |
| 18 |
// The existDB constructor is smart enough to read either a pre-built |
| 19 |
// image or a regular multi-fasta file. |
| 20 |
|
| 21 |
|
| 22 |
int |
| 23 |
testFiles(char *filename, char *prefix, uint32 merSize) { |
| 24 |
char *prefixfilename = new char [strlen(prefix) + 32]; |
| 25 |
|
| 26 |
// Create existDB e and save it to disk |
| 27 |
// |
| 28 |
existDB *e = new existDB(filename, merSize, existDBnoFlags | existDBcounts, 0, ~uint32ZERO); |
| 29 |
sprintf(prefixfilename, "%s.1", prefix); |
| 30 |
e->saveState(prefixfilename); |
| 31 |
|
| 32 |
// Create existDB f by loading the saved copy from disk |
| 33 |
// |
| 34 |
existDB *f = new existDB(prefixfilename); |
| 35 |
|
| 36 |
// Create a fresh existDB g (to check if we corrup the original when saved) |
| 37 |
// |
| 38 |
existDB *g = new existDB(filename, merSize, existDBnoFlags | existDBcounts, 0, ~uint32ZERO); |
| 39 |
|
| 40 |
speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, true); |
| 41 |
fprintf(stderr, "Need to iterate over %7.2f Mmers.\n", (uint64MASK(2 * merSize) + 1) / 1000000.0); |
| 42 |
|
| 43 |
for (uint64 d=0, m=uint64MASK(2 * merSize); m--; ) { |
| 44 |
bool ee = e->exists(m); |
| 45 |
bool ef = f->exists(m); |
| 46 |
bool eg = g->exists(m); |
| 47 |
|
| 48 |
uint32 ce = e->count(m); |
| 49 |
uint32 cf = f->count(m); |
| 50 |
uint32 cg = g->count(m); |
| 51 |
|
| 52 |
if ((ee != ef) || (ef != eg) || (ee != eg)) |
| 53 |
fprintf(stderr, "mer "uint64HEX" not found : e=%d f=%d g=%d\n", m, ee, ef, eg); |
| 54 |
|
| 55 |
if ((ce != cf) || (cf != cg) || (ce != cg)) |
| 56 |
fprintf(stderr, "mer "uint64HEX" count differs : e=%u f=%u g=%u (exists=%d)\n", m, ce, cf, cg, ee); |
| 57 |
|
| 58 |
if ((m & 0xffffff) == 0) { |
| 59 |
// Been a while since a report, so report. |
| 60 |
d = 1; |
| 61 |
} |
| 62 |
|
| 63 |
if ((ce > 1) && (d == 1)) { |
| 64 |
// Report anything not unique, to make sure that we're testing real counts and not just existence. |
| 65 |
fprintf(stderr, "mer "uint64HEX" : e=%u f=%u g=%u (exists=%d)\n", m, ce, cf, cg, ee); |
| 66 |
d = 0; |
| 67 |
} |
| 68 |
|
| 69 |
C->tick(); |
| 70 |
} |
| 71 |
|
| 72 |
delete e; |
| 73 |
delete C; |
| 74 |
|
| 75 |
return(0); |
| 76 |
} |
| 77 |
|
| 78 |
|
| 79 |
int |
| 80 |
testExistence(char *filename, uint32 merSize) { |
| 81 |
existDB *E = new existDB(filename, merSize, existDBnoFlags, 0, ~uint32ZERO); |
| 82 |
merStream *M = new merStream(new kMerBuilder(merSize), new seqStream(filename), true, true); |
| 83 |
uint64 tried = 0; |
| 84 |
uint64 lost = 0; |
| 85 |
|
| 86 |
while (M->nextMer()) { |
| 87 |
tried++; |
| 88 |
if (!E->exists(M->theFMer())) |
| 89 |
lost++; |
| 90 |
} |
| 91 |
|
| 92 |
delete M; |
| 93 |
delete E; |
| 94 |
|
| 95 |
if (lost) { |
| 96 |
fprintf(stderr, "Tried "uint64FMT", didn't find "uint64FMT" merStream mers in the existDB.\n", tried, lost); |
| 97 |
return(1); |
| 98 |
} else { |
| 99 |
return(0); |
| 100 |
} |
| 101 |
} |
| 102 |
|
| 103 |
|
| 104 |
|
| 105 |
int |
| 106 |
testExhaustive(char *filename, char *merylname, uint32 merSize) { |
| 107 |
existDB *E = new existDB(filename, merSize, existDBnoFlags, 0, ~uint32ZERO); |
| 108 |
merylStreamReader *M = new merylStreamReader(merylname); |
| 109 |
speedCounter *C = new speedCounter(" %7.2f Mmers -- %5.2f Mmers/second\r", 1000000.0, 0x1fffff, true); |
| 110 |
uint64 found = uint64ZERO; |
| 111 |
uint64 expected = uint64ZERO; |
| 112 |
|
| 113 |
FILE *DUMP = 0L; |
| 114 |
|
| 115 |
DUMP = fopen("testExhaustive.ms.dump", "w"); |
| 116 |
|
| 117 |
while (M->nextMer()) { |
| 118 |
if (E->exists(M->theFMer())) { |
| 119 |
expected++; |
| 120 |
fprintf(DUMP, uint64HEX"\n", (uint64)M->theFMer()); |
| 121 |
} else { |
| 122 |
fprintf(DUMP, uint64HEX" MISSED!\n", (uint64)M->theFMer()); |
| 123 |
} |
| 124 |
} |
| 125 |
|
| 126 |
fclose(DUMP); |
| 127 |
|
| 128 |
fprintf(stderr, "Found "uint64FMT" mers in the meryl database.\n", expected); |
| 129 |
fprintf(stderr, "Need to iterate over %7.2f Mmers.\n", (uint64MASK(2 * merSize) + 1) / 1000000.0); |
| 130 |
|
| 131 |
DUMP = fopen("testExhaustive.ck.dump", "w"); |
| 132 |
|
| 133 |
for (uint64 m = uint64MASK(2 * merSize); m--; ) { |
| 134 |
if (E->exists(m)) { |
| 135 |
found++; |
| 136 |
fprintf(DUMP, uint64HEX"\n", m); |
| 137 |
} |
| 138 |
C->tick(); |
| 139 |
} |
| 140 |
|
| 141 |
fclose(DUMP); |
| 142 |
|
| 143 |
delete C; |
| 144 |
delete E; |
| 145 |
delete M; |
| 146 |
|
| 147 |
if (expected != found) { |
| 148 |
fprintf(stderr, "Expected to find "uint64FMT" mers, but found "uint64FMT" instead.\n", |
| 149 |
expected, found); |
| 150 |
return(1); |
| 151 |
} else { |
| 152 |
return(0); |
| 153 |
} |
| 154 |
} |
| 155 |
|
| 156 |
|
| 157 |
const char *usage = |
| 158 |
"usage: %s [stuff]\n" |
| 159 |
" -mersize mersize\n" |
| 160 |
" -- Use the specified mersize when building existDB tables.\n" |
| 161 |
"\n" |
| 162 |
" -build some.fasta prefix\n" |
| 163 |
" -- Build an existDB on all mers in some.fasta and save\n" |
| 164 |
" the tables into prefix.\n" |
| 165 |
"\n" |
| 166 |
" -describe prefix\n" |
| 167 |
" -- Reports the state of some existDB file.\n" |
| 168 |
"\n" |
| 169 |
" -testfiles some.fasta prefix\n" |
| 170 |
" -- Build an existDB table from some.fasta. Write that table to disk.\n" |
| 171 |
" Load the table back. Compare that each mer in some.fasta is present\n" |
| 172 |
" in all three existDB tables created earlier.\n" |
| 173 |
"\n" |
| 174 |
" -testexistence some.fasta\n" |
| 175 |
" -- Build an existDB table from some.fasta, check that every\n" |
| 176 |
" mer in some.fasta can be found in the table. Does not\n" |
| 177 |
" guarantee that every mer in the table is found in the file.\n" |
| 178 |
"\n" |
| 179 |
" -testexhaustive some.fasta some.meryl\n" |
| 180 |
" -- Build an existDB table from some.fasta, check _EVERY_ mer\n" |
| 181 |
" for existance. Complain if a mer exists in the table but\n" |
| 182 |
" not in the meryl database. Assumes 'some.meryl' is the\n" |
| 183 |
" mercount of some.fasta.\n" |
| 184 |
"\n"; |
| 185 |
|
| 186 |
int |
| 187 |
main(int argc, char **argv) { |
| 188 |
uint32 mersize = 20; |
| 189 |
|
| 190 |
if (argc < 3) { |
| 191 |
fprintf(stderr, usage, argv[0]); |
| 192 |
exit(1); |
| 193 |
} |
| 194 |
|
| 195 |
int arg = 1; |
| 196 |
while (arg < argc) { |
| 197 |
if (strncmp(argv[arg], "-mersize", 2) == 0) { |
| 198 |
arg++; |
| 199 |
mersize = atoi(argv[arg]); |
| 200 |
|
| 201 |
} else if (strncmp(argv[arg], "-describe", 2) == 0) { |
| 202 |
existDB *e = new existDB(argv[argc-1], false); |
| 203 |
e->printState(stdout); |
| 204 |
delete e; |
| 205 |
exit(0); |
| 206 |
|
| 207 |
} else if (strncmp(argv[arg], "-testfiles", 8) == 0) { |
| 208 |
exit(testFiles(argv[arg+1], argv[arg+2], mersize)); |
| 209 |
|
| 210 |
} else if (strncmp(argv[arg], "-testexistence", 8) == 0) { |
| 211 |
exit(testExistence(argv[arg+1], mersize)); |
| 212 |
|
| 213 |
} else if (strncmp(argv[arg], "-testexhaustive", 8) == 0) { |
| 214 |
exit(testExhaustive(argv[arg+1], argv[arg+2], mersize)); |
| 215 |
|
| 216 |
} else if (strncmp(argv[arg], "-build", 2) == 0) { |
| 217 |
existDB *e = new existDB(argv[argc-2], mersize, existDBnoFlags, 0, ~uint32ZERO); |
| 218 |
e->saveState(argv[argc-1]); |
| 219 |
delete e; |
| 220 |
exit(0); |
| 221 |
} |
| 222 |
|
| 223 |
arg++; |
| 224 |
} |
| 225 |
|
| 226 |
exit(0); |
| 227 |
} |