| 1 |
#include <stddef.h> |
| 2 |
#include <stdint.h> |
| 3 |
#include <stdlib.h> // malloc |
| 4 |
#include <stdio.h> |
| 5 |
#include <assert.h> |
| 6 |
#include <string.h> |
| 7 |
|
| 8 |
#include "../zstd_seekable.h" |
| 9 |
|
| 10 |
|
| 11 |
/* ZSTD_seekable_customFile implementation that reads/seeks a buffer while keeping track of total bytes read */ |
| 12 |
typedef struct { |
| 13 |
const void *ptr; |
| 14 |
size_t size; |
| 15 |
size_t pos; |
| 16 |
size_t totalRead; |
| 17 |
} buffWrapperWithTotal_t; |
| 18 |
|
| 19 |
static int readBuffWithTotal(void* opaque, void* buffer, size_t n) |
| 20 |
{ |
| 21 |
buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*)opaque; |
| 22 |
assert(buff != NULL); |
| 23 |
if (buff->pos + n > buff->size) return -1; |
| 24 |
memcpy(buffer, (const char*)buff->ptr + buff->pos, n); |
| 25 |
buff->pos += n; |
| 26 |
buff->totalRead += n; |
| 27 |
return 0; |
| 28 |
} |
| 29 |
|
| 30 |
static int seekBuffWithTotal(void* opaque, long long offset, int origin) |
| 31 |
{ |
| 32 |
buffWrapperWithTotal_t* const buff = (buffWrapperWithTotal_t*) opaque; |
| 33 |
unsigned long long newOffset; |
| 34 |
assert(buff != NULL); |
| 35 |
switch (origin) { |
| 36 |
case SEEK_SET: |
| 37 |
assert(offset >= 0); |
| 38 |
newOffset = (unsigned long long)offset; |
| 39 |
break; |
| 40 |
case SEEK_CUR: |
| 41 |
newOffset = (unsigned long long)((long long)buff->pos + offset); |
| 42 |
break; |
| 43 |
case SEEK_END: |
| 44 |
newOffset = (unsigned long long)((long long)buff->size + offset); |
| 45 |
break; |
| 46 |
default: |
| 47 |
assert(0); /* not possible */ |
| 48 |
} |
| 49 |
if (newOffset > buff->size) { |
| 50 |
return -1; |
| 51 |
} |
| 52 |
buff->pos = newOffset; |
| 53 |
return 0; |
| 54 |
} |
| 55 |
|
| 56 |
/* Basic unit tests for zstd seekable format */ |
| 57 |
int main(int argc, const char** argv) |
| 58 |
{ |
| 59 |
unsigned testNb = 1; |
| 60 |
(void)argc; (void)argv; |
| 61 |
printf("Beginning zstd seekable format tests...\n"); |
| 62 |
|
| 63 |
printf("Test %u - simple round trip: ", testNb++); |
| 64 |
{ size_t const inSize = 4000; |
| 65 |
void* const inBuffer = malloc(inSize); |
| 66 |
assert(inBuffer != NULL); |
| 67 |
|
| 68 |
size_t const seekCapacity = 5000; |
| 69 |
void* const seekBuffer = malloc(seekCapacity); |
| 70 |
assert(seekBuffer != NULL); |
| 71 |
size_t seekSize; |
| 72 |
|
| 73 |
size_t const outCapacity = inSize; |
| 74 |
void* const outBuffer = malloc(outCapacity); |
| 75 |
assert(outBuffer != NULL); |
| 76 |
|
| 77 |
ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream(); |
| 78 |
assert(zscs != NULL); |
| 79 |
|
| 80 |
{ size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, (unsigned)inSize /* maxFrameSize */); |
| 81 |
assert(!ZSTD_isError(initStatus)); |
| 82 |
} |
| 83 |
|
| 84 |
{ ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity }; |
| 85 |
ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize }; |
| 86 |
|
| 87 |
size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb); |
| 88 |
assert(!ZSTD_isError(cStatus)); |
| 89 |
assert(inb.pos == inb.size); |
| 90 |
|
| 91 |
size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb); |
| 92 |
assert(!ZSTD_isError(endStatus)); |
| 93 |
seekSize = outb.pos; |
| 94 |
} |
| 95 |
|
| 96 |
ZSTD_seekable* const stream = ZSTD_seekable_create(); |
| 97 |
assert(stream != NULL); |
| 98 |
{ size_t const initStatus = ZSTD_seekable_initBuff(stream, seekBuffer, seekSize); |
| 99 |
assert(!ZSTD_isError(initStatus)); } |
| 100 |
|
| 101 |
{ size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, outCapacity, 0); |
| 102 |
assert(decStatus == inSize); } |
| 103 |
|
| 104 |
/* unit test ZSTD_seekTable functions */ |
| 105 |
ZSTD_seekTable* const zst = ZSTD_seekTable_create_fromSeekable(stream); |
| 106 |
assert(zst != NULL); |
| 107 |
|
| 108 |
unsigned const nbFrames = ZSTD_seekTable_getNumFrames(zst); |
| 109 |
assert(nbFrames > 0); |
| 110 |
|
| 111 |
unsigned long long const frame0Offset = ZSTD_seekTable_getFrameCompressedOffset(zst, 0); |
| 112 |
assert(frame0Offset == 0); |
| 113 |
|
| 114 |
unsigned long long const content0Offset = ZSTD_seekTable_getFrameDecompressedOffset(zst, 0); |
| 115 |
assert(content0Offset == 0); |
| 116 |
|
| 117 |
size_t const cSize = ZSTD_seekTable_getFrameCompressedSize(zst, 0); |
| 118 |
assert(!ZSTD_isError(cSize)); |
| 119 |
assert(cSize <= seekCapacity); |
| 120 |
|
| 121 |
size_t const origSize = ZSTD_seekTable_getFrameDecompressedSize(zst, 0); |
| 122 |
assert(origSize == inSize); |
| 123 |
|
| 124 |
unsigned const fo1idx = ZSTD_seekTable_offsetToFrameIndex(zst, 1); |
| 125 |
assert(fo1idx == 0); |
| 126 |
|
| 127 |
free(inBuffer); |
| 128 |
free(seekBuffer); |
| 129 |
free(outBuffer); |
| 130 |
ZSTD_seekable_freeCStream(zscs); |
| 131 |
ZSTD_seekTable_free(zst); |
| 132 |
ZSTD_seekable_free(stream); |
| 133 |
} |
| 134 |
printf("Success!\n"); |
| 135 |
|
| 136 |
|
| 137 |
printf("Test %u - check that seekable decompress does not hang: ", testNb++); |
| 138 |
{ /* Github issue #2335 */ |
| 139 |
const size_t compressed_size = 17; |
| 140 |
const uint8_t compressed_data[17] = { |
| 141 |
'^', |
| 142 |
'*', |
| 143 |
'M', |
| 144 |
'\x18', |
| 145 |
'\t', |
| 146 |
'\x00', |
| 147 |
'\x00', |
| 148 |
'\x00', |
| 149 |
'\x00', |
| 150 |
'\x00', |
| 151 |
'\x00', |
| 152 |
'\x00', |
| 153 |
(uint8_t)('\x03'), |
| 154 |
(uint8_t)('\xb1'), |
| 155 |
(uint8_t)('\xea'), |
| 156 |
(uint8_t)('\x92'), |
| 157 |
(uint8_t)('\x8f'), |
| 158 |
}; |
| 159 |
const size_t uncompressed_size = 32; |
| 160 |
uint8_t uncompressed_data[32]; |
| 161 |
|
| 162 |
ZSTD_seekable* const stream = ZSTD_seekable_create(); |
| 163 |
assert(stream != NULL); |
| 164 |
{ size_t const status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); |
| 165 |
if (ZSTD_isError(status)) { |
| 166 |
ZSTD_seekable_free(stream); |
| 167 |
goto _test_error; |
| 168 |
} } |
| 169 |
|
| 170 |
/* Should return an error, but not hang */ |
| 171 |
{ const size_t offset = 2; |
| 172 |
size_t const status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); |
| 173 |
if (!ZSTD_isError(status)) { |
| 174 |
ZSTD_seekable_free(stream); |
| 175 |
goto _test_error; |
| 176 |
} } |
| 177 |
|
| 178 |
ZSTD_seekable_free(stream); |
| 179 |
} |
| 180 |
printf("Success!\n"); |
| 181 |
|
| 182 |
printf("Test %u - check #2 that seekable decompress does not hang: ", testNb++); |
| 183 |
{ /* Github issue #FIXME */ |
| 184 |
const size_t compressed_size = 27; |
| 185 |
const uint8_t compressed_data[27] = { |
| 186 |
(uint8_t)'\x28', |
| 187 |
(uint8_t)'\xb5', |
| 188 |
(uint8_t)'\x2f', |
| 189 |
(uint8_t)'\xfd', |
| 190 |
(uint8_t)'\x00', |
| 191 |
(uint8_t)'\x32', |
| 192 |
(uint8_t)'\x91', |
| 193 |
(uint8_t)'\x00', |
| 194 |
(uint8_t)'\x00', |
| 195 |
(uint8_t)'\x00', |
| 196 |
(uint8_t)'\x5e', |
| 197 |
(uint8_t)'\x2a', |
| 198 |
(uint8_t)'\x4d', |
| 199 |
(uint8_t)'\x18', |
| 200 |
(uint8_t)'\x09', |
| 201 |
(uint8_t)'\x00', |
| 202 |
(uint8_t)'\x00', |
| 203 |
(uint8_t)'\x00', |
| 204 |
(uint8_t)'\x00', |
| 205 |
(uint8_t)'\x00', |
| 206 |
(uint8_t)'\x00', |
| 207 |
(uint8_t)'\x00', |
| 208 |
(uint8_t)'\x00', |
| 209 |
(uint8_t)'\xb1', |
| 210 |
(uint8_t)'\xea', |
| 211 |
(uint8_t)'\x92', |
| 212 |
(uint8_t)'\x8f', |
| 213 |
}; |
| 214 |
const size_t uncompressed_size = 400; |
| 215 |
uint8_t uncompressed_data[400]; |
| 216 |
|
| 217 |
ZSTD_seekable* stream = ZSTD_seekable_create(); |
| 218 |
size_t status = ZSTD_seekable_initBuff(stream, compressed_data, compressed_size); |
| 219 |
if (ZSTD_isError(status)) { |
| 220 |
ZSTD_seekable_free(stream); |
| 221 |
goto _test_error; |
| 222 |
} |
| 223 |
|
| 224 |
const size_t offset = 2; |
| 225 |
/* Should return an error, but not hang */ |
| 226 |
status = ZSTD_seekable_decompress(stream, uncompressed_data, uncompressed_size, offset); |
| 227 |
if (!ZSTD_isError(status)) { |
| 228 |
ZSTD_seekable_free(stream); |
| 229 |
goto _test_error; |
| 230 |
} |
| 231 |
|
| 232 |
ZSTD_seekable_free(stream); |
| 233 |
} |
| 234 |
printf("Success!\n"); |
| 235 |
|
| 236 |
|
| 237 |
printf("Test %u - check ZSTD magic in compressing empty string: ", testNb++); |
| 238 |
{ // compressing empty string should return a zstd header |
| 239 |
size_t const capacity = 255; |
| 240 |
char* inBuffer = malloc(capacity); |
| 241 |
assert(inBuffer != NULL); |
| 242 |
inBuffer[0] = '\0'; |
| 243 |
void* const outBuffer = malloc(capacity); |
| 244 |
assert(outBuffer != NULL); |
| 245 |
|
| 246 |
ZSTD_seekable_CStream *s = ZSTD_seekable_createCStream(); |
| 247 |
ZSTD_seekable_initCStream(s, 1, 1, 255); |
| 248 |
|
| 249 |
ZSTD_inBuffer input = { .src=inBuffer, .pos=0, .size=0 }; |
| 250 |
ZSTD_outBuffer output = { .dst=outBuffer, .pos=0, .size=capacity }; |
| 251 |
|
| 252 |
ZSTD_seekable_compressStream(s, &output, &input); |
| 253 |
ZSTD_seekable_endStream(s, &output); |
| 254 |
|
| 255 |
if((((char*)output.dst)[0] != '\x28') | (((char*)output.dst)[1] != '\xb5') | (((char*)output.dst)[2] != '\x2f') | (((char*)output.dst)[3] != '\xfd')) { |
| 256 |
printf("%#02x %#02x %#02x %#02x\n", ((char*)output.dst)[0], ((char*)output.dst)[1] , ((char*)output.dst)[2] , ((char*)output.dst)[3] ); |
| 257 |
|
| 258 |
free(inBuffer); |
| 259 |
free(outBuffer); |
| 260 |
ZSTD_seekable_freeCStream(s); |
| 261 |
goto _test_error; |
| 262 |
} |
| 263 |
|
| 264 |
free(inBuffer); |
| 265 |
free(outBuffer); |
| 266 |
ZSTD_seekable_freeCStream(s); |
| 267 |
} |
| 268 |
printf("Success!\n"); |
| 269 |
|
| 270 |
|
| 271 |
printf("Test %u - multiple decompress calls: ", testNb++); |
| 272 |
{ char const inBuffer[] = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt"; |
| 273 |
size_t const inSize = sizeof(inBuffer); |
| 274 |
|
| 275 |
size_t const seekCapacity = 5000; |
| 276 |
void* const seekBuffer = malloc(seekCapacity); |
| 277 |
assert(seekBuffer != NULL); |
| 278 |
size_t seekSize; |
| 279 |
|
| 280 |
size_t const outCapacity = inSize; |
| 281 |
char* const outBuffer = malloc(outCapacity); |
| 282 |
assert(outBuffer != NULL); |
| 283 |
|
| 284 |
ZSTD_seekable_CStream* const zscs = ZSTD_seekable_createCStream(); |
| 285 |
assert(zscs != NULL); |
| 286 |
|
| 287 |
/* compress test data with a small frame size to ensure multiple frames in the output */ |
| 288 |
unsigned const maxFrameSize = 40; |
| 289 |
{ size_t const initStatus = ZSTD_seekable_initCStream(zscs, 9, 0 /* checksumFlag */, maxFrameSize); |
| 290 |
assert(!ZSTD_isError(initStatus)); |
| 291 |
} |
| 292 |
|
| 293 |
{ ZSTD_outBuffer outb = { .dst=seekBuffer, .pos=0, .size=seekCapacity }; |
| 294 |
ZSTD_inBuffer inb = { .src=inBuffer, .pos=0, .size=inSize }; |
| 295 |
|
| 296 |
while (inb.pos < inb.size) { |
| 297 |
size_t const cStatus = ZSTD_seekable_compressStream(zscs, &outb, &inb); |
| 298 |
assert(!ZSTD_isError(cStatus)); |
| 299 |
} |
| 300 |
|
| 301 |
size_t const endStatus = ZSTD_seekable_endStream(zscs, &outb); |
| 302 |
assert(!ZSTD_isError(endStatus)); |
| 303 |
seekSize = outb.pos; |
| 304 |
} |
| 305 |
|
| 306 |
ZSTD_seekable* const stream = ZSTD_seekable_create(); |
| 307 |
assert(stream != NULL); |
| 308 |
buffWrapperWithTotal_t buffWrapper = {seekBuffer, seekSize, 0, 0}; |
| 309 |
{ ZSTD_seekable_customFile srcFile = {&buffWrapper, &readBuffWithTotal, &seekBuffWithTotal}; |
| 310 |
size_t const initStatus = ZSTD_seekable_initAdvanced(stream, srcFile); |
| 311 |
assert(!ZSTD_isError(initStatus)); } |
| 312 |
|
| 313 |
/* Perform a series of small reads and seeks (repeatedly read 1 byte and skip 1 byte) |
| 314 |
and check that we didn't reread input data unnecessarily */ |
| 315 |
size_t pos; |
| 316 |
for (pos = 0; pos < inSize; pos += 2) { |
| 317 |
size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, 1, pos); |
| 318 |
if (decStatus != 1 || outBuffer[0] != inBuffer[pos]) { |
| 319 |
goto _test_error; |
| 320 |
} |
| 321 |
} |
| 322 |
if (buffWrapper.totalRead > seekSize) { |
| 323 |
/* We read more than the compressed size, meaning there were some rereads. |
| 324 |
This is unneeded because we only seeked forward. */ |
| 325 |
printf("Too much data read: %zu read, with compressed size %zu\n", buffWrapper.totalRead, seekSize); |
| 326 |
goto _test_error; |
| 327 |
} |
| 328 |
|
| 329 |
/* Perform some reads and seeks to ensure correctness */ |
| 330 |
struct { |
| 331 |
size_t offset; |
| 332 |
size_t size; |
| 333 |
} const tests[] = { /* Assume the frame size is 40 */ |
| 334 |
{20, 40}, /* read partial data from two frames */ |
| 335 |
{60, 10}, /* continue reading from the same offset */ |
| 336 |
{50, 20}, /* seek backward within the same frame */ |
| 337 |
{10, 10}, /* seek backward to a different frame */ |
| 338 |
{25, 10}, /* seek forward within the same frame */ |
| 339 |
{60, 10}, /* seek forward to a different frame */ |
| 340 |
}; |
| 341 |
size_t idx; |
| 342 |
for (idx = 0; idx < sizeof(tests) / sizeof(tests[0]); idx++) { |
| 343 |
size_t const decStatus = ZSTD_seekable_decompress(stream, outBuffer, tests[idx].size, tests[idx].offset); |
| 344 |
if (decStatus != tests[idx].size || memcmp(outBuffer, inBuffer + tests[idx].offset, tests[idx].size) != 0) { |
| 345 |
goto _test_error; |
| 346 |
} |
| 347 |
} |
| 348 |
|
| 349 |
free(seekBuffer); |
| 350 |
free(outBuffer); |
| 351 |
ZSTD_seekable_freeCStream(zscs); |
| 352 |
ZSTD_seekable_free(stream); |
| 353 |
} |
| 354 |
printf("Success!\n"); |
| 355 |
|
| 356 |
/* TODO: Add more tests */ |
| 357 |
printf("Finished tests\n"); |
| 358 |
return 0; |
| 359 |
|
| 360 |
_test_error: |
| 361 |
printf("test failed! Exiting..\n"); |
| 362 |
return 1; |
| 363 |
} |