| 1 |
/* |
| 2 |
* This file is an example of an HDF5 filter plugin. |
| 3 |
* The plugin can be used with the HDF5 library vesrion 1.8.11+ to read |
| 4 |
* HDF5 datasets compressed with lz4. |
| 5 |
*/ |
| 6 |
|
| 7 |
#include <sys/types.h> |
| 8 |
#include <stdlib.h> |
| 9 |
#include <string.h> |
| 10 |
#include <assert.h> |
| 11 |
#include <stdio.h> |
| 12 |
#if defined(_WIN32) |
| 13 |
#include <Winsock2.h> |
| 14 |
#else |
| 15 |
#include <arpa/inet.h> |
| 16 |
#endif |
| 17 |
#include <H5PLextern.h> |
| 18 |
#include <lz4.h> |
| 19 |
#include "lz4_h5filter.h" |
| 20 |
|
| 21 |
#define PUSH_ERR(func, minor, str) \ |
| 22 |
H5Epush1(__FILE__, func, __LINE__, H5E_PLINE, minor, str) |
| 23 |
|
| 24 |
static size_t H5Z_filter_lz4(unsigned int flags, size_t cd_nelmts, |
| 25 |
const unsigned int cd_values[], size_t nbytes, |
| 26 |
size_t *buf_size, void **buf); |
| 27 |
|
| 28 |
#define H5Z_FILTER_LZ4 32004 |
| 29 |
|
| 30 |
#define htonll(x) ( ( (uint64_t)(htonl( (uint32_t)((x << 32) >> 32)))<< 32) | htonl( ((uint32_t)(x >> 32)) )) |
| 31 |
#define ntohll(x) htonll(x) |
| 32 |
|
| 33 |
#define htobe16t(x) htons(x) |
| 34 |
#define htobe32t(x) htonl(x) |
| 35 |
#define htobe64t(x) htonll(x) |
| 36 |
#define be16toht(x) ntohs(x) |
| 37 |
#define be32toht(x) ntohl(x) |
| 38 |
#define be64toht(x) ntohll(x) |
| 39 |
|
| 40 |
|
| 41 |
#define DEFAULT_BLOCK_SIZE 1<<30; /* 1GB. LZ4 needs blocks < 1.9GB. */ |
| 42 |
|
| 43 |
const H5Z_class2_t H5Z_LZ4[1] = {{ |
| 44 |
H5Z_CLASS_T_VERS, /* H5Z_class_t version */ |
| 45 |
(H5Z_filter_t)H5Z_FILTER_LZ4, /* Filter id number */ |
| 46 |
1, /* encoder_present flag (set to true) */ |
| 47 |
1, /* decoder_present flag (set to true) */ |
| 48 |
"HDF5 lz4 filter; see http://www.hdfgroup.org/services/contributions.html", |
| 49 |
/* Filter name for debugging */ |
| 50 |
NULL, /* The "can apply" callback */ |
| 51 |
NULL, /* The "set local" callback */ |
| 52 |
(H5Z_func_t)H5Z_filter_lz4, /* The actual filter function */ |
| 53 |
}}; |
| 54 |
|
| 55 |
static size_t H5Z_filter_lz4(unsigned int flags, size_t cd_nelmts, |
| 56 |
const unsigned int cd_values[], size_t nbytes, |
| 57 |
size_t *buf_size, void **buf) |
| 58 |
{ |
| 59 |
void * outBuf = NULL; |
| 60 |
size_t ret_value; |
| 61 |
|
| 62 |
if (flags & H5Z_FLAG_REVERSE) |
| 63 |
{ |
| 64 |
uint32_t *i32Buf; |
| 65 |
uint32_t blockSize; |
| 66 |
char *roBuf; /* pointer to current write position */ |
| 67 |
uint64_t decompSize; |
| 68 |
const char* rpos = (char*)*buf; /* pointer to current read position */ |
| 69 |
const uint64_t * const i64Buf = (uint64_t *) rpos; |
| 70 |
const uint64_t origSize = (uint64_t)(be64toht(*i64Buf));/* is saved in be format */ |
| 71 |
rpos += 8; /* advance the pointer */ |
| 72 |
|
| 73 |
i32Buf = (uint32_t*)rpos; |
| 74 |
blockSize = (uint32_t)(be32toht(*i32Buf)); |
| 75 |
rpos += 4; |
| 76 |
if(blockSize>origSize) |
| 77 |
blockSize = origSize; |
| 78 |
|
| 79 |
if (NULL==(outBuf = H5allocate_memory(origSize, false))) |
| 80 |
{ |
| 81 |
printf("error calling H5allocate_memory\n"); |
| 82 |
goto error; |
| 83 |
} |
| 84 |
roBuf = (char*)outBuf; /* pointer to current write position */ |
| 85 |
decompSize = 0; |
| 86 |
/// start with the first block /// |
| 87 |
while(decompSize < origSize) |
| 88 |
{ |
| 89 |
uint32_t compressedBlockSize; /// is saved in be format |
| 90 |
|
| 91 |
if(origSize-decompSize < blockSize) /* the last block can be smaller than blockSize. */ |
| 92 |
blockSize = origSize-decompSize; |
| 93 |
i32Buf = (uint32_t*)rpos; |
| 94 |
compressedBlockSize = be32toht(*i32Buf); /// is saved in be format |
| 95 |
rpos += 4; |
| 96 |
if(compressedBlockSize == blockSize) /* there was no compression */ |
| 97 |
{ |
| 98 |
memcpy(roBuf, rpos, blockSize); |
| 99 |
} |
| 100 |
else /* do the decompression */ |
| 101 |
{ |
| 102 |
#if LZ4_VERSION_NUMBER > 10300 |
| 103 |
int compressedBytes = LZ4_decompress_fast(rpos, roBuf, blockSize); |
| 104 |
#else |
| 105 |
int compressedBytes = LZ4_uncompress(rpos, roBuf, blockSize); |
| 106 |
#endif |
| 107 |
if(compressedBytes != compressedBlockSize) |
| 108 |
{ |
| 109 |
printf("decompressed size not the same: %d, != %d\n", compressedBytes, compressedBlockSize); |
| 110 |
goto error; |
| 111 |
} |
| 112 |
} |
| 113 |
|
| 114 |
rpos += compressedBlockSize; /* advance the read pointer to the next block */ |
| 115 |
roBuf += blockSize; /* advance the write pointer */ |
| 116 |
decompSize += blockSize; |
| 117 |
} |
| 118 |
H5free_memory(*buf); |
| 119 |
*buf = outBuf; |
| 120 |
outBuf = NULL; |
| 121 |
ret_value = (size_t)origSize; // should always work, as orig_size cannot be > 2GB (sizeof(size_t) < 4GB) |
| 122 |
} |
| 123 |
else /* forward filter */ |
| 124 |
{ |
| 125 |
size_t blockSize; |
| 126 |
size_t nBlocks; |
| 127 |
size_t outSize; /* size of the output buffer. Header size (12 bytes) is included */ |
| 128 |
size_t block; |
| 129 |
uint64_t *i64Buf; |
| 130 |
uint32_t *i32Buf; |
| 131 |
size_t maxDestSize; |
| 132 |
char *rpos; /* pointer to current read position */ |
| 133 |
char *roBuf; /* pointer to current write position */ |
| 134 |
|
| 135 |
if (nbytes > INT32_MAX) |
| 136 |
{ |
| 137 |
/* can only compress chunks up to 2GB */ |
| 138 |
goto error; |
| 139 |
} |
| 140 |
|
| 141 |
if(cd_nelmts > 0 && cd_values[0] > 0) |
| 142 |
{ |
| 143 |
blockSize = cd_values[0]; |
| 144 |
} |
| 145 |
else |
| 146 |
{ |
| 147 |
blockSize = DEFAULT_BLOCK_SIZE; |
| 148 |
} |
| 149 |
if(blockSize > nbytes) |
| 150 |
{ |
| 151 |
blockSize = nbytes; |
| 152 |
} |
| 153 |
nBlocks = (nbytes-1)/blockSize +1; |
| 154 |
maxDestSize = nBlocks * LZ4_compressBound(blockSize) + 4 + 8 + nBlocks*4; |
| 155 |
outBuf = H5allocate_memory(maxDestSize, false); |
| 156 |
if (NULL == outBuf) |
| 157 |
{ |
| 158 |
goto error; |
| 159 |
} |
| 160 |
|
| 161 |
rpos = (char*)*buf; /* pointer to current read position */ |
| 162 |
roBuf = (char*)outBuf; /* pointer to current write position */ |
| 163 |
/* header */ |
| 164 |
i64Buf = (uint64_t *) (roBuf); |
| 165 |
i64Buf[0] = htobe64t((uint64_t)nbytes); /* Store decompressed size in be format */ |
| 166 |
roBuf += 8; |
| 167 |
|
| 168 |
i32Buf = (uint32_t *) (roBuf); |
| 169 |
i32Buf[0] = htobe32t((uint32_t)blockSize); /* Store the block size in be format */ |
| 170 |
roBuf += 4; |
| 171 |
|
| 172 |
outSize = 12; /* size of the output buffer. Header size (12 bytes) is included */ |
| 173 |
|
| 174 |
for(block = 0; block < nBlocks; ++block) |
| 175 |
{ |
| 176 |
uint32_t compBlockSize; /// reserve space for compBlockSize |
| 177 |
size_t origWritten = block*blockSize; |
| 178 |
if(nbytes - origWritten < blockSize) /* the last block may be < blockSize */ |
| 179 |
blockSize = nbytes - origWritten; |
| 180 |
|
| 181 |
#if LZ4_VERSION_NUMBER > 10300 |
| 182 |
compBlockSize = LZ4_compress_default(rpos, roBuf+4, blockSize, LZ4_compressBound(blockSize)); /// reserve space for compBlockSize |
| 183 |
#else |
| 184 |
compBlockSize = LZ4_compress(rpos, roBuf+4, blockSize); /// reserve space for compBlockSize |
| 185 |
#endif |
| 186 |
if(!compBlockSize) |
| 187 |
goto error; |
| 188 |
if(compBlockSize >= blockSize) /* compression did not save any space, do a memcpy instead */ |
| 189 |
{ |
| 190 |
compBlockSize = blockSize; |
| 191 |
memcpy(roBuf+4, rpos, blockSize); |
| 192 |
} |
| 193 |
|
| 194 |
i32Buf = (uint32_t *) (roBuf); |
| 195 |
i32Buf[0] = htobe32t((uint32_t)compBlockSize); /* write blocksize */ |
| 196 |
roBuf += 4; |
| 197 |
|
| 198 |
rpos += blockSize; /* advance read pointer */ |
| 199 |
roBuf += compBlockSize; /* advance write pointer */ |
| 200 |
outSize += compBlockSize + 4; |
| 201 |
} |
| 202 |
|
| 203 |
H5free_memory(*buf); |
| 204 |
*buf = outBuf; |
| 205 |
*buf_size = outSize; |
| 206 |
outBuf = NULL; |
| 207 |
ret_value = outSize; |
| 208 |
|
| 209 |
} |
| 210 |
/* done: */ |
| 211 |
if(outBuf) |
| 212 |
H5free_memory(outBuf); |
| 213 |
return ret_value; |
| 214 |
|
| 215 |
|
| 216 |
error: |
| 217 |
if(outBuf) |
| 218 |
H5free_memory(outBuf); |
| 219 |
outBuf = NULL; |
| 220 |
return 0; |
| 221 |
} |
| 222 |
|
| 223 |
int lz4_register_h5filter(void){ |
| 224 |
|
| 225 |
int retval; |
| 226 |
|
| 227 |
retval = H5Zregister(H5Z_LZ4); |
| 228 |
if(retval<0){ |
| 229 |
PUSH_ERR("lz4_register_h5filter", |
| 230 |
H5E_CANTREGISTER, "Can't register lz4 filter"); |
| 231 |
} |
| 232 |
return retval; |
| 233 |
} |