Debian Codegraph @

LZ4/src/H5Zlz4.c

raw
1 /*
2 * This file is an example of an HDF5 filter plugin.
3 * The plugin can be used with the HDF5 library vesrion 1.8.11+ to read
4 * HDF5 datasets compressed with lz4.
5 */
6
7 #include <sys/types.h>
8 #include <stdlib.h>
9 #include <string.h>
10 #include <assert.h>
11 #include <stdio.h>
12 #if defined(_WIN32)
13 #include <Winsock2.h>
14 #else
15 #include <arpa/inet.h>
16 #endif
17 #include <H5PLextern.h>
18 #include <lz4.h>
19 #include "lz4_h5filter.h"
20
21 #define PUSH_ERR(func, minor, str) \
22 H5Epush1(__FILE__, func, __LINE__, H5E_PLINE, minor, str)
23
24 static size_t H5Z_filter_lz4(unsigned int flags, size_t cd_nelmts,
25 const unsigned int cd_values[], size_t nbytes,
26 size_t *buf_size, void **buf);
27
28 #define H5Z_FILTER_LZ4 32004
29
30 #define htonll(x) ( ( (uint64_t)(htonl( (uint32_t)((x << 32) >> 32)))<< 32) | htonl( ((uint32_t)(x >> 32)) ))
31 #define ntohll(x) htonll(x)
32
33 #define htobe16t(x) htons(x)
34 #define htobe32t(x) htonl(x)
35 #define htobe64t(x) htonll(x)
36 #define be16toht(x) ntohs(x)
37 #define be32toht(x) ntohl(x)
38 #define be64toht(x) ntohll(x)
39
40
41 #define DEFAULT_BLOCK_SIZE 1<<30; /* 1GB. LZ4 needs blocks < 1.9GB. */
42
43 const H5Z_class2_t H5Z_LZ4[1] = {{
44 H5Z_CLASS_T_VERS, /* H5Z_class_t version */
45 (H5Z_filter_t)H5Z_FILTER_LZ4, /* Filter id number */
46 1, /* encoder_present flag (set to true) */
47 1, /* decoder_present flag (set to true) */
48 "HDF5 lz4 filter; see http://www.hdfgroup.org/services/contributions.html",
49 /* Filter name for debugging */
50 NULL, /* The "can apply" callback */
51 NULL, /* The "set local" callback */
52 (H5Z_func_t)H5Z_filter_lz4, /* The actual filter function */
53 }};
54
55 static size_t H5Z_filter_lz4(unsigned int flags, size_t cd_nelmts,
56 const unsigned int cd_values[], size_t nbytes,
57 size_t *buf_size, void **buf)
58 {
59 void * outBuf = NULL;
60 size_t ret_value;
61
62 if (flags & H5Z_FLAG_REVERSE)
63 {
64 uint32_t *i32Buf;
65 uint32_t blockSize;
66 char *roBuf; /* pointer to current write position */
67 uint64_t decompSize;
68 const char* rpos = (char*)*buf; /* pointer to current read position */
69 const uint64_t * const i64Buf = (uint64_t *) rpos;
70 const uint64_t origSize = (uint64_t)(be64toht(*i64Buf));/* is saved in be format */
71 rpos += 8; /* advance the pointer */
72
73 i32Buf = (uint32_t*)rpos;
74 blockSize = (uint32_t)(be32toht(*i32Buf));
75 rpos += 4;
76 if(blockSize>origSize)
77 blockSize = origSize;
78
79 if (NULL==(outBuf = H5allocate_memory(origSize, false)))
80 {
81 printf("error calling H5allocate_memory\n");
82 goto error;
83 }
84 roBuf = (char*)outBuf; /* pointer to current write position */
85 decompSize = 0;
86 /// start with the first block ///
87 while(decompSize < origSize)
88 {
89 uint32_t compressedBlockSize; /// is saved in be format
90
91 if(origSize-decompSize < blockSize) /* the last block can be smaller than blockSize. */
92 blockSize = origSize-decompSize;
93 i32Buf = (uint32_t*)rpos;
94 compressedBlockSize = be32toht(*i32Buf); /// is saved in be format
95 rpos += 4;
96 if(compressedBlockSize == blockSize) /* there was no compression */
97 {
98 memcpy(roBuf, rpos, blockSize);
99 }
100 else /* do the decompression */
101 {
102 #if LZ4_VERSION_NUMBER > 10300
103 int compressedBytes = LZ4_decompress_fast(rpos, roBuf, blockSize);
104 #else
105 int compressedBytes = LZ4_uncompress(rpos, roBuf, blockSize);
106 #endif
107 if(compressedBytes != compressedBlockSize)
108 {
109 printf("decompressed size not the same: %d, != %d\n", compressedBytes, compressedBlockSize);
110 goto error;
111 }
112 }
113
114 rpos += compressedBlockSize; /* advance the read pointer to the next block */
115 roBuf += blockSize; /* advance the write pointer */
116 decompSize += blockSize;
117 }
118 H5free_memory(*buf);
119 *buf = outBuf;
120 outBuf = NULL;
121 ret_value = (size_t)origSize; // should always work, as orig_size cannot be > 2GB (sizeof(size_t) < 4GB)
122 }
123 else /* forward filter */
124 {
125 size_t blockSize;
126 size_t nBlocks;
127 size_t outSize; /* size of the output buffer. Header size (12 bytes) is included */
128 size_t block;
129 uint64_t *i64Buf;
130 uint32_t *i32Buf;
131 size_t maxDestSize;
132 char *rpos; /* pointer to current read position */
133 char *roBuf; /* pointer to current write position */
134
135 if (nbytes > INT32_MAX)
136 {
137 /* can only compress chunks up to 2GB */
138 goto error;
139 }
140
141 if(cd_nelmts > 0 && cd_values[0] > 0)
142 {
143 blockSize = cd_values[0];
144 }
145 else
146 {
147 blockSize = DEFAULT_BLOCK_SIZE;
148 }
149 if(blockSize > nbytes)
150 {
151 blockSize = nbytes;
152 }
153 nBlocks = (nbytes-1)/blockSize +1;
154 maxDestSize = nBlocks * LZ4_compressBound(blockSize) + 4 + 8 + nBlocks*4;
155 outBuf = H5allocate_memory(maxDestSize, false);
156 if (NULL == outBuf)
157 {
158 goto error;
159 }
160
161 rpos = (char*)*buf; /* pointer to current read position */
162 roBuf = (char*)outBuf; /* pointer to current write position */
163 /* header */
164 i64Buf = (uint64_t *) (roBuf);
165 i64Buf[0] = htobe64t((uint64_t)nbytes); /* Store decompressed size in be format */
166 roBuf += 8;
167
168 i32Buf = (uint32_t *) (roBuf);
169 i32Buf[0] = htobe32t((uint32_t)blockSize); /* Store the block size in be format */
170 roBuf += 4;
171
172 outSize = 12; /* size of the output buffer. Header size (12 bytes) is included */
173
174 for(block = 0; block < nBlocks; ++block)
175 {
176 uint32_t compBlockSize; /// reserve space for compBlockSize
177 size_t origWritten = block*blockSize;
178 if(nbytes - origWritten < blockSize) /* the last block may be < blockSize */
179 blockSize = nbytes - origWritten;
180
181 #if LZ4_VERSION_NUMBER > 10300
182 compBlockSize = LZ4_compress_default(rpos, roBuf+4, blockSize, LZ4_compressBound(blockSize)); /// reserve space for compBlockSize
183 #else
184 compBlockSize = LZ4_compress(rpos, roBuf+4, blockSize); /// reserve space for compBlockSize
185 #endif
186 if(!compBlockSize)
187 goto error;
188 if(compBlockSize >= blockSize) /* compression did not save any space, do a memcpy instead */
189 {
190 compBlockSize = blockSize;
191 memcpy(roBuf+4, rpos, blockSize);
192 }
193
194 i32Buf = (uint32_t *) (roBuf);
195 i32Buf[0] = htobe32t((uint32_t)compBlockSize); /* write blocksize */
196 roBuf += 4;
197
198 rpos += blockSize; /* advance read pointer */
199 roBuf += compBlockSize; /* advance write pointer */
200 outSize += compBlockSize + 4;
201 }
202
203 H5free_memory(*buf);
204 *buf = outBuf;
205 *buf_size = outSize;
206 outBuf = NULL;
207 ret_value = outSize;
208
209 }
210 /* done: */
211 if(outBuf)
212 H5free_memory(outBuf);
213 return ret_value;
214
215
216 error:
217 if(outBuf)
218 H5free_memory(outBuf);
219 outBuf = NULL;
220 return 0;
221 }
222
223 int lz4_register_h5filter(void){
224
225 int retval;
226
227 retval = H5Zregister(H5Z_LZ4);
228 if(retval<0){
229 PUSH_ERR("lz4_register_h5filter",
230 H5E_CANTREGISTER, "Can't register lz4 filter");
231 }
232 return retval;
233 }