| 1 |
#ifndef SEEKABLE_H |
| 2 |
#define SEEKABLE_H |
| 3 |
|
| 4 |
#include <stdio.h> |
| 5 |
#include "zstd.h" /* ZSTDLIB_API */ |
| 6 |
|
| 7 |
#if defined (__cplusplus) |
| 8 |
extern "C" { |
| 9 |
#endif |
| 10 |
|
| 11 |
|
| 12 |
#define ZSTD_seekTableFooterSize 9 |
| 13 |
|
| 14 |
#define ZSTD_SEEKABLE_MAGICNUMBER 0x8F92EAB1 |
| 15 |
|
| 16 |
#define ZSTD_SEEKABLE_MAXFRAMES 0x8000000U |
| 17 |
|
| 18 |
/* Limit maximum size to avoid potential issues storing the compressed size */ |
| 19 |
#define ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE 0x40000000U |
| 20 |
|
| 21 |
/*-**************************************************************************** |
| 22 |
* Seekable Format |
| 23 |
* |
| 24 |
* The seekable format splits the compressed data into a series of "frames", |
| 25 |
* each compressed individually so that decompression of a section in the |
| 26 |
* middle of an archive only requires zstd to decompress at most a frame's |
| 27 |
* worth of extra data, instead of the entire archive. |
| 28 |
******************************************************************************/ |
| 29 |
|
| 30 |
typedef struct ZSTD_seekable_CStream_s ZSTD_seekable_CStream; |
| 31 |
typedef struct ZSTD_seekable_s ZSTD_seekable; |
| 32 |
typedef struct ZSTD_seekTable_s ZSTD_seekTable; |
| 33 |
|
| 34 |
/*-**************************************************************************** |
| 35 |
* Seekable compression - HowTo |
| 36 |
* A ZSTD_seekable_CStream object is required to tracking streaming operation. |
| 37 |
* Use ZSTD_seekable_createCStream() and ZSTD_seekable_freeCStream() to create/ |
| 38 |
* release resources. |
| 39 |
* |
| 40 |
* Streaming objects are reusable to avoid allocation and deallocation, |
| 41 |
* to start a new compression operation call ZSTD_seekable_initCStream() on the |
| 42 |
* compressor. |
| 43 |
* |
| 44 |
* Data streamed to the seekable compressor will automatically be split into |
| 45 |
* frames of size `maxFrameSize` (provided in ZSTD_seekable_initCStream()), |
| 46 |
* or if none is provided, will be cut off whenever ZSTD_seekable_endFrame() is |
| 47 |
* called or when the default maximum frame size (2GB) is reached. |
| 48 |
* |
| 49 |
* Use ZSTD_seekable_initCStream() to initialize a ZSTD_seekable_CStream object |
| 50 |
* for a new compression operation. |
| 51 |
* - `maxFrameSize` indicates the size at which to automatically start a new |
| 52 |
* seekable frame. |
| 53 |
* `maxFrameSize == 0` implies the default maximum size. |
| 54 |
* Smaller frame sizes allow faster decompression of small segments, |
| 55 |
* since retrieving a single byte requires decompression of |
| 56 |
* the full frame where the byte belongs. |
| 57 |
* In general, size the frames to roughly correspond to |
| 58 |
* the access granularity (when it's known). |
| 59 |
* But small sizes also reduce compression ratio. |
| 60 |
* Avoid really tiny frame sizes (< 1 KB), |
| 61 |
* that would hurt compression ratio considerably. |
| 62 |
* - `checksumFlag` indicates whether or not the seek table should include frame |
| 63 |
* checksums on the uncompressed data for verification. |
| 64 |
* @return : a size hint for input to provide for compression, or an error code |
| 65 |
* checkable with ZSTD_isError() |
| 66 |
* |
| 67 |
* Use ZSTD_seekable_compressStream() repetitively to consume input stream. |
| 68 |
* The function will automatically update both `pos` fields. |
| 69 |
* Note that it may not consume the entire input, in which case `pos < size`, |
| 70 |
* and it's up to the caller to present again remaining data. |
| 71 |
* @return : a size hint, preferred nb of bytes to use as input for next |
| 72 |
* function call or an error code, which can be tested using |
| 73 |
* ZSTD_isError(). |
| 74 |
* Note 1 : it's just a hint, to help latency a little, any other |
| 75 |
* value will work fine. |
| 76 |
* |
| 77 |
* At any time, call ZSTD_seekable_endFrame() to end the current frame and |
| 78 |
* start a new one. |
| 79 |
* |
| 80 |
* ZSTD_seekable_endStream() will end the current frame, and then write the seek |
| 81 |
* table so that decompressors can efficiently find compressed frames. |
| 82 |
* ZSTD_seekable_endStream() may return a number > 0 if it was unable to flush |
| 83 |
* all the necessary data to `output`. In this case, it should be called again |
| 84 |
* until all remaining data is flushed out and 0 is returned. |
| 85 |
******************************************************************************/ |
| 86 |
|
| 87 |
/*===== Seekable compressor management =====*/ |
| 88 |
ZSTDLIB_API ZSTD_seekable_CStream* ZSTD_seekable_createCStream(void); |
| 89 |
ZSTDLIB_API size_t ZSTD_seekable_freeCStream(ZSTD_seekable_CStream* zcs); |
| 90 |
|
| 91 |
/*===== Seekable compression functions =====*/ |
| 92 |
ZSTDLIB_API size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream* zcs, int compressionLevel, int checksumFlag, unsigned maxFrameSize); |
| 93 |
ZSTDLIB_API size_t ZSTD_seekable_compressStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); |
| 94 |
ZSTDLIB_API size_t ZSTD_seekable_endFrame(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output); |
| 95 |
ZSTDLIB_API size_t ZSTD_seekable_endStream(ZSTD_seekable_CStream* zcs, ZSTD_outBuffer* output); |
| 96 |
|
| 97 |
/*= Raw seek table API |
| 98 |
* These functions allow for the seek table to be constructed directly. |
| 99 |
* This table can then be appended to a file of concatenated frames. |
| 100 |
* This allows the frames to be compressed independently, even in parallel, |
| 101 |
* and compiled together afterward into a seekable archive. |
| 102 |
* |
| 103 |
* Use ZSTD_seekable_createFrameLog() to allocate and initialize a tracking |
| 104 |
* structure. |
| 105 |
* |
| 106 |
* Call ZSTD_seekable_logFrame() once for each frame in the archive. |
| 107 |
* checksum is optional, and will not be used if checksumFlag was 0 when the |
| 108 |
* frame log was created. If present, it should be the least significant 32 |
| 109 |
* bits of the XXH64 hash of the uncompressed data. |
| 110 |
* |
| 111 |
* Call ZSTD_seekable_writeSeekTable to serialize the data into a seek table. |
| 112 |
* If the entire table was written, the return value will be 0. Otherwise, |
| 113 |
* it will be equal to the number of bytes left to write. */ |
| 114 |
typedef struct ZSTD_frameLog_s ZSTD_frameLog; |
| 115 |
ZSTDLIB_API ZSTD_frameLog* ZSTD_seekable_createFrameLog(int checksumFlag); |
| 116 |
ZSTDLIB_API size_t ZSTD_seekable_freeFrameLog(ZSTD_frameLog* fl); |
| 117 |
ZSTDLIB_API size_t ZSTD_seekable_logFrame(ZSTD_frameLog* fl, unsigned compressedSize, unsigned decompressedSize, unsigned checksum); |
| 118 |
ZSTDLIB_API size_t ZSTD_seekable_writeSeekTable(ZSTD_frameLog* fl, ZSTD_outBuffer* output); |
| 119 |
|
| 120 |
|
| 121 |
/*-**************************************************************************** |
| 122 |
* Seekable decompression - HowTo |
| 123 |
* A ZSTD_seekable object is required to tracking the seekTable. |
| 124 |
* |
| 125 |
* Call ZSTD_seekable_init* to initialize a ZSTD_seekable object with the |
| 126 |
* the seek table provided in the input. |
| 127 |
* There are three modes for ZSTD_seekable_init: |
| 128 |
* - ZSTD_seekable_initBuff() : An in-memory API. The data contained in |
| 129 |
* `src` should be the entire seekable file, including the seek table. |
| 130 |
* `src` should be kept alive and unmodified until the ZSTD_seekable object |
| 131 |
* is freed or reset. |
| 132 |
* - ZSTD_seekable_initFile() : A simplified file API using stdio. fread and |
| 133 |
* fseek will be used to access the required data for building the seek |
| 134 |
* table and doing decompression operations. `src` should not be closed |
| 135 |
* or modified until the ZSTD_seekable object is freed or reset. |
| 136 |
* - ZSTD_seekable_initAdvanced() : A general API allowing the client to |
| 137 |
* provide its own read and seek callbacks. |
| 138 |
* + ZSTD_seekable_read() : read exactly `n` bytes into `buffer`. |
| 139 |
* Premature EOF should be treated as an error. |
| 140 |
* + ZSTD_seekable_seek() : seek the read head to `offset` from `origin`, |
| 141 |
* where origin is either SEEK_SET (beginning of |
| 142 |
* file), or SEEK_END (end of file). |
| 143 |
* Both functions should return a non-negative value in case of success, and a |
| 144 |
* negative value in case of failure. If implementing using this API and |
| 145 |
* stdio, be careful with files larger than 4GB and fseek. All of these |
| 146 |
* functions return an error code checkable with ZSTD_isError(). |
| 147 |
* |
| 148 |
* Call ZSTD_seekable_decompress to decompress `dstSize` bytes at decompressed |
| 149 |
* offset `offset`. ZSTD_seekable_decompress may have to decompress the entire |
| 150 |
* prefix of the frame before the desired data if it has not already processed |
| 151 |
* this section. If ZSTD_seekable_decompress is called multiple times for a |
| 152 |
* consecutive range of data, it will efficiently retain the decompressor object |
| 153 |
* and avoid redecompressing frame prefixes. The return value is the number of |
| 154 |
* bytes decompressed, or an error code checkable with ZSTD_isError(). |
| 155 |
* |
| 156 |
* The seek table access functions can be used to obtain the data contained |
| 157 |
* in the seek table. If frameIndex is larger than the value returned by |
| 158 |
* ZSTD_seekable_getNumFrames(), they will return error codes checkable with |
| 159 |
* ZSTD_isError(). Note that since the offset access functions return |
| 160 |
* unsigned long long instead of size_t, in this case they will instead return |
| 161 |
* the value ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE. |
| 162 |
******************************************************************************/ |
| 163 |
|
| 164 |
/*===== Seekable decompressor management =====*/ |
| 165 |
ZSTDLIB_API ZSTD_seekable* ZSTD_seekable_create(void); |
| 166 |
ZSTDLIB_API size_t ZSTD_seekable_free(ZSTD_seekable* zs); |
| 167 |
|
| 168 |
/*===== Seekable decompression functions =====*/ |
| 169 |
ZSTDLIB_API size_t ZSTD_seekable_initBuff(ZSTD_seekable* zs, const void* src, size_t srcSize); |
| 170 |
ZSTDLIB_API size_t ZSTD_seekable_initFile(ZSTD_seekable* zs, FILE* src); |
| 171 |
ZSTDLIB_API size_t ZSTD_seekable_decompress(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned long long offset); |
| 172 |
ZSTDLIB_API size_t ZSTD_seekable_decompressFrame(ZSTD_seekable* zs, void* dst, size_t dstSize, unsigned frameIndex); |
| 173 |
|
| 174 |
#define ZSTD_SEEKABLE_FRAMEINDEX_TOOLARGE (0ULL-2) |
| 175 |
/*===== Seekable seek table access functions =====*/ |
| 176 |
ZSTDLIB_API unsigned ZSTD_seekable_getNumFrames(const ZSTD_seekable* zs); |
| 177 |
ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameCompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex); |
| 178 |
ZSTDLIB_API unsigned long long ZSTD_seekable_getFrameDecompressedOffset(const ZSTD_seekable* zs, unsigned frameIndex); |
| 179 |
ZSTDLIB_API size_t ZSTD_seekable_getFrameCompressedSize(const ZSTD_seekable* zs, unsigned frameIndex); |
| 180 |
ZSTDLIB_API size_t ZSTD_seekable_getFrameDecompressedSize(const ZSTD_seekable* zs, unsigned frameIndex); |
| 181 |
ZSTDLIB_API unsigned ZSTD_seekable_offsetToFrameIndex(const ZSTD_seekable* zs, unsigned long long offset); |
| 182 |
|
| 183 |
|
| 184 |
/*-**************************************************************************** |
| 185 |
* Direct exploitation of the seekTable |
| 186 |
* |
| 187 |
* Memory constrained use cases that manage multiple archives |
| 188 |
* benefit from retaining multiple archive seek tables |
| 189 |
* without retaining a ZSTD_seekable instance for each. |
| 190 |
* |
| 191 |
* Below API allow the above-mentioned use cases |
| 192 |
* to initialize a ZSTD_seekable, extract its (smaller) ZSTD_seekTable, |
| 193 |
* then throw the ZSTD_seekable away to save memory. |
| 194 |
* |
| 195 |
* Standard ZSTD operations can then be used |
| 196 |
* to decompress frames based on seek table offsets. |
| 197 |
******************************************************************************/ |
| 198 |
|
| 199 |
/*===== Independent seek table management =====*/ |
| 200 |
ZSTDLIB_API ZSTD_seekTable* ZSTD_seekTable_create_fromSeekable(const ZSTD_seekable* zs); |
| 201 |
ZSTDLIB_API size_t ZSTD_seekTable_free(ZSTD_seekTable* st); |
| 202 |
|
| 203 |
/*===== Direct seek table access functions =====*/ |
| 204 |
ZSTDLIB_API unsigned ZSTD_seekTable_getNumFrames(const ZSTD_seekTable* st); |
| 205 |
ZSTDLIB_API unsigned long long ZSTD_seekTable_getFrameCompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex); |
| 206 |
ZSTDLIB_API unsigned long long ZSTD_seekTable_getFrameDecompressedOffset(const ZSTD_seekTable* st, unsigned frameIndex); |
| 207 |
ZSTDLIB_API size_t ZSTD_seekTable_getFrameCompressedSize(const ZSTD_seekTable* st, unsigned frameIndex); |
| 208 |
ZSTDLIB_API size_t ZSTD_seekTable_getFrameDecompressedSize(const ZSTD_seekTable* st, unsigned frameIndex); |
| 209 |
ZSTDLIB_API unsigned ZSTD_seekTable_offsetToFrameIndex(const ZSTD_seekTable* st, unsigned long long offset); |
| 210 |
|
| 211 |
|
| 212 |
/*===== Seekable advanced I/O API =====*/ |
| 213 |
typedef int(ZSTD_seekable_read)(void* opaque, void* buffer, size_t n); |
| 214 |
typedef int(ZSTD_seekable_seek)(void* opaque, long long offset, int origin); |
| 215 |
typedef struct { |
| 216 |
void* opaque; |
| 217 |
ZSTD_seekable_read* read; |
| 218 |
ZSTD_seekable_seek* seek; |
| 219 |
} ZSTD_seekable_customFile; |
| 220 |
ZSTDLIB_API size_t ZSTD_seekable_initAdvanced(ZSTD_seekable* zs, ZSTD_seekable_customFile src); |
| 221 |
|
| 222 |
#if defined (__cplusplus) |
| 223 |
} |
| 224 |
#endif |
| 225 |
|
| 226 |
#endif |