| 1 |
/* |
| 2 |
* Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. |
| 3 |
* All rights reserved. |
| 4 |
* |
| 5 |
* This source code is licensed under both the BSD-style license (found in the |
| 6 |
* LICENSE file in the root directory of this source tree) and the GPLv2 (found |
| 7 |
* in the COPYING file in the root directory of this source tree). |
| 8 |
* You may select, at your option, one of the above-listed licenses. |
| 9 |
*/ |
| 10 |
|
| 11 |
/* zstd_ddict.c : |
| 12 |
* concentrates all logic that needs to know the internals of ZSTD_DDict object */ |
| 13 |
|
| 14 |
/*-******************************************************* |
| 15 |
* Dependencies |
| 16 |
*********************************************************/ |
| 17 |
#include <string.h> /* memcpy, memmove, memset */ |
| 18 |
#include "../common/cpu.h" /* bmi2 */ |
| 19 |
#include "../common/mem.h" /* low level memory routines */ |
| 20 |
#define FSE_STATIC_LINKING_ONLY |
| 21 |
#include "../common/fse.h" |
| 22 |
#define HUF_STATIC_LINKING_ONLY |
| 23 |
#include "../common/huf.h" |
| 24 |
#include "zstd_decompress_internal.h" |
| 25 |
#include "zstd_ddict.h" |
| 26 |
|
| 27 |
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) |
| 28 |
# include "../legacy/zstd_legacy.h" |
| 29 |
#endif |
| 30 |
|
| 31 |
|
| 32 |
|
| 33 |
/*-******************************************************* |
| 34 |
* Types |
| 35 |
*********************************************************/ |
| 36 |
struct ZSTD_DDict_s { |
| 37 |
void* dictBuffer; |
| 38 |
const void* dictContent; |
| 39 |
size_t dictSize; |
| 40 |
ZSTD_entropyDTables_t entropy; |
| 41 |
U32 dictID; |
| 42 |
U32 entropyPresent; |
| 43 |
ZSTD_customMem cMem; |
| 44 |
}; /* typedef'd to ZSTD_DDict within "zstd.h" */ |
| 45 |
|
| 46 |
const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) |
| 47 |
{ |
| 48 |
assert(ddict != NULL); |
| 49 |
return ddict->dictContent; |
| 50 |
} |
| 51 |
|
| 52 |
size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) |
| 53 |
{ |
| 54 |
assert(ddict != NULL); |
| 55 |
return ddict->dictSize; |
| 56 |
} |
| 57 |
|
| 58 |
void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) |
| 59 |
{ |
| 60 |
DEBUGLOG(4, "ZSTD_copyDDictParameters"); |
| 61 |
assert(dctx != NULL); |
| 62 |
assert(ddict != NULL); |
| 63 |
dctx->dictID = ddict->dictID; |
| 64 |
dctx->prefixStart = ddict->dictContent; |
| 65 |
dctx->virtualStart = ddict->dictContent; |
| 66 |
dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; |
| 67 |
dctx->previousDstEnd = dctx->dictEnd; |
| 68 |
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION |
| 69 |
dctx->dictContentBeginForFuzzing = dctx->prefixStart; |
| 70 |
dctx->dictContentEndForFuzzing = dctx->previousDstEnd; |
| 71 |
#endif |
| 72 |
if (ddict->entropyPresent) { |
| 73 |
dctx->litEntropy = 1; |
| 74 |
dctx->fseEntropy = 1; |
| 75 |
dctx->LLTptr = ddict->entropy.LLTable; |
| 76 |
dctx->MLTptr = ddict->entropy.MLTable; |
| 77 |
dctx->OFTptr = ddict->entropy.OFTable; |
| 78 |
dctx->HUFptr = ddict->entropy.hufTable; |
| 79 |
dctx->entropy.rep[0] = ddict->entropy.rep[0]; |
| 80 |
dctx->entropy.rep[1] = ddict->entropy.rep[1]; |
| 81 |
dctx->entropy.rep[2] = ddict->entropy.rep[2]; |
| 82 |
} else { |
| 83 |
dctx->litEntropy = 0; |
| 84 |
dctx->fseEntropy = 0; |
| 85 |
} |
| 86 |
} |
| 87 |
|
| 88 |
|
| 89 |
static size_t |
| 90 |
ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, |
| 91 |
ZSTD_dictContentType_e dictContentType) |
| 92 |
{ |
| 93 |
ddict->dictID = 0; |
| 94 |
ddict->entropyPresent = 0; |
| 95 |
if (dictContentType == ZSTD_dct_rawContent) return 0; |
| 96 |
|
| 97 |
if (ddict->dictSize < 8) { |
| 98 |
if (dictContentType == ZSTD_dct_fullDict) |
| 99 |
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ |
| 100 |
return 0; /* pure content mode */ |
| 101 |
} |
| 102 |
{ U32 const magic = MEM_readLE32(ddict->dictContent); |
| 103 |
if (magic != ZSTD_MAGIC_DICTIONARY) { |
| 104 |
if (dictContentType == ZSTD_dct_fullDict) |
| 105 |
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ |
| 106 |
return 0; /* pure content mode */ |
| 107 |
} |
| 108 |
} |
| 109 |
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); |
| 110 |
|
| 111 |
/* load entropy tables */ |
| 112 |
RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( |
| 113 |
&ddict->entropy, ddict->dictContent, ddict->dictSize)), |
| 114 |
dictionary_corrupted, ""); |
| 115 |
ddict->entropyPresent = 1; |
| 116 |
return 0; |
| 117 |
} |
| 118 |
|
| 119 |
|
| 120 |
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, |
| 121 |
const void* dict, size_t dictSize, |
| 122 |
ZSTD_dictLoadMethod_e dictLoadMethod, |
| 123 |
ZSTD_dictContentType_e dictContentType) |
| 124 |
{ |
| 125 |
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { |
| 126 |
ddict->dictBuffer = NULL; |
| 127 |
ddict->dictContent = dict; |
| 128 |
if (!dict) dictSize = 0; |
| 129 |
} else { |
| 130 |
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); |
| 131 |
ddict->dictBuffer = internalBuffer; |
| 132 |
ddict->dictContent = internalBuffer; |
| 133 |
if (!internalBuffer) return ERROR(memory_allocation); |
| 134 |
memcpy(internalBuffer, dict, dictSize); |
| 135 |
} |
| 136 |
ddict->dictSize = dictSize; |
| 137 |
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ |
| 138 |
|
| 139 |
/* parse dictionary content */ |
| 140 |
FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); |
| 141 |
|
| 142 |
return 0; |
| 143 |
} |
| 144 |
|
| 145 |
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, |
| 146 |
ZSTD_dictLoadMethod_e dictLoadMethod, |
| 147 |
ZSTD_dictContentType_e dictContentType, |
| 148 |
ZSTD_customMem customMem) |
| 149 |
{ |
| 150 |
if (!customMem.customAlloc ^ !customMem.customFree) return NULL; |
| 151 |
|
| 152 |
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); |
| 153 |
if (ddict == NULL) return NULL; |
| 154 |
ddict->cMem = customMem; |
| 155 |
{ size_t const initResult = ZSTD_initDDict_internal(ddict, |
| 156 |
dict, dictSize, |
| 157 |
dictLoadMethod, dictContentType); |
| 158 |
if (ZSTD_isError(initResult)) { |
| 159 |
ZSTD_freeDDict(ddict); |
| 160 |
return NULL; |
| 161 |
} } |
| 162 |
return ddict; |
| 163 |
} |
| 164 |
} |
| 165 |
|
| 166 |
/*! ZSTD_createDDict() : |
| 167 |
* Create a digested dictionary, to start decompression without startup delay. |
| 168 |
* `dict` content is copied inside DDict. |
| 169 |
* Consequently, `dict` can be released after `ZSTD_DDict` creation */ |
| 170 |
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) |
| 171 |
{ |
| 172 |
ZSTD_customMem const allocator = { NULL, NULL, NULL }; |
| 173 |
return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); |
| 174 |
} |
| 175 |
|
| 176 |
/*! ZSTD_createDDict_byReference() : |
| 177 |
* Create a digested dictionary, to start decompression without startup delay. |
| 178 |
* Dictionary content is simply referenced, it will be accessed during decompression. |
| 179 |
* Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ |
| 180 |
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) |
| 181 |
{ |
| 182 |
ZSTD_customMem const allocator = { NULL, NULL, NULL }; |
| 183 |
return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); |
| 184 |
} |
| 185 |
|
| 186 |
|
| 187 |
const ZSTD_DDict* ZSTD_initStaticDDict( |
| 188 |
void* sBuffer, size_t sBufferSize, |
| 189 |
const void* dict, size_t dictSize, |
| 190 |
ZSTD_dictLoadMethod_e dictLoadMethod, |
| 191 |
ZSTD_dictContentType_e dictContentType) |
| 192 |
{ |
| 193 |
size_t const neededSpace = sizeof(ZSTD_DDict) |
| 194 |
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); |
| 195 |
ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; |
| 196 |
assert(sBuffer != NULL); |
| 197 |
assert(dict != NULL); |
| 198 |
if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ |
| 199 |
if (sBufferSize < neededSpace) return NULL; |
| 200 |
if (dictLoadMethod == ZSTD_dlm_byCopy) { |
| 201 |
memcpy(ddict+1, dict, dictSize); /* local copy */ |
| 202 |
dict = ddict+1; |
| 203 |
} |
| 204 |
if (ZSTD_isError( ZSTD_initDDict_internal(ddict, |
| 205 |
dict, dictSize, |
| 206 |
ZSTD_dlm_byRef, dictContentType) )) |
| 207 |
return NULL; |
| 208 |
return ddict; |
| 209 |
} |
| 210 |
|
| 211 |
|
| 212 |
size_t ZSTD_freeDDict(ZSTD_DDict* ddict) |
| 213 |
{ |
| 214 |
if (ddict==NULL) return 0; /* support free on NULL */ |
| 215 |
{ ZSTD_customMem const cMem = ddict->cMem; |
| 216 |
ZSTD_free(ddict->dictBuffer, cMem); |
| 217 |
ZSTD_free(ddict, cMem); |
| 218 |
return 0; |
| 219 |
} |
| 220 |
} |
| 221 |
|
| 222 |
/*! ZSTD_estimateDDictSize() : |
| 223 |
* Estimate amount of memory that will be needed to create a dictionary for decompression. |
| 224 |
* Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ |
| 225 |
size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) |
| 226 |
{ |
| 227 |
return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); |
| 228 |
} |
| 229 |
|
| 230 |
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) |
| 231 |
{ |
| 232 |
if (ddict==NULL) return 0; /* support sizeof on NULL */ |
| 233 |
return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; |
| 234 |
} |
| 235 |
|
| 236 |
/*! ZSTD_getDictID_fromDDict() : |
| 237 |
* Provides the dictID of the dictionary loaded into `ddict`. |
| 238 |
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. |
| 239 |
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ |
| 240 |
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) |
| 241 |
{ |
| 242 |
if (ddict==NULL) return 0; |
| 243 |
return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); |
| 244 |
} |