Debian Codegraph @

src/backend/access/common/toast_compression.c

raw
1 /*-------------------------------------------------------------------------
2 *
3 * toast_compression.c
4 * Functions for toast compression.
5 *
6 * Copyright (c) 2021-2025, PostgreSQL Global Development Group
7 *
8 *
9 * IDENTIFICATION
10 * src/backend/access/common/toast_compression.c
11 *
12 *-------------------------------------------------------------------------
13 */
14 #include "postgres.h"
15
16 #ifdef USE_LZ4
17 #include <lz4.h>
18 #endif
19
20 #include "access/detoast.h"
21 #include "access/toast_compression.h"
22 #include "common/pg_lzcompress.h"
23 #include "varatt.h"
24
25 /* GUC */
26 int default_toast_compression = TOAST_PGLZ_COMPRESSION;
27
28 #define NO_LZ4_SUPPORT() \
29 ereport(ERROR, \
30 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
31 errmsg("compression method lz4 not supported"), \
32 errdetail("This functionality requires the server to be built with lz4 support.")))
33
34 /*
35 * Compress a varlena using PGLZ.
36 *
37 * Returns the compressed varlena, or NULL if compression fails.
38 */
39 struct varlena *
40 pglz_compress_datum(const struct varlena *value)
41 {
42 int32 valsize,
43 len;
44 struct varlena *tmp = NULL;
45
46 valsize = VARSIZE_ANY_EXHDR(value);
47
48 /*
49 * No point in wasting a palloc cycle if value size is outside the allowed
50 * range for compression.
51 */
52 if (valsize < PGLZ_strategy_default->min_input_size ||
53 valsize > PGLZ_strategy_default->max_input_size)
54 return NULL;
55
56 /*
57 * Figure out the maximum possible size of the pglz output, add the bytes
58 * that will be needed for varlena overhead, and allocate that amount.
59 */
60 tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
61 VARHDRSZ_COMPRESSED);
62
63 len = pglz_compress(VARDATA_ANY(value),
64 valsize,
65 (char *) tmp + VARHDRSZ_COMPRESSED,
66 NULL);
67 if (len < 0)
68 {
69 pfree(tmp);
70 return NULL;
71 }
72
73 SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED);
74
75 return tmp;
76 }
77
78 /*
79 * Decompress a varlena that was compressed using PGLZ.
80 */
81 struct varlena *
82 pglz_decompress_datum(const struct varlena *value)
83 {
84 struct varlena *result;
85 int32 rawsize;
86
87 /* allocate memory for the uncompressed data */
88 result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
89
90 /* decompress the data */
91 rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
92 VARSIZE(value) - VARHDRSZ_COMPRESSED,
93 VARDATA(result),
94 VARDATA_COMPRESSED_GET_EXTSIZE(value), true);
95 if (rawsize < 0)
96 ereport(ERROR,
97 (errcode(ERRCODE_DATA_CORRUPTED),
98 errmsg_internal("compressed pglz data is corrupt")));
99
100 SET_VARSIZE(result, rawsize + VARHDRSZ);
101
102 return result;
103 }
104
105 /*
106 * Decompress part of a varlena that was compressed using PGLZ.
107 */
108 struct varlena *
109 pglz_decompress_datum_slice(const struct varlena *value,
110 int32 slicelength)
111 {
112 struct varlena *result;
113 int32 rawsize;
114
115 /* allocate memory for the uncompressed data */
116 result = (struct varlena *) palloc(slicelength + VARHDRSZ);
117
118 /* decompress the data */
119 rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
120 VARSIZE(value) - VARHDRSZ_COMPRESSED,
121 VARDATA(result),
122 slicelength, false);
123 if (rawsize < 0)
124 ereport(ERROR,
125 (errcode(ERRCODE_DATA_CORRUPTED),
126 errmsg_internal("compressed pglz data is corrupt")));
127
128 SET_VARSIZE(result, rawsize + VARHDRSZ);
129
130 return result;
131 }
132
133 /*
134 * Compress a varlena using LZ4.
135 *
136 * Returns the compressed varlena, or NULL if compression fails.
137 */
138 struct varlena *
139 lz4_compress_datum(const struct varlena *value)
140 {
141 #ifndef USE_LZ4
142 NO_LZ4_SUPPORT();
143 return NULL; /* keep compiler quiet */
144 #else
145 int32 valsize;
146 int32 len;
147 int32 max_size;
148 struct varlena *tmp = NULL;
149
150 valsize = VARSIZE_ANY_EXHDR(value);
151
152 /*
153 * Figure out the maximum possible size of the LZ4 output, add the bytes
154 * that will be needed for varlena overhead, and allocate that amount.
155 */
156 max_size = LZ4_compressBound(valsize);
157 tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
158
159 len = LZ4_compress_default(VARDATA_ANY(value),
160 (char *) tmp + VARHDRSZ_COMPRESSED,
161 valsize, max_size);
162 if (len <= 0)
163 elog(ERROR, "lz4 compression failed");
164
165 /* data is incompressible so just free the memory and return NULL */
166 if (len > valsize)
167 {
168 pfree(tmp);
169 return NULL;
170 }
171
172 SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED);
173
174 return tmp;
175 #endif
176 }
177
178 /*
179 * Decompress a varlena that was compressed using LZ4.
180 */
181 struct varlena *
182 lz4_decompress_datum(const struct varlena *value)
183 {
184 #ifndef USE_LZ4
185 NO_LZ4_SUPPORT();
186 return NULL; /* keep compiler quiet */
187 #else
188 int32 rawsize;
189 struct varlena *result;
190
191 /* allocate memory for the uncompressed data */
192 result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
193
194 /* decompress the data */
195 rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESSED,
196 VARDATA(result),
197 VARSIZE(value) - VARHDRSZ_COMPRESSED,
198 VARDATA_COMPRESSED_GET_EXTSIZE(value));
199 if (rawsize < 0)
200 ereport(ERROR,
201 (errcode(ERRCODE_DATA_CORRUPTED),
202 errmsg_internal("compressed lz4 data is corrupt")));
203
204
205 SET_VARSIZE(result, rawsize + VARHDRSZ);
206
207 return result;
208 #endif
209 }
210
211 /*
212 * Decompress part of a varlena that was compressed using LZ4.
213 */
214 struct varlena *
215 lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
216 {
217 #ifndef USE_LZ4
218 NO_LZ4_SUPPORT();
219 return NULL; /* keep compiler quiet */
220 #else
221 int32 rawsize;
222 struct varlena *result;
223
224 /* slice decompression not supported prior to 1.8.3 */
225 if (LZ4_versionNumber() < 10803)
226 return lz4_decompress_datum(value);
227
228 /* allocate memory for the uncompressed data */
229 result = (struct varlena *) palloc(slicelength + VARHDRSZ);
230
231 /* decompress the data */
232 rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESSED,
233 VARDATA(result),
234 VARSIZE(value) - VARHDRSZ_COMPRESSED,
235 slicelength,
236 slicelength);
237 if (rawsize < 0)
238 ereport(ERROR,
239 (errcode(ERRCODE_DATA_CORRUPTED),
240 errmsg_internal("compressed lz4 data is corrupt")));
241
242 SET_VARSIZE(result, rawsize + VARHDRSZ);
243
244 return result;
245 #endif
246 }
247
248 /*
249 * Extract compression ID from a varlena.
250 *
251 * Returns TOAST_INVALID_COMPRESSION_ID if the varlena is not compressed.
252 */
253 ToastCompressionId
254 toast_get_compression_id(struct varlena *attr)
255 {
256 ToastCompressionId cmid = TOAST_INVALID_COMPRESSION_ID;
257
258 /*
259 * If it is stored externally then fetch the compression method id from
260 * the external toast pointer. If compressed inline, fetch it from the
261 * toast compression header.
262 */
263 if (VARATT_IS_EXTERNAL_ONDISK(attr))
264 {
265 struct varatt_external toast_pointer;
266
267 VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
268
269 if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
270 cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
271 }
272 else if (VARATT_IS_COMPRESSED(attr))
273 cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
274
275 return cmid;
276 }
277
278 /*
279 * CompressionNameToMethod - Get compression method from compression name
280 *
281 * Search in the available built-in methods. If the compression not found
282 * in the built-in methods then return InvalidCompressionMethod.
283 */
284 char
285 CompressionNameToMethod(const char *compression)
286 {
287 if (strcmp(compression, "pglz") == 0)
288 return TOAST_PGLZ_COMPRESSION;
289 else if (strcmp(compression, "lz4") == 0)
290 {
291 #ifndef USE_LZ4
292 NO_LZ4_SUPPORT();
293 #endif
294 return TOAST_LZ4_COMPRESSION;
295 }
296
297 return InvalidCompressionMethod;
298 }
299
300 /*
301 * GetCompressionMethodName - Get compression method name
302 */
303 const char *
304 GetCompressionMethodName(char method)
305 {
306 switch (method)
307 {
308 case TOAST_PGLZ_COMPRESSION:
309 return "pglz";
310 case TOAST_LZ4_COMPRESSION:
311 return "lz4";
312 default:
313 elog(ERROR, "invalid compression method %c", method);
314 return NULL; /* keep compiler quiet */
315 }
316 }