Debian Codegraph @

src/basic/compress.c

raw
1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
2
3 #include <fcntl.h>
4 #include <stdio.h>
5 #include <sys/stat.h>
6 #include <unistd.h>
7
8 #if HAVE_XZ
9 #include <lzma.h>
10 #endif
11
12 #if HAVE_LZ4
13 #include <lz4.h>
14 #include <lz4frame.h>
15 #include <lz4hc.h>
16 #endif
17
18 #if HAVE_ZSTD
19 #include <zstd.h>
20 #include <zstd_errors.h>
21 #endif
22
23 #if HAVE_ZLIB
24 #include <zlib.h>
25 #endif
26
27 #if HAVE_BZIP2
28 #include <bzlib.h>
29 #endif
30
31 #include "sd-dlopen.h"
32
33 #include "alloc-util.h"
34 #include "bitfield.h"
35 #include "compress.h"
36 #include "dlfcn-util.h"
37 #include "io-util.h"
38 #include "log.h"
39 #include "string-table.h"
40 #include "unaligned.h"
41
42 #if HAVE_XZ
43 static DLSYM_PROTOTYPE(lzma_code) = NULL;
44 static DLSYM_PROTOTYPE(lzma_easy_encoder) = NULL;
45 static DLSYM_PROTOTYPE(lzma_end) = NULL;
46 static DLSYM_PROTOTYPE(lzma_stream_buffer_encode) = NULL;
47 static DLSYM_PROTOTYPE(lzma_stream_decoder) = NULL;
48 static DLSYM_PROTOTYPE(lzma_lzma_preset) = NULL;
49
50 /* We can’t just do _cleanup_(sym_lzma_end) because a compiler bug makes
51 * this fail with:
52 * ../src/basic/compress.c: In function ‘decompress_blob_xz’:
53 * ../src/basic/compress.c:304:9: error: cleanup argument not a function
54 * 304 | _cleanup_(sym_lzma_end) lzma_stream s = LZMA_STREAM_INIT;
55 * | ^~~~~~~~~
56 */
57 static inline void lzma_end_wrapper(lzma_stream *ls) {
58 sym_lzma_end(ls);
59 }
60 #endif
61
62 #if HAVE_LZ4
63 static DLSYM_PROTOTYPE(LZ4F_compressBegin) = NULL;
64 static DLSYM_PROTOTYPE(LZ4F_compressBound) = NULL;
65 static DLSYM_PROTOTYPE(LZ4F_compressEnd) = NULL;
66 static DLSYM_PROTOTYPE(LZ4F_compressUpdate) = NULL;
67 static DLSYM_PROTOTYPE(LZ4F_createCompressionContext) = NULL;
68 static DLSYM_PROTOTYPE(LZ4F_createDecompressionContext) = NULL;
69 static DLSYM_PROTOTYPE(LZ4F_decompress) = NULL;
70 static DLSYM_PROTOTYPE(LZ4F_freeCompressionContext) = NULL;
71 static DLSYM_PROTOTYPE(LZ4F_freeDecompressionContext) = NULL;
72 static DLSYM_PROTOTYPE(LZ4F_isError) = NULL;
73 static DLSYM_PROTOTYPE(LZ4_compress_HC) = NULL;
74 static DLSYM_PROTOTYPE(LZ4_compress_default) = NULL;
75 static DLSYM_PROTOTYPE(LZ4_decompress_safe) = NULL;
76 static DLSYM_PROTOTYPE(LZ4_decompress_safe_partial) = NULL;
77 static DLSYM_PROTOTYPE(LZ4_versionNumber) = NULL;
78
79 static const LZ4F_preferences_t lz4_preferences = {
80 .frameInfo.blockSizeID = 5,
81 };
82 #endif
83
84 #if HAVE_ZSTD
85 static DLSYM_PROTOTYPE(ZSTD_CCtx_setParameter) = NULL;
86 static DLSYM_PROTOTYPE(ZSTD_compress) = NULL;
87 static DLSYM_PROTOTYPE(ZSTD_compressStream2) = NULL;
88 static DLSYM_PROTOTYPE(ZSTD_createCCtx) = NULL;
89 static DLSYM_PROTOTYPE(ZSTD_createDCtx) = NULL;
90 static DLSYM_PROTOTYPE(ZSTD_CStreamInSize) = NULL;
91 static DLSYM_PROTOTYPE(ZSTD_CStreamOutSize) = NULL;
92 static DLSYM_PROTOTYPE(ZSTD_decompressStream) = NULL;
93 static DLSYM_PROTOTYPE(ZSTD_DStreamInSize) = NULL;
94 static DLSYM_PROTOTYPE(ZSTD_DStreamOutSize) = NULL;
95 static DLSYM_PROTOTYPE(ZSTD_freeCCtx) = NULL;
96 static DLSYM_PROTOTYPE(ZSTD_freeDCtx) = NULL;
97 static DLSYM_PROTOTYPE(ZSTD_getErrorCode) = NULL;
98 static DLSYM_PROTOTYPE(ZSTD_getErrorName) = NULL;
99 static DLSYM_PROTOTYPE(ZSTD_getFrameContentSize) = NULL;
100 static DLSYM_PROTOTYPE(ZSTD_isError) = NULL;
101
102 DEFINE_TRIVIAL_CLEANUP_FUNC_FULL_RENAME(ZSTD_DCtx*, sym_ZSTD_freeDCtx, ZSTD_freeDCtxp, NULL);
103
104 static int zstd_ret_to_errno(size_t ret) {
105 switch (sym_ZSTD_getErrorCode(ret)) {
106 case ZSTD_error_dstSize_tooSmall:
107 return -ENOBUFS;
108 case ZSTD_error_memory_allocation:
109 return -ENOMEM;
110 default:
111 return -EBADMSG;
112 }
113 }
114 #endif
115
116 #if HAVE_ZLIB
117 static DLSYM_PROTOTYPE(deflateInit2_) = NULL;
118 static DLSYM_PROTOTYPE(deflate) = NULL;
119 static DLSYM_PROTOTYPE(deflateEnd) = NULL;
120 static DLSYM_PROTOTYPE(inflateInit2_) = NULL;
121 static DLSYM_PROTOTYPE(inflate) = NULL;
122 static DLSYM_PROTOTYPE(inflateEnd) = NULL;
123
124 static inline void deflateEnd_wrapper(z_stream *s) {
125 sym_deflateEnd(s);
126 }
127
128 static inline void inflateEnd_wrapper(z_stream *s) {
129 sym_inflateEnd(s);
130 }
131 #endif
132
133 #if HAVE_BZIP2
134 static DLSYM_PROTOTYPE(BZ2_bzCompressInit) = NULL;
135 static DLSYM_PROTOTYPE(BZ2_bzCompress) = NULL;
136 static DLSYM_PROTOTYPE(BZ2_bzCompressEnd) = NULL;
137 static DLSYM_PROTOTYPE(BZ2_bzDecompressInit) = NULL;
138 static DLSYM_PROTOTYPE(BZ2_bzDecompress) = NULL;
139 static DLSYM_PROTOTYPE(BZ2_bzDecompressEnd) = NULL;
140
141 static inline void BZ2_bzCompressEnd_wrapper(bz_stream *s) {
142 sym_BZ2_bzCompressEnd(s);
143 }
144
145 static inline void BZ2_bzDecompressEnd_wrapper(bz_stream *s) {
146 sym_BZ2_bzDecompressEnd(s);
147 }
148 #endif
149
150 /* Opaque Compressor/Decompressor struct definition */
151 struct Compressor {
152 Compression type;
153 bool encoding;
154 union {
155 #if HAVE_XZ
156 lzma_stream xz;
157 #endif
158 #if HAVE_LZ4
159 struct {
160 LZ4F_compressionContext_t c_lz4;
161 void *lz4_header; /* stashed frame header from LZ4F_compressBegin */
162 size_t lz4_header_size;
163 };
164 LZ4F_decompressionContext_t d_lz4;
165 #endif
166 #if HAVE_ZSTD
167 ZSTD_CCtx *c_zstd;
168 ZSTD_DCtx *d_zstd;
169 #endif
170 #if HAVE_ZLIB
171 z_stream gzip;
172 #endif
173 #if HAVE_BZIP2
174 bz_stream bzip2;
175 #endif
176 };
177 };
178
179 #define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
180
181 /* zlib windowBits value for gzip format: MAX_WBITS (15) + 16 to enable gzip header detection/generation */
182 #define ZLIB_WBITS_GZIP (15 + 16)
183
184 static const char* const compression_table[_COMPRESSION_MAX] = {
185 [COMPRESSION_NONE] = "uncompressed", /* backwards compatibility with importd */
186 [COMPRESSION_XZ] = "xz",
187 [COMPRESSION_LZ4] = "lz4",
188 [COMPRESSION_ZSTD] = "zstd",
189 [COMPRESSION_GZIP] = "gzip",
190 [COMPRESSION_BZIP2] = "bzip2",
191 };
192
193 static const char* const compression_uppercase_table[_COMPRESSION_MAX] = {
194 [COMPRESSION_NONE] = "NONE", /* backwards compatibility with SYSTEMD_JOURNAL_COMPRESS=NONE */
195 [COMPRESSION_XZ] = "XZ",
196 [COMPRESSION_LZ4] = "LZ4",
197 [COMPRESSION_ZSTD] = "ZSTD",
198 [COMPRESSION_GZIP] = "GZIP",
199 [COMPRESSION_BZIP2] = "BZIP2",
200 };
201
202 static const char* const compression_extension_table[_COMPRESSION_MAX] = {
203 [COMPRESSION_NONE] = "",
204 [COMPRESSION_XZ] = ".xz",
205 [COMPRESSION_LZ4] = ".lz4",
206 [COMPRESSION_ZSTD] = ".zst",
207 [COMPRESSION_GZIP] = ".gz",
208 [COMPRESSION_BZIP2] = ".bz2",
209 };
210
211 DEFINE_STRING_TABLE_LOOKUP(compression, Compression);
212 DEFINE_STRING_TABLE_LOOKUP(compression_uppercase, Compression);
213 DEFINE_STRING_TABLE_LOOKUP(compression_extension, Compression);
214
215 Compression compression_from_string_harder(const char *s) {
216 Compression c;
217
218 assert(s);
219
220 c = compression_from_string(s);
221 if (c >= 0)
222 return c;
223
224 return compression_uppercase_from_string(s);
225 }
226
227 Compression compression_from_filename(const char *filename) {
228 Compression c;
229 const char *e;
230
231 assert(filename);
232
233 e = strrchr(filename, '.');
234 if (!e)
235 return COMPRESSION_NONE;
236
237 c = compression_extension_from_string(e);
238 if (c < 0)
239 return COMPRESSION_NONE;
240
241 return c;
242 }
243
244 bool compression_supported(Compression c) {
245 static const unsigned supported =
246 (1U << COMPRESSION_NONE) |
247 (1U << COMPRESSION_XZ) * HAVE_XZ |
248 (1U << COMPRESSION_LZ4) * HAVE_LZ4 |
249 (1U << COMPRESSION_ZSTD) * HAVE_ZSTD |
250 (1U << COMPRESSION_GZIP) * HAVE_ZLIB |
251 (1U << COMPRESSION_BZIP2) * HAVE_BZIP2;
252
253 assert(c >= 0);
254 assert(c < _COMPRESSION_MAX);
255
256 return BIT_SET(supported, c);
257 }
258
259 Compression compression_detect_from_magic(const uint8_t data[static COMPRESSION_MAGIC_BYTES_MAX]) {
260 /* Magic signatures per RFC 1952 (gzip), tukaani.org/xz/xz-file-format.txt (xz),
261 * RFC 8878 (zstd), lz4/doc/lz4_Frame_format.md (lz4), and the bzip2 file format.
262 * Make sure to update COMPRESSION_MAGIC_BYTES_MAX if needed when adding a new magic. */
263 if (memcmp(data, (const uint8_t[]) { 0x1f, 0x8b }, 2) == 0)
264 return COMPRESSION_GZIP;
265 if (memcmp(data, (const uint8_t[]) { 0xfd, '7', 'z', 'X', 'Z', 0x00 }, 6) == 0)
266 return COMPRESSION_XZ;
267 if (memcmp(data, (const uint8_t[]) { 0x28, 0xb5, 0x2f, 0xfd }, 4) == 0)
268 return COMPRESSION_ZSTD;
269 if (memcmp(data, (const uint8_t[]) { 0x04, 0x22, 0x4d, 0x18 }, 4) == 0)
270 return COMPRESSION_LZ4;
271 if (memcmp(data, (const uint8_t[]) { 'B', 'Z', 'h' }, 3) == 0)
272 return COMPRESSION_BZIP2;
273
274 return _COMPRESSION_INVALID;
275 }
276
277 int dlopen_xz(int log_level) {
278 #if HAVE_XZ
279 static void *lzma_dl = NULL;
280
281 SD_ELF_NOTE_DLOPEN(
282 "lzma",
283 "Support lzma compression in journal and coredump files",
284 COMPRESSION_PRIORITY_XZ,
285 "liblzma.so.5");
286
287 return dlopen_many_sym_or_warn(
288 &lzma_dl,
289 "liblzma.so.5", log_level,
290 DLSYM_ARG(lzma_code),
291 DLSYM_ARG(lzma_easy_encoder),
292 DLSYM_ARG(lzma_end),
293 DLSYM_ARG(lzma_stream_buffer_encode),
294 DLSYM_ARG(lzma_lzma_preset),
295 DLSYM_ARG(lzma_stream_decoder));
296 #else
297 return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
298 "lzma support is not compiled in.");
299 #endif
300 }
301
302 int dlopen_lz4(int log_level) {
303 #if HAVE_LZ4
304 static void *lz4_dl = NULL;
305
306 SD_ELF_NOTE_DLOPEN(
307 "lz4",
308 "Support lz4 compression in journal and coredump files",
309 COMPRESSION_PRIORITY_LZ4,
310 "liblz4.so.1");
311
312 return dlopen_many_sym_or_warn(
313 &lz4_dl,
314 "liblz4.so.1", log_level,
315 DLSYM_ARG(LZ4F_compressBegin),
316 DLSYM_ARG(LZ4F_compressBound),
317 DLSYM_ARG(LZ4F_compressEnd),
318 DLSYM_ARG(LZ4F_compressUpdate),
319 DLSYM_ARG(LZ4F_createCompressionContext),
320 DLSYM_ARG(LZ4F_createDecompressionContext),
321 DLSYM_ARG(LZ4F_decompress),
322 DLSYM_ARG(LZ4F_freeCompressionContext),
323 DLSYM_ARG(LZ4F_freeDecompressionContext),
324 DLSYM_ARG(LZ4F_isError),
325 DLSYM_ARG(LZ4_compress_default),
326 DLSYM_ARG(LZ4_compress_HC),
327 DLSYM_ARG(LZ4_decompress_safe),
328 DLSYM_ARG(LZ4_decompress_safe_partial),
329 DLSYM_ARG(LZ4_versionNumber));
330 #else
331 return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
332 "lz4 support is not compiled in.");
333 #endif
334 }
335
336 int dlopen_zstd(int log_level) {
337 #if HAVE_ZSTD
338 static void *zstd_dl = NULL;
339
340 SD_ELF_NOTE_DLOPEN(
341 "zstd",
342 "Support zstd compression in journal and coredump files",
343 COMPRESSION_PRIORITY_ZSTD,
344 "libzstd.so.1");
345
346 return dlopen_many_sym_or_warn(
347 &zstd_dl,
348 "libzstd.so.1", log_level,
349 DLSYM_ARG(ZSTD_getErrorCode),
350 DLSYM_ARG(ZSTD_compress),
351 DLSYM_ARG(ZSTD_getFrameContentSize),
352 DLSYM_ARG(ZSTD_decompressStream),
353 DLSYM_ARG(ZSTD_getErrorName),
354 DLSYM_ARG(ZSTD_DStreamOutSize),
355 DLSYM_ARG(ZSTD_CStreamInSize),
356 DLSYM_ARG(ZSTD_CStreamOutSize),
357 DLSYM_ARG(ZSTD_CCtx_setParameter),
358 DLSYM_ARG(ZSTD_compressStream2),
359 DLSYM_ARG(ZSTD_DStreamInSize),
360 DLSYM_ARG(ZSTD_freeCCtx),
361 DLSYM_ARG(ZSTD_freeDCtx),
362 DLSYM_ARG(ZSTD_isError),
363 DLSYM_ARG(ZSTD_createDCtx),
364 DLSYM_ARG(ZSTD_createCCtx));
365 #else
366 return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
367 "zstd support is not compiled in.");
368 #endif
369 }
370
371 int dlopen_zlib(int log_level) {
372 #if HAVE_ZLIB
373 static void *zlib_dl = NULL;
374
375 SD_ELF_NOTE_DLOPEN(
376 "zlib",
377 "Support gzip compression and decompression",
378 SD_ELF_NOTE_DLOPEN_PRIORITY_SUGGESTED,
379 "libz.so.1");
380
381 return dlopen_many_sym_or_warn(
382 &zlib_dl,
383 "libz.so.1", log_level,
384 DLSYM_ARG(deflateInit2_),
385 DLSYM_ARG(deflate),
386 DLSYM_ARG(deflateEnd),
387 DLSYM_ARG(inflateInit2_),
388 DLSYM_ARG(inflate),
389 DLSYM_ARG(inflateEnd));
390 #else
391 return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
392 "zlib support is not compiled in.");
393 #endif
394 }
395
396 int dlopen_bzip2(int log_level) {
397 #if HAVE_BZIP2
398 static void *bzip2_dl = NULL;
399
400 SD_ELF_NOTE_DLOPEN(
401 "bzip2",
402 "Support bzip2 compression and decompression",
403 SD_ELF_NOTE_DLOPEN_PRIORITY_SUGGESTED,
404 "libbz2.so.1");
405
406 return dlopen_many_sym_or_warn(
407 &bzip2_dl,
408 "libbz2.so.1", log_level,
409 DLSYM_ARG(BZ2_bzCompressInit),
410 DLSYM_ARG(BZ2_bzCompress),
411 DLSYM_ARG(BZ2_bzCompressEnd),
412 DLSYM_ARG(BZ2_bzDecompressInit),
413 DLSYM_ARG(BZ2_bzDecompress),
414 DLSYM_ARG(BZ2_bzDecompressEnd));
415 #else
416 return log_full_errno(log_level, SYNTHETIC_ERRNO(EOPNOTSUPP),
417 "bzip2 support is not compiled in.");
418 #endif
419 }
420
421 static int compress_blob_xz(
422 const void *src,
423 uint64_t src_size,
424 void *dst,
425 size_t dst_alloc_size,
426 size_t *dst_size,
427 int level) {
428
429 assert(src);
430 assert(src_size > 0);
431 assert(dst);
432 assert(dst_alloc_size > 0);
433 assert(dst_size);
434
435 #if HAVE_XZ
436 lzma_options_lzma opt = {
437 1u << 20u, NULL, 0, LZMA_LC_DEFAULT, LZMA_LP_DEFAULT,
438 LZMA_PB_DEFAULT, LZMA_MODE_FAST, 128, LZMA_MF_HC3, 4
439 };
440 lzma_filter filters[] = {
441 { LZMA_FILTER_LZMA2, &opt },
442 { LZMA_VLI_UNKNOWN, NULL }
443 };
444 lzma_ret ret;
445 size_t out_pos = 0;
446 int r;
447
448 r = dlopen_xz(LOG_DEBUG);
449 if (r < 0)
450 return r;
451
452 if (level >= 0) {
453 r = sym_lzma_lzma_preset(&opt, (uint32_t) level);
454 if (r < 0)
455 return r;
456 }
457
458 /* Returns < 0 if we couldn't compress the data or the
459 * compressed result is longer than the original */
460
461 if (src_size < 80)
462 return -ENOBUFS;
463
464 ret = sym_lzma_stream_buffer_encode(filters, LZMA_CHECK_NONE, NULL,
465 src, src_size, dst, &out_pos, dst_alloc_size);
466 if (ret != LZMA_OK)
467 return -ENOBUFS;
468
469 *dst_size = out_pos;
470 return 0;
471 #else
472 return -EPROTONOSUPPORT;
473 #endif
474 }
475
476 static int compress_blob_lz4(
477 const void *src,
478 uint64_t src_size,
479 void *dst,
480 size_t dst_alloc_size,
481 size_t *dst_size,
482 int level) {
483
484 assert(src);
485 assert(src_size > 0);
486 assert(dst);
487 assert(dst_alloc_size > 0);
488 assert(dst_size);
489
490 #if HAVE_LZ4
491 int r;
492
493 r = dlopen_lz4(LOG_DEBUG);
494 if (r < 0)
495 return r;
496 /* Returns < 0 if we couldn't compress the data or the
497 * compressed result is longer than the original */
498
499 if (src_size < 9)
500 return -ENOBUFS;
501
502 if (src_size > INT_MAX)
503 return -EFBIG;
504 if (dst_alloc_size > INT_MAX)
505 dst_alloc_size = INT_MAX;
506
507 if (level <= 0)
508 r = sym_LZ4_compress_default(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8);
509 else
510 r = sym_LZ4_compress_HC(src, (char*)dst + 8, src_size, (int) dst_alloc_size - 8, level);
511 if (r <= 0)
512 return -ENOBUFS;
513
514 unaligned_write_le64(dst, src_size);
515 *dst_size = r + 8;
516
517 return 0;
518 #else
519 return -EPROTONOSUPPORT;
520 #endif
521 }
522
523 static int compress_blob_zstd(
524 const void *src,
525 uint64_t src_size,
526 void *dst,
527 size_t dst_alloc_size,
528 size_t *dst_size,
529 int level) {
530
531 assert(src);
532 assert(src_size > 0);
533 assert(dst);
534 assert(dst_alloc_size > 0);
535 assert(dst_size);
536
537 #if HAVE_ZSTD
538 size_t k;
539 int r;
540
541 r = dlopen_zstd(LOG_DEBUG);
542 if (r < 0)
543 return r;
544
545 k = sym_ZSTD_compress(dst, dst_alloc_size, src, src_size, level < 0 ? 0 : level);
546 if (sym_ZSTD_isError(k))
547 return zstd_ret_to_errno(k);
548
549 *dst_size = k;
550 return 0;
551 #else
552 return -EPROTONOSUPPORT;
553 #endif
554 }
555
556 static int compress_blob_gzip(const void *src, uint64_t src_size,
557 void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
558
559 assert(src);
560 assert(src_size > 0);
561 assert(dst);
562 assert(dst_alloc_size > 0);
563 assert(dst_size);
564
565 #if HAVE_ZLIB
566 int r;
567
568 r = dlopen_zlib(LOG_DEBUG);
569 if (r < 0)
570 return r;
571
572 if (src_size > UINT_MAX)
573 return -EFBIG;
574 if (dst_alloc_size > UINT_MAX)
575 dst_alloc_size = UINT_MAX;
576
577 _cleanup_(deflateEnd_wrapper) z_stream s = {};
578
579 r = sym_deflateInit2_(&s, level < 0 ? Z_DEFAULT_COMPRESSION : level,
580 /* method= */ Z_DEFLATED,
581 /* windowBits= */ ZLIB_WBITS_GZIP,
582 /* memLevel= */ 8,
583 /* strategy= */ Z_DEFAULT_STRATEGY,
584 ZLIB_VERSION, (int) sizeof(s));
585 if (r != Z_OK)
586 return -ENOMEM;
587
588 s.next_in = (void*) src;
589 s.avail_in = src_size;
590 s.next_out = dst;
591 s.avail_out = dst_alloc_size;
592
593 r = sym_deflate(&s, Z_FINISH);
594 if (r != Z_STREAM_END)
595 return -ENOBUFS;
596
597 *dst_size = dst_alloc_size - s.avail_out;
598 return 0;
599 #else
600 return -EPROTONOSUPPORT;
601 #endif
602 }
603
604 static int compress_blob_bzip2(
605 const void *src, uint64_t src_size,
606 void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
607
608 assert(src);
609 assert(src_size > 0);
610 assert(dst);
611 assert(dst_alloc_size > 0);
612 assert(dst_size);
613
614 #if HAVE_BZIP2
615 int r;
616
617 r = dlopen_bzip2(LOG_DEBUG);
618 if (r < 0)
619 return r;
620
621 if (src_size > UINT_MAX)
622 return -EFBIG;
623 if (dst_alloc_size > UINT_MAX)
624 dst_alloc_size = UINT_MAX;
625
626 _cleanup_(BZ2_bzCompressEnd_wrapper) bz_stream s = {};
627
628 r = sym_BZ2_bzCompressInit(&s, level < 0 ? 9 : level, /* verbosity= */ 0, /* workFactor= */ 0);
629 if (r != BZ_OK)
630 return -ENOMEM;
631
632 s.next_in = (char*) src;
633 s.avail_in = src_size;
634 s.next_out = (char*) dst;
635 s.avail_out = dst_alloc_size;
636
637 r = sym_BZ2_bzCompress(&s, BZ_FINISH);
638
639 if (r != BZ_STREAM_END)
640 return -ENOBUFS;
641
642 *dst_size = dst_alloc_size - s.avail_out;
643 return 0;
644 #else
645 return -EPROTONOSUPPORT;
646 #endif
647 }
648
649 int compress_blob(
650 Compression compression,
651 const void *src, uint64_t src_size,
652 void *dst, size_t dst_alloc_size, size_t *dst_size, int level) {
653
654 switch (compression) {
655 case COMPRESSION_XZ:
656 return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size, level);
657 case COMPRESSION_LZ4:
658 return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size, level);
659 case COMPRESSION_ZSTD:
660 return compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size, level);
661 case COMPRESSION_GZIP:
662 return compress_blob_gzip(src, src_size, dst, dst_alloc_size, dst_size, level);
663 case COMPRESSION_BZIP2:
664 return compress_blob_bzip2(src, src_size, dst, dst_alloc_size, dst_size, level);
665 default:
666 return -EOPNOTSUPP;
667 }
668 }
669
670 static int decompress_blob_xz(
671 const void *src,
672 uint64_t src_size,
673 void **dst,
674 size_t *dst_size,
675 size_t dst_max) {
676
677 assert(src);
678 assert(src_size > 0);
679 assert(dst);
680 assert(dst_size);
681
682 #if HAVE_XZ
683 int r;
684
685 r = dlopen_xz(LOG_DEBUG);
686 if (r < 0)
687 return r;
688
689 _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
690 lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, /* flags= */ 0);
691 if (ret != LZMA_OK)
692 return -ENOMEM;
693
694 size_t space = MIN(src_size * 2, dst_max ?: SIZE_MAX);
695 if (!greedy_realloc(dst, space, 1))
696 return -ENOMEM;
697
698 s.next_in = src;
699 s.avail_in = src_size;
700
701 s.next_out = *dst;
702 s.avail_out = space;
703
704 for (;;) {
705 size_t used;
706
707 ret = sym_lzma_code(&s, LZMA_FINISH);
708 if (ret == LZMA_STREAM_END)
709 break;
710 if (ret != LZMA_OK)
711 return -ENOMEM;
712
713 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
714 break;
715 if (dst_max > 0 && space == dst_max)
716 return -ENOBUFS;
717
718 used = space - s.avail_out;
719 /* Silence static analyzers, space is bounded by allocation size */
720 assert(space <= SIZE_MAX / 2);
721 space = MIN(2 * space, dst_max ?: SIZE_MAX);
722 if (!greedy_realloc(dst, space, 1))
723 return -ENOMEM;
724
725 s.avail_out = space - used;
726 s.next_out = *(uint8_t**)dst + used;
727 }
728
729 *dst_size = space - s.avail_out;
730 return 0;
731 #else
732 return -EPROTONOSUPPORT;
733 #endif
734 }
735
736 static int decompress_blob_lz4(
737 const void *src,
738 uint64_t src_size,
739 void **dst,
740 size_t *dst_size,
741 size_t dst_max) {
742
743 assert(src);
744 assert(src_size > 0);
745 assert(dst);
746 assert(dst_size);
747
748 #if HAVE_LZ4
749 char* out;
750 int r, size; /* LZ4 uses int for size */
751
752 r = dlopen_lz4(LOG_DEBUG);
753 if (r < 0)
754 return r;
755
756 if (src_size <= 8)
757 return -EBADMSG;
758
759 if (src_size - 8 > INT_MAX)
760 return -EFBIG;
761
762 size = unaligned_read_le64(src);
763 if (size < 0 || (unsigned) size != unaligned_read_le64(src))
764 return -EFBIG;
765 if (dst_max > 0 && (size_t) size > dst_max)
766 return -ENOBUFS;
767 out = greedy_realloc(dst, size, 1);
768 if (!out)
769 return -ENOMEM;
770
771 r = sym_LZ4_decompress_safe((char*)src + 8, out, src_size - 8, size);
772 if (r < 0 || r != size)
773 return -EBADMSG;
774
775 *dst_size = size;
776 return 0;
777 #else
778 return -EPROTONOSUPPORT;
779 #endif
780 }
781
782 static int decompress_blob_zstd(
783 const void *src,
784 uint64_t src_size,
785 void **dst,
786 size_t *dst_size,
787 size_t dst_max) {
788
789 assert(src);
790 assert(src_size > 0);
791 assert(dst);
792 assert(dst_size);
793
794 #if HAVE_ZSTD
795 uint64_t size;
796 int r;
797
798 r = dlopen_zstd(LOG_DEBUG);
799 if (r < 0)
800 return r;
801
802 size = sym_ZSTD_getFrameContentSize(src, src_size);
803 if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN))
804 return -EBADMSG;
805
806 if (dst_max > 0 && size > dst_max)
807 size = dst_max;
808 if (size > SIZE_MAX)
809 return -E2BIG;
810
811 if (!(greedy_realloc(dst, MAX(sym_ZSTD_DStreamOutSize(), size), 1)))
812 return -ENOMEM;
813
814 _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx();
815 if (!dctx)
816 return -ENOMEM;
817
818 ZSTD_inBuffer input = {
819 .src = src,
820 .size = src_size,
821 };
822 ZSTD_outBuffer output = {
823 .dst = *dst,
824 .size = MALLOC_SIZEOF_SAFE(*dst),
825 };
826
827 size_t k = sym_ZSTD_decompressStream(dctx, &output, &input);
828 if (sym_ZSTD_isError(k))
829 return log_debug_errno(zstd_ret_to_errno(k), "ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k));
830 if (output.pos < size)
831 return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD decoded less data than indicated, probably corrupted stream.");
832
833 *dst_size = size;
834 return 0;
835 #else
836 return -EPROTONOSUPPORT;
837 #endif
838 }
839
840 static int decompress_blob_gzip(
841 const void *src,
842 uint64_t src_size,
843 void **dst,
844 size_t *dst_size,
845 size_t dst_max) {
846
847 assert(src);
848 assert(src_size > 0);
849 assert(dst);
850 assert(dst_size);
851
852 #if HAVE_ZLIB
853 int r;
854
855 r = dlopen_zlib(LOG_DEBUG);
856 if (r < 0)
857 return r;
858
859 if (src_size > UINT_MAX)
860 return -EFBIG;
861
862 _cleanup_(inflateEnd_wrapper) z_stream s = {};
863
864 r = sym_inflateInit2_(&s, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(s));
865 if (r != Z_OK)
866 return -ENOMEM;
867
868 size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
869 if (!greedy_realloc(dst, space, 1))
870 return -ENOMEM;
871
872 s.next_in = (void*) src;
873 s.avail_in = src_size;
874 s.next_out = *dst;
875 s.avail_out = space;
876
877 for (;;) {
878 size_t used;
879
880 r = sym_inflate(&s, Z_NO_FLUSH);
881 if (r == Z_STREAM_END)
882 break;
883 if (!IN_SET(r, Z_OK, Z_BUF_ERROR))
884 return -EBADMSG;
885
886 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
887 break;
888 if (dst_max > 0 && space == dst_max)
889 return -ENOBUFS;
890
891 used = space - s.avail_out;
892 space = MIN3(2 * space, dst_max ?: SIZE_MAX, UINT_MAX);
893 if (!greedy_realloc(dst, space, 1))
894 return -ENOMEM;
895
896 s.avail_out = space - used;
897 s.next_out = *(uint8_t**)dst + used;
898 }
899
900 *dst_size = space - s.avail_out;
901 return 0;
902 #else
903 return -EPROTONOSUPPORT;
904 #endif
905 }
906
907 static int decompress_blob_bzip2(
908 const void *src,
909 uint64_t src_size,
910 void **dst,
911 size_t *dst_size,
912 size_t dst_max) {
913
914 assert(src);
915 assert(src_size > 0);
916 assert(dst);
917 assert(dst_size);
918
919 #if HAVE_BZIP2
920 int r;
921
922 r = dlopen_bzip2(LOG_DEBUG);
923 if (r < 0)
924 return r;
925
926 if (src_size > UINT_MAX)
927 return -EFBIG;
928
929 _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {};
930
931 r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0);
932 if (r != BZ_OK)
933 return -ENOMEM;
934
935 size_t space = MIN3(src_size * 2, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
936 if (!greedy_realloc(dst, space, 1))
937 return -ENOMEM;
938
939 s.next_in = (char*) src;
940 s.avail_in = src_size;
941 s.next_out = (char*) *dst;
942 s.avail_out = space;
943
944 for (;;) {
945 size_t used;
946
947 r = sym_BZ2_bzDecompress(&s);
948 if (r == BZ_STREAM_END)
949 break;
950 if (r != BZ_OK)
951 return -EBADMSG;
952
953 if (dst_max > 0 && (space - s.avail_out) >= dst_max)
954 break;
955 if (dst_max > 0 && space == dst_max)
956 return -ENOBUFS;
957
958 used = space - s.avail_out;
959 space = MIN3(2 * space, dst_max ?: SIZE_MAX, (size_t) UINT_MAX);
960 if (!greedy_realloc(dst, space, 1))
961 return -ENOMEM;
962
963 s.avail_out = space - used;
964 s.next_out = (char*) *dst + used;
965 }
966
967 *dst_size = space - s.avail_out;
968 return 0;
969 #else
970 return -EPROTONOSUPPORT;
971 #endif
972 }
973
974 int decompress_blob(
975 Compression compression,
976 const void *src,
977 uint64_t src_size,
978 void **dst,
979 size_t *dst_size,
980 size_t dst_max) {
981
982 switch (compression) {
983 case COMPRESSION_XZ:
984 return decompress_blob_xz(
985 src, src_size,
986 dst, dst_size, dst_max);
987 case COMPRESSION_LZ4:
988 return decompress_blob_lz4(
989 src, src_size,
990 dst, dst_size, dst_max);
991 case COMPRESSION_ZSTD:
992 return decompress_blob_zstd(
993 src, src_size,
994 dst, dst_size, dst_max);
995 case COMPRESSION_GZIP:
996 return decompress_blob_gzip(
997 src, src_size,
998 dst, dst_size, dst_max);
999 case COMPRESSION_BZIP2:
1000 return decompress_blob_bzip2(
1001 src, src_size,
1002 dst, dst_size, dst_max);
1003 default:
1004 return -EPROTONOSUPPORT;
1005 }
1006 }
1007
1008 int decompress_zlib_raw(
1009 const void *src,
1010 uint64_t src_size,
1011 void *dst,
1012 size_t dst_size,
1013 int wbits) {
1014
1015 #if HAVE_ZLIB
1016 int r;
1017
1018 r = dlopen_zlib(LOG_DEBUG);
1019 if (r < 0)
1020 return r;
1021
1022 if (src_size > UINT_MAX)
1023 return -EFBIG;
1024 if (dst_size > UINT_MAX)
1025 return -EFBIG;
1026
1027 _cleanup_(inflateEnd_wrapper) z_stream s = {
1028 .next_in = (void*) src,
1029 .avail_in = src_size,
1030 .next_out = dst,
1031 .avail_out = dst_size,
1032 };
1033
1034 r = sym_inflateInit2_(&s, /* windowBits= */ wbits, ZLIB_VERSION, (int) sizeof(s));
1035 if (r != Z_OK)
1036 return -EIO;
1037
1038 r = sym_inflate(&s, Z_FINISH);
1039 size_t produced = (uint8_t*) s.next_out - (uint8_t*) dst;
1040
1041 if (r != Z_STREAM_END || produced != dst_size)
1042 return -EBADMSG;
1043
1044 return 0;
1045 #else
1046 return -EPROTONOSUPPORT;
1047 #endif
1048 }
1049
1050 static int decompress_startswith_xz(
1051 const void *src,
1052 uint64_t src_size,
1053 void **buffer,
1054 const void *prefix,
1055 size_t prefix_len,
1056 uint8_t extra) {
1057
1058 /* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to
1059 * follow the prefix */
1060
1061 assert(src);
1062 assert(src_size > 0);
1063 assert(buffer);
1064 assert(prefix);
1065
1066 #if HAVE_XZ
1067 int r;
1068
1069 r = dlopen_xz(LOG_DEBUG);
1070 if (r < 0)
1071 return r;
1072
1073 _cleanup_(lzma_end_wrapper) lzma_stream s = LZMA_STREAM_INIT;
1074 lzma_ret ret = sym_lzma_stream_decoder(&s, UINT64_MAX, /* flags= */ 0);
1075 if (ret != LZMA_OK)
1076 return -EBADMSG;
1077
1078 if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
1079 return -ENOMEM;
1080
1081 size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);
1082
1083 s.next_in = src;
1084 s.avail_in = src_size;
1085
1086 s.next_out = *buffer;
1087 s.avail_out = allocated;
1088
1089 for (;;) {
1090 ret = sym_lzma_code(&s, LZMA_FINISH);
1091
1092 if (!IN_SET(ret, LZMA_OK, LZMA_STREAM_END))
1093 return -EBADMSG;
1094
1095 if (allocated - s.avail_out >= prefix_len + 1)
1096 return memcmp(*buffer, prefix, prefix_len) == 0 &&
1097 ((const uint8_t*) *buffer)[prefix_len] == extra;
1098
1099 if (ret == LZMA_STREAM_END)
1100 return 0;
1101
1102 s.avail_out += allocated;
1103
1104 if (!(greedy_realloc(buffer, allocated * 2, 1)))
1105 return -ENOMEM;
1106
1107 allocated = MALLOC_SIZEOF_SAFE(*buffer);
1108 s.next_out = *(uint8_t**)buffer + allocated - s.avail_out;
1109 }
1110
1111 #else
1112 return -EPROTONOSUPPORT;
1113 #endif
1114 }
1115
1116 static int decompress_startswith_lz4(
1117 const void *src,
1118 uint64_t src_size,
1119 void **buffer,
1120 const void *prefix,
1121 size_t prefix_len,
1122 uint8_t extra) {
1123
1124 /* Checks whether the decompressed blob starts with the mentioned prefix. The byte extra needs to
1125 * follow the prefix */
1126
1127 assert(src);
1128 assert(src_size > 0);
1129 assert(buffer);
1130 assert(prefix);
1131
1132 #if HAVE_LZ4
1133 size_t allocated;
1134 int r;
1135
1136 r = dlopen_lz4(LOG_DEBUG);
1137 if (r < 0)
1138 return r;
1139
1140 if (src_size <= 8)
1141 return -EBADMSG;
1142
1143 if (src_size - 8 > INT_MAX)
1144 return -EFBIG;
1145
1146 if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
1147 return -ENOMEM;
1148 allocated = MALLOC_SIZEOF_SAFE(*buffer);
1149
1150 r = sym_LZ4_decompress_safe_partial(
1151 (char*)src + 8,
1152 *buffer,
1153 src_size - 8,
1154 prefix_len + 1,
1155 allocated);
1156
1157 /* One lz4 < 1.8.3, we might get "failure" (r < 0), or "success" where just a part of the buffer is
1158 * decompressed. But if we get a smaller amount of bytes than requested, we don't know whether there
1159 * isn't enough data to fill the requested size or whether we just got a partial answer.
1160 */
1161 if (r < 0 || (size_t) r < prefix_len + 1) {
1162 size_t size;
1163
1164 if (sym_LZ4_versionNumber() >= 10803)
1165 /* We trust that the newer lz4 decompresses the number of bytes we
1166 * requested if available in the compressed string. */
1167 return 0;
1168
1169 if (r > 0)
1170 /* Compare what we have first, in case of mismatch we can
1171 * shortcut the full comparison. */
1172 if (memcmp(*buffer, prefix, r) != 0)
1173 return 0;
1174
1175 /* Before version 1.8.3, lz4 always tries to decode full a "sequence",
1176 * so in pathological cases might need to decompress the full field. */
1177 r = decompress_blob_lz4(src, src_size, buffer, &size, 0);
1178 if (r < 0)
1179 return r;
1180
1181 if (size < prefix_len + 1)
1182 return 0;
1183 }
1184
1185 return memcmp(*buffer, prefix, prefix_len) == 0 &&
1186 ((const uint8_t*) *buffer)[prefix_len] == extra;
1187 #else
1188 return -EPROTONOSUPPORT;
1189 #endif
1190 }
1191
1192 static int decompress_startswith_zstd(
1193 const void *src,
1194 uint64_t src_size,
1195 void **buffer,
1196 const void *prefix,
1197 size_t prefix_len,
1198 uint8_t extra) {
1199
1200 assert(src);
1201 assert(src_size > 0);
1202 assert(buffer);
1203 assert(prefix);
1204
1205 #if HAVE_ZSTD
1206 int r;
1207
1208 r = dlopen_zstd(LOG_DEBUG);
1209 if (r < 0)
1210 return r;
1211
1212 uint64_t size = sym_ZSTD_getFrameContentSize(src, src_size);
1213 if (IN_SET(size, ZSTD_CONTENTSIZE_ERROR, ZSTD_CONTENTSIZE_UNKNOWN))
1214 return -EBADMSG;
1215
1216 if (size < prefix_len + 1)
1217 return 0; /* Decompressed text too short to match the prefix and extra */
1218
1219 _cleanup_(ZSTD_freeDCtxp) ZSTD_DCtx *dctx = sym_ZSTD_createDCtx();
1220 if (!dctx)
1221 return -ENOMEM;
1222
1223 if (!(greedy_realloc(buffer, MAX(sym_ZSTD_DStreamOutSize(), prefix_len + 1), 1)))
1224 return -ENOMEM;
1225
1226 ZSTD_inBuffer input = {
1227 .src = src,
1228 .size = src_size,
1229 };
1230 ZSTD_outBuffer output = {
1231 .dst = *buffer,
1232 .size = MALLOC_SIZEOF_SAFE(*buffer),
1233 };
1234 size_t k;
1235
1236 k = sym_ZSTD_decompressStream(dctx, &output, &input);
1237 if (sym_ZSTD_isError(k))
1238 return log_debug_errno(zstd_ret_to_errno(k), "ZSTD decoder failed: %s", sym_ZSTD_getErrorName(k));
1239 if (output.pos < prefix_len + 1)
1240 return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "ZSTD decoded less data than indicated, probably corrupted stream.");
1241
1242 return memcmp(*buffer, prefix, prefix_len) == 0 &&
1243 ((const uint8_t*) *buffer)[prefix_len] == extra;
1244 #else
1245 return -EPROTONOSUPPORT;
1246 #endif
1247 }
1248
1249 static int decompress_startswith_gzip(
1250 const void *src,
1251 uint64_t src_size,
1252 void **buffer,
1253 const void *prefix,
1254 size_t prefix_len,
1255 uint8_t extra) {
1256
1257 assert(src);
1258 assert(src_size > 0);
1259 assert(buffer);
1260 assert(prefix);
1261
1262 #if HAVE_ZLIB
1263 int r;
1264
1265 r = dlopen_zlib(LOG_DEBUG);
1266 if (r < 0)
1267 return r;
1268
1269 if (src_size > UINT_MAX)
1270 return -EFBIG;
1271
1272 _cleanup_(inflateEnd_wrapper) z_stream s = {};
1273
1274 r = sym_inflateInit2_(&s, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(s));
1275 if (r != Z_OK)
1276 return -EBADMSG;
1277
1278 if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
1279 return -ENOMEM;
1280
1281 size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);
1282
1283 s.next_in = (void*) src;
1284 s.avail_in = src_size;
1285
1286 s.next_out = *buffer;
1287 s.avail_out = MIN(allocated, (size_t) UINT_MAX);
1288
1289 for (;;) {
1290 r = sym_inflate(&s, Z_FINISH);
1291
1292 if (!IN_SET(r, Z_OK, Z_STREAM_END, Z_BUF_ERROR))
1293 return -EBADMSG;
1294
1295 if (allocated - s.avail_out >= prefix_len + 1)
1296 return memcmp(*buffer, prefix, prefix_len) == 0 &&
1297 ((const uint8_t*) *buffer)[prefix_len] == extra;
1298
1299 if (r == Z_STREAM_END)
1300 return 0;
1301
1302 size_t used = allocated - s.avail_out;
1303
1304 if (!(greedy_realloc(buffer, allocated * 2, 1)))
1305 return -ENOMEM;
1306
1307 allocated = MALLOC_SIZEOF_SAFE(*buffer);
1308 s.avail_out = MIN(allocated - used, (size_t) UINT_MAX);
1309 s.next_out = *(uint8_t**)buffer + used;
1310 }
1311 #else
1312 return -EPROTONOSUPPORT;
1313 #endif
1314 }
1315
1316 static int decompress_startswith_bzip2(
1317 const void *src,
1318 uint64_t src_size,
1319 void **buffer,
1320 const void *prefix,
1321 size_t prefix_len,
1322 uint8_t extra) {
1323
1324 assert(src);
1325 assert(src_size > 0);
1326 assert(buffer);
1327 assert(prefix);
1328
1329 #if HAVE_BZIP2
1330 int r;
1331
1332 r = dlopen_bzip2(LOG_DEBUG);
1333 if (r < 0)
1334 return r;
1335
1336 if (src_size > UINT_MAX)
1337 return -EFBIG;
1338
1339 _cleanup_(BZ2_bzDecompressEnd_wrapper) bz_stream s = {};
1340
1341 r = sym_BZ2_bzDecompressInit(&s, /* verbosity= */ 0, /* small= */ 0);
1342 if (r != BZ_OK)
1343 return -EBADMSG;
1344
1345 if (!(greedy_realloc(buffer, ALIGN_8(prefix_len + 1), 1)))
1346 return -ENOMEM;
1347
1348 size_t allocated = MALLOC_SIZEOF_SAFE(*buffer);
1349
1350 s.next_in = (char*) src;
1351 s.avail_in = src_size;
1352
1353 s.next_out = *buffer;
1354 s.avail_out = MIN(allocated, (size_t) UINT_MAX);
1355
1356 for (;;) {
1357 r = sym_BZ2_bzDecompress(&s);
1358
1359 if (!IN_SET(r, BZ_OK, BZ_STREAM_END))
1360 return -EBADMSG;
1361
1362 if (allocated - s.avail_out >= prefix_len + 1)
1363 return memcmp(*buffer, prefix, prefix_len) == 0 &&
1364 ((const uint8_t*) *buffer)[prefix_len] == extra;
1365
1366 if (r == BZ_STREAM_END)
1367 return 0;
1368
1369 size_t used = allocated - s.avail_out;
1370
1371 if (!(greedy_realloc(buffer, allocated * 2, 1)))
1372 return -ENOMEM;
1373
1374 allocated = MALLOC_SIZEOF_SAFE(*buffer);
1375 s.avail_out = MIN(allocated - used, (size_t) UINT_MAX);
1376 s.next_out = (char*) *buffer + used;
1377 }
1378 #else
1379 return -EPROTONOSUPPORT;
1380 #endif
1381 }
1382
1383 int decompress_startswith(
1384 Compression compression,
1385 const void *src, uint64_t src_size,
1386 void **buffer,
1387 const void *prefix, size_t prefix_len,
1388 uint8_t extra) {
1389
1390 switch (compression) {
1391 case COMPRESSION_XZ:
1392 return decompress_startswith_xz(src, src_size, buffer, prefix, prefix_len, extra);
1393 case COMPRESSION_LZ4:
1394 return decompress_startswith_lz4(src, src_size, buffer, prefix, prefix_len, extra);
1395 case COMPRESSION_ZSTD:
1396 return decompress_startswith_zstd(src, src_size, buffer, prefix, prefix_len, extra);
1397 case COMPRESSION_GZIP:
1398 return decompress_startswith_gzip(src, src_size, buffer, prefix, prefix_len, extra);
1399 case COMPRESSION_BZIP2:
1400 return decompress_startswith_bzip2(src, src_size, buffer, prefix, prefix_len, extra);
1401 default:
1402 return -EOPNOTSUPP;
1403 }
1404 }
1405
1406 int compress_stream(
1407 Compression type,
1408 int fdf, int fdt,
1409 uint64_t max_bytes,
1410 uint64_t *ret_uncompressed_size) {
1411
1412 _cleanup_(compressor_freep) Compressor *c = NULL;
1413 _cleanup_free_ void *buf = NULL;
1414 _cleanup_free_ uint8_t *input = NULL;
1415 size_t buf_size = 0, buf_alloc = 0;
1416 uint64_t total_in = 0, total_out = 0;
1417 int r;
1418
1419 assert(fdf >= 0);
1420 assert(fdt >= 0);
1421
1422 r = compressor_new(&c, type);
1423 if (r < 0)
1424 return r;
1425
1426 input = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
1427 if (!input)
1428 return -ENOMEM;
1429
1430 for (;;) {
1431 size_t m = COMPRESS_PIPE_BUFFER_SIZE;
1432 ssize_t n;
1433
1434 if (max_bytes != UINT64_MAX && (uint64_t) m > max_bytes)
1435 m = (size_t) max_bytes;
1436
1437 n = read(fdf, input, m);
1438 if (n < 0)
1439 return -errno;
1440
1441 if (n == 0) {
1442 r = compressor_finish(c, &buf, &buf_size, &buf_alloc);
1443 if (r < 0)
1444 return r;
1445
1446 if (buf_size > 0) {
1447 r = loop_write(fdt, buf, buf_size);
1448 if (r < 0)
1449 return r;
1450 total_out += buf_size;
1451 }
1452 break;
1453 }
1454
1455 total_in += n;
1456 if (max_bytes != UINT64_MAX) {
1457 assert(max_bytes >= (uint64_t) n);
1458 max_bytes -= n;
1459 }
1460
1461 r = compressor_start(c, input, n, &buf, &buf_size, &buf_alloc);
1462 if (r < 0)
1463 return r;
1464
1465 if (buf_size > 0) {
1466 r = loop_write(fdt, buf, buf_size);
1467 if (r < 0)
1468 return r;
1469 total_out += buf_size;
1470 }
1471 }
1472
1473 if (ret_uncompressed_size)
1474 *ret_uncompressed_size = total_in;
1475
1476 if (total_in == 0)
1477 log_debug("%s compression finished (no input data)", compression_to_string(type));
1478 else
1479 log_debug("%s compression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
1480 compression_to_string(type), total_in, total_out, (double) total_out / total_in * 100);
1481
1482 return 0;
1483 }
1484
1485 /* Determine whether sparse writes should be used for this fd. Sparse writes are only safe on
1486 * regular files without O_APPEND (O_APPEND ignores lseek position, which would collapse holes). */
1487 static int should_sparse(int fd) {
1488 struct stat st;
1489
1490 assert(fd >= 0);
1491
1492 if (fstat(fd, &st) < 0)
1493 return -errno;
1494
1495 int flags = fcntl(fd, F_GETFL);
1496 if (flags < 0)
1497 return -errno;
1498
1499 return S_ISREG(st.st_mode) && !FLAGS_SET(flags, O_APPEND);
1500 }
1501
1502 /* After sparse decompression, set the file size to the current position to account for
1503 * trailing holes that sparse_write() created via lseek but never extended the file size for. */
1504 static int finalize_sparse(int fd) {
1505 off_t pos;
1506
1507 assert(fd >= 0);
1508
1509 pos = lseek(fd, 0, SEEK_CUR);
1510 if (pos < 0)
1511 return -errno;
1512
1513 if (ftruncate(fd, pos) < 0)
1514 return -errno;
1515
1516 return 0;
1517 }
1518
1519 /* Common helper for decompress_stream_*() wrappers */
1520
1521 struct decompress_stream_userdata {
1522 int fd;
1523 uint64_t max_bytes;
1524 uint64_t total_out;
1525 bool sparse;
1526 };
1527
1528 static int decompress_stream_write_callback(const void *data, size_t size, void *userdata) {
1529 struct decompress_stream_userdata *u = ASSERT_PTR(userdata);
1530
1531 if (u->max_bytes != UINT64_MAX) {
1532 if (u->max_bytes < size)
1533 return -EFBIG;
1534 u->max_bytes -= size;
1535 }
1536
1537 u->total_out += size;
1538
1539 if (u->sparse) {
1540 /* Note: sparse_write() does not retry on EINTR and converts short writes to -EIO.
1541 * This is fine here since sparse mode is only used on regular files, where short
1542 * writes and EINTR are not expected in practice. */
1543 ssize_t k = sparse_write(u->fd, data, size, 64);
1544 if (k < 0)
1545 return (int) k;
1546 return 0;
1547 }
1548
1549 return loop_write(u->fd, data, size);
1550 }
1551
1552 static int decompressor_new(Decompressor **ret, Compression type) {
1553 #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
1554 int r;
1555 #endif
1556
1557 assert(ret);
1558
1559 _cleanup_(compressor_freep) Decompressor *c = new0(Decompressor, 1);
1560 if (!c)
1561 return -ENOMEM;
1562
1563 c->type = _COMPRESSION_INVALID;
1564
1565 switch (type) {
1566
1567 #if HAVE_XZ
1568 case COMPRESSION_XZ:
1569 r = dlopen_xz(LOG_DEBUG);
1570 if (r < 0)
1571 return r;
1572
1573 if (sym_lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED) != LZMA_OK)
1574 return -EIO;
1575 break;
1576 #endif
1577
1578 #if HAVE_LZ4
1579 case COMPRESSION_LZ4: {
1580 r = dlopen_lz4(LOG_DEBUG);
1581 if (r < 0)
1582 return r;
1583
1584 size_t rc = sym_LZ4F_createDecompressionContext(&c->d_lz4, LZ4F_VERSION);
1585 if (sym_LZ4F_isError(rc))
1586 return -ENOMEM;
1587
1588 break;
1589 }
1590 #endif
1591
1592 #if HAVE_ZSTD
1593 case COMPRESSION_ZSTD:
1594 r = dlopen_zstd(LOG_DEBUG);
1595 if (r < 0)
1596 return r;
1597
1598 c->d_zstd = sym_ZSTD_createDCtx();
1599 if (!c->d_zstd)
1600 return -ENOMEM;
1601 break;
1602 #endif
1603
1604 #if HAVE_ZLIB
1605 case COMPRESSION_GZIP:
1606 r = dlopen_zlib(LOG_DEBUG);
1607 if (r < 0)
1608 return r;
1609
1610 r = sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip));
1611 if (r != Z_OK)
1612 return -EIO;
1613 break;
1614 #endif
1615
1616 #if HAVE_BZIP2
1617 case COMPRESSION_BZIP2:
1618 r = dlopen_bzip2(LOG_DEBUG);
1619 if (r < 0)
1620 return r;
1621
1622 r = sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0);
1623 if (r != BZ_OK)
1624 return -EIO;
1625 break;
1626 #endif
1627
1628 default:
1629 return -EOPNOTSUPP;
1630 }
1631
1632 c->type = type;
1633 c->encoding = false;
1634 *ret = TAKE_PTR(c);
1635 return 0;
1636 }
1637
1638 int decompress_stream(
1639 Compression type,
1640 int fdf, int fdt,
1641 uint64_t max_bytes) {
1642
1643 _cleanup_(compressor_freep) Decompressor *c = NULL;
1644 _cleanup_free_ uint8_t *buf = NULL;
1645 uint64_t total_in = 0;
1646 int r;
1647
1648 assert(fdf >= 0);
1649 assert(fdt >= 0);
1650
1651 r = decompressor_new(&c, type);
1652 if (r < 0)
1653 return r;
1654
1655 struct decompress_stream_userdata userdata = {
1656 .fd = fdt,
1657 .max_bytes = max_bytes,
1658 .sparse = should_sparse(fdt) > 0,
1659 };
1660
1661 buf = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
1662 if (!buf)
1663 return -ENOMEM;
1664
1665 for (;;) {
1666 ssize_t n;
1667
1668 n = read(fdf, buf, COMPRESS_PIPE_BUFFER_SIZE);
1669 if (n < 0)
1670 return -errno;
1671 if (n == 0)
1672 break;
1673
1674 total_in += n;
1675
1676 r = decompressor_push(c, buf, n, decompress_stream_write_callback, &userdata);
1677 if (r < 0)
1678 return r;
1679 }
1680
1681 if (total_in == 0)
1682 return log_debug_errno(SYNTHETIC_ERRNO(EBADMSG), "%s decompression failed: no data read",
1683 compression_to_string(type));
1684
1685 if (userdata.sparse) {
1686 r = finalize_sparse(fdt);
1687 if (r < 0)
1688 return r;
1689 }
1690
1691 log_debug("%s decompression finished (%" PRIu64 " -> %" PRIu64 " bytes, %.1f%%)",
1692 compression_to_string(type), total_in, userdata.total_out,
1693 (double) userdata.total_out / total_in * 100);
1694
1695 return 0;
1696 }
1697
1698 int decompress_stream_by_filename(const char *filename, int fdf, int fdt, uint64_t max_bytes) {
1699 Compression c = compression_from_filename(filename);
1700 if (c == COMPRESSION_NONE)
1701 return -EPROTONOSUPPORT;
1702
1703 return decompress_stream(c, fdf, fdt, max_bytes);
1704 }
1705
1706 /* Push-based streaming compression/decompression context API */
1707
1708 Compressor* compressor_free(Compressor *c) {
1709 if (!c)
1710 return NULL;
1711
1712 switch (c->type) {
1713
1714 #if HAVE_XZ
1715 case COMPRESSION_XZ:
1716 sym_lzma_end(&c->xz);
1717 break;
1718 #endif
1719
1720 #if HAVE_LZ4
1721 case COMPRESSION_LZ4:
1722 if (c->encoding) {
1723 sym_LZ4F_freeCompressionContext(c->c_lz4);
1724 c->c_lz4 = NULL;
1725 c->lz4_header = mfree(c->lz4_header);
1726 } else {
1727 sym_LZ4F_freeDecompressionContext(c->d_lz4);
1728 c->d_lz4 = NULL;
1729 }
1730 break;
1731 #endif
1732
1733 #if HAVE_ZSTD
1734 case COMPRESSION_ZSTD:
1735 if (c->encoding) {
1736 sym_ZSTD_freeCCtx(c->c_zstd);
1737 c->c_zstd = NULL;
1738 } else {
1739 sym_ZSTD_freeDCtx(c->d_zstd);
1740 c->d_zstd = NULL;
1741 }
1742 break;
1743 #endif
1744
1745 #if HAVE_ZLIB
1746 case COMPRESSION_GZIP:
1747 if (c->encoding)
1748 sym_deflateEnd(&c->gzip);
1749 else
1750 sym_inflateEnd(&c->gzip);
1751 break;
1752 #endif
1753
1754 #if HAVE_BZIP2
1755 case COMPRESSION_BZIP2:
1756 if (c->encoding)
1757 sym_BZ2_bzCompressEnd(&c->bzip2);
1758 else
1759 sym_BZ2_bzDecompressEnd(&c->bzip2);
1760 break;
1761 #endif
1762
1763 default:
1764 break;
1765 }
1766
1767 return mfree(c);
1768 }
1769
1770 Compression compressor_type(const Compressor *c) {
1771 return c ? c->type : _COMPRESSION_INVALID;
1772 }
1773
1774 int decompressor_detect(Decompressor **ret, const void *data, size_t size) {
1775 #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
1776 int r;
1777 #endif
1778
1779 assert(ret);
1780
1781 if (*ret)
1782 return 1;
1783
1784 if (size < COMPRESSION_MAGIC_BYTES_MAX)
1785 return 0;
1786
1787 assert(data);
1788
1789 Compression type = compression_detect_from_magic(data);
1790
1791 _cleanup_(compressor_freep) Decompressor *c = new0(Decompressor, 1);
1792 if (!c)
1793 return -ENOMEM;
1794
1795 switch (type) {
1796
1797 #if HAVE_XZ
1798 case COMPRESSION_XZ: {
1799 r = dlopen_xz(LOG_DEBUG);
1800 if (r < 0)
1801 return r;
1802
1803 lzma_ret xzr = sym_lzma_stream_decoder(&c->xz, UINT64_MAX, LZMA_TELL_UNSUPPORTED_CHECK | LZMA_CONCATENATED);
1804 if (xzr != LZMA_OK)
1805 return -EIO;
1806
1807 break;
1808 }
1809 #endif
1810
1811 #if HAVE_LZ4
1812 case COMPRESSION_LZ4: {
1813 r = dlopen_lz4(LOG_DEBUG);
1814 if (r < 0)
1815 return r;
1816
1817 size_t rc = sym_LZ4F_createDecompressionContext(&c->d_lz4, LZ4F_VERSION);
1818 if (sym_LZ4F_isError(rc))
1819 return -ENOMEM;
1820
1821 break;
1822 }
1823 #endif
1824
1825 #if HAVE_ZSTD
1826 case COMPRESSION_ZSTD: {
1827 r = dlopen_zstd(LOG_DEBUG);
1828 if (r < 0)
1829 return r;
1830
1831 c->d_zstd = sym_ZSTD_createDCtx();
1832 if (!c->d_zstd)
1833 return -ENOMEM;
1834
1835 break;
1836 }
1837 #endif
1838
1839 #if HAVE_ZLIB
1840 case COMPRESSION_GZIP: {
1841 r = dlopen_zlib(LOG_DEBUG);
1842 if (r < 0)
1843 return r;
1844
1845 r = sym_inflateInit2_(&c->gzip, /* windowBits= */ ZLIB_WBITS_GZIP, ZLIB_VERSION, (int) sizeof(c->gzip));
1846 if (r != Z_OK)
1847 return -EIO;
1848
1849 break;
1850 }
1851 #endif
1852
1853 #if HAVE_BZIP2
1854 case COMPRESSION_BZIP2: {
1855 r = dlopen_bzip2(LOG_DEBUG);
1856 if (r < 0)
1857 return r;
1858
1859 r = sym_BZ2_bzDecompressInit(&c->bzip2, /* verbosity= */ 0, /* small= */ 0);
1860 if (r != BZ_OK)
1861 return -EIO;
1862
1863 break;
1864 }
1865 #endif
1866
1867 default:
1868 if (type != _COMPRESSION_INVALID)
1869 return log_debug_errno(SYNTHETIC_ERRNO(EOPNOTSUPP),
1870 "Detected %s compression, but support is not compiled in.",
1871 compression_to_string(type));
1872 type = COMPRESSION_NONE;
1873 break;
1874 }
1875
1876 c->type = type;
1877 c->encoding = false;
1878
1879 log_debug("Detected compression type: %s", compression_to_string(c->type));
1880 *ret = TAKE_PTR(c);
1881 return 1;
1882 }
1883
1884 int decompressor_force_off(Decompressor **ret) {
1885 assert(ret);
1886
1887 *ret = compressor_free(*ret);
1888
1889 Decompressor *c = new0(Decompressor, 1);
1890 if (!c)
1891 return -ENOMEM;
1892
1893 c->type = COMPRESSION_NONE;
1894 c->encoding = false;
1895 *ret = c;
1896 return 0;
1897 }
1898
1899 int decompressor_push(Decompressor *c, const void *data, size_t size, DecompressorCallback callback, void *userdata) {
1900 #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
1901 _cleanup_free_ uint8_t *buffer = NULL;
1902 #endif
1903 int r;
1904
1905 assert(c);
1906 assert(callback);
1907
1908 if (c->encoding)
1909 return -EINVAL;
1910
1911 if (size == 0)
1912 return 1;
1913
1914 assert(data);
1915
1916 #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
1917 if (c->type != COMPRESSION_NONE) {
1918 buffer = new(uint8_t, COMPRESS_PIPE_BUFFER_SIZE);
1919 if (!buffer)
1920 return -ENOMEM;
1921 }
1922 #endif
1923
1924 switch (c->type) {
1925
1926 case COMPRESSION_NONE:
1927 r = callback(data, size, userdata);
1928 if (r < 0)
1929 return r;
1930
1931 break;
1932
1933 #if HAVE_XZ
1934 case COMPRESSION_XZ:
1935 c->xz.next_in = data;
1936 c->xz.avail_in = size;
1937
1938 while (c->xz.avail_in > 0) {
1939 c->xz.next_out = buffer;
1940 c->xz.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
1941
1942 lzma_ret lzr = sym_lzma_code(&c->xz, LZMA_RUN);
1943 if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
1944 return -EBADMSG;
1945
1946 if (c->xz.avail_out < COMPRESS_PIPE_BUFFER_SIZE) {
1947 r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->xz.avail_out, userdata);
1948 if (r < 0)
1949 return r;
1950 }
1951 }
1952
1953 break;
1954 #endif
1955
1956 #if HAVE_LZ4
1957 case COMPRESSION_LZ4: {
1958 const uint8_t *src = data;
1959 size_t src_remaining = size;
1960
1961 while (src_remaining > 0) {
1962 size_t produced = COMPRESS_PIPE_BUFFER_SIZE;
1963 size_t consumed = src_remaining;
1964
1965 size_t rc = sym_LZ4F_decompress(c->d_lz4, buffer, &produced, src, &consumed, NULL);
1966 if (sym_LZ4F_isError(rc))
1967 return -EBADMSG;
1968
1969 if (consumed == 0 && produced == 0)
1970 break; /* No progress possible with current input */
1971
1972 src += consumed;
1973 src_remaining -= consumed;
1974
1975 if (produced > 0) {
1976 r = callback(buffer, produced, userdata);
1977 if (r < 0)
1978 return r;
1979 }
1980 }
1981
1982 break;
1983 }
1984 #endif
1985
1986 #if HAVE_ZSTD
1987 case COMPRESSION_ZSTD: {
1988 ZSTD_inBuffer input = {
1989 .src = (void*) data,
1990 .size = size,
1991 };
1992
1993 while (input.pos < input.size) {
1994 ZSTD_outBuffer output = {
1995 .dst = buffer,
1996 .size = COMPRESS_PIPE_BUFFER_SIZE,
1997 };
1998
1999 size_t res = sym_ZSTD_decompressStream(c->d_zstd, &output, &input);
2000 if (sym_ZSTD_isError(res))
2001 return -EBADMSG;
2002
2003 if (output.pos > 0) {
2004 r = callback(output.dst, output.pos, userdata);
2005 if (r < 0)
2006 return r;
2007 }
2008 }
2009
2010 break;
2011 }
2012 #endif
2013
2014 #if HAVE_ZLIB
2015 case COMPRESSION_GZIP:
2016 if (size > UINT_MAX)
2017 return -EFBIG;
2018
2019 c->gzip.next_in = (void*) data;
2020 c->gzip.avail_in = size;
2021
2022 while (c->gzip.avail_in > 0) {
2023 c->gzip.next_out = buffer;
2024 c->gzip.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
2025
2026 int zr = sym_inflate(&c->gzip, Z_NO_FLUSH);
2027 if (!IN_SET(zr, Z_OK, Z_STREAM_END))
2028 return -EBADMSG;
2029
2030 if (c->gzip.avail_out < COMPRESS_PIPE_BUFFER_SIZE) {
2031 r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->gzip.avail_out, userdata);
2032 if (r < 0)
2033 return r;
2034 }
2035
2036 if (zr == Z_STREAM_END)
2037 break;
2038 }
2039
2040 break;
2041 #endif
2042
2043 #if HAVE_BZIP2
2044 case COMPRESSION_BZIP2:
2045 if (size > UINT_MAX)
2046 return -EFBIG;
2047
2048 c->bzip2.next_in = (char*) data;
2049 c->bzip2.avail_in = size;
2050
2051 while (c->bzip2.avail_in > 0) {
2052 c->bzip2.next_out = (char*) buffer;
2053 c->bzip2.avail_out = COMPRESS_PIPE_BUFFER_SIZE;
2054
2055 int bzr = sym_BZ2_bzDecompress(&c->bzip2);
2056 if (!IN_SET(bzr, BZ_OK, BZ_STREAM_END))
2057 return -EBADMSG;
2058
2059 if (c->bzip2.avail_out < COMPRESS_PIPE_BUFFER_SIZE) {
2060 r = callback(buffer, COMPRESS_PIPE_BUFFER_SIZE - c->bzip2.avail_out, userdata);
2061 if (r < 0)
2062 return r;
2063 }
2064
2065 if (bzr == BZ_STREAM_END)
2066 break;
2067 }
2068
2069 break;
2070 #endif
2071
2072 default:
2073 assert_not_reached();
2074 }
2075
2076 return 1;
2077 }
2078
2079 int compressor_new(Compressor **ret, Compression type) {
2080 #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
2081 int r;
2082 #endif
2083
2084 assert(ret);
2085
2086 _cleanup_(compressor_freep) Compressor *c = new0(Compressor, 1);
2087 if (!c)
2088 return -ENOMEM;
2089
2090 c->type = _COMPRESSION_INVALID;
2091 /* Set encoding early so that compressor_freep calls the correct cleanup (compression vs
2092 * decompression) if any operation in the switch fails after setting c->type. This is safe
2093 * because _COMPRESSION_INVALID hits the default: break case regardless of the encoding flag. */
2094 c->encoding = true;
2095
2096 switch (type) {
2097
2098 #if HAVE_XZ
2099 case COMPRESSION_XZ: {
2100 r = dlopen_xz(LOG_DEBUG);
2101 if (r < 0)
2102 return r;
2103
2104 lzma_ret xzr = sym_lzma_easy_encoder(&c->xz, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC64);
2105 if (xzr != LZMA_OK)
2106 return -EIO;
2107
2108 c->type = COMPRESSION_XZ;
2109 break;
2110 }
2111 #endif
2112
2113 #if HAVE_LZ4
2114 case COMPRESSION_LZ4: {
2115 r = dlopen_lz4(LOG_DEBUG);
2116 if (r < 0)
2117 return r;
2118
2119 size_t rc = sym_LZ4F_createCompressionContext(&c->c_lz4, LZ4F_VERSION);
2120 if (sym_LZ4F_isError(rc))
2121 return -ENOMEM;
2122
2123 c->type = COMPRESSION_LZ4;
2124
2125 /* Generate the frame header and stash it for the first compressor_start call */
2126 size_t header_bound = sym_LZ4F_compressBound(0, &lz4_preferences);
2127 c->lz4_header = malloc(header_bound);
2128 if (!c->lz4_header)
2129 return -ENOMEM;
2130
2131 c->lz4_header_size = sym_LZ4F_compressBegin(c->c_lz4, c->lz4_header, header_bound, &lz4_preferences);
2132 if (sym_LZ4F_isError(c->lz4_header_size))
2133 return -EINVAL;
2134
2135 break;
2136 }
2137 #endif
2138
2139 #if HAVE_ZSTD
2140 case COMPRESSION_ZSTD:
2141 r = dlopen_zstd(LOG_DEBUG);
2142 if (r < 0)
2143 return r;
2144
2145 c->c_zstd = sym_ZSTD_createCCtx();
2146 if (!c->c_zstd)
2147 return -ENOMEM;
2148
2149 c->type = COMPRESSION_ZSTD;
2150
2151 size_t z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_compressionLevel, ZSTD_CLEVEL_DEFAULT);
2152 if (sym_ZSTD_isError(z))
2153 return -EIO;
2154
2155 z = sym_ZSTD_CCtx_setParameter(c->c_zstd, ZSTD_c_checksumFlag, /* enable= */ 1);
2156 if (sym_ZSTD_isError(z))
2157 log_debug("Failed to enable ZSTD checksum, ignoring: %s", sym_ZSTD_getErrorName(z));
2158
2159 break;
2160 #endif
2161
2162 #if HAVE_ZLIB
2163 case COMPRESSION_GZIP:
2164 r = dlopen_zlib(LOG_DEBUG);
2165 if (r < 0)
2166 return r;
2167
2168 r = sym_deflateInit2_(&c->gzip,
2169 Z_DEFAULT_COMPRESSION,
2170 /* method= */ Z_DEFLATED,
2171 /* windowBits= */ ZLIB_WBITS_GZIP,
2172 /* memLevel= */ 8,
2173 /* strategy= */ Z_DEFAULT_STRATEGY,
2174 ZLIB_VERSION, (int) sizeof(c->gzip));
2175 if (r != Z_OK)
2176 return -EIO;
2177
2178 c->type = COMPRESSION_GZIP;
2179 break;
2180 #endif
2181
2182 #if HAVE_BZIP2
2183 case COMPRESSION_BZIP2:
2184 r = dlopen_bzip2(LOG_DEBUG);
2185 if (r < 0)
2186 return r;
2187
2188 r = sym_BZ2_bzCompressInit(&c->bzip2, /* blockSize100k= */ 9, /* verbosity= */ 0, /* workFactor= */ 0);
2189 if (r != BZ_OK)
2190 return -EIO;
2191
2192 c->type = COMPRESSION_BZIP2;
2193 break;
2194 #endif
2195
2196 case COMPRESSION_NONE:
2197 c->type = COMPRESSION_NONE;
2198 break;
2199
2200 default:
2201 return -EOPNOTSUPP;
2202 }
2203
2204 *ret = TAKE_PTR(c);
2205 return 0;
2206 }
2207
2208 #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
2209 static int enlarge_buffer(void **buffer, size_t *buffer_size, size_t *buffer_allocated, size_t need) {
2210 assert(buffer);
2211 assert(buffer_size);
2212 assert(buffer_allocated);
2213
2214 need = MAX3(need, *buffer_size + 1, (size_t) COMPRESS_PIPE_BUFFER_SIZE);
2215 if (*buffer_allocated >= need)
2216 return 0;
2217
2218 if (!greedy_realloc(buffer, need, 1))
2219 return -ENOMEM;
2220
2221 *buffer_allocated = MALLOC_SIZEOF_SAFE(*buffer);
2222 return 1;
2223 }
2224 #endif
2225
2226 int compressor_start(
2227 Compressor *c,
2228 const void *data,
2229 size_t size,
2230 void **buffer,
2231 size_t *buffer_size,
2232 size_t *buffer_allocated) {
2233
2234 #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
2235 int r;
2236 #endif
2237
2238 assert(c);
2239 assert(buffer);
2240 assert(buffer_size);
2241 assert(buffer_allocated);
2242
2243 if (!c->encoding)
2244 return -EINVAL;
2245
2246 if (size == 0)
2247 return 0;
2248
2249 assert(data);
2250
2251 *buffer_size = 0;
2252
2253 switch (c->type) {
2254
2255 #if HAVE_XZ
2256 case COMPRESSION_XZ:
2257
2258 c->xz.next_in = data;
2259 c->xz.avail_in = size;
2260
2261 while (c->xz.avail_in > 0) {
2262 lzma_ret lzr;
2263
2264 r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
2265 if (r < 0)
2266 return r;
2267
2268 c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
2269 c->xz.avail_out = *buffer_allocated - *buffer_size;
2270
2271 lzr = sym_lzma_code(&c->xz, LZMA_RUN);
2272 if (lzr != LZMA_OK)
2273 return -EIO;
2274
2275 *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
2276 }
2277
2278 break;
2279 #endif
2280
2281 #if HAVE_LZ4
2282 case COMPRESSION_LZ4: {
2283 /* Prepend any stashed frame header from compressor_new */
2284 if (c->lz4_header_size > 0) {
2285 r = enlarge_buffer(buffer, buffer_size, buffer_allocated, c->lz4_header_size);
2286 if (r < 0)
2287 return r;
2288
2289 memcpy(*buffer, c->lz4_header, c->lz4_header_size);
2290 *buffer_size = c->lz4_header_size;
2291 c->lz4_header = mfree(c->lz4_header);
2292 c->lz4_header_size = 0;
2293 }
2294
2295 size_t bound = sym_LZ4F_compressBound(size, &lz4_preferences);
2296 r = enlarge_buffer(buffer, buffer_size, buffer_allocated, *buffer_size + bound);
2297 if (r < 0)
2298 return r;
2299
2300 size_t n = sym_LZ4F_compressUpdate(c->c_lz4,
2301 (uint8_t*) *buffer + *buffer_size,
2302 *buffer_allocated - *buffer_size,
2303 data, size, NULL);
2304 if (sym_LZ4F_isError(n))
2305 return -EIO;
2306
2307 *buffer_size += n;
2308 break;
2309 }
2310 #endif
2311
2312 #if HAVE_ZSTD
2313 case COMPRESSION_ZSTD: {
2314 ZSTD_inBuffer input = {
2315 .src = data,
2316 .size = size,
2317 };
2318
2319 while (input.pos < input.size) {
2320 r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
2321 if (r < 0)
2322 return r;
2323
2324 ZSTD_outBuffer output = {
2325 .dst = ((uint8_t *) *buffer + *buffer_size),
2326 .size = *buffer_allocated - *buffer_size,
2327 };
2328
2329 size_t res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_continue);
2330 if (sym_ZSTD_isError(res))
2331 return -EIO;
2332
2333 *buffer_size += output.pos;
2334 }
2335
2336 break;
2337 }
2338 #endif
2339
2340 #if HAVE_ZLIB
2341 case COMPRESSION_GZIP:
2342 if (size > UINT_MAX)
2343 return -EFBIG;
2344
2345 c->gzip.next_in = (void*) data;
2346 c->gzip.avail_in = size;
2347
2348 while (c->gzip.avail_in > 0) {
2349 r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
2350 if (r < 0)
2351 return r;
2352
2353 size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
2354 c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
2355 c->gzip.avail_out = avail;
2356
2357 r = sym_deflate(&c->gzip, Z_NO_FLUSH);
2358 if (r != Z_OK)
2359 return -EIO;
2360
2361 *buffer_size += avail - c->gzip.avail_out;
2362 }
2363
2364 break;
2365 #endif
2366
2367 #if HAVE_BZIP2
2368 case COMPRESSION_BZIP2:
2369 if (size > UINT_MAX)
2370 return -EFBIG;
2371
2372 c->bzip2.next_in = (void*) data;
2373 c->bzip2.avail_in = size;
2374
2375 while (c->bzip2.avail_in > 0) {
2376 r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
2377 if (r < 0)
2378 return r;
2379
2380 size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
2381 c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
2382 c->bzip2.avail_out = avail;
2383
2384 r = sym_BZ2_bzCompress(&c->bzip2, BZ_RUN);
2385 if (r != BZ_RUN_OK)
2386 return -EIO;
2387
2388 *buffer_size += avail - c->bzip2.avail_out;
2389 }
2390
2391 break;
2392 #endif
2393
2394 case COMPRESSION_NONE:
2395
2396 if (*buffer_allocated < size) {
2397 void *p;
2398
2399 p = realloc(*buffer, size);
2400 if (!p)
2401 return -ENOMEM;
2402
2403 *buffer = p;
2404 *buffer_allocated = size;
2405 }
2406
2407 memcpy(*buffer, data, size);
2408 *buffer_size = size;
2409 break;
2410
2411 default:
2412 return -EOPNOTSUPP;
2413 }
2414
2415 return 0;
2416 }
2417
2418 int compressor_finish(Compressor *c, void **buffer, size_t *buffer_size, size_t *buffer_allocated) {
2419 #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD || HAVE_ZLIB || HAVE_BZIP2
2420 int r;
2421 #endif
2422
2423 assert(c);
2424 assert(buffer);
2425 assert(buffer_size);
2426 assert(buffer_allocated);
2427
2428 if (!c->encoding)
2429 return -EINVAL;
2430
2431 *buffer_size = 0;
2432
2433 switch (c->type) {
2434
2435 #if HAVE_XZ
2436 case COMPRESSION_XZ: {
2437 lzma_ret lzr;
2438
2439 c->xz.avail_in = 0;
2440
2441 do {
2442 r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
2443 if (r < 0)
2444 return r;
2445
2446 c->xz.next_out = (uint8_t*) *buffer + *buffer_size;
2447 c->xz.avail_out = *buffer_allocated - *buffer_size;
2448
2449 lzr = sym_lzma_code(&c->xz, LZMA_FINISH);
2450 if (!IN_SET(lzr, LZMA_OK, LZMA_STREAM_END))
2451 return -EIO;
2452
2453 *buffer_size += (*buffer_allocated - *buffer_size) - c->xz.avail_out;
2454 } while (lzr != LZMA_STREAM_END);
2455
2456 break;
2457 }
2458 #endif
2459
2460 #if HAVE_LZ4
2461 case COMPRESSION_LZ4: {
2462 size_t bound = sym_LZ4F_compressBound(0, &lz4_preferences);
2463 r = enlarge_buffer(buffer, buffer_size, buffer_allocated, bound);
2464 if (r < 0)
2465 return r;
2466
2467 size_t n = sym_LZ4F_compressEnd(c->c_lz4, *buffer, *buffer_allocated, NULL);
2468 if (sym_LZ4F_isError(n))
2469 return -EIO;
2470
2471 *buffer_size = n;
2472 break;
2473 }
2474 #endif
2475
2476 #if HAVE_ZSTD
2477 case COMPRESSION_ZSTD: {
2478 ZSTD_inBuffer input = {};
2479 size_t res;
2480
2481 do {
2482 r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
2483 if (r < 0)
2484 return r;
2485
2486 ZSTD_outBuffer output = {
2487 .dst = ((uint8_t *) *buffer + *buffer_size),
2488 .size = *buffer_allocated - *buffer_size,
2489 };
2490
2491 res = sym_ZSTD_compressStream2(c->c_zstd, &output, &input, ZSTD_e_end);
2492 if (sym_ZSTD_isError(res))
2493 return -EIO;
2494
2495 *buffer_size += output.pos;
2496 } while (res != 0);
2497
2498 break;
2499 }
2500 #endif
2501
2502 #if HAVE_ZLIB
2503 case COMPRESSION_GZIP:
2504 c->gzip.avail_in = 0;
2505
2506 do {
2507 r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
2508 if (r < 0)
2509 return r;
2510
2511 size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
2512 c->gzip.next_out = (uint8_t*) *buffer + *buffer_size;
2513 c->gzip.avail_out = avail;
2514
2515 r = sym_deflate(&c->gzip, Z_FINISH);
2516 if (!IN_SET(r, Z_OK, Z_STREAM_END))
2517 return -EIO;
2518
2519 *buffer_size += avail - c->gzip.avail_out;
2520 } while (r != Z_STREAM_END);
2521
2522 break;
2523 #endif
2524
2525 #if HAVE_BZIP2
2526 case COMPRESSION_BZIP2:
2527 c->bzip2.avail_in = 0;
2528
2529 do {
2530 r = enlarge_buffer(buffer, buffer_size, buffer_allocated, /* need= */ 0);
2531 if (r < 0)
2532 return r;
2533
2534 size_t avail = MIN(*buffer_allocated - *buffer_size, (size_t) UINT_MAX);
2535 c->bzip2.next_out = (void*) ((uint8_t*) *buffer + *buffer_size);
2536 c->bzip2.avail_out = avail;
2537
2538 r = sym_BZ2_bzCompress(&c->bzip2, BZ_FINISH);
2539 if (!IN_SET(r, BZ_FINISH_OK, BZ_STREAM_END))
2540 return -EIO;
2541
2542 *buffer_size += avail - c->bzip2.avail_out;
2543 } while (r != BZ_STREAM_END);
2544
2545 break;
2546 #endif
2547
2548 case COMPRESSION_NONE:
2549 break;
2550
2551 default:
2552 return -EOPNOTSUPP;
2553 }
2554
2555 return 0;
2556 }