2 * Copyright (c) 2014 Michihiro NAKAJIMA
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 #include "archive_platform.h"
28 __FBSDID("$FreeBSD$");
48 #include "archive_endian.h"
49 #include "archive_private.h"
50 #include "archive_write_private.h"
51 #include "archive_xxhash.h"
53 #define LZ4_MAGICNUMBER 0x184d2204
56 int compression_level;
57 unsigned header_written:1;
58 unsigned version_number:1;
59 unsigned block_independence:1;
60 unsigned block_checksum:1;
61 unsigned stream_size:1;
62 unsigned stream_checksum:1;
63 unsigned preset_dictionary:1;
64 unsigned block_maximum_size:3;
65 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
69 size_t out_buffer_size;
70 size_t out_block_size;
72 char *in_buffer_allocated;
74 size_t in_buffer_size;
80 struct archive_write_program_data *pdata;
84 static int archive_filter_lz4_close(struct archive_write_filter *);
85 static int archive_filter_lz4_free(struct archive_write_filter *);
86 static int archive_filter_lz4_open(struct archive_write_filter *);
87 static int archive_filter_lz4_options(struct archive_write_filter *,
88 const char *, const char *);
89 static int archive_filter_lz4_write(struct archive_write_filter *,
90 const void *, size_t);
93 * Add a lz4 compression filter to this write handle.
96 archive_write_add_filter_lz4(struct archive *_a)
98 struct archive_write *a = (struct archive_write *)_a;
99 struct archive_write_filter *f = __archive_write_allocate_filter(_a);
100 struct private_data *data;
102 archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,
103 ARCHIVE_STATE_NEW, "archive_write_add_filter_lz4");
105 data = calloc(1, sizeof(*data));
107 archive_set_error(&a->archive, ENOMEM, "Out of memory");
108 return (ARCHIVE_FATAL);
112 * Setup default settings.
114 data->compression_level = 1;
115 data->version_number = 0x01;
116 data->block_independence = 1;
117 data->block_checksum = 0;
118 data->stream_size = 0;
119 data->stream_checksum = 1;
120 data->preset_dictionary = 0;
121 data->block_maximum_size = 7;
124 * Setup a filter setting.
127 f->options = &archive_filter_lz4_options;
128 f->close = &archive_filter_lz4_close;
129 f->free = &archive_filter_lz4_free;
130 f->open = &archive_filter_lz4_open;
131 f->code = ARCHIVE_FILTER_LZ4;
133 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
137 * We don't have lz4 library, and execute external lz4 program
140 data->pdata = __archive_write_program_allocate("lz4");
141 if (data->pdata == NULL) {
143 archive_set_error(&a->archive, ENOMEM, "Out of memory");
144 return (ARCHIVE_FATAL);
146 data->compression_level = 0;
147 archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
148 "Using external lz4 program");
149 return (ARCHIVE_WARN);
157 archive_filter_lz4_options(struct archive_write_filter *f,
158 const char *key, const char *value)
160 struct private_data *data = (struct private_data *)f->data;
162 if (strcmp(key, "compression-level") == 0) {
164 if (value == NULL || !((val = value[0] - '0') >= 1 && val <= 9) ||
166 return (ARCHIVE_WARN);
171 archive_set_error(f->archive, ARCHIVE_ERRNO_PROGRAMMER,
172 "High compression not included in this build");
173 return (ARCHIVE_FATAL);
176 data->compression_level = val;
179 if (strcmp(key, "stream-checksum") == 0) {
180 data->stream_checksum = value != NULL;
183 if (strcmp(key, "block-checksum") == 0) {
184 data->block_checksum = value != NULL;
187 if (strcmp(key, "block-size") == 0) {
188 if (value == NULL || !(value[0] >= '4' && value[0] <= '7') ||
190 return (ARCHIVE_WARN);
191 data->block_maximum_size = value[0] - '0';
194 if (strcmp(key, "block-dependence") == 0) {
195 data->block_independence = value == NULL;
199 /* Note: The "warn" return is just to inform the options
200 * supervisor that we didn't handle it. It will generate
201 * a suitable error if no one used this option. */
202 return (ARCHIVE_WARN);
205 #if defined(HAVE_LIBLZ4) && LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 2
206 /* Don't compile this if we don't have liblz4. */
208 static int drive_compressor(struct archive_write_filter *, const char *,
210 static int drive_compressor_independence(struct archive_write_filter *,
211 const char *, size_t);
212 static int drive_compressor_dependence(struct archive_write_filter *,
213 const char *, size_t);
214 static int lz4_write_stream_descriptor(struct archive_write_filter *);
215 static ssize_t lz4_write_one_block(struct archive_write_filter *, const char *,
223 archive_filter_lz4_open(struct archive_write_filter *f)
225 struct private_data *data = (struct private_data *)f->data;
226 size_t required_size;
227 static size_t const bkmap[] = { 64 * 1024, 256 * 1024, 1 * 1024 * 1024,
229 size_t pre_block_size;
231 if (data->block_maximum_size < 4)
232 data->block_size = bkmap[0];
234 data->block_size = bkmap[data->block_maximum_size - 4];
236 required_size = 4 + 15 + 4 + data->block_size + 4 + 4;
237 if (data->out_buffer_size < required_size) {
238 size_t bs = required_size, bpb;
239 free(data->out_buffer);
240 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
241 /* Buffer size should be a multiple number of
242 * the of bytes per block for performance. */
243 bpb = archive_write_get_bytes_per_block(f->archive);
251 data->out_block_size = bs;
253 data->out_buffer = malloc(bs);
254 data->out = data->out_buffer;
255 data->out_buffer_size = bs;
258 pre_block_size = (data->block_independence)? 0: 64 * 1024;
259 if (data->in_buffer_size < data->block_size + pre_block_size) {
260 free(data->in_buffer_allocated);
261 data->in_buffer_size = data->block_size;
262 data->in_buffer_allocated =
263 malloc(data->in_buffer_size + pre_block_size);
264 data->in_buffer = data->in_buffer_allocated + pre_block_size;
265 if (!data->block_independence && data->compression_level >= 3)
266 data->in_buffer = data->in_buffer_allocated;
267 data->in = data->in_buffer;
268 data->in_buffer_size = data->block_size;
271 if (data->out_buffer == NULL || data->in_buffer_allocated == NULL) {
272 archive_set_error(f->archive, ENOMEM,
273 "Can't allocate data for compression buffer");
274 return (ARCHIVE_FATAL);
277 f->write = archive_filter_lz4_write;
283 * Write data to the out stream.
285 * Returns ARCHIVE_OK if all data written, error otherwise.
288 archive_filter_lz4_write(struct archive_write_filter *f,
289 const void *buff, size_t length)
291 struct private_data *data = (struct private_data *)f->data;
292 int ret = ARCHIVE_OK;
297 /* If we haven't written a stream descriptor, we have to do it first. */
298 if (!data->header_written) {
299 ret = lz4_write_stream_descriptor(f);
300 if (ret != ARCHIVE_OK)
302 data->header_written = 1;
305 /* Update statistics */
306 data->total_in += length;
308 p = (const char *)buff;
312 /* Compress input data to output buffer */
313 size = lz4_write_one_block(f, p, remaining);
314 if (size < ARCHIVE_OK)
315 return (ARCHIVE_FATAL);
316 l = data->out - data->out_buffer;
317 if (l >= data->out_block_size) {
318 ret = __archive_write_filter(f->next_filter,
319 data->out_buffer, data->out_block_size);
320 l -= data->out_block_size;
321 memcpy(data->out_buffer,
322 data->out_buffer + data->out_block_size, l);
323 data->out = data->out_buffer + l;
324 if (ret < ARCHIVE_WARN)
335 * Finish the compression.
338 archive_filter_lz4_close(struct archive_write_filter *f)
340 struct private_data *data = (struct private_data *)f->data;
343 /* Finish compression cycle. */
344 ret = (int)lz4_write_one_block(f, NULL, 0);
347 * Write the last block and the end of the stream data.
350 /* Write End Of Stream. */
351 memset(data->out, 0, 4); data->out += 4;
352 /* Write Stream checksum if needed. */
353 if (data->stream_checksum) {
354 unsigned int checksum;
355 checksum = __archive_xxhash.XXH32_digest(
357 data->xxh32_state = NULL;
358 archive_le32enc(data->out, checksum);
361 ret = __archive_write_filter(f->next_filter,
362 data->out_buffer, data->out - data->out_buffer);
368 archive_filter_lz4_free(struct archive_write_filter *f)
370 struct private_data *data = (struct private_data *)f->data;
372 if (data->lz4_stream != NULL) {
374 if (data->compression_level >= 3)
375 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
376 LZ4_freeStreamHC(data->lz4_stream);
378 LZ4_freeHC(data->lz4_stream);
382 #if LZ4_VERSION_MINOR >= 3
383 LZ4_freeStream(data->lz4_stream);
385 LZ4_free(data->lz4_stream);
388 free(data->out_buffer);
389 free(data->in_buffer_allocated);
390 free(data->xxh32_state);
397 lz4_write_stream_descriptor(struct archive_write_filter *f)
399 struct private_data *data = (struct private_data *)f->data;
402 sd = (uint8_t *)data->out;
403 /* Write Magic Number. */
404 archive_le32enc(&sd[0], LZ4_MAGICNUMBER);
406 sd[4] = (data->version_number << 6)
407 | (data->block_independence << 5)
408 | (data->block_checksum << 4)
409 | (data->stream_size << 3)
410 | (data->stream_checksum << 2)
411 | (data->preset_dictionary << 0);
413 sd[5] = (data->block_maximum_size << 4);
414 sd[6] = (__archive_xxhash.XXH32(&sd[4], 2, 0) >> 8) & 0xff;
416 if (data->stream_checksum)
417 data->xxh32_state = __archive_xxhash.XXH32_init(0);
419 data->xxh32_state = NULL;
424 lz4_write_one_block(struct archive_write_filter *f, const char *p,
427 struct private_data *data = (struct private_data *)f->data;
431 /* Compress remaining uncompressed data. */
432 if (data->in_buffer == data->in)
435 size_t l = data->in - data->in_buffer;
436 r = drive_compressor(f, data->in_buffer, l);
440 } else if ((data->block_independence || data->compression_level < 3) &&
441 data->in_buffer == data->in && length >= data->block_size) {
442 r = drive_compressor(f, p, data->block_size);
444 r = (ssize_t)data->block_size;
446 size_t remaining_size = data->in_buffer_size -
447 (data->in - data->in_buffer);
448 size_t l = (remaining_size > length)? length: remaining_size;
449 memcpy(data->in, p, l);
451 if (l == remaining_size) {
452 r = drive_compressor(f, data->in_buffer,
456 data->in = data->in_buffer;
466 * Utility function to push input data through compressor, writing
467 * full output blocks as necessary.
469 * Note that this handles both the regular write case (finishing ==
470 * false) and the end-of-archive case (finishing == true).
473 drive_compressor(struct archive_write_filter *f, const char *p, size_t length)
475 struct private_data *data = (struct private_data *)f->data;
477 if (data->stream_checksum)
478 __archive_xxhash.XXH32_update(data->xxh32_state,
480 if (data->block_independence)
481 return drive_compressor_independence(f, p, length);
483 return drive_compressor_dependence(f, p, length);
487 drive_compressor_independence(struct archive_write_filter *f, const char *p,
490 struct private_data *data = (struct private_data *)f->data;
491 unsigned int outsize;
494 if (data->compression_level >= 3)
495 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
496 outsize = LZ4_compress_HC(p, data->out + 4,
497 (int)length, (int)data->block_size,
498 data->compression_level);
500 outsize = LZ4_compressHC2_limitedOutput(p, data->out + 4,
501 (int)length, (int)data->block_size,
502 data->compression_level);
506 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
507 outsize = LZ4_compress_default(p, data->out + 4,
508 (int)length, (int)data->block_size);
510 outsize = LZ4_compress_limitedOutput(p, data->out + 4,
511 (int)length, (int)data->block_size);
515 /* The buffer is compressed. */
516 archive_le32enc(data->out, outsize);
519 /* The buffer is not compressed. The compressed size was
520 * bigger than its uncompressed size. */
521 archive_le32enc(data->out, length | 0x80000000);
523 memcpy(data->out, p, length);
526 data->out += outsize;
527 if (data->block_checksum) {
528 unsigned int checksum =
529 __archive_xxhash.XXH32(data->out - outsize, outsize, 0);
530 archive_le32enc(data->out, checksum);
537 drive_compressor_dependence(struct archive_write_filter *f, const char *p,
540 struct private_data *data = (struct private_data *)f->data;
543 #define DICT_SIZE (64 * 1024)
545 if (data->compression_level >= 3) {
546 if (data->lz4_stream == NULL) {
547 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
548 data->lz4_stream = LZ4_createStreamHC();
549 LZ4_resetStreamHC(data->lz4_stream, data->compression_level);
552 LZ4_createHC(data->in_buffer_allocated);
554 if (data->lz4_stream == NULL) {
555 archive_set_error(f->archive, ENOMEM,
556 "Can't allocate data for compression"
558 return (ARCHIVE_FATAL);
562 LZ4_loadDictHC(data->lz4_stream, data->in_buffer_allocated, DICT_SIZE);
564 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
565 outsize = LZ4_compress_HC_continue(
566 data->lz4_stream, p, data->out + 4, (int)length,
567 (int)data->block_size);
569 outsize = LZ4_compressHC2_limitedOutput_continue(
570 data->lz4_stream, p, data->out + 4, (int)length,
571 (int)data->block_size, data->compression_level);
576 if (data->lz4_stream == NULL) {
577 data->lz4_stream = LZ4_createStream();
578 if (data->lz4_stream == NULL) {
579 archive_set_error(f->archive, ENOMEM,
580 "Can't allocate data for compression"
582 return (ARCHIVE_FATAL);
586 LZ4_loadDict(data->lz4_stream, data->in_buffer_allocated, DICT_SIZE);
588 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
589 outsize = LZ4_compress_fast_continue(
590 data->lz4_stream, p, data->out + 4, (int)length,
591 (int)data->block_size, 1);
593 outsize = LZ4_compress_limitedOutput_continue(
594 data->lz4_stream, p, data->out + 4, (int)length,
595 (int)data->block_size);
600 /* The buffer is compressed. */
601 archive_le32enc(data->out, outsize);
604 /* The buffer is not compressed. The compressed size was
605 * bigger than its uncompressed size. */
606 archive_le32enc(data->out, length | 0x80000000);
608 memcpy(data->out, p, length);
611 data->out += outsize;
612 if (data->block_checksum) {
613 unsigned int checksum =
614 __archive_xxhash.XXH32(data->out - outsize, outsize, 0);
615 archive_le32enc(data->out, checksum);
619 if (length == data->block_size) {
621 if (data->compression_level >= 3) {
622 #if LZ4_VERSION_MAJOR >= 1 && LZ4_VERSION_MINOR >= 7
623 LZ4_saveDictHC(data->lz4_stream, data->in_buffer_allocated, DICT_SIZE);
625 LZ4_slideInputBufferHC(data->lz4_stream);
627 data->in_buffer = data->in_buffer_allocated + DICT_SIZE;
631 LZ4_saveDict(data->lz4_stream,
632 data->in_buffer_allocated, DICT_SIZE);
638 #else /* HAVE_LIBLZ4 */
641 archive_filter_lz4_open(struct archive_write_filter *f)
643 struct private_data *data = (struct private_data *)f->data;
644 struct archive_string as;
647 archive_string_init(&as);
648 archive_strcpy(&as, "lz4 -z -q -q");
650 /* Specify a compression level. */
651 if (data->compression_level > 0) {
652 archive_strcat(&as, " -");
653 archive_strappend_char(&as, '0' + data->compression_level);
655 /* Specify a block size. */
656 archive_strcat(&as, " -B");
657 archive_strappend_char(&as, '0' + data->block_maximum_size);
659 if (data->block_checksum)
660 archive_strcat(&as, " -BX");
661 if (data->stream_checksum == 0)
662 archive_strcat(&as, " --no-frame-crc");
663 if (data->block_independence == 0)
664 archive_strcat(&as, " -BD");
666 f->write = archive_filter_lz4_write;
668 r = __archive_write_program_open(f, data->pdata, as.s);
669 archive_string_free(&as);
674 archive_filter_lz4_write(struct archive_write_filter *f, const void *buff,
677 struct private_data *data = (struct private_data *)f->data;
679 return __archive_write_program_write(f, data->pdata, buff, length);
683 archive_filter_lz4_close(struct archive_write_filter *f)
685 struct private_data *data = (struct private_data *)f->data;
687 return __archive_write_program_close(f, data->pdata);
691 archive_filter_lz4_free(struct archive_write_filter *f)
693 struct private_data *data = (struct private_data *)f->data;
695 __archive_write_program_free(data->pdata);
700 #endif /* HAVE_LIBLZ4 */