diff --git a/allocator.h b/allocator.h index eb52485..eb21cee 100644 --- a/allocator.h +++ b/allocator.h @@ -24,6 +24,8 @@ #ifndef __ALLOCATOR_H__ #define __ALLOCATOR_H__ +#include + void slab_init(); void slab_cleanup(int quiet); void *slab_alloc(void *p, size_t size); diff --git a/lzfx/lzfx.c b/lzfx/lzfx.c index ca8138e..e0f5a6d 100644 --- a/lzfx/lzfx.c +++ b/lzfx/lzfx.c @@ -56,8 +56,7 @@ */ #include "lzfx.h" - -#define LZFX_HSIZE (1 << (LZFX_HLOG)) +#include /* We need this for memset */ #ifdef __cplusplus @@ -74,13 +73,10 @@ # define fx_expect_true(expr) (expr) #endif -typedef unsigned char u8; -typedef const u8 *LZSTATE[LZFX_HSIZE]; - /* Define the hash function */ #define LZFX_FRST(p) (((p[0]) << 8) | p[1]) #define LZFX_NEXT(v,p) (((v) << 8) | p[2]) -#define LZFX_IDX(h) ((( h >> (3*8 - LZFX_HLOG)) - h ) & (LZFX_HSIZE - 1)) +#define LZFX_IDX(h, bits) ((( h >> (3*8 - bits)) - h ) & (LZFX_HTAB_SIZE(bits) - 1)) /* These cannot be changed, as they are related to the compressed format. */ #define LZFX_MAX_LIT (1 << 5) @@ -110,11 +106,12 @@ int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen); */ #include int lzfx_compress(const void *const ibuf, const unsigned int ilen, - void *obuf, unsigned int *const olen){ + void *obuf, unsigned int *const olen, + unsigned int htab_bits){ /* Hash table; an array of u8*'s which point to various locations in the input buffer */ - const u8 *htab[LZFX_HSIZE]; + const u8 **htab; const u8 **hslot; /* Pointer to entry in hash table */ unsigned int hval; /* Hash value generated by macros above */ @@ -145,7 +142,8 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen, return lzfx_getsize(ibuf, ilen, olen); } - memset(htab, 0, sizeof(htab)); + htab = (const u8 **)slab_calloc(NULL, LZFX_HTAB_SIZE(htab_bits), sizeof (u8*)); + if (htab == NULL) return LZFX_ENOMEM; /* Start a literal run. Whenever we do this the output pointer is advanced because the current byte will hold the encoded length. */ @@ -156,7 +154,7 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen, while(ip + 2 < in_end){ /* The NEXT macro reads 2 bytes ahead */ hval = LZFX_NEXT(hval, ip); - hslot = htab + LZFX_IDX(hval); + hslot = htab + LZFX_IDX(hval, htab_bits); ref = *hslot; *hslot = ip; @@ -174,8 +172,10 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen, /* lit == 0: op + 3 must be < out_end (because we undo the run) lit != 0: op + 3 + 1 must be < out_end */ - if(fx_expect_false(op - !lit + 3 + 1 >= out_end)) + if(fx_expect_false(op - !lit + 3 + 1 >= out_end)) { + slab_free(NULL, htab); return LZFX_ESIZE; + } op [- lit - 1] = lit - 1; /* Terminate literal run */ op -= !lit; /* Undo run if length is zero */ @@ -209,14 +209,17 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen, hval = LZFX_FRST (ip); hval = LZFX_NEXT (hval, ip); - htab[LZFX_IDX (hval)] = ip; + htab[LZFX_IDX (hval, htab_bits)] = ip; ip++; /* ip = initial ip + #octets */ } else { /* Keep copying literal bytes */ - if (fx_expect_false (op >= out_end)) return LZFX_ESIZE; + if (fx_expect_false (op >= out_end)) { + slab_free(NULL, htab); + return LZFX_ESIZE; + } lit++; *op++ = *ip++; @@ -231,7 +234,10 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen, /* At most 3 bytes remain in input. We therefore need 4 bytes available in the output buffer to store them (3 data + ctrl byte).*/ - if (op + 3 > out_end) return LZFX_ESIZE; + if (op + 3 > out_end) { + slab_free(NULL, htab); + return LZFX_ESIZE; + } while (ip < in_end) { @@ -247,6 +253,7 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen, op -= !lit; *olen = op - (u8 *)obuf; + slab_free(NULL, htab); return 0; } diff --git a/lzfx/lzfx.h b/lzfx/lzfx.h index 3aedd42..4858ec2 100644 --- a/lzfx/lzfx.h +++ b/lzfx/lzfx.h @@ -72,15 +72,15 @@ extern "C" { #define LZFX_VERSION_MINOR 1 #define LZFX_VERSION_STRING "0.1" -/* Hashtable size (2**LZFX_HLOG entries) */ -#ifndef LZFX_HLOG -# define LZFX_HLOG 16 -#endif +#define LZFX_HTAB_SIZE(x) (1 << x) /* Predefined errors. */ #define LZFX_ESIZE -1 /* Output buffer too small */ #define LZFX_ECORRUPT -2 /* Invalid data for decompression */ #define LZFX_EARGS -3 /* Arguments invalid (NULL) */ +#define LZFX_ENOMEM -4 /* Out of memory when allocating hashtable */ + +typedef unsigned char u8; /* Buffer-to buffer compression. @@ -92,7 +92,8 @@ extern "C" { value is returned and olen is not modified. */ int lzfx_compress(const void* ibuf, unsigned int ilen, - void* obuf, unsigned int *olen); + void* obuf, unsigned int *olen, + unsigned int htab_bits); /* Buffer-to-buffer decompression. diff --git a/lzfx_compress.c b/lzfx_compress.c index 045b663..38517a8 100644 --- a/lzfx_compress.c +++ b/lzfx_compress.c @@ -28,6 +28,11 @@ #include #include #include +#include + +struct lzfx_params { + uint32_t htab_bits; +}; void lz_fx_stats(int show) @@ -37,11 +42,33 @@ lz_fx_stats(int show) int lz_fx_init(void **data, int *level, ssize_t chunksize) { - if (*level > 9) *level = 9; + struct lzfx_params *lzdat; + int lev; + if (chunksize > UINT_MAX) { fprintf(stderr, "Chunk size too big for LZFX.\n"); return (1); } + lzdat = slab_alloc(NULL, sizeof (struct lzfx_params)); + + lev = *level; + if (lev > 5) lev = 5; + lzdat->htab_bits = 16 + (lev-1); + *data = lzdat; + + if (*level > 9) *level = 9; + return (0); +} + +int +lz_fx_deinit(void **data) +{ + struct lzfx_params *lzdat = (struct lzfx_params *)(*data); + + if (lzdat) { + slab_free(NULL, lzdat); + } + *data = NULL; return (0); } @@ -58,6 +85,9 @@ lz_fx_err(int err) case LZFX_EARGS: fprintf(stderr, "LZFX: Invalid arguments.\n"); break; + case LZFX_ENOMEM: + fprintf(stderr, "LZFX: Out of memory when allocating hashtable.\n"); + break; default: fprintf(stderr, "LZFX: Unknown error code: %d\n", err); } @@ -68,10 +98,11 @@ lz_fx_compress(void *src, size_t srclen, void *dst, size_t *dstlen, int level, uchar_t chdr, void *data) { int rv; + struct lzfx_params *lzdat = (struct lzfx_params *)data; unsigned int _srclen = srclen; unsigned int _dstlen = *dstlen; - rv = lzfx_compress(src, _srclen, dst, &_dstlen); + rv = lzfx_compress(src, _srclen, dst, &_dstlen, lzdat->htab_bits); if (rv == -1) { lz_fx_err(rv); return (-1); diff --git a/main.c b/main.c index 93dea47..776ce4a 100644 --- a/main.c +++ b/main.c @@ -442,8 +442,11 @@ start_decompress(const char *filename, const char *to_filename) sem_init(&(tdat->start_sem), 0, 0); sem_init(&(tdat->cmp_done_sem), 0, 0); sem_init(&(tdat->write_done_sem), 0, 1); - if (_init_func) - _init_func(&(tdat->data), &(tdat->level), chunksize); + if (_init_func) { + if (_init_func(&(tdat->data), &(tdat->level), chunksize) != 0) { + UNCOMP_BAIL; + } + } if (enable_rabin_scan) tdat->rctx = create_rabin_context(chunksize, compressed_chunksize, algo, enable_delta_encode); @@ -918,8 +921,11 @@ start_compress(const char *filename, uint64_t chunksize, int level) sem_init(&(tdat->start_sem), 0, 0); sem_init(&(tdat->cmp_done_sem), 0, 0); sem_init(&(tdat->write_done_sem), 0, 1); - if (_init_func) - _init_func(&(tdat->data), &(tdat->level), chunksize); + if (_init_func) { + if (_init_func(&(tdat->data), &(tdat->level), chunksize) != 0) { + COMP_BAIL; + } + } if (enable_rabin_scan) tdat->rctx = create_rabin_context(chunksize, compressed_chunksize, algo, enable_delta_encode); @@ -1189,7 +1195,7 @@ init_algo(const char *algo, int bail) _compress_func = lz_fx_compress; _decompress_func = lz_fx_decompress; _init_func = lz_fx_init; - _deinit_func = NULL; + _deinit_func = lz_fx_deinit; _stats_func = lz_fx_stats; rv = 0; diff --git a/pcompress.h b/pcompress.h index f391d25..4f6e114 100644 --- a/pcompress.h +++ b/pcompress.h @@ -92,6 +92,7 @@ extern int lz_fx_init(void **data, int *level, ssize_t chunksize); extern int adapt_deinit(void **data); extern int lzma_deinit(void **data); extern int ppmd_deinit(void **data); +extern int lz_fx_deinit(void **data); extern void adapt_stats(int show); extern void ppmd_stats(int show); diff --git a/rabin/rabin_polynomial.c b/rabin/rabin_polynomial.c index 489bd3f..7dc0789 100755 --- a/rabin/rabin_polynomial.c +++ b/rabin/rabin_polynomial.c @@ -111,12 +111,20 @@ create_rabin_context(uint64_t chunksize, uint64_t real_chunksize, const char *al */ ctx = (rabin_context_t *)slab_alloc(NULL, sizeof (rabin_context_t)); ctx->rabin_poly_max_block_size = RAB_POLYNOMIAL_MAX_BLOCK_SIZE; + if (((memcmp(algo, "lzma", 4) == 0 || memcmp(algo, "adapt", 5) == 0) && chunksize <= LZMA_WINDOW_MAX) || delta_flag) { - ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE; - ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK; - ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE; - ctx->rabin_break_patt = RAB_POLYNOMIAL_CONST; + if (memcmp(algo, "lzfx", 4) == 0 || memcmp(algo, "lz4", 3) == 0) { + ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE2; + ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK2; + ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE2; + ctx->rabin_break_patt = 0; + } else { + ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE; + ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK; + ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE; + ctx->rabin_break_patt = RAB_POLYNOMIAL_CONST; + } } else { ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE2; ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK2;