Make LZFX Hash size dynamic.
Use smaller min rabin block when using fast compression algos. Add missing check for algo init function return value.
This commit is contained in:
parent
8cfd54fe34
commit
53d4311534
7 changed files with 86 additions and 30 deletions
|
@ -24,6 +24,8 @@
|
|||
#ifndef __ALLOCATOR_H__
|
||||
#define __ALLOCATOR_H__
|
||||
|
||||
#include <sys/types.h>
|
||||
|
||||
void slab_init();
|
||||
void slab_cleanup(int quiet);
|
||||
void *slab_alloc(void *p, size_t size);
|
||||
|
|
35
lzfx/lzfx.c
35
lzfx/lzfx.c
|
@ -56,8 +56,7 @@
|
|||
*/
|
||||
|
||||
#include "lzfx.h"
|
||||
|
||||
#define LZFX_HSIZE (1 << (LZFX_HLOG))
|
||||
#include <allocator.h>
|
||||
|
||||
/* We need this for memset */
|
||||
#ifdef __cplusplus
|
||||
|
@ -74,13 +73,10 @@
|
|||
# define fx_expect_true(expr) (expr)
|
||||
#endif
|
||||
|
||||
typedef unsigned char u8;
|
||||
typedef const u8 *LZSTATE[LZFX_HSIZE];
|
||||
|
||||
/* Define the hash function */
|
||||
#define LZFX_FRST(p) (((p[0]) << 8) | p[1])
|
||||
#define LZFX_NEXT(v,p) (((v) << 8) | p[2])
|
||||
#define LZFX_IDX(h) ((( h >> (3*8 - LZFX_HLOG)) - h ) & (LZFX_HSIZE - 1))
|
||||
#define LZFX_IDX(h, bits) ((( h >> (3*8 - bits)) - h ) & (LZFX_HTAB_SIZE(bits) - 1))
|
||||
|
||||
/* These cannot be changed, as they are related to the compressed format. */
|
||||
#define LZFX_MAX_LIT (1 << 5)
|
||||
|
@ -110,11 +106,12 @@ int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen);
|
|||
*/
|
||||
#include <stdio.h>
|
||||
int lzfx_compress(const void *const ibuf, const unsigned int ilen,
|
||||
void *obuf, unsigned int *const olen){
|
||||
void *obuf, unsigned int *const olen,
|
||||
unsigned int htab_bits){
|
||||
|
||||
/* Hash table; an array of u8*'s which point
|
||||
to various locations in the input buffer */
|
||||
const u8 *htab[LZFX_HSIZE];
|
||||
const u8 **htab;
|
||||
|
||||
const u8 **hslot; /* Pointer to entry in hash table */
|
||||
unsigned int hval; /* Hash value generated by macros above */
|
||||
|
@ -145,7 +142,8 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
|
|||
return lzfx_getsize(ibuf, ilen, olen);
|
||||
}
|
||||
|
||||
memset(htab, 0, sizeof(htab));
|
||||
htab = (const u8 **)slab_calloc(NULL, LZFX_HTAB_SIZE(htab_bits), sizeof (u8*));
|
||||
if (htab == NULL) return LZFX_ENOMEM;
|
||||
|
||||
/* Start a literal run. Whenever we do this the output pointer is
|
||||
advanced because the current byte will hold the encoded length. */
|
||||
|
@ -156,7 +154,7 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
|
|||
while(ip + 2 < in_end){ /* The NEXT macro reads 2 bytes ahead */
|
||||
|
||||
hval = LZFX_NEXT(hval, ip);
|
||||
hslot = htab + LZFX_IDX(hval);
|
||||
hslot = htab + LZFX_IDX(hval, htab_bits);
|
||||
|
||||
ref = *hslot; *hslot = ip;
|
||||
|
||||
|
@ -174,8 +172,10 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
|
|||
|
||||
/* lit == 0: op + 3 must be < out_end (because we undo the run)
|
||||
lit != 0: op + 3 + 1 must be < out_end */
|
||||
if(fx_expect_false(op - !lit + 3 + 1 >= out_end))
|
||||
if(fx_expect_false(op - !lit + 3 + 1 >= out_end)) {
|
||||
slab_free(NULL, htab);
|
||||
return LZFX_ESIZE;
|
||||
}
|
||||
|
||||
op [- lit - 1] = lit - 1; /* Terminate literal run */
|
||||
op -= !lit; /* Undo run if length is zero */
|
||||
|
@ -209,14 +209,17 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
|
|||
|
||||
hval = LZFX_FRST (ip);
|
||||
hval = LZFX_NEXT (hval, ip);
|
||||
htab[LZFX_IDX (hval)] = ip;
|
||||
htab[LZFX_IDX (hval, htab_bits)] = ip;
|
||||
|
||||
ip++; /* ip = initial ip + #octets */
|
||||
|
||||
} else {
|
||||
/* Keep copying literal bytes */
|
||||
|
||||
if (fx_expect_false (op >= out_end)) return LZFX_ESIZE;
|
||||
if (fx_expect_false (op >= out_end)) {
|
||||
slab_free(NULL, htab);
|
||||
return LZFX_ESIZE;
|
||||
}
|
||||
|
||||
lit++; *op++ = *ip++;
|
||||
|
||||
|
@ -231,7 +234,10 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
|
|||
|
||||
/* At most 3 bytes remain in input. We therefore need 4 bytes available
|
||||
in the output buffer to store them (3 data + ctrl byte).*/
|
||||
if (op + 3 > out_end) return LZFX_ESIZE;
|
||||
if (op + 3 > out_end) {
|
||||
slab_free(NULL, htab);
|
||||
return LZFX_ESIZE;
|
||||
}
|
||||
|
||||
while (ip < in_end) {
|
||||
|
||||
|
@ -247,6 +253,7 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
|
|||
op -= !lit;
|
||||
|
||||
*olen = op - (u8 *)obuf;
|
||||
slab_free(NULL, htab);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
11
lzfx/lzfx.h
11
lzfx/lzfx.h
|
@ -72,15 +72,15 @@ extern "C" {
|
|||
#define LZFX_VERSION_MINOR 1
|
||||
#define LZFX_VERSION_STRING "0.1"
|
||||
|
||||
/* Hashtable size (2**LZFX_HLOG entries) */
|
||||
#ifndef LZFX_HLOG
|
||||
# define LZFX_HLOG 16
|
||||
#endif
|
||||
#define LZFX_HTAB_SIZE(x) (1 << x)
|
||||
|
||||
/* Predefined errors. */
|
||||
#define LZFX_ESIZE -1 /* Output buffer too small */
|
||||
#define LZFX_ECORRUPT -2 /* Invalid data for decompression */
|
||||
#define LZFX_EARGS -3 /* Arguments invalid (NULL) */
|
||||
#define LZFX_ENOMEM -4 /* Out of memory when allocating hashtable */
|
||||
|
||||
typedef unsigned char u8;
|
||||
|
||||
/* Buffer-to buffer compression.
|
||||
|
||||
|
@ -92,7 +92,8 @@ extern "C" {
|
|||
value is returned and olen is not modified.
|
||||
*/
|
||||
int lzfx_compress(const void* ibuf, unsigned int ilen,
|
||||
void* obuf, unsigned int *olen);
|
||||
void* obuf, unsigned int *olen,
|
||||
unsigned int htab_bits);
|
||||
|
||||
/* Buffer-to-buffer decompression.
|
||||
|
||||
|
|
|
@ -28,6 +28,11 @@
|
|||
#include <utils.h>
|
||||
#include <pcompress.h>
|
||||
#include <lzfx.h>
|
||||
#include <allocator.h>
|
||||
|
||||
struct lzfx_params {
|
||||
uint32_t htab_bits;
|
||||
};
|
||||
|
||||
void
|
||||
lz_fx_stats(int show)
|
||||
|
@ -37,11 +42,33 @@ lz_fx_stats(int show)
|
|||
int
|
||||
lz_fx_init(void **data, int *level, ssize_t chunksize)
|
||||
{
|
||||
if (*level > 9) *level = 9;
|
||||
struct lzfx_params *lzdat;
|
||||
int lev;
|
||||
|
||||
if (chunksize > UINT_MAX) {
|
||||
fprintf(stderr, "Chunk size too big for LZFX.\n");
|
||||
return (1);
|
||||
}
|
||||
lzdat = slab_alloc(NULL, sizeof (struct lzfx_params));
|
||||
|
||||
lev = *level;
|
||||
if (lev > 5) lev = 5;
|
||||
lzdat->htab_bits = 16 + (lev-1);
|
||||
*data = lzdat;
|
||||
|
||||
if (*level > 9) *level = 9;
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
lz_fx_deinit(void **data)
|
||||
{
|
||||
struct lzfx_params *lzdat = (struct lzfx_params *)(*data);
|
||||
|
||||
if (lzdat) {
|
||||
slab_free(NULL, lzdat);
|
||||
}
|
||||
*data = NULL;
|
||||
return (0);
|
||||
}
|
||||
|
||||
|
@ -58,6 +85,9 @@ lz_fx_err(int err)
|
|||
case LZFX_EARGS:
|
||||
fprintf(stderr, "LZFX: Invalid arguments.\n");
|
||||
break;
|
||||
case LZFX_ENOMEM:
|
||||
fprintf(stderr, "LZFX: Out of memory when allocating hashtable.\n");
|
||||
break;
|
||||
default:
|
||||
fprintf(stderr, "LZFX: Unknown error code: %d\n", err);
|
||||
}
|
||||
|
@ -68,10 +98,11 @@ lz_fx_compress(void *src, size_t srclen, void *dst, size_t *dstlen,
|
|||
int level, uchar_t chdr, void *data)
|
||||
{
|
||||
int rv;
|
||||
struct lzfx_params *lzdat = (struct lzfx_params *)data;
|
||||
unsigned int _srclen = srclen;
|
||||
unsigned int _dstlen = *dstlen;
|
||||
|
||||
rv = lzfx_compress(src, _srclen, dst, &_dstlen);
|
||||
rv = lzfx_compress(src, _srclen, dst, &_dstlen, lzdat->htab_bits);
|
||||
if (rv == -1) {
|
||||
lz_fx_err(rv);
|
||||
return (-1);
|
||||
|
|
16
main.c
16
main.c
|
@ -442,8 +442,11 @@ start_decompress(const char *filename, const char *to_filename)
|
|||
sem_init(&(tdat->start_sem), 0, 0);
|
||||
sem_init(&(tdat->cmp_done_sem), 0, 0);
|
||||
sem_init(&(tdat->write_done_sem), 0, 1);
|
||||
if (_init_func)
|
||||
_init_func(&(tdat->data), &(tdat->level), chunksize);
|
||||
if (_init_func) {
|
||||
if (_init_func(&(tdat->data), &(tdat->level), chunksize) != 0) {
|
||||
UNCOMP_BAIL;
|
||||
}
|
||||
}
|
||||
if (enable_rabin_scan)
|
||||
tdat->rctx = create_rabin_context(chunksize, compressed_chunksize,
|
||||
algo, enable_delta_encode);
|
||||
|
@ -918,8 +921,11 @@ start_compress(const char *filename, uint64_t chunksize, int level)
|
|||
sem_init(&(tdat->start_sem), 0, 0);
|
||||
sem_init(&(tdat->cmp_done_sem), 0, 0);
|
||||
sem_init(&(tdat->write_done_sem), 0, 1);
|
||||
if (_init_func)
|
||||
_init_func(&(tdat->data), &(tdat->level), chunksize);
|
||||
if (_init_func) {
|
||||
if (_init_func(&(tdat->data), &(tdat->level), chunksize) != 0) {
|
||||
COMP_BAIL;
|
||||
}
|
||||
}
|
||||
if (enable_rabin_scan)
|
||||
tdat->rctx = create_rabin_context(chunksize, compressed_chunksize,
|
||||
algo, enable_delta_encode);
|
||||
|
@ -1189,7 +1195,7 @@ init_algo(const char *algo, int bail)
|
|||
_compress_func = lz_fx_compress;
|
||||
_decompress_func = lz_fx_decompress;
|
||||
_init_func = lz_fx_init;
|
||||
_deinit_func = NULL;
|
||||
_deinit_func = lz_fx_deinit;
|
||||
_stats_func = lz_fx_stats;
|
||||
rv = 0;
|
||||
|
||||
|
|
|
@ -92,6 +92,7 @@ extern int lz_fx_init(void **data, int *level, ssize_t chunksize);
|
|||
extern int adapt_deinit(void **data);
|
||||
extern int lzma_deinit(void **data);
|
||||
extern int ppmd_deinit(void **data);
|
||||
extern int lz_fx_deinit(void **data);
|
||||
|
||||
extern void adapt_stats(int show);
|
||||
extern void ppmd_stats(int show);
|
||||
|
|
|
@ -111,12 +111,20 @@ create_rabin_context(uint64_t chunksize, uint64_t real_chunksize, const char *al
|
|||
*/
|
||||
ctx = (rabin_context_t *)slab_alloc(NULL, sizeof (rabin_context_t));
|
||||
ctx->rabin_poly_max_block_size = RAB_POLYNOMIAL_MAX_BLOCK_SIZE;
|
||||
|
||||
if (((memcmp(algo, "lzma", 4) == 0 || memcmp(algo, "adapt", 5) == 0) &&
|
||||
chunksize <= LZMA_WINDOW_MAX) || delta_flag) {
|
||||
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE;
|
||||
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK;
|
||||
ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE;
|
||||
ctx->rabin_break_patt = RAB_POLYNOMIAL_CONST;
|
||||
if (memcmp(algo, "lzfx", 4) == 0 || memcmp(algo, "lz4", 3) == 0) {
|
||||
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE2;
|
||||
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK2;
|
||||
ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE2;
|
||||
ctx->rabin_break_patt = 0;
|
||||
} else {
|
||||
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE;
|
||||
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK;
|
||||
ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE;
|
||||
ctx->rabin_break_patt = RAB_POLYNOMIAL_CONST;
|
||||
}
|
||||
} else {
|
||||
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE2;
|
||||
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK2;
|
||||
|
|
Loading…
Reference in a new issue