greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Make LZFX Hash size dynamic.

Browse source 
Use smaller min rabin block when using fast compression algos.
Add missing check for algo init function return value.
This commit is contained in:
Moinak Ghosh 2012-07-23 21:43:12 +05:30
parent 8cfd54fe34
commit 53d4311534
7 changed files with 86 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
allocator.h
View file

@ -24,6 +24,8 @@
#ifndef __ALLOCATOR_H__
#define __ALLOCATOR_H__
#include <sys/types.h>
void slab_init();
void slab_cleanup(int quiet);
void *slab_alloc(void *p, size_t size);

35
lzfx/lzfx.c
View file

@ -56,8 +56,7 @@
*/
#include "lzfx.h"
#define LZFX_HSIZE (1 << (LZFX_HLOG))
#include <allocator.h>
/* We need this for memset */
#ifdef __cplusplus
@ -74,13 +73,10 @@
# define fx_expect_true(expr) (expr)
#endif
typedef unsigned char u8;
typedef const u8 *LZSTATE[LZFX_HSIZE];
/* Define the hash function */
#define LZFX_FRST(p) (((p[0]) << 8) | p[1])
#define LZFX_NEXT(v,p) (((v) << 8) | p[2])
#define LZFX_IDX(h) ((( h >> (3*8 - LZFX_HLOG)) - h ) & (LZFX_HSIZE - 1))
#define LZFX_IDX(h, bits) ((( h >> (3*8 - bits)) - h ) & (LZFX_HTAB_SIZE(bits) - 1))
/* These cannot be changed, as they are related to the compressed format. */
#define LZFX_MAX_LIT (1 << 5)
@ -110,11 +106,12 @@ int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen);
*/
#include <stdio.h>
int lzfx_compress(const void *const ibuf, const unsigned int ilen,
void *obuf, unsigned int *const olen){
void *obuf, unsigned int *const olen,
unsigned int htab_bits){
/* Hash table; an array of u8*'s which point
to various locations in the input buffer */
const u8 *htab[LZFX_HSIZE];
const u8 **htab;
const u8 **hslot; /* Pointer to entry in hash table */
unsigned int hval; /* Hash value generated by macros above */
@ -145,7 +142,8 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
return lzfx_getsize(ibuf, ilen, olen);
}
memset(htab, 0, sizeof(htab));
htab = (const u8 **)slab_calloc(NULL, LZFX_HTAB_SIZE(htab_bits), sizeof (u8*));
if (htab == NULL) return LZFX_ENOMEM;
/* Start a literal run. Whenever we do this the output pointer is
advanced because the current byte will hold the encoded length. */
@ -156,7 +154,7 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
while(ip + 2 < in_end){ /* The NEXT macro reads 2 bytes ahead */
hval = LZFX_NEXT(hval, ip);
hslot = htab + LZFX_IDX(hval);
hslot = htab + LZFX_IDX(hval, htab_bits);
ref = *hslot; *hslot = ip;
@ -174,8 +172,10 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
/* lit == 0: op + 3 must be < out_end (because we undo the run)
lit != 0: op + 3 + 1 must be < out_end */
if(fx_expect_false(op - !lit + 3 + 1 >= out_end))
if(fx_expect_false(op - !lit + 3 + 1 >= out_end)) {
slab_free(NULL, htab);
return LZFX_ESIZE;
}
op [- lit - 1] = lit - 1; /* Terminate literal run */
op -= !lit; /* Undo run if length is zero */
@ -209,14 +209,17 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
hval = LZFX_FRST (ip);
hval = LZFX_NEXT (hval, ip);
htab[LZFX_IDX (hval)] = ip;
htab[LZFX_IDX (hval, htab_bits)] = ip;
ip++; /* ip = initial ip + #octets */
} else {
/* Keep copying literal bytes */
if (fx_expect_false (op >= out_end)) return LZFX_ESIZE;
if (fx_expect_false (op >= out_end)) {
slab_free(NULL, htab);
return LZFX_ESIZE;
}
lit++; *op++ = *ip++;
@ -231,7 +234,10 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
/* At most 3 bytes remain in input. We therefore need 4 bytes available
in the output buffer to store them (3 data + ctrl byte).*/
if (op + 3 > out_end) return LZFX_ESIZE;
if (op + 3 > out_end) {
slab_free(NULL, htab);
return LZFX_ESIZE;
}
while (ip < in_end) {
@ -247,6 +253,7 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
op -= !lit;
*olen = op - (u8 *)obuf;
slab_free(NULL, htab);
return 0;
}

11
lzfx/lzfx.h
View file

@ -72,15 +72,15 @@ extern "C" {
#define LZFX_VERSION_MINOR 1
#define LZFX_VERSION_STRING "0.1"
/* Hashtable size (2**LZFX_HLOG entries) */
#ifndef LZFX_HLOG
# define LZFX_HLOG 16
#endif
#define LZFX_HTAB_SIZE(x) (1 << x)
/* Predefined errors. */
#define LZFX_ESIZE -1 /* Output buffer too small */
#define LZFX_ECORRUPT -2 /* Invalid data for decompression */
#define LZFX_EARGS -3 /* Arguments invalid (NULL) */
#define LZFX_ENOMEM -4 /* Out of memory when allocating hashtable */
typedef unsigned char u8;
/* Buffer-to buffer compression.
@ -92,7 +92,8 @@ extern "C" {
value is returned and olen is not modified.
*/
int lzfx_compress(const void* ibuf, unsigned int ilen,
void* obuf, unsigned int *olen);
void* obuf, unsigned int *olen,
unsigned int htab_bits);
/* Buffer-to-buffer decompression.

35
lzfx_compress.c
View file

@ -28,6 +28,11 @@
#include <utils.h>
#include <pcompress.h>
#include <lzfx.h>
#include <allocator.h>
struct lzfx_params {
uint32_t htab_bits;
};
void
lz_fx_stats(int show)
@ -37,11 +42,33 @@ lz_fx_stats(int show)
int
lz_fx_init(void **data, int *level, ssize_t chunksize)
{
if (*level > 9) *level = 9;
struct lzfx_params *lzdat;
int lev;
if (chunksize > UINT_MAX) {
fprintf(stderr, "Chunk size too big for LZFX.\n");
return (1);
}
lzdat = slab_alloc(NULL, sizeof (struct lzfx_params));
lev = *level;
if (lev > 5) lev = 5;
lzdat->htab_bits = 16 + (lev-1);
*data = lzdat;
if (*level > 9) *level = 9;
return (0);
}
int
lz_fx_deinit(void **data)
{
struct lzfx_params *lzdat = (struct lzfx_params *)(*data);
if (lzdat) {
slab_free(NULL, lzdat);
}
*data = NULL;
return (0);
}
@ -58,6 +85,9 @@ lz_fx_err(int err)
case LZFX_EARGS:
fprintf(stderr, "LZFX: Invalid arguments.\n");
break;
case LZFX_ENOMEM:
fprintf(stderr, "LZFX: Out of memory when allocating hashtable.\n");
break;
default:
fprintf(stderr, "LZFX: Unknown error code: %d\n", err);
}
@ -68,10 +98,11 @@ lz_fx_compress(void *src, size_t srclen, void *dst, size_t *dstlen,
int level, uchar_t chdr, void *data)
{
int rv;
struct lzfx_params *lzdat = (struct lzfx_params *)data;
unsigned int _srclen = srclen;
unsigned int _dstlen = *dstlen;
rv = lzfx_compress(src, _srclen, dst, &_dstlen);
rv = lzfx_compress(src, _srclen, dst, &_dstlen, lzdat->htab_bits);
if (rv == -1) {
lz_fx_err(rv);
return (-1);

16
main.c
View file

@ -442,8 +442,11 @@ start_decompress(const char *filename, const char *to_filename)
sem_init(&(tdat->start_sem), 0, 0);
sem_init(&(tdat->cmp_done_sem), 0, 0);
sem_init(&(tdat->write_done_sem), 0, 1);
if (_init_func)
_init_func(&(tdat->data), &(tdat->level), chunksize);
if (_init_func) {
if (_init_func(&(tdat->data), &(tdat->level), chunksize) != 0) {
UNCOMP_BAIL;
}
}
if (enable_rabin_scan)
tdat->rctx = create_rabin_context(chunksize, compressed_chunksize,
algo, enable_delta_encode);
@ -918,8 +921,11 @@ start_compress(const char *filename, uint64_t chunksize, int level)
sem_init(&(tdat->start_sem), 0, 0);
sem_init(&(tdat->cmp_done_sem), 0, 0);
sem_init(&(tdat->write_done_sem), 0, 1);
if (_init_func)
_init_func(&(tdat->data), &(tdat->level), chunksize);
if (_init_func) {
if (_init_func(&(tdat->data), &(tdat->level), chunksize) != 0) {
COMP_BAIL;
}
}
if (enable_rabin_scan)
tdat->rctx = create_rabin_context(chunksize, compressed_chunksize,
algo, enable_delta_encode);
@ -1189,7 +1195,7 @@ init_algo(const char *algo, int bail)
_compress_func = lz_fx_compress;
_decompress_func = lz_fx_decompress;
_init_func = lz_fx_init;
_deinit_func = NULL;
_deinit_func = lz_fx_deinit;
_stats_func = lz_fx_stats;
rv = 0;

1
pcompress.h
View file

@ -92,6 +92,7 @@ extern int lz_fx_init(void **data, int *level, ssize_t chunksize);
extern int adapt_deinit(void **data);
extern int lzma_deinit(void **data);
extern int ppmd_deinit(void **data);
extern int lz_fx_deinit(void **data);
extern void adapt_stats(int show);
extern void ppmd_stats(int show);

16
rabin/rabin_polynomial.c
View file

@ -111,12 +111,20 @@ create_rabin_context(uint64_t chunksize, uint64_t real_chunksize, const char *al
*/
ctx = (rabin_context_t *)slab_alloc(NULL, sizeof (rabin_context_t));
ctx->rabin_poly_max_block_size = RAB_POLYNOMIAL_MAX_BLOCK_SIZE;
if (((memcmp(algo, "lzma", 4) == 0 || memcmp(algo, "adapt", 5) == 0) &&
chunksize <= LZMA_WINDOW_MAX) || delta_flag) {
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE;
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK;
ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE;
ctx->rabin_break_patt = RAB_POLYNOMIAL_CONST;
if (memcmp(algo, "lzfx", 4) == 0 || memcmp(algo, "lz4", 3) == 0) {
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE2;
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK2;
ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE2;
ctx->rabin_break_patt = 0;
} else {
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE;
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK;
ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE;
ctx->rabin_break_patt = RAB_POLYNOMIAL_CONST;
}
} else {
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE2;
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK2;