Make LZFX Hash size dynamic.

Use smaller min rabin block when using fast compression algos.
Add missing check for algo init function return value.

allocator.h

@@ -24,6 +24,8 @@
 #ifndef __ALLOCATOR_H__
 #define __ALLOCATOR_H__
 
+#include <sys/types.h>
+
 void slab_init();
 void slab_cleanup(int quiet);
 void *slab_alloc(void *p, size_t size);

lzfx/lzfx.c

@@ -56,8 +56,7 @@
  */
 
 #include "lzfx.h"
-
+#include <allocator.h>
-#define LZFX_HSIZE (1 << (LZFX_HLOG))
 
 /* We need this for memset */
 #ifdef __cplusplus
@@ -74,13 +73,10 @@
 # define fx_expect_true(expr)   (expr)
 #endif
 
-typedef unsigned char u8;
-typedef const u8 *LZSTATE[LZFX_HSIZE];
-
 /* Define the hash function */
 #define LZFX_FRST(p)     (((p[0]) << 8) | p[1])
 #define LZFX_NEXT(v,p)   (((v) << 8) | p[2])
-#define LZFX_IDX(h)      ((( h >> (3*8 - LZFX_HLOG)) - h  ) & (LZFX_HSIZE - 1))
+#define LZFX_IDX(h, bits)      ((( h >> (3*8 - bits)) - h  ) & (LZFX_HTAB_SIZE(bits) - 1))
 
 /* These cannot be changed, as they are related to the compressed format. */
 #define LZFX_MAX_LIT        (1 <<  5)
@@ -110,11 +106,12 @@ int lzfx_getsize(const void* ibuf, unsigned int ilen, unsigned int *olen);
 */
 #include <stdio.h>
 int lzfx_compress(const void *const ibuf, const unsigned int ilen,
-                              void *obuf, unsigned int *const olen){
+                              void *obuf, unsigned int *const olen,
+                              unsigned int htab_bits){
 
     /* Hash table; an array of u8*'s which point
        to various locations in the input buffer */
-    const u8 *htab[LZFX_HSIZE];
+    const u8 **htab;
 
     const u8 **hslot;       /* Pointer to entry in hash table */
     unsigned int hval;      /* Hash value generated by macros above */
@@ -145,7 +142,8 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
         return lzfx_getsize(ibuf, ilen, olen);
     }
 
-    memset(htab, 0, sizeof(htab));
+    htab = (const u8 **)slab_calloc(NULL, LZFX_HTAB_SIZE(htab_bits), sizeof (u8*));
+    if (htab == NULL) return LZFX_ENOMEM;
 
     /*  Start a literal run.  Whenever we do this the output pointer is
         advanced because the current byte will hold the encoded length. */
@@ -156,7 +154,7 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
     while(ip + 2 < in_end){   /* The NEXT macro reads 2 bytes ahead */
 
         hval = LZFX_NEXT(hval, ip);
-        hslot = htab + LZFX_IDX(hval);
+        hslot = htab + LZFX_IDX(hval, htab_bits);
 
         ref = *hslot; *hslot = ip;
 
@@ -174,8 +172,10 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
 
             /* lit == 0:  op + 3 must be < out_end (because we undo the run)
                lit != 0:  op + 3 + 1 must be < out_end */
-            if(fx_expect_false(op - !lit + 3 + 1 >= out_end))
+            if(fx_expect_false(op - !lit + 3 + 1 >= out_end)) {
+                slab_free(NULL, htab);
                 return LZFX_ESIZE;
+	    }
             
             op [- lit - 1] = lit - 1; /* Terminate literal run */
             op -= !lit;               /* Undo run if length is zero */
@@ -209,14 +209,17 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
 
             hval = LZFX_FRST (ip);
             hval = LZFX_NEXT (hval, ip);
-            htab[LZFX_IDX (hval)] = ip;
+            htab[LZFX_IDX (hval, htab_bits)] = ip;
 
             ip++;   /* ip = initial ip + #octets */
 
         } else {
               /* Keep copying literal bytes */
 
-              if (fx_expect_false (op >= out_end)) return LZFX_ESIZE;
+              if (fx_expect_false (op >= out_end)) {
+                  slab_free(NULL, htab);
+                  return LZFX_ESIZE;
+	      }
 
               lit++; *op++ = *ip++;
 
@@ -231,7 +234,10 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
 
     /*  At most 3 bytes remain in input.  We therefore need 4 bytes available
         in the output buffer to store them (3 data + ctrl byte).*/
-    if (op + 3 > out_end) return LZFX_ESIZE;
+    if (op + 3 > out_end) {
+        slab_free(NULL, htab);
+        return LZFX_ESIZE;
+    }
 
     while (ip < in_end) {
 
@@ -247,6 +253,7 @@ int lzfx_compress(const void *const ibuf, const unsigned int ilen,
     op -= !lit;
 
     *olen = op - (u8 *)obuf;
+    slab_free(NULL, htab);
     return 0;
 }
 

lzfx/lzfx.h

@@ -72,15 +72,15 @@ extern "C" {
 #define LZFX_VERSION_MINOR      1
 #define LZFX_VERSION_STRING     "0.1"
 
-/* Hashtable size (2**LZFX_HLOG entries) */
+#define LZFX_HTAB_SIZE(x) (1 << x)
-#ifndef LZFX_HLOG
-# define LZFX_HLOG 16
-#endif
 
 /* Predefined errors. */
 #define LZFX_ESIZE      -1      /* Output buffer too small */
 #define LZFX_ECORRUPT   -2      /* Invalid data for decompression */
 #define LZFX_EARGS      -3      /* Arguments invalid (NULL) */
+#define LZFX_ENOMEM     -4      /* Out of memory when allocating hashtable */
+
+typedef unsigned char u8;
 
 /*  Buffer-to buffer compression.
 
@@ -92,7 +92,8 @@ extern "C" {
     value is returned and olen is not modified.
 */
 int lzfx_compress(const void* ibuf, unsigned int ilen,
-                        void* obuf, unsigned int *olen);
+                        void* obuf, unsigned int *olen,
+                        unsigned int htab_bits);
 
 /*  Buffer-to-buffer decompression.
 

lzfx_compress.c

@@ -28,6 +28,11 @@
 #include <utils.h>
 #include <pcompress.h>
 #include <lzfx.h>
+#include <allocator.h>
+
+struct lzfx_params {
+	uint32_t htab_bits;
+};
 
 void
 lz_fx_stats(int show)
@@ -37,11 +42,33 @@ lz_fx_stats(int show)
 int
 lz_fx_init(void **data, int *level, ssize_t chunksize)
 {
-	if (*level > 9) *level = 9;
+	struct lzfx_params *lzdat;
+	int lev;
+
 	if (chunksize > UINT_MAX) {
 		fprintf(stderr, "Chunk size too big for LZFX.\n");
 		return (1);
 	}
+	lzdat = slab_alloc(NULL, sizeof (struct lzfx_params));
+
+	lev = *level;
+	if (lev > 5) lev = 5;
+	lzdat->htab_bits = 16 + (lev-1);
+	*data = lzdat;
+
+	if (*level > 9) *level = 9;
+	return (0);
+}
+
+int
+lz_fx_deinit(void **data)
+{
+	struct lzfx_params *lzdat = (struct lzfx_params *)(*data);
+	
+	if (lzdat) {
+		slab_free(NULL, lzdat);
+	}
+	*data = NULL;
 	return (0);
 }
 
@@ -58,6 +85,9 @@ lz_fx_err(int err)
 	    case LZFX_EARGS:
 		fprintf(stderr, "LZFX: Invalid arguments.\n");
 		break;
+	    case LZFX_ENOMEM:
+		fprintf(stderr, "LZFX: Out of memory when allocating hashtable.\n");
+		break;
 	    default:
 		fprintf(stderr, "LZFX: Unknown error code: %d\n", err);
 	}
@@ -68,10 +98,11 @@ lz_fx_compress(void *src, size_t srclen, void *dst, size_t *dstlen,
 	       int level, uchar_t chdr, void *data)
 {
 	int rv;
+	struct lzfx_params *lzdat = (struct lzfx_params *)data;
 	unsigned int _srclen = srclen;
 	unsigned int _dstlen = *dstlen;
 
-	rv = lzfx_compress(src, _srclen, dst, &_dstlen);
+	rv = lzfx_compress(src, _srclen, dst, &_dstlen, lzdat->htab_bits);
 	if (rv == -1) {
 		lz_fx_err(rv);
 		return (-1);

main.c

@@ -442,8 +442,11 @@ start_decompress(const char *filename, const char *to_filename)
 		sem_init(&(tdat->start_sem), 0, 0);
 		sem_init(&(tdat->cmp_done_sem), 0, 0);
 		sem_init(&(tdat->write_done_sem), 0, 1);
-		if (_init_func)
+		if (_init_func) {
-			_init_func(&(tdat->data), &(tdat->level), chunksize);
+			if (_init_func(&(tdat->data), &(tdat->level), chunksize) != 0) {
+				UNCOMP_BAIL;
+			}
+		}
 		if (enable_rabin_scan)
 			tdat->rctx = create_rabin_context(chunksize, compressed_chunksize,
 			    algo, enable_delta_encode);
@@ -918,8 +921,11 @@ start_compress(const char *filename, uint64_t chunksize, int level)
 		sem_init(&(tdat->start_sem), 0, 0);
 		sem_init(&(tdat->cmp_done_sem), 0, 0);
 		sem_init(&(tdat->write_done_sem), 0, 1);
-		if (_init_func)
+		if (_init_func) {
-			_init_func(&(tdat->data), &(tdat->level), chunksize);
+			if (_init_func(&(tdat->data), &(tdat->level), chunksize) != 0) {
+				COMP_BAIL;
+			}
+		}
 		if (enable_rabin_scan)
 			tdat->rctx = create_rabin_context(chunksize, compressed_chunksize,
 			    algo, enable_delta_encode);
@@ -1189,7 +1195,7 @@ init_algo(const char *algo, int bail)
 		_compress_func = lz_fx_compress;
 		_decompress_func = lz_fx_decompress;
 		_init_func = lz_fx_init;
-		_deinit_func = NULL;
+		_deinit_func = lz_fx_deinit;
 		_stats_func = lz_fx_stats;
 		rv = 0;
 

pcompress.h

@@ -92,6 +92,7 @@ extern int lz_fx_init(void **data, int *level, ssize_t chunksize);
 extern int adapt_deinit(void **data);
 extern int lzma_deinit(void **data);
 extern int ppmd_deinit(void **data);
+extern int lz_fx_deinit(void **data);
 
 extern void adapt_stats(int show);
 extern void ppmd_stats(int show);

rabin/rabin_polynomial.c

@@ -111,12 +111,20 @@ create_rabin_context(uint64_t chunksize, uint64_t real_chunksize, const char *al
 	 */
 	ctx = (rabin_context_t *)slab_alloc(NULL, sizeof (rabin_context_t));
 	ctx->rabin_poly_max_block_size = RAB_POLYNOMIAL_MAX_BLOCK_SIZE;
+
 	if (((memcmp(algo, "lzma", 4) == 0 || memcmp(algo, "adapt", 5) == 0) &&
 	      chunksize <= LZMA_WINDOW_MAX) || delta_flag) {
+		if (memcmp(algo, "lzfx", 4) == 0 || memcmp(algo, "lz4", 3) == 0) {
+			ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE2;
+			ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK2;
+			ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE2;
+			ctx->rabin_break_patt = 0;
+		} else {
 			ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE;
 			ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK;
 			ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE;
 			ctx->rabin_break_patt = RAB_POLYNOMIAL_CONST;
+		}
 	} else {
 		ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE2;
 		ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK2;