From a851bac247c27566a32e9444cd98f2154a70e1c5 Mon Sep 17 00:00:00 2001 From: Moinak Ghosh Date: Fri, 13 Dec 2013 19:53:14 +0530 Subject: [PATCH] Check harder with more strides in Delta2 for extreme compression levels. --- filters/delta2/delta2.c | 21 +++++++++++++-------- filters/delta2/delta2.h | 7 ++++++- pcompress.c | 6 ++++-- pcompress.h | 1 + 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/filters/delta2/delta2.c b/filters/delta2/delta2.c index 9c9387f..723807e 100644 --- a/filters/delta2/delta2.c +++ b/filters/delta2/delta2.c @@ -94,12 +94,12 @@ * upto 8 bytes (uint64_t) are supported and common type lengths only * are checked. */ -#define NSTRIDES 3 -static uchar_t strides[NSTRIDES] = {2, 4, 8}; +#define NSTRIDES NSTRIDES_EXTRA +static uchar_t strides[NSTRIDES] = {2, 4, 8, 3, 5, 6, 7}; static int delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, - int rle_thresh, int last_encode, int *hdr_ovr); + int rle_thresh, int last_encode, int *hdr_ovr, int nstrides); /* * Perform Delta2 encoding of the given data buffer in src. Delta Encoding @@ -111,13 +111,14 @@ static int delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint6 * buffer then delta encoding is aborted. */ int -delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh) +delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh, int nstrides) { if (srclen > MAX_THRESH) { DEBUG_STAT_EN(fprintf(stderr, "DELTA2: srclen: %" PRIu64 " is too big.\n", srclen)); return (-1); } + if (nstrides > NSTRIDES) return (-1); if (srclen <= (MAIN_HDR + LIT_HDR + DELTA_HDR)) return (-1); @@ -131,7 +132,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int hdr_ovr = 0; U64_P(dst) = LE64(srclen); dst += MAIN_HDR; - rv = delta2_encode_real(src, srclen, dst, dstlen, rle_thresh, 1, &hdr_ovr); + rv = delta2_encode_real(src, srclen, dst, dstlen, rle_thresh, 1, &hdr_ovr, nstrides); if (rv == -1) return (rv); *dstlen += MAIN_HDR; @@ -165,7 +166,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int } dsz = sz; rem = delta2_encode_real(srcpos, sz, dstpos, &dsz, rle_thresh, lenc, - &hdr_ovr); + &hdr_ovr, nstrides); if (rem == -1) { if (pending == 0) { lastdst = dstpos; @@ -217,7 +218,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int */ static int delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, - int rle_thresh, int last_encode, int *hdr_ovr) + int rle_thresh, int last_encode, int *hdr_ovr, int nstrides) { uint64_t snum, gtot1, gtot2, tot; uint64_t cnt, val, sval; @@ -233,7 +234,7 @@ delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen /* * Estimate which stride length gives the max reduction given rle_thresh. */ - for (st = 0; st < NSTRIDES; st++) { + for (st = 0; st < nstrides; st++) { int gt; snum = 0; @@ -479,6 +480,10 @@ delta2_decode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen) } else { stride = flags; + if (stride > STRIDE_MAX) { + log_msg(LOG_ERR, 0, "DELTA2 Decode(delta): Invalid stride length: %d. Corrupt data.\n", stride); + return (-1); + } rcnt = val & MSB_SETZERO_MASK; pos += sizeof (rcnt); sval = LE64(U64_P(pos)); diff --git a/filters/delta2/delta2.h b/filters/delta2/delta2.h index a76eb18..0674403 100644 --- a/filters/delta2/delta2.h +++ b/filters/delta2/delta2.h @@ -34,7 +34,12 @@ extern "C" { #endif -int delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh); +#define NSTRIDES_STANDARD 3 +#define NSTRIDES_EXTRA 7 +#define STRIDE_MAX 8 +#define STRIDE_MIN 2 + +int delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh, int nstrides); int delta2_decode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen); #define ULL_MAX (18446744073709551615ULL) diff --git a/pcompress.c b/pcompress.c index d660e06..ff83e79 100644 --- a/pcompress.c +++ b/pcompress.c @@ -261,7 +261,7 @@ preproc_compress(pc_ctx_t *pctx, compress_func_ptr cmp_func, void *src, uint64_t stype != TYPE_TIFF && stype != TYPE_MP4) { _dstlen = fromlen; result = delta2_encode((uchar_t *)from, fromlen, to, - &_dstlen, props->delta2_span); + &_dstlen, props->delta2_span, pctx->delta2_nstrides); if (result != -1) { uchar_t *tmp; tmp = from; @@ -2760,6 +2760,7 @@ create_pc_context(void) ctx->archive_temp_fd = -1; ctx->pagesize = sysconf(_SC_PAGE_SIZE); ctx->btype = TYPE_UNKNOWN; + ctx->delta2_nstrides = NSTRIDES_STANDARD; return (ctx); } @@ -3272,9 +3273,10 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) pctx->enable_rabin_scan = 1; pctx->enable_rabin_split = 1; pctx->rab_blk_size = 2; - if (pctx->level > 4) pctx->rab_blk_size = 1; + if (pctx->level > 5) pctx->rab_blk_size = 1; if (pctx->level > 8) pctx->rab_blk_size = 0; } + if (pctx->level > 9) pctx->delta2_nstrides = NSTRIDES_EXTRA; } if (pctx->lzp_preprocess || pctx->enable_delta2_encode || pctx->dispack_preprocess) { pctx->preprocess_mode = 1; diff --git a/pcompress.h b/pcompress.h index d35566f..065dac7 100644 --- a/pcompress.h +++ b/pcompress.h @@ -202,6 +202,7 @@ typedef struct pc_ctx { int enable_rabin_global; int enable_delta_encode; int enable_delta2_encode; + int delta2_nstrides; int enable_rabin_split; int enable_fixed_scan; int preprocess_mode;