greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Check harder with more strides in Delta2 for extreme compression levels.

Browse source
This commit is contained in:
Moinak Ghosh 2013-12-13 19:53:14 +05:30
parent bb08b24989
commit a851bac247
4 changed files with 24 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
filters/delta2/delta2.c
View file

@ -94,12 +94,12 @@
* upto 8 bytes (uint64_t) are supported and common type lengths only * upto 8 bytes (uint64_t) are supported and common type lengths only
* are checked. * are checked.
*/ */
#define NSTRIDES 3 #define NSTRIDES NSTRIDES_EXTRA
static uchar_t strides[NSTRIDES] = {2, 4, 8}; static uchar_t strides[NSTRIDES] = {2, 4, 8, 3, 5, 6, 7};
static int delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, static int delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen,
int rle_thresh, int last_encode, int *hdr_ovr); int rle_thresh, int last_encode, int *hdr_ovr, int nstrides);
/* /*
* Perform Delta2 encoding of the given data buffer in src. Delta Encoding * Perform Delta2 encoding of the given data buffer in src. Delta Encoding
@ -111,13 +111,14 @@ static int delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint6
* buffer then delta encoding is aborted. * buffer then delta encoding is aborted.
*/ */
int int
delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh) delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh, int nstrides)
{ {
if (srclen > MAX_THRESH) { if (srclen > MAX_THRESH) {
DEBUG_STAT_EN(fprintf(stderr, "DELTA2: srclen: %" PRIu64 " is too big.\n", srclen)); DEBUG_STAT_EN(fprintf(stderr, "DELTA2: srclen: %" PRIu64 " is too big.\n", srclen));
return (-1); return (-1);
} }
if (nstrides > NSTRIDES) return (-1);
if (srclen <= (MAIN_HDR + LIT_HDR + DELTA_HDR)) if (srclen <= (MAIN_HDR + LIT_HDR + DELTA_HDR))
return (-1); return (-1);
@ -131,7 +132,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int
hdr_ovr = 0; hdr_ovr = 0;
U64_P(dst) = LE64(srclen); U64_P(dst) = LE64(srclen);
dst += MAIN_HDR; dst += MAIN_HDR;
rv = delta2_encode_real(src, srclen, dst, dstlen, rle_thresh, 1, &hdr_ovr); rv = delta2_encode_real(src, srclen, dst, dstlen, rle_thresh, 1, &hdr_ovr, nstrides);
if (rv == -1) if (rv == -1)
return (rv); return (rv);
*dstlen += MAIN_HDR; *dstlen += MAIN_HDR;
@ -165,7 +166,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int
} }
dsz = sz; dsz = sz;
rem = delta2_encode_real(srcpos, sz, dstpos, &dsz, rle_thresh, lenc, rem = delta2_encode_real(srcpos, sz, dstpos, &dsz, rle_thresh, lenc,
&hdr_ovr); &hdr_ovr, nstrides);
if (rem == -1) { if (rem == -1) {
if (pending == 0) { if (pending == 0) {
lastdst = dstpos; lastdst = dstpos;
@ -217,7 +218,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int
*/ */
static int static int
delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen,
int rle_thresh, int last_encode, int *hdr_ovr) int rle_thresh, int last_encode, int *hdr_ovr, int nstrides)
{ {
uint64_t snum, gtot1, gtot2, tot; uint64_t snum, gtot1, gtot2, tot;
uint64_t cnt, val, sval; uint64_t cnt, val, sval;
@ -233,7 +234,7 @@ delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen
/* /*
* Estimate which stride length gives the max reduction given rle_thresh. * Estimate which stride length gives the max reduction given rle_thresh.
*/ */
for (st = 0; st < NSTRIDES; st++) { for (st = 0; st < nstrides; st++) {
int gt; int gt;
snum = 0; snum = 0;
@ -479,6 +480,10 @@ delta2_decode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen)
} else { } else {
stride = flags; stride = flags;
if (stride > STRIDE_MAX) {
log_msg(LOG_ERR, 0, "DELTA2 Decode(delta): Invalid stride length: %d. Corrupt data.\n", stride);
return (-1);
}
rcnt = val & MSB_SETZERO_MASK; rcnt = val & MSB_SETZERO_MASK;
pos += sizeof (rcnt); pos += sizeof (rcnt);
sval = LE64(U64_P(pos)); sval = LE64(U64_P(pos));

7
filters/delta2/delta2.h
View file

@ -34,7 +34,12 @@
extern "C" { extern "C" {
#endif #endif
int delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh); #define NSTRIDES_STANDARD 3
#define NSTRIDES_EXTRA 7
#define STRIDE_MAX 8
#define STRIDE_MIN 2
int delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh, int nstrides);
int delta2_decode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen); int delta2_decode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen);
#define ULL_MAX (18446744073709551615ULL) #define ULL_MAX (18446744073709551615ULL)

6
pcompress.c
View file

@ -261,7 +261,7 @@ preproc_compress(pc_ctx_t *pctx, compress_func_ptr cmp_func, void *src, uint64_t
stype != TYPE_TIFF && stype != TYPE_MP4) { stype != TYPE_TIFF && stype != TYPE_MP4) {
_dstlen = fromlen; _dstlen = fromlen;
result = delta2_encode((uchar_t *)from, fromlen, to, result = delta2_encode((uchar_t *)from, fromlen, to,
&_dstlen, props->delta2_span); &_dstlen, props->delta2_span, pctx->delta2_nstrides);
if (result != -1) { if (result != -1) {
uchar_t *tmp; uchar_t *tmp;
tmp = from; tmp = from;
@ -2760,6 +2760,7 @@ create_pc_context(void)
ctx->archive_temp_fd = -1; ctx->archive_temp_fd = -1;
ctx->pagesize = sysconf(_SC_PAGE_SIZE); ctx->pagesize = sysconf(_SC_PAGE_SIZE);
ctx->btype = TYPE_UNKNOWN; ctx->btype = TYPE_UNKNOWN;
ctx->delta2_nstrides = NSTRIDES_STANDARD;
return (ctx); return (ctx);
} }
@ -3272,9 +3273,10 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
pctx->enable_rabin_scan = 1; pctx->enable_rabin_scan = 1;
pctx->enable_rabin_split = 1; pctx->enable_rabin_split = 1;
pctx->rab_blk_size = 2; pctx->rab_blk_size = 2;
if (pctx->level > 4) pctx->rab_blk_size = 1; if (pctx->level > 5) pctx->rab_blk_size = 1;
if (pctx->level > 8) pctx->rab_blk_size = 0; if (pctx->level > 8) pctx->rab_blk_size = 0;
} }
if (pctx->level > 9) pctx->delta2_nstrides = NSTRIDES_EXTRA;
} }
if (pctx->lzp_preprocess || pctx->enable_delta2_encode || pctx->dispack_preprocess) { if (pctx->lzp_preprocess || pctx->enable_delta2_encode || pctx->dispack_preprocess) {
pctx->preprocess_mode = 1; pctx->preprocess_mode = 1;

1
pcompress.h
View file

@ -202,6 +202,7 @@ typedef struct pc_ctx {
int enable_rabin_global; int enable_rabin_global;
int enable_delta_encode; int enable_delta_encode;
int enable_delta2_encode; int enable_delta2_encode;
int delta2_nstrides;
int enable_rabin_split; int enable_rabin_split;
int enable_fixed_scan; int enable_fixed_scan;
int preprocess_mode; int preprocess_mode;