greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Check harder with more strides in Delta2 for extreme compression levels.

Browse source
This commit is contained in:
Moinak Ghosh 2013-12-13 19:53:14 +05:30
parent bb08b24989
commit a851bac247
4 changed files with 24 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
filters/delta2/delta2.c
View file

@ -94,12 +94,12 @@
* upto 8 bytes (uint64_t) are supported and common type lengths only
* are checked.
*/
#define NSTRIDES 3
static uchar_t strides[NSTRIDES] = {2, 4, 8};
#define NSTRIDES NSTRIDES_EXTRA
static uchar_t strides[NSTRIDES] = {2, 4, 8, 3, 5, 6, 7};
static int delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen,
int rle_thresh, int last_encode, int *hdr_ovr);
int rle_thresh, int last_encode, int *hdr_ovr, int nstrides);
/*
* Perform Delta2 encoding of the given data buffer in src. Delta Encoding
@ -111,13 +111,14 @@ static int delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint6
* buffer then delta encoding is aborted.
*/
int
delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh)
delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh, int nstrides)
{
if (srclen > MAX_THRESH) {
DEBUG_STAT_EN(fprintf(stderr, "DELTA2: srclen: %" PRIu64 " is too big.\n", srclen));
return (-1);
}
if (nstrides > NSTRIDES) return (-1);
if (srclen <= (MAIN_HDR + LIT_HDR + DELTA_HDR))
return (-1);
@ -131,7 +132,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int
hdr_ovr = 0;
U64_P(dst) = LE64(srclen);
dst += MAIN_HDR;
rv = delta2_encode_real(src, srclen, dst, dstlen, rle_thresh, 1, &hdr_ovr);
rv = delta2_encode_real(src, srclen, dst, dstlen, rle_thresh, 1, &hdr_ovr, nstrides);
if (rv == -1)
return (rv);
*dstlen += MAIN_HDR;
@ -165,7 +166,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int
}
dsz = sz;
rem = delta2_encode_real(srcpos, sz, dstpos, &dsz, rle_thresh, lenc,
&hdr_ovr);
&hdr_ovr, nstrides);
if (rem == -1) {
if (pending == 0) {
lastdst = dstpos;
@ -217,7 +218,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int
*/
static int
delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen,
int rle_thresh, int last_encode, int *hdr_ovr)
int rle_thresh, int last_encode, int *hdr_ovr, int nstrides)
{
uint64_t snum, gtot1, gtot2, tot;
uint64_t cnt, val, sval;
@ -233,7 +234,7 @@ delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen
/*
* Estimate which stride length gives the max reduction given rle_thresh.
*/
for (st = 0; st < NSTRIDES; st++) {
for (st = 0; st < nstrides; st++) {
int gt;
snum = 0;
@ -479,6 +480,10 @@ delta2_decode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen)
} else {
stride = flags;
if (stride > STRIDE_MAX) {
log_msg(LOG_ERR, 0, "DELTA2 Decode(delta): Invalid stride length: %d. Corrupt data.\n", stride);
return (-1);
}
rcnt = val & MSB_SETZERO_MASK;
pos += sizeof (rcnt);
sval = LE64(U64_P(pos));

7
filters/delta2/delta2.h
View file

@ -34,7 +34,12 @@
extern "C" {
#endif
int delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh);
#define NSTRIDES_STANDARD 3
#define NSTRIDES_EXTRA 7
#define STRIDE_MAX 8
#define STRIDE_MIN 2
int delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh, int nstrides);
int delta2_decode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen);
#define ULL_MAX (18446744073709551615ULL)

6
pcompress.c
View file

@ -261,7 +261,7 @@ preproc_compress(pc_ctx_t *pctx, compress_func_ptr cmp_func, void *src, uint64_t
stype != TYPE_TIFF && stype != TYPE_MP4) {
_dstlen = fromlen;
result = delta2_encode((uchar_t *)from, fromlen, to,
&_dstlen, props->delta2_span);
&_dstlen, props->delta2_span, pctx->delta2_nstrides);
if (result != -1) {
uchar_t *tmp;
tmp = from;
@ -2760,6 +2760,7 @@ create_pc_context(void)
ctx->archive_temp_fd = -1;
ctx->pagesize = sysconf(_SC_PAGE_SIZE);
ctx->btype = TYPE_UNKNOWN;
ctx->delta2_nstrides = NSTRIDES_STANDARD;
return (ctx);
}
@ -3272,9 +3273,10 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
pctx->enable_rabin_scan = 1;
pctx->enable_rabin_split = 1;
pctx->rab_blk_size = 2;
if (pctx->level > 4) pctx->rab_blk_size = 1;
if (pctx->level > 5) pctx->rab_blk_size = 1;
if (pctx->level > 8) pctx->rab_blk_size = 0;
}
if (pctx->level > 9) pctx->delta2_nstrides = NSTRIDES_EXTRA;
}
if (pctx->lzp_preprocess || pctx->enable_delta2_encode || pctx->dispack_preprocess) {
pctx->preprocess_mode = 1;

1
pcompress.h
View file

@ -202,6 +202,7 @@ typedef struct pc_ctx {
int enable_rabin_global;
int enable_delta_encode;
int enable_delta2_encode;
int delta2_nstrides;
int enable_rabin_split;
int enable_fixed_scan;
int preprocess_mode;