Check harder with more strides in Delta2 for extreme compression levels.
This commit is contained in:
parent
bb08b24989
commit
a851bac247
4 changed files with 24 additions and 11 deletions
|
@ -94,12 +94,12 @@
|
|||
* upto 8 bytes (uint64_t) are supported and common type lengths only
|
||||
* are checked.
|
||||
*/
|
||||
#define NSTRIDES 3
|
||||
static uchar_t strides[NSTRIDES] = {2, 4, 8};
|
||||
#define NSTRIDES NSTRIDES_EXTRA
|
||||
static uchar_t strides[NSTRIDES] = {2, 4, 8, 3, 5, 6, 7};
|
||||
|
||||
|
||||
static int delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen,
|
||||
int rle_thresh, int last_encode, int *hdr_ovr);
|
||||
int rle_thresh, int last_encode, int *hdr_ovr, int nstrides);
|
||||
|
||||
/*
|
||||
* Perform Delta2 encoding of the given data buffer in src. Delta Encoding
|
||||
|
@ -111,13 +111,14 @@ static int delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint6
|
|||
* buffer then delta encoding is aborted.
|
||||
*/
|
||||
int
|
||||
delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh)
|
||||
delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh, int nstrides)
|
||||
{
|
||||
if (srclen > MAX_THRESH) {
|
||||
DEBUG_STAT_EN(fprintf(stderr, "DELTA2: srclen: %" PRIu64 " is too big.\n", srclen));
|
||||
return (-1);
|
||||
}
|
||||
|
||||
if (nstrides > NSTRIDES) return (-1);
|
||||
if (srclen <= (MAIN_HDR + LIT_HDR + DELTA_HDR))
|
||||
return (-1);
|
||||
|
||||
|
@ -131,7 +132,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int
|
|||
hdr_ovr = 0;
|
||||
U64_P(dst) = LE64(srclen);
|
||||
dst += MAIN_HDR;
|
||||
rv = delta2_encode_real(src, srclen, dst, dstlen, rle_thresh, 1, &hdr_ovr);
|
||||
rv = delta2_encode_real(src, srclen, dst, dstlen, rle_thresh, 1, &hdr_ovr, nstrides);
|
||||
if (rv == -1)
|
||||
return (rv);
|
||||
*dstlen += MAIN_HDR;
|
||||
|
@ -165,7 +166,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int
|
|||
}
|
||||
dsz = sz;
|
||||
rem = delta2_encode_real(srcpos, sz, dstpos, &dsz, rle_thresh, lenc,
|
||||
&hdr_ovr);
|
||||
&hdr_ovr, nstrides);
|
||||
if (rem == -1) {
|
||||
if (pending == 0) {
|
||||
lastdst = dstpos;
|
||||
|
@ -217,7 +218,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int
|
|||
*/
|
||||
static int
|
||||
delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen,
|
||||
int rle_thresh, int last_encode, int *hdr_ovr)
|
||||
int rle_thresh, int last_encode, int *hdr_ovr, int nstrides)
|
||||
{
|
||||
uint64_t snum, gtot1, gtot2, tot;
|
||||
uint64_t cnt, val, sval;
|
||||
|
@ -233,7 +234,7 @@ delta2_encode_real(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen
|
|||
/*
|
||||
* Estimate which stride length gives the max reduction given rle_thresh.
|
||||
*/
|
||||
for (st = 0; st < NSTRIDES; st++) {
|
||||
for (st = 0; st < nstrides; st++) {
|
||||
int gt;
|
||||
|
||||
snum = 0;
|
||||
|
@ -479,6 +480,10 @@ delta2_decode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen)
|
|||
|
||||
} else {
|
||||
stride = flags;
|
||||
if (stride > STRIDE_MAX) {
|
||||
log_msg(LOG_ERR, 0, "DELTA2 Decode(delta): Invalid stride length: %d. Corrupt data.\n", stride);
|
||||
return (-1);
|
||||
}
|
||||
rcnt = val & MSB_SETZERO_MASK;
|
||||
pos += sizeof (rcnt);
|
||||
sval = LE64(U64_P(pos));
|
||||
|
|
|
@ -34,7 +34,12 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
int delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh);
|
||||
#define NSTRIDES_STANDARD 3
|
||||
#define NSTRIDES_EXTRA 7
|
||||
#define STRIDE_MAX 8
|
||||
#define STRIDE_MIN 2
|
||||
|
||||
int delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int rle_thresh, int nstrides);
|
||||
int delta2_decode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen);
|
||||
|
||||
#define ULL_MAX (18446744073709551615ULL)
|
||||
|
|
|
@ -261,7 +261,7 @@ preproc_compress(pc_ctx_t *pctx, compress_func_ptr cmp_func, void *src, uint64_t
|
|||
stype != TYPE_TIFF && stype != TYPE_MP4) {
|
||||
_dstlen = fromlen;
|
||||
result = delta2_encode((uchar_t *)from, fromlen, to,
|
||||
&_dstlen, props->delta2_span);
|
||||
&_dstlen, props->delta2_span, pctx->delta2_nstrides);
|
||||
if (result != -1) {
|
||||
uchar_t *tmp;
|
||||
tmp = from;
|
||||
|
@ -2760,6 +2760,7 @@ create_pc_context(void)
|
|||
ctx->archive_temp_fd = -1;
|
||||
ctx->pagesize = sysconf(_SC_PAGE_SIZE);
|
||||
ctx->btype = TYPE_UNKNOWN;
|
||||
ctx->delta2_nstrides = NSTRIDES_STANDARD;
|
||||
|
||||
return (ctx);
|
||||
}
|
||||
|
@ -3272,9 +3273,10 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
|
|||
pctx->enable_rabin_scan = 1;
|
||||
pctx->enable_rabin_split = 1;
|
||||
pctx->rab_blk_size = 2;
|
||||
if (pctx->level > 4) pctx->rab_blk_size = 1;
|
||||
if (pctx->level > 5) pctx->rab_blk_size = 1;
|
||||
if (pctx->level > 8) pctx->rab_blk_size = 0;
|
||||
}
|
||||
if (pctx->level > 9) pctx->delta2_nstrides = NSTRIDES_EXTRA;
|
||||
}
|
||||
if (pctx->lzp_preprocess || pctx->enable_delta2_encode || pctx->dispack_preprocess) {
|
||||
pctx->preprocess_mode = 1;
|
||||
|
|
|
@ -202,6 +202,7 @@ typedef struct pc_ctx {
|
|||
int enable_rabin_global;
|
||||
int enable_delta_encode;
|
||||
int enable_delta2_encode;
|
||||
int delta2_nstrides;
|
||||
int enable_rabin_split;
|
||||
int enable_fixed_scan;
|
||||
int preprocess_mode;
|
||||
|
|
Loading…
Reference in a new issue