greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Compute and compare Mean sketch cksum to improve similarity comparison

Browse source 
Fix optflags settings in Makefile
Small optimization in zero RLE encoder to avoid scanning during lookahead
Some minor fixes
This commit is contained in:
Moinak Ghosh 2012-08-09 23:57:24 +05:30
parent 400d0bfa72
commit f2ffcad2fd
4 changed files with 34 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
Makefile
View file

@ -64,19 +64,22 @@ CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -D_7ZIP_ST -DNODEF
-DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32
VEC_FLAGS = -ftree-vectorize
LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block
GEN_OPT = -O3
RABIN_OPT = -O2
LDLIBS = -ldl -lbz2 $(ZLIB_DIR) -lz -lm
ifdef DEBUG
LINK = g++ -m64 -pthread -msse3
COMPILE = gcc -m64 -O -g -msse3 -c
COMPILE_cpp = g++ -m64 -O -g -msse3 -c
COMPILE = gcc -m64 -g -msse3 -c
COMPILE_cpp = g++ -m64 -g -msse3 -c
VEC_FLAGS =
LOOP_OPTFLAGS =
GEN_OPT = -O
RABIN_OPT = -O
ifdef DEBUG_NO_SLAB
CPPFLAGS += -DDEBUG_NO_SLAB
endif
else
GEN_OPT = -O3
RABIN_OPT = -O2
LINK = g++ -m64 -pthread -msse3
COMPILE = gcc -m64 -msse3 -c
COMPILE_cpp = g++ -m64 -msse3 -c

10
bsdiff/rle_encoder.c
View file

@ -47,7 +47,7 @@ zero_rle_encode(const void *const ibuf, const unsigned int ilen,
*((unsigned short *)(ob + pos2)) = htons(count);
pos2 += 2;
} else {
unsigned int pos3, pos4, cnt, state;
unsigned int pos3, pos4, state;
pos3 = pos2;
pos2 += 2;
if (pos2 > *olen) break;
@ -56,12 +56,12 @@ zero_rle_encode(const void *const ibuf, const unsigned int ilen,
for (;pos1<ilen && pos2<*olen && count<COUNT_MAX;) {
if (ib[pos1] != 0) state = 0;
if (ib[pos1] == 0 && !state) {
cnt = 0;
pos4 = pos1;
state = 1;
// Lookahead if there are at least 4 consecutive zeroes
for (;pos4<ilen && ib[pos4] == 0; pos4++) cnt++;
if (cnt >= 4) break;
if (ilen > 3) {
pos4 = *((unsigned int *)(ib+pos1));
if (!pos4) break;
}
}
ob[pos2++] = ib[pos1++];
count++;

31
rabin/rabin_polynomial.c
View file

@ -297,7 +297,6 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
length >= rabin_polynomial_max_block_size) {
last_offset = i+1;
length = 0;
j = 0;
}
}
@ -342,7 +341,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
splits = (uint32_t *)(&fplist[fpos[1]]);
#if BYTE_ORDER == BIG_ENDIAN
splits[0]++;
splits[0] += cur_roll_checksum & ctx->fp_mask;
splits[1] += cur_roll_checksum & ctx->fp_mask;
#else
splits[1]++;
splits[0] += cur_roll_checksum & ctx->fp_mask;
@ -362,6 +361,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
memset(fplist, 0, fplist_sz);
fpos[0] = 0;
len1 = 0;
j++;
}
/*
* Window pos has to rotate from 0 .. RAB_POLYNOMIAL_WIN_SIZE-1
@ -382,6 +382,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
ctx->blocks[blknum].ref = 0;
ctx->blocks[blknum].similar = 0;
ctx->blocks[blknum].cksum_n_offset = cur_sketch;
ctx->blocks[blknum].mean_n_length = cur_sketch / j;
memset(fplist, 0, fplist_sz);
fpos[0] = 0;
len1 = 0;
@ -466,23 +467,33 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
}
if (ctx->delta_flag) {
uint64_t prev_mean;
prev_mean = 0;
prev_cksum = 0;
prev_length = 0;
for (blk = 0; blk < blknum; blk++) {
if (ctx->blocks[blk].similar) continue;
/*
* Compare blocks for similarity.
* Note: Block list by now is sorted by length as well.
*/
if (blk > 0 && ctx->blocks[blk].ref == 0 &&
ctx->blocks[blk].cksum_n_offset == prev_cksum &&
ctx->blocks[blk].length - prev_length < 512
ctx->blocks[blk].length - prev_length < 512 &&
ctx->blocks[blk].mean_n_length == prev_mean
) {
ctx->blocks[blk].index = prev_index;
ctx->blocks[blk].similar = SIMILAR_PARTIAL;
ctx->blocks[prev_blk].ref = 1;
matchlen += prev_length/2;
matchlen += (prev_length>>1);
continue;
}
prev_offset = buf1 + ctx->blocks[blk].offset;
prev_cksum = ctx->blocks[blk].cksum_n_offset;
prev_length = ctx->blocks[blk].length;
prev_index = ctx->blocks[blk].index;
prev_mean = ctx->blocks[blk].mean_n_length;
prev_blk = blk;
}
}
@ -547,7 +558,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
prev_index = 0;
prev_length = 0;
ctx->blocks[pos].cksum_n_offset = be->offset;
ctx->blocks[pos].new_length = be->length;
ctx->blocks[pos].mean_n_length = be->length;
trans[blk] = pos;
if (be->similar == SIMILAR_EXACT) {
@ -589,12 +600,12 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
matchlen = ctx->real_chunksize - *size;
bsz = bsdiff(old, ctx->blocks[j].length, new,
ctx->blocks[blk].new_length, ctx->cbuf + pos1,
ctx->blocks[blk].mean_n_length, ctx->cbuf + pos1,
buf1 + *size, matchlen);
if (bsz == 0) {
memcpy(ctx->cbuf + pos1, new, ctx->blocks[blk].new_length);
rabin_index[blk] = htonl(ctx->blocks[blk].new_length);
pos1 += ctx->blocks[blk].new_length;
memcpy(ctx->cbuf + pos1, new, ctx->blocks[blk].mean_n_length);
rabin_index[blk] = htonl(ctx->blocks[blk].mean_n_length);
pos1 += ctx->blocks[blk].mean_n_length;
} else {
rabin_index[blk] = htonl(trans[i] |
RABIN_INDEX_FLAG | SET_SIMILARITY_FLAG);

2
rabin/rabin_polynomial.h
View file

@ -133,9 +133,9 @@
typedef struct {
ssize_t offset;
uint64_t cksum_n_offset; // Dual purpose variable
uint64_t mean_n_length; // Dual purpose variable
unsigned int index;
unsigned int length;
unsigned int new_length;
unsigned char ref, similar;
} rabin_blockentry_t;