From f2ffcad2fde41b60a9796b6a9b5bdbaa9375abdd Mon Sep 17 00:00:00 2001 From: Moinak Ghosh Date: Thu, 9 Aug 2012 23:57:24 +0530 Subject: [PATCH] Compute and compare Mean sketch cksum to improve similarity comparison Fix optflags settings in Makefile Small optimization in zero RLE encoder to avoid scanning during lookahead Some minor fixes --- Makefile | 11 +++++++---- bsdiff/rle_encoder.c | 10 +++++----- rabin/rabin_polynomial.c | 31 +++++++++++++++++++++---------- rabin/rabin_polynomial.h | 2 +- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index d1bceac..88fdf83 100644 --- a/Makefile +++ b/Makefile @@ -64,19 +64,22 @@ CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -D_7ZIP_ST -DNODEF -DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32 VEC_FLAGS = -ftree-vectorize LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block -GEN_OPT = -O3 -RABIN_OPT = -O2 LDLIBS = -ldl -lbz2 $(ZLIB_DIR) -lz -lm ifdef DEBUG LINK = g++ -m64 -pthread -msse3 -COMPILE = gcc -m64 -O -g -msse3 -c -COMPILE_cpp = g++ -m64 -O -g -msse3 -c +COMPILE = gcc -m64 -g -msse3 -c +COMPILE_cpp = g++ -m64 -g -msse3 -c VEC_FLAGS = +LOOP_OPTFLAGS = +GEN_OPT = -O +RABIN_OPT = -O ifdef DEBUG_NO_SLAB CPPFLAGS += -DDEBUG_NO_SLAB endif else +GEN_OPT = -O3 +RABIN_OPT = -O2 LINK = g++ -m64 -pthread -msse3 COMPILE = gcc -m64 -msse3 -c COMPILE_cpp = g++ -m64 -msse3 -c diff --git a/bsdiff/rle_encoder.c b/bsdiff/rle_encoder.c index 24f581e..70905aa 100644 --- a/bsdiff/rle_encoder.c +++ b/bsdiff/rle_encoder.c @@ -47,7 +47,7 @@ zero_rle_encode(const void *const ibuf, const unsigned int ilen, *((unsigned short *)(ob + pos2)) = htons(count); pos2 += 2; } else { - unsigned int pos3, pos4, cnt, state; + unsigned int pos3, pos4, state; pos3 = pos2; pos2 += 2; if (pos2 > *olen) break; @@ -56,12 +56,12 @@ zero_rle_encode(const void *const ibuf, const unsigned int ilen, for (;pos1= 4) break; + if (ilen > 3) { + pos4 = *((unsigned int *)(ib+pos1)); + if (!pos4) break; + } } ob[pos2++] = ib[pos1++]; count++; diff --git a/rabin/rabin_polynomial.c b/rabin/rabin_polynomial.c index ebebd9c..2d3b6ce 100755 --- a/rabin/rabin_polynomial.c +++ b/rabin/rabin_polynomial.c @@ -297,7 +297,6 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s length >= rabin_polynomial_max_block_size) { last_offset = i+1; length = 0; - j = 0; } } @@ -342,7 +341,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s splits = (uint32_t *)(&fplist[fpos[1]]); #if BYTE_ORDER == BIG_ENDIAN splits[0]++; - splits[0] += cur_roll_checksum & ctx->fp_mask; + splits[1] += cur_roll_checksum & ctx->fp_mask; #else splits[1]++; splits[0] += cur_roll_checksum & ctx->fp_mask; @@ -362,6 +361,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s memset(fplist, 0, fplist_sz); fpos[0] = 0; len1 = 0; + j++; } /* * Window pos has to rotate from 0 .. RAB_POLYNOMIAL_WIN_SIZE-1 @@ -382,6 +382,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s ctx->blocks[blknum].ref = 0; ctx->blocks[blknum].similar = 0; ctx->blocks[blknum].cksum_n_offset = cur_sketch; + ctx->blocks[blknum].mean_n_length = cur_sketch / j; memset(fplist, 0, fplist_sz); fpos[0] = 0; len1 = 0; @@ -466,23 +467,33 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s } if (ctx->delta_flag) { + uint64_t prev_mean; + prev_mean = 0; + prev_cksum = 0; + prev_length = 0; + for (blk = 0; blk < blknum; blk++) { if (ctx->blocks[blk].similar) continue; + /* + * Compare blocks for similarity. + * Note: Block list by now is sorted by length as well. + */ if (blk > 0 && ctx->blocks[blk].ref == 0 && ctx->blocks[blk].cksum_n_offset == prev_cksum && - ctx->blocks[blk].length - prev_length < 512 + ctx->blocks[blk].length - prev_length < 512 && + ctx->blocks[blk].mean_n_length == prev_mean ) { ctx->blocks[blk].index = prev_index; ctx->blocks[blk].similar = SIMILAR_PARTIAL; ctx->blocks[prev_blk].ref = 1; - matchlen += prev_length/2; + matchlen += (prev_length>>1); continue; } - prev_offset = buf1 + ctx->blocks[blk].offset; prev_cksum = ctx->blocks[blk].cksum_n_offset; prev_length = ctx->blocks[blk].length; prev_index = ctx->blocks[blk].index; + prev_mean = ctx->blocks[blk].mean_n_length; prev_blk = blk; } } @@ -547,7 +558,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s prev_index = 0; prev_length = 0; ctx->blocks[pos].cksum_n_offset = be->offset; - ctx->blocks[pos].new_length = be->length; + ctx->blocks[pos].mean_n_length = be->length; trans[blk] = pos; if (be->similar == SIMILAR_EXACT) { @@ -589,12 +600,12 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s matchlen = ctx->real_chunksize - *size; bsz = bsdiff(old, ctx->blocks[j].length, new, - ctx->blocks[blk].new_length, ctx->cbuf + pos1, + ctx->blocks[blk].mean_n_length, ctx->cbuf + pos1, buf1 + *size, matchlen); if (bsz == 0) { - memcpy(ctx->cbuf + pos1, new, ctx->blocks[blk].new_length); - rabin_index[blk] = htonl(ctx->blocks[blk].new_length); - pos1 += ctx->blocks[blk].new_length; + memcpy(ctx->cbuf + pos1, new, ctx->blocks[blk].mean_n_length); + rabin_index[blk] = htonl(ctx->blocks[blk].mean_n_length); + pos1 += ctx->blocks[blk].mean_n_length; } else { rabin_index[blk] = htonl(trans[i] | RABIN_INDEX_FLAG | SET_SIMILARITY_FLAG); diff --git a/rabin/rabin_polynomial.h b/rabin/rabin_polynomial.h index bad5d9e..dc15984 100644 --- a/rabin/rabin_polynomial.h +++ b/rabin/rabin_polynomial.h @@ -133,9 +133,9 @@ typedef struct { ssize_t offset; uint64_t cksum_n_offset; // Dual purpose variable + uint64_t mean_n_length; // Dual purpose variable unsigned int index; unsigned int length; - unsigned int new_length; unsigned char ref, similar; } rabin_blockentry_t;