Compute and compare Mean sketch cksum to improve similarity comparison

Fix optflags settings in Makefile
Small optimization in zero RLE encoder to avoid scanning during lookahead
Some minor fixes

Makefile

@@ -64,19 +64,22 @@ CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -D_7ZIP_ST -DNODEF
 	-DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32
 VEC_FLAGS = -ftree-vectorize
 LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block
-GEN_OPT = -O3
-RABIN_OPT = -O2
 LDLIBS = -ldl -lbz2 $(ZLIB_DIR) -lz -lm
 
 ifdef DEBUG
 LINK = g++ -m64 -pthread -msse3
-COMPILE = gcc -m64 -O -g -msse3 -c
+COMPILE = gcc -m64 -g -msse3 -c
-COMPILE_cpp = g++ -m64 -O -g -msse3 -c
+COMPILE_cpp = g++ -m64 -g -msse3 -c
 VEC_FLAGS = 
+LOOP_OPTFLAGS = 
+GEN_OPT = -O
+RABIN_OPT = -O
 ifdef DEBUG_NO_SLAB
 CPPFLAGS += -DDEBUG_NO_SLAB
 endif
 else
+GEN_OPT = -O3
+RABIN_OPT = -O2
 LINK = g++ -m64 -pthread -msse3
 COMPILE = gcc -m64 -msse3 -c
 COMPILE_cpp = g++ -m64 -msse3 -c

bsdiff/rle_encoder.c

@@ -47,7 +47,7 @@ zero_rle_encode(const void *const ibuf, const unsigned int ilen,
 			*((unsigned short *)(ob + pos2)) = htons(count);
 			pos2 += 2;
 		} else {
-			unsigned int pos3, pos4, cnt, state;
+			unsigned int pos3, pos4, state;
 			pos3 = pos2;
 			pos2 += 2;
 			if (pos2 > *olen) break;
@@ -56,12 +56,12 @@ zero_rle_encode(const void *const ibuf, const unsigned int ilen,
 			for (;pos1<ilen && pos2<*olen && count<COUNT_MAX;) {
 				if (ib[pos1] != 0) state = 0;
 				if (ib[pos1] == 0 && !state) {
-					cnt = 0;
-					pos4 = pos1;
 					state = 1;
 					// Lookahead if there are at least 4 consecutive zeroes
-					for (;pos4<ilen && ib[pos4] == 0; pos4++) cnt++;
+					if (ilen > 3) {
-					if (cnt >= 4) break;
+						pos4 = *((unsigned int *)(ib+pos1));
+						if (!pos4) break;
+					}
 				}
 				ob[pos2++] = ib[pos1++];
 				count++;

rabin/rabin_polynomial.c

@@ -297,7 +297,6 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
 			length >= rabin_polynomial_max_block_size) {
 				last_offset = i+1;
 				length = 0;
-				j = 0;
 			}
 		}
 
@@ -342,7 +341,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
 		splits = (uint32_t *)(&fplist[fpos[1]]);
 #if BYTE_ORDER == BIG_ENDIAN
 		splits[0]++;
-		splits[0] += cur_roll_checksum & ctx->fp_mask;
+		splits[1] += cur_roll_checksum & ctx->fp_mask;
 #else
 		splits[1]++;
 		splits[0] += cur_roll_checksum & ctx->fp_mask;
@@ -362,6 +361,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
 			memset(fplist, 0, fplist_sz);
 			fpos[0] = 0;
 			len1 = 0;
+			j++;
 		}
 		/*
 		 * Window pos has to rotate from 0 .. RAB_POLYNOMIAL_WIN_SIZE-1
@@ -382,6 +382,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
 			ctx->blocks[blknum].ref = 0;
 			ctx->blocks[blknum].similar = 0;
 			ctx->blocks[blknum].cksum_n_offset = cur_sketch;
+			ctx->blocks[blknum].mean_n_length = cur_sketch / j;
 			memset(fplist, 0, fplist_sz);
 			fpos[0] = 0;
 			len1 = 0;
@@ -466,23 +467,33 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
 		}
 
 		if (ctx->delta_flag) {
+			uint64_t prev_mean;
+			prev_mean = 0;
+			prev_cksum = 0;
+			prev_length = 0;
+
 			for (blk = 0; blk < blknum; blk++) {
 				if (ctx->blocks[blk].similar) continue;
 
+				/*
+				 * Compare blocks for similarity.
+				 * Note: Block list by now is sorted by length as well.
+				 */
 				if (blk > 0 && ctx->blocks[blk].ref == 0 &&
 				    ctx->blocks[blk].cksum_n_offset == prev_cksum &&
-				    ctx->blocks[blk].length - prev_length < 512
+				    ctx->blocks[blk].length - prev_length < 512 &&
+				    ctx->blocks[blk].mean_n_length == prev_mean
 				) {
 					ctx->blocks[blk].index = prev_index;
 					ctx->blocks[blk].similar = SIMILAR_PARTIAL;
 					ctx->blocks[prev_blk].ref = 1;
-					matchlen += prev_length/2;
+					matchlen += (prev_length>>1);
 					continue;
 				}
-				prev_offset = buf1 + ctx->blocks[blk].offset;
 				prev_cksum = ctx->blocks[blk].cksum_n_offset;
 				prev_length = ctx->blocks[blk].length;
 				prev_index = ctx->blocks[blk].index;
+				prev_mean = ctx->blocks[blk].mean_n_length;
 				prev_blk = blk;
 			}
 		}
@@ -547,7 +558,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
 				prev_index = 0;
 				prev_length = 0;
 				ctx->blocks[pos].cksum_n_offset = be->offset;
-				ctx->blocks[pos].new_length = be->length;
+				ctx->blocks[pos].mean_n_length = be->length;
 				trans[blk] = pos;
 
 				if (be->similar == SIMILAR_EXACT) {
@@ -589,12 +600,12 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
 					matchlen = ctx->real_chunksize - *size;
 
 					bsz = bsdiff(old, ctx->blocks[j].length, new,
-					    ctx->blocks[blk].new_length, ctx->cbuf + pos1,
+					    ctx->blocks[blk].mean_n_length, ctx->cbuf + pos1,
 					    buf1 + *size, matchlen);
 					if (bsz == 0) {
-						memcpy(ctx->cbuf + pos1, new, ctx->blocks[blk].new_length);
+						memcpy(ctx->cbuf + pos1, new, ctx->blocks[blk].mean_n_length);
-						rabin_index[blk] = htonl(ctx->blocks[blk].new_length);
+						rabin_index[blk] = htonl(ctx->blocks[blk].mean_n_length);
-						pos1 += ctx->blocks[blk].new_length;
+						pos1 += ctx->blocks[blk].mean_n_length;
 					} else {
 						rabin_index[blk] = htonl(trans[i] |
 						    RABIN_INDEX_FLAG | SET_SIMILARITY_FLAG);

rabin/rabin_polynomial.h

@@ -133,9 +133,9 @@
 typedef struct {
 	ssize_t offset;
 	uint64_t cksum_n_offset; // Dual purpose variable
+	uint64_t mean_n_length;  // Dual purpose variable
 	unsigned int index;
 	unsigned int length;
-	unsigned int new_length;
 	unsigned char ref, similar;
 } rabin_blockentry_t;