Improve Rabin computations using an irreducible polynomial

Slight improvement to similarity computation
A simple mechanism to include DEBUG mode stats
Include stdint for common int types
This commit is contained in:
Moinak Ghosh 2012-08-15 20:13:40 +05:30
parent 3150bdbed7
commit 55d0485d34
4 changed files with 69 additions and 19 deletions

View file

@ -74,9 +74,6 @@ VEC_FLAGS =
LOOP_OPTFLAGS =
GEN_OPT = -O
RABIN_OPT = -O
ifdef DEBUG_NO_SLAB
CPPFLAGS += -DDEBUG_NO_SLAB
endif
else
GEN_OPT = -O3
RABIN_OPT = -O2
@ -84,9 +81,14 @@ LINK = g++ -m64 -pthread -msse3
COMPILE = gcc -m64 -msse3 -c
COMPILE_cpp = g++ -m64 -msse3 -c
CPPFLAGS += -DNDEBUG
endif
ifdef DEBUG_NO_SLAB
CPPFLAGS += -DDEBUG_NO_SLAB
endif
ifdef DEBUG_STATS
CPPFLAGS += -DDEBUG_STATS
endif
all: $(PROG)

View file

@ -64,6 +64,7 @@
#include <string.h>
#include <allocator.h>
#include <utils.h>
#include <pthread.h>
#include "rabin_polynomial.h"
@ -81,6 +82,10 @@ extern int bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new,
uint32_t rabin_polynomial_max_block_size = RAB_POLYNOMIAL_MAX_BLOCK_SIZE;
static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
uint64_t ir[256];
static int inited = 0;
uint32_t
rabin_buf_extra(uint64_t chunksize)
{
@ -96,6 +101,31 @@ create_rabin_context(uint64_t chunksize, uint64_t real_chunksize, const char *al
unsigned char *current_window_data;
uint32_t blknum;
/*
* Pre-compute a table of irreducible polynomial evaluations for each
* possible byte value.
*/
pthread_mutex_lock(&init_lock);
if (!inited) {
int i;
int term;
uint64_t val;
for (blknum = 0; blknum < 256; blknum++) {
term = 1;
val = 0;
for (i=0; i<RAB_POLYNOMIAL_WIN_SIZE; i++) {
if (term & FP_POLY) {
val = (val << 1) + blknum;
}
term <<= 1;
}
ir[blknum] = val;
}
inited = 1;
}
pthread_mutex_unlock(&init_lock);
/*
* Rabin window size must be power of 2 for optimization.
*/
@ -249,10 +279,10 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
uint32_t blknum;
char *buf1 = (char *)buf;
uint32_t length;
uint64_t cur_roll_checksum, cur_sketch;
uint64_t cur_roll_checksum, cur_pos_checksum, cur_sketch;
uint64_t *fplist;
uint32_t len1, fpos[2], cur_sketch2;
uint32_t *charcounts;
uint32_t *charcounts, byts;
if (rabin_pos == NULL) {
/*
@ -292,13 +322,14 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
ctx->current_window_data[ctx->window_pos] = cur_byte;
cur_roll_checksum = (cur_roll_checksum << 1) + cur_byte;
cur_roll_checksum -= (pushed_out << RAB_POLYNOMIAL_WIN_SIZE);
cur_pos_checksum = cur_roll_checksum ^ ir[pushed_out];
ctx->window_pos = (ctx->window_pos + 1) & (RAB_POLYNOMIAL_WIN_SIZE-1);
length++;
if (length < ctx->rabin_poly_min_block_size) continue;
// If we hit our special value or reached the max block size update block offset
if ((cur_roll_checksum & ctx->rabin_avg_block_mask) == ctx->rabin_break_patt ||
if ((cur_pos_checksum & ctx->rabin_avg_block_mask) == ctx->rabin_break_patt ||
length >= rabin_polynomial_max_block_size) {
last_offset = i+1;
length = 0;
@ -326,6 +357,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
*/
cur_roll_checksum = (cur_roll_checksum << 1) + cur_byte;
cur_roll_checksum -= (pushed_out << RAB_POLYNOMIAL_WIN_SIZE);
cur_pos_checksum = cur_roll_checksum ^ ir[pushed_out];
/*
* Compute a super sketch value of the block. We store a sum of relative
@ -345,14 +377,14 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
* http://www.armedia.com/wp/SimilarityIndex.pdf
*/
len1++;
fpos[1] = cur_roll_checksum & ctx->rabin_avg_block_mask;
fpos[1] = cur_pos_checksum & ctx->rabin_avg_block_mask;
splits = (uint32_t *)(&fplist[fpos[1]]);
#if BYTE_ORDER == BIG_ENDIAN
splits[0]++;
splits[1] += cur_roll_checksum & ctx->fp_mask;
splits[1] += cur_pos_checksum & ctx->fp_mask;
#else
splits[1]++;
splits[0] += cur_roll_checksum & ctx->fp_mask;
splits[0] += cur_pos_checksum & ctx->fp_mask;
#endif
charcounts[cur_byte]++;
@ -387,7 +419,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
if (ctx->delta_flag == DELTA_LESS_FUZZY) {
cur_sketch2 += ((p1 << 8) | p2);
} else {
cur_sketch2 = p2;
cur_sketch2 += p2;
}
len1 = 0;
j++;
@ -400,11 +432,10 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
*/
ctx->window_pos = (ctx->window_pos + 1) & (RAB_POLYNOMIAL_WIN_SIZE-1);
length++;
if (length < ctx->rabin_poly_min_block_size) continue;
// If we hit our special value or reached the max block size update block offset
if ((cur_roll_checksum & ctx->rabin_avg_block_mask) == ctx->rabin_break_patt ||
if ((cur_pos_checksum & ctx->rabin_avg_block_mask) == ctx->rabin_break_patt ||
length >= rabin_polynomial_max_block_size) {
ctx->blocks[blknum].offset = last_offset;
ctx->blocks[blknum].index = blknum; // Need to store for sorting
@ -413,7 +444,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
ctx->blocks[blknum].similar = 0;
// Accumulate the 2 sketch values into a combined similarity checksum
ctx->blocks[blknum].cksum_n_offset = cur_sketch + cur_sketch2 / j;
ctx->blocks[blknum].cksum_n_offset = (cur_sketch + cur_sketch2) / 2;
ctx->blocks[blknum].mean_n_length = cur_sketch / j;
memset(fplist, 0, fplist_sz);
fpos[0] = 0;
@ -427,6 +458,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
}
}
DEBUG_STAT_EN(printf("Original size: %lld, blknum: %u\n", *size, blknum));
// If we found at least a few chunks, perform dedup.
if (blknum > 2) {
uint64_t prev_cksum;
@ -436,6 +468,9 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
char *tmp, *prev_offset;
uint32_t *blkarr, *trans, *rabin_index, prev_index, prev_blk;
ssize_t rabin_index_sz;
DEBUG_STAT_EN(uint32_t delta_calls, delta_fails);
DEBUG_STAT_EN(delta_calls = 0);
DEBUG_STAT_EN(delta_fails = 0);
// Insert the last left-over trailing bytes, if any, into a block.
if (last_offset < *size) {
@ -445,8 +480,8 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
ctx->blocks[blknum].ref = 0;
ctx->blocks[blknum].similar = 0;
j = (j > 0 ? j:1);
ctx->blocks[blknum].cksum_n_offset = cur_sketch;
ctx->blocks[blknum].mean_n_length = cur_sketch / j + cur_sketch2 / j;
ctx->blocks[blknum].cksum_n_offset = (cur_sketch + cur_sketch2) / 2;
ctx->blocks[blknum].mean_n_length = cur_sketch / j;
blknum++;
last_offset = *size;
}
@ -633,11 +668,13 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset, s
old = buf1 + ctx->blocks[j].offset;
new = buf1 + ctx->blocks[blk].cksum_n_offset;
matchlen = ctx->real_chunksize - *size;
DEBUG_STAT_EN(delta_calls++);
bsz = bsdiff(old, ctx->blocks[j].length, new,
ctx->blocks[blk].mean_n_length, ctx->cbuf + pos1,
buf1 + *size, matchlen);
if (bsz == 0) {
DEBUG_STAT_EN(delta_fails++);
memcpy(ctx->cbuf + pos1, new, ctx->blocks[blk].mean_n_length);
rabin_index[blk] = htonl(ctx->blocks[blk].mean_n_length);
pos1 += ctx->blocks[blk].mean_n_length;
@ -669,6 +706,8 @@ cont:
entries[2] = htonll(pos1 - rabin_index_sz - RABIN_HDR_SIZE);
*size = pos1;
ctx->valid = 1;
DEBUG_STAT_EN(printf("Deduped size: %lld, blknum: %u, delta_calls: %u, delta_fails: %u\n",
*size, blknum, delta_calls, delta_fails));
/*
* Remaining header entries: size of compressed index and size of

View file

@ -132,6 +132,12 @@
#define DELTA_FUZZY 1
#define DELTA_LESS_FUZZY 2
/*
* Irreducible polynomial for Rabin modulus. This value is from the
* Low Bandwidth Filesystem.
*/
#define FP_POLY 0xbfe6b8a5bf378d83ULL
typedef struct {
ssize_t offset;
uint64_t cksum_n_offset; // Dual purpose variable

11
utils.h
View file

@ -26,6 +26,7 @@
#include <arpa/nameser_compat.h>
#include <sys/types.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
@ -35,11 +36,7 @@ extern "C" {
#define DATA_BINARY 2
#if !defined(sun) && !defined(__sun)
#define ulong_t u_long
#define uchar_t u_char
#define uint8_t u_char
#define uint64_t u_int64_t
#define uint32_t u_int32_t
#endif
#if ULONG_MAX == 4294967295UL
@ -98,6 +95,12 @@ typedef ssize_t bsize_t;
#define ISP2(x) ((x != 0) && ((x & (~x + 1)) == x))
#ifdef DEBUG_STATS
#define DEBUG_STAT_EN(...) __VA_ARGS__;
#else
#define DEBUG_STAT_EN(...)
#endif
extern void err_exit(int show_errno, const char *format, ...);
extern const char *get_execname(const char *);
extern int parse_numeric(ssize_t *val, const char *str);