Compare commits

...

7 commits

Author SHA1 Message Date
Moinak Ghosh
043cdfc05c Fix Dedupe Mode initialization. 2013-07-12 18:23:31 +05:30
Moinak Ghosh
ae3ba0858c Avoid CRC64 for Similarity IDs when using 256-bit hash. 2013-06-29 19:42:45 +05:30
Moinak Ghosh
a2d74dab50 Print index type. 2013-06-29 11:57:32 +05:30
Moinak Ghosh
e732e86b91 New option to capture statistics for all rolling hash breakpoints. 2013-06-23 21:30:32 +05:30
Moinak Ghosh
84944932b0 Add more statistics. 2013-06-22 23:54:54 +05:30
Moinak Ghosh
b8f4a5d411 Compute rolling hash coverage metrics. 2013-06-20 23:57:18 +05:30
Moinak Ghosh
916f31d62b Add measurements for Chunking properties. 2013-06-20 22:08:07 +05:30
5 changed files with 143 additions and 47 deletions

View file

@ -1103,12 +1103,7 @@ start_decompress(pc_ctx_t *pctx, const char *filename, const char *to_filename)
} }
} }
nprocs = sysconf(_SC_NPROCESSORS_ONLN); nprocs = 1;
if (pctx->nthreads > 0 && pctx->nthreads < nprocs)
nprocs = pctx->nthreads;
else
pctx->nthreads = nprocs;
set_threadcounts(&props, &(pctx->nthreads), nprocs, DECOMPRESS_THREADS); set_threadcounts(&props, &(pctx->nthreads), nprocs, DECOMPRESS_THREADS);
if (props.is_single_chunk) if (props.is_single_chunk)
pctx->nthreads = 1; pctx->nthreads = 1;
@ -1662,7 +1657,8 @@ repeat:
pctx->avg_chunk += tdat->len_cmp; pctx->avg_chunk += tdat->len_cmp;
} }
wbytes = Write(w->wfd, tdat->cmp_seg, tdat->len_cmp); //wbytes = Write(w->wfd, tdat->cmp_seg, tdat->len_cmp);
wbytes = tdat->len_cmp;
if (unlikely(wbytes != tdat->len_cmp)) { if (unlikely(wbytes != tdat->len_cmp)) {
perror("Chunk Write: "); perror("Chunk Write: ");
do_cancel: do_cancel:
@ -1785,12 +1781,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
thread = 0; thread = 0;
single_chunk = 0; single_chunk = 0;
rctx = NULL; rctx = NULL;
nprocs = 1;
nprocs = sysconf(_SC_NPROCESSORS_ONLN);
if (pctx->nthreads > 0 && pctx->nthreads < nprocs)
nprocs = pctx->nthreads;
else
pctx->nthreads = nprocs;
/* A host of sanity checks. */ /* A host of sanity checks. */
if (!pctx->pipe_mode) { if (!pctx->pipe_mode) {
@ -2057,6 +2048,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
COMP_BAIL; COMP_BAIL;
} }
tdat->rctx->full_chunking = pctx->full_chunking;
tdat->rctx->index_sem = &(tdat->index_sem); tdat->rctx->index_sem = &(tdat->index_sem);
tdat->rctx->id = i; tdat->rctx->id = i;
} }
@ -2182,6 +2174,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
if (pctx->enable_rabin_split) { if (pctx->enable_rabin_split) {
rctx = create_dedupe_context(chunksize, 0, 0, pctx->algo, &props, pctx->enable_delta_encode, rctx = create_dedupe_context(chunksize, 0, 0, pctx->algo, &props, pctx->enable_delta_encode,
pctx->enable_fixed_scan, VERSION, COMPRESS, 0, NULL, pctx->pipe_mode, nprocs); pctx->enable_fixed_scan, VERSION, COMPRESS, 0, NULL, pctx->pipe_mode, nprocs);
rctx->full_chunking = 0;
rbytes = Read_Adjusted(uncompfd, cread_buf, chunksize, &rabin_count, rctx); rbytes = Read_Adjusted(uncompfd, cread_buf, chunksize, &rabin_count, rctx);
} else { } else {
rbytes = Read(uncompfd, cread_buf, chunksize); rbytes = Read(uncompfd, cread_buf, chunksize);
@ -2387,6 +2380,7 @@ comp_done:
if (uncompfd != -1) close(uncompfd); if (uncompfd != -1) close(uncompfd);
} }
dump_frequencies();
if (!pctx->hide_cmp_stats) show_compression_stats(pctx); if (!pctx->hide_cmp_stats) show_compression_stats(pctx);
pctx->_stats_func(!pctx->hide_cmp_stats); pctx->_stats_func(!pctx->hide_cmp_stats);
@ -2533,6 +2527,7 @@ create_pc_context(void)
ctx->hide_mem_stats = 1; ctx->hide_mem_stats = 1;
ctx->hide_cmp_stats = 1; ctx->hide_cmp_stats = 1;
ctx->enable_rabin_split = 1; ctx->enable_rabin_split = 1;
ctx->rab_blk_size = 1;
return (ctx); return (ctx);
} }
@ -2579,7 +2574,7 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
pctx->chunksize = DEFAULT_CHUNKSIZE; pctx->chunksize = DEFAULT_CHUNKSIZE;
pthread_mutex_lock(&opt_parse); pthread_mutex_lock(&opt_parse);
while ((opt = getopt(argc, argv, "dc:s:l:pt:MCDGEe:w:rLPS:B:Fk:")) != -1) { while ((opt = getopt(argc, argv, "dc:s:l:pt:MCDGEe:w:rLPS:B:Fk:f")) != -1) {
int ovr; int ovr;
int64_t chunksize; int64_t chunksize;
@ -2716,6 +2711,10 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
} }
break; break;
case 'f':
pctx->full_chunking = 1;
break;
case '?': case '?':
default: default:
return (2); return (2);
@ -2726,6 +2725,7 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
optind = 0; optind = 0;
pthread_mutex_unlock(&opt_parse); pthread_mutex_unlock(&opt_parse);
pctx->nthreads = 1;
if ((pctx->do_compress && pctx->do_uncompress) || (!pctx->do_compress && !pctx->do_uncompress)) { if ((pctx->do_compress && pctx->do_uncompress) || (!pctx->do_compress && !pctx->do_uncompress)) {
return (2); return (2);
} }

View file

@ -197,6 +197,7 @@ typedef struct pc_ctx {
int enable_fixed_scan; int enable_fixed_scan;
int lzp_preprocess; int lzp_preprocess;
int encrypt_type; int encrypt_type;
int full_chunking;
unsigned int chunk_num; unsigned int chunk_num;
uint64_t largest_chunk, smallest_chunk, avg_chunk; uint64_t largest_chunk, smallest_chunk, avg_chunk;
uint64_t chunksize; uint64_t chunksize;

View file

@ -373,7 +373,7 @@ set_config_s(archive_config_t *cfg, const char *algo, cksum_t ck, cksum_t ck_sim
cfg->archive_sz = file_sz; cfg->archive_sz = file_sz;
cfg->dedupe_mode = MODE_SIMILARITY; cfg->dedupe_mode = MODE_SIMILARITY;
if (cfg->archive_sz <= SIXTEEN_GB && (pct_interval == 0 || pct_interval == 100)) { if (pct_interval == 0 || pct_interval == 100) {
cfg->dedupe_mode = MODE_SIMPLE; cfg->dedupe_mode = MODE_SIMPLE;
cfg->segment_sz_bytes = user_chunk_sz; cfg->segment_sz_bytes = user_chunk_sz;
cfg->similarity_cksum_sz = cfg->chunk_cksum_sz; cfg->similarity_cksum_sz = cfg->chunk_cksum_sz;

View file

@ -114,6 +114,12 @@ uint64_t ir[256], out[256];
static int inited = 0; static int inited = 0;
archive_config_t *arc = NULL; archive_config_t *arc = NULL;
static uint64_t freqs[RAB_POLYNOMIAL_MAX_BLOCK_SIZE+1];
static uint64_t tot_chunks = 0, min_chunk;
static uint64_t tot_size = 0, non_hashed_size = 0;
static double tot_time = 0;
static int full_chunking = 0;
static uint32_t static uint32_t
dedupe_min_blksz(int rab_blk_sz) dedupe_min_blksz(int rab_blk_sz)
{ {
@ -132,6 +138,46 @@ dedupe_buf_extra(uint64_t chunksize, int rab_blk_sz, const char *algo, int delta
return ((chunksize / dedupe_min_blksz(rab_blk_sz)) * sizeof (uint32_t)); return ((chunksize / dedupe_min_blksz(rab_blk_sz)) * sizeof (uint32_t));
} }
void
dump_frequencies()
{
int i, j, limit;
uint64_t tot;
double tot_c, tot_s, bytes_sec;
printf("\nChunk Frequency Distribution\n");
printf("====================================\n");
printf("Min chunk size: %" PRIu64 "\n", min_chunk);
if (full_chunking)
limit = 1024;
else
limit = 4096;
for (i = 1; i <= RAB_POLYNOMIAL_MAX_BLOCK_SIZE;) {
tot = 0;
for (j = 0; j < limit; j++) tot += freqs[i++];
if (tot > 0) {
printf("%3d KB: %" PRIu64 "\n", i/1024, tot);
}
}
printf("====================================\n");
printf("Number of chunks : %" PRIu64 "\n", tot_chunks);
tot_c = tot_chunks;
tot_s = tot_size;
printf("Average chunk size: %.2F Bytes\n", tot_s / tot_c);
bytes_sec = tot_s / tot_time * 1000;
printf("Average chunking speed: %.3f MB/s\n", BYTES_TO_MB(bytes_sec));
tot_c = non_hashed_size;
printf("Total data length: %" PRIu64 "\n", tot_size);
printf("Hashed data length: %" PRIu64 "\n", tot_size - non_hashed_size);
printf("%%age of roll hash coverage: %.2f%%\n", (1 - tot_c / tot_s) * 100);
printf("====================================\n");
}
/* /*
* Helper function to let caller size the the user specific compression chunk/segment * Helper function to let caller size the the user specific compression chunk/segment
* to align with deduplication requirements. * to align with deduplication requirements.
@ -185,6 +231,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
int term, pow, j; int term, pow, j;
uint64_t val, poly_pow; uint64_t val, poly_pow;
memset(freqs, 0, sizeof (freqs));
poly_pow = 1; poly_pow = 1;
for (j = 0; j < RAB_POLYNOMIAL_WIN_SIZE; j++) { for (j = 0; j < RAB_POLYNOMIAL_WIN_SIZE; j++) {
poly_pow = (poly_pow * RAB_POLYNOMIAL_CONST) & POLY_MASK; poly_pow = (poly_pow * RAB_POLYNOMIAL_CONST) & POLY_MASK;
@ -281,6 +328,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
ctx->rabin_poly_avg_block_size = RAB_BLK_AVG_SZ(rab_blk_sz); ctx->rabin_poly_avg_block_size = RAB_BLK_AVG_SZ(rab_blk_sz);
ctx->rabin_avg_block_mask = RAB_BLK_MASK; ctx->rabin_avg_block_mask = RAB_BLK_MASK;
ctx->rabin_poly_min_block_size = dedupe_min_blksz(rab_blk_sz); ctx->rabin_poly_min_block_size = dedupe_min_blksz(rab_blk_sz);
min_chunk = ctx->rabin_poly_min_block_size;
ctx->delta_flag = 0; ctx->delta_flag = 0;
ctx->deltac_min_distance = props->deltac_min_distance; ctx->deltac_min_distance = props->deltac_min_distance;
ctx->pagesize = sysconf(_SC_PAGE_SIZE); ctx->pagesize = sysconf(_SC_PAGE_SIZE);
@ -478,9 +526,9 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
uint32_t *ctx_heap; uint32_t *ctx_heap;
rabin_blockentry_t **htab; rabin_blockentry_t **htab;
MinHeap heap; MinHeap heap;
DEBUG_STAT_EN(uint32_t max_count); DEBUG_STAT_EN(uint32_t max_count = 0);
DEBUG_STAT_EN(max_count = 0); DEBUG_STAT_EN(double en);
DEBUG_STAT_EN(double strt, en_1, en); double strt, en_1;
length = offset; length = offset;
last_offset = 0; last_offset = 0;
@ -488,8 +536,9 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
window_pos = 0; window_pos = 0;
ctx->valid = 0; ctx->valid = 0;
cur_roll_checksum = 0; cur_roll_checksum = 0;
full_chunking = ctx->full_chunking;
if (*size < ctx->rabin_poly_avg_block_size) return (0); if (*size < ctx->rabin_poly_avg_block_size) return (0);
DEBUG_STAT_EN(strt = get_wtime_millis()); strt = get_wtime_millis();
if (ctx->dedupe_flag == RABIN_DEDUPE_FIXED) { if (ctx->dedupe_flag == RABIN_DEDUPE_FIXED) {
blknum = *size / ctx->rabin_poly_avg_block_size; blknum = *size / ctx->rabin_poly_avg_block_size;
@ -516,7 +565,13 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
ctx->blocks[i]->hash = XXH32(buf1+last_offset, length, 0); ctx->blocks[i]->hash = XXH32(buf1+last_offset, length, 0);
ctx->blocks[i]->similarity_hash = ctx->blocks[i]->hash; ctx->blocks[i]->similarity_hash = ctx->blocks[i]->hash;
last_offset += length; last_offset += length;
tot_chunks++;
tot_size += length;
non_hashed_size += length;
} }
en_1 = get_wtime_millis();
tot_time += en_1 - strt;
for (i=0; i<blknum; i++) freqs[ctx->blocks[i]->length]++;
goto process_blocks; goto process_blocks;
} }
@ -538,6 +593,9 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
ary_sz += ctx->rabin_poly_max_block_size; ary_sz += ctx->rabin_poly_max_block_size;
ctx_heap = (uint32_t *)(ctx->cbuf + ctx->real_chunksize - ary_sz); ctx_heap = (uint32_t *)(ctx->cbuf + ctx->real_chunksize - ary_sz);
} }
if (ctx->full_chunking) {
ctx->rabin_poly_min_block_size = 1;
}
#ifndef SSE_MODE #ifndef SSE_MODE
memset(ctx->current_window_data, 0, RAB_POLYNOMIAL_WIN_SIZE); memset(ctx->current_window_data, 0, RAB_POLYNOMIAL_WIN_SIZE);
#else #else
@ -600,8 +658,13 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
* Start our sliding window at a fixed number of bytes before the min window size. * Start our sliding window at a fixed number of bytes before the min window size.
* It is pointless to slide the window over the whole length of the chunk. * It is pointless to slide the window over the whole length of the chunk.
*/ */
offset = ctx->rabin_poly_min_block_size - RAB_WINDOW_SLIDE_OFFSET; if (ctx->full_chunking) {
offset = 0;
} else {
offset = ctx->rabin_poly_min_block_size - RAB_WINDOW_SLIDE_OFFSET;
}
length = offset; length = offset;
non_hashed_size += offset;
for (i=offset; i<j; i++) { for (i=offset; i<j; i++) {
uint64_t pc[4]; uint64_t pc[4];
uint32_t cur_byte = buf1[i]; uint32_t cur_byte = buf1[i];
@ -648,17 +711,23 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
if ((cur_pos_checksum & ctx->rabin_avg_block_mask) == ctx->rabin_break_patt || if ((cur_pos_checksum & ctx->rabin_avg_block_mask) == ctx->rabin_break_patt ||
length >= ctx->rabin_poly_max_block_size) { length >= ctx->rabin_poly_max_block_size) {
if (!(ctx->arc)) { if (!(ctx->full_chunking)) {
if (ctx->blocks[blknum] == 0) if (!(ctx->arc)) {
ctx->blocks[blknum] = (rabin_blockentry_t *)slab_alloc(NULL, if (ctx->blocks[blknum] == 0)
sizeof (rabin_blockentry_t)); ctx->blocks[blknum] = (rabin_blockentry_t *)slab_alloc(NULL,
ctx->blocks[blknum]->offset = last_offset; sizeof (rabin_blockentry_t));
ctx->blocks[blknum]->index = blknum; // Need to store for sorting ctx->blocks[blknum]->offset = last_offset;
ctx->blocks[blknum]->length = length; ctx->blocks[blknum]->index = blknum; // Need to store for sorting
ctx->blocks[blknum]->length = length;
} else {
ctx->g_blocks[blknum].length = length;
ctx->g_blocks[blknum].offset = last_offset;
}
} else { } else {
ctx->g_blocks[blknum].length = length; freqs[length]++;
ctx->g_blocks[blknum].offset = last_offset;
} }
tot_chunks++;
tot_size += length;
DEBUG_STAT_EN(if (length >= ctx->rabin_poly_max_block_size) ++max_count); DEBUG_STAT_EN(if (length >= ctx->rabin_poly_max_block_size) ++max_count);
/* /*
@ -688,26 +757,36 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
last_offset = i+1; last_offset = i+1;
length = 0; length = 0;
if (*size - last_offset <= ctx->rabin_poly_min_block_size) break; if (*size - last_offset <= ctx->rabin_poly_min_block_size) break;
length = ctx->rabin_poly_min_block_size - RAB_WINDOW_SLIDE_OFFSET; if (ctx->full_chunking == 0) {
i = i + length; length = ctx->rabin_poly_min_block_size - RAB_WINDOW_SLIDE_OFFSET;
i = i + length;
}
non_hashed_size += length;
} }
} }
// Insert the last left-over trailing bytes, if any, into a block. // Insert the last left-over trailing bytes, if any, into a block.
if (last_offset < *size) { if (last_offset < *size) {
length = *size - last_offset; length = *size - last_offset;
if (!(ctx->arc)) { non_hashed_size += length;
if (ctx->blocks[blknum] == 0) if (!(ctx->full_chunking)) {
ctx->blocks[blknum] = (rabin_blockentry_t *)slab_alloc(NULL, if (!(ctx->arc)) {
sizeof (rabin_blockentry_t)); if (ctx->blocks[blknum] == 0)
ctx->blocks[blknum]->offset = last_offset; ctx->blocks[blknum] = (rabin_blockentry_t *)slab_alloc(NULL,
ctx->blocks[blknum]->index = blknum; sizeof (rabin_blockentry_t));
ctx->blocks[blknum]->length = length; ctx->blocks[blknum]->offset = last_offset;
ctx->blocks[blknum]->index = blknum;
ctx->blocks[blknum]->length = length;
} else {
ctx->g_blocks[blknum].length = length;
ctx->g_blocks[blknum].offset = last_offset;
}
} else { } else {
ctx->g_blocks[blknum].length = length; freqs[length]++;
ctx->g_blocks[blknum].offset = last_offset;
} }
tot_chunks++;
tot_size += length;
if (ctx->delta_flag) { if (ctx->delta_flag) {
uint64_t cur_sketch; uint64_t cur_sketch;
uint64_t pc[4]; uint64_t pc[4];
@ -735,9 +814,11 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
process_blocks: process_blocks:
// If we found at least a few chunks, perform dedup. // If we found at least a few chunks, perform dedup.
DEBUG_STAT_EN(en_1 = get_wtime_millis()); en_1 = get_wtime_millis();
tot_time += en_1 - strt;
DEBUG_STAT_EN(fprintf(stderr, "Original size: %" PRId64 ", blknum: %u\n", *size, blknum)); DEBUG_STAT_EN(fprintf(stderr, "Original size: %" PRId64 ", blknum: %u\n", *size, blknum));
DEBUG_STAT_EN(fprintf(stderr, "Number of maxlen blocks: %u\n", max_count)); DEBUG_STAT_EN(fprintf(stderr, "Number of maxlen blocks: %u\n", max_count));
if (ctx->full_chunking) blknum = 0;
if (blknum <=2 && ctx->arc) { if (blknum <=2 && ctx->arc) {
sem_wait(ctx->index_sem); sem_wait(ctx->index_sem);
sem_post(ctx->index_sem_next); sem_post(ctx->index_sem_next);
@ -778,6 +859,7 @@ process_blocks:
*/ */
g_dedupe_idx = ctx->cbuf + RABIN_HDR_SIZE; g_dedupe_idx = ctx->cbuf + RABIN_HDR_SIZE;
dedupe_index_sz = 0; dedupe_index_sz = 0;
for (i=0; i<blknum; i++) freqs[ctx->g_blocks[i].length]++;
/* /*
* First entry in table is the original file offset where this * First entry in table is the original file offset where this
@ -800,6 +882,7 @@ process_blocks:
* predictable serialization of index access in a sequence of * predictable serialization of index access in a sequence of
* threads without locking. * threads without locking.
*/ */
printf("Using simple dedupe index.\n");
length = 0; length = 0;
DEBUG_STAT_EN(w1 = get_wtime_millis()); DEBUG_STAT_EN(w1 = get_wtime_millis());
sem_wait(ctx->index_sem); sem_wait(ctx->index_sem);
@ -886,6 +969,7 @@ process_blocks:
* ====================================================================== * ======================================================================
*/ */
printf("Using similarity based dedupe index.\n");
cfg = ctx->arc; cfg = ctx->arc;
assert(cfg->similarity_cksum_sz == sizeof (uint64_t)); assert(cfg->similarity_cksum_sz == sizeof (uint64_t));
seg_heap = (uchar_t *)(ctx->g_blocks) - cfg->segment_sz * cfg->chunk_cksum_sz; seg_heap = (uchar_t *)(ctx->g_blocks) - cfg->segment_sz * cfg->chunk_cksum_sz;
@ -957,11 +1041,19 @@ process_blocks:
increment = cfg->chunk_cksum_sz / 2; increment = cfg->chunk_cksum_sz / 2;
if (increment * sub_i > length) if (increment * sub_i > length)
sub_i = length / increment; sub_i = length / increment;
for (j = 0; j<sub_i; j++) { if (increment/2 == sizeof (uint64_t)) {
crc = lzma_crc64(tgt, increment/2, 0); for (j = 0; j<sub_i; j++) {
*((uint64_t *)sim_ck) = crc; *((uint64_t *)sim_ck) = *((uint64_t *)tgt);
tgt += increment; tgt += increment;
sim_ck += cfg->similarity_cksum_sz; sim_ck += cfg->similarity_cksum_sz;
}
} else {
for (j = 0; j<sub_i; j++) {
crc = lzma_crc64(tgt, increment/2, 0);
*((uint64_t *)sim_ck) = crc;
tgt += increment;
sim_ck += cfg->similarity_cksum_sz;
}
} }
/* /*
@ -1288,6 +1380,7 @@ next_ent:
} }
} }
for (i=0; i<blknum; i++) freqs[ctx->blocks[i]->length]++;
ary_sz = (blknum << 1) * sizeof (rabin_blockentry_t *); ary_sz = (blknum << 1) * sizeof (rabin_blockentry_t *);
htab = (rabin_blockentry_t **)(ctx->cbuf + ctx->real_chunksize - ary_sz); htab = (rabin_blockentry_t **)(ctx->cbuf + ctx->real_chunksize - ary_sz);
memset(htab, 0, ary_sz); memset(htab, 0, ary_sz);

View file

@ -80,7 +80,7 @@
#define RAB_POLYNOMIAL_WIN_SIZE 16 #define RAB_POLYNOMIAL_WIN_SIZE 16
#define RAB_POLYNOMIAL_MIN_WIN_SIZE 8 #define RAB_POLYNOMIAL_MIN_WIN_SIZE 8
#define RAB_POLYNOMIAL_MAX_WIN_SIZE 64 #define RAB_POLYNOMIAL_MAX_WIN_SIZE 64
#define RAB_POLYNOMIAL_MAX_BLOCK_SIZE (128 * 1024) #define RAB_POLYNOMIAL_MAX_BLOCK_SIZE (64 * 1024)
#define RAB_BLK_MASK (((1 << RAB_BLK_MIN_BITS) - 1) >> 1) #define RAB_BLK_MASK (((1 << RAB_BLK_MIN_BITS) - 1) >> 1)
#define RAB_BLK_AVG_SZ(x) (1 << ((x) + RAB_BLK_MIN_BITS)) #define RAB_BLK_AVG_SZ(x) (1 << ((x) + RAB_BLK_MIN_BITS))
@ -178,6 +178,7 @@ typedef struct {
short valid; short valid;
void *lzma_data; void *lzma_data;
int level, delta_flag, dedupe_flag, deltac_min_distance; int level, delta_flag, dedupe_flag, deltac_min_distance;
int full_chunking;
uint64_t file_offset; // For global dedupe uint64_t file_offset; // For global dedupe
archive_config_t *arc; archive_config_t *arc;
sem_t *index_sem; sem_t *index_sem;
@ -206,5 +207,6 @@ extern uint32_t dedupe_buf_extra(uint64_t chunksize, int rab_blk_sz, const char
extern int global_dedupe_bufadjust(uint32_t rab_blk_sz, uint64_t *user_chunk_sz, int pct_interval, extern int global_dedupe_bufadjust(uint32_t rab_blk_sz, uint64_t *user_chunk_sz, int pct_interval,
const char *algo, cksum_t ck, cksum_t ck_sim, size_t file_sz, const char *algo, cksum_t ck, cksum_t ck_sim, size_t file_sz,
size_t memlimit, int nthreads, int pipe_mode); size_t memlimit, int nthreads, int pipe_mode);
extern void dump_frequencies();
#endif /* _RABIN_POLY_H_ */ #endif /* _RABIN_POLY_H_ */