greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Make data partitioning between threads more effective.

Browse source 
Remove unnecessary computation to make Fixed block chunking faster.
This commit is contained in:
Moinak Ghosh 2013-07-21 09:31:59 +05:30
parent 2a218e9da5
commit f34cfb1aa6
3 changed files with 4 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
pcompress.c
View file

@ -2180,8 +2180,9 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
*/ */
file_offset = 0; file_offset = 0;
if (pctx->enable_rabin_split) { if (pctx->enable_rabin_split) {
rctx = create_dedupe_context(chunksize, 0, 0, pctx->algo, &props, pctx->enable_delta_encode, rctx = create_dedupe_context(chunksize, 0, pctx->rab_blk_size, pctx->algo, &props,
pctx->enable_fixed_scan, VERSION, COMPRESS, 0, NULL, pctx->pipe_mode, nprocs); pctx->enable_delta_encode, pctx->enable_fixed_scan, VERSION, COMPRESS, 0, NULL,
pctx->pipe_mode, nprocs);
rbytes = Read_Adjusted(uncompfd, cread_buf, chunksize, &rabin_count, rctx); rbytes = Read_Adjusted(uncompfd, cread_buf, chunksize, &rabin_count, rctx);
} else { } else {
rbytes = Read(uncompfd, cread_buf, chunksize); rbytes = Read(uncompfd, cread_buf, chunksize);

2
rabin/rabin_dedup.c
View file

@ -512,8 +512,6 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
ctx->blocks[i]->index = i; // Need to store for sorting ctx->blocks[i]->index = i; // Need to store for sorting
ctx->blocks[i]->length = length; ctx->blocks[i]->length = length;
ctx->blocks[i]->similar = 0; ctx->blocks[i]->similar = 0;
ctx->blocks[i]->hash = XXH32(buf1+last_offset, length, 0);
ctx->blocks[i]->similarity_hash = ctx->blocks[i]->hash;
last_offset += length; last_offset += length;
} }
goto process_blocks; goto process_blocks;

2
rabin/rabin_dedup.h
View file

@ -80,7 +80,7 @@
#define RAB_POLYNOMIAL_WIN_SIZE 16 #define RAB_POLYNOMIAL_WIN_SIZE 16
#define RAB_POLYNOMIAL_MIN_WIN_SIZE 8 #define RAB_POLYNOMIAL_MIN_WIN_SIZE 8
#define RAB_POLYNOMIAL_MAX_WIN_SIZE 64 #define RAB_POLYNOMIAL_MAX_WIN_SIZE 64
#define RAB_POLYNOMIAL_MAX_BLOCK_SIZE (128 * 1024) #define RAB_POLYNOMIAL_MAX_BLOCK_SIZE (64 * 1024)
#define RAB_BLK_MASK (((1 << RAB_BLK_MIN_BITS) - 1) >> 1) #define RAB_BLK_MASK (((1 << RAB_BLK_MIN_BITS) - 1) >> 1)
#define RAB_BLK_AVG_SZ(x) (1 << ((x) + RAB_BLK_MIN_BITS)) #define RAB_BLK_AVG_SZ(x) (1 << ((x) + RAB_BLK_MIN_BITS))