greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Improve Dedupe performance.

Browse source 
Add more debug timing stats.
Change default checksum to Keccak 256 (SIMD version 4x faster than Skein).
Fix compiler warning in allocator code.
This commit is contained in:
Moinak Ghosh 2012-12-29 23:43:41 +05:30
parent 36d95276ee
commit 28224d29d3
4 changed files with 20 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
allocator.c
View file

@ -522,6 +522,9 @@ slab_free(void *p, void *address)
}
int
slab_cache_add(uint64_t size) {}
slab_cache_add(uint64_t size)
{
return (0);
}
#endif

5
crypto/crypto_utils.c
View file

@ -76,6 +76,9 @@ extern uint64_t lzma_crc64_8bchk(const uint8_t *buf, uint64_t size,
int
compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, int64_t bytes)
{
DEBUG_STAT_EN(double strt, en);
DEBUG_STAT_EN(strt = get_wtime_millis());
if (cksum == CKSUM_CRC64) {
uint64_t *ck = (uint64_t *)cksum_buf;
*ck = lzma_crc64(buf, bytes, 0);
@ -125,6 +128,8 @@ compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, int64_t bytes)
} else {
return (-1);
}
DEBUG_STAT_EN(en = get_wtime_millis());
DEBUG_STAT_EN(fprintf(stderr, "Checksum computed at %.3f MB/s\n", get_mb_s(bytes, strt, en)));
return (0);
}

2
crypto/crypto_utils.h
View file

@ -35,7 +35,7 @@ extern "C" {
#define MAX_PW_LEN 16
#define CKSUM_MASK 0x700
#define CKSUM_MAX_BYTES 64
#define DEFAULT_CKSUM "SKEIN256"
#define DEFAULT_CKSUM "KECCAK256"
#define ENCRYPT_FLAG 1
#define DECRYPT_FLAG 0

17
rabin/rabin_dedup.c
View file

@ -296,7 +296,7 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
heap_t heap;
DEBUG_STAT_EN(uint32_t max_count);
DEBUG_STAT_EN(max_count = 0);
DEBUG_STAT_EN(double strt, en);
DEBUG_STAT_EN(double strt, en_1, en);
length = offset;
last_offset = 0;
@ -342,7 +342,7 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
*/
ary_sz = 4 * ctx->rabin_poly_max_block_size;
fplist = (uint32_t *)(ctx->cbuf + ctx->real_chunksize - ary_sz);
memset(fplist, 0, ary_sz);
if (ctx->delta_flag) memset(fplist, 0, ary_sz);
}
memset(ctx->current_window_data, 0, RAB_POLYNOMIAL_WIN_SIZE);
@ -408,8 +408,10 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
* This is called minhashing and is used widely, for example in various
* search engines to detect similar documents.
*/
fplist[j] = cur_pos_checksum & 0xFFFFFFFFUL;
j++;
if (ctx->delta_flag) {
fplist[j] = cur_pos_checksum & 0xFFFFFFFFUL;
j++;
}
/*
* Window pos has to rotate from 0 .. RAB_POLYNOMIAL_WIN_SIZE-1
@ -484,13 +486,13 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
}
ctx->blocks[blknum]->similarity_hash = cur_sketch;
}
blknum++;
last_offset = *size;
}
process_blocks:
// If we found at least a few chunks, perform dedup.
DEBUG_STAT_EN(en_1 = get_wtime_millis());
DEBUG_STAT_EN(fprintf(stderr, "Original size: %" PRId64 ", blknum: %u\n", *size, blknum));
DEBUG_STAT_EN(fprintf(stderr, "Number of maxlen blocks: %u\n", max_count));
if (blknum > 2) {
@ -578,7 +580,7 @@ process_blocks:
break;
}
if (!length && ctx->delta_flag) {
if (ctx->delta_flag && !length) {
/*
* Look for similar blocks.
*/
@ -710,7 +712,8 @@ process_blocks:
DEBUG_STAT_EN(en = get_wtime_millis());
DEBUG_STAT_EN(fprintf(stderr, "Deduped size: %" PRId64 ", blknum: %u, delta_calls: %u, delta_fails: %u\n",
*size, blknum, delta_calls, delta_fails));
DEBUG_STAT_EN(fprintf(stderr, "Dedupe speed %.3f MB/s\n", get_mb_s(sz, strt, en)));
DEBUG_STAT_EN(fprintf(stderr, "Chunking speed %.3f MB/s, Overall Dedupe speed %.3f MB/s\n",
get_mb_s(sz, strt, en_1), get_mb_s(sz, strt, en)));
/*
* Remaining header entries: size of compressed index and size of
* compressed data are inserted later via rabin_update_hdr, after actual compression!