Improve Dedupe performance.

Add more debug timing stats.
Change default checksum to Keccak 256 (SIMD version 4x faster than Skein).
Fix compiler warning in allocator code.
This commit is contained in:
Moinak Ghosh 2012-12-29 23:43:41 +05:30
parent 36d95276ee
commit 28224d29d3
4 changed files with 20 additions and 9 deletions

View file

@ -522,6 +522,9 @@ slab_free(void *p, void *address)
} }
int int
slab_cache_add(uint64_t size) {} slab_cache_add(uint64_t size)
{
return (0);
}
#endif #endif

View file

@ -76,6 +76,9 @@ extern uint64_t lzma_crc64_8bchk(const uint8_t *buf, uint64_t size,
int int
compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, int64_t bytes) compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, int64_t bytes)
{ {
DEBUG_STAT_EN(double strt, en);
DEBUG_STAT_EN(strt = get_wtime_millis());
if (cksum == CKSUM_CRC64) { if (cksum == CKSUM_CRC64) {
uint64_t *ck = (uint64_t *)cksum_buf; uint64_t *ck = (uint64_t *)cksum_buf;
*ck = lzma_crc64(buf, bytes, 0); *ck = lzma_crc64(buf, bytes, 0);
@ -125,6 +128,8 @@ compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, int64_t bytes)
} else { } else {
return (-1); return (-1);
} }
DEBUG_STAT_EN(en = get_wtime_millis());
DEBUG_STAT_EN(fprintf(stderr, "Checksum computed at %.3f MB/s\n", get_mb_s(bytes, strt, en)));
return (0); return (0);
} }

View file

@ -35,7 +35,7 @@ extern "C" {
#define MAX_PW_LEN 16 #define MAX_PW_LEN 16
#define CKSUM_MASK 0x700 #define CKSUM_MASK 0x700
#define CKSUM_MAX_BYTES 64 #define CKSUM_MAX_BYTES 64
#define DEFAULT_CKSUM "SKEIN256" #define DEFAULT_CKSUM "KECCAK256"
#define ENCRYPT_FLAG 1 #define ENCRYPT_FLAG 1
#define DECRYPT_FLAG 0 #define DECRYPT_FLAG 0

View file

@ -296,7 +296,7 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
heap_t heap; heap_t heap;
DEBUG_STAT_EN(uint32_t max_count); DEBUG_STAT_EN(uint32_t max_count);
DEBUG_STAT_EN(max_count = 0); DEBUG_STAT_EN(max_count = 0);
DEBUG_STAT_EN(double strt, en); DEBUG_STAT_EN(double strt, en_1, en);
length = offset; length = offset;
last_offset = 0; last_offset = 0;
@ -342,7 +342,7 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
*/ */
ary_sz = 4 * ctx->rabin_poly_max_block_size; ary_sz = 4 * ctx->rabin_poly_max_block_size;
fplist = (uint32_t *)(ctx->cbuf + ctx->real_chunksize - ary_sz); fplist = (uint32_t *)(ctx->cbuf + ctx->real_chunksize - ary_sz);
memset(fplist, 0, ary_sz); if (ctx->delta_flag) memset(fplist, 0, ary_sz);
} }
memset(ctx->current_window_data, 0, RAB_POLYNOMIAL_WIN_SIZE); memset(ctx->current_window_data, 0, RAB_POLYNOMIAL_WIN_SIZE);
@ -408,8 +408,10 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
* This is called minhashing and is used widely, for example in various * This is called minhashing and is used widely, for example in various
* search engines to detect similar documents. * search engines to detect similar documents.
*/ */
fplist[j] = cur_pos_checksum & 0xFFFFFFFFUL; if (ctx->delta_flag) {
j++; fplist[j] = cur_pos_checksum & 0xFFFFFFFFUL;
j++;
}
/* /*
* Window pos has to rotate from 0 .. RAB_POLYNOMIAL_WIN_SIZE-1 * Window pos has to rotate from 0 .. RAB_POLYNOMIAL_WIN_SIZE-1
@ -484,13 +486,13 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
} }
ctx->blocks[blknum]->similarity_hash = cur_sketch; ctx->blocks[blknum]->similarity_hash = cur_sketch;
} }
blknum++; blknum++;
last_offset = *size; last_offset = *size;
} }
process_blocks: process_blocks:
// If we found at least a few chunks, perform dedup. // If we found at least a few chunks, perform dedup.
DEBUG_STAT_EN(en_1 = get_wtime_millis());
DEBUG_STAT_EN(fprintf(stderr, "Original size: %" PRId64 ", blknum: %u\n", *size, blknum)); DEBUG_STAT_EN(fprintf(stderr, "Original size: %" PRId64 ", blknum: %u\n", *size, blknum));
DEBUG_STAT_EN(fprintf(stderr, "Number of maxlen blocks: %u\n", max_count)); DEBUG_STAT_EN(fprintf(stderr, "Number of maxlen blocks: %u\n", max_count));
if (blknum > 2) { if (blknum > 2) {
@ -578,7 +580,7 @@ process_blocks:
break; break;
} }
if (!length && ctx->delta_flag) { if (ctx->delta_flag && !length) {
/* /*
* Look for similar blocks. * Look for similar blocks.
*/ */
@ -710,7 +712,8 @@ process_blocks:
DEBUG_STAT_EN(en = get_wtime_millis()); DEBUG_STAT_EN(en = get_wtime_millis());
DEBUG_STAT_EN(fprintf(stderr, "Deduped size: %" PRId64 ", blknum: %u, delta_calls: %u, delta_fails: %u\n", DEBUG_STAT_EN(fprintf(stderr, "Deduped size: %" PRId64 ", blknum: %u, delta_calls: %u, delta_fails: %u\n",
*size, blknum, delta_calls, delta_fails)); *size, blknum, delta_calls, delta_fails));
DEBUG_STAT_EN(fprintf(stderr, "Dedupe speed %.3f MB/s\n", get_mb_s(sz, strt, en))); DEBUG_STAT_EN(fprintf(stderr, "Chunking speed %.3f MB/s, Overall Dedupe speed %.3f MB/s\n",
get_mb_s(sz, strt, en_1), get_mb_s(sz, strt, en)));
/* /*
* Remaining header entries: size of compressed index and size of * Remaining header entries: size of compressed index and size of
* compressed data are inserted later via rabin_update_hdr, after actual compression! * compressed data are inserted later via rabin_update_hdr, after actual compression!