Improve Dedupe performance.
Add more debug timing stats. Change default checksum to Keccak 256 (SIMD version 4x faster than Skein). Fix compiler warning in allocator code.
This commit is contained in:
parent
36d95276ee
commit
28224d29d3
4 changed files with 20 additions and 9 deletions
|
@ -522,6 +522,9 @@ slab_free(void *p, void *address)
|
||||||
}
|
}
|
||||||
|
|
||||||
int
|
int
|
||||||
slab_cache_add(uint64_t size) {}
|
slab_cache_add(uint64_t size)
|
||||||
|
{
|
||||||
|
return (0);
|
||||||
|
}
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -76,6 +76,9 @@ extern uint64_t lzma_crc64_8bchk(const uint8_t *buf, uint64_t size,
|
||||||
int
|
int
|
||||||
compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, int64_t bytes)
|
compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, int64_t bytes)
|
||||||
{
|
{
|
||||||
|
DEBUG_STAT_EN(double strt, en);
|
||||||
|
|
||||||
|
DEBUG_STAT_EN(strt = get_wtime_millis());
|
||||||
if (cksum == CKSUM_CRC64) {
|
if (cksum == CKSUM_CRC64) {
|
||||||
uint64_t *ck = (uint64_t *)cksum_buf;
|
uint64_t *ck = (uint64_t *)cksum_buf;
|
||||||
*ck = lzma_crc64(buf, bytes, 0);
|
*ck = lzma_crc64(buf, bytes, 0);
|
||||||
|
@ -125,6 +128,8 @@ compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, int64_t bytes)
|
||||||
} else {
|
} else {
|
||||||
return (-1);
|
return (-1);
|
||||||
}
|
}
|
||||||
|
DEBUG_STAT_EN(en = get_wtime_millis());
|
||||||
|
DEBUG_STAT_EN(fprintf(stderr, "Checksum computed at %.3f MB/s\n", get_mb_s(bytes, strt, en)));
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -35,7 +35,7 @@ extern "C" {
|
||||||
#define MAX_PW_LEN 16
|
#define MAX_PW_LEN 16
|
||||||
#define CKSUM_MASK 0x700
|
#define CKSUM_MASK 0x700
|
||||||
#define CKSUM_MAX_BYTES 64
|
#define CKSUM_MAX_BYTES 64
|
||||||
#define DEFAULT_CKSUM "SKEIN256"
|
#define DEFAULT_CKSUM "KECCAK256"
|
||||||
|
|
||||||
#define ENCRYPT_FLAG 1
|
#define ENCRYPT_FLAG 1
|
||||||
#define DECRYPT_FLAG 0
|
#define DECRYPT_FLAG 0
|
||||||
|
|
|
@ -296,7 +296,7 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
|
||||||
heap_t heap;
|
heap_t heap;
|
||||||
DEBUG_STAT_EN(uint32_t max_count);
|
DEBUG_STAT_EN(uint32_t max_count);
|
||||||
DEBUG_STAT_EN(max_count = 0);
|
DEBUG_STAT_EN(max_count = 0);
|
||||||
DEBUG_STAT_EN(double strt, en);
|
DEBUG_STAT_EN(double strt, en_1, en);
|
||||||
|
|
||||||
length = offset;
|
length = offset;
|
||||||
last_offset = 0;
|
last_offset = 0;
|
||||||
|
@ -342,7 +342,7 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
|
||||||
*/
|
*/
|
||||||
ary_sz = 4 * ctx->rabin_poly_max_block_size;
|
ary_sz = 4 * ctx->rabin_poly_max_block_size;
|
||||||
fplist = (uint32_t *)(ctx->cbuf + ctx->real_chunksize - ary_sz);
|
fplist = (uint32_t *)(ctx->cbuf + ctx->real_chunksize - ary_sz);
|
||||||
memset(fplist, 0, ary_sz);
|
if (ctx->delta_flag) memset(fplist, 0, ary_sz);
|
||||||
}
|
}
|
||||||
memset(ctx->current_window_data, 0, RAB_POLYNOMIAL_WIN_SIZE);
|
memset(ctx->current_window_data, 0, RAB_POLYNOMIAL_WIN_SIZE);
|
||||||
|
|
||||||
|
@ -408,8 +408,10 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
|
||||||
* This is called minhashing and is used widely, for example in various
|
* This is called minhashing and is used widely, for example in various
|
||||||
* search engines to detect similar documents.
|
* search engines to detect similar documents.
|
||||||
*/
|
*/
|
||||||
fplist[j] = cur_pos_checksum & 0xFFFFFFFFUL;
|
if (ctx->delta_flag) {
|
||||||
j++;
|
fplist[j] = cur_pos_checksum & 0xFFFFFFFFUL;
|
||||||
|
j++;
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Window pos has to rotate from 0 .. RAB_POLYNOMIAL_WIN_SIZE-1
|
* Window pos has to rotate from 0 .. RAB_POLYNOMIAL_WIN_SIZE-1
|
||||||
|
@ -484,13 +486,13 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t of
|
||||||
}
|
}
|
||||||
ctx->blocks[blknum]->similarity_hash = cur_sketch;
|
ctx->blocks[blknum]->similarity_hash = cur_sketch;
|
||||||
}
|
}
|
||||||
|
|
||||||
blknum++;
|
blknum++;
|
||||||
last_offset = *size;
|
last_offset = *size;
|
||||||
}
|
}
|
||||||
|
|
||||||
process_blocks:
|
process_blocks:
|
||||||
// If we found at least a few chunks, perform dedup.
|
// If we found at least a few chunks, perform dedup.
|
||||||
|
DEBUG_STAT_EN(en_1 = get_wtime_millis());
|
||||||
DEBUG_STAT_EN(fprintf(stderr, "Original size: %" PRId64 ", blknum: %u\n", *size, blknum));
|
DEBUG_STAT_EN(fprintf(stderr, "Original size: %" PRId64 ", blknum: %u\n", *size, blknum));
|
||||||
DEBUG_STAT_EN(fprintf(stderr, "Number of maxlen blocks: %u\n", max_count));
|
DEBUG_STAT_EN(fprintf(stderr, "Number of maxlen blocks: %u\n", max_count));
|
||||||
if (blknum > 2) {
|
if (blknum > 2) {
|
||||||
|
@ -578,7 +580,7 @@ process_blocks:
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!length && ctx->delta_flag) {
|
if (ctx->delta_flag && !length) {
|
||||||
/*
|
/*
|
||||||
* Look for similar blocks.
|
* Look for similar blocks.
|
||||||
*/
|
*/
|
||||||
|
@ -710,7 +712,8 @@ process_blocks:
|
||||||
DEBUG_STAT_EN(en = get_wtime_millis());
|
DEBUG_STAT_EN(en = get_wtime_millis());
|
||||||
DEBUG_STAT_EN(fprintf(stderr, "Deduped size: %" PRId64 ", blknum: %u, delta_calls: %u, delta_fails: %u\n",
|
DEBUG_STAT_EN(fprintf(stderr, "Deduped size: %" PRId64 ", blknum: %u, delta_calls: %u, delta_fails: %u\n",
|
||||||
*size, blknum, delta_calls, delta_fails));
|
*size, blknum, delta_calls, delta_fails));
|
||||||
DEBUG_STAT_EN(fprintf(stderr, "Dedupe speed %.3f MB/s\n", get_mb_s(sz, strt, en)));
|
DEBUG_STAT_EN(fprintf(stderr, "Chunking speed %.3f MB/s, Overall Dedupe speed %.3f MB/s\n",
|
||||||
|
get_mb_s(sz, strt, en_1), get_mb_s(sz, strt, en)));
|
||||||
/*
|
/*
|
||||||
* Remaining header entries: size of compressed index and size of
|
* Remaining header entries: size of compressed index and size of
|
||||||
* compressed data are inserted later via rabin_update_hdr, after actual compression!
|
* compressed data are inserted later via rabin_update_hdr, after actual compression!
|
||||||
|
|
Loading…
Reference in a new issue