greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use OpenMP parallelism when computing xxHashes for chunks.

Browse source
This commit is contained in:
Moinak Ghosh 2013-02-02 09:27:58 +05:30
parent 6bfd044311
commit 3e1737b4ab
4 changed files with 18 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
main.c
View file

@ -1224,7 +1224,8 @@ redo:
rctx = tdat->rctx; rctx = tdat->rctx;
reset_dedupe_context(tdat->rctx); reset_dedupe_context(tdat->rctx);
rctx->cbuf = tdat->uncompressed_chunk; rctx->cbuf = tdat->uncompressed_chunk;
dedupe_index_sz = dedupe_compress(tdat->rctx, tdat->cmp_seg, &(tdat->rbytes), 0, NULL); dedupe_index_sz = dedupe_compress(tdat->rctx, tdat->cmp_seg, &(tdat->rbytes), 0,
NULL, tdat->cksum_mt);
if (!rctx->valid) { if (!rctx->valid) {
memcpy(tdat->uncompressed_chunk, tdat->cmp_seg, rbytes); memcpy(tdat->uncompressed_chunk, tdat->cmp_seg, rbytes);
tdat->rbytes = rbytes; tdat->rbytes = rbytes;

17
rabin/rabin_dedup.c
View file

@ -76,6 +76,10 @@
# define SSE_MODE 1 # define SSE_MODE 1
#endif #endif
#if defined(_OPENMP)
#include <omp.h>
#endif
#define DELTA_EXTRA2_PCT(x) ((x) >> 1) #define DELTA_EXTRA2_PCT(x) ((x) >> 1)
#define DELTA_EXTRA_PCT(x) (((x) >> 1) + ((x) >> 3)) #define DELTA_EXTRA_PCT(x) (((x) >> 1) + ((x) >> 3))
#define DELTA_NORMAL_PCT(x) (((x) >> 1) + ((x) >> 2) + ((x) >> 3)) #define DELTA_NORMAL_PCT(x) (((x) >> 1) + ((x) >> 2) + ((x) >> 3))
@ -298,7 +302,8 @@ destroy_dedupe_context(dedupe_context_t *ctx)
* from 4K-128K. * from 4K-128K.
*/ */
uint32_t uint32_t
dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t offset, uint64_t *rabin_pos) dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t offset,
uint64_t *rabin_pos, int mt)
{ {
uint64_t i, last_offset, j, ary_sz; uint64_t i, last_offset, j, ary_sz;
uint32_t blknum, window_pos; uint32_t blknum, window_pos;
@ -569,14 +574,20 @@ process_blocks:
* have a fast linear scan through the buffer. * have a fast linear scan through the buffer.
*/ */
if (ctx->delta_flag) { if (ctx->delta_flag) {
#if defined(_OPENMP)
# pragma omp parallel for if (mt)
#endif
for (i=0; i<blknum; i++) { for (i=0; i<blknum; i++) {
ctx->blocks[i]->hash = XXH32(buf1+ctx->blocks[i]->offset, ctx->blocks[i]->hash = XXH32(buf1+ctx->blocks[i]->offset,
ctx->blocks[i]->length, 0); ctx->blocks[i]->length, 0);
} }
} else { } else {
#if defined(_OPENMP)
# pragma omp parallel for if (mt)
#endif
for (i=0; i<blknum; i++) { for (i=0; i<blknum; i++) {
ctx->blocks[i]->hash = XXH32(buf1+ctx->blocks[i]->offset, ctx->blocks[i]->hash = XXH32(buf1+ctx->blocks[i]->offset,
ctx->blocks[i]->length, 0); ctx->blocks[i]->length, 0);
ctx->blocks[i]->similarity_hash = ctx->blocks[i]->hash; ctx->blocks[i]->similarity_hash = ctx->blocks[i]->hash;
} }
} }

2
rabin/rabin_dedup.h
View file

@ -170,7 +170,7 @@ extern dedupe_context_t *create_dedupe_context(uint64_t chunksize, uint64_t real
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern void destroy_dedupe_context(dedupe_context_t *ctx); extern void destroy_dedupe_context(dedupe_context_t *ctx);
extern unsigned int dedupe_compress(dedupe_context_t *ctx, unsigned char *buf, extern unsigned int dedupe_compress(dedupe_context_t *ctx, unsigned char *buf,
uint64_t *size, uint64_t offset, uint64_t *rabin_pos); uint64_t *size, uint64_t offset, uint64_t *rabin_pos, int mt);
extern void dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size); extern void dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size);
extern void parse_dedupe_hdr(uchar_t *buf, unsigned int *blknum, uint64_t *dedupe_index_sz, extern void parse_dedupe_hdr(uchar_t *buf, unsigned int *blknum, uint64_t *dedupe_index_sz,
uint64_t *dedupe_data_sz, uint64_t *rabin_index_sz_cmp, uint64_t *dedupe_data_sz, uint64_t *rabin_index_sz_cmp,

2
utils/utils.c
View file

@ -239,7 +239,7 @@ Read_Adjusted(int fd, uchar_t *buf, uint64_t count, int64_t *rabin_count, void *
* This call does not actually dedupe but finds the last rabin boundary * This call does not actually dedupe but finds the last rabin boundary
* in the buf. * in the buf.
*/ */
dedupe_compress(rctx, buf, &rc, 0, &rbc); dedupe_compress(rctx, buf, &rc, 0, &rbc, 0);
rcount = rc; rcount = rc;
*rabin_count = rbc; *rabin_count = rbc;
} else { } else {