Use OpenMP parallelism when computing xxHashes for chunks.
This commit is contained in:
parent
6bfd044311
commit
3e1737b4ab
4 changed files with 18 additions and 6 deletions
3
main.c
3
main.c
|
@ -1224,7 +1224,8 @@ redo:
|
||||||
rctx = tdat->rctx;
|
rctx = tdat->rctx;
|
||||||
reset_dedupe_context(tdat->rctx);
|
reset_dedupe_context(tdat->rctx);
|
||||||
rctx->cbuf = tdat->uncompressed_chunk;
|
rctx->cbuf = tdat->uncompressed_chunk;
|
||||||
dedupe_index_sz = dedupe_compress(tdat->rctx, tdat->cmp_seg, &(tdat->rbytes), 0, NULL);
|
dedupe_index_sz = dedupe_compress(tdat->rctx, tdat->cmp_seg, &(tdat->rbytes), 0,
|
||||||
|
NULL, tdat->cksum_mt);
|
||||||
if (!rctx->valid) {
|
if (!rctx->valid) {
|
||||||
memcpy(tdat->uncompressed_chunk, tdat->cmp_seg, rbytes);
|
memcpy(tdat->uncompressed_chunk, tdat->cmp_seg, rbytes);
|
||||||
tdat->rbytes = rbytes;
|
tdat->rbytes = rbytes;
|
||||||
|
|
|
@ -76,6 +76,10 @@
|
||||||
# define SSE_MODE 1
|
# define SSE_MODE 1
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#if defined(_OPENMP)
|
||||||
|
#include <omp.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
#define DELTA_EXTRA2_PCT(x) ((x) >> 1)
|
#define DELTA_EXTRA2_PCT(x) ((x) >> 1)
|
||||||
#define DELTA_EXTRA_PCT(x) (((x) >> 1) + ((x) >> 3))
|
#define DELTA_EXTRA_PCT(x) (((x) >> 1) + ((x) >> 3))
|
||||||
#define DELTA_NORMAL_PCT(x) (((x) >> 1) + ((x) >> 2) + ((x) >> 3))
|
#define DELTA_NORMAL_PCT(x) (((x) >> 1) + ((x) >> 2) + ((x) >> 3))
|
||||||
|
@ -298,7 +302,8 @@ destroy_dedupe_context(dedupe_context_t *ctx)
|
||||||
* from 4K-128K.
|
* from 4K-128K.
|
||||||
*/
|
*/
|
||||||
uint32_t
|
uint32_t
|
||||||
dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t offset, uint64_t *rabin_pos)
|
dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t offset,
|
||||||
|
uint64_t *rabin_pos, int mt)
|
||||||
{
|
{
|
||||||
uint64_t i, last_offset, j, ary_sz;
|
uint64_t i, last_offset, j, ary_sz;
|
||||||
uint32_t blknum, window_pos;
|
uint32_t blknum, window_pos;
|
||||||
|
@ -569,11 +574,17 @@ process_blocks:
|
||||||
* have a fast linear scan through the buffer.
|
* have a fast linear scan through the buffer.
|
||||||
*/
|
*/
|
||||||
if (ctx->delta_flag) {
|
if (ctx->delta_flag) {
|
||||||
|
#if defined(_OPENMP)
|
||||||
|
# pragma omp parallel for if (mt)
|
||||||
|
#endif
|
||||||
for (i=0; i<blknum; i++) {
|
for (i=0; i<blknum; i++) {
|
||||||
ctx->blocks[i]->hash = XXH32(buf1+ctx->blocks[i]->offset,
|
ctx->blocks[i]->hash = XXH32(buf1+ctx->blocks[i]->offset,
|
||||||
ctx->blocks[i]->length, 0);
|
ctx->blocks[i]->length, 0);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
#if defined(_OPENMP)
|
||||||
|
# pragma omp parallel for if (mt)
|
||||||
|
#endif
|
||||||
for (i=0; i<blknum; i++) {
|
for (i=0; i<blknum; i++) {
|
||||||
ctx->blocks[i]->hash = XXH32(buf1+ctx->blocks[i]->offset,
|
ctx->blocks[i]->hash = XXH32(buf1+ctx->blocks[i]->offset,
|
||||||
ctx->blocks[i]->length, 0);
|
ctx->blocks[i]->length, 0);
|
||||||
|
|
|
@ -170,7 +170,7 @@ extern dedupe_context_t *create_dedupe_context(uint64_t chunksize, uint64_t real
|
||||||
int file_version, compress_op_t op);
|
int file_version, compress_op_t op);
|
||||||
extern void destroy_dedupe_context(dedupe_context_t *ctx);
|
extern void destroy_dedupe_context(dedupe_context_t *ctx);
|
||||||
extern unsigned int dedupe_compress(dedupe_context_t *ctx, unsigned char *buf,
|
extern unsigned int dedupe_compress(dedupe_context_t *ctx, unsigned char *buf,
|
||||||
uint64_t *size, uint64_t offset, uint64_t *rabin_pos);
|
uint64_t *size, uint64_t offset, uint64_t *rabin_pos, int mt);
|
||||||
extern void dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size);
|
extern void dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size);
|
||||||
extern void parse_dedupe_hdr(uchar_t *buf, unsigned int *blknum, uint64_t *dedupe_index_sz,
|
extern void parse_dedupe_hdr(uchar_t *buf, unsigned int *blknum, uint64_t *dedupe_index_sz,
|
||||||
uint64_t *dedupe_data_sz, uint64_t *rabin_index_sz_cmp,
|
uint64_t *dedupe_data_sz, uint64_t *rabin_index_sz_cmp,
|
||||||
|
|
|
@ -239,7 +239,7 @@ Read_Adjusted(int fd, uchar_t *buf, uint64_t count, int64_t *rabin_count, void *
|
||||||
* This call does not actually dedupe but finds the last rabin boundary
|
* This call does not actually dedupe but finds the last rabin boundary
|
||||||
* in the buf.
|
* in the buf.
|
||||||
*/
|
*/
|
||||||
dedupe_compress(rctx, buf, &rc, 0, &rbc);
|
dedupe_compress(rctx, buf, &rc, 0, &rbc, 0);
|
||||||
rcount = rc;
|
rcount = rc;
|
||||||
*rabin_count = rbc;
|
*rabin_count = rbc;
|
||||||
} else {
|
} else {
|
||||||
|
|
Loading…
Reference in a new issue