Fix multiple crashes for some corner cases.
Increase max block size for variable dedup block sizes greater than 16KB. Update test cases and fix a test script bug.
This commit is contained in:
parent
fe18afbcf4
commit
f35d0ff4ef
4 changed files with 27 additions and 13 deletions
11
pcompress.c
11
pcompress.c
|
@ -1700,7 +1700,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
|
|||
unsigned short version, flags;
|
||||
struct stat sbuf;
|
||||
int compfd = -1, uncompfd = -1, err;
|
||||
int thread, bail, single_chunk;
|
||||
int thread, wthread, bail, single_chunk;
|
||||
uint32_t i, nprocs, np, p, dedupe_flag;
|
||||
struct cmp_data **dary = NULL, *tdat;
|
||||
pthread_t writer_thr;
|
||||
|
@ -1714,6 +1714,9 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
|
|||
cread_buf = NULL;
|
||||
flags = 0;
|
||||
sbuf.st_size = 0;
|
||||
err = 0;
|
||||
thread = 0;
|
||||
wthread = 0;
|
||||
dedupe_flag = RABIN_DEDUPE_SEGMENTED; // Silence the compiler
|
||||
|
||||
if (pctx->encrypt_type) {
|
||||
|
@ -1781,8 +1784,6 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
|
|||
}
|
||||
}
|
||||
|
||||
err = 0;
|
||||
thread = 0;
|
||||
single_chunk = 0;
|
||||
rctx = NULL;
|
||||
|
||||
|
@ -2078,6 +2079,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
|
|||
perror("Error in thread creation: ");
|
||||
COMP_BAIL;
|
||||
}
|
||||
wthread = 1;
|
||||
|
||||
/*
|
||||
* Write out file header. First insert hdr elements into mem buffer
|
||||
|
@ -2322,7 +2324,8 @@ comp_done:
|
|||
if (pctx->encrypt_type)
|
||||
hmac_cleanup(&tdat->chunk_hmac);
|
||||
}
|
||||
pthread_join(writer_thr, NULL);
|
||||
if (wthread)
|
||||
pthread_join(writer_thr, NULL);
|
||||
}
|
||||
|
||||
if (err) {
|
||||
|
|
|
@ -227,7 +227,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
|
|||
if (get_checksum_props(ck, &chunk_cksum, &cksum_bytes, &mac_bytes, 1) != 0 ||
|
||||
strcmp(ck, "CRC64") == 0) {
|
||||
fprintf(stderr, "Invalid PCOMPRESS_CHUNK_HASH_GLOBAL.\n");
|
||||
chunk_cksum = -1;
|
||||
chunk_cksum = DEFAULT_CHUNK_CKSUM;
|
||||
pthread_mutex_unlock(&init_lock);
|
||||
return (NULL);
|
||||
}
|
||||
|
@ -236,6 +236,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
|
|||
chunk_cksum = DEFAULT_CHUNK_CKSUM;
|
||||
if (get_checksum_props(NULL, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0) {
|
||||
fprintf(stderr, "Invalid default chunk checksum: %d\n", DEFAULT_CHUNK_CKSUM);
|
||||
pthread_mutex_unlock(&init_lock);
|
||||
return (NULL);
|
||||
}
|
||||
}
|
||||
|
@ -259,10 +260,18 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
|
|||
return (NULL);
|
||||
}
|
||||
|
||||
if (chunksize < RAB_MIN_CHUNK_SIZE) {
|
||||
fprintf(stderr, "Minimum chunk size for Dedup must be %" PRIu64 " bytes\n",
|
||||
RAB_MIN_CHUNK_SIZE);
|
||||
return (NULL);
|
||||
if (arc) {
|
||||
if (chunksize < RAB_MIN_CHUNK_SIZE_GLOBAL) {
|
||||
fprintf(stderr, "Minimum chunk size for Global Dedup must be %" PRIu64 " bytes\n",
|
||||
RAB_MIN_CHUNK_SIZE_GLOBAL);
|
||||
return (NULL);
|
||||
}
|
||||
} else {
|
||||
if (chunksize < RAB_MIN_CHUNK_SIZE) {
|
||||
fprintf(stderr, "Minimum chunk size for Dedup must be %" PRIu64 " bytes\n",
|
||||
RAB_MIN_CHUNK_SIZE);
|
||||
return (NULL);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -286,7 +295,8 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
|
|||
ctx->similarity_cksums = NULL;
|
||||
if (arc) {
|
||||
arc->pagesize = ctx->pagesize;
|
||||
ctx->rabin_poly_max_block_size = RAB_POLY_MAX_BLOCK_SIZE_GLOBAL;
|
||||
if (rab_blk_sz < 3)
|
||||
ctx->rabin_poly_max_block_size = RAB_POLY_MAX_BLOCK_SIZE_GLOBAL;
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
@ -89,8 +89,9 @@
|
|||
// to slide the window over every byte in the chunk.
|
||||
#define RAB_WINDOW_SLIDE_OFFSET (64)
|
||||
|
||||
// Minimum practical chunk size when doing dedup
|
||||
// Minimum practical chunk sizes when doing dedup
|
||||
#define RAB_MIN_CHUNK_SIZE (1048576L)
|
||||
#define RAB_MIN_CHUNK_SIZE_GLOBAL (2097152L)
|
||||
|
||||
// An entry in the Rabin block array in the chunk.
|
||||
// It is either a length value <= RABIN_MAX_BLOCK_SIZE or an index value with
|
||||
|
|
|
@ -48,12 +48,12 @@ done
|
|||
|
||||
for algo in lz4 zlib
|
||||
do
|
||||
for dopts "" "-G -D" "-G -F" "-D"
|
||||
for dopts in "" "-G -D" "-G -F" "-D"
|
||||
do
|
||||
for tf in `cat files.lst`
|
||||
do
|
||||
rm -f ${tf}.*
|
||||
for seg in 1m 21m
|
||||
for seg in 2m 21m
|
||||
do
|
||||
cmd="../../pcompress -c ${algo} -l6 -s ${seg} ${dopts} ${tf} - > ${tf}.pz"
|
||||
echo "Running $cmd"
|
||||
|
|
Loading…
Reference in a new issue