Fix multiple crashes for some corner cases.
Increase max block size for variable dedup block sizes greater than 16KB. Update test cases and fix a test script bug.
This commit is contained in:
parent
fe18afbcf4
commit
f35d0ff4ef
4 changed files with 27 additions and 13 deletions
|
@ -1700,7 +1700,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
|
||||||
unsigned short version, flags;
|
unsigned short version, flags;
|
||||||
struct stat sbuf;
|
struct stat sbuf;
|
||||||
int compfd = -1, uncompfd = -1, err;
|
int compfd = -1, uncompfd = -1, err;
|
||||||
int thread, bail, single_chunk;
|
int thread, wthread, bail, single_chunk;
|
||||||
uint32_t i, nprocs, np, p, dedupe_flag;
|
uint32_t i, nprocs, np, p, dedupe_flag;
|
||||||
struct cmp_data **dary = NULL, *tdat;
|
struct cmp_data **dary = NULL, *tdat;
|
||||||
pthread_t writer_thr;
|
pthread_t writer_thr;
|
||||||
|
@ -1714,6 +1714,9 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
|
||||||
cread_buf = NULL;
|
cread_buf = NULL;
|
||||||
flags = 0;
|
flags = 0;
|
||||||
sbuf.st_size = 0;
|
sbuf.st_size = 0;
|
||||||
|
err = 0;
|
||||||
|
thread = 0;
|
||||||
|
wthread = 0;
|
||||||
dedupe_flag = RABIN_DEDUPE_SEGMENTED; // Silence the compiler
|
dedupe_flag = RABIN_DEDUPE_SEGMENTED; // Silence the compiler
|
||||||
|
|
||||||
if (pctx->encrypt_type) {
|
if (pctx->encrypt_type) {
|
||||||
|
@ -1781,8 +1784,6 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = 0;
|
|
||||||
thread = 0;
|
|
||||||
single_chunk = 0;
|
single_chunk = 0;
|
||||||
rctx = NULL;
|
rctx = NULL;
|
||||||
|
|
||||||
|
@ -2078,6 +2079,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
|
||||||
perror("Error in thread creation: ");
|
perror("Error in thread creation: ");
|
||||||
COMP_BAIL;
|
COMP_BAIL;
|
||||||
}
|
}
|
||||||
|
wthread = 1;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Write out file header. First insert hdr elements into mem buffer
|
* Write out file header. First insert hdr elements into mem buffer
|
||||||
|
@ -2322,6 +2324,7 @@ comp_done:
|
||||||
if (pctx->encrypt_type)
|
if (pctx->encrypt_type)
|
||||||
hmac_cleanup(&tdat->chunk_hmac);
|
hmac_cleanup(&tdat->chunk_hmac);
|
||||||
}
|
}
|
||||||
|
if (wthread)
|
||||||
pthread_join(writer_thr, NULL);
|
pthread_join(writer_thr, NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -227,7 +227,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
|
||||||
if (get_checksum_props(ck, &chunk_cksum, &cksum_bytes, &mac_bytes, 1) != 0 ||
|
if (get_checksum_props(ck, &chunk_cksum, &cksum_bytes, &mac_bytes, 1) != 0 ||
|
||||||
strcmp(ck, "CRC64") == 0) {
|
strcmp(ck, "CRC64") == 0) {
|
||||||
fprintf(stderr, "Invalid PCOMPRESS_CHUNK_HASH_GLOBAL.\n");
|
fprintf(stderr, "Invalid PCOMPRESS_CHUNK_HASH_GLOBAL.\n");
|
||||||
chunk_cksum = -1;
|
chunk_cksum = DEFAULT_CHUNK_CKSUM;
|
||||||
pthread_mutex_unlock(&init_lock);
|
pthread_mutex_unlock(&init_lock);
|
||||||
return (NULL);
|
return (NULL);
|
||||||
}
|
}
|
||||||
|
@ -236,6 +236,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
|
||||||
chunk_cksum = DEFAULT_CHUNK_CKSUM;
|
chunk_cksum = DEFAULT_CHUNK_CKSUM;
|
||||||
if (get_checksum_props(NULL, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0) {
|
if (get_checksum_props(NULL, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0) {
|
||||||
fprintf(stderr, "Invalid default chunk checksum: %d\n", DEFAULT_CHUNK_CKSUM);
|
fprintf(stderr, "Invalid default chunk checksum: %d\n", DEFAULT_CHUNK_CKSUM);
|
||||||
|
pthread_mutex_unlock(&init_lock);
|
||||||
return (NULL);
|
return (NULL);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -259,11 +260,19 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
|
||||||
return (NULL);
|
return (NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (arc) {
|
||||||
|
if (chunksize < RAB_MIN_CHUNK_SIZE_GLOBAL) {
|
||||||
|
fprintf(stderr, "Minimum chunk size for Global Dedup must be %" PRIu64 " bytes\n",
|
||||||
|
RAB_MIN_CHUNK_SIZE_GLOBAL);
|
||||||
|
return (NULL);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
if (chunksize < RAB_MIN_CHUNK_SIZE) {
|
if (chunksize < RAB_MIN_CHUNK_SIZE) {
|
||||||
fprintf(stderr, "Minimum chunk size for Dedup must be %" PRIu64 " bytes\n",
|
fprintf(stderr, "Minimum chunk size for Dedup must be %" PRIu64 " bytes\n",
|
||||||
RAB_MIN_CHUNK_SIZE);
|
RAB_MIN_CHUNK_SIZE);
|
||||||
return (NULL);
|
return (NULL);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* For LZMA with chunksize <= LZMA Window size and/or Delta enabled we
|
* For LZMA with chunksize <= LZMA Window size and/or Delta enabled we
|
||||||
|
@ -286,6 +295,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
|
||||||
ctx->similarity_cksums = NULL;
|
ctx->similarity_cksums = NULL;
|
||||||
if (arc) {
|
if (arc) {
|
||||||
arc->pagesize = ctx->pagesize;
|
arc->pagesize = ctx->pagesize;
|
||||||
|
if (rab_blk_sz < 3)
|
||||||
ctx->rabin_poly_max_block_size = RAB_POLY_MAX_BLOCK_SIZE_GLOBAL;
|
ctx->rabin_poly_max_block_size = RAB_POLY_MAX_BLOCK_SIZE_GLOBAL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -89,8 +89,9 @@
|
||||||
// to slide the window over every byte in the chunk.
|
// to slide the window over every byte in the chunk.
|
||||||
#define RAB_WINDOW_SLIDE_OFFSET (64)
|
#define RAB_WINDOW_SLIDE_OFFSET (64)
|
||||||
|
|
||||||
// Minimum practical chunk size when doing dedup
|
// Minimum practical chunk sizes when doing dedup
|
||||||
#define RAB_MIN_CHUNK_SIZE (1048576L)
|
#define RAB_MIN_CHUNK_SIZE (1048576L)
|
||||||
|
#define RAB_MIN_CHUNK_SIZE_GLOBAL (2097152L)
|
||||||
|
|
||||||
// An entry in the Rabin block array in the chunk.
|
// An entry in the Rabin block array in the chunk.
|
||||||
// It is either a length value <= RABIN_MAX_BLOCK_SIZE or an index value with
|
// It is either a length value <= RABIN_MAX_BLOCK_SIZE or an index value with
|
||||||
|
|
|
@ -48,12 +48,12 @@ done
|
||||||
|
|
||||||
for algo in lz4 zlib
|
for algo in lz4 zlib
|
||||||
do
|
do
|
||||||
for dopts "" "-G -D" "-G -F" "-D"
|
for dopts in "" "-G -D" "-G -F" "-D"
|
||||||
do
|
do
|
||||||
for tf in `cat files.lst`
|
for tf in `cat files.lst`
|
||||||
do
|
do
|
||||||
rm -f ${tf}.*
|
rm -f ${tf}.*
|
||||||
for seg in 1m 21m
|
for seg in 2m 21m
|
||||||
do
|
do
|
||||||
cmd="../../pcompress -c ${algo} -l6 -s ${seg} ${dopts} ${tf} - > ${tf}.pz"
|
cmd="../../pcompress -c ${algo} -l6 -s ${seg} ${dopts} ${tf} - > ${tf}.pz"
|
||||||
echo "Running $cmd"
|
echo "Running $cmd"
|
||||||
|
|
Loading…
Reference in a new issue