Fix multiple crashes for some corner cases.

Increase max block size for variable dedup block sizes greater than 16KB.
Update test cases and fix a test script bug.
This commit is contained in:
Moinak Ghosh 2013-08-09 21:55:06 +05:30
parent fe18afbcf4
commit f35d0ff4ef
4 changed files with 27 additions and 13 deletions

View file

@ -1700,7 +1700,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
unsigned short version, flags; unsigned short version, flags;
struct stat sbuf; struct stat sbuf;
int compfd = -1, uncompfd = -1, err; int compfd = -1, uncompfd = -1, err;
int thread, bail, single_chunk; int thread, wthread, bail, single_chunk;
uint32_t i, nprocs, np, p, dedupe_flag; uint32_t i, nprocs, np, p, dedupe_flag;
struct cmp_data **dary = NULL, *tdat; struct cmp_data **dary = NULL, *tdat;
pthread_t writer_thr; pthread_t writer_thr;
@ -1714,6 +1714,9 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
cread_buf = NULL; cread_buf = NULL;
flags = 0; flags = 0;
sbuf.st_size = 0; sbuf.st_size = 0;
err = 0;
thread = 0;
wthread = 0;
dedupe_flag = RABIN_DEDUPE_SEGMENTED; // Silence the compiler dedupe_flag = RABIN_DEDUPE_SEGMENTED; // Silence the compiler
if (pctx->encrypt_type) { if (pctx->encrypt_type) {
@ -1781,8 +1784,6 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
} }
} }
err = 0;
thread = 0;
single_chunk = 0; single_chunk = 0;
rctx = NULL; rctx = NULL;
@ -2078,6 +2079,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
perror("Error in thread creation: "); perror("Error in thread creation: ");
COMP_BAIL; COMP_BAIL;
} }
wthread = 1;
/* /*
* Write out file header. First insert hdr elements into mem buffer * Write out file header. First insert hdr elements into mem buffer
@ -2322,6 +2324,7 @@ comp_done:
if (pctx->encrypt_type) if (pctx->encrypt_type)
hmac_cleanup(&tdat->chunk_hmac); hmac_cleanup(&tdat->chunk_hmac);
} }
if (wthread)
pthread_join(writer_thr, NULL); pthread_join(writer_thr, NULL);
} }

View file

@ -227,7 +227,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
if (get_checksum_props(ck, &chunk_cksum, &cksum_bytes, &mac_bytes, 1) != 0 || if (get_checksum_props(ck, &chunk_cksum, &cksum_bytes, &mac_bytes, 1) != 0 ||
strcmp(ck, "CRC64") == 0) { strcmp(ck, "CRC64") == 0) {
fprintf(stderr, "Invalid PCOMPRESS_CHUNK_HASH_GLOBAL.\n"); fprintf(stderr, "Invalid PCOMPRESS_CHUNK_HASH_GLOBAL.\n");
chunk_cksum = -1; chunk_cksum = DEFAULT_CHUNK_CKSUM;
pthread_mutex_unlock(&init_lock); pthread_mutex_unlock(&init_lock);
return (NULL); return (NULL);
} }
@ -236,6 +236,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
chunk_cksum = DEFAULT_CHUNK_CKSUM; chunk_cksum = DEFAULT_CHUNK_CKSUM;
if (get_checksum_props(NULL, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0) { if (get_checksum_props(NULL, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0) {
fprintf(stderr, "Invalid default chunk checksum: %d\n", DEFAULT_CHUNK_CKSUM); fprintf(stderr, "Invalid default chunk checksum: %d\n", DEFAULT_CHUNK_CKSUM);
pthread_mutex_unlock(&init_lock);
return (NULL); return (NULL);
} }
} }
@ -259,11 +260,19 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
return (NULL); return (NULL);
} }
if (arc) {
if (chunksize < RAB_MIN_CHUNK_SIZE_GLOBAL) {
fprintf(stderr, "Minimum chunk size for Global Dedup must be %" PRIu64 " bytes\n",
RAB_MIN_CHUNK_SIZE_GLOBAL);
return (NULL);
}
} else {
if (chunksize < RAB_MIN_CHUNK_SIZE) { if (chunksize < RAB_MIN_CHUNK_SIZE) {
fprintf(stderr, "Minimum chunk size for Dedup must be %" PRIu64 " bytes\n", fprintf(stderr, "Minimum chunk size for Dedup must be %" PRIu64 " bytes\n",
RAB_MIN_CHUNK_SIZE); RAB_MIN_CHUNK_SIZE);
return (NULL); return (NULL);
} }
}
/* /*
* For LZMA with chunksize <= LZMA Window size and/or Delta enabled we * For LZMA with chunksize <= LZMA Window size and/or Delta enabled we
@ -286,6 +295,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
ctx->similarity_cksums = NULL; ctx->similarity_cksums = NULL;
if (arc) { if (arc) {
arc->pagesize = ctx->pagesize; arc->pagesize = ctx->pagesize;
if (rab_blk_sz < 3)
ctx->rabin_poly_max_block_size = RAB_POLY_MAX_BLOCK_SIZE_GLOBAL; ctx->rabin_poly_max_block_size = RAB_POLY_MAX_BLOCK_SIZE_GLOBAL;
} }

View file

@ -89,8 +89,9 @@
// to slide the window over every byte in the chunk. // to slide the window over every byte in the chunk.
#define RAB_WINDOW_SLIDE_OFFSET (64) #define RAB_WINDOW_SLIDE_OFFSET (64)
// Minimum practical chunk size when doing dedup // Minimum practical chunk sizes when doing dedup
#define RAB_MIN_CHUNK_SIZE (1048576L) #define RAB_MIN_CHUNK_SIZE (1048576L)
#define RAB_MIN_CHUNK_SIZE_GLOBAL (2097152L)
// An entry in the Rabin block array in the chunk. // An entry in the Rabin block array in the chunk.
// It is either a length value <= RABIN_MAX_BLOCK_SIZE or an index value with // It is either a length value <= RABIN_MAX_BLOCK_SIZE or an index value with

View file

@ -48,12 +48,12 @@ done
for algo in lz4 zlib for algo in lz4 zlib
do do
for dopts "" "-G -D" "-G -F" "-D" for dopts in "" "-G -D" "-G -F" "-D"
do do
for tf in `cat files.lst` for tf in `cat files.lst`
do do
rm -f ${tf}.* rm -f ${tf}.*
for seg in 1m 21m for seg in 2m 21m
do do
cmd="../../pcompress -c ${algo} -l6 -s ${seg} ${dopts} ${tf} - > ${tf}.pz" cmd="../../pcompress -c ${algo} -l6 -s ${seg} ${dopts} ${tf} - > ${tf}.pz"
echo "Running $cmd" echo "Running $cmd"