greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix multiple crashes for some corner cases.

Browse source 
Increase max block size for variable dedup block sizes greater than 16KB.
Update test cases and fix a test script bug.
This commit is contained in:
Moinak Ghosh 2013-08-09 21:55:06 +05:30
parent fe18afbcf4
commit f35d0ff4ef
4 changed files with 27 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
pcompress.c
View file

@ -1700,7 +1700,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
unsigned short version, flags; unsigned short version, flags;
struct stat sbuf; struct stat sbuf;
int compfd = -1, uncompfd = -1, err; int compfd = -1, uncompfd = -1, err;
int thread, bail, single_chunk; int thread, wthread, bail, single_chunk;
uint32_t i, nprocs, np, p, dedupe_flag; uint32_t i, nprocs, np, p, dedupe_flag;
struct cmp_data **dary = NULL, *tdat; struct cmp_data **dary = NULL, *tdat;
pthread_t writer_thr; pthread_t writer_thr;
@ -1714,6 +1714,9 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
cread_buf = NULL; cread_buf = NULL;
flags = 0; flags = 0;
sbuf.st_size = 0; sbuf.st_size = 0;
err = 0;
thread = 0;
wthread = 0;
dedupe_flag = RABIN_DEDUPE_SEGMENTED; // Silence the compiler dedupe_flag = RABIN_DEDUPE_SEGMENTED; // Silence the compiler
if (pctx->encrypt_type) { if (pctx->encrypt_type) {
@ -1781,8 +1784,6 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
} }
} }
err = 0;
thread = 0;
single_chunk = 0; single_chunk = 0;
rctx = NULL; rctx = NULL;
@ -2078,6 +2079,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
perror("Error in thread creation: "); perror("Error in thread creation: ");
COMP_BAIL; COMP_BAIL;
} }
wthread = 1;
/* /*
* Write out file header. First insert hdr elements into mem buffer * Write out file header. First insert hdr elements into mem buffer
@ -2322,6 +2324,7 @@ comp_done:
if (pctx->encrypt_type) if (pctx->encrypt_type)
hmac_cleanup(&tdat->chunk_hmac); hmac_cleanup(&tdat->chunk_hmac);
} }
if (wthread)
pthread_join(writer_thr, NULL); pthread_join(writer_thr, NULL);
} }

12
rabin/rabin_dedup.c
View file

@ -227,7 +227,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
if (get_checksum_props(ck, &chunk_cksum, &cksum_bytes, &mac_bytes, 1) != 0 || if (get_checksum_props(ck, &chunk_cksum, &cksum_bytes, &mac_bytes, 1) != 0 ||
strcmp(ck, "CRC64") == 0) { strcmp(ck, "CRC64") == 0) {
fprintf(stderr, "Invalid PCOMPRESS_CHUNK_HASH_GLOBAL.\n"); fprintf(stderr, "Invalid PCOMPRESS_CHUNK_HASH_GLOBAL.\n");
chunk_cksum = -1; chunk_cksum = DEFAULT_CHUNK_CKSUM;
pthread_mutex_unlock(&init_lock); pthread_mutex_unlock(&init_lock);
return (NULL); return (NULL);
} }
@ -236,6 +236,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
chunk_cksum = DEFAULT_CHUNK_CKSUM; chunk_cksum = DEFAULT_CHUNK_CKSUM;
if (get_checksum_props(NULL, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0) { if (get_checksum_props(NULL, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0) {
fprintf(stderr, "Invalid default chunk checksum: %d\n", DEFAULT_CHUNK_CKSUM); fprintf(stderr, "Invalid default chunk checksum: %d\n", DEFAULT_CHUNK_CKSUM);
pthread_mutex_unlock(&init_lock);
return (NULL); return (NULL);
} }
} }
@ -259,11 +260,19 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
return (NULL); return (NULL);
} }
if (arc) {
if (chunksize < RAB_MIN_CHUNK_SIZE_GLOBAL) {
fprintf(stderr, "Minimum chunk size for Global Dedup must be %" PRIu64 " bytes\n",
RAB_MIN_CHUNK_SIZE_GLOBAL);
return (NULL);
}
} else {
if (chunksize < RAB_MIN_CHUNK_SIZE) { if (chunksize < RAB_MIN_CHUNK_SIZE) {
fprintf(stderr, "Minimum chunk size for Dedup must be %" PRIu64 " bytes\n", fprintf(stderr, "Minimum chunk size for Dedup must be %" PRIu64 " bytes\n",
RAB_MIN_CHUNK_SIZE); RAB_MIN_CHUNK_SIZE);
return (NULL); return (NULL);
} }
}
/* /*
* For LZMA with chunksize <= LZMA Window size and/or Delta enabled we * For LZMA with chunksize <= LZMA Window size and/or Delta enabled we
@ -286,6 +295,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
ctx->similarity_cksums = NULL; ctx->similarity_cksums = NULL;
if (arc) { if (arc) {
arc->pagesize = ctx->pagesize; arc->pagesize = ctx->pagesize;
if (rab_blk_sz < 3)
ctx->rabin_poly_max_block_size = RAB_POLY_MAX_BLOCK_SIZE_GLOBAL; ctx->rabin_poly_max_block_size = RAB_POLY_MAX_BLOCK_SIZE_GLOBAL;
} }

3
rabin/rabin_dedup.h
View file

@ -89,8 +89,9 @@
// to slide the window over every byte in the chunk. // to slide the window over every byte in the chunk.
#define RAB_WINDOW_SLIDE_OFFSET (64) #define RAB_WINDOW_SLIDE_OFFSET (64)
// Minimum practical chunk size when doing dedup // Minimum practical chunk sizes when doing dedup
#define RAB_MIN_CHUNK_SIZE (1048576L) #define RAB_MIN_CHUNK_SIZE (1048576L)
#define RAB_MIN_CHUNK_SIZE_GLOBAL (2097152L)
// An entry in the Rabin block array in the chunk. // An entry in the Rabin block array in the chunk.
// It is either a length value <= RABIN_MAX_BLOCK_SIZE or an index value with // It is either a length value <= RABIN_MAX_BLOCK_SIZE or an index value with

4
test/t6.tst
View file

@ -48,12 +48,12 @@ done
for algo in lz4 zlib for algo in lz4 zlib
do do
for dopts "" "-G -D" "-G -F" "-D" for dopts in "" "-G -D" "-G -F" "-D"
do do
for tf in `cat files.lst` for tf in `cat files.lst`
do do
rm -f ${tf}.* rm -f ${tf}.*
for seg in 1m 21m for seg in 2m 21m
do do
cmd="../../pcompress -c ${algo} -l6 -s ${seg} ${dopts} ${tf} - > ${tf}.pz" cmd="../../pcompress -c ${algo} -l6 -s ${seg} ${dopts} ${tf} - > ${tf}.pz"
echo "Running $cmd" echo "Running $cmd"