Fix multiple crashes for some corner cases.

Increase max block size for variable dedup block sizes greater than 16KB.
Update test cases and fix a test script bug.

pcompress.c

@@ -1700,7 +1700,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
 	unsigned short version, flags;
 	struct stat sbuf;
 	int compfd = -1, uncompfd = -1, err;
-	int thread, bail, single_chunk;
+	int thread, wthread, bail, single_chunk;
 	uint32_t i, nprocs, np, p, dedupe_flag;
 	struct cmp_data **dary = NULL, *tdat;
 	pthread_t writer_thr;
@@ -1714,6 +1714,9 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
 	cread_buf = NULL;
 	flags = 0;
 	sbuf.st_size = 0;
+	err = 0;
+	thread = 0;
+	wthread = 0;
 	dedupe_flag = RABIN_DEDUPE_SEGMENTED; // Silence the compiler
 
 	if (pctx->encrypt_type) {
@@ -1781,8 +1784,6 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
 		}
 	}
 
-	err = 0;
-	thread = 0;
 	single_chunk = 0;
 	rctx = NULL;
 
@@ -2078,6 +2079,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
 		perror("Error in thread creation: ");
 		COMP_BAIL;
 	}
+	wthread = 1;
 
 	/*
 	 * Write out file header. First insert hdr elements into mem buffer
@@ -2322,7 +2324,8 @@ comp_done:
 			if (pctx->encrypt_type)
 				hmac_cleanup(&tdat->chunk_hmac);
 		}
-		pthread_join(writer_thr, NULL);
+		if (wthread)
+			pthread_join(writer_thr, NULL);
 	}
 
 	if (err) {

rabin/rabin_dedup.c

@@ -227,7 +227,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
 				if (get_checksum_props(ck, &chunk_cksum, &cksum_bytes, &mac_bytes, 1) != 0 ||
 				    strcmp(ck, "CRC64") == 0) {
 					fprintf(stderr, "Invalid PCOMPRESS_CHUNK_HASH_GLOBAL.\n");
-					chunk_cksum = -1;
+					chunk_cksum = DEFAULT_CHUNK_CKSUM;
 					pthread_mutex_unlock(&init_lock);
 					return (NULL);
 				}
@@ -236,6 +236,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
 				chunk_cksum = DEFAULT_CHUNK_CKSUM;
 				if (get_checksum_props(NULL, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0) {
 					fprintf(stderr, "Invalid default chunk checksum: %d\n", DEFAULT_CHUNK_CKSUM);
+					pthread_mutex_unlock(&init_lock);
 					return (NULL);
 				}
 			}
@@ -259,10 +260,18 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
 		return (NULL);
 	}
 
-	if (chunksize < RAB_MIN_CHUNK_SIZE) {
-		fprintf(stderr, "Minimum chunk size for Dedup must be %" PRIu64 " bytes\n",
-		    RAB_MIN_CHUNK_SIZE);
-		return (NULL);
+	if (arc) {
+		if (chunksize < RAB_MIN_CHUNK_SIZE_GLOBAL) {
+			fprintf(stderr, "Minimum chunk size for Global Dedup must be %" PRIu64 " bytes\n",
+			RAB_MIN_CHUNK_SIZE_GLOBAL);
+			return (NULL);
+		}
+	} else {
+		if (chunksize < RAB_MIN_CHUNK_SIZE) {
+			fprintf(stderr, "Minimum chunk size for Dedup must be %" PRIu64 " bytes\n",
+			RAB_MIN_CHUNK_SIZE);
+			return (NULL);
+		}
 	}
 
 	/*
@@ -286,7 +295,8 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
 	ctx->similarity_cksums = NULL;
 	if (arc) {
 		arc->pagesize = ctx->pagesize;
-		ctx->rabin_poly_max_block_size = RAB_POLY_MAX_BLOCK_SIZE_GLOBAL;
+		if (rab_blk_sz < 3)
+			ctx->rabin_poly_max_block_size = RAB_POLY_MAX_BLOCK_SIZE_GLOBAL;
 	}
 
 	/*

rabin/rabin_dedup.h

@@ -89,8 +89,9 @@
 // to slide the window over every byte in the chunk.
 #define	RAB_WINDOW_SLIDE_OFFSET	(64)
 
-// Minimum practical chunk size when doing dedup
+// Minimum practical chunk sizes when doing dedup
 #define	RAB_MIN_CHUNK_SIZE (1048576L)
+#define	RAB_MIN_CHUNK_SIZE_GLOBAL (2097152L)
 
 // An entry in the Rabin block array in the chunk.
 // It is either a length value <= RABIN_MAX_BLOCK_SIZE or an index value with

test/t6.tst

@@ -48,12 +48,12 @@ done
 
 for algo in lz4 zlib
 do
-	for dopts "" "-G -D" "-G -F" "-D"
+	for dopts in "" "-G -D" "-G -F" "-D"
 	do
 		for tf in `cat files.lst`
 		do
 			rm -f ${tf}.*
-			for seg in 1m 21m
+			for seg in 2m 21m
 			do
 				cmd="../../pcompress -c ${algo} -l6 -s ${seg} ${dopts} ${tf} - > ${tf}.pz"
 				echo "Running $cmd"