Update Changelog and tweak free memory detection for 2.4 release.

Add identifiers to error messages for clarity.
Fix init of dedupe block size.
Tweak free memory detection to include swap and shared memory consideration.

Changelog

@@ -1,3 +1,28 @@
+== 2.4 Bugfix Release ==
+Add identifiers to error messages for clarity.
+Fix init of dedupe block size.
+Tweak free memory detection to include swap and shared memory consideration.
+Fix incorrect chunk size initialization from a previous commit.
+Remove confusing option with little practical utility.
+Update test cases and documentation.
+Additional error checks in RLE encoding for bsdiff extra data.
+Add a buffer overflow check in RLE encoder.
+Avoid calling RLE encoding if extra data length is zero.
+Make 2KB block size default for non-global deduplication.
+Update test cases for new 2KB block size support.
+Truncate password file after zeroing.
+Add more example usage.
+Avoid calling compression routine when dedupe reduces data size to zero.
+Default compression level only when compressing.
+Fix issue #11.
+Increase default chunk size to 8MB.
+Use default compression level of 1 (fast mode) for LZ4.
+Support for deduplication using 2KB block size.
+Add basic file format documentation.
+Reduce memory threshold for switching to Similarity based Deduplication.
+Avoid unnecessary re-hashing of 64-bit keys of the segment index.
+Update free memory computation to include cached buffers.
+Fix a potential rare corner case.
 == 2.3 Update Release ==
 Fix multiple crashes for some corner cases.
 Increase max block size for variable dedup block sizes greater than 16KB.

pcompress.c

@@ -1128,7 +1128,7 @@ start_decompress(pc_ctx_t *pctx, const char *filename, const char *to_filename)
 	for (i = 0; i < nprocs; i++) {
 		dary[i] = (struct cmp_data *)slab_alloc(NULL, sizeof (struct cmp_data));
 		if (!dary[i]) {
-			fprintf(stderr, "Out of memory\n");
+			fprintf(stderr, "1: Out of memory\n");
 			UNCOMP_BAIL;
 		}
 		tdat = dary[i];
@@ -1287,7 +1287,7 @@ start_decompress(pc_ctx_t *pctx, const char *filename, const char *to_filename)
 					tdat->uncompressed_chunk = (uchar_t *)slab_alloc(NULL,
 					    chunksize);
 				if (!tdat->compressed_chunk || !tdat->uncompressed_chunk) {
-					fprintf(stderr, "Out of memory\n");
+					fprintf(stderr, "2: Out of memory\n");
 					UNCOMP_BAIL;
 				}
 				tdat->cmp_seg = tdat->uncompressed_chunk;
@@ -1995,21 +1995,20 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
 	if (pctx->nthreads * props.nthreads > 1) fprintf(stderr, "s");
 	nprocs = pctx->nthreads;
 	fprintf(stderr, "\n");
-
 	dary = (struct cmp_data **)slab_calloc(NULL, nprocs, sizeof (struct cmp_data *));
 	if ((pctx->enable_rabin_scan || pctx->enable_fixed_scan))
 		cread_buf = (uchar_t *)slab_alloc(NULL, compressed_chunksize);
 	else
 		cread_buf = (uchar_t *)slab_alloc(NULL, chunksize);
 	if (!cread_buf) {
-		fprintf(stderr, "Out of memory\n");
+		fprintf(stderr, "3: Out of memory\n");
 		COMP_BAIL;
 	}
 
 	for (i = 0; i < nprocs; i++) {
 		dary[i] = (struct cmp_data *)slab_alloc(NULL, sizeof (struct cmp_data));
 		if (!dary[i]) {
-			fprintf(stderr, "Out of memory\n");
+			fprintf(stderr, "4: Out of memory\n");
 			COMP_BAIL;
 		}
 		tdat = dary[i];
@@ -2241,7 +2240,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev
 				tdat->compressed_chunk = tdat->cmp_seg + COMPRESSED_CHUNKSZ +
 				    pctx->cksum_bytes + pctx->mac_bytes;
 				if (!tdat->cmp_seg || !tdat->uncompressed_chunk) {
-					fprintf(stderr, "Out of memory\n");
+					fprintf(stderr, "5: Out of memory\n");
 					COMP_BAIL;
 				}
 			}
@@ -2759,6 +2758,8 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
 	if (pctx->rab_blk_size == -1) {
 		if (!pctx->enable_rabin_global)
 			pctx->rab_blk_size = 0;
+		else
+			pctx->rab_blk_size = RAB_BLK_DEFAULT;
 	}
 	/*
 	 * Remaining mandatory arguments are the filenames.

ppmd_compress.c

@@ -86,7 +86,7 @@ ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
 	_ppmd->Base = 0;
 	_ppmd->Size = 0;
 	if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[*level], &g_Alloc)) {
-		fprintf(stderr, "Out of memory.\n");
+		fprintf(stderr, "PPMD: Out of memory.\n");
 		return (-1);
 	}
 	Ppmd8_Construct(_ppmd);

utils/utils.c

@@ -389,6 +389,7 @@ void
 get_sys_limits(my_sysinfo *msys_info)
 {
 	struct sysinfo sys_info;
+	unsigned long totram;
 	int rv;
 	char *val;
 
@@ -402,6 +403,18 @@ get_sys_limits(my_sysinfo *msys_info)
 	msys_info->totalswap = sys_info.totalswap * sys_info.mem_unit;
 	msys_info->freeswap = sys_info.freeswap * sys_info.mem_unit;
 	msys_info->mem_unit = sys_info.mem_unit;
+	msys_info->sharedram = sys_info.sharedram * sys_info.mem_unit;
+
+	/*
+	 * If free memory is less than half of total memory (excluding shared allocations),
+	 * and at least 75% of swap is free then adjust free memory value to 75% of
+	 * total memory excluding shared allocations.
+	 */
+	totram = msys_info->totalram - sys_info.sharedram;
+	if (msys_info->freeram <= (totram >> 1) &&
+	    msys_info->freeswap >= ((msys_info->totalswap >> 1) + (msys_info->totalswap >> 2))) {
+		msys_info->freeram = (totram >> 1) + (totram >> 2);
+	}
 
 	if ((val = getenv("PCOMPRESS_INDEX_MEM")) != NULL) {
 		uint64_t mem;

utils/utils.h

@@ -212,6 +212,7 @@ typedef struct{
 	int64_t totalswap;
 	int64_t freeswap;
 	int64_t mem_unit;
+	int64_t sharedram;
 } my_sysinfo;
 
 #ifndef _IN_UTILS_