diff --git a/Changelog b/Changelog index 0857f5e..e152506 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,28 @@ +== 2.4 Bugfix Release == +Add identifiers to error messages for clarity. +Fix init of dedupe block size. +Tweak free memory detection to include swap and shared memory consideration. +Fix incorrect chunk size initialization from a previous commit. +Remove confusing option with little practical utility. +Update test cases and documentation. +Additional error checks in RLE encoding for bsdiff extra data. +Add a buffer overflow check in RLE encoder. +Avoid calling RLE encoding if extra data length is zero. +Make 2KB block size default for non-global deduplication. +Update test cases for new 2KB block size support. +Truncate password file after zeroing. +Add more example usage. +Avoid calling compression routine when dedupe reduces data size to zero. +Default compression level only when compressing. +Fix issue #11. +Increase default chunk size to 8MB. +Use default compression level of 1 (fast mode) for LZ4. +Support for deduplication using 2KB block size. +Add basic file format documentation. +Reduce memory threshold for switching to Similarity based Deduplication. +Avoid unnecessary re-hashing of 64-bit keys of the segment index. +Update free memory computation to include cached buffers. +Fix a potential rare corner case. == 2.3 Update Release == Fix multiple crashes for some corner cases. Increase max block size for variable dedup block sizes greater than 16KB. diff --git a/pcompress.c b/pcompress.c index aa21414..f24b896 100644 --- a/pcompress.c +++ b/pcompress.c @@ -1128,7 +1128,7 @@ start_decompress(pc_ctx_t *pctx, const char *filename, const char *to_filename) for (i = 0; i < nprocs; i++) { dary[i] = (struct cmp_data *)slab_alloc(NULL, sizeof (struct cmp_data)); if (!dary[i]) { - fprintf(stderr, "Out of memory\n"); + fprintf(stderr, "1: Out of memory\n"); UNCOMP_BAIL; } tdat = dary[i]; @@ -1287,7 +1287,7 @@ start_decompress(pc_ctx_t *pctx, const char *filename, const char *to_filename) tdat->uncompressed_chunk = (uchar_t *)slab_alloc(NULL, chunksize); if (!tdat->compressed_chunk || !tdat->uncompressed_chunk) { - fprintf(stderr, "Out of memory\n"); + fprintf(stderr, "2: Out of memory\n"); UNCOMP_BAIL; } tdat->cmp_seg = tdat->uncompressed_chunk; @@ -1995,21 +1995,20 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev if (pctx->nthreads * props.nthreads > 1) fprintf(stderr, "s"); nprocs = pctx->nthreads; fprintf(stderr, "\n"); - dary = (struct cmp_data **)slab_calloc(NULL, nprocs, sizeof (struct cmp_data *)); if ((pctx->enable_rabin_scan || pctx->enable_fixed_scan)) cread_buf = (uchar_t *)slab_alloc(NULL, compressed_chunksize); else cread_buf = (uchar_t *)slab_alloc(NULL, chunksize); if (!cread_buf) { - fprintf(stderr, "Out of memory\n"); + fprintf(stderr, "3: Out of memory\n"); COMP_BAIL; } for (i = 0; i < nprocs; i++) { dary[i] = (struct cmp_data *)slab_alloc(NULL, sizeof (struct cmp_data)); if (!dary[i]) { - fprintf(stderr, "Out of memory\n"); + fprintf(stderr, "4: Out of memory\n"); COMP_BAIL; } tdat = dary[i]; @@ -2241,7 +2240,7 @@ start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int lev tdat->compressed_chunk = tdat->cmp_seg + COMPRESSED_CHUNKSZ + pctx->cksum_bytes + pctx->mac_bytes; if (!tdat->cmp_seg || !tdat->uncompressed_chunk) { - fprintf(stderr, "Out of memory\n"); + fprintf(stderr, "5: Out of memory\n"); COMP_BAIL; } } @@ -2759,6 +2758,8 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) if (pctx->rab_blk_size == -1) { if (!pctx->enable_rabin_global) pctx->rab_blk_size = 0; + else + pctx->rab_blk_size = RAB_BLK_DEFAULT; } /* * Remaining mandatory arguments are the filenames. diff --git a/ppmd_compress.c b/ppmd_compress.c index e2cd618..88eca76 100644 --- a/ppmd_compress.c +++ b/ppmd_compress.c @@ -86,7 +86,7 @@ ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize, _ppmd->Base = 0; _ppmd->Size = 0; if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[*level], &g_Alloc)) { - fprintf(stderr, "Out of memory.\n"); + fprintf(stderr, "PPMD: Out of memory.\n"); return (-1); } Ppmd8_Construct(_ppmd); diff --git a/utils/utils.c b/utils/utils.c index f647db3..aa25b18 100644 --- a/utils/utils.c +++ b/utils/utils.c @@ -389,6 +389,7 @@ void get_sys_limits(my_sysinfo *msys_info) { struct sysinfo sys_info; + unsigned long totram; int rv; char *val; @@ -402,6 +403,18 @@ get_sys_limits(my_sysinfo *msys_info) msys_info->totalswap = sys_info.totalswap * sys_info.mem_unit; msys_info->freeswap = sys_info.freeswap * sys_info.mem_unit; msys_info->mem_unit = sys_info.mem_unit; + msys_info->sharedram = sys_info.sharedram * sys_info.mem_unit; + + /* + * If free memory is less than half of total memory (excluding shared allocations), + * and at least 75% of swap is free then adjust free memory value to 75% of + * total memory excluding shared allocations. + */ + totram = msys_info->totalram - sys_info.sharedram; + if (msys_info->freeram <= (totram >> 1) && + msys_info->freeswap >= ((msys_info->totalswap >> 1) + (msys_info->totalswap >> 2))) { + msys_info->freeram = (totram >> 1) + (totram >> 2); + } if ((val = getenv("PCOMPRESS_INDEX_MEM")) != NULL) { uint64_t mem; diff --git a/utils/utils.h b/utils/utils.h index 3786940..d466623 100644 --- a/utils/utils.h +++ b/utils/utils.h @@ -212,6 +212,7 @@ typedef struct{ int64_t totalswap; int64_t freeswap; int64_t mem_unit; + int64_t sharedram; } my_sysinfo; #ifndef _IN_UTILS_