Free PPMD buffer after compression, rather than caching.

Introduce new API in allocator to release buffer to OS.
Release LZMA buffers after use.

adaptive_compress.c

@@ -82,6 +82,10 @@ extern int lz4_init(void **data, int *level, int nthreads, uint64_t chunksize,
 		       int file_version, compress_op_t op);
 extern int lz4_deinit(void **data);
 
+extern int ppmd_alloc(void *data);
+extern void ppmd_free(void *data);
+extern int ppmd_state_init(void **data, int *level, int alloc);
+
 struct adapt_data {
 	void *lzma_data;
 	void *ppmd_data;
@@ -129,7 +133,7 @@ adapt_init(void **data, int *level, int nthreads, uint64_t chunksize,
 	if (!adat) {
 		adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data));
 		adat->adapt_mode = 1;
-		rv = ppmd_init(&(adat->ppmd_data), level, nthreads, chunksize, file_version, op);
+		rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
 
 		/*
 		 * LZ4 is used to tackle some embedded archive headers and/or zero paddings in
@@ -165,7 +169,7 @@ adapt2_init(void **data, int *level, int nthreads, uint64_t chunksize,
 		adat->bsc_data = NULL;
 		lv = *level;
 		if (lv > 10) lv = 10;
-		rv = ppmd_init(&(adat->ppmd_data), &lv, nthreads, chunksize, file_version, op);
+		rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
 		lv = *level;
 		if (rv == 0)
 			rv = lzma_init(&(adat->lzma_data), &lv, nthreads, chunksize, file_version, op);
@@ -305,7 +309,11 @@ adapt_compress(void *src, uint64_t srclen, void *dst,
 			bsc_count++;
 		} else {
 #endif
+			rv = ppmd_alloc(adat->ppmd_data);
+			if (rv < 0)
+				return (rv);
 			rv = ppmd_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->ppmd_data);
+			ppmd_free(adat->ppmd_data);
 			if (rv < 0)
 				return (rv);
 			rv = ADAPT_COMPRESS_PPMD;

allocator.c

@@ -444,8 +444,8 @@ slab_alloc(void *p, uint64_t size)
 	}
 }
 
-void
+static void
-slab_free(void *p, void *address)
+slab_free_real(void *p, void *address, int do_free)
 {
 	struct bufentry *buf, *pbuf;
 	int found = 0;
@@ -471,7 +471,7 @@ slab_free(void *p, void *address)
 			pthread_mutex_unlock(&hbucket_locks[hindx]);
 			ATOMIC_SUB(hash_entries, 1);
 
-			if (buf->slab == NULL) {
+			if (buf->slab == NULL || do_free) {
 				free(buf->ptr);
 				free(buf);
 				found = 1;
@@ -497,6 +497,18 @@ slab_free(void *p, void *address)
 	}
 }
 
+void
+slab_free(void *p, void *address)
+{
+	slab_free_real(p, address, 0);
+}
+
+void
+slab_release(void *p, void *address)
+{
+	slab_free_real(p, address, 1);
+}
+
 #else
 void
 slab_init() {}
@@ -522,6 +534,12 @@ slab_free(void *p, void *address)
 	free(address);
 }
 
+void
+slab_release(void *p, void *address)
+{
+	free(address);
+}
+
 int
 slab_cache_add(uint64_t size)
 {

allocator.h

@@ -34,6 +34,7 @@ void slab_cleanup(int quiet);
 void *slab_alloc(void *p, uint64_t size);
 void *slab_calloc(void *p, uint64_t items, uint64_t size);
 void slab_free(void *p, void *address);
+void slab_release(void *p, void *address);
 int slab_cache_add(uint64_t size);
 
 #endif

lzma/Ppmd8.c

@@ -1,28 +1,28 @@
-/*
+/*
- * This file is a part of Pcompress, a chunked parallel multi-
+ * This file is a part of Pcompress, a chunked parallel multi-
- * algorithm lossless compression and decompression program.
+ * algorithm lossless compression and decompression program.
- *
+ *
- * Copyright (C) 2012-2013 Moinak Ghosh. All rights reserved.
+ * Copyright (C) 2012-2013 Moinak Ghosh. All rights reserved.
- * Use is subject to license terms.
+ * Use is subject to license terms.
- *
+ *
- * This program is free software; you can redistribute it and/or
+ * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
+ * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
+ * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
+ * version 3 of the License, or (at your option) any later version.
- *
+ *
- * This program is distributed in the hope that it will be useful,
+ * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
+ * Lesser General Public License for more details.
- *
+ *
- * You should have received a copy of the GNU Lesser General Public
+ * You should have received a copy of the GNU Lesser General Public
- * License along with this program.
+ * License along with this program.
- * If not, see <http://www.gnu.org/licenses/>.
+ * If not, see <http://www.gnu.org/licenses/>.
- *
+ *
- * moinakg@belenix.org, http://moinakg.wordpress.com/
+ * moinakg@belenix.org, http://moinakg.wordpress.com/
- *      
+ *      
- */
+ */
-
+
 /* Ppmd8.c -- PPMdI codec
 2010-03-24 : Igor Pavlov : Public domain
 This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */
@@ -109,9 +109,11 @@ void Ppmd8_Construct(CPpmd8 *p)
 
 void Ppmd8_Free(CPpmd8 *p, ISzAlloc *alloc)
 {
-  alloc->Free(alloc, p->Base);
+  if (p->Base != 0) {
-  p->Size = 0;
+    alloc->Free(alloc, p->Base);
-  p->Base = 0;
+    p->Size = 0;
+    p->Base = 0;
+  }
 }
 
 Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAlloc *alloc)

lzma_compress.c

@@ -39,7 +39,7 @@ CLzmaEncProps *p = NULL;
 
 static ISzAlloc g_Alloc = {
 	slab_alloc,
-	slab_free,
+	slab_release,
 	NULL
 };
 
@@ -140,7 +140,7 @@ int
 lzma_deinit(void **data)
 {
 	if (p) {
-		slab_free(NULL, p);
+		slab_release(NULL, p);
 		p = NULL;
 	}
 	*data = NULL;

ppmd_compress.c

@@ -58,10 +58,59 @@ static int mem_inited = 0;
 
 static ISzAlloc g_Alloc = {
 	slab_alloc,
-	slab_free,
+	slab_release,
 	NULL
 };
 
+int
+ppmd_alloc(void *data)
+{
+	CPpmd8 *_ppmd = (CPpmd8 *)data;
+
+	if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[_ppmd->Order], &g_Alloc)) {
+		log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
+		return (-1);
+	}
+	Ppmd8_Construct(_ppmd);
+	return (0);
+}
+
+void
+ppmd_free(void *data)
+{
+	CPpmd8 *_ppmd = (CPpmd8 *)data;
+	Ppmd8_Free(_ppmd, &g_Alloc);
+}
+
+int
+ppmd_state_init(void **data, int *level, int alloc)
+{
+	CPpmd8 *_ppmd;
+
+	pthread_mutex_lock(&mem_init_lock);
+	if (!mem_inited) {
+		slab_cache_add(sizeof (CPpmd8));
+		slab_cache_add(ppmd8_mem_sz[*level]);
+	}
+	pthread_mutex_unlock(&mem_init_lock);
+	_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
+	if (!_ppmd)
+		return (-1);
+
+	/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
+	if (*level > 14) *level = 14;
+	_ppmd->Order = *level;
+
+	_ppmd->Base = 0;
+	_ppmd->Size = 0;
+	*data = _ppmd;
+	if (*level > 9) *level = 9;
+
+	if (alloc)
+		return (ppmd_alloc(*data));
+	return (0);
+}
+
 void
 ppmd_stats(int show)
 {
@@ -77,33 +126,7 @@ int
 ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
 	  int file_version, compress_op_t op)
 {
-	CPpmd8 *_ppmd;
+	return (ppmd_state_init(data, level, 1));
-
-	pthread_mutex_lock(&mem_init_lock);
-	if (!mem_inited) {
-		slab_cache_add(sizeof (CPpmd8));
-		slab_cache_add(ppmd8_mem_sz[*level]);
-		mem_inited = 1;
-	}
-	pthread_mutex_unlock(&mem_init_lock);
-	_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
-	if (!_ppmd)
-		return (-1);
-
-	/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
-	if (*level > 14) *level = 14;
-	_ppmd->Order = *level;
-
-	_ppmd->Base = 0;
-	_ppmd->Size = 0;
-	if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[*level], &g_Alloc)) {
-		log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
-		return (-1);
-	}
-	Ppmd8_Construct(_ppmd);
-	*data = _ppmd;
-	if (*level > 9) *level = 9;
-	return (0);
 }
 
 int
@@ -111,8 +134,8 @@ ppmd_deinit(void **data)
 {
 	CPpmd8 *_ppmd = (CPpmd8 *)(*data);
 	if (_ppmd) {
-		Ppmd8_Free(_ppmd, &g_Alloc);
+		ppmd_free(*data);
-		slab_free(NULL, _ppmd);
+		slab_release(NULL, _ppmd);
 	}
 	*data = NULL;
 	return (0);