greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Free PPMD buffer after compression, rather than caching.

Browse source 
Introduce new API in allocator to release buffer to OS.
Release LZMA buffers after use.
This commit is contained in:
Moinak Ghosh 2013-12-21 23:32:27 +05:30
parent 271414535e
commit a022a958c3
6 changed files with 117 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
adaptive_compress.c
View file

@ -82,6 +82,10 @@ extern int lz4_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int lz4_deinit(void **data);
extern int ppmd_alloc(void *data);
extern void ppmd_free(void *data);
extern int ppmd_state_init(void **data, int *level, int alloc);
struct adapt_data {
void *lzma_data;
void *ppmd_data;
@ -129,7 +133,7 @@ adapt_init(void **data, int *level, int nthreads, uint64_t chunksize,
if (!adat) {
adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data));
adat->adapt_mode = 1;
rv = ppmd_init(&(adat->ppmd_data), level, nthreads, chunksize, file_version, op);
rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
/*
* LZ4 is used to tackle some embedded archive headers and/or zero paddings in
@ -165,7 +169,7 @@ adapt2_init(void **data, int *level, int nthreads, uint64_t chunksize,
adat->bsc_data = NULL;
lv = *level;
if (lv > 10) lv = 10;
rv = ppmd_init(&(adat->ppmd_data), &lv, nthreads, chunksize, file_version, op);
rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
lv = *level;
if (rv == 0)
rv = lzma_init(&(adat->lzma_data), &lv, nthreads, chunksize, file_version, op);
@ -305,7 +309,11 @@ adapt_compress(void *src, uint64_t srclen, void *dst,
bsc_count++;
} else {
#endif
rv = ppmd_alloc(adat->ppmd_data);
if (rv < 0)
return (rv);
rv = ppmd_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->ppmd_data);
ppmd_free(adat->ppmd_data);
if (rv < 0)
return (rv);
rv = ADAPT_COMPRESS_PPMD;

24
allocator.c
View file

@ -444,8 +444,8 @@ slab_alloc(void *p, uint64_t size)
}
}
void
slab_free(void *p, void *address)
static void
slab_free_real(void *p, void *address, int do_free)
{
struct bufentry *buf, *pbuf;
int found = 0;
@ -471,7 +471,7 @@ slab_free(void *p, void *address)
pthread_mutex_unlock(&hbucket_locks[hindx]);
ATOMIC_SUB(hash_entries, 1);
if (buf->slab == NULL) {
if (buf->slab == NULL || do_free) {
free(buf->ptr);
free(buf);
found = 1;
@ -497,6 +497,18 @@ slab_free(void *p, void *address)
}
}
void
slab_free(void *p, void *address)
{
slab_free_real(p, address, 0);
}
void
slab_release(void *p, void *address)
{
slab_free_real(p, address, 1);
}
#else
void
slab_init() {}
@ -522,6 +534,12 @@ slab_free(void *p, void *address)
free(address);
}
void
slab_release(void *p, void *address)
{
free(address);
}
int
slab_cache_add(uint64_t size)
{

1
allocator.h
View file

@ -34,6 +34,7 @@ void slab_cleanup(int quiet);
void *slab_alloc(void *p, uint64_t size);
void *slab_calloc(void *p, uint64_t items, uint64_t size);
void slab_free(void *p, void *address);
void slab_release(void *p, void *address);
int slab_cache_add(uint64_t size);
#endif

58
lzma/Ppmd8.c
View file

@ -1,28 +1,28 @@
/*
* This file is a part of Pcompress, a chunked parallel multi-
* algorithm lossless compression and decompression program.
*
* Copyright (C) 2012-2013 Moinak Ghosh. All rights reserved.
* Use is subject to license terms.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program.
* If not, see <http://www.gnu.org/licenses/>.
*
* moinakg@belenix.org, http://moinakg.wordpress.com/
*
*/
/*
* This file is a part of Pcompress, a chunked parallel multi-
* algorithm lossless compression and decompression program.
*
* Copyright (C) 2012-2013 Moinak Ghosh. All rights reserved.
* Use is subject to license terms.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this program.
* If not, see <http://www.gnu.org/licenses/>.
*
* moinakg@belenix.org, http://moinakg.wordpress.com/
*
*/
/* Ppmd8.c -- PPMdI codec
2010-03-24 : Igor Pavlov : Public domain
This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */
@ -109,9 +109,11 @@ void Ppmd8_Construct(CPpmd8 *p)
void Ppmd8_Free(CPpmd8 *p, ISzAlloc *alloc)
{
alloc->Free(alloc, p->Base);
p->Size = 0;
p->Base = 0;
if (p->Base != 0) {
alloc->Free(alloc, p->Base);
p->Size = 0;
p->Base = 0;
}
}
Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAlloc *alloc)

4
lzma_compress.c
View file

@ -39,7 +39,7 @@ CLzmaEncProps *p = NULL;
static ISzAlloc g_Alloc = {
slab_alloc,
slab_free,
slab_release,
NULL
};
@ -140,7 +140,7 @@ int
lzma_deinit(void **data)
{
if (p) {
slab_free(NULL, p);
slab_release(NULL, p);
p = NULL;
}
*data = NULL;

83
ppmd_compress.c
View file

@ -58,10 +58,59 @@ static int mem_inited = 0;
static ISzAlloc g_Alloc = {
slab_alloc,
slab_free,
slab_release,
NULL
};
int
ppmd_alloc(void *data)
{
CPpmd8 *_ppmd = (CPpmd8 *)data;
if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[_ppmd->Order], &g_Alloc)) {
log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
return (-1);
}
Ppmd8_Construct(_ppmd);
return (0);
}
void
ppmd_free(void *data)
{
CPpmd8 *_ppmd = (CPpmd8 *)data;
Ppmd8_Free(_ppmd, &g_Alloc);
}
int
ppmd_state_init(void **data, int *level, int alloc)
{
CPpmd8 *_ppmd;
pthread_mutex_lock(&mem_init_lock);
if (!mem_inited) {
slab_cache_add(sizeof (CPpmd8));
slab_cache_add(ppmd8_mem_sz[*level]);
}
pthread_mutex_unlock(&mem_init_lock);
_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
if (!_ppmd)
return (-1);
/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
if (*level > 14) *level = 14;
_ppmd->Order = *level;
_ppmd->Base = 0;
_ppmd->Size = 0;
*data = _ppmd;
if (*level > 9) *level = 9;
if (alloc)
return (ppmd_alloc(*data));
return (0);
}
void
ppmd_stats(int show)
{
@ -77,33 +126,7 @@ int
ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
CPpmd8 *_ppmd;
pthread_mutex_lock(&mem_init_lock);
if (!mem_inited) {
slab_cache_add(sizeof (CPpmd8));
slab_cache_add(ppmd8_mem_sz[*level]);
mem_inited = 1;
}
pthread_mutex_unlock(&mem_init_lock);
_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
if (!_ppmd)
return (-1);
/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
if (*level > 14) *level = 14;
_ppmd->Order = *level;
_ppmd->Base = 0;
_ppmd->Size = 0;
if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[*level], &g_Alloc)) {
log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
return (-1);
}
Ppmd8_Construct(_ppmd);
*data = _ppmd;
if (*level > 9) *level = 9;
return (0);
return (ppmd_state_init(data, level, 1));
}
int
@ -111,8 +134,8 @@ ppmd_deinit(void **data)
{
CPpmd8 *_ppmd = (CPpmd8 *)(*data);
if (_ppmd) {
Ppmd8_Free(_ppmd, &g_Alloc);
slab_free(NULL, _ppmd);
ppmd_free(*data);
slab_release(NULL, _ppmd);
}
*data = NULL;
return (0);