Free PPMD buffer after compression, rather than caching.

Introduce new API in allocator to release buffer to OS.
Release LZMA buffers after use.
This commit is contained in:
Moinak Ghosh 2013-12-21 23:32:27 +05:30
parent 271414535e
commit a022a958c3
6 changed files with 117 additions and 65 deletions

View file

@ -82,6 +82,10 @@ extern int lz4_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int lz4_deinit(void **data);
extern int ppmd_alloc(void *data);
extern void ppmd_free(void *data);
extern int ppmd_state_init(void **data, int *level, int alloc);
struct adapt_data {
void *lzma_data;
void *ppmd_data;
@ -129,7 +133,7 @@ adapt_init(void **data, int *level, int nthreads, uint64_t chunksize,
if (!adat) {
adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data));
adat->adapt_mode = 1;
rv = ppmd_init(&(adat->ppmd_data), level, nthreads, chunksize, file_version, op);
rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
/*
* LZ4 is used to tackle some embedded archive headers and/or zero paddings in
@ -165,7 +169,7 @@ adapt2_init(void **data, int *level, int nthreads, uint64_t chunksize,
adat->bsc_data = NULL;
lv = *level;
if (lv > 10) lv = 10;
rv = ppmd_init(&(adat->ppmd_data), &lv, nthreads, chunksize, file_version, op);
rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
lv = *level;
if (rv == 0)
rv = lzma_init(&(adat->lzma_data), &lv, nthreads, chunksize, file_version, op);
@ -305,7 +309,11 @@ adapt_compress(void *src, uint64_t srclen, void *dst,
bsc_count++;
} else {
#endif
rv = ppmd_alloc(adat->ppmd_data);
if (rv < 0)
return (rv);
rv = ppmd_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->ppmd_data);
ppmd_free(adat->ppmd_data);
if (rv < 0)
return (rv);
rv = ADAPT_COMPRESS_PPMD;

View file

@ -444,8 +444,8 @@ slab_alloc(void *p, uint64_t size)
}
}
void
slab_free(void *p, void *address)
static void
slab_free_real(void *p, void *address, int do_free)
{
struct bufentry *buf, *pbuf;
int found = 0;
@ -471,7 +471,7 @@ slab_free(void *p, void *address)
pthread_mutex_unlock(&hbucket_locks[hindx]);
ATOMIC_SUB(hash_entries, 1);
if (buf->slab == NULL) {
if (buf->slab == NULL || do_free) {
free(buf->ptr);
free(buf);
found = 1;
@ -497,6 +497,18 @@ slab_free(void *p, void *address)
}
}
void
slab_free(void *p, void *address)
{
slab_free_real(p, address, 0);
}
void
slab_release(void *p, void *address)
{
slab_free_real(p, address, 1);
}
#else
void
slab_init() {}
@ -522,6 +534,12 @@ slab_free(void *p, void *address)
free(address);
}
void
slab_release(void *p, void *address)
{
free(address);
}
int
slab_cache_add(uint64_t size)
{

View file

@ -34,6 +34,7 @@ void slab_cleanup(int quiet);
void *slab_alloc(void *p, uint64_t size);
void *slab_calloc(void *p, uint64_t items, uint64_t size);
void slab_free(void *p, void *address);
void slab_release(void *p, void *address);
int slab_cache_add(uint64_t size);
#endif

View file

@ -109,9 +109,11 @@ void Ppmd8_Construct(CPpmd8 *p)
void Ppmd8_Free(CPpmd8 *p, ISzAlloc *alloc)
{
alloc->Free(alloc, p->Base);
p->Size = 0;
p->Base = 0;
if (p->Base != 0) {
alloc->Free(alloc, p->Base);
p->Size = 0;
p->Base = 0;
}
}
Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAlloc *alloc)

View file

@ -39,7 +39,7 @@ CLzmaEncProps *p = NULL;
static ISzAlloc g_Alloc = {
slab_alloc,
slab_free,
slab_release,
NULL
};
@ -140,7 +140,7 @@ int
lzma_deinit(void **data)
{
if (p) {
slab_free(NULL, p);
slab_release(NULL, p);
p = NULL;
}
*data = NULL;

View file

@ -58,10 +58,59 @@ static int mem_inited = 0;
static ISzAlloc g_Alloc = {
slab_alloc,
slab_free,
slab_release,
NULL
};
int
ppmd_alloc(void *data)
{
CPpmd8 *_ppmd = (CPpmd8 *)data;
if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[_ppmd->Order], &g_Alloc)) {
log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
return (-1);
}
Ppmd8_Construct(_ppmd);
return (0);
}
void
ppmd_free(void *data)
{
CPpmd8 *_ppmd = (CPpmd8 *)data;
Ppmd8_Free(_ppmd, &g_Alloc);
}
int
ppmd_state_init(void **data, int *level, int alloc)
{
CPpmd8 *_ppmd;
pthread_mutex_lock(&mem_init_lock);
if (!mem_inited) {
slab_cache_add(sizeof (CPpmd8));
slab_cache_add(ppmd8_mem_sz[*level]);
}
pthread_mutex_unlock(&mem_init_lock);
_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
if (!_ppmd)
return (-1);
/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
if (*level > 14) *level = 14;
_ppmd->Order = *level;
_ppmd->Base = 0;
_ppmd->Size = 0;
*data = _ppmd;
if (*level > 9) *level = 9;
if (alloc)
return (ppmd_alloc(*data));
return (0);
}
void
ppmd_stats(int show)
{
@ -77,33 +126,7 @@ int
ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
CPpmd8 *_ppmd;
pthread_mutex_lock(&mem_init_lock);
if (!mem_inited) {
slab_cache_add(sizeof (CPpmd8));
slab_cache_add(ppmd8_mem_sz[*level]);
mem_inited = 1;
}
pthread_mutex_unlock(&mem_init_lock);
_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
if (!_ppmd)
return (-1);
/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
if (*level > 14) *level = 14;
_ppmd->Order = *level;
_ppmd->Base = 0;
_ppmd->Size = 0;
if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[*level], &g_Alloc)) {
log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
return (-1);
}
Ppmd8_Construct(_ppmd);
*data = _ppmd;
if (*level > 9) *level = 9;
return (0);
return (ppmd_state_init(data, level, 1));
}
int
@ -111,8 +134,8 @@ ppmd_deinit(void **data)
{
CPpmd8 *_ppmd = (CPpmd8 *)(*data);
if (_ppmd) {
Ppmd8_Free(_ppmd, &g_Alloc);
slab_free(NULL, _ppmd);
ppmd_free(*data);
slab_release(NULL, _ppmd);
}
*data = NULL;
return (0);