Free PPMD buffer after compression, rather than caching.

Introduce new API in allocator to release buffer to OS.
Release LZMA buffers after use.
This commit is contained in:
Moinak Ghosh 2013-12-21 23:32:27 +05:30
parent 271414535e
commit a022a958c3
6 changed files with 117 additions and 65 deletions

View file

@ -82,6 +82,10 @@ extern int lz4_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int lz4_deinit(void **data); extern int lz4_deinit(void **data);
extern int ppmd_alloc(void *data);
extern void ppmd_free(void *data);
extern int ppmd_state_init(void **data, int *level, int alloc);
struct adapt_data { struct adapt_data {
void *lzma_data; void *lzma_data;
void *ppmd_data; void *ppmd_data;
@ -129,7 +133,7 @@ adapt_init(void **data, int *level, int nthreads, uint64_t chunksize,
if (!adat) { if (!adat) {
adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data)); adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data));
adat->adapt_mode = 1; adat->adapt_mode = 1;
rv = ppmd_init(&(adat->ppmd_data), level, nthreads, chunksize, file_version, op); rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
/* /*
* LZ4 is used to tackle some embedded archive headers and/or zero paddings in * LZ4 is used to tackle some embedded archive headers and/or zero paddings in
@ -165,7 +169,7 @@ adapt2_init(void **data, int *level, int nthreads, uint64_t chunksize,
adat->bsc_data = NULL; adat->bsc_data = NULL;
lv = *level; lv = *level;
if (lv > 10) lv = 10; if (lv > 10) lv = 10;
rv = ppmd_init(&(adat->ppmd_data), &lv, nthreads, chunksize, file_version, op); rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
lv = *level; lv = *level;
if (rv == 0) if (rv == 0)
rv = lzma_init(&(adat->lzma_data), &lv, nthreads, chunksize, file_version, op); rv = lzma_init(&(adat->lzma_data), &lv, nthreads, chunksize, file_version, op);
@ -305,7 +309,11 @@ adapt_compress(void *src, uint64_t srclen, void *dst,
bsc_count++; bsc_count++;
} else { } else {
#endif #endif
rv = ppmd_alloc(adat->ppmd_data);
if (rv < 0)
return (rv);
rv = ppmd_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->ppmd_data); rv = ppmd_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->ppmd_data);
ppmd_free(adat->ppmd_data);
if (rv < 0) if (rv < 0)
return (rv); return (rv);
rv = ADAPT_COMPRESS_PPMD; rv = ADAPT_COMPRESS_PPMD;

View file

@ -444,8 +444,8 @@ slab_alloc(void *p, uint64_t size)
} }
} }
void static void
slab_free(void *p, void *address) slab_free_real(void *p, void *address, int do_free)
{ {
struct bufentry *buf, *pbuf; struct bufentry *buf, *pbuf;
int found = 0; int found = 0;
@ -471,7 +471,7 @@ slab_free(void *p, void *address)
pthread_mutex_unlock(&hbucket_locks[hindx]); pthread_mutex_unlock(&hbucket_locks[hindx]);
ATOMIC_SUB(hash_entries, 1); ATOMIC_SUB(hash_entries, 1);
if (buf->slab == NULL) { if (buf->slab == NULL || do_free) {
free(buf->ptr); free(buf->ptr);
free(buf); free(buf);
found = 1; found = 1;
@ -497,6 +497,18 @@ slab_free(void *p, void *address)
} }
} }
void
slab_free(void *p, void *address)
{
slab_free_real(p, address, 0);
}
void
slab_release(void *p, void *address)
{
slab_free_real(p, address, 1);
}
#else #else
void void
slab_init() {} slab_init() {}
@ -522,6 +534,12 @@ slab_free(void *p, void *address)
free(address); free(address);
} }
void
slab_release(void *p, void *address)
{
free(address);
}
int int
slab_cache_add(uint64_t size) slab_cache_add(uint64_t size)
{ {

View file

@ -34,6 +34,7 @@ void slab_cleanup(int quiet);
void *slab_alloc(void *p, uint64_t size); void *slab_alloc(void *p, uint64_t size);
void *slab_calloc(void *p, uint64_t items, uint64_t size); void *slab_calloc(void *p, uint64_t items, uint64_t size);
void slab_free(void *p, void *address); void slab_free(void *p, void *address);
void slab_release(void *p, void *address);
int slab_cache_add(uint64_t size); int slab_cache_add(uint64_t size);
#endif #endif

View file

@ -109,10 +109,12 @@ void Ppmd8_Construct(CPpmd8 *p)
void Ppmd8_Free(CPpmd8 *p, ISzAlloc *alloc) void Ppmd8_Free(CPpmd8 *p, ISzAlloc *alloc)
{ {
if (p->Base != 0) {
alloc->Free(alloc, p->Base); alloc->Free(alloc, p->Base);
p->Size = 0; p->Size = 0;
p->Base = 0; p->Base = 0;
} }
}
Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAlloc *alloc) Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAlloc *alloc)
{ {

View file

@ -39,7 +39,7 @@ CLzmaEncProps *p = NULL;
static ISzAlloc g_Alloc = { static ISzAlloc g_Alloc = {
slab_alloc, slab_alloc,
slab_free, slab_release,
NULL NULL
}; };
@ -140,7 +140,7 @@ int
lzma_deinit(void **data) lzma_deinit(void **data)
{ {
if (p) { if (p) {
slab_free(NULL, p); slab_release(NULL, p);
p = NULL; p = NULL;
} }
*data = NULL; *data = NULL;

View file

@ -58,10 +58,59 @@ static int mem_inited = 0;
static ISzAlloc g_Alloc = { static ISzAlloc g_Alloc = {
slab_alloc, slab_alloc,
slab_free, slab_release,
NULL NULL
}; };
int
ppmd_alloc(void *data)
{
CPpmd8 *_ppmd = (CPpmd8 *)data;
if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[_ppmd->Order], &g_Alloc)) {
log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
return (-1);
}
Ppmd8_Construct(_ppmd);
return (0);
}
void
ppmd_free(void *data)
{
CPpmd8 *_ppmd = (CPpmd8 *)data;
Ppmd8_Free(_ppmd, &g_Alloc);
}
int
ppmd_state_init(void **data, int *level, int alloc)
{
CPpmd8 *_ppmd;
pthread_mutex_lock(&mem_init_lock);
if (!mem_inited) {
slab_cache_add(sizeof (CPpmd8));
slab_cache_add(ppmd8_mem_sz[*level]);
}
pthread_mutex_unlock(&mem_init_lock);
_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
if (!_ppmd)
return (-1);
/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
if (*level > 14) *level = 14;
_ppmd->Order = *level;
_ppmd->Base = 0;
_ppmd->Size = 0;
*data = _ppmd;
if (*level > 9) *level = 9;
if (alloc)
return (ppmd_alloc(*data));
return (0);
}
void void
ppmd_stats(int show) ppmd_stats(int show)
{ {
@ -77,33 +126,7 @@ int
ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize, ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op) int file_version, compress_op_t op)
{ {
CPpmd8 *_ppmd; return (ppmd_state_init(data, level, 1));
pthread_mutex_lock(&mem_init_lock);
if (!mem_inited) {
slab_cache_add(sizeof (CPpmd8));
slab_cache_add(ppmd8_mem_sz[*level]);
mem_inited = 1;
}
pthread_mutex_unlock(&mem_init_lock);
_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
if (!_ppmd)
return (-1);
/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
if (*level > 14) *level = 14;
_ppmd->Order = *level;
_ppmd->Base = 0;
_ppmd->Size = 0;
if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[*level], &g_Alloc)) {
log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
return (-1);
}
Ppmd8_Construct(_ppmd);
*data = _ppmd;
if (*level > 9) *level = 9;
return (0);
} }
int int
@ -111,8 +134,8 @@ ppmd_deinit(void **data)
{ {
CPpmd8 *_ppmd = (CPpmd8 *)(*data); CPpmd8 *_ppmd = (CPpmd8 *)(*data);
if (_ppmd) { if (_ppmd) {
Ppmd8_Free(_ppmd, &g_Alloc); ppmd_free(*data);
slab_free(NULL, _ppmd); slab_release(NULL, _ppmd);
} }
*data = NULL; *data = NULL;
return (0); return (0);