Free PPMD buffer after compression, rather than caching.
Introduce new API in allocator to release buffer to OS. Release LZMA buffers after use.
This commit is contained in:
parent
271414535e
commit
a022a958c3
6 changed files with 117 additions and 65 deletions
|
@ -82,6 +82,10 @@ extern int lz4_init(void **data, int *level, int nthreads, uint64_t chunksize,
|
||||||
int file_version, compress_op_t op);
|
int file_version, compress_op_t op);
|
||||||
extern int lz4_deinit(void **data);
|
extern int lz4_deinit(void **data);
|
||||||
|
|
||||||
|
extern int ppmd_alloc(void *data);
|
||||||
|
extern void ppmd_free(void *data);
|
||||||
|
extern int ppmd_state_init(void **data, int *level, int alloc);
|
||||||
|
|
||||||
struct adapt_data {
|
struct adapt_data {
|
||||||
void *lzma_data;
|
void *lzma_data;
|
||||||
void *ppmd_data;
|
void *ppmd_data;
|
||||||
|
@ -129,7 +133,7 @@ adapt_init(void **data, int *level, int nthreads, uint64_t chunksize,
|
||||||
if (!adat) {
|
if (!adat) {
|
||||||
adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data));
|
adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data));
|
||||||
adat->adapt_mode = 1;
|
adat->adapt_mode = 1;
|
||||||
rv = ppmd_init(&(adat->ppmd_data), level, nthreads, chunksize, file_version, op);
|
rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* LZ4 is used to tackle some embedded archive headers and/or zero paddings in
|
* LZ4 is used to tackle some embedded archive headers and/or zero paddings in
|
||||||
|
@ -165,7 +169,7 @@ adapt2_init(void **data, int *level, int nthreads, uint64_t chunksize,
|
||||||
adat->bsc_data = NULL;
|
adat->bsc_data = NULL;
|
||||||
lv = *level;
|
lv = *level;
|
||||||
if (lv > 10) lv = 10;
|
if (lv > 10) lv = 10;
|
||||||
rv = ppmd_init(&(adat->ppmd_data), &lv, nthreads, chunksize, file_version, op);
|
rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
|
||||||
lv = *level;
|
lv = *level;
|
||||||
if (rv == 0)
|
if (rv == 0)
|
||||||
rv = lzma_init(&(adat->lzma_data), &lv, nthreads, chunksize, file_version, op);
|
rv = lzma_init(&(adat->lzma_data), &lv, nthreads, chunksize, file_version, op);
|
||||||
|
@ -305,7 +309,11 @@ adapt_compress(void *src, uint64_t srclen, void *dst,
|
||||||
bsc_count++;
|
bsc_count++;
|
||||||
} else {
|
} else {
|
||||||
#endif
|
#endif
|
||||||
|
rv = ppmd_alloc(adat->ppmd_data);
|
||||||
|
if (rv < 0)
|
||||||
|
return (rv);
|
||||||
rv = ppmd_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->ppmd_data);
|
rv = ppmd_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->ppmd_data);
|
||||||
|
ppmd_free(adat->ppmd_data);
|
||||||
if (rv < 0)
|
if (rv < 0)
|
||||||
return (rv);
|
return (rv);
|
||||||
rv = ADAPT_COMPRESS_PPMD;
|
rv = ADAPT_COMPRESS_PPMD;
|
||||||
|
|
24
allocator.c
24
allocator.c
|
@ -444,8 +444,8 @@ slab_alloc(void *p, uint64_t size)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
static void
|
||||||
slab_free(void *p, void *address)
|
slab_free_real(void *p, void *address, int do_free)
|
||||||
{
|
{
|
||||||
struct bufentry *buf, *pbuf;
|
struct bufentry *buf, *pbuf;
|
||||||
int found = 0;
|
int found = 0;
|
||||||
|
@ -471,7 +471,7 @@ slab_free(void *p, void *address)
|
||||||
pthread_mutex_unlock(&hbucket_locks[hindx]);
|
pthread_mutex_unlock(&hbucket_locks[hindx]);
|
||||||
ATOMIC_SUB(hash_entries, 1);
|
ATOMIC_SUB(hash_entries, 1);
|
||||||
|
|
||||||
if (buf->slab == NULL) {
|
if (buf->slab == NULL || do_free) {
|
||||||
free(buf->ptr);
|
free(buf->ptr);
|
||||||
free(buf);
|
free(buf);
|
||||||
found = 1;
|
found = 1;
|
||||||
|
@ -497,6 +497,18 @@ slab_free(void *p, void *address)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
slab_free(void *p, void *address)
|
||||||
|
{
|
||||||
|
slab_free_real(p, address, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
slab_release(void *p, void *address)
|
||||||
|
{
|
||||||
|
slab_free_real(p, address, 1);
|
||||||
|
}
|
||||||
|
|
||||||
#else
|
#else
|
||||||
void
|
void
|
||||||
slab_init() {}
|
slab_init() {}
|
||||||
|
@ -522,6 +534,12 @@ slab_free(void *p, void *address)
|
||||||
free(address);
|
free(address);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
slab_release(void *p, void *address)
|
||||||
|
{
|
||||||
|
free(address);
|
||||||
|
}
|
||||||
|
|
||||||
int
|
int
|
||||||
slab_cache_add(uint64_t size)
|
slab_cache_add(uint64_t size)
|
||||||
{
|
{
|
||||||
|
|
|
@ -34,6 +34,7 @@ void slab_cleanup(int quiet);
|
||||||
void *slab_alloc(void *p, uint64_t size);
|
void *slab_alloc(void *p, uint64_t size);
|
||||||
void *slab_calloc(void *p, uint64_t items, uint64_t size);
|
void *slab_calloc(void *p, uint64_t items, uint64_t size);
|
||||||
void slab_free(void *p, void *address);
|
void slab_free(void *p, void *address);
|
||||||
|
void slab_release(void *p, void *address);
|
||||||
int slab_cache_add(uint64_t size);
|
int slab_cache_add(uint64_t size);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -109,10 +109,12 @@ void Ppmd8_Construct(CPpmd8 *p)
|
||||||
|
|
||||||
void Ppmd8_Free(CPpmd8 *p, ISzAlloc *alloc)
|
void Ppmd8_Free(CPpmd8 *p, ISzAlloc *alloc)
|
||||||
{
|
{
|
||||||
|
if (p->Base != 0) {
|
||||||
alloc->Free(alloc, p->Base);
|
alloc->Free(alloc, p->Base);
|
||||||
p->Size = 0;
|
p->Size = 0;
|
||||||
p->Base = 0;
|
p->Base = 0;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAlloc *alloc)
|
Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAlloc *alloc)
|
||||||
{
|
{
|
||||||
|
|
|
@ -39,7 +39,7 @@ CLzmaEncProps *p = NULL;
|
||||||
|
|
||||||
static ISzAlloc g_Alloc = {
|
static ISzAlloc g_Alloc = {
|
||||||
slab_alloc,
|
slab_alloc,
|
||||||
slab_free,
|
slab_release,
|
||||||
NULL
|
NULL
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -140,7 +140,7 @@ int
|
||||||
lzma_deinit(void **data)
|
lzma_deinit(void **data)
|
||||||
{
|
{
|
||||||
if (p) {
|
if (p) {
|
||||||
slab_free(NULL, p);
|
slab_release(NULL, p);
|
||||||
p = NULL;
|
p = NULL;
|
||||||
}
|
}
|
||||||
*data = NULL;
|
*data = NULL;
|
||||||
|
|
|
@ -58,10 +58,59 @@ static int mem_inited = 0;
|
||||||
|
|
||||||
static ISzAlloc g_Alloc = {
|
static ISzAlloc g_Alloc = {
|
||||||
slab_alloc,
|
slab_alloc,
|
||||||
slab_free,
|
slab_release,
|
||||||
NULL
|
NULL
|
||||||
};
|
};
|
||||||
|
|
||||||
|
int
|
||||||
|
ppmd_alloc(void *data)
|
||||||
|
{
|
||||||
|
CPpmd8 *_ppmd = (CPpmd8 *)data;
|
||||||
|
|
||||||
|
if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[_ppmd->Order], &g_Alloc)) {
|
||||||
|
log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
|
||||||
|
return (-1);
|
||||||
|
}
|
||||||
|
Ppmd8_Construct(_ppmd);
|
||||||
|
return (0);
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
ppmd_free(void *data)
|
||||||
|
{
|
||||||
|
CPpmd8 *_ppmd = (CPpmd8 *)data;
|
||||||
|
Ppmd8_Free(_ppmd, &g_Alloc);
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
ppmd_state_init(void **data, int *level, int alloc)
|
||||||
|
{
|
||||||
|
CPpmd8 *_ppmd;
|
||||||
|
|
||||||
|
pthread_mutex_lock(&mem_init_lock);
|
||||||
|
if (!mem_inited) {
|
||||||
|
slab_cache_add(sizeof (CPpmd8));
|
||||||
|
slab_cache_add(ppmd8_mem_sz[*level]);
|
||||||
|
}
|
||||||
|
pthread_mutex_unlock(&mem_init_lock);
|
||||||
|
_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
|
||||||
|
if (!_ppmd)
|
||||||
|
return (-1);
|
||||||
|
|
||||||
|
/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
|
||||||
|
if (*level > 14) *level = 14;
|
||||||
|
_ppmd->Order = *level;
|
||||||
|
|
||||||
|
_ppmd->Base = 0;
|
||||||
|
_ppmd->Size = 0;
|
||||||
|
*data = _ppmd;
|
||||||
|
if (*level > 9) *level = 9;
|
||||||
|
|
||||||
|
if (alloc)
|
||||||
|
return (ppmd_alloc(*data));
|
||||||
|
return (0);
|
||||||
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
ppmd_stats(int show)
|
ppmd_stats(int show)
|
||||||
{
|
{
|
||||||
|
@ -77,33 +126,7 @@ int
|
||||||
ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
|
ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
|
||||||
int file_version, compress_op_t op)
|
int file_version, compress_op_t op)
|
||||||
{
|
{
|
||||||
CPpmd8 *_ppmd;
|
return (ppmd_state_init(data, level, 1));
|
||||||
|
|
||||||
pthread_mutex_lock(&mem_init_lock);
|
|
||||||
if (!mem_inited) {
|
|
||||||
slab_cache_add(sizeof (CPpmd8));
|
|
||||||
slab_cache_add(ppmd8_mem_sz[*level]);
|
|
||||||
mem_inited = 1;
|
|
||||||
}
|
|
||||||
pthread_mutex_unlock(&mem_init_lock);
|
|
||||||
_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
|
|
||||||
if (!_ppmd)
|
|
||||||
return (-1);
|
|
||||||
|
|
||||||
/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
|
|
||||||
if (*level > 14) *level = 14;
|
|
||||||
_ppmd->Order = *level;
|
|
||||||
|
|
||||||
_ppmd->Base = 0;
|
|
||||||
_ppmd->Size = 0;
|
|
||||||
if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[*level], &g_Alloc)) {
|
|
||||||
log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
|
|
||||||
return (-1);
|
|
||||||
}
|
|
||||||
Ppmd8_Construct(_ppmd);
|
|
||||||
*data = _ppmd;
|
|
||||||
if (*level > 9) *level = 9;
|
|
||||||
return (0);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
int
|
int
|
||||||
|
@ -111,8 +134,8 @@ ppmd_deinit(void **data)
|
||||||
{
|
{
|
||||||
CPpmd8 *_ppmd = (CPpmd8 *)(*data);
|
CPpmd8 *_ppmd = (CPpmd8 *)(*data);
|
||||||
if (_ppmd) {
|
if (_ppmd) {
|
||||||
Ppmd8_Free(_ppmd, &g_Alloc);
|
ppmd_free(*data);
|
||||||
slab_free(NULL, _ppmd);
|
slab_release(NULL, _ppmd);
|
||||||
}
|
}
|
||||||
*data = NULL;
|
*data = NULL;
|
||||||
return (0);
|
return (0);
|
||||||
|
|
Loading…
Reference in a new issue