greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Free PPMD buffer after compression, rather than caching.

Browse source 
Introduce new API in allocator to release buffer to OS.
Release LZMA buffers after use.
This commit is contained in:
Moinak Ghosh 2013-12-21 23:32:27 +05:30
parent 271414535e
commit a022a958c3
6 changed files with 117 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
adaptive_compress.c
View file

@ -82,6 +82,10 @@ extern int lz4_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int lz4_deinit(void **data);
extern int ppmd_alloc(void *data);
extern void ppmd_free(void *data);
extern int ppmd_state_init(void **data, int *level, int alloc);
struct adapt_data {
void *lzma_data;
void *ppmd_data;
@ -129,7 +133,7 @@ adapt_init(void **data, int *level, int nthreads, uint64_t chunksize,
if (!adat) {
adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data));
adat->adapt_mode = 1;
rv = ppmd_init(&(adat->ppmd_data), level, nthreads, chunksize, file_version, op);
rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
/*
* LZ4 is used to tackle some embedded archive headers and/or zero paddings in
@ -165,7 +169,7 @@ adapt2_init(void **data, int *level, int nthreads, uint64_t chunksize,
adat->bsc_data = NULL;
lv = *level;
if (lv > 10) lv = 10;
rv = ppmd_init(&(adat->ppmd_data), &lv, nthreads, chunksize, file_version, op);
rv = ppmd_state_init(&(adat->ppmd_data), level, 0);
lv = *level;
if (rv == 0)
rv = lzma_init(&(adat->lzma_data), &lv, nthreads, chunksize, file_version, op);
@ -305,7 +309,11 @@ adapt_compress(void *src, uint64_t srclen, void *dst,
bsc_count++;
} else {
#endif
rv = ppmd_alloc(adat->ppmd_data);
if (rv < 0)
return (rv);
rv = ppmd_compress(src, srclen, dst, dstlen, level, chdr, btype, adat->ppmd_data);
ppmd_free(adat->ppmd_data);
if (rv < 0)
return (rv);
rv = ADAPT_COMPRESS_PPMD;

24
allocator.c
View file

@ -444,8 +444,8 @@ slab_alloc(void *p, uint64_t size)
}
}
void
slab_free(void *p, void *address)
static void
slab_free_real(void *p, void *address, int do_free)
{
struct bufentry *buf, *pbuf;
int found = 0;
@ -471,7 +471,7 @@ slab_free(void *p, void *address)
pthread_mutex_unlock(&hbucket_locks[hindx]);
ATOMIC_SUB(hash_entries, 1);
if (buf->slab == NULL) {
if (buf->slab == NULL || do_free) {
free(buf->ptr);
free(buf);
found = 1;
@ -497,6 +497,18 @@ slab_free(void *p, void *address)
}
}
void
slab_free(void *p, void *address)
{
slab_free_real(p, address, 0);
}
void
slab_release(void *p, void *address)
{
slab_free_real(p, address, 1);
}
#else
void
slab_init() {}
@ -522,6 +534,12 @@ slab_free(void *p, void *address)
free(address);
}
void
slab_release(void *p, void *address)
{
free(address);
}
int
slab_cache_add(uint64_t size)
{

1
allocator.h
View file

@ -34,6 +34,7 @@ void slab_cleanup(int quiet);
void *slab_alloc(void *p, uint64_t size);
void *slab_calloc(void *p, uint64_t items, uint64_t size);
void slab_free(void *p, void *address);
void slab_release(void *p, void *address);
int slab_cache_add(uint64_t size);
#endif

8
lzma/Ppmd8.c
View file

@ -109,9 +109,11 @@ void Ppmd8_Construct(CPpmd8 *p)
void Ppmd8_Free(CPpmd8 *p, ISzAlloc *alloc)
{
alloc->Free(alloc, p->Base);
p->Size = 0;
p->Base = 0;
if (p->Base != 0) {
alloc->Free(alloc, p->Base);
p->Size = 0;
p->Base = 0;
}
}
Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAlloc *alloc)

4
lzma_compress.c
View file

@ -39,7 +39,7 @@ CLzmaEncProps *p = NULL;
static ISzAlloc g_Alloc = {
slab_alloc,
slab_free,
slab_release,
NULL
};
@ -140,7 +140,7 @@ int
lzma_deinit(void **data)
{
if (p) {
slab_free(NULL, p);
slab_release(NULL, p);
p = NULL;
}
*data = NULL;

83
ppmd_compress.c
View file

@ -58,10 +58,59 @@ static int mem_inited = 0;
static ISzAlloc g_Alloc = {
slab_alloc,
slab_free,
slab_release,
NULL
};
int
ppmd_alloc(void *data)
{
CPpmd8 *_ppmd = (CPpmd8 *)data;
if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[_ppmd->Order], &g_Alloc)) {
log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
return (-1);
}
Ppmd8_Construct(_ppmd);
return (0);
}
void
ppmd_free(void *data)
{
CPpmd8 *_ppmd = (CPpmd8 *)data;
Ppmd8_Free(_ppmd, &g_Alloc);
}
int
ppmd_state_init(void **data, int *level, int alloc)
{
CPpmd8 *_ppmd;
pthread_mutex_lock(&mem_init_lock);
if (!mem_inited) {
slab_cache_add(sizeof (CPpmd8));
slab_cache_add(ppmd8_mem_sz[*level]);
}
pthread_mutex_unlock(&mem_init_lock);
_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
if (!_ppmd)
return (-1);
/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
if (*level > 14) *level = 14;
_ppmd->Order = *level;
_ppmd->Base = 0;
_ppmd->Size = 0;
*data = _ppmd;
if (*level > 9) *level = 9;
if (alloc)
return (ppmd_alloc(*data));
return (0);
}
void
ppmd_stats(int show)
{
@ -77,33 +126,7 @@ int
ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
CPpmd8 *_ppmd;
pthread_mutex_lock(&mem_init_lock);
if (!mem_inited) {
slab_cache_add(sizeof (CPpmd8));
slab_cache_add(ppmd8_mem_sz[*level]);
mem_inited = 1;
}
pthread_mutex_unlock(&mem_init_lock);
_ppmd = (CPpmd8 *)slab_alloc(NULL, sizeof (CPpmd8));
if (!_ppmd)
return (-1);
/* Levels 0 - 14 correspond to PPMd model orders 0 - 14. */
if (*level > 14) *level = 14;
_ppmd->Order = *level;
_ppmd->Base = 0;
_ppmd->Size = 0;
if (!Ppmd8_Alloc(_ppmd, ppmd8_mem_sz[*level], &g_Alloc)) {
log_msg(LOG_ERR, 0, "PPMD: Out of memory.\n");
return (-1);
}
Ppmd8_Construct(_ppmd);
*data = _ppmd;
if (*level > 9) *level = 9;
return (0);
return (ppmd_state_init(data, level, 1));
}
int
@ -111,8 +134,8 @@ ppmd_deinit(void **data)
{
CPpmd8 *_ppmd = (CPpmd8 *)(*data);
if (_ppmd) {
Ppmd8_Free(_ppmd, &g_Alloc);
slab_free(NULL, _ppmd);
ppmd_free(*data);
slab_release(NULL, _ppmd);
}
*data = NULL;
return (0);