greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Enable auto-filtering of archive entries based on compression level.

Browse source 
Miscellaneous fixes.
This commit is contained in:
Moinak Ghosh 2013-11-14 21:54:46 +05:30
parent e90c52e516
commit c567a1d2f5
5 changed files with 80 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
archive/pc_arc_filter.c
View file

@ -55,27 +55,28 @@ extern size_t packjpg_filter_process(uchar_t *in_buf, size_t len, uchar_t **out_
int64_t packjpg_filter(struct filter_info *fi, void *filter_private); int64_t packjpg_filter(struct filter_info *fi, void *filter_private);
void void
add_filters_by_type(struct type_data *typetab) add_filters_by_type(struct type_data *typetab, struct filter_flags *ff)
{ {
struct packjpg_filter_data *pjdat; struct packjpg_filter_data *pjdat;
int slot; int slot;
pjdat = (struct packjpg_filter_data *)malloc(sizeof (struct packjpg_filter_data)); if (ff->enable_packjpg) {
pjdat->buff = (uchar_t *)malloc(PACKJPG_DEF_BUFSIZ); pjdat = (struct packjpg_filter_data *)malloc(sizeof (struct packjpg_filter_data));
pjdat->bufflen = PACKJPG_DEF_BUFSIZ; pjdat->buff = (uchar_t *)malloc(PACKJPG_DEF_BUFSIZ);
pjdat->in_buff = NULL; pjdat->bufflen = PACKJPG_DEF_BUFSIZ;
pjdat->in_bufflen = 0; pjdat->in_buff = NULL;
pjdat->in_bufflen = 0;
slot = TYPE_JPEG >> 3; slot = TYPE_JPEG >> 3;
typetab[slot].filter_private = pjdat; typetab[slot].filter_private = pjdat;
typetab[slot].filter_func = packjpg_filter; typetab[slot].filter_func = packjpg_filter;
typetab[slot].filter_name = "packJPG"; typetab[slot].filter_name = "packJPG";
slot = TYPE_PACKJPG >> 3; }
typetab[slot].filter_private = pjdat;
typetab[slot].filter_func = packjpg_filter;
typetab[slot].filter_name = "packJPG";
} }
/*
* Copy current entry data from the archive being extracted into the given buffer.
*/
static ssize_t static ssize_t
copy_archive_data(struct archive *ar, uchar_t *out_buf) copy_archive_data(struct archive *ar, uchar_t *out_buf)
{ {
@ -97,6 +98,9 @@ copy_archive_data(struct archive *ar, uchar_t *out_buf)
return (tot); return (tot);
} }
/*
* Copy the given buffer into the archive stream.
*/
static ssize_t static ssize_t
write_archive_data(struct archive *aw, uchar_t *out_buf, size_t len, int block_size) write_archive_data(struct archive *aw, uchar_t *out_buf, size_t len, int block_size)
{ {
@ -119,6 +123,7 @@ write_archive_data(struct archive *aw, uchar_t *out_buf, size_t len, int block_s
} }
offset += block_size; offset += block_size;
len -= block_size; len -= block_size;
buff += block_size;
} }
return (tot); return (tot);
} }
@ -131,9 +136,10 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
{ {
struct packjpg_filter_data *pjdat = (struct packjpg_filter_data *)filter_private; struct packjpg_filter_data *pjdat = (struct packjpg_filter_data *)filter_private;
uchar_t *mapbuf, *out; uchar_t *mapbuf, *out;
size_t len, in_size = 0; size_t len, in_size = 0, len1;
len = archive_entry_size(fi->entry); len = archive_entry_size(fi->entry);
len1 = len;
if (len > JPG_SIZE_LIMIT) // Bork on massive JPEGs if (len > JPG_SIZE_LIMIT) // Bork on massive JPEGs
return (FILTER_RETURN_SKIP); return (FILTER_RETURN_SKIP);
@ -173,21 +179,29 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
log_msg(LOG_ERR, 0, "Failed to read archive data."); log_msg(LOG_ERR, 0, "Failed to read archive data.");
return (FILTER_RETURN_ERROR); return (FILTER_RETURN_ERROR);
} }
/*
* First 8 bytes in the data is the compressed size of the entry.
* LibArchive always zero-pads entries to their original size so
* we need to separately store the compressed size.
*/
in_size = U64_P(pjdat->in_buff); in_size = U64_P(pjdat->in_buff);
mapbuf = pjdat->in_buff + 8; mapbuf = pjdat->in_buff + 8;
/* /*
* We are trying to decompress and this is not a packJPG file. * We are trying to decompress and this is not a packJPG file, or is
* a normal Jpeg.
* Write the raw data and skip. * Write the raw data and skip.
*/ */
if (mapbuf[0] != 'J' && mapbuf[1] != 'S') { if ((mapbuf[0] != 'J' && mapbuf[1] != 'S') ||
(pjdat->in_buff[0] == 0xFF && pjdat->in_buff[1] == 0xD8)) {
return (write_archive_data(fi->target_arc, mapbuf, in_size, return (write_archive_data(fi->target_arc, mapbuf, in_size,
fi->block_size)); fi->block_size));
} }
} }
if (pjdat->bufflen < len) { if (pjdat->bufflen < len) {
free(pjdat->buff); free(pjdat->buff);
pjdat->bufflen = len; // Include size for compressed len pjdat->bufflen = len;
pjdat->buff = malloc(pjdat->bufflen); pjdat->buff = malloc(pjdat->bufflen);
if (pjdat->buff == NULL) { if (pjdat->buff == NULL) {
log_msg(LOG_ERR, 1, "Out of memory."); log_msg(LOG_ERR, 1, "Out of memory.");
@ -218,7 +232,13 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
*/ */
out = pjdat->buff; out = pjdat->buff;
if ((len = packjpg_filter_process(mapbuf, in_size, &out)) == 0) { if ((len = packjpg_filter_process(mapbuf, in_size, &out)) == 0) {
return (FILTER_RETURN_ERROR); /*
* If filter failed we write out the original data and indicate skip
* to continue the archive extraction.
*/
if (write_archive_data(fi->target_arc, mapbuf, len1, fi->block_size) < len1)
return (FILTER_RETURN_ERROR);
return (FILTER_RETURN_SKIP);
} }
return (write_archive_data(fi->target_arc, out, len, fi->block_size)); return (write_archive_data(fi->target_arc, out, len, fi->block_size));
} }

6
archive/pc_arc_filter.h
View file

@ -48,6 +48,10 @@ struct filter_info {
int compressing, block_size; int compressing, block_size;
}; };
struct filter_flags {
int enable_packjpg;
};
typedef ssize_t (*filter_func_ptr)(struct filter_info *fi, void *filter_private); typedef ssize_t (*filter_func_ptr)(struct filter_info *fi, void *filter_private);
struct type_data { struct type_data {
@ -56,7 +60,7 @@ struct type_data {
char *filter_name; char *filter_name;
}; };
void add_filters_by_type(struct type_data *typetab); void add_filters_by_type(struct type_data *typetab, struct filter_flags *ff);
#ifdef __cplusplus #ifdef __cplusplus
} }

22
archive/pc_archive.c
View file

@ -55,14 +55,14 @@
#include <ftw.h> #include <ftw.h>
#include <stdint.h> #include <stdint.h>
static int inited = 0; static int inited = 0, filters_inited = 0;
pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER;
static struct ext_hash_entry { static struct ext_hash_entry {
uint64_t extnum; uint64_t extnum;
int type; int type;
} *exthtab = NULL; } *exthtab = NULL;
struct type_data typetab[NUM_SUB_TYPES]; static struct type_data typetab[NUM_SUB_TYPES];
/* /*
AE_IFREG Regular file AE_IFREG Regular file
@ -1140,6 +1140,10 @@ copy_data_out(struct archive *ar, struct archive *aw, struct archive_entry *entr
archive_set_error(ar, archive_errno(aw), archive_set_error(ar, archive_errno(aw),
"%s", archive_error_string(aw)); "%s", archive_error_string(aw));
return (ARCHIVE_FATAL); return (ARCHIVE_FATAL);
} else if (rv == FILTER_RETURN_SKIP) {
log_msg(LOG_WARN, 0, "Filter function failed for entry.");
return (ARCHIVE_WARN);
} else { } else {
return (ARCHIVE_OK); return (ARCHIVE_OK);
} }
@ -1356,7 +1360,6 @@ init_archive_mod() {
} }
memset(typetab, 0, sizeof (typetab)); memset(typetab, 0, sizeof (typetab));
add_filters_by_type(typetab);
inited = 1; inited = 1;
} else { } else {
rv = 1; rv = 1;
@ -1366,6 +1369,17 @@ init_archive_mod() {
return (rv); return (rv);
} }
void
init_filters(struct filter_flags *ff)
{
pthread_mutex_lock(&init_mutex);
if (!filters_inited) {
add_filters_by_type(typetab, ff);
filters_inited = 1;
}
pthread_mutex_unlock(&init_mutex);
}
/* /*
* Identify file type based on extension. Lookup is fast as we have a perfect hash function. * Identify file type based on extension. Lookup is fast as we have a perfect hash function.
* If the given extension maps to a slot which has a different extension or maps to a slot * If the given extension maps to a slot which has a different extension or maps to a slot

1
archive/pc_archive.h
View file

@ -54,6 +54,7 @@ int64_t archiver_write(void *ctx, void *buf, uint64_t count);
int archiver_close(void *ctx); int archiver_close(void *ctx);
int init_archive_mod(); int init_archive_mod();
int insert_filter_data(filter_func_ptr func, void *filter_private, const char *ext); int insert_filter_data(filter_func_ptr func, void *filter_private, const char *ext);
void init_filters(struct filter_flags *ff);
#ifdef __cplusplus #ifdef __cplusplus
} }

17
pcompress.c
View file

@ -3123,11 +3123,28 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
if (pctx->do_compress) { if (pctx->do_compress) {
struct stat sbuf; struct stat sbuf;
struct filter_flags ff;
if (pctx->filename && stat(pctx->filename, &sbuf) == -1) { if (pctx->filename && stat(pctx->filename, &sbuf) == -1) {
log_msg(LOG_ERR, 1, "Cannot stat: %s", pctx->filename); log_msg(LOG_ERR, 1, "Cannot stat: %s", pctx->filename);
return (1); return (1);
} }
/*
* Selectively enable filters while compressing.
*/
ff.enable_packjpg = 0;
if (pctx->level > 9) ff.enable_packjpg = 1;
init_filters(&ff);
} else if (pctx->do_uncompress) {
struct filter_flags ff;
/*
* Enable all filters while decompressing. Obviously!
*/
ff.enable_packjpg = 1;
init_filters(&ff);
} }
pctx->inited = 1; pctx->inited = 1;