From c567a1d2f5990e47f4912d3802c646f9c88dfb8c Mon Sep 17 00:00:00 2001 From: Moinak Ghosh Date: Thu, 14 Nov 2013 21:54:46 +0530 Subject: [PATCH] Enable auto-filtering of archive entries based on compression level. Miscellaneous fixes. --- archive/pc_arc_filter.c | 58 +++++++++++++++++++++++++++-------------- archive/pc_arc_filter.h | 6 ++++- archive/pc_archive.c | 22 +++++++++++++--- archive/pc_archive.h | 1 + pcompress.c | 17 ++++++++++++ 5 files changed, 80 insertions(+), 24 deletions(-) diff --git a/archive/pc_arc_filter.c b/archive/pc_arc_filter.c index 051b5fa..a3248f3 100644 --- a/archive/pc_arc_filter.c +++ b/archive/pc_arc_filter.c @@ -55,27 +55,28 @@ extern size_t packjpg_filter_process(uchar_t *in_buf, size_t len, uchar_t **out_ int64_t packjpg_filter(struct filter_info *fi, void *filter_private); void -add_filters_by_type(struct type_data *typetab) +add_filters_by_type(struct type_data *typetab, struct filter_flags *ff) { struct packjpg_filter_data *pjdat; int slot; - pjdat = (struct packjpg_filter_data *)malloc(sizeof (struct packjpg_filter_data)); - pjdat->buff = (uchar_t *)malloc(PACKJPG_DEF_BUFSIZ); - pjdat->bufflen = PACKJPG_DEF_BUFSIZ; - pjdat->in_buff = NULL; - pjdat->in_bufflen = 0; + if (ff->enable_packjpg) { + pjdat = (struct packjpg_filter_data *)malloc(sizeof (struct packjpg_filter_data)); + pjdat->buff = (uchar_t *)malloc(PACKJPG_DEF_BUFSIZ); + pjdat->bufflen = PACKJPG_DEF_BUFSIZ; + pjdat->in_buff = NULL; + pjdat->in_bufflen = 0; - slot = TYPE_JPEG >> 3; - typetab[slot].filter_private = pjdat; - typetab[slot].filter_func = packjpg_filter; - typetab[slot].filter_name = "packJPG"; - slot = TYPE_PACKJPG >> 3; - typetab[slot].filter_private = pjdat; - typetab[slot].filter_func = packjpg_filter; - typetab[slot].filter_name = "packJPG"; + slot = TYPE_JPEG >> 3; + typetab[slot].filter_private = pjdat; + typetab[slot].filter_func = packjpg_filter; + typetab[slot].filter_name = "packJPG"; + } } +/* + * Copy current entry data from the archive being extracted into the given buffer. + */ static ssize_t copy_archive_data(struct archive *ar, uchar_t *out_buf) { @@ -97,6 +98,9 @@ copy_archive_data(struct archive *ar, uchar_t *out_buf) return (tot); } +/* + * Copy the given buffer into the archive stream. + */ static ssize_t write_archive_data(struct archive *aw, uchar_t *out_buf, size_t len, int block_size) { @@ -119,6 +123,7 @@ write_archive_data(struct archive *aw, uchar_t *out_buf, size_t len, int block_s } offset += block_size; len -= block_size; + buff += block_size; } return (tot); } @@ -131,9 +136,10 @@ packjpg_filter(struct filter_info *fi, void *filter_private) { struct packjpg_filter_data *pjdat = (struct packjpg_filter_data *)filter_private; uchar_t *mapbuf, *out; - size_t len, in_size = 0; + size_t len, in_size = 0, len1; len = archive_entry_size(fi->entry); + len1 = len; if (len > JPG_SIZE_LIMIT) // Bork on massive JPEGs return (FILTER_RETURN_SKIP); @@ -173,21 +179,29 @@ packjpg_filter(struct filter_info *fi, void *filter_private) log_msg(LOG_ERR, 0, "Failed to read archive data."); return (FILTER_RETURN_ERROR); } + + /* + * First 8 bytes in the data is the compressed size of the entry. + * LibArchive always zero-pads entries to their original size so + * we need to separately store the compressed size. + */ in_size = U64_P(pjdat->in_buff); mapbuf = pjdat->in_buff + 8; /* - * We are trying to decompress and this is not a packJPG file. + * We are trying to decompress and this is not a packJPG file, or is + * a normal Jpeg. * Write the raw data and skip. */ - if (mapbuf[0] != 'J' && mapbuf[1] != 'S') { + if ((mapbuf[0] != 'J' && mapbuf[1] != 'S') || + (pjdat->in_buff[0] == 0xFF && pjdat->in_buff[1] == 0xD8)) { return (write_archive_data(fi->target_arc, mapbuf, in_size, fi->block_size)); } } if (pjdat->bufflen < len) { free(pjdat->buff); - pjdat->bufflen = len; // Include size for compressed len + pjdat->bufflen = len; pjdat->buff = malloc(pjdat->bufflen); if (pjdat->buff == NULL) { log_msg(LOG_ERR, 1, "Out of memory."); @@ -218,7 +232,13 @@ packjpg_filter(struct filter_info *fi, void *filter_private) */ out = pjdat->buff; if ((len = packjpg_filter_process(mapbuf, in_size, &out)) == 0) { - return (FILTER_RETURN_ERROR); + /* + * If filter failed we write out the original data and indicate skip + * to continue the archive extraction. + */ + if (write_archive_data(fi->target_arc, mapbuf, len1, fi->block_size) < len1) + return (FILTER_RETURN_ERROR); + return (FILTER_RETURN_SKIP); } return (write_archive_data(fi->target_arc, out, len, fi->block_size)); } diff --git a/archive/pc_arc_filter.h b/archive/pc_arc_filter.h index 60c22ca..fc5076c 100644 --- a/archive/pc_arc_filter.h +++ b/archive/pc_arc_filter.h @@ -48,6 +48,10 @@ struct filter_info { int compressing, block_size; }; +struct filter_flags { + int enable_packjpg; +}; + typedef ssize_t (*filter_func_ptr)(struct filter_info *fi, void *filter_private); struct type_data { @@ -56,7 +60,7 @@ struct type_data { char *filter_name; }; -void add_filters_by_type(struct type_data *typetab); +void add_filters_by_type(struct type_data *typetab, struct filter_flags *ff); #ifdef __cplusplus } diff --git a/archive/pc_archive.c b/archive/pc_archive.c index 5d0575e..17f0f97 100644 --- a/archive/pc_archive.c +++ b/archive/pc_archive.c @@ -55,14 +55,14 @@ #include #include -static int inited = 0; -pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER; +static int inited = 0, filters_inited = 0; +static pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER; static struct ext_hash_entry { uint64_t extnum; int type; } *exthtab = NULL; -struct type_data typetab[NUM_SUB_TYPES]; +static struct type_data typetab[NUM_SUB_TYPES]; /* AE_IFREG Regular file @@ -1140,6 +1140,10 @@ copy_data_out(struct archive *ar, struct archive *aw, struct archive_entry *entr archive_set_error(ar, archive_errno(aw), "%s", archive_error_string(aw)); return (ARCHIVE_FATAL); + + } else if (rv == FILTER_RETURN_SKIP) { + log_msg(LOG_WARN, 0, "Filter function failed for entry."); + return (ARCHIVE_WARN); } else { return (ARCHIVE_OK); } @@ -1356,7 +1360,6 @@ init_archive_mod() { } memset(typetab, 0, sizeof (typetab)); - add_filters_by_type(typetab); inited = 1; } else { rv = 1; @@ -1366,6 +1369,17 @@ init_archive_mod() { return (rv); } +void +init_filters(struct filter_flags *ff) +{ + pthread_mutex_lock(&init_mutex); + if (!filters_inited) { + add_filters_by_type(typetab, ff); + filters_inited = 1; + } + pthread_mutex_unlock(&init_mutex); +} + /* * Identify file type based on extension. Lookup is fast as we have a perfect hash function. * If the given extension maps to a slot which has a different extension or maps to a slot diff --git a/archive/pc_archive.h b/archive/pc_archive.h index 31d8753..fb2fbdc 100644 --- a/archive/pc_archive.h +++ b/archive/pc_archive.h @@ -54,6 +54,7 @@ int64_t archiver_write(void *ctx, void *buf, uint64_t count); int archiver_close(void *ctx); int init_archive_mod(); int insert_filter_data(filter_func_ptr func, void *filter_private, const char *ext); +void init_filters(struct filter_flags *ff); #ifdef __cplusplus } diff --git a/pcompress.c b/pcompress.c index 627e16e..aba558c 100644 --- a/pcompress.c +++ b/pcompress.c @@ -3123,11 +3123,28 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) if (pctx->do_compress) { struct stat sbuf; + struct filter_flags ff; if (pctx->filename && stat(pctx->filename, &sbuf) == -1) { log_msg(LOG_ERR, 1, "Cannot stat: %s", pctx->filename); return (1); } + + /* + * Selectively enable filters while compressing. + */ + ff.enable_packjpg = 0; + if (pctx->level > 9) ff.enable_packjpg = 1; + init_filters(&ff); + + } else if (pctx->do_uncompress) { + struct filter_flags ff; + + /* + * Enable all filters while decompressing. Obviously! + */ + ff.enable_packjpg = 1; + init_filters(&ff); } pctx->inited = 1;