From bb08b24989fb00fd357080eb03371eaef0976f41 Mon Sep 17 00:00:00 2001 From: Moinak Ghosh Date: Thu, 12 Dec 2013 00:22:15 +0530 Subject: [PATCH] Make LibArchive filter process buffer more generic. Include explicit CLI flags for PackJPG and Dispack. Avoid auto-selection of filters if advanced options are specified. --- archive/pc_arc_filter.c | 44 +++++++++++++++++-------------- pcompress.c | 57 +++++++++++++++++++++++++++++++++-------- 2 files changed, 72 insertions(+), 29 deletions(-) diff --git a/archive/pc_arc_filter.c b/archive/pc_arc_filter.c index c691f75..3b5b603 100644 --- a/archive/pc_arc_filter.c +++ b/archive/pc_arc_filter.c @@ -47,7 +47,7 @@ #define PJG_APPVERSION1 (25) #define PJG_APPVERSION2 (25) -struct packjpg_filter_data { +struct scratch_buffer { uchar_t *in_buff; size_t in_bufflen; }; @@ -59,21 +59,31 @@ int64_t packjpg_filter(struct filter_info *fi, void *filter_private); void add_filters_by_type(struct type_data *typetab, struct filter_flags *ff) { - struct packjpg_filter_data *pjdat; + struct scratch_buffer *sdat; int slot; if (ff->enable_packjpg) { - pjdat = (struct packjpg_filter_data *)malloc(sizeof (struct packjpg_filter_data)); - pjdat->in_buff = NULL; - pjdat->in_bufflen = 0; + sdat = (struct scratch_buffer *)malloc(sizeof (struct scratch_buffer)); + sdat->in_buff = NULL; + sdat->in_bufflen = 0; slot = TYPE_JPEG >> 3; - typetab[slot].filter_private = pjdat; + typetab[slot].filter_private = sdat; typetab[slot].filter_func = packjpg_filter; typetab[slot].filter_name = "packJPG"; } } +static void +ensure_buffer(struct scratch_buffer *sdat, uint64_t len) +{ + if (sdat->in_bufflen < len) { + if (sdat->in_buff) free(sdat->in_buff); + sdat->in_bufflen = len; + sdat->in_buff = malloc(sdat->in_bufflen); + } +} + /* * Copy current entry data from the archive being extracted into the given buffer. */ @@ -137,7 +147,7 @@ pjg_version_supported(char ver) ssize_t packjpg_filter(struct filter_info *fi, void *filter_private) { - struct packjpg_filter_data *pjdat = (struct packjpg_filter_data *)filter_private; + struct scratch_buffer *sdat = (struct scratch_buffer *)filter_private; uchar_t *mapbuf, *out; uint64_t len, in_size = 0, len1; ssize_t rv; @@ -171,17 +181,13 @@ packjpg_filter(struct filter_info *fi, void *filter_private) * Allocate input buffer and read archive data stream for the entry * into this buffer. */ - if (pjdat->in_bufflen < len) { - if (pjdat->in_buff) free(pjdat->in_buff); - pjdat->in_bufflen = len; - pjdat->in_buff = malloc(pjdat->in_bufflen); - if (pjdat->in_buff == NULL) { - log_msg(LOG_ERR, 1, "Out of memory."); - return (FILTER_RETURN_ERROR); - } + ensure_buffer(sdat, len); + if (sdat->in_buff == NULL) { + log_msg(LOG_ERR, 1, "Out of memory."); + return (FILTER_RETURN_ERROR); } - in_size = copy_archive_data(fi->source_arc, pjdat->in_buff); + in_size = copy_archive_data(fi->source_arc, sdat->in_buff); if (in_size != len) { log_msg(LOG_ERR, 0, "Failed to read archive data."); return (FILTER_RETURN_ERROR); @@ -192,8 +198,8 @@ packjpg_filter(struct filter_info *fi, void *filter_private) * LibArchive always zero-pads entries to their original size so * we need to separately store the compressed size. */ - in_size = LE64(U64_P(pjdat->in_buff)); - mapbuf = pjdat->in_buff + 8; + in_size = LE64(U64_P(sdat->in_buff)); + mapbuf = sdat->in_buff + 8; /* * We are trying to decompress and this is not a packJPG file. @@ -201,7 +207,7 @@ packjpg_filter(struct filter_info *fi, void *filter_private) * version number. We also check if it is supported. */ if (mapbuf[0] != 'J' || mapbuf[1] != 'S' || !pjg_version_supported(mapbuf[2])) { - return (write_archive_data(fi->target_arc, pjdat->in_buff, + return (write_archive_data(fi->target_arc, sdat->in_buff, len, fi->block_size)); } } diff --git a/pcompress.c b/pcompress.c index d36f906..d660e06 100644 --- a/pcompress.c +++ b/pcompress.c @@ -24,7 +24,8 @@ */ /* - * pcompress - Do a chunked parallel compression/decompression of a file. + * pcompress - Do a chunked parallel compression/decompression and archiving + * of one or more files. */ #include @@ -2797,8 +2798,10 @@ init_pc_context_argstr(pc_ctx_t *pctx, char *args) int DLL_EXPORT init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) { - int opt, num_rem, err, my_optind; + int opt, num_rem, err, my_optind, advanced_opts; char *pos; + struct filter_flags ff; + pctx->level = -1; err = 0; @@ -2808,6 +2811,8 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) while (*pos != '/' && pos > argv[0]) pos--; if (*pos == '/') pos++; strcpy(pctx->exec_name, pos); + advanced_opts = 0; + ff.enable_packjpg = 0; pthread_mutex_lock(&opt_parse); while ((opt = getopt(argc, argv, "dc:s:l:pt:MCDGEe:w:LPS:B:Fk:avnmK")) != -1) { @@ -2859,6 +2864,7 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) break; case 'B': + advanced_opts = 1; pctx->rab_blk_size = atoi(optarg); if (pctx->rab_blk_size < 0 || pctx->rab_blk_size > 5) { log_msg(LOG_ERR, 0, "Average Dedupe block size must be in range 0 (2k), 1 (4k) .. 5 (64k)"); @@ -2887,14 +2893,17 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) break; case 'D': + advanced_opts = 1; pctx->enable_rabin_scan = 1; break; case 'G': + advanced_opts = 1; pctx->enable_rabin_global = 1; break; case 'E': + advanced_opts = 1; pctx->enable_rabin_scan = 1; if (!pctx->enable_delta_encode) pctx->enable_delta_encode = DELTA_NORMAL; @@ -2916,15 +2925,18 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) break; case 'F': + advanced_opts = 1; pctx->enable_fixed_scan = 1; pctx->enable_rabin_split = 0; break; case 'L': + advanced_opts = 1; pctx->lzp_preprocess = 1; break; case 'P': + advanced_opts = 1; pctx->enable_delta2_encode = 1; break; @@ -2965,6 +2977,16 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) pctx->no_overwrite_newer = 1; break; + case 'j': + advanced_opts = 1; + ff.enable_packjpg = 1; + break; + + case 'x': + advanced_opts = 1; + pctx->dispack_preprocess = 1; + break; + case '?': default: return (2); @@ -3070,6 +3092,14 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) return (1); } + /* + * Dispack and PackJPG are only valid when archiving files. + */ + if ((pctx->dispack_preprocess || ff.enable_packjpg) && !pctx->archive_mode) { + log_msg(LOG_ERR, 0, "Dispack Executable Preprocessor and PackJPG are only valid when archiving."); + return (1); + } + if (num_rem == 0 && !pctx->pipe_mode) { log_msg(LOG_ERR, 0, "Expected at least one filename."); return (1); @@ -3218,16 +3248,23 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) } /* - * Selectively enable filters while archiving, depending on compression level. + * Auto-select filters and preprocessing modes based on compresion level. + * This is not done if user explicitly specified advanced options. */ - if (pctx->archive_mode) { - struct filter_flags ff; + if (!advanced_opts) { + /* + * Selectively enable filters while archiving, depending on compression level. + */ + if (pctx->archive_mode) { + if (pctx->level > 10) ff.enable_packjpg = 1; + init_filters(&ff); + pctx->enable_packjpg = ff.enable_packjpg; + if (pctx->level > 8) pctx->dispack_preprocess = 1; + } - ff.enable_packjpg = 0; - if (pctx->level > 10) ff.enable_packjpg = 1; - init_filters(&ff); - pctx->enable_packjpg = ff.enable_packjpg; - if (pctx->level > 8) pctx->dispack_preprocess = 1; + /* + * Enable other preprocessors based on compresion level. + */ if (pctx->level > 4) pctx->enable_delta2_encode = 1; if (pctx->level > 9) pctx->lzp_preprocess = 1; if (pctx->level > 3) {