greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Make LibArchive filter process buffer more generic.

Browse source 
Include explicit CLI flags for PackJPG and Dispack.
Avoid auto-selection of filters if advanced options are specified.
This commit is contained in:
Moinak Ghosh 2013-12-12 00:22:15 +05:30
parent 393fd790b0
commit bb08b24989
2 changed files with 72 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
archive/pc_arc_filter.c
View file

@ -47,7 +47,7 @@
#define PJG_APPVERSION1 (25) #define PJG_APPVERSION1 (25)
#define PJG_APPVERSION2 (25) #define PJG_APPVERSION2 (25)
struct packjpg_filter_data { struct scratch_buffer {
uchar_t *in_buff; uchar_t *in_buff;
size_t in_bufflen; size_t in_bufflen;
}; };
@ -59,21 +59,31 @@ int64_t packjpg_filter(struct filter_info *fi, void *filter_private);
void void
add_filters_by_type(struct type_data *typetab, struct filter_flags *ff) add_filters_by_type(struct type_data *typetab, struct filter_flags *ff)
{ {
struct packjpg_filter_data *pjdat; struct scratch_buffer *sdat;
int slot; int slot;
if (ff->enable_packjpg) { if (ff->enable_packjpg) {
pjdat = (struct packjpg_filter_data *)malloc(sizeof (struct packjpg_filter_data)); sdat = (struct scratch_buffer *)malloc(sizeof (struct scratch_buffer));
pjdat->in_buff = NULL; sdat->in_buff = NULL;
pjdat->in_bufflen = 0; sdat->in_bufflen = 0;
slot = TYPE_JPEG >> 3; slot = TYPE_JPEG >> 3;
typetab[slot].filter_private = pjdat; typetab[slot].filter_private = sdat;
typetab[slot].filter_func = packjpg_filter; typetab[slot].filter_func = packjpg_filter;
typetab[slot].filter_name = "packJPG"; typetab[slot].filter_name = "packJPG";
} }
} }
static void
ensure_buffer(struct scratch_buffer *sdat, uint64_t len)
{
if (sdat->in_bufflen < len) {
if (sdat->in_buff) free(sdat->in_buff);
sdat->in_bufflen = len;
sdat->in_buff = malloc(sdat->in_bufflen);
}
}
/* /*
* Copy current entry data from the archive being extracted into the given buffer. * Copy current entry data from the archive being extracted into the given buffer.
*/ */
@ -137,7 +147,7 @@ pjg_version_supported(char ver)
ssize_t ssize_t
packjpg_filter(struct filter_info *fi, void *filter_private) packjpg_filter(struct filter_info *fi, void *filter_private)
{ {
struct packjpg_filter_data *pjdat = (struct packjpg_filter_data *)filter_private; struct scratch_buffer *sdat = (struct scratch_buffer *)filter_private;
uchar_t *mapbuf, *out; uchar_t *mapbuf, *out;
uint64_t len, in_size = 0, len1; uint64_t len, in_size = 0, len1;
ssize_t rv; ssize_t rv;
@ -171,17 +181,13 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
* Allocate input buffer and read archive data stream for the entry * Allocate input buffer and read archive data stream for the entry
* into this buffer. * into this buffer.
*/ */
if (pjdat->in_bufflen < len) { ensure_buffer(sdat, len);
if (pjdat->in_buff) free(pjdat->in_buff); if (sdat->in_buff == NULL) {
pjdat->in_bufflen = len; log_msg(LOG_ERR, 1, "Out of memory.");
pjdat->in_buff = malloc(pjdat->in_bufflen); return (FILTER_RETURN_ERROR);
if (pjdat->in_buff == NULL) {
log_msg(LOG_ERR, 1, "Out of memory.");
return (FILTER_RETURN_ERROR);
}
} }
in_size = copy_archive_data(fi->source_arc, pjdat->in_buff); in_size = copy_archive_data(fi->source_arc, sdat->in_buff);
if (in_size != len) { if (in_size != len) {
log_msg(LOG_ERR, 0, "Failed to read archive data."); log_msg(LOG_ERR, 0, "Failed to read archive data.");
return (FILTER_RETURN_ERROR); return (FILTER_RETURN_ERROR);
@ -192,8 +198,8 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
* LibArchive always zero-pads entries to their original size so * LibArchive always zero-pads entries to their original size so
* we need to separately store the compressed size. * we need to separately store the compressed size.
*/ */
in_size = LE64(U64_P(pjdat->in_buff)); in_size = LE64(U64_P(sdat->in_buff));
mapbuf = pjdat->in_buff + 8; mapbuf = sdat->in_buff + 8;
/* /*
* We are trying to decompress and this is not a packJPG file. * We are trying to decompress and this is not a packJPG file.
@ -201,7 +207,7 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
* version number. We also check if it is supported. * version number. We also check if it is supported.
*/ */
if (mapbuf[0] != 'J' || mapbuf[1] != 'S' || !pjg_version_supported(mapbuf[2])) { if (mapbuf[0] != 'J' || mapbuf[1] != 'S' || !pjg_version_supported(mapbuf[2])) {
return (write_archive_data(fi->target_arc, pjdat->in_buff, return (write_archive_data(fi->target_arc, sdat->in_buff,
len, fi->block_size)); len, fi->block_size));
} }
} }

57
pcompress.c
View file

@ -24,7 +24,8 @@
*/ */
/* /*
* pcompress - Do a chunked parallel compression/decompression of a file. * pcompress - Do a chunked parallel compression/decompression and archiving
* of one or more files.
*/ */
#include <stdio.h> #include <stdio.h>
@ -2797,8 +2798,10 @@ init_pc_context_argstr(pc_ctx_t *pctx, char *args)
int DLL_EXPORT int DLL_EXPORT
init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]) init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
{ {
int opt, num_rem, err, my_optind; int opt, num_rem, err, my_optind, advanced_opts;
char *pos; char *pos;
struct filter_flags ff;
pctx->level = -1; pctx->level = -1;
err = 0; err = 0;
@ -2808,6 +2811,8 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
while (*pos != '/' && pos > argv[0]) pos--; while (*pos != '/' && pos > argv[0]) pos--;
if (*pos == '/') pos++; if (*pos == '/') pos++;
strcpy(pctx->exec_name, pos); strcpy(pctx->exec_name, pos);
advanced_opts = 0;
ff.enable_packjpg = 0;
pthread_mutex_lock(&opt_parse); pthread_mutex_lock(&opt_parse);
while ((opt = getopt(argc, argv, "dc:s:l:pt:MCDGEe:w:LPS:B:Fk:avnmK")) != -1) { while ((opt = getopt(argc, argv, "dc:s:l:pt:MCDGEe:w:LPS:B:Fk:avnmK")) != -1) {
@ -2859,6 +2864,7 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
break; break;
case 'B': case 'B':
advanced_opts = 1;
pctx->rab_blk_size = atoi(optarg); pctx->rab_blk_size = atoi(optarg);
if (pctx->rab_blk_size < 0 || pctx->rab_blk_size > 5) { if (pctx->rab_blk_size < 0 || pctx->rab_blk_size > 5) {
log_msg(LOG_ERR, 0, "Average Dedupe block size must be in range 0 (2k), 1 (4k) .. 5 (64k)"); log_msg(LOG_ERR, 0, "Average Dedupe block size must be in range 0 (2k), 1 (4k) .. 5 (64k)");
@ -2887,14 +2893,17 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
break; break;
case 'D': case 'D':
advanced_opts = 1;
pctx->enable_rabin_scan = 1; pctx->enable_rabin_scan = 1;
break; break;
case 'G': case 'G':
advanced_opts = 1;
pctx->enable_rabin_global = 1; pctx->enable_rabin_global = 1;
break; break;
case 'E': case 'E':
advanced_opts = 1;
pctx->enable_rabin_scan = 1; pctx->enable_rabin_scan = 1;
if (!pctx->enable_delta_encode) if (!pctx->enable_delta_encode)
pctx->enable_delta_encode = DELTA_NORMAL; pctx->enable_delta_encode = DELTA_NORMAL;
@ -2916,15 +2925,18 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
break; break;
case 'F': case 'F':
advanced_opts = 1;
pctx->enable_fixed_scan = 1; pctx->enable_fixed_scan = 1;
pctx->enable_rabin_split = 0; pctx->enable_rabin_split = 0;
break; break;
case 'L': case 'L':
advanced_opts = 1;
pctx->lzp_preprocess = 1; pctx->lzp_preprocess = 1;
break; break;
case 'P': case 'P':
advanced_opts = 1;
pctx->enable_delta2_encode = 1; pctx->enable_delta2_encode = 1;
break; break;
@ -2965,6 +2977,16 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
pctx->no_overwrite_newer = 1; pctx->no_overwrite_newer = 1;
break; break;
case 'j':
advanced_opts = 1;
ff.enable_packjpg = 1;
break;
case 'x':
advanced_opts = 1;
pctx->dispack_preprocess = 1;
break;
case '?': case '?':
default: default:
return (2); return (2);
@ -3070,6 +3092,14 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
return (1); return (1);
} }
/*
* Dispack and PackJPG are only valid when archiving files.
*/
if ((pctx->dispack_preprocess || ff.enable_packjpg) && !pctx->archive_mode) {
log_msg(LOG_ERR, 0, "Dispack Executable Preprocessor and PackJPG are only valid when archiving.");
return (1);
}
if (num_rem == 0 && !pctx->pipe_mode) { if (num_rem == 0 && !pctx->pipe_mode) {
log_msg(LOG_ERR, 0, "Expected at least one filename."); log_msg(LOG_ERR, 0, "Expected at least one filename.");
return (1); return (1);
@ -3218,16 +3248,23 @@ init_pc_context(pc_ctx_t *pctx, int argc, char *argv[])
} }
/* /*
* Selectively enable filters while archiving, depending on compression level. * Auto-select filters and preprocessing modes based on compresion level.
* This is not done if user explicitly specified advanced options.
*/ */
if (pctx->archive_mode) { if (!advanced_opts) {
struct filter_flags ff; /*
* Selectively enable filters while archiving, depending on compression level.
*/
if (pctx->archive_mode) {
if (pctx->level > 10) ff.enable_packjpg = 1;
init_filters(&ff);
pctx->enable_packjpg = ff.enable_packjpg;
if (pctx->level > 8) pctx->dispack_preprocess = 1;
}
ff.enable_packjpg = 0; /*
if (pctx->level > 10) ff.enable_packjpg = 1; * Enable other preprocessors based on compresion level.
init_filters(&ff); */
pctx->enable_packjpg = ff.enable_packjpg;
if (pctx->level > 8) pctx->dispack_preprocess = 1;
if (pctx->level > 4) pctx->enable_delta2_encode = 1; if (pctx->level > 4) pctx->enable_delta2_encode = 1;
if (pctx->level > 9) pctx->lzp_preprocess = 1; if (pctx->level > 9) pctx->lzp_preprocess = 1;
if (pctx->level > 3) { if (pctx->level > 3) {