greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Revamp Filter handling code.

Browse source 
1) Really avoid adding filter xattr for non-processed files.
2) Clean up filter error handling.
3) Avoid libarchive data writes in filter callbacks.
4) Have libarchive data writes in a single place.
5) Properly handle skipping filter processing for a file.
6) Fix temporary file pathname handling.
This commit is contained in:
Moinak Ghosh 2014-12-14 23:37:40 +05:30
parent dfe18ef48f
commit 2cd41ec257
6 changed files with 223 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

125
archive/pc_arc_filter.c
View file

@ -168,7 +168,6 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
struct scratch_buffer *sdat = (struct scratch_buffer *)filter_private;
uchar_t *mapbuf, *out;
uint64_t len, in_size = 0, len1;
ssize_t rv;
len = archive_entry_size(fi->entry);
len1 = len;
@ -225,7 +224,13 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
* version number. We also check if it is supported.
*/
if (mapbuf[0] != 'J' || mapbuf[1] != 'S' || !pjg_version_supported(mapbuf[2])) {
return (archive_write_data(fi->target_arc, sdat->in_buff, len));
uint8_t *out = malloc(len);
memcpy(out, sdat->in_buff, len);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
return (FILTER_RETURN_SOFT_ERROR);
}
}
@ -242,13 +247,11 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
}
munmap(mapbuf, len1);
in_size = LE64(len);
rv = archive_write_data(fi->target_arc, &in_size, 8);
if (rv != 8)
return (rv);
rv = archive_write_data(fi->target_arc, out, len);
free(out);
return (rv);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
fi->fout->hdr.in_size = LE64(len);
return (ARCHIVE_OK);
}
/*
@ -257,17 +260,23 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
out = NULL;
if ((len = packjpg_filter_process(mapbuf, in_size, &out)) == 0) {
/*
* If filter failed we write out the original data and indicate a
* soft error to continue the archive extraction.
* If filter failed we indicate a soft error to continue the
* archive extraction.
*/
free(out);
if (archive_write_data(fi->target_arc, mapbuf, len1) < len1)
return (FILTER_RETURN_ERROR);
out = malloc(len);
memcpy(out, sdat->in_buff, len);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
return (FILTER_RETURN_SOFT_ERROR);
}
rv = archive_write_data(fi->target_arc, out, len);
free(out);
return (rv);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
return (ARCHIVE_OK);
}
ssize_t
@ -276,7 +285,6 @@ packpnm_filter(struct filter_info *fi, void *filter_private)
struct scratch_buffer *sdat = (struct scratch_buffer *)filter_private;
uchar_t *mapbuf, *out;
uint64_t len, in_size = 0, len1;
ssize_t rv;
len = archive_entry_size(fi->entry);
len1 = len;
@ -327,7 +335,13 @@ packpnm_filter(struct filter_info *fi, void *filter_private)
* Write the raw data and skip.
*/
if (identify_pnm_type(mapbuf, len - 8) != 2) {
return (archive_write_data(fi->target_arc, sdat->in_buff, len));
uint8_t *out = malloc(len);
memcpy(out, sdat->in_buff, len);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
return (FILTER_RETURN_SOFT_ERROR);
}
}
@ -344,13 +358,11 @@ packpnm_filter(struct filter_info *fi, void *filter_private)
}
munmap(mapbuf, len1);
in_size = LE64(len);
rv = archive_write_data(fi->target_arc, &in_size, 8);
if (rv != 8)
return (rv);
rv = archive_write_data(fi->target_arc, out, len);
free(out);
return (rv);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
fi->fout->hdr.in_size = LE64(len);
return (ARCHIVE_OK);
}
/*
@ -359,17 +371,23 @@ packpnm_filter(struct filter_info *fi, void *filter_private)
out = NULL;
if ((len = packpnm_filter_process(mapbuf, in_size, &out)) == 0) {
/*
* If filter failed we write out the original data and indicate a
* soft error to continue the archive extraction.
* If filter failed we indicate a soft error to continue the
* archive extraction.
*/
free(out);
if (archive_write_data(fi->target_arc, mapbuf, len1) < len1)
return (FILTER_RETURN_ERROR);
out = malloc(len);
memcpy(out, sdat->in_buff, len);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
return (FILTER_RETURN_SOFT_ERROR);
}
rv = archive_write_data(fi->target_arc, out, len);
free(out);
return (rv);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
return (ARCHIVE_OK);
}
#endif /* _MPLV2_LICENSE_ */
@ -380,7 +398,6 @@ wavpack_filter(struct filter_info *fi, void *filter_private)
struct scratch_buffer *sdat = (struct scratch_buffer *)filter_private;
uchar_t *mapbuf, *out;
uint64_t len, in_size = 0, len1;
ssize_t rv;
len = archive_entry_size(fi->entry);
len1 = len;
@ -434,7 +451,13 @@ wavpack_filter(struct filter_info *fi, void *filter_private)
*/
wpkstr = (char *)mapbuf;
if (strncmp(wpkstr, "wvpk", 4) != 0) {
return (archive_write_data(fi->target_arc, sdat->in_buff, len));
uint8_t *out = malloc(len);
memcpy(out, sdat->in_buff, len);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
return (FILTER_RETURN_SOFT_ERROR);
}
}
@ -451,32 +474,36 @@ wavpack_filter(struct filter_info *fi, void *filter_private)
}
munmap(mapbuf, len1);
in_size = LE64(len1);
rv = archive_write_data(fi->target_arc, &in_size, 8);
if (rv != 8)
return (rv);
rv = archive_write_data(fi->target_arc, out, len);
free(out);
return (rv);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
fi->fout->hdr.in_size = LE64(len1);
return (ARCHIVE_OK);
}
/*
* Decompression case.
*/
out = NULL;
if ((len = wavpack_filter_decode(mapbuf, len, &out, in_size)) == 0) {
if ((len = wavpack_filter_decode(mapbuf, in_size, &out, len)) == 0) {
/*
* If filter failed we write out the original data and indicate a
* soft error to continue the archive extraction.
* If filter failed we indicate a soft error to continue the
* archive extraction.
*/
free(out);
if (archive_write_data(fi->target_arc, mapbuf, len1) < len1)
return (FILTER_RETURN_ERROR);
out = malloc(len);
memcpy(out, sdat->in_buff, len);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
return (FILTER_RETURN_SOFT_ERROR);
}
rv = archive_write_data(fi->target_arc, out, len);
free(out);
return (rv);
fi->fout->output_type = FILTER_OUTPUT_MEM;
fi->fout->out = out;
fi->fout->out_size = len;
return (ARCHIVE_OK);
}
#endif /* _ENABLE_WAVPACK_ */

18
archive/pc_arc_filter.h
View file

@ -42,6 +42,9 @@ extern "C" {
#define FILTER_RETURN_SOFT_ERROR (-2)
#define FILTER_XATTR_ENTRY "_._pc_filter_xattr"
#define FILTER_OUTPUT_MEM 1
#define FILTER_OUTPUT_FILE 2
#define HELPER_DEF_BUFSIZ (512 * 1024)
#define WVPK_FILE_SIZE_LIMIT (18 * 1024 * 1024)
@ -57,6 +60,20 @@ extern "C" {
# define PJG_APPVERSION2 (25)
#endif
#pragma pack(1)
typedef struct _filter_std_hdr {
uint64_t in_size;
} filter_std_hdr_t;
#pragma pack()
typedef struct _filter_output {
int output_type;
uint8_t *out;
size_t out_size;
int out_fd;
filter_std_hdr_t hdr;
} filter_output_t;
struct filter_info {
struct archive *source_arc;
struct archive *target_arc;
@ -65,6 +82,7 @@ struct filter_info {
int compressing, block_size;
int *type_ptr;
int cmp_level;
filter_output_t *fout;
uchar_t scratch_buffer;
size_t scratch_buffer_size;
};

139
archive/pc_archive.c
View file

@ -1022,8 +1022,8 @@ setup_extractor(pc_ctx_t *pctx)
static ssize_t
process_by_filter(int fd, int *typ, struct archive *target_arc,
struct archive *source_arc, struct archive_entry *entry, int cmp,
int level)
struct archive *source_arc, struct archive_entry *entry,
filter_output_t *fout, int cmp, int level)
{
struct filter_info fi;
int64_t wrtn;
@ -1036,9 +1036,10 @@ process_by_filter(int fd, int *typ, struct archive *target_arc,
fi.block_size = AW_BLOCK_SIZE;
fi.type_ptr = typ;
fi.cmp_level = level;
fi.fout = fout;
wrtn = (*(typetab[(*typ >> 3)].filter_func))(&fi, typetab[(*typ >> 3)].filter_private);
if (wrtn == FILTER_RETURN_ERROR) {
log_msg(LOG_ERR, 0, "Error invoking filter module: %s",
log_msg(LOG_ERR, 0, "Warning: Error invoking filter: %s (skipping)",
typetab[(*typ >> 3)].filter_name);
}
return (wrtn);
@ -1075,6 +1076,7 @@ copy_file_data(pc_ctx_t *pctx, struct archive *arc, struct archive_entry *entry,
uchar_t *mapbuf;
int rv, fd, typ1;
const char *fpath;
filter_output_t fout;
typ1 = typ;
offset = 0;
@ -1093,21 +1095,40 @@ copy_file_data(pc_ctx_t *pctx, struct archive *arc, struct archive_entry *entry,
int64_t rv;
char *fname = typetab[(typ >> 3)].filter_name;
pctx->ctype = typ;
rv = process_by_filter(fd, &(pctx->ctype), arc, NULL, entry,
&fout, 1, pctx->level);
if (rv != FILTER_RETURN_SKIP &&
rv != FILTER_RETURN_ERROR) {
if (fout.output_type == FILTER_OUTPUT_MEM) {
archive_entry_xattr_add_entry(entry, FILTER_XATTR_ENTRY,
fname, strlen(fname));
if (write_header(arc, entry) == -1) {
close(fd);
return (-1);
}
pctx->ctype = typ;
rv = process_by_filter(fd, &(pctx->ctype), arc, NULL, entry,
1, pctx->level);
if (rv == FILTER_RETURN_ERROR) {
rv = archive_write_data(arc, &(fout.hdr),
sizeof (fout.hdr));
if (rv != sizeof (fout.hdr))
return (rv);
rv = archive_write_data(arc, fout.out,
fout.out_size);
free(fout.out);
close(fd);
if (rv != fout.out_size)
return (ARCHIVE_FATAL);
else
return (ARCHIVE_OK);
} else {
log_msg(LOG_WARN, 0,
"Unsupported filter output for entry: %s.",
archive_entry_pathname(entry));
return (ARCHIVE_FATAL);
}
}
if (write_header(arc, entry) == -1) {
close(fd);
return (-1);
} else if (rv != FILTER_RETURN_SKIP) {
close(fd);
return (ARCHIVE_OK);
}
} else {
if (write_header(arc, entry) == -1) {
@ -1150,26 +1171,46 @@ do_map:
int64_t rv;
char *fname = typetab[(typ >> 3)].filter_name;
archive_entry_xattr_add_entry(entry, FILTER_XATTR_ENTRY,
munmap(mapbuf, len);
rv = process_by_filter(fd, &(pctx->ctype), arc, NULL, entry,
&fout, 1, pctx->level);
if (rv != FILTER_RETURN_SKIP &&
rv != FILTER_RETURN_ERROR) {
if (fout.output_type == FILTER_OUTPUT_MEM) {
archive_entry_xattr_add_entry(entry,
FILTER_XATTR_ENTRY,
fname, strlen(fname));
if (write_header(arc, entry) == -1) {
close(fd);
return (-1);
}
munmap(mapbuf, len);
rv = process_by_filter(fd, &(pctx->ctype), arc, NULL, entry,
1, pctx->level);
if (rv == FILTER_RETURN_ERROR) {
rv = archive_write_data(arc, &(fout.hdr),
sizeof (fout.hdr));
if (rv != sizeof (fout.hdr))
return (rv);
rv = archive_write_data(arc, fout.out,
fout.out_size);
free(fout.out);
close(fd);
if (rv != fout.out_size)
return (ARCHIVE_FATAL);
else
return (ARCHIVE_OK);
} else {
log_msg(LOG_WARN, 0,
"Unsupported filter output for entry: %s.",
archive_entry_pathname(entry));
return (ARCHIVE_FATAL);
}
}
if (write_header(arc, entry) == -1) {
close(fd);
return (-1);
} else if (rv == FILTER_RETURN_SKIP) {
}
lseek(fd, 0, SEEK_SET);
typ = TYPE_COMPRESSED;
offset = 0;
goto do_map;
} else {
return (ARCHIVE_OK);
}
} else {
if (write_header(arc, entry) == -1) {
close(fd);
@ -1264,7 +1305,8 @@ archiver_thread_func(void *dat) {
if (rbytes == -1) break;
archive_entry_copy_sourcepath(entry, fpath);
if (archive_read_disk_entry_from_file(ard, entry, -1, NULL) != ARCHIVE_OK) {
log_msg(LOG_WARN, 1, "archive_read_disk_entry_from_file:\n %s", archive_error_string(ard));
log_msg(LOG_WARN, 1, "archive_read_disk_entry_from_file:\n %s",
archive_error_string(ard));
archive_entry_clear(entry);
continue;
}
@ -1328,6 +1370,9 @@ archiver_thread_func(void *dat) {
ent = entry;
while (ent != NULL) {
if (write_entry(pctx, arc, ent, typ) != 0) {
log_msg(LOG_WARN, 1, "Error archiving entry: %s\n%s",
archive_entry_pathname(entry),
archive_error_string(ard));
goto done;
}
ent = spare_entry;
@ -1370,42 +1415,63 @@ copy_data_out(struct archive *ar, struct archive *aw, struct archive_entry *entr
int64_t offset;
const void *buff;
size_t size;
int r;
int r, ret;
filter_output_t fout;
ret = ARCHIVE_OK;
if (typ != TYPE_UNKNOWN) {
if (typetab[(typ >> 3)].filter_func != NULL) {
int64_t rv;
rv = process_by_filter(-1, &typ, aw, ar, entry, 0, 0);
rv = process_by_filter(-1, &typ, aw, ar, entry, &fout, 0, 0);
if (rv == FILTER_RETURN_ERROR) {
archive_set_error(ar, archive_errno(aw),
"%s", archive_error_string(aw));
return (ARCHIVE_FATAL);
} else if (rv == FILTER_RETURN_SKIP) {
log_msg(LOG_WARN, 0, "Filter function skipped.");
return (ARCHIVE_WARN);
} else if (rv == FILTER_RETURN_SOFT_ERROR) {
log_msg(LOG_WARN, 0, "Filter function failed for entry: %s.",
} else if (rv == FILTER_RETURN_SOFT_ERROR ||
rv == FILTER_RETURN_SKIP) {
if (rv == FILTER_RETURN_SKIP) {
log_msg(LOG_WARN, 0, "Filter function skipped"
" for entry: %s.",
archive_entry_pathname(entry));
} else {
log_msg(LOG_WARN, 0, "Filter function failed"
" for entry: %s.",
archive_entry_pathname(entry));
}
pctx->errored_count++;
if (pctx->err_paths_fd) {
fprintf(pctx->err_paths_fd, "%s,%s",
fprintf(pctx->err_paths_fd, "%s,%s\n",
archive_entry_pathname(entry),
typetab[(typ >> 3)].filter_name);
}
return (ARCHIVE_WARN);
ret = ARCHIVE_WARN;
}
if (fout.output_type == FILTER_OUTPUT_MEM) {
int rv;
rv = archive_write_data(aw, fout.out, fout.out_size);
free(fout.out);
if (rv < ret)
ret = rv;
return (ret);
} else {
return (ARCHIVE_OK);
log_msg(LOG_WARN, 0,
"Unsupported filter output for entry: %s.",
archive_entry_pathname(entry));
return (ARCHIVE_FATAL);
}
}
/*
* If the filter above fails we fall through below to consume
* the data for the entry.
*/
}
for (;;) {
r = archive_read_data_block(ar, &buff, &size, &offset);
if (r == ARCHIVE_EOF)
return (ARCHIVE_OK);
break;
if (r != ARCHIVE_OK)
return (r);
r = (int)archive_write_data_block(aw, buff, size, offset);
@ -1417,7 +1483,7 @@ copy_data_out(struct archive *ar, struct archive *aw, struct archive_entry *entr
return (r);
}
}
return (ARCHIVE_OK);
return (ret);
}
static int
@ -1613,11 +1679,6 @@ extractor_thread_func(void *dat) {
}
typ = TYPE_UNKNOWN;
if (archive_entry_filetype(entry) == AE_IFREG) {
const char *fpath = archive_entry_pathname(entry);
typ = detect_type_by_ext(fpath, strlen(fpath));
}
/*
* Workaround for libarchive weirdness on Non MAC OS X platforms for filenames
* starting with '._'. See above ...

2
archive/wavpack_helper.c
View file

@ -327,7 +327,7 @@ wavpack_filter_encode(uchar_t *in_buf, size_t len, uchar_t **out_buf, int cmp_le
if (cmp_level < 6) {
loc_config.flags |= CONFIG_FAST_FLAG;
} else if (cmp_level > 8) {
} else if (cmp_level > 7) {
loc_config.flags |= CONFIG_HIGH_FLAG;
}

16
pcompress.c
View file

@ -1320,8 +1320,20 @@ start_decompress(pc_ctx_t *pctx, const char *filename, char *to_filename)
if (flags & FLAG_ARCHIVE) {
if (pctx->enable_rabin_global) {
strcpy(pctx->archive_temp_file, to_filename);
strcat(pctx->archive_temp_file, "/.data");
char cwd[MAXPATHLEN];
if (to_filename[0] != PATHSEP_CHAR) {
if (getcwd(cwd, MAXPATHLEN) == NULL) {
log_msg(LOG_ERR, 1, "Cannot get current dir");
UNCOMP_BAIL;
}
snprintf(pctx->archive_temp_file, sizeof (pctx->archive_temp_file),
"%s" PATHSEP_STR "%s" PATHSEP_STR ".data", cwd, to_filename);
} else {
snprintf(pctx->archive_temp_file, sizeof (pctx->archive_temp_file),
"%s" PATHSEP_STR ".data", to_filename);
}
if ((pctx->archive_temp_fd = open(pctx->archive_temp_file,
O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR)) == -1) {
log_msg(LOG_ERR, 1, "Cannot open temporary data file in "

1
utils/utils.h
View file

@ -182,6 +182,7 @@ typedef int32_t bsize_t;
#endif
#define PATHSEP_CHAR '/'
#define PATHSEP_STR "/"
#define BYTES_TO_MB(x) ((x) / (1024 * 1024))
#define U64_P(x) *((uint64_t *)(x))
#define U32_P(x) *((uint32_t *)(x))