Revamp Filter handling code.

1) Really avoid adding filter xattr for non-processed files.
2) Clean up filter error handling.
3) Avoid libarchive data writes in filter callbacks.
4) Have libarchive data writes in a single place.
5) Properly handle skipping filter processing for a file.
6) Fix temporary file pathname handling.

archive/pc_arc_filter.c

@@ -168,7 +168,6 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
 	struct scratch_buffer *sdat = (struct scratch_buffer *)filter_private;
 	uchar_t *mapbuf, *out;
 	uint64_t len, in_size = 0, len1;
-	ssize_t rv;
 
 	len = archive_entry_size(fi->entry);
 	len1 = len;
@@ -225,7 +224,13 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
 		 * version number. We also check if it is supported.
 		 */
 		if (mapbuf[0] != 'J' || mapbuf[1] != 'S' || !pjg_version_supported(mapbuf[2])) {
-			return (archive_write_data(fi->target_arc, sdat->in_buff, len));
+			uint8_t *out = malloc(len);
+
+			memcpy(out, sdat->in_buff, len);
+			fi->fout->output_type = FILTER_OUTPUT_MEM;
+			fi->fout->out = out;
+			fi->fout->out_size = len;
+			return (FILTER_RETURN_SOFT_ERROR);
 		}
 	}
 
@@ -242,13 +247,11 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
 		}
 		munmap(mapbuf, len1);
 
-		in_size = LE64(len);
+		fi->fout->output_type = FILTER_OUTPUT_MEM;
-		rv = archive_write_data(fi->target_arc, &in_size, 8);
+		fi->fout->out = out;
-		if (rv != 8)
+		fi->fout->out_size = len;
-			return (rv);
+		fi->fout->hdr.in_size = LE64(len);
-		rv = archive_write_data(fi->target_arc, out, len);
+		return (ARCHIVE_OK);
-		free(out);
-		return (rv);
 	}
 
 	/*
@@ -257,17 +260,23 @@ packjpg_filter(struct filter_info *fi, void *filter_private)
 	out = NULL;
 	if ((len = packjpg_filter_process(mapbuf, in_size, &out)) == 0) {
 		/*
-		 * If filter failed we write out the original data and indicate a
+		 * If filter failed we indicate a soft error to continue the
-		 * soft error to continue the archive extraction.
+		 * archive extraction.
 		 */
 		free(out);
-		if (archive_write_data(fi->target_arc, mapbuf, len1) < len1)
+		out = malloc(len);
-			return (FILTER_RETURN_ERROR);
+		memcpy(out, sdat->in_buff, len);
+
+		fi->fout->output_type = FILTER_OUTPUT_MEM;
+		fi->fout->out = out;
+		fi->fout->out_size = len;
 		return (FILTER_RETURN_SOFT_ERROR);
 	}
-	rv = archive_write_data(fi->target_arc, out, len);
+
-	free(out);
+	fi->fout->output_type = FILTER_OUTPUT_MEM;
-	return (rv);
+	fi->fout->out = out;
+	fi->fout->out_size = len;
+	return (ARCHIVE_OK);
 }
 
 ssize_t
@@ -276,7 +285,6 @@ packpnm_filter(struct filter_info *fi, void *filter_private)
 	struct scratch_buffer *sdat = (struct scratch_buffer *)filter_private;
 	uchar_t *mapbuf, *out;
 	uint64_t len, in_size = 0, len1;
-	ssize_t rv;
 
 	len = archive_entry_size(fi->entry);
 	len1 = len;
@@ -327,7 +335,13 @@ packpnm_filter(struct filter_info *fi, void *filter_private)
 		 * Write the raw data and skip.
 		 */
 		if (identify_pnm_type(mapbuf, len - 8) != 2) {
-			return (archive_write_data(fi->target_arc, sdat->in_buff, len));
+			uint8_t *out = malloc(len);
+
+			memcpy(out, sdat->in_buff, len);
+			fi->fout->output_type = FILTER_OUTPUT_MEM;
+			fi->fout->out = out;
+			fi->fout->out_size = len;
+			return (FILTER_RETURN_SOFT_ERROR);
 		}
 	}
 
@@ -344,13 +358,11 @@ packpnm_filter(struct filter_info *fi, void *filter_private)
 		}
 		munmap(mapbuf, len1);
 
-		in_size = LE64(len);
+		fi->fout->output_type = FILTER_OUTPUT_MEM;
-		rv = archive_write_data(fi->target_arc, &in_size, 8);
+		fi->fout->out = out;
-		if (rv != 8)
+		fi->fout->out_size = len;
-			return (rv);
+		fi->fout->hdr.in_size = LE64(len);
-		rv = archive_write_data(fi->target_arc, out, len);
+		return (ARCHIVE_OK);
-		free(out);
-		return (rv);
 	}
 
 	/*
@@ -359,17 +371,23 @@ packpnm_filter(struct filter_info *fi, void *filter_private)
 	out = NULL;
 	if ((len = packpnm_filter_process(mapbuf, in_size, &out)) == 0) {
 		/*
-		 * If filter failed we write out the original data and indicate a
+		 * If filter failed we indicate a soft error to continue the
-		 * soft error to continue the archive extraction.
+		 * archive extraction.
 		 */
 		free(out);
-		if (archive_write_data(fi->target_arc, mapbuf, len1) < len1)
+		out = malloc(len);
-			return (FILTER_RETURN_ERROR);
+		memcpy(out, sdat->in_buff, len);
+
+		fi->fout->output_type = FILTER_OUTPUT_MEM;
+		fi->fout->out = out;
+		fi->fout->out_size = len;
 		return (FILTER_RETURN_SOFT_ERROR);
 	}
-	rv = archive_write_data(fi->target_arc, out, len);
+
-	free(out);
+	fi->fout->output_type = FILTER_OUTPUT_MEM;
-	return (rv);
+	fi->fout->out = out;
+	fi->fout->out_size = len;
+	return (ARCHIVE_OK);
 }
 #endif /* _MPLV2_LICENSE_ */
 
@@ -380,7 +398,6 @@ wavpack_filter(struct filter_info *fi, void *filter_private)
 	struct scratch_buffer *sdat = (struct scratch_buffer *)filter_private;
 	uchar_t *mapbuf, *out;
 	uint64_t len, in_size = 0, len1;
-	ssize_t rv;
 
 	len = archive_entry_size(fi->entry);
 	len1 = len;
@@ -434,7 +451,13 @@ wavpack_filter(struct filter_info *fi, void *filter_private)
 		 */
 		wpkstr = (char *)mapbuf;
 		if (strncmp(wpkstr, "wvpk", 4) != 0) {
-			return (archive_write_data(fi->target_arc, sdat->in_buff, len));
+			uint8_t *out = malloc(len);
+
+			memcpy(out, sdat->in_buff, len);
+			fi->fout->output_type = FILTER_OUTPUT_MEM;
+			fi->fout->out = out;
+			fi->fout->out_size = len;
+			return (FILTER_RETURN_SOFT_ERROR);
 		}
 	}
 
@@ -451,32 +474,36 @@ wavpack_filter(struct filter_info *fi, void *filter_private)
 		}
 		munmap(mapbuf, len1);
 
-		in_size = LE64(len1);
+		fi->fout->output_type = FILTER_OUTPUT_MEM;
-		rv = archive_write_data(fi->target_arc, &in_size, 8);
+		fi->fout->out = out;
-		if (rv != 8)
+		fi->fout->out_size = len;
-			return (rv);
+		fi->fout->hdr.in_size = LE64(len1);
-		rv = archive_write_data(fi->target_arc, out, len);
+		return (ARCHIVE_OK);
-		free(out);
-		return (rv);
 	}
 
 	/*
 	 * Decompression case.
 	 */
 	out = NULL;
-	if ((len = wavpack_filter_decode(mapbuf, len, &out, in_size)) == 0) {
+	if ((len = wavpack_filter_decode(mapbuf, in_size, &out, len)) == 0) {
 		/*
-		 * If filter failed we write out the original data and indicate a
+		 * If filter failed we indicate a soft error to continue the
-		 * soft error to continue the archive extraction.
+		 * archive extraction.
 		 */
 		free(out);
-		if (archive_write_data(fi->target_arc, mapbuf, len1) < len1)
+		out = malloc(len);
-			return (FILTER_RETURN_ERROR);
+		memcpy(out, sdat->in_buff, len);
+
+		fi->fout->output_type = FILTER_OUTPUT_MEM;
+		fi->fout->out = out;
+		fi->fout->out_size = len;
 		return (FILTER_RETURN_SOFT_ERROR);
 	}
-	rv = archive_write_data(fi->target_arc, out, len);
+
-	free(out);
+	fi->fout->output_type = FILTER_OUTPUT_MEM;
-	return (rv);
+	fi->fout->out = out;
+	fi->fout->out_size = len;
+	return (ARCHIVE_OK);
 }
 #endif /* _ENABLE_WAVPACK_ */
 

archive/pc_arc_filter.h

@@ -42,6 +42,9 @@ extern "C" {
 #define	FILTER_RETURN_SOFT_ERROR	(-2)
 #define FILTER_XATTR_ENTRY  "_._pc_filter_xattr"
 
+#define	FILTER_OUTPUT_MEM	1
+#define	FILTER_OUTPUT_FILE	2
+
 #define HELPER_DEF_BUFSIZ       (512 * 1024)
 #define WVPK_FILE_SIZE_LIMIT    (18 * 1024 * 1024)
 
@@ -57,6 +60,20 @@ extern "C" {
 #	define  PJG_APPVERSION2         (25)
 #endif
 
+#pragma	pack(1)
+typedef struct _filter_std_hdr {
+	uint64_t in_size;
+} filter_std_hdr_t;
+#pragma	pack()
+
+typedef struct _filter_output {
+	int output_type;
+	uint8_t *out;
+	size_t out_size;
+	int out_fd;
+	filter_std_hdr_t hdr;
+} filter_output_t;
+
 struct filter_info {
 	struct archive *source_arc;
 	struct archive *target_arc;
@@ -65,6 +82,7 @@ struct filter_info {
 	int compressing, block_size;
 	int *type_ptr;
 	int cmp_level;
+	filter_output_t *fout;
 	uchar_t scratch_buffer;
 	size_t scratch_buffer_size;
 };

archive/pc_archive.c

@@ -1022,8 +1022,8 @@ setup_extractor(pc_ctx_t *pctx)
 
 static ssize_t
 process_by_filter(int fd, int *typ, struct archive *target_arc,
-    struct archive *source_arc, struct archive_entry *entry, int cmp,
+    struct archive *source_arc, struct archive_entry *entry,
-    int level)
+    filter_output_t *fout, int cmp, int level)
 {
 	struct filter_info fi;
 	int64_t wrtn;
@@ -1036,9 +1036,10 @@ process_by_filter(int fd, int *typ, struct archive *target_arc,
 	fi.block_size = AW_BLOCK_SIZE;
 	fi.type_ptr = typ;
 	fi.cmp_level = level;
+	fi.fout = fout;
 	wrtn = (*(typetab[(*typ >> 3)].filter_func))(&fi, typetab[(*typ >> 3)].filter_private);
 	if (wrtn == FILTER_RETURN_ERROR) {
-		log_msg(LOG_ERR, 0, "Error invoking filter module: %s",
+		log_msg(LOG_ERR, 0, "Warning: Error invoking filter: %s (skipping)",
 		    typetab[(*typ >> 3)].filter_name);
 	}
 	return (wrtn);
@@ -1075,6 +1076,7 @@ copy_file_data(pc_ctx_t *pctx, struct archive *arc, struct archive_entry *entry,
 	uchar_t *mapbuf;
 	int rv, fd, typ1;
 	const char *fpath;
+	filter_output_t fout;
 
 	typ1 = typ;
 	offset = 0;
@@ -1093,22 +1095,41 @@ copy_file_data(pc_ctx_t *pctx, struct archive *arc, struct archive_entry *entry,
 			int64_t rv;
 			char *fname = typetab[(typ >> 3)].filter_name;
 
-			archive_entry_xattr_add_entry(entry, FILTER_XATTR_ENTRY,
+			pctx->ctype = typ;
-			    fname, strlen(fname));
+			rv = process_by_filter(fd, &(pctx->ctype), arc, NULL, entry,
+			    &fout, 1, pctx->level);
+			if (rv != FILTER_RETURN_SKIP &&
+			    rv != FILTER_RETURN_ERROR) {
+				if (fout.output_type == FILTER_OUTPUT_MEM) {
+					archive_entry_xattr_add_entry(entry, FILTER_XATTR_ENTRY,
+								      fname, strlen(fname));
+					if (write_header(arc, entry) == -1) {
+						close(fd);
+						return (-1);
+					}
+					rv = archive_write_data(arc, &(fout.hdr),
+								sizeof (fout.hdr));
+					if (rv != sizeof (fout.hdr))
+						return (rv);
+					rv = archive_write_data(arc, fout.out,
+								fout.out_size);
+					free(fout.out);
+					close(fd);
+					if (rv != fout.out_size)
+						return (ARCHIVE_FATAL);
+					else
+						return (ARCHIVE_OK);
+				} else {
+					log_msg(LOG_WARN, 0,
+						"Unsupported filter output for entry: %s.",
+						archive_entry_pathname(entry));
+					return (ARCHIVE_FATAL);
+				}
+			}
 			if (write_header(arc, entry) == -1) {
 				close(fd);
 				return (-1);
 			}
-			pctx->ctype = typ;
-			rv = process_by_filter(fd, &(pctx->ctype), arc, NULL, entry,
-			    1, pctx->level);
-			if (rv == FILTER_RETURN_ERROR) {
-				close(fd);
-				return (-1);
-			} else if (rv != FILTER_RETURN_SKIP) {
-				close(fd);
-				return (ARCHIVE_OK);
-			}
 		} else {
 			if (write_header(arc, entry) == -1) {
 				close(fd);
@@ -1150,26 +1171,46 @@ do_map:
 					int64_t rv;
 					char *fname = typetab[(typ >> 3)].filter_name;
 
-					archive_entry_xattr_add_entry(entry, FILTER_XATTR_ENTRY,
+					munmap(mapbuf, len);
-					    fname, strlen(fname));
+					rv = process_by_filter(fd, &(pctx->ctype), arc, NULL, entry,
+					    &fout, 1, pctx->level);
+					if (rv != FILTER_RETURN_SKIP &&
+					    rv != FILTER_RETURN_ERROR) {
+						if (fout.output_type == FILTER_OUTPUT_MEM) {
+							archive_entry_xattr_add_entry(entry,
+										      FILTER_XATTR_ENTRY,
+										      fname, strlen(fname));
+							if (write_header(arc, entry) == -1) {
+								close(fd);
+								return (-1);
+							}
+							rv = archive_write_data(arc, &(fout.hdr),
+										sizeof (fout.hdr));
+							if (rv != sizeof (fout.hdr))
+								return (rv);
+							rv = archive_write_data(arc, fout.out,
+										fout.out_size);
+							free(fout.out);
+							close(fd);
+							if (rv != fout.out_size)
+								return (ARCHIVE_FATAL);
+							else
+								return (ARCHIVE_OK);
+						} else {
+							log_msg(LOG_WARN, 0,
+								"Unsupported filter output for entry: %s.",
+								archive_entry_pathname(entry));
+							return (ARCHIVE_FATAL);
+						}
+					}
 					if (write_header(arc, entry) == -1) {
 						close(fd);
 						return (-1);
 					}
-
+					lseek(fd, 0, SEEK_SET);
-					munmap(mapbuf, len);
+					typ = TYPE_COMPRESSED;
-					rv = process_by_filter(fd, &(pctx->ctype), arc, NULL, entry,
+					offset = 0;
-					    1, pctx->level);
+					goto do_map;
-					if (rv == FILTER_RETURN_ERROR) {
-						return (-1);
-					} else if (rv == FILTER_RETURN_SKIP) {
-						lseek(fd, 0, SEEK_SET);
-						typ = TYPE_COMPRESSED;
-						offset = 0;
-						goto do_map;
-					} else {
-						return (ARCHIVE_OK);
-					}
 				} else {
 					if (write_header(arc, entry) == -1) {
 						close(fd);
@@ -1264,7 +1305,8 @@ archiver_thread_func(void *dat) {
 		if (rbytes == -1) break;
 		archive_entry_copy_sourcepath(entry, fpath);
 		if (archive_read_disk_entry_from_file(ard, entry, -1, NULL) != ARCHIVE_OK) {
-			log_msg(LOG_WARN, 1, "archive_read_disk_entry_from_file:\n  %s", archive_error_string(ard));
+			log_msg(LOG_WARN, 1, "archive_read_disk_entry_from_file:\n  %s",
+			    archive_error_string(ard));
 			archive_entry_clear(entry);
 			continue;
 		}
@@ -1328,6 +1370,9 @@ archiver_thread_func(void *dat) {
 		ent = entry;
 		while (ent != NULL) {
 			if (write_entry(pctx, arc, ent, typ) != 0) {
+				log_msg(LOG_WARN, 1, "Error archiving entry: %s\n%s",
+				    archive_entry_pathname(entry),
+				    archive_error_string(ard));
 				goto done;
 			}
 			ent = spare_entry;
@@ -1370,42 +1415,63 @@ copy_data_out(struct archive *ar, struct archive *aw, struct archive_entry *entr
 	int64_t offset;
 	const void *buff;
 	size_t size;
-	int r;
+	int r, ret;
+	filter_output_t fout;
 
+	ret = ARCHIVE_OK;
 	if (typ != TYPE_UNKNOWN) {
 		if (typetab[(typ >> 3)].filter_func != NULL) {
 			int64_t rv;
 
-			rv = process_by_filter(-1, &typ, aw, ar, entry, 0, 0);
+			rv = process_by_filter(-1, &typ, aw, ar, entry, &fout, 0, 0);
 			if (rv == FILTER_RETURN_ERROR) {
 				archive_set_error(ar, archive_errno(aw),
 				    "%s", archive_error_string(aw));
 				return (ARCHIVE_FATAL);
 
-			} else if (rv == FILTER_RETURN_SKIP) {
+			} else if (rv == FILTER_RETURN_SOFT_ERROR ||
-				log_msg(LOG_WARN, 0, "Filter function skipped.");
+				   rv == FILTER_RETURN_SKIP) {
-				return (ARCHIVE_WARN);
+				if (rv == FILTER_RETURN_SKIP) {
-
+					log_msg(LOG_WARN, 0, "Filter function skipped"
-			} else if (rv == FILTER_RETURN_SOFT_ERROR) {
+						" for entry: %s.",
-				log_msg(LOG_WARN, 0, "Filter function failed for entry: %s.",
+						archive_entry_pathname(entry));
-				    archive_entry_pathname(entry));
+				} else {
+					log_msg(LOG_WARN, 0, "Filter function failed"
+						" for entry: %s.",
+						archive_entry_pathname(entry));
+				}
 				pctx->errored_count++;
 				if (pctx->err_paths_fd) {
-					fprintf(pctx->err_paths_fd, "%s,%s",
+					fprintf(pctx->err_paths_fd, "%s,%s\n",
 					    archive_entry_pathname(entry),
 					    typetab[(typ >> 3)].filter_name);
 				}
-				return (ARCHIVE_WARN);
+				ret = ARCHIVE_WARN;
+			}
+			if (fout.output_type == FILTER_OUTPUT_MEM) {
+				int rv;
+				rv = archive_write_data(aw, fout.out, fout.out_size);
+				free(fout.out);
+				if (rv < ret)
+					ret = rv;
+				return (ret);
 			} else {
-				return (ARCHIVE_OK);
+				log_msg(LOG_WARN, 0,
+					"Unsupported filter output for entry: %s.",
+					archive_entry_pathname(entry));
+				return (ARCHIVE_FATAL);
 			}
 		}
+		/*
+		 * If the filter above fails we fall through below to consume
+		 * the data for the entry.
+		 */
 	}
 
 	for (;;) {
 		r = archive_read_data_block(ar, &buff, &size, &offset);
 		if (r == ARCHIVE_EOF)
-			return (ARCHIVE_OK);
+			break;
 		if (r != ARCHIVE_OK)
 			return (r);
 		r = (int)archive_write_data_block(aw, buff, size, offset);
@@ -1417,7 +1483,7 @@ copy_data_out(struct archive *ar, struct archive *aw, struct archive_entry *entr
 			return (r);
 		}
 	}
-	return (ARCHIVE_OK);
+	return (ret);
 }
 
 static int
@@ -1613,11 +1679,6 @@ extractor_thread_func(void *dat) {
 		}
 
 		typ = TYPE_UNKNOWN;
-		if (archive_entry_filetype(entry) == AE_IFREG) {
-			const char *fpath = archive_entry_pathname(entry);
-			typ = detect_type_by_ext(fpath, strlen(fpath));
-		}
-
 		/*
 		 * Workaround for libarchive weirdness on Non MAC OS X platforms for filenames
 		 * starting with '._'. See above ...

archive/wavpack_helper.c

@@ -327,7 +327,7 @@ wavpack_filter_encode(uchar_t *in_buf, size_t len, uchar_t **out_buf, int cmp_le
 
 	if (cmp_level < 6) {
 		loc_config.flags |= CONFIG_FAST_FLAG;
-	} else if (cmp_level > 8) {
+	} else if (cmp_level > 7) {
 		loc_config.flags |= CONFIG_HIGH_FLAG;
 	}
 

pcompress.c

@@ -1320,10 +1320,22 @@ start_decompress(pc_ctx_t *pctx, const char *filename, char *to_filename)
 
 	if (flags & FLAG_ARCHIVE) {
 		if (pctx->enable_rabin_global) {
-			strcpy(pctx->archive_temp_file, to_filename);
+			char cwd[MAXPATHLEN];
-			strcat(pctx->archive_temp_file, "/.data");
+
+			if (to_filename[0] != PATHSEP_CHAR) {
+				if (getcwd(cwd, MAXPATHLEN) == NULL) {
+					log_msg(LOG_ERR, 1, "Cannot get current dir");
+					UNCOMP_BAIL;
+				}
+
+				snprintf(pctx->archive_temp_file, sizeof (pctx->archive_temp_file),
+				    "%s" PATHSEP_STR "%s" PATHSEP_STR ".data", cwd, to_filename);
+			} else {
+				snprintf(pctx->archive_temp_file, sizeof (pctx->archive_temp_file),
+					 "%s" PATHSEP_STR ".data", to_filename);
+			}
 			if ((pctx->archive_temp_fd = open(pctx->archive_temp_file,
-			O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR)) == -1) {
+			    O_WRONLY|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR)) == -1) {
 				log_msg(LOG_ERR, 1, "Cannot open temporary data file in "
 				    "target directory.");
 				UNCOMP_BAIL;

utils/utils.h

@@ -182,6 +182,7 @@ typedef int32_t bsize_t;
 #endif
 
 #define	PATHSEP_CHAR	'/'
+#define	PATHSEP_STR	"/"
 #define	BYTES_TO_MB(x) ((x) / (1024 * 1024))
 #define	U64_P(x) *((uint64_t *)(x))
 #define	U32_P(x) *((uint32_t *)(x))