Add compatibility to decode old-format parallel hashes created with version 1.2.

Bump archive version to 7 as parallel hashes are now merkle style hashes.

Makefile.in

@@ -35,7 +35,10 @@ CRYPTO_SRCS = crypto/aes/crypto_aes.c crypto/scrypt/crypto_scrypt-nosse.c \
 CRYPTO_HDRS = crypto/crypto_utils.h crypto/scrypt/crypto_scrypt.h \
 	crypto/scrypt/sha256.h crypto/scrypt/crypto_aesctr.h crypto/aes/crypto_aes.h \
 	crypto/sha2_utils.h crypto/sha3_utils.h $(MAINHDRS)
+CRYPTO_COMPAT_SRCS = crypto/old/sha2_utils_old.c crypto/old/sha3_utils_old.c
+CRYPTO_COMPAT_HDRS = crypto/old/sha2_utils_old.h crypto/old/sha3_utils_old.h
 CRYPTO_OBJS = $(CRYPTO_SRCS:.c=.o)
+CRYPTO_COMPAT_OBJS = $(CRYPTO_COMPAT_SRCS:.c=.o)
 CRYPTO_CPPFLAGS=-I@OPENSSL_INCDIR@
 
 XXHASH_SRCS = utils/xxhash.c
@@ -170,7 +173,8 @@ COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT
 	-I./lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I./crypto/sha2 \
 	-I./crypto/scrypt -I./crypto/aes -I./crypto @KEYLEN@ -I./rabin/global \
 	-I./crypto/keccak -I./transpose -I./crypto/blake2 $(EXTRA_CPPFLAGS) -pedantic -Wall -std=gnu99 \
-	-fno-strict-aliasing -Wno-unused-but-set-variable -Wno-enum-compare
+	-fno-strict-aliasing -Wno-unused-but-set-variable -Wno-enum-compare \
+	@COMPAT_CPPFLAGS@
 COMMON_VEC_FLAGS = -ftree-vectorize
 COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block
 LDLIBS = -ldl -L./buildtmp -Wl,-R@LIBBZ2_DIR@ -lbz2 -L./buildtmp -Wl,-R@LIBZ_DIR@ -lz -lm @LIBBSCLFLAGS@ \
@@ -178,7 +182,8 @@ LDLIBS = -ldl -L./buildtmp -Wl,-R@LIBBZ2_DIR@ -lbz2 -L./buildtmp -Wl,-R@LIBZ_DIR
 OBJS = $(MAINOBJS) $(LZMAOBJS) $(PPMDOBJS) $(LZFXOBJS) $(LZ4OBJS) $(CRCOBJS) \
 $(RABINOBJS) $(BSDIFFOBJS) $(LZPOBJS) $(DELTA2OBJS) @LIBBSCWRAPOBJ@ $(SKEINOBJS) \
 $(SKEIN_BLOCK_OBJ) @SHA2ASM_OBJS@ @SHA2_OBJS@ $(KECCAK_OBJS) $(KECCAK_OBJS_ASM) \
-$(TRANSP_OBJS) $(CRYPTO_OBJS) $(ZLIB_OBJS) $(BZLIB_OBJS) $(XXHASH_OBJS) $(BLAKE2_OBJS)
+$(TRANSP_OBJS) $(CRYPTO_OBJS) $(ZLIB_OBJS) $(BZLIB_OBJS) $(XXHASH_OBJS) $(BLAKE2_OBJS) \
+@CRYPTO_COMPAT_OBJS@
 
 DEBUG_LINK = g++ -pthread @LIBBSCGEN_OPT@ @EXTRA_OPT_FLAGS@ -fopenmp
 DEBUG_COMPILE = gcc -g -c @EXTRA_OPT_FLAGS@
@@ -278,6 +283,9 @@ $(TRANSP_OBJS): $(TRANSP_SRCS) $(TRANSP_HDRS)
 $(CRYPTO_OBJS): $(CRYPTO_SRCS) $(CRYPTO_HDRS)
 	$(COMPILE) $(GEN_OPT) $(CRYPTO_CPPFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
 
+$(CRYPTO_COMPAT_OBJS): $(CRYPTO_COMPAT_SRCS) $(CRYPTO_COMPAT_HDRS)
+	$(COMPILE) $(GEN_OPT) $(CRYPTO_CPPFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
+
 $(ZLIB_OBJS): $(ZLIB_SRCS) $(ZLIB_HDRS)
 	$(COMPILE) $(GEN_OPT) $(ZLIB_CPPFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
 

config

@@ -22,6 +22,10 @@ ${prog} [<options>]
 --with-bzlib=<path to Bzip2 library installation tree> (Default: System)
                         Enable building against an alternate Bzip2 and library installation.
 --no-sse-detect		Do NOT attempt to probe the system's SSE/AVX capability for build flags. 
+--no-1.3-archive-compat Disable compatibility with compressed archives created with Pcompress
+                        version 1.3 (default: retain compatibility). Hash formats changed from
+                        version 1.3 to 1.4 so this option is required if files created using
+                        1.3 need to be decompressed by version 1.4 onwards.
 --use-key256		Use 256-bit encryption keys. Default key length is 128-bit.
 --help			Display this help message.
 
@@ -56,6 +60,8 @@ zlib_prefix=
 bzlib_prefix=
 sse_detect=1
 sse_opt_flags="-msse2"
+crypto_compat_objs='\$\(CRYPTO_COMPAT_OBJS\)'
+crypto_compat_flags="-D__HASH_COMPATIBILITY_"
 
 rm -rf ./buildtmp
 mkdir ./buildtmp
@@ -144,6 +150,10 @@ do
 	--no-sse-detect)
 		sse_detect=0
 	;;
+	--no-1.3-archive-compat)
+		crypto_compat_objs=""
+		crypto_compat_flags=""
+	;;
 	--help) usage $0;;
 	*)
 		echo "Unrecognized option: ${arg1}"
@@ -510,6 +520,9 @@ keccak_srcs_var="KECCAK_SRCS"
 keccak_hdrs_var="KECCAK_HDRS"
 keccak_srcs_asm_var="KECCAK_SRCS_ASM"
 
+crypto_compat_objs_var="CRYPTO_COMPAT_OBJS"
+crypto_compat_flags_var="COMPAT_CPPFLAGS"
+
 noslabcppflagsval=
 debugstatscppflagsval=
 
@@ -551,5 +564,7 @@ s#@${keccak_srcs_var}@#${keccak_srcs}#g
 s#@${keccak_srcs_asm_var}@#${keccak_srcs_asm}#g
 s#@${extra_opt_flags_var}@#${extra_opt_flags}#g
 s#@${sse_opt_flags_var}@#${sse_opt_flags}#g
+s#@${crypto_compat_objs_var}@#${crypto_compat_objs}#g
+s#@${crypto_compat_flags_var}@#${crypto_compat_flags}#g
 " > Makefile
 

crypto/crypto_utils.c

@@ -46,6 +46,11 @@
 #include "sha2_utils.h"
 #include "sha3_utils.h"
 
+#ifdef __HASH_COMPATIBILITY_
+#include "old/sha2_utils_old.h"
+#include "old/sha3_utils_old.h"
+#endif
+
 #define	PROVIDER_OPENSSL	0
 #define	PROVIDER_X64_OPT	1
 
@@ -181,6 +186,12 @@ compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, uint64_t bytes, in
 {
 	DEBUG_STAT_EN(double strt, en);
 
+#ifdef __HASH_COMPATIBILITY_
+	assert(mt == 0 || mt == 1 || mt == 2);
+#else
+	assert(mt == 0 || mt == 1);
+#endif
+
 	DEBUG_STAT_EN(strt = get_wtime_millis());
 	if (cksum == CKSUM_CRC64) {
 		uint64_t *ck = (uint64_t *)cksum_buf;
@@ -224,41 +235,67 @@ compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, uint64_t bytes, in
 		if (cksum_provider == PROVIDER_OPENSSL) {
 			if (!mt)
 				ossl_SHA256(cksum_buf, buf, bytes);
-			else
+			else if (mt == 1)
 				ossl_SHA256_par(cksum_buf, buf, bytes);
+#ifdef __HASH_COMPATIBILITY_
+			else
+				ossl_SHA256_par_old(cksum_buf, buf, bytes);
+#endif
 		} else {
 			if (!mt)
 				opt_SHA512t256(cksum_buf, buf, bytes);
-			else
+			else if (mt == 1)
 				opt_SHA512t256_par(cksum_buf, buf, bytes);
+#ifdef __HASH_COMPATIBILITY_
+			else
+				opt_SHA512t256_par_old(cksum_buf, buf, bytes);
+#endif
 		}
 	} else if (cksum == CKSUM_SHA512) {
 		if (cksum_provider == PROVIDER_OPENSSL) {
 			if (!mt)
 				ossl_SHA512(cksum_buf, buf, bytes);
-			else
+			else if (mt == 1)
 				ossl_SHA512_par(cksum_buf, buf, bytes);
+#ifdef __HASH_COMPATIBILITY_
+			else
+				ossl_SHA512_par_old(cksum_buf, buf, bytes);
+#endif
 		} else {
 			if (!mt)
 				opt_SHA512(cksum_buf, buf, bytes);
-			else
+			else if (mt == 1)
 				opt_SHA512_par(cksum_buf, buf, bytes);
+#ifdef __HASH_COMPATIBILITY_
+			else
+				opt_SHA512_par_old(cksum_buf, buf, bytes);
+#endif
 		}
 	} else if (cksum == CKSUM_KECCAK256) {
 		if (!mt) {
 			if (Keccak256(cksum_buf, buf, bytes) != 0)
 				return (-1);
-		} else {
+		} else if (mt == 1) {
 			if (Keccak256_par(cksum_buf, buf, bytes) != 0)
 				return (-1);
+#ifdef __HASH_COMPATIBILITY_
+		} else {
+			if (Keccak256_par_old(cksum_buf, buf, bytes) != 0)
+				return (-1);
+#endif
 		}
 	} else if (cksum == CKSUM_KECCAK512) {
 		if (!mt) {
 			if (Keccak512(cksum_buf, buf, bytes) != 0)
 				return (-1);
-		} else {
+		} else if (mt == 1) {
 			if (Keccak512_par(cksum_buf, buf, bytes) != 0)
 				return (-1);
+#ifdef __HASH_COMPATIBILITY_
+		} else {
+			if (Keccak512_par_old(cksum_buf, buf, bytes) != 0)
+				return (-1);
+#endif
 		}
 	} else {
 		return (-1);

crypto/old/sha2_utils_old.c

@@ -0,0 +1,201 @@
+/*
+ * This file is a part of Pcompress, a chunked parallel multi-
+ * algorithm lossless compression and decompression program.
+ *
+ * Copyright (C) 2012 Moinak Ghosh. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * moinakg@belenix.org, http://moinakg.wordpress.com/
+ */
+
+#include <sys/types.h>
+#include <stdlib.h>
+#include <openssl/sha.h>
+#include <sha512.h>
+#include <stdio.h>
+#include <string.h>
+
+#if defined(_OPENMP)
+#include <omp.h>
+#endif
+#include <utils.h>
+
+void
+ossl_SHA256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
+{
+	uchar_t *pos[2];
+	uint64_t len[2];
+	uchar_t cksum[2][32];
+	int i;
+	SHA256_CTX *mctx;
+
+	/*
+	 * Is it worth doing the overhead of parallelism ? Buffer large enough ?
+	 * If not then just do a simple serial hashing.
+	 */
+	if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) {
+		mctx = (SHA256_CTX *)malloc(sizeof (SHA256_CTX));
+		SHA256_Init(mctx);
+		SHA256_Update(mctx, buf, bytes);
+		SHA256_Final(cksum_buf, mctx);
+		free(mctx);
+		return;
+	}
+	pos[0] = buf;
+	len[0] = bytes/2;
+	buf += bytes/2;
+	pos[1] = buf;
+	len[1] = bytes - bytes/2;
+#if defined(_OPENMP)
+#	pragma omp parallel for
+#endif
+	for(i = 0; i < 2; ++i)
+	{
+		SHA256_CTX ctx;
+		SHA256_Init(&ctx);
+		SHA256_Update(&ctx, pos[i], len[i]);
+		SHA256_Final(cksum[i], &ctx);
+	}
+	mctx = (SHA256_CTX *)malloc(sizeof (SHA256_CTX));
+	SHA256_Init(mctx);
+	SHA256_Update(mctx, cksum, 2 * 32);
+	SHA256_Final(cksum_buf, mctx);
+	free(mctx);
+}
+
+void
+ossl_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
+{
+	uchar_t *pos[2];
+	uint64_t len[2];
+	uchar_t cksum[2][64];
+	int i;
+	SHA512_CTX *mctx;
+
+	/*
+	 * Is it worth doing the overhead of parallelism ? Buffer large enough ?
+	 * If not then just do a simple hashing.
+	 */
+	if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) {
+		mctx = (SHA512_CTX *)malloc(sizeof (SHA512_CTX));
+		SHA512_Init(mctx);
+		SHA512_Update(mctx, buf, bytes);
+		SHA512_Final(cksum_buf, mctx);
+		free(mctx);
+		return;
+	}
+	pos[0] = buf;
+	len[0] = bytes/2;
+	pos[1] = buf + bytes/2;
+	len[1] = bytes - bytes/2;
+#if defined(_OPENMP)
+#	pragma omp parallel for
+#endif
+	for(i = 0; i < 2; ++i)
+	{
+		SHA512_CTX ctx;
+		SHA512_Init(&ctx);
+		SHA512_Update(&ctx, pos[i], len[i]);
+		SHA512_Final(cksum[i], &ctx);
+	}
+	mctx = (SHA512_CTX *)malloc(sizeof (SHA512_CTX));
+	SHA512_Init(mctx);
+	SHA512_Update(mctx, cksum, 2 * 64);
+	SHA512_Final(cksum_buf, mctx);
+	free(mctx);
+}
+
+void
+opt_SHA512t256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
+{
+	uchar_t *pos[2];
+	uint64_t len[2];
+	uchar_t cksum[2][32];
+	int i;
+	SHA512_Context *mctx;
+
+	/*
+	 * Is it worth doing the overhead of parallelism ? Buffer large enough ?
+	 * If not then just do a simple serial hashing.
+	 */
+	if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) {
+		mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context));
+		opt_SHA512t256_Init(mctx);
+		opt_SHA512t256_Update(mctx, buf, bytes);
+		opt_SHA512t256_Final(mctx, cksum_buf);
+		free(mctx);
+		return;
+	}
+	pos[0] = buf;
+	len[0] = bytes/2;
+	pos[1] = buf + bytes/2;
+	len[1] = bytes - bytes/2;
+#if defined(_OPENMP)
+#	pragma omp parallel for
+#endif
+	for(i = 0; i < 2; ++i)
+	{
+		SHA512_Context ctx;
+		opt_SHA512t256_Init(&ctx);
+		opt_SHA512t256_Update(&ctx, pos[i], len[i]);
+		opt_SHA512t256_Final(&ctx, cksum[i]);
+	}
+	mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context));
+	opt_SHA512t256_Init(mctx);
+	opt_SHA512t256_Update(mctx, cksum, 2 * 32);
+	opt_SHA512t256_Final(mctx, cksum_buf);
+	free(mctx);
+}
+
+void
+opt_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
+{
+	uchar_t *pos[2];
+	uint64_t len[2];
+	uchar_t cksum[2][64];
+	int i;
+	SHA512_Context *mctx;
+
+	/*
+	 * Is it worth doing the overhead of parallelism ? Buffer large enough ?
+	 * If not then just do a simple serial hashing.
+	 */
+	if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) {
+		mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context));
+		opt_SHA512_Init(mctx);
+		opt_SHA512_Update(mctx, buf, bytes);
+		opt_SHA512_Final(mctx, cksum_buf);
+		free(mctx);
+		return;
+	}
+	pos[0] = buf;
+	len[0] = bytes/2;
+	pos[1] = buf + bytes/2;
+	len[1] = bytes - bytes/2;
+#if defined(_OPENMP)
+#	pragma omp parallel for
+#endif
+	for(i = 0; i < 2; ++i)
+	{
+		SHA512_Context ctx;
+		opt_SHA512_Init(&ctx);
+		opt_SHA512_Update(&ctx, pos[i], len[i]);
+		opt_SHA512_Final(&ctx, cksum[i]);
+	}
+	mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context));
+	opt_SHA512_Init(mctx);
+	opt_SHA512_Update(mctx, cksum, 2 * 64);
+	opt_SHA512_Final(mctx, cksum_buf);
+	free(mctx);
+}
+

crypto/old/sha2_utils_old.h

@@ -0,0 +1,11 @@
+#ifndef _SHA2_UTILS_OLD_H_
+#define	_SHA2_UTILS_OLD_H_
+
+void ossl_SHA256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
+void opt_SHA512t256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
+
+void ossl_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
+void opt_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
+
+#endif
+

crypto/old/sha3_utils_old.c

@@ -0,0 +1,99 @@
+/*
+ * This file is a part of Pcompress, a chunked parallel multi-
+ * algorithm lossless compression and decompression program.
+ *
+ * Copyright (C) 2012 Moinak Ghosh. All rights reserved.
+ * Use is subject to license terms.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * moinakg@belenix.org, http://moinakg.wordpress.com/
+ */
+
+#include <sys/types.h>
+#include <KeccakNISTInterface.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#if defined(_OPENMP)
+#include <omp.h>
+#endif
+#include <utils.h>
+
+#define	KECCAK_BLOCK_SIZE	1024
+
+/*
+ * Helper functions for single-call SHA3 (Keccak) hashing. Both serial
+ * and parallel versions are provided.
+ */
+
+int
+Keccak256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
+{
+	uchar_t *pos[2];
+	uint64_t len[2];
+	uchar_t cksum[2][32];
+	int i, rv[2];
+
+	/*
+	 * Is it worth doing the overhead of parallelism ? Buffer large enough ?
+	 * If not then just do a simple serial hashing.
+	 */
+	if (bytes / 2 <= KECCAK_BLOCK_SIZE * 2) {
+		return (Keccak_Hash(256, buf, bytes * 8, cksum_buf));
+	}
+	pos[0] = buf;
+	len[0] = bytes/2;
+	pos[1] = buf + bytes/2;
+	len[1] = bytes - bytes/2;
+#if defined(_OPENMP)
+#	pragma omp parallel for
+#endif
+	for(i = 0; i < 2; ++i)
+	{
+		rv[i] = Keccak_Hash(256, pos[i], len[i] * 8, cksum[i]);
+	}
+	if (rv[0] != 0 || rv[1] != 0)
+		return (-1);
+	return (Keccak_Hash(256, (const BitSequence *)cksum, 2 * 32 * 8, cksum_buf));
+}
+
+int
+Keccak512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
+{
+	uchar_t *pos[2];
+	uint64_t len[2];
+	uchar_t cksum[2][64];
+	int i, rv[2];
+
+	/*
+	 * Is it worth doing the overhead of parallelism ? Buffer large enough ?
+	 * If not then just do a simple serial hashing.
+	 */
+	if (bytes / 2 <= KECCAK_BLOCK_SIZE * 2) {
+		return (Keccak_Hash(512, buf, bytes * 8, cksum_buf));
+	}
+	pos[0] = buf;
+	len[0] = bytes/2;
+	pos[1] = buf + bytes/2;
+	len[1] = bytes - bytes/2;
+#if defined(_OPENMP)
+#	pragma omp parallel for
+#endif
+	for(i = 0; i < 2; ++i)
+	{
+		rv[i] = Keccak_Hash(512, pos[i], len[i] * 8, cksum[i]);
+	}
+	if (rv[0] != 0 || rv[1] != 0)
+		return (-1);
+	return (Keccak_Hash(512, (const BitSequence *)cksum, 2 * 64 * 8, cksum_buf));
+}

crypto/old/sha3_utils_old.h

@@ -0,0 +1,9 @@
+#ifndef _SHA3_UTILS_OLD_H_
+#define	_SHA3_UTILS_OLD_H_
+
+int Keccak256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
+
+int Keccak512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
+
+#endif
+

main.c

@@ -977,10 +977,14 @@ start_decompress(const char *filename, const char *to_filename)
 		tdat->compress = _compress_func;
 		tdat->decompress = _decompress_func;
 		tdat->cancel = 0;
-		if (props.is_single_chunk)
+		if (props.is_single_chunk) {
 			tdat->cksum_mt = 1;
-		else
+			if (version == 6) {
+				tdat->cksum_mt = 2; // Indicate old format parallel hash
+			}
+		} else {
 			tdat->cksum_mt = 0;
+		}
 		tdat->level = level;
 		tdat->data = NULL;
 		tdat->props = &props;

pcompress.h

@@ -38,7 +38,7 @@ extern "C" {
 #define	CHUNK_FLAG_SZ	1
 #define	ALGO_SZ		8
 #define	MIN_CHUNK	2048
-#define	VERSION		6
+#define	VERSION		7
 #define	FLAG_DEDUP	1
 #define	FLAG_DEDUP_FIXED	2
 #define	FLAG_SINGLE_CHUNK	4