From efe5232cdcd6f7a07023fb5c9041390a1ee2422c Mon Sep 17 00:00:00 2001 From: Moinak Ghosh Date: Sun, 24 Feb 2013 20:05:16 +0530 Subject: [PATCH] Add compatibility to decode old-format parallel hashes created with version 1.2. Bump archive version to 7 as parallel hashes are now merkle style hashes. --- Makefile.in | 12 ++- config | 15 +++ crypto/crypto_utils.c | 49 +++++++-- crypto/old/sha2_utils_old.c | 201 ++++++++++++++++++++++++++++++++++++ crypto/old/sha2_utils_old.h | 11 ++ crypto/old/sha3_utils_old.c | 99 ++++++++++++++++++ crypto/old/sha3_utils_old.h | 9 ++ main.c | 8 +- pcompress.h | 2 +- 9 files changed, 395 insertions(+), 11 deletions(-) create mode 100644 crypto/old/sha2_utils_old.c create mode 100644 crypto/old/sha2_utils_old.h create mode 100644 crypto/old/sha3_utils_old.c create mode 100644 crypto/old/sha3_utils_old.h diff --git a/Makefile.in b/Makefile.in index 01d6b00..43290a1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -35,7 +35,10 @@ CRYPTO_SRCS = crypto/aes/crypto_aes.c crypto/scrypt/crypto_scrypt-nosse.c \ CRYPTO_HDRS = crypto/crypto_utils.h crypto/scrypt/crypto_scrypt.h \ crypto/scrypt/sha256.h crypto/scrypt/crypto_aesctr.h crypto/aes/crypto_aes.h \ crypto/sha2_utils.h crypto/sha3_utils.h $(MAINHDRS) +CRYPTO_COMPAT_SRCS = crypto/old/sha2_utils_old.c crypto/old/sha3_utils_old.c +CRYPTO_COMPAT_HDRS = crypto/old/sha2_utils_old.h crypto/old/sha3_utils_old.h CRYPTO_OBJS = $(CRYPTO_SRCS:.c=.o) +CRYPTO_COMPAT_OBJS = $(CRYPTO_COMPAT_SRCS:.c=.o) CRYPTO_CPPFLAGS=-I@OPENSSL_INCDIR@ XXHASH_SRCS = utils/xxhash.c @@ -170,7 +173,8 @@ COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT -I./lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I./crypto/sha2 \ -I./crypto/scrypt -I./crypto/aes -I./crypto @KEYLEN@ -I./rabin/global \ -I./crypto/keccak -I./transpose -I./crypto/blake2 $(EXTRA_CPPFLAGS) -pedantic -Wall -std=gnu99 \ - -fno-strict-aliasing -Wno-unused-but-set-variable -Wno-enum-compare + -fno-strict-aliasing -Wno-unused-but-set-variable -Wno-enum-compare \ + @COMPAT_CPPFLAGS@ COMMON_VEC_FLAGS = -ftree-vectorize COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block LDLIBS = -ldl -L./buildtmp -Wl,-R@LIBBZ2_DIR@ -lbz2 -L./buildtmp -Wl,-R@LIBZ_DIR@ -lz -lm @LIBBSCLFLAGS@ \ @@ -178,7 +182,8 @@ LDLIBS = -ldl -L./buildtmp -Wl,-R@LIBBZ2_DIR@ -lbz2 -L./buildtmp -Wl,-R@LIBZ_DIR OBJS = $(MAINOBJS) $(LZMAOBJS) $(PPMDOBJS) $(LZFXOBJS) $(LZ4OBJS) $(CRCOBJS) \ $(RABINOBJS) $(BSDIFFOBJS) $(LZPOBJS) $(DELTA2OBJS) @LIBBSCWRAPOBJ@ $(SKEINOBJS) \ $(SKEIN_BLOCK_OBJ) @SHA2ASM_OBJS@ @SHA2_OBJS@ $(KECCAK_OBJS) $(KECCAK_OBJS_ASM) \ -$(TRANSP_OBJS) $(CRYPTO_OBJS) $(ZLIB_OBJS) $(BZLIB_OBJS) $(XXHASH_OBJS) $(BLAKE2_OBJS) +$(TRANSP_OBJS) $(CRYPTO_OBJS) $(ZLIB_OBJS) $(BZLIB_OBJS) $(XXHASH_OBJS) $(BLAKE2_OBJS) \ +@CRYPTO_COMPAT_OBJS@ DEBUG_LINK = g++ -pthread @LIBBSCGEN_OPT@ @EXTRA_OPT_FLAGS@ -fopenmp DEBUG_COMPILE = gcc -g -c @EXTRA_OPT_FLAGS@ @@ -278,6 +283,9 @@ $(TRANSP_OBJS): $(TRANSP_SRCS) $(TRANSP_HDRS) $(CRYPTO_OBJS): $(CRYPTO_SRCS) $(CRYPTO_HDRS) $(COMPILE) $(GEN_OPT) $(CRYPTO_CPPFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@ +$(CRYPTO_COMPAT_OBJS): $(CRYPTO_COMPAT_SRCS) $(CRYPTO_COMPAT_HDRS) + $(COMPILE) $(GEN_OPT) $(CRYPTO_CPPFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@ + $(ZLIB_OBJS): $(ZLIB_SRCS) $(ZLIB_HDRS) $(COMPILE) $(GEN_OPT) $(ZLIB_CPPFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@ diff --git a/config b/config index a3550e5..c68901e 100755 --- a/config +++ b/config @@ -22,6 +22,10 @@ ${prog} [] --with-bzlib= (Default: System) Enable building against an alternate Bzip2 and library installation. --no-sse-detect Do NOT attempt to probe the system's SSE/AVX capability for build flags. +--no-1.3-archive-compat Disable compatibility with compressed archives created with Pcompress + version 1.3 (default: retain compatibility). Hash formats changed from + version 1.3 to 1.4 so this option is required if files created using + 1.3 need to be decompressed by version 1.4 onwards. --use-key256 Use 256-bit encryption keys. Default key length is 128-bit. --help Display this help message. @@ -56,6 +60,8 @@ zlib_prefix= bzlib_prefix= sse_detect=1 sse_opt_flags="-msse2" +crypto_compat_objs='\$\(CRYPTO_COMPAT_OBJS\)' +crypto_compat_flags="-D__HASH_COMPATIBILITY_" rm -rf ./buildtmp mkdir ./buildtmp @@ -144,6 +150,10 @@ do --no-sse-detect) sse_detect=0 ;; + --no-1.3-archive-compat) + crypto_compat_objs="" + crypto_compat_flags="" + ;; --help) usage $0;; *) echo "Unrecognized option: ${arg1}" @@ -510,6 +520,9 @@ keccak_srcs_var="KECCAK_SRCS" keccak_hdrs_var="KECCAK_HDRS" keccak_srcs_asm_var="KECCAK_SRCS_ASM" +crypto_compat_objs_var="CRYPTO_COMPAT_OBJS" +crypto_compat_flags_var="COMPAT_CPPFLAGS" + noslabcppflagsval= debugstatscppflagsval= @@ -551,5 +564,7 @@ s#@${keccak_srcs_var}@#${keccak_srcs}#g s#@${keccak_srcs_asm_var}@#${keccak_srcs_asm}#g s#@${extra_opt_flags_var}@#${extra_opt_flags}#g s#@${sse_opt_flags_var}@#${sse_opt_flags}#g +s#@${crypto_compat_objs_var}@#${crypto_compat_objs}#g +s#@${crypto_compat_flags_var}@#${crypto_compat_flags}#g " > Makefile diff --git a/crypto/crypto_utils.c b/crypto/crypto_utils.c index a945741..f5ce197 100644 --- a/crypto/crypto_utils.c +++ b/crypto/crypto_utils.c @@ -46,6 +46,11 @@ #include "sha2_utils.h" #include "sha3_utils.h" +#ifdef __HASH_COMPATIBILITY_ +#include "old/sha2_utils_old.h" +#include "old/sha3_utils_old.h" +#endif + #define PROVIDER_OPENSSL 0 #define PROVIDER_X64_OPT 1 @@ -181,6 +186,12 @@ compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, uint64_t bytes, in { DEBUG_STAT_EN(double strt, en); +#ifdef __HASH_COMPATIBILITY_ + assert(mt == 0 || mt == 1 || mt == 2); +#else + assert(mt == 0 || mt == 1); +#endif + DEBUG_STAT_EN(strt = get_wtime_millis()); if (cksum == CKSUM_CRC64) { uint64_t *ck = (uint64_t *)cksum_buf; @@ -224,41 +235,67 @@ compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, uint64_t bytes, in if (cksum_provider == PROVIDER_OPENSSL) { if (!mt) ossl_SHA256(cksum_buf, buf, bytes); - else + else if (mt == 1) ossl_SHA256_par(cksum_buf, buf, bytes); +#ifdef __HASH_COMPATIBILITY_ + else + ossl_SHA256_par_old(cksum_buf, buf, bytes); +#endif } else { if (!mt) opt_SHA512t256(cksum_buf, buf, bytes); - else + else if (mt == 1) opt_SHA512t256_par(cksum_buf, buf, bytes); +#ifdef __HASH_COMPATIBILITY_ + else + opt_SHA512t256_par_old(cksum_buf, buf, bytes); +#endif } } else if (cksum == CKSUM_SHA512) { if (cksum_provider == PROVIDER_OPENSSL) { if (!mt) ossl_SHA512(cksum_buf, buf, bytes); - else + else if (mt == 1) ossl_SHA512_par(cksum_buf, buf, bytes); +#ifdef __HASH_COMPATIBILITY_ + else + ossl_SHA512_par_old(cksum_buf, buf, bytes); +#endif } else { if (!mt) opt_SHA512(cksum_buf, buf, bytes); - else + else if (mt == 1) opt_SHA512_par(cksum_buf, buf, bytes); +#ifdef __HASH_COMPATIBILITY_ + else + opt_SHA512_par_old(cksum_buf, buf, bytes); +#endif } } else if (cksum == CKSUM_KECCAK256) { if (!mt) { if (Keccak256(cksum_buf, buf, bytes) != 0) return (-1); - } else { + } else if (mt == 1) { if (Keccak256_par(cksum_buf, buf, bytes) != 0) return (-1); +#ifdef __HASH_COMPATIBILITY_ + } else { + if (Keccak256_par_old(cksum_buf, buf, bytes) != 0) + return (-1); +#endif } } else if (cksum == CKSUM_KECCAK512) { if (!mt) { if (Keccak512(cksum_buf, buf, bytes) != 0) return (-1); - } else { + } else if (mt == 1) { if (Keccak512_par(cksum_buf, buf, bytes) != 0) return (-1); +#ifdef __HASH_COMPATIBILITY_ + } else { + if (Keccak512_par_old(cksum_buf, buf, bytes) != 0) + return (-1); +#endif } } else { return (-1); diff --git a/crypto/old/sha2_utils_old.c b/crypto/old/sha2_utils_old.c new file mode 100644 index 0000000..2155f58 --- /dev/null +++ b/crypto/old/sha2_utils_old.c @@ -0,0 +1,201 @@ +/* + * This file is a part of Pcompress, a chunked parallel multi- + * algorithm lossless compression and decompression program. + * + * Copyright (C) 2012 Moinak Ghosh. All rights reserved. + * Use is subject to license terms. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * moinakg@belenix.org, http://moinakg.wordpress.com/ + */ + +#include +#include +#include +#include +#include +#include + +#if defined(_OPENMP) +#include +#endif +#include + +void +ossl_SHA256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes) +{ + uchar_t *pos[2]; + uint64_t len[2]; + uchar_t cksum[2][32]; + int i; + SHA256_CTX *mctx; + + /* + * Is it worth doing the overhead of parallelism ? Buffer large enough ? + * If not then just do a simple serial hashing. + */ + if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) { + mctx = (SHA256_CTX *)malloc(sizeof (SHA256_CTX)); + SHA256_Init(mctx); + SHA256_Update(mctx, buf, bytes); + SHA256_Final(cksum_buf, mctx); + free(mctx); + return; + } + pos[0] = buf; + len[0] = bytes/2; + buf += bytes/2; + pos[1] = buf; + len[1] = bytes - bytes/2; +#if defined(_OPENMP) +# pragma omp parallel for +#endif + for(i = 0; i < 2; ++i) + { + SHA256_CTX ctx; + SHA256_Init(&ctx); + SHA256_Update(&ctx, pos[i], len[i]); + SHA256_Final(cksum[i], &ctx); + } + mctx = (SHA256_CTX *)malloc(sizeof (SHA256_CTX)); + SHA256_Init(mctx); + SHA256_Update(mctx, cksum, 2 * 32); + SHA256_Final(cksum_buf, mctx); + free(mctx); +} + +void +ossl_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes) +{ + uchar_t *pos[2]; + uint64_t len[2]; + uchar_t cksum[2][64]; + int i; + SHA512_CTX *mctx; + + /* + * Is it worth doing the overhead of parallelism ? Buffer large enough ? + * If not then just do a simple hashing. + */ + if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) { + mctx = (SHA512_CTX *)malloc(sizeof (SHA512_CTX)); + SHA512_Init(mctx); + SHA512_Update(mctx, buf, bytes); + SHA512_Final(cksum_buf, mctx); + free(mctx); + return; + } + pos[0] = buf; + len[0] = bytes/2; + pos[1] = buf + bytes/2; + len[1] = bytes - bytes/2; +#if defined(_OPENMP) +# pragma omp parallel for +#endif + for(i = 0; i < 2; ++i) + { + SHA512_CTX ctx; + SHA512_Init(&ctx); + SHA512_Update(&ctx, pos[i], len[i]); + SHA512_Final(cksum[i], &ctx); + } + mctx = (SHA512_CTX *)malloc(sizeof (SHA512_CTX)); + SHA512_Init(mctx); + SHA512_Update(mctx, cksum, 2 * 64); + SHA512_Final(cksum_buf, mctx); + free(mctx); +} + +void +opt_SHA512t256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes) +{ + uchar_t *pos[2]; + uint64_t len[2]; + uchar_t cksum[2][32]; + int i; + SHA512_Context *mctx; + + /* + * Is it worth doing the overhead of parallelism ? Buffer large enough ? + * If not then just do a simple serial hashing. + */ + if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) { + mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context)); + opt_SHA512t256_Init(mctx); + opt_SHA512t256_Update(mctx, buf, bytes); + opt_SHA512t256_Final(mctx, cksum_buf); + free(mctx); + return; + } + pos[0] = buf; + len[0] = bytes/2; + pos[1] = buf + bytes/2; + len[1] = bytes - bytes/2; +#if defined(_OPENMP) +# pragma omp parallel for +#endif + for(i = 0; i < 2; ++i) + { + SHA512_Context ctx; + opt_SHA512t256_Init(&ctx); + opt_SHA512t256_Update(&ctx, pos[i], len[i]); + opt_SHA512t256_Final(&ctx, cksum[i]); + } + mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context)); + opt_SHA512t256_Init(mctx); + opt_SHA512t256_Update(mctx, cksum, 2 * 32); + opt_SHA512t256_Final(mctx, cksum_buf); + free(mctx); +} + +void +opt_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes) +{ + uchar_t *pos[2]; + uint64_t len[2]; + uchar_t cksum[2][64]; + int i; + SHA512_Context *mctx; + + /* + * Is it worth doing the overhead of parallelism ? Buffer large enough ? + * If not then just do a simple serial hashing. + */ + if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) { + mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context)); + opt_SHA512_Init(mctx); + opt_SHA512_Update(mctx, buf, bytes); + opt_SHA512_Final(mctx, cksum_buf); + free(mctx); + return; + } + pos[0] = buf; + len[0] = bytes/2; + pos[1] = buf + bytes/2; + len[1] = bytes - bytes/2; +#if defined(_OPENMP) +# pragma omp parallel for +#endif + for(i = 0; i < 2; ++i) + { + SHA512_Context ctx; + opt_SHA512_Init(&ctx); + opt_SHA512_Update(&ctx, pos[i], len[i]); + opt_SHA512_Final(&ctx, cksum[i]); + } + mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context)); + opt_SHA512_Init(mctx); + opt_SHA512_Update(mctx, cksum, 2 * 64); + opt_SHA512_Final(mctx, cksum_buf); + free(mctx); +} + diff --git a/crypto/old/sha2_utils_old.h b/crypto/old/sha2_utils_old.h new file mode 100644 index 0000000..d4b2ac1 --- /dev/null +++ b/crypto/old/sha2_utils_old.h @@ -0,0 +1,11 @@ +#ifndef _SHA2_UTILS_OLD_H_ +#define _SHA2_UTILS_OLD_H_ + +void ossl_SHA256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes); +void opt_SHA512t256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes); + +void ossl_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes); +void opt_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes); + +#endif + diff --git a/crypto/old/sha3_utils_old.c b/crypto/old/sha3_utils_old.c new file mode 100644 index 0000000..015f7ac --- /dev/null +++ b/crypto/old/sha3_utils_old.c @@ -0,0 +1,99 @@ +/* + * This file is a part of Pcompress, a chunked parallel multi- + * algorithm lossless compression and decompression program. + * + * Copyright (C) 2012 Moinak Ghosh. All rights reserved. + * Use is subject to license terms. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * moinakg@belenix.org, http://moinakg.wordpress.com/ + */ + +#include +#include +#include +#include +#include + +#if defined(_OPENMP) +#include +#endif +#include + +#define KECCAK_BLOCK_SIZE 1024 + +/* + * Helper functions for single-call SHA3 (Keccak) hashing. Both serial + * and parallel versions are provided. + */ + +int +Keccak256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes) +{ + uchar_t *pos[2]; + uint64_t len[2]; + uchar_t cksum[2][32]; + int i, rv[2]; + + /* + * Is it worth doing the overhead of parallelism ? Buffer large enough ? + * If not then just do a simple serial hashing. + */ + if (bytes / 2 <= KECCAK_BLOCK_SIZE * 2) { + return (Keccak_Hash(256, buf, bytes * 8, cksum_buf)); + } + pos[0] = buf; + len[0] = bytes/2; + pos[1] = buf + bytes/2; + len[1] = bytes - bytes/2; +#if defined(_OPENMP) +# pragma omp parallel for +#endif + for(i = 0; i < 2; ++i) + { + rv[i] = Keccak_Hash(256, pos[i], len[i] * 8, cksum[i]); + } + if (rv[0] != 0 || rv[1] != 0) + return (-1); + return (Keccak_Hash(256, (const BitSequence *)cksum, 2 * 32 * 8, cksum_buf)); +} + +int +Keccak512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes) +{ + uchar_t *pos[2]; + uint64_t len[2]; + uchar_t cksum[2][64]; + int i, rv[2]; + + /* + * Is it worth doing the overhead of parallelism ? Buffer large enough ? + * If not then just do a simple serial hashing. + */ + if (bytes / 2 <= KECCAK_BLOCK_SIZE * 2) { + return (Keccak_Hash(512, buf, bytes * 8, cksum_buf)); + } + pos[0] = buf; + len[0] = bytes/2; + pos[1] = buf + bytes/2; + len[1] = bytes - bytes/2; +#if defined(_OPENMP) +# pragma omp parallel for +#endif + for(i = 0; i < 2; ++i) + { + rv[i] = Keccak_Hash(512, pos[i], len[i] * 8, cksum[i]); + } + if (rv[0] != 0 || rv[1] != 0) + return (-1); + return (Keccak_Hash(512, (const BitSequence *)cksum, 2 * 64 * 8, cksum_buf)); +} diff --git a/crypto/old/sha3_utils_old.h b/crypto/old/sha3_utils_old.h new file mode 100644 index 0000000..98bcfb0 --- /dev/null +++ b/crypto/old/sha3_utils_old.h @@ -0,0 +1,9 @@ +#ifndef _SHA3_UTILS_OLD_H_ +#define _SHA3_UTILS_OLD_H_ + +int Keccak256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes); + +int Keccak512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes); + +#endif + diff --git a/main.c b/main.c index 4d642a7..de2b1b2 100644 --- a/main.c +++ b/main.c @@ -977,10 +977,14 @@ start_decompress(const char *filename, const char *to_filename) tdat->compress = _compress_func; tdat->decompress = _decompress_func; tdat->cancel = 0; - if (props.is_single_chunk) + if (props.is_single_chunk) { tdat->cksum_mt = 1; - else + if (version == 6) { + tdat->cksum_mt = 2; // Indicate old format parallel hash + } + } else { tdat->cksum_mt = 0; + } tdat->level = level; tdat->data = NULL; tdat->props = &props; diff --git a/pcompress.h b/pcompress.h index 3296483..92ce825 100644 --- a/pcompress.h +++ b/pcompress.h @@ -38,7 +38,7 @@ extern "C" { #define CHUNK_FLAG_SZ 1 #define ALGO_SZ 8 #define MIN_CHUNK 2048 -#define VERSION 6 +#define VERSION 7 #define FLAG_DEDUP 1 #define FLAG_DEDUP_FIXED 2 #define FLAG_SINGLE_CHUNK 4