greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add compatibility to decode old-format parallel hashes created with version 1.2.

Browse source 
Bump archive version to 7 as parallel hashes are now merkle style hashes.
This commit is contained in:
Moinak Ghosh 2013-02-24 20:05:16 +05:30
parent 5f6217bb1f
commit efe5232cdc
9 changed files with 395 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
Makefile.in
View file

@ -35,7 +35,10 @@ CRYPTO_SRCS = crypto/aes/crypto_aes.c crypto/scrypt/crypto_scrypt-nosse.c \
CRYPTO_HDRS = crypto/crypto_utils.h crypto/scrypt/crypto_scrypt.h \
crypto/scrypt/sha256.h crypto/scrypt/crypto_aesctr.h crypto/aes/crypto_aes.h \
crypto/sha2_utils.h crypto/sha3_utils.h $(MAINHDRS)
CRYPTO_COMPAT_SRCS = crypto/old/sha2_utils_old.c crypto/old/sha3_utils_old.c
CRYPTO_COMPAT_HDRS = crypto/old/sha2_utils_old.h crypto/old/sha3_utils_old.h
CRYPTO_OBJS = $(CRYPTO_SRCS:.c=.o)
CRYPTO_COMPAT_OBJS = $(CRYPTO_COMPAT_SRCS:.c=.o)
CRYPTO_CPPFLAGS=-I@OPENSSL_INCDIR@
XXHASH_SRCS = utils/xxhash.c
@ -170,7 +173,8 @@ COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT
-I./lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I./crypto/sha2 \
-I./crypto/scrypt -I./crypto/aes -I./crypto @KEYLEN@ -I./rabin/global \
-I./crypto/keccak -I./transpose -I./crypto/blake2 $(EXTRA_CPPFLAGS) -pedantic -Wall -std=gnu99 \
-fno-strict-aliasing -Wno-unused-but-set-variable -Wno-enum-compare
-fno-strict-aliasing -Wno-unused-but-set-variable -Wno-enum-compare \
@COMPAT_CPPFLAGS@
COMMON_VEC_FLAGS = -ftree-vectorize
COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block
LDLIBS = -ldl -L./buildtmp -Wl,-R@LIBBZ2_DIR@ -lbz2 -L./buildtmp -Wl,-R@LIBZ_DIR@ -lz -lm @LIBBSCLFLAGS@ \
@ -178,7 +182,8 @@ LDLIBS = -ldl -L./buildtmp -Wl,-R@LIBBZ2_DIR@ -lbz2 -L./buildtmp -Wl,-R@LIBZ_DIR
OBJS = $(MAINOBJS) $(LZMAOBJS) $(PPMDOBJS) $(LZFXOBJS) $(LZ4OBJS) $(CRCOBJS) \
$(RABINOBJS) $(BSDIFFOBJS) $(LZPOBJS) $(DELTA2OBJS) @LIBBSCWRAPOBJ@ $(SKEINOBJS) \
$(SKEIN_BLOCK_OBJ) @SHA2ASM_OBJS@ @SHA2_OBJS@ $(KECCAK_OBJS) $(KECCAK_OBJS_ASM) \
$(TRANSP_OBJS) $(CRYPTO_OBJS) $(ZLIB_OBJS) $(BZLIB_OBJS) $(XXHASH_OBJS) $(BLAKE2_OBJS)
$(TRANSP_OBJS) $(CRYPTO_OBJS) $(ZLIB_OBJS) $(BZLIB_OBJS) $(XXHASH_OBJS) $(BLAKE2_OBJS) \
@CRYPTO_COMPAT_OBJS@
DEBUG_LINK = g++ -pthread @LIBBSCGEN_OPT@ @EXTRA_OPT_FLAGS@ -fopenmp
DEBUG_COMPILE = gcc -g -c @EXTRA_OPT_FLAGS@
@ -278,6 +283,9 @@ $(TRANSP_OBJS): $(TRANSP_SRCS) $(TRANSP_HDRS)
$(CRYPTO_OBJS): $(CRYPTO_SRCS) $(CRYPTO_HDRS)
$(COMPILE) $(GEN_OPT) $(CRYPTO_CPPFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
$(CRYPTO_COMPAT_OBJS): $(CRYPTO_COMPAT_SRCS) $(CRYPTO_COMPAT_HDRS)
$(COMPILE) $(GEN_OPT) $(CRYPTO_CPPFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
$(ZLIB_OBJS): $(ZLIB_SRCS) $(ZLIB_HDRS)
$(COMPILE) $(GEN_OPT) $(ZLIB_CPPFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@

15
config
View file

@ -22,6 +22,10 @@ ${prog} [<options>]
--with-bzlib=<path to Bzip2 library installation tree> (Default: System)
Enable building against an alternate Bzip2 and library installation.
--no-sse-detect Do NOT attempt to probe the system's SSE/AVX capability for build flags.
--no-1.3-archive-compat Disable compatibility with compressed archives created with Pcompress
version 1.3 (default: retain compatibility). Hash formats changed from
version 1.3 to 1.4 so this option is required if files created using
1.3 need to be decompressed by version 1.4 onwards.
--use-key256 Use 256-bit encryption keys. Default key length is 128-bit.
--help Display this help message.
@ -56,6 +60,8 @@ zlib_prefix=
bzlib_prefix=
sse_detect=1
sse_opt_flags="-msse2"
crypto_compat_objs='\$\(CRYPTO_COMPAT_OBJS\)'
crypto_compat_flags="-D__HASH_COMPATIBILITY_"
rm -rf ./buildtmp
mkdir ./buildtmp
@ -144,6 +150,10 @@ do
--no-sse-detect)
sse_detect=0
;;
--no-1.3-archive-compat)
crypto_compat_objs=""
crypto_compat_flags=""
;;
--help) usage $0;;
*)
echo "Unrecognized option: ${arg1}"
@ -510,6 +520,9 @@ keccak_srcs_var="KECCAK_SRCS"
keccak_hdrs_var="KECCAK_HDRS"
keccak_srcs_asm_var="KECCAK_SRCS_ASM"
crypto_compat_objs_var="CRYPTO_COMPAT_OBJS"
crypto_compat_flags_var="COMPAT_CPPFLAGS"
noslabcppflagsval=
debugstatscppflagsval=
@ -551,5 +564,7 @@ s#@${keccak_srcs_var}@#${keccak_srcs}#g
s#@${keccak_srcs_asm_var}@#${keccak_srcs_asm}#g
s#@${extra_opt_flags_var}@#${extra_opt_flags}#g
s#@${sse_opt_flags_var}@#${sse_opt_flags}#g
s#@${crypto_compat_objs_var}@#${crypto_compat_objs}#g
s#@${crypto_compat_flags_var}@#${crypto_compat_flags}#g
" > Makefile

49
crypto/crypto_utils.c
View file

@ -46,6 +46,11 @@
#include "sha2_utils.h"
#include "sha3_utils.h"
#ifdef __HASH_COMPATIBILITY_
#include "old/sha2_utils_old.h"
#include "old/sha3_utils_old.h"
#endif
#define PROVIDER_OPENSSL 0
#define PROVIDER_X64_OPT 1
@ -181,6 +186,12 @@ compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, uint64_t bytes, in
{
DEBUG_STAT_EN(double strt, en);
#ifdef __HASH_COMPATIBILITY_
assert(mt == 0 || mt == 1 || mt == 2);
#else
assert(mt == 0 || mt == 1);
#endif
DEBUG_STAT_EN(strt = get_wtime_millis());
if (cksum == CKSUM_CRC64) {
uint64_t *ck = (uint64_t *)cksum_buf;
@ -224,41 +235,67 @@ compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, uint64_t bytes, in
if (cksum_provider == PROVIDER_OPENSSL) {
if (!mt)
ossl_SHA256(cksum_buf, buf, bytes);
else
else if (mt == 1)
ossl_SHA256_par(cksum_buf, buf, bytes);
#ifdef __HASH_COMPATIBILITY_
else
ossl_SHA256_par_old(cksum_buf, buf, bytes);
#endif
} else {
if (!mt)
opt_SHA512t256(cksum_buf, buf, bytes);
else
else if (mt == 1)
opt_SHA512t256_par(cksum_buf, buf, bytes);
#ifdef __HASH_COMPATIBILITY_
else
opt_SHA512t256_par_old(cksum_buf, buf, bytes);
#endif
}
} else if (cksum == CKSUM_SHA512) {
if (cksum_provider == PROVIDER_OPENSSL) {
if (!mt)
ossl_SHA512(cksum_buf, buf, bytes);
else
else if (mt == 1)
ossl_SHA512_par(cksum_buf, buf, bytes);
#ifdef __HASH_COMPATIBILITY_
else
ossl_SHA512_par_old(cksum_buf, buf, bytes);
#endif
} else {
if (!mt)
opt_SHA512(cksum_buf, buf, bytes);
else
else if (mt == 1)
opt_SHA512_par(cksum_buf, buf, bytes);
#ifdef __HASH_COMPATIBILITY_
else
opt_SHA512_par_old(cksum_buf, buf, bytes);
#endif
}
} else if (cksum == CKSUM_KECCAK256) {
if (!mt) {
if (Keccak256(cksum_buf, buf, bytes) != 0)
return (-1);
} else {
} else if (mt == 1) {
if (Keccak256_par(cksum_buf, buf, bytes) != 0)
return (-1);
#ifdef __HASH_COMPATIBILITY_
} else {
if (Keccak256_par_old(cksum_buf, buf, bytes) != 0)
return (-1);
#endif
}
} else if (cksum == CKSUM_KECCAK512) {
if (!mt) {
if (Keccak512(cksum_buf, buf, bytes) != 0)
return (-1);
} else {
} else if (mt == 1) {
if (Keccak512_par(cksum_buf, buf, bytes) != 0)
return (-1);
#ifdef __HASH_COMPATIBILITY_
} else {
if (Keccak512_par_old(cksum_buf, buf, bytes) != 0)
return (-1);
#endif
}
} else {
return (-1);

201
crypto/old/sha2_utils_old.c Normal file
View file

@ -0,0 +1,201 @@
/*
* This file is a part of Pcompress, a chunked parallel multi-
* algorithm lossless compression and decompression program.
*
* Copyright (C) 2012 Moinak Ghosh. All rights reserved.
* Use is subject to license terms.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* moinakg@belenix.org, http://moinakg.wordpress.com/
*/
#include <sys/types.h>
#include <stdlib.h>
#include <openssl/sha.h>
#include <sha512.h>
#include <stdio.h>
#include <string.h>
#if defined(_OPENMP)
#include <omp.h>
#endif
#include <utils.h>
void
ossl_SHA256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
{
uchar_t *pos[2];
uint64_t len[2];
uchar_t cksum[2][32];
int i;
SHA256_CTX *mctx;
/*
* Is it worth doing the overhead of parallelism ? Buffer large enough ?
* If not then just do a simple serial hashing.
*/
if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) {
mctx = (SHA256_CTX *)malloc(sizeof (SHA256_CTX));
SHA256_Init(mctx);
SHA256_Update(mctx, buf, bytes);
SHA256_Final(cksum_buf, mctx);
free(mctx);
return;
}
pos[0] = buf;
len[0] = bytes/2;
buf += bytes/2;
pos[1] = buf;
len[1] = bytes - bytes/2;
#if defined(_OPENMP)
# pragma omp parallel for
#endif
for(i = 0; i < 2; ++i)
{
SHA256_CTX ctx;
SHA256_Init(&ctx);
SHA256_Update(&ctx, pos[i], len[i]);
SHA256_Final(cksum[i], &ctx);
}
mctx = (SHA256_CTX *)malloc(sizeof (SHA256_CTX));
SHA256_Init(mctx);
SHA256_Update(mctx, cksum, 2 * 32);
SHA256_Final(cksum_buf, mctx);
free(mctx);
}
void
ossl_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
{
uchar_t *pos[2];
uint64_t len[2];
uchar_t cksum[2][64];
int i;
SHA512_CTX *mctx;
/*
* Is it worth doing the overhead of parallelism ? Buffer large enough ?
* If not then just do a simple hashing.
*/
if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) {
mctx = (SHA512_CTX *)malloc(sizeof (SHA512_CTX));
SHA512_Init(mctx);
SHA512_Update(mctx, buf, bytes);
SHA512_Final(cksum_buf, mctx);
free(mctx);
return;
}
pos[0] = buf;
len[0] = bytes/2;
pos[1] = buf + bytes/2;
len[1] = bytes - bytes/2;
#if defined(_OPENMP)
# pragma omp parallel for
#endif
for(i = 0; i < 2; ++i)
{
SHA512_CTX ctx;
SHA512_Init(&ctx);
SHA512_Update(&ctx, pos[i], len[i]);
SHA512_Final(cksum[i], &ctx);
}
mctx = (SHA512_CTX *)malloc(sizeof (SHA512_CTX));
SHA512_Init(mctx);
SHA512_Update(mctx, cksum, 2 * 64);
SHA512_Final(cksum_buf, mctx);
free(mctx);
}
void
opt_SHA512t256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
{
uchar_t *pos[2];
uint64_t len[2];
uchar_t cksum[2][32];
int i;
SHA512_Context *mctx;
/*
* Is it worth doing the overhead of parallelism ? Buffer large enough ?
* If not then just do a simple serial hashing.
*/
if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) {
mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context));
opt_SHA512t256_Init(mctx);
opt_SHA512t256_Update(mctx, buf, bytes);
opt_SHA512t256_Final(mctx, cksum_buf);
free(mctx);
return;
}
pos[0] = buf;
len[0] = bytes/2;
pos[1] = buf + bytes/2;
len[1] = bytes - bytes/2;
#if defined(_OPENMP)
# pragma omp parallel for
#endif
for(i = 0; i < 2; ++i)
{
SHA512_Context ctx;
opt_SHA512t256_Init(&ctx);
opt_SHA512t256_Update(&ctx, pos[i], len[i]);
opt_SHA512t256_Final(&ctx, cksum[i]);
}
mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context));
opt_SHA512t256_Init(mctx);
opt_SHA512t256_Update(mctx, cksum, 2 * 32);
opt_SHA512t256_Final(mctx, cksum_buf);
free(mctx);
}
void
opt_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
{
uchar_t *pos[2];
uint64_t len[2];
uchar_t cksum[2][64];
int i;
SHA512_Context *mctx;
/*
* Is it worth doing the overhead of parallelism ? Buffer large enough ?
* If not then just do a simple serial hashing.
*/
if (bytes / 2 <= SHA512_BLOCK_SIZE * 4) {
mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context));
opt_SHA512_Init(mctx);
opt_SHA512_Update(mctx, buf, bytes);
opt_SHA512_Final(mctx, cksum_buf);
free(mctx);
return;
}
pos[0] = buf;
len[0] = bytes/2;
pos[1] = buf + bytes/2;
len[1] = bytes - bytes/2;
#if defined(_OPENMP)
# pragma omp parallel for
#endif
for(i = 0; i < 2; ++i)
{
SHA512_Context ctx;
opt_SHA512_Init(&ctx);
opt_SHA512_Update(&ctx, pos[i], len[i]);
opt_SHA512_Final(&ctx, cksum[i]);
}
mctx = (SHA512_Context *)malloc(sizeof (SHA512_Context));
opt_SHA512_Init(mctx);
opt_SHA512_Update(mctx, cksum, 2 * 64);
opt_SHA512_Final(mctx, cksum_buf);
free(mctx);
}

11
crypto/old/sha2_utils_old.h Normal file
View file

@ -0,0 +1,11 @@
#ifndef _SHA2_UTILS_OLD_H_
#define _SHA2_UTILS_OLD_H_
void ossl_SHA256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
void opt_SHA512t256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
void ossl_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
void opt_SHA512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
#endif

99
crypto/old/sha3_utils_old.c Normal file
View file

@ -0,0 +1,99 @@
/*
* This file is a part of Pcompress, a chunked parallel multi-
* algorithm lossless compression and decompression program.
*
* Copyright (C) 2012 Moinak Ghosh. All rights reserved.
* Use is subject to license terms.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* moinakg@belenix.org, http://moinakg.wordpress.com/
*/
#include <sys/types.h>
#include <KeccakNISTInterface.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#if defined(_OPENMP)
#include <omp.h>
#endif
#include <utils.h>
#define KECCAK_BLOCK_SIZE 1024
/*
* Helper functions for single-call SHA3 (Keccak) hashing. Both serial
* and parallel versions are provided.
*/
int
Keccak256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
{
uchar_t *pos[2];
uint64_t len[2];
uchar_t cksum[2][32];
int i, rv[2];
/*
* Is it worth doing the overhead of parallelism ? Buffer large enough ?
* If not then just do a simple serial hashing.
*/
if (bytes / 2 <= KECCAK_BLOCK_SIZE * 2) {
return (Keccak_Hash(256, buf, bytes * 8, cksum_buf));
}
pos[0] = buf;
len[0] = bytes/2;
pos[1] = buf + bytes/2;
len[1] = bytes - bytes/2;
#if defined(_OPENMP)
# pragma omp parallel for
#endif
for(i = 0; i < 2; ++i)
{
rv[i] = Keccak_Hash(256, pos[i], len[i] * 8, cksum[i]);
}
if (rv[0] != 0 || rv[1] != 0)
return (-1);
return (Keccak_Hash(256, (const BitSequence *)cksum, 2 * 32 * 8, cksum_buf));
}
int
Keccak512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes)
{
uchar_t *pos[2];
uint64_t len[2];
uchar_t cksum[2][64];
int i, rv[2];
/*
* Is it worth doing the overhead of parallelism ? Buffer large enough ?
* If not then just do a simple serial hashing.
*/
if (bytes / 2 <= KECCAK_BLOCK_SIZE * 2) {
return (Keccak_Hash(512, buf, bytes * 8, cksum_buf));
}
pos[0] = buf;
len[0] = bytes/2;
pos[1] = buf + bytes/2;
len[1] = bytes - bytes/2;
#if defined(_OPENMP)
# pragma omp parallel for
#endif
for(i = 0; i < 2; ++i)
{
rv[i] = Keccak_Hash(512, pos[i], len[i] * 8, cksum[i]);
}
if (rv[0] != 0 || rv[1] != 0)
return (-1);
return (Keccak_Hash(512, (const BitSequence *)cksum, 2 * 64 * 8, cksum_buf));
}

9
crypto/old/sha3_utils_old.h Normal file
View file

@ -0,0 +1,9 @@
#ifndef _SHA3_UTILS_OLD_H_
#define _SHA3_UTILS_OLD_H_
int Keccak256_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
int Keccak512_par_old(uchar_t *cksum_buf, uchar_t *buf, uint64_t bytes);
#endif

8
main.c
View file

@ -977,10 +977,14 @@ start_decompress(const char *filename, const char *to_filename)
tdat->compress = _compress_func;
tdat->decompress = _decompress_func;
tdat->cancel = 0;
if (props.is_single_chunk)
if (props.is_single_chunk) {
tdat->cksum_mt = 1;
else
if (version == 6) {
tdat->cksum_mt = 2; // Indicate old format parallel hash
}
} else {
tdat->cksum_mt = 0;
}
tdat->level = level;
tdat->data = NULL;
tdat->props = &props;

2
pcompress.h
View file

@ -38,7 +38,7 @@ extern "C" {
#define CHUNK_FLAG_SZ 1
#define ALGO_SZ 8
#define MIN_CHUNK 2048
#define VERSION 6
#define VERSION 7
#define FLAG_DEDUP 1
#define FLAG_DEDUP_FIXED 2
#define FLAG_SINGLE_CHUNK 4