Add support for SHA256 and SHA512 digests from OpenSSL library.

This commit is contained in:
Moinak Ghosh 2012-10-03 22:43:02 +05:30
parent 3492949aff
commit 2578676b1b
6 changed files with 94 additions and 9 deletions

View file

@ -85,10 +85,10 @@ BAKFILES = *~ lzma/*~ lzfx/*~ lz4/*~ rabin/*~ bsdiff/*~ lzp/*~ utils/*~
RM = rm -f RM = rm -f
COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT_PROPS \ COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT_PROPS \
-DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32 \ -DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32 \
-I./lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I./lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I@OPENSSL_INCDIR@
COMMON_VEC_FLAGS = -ftree-vectorize COMMON_VEC_FLAGS = -ftree-vectorize
COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block
LDLIBS = -ldl -lbz2 $(ZLIB_DIR) -lz -lm @LIBBSCLFLAGS@ LDLIBS = -ldl -lbz2 $(ZLIB_DIR) -lz -lm @LIBBSCLFLAGS@ -L@OPENSSL_LIBDIR@ -lcrypto
OBJS = $(MAINOBJS) $(LZMAOBJS) $(PPMDOBJS) $(LZFXOBJS) $(LZ4OBJS) $(CRCOBJS) \ OBJS = $(MAINOBJS) $(LZMAOBJS) $(PPMDOBJS) $(LZFXOBJS) $(LZ4OBJS) $(CRCOBJS) \
$(RABINOBJS) $(BSDIFFOBJS) $(LZPOBJS) @LIBBSCWRAPOBJ@ $(SKEINOBJS) $(SKEIN_BLOCK_OBJ) $(RABINOBJS) $(BSDIFFOBJS) $(LZPOBJS) @LIBBSCWRAPOBJ@ $(SKEINOBJS) $(SKEIN_BLOCK_OBJ)

View file

@ -98,8 +98,8 @@ NOTE: The option "libbsc" uses Ilya Grebnov's block sorting compression library
algorithms with some extra CPU and very low RAM overhead. Using algorithms with some extra CPU and very low RAM overhead. Using
delta encoding in conjunction with this may not always be beneficial. delta encoding in conjunction with this may not always be beneficial.
'-S' <cksum> '-S' <cksum>
- Specify chunk checksum to use: CRC64, SKEIN256, SKEIN512 - Specify chunk checksum to use: CRC64, SKEIN256, SKEIN512, SHA256 and
Default one is SKEIN256. The implementation actually uses SKEIN SHA512. Default one is SKEIN256. The implementation actually uses SKEIN
512-256. This is 25% slower than simple CRC64 but is many times more 512-256. This is 25% slower than simple CRC64 but is many times more
robust than CRC64 in detecting data integrity errors. SKEIN is a robust than CRC64 in detecting data integrity errors. SKEIN is a
finalist in the NIST SHA-3 standard selection process and is one of finalist in the NIST SHA-3 standard selection process and is one of

68
config
View file

@ -11,9 +11,12 @@ ${prog} [<options>]
--enable-debug Enable debug mode compilation (default: disabled). --enable-debug Enable debug mode compilation (default: disabled).
--disable-allocator Disable use of internal memory allocator mechanism (default: enabled). --disable-allocator Disable use of internal memory allocator mechanism (default: enabled).
--enable-debug-stats Enable printing of some verbose debug info (default: disabled). --enable-debug-stats Enable printing of some verbose debug info (default: disabled).
--with-libbsc=<path_to_libbsc_source> --with-libbsc=<path to libbsc source>
Enable support for libbsc (See: libbsc.com). Full path to the libbsc Enable support for libbsc (See: libbsc.com). Full path to the libbsc
source tree must be provided. It links the library statically. source tree must be provided. It links the library statically.
--with-openssl=<path to OpenSSL installation tree> (Default: System)
This defaults to the system's OpenSSL library. You can use this option
if you want to use an alternate OpenSSL installation.
--help Display this help message. --help Display this help message.
_EOF _EOF
@ -30,6 +33,9 @@ libbsclflags=
libbscwrapobj= libbscwrapobj=
libbscgenopt= libbscgenopt=
libbsccppflags= libbsccppflags=
openssl_prefix=
openssl_libdir=
openssl_incdir=
while [ "${arg1}" != "" ] while [ "${arg1}" != "" ]
do do
@ -55,6 +61,9 @@ do
echo "Libbsc not found in ${path}, not enabling libbsc support.\n" echo "Libbsc not found in ${path}, not enabling libbsc support.\n"
fi fi
;; ;;
--with-openssl=*)
openssl_prefix=`echo ${arg1} | cut -f2 -d"="`
;;
--help) usage $0;; --help) usage $0;;
*) *)
echo "Unrecognized option: ${arg1}" echo "Unrecognized option: ${arg1}"
@ -91,6 +100,58 @@ then
skeinblock='\$\(SKEIN_BLOCK_ASM\)' skeinblock='\$\(SKEIN_BLOCK_ASM\)'
fi fi
# Detect OpenSSL library
for lib in "${openssl_prefix}/lib64" "${openssl_prefix}/usr/lib64" \
"${openssl_prefix}/lib" "${openssl_prefix}/usr/lib" \
"${openssl_prefix}/ssl/lib64" "${openssl_prefix}/ssl/lib"
do
if [ -d ${lib} ]
then
if [ -f "${lib}/libcrypto.so" ]
then
openssl_libdir=${lib}
break
else
if [ -f "${lib}/libcrypto.a" ]
then
openssl_libdir=${lib}
break
fi
fi
fi
done
if [ "x${openssl_libdir}" = "x" ]
then
echo "ERROR: OpenSSL libraries not detected."
exit 1
fi
# Detect OpenSSL headers
for inc in "${openssl_prefix}/include" \
"${openssl_prefix}/usr/include" \
"${openssl_prefix}/ssl/include"
do
if [ -d ${inc} ]
then
if [ -f "${inc}/openssl/sha.h" ]
then
openssl_incdir=${inc}
break
fi
fi
done
if [ "x${openssl_incdir}" = "x" ]
then
echo "ERROR: OpenSSL header files not detected."
if [ "x${openssl_prefix}" = "x" ]
then
echo "Depending on your system you may need to install the openssl-devel package."
fi
exit 1
fi
linkvar="LINK" linkvar="LINK"
compilevar="COMPILE" compilevar="COMPILE"
compilecppvar="COMPILE_cpp" compilecppvar="COMPILE_cpp"
@ -111,6 +172,9 @@ libbscwrapobjvar="LIBBSCWRAPOBJ"
libbscgenoptvar="LIBBSCGEN_OPT" libbscgenoptvar="LIBBSCGEN_OPT"
libbsccppflagsvar="LIBBSCCPPFLAGS" libbsccppflagsvar="LIBBSCCPPFLAGS"
openssllibdirvar="OPENSSL_LIBDIR"
opensslincdirvar="OPENSSL_INCDIR"
noslabcppflagsval= noslabcppflagsval=
debugstatscppflagsval= debugstatscppflagsval=
@ -136,5 +200,7 @@ s#@${libbscwrapobjvar}@#${libbscwrapobj}#g
s#@${libbscgenoptvar}@#${libbscgenopt}#g s#@${libbscgenoptvar}@#${libbscgenopt}#g
s#@${libbsccppflagsvar}@#${libbsccppflags}#g s#@${libbsccppflagsvar}@#${libbsccppflags}#g
s#@${skeinblockvar}@#${skeinblock}#g s#@${skeinblockvar}@#${skeinblock}#g
s#@${openssllibdirvar}@#${openssl_libdir}#g
s#@${opensslincdirvar}@#${openssl_incdir}#g
" > Makefile " > Makefile

4
main.c
View file

@ -144,8 +144,8 @@ usage(void)
" '-L' - Enable LZP pre-compression. This improves compression ratio of all\n" " '-L' - Enable LZP pre-compression. This improves compression ratio of all\n"
" algorithms with some extra CPU and very low RAM overhead.\n" " algorithms with some extra CPU and very low RAM overhead.\n"
" '-S' <cksum>\n" " '-S' <cksum>\n"
" - Specify chunk checksum to use: CRC64, SKEIN256, SKEIN512\n" " - Specify chunk checksum to use: CRC64, SKEIN256, SKEIN512, SHA256 and\n"
" Default one is SKEIN256.\n" " SHA512. Default one is SKEIN256.\n"
" '-F' - Perform Fixed-Block Deduplication. Faster than '-D' in some cases\n" " '-F' - Perform Fixed-Block Deduplication. Faster than '-D' in some cases\n"
" but with lower deduplication ratio.\n" " but with lower deduplication ratio.\n"
" '-B' <1..5>\n" " '-B' <1..5>\n"

View file

@ -33,6 +33,7 @@
#include <link.h> #include <link.h>
#include <rabin_dedup.h> #include <rabin_dedup.h>
#include <skein.h> #include <skein.h>
#include <openssl/sha.h>
#include "utils.h" #include "utils.h"
@ -46,7 +47,9 @@ static struct {
} cksum_props[] = { } cksum_props[] = {
{"CRC64", CKSUM_CRC64, 8}, {"CRC64", CKSUM_CRC64, 8},
{"SKEIN256", CKSUM_SKEIN256, 32}, {"SKEIN256", CKSUM_SKEIN256, 32},
{"SKEIN512", CKSUM_SKEIN512, 64} {"SKEIN512", CKSUM_SKEIN512, 64},
{"SHA256", CKSUM_SHA256, 32},
{"SHA512", CKSUM_SHA512, 64}
}; };
extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc); extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc);
@ -334,6 +337,20 @@ compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, ssize_t bytes)
Skein_512_Init(&ctx, 512); Skein_512_Init(&ctx, 512);
Skein_512_Update(&ctx, buf, bytes); Skein_512_Update(&ctx, buf, bytes);
Skein_512_Final(&ctx, cksum_buf); Skein_512_Final(&ctx, cksum_buf);
} else if (cksum == CKSUM_SHA256) {
SHA256_CTX ctx;
SHA256_Init(&ctx);
SHA256_Update(&ctx, buf, bytes);
SHA256_Final(cksum_buf, &ctx);
} else if (cksum == CKSUM_SKEIN512) {
SHA512_CTX ctx;
SHA512_Init(&ctx);
SHA512_Update(&ctx, buf, bytes);
SHA512_Final(cksum_buf, &ctx);
} else { } else {
return (-1); return (-1);
} }

View file

@ -108,7 +108,9 @@ typedef ssize_t bsize_t;
typedef enum { typedef enum {
CKSUM_CRC64 = 0x100, CKSUM_CRC64 = 0x100,
CKSUM_SKEIN256 = 0x200, CKSUM_SKEIN256 = 0x200,
CKSUM_SKEIN512 = 0x300 CKSUM_SKEIN512 = 0x300,
CKSUM_SHA256 = 0x400,
CKSUM_SHA512 = 0x500
} cksum_t; } cksum_t;
#define CKSUM_MASK 0x700 #define CKSUM_MASK 0x700