greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Introduce strict compiler flags and fix scores of warnings/issues.

Browse source 
Avoid different optimization flags for Dedupe sources.
Fix liberal mixing of uint64_t and int64_t (should all be uint64_t).
Fix corner case crash when decompressing.
This commit is contained in:
Moinak Ghosh 2012-12-27 23:06:48 +05:30
parent 063624d0d3
commit 26a4f42506
38 changed files with 407 additions and 383 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
Makefile.in
View file

@ -139,7 +139,7 @@ KECCAK_OBJS_ASM = $(KECCAK_SRCS_ASM:.s=.o)
BAKFILES = *~ lzma/*~ lzfx/*~ lz4/*~ rabin/*~ bsdiff/*~ lzp/*~ utils/*~ crypto/sha2/*~ \ BAKFILES = *~ lzma/*~ lzfx/*~ lz4/*~ rabin/*~ bsdiff/*~ lzp/*~ utils/*~ crypto/sha2/*~ \
crypto/sha2/intel/*~ crypto/aes/*~ crypto/scrypt/*~ crypto/*~ rabin/global/*~ \ crypto/sha2/intel/*~ crypto/aes/*~ crypto/scrypt/*~ crypto/*~ rabin/global/*~ \
delta2/*~ crypto/keccak/*~ transpose/*~ delta2/*~ crypto/keccak/*~ transpose/*~ crypto/skein/*~ crypto/keccak/*.o
RM = rm -f RM = rm -f
RM_RF = rm -rf RM_RF = rm -rf
@ -147,7 +147,8 @@ COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT
-DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32 \ -DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32 \
-I./lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I./crypto/sha2 \ -I./lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I./crypto/sha2 \
-I./crypto/scrypt -I./crypto/aes -I./crypto @KEYLEN@ \ -I./crypto/scrypt -I./crypto/aes -I./crypto @KEYLEN@ \
-I./crypto/keccak -I./transpose $(EXTRA_CPPFLAGS) -I./crypto/keccak -I./transpose $(EXTRA_CPPFLAGS) -pedantic -Wall -Werror -std=gnu99 \
-fno-strict-aliasing -Wno-unused-but-set-variable -Wno-enum-compare
COMMON_VEC_FLAGS = -ftree-vectorize COMMON_VEC_FLAGS = -ftree-vectorize
COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block
LDLIBS = -ldl -L@LIBBZ2_DIR@ -lbz2 -L@LIBZ_DIR@ -lz -lm @LIBBSCLFLAGS@ \ LDLIBS = -ldl -L@LIBBZ2_DIR@ -lbz2 -L@LIBZ_DIR@ -lz -lm @LIBBSCLFLAGS@ \
@ -163,7 +164,6 @@ DEBUG_COMPILE_cpp = g++ -m64 -g -msse3 -c
DEBUG_VEC_FLAGS = DEBUG_VEC_FLAGS =
DEBUG_LOOP_OPTFLAGS = DEBUG_LOOP_OPTFLAGS =
DEBUG_GEN_OPT = -O -fno-omit-frame-pointer @LIBBSCGEN_OPT@ DEBUG_GEN_OPT = -O -fno-omit-frame-pointer @LIBBSCGEN_OPT@
DEBUG_RABIN_OPT = -O -fno-omit-frame-pointer
DEBUG_CPPFLAGS = $(COMMON_CPPFLAGS) DEBUG_CPPFLAGS = $(COMMON_CPPFLAGS)
DEBUG_FPTR_FLAG = DEBUG_FPTR_FLAG =
@ -174,7 +174,6 @@ RELEASE_VEC_FLAGS = $(COMMON_VEC_FLAGS)
RELEASE_LOOP_OPTFLAGS = $(COMMON_LOOP_OPTFLAGS) RELEASE_LOOP_OPTFLAGS = $(COMMON_LOOP_OPTFLAGS)
RELEASE_CPPFLAGS = $(COMMON_CPPFLAGS) -DNDEBUG RELEASE_CPPFLAGS = $(COMMON_CPPFLAGS) -DNDEBUG
RELEASE_GEN_OPT = -O3 @LIBBSCGEN_OPT@ RELEASE_GEN_OPT = -O3 @LIBBSCGEN_OPT@
RELEASE_RABIN_OPT = -O2
RELEASE_FPTR_FLAG = -fomit-frame-pointer RELEASE_FPTR_FLAG = -fomit-frame-pointer
NO_SLAB_CPPFLAGS = -DDEBUG_NO_SLAB NO_SLAB_CPPFLAGS = -DDEBUG_NO_SLAB
@ -187,7 +186,6 @@ VEC_FLAGS = @VEC_FLAGS@
LOOP_OPTFLAGS = @LOOP_OPTFLAGS@ LOOP_OPTFLAGS = @LOOP_OPTFLAGS@
CPPFLAGS = @CPPFLAGS@ @NO_SLAB_CPPFLAGS@ @DEBUG_STATS_CPPFLAGS@ CPPFLAGS = @CPPFLAGS@ @NO_SLAB_CPPFLAGS@ @DEBUG_STATS_CPPFLAGS@
GEN_OPT = @GEN_OPT@ GEN_OPT = @GEN_OPT@
RABIN_OPT = @RABIN_OPT@
PREFIX=@PREFIX@ PREFIX=@PREFIX@
SKEIN_FLAGS = $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) @FPTR_FLAG@ SKEIN_FLAGS = $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) @FPTR_FLAG@
@ -206,7 +204,7 @@ $(PPMDOBJS): $(PPMDSRCS) $(PPMDHDRS)
$(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@ $(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
$(RABINOBJS): $(RABINSRCS) $(RABINHDRS) $(RABINOBJS): $(RABINSRCS) $(RABINHDRS)
$(COMPILE) $(RABIN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@ $(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
$(BSDIFFOBJS): $(BSDIFFSRCS) $(BSDIFFHDRS) $(BSDIFFOBJS): $(BSDIFFSRCS) $(BSDIFFHDRS)
$(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@ $(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@

20
adaptive_compress.c
View file

@ -62,13 +62,13 @@ extern int ppmd_decompress(void *src, uint64_t srclen, void *dst,
extern int libbsc_decompress(void *src, uint64_t srclen, void *dst, extern int libbsc_decompress(void *src, uint64_t srclen, void *dst,
uint64_t *dstlen, int level, uchar_t chdr, void *data); uint64_t *dstlen, int level, uchar_t chdr, void *data);
extern int lzma_init(void **data, int *level, int nthreads, int64_t chunksize, extern int lzma_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int lzma_deinit(void **data); extern int lzma_deinit(void **data);
extern int ppmd_init(void **data, int *level, int nthreads, int64_t chunksize, extern int ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int ppmd_deinit(void **data); extern int ppmd_deinit(void **data);
extern int libbsc_init(void **data, int *level, int nthreads, int64_t chunksize, extern int libbsc_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int libbsc_deinit(void **data); extern int libbsc_deinit(void **data);
@ -96,17 +96,17 @@ adapt_stats(int show)
} }
void void
adapt_props(algo_props_t *data, int level, int64_t chunksize) adapt_props(algo_props_t *data, int level, uint64_t chunksize)
{ {
data->delta2_span = 200; data->delta2_span = 200;
} }
int int
adapt_init(void **data, int *level, int nthreads, int64_t chunksize, adapt_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op) int file_version, compress_op_t op)
{ {
struct adapt_data *adat = (struct adapt_data *)(*data); struct adapt_data *adat = (struct adapt_data *)(*data);
int rv; int rv = 0;
if (!adat) { if (!adat) {
adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data)); adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data));
@ -125,11 +125,11 @@ adapt_init(void **data, int *level, int nthreads, int64_t chunksize,
} }
int int
adapt2_init(void **data, int *level, int nthreads, int64_t chunksize, adapt2_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op) int file_version, compress_op_t op)
{ {
struct adapt_data *adat = (struct adapt_data *)(*data); struct adapt_data *adat = (struct adapt_data *)(*data);
int rv, lv; int rv = 0, lv;
if (!adat) { if (!adat) {
adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data)); adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data));
@ -160,7 +160,7 @@ int
adapt_deinit(void **data) adapt_deinit(void **data)
{ {
struct adapt_data *adat = (struct adapt_data *)(*data); struct adapt_data *adat = (struct adapt_data *)(*data);
int rv; int rv = 0;
if (adat) { if (adat) {
rv = ppmd_deinit(&(adat->ppmd_data)); rv = ppmd_deinit(&(adat->ppmd_data));
@ -179,7 +179,7 @@ adapt_compress(void *src, uint64_t srclen, void *dst,
struct adapt_data *adat = (struct adapt_data *)(data); struct adapt_data *adat = (struct adapt_data *)(data);
uchar_t *src1 = (uchar_t *)src; uchar_t *src1 = (uchar_t *)src;
uint64_t i, tot8b, tagcnt; uint64_t i, tot8b, tagcnt;
int rv, tag; int rv = 0, tag;
/* /*
* Count number of 8-bit binary bytes and XML tags in source. * Count number of 8-bit binary bytes and XML tags in source.

27
allocator.c
View file

@ -94,17 +94,33 @@ static struct slabentry slabheads[NUM_SLABS];
static struct bufentry **htable; static struct bufentry **htable;
static pthread_mutex_t *hbucket_locks; static pthread_mutex_t *hbucket_locks;
static pthread_mutex_t htable_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t htable_lock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t slab_table_lock = PTHREAD_MUTEX_INITIALIZER;
static int inited = 0, bypass = 0; static int inited = 0, bypass = 0;
static uint64_t total_allocs, oversize_allocs, hash_collisions, hash_entries; static uint64_t total_allocs, oversize_allocs, hash_collisions, hash_entries;
/*
* Hash function for 64Bit pointers/numbers that generates
* a 32Bit hash value.
* Taken from Thomas Wang's Integer hashing paper:
* http://www.cris.com/~Ttwang/tech/inthash.htm
*/
static uint32_t
hash6432shift(uint64_t key)
{
key = (~key) + (key << 18); // key = (key << 18) - key - 1;
key = key ^ (key >> 31);
key = key * 21; // key = (key + (key << 2)) + (key << 4);
key = key ^ (key >> 11);
key = key + (key << 6);
key = key ^ (key >> 22);
return (uint32_t) key;
}
void void
slab_init() slab_init()
{ {
int i; int i;
uint64_t slab_sz; uint64_t slab_sz;
int nprocs;
/* Check bypass env variable. */ /* Check bypass env variable. */
if (getenv("ALLOCATOR_BYPASS") != NULL) { if (getenv("ALLOCATOR_BYPASS") != NULL) {
@ -278,7 +294,7 @@ slab_calloc(void *p, uint64_t items, uint64_t size) {
static unsigned int static unsigned int
find_slot(unsigned int v) find_slot(unsigned int v)
{ {
unsigned int r, i; unsigned int r;
/* Round up to nearest power of 2 */ /* Round up to nearest power of 2 */
v = roundup_pow_two(v); v = roundup_pow_two(v);
@ -304,7 +320,7 @@ find_slot(unsigned int v)
return (r); return (r);
} }
static void * static struct slabentry *
try_dynamic_slab(uint64_t size) try_dynamic_slab(uint64_t size)
{ {
uint32_t sindx; uint32_t sindx;
@ -361,10 +377,7 @@ slab_cache_add(uint64_t size)
void * void *
slab_alloc(void *p, uint64_t size) slab_alloc(void *p, uint64_t size)
{ {
uint64_t slab_sz = SLAB_START_SZ;
int i;
uint64_t div; uint64_t div;
void *ptr;
struct slabentry *slab; struct slabentry *slab;
if (bypass) return (malloc(size)); if (bypass) return (malloc(size));

12
bsdiff/bscommon.h
View file

@ -64,6 +64,8 @@ BUFOPEN(bufio_t *bio, uchar_t *buf, bsize_t len)
bio->buf = buf; bio->pos = 0; bio->buflen = len; bio->buf = buf; bio->pos = 0; bio->buflen = len;
return (0); return (0);
} }
#ifndef __IN_BSPATCH__
static bsize_t static bsize_t
BUFWRITE(bufio_t *bio, uchar_t *buf, bsize_t len) BUFWRITE(bufio_t *bio, uchar_t *buf, bsize_t len)
{ {
@ -75,12 +77,13 @@ BUFWRITE(bufio_t *bio, uchar_t *buf, bsize_t len)
return (-1); return (-1);
} }
} }
#endif
#ifndef __IN_BSDIFF__
static bsize_t static bsize_t
BUFREAD(bufio_t *bio, uchar_t *buf, bsize_t len) BUFREAD(bufio_t *bio, uchar_t *buf, bsize_t len)
{ {
bsize_t actual; bsize_t actual;
int i;
actual = len; actual = len;
if (bio->pos + len > bio->buflen) { if (bio->pos + len > bio->buflen) {
@ -91,7 +94,9 @@ int i;
bio->pos += actual; bio->pos += actual;
return (actual); return (actual);
} }
#endif
#ifndef __IN_BSPATCH__
static bsize_t static bsize_t
BUFTELL(bufio_t *bio) BUFTELL(bufio_t *bio)
{ {
@ -123,9 +128,10 @@ BUFSEEK(bufio_t *bio, bsize_t pos, int typ)
} }
return (0); return (0);
} }
#endif
extern int zero_rle_encode(const void *const ibuf, const unsigned int ilen, extern int zero_rle_encode(const void *ibuf, const unsigned int ilen,
void *obuf, unsigned int *const olen); void *obuf, unsigned int *olen);
extern int zero_rle_decode(const void* ibuf, unsigned int ilen, extern int zero_rle_decode(const void* ibuf, unsigned int ilen,
void* obuf, unsigned int *olen); void* obuf, unsigned int *olen);

70
bsdiff/bsdiff.c
View file

@ -66,6 +66,7 @@ __FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05
#include <emmintrin.h> #include <emmintrin.h>
#endif #endif
#define __IN_BSDIFF__
#include "bscommon.h" #include "bscommon.h"
#define MIN(x,y) (((x)<(y)) ? (x) : (y)) #define MIN(x,y) (((x)<(y)) ? (x) : (y))
@ -131,7 +132,7 @@ static void split(bsize_t *I,bsize_t *V,bsize_t start,bsize_t len,bsize_t h)
if(start+len>kk) split(I,V,kk,start+len-kk,h); if(start+len>kk) split(I,V,kk,start+len-kk,h);
} }
static void qsufsort(bsize_t *I,bsize_t *V,u_char *old,bsize_t oldsize) static void qsufsort(bsize_t *I,bsize_t *V,u_char *oldbuf,bsize_t oldsize)
{ {
bsize_t buckets[256]; bsize_t buckets[256];
bsize_t i,h,len; bsize_t i,h,len;
@ -158,14 +159,14 @@ static void qsufsort(bsize_t *I,bsize_t *V,u_char *old,bsize_t oldsize)
#ifdef __USE_SSE_INTRIN__ #ifdef __USE_SSE_INTRIN__
} }
#endif #endif
for(i=0;i<oldsize;i++) buckets[old[i]]++; for(i=0;i<oldsize;i++) buckets[oldbuf[i]]++;
for(i=1;i<256;i++) buckets[i]+=buckets[i-1]; for(i=1;i<256;i++) buckets[i]+=buckets[i-1];
for(i=255;i>0;i--) buckets[i]=buckets[i-1]; for(i=255;i>0;i--) buckets[i]=buckets[i-1];
buckets[0]=0; buckets[0]=0;
for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i; for(i=0;i<oldsize;i++) I[++buckets[oldbuf[i]]]=i;
I[0]=oldsize; I[0]=oldsize;
for(i=0;i<oldsize;i++) V[i]=buckets[old[i]]; for(i=0;i<oldsize;i++) V[i]=buckets[oldbuf[i]];
V[oldsize]=0; V[oldsize]=0;
for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1; for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1;
I[0]=-1; I[0]=-1;
@ -190,24 +191,24 @@ static void qsufsort(bsize_t *I,bsize_t *V,u_char *old,bsize_t oldsize)
for(i=0;i<oldsize+1;i++) I[V[i]]=i; for(i=0;i<oldsize+1;i++) I[V[i]]=i;
} }
static bsize_t matchlen(u_char *old,bsize_t oldsize,u_char *new,bsize_t newsize) static bsize_t matchlen(u_char *oldbuf,bsize_t oldsize,u_char *newbuf,bsize_t newsize)
{ {
bsize_t i; bsize_t i;
for(i=0;(i<oldsize)&&(i<newsize);i++) for(i=0;(i<oldsize)&&(i<newsize);i++)
if(old[i]!=new[i]) break; if(oldbuf[i]!=newbuf[i]) break;
return i; return i;
} }
static bsize_t search(bsize_t *I,u_char *old,bsize_t oldsize, static bsize_t search(bsize_t *I,u_char *oldbuf,bsize_t oldsize,
u_char *new,bsize_t newsize,bsize_t st,bsize_t en,bsize_t *pos) u_char *newbuf,bsize_t newsize,bsize_t st,bsize_t en,bsize_t *pos)
{ {
bsize_t x,y; bsize_t x,y;
if(en-st<2) { if(en-st<2) {
x=matchlen(old+I[st],oldsize-I[st],new,newsize); x=matchlen(oldbuf+I[st],oldsize-I[st],newbuf,newsize);
y=matchlen(old+I[en],oldsize-I[en],new,newsize); y=matchlen(oldbuf+I[en],oldsize-I[en],newbuf,newsize);
if(x>y) { if(x>y) {
*pos=I[st]; *pos=I[st];
@ -219,19 +220,13 @@ static bsize_t search(bsize_t *I,u_char *old,bsize_t oldsize,
}; };
x=st+(en-st)/2; x=st+(en-st)/2;
if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) { if(memcmp(oldbuf+I[x],newbuf,MIN(oldsize-I[x],newsize))<0) {
return search(I,old,oldsize,new,newsize,x,en,pos); return search(I,oldbuf,oldsize,newbuf,newsize,x,en,pos);
} else { } else {
return search(I,old,oldsize,new,newsize,st,x,pos); return search(I,oldbuf,oldsize,newbuf,newsize,st,x,pos);
}; };
} }
static void
valout(bsize_t x, u_char *buf)
{
*((bsize_t *)buf) = htonll(x);
}
static void static void
valouti32(bsize_t x, u_char *buf) valouti32(bsize_t x, u_char *buf)
{ {
@ -241,7 +236,7 @@ valouti32(bsize_t x, u_char *buf)
} }
bsize_t bsize_t
bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize, bsdiff(u_char *oldbuf, bsize_t oldsize, u_char *newbuf, bsize_t newsize,
u_char *diff, u_char *scratch, bsize_t scratchsize) u_char *diff, u_char *scratch, bsize_t scratchsize)
{ {
bsize_t *I,*V; bsize_t *I,*V;
@ -259,15 +254,15 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
bufio_t pf; bufio_t pf;
sz = sizeof (bsize_t); sz = sizeof (bsize_t);
I = slab_alloc(NULL, (oldsize+1)*sz); I = (bsize_t *)slab_alloc(NULL, (oldsize+1)*sz);
V = slab_alloc(NULL, (oldsize+1)*sz); V = (bsize_t *)slab_alloc(NULL, (oldsize+1)*sz);
if(I == NULL || V == NULL) return (0); if(I == NULL || V == NULL) return (0);
qsufsort(I,V,old,oldsize); qsufsort(I,V,oldbuf,oldsize);
slab_free(NULL, V); slab_free(NULL, V);
if(((db=slab_alloc(NULL, newsize+1))==NULL) || if(((db=(u_char *)slab_alloc(NULL, newsize+1))==NULL) ||
((eb=slab_alloc(NULL, newsize+1))==NULL)) { ((eb=(u_char *)slab_alloc(NULL, newsize+1))==NULL)) {
fprintf(stderr, "bsdiff: Memory allocation error.\n"); fprintf(stderr, "bsdiff: Memory allocation error.\n");
slab_free(NULL, I); slab_free(NULL, I);
slab_free(NULL, V); slab_free(NULL, V);
@ -307,30 +302,31 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
/* Compute the differences, writing ctrl as we go */ /* Compute the differences, writing ctrl as we go */
scan=0;len=0; scan=0;len=0;
lastscan=0;lastpos=0;lastoffset=0; lastscan=0;lastpos=0;lastoffset=0;
pos=0;
while(scan<newsize) { while(scan<newsize) {
oldscore=0; oldscore=0;
for(scsc=scan+=len;scan<newsize;scan++) { for(scsc=scan+=len;scan<newsize;scan++) {
len=search(I,old,oldsize,new+scan,newsize-scan, len=search(I,oldbuf,oldsize,newbuf+scan,newsize-scan,
0,oldsize,&pos); 0,oldsize,&pos);
for(;scsc<scan+len;scsc++) for(;scsc<scan+len;scsc++)
if((scsc+lastoffset<oldsize) && if((scsc+lastoffset<oldsize) &&
(old[scsc+lastoffset] == new[scsc])) (oldbuf[scsc+lastoffset] == newbuf[scsc]))
oldscore++; oldscore++;
if(((len==oldscore) && (len!=0)) || if(((len==oldscore) && (len!=0)) ||
(len>oldscore+sz)) break; (len>oldscore+sz)) break;
if((scan+lastoffset<oldsize) && if((scan+lastoffset<oldsize) &&
(old[scan+lastoffset] == new[scan])) (oldbuf[scan+lastoffset] == newbuf[scan]))
oldscore--; oldscore--;
}; };
if((len!=oldscore) || (scan==newsize)) { if((len!=oldscore) || (scan==newsize)) {
s=0;Sf=0;lenf=0; s=0;Sf=0;lenf=0;
for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) { for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) {
if(old[lastpos+i]==new[lastscan+i]) s++; if(oldbuf[lastpos+i]==newbuf[lastscan+i]) s++;
i++; i++;
if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; }; if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; };
}; };
@ -339,7 +335,7 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
if(scan<newsize) { if(scan<newsize) {
s=0;Sb=0; s=0;Sb=0;
for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) { for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) {
if(old[pos-i]==new[scan-i]) s++; if(oldbuf[pos-i]==newbuf[scan-i]) s++;
if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; }; if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; };
}; };
}; };
@ -348,10 +344,10 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
overlap=(lastscan+lenf)-(scan-lenb); overlap=(lastscan+lenf)-(scan-lenb);
s=0;Ss=0;lens=0; s=0;Ss=0;lens=0;
for(i=0;i<overlap;i++) { for(i=0;i<overlap;i++) {
if(new[lastscan+lenf-overlap+i]== if(newbuf[lastscan+lenf-overlap+i]==
old[lastpos+lenf-overlap+i]) s++; oldbuf[lastpos+lenf-overlap+i]) s++;
if(new[scan-lenb+i]== if(newbuf[scan-lenb+i]==
old[pos-lenb+i]) s--; oldbuf[pos-lenb+i]) s--;
if(s>Ss) { Ss=s; lens=i+1; }; if(s>Ss) { Ss=s; lens=i+1; };
}; };
@ -360,9 +356,9 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
}; };
for(i=0;i<lenf;i++) for(i=0;i<lenf;i++)
db[dblen+i]=new[lastscan+i]-old[lastpos+i]; db[dblen+i]=newbuf[lastscan+i]-oldbuf[lastpos+i];
for(i=0;i<(scan-lenb)-(lastscan+lenf);i++) for(i=0;i<(scan-lenb)-(lastscan+lenf);i++)
eb[eblen+i]=new[lastscan+lenf+i]; eb[eblen+i]=newbuf[lastscan+lenf+i];
dblen+=lenf; dblen+=lenf;
eblen+=(scan-lenb)-(lastscan+lenf); eblen+=(scan-lenb)-(lastscan+lenf);
@ -391,7 +387,7 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
/* If our data can fit in the scratch area use it otherwise alloc. */ /* If our data can fit in the scratch area use it otherwise alloc. */
if (ulen > scratchsize) { if (ulen > scratchsize) {
cb = slab_alloc(NULL, ulen); cb = (u_char *)slab_alloc(NULL, ulen);
} else { } else {
cb = scratch; cb = scratch;
} }

24
bsdiff/bspatch.c
View file

@ -37,13 +37,9 @@ __FBSDID("$FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.c,v 1.1 2005/08/06 01:59:
#include <fcntl.h> #include <fcntl.h>
#include <allocator.h> #include <allocator.h>
#include <utils.h> #include <utils.h>
#include "bscommon.h"
static bsize_t #define __IN_BSPATCH__
valin(u_char *buf) #include "bscommon.h"
{
return ntohll(*((bsize_t *)buf));
}
static int32_t static int32_t
valini32(u_char *buf) valini32(u_char *buf)
@ -55,7 +51,7 @@ bsize_t
get_bsdiff_sz(u_char *pbuf) { get_bsdiff_sz(u_char *pbuf) {
bsize_t newsize; bsize_t newsize;
bsize_t lzctrllen, ctrllen, lzdatalen, datalen, lzextralen, extralen; bsize_t lzctrllen, ctrllen, lzdatalen, datalen, lzextralen, extralen;
int hdrsz, rv; int hdrsz;
hdrsz = 4*7; hdrsz = 4*7;
@ -70,7 +66,7 @@ get_bsdiff_sz(u_char *pbuf) {
} }
int int
bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsize) bspatch(u_char *pbuf, u_char *oldbuf, bsize_t oldsize, u_char *newbuf, bsize_t *_newsize)
{ {
bsize_t newsize; bsize_t newsize;
bsize_t lzctrllen, ctrllen, lzdatalen, datalen, lzextralen, extralen; bsize_t lzctrllen, ctrllen, lzdatalen, datalen, lzextralen, extralen;
@ -123,8 +119,8 @@ bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsi
*_newsize = newsize; *_newsize = newsize;
/* Allocate buffers. */ /* Allocate buffers. */
diffdata = slab_alloc(NULL, datalen); diffdata = (u_char *)slab_alloc(NULL, datalen);
extradata = slab_alloc(NULL, extralen); extradata = (u_char *)slab_alloc(NULL, extralen);
if (diffdata == NULL || extradata == NULL) { if (diffdata == NULL || extradata == NULL) {
fprintf(stderr, "bspatch: Out of memory.\n"); fprintf(stderr, "bspatch: Out of memory.\n");
if (diffdata) slab_free(NULL, diffdata); if (diffdata) slab_free(NULL, diffdata);
@ -135,7 +131,7 @@ bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsi
/* Decompress ctrldata, diffdata and extradata. */ /* Decompress ctrldata, diffdata and extradata. */
if (lzctrllen < ctrllen) { if (lzctrllen < ctrllen) {
/* Ctrl data will be RLE-d if RLE size is less. */ /* Ctrl data will be RLE-d if RLE size is less. */
ctrldata = slab_alloc(NULL, ctrllen); ctrldata = (u_char *)slab_alloc(NULL, ctrllen);
if (ctrldata == NULL) { if (ctrldata == NULL) {
fprintf(stderr, "bspatch: Out of memory.\n"); fprintf(stderr, "bspatch: Out of memory.\n");
slab_free(NULL, diffdata); slab_free(NULL, diffdata);
@ -195,7 +191,7 @@ bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsi
} }
/* Read diff string */ /* Read diff string */
lenread = BUFREAD(&dpf, new + newpos, ctrl[0]); lenread = BUFREAD(&dpf, newbuf + newpos, ctrl[0]);
if (lenread < ctrl[0]) { if (lenread < ctrl[0]) {
fprintf(stderr, "4: Corrupt diff data\n"); fprintf(stderr, "4: Corrupt diff data\n");
rv = 0; rv = 0;
@ -205,7 +201,7 @@ bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsi
/* Add old data to diff string */ /* Add old data to diff string */
for(i=0;i<ctrl[0];i++) for(i=0;i<ctrl[0];i++)
if((oldpos+i>=0) && (oldpos+i<oldsize)) if((oldpos+i>=0) && (oldpos+i<oldsize))
new[newpos+i]+=old[oldpos+i]; newbuf[newpos+i]+=oldbuf[oldpos+i];
/* Adjust pointers */ /* Adjust pointers */
newpos+=ctrl[0]; newpos+=ctrl[0];
@ -219,7 +215,7 @@ bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsi
} }
/* Read extra string */ /* Read extra string */
lenread = BUFREAD(&epf, new + newpos, ctrl[1]); lenread = BUFREAD(&epf, newbuf + newpos, ctrl[1]);
if (lenread < ctrl[1]) { if (lenread < ctrl[1]) {
fprintf(stderr, "6: Corrupt diff data\n"); fprintf(stderr, "6: Corrupt diff data\n");
rv = 0; rv = 0;

12
bsdiff/rle_encoder.c
View file

@ -30,13 +30,13 @@
#define COUNT_MAX (32767) #define COUNT_MAX (32767)
int int
zero_rle_encode(const void *const ibuf, const unsigned int ilen, zero_rle_encode(const void *ibuf, const unsigned int ilen,
void *obuf, unsigned int *const olen) void *obuf, unsigned int *olen)
{ {
unsigned int pos1, pos2; unsigned int pos1, pos2;
unsigned short count; unsigned short count;
const uchar_t *const ib = ibuf; const uchar_t *ib = (const uchar_t *)ibuf;
uchar_t *ob = obuf; uchar_t *ob = (uchar_t *)obuf;
pos2 = 0; pos2 = 0;
for (pos1=0; pos1<ilen && pos2<*olen;) { for (pos1=0; pos1<ilen && pos2<*olen;) {
@ -83,8 +83,8 @@ zero_rle_decode(const void* ibuf, unsigned int ilen,
{ {
unsigned int pos1, pos2, i; unsigned int pos1, pos2, i;
unsigned short count; unsigned short count;
const uchar_t *ib = ibuf; const uchar_t *ib = (const uchar_t *)ibuf;
uchar_t *ob = obuf; uchar_t *ob = (uchar_t *)obuf;
pos2 = 0; pos2 = 0;
pos1 = 0; pos1 = 0;

12
bzip2_compress.c
View file

@ -49,12 +49,12 @@ bzip2_stats(int show)
} }
void void
bzip2_props(algo_props_t *data, int level, int64_t chunksize) { bzip2_props(algo_props_t *data, int level, uint64_t chunksize) {
data->delta2_span = 200; data->delta2_span = 200;
} }
int int
bzip2_init(void **data, int *level, int nthreads, int64_t chunksize, bzip2_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op) int file_version, compress_op_t op)
{ {
if (*level > 9) *level = 9; if (*level > 9) *level = 9;
@ -100,8 +100,8 @@ bzip2_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
unsigned int slen, dlen; unsigned int slen, dlen;
uint64_t _srclen = srclen; uint64_t _srclen = srclen;
uint64_t _dstlen = *dstlen; uint64_t _dstlen = *dstlen;
uchar_t *dst1 = dst; char *dst1 = (char *)dst;
uchar_t *src1 = src; char *src1 = (char *)src;
bzs.bzalloc = slab_alloc_i; bzs.bzalloc = slab_alloc_i;
bzs.bzfree = slab_free; bzs.bzfree = slab_free;
@ -169,8 +169,8 @@ bzip2_decompress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
unsigned int slen, dlen; unsigned int slen, dlen;
uint64_t _srclen = srclen; uint64_t _srclen = srclen;
uint64_t _dstlen = *dstlen; uint64_t _dstlen = *dstlen;
uchar_t *dst1 = dst; char *dst1 = (char *)dst;
uchar_t *src1 = src; char *src1 = (char *)src;
bzs.bzalloc = slab_alloc_i; bzs.bzalloc = slab_alloc_i;
bzs.bzfree = slab_free; bzs.bzfree = slab_free;

2
config
View file

@ -383,7 +383,6 @@ vecflagsvar="VEC_FLAGS"
loopoptflagsvar="LOOP_OPTFLAGS" loopoptflagsvar="LOOP_OPTFLAGS"
cppflagsvar="CPPFLAGS" cppflagsvar="CPPFLAGS"
genoptvar="GEN_OPT" genoptvar="GEN_OPT"
rabinoptvar="RABIN_OPT"
noslabcppflagsvar="NO_SLAB_CPPFLAGS" noslabcppflagsvar="NO_SLAB_CPPFLAGS"
debugstatscppflagsvar="DEBUG_STATS_CPPFLAGS" debugstatscppflagsvar="DEBUG_STATS_CPPFLAGS"
prefixvar="PREFIX" prefixvar="PREFIX"
@ -427,7 +426,6 @@ s#@${vecflagsvar}@#\\\$\\(${typ}_${vecflagsvar}\\)#g
s#@${loopoptflagsvar}@#\\\$\\(${typ}_${loopoptflagsvar}\\)#g s#@${loopoptflagsvar}@#\\\$\\(${typ}_${loopoptflagsvar}\\)#g
s#@${cppflagsvar}@#\\\$\\(${typ}_${cppflagsvar}\\)#g s#@${cppflagsvar}@#\\\$\\(${typ}_${cppflagsvar}\\)#g
s#@${genoptvar}@#\\\$\\(${typ}_${genoptvar}\\)#g s#@${genoptvar}@#\\\$\\(${typ}_${genoptvar}\\)#g
s#@${rabinoptvar}@#\\\$\\(${typ}_${rabinoptvar}\\)#g
s#@${fptr_flag_var}@#\\\$\\(${typ}_${fptr_flag_var}\\)#g s#@${fptr_flag_var}@#\\\$\\(${typ}_${fptr_flag_var}\\)#g
s#@${noslabcppflagsvar}@#${noslabcppflagsval}#g s#@${noslabcppflagsvar}@#${noslabcppflagsval}#g
s#@${debugstatscppflagsvar}@#${debugstatscppflagsval}#g s#@${debugstatscppflagsvar}@#${debugstatscppflagsval}#g

12
crypto/aes/crypto_aes.c
View file

@ -50,6 +50,10 @@
* online backup system. * online backup system.
*/ */
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS 1
#endif
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
@ -106,7 +110,6 @@ int
aes_init(aes_ctx_t *ctx, uchar_t *salt, int saltlen, uchar_t *pwd, int pwd_len, aes_init(aes_ctx_t *ctx, uchar_t *salt, int saltlen, uchar_t *pwd, int pwd_len,
uint64_t nonce, int enc) uint64_t nonce, int enc)
{ {
int rv;
struct timespec tp; struct timespec tp;
uint64_t tv; uint64_t tv;
uchar_t num[25]; uchar_t num[25];
@ -141,8 +144,9 @@ aes_init(aes_ctx_t *ctx, uchar_t *salt, int saltlen, uchar_t *pwd, int pwd_len,
} else { } else {
tv = tp.tv_sec * 1000UL + tp.tv_nsec; tv = tp.tv_sec * 1000UL + tp.tv_nsec;
} }
sprintf(num, "%" PRIu64, tv); sprintf((char *)num, "%" PRIu64, tv);
PKCS5_PBKDF2_HMAC(num, strlen(num), salt, saltlen, PBE_ROUNDS, EVP_sha256(), 32, IV); PKCS5_PBKDF2_HMAC((const char *)num, strlen((char *)num), salt,
saltlen, PBE_ROUNDS, EVP_sha256(), 32, IV);
ctx->nonce = lzma_crc64(IV, 32, 0) & 0xffffffff00000000ULL; ctx->nonce = lzma_crc64(IV, 32, 0) & 0xffffffff00000000ULL;
// Nullify stack components // Nullify stack components
memset(num, 0, 25); memset(num, 0, 25);
@ -177,6 +181,7 @@ aes_encrypt(aes_ctx_t *ctx, uchar_t *plaintext, uchar_t *ciphertext, ssize_t len
crypto_aesctr_stream(strm, plaintext, ciphertext, len); crypto_aesctr_stream(strm, plaintext, ciphertext, len);
crypto_aesctr_free(strm); crypto_aesctr_free(strm);
memset(&key, 0, sizeof (key)); memset(&key, 0, sizeof (key));
return (0);
} }
int int
@ -200,6 +205,7 @@ aes_decrypt(aes_ctx_t *ctx, uchar_t *ciphertext, uchar_t *plaintext, ssize_t len
crypto_aesctr_stream(strm, ciphertext, plaintext, len); crypto_aesctr_stream(strm, ciphertext, plaintext, len);
crypto_aesctr_free(strm); crypto_aesctr_free(strm);
memset(&key, 0, sizeof (key)); memset(&key, 0, sizeof (key));
return (0);
} }
uint64_t uint64_t

95
crypto/crypto_utils.c
View file

@ -39,9 +39,9 @@
#include <sha256.h> #include <sha256.h>
#include <crypto_aes.h> #include <crypto_aes.h>
#include <KeccakNISTInterface.h> #include <KeccakNISTInterface.h>
#include <cpuid.h>
#include "crypto_utils.h" #include "crypto_utils.h"
#include "cpuid.h"
#define PROVIDER_OPENSSL 0 #define PROVIDER_OPENSSL 0
#define PROVIDER_X64_OPT 1 #define PROVIDER_X64_OPT 1
@ -53,7 +53,7 @@ static int geturandom_bytes(uchar_t rbytes[32]);
*/ */
typedef void (*ckinit_func_ptr)(void); typedef void (*ckinit_func_ptr)(void);
static struct { static struct {
char *name; const char *name;
cksum_t cksum_id; cksum_t cksum_id;
int bytes, mac_bytes; int bytes, mac_bytes;
ckinit_func_ptr init_func; ckinit_func_ptr init_func;
@ -67,7 +67,7 @@ static struct {
{"KECCAK512", CKSUM_KECCAK512, 64, 64, NULL} {"KECCAK512", CKSUM_KECCAK512, 64, 64, NULL}
}; };
static int cksum_provider = PROVIDER_OPENSSL, ossl_inited = 0; static int cksum_provider = PROVIDER_OPENSSL;
extern uint64_t lzma_crc64(const uint8_t *buf, uint64_t size, uint64_t crc); extern uint64_t lzma_crc64(const uint8_t *buf, uint64_t size, uint64_t crc);
extern uint64_t lzma_crc64_8bchk(const uint8_t *buf, uint64_t size, extern uint64_t lzma_crc64_8bchk(const uint8_t *buf, uint64_t size,
@ -153,7 +153,7 @@ init_sha256(void)
* return it's properties. * return it's properties.
*/ */
int int
get_checksum_props(char *name, int *cksum, int *cksum_bytes, int *mac_bytes) get_checksum_props(const char *name, int *cksum, int *cksum_bytes, int *mac_bytes)
{ {
int i; int i;
@ -212,12 +212,12 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
mctx->mac_cksum = cksum; mctx->mac_cksum = cksum;
if (cksum == CKSUM_SKEIN256) { if (cksum == CKSUM_SKEIN256) {
Skein_512_Ctxt_t *ctx = malloc(sizeof (Skein_512_Ctxt_t)); Skein_512_Ctxt_t *ctx = (Skein_512_Ctxt_t *)malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) return (-1); if (!ctx) return (-1);
Skein_512_InitExt(ctx, 256, SKEIN_CFG_TREE_INFO_SEQUENTIAL, Skein_512_InitExt(ctx, 256, SKEIN_CFG_TREE_INFO_SEQUENTIAL,
actx->pkey, KEYLEN); actx->pkey, KEYLEN);
mctx->mac_ctx = ctx; mctx->mac_ctx = ctx;
ctx = malloc(sizeof (Skein_512_Ctxt_t)); ctx = (Skein_512_Ctxt_t *)malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) { if (!ctx) {
free(mctx->mac_ctx); free(mctx->mac_ctx);
return (-1); return (-1);
@ -226,12 +226,12 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
mctx->mac_ctx_reinit = ctx; mctx->mac_ctx_reinit = ctx;
} else if (cksum == CKSUM_SKEIN512) { } else if (cksum == CKSUM_SKEIN512) {
Skein_512_Ctxt_t *ctx = malloc(sizeof (Skein_512_Ctxt_t)); Skein_512_Ctxt_t *ctx = (Skein_512_Ctxt_t *)malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) return (-1); if (!ctx) return (-1);
Skein_512_InitExt(ctx, 512, SKEIN_CFG_TREE_INFO_SEQUENTIAL, Skein_512_InitExt(ctx, 512, SKEIN_CFG_TREE_INFO_SEQUENTIAL,
actx->pkey, KEYLEN); actx->pkey, KEYLEN);
mctx->mac_ctx = ctx; mctx->mac_ctx = ctx;
ctx = malloc(sizeof (Skein_512_Ctxt_t)); ctx = (Skein_512_Ctxt_t *)malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) { if (!ctx) {
free(mctx->mac_ctx); free(mctx->mac_ctx);
return (-1); return (-1);
@ -241,30 +241,30 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) { } else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) { if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_CTX *ctx = malloc(sizeof (HMAC_CTX)); HMAC_CTX *ctx = (HMAC_CTX *)malloc(sizeof (HMAC_CTX));
if (!ctx) return (-1); if (!ctx) return (-1);
HMAC_CTX_init(ctx); HMAC_CTX_init(ctx);
HMAC_Init_ex(ctx, actx->pkey, KEYLEN, EVP_sha256(), NULL); HMAC_Init_ex(ctx, actx->pkey, KEYLEN, EVP_sha256(), NULL);
mctx->mac_ctx = ctx; mctx->mac_ctx = ctx;
ctx = malloc(sizeof (HMAC_CTX)); ctx = (HMAC_CTX *)malloc(sizeof (HMAC_CTX));
if (!ctx) { if (!ctx) {
free(mctx->mac_ctx); free(mctx->mac_ctx);
return (-1); return (-1);
} }
if (!HMAC_CTX_copy(ctx, mctx->mac_ctx)) { if (!HMAC_CTX_copy(ctx, (HMAC_CTX *)(mctx->mac_ctx))) {
free(ctx); free(ctx);
free(mctx->mac_ctx); free(mctx->mac_ctx);
return (-1); return (-1);
} }
mctx->mac_ctx_reinit = ctx; mctx->mac_ctx_reinit = ctx;
} else { } else {
HMAC_SHA256_Context *ctx = malloc(sizeof (HMAC_SHA256_Context)); HMAC_SHA256_Context *ctx = (HMAC_SHA256_Context *)malloc(sizeof (HMAC_SHA256_Context));
if (!ctx) return (-1); if (!ctx) return (-1);
opt_HMAC_SHA256_Init(ctx, actx->pkey, KEYLEN); opt_HMAC_SHA256_Init(ctx, actx->pkey, KEYLEN);
mctx->mac_ctx = ctx; mctx->mac_ctx = ctx;
ctx = malloc(sizeof (HMAC_SHA256_Context)); ctx = (HMAC_SHA256_Context *)malloc(sizeof (HMAC_SHA256_Context));
if (!ctx) { if (!ctx) {
free(mctx->mac_ctx); free(mctx->mac_ctx);
return (-1); return (-1);
@ -273,18 +273,18 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
mctx->mac_ctx_reinit = ctx; mctx->mac_ctx_reinit = ctx;
} }
} else if (cksum == CKSUM_SHA512) { } else if (cksum == CKSUM_SHA512) {
HMAC_CTX *ctx = malloc(sizeof (HMAC_CTX)); HMAC_CTX *ctx = (HMAC_CTX *)malloc(sizeof (HMAC_CTX));
if (!ctx) return (-1); if (!ctx) return (-1);
HMAC_CTX_init(ctx); HMAC_CTX_init(ctx);
HMAC_Init_ex(ctx, actx->pkey, KEYLEN, EVP_sha512(), NULL); HMAC_Init_ex(ctx, actx->pkey, KEYLEN, EVP_sha512(), NULL);
mctx->mac_ctx = ctx; mctx->mac_ctx = ctx;
ctx = malloc(sizeof (HMAC_CTX)); ctx = (HMAC_CTX *)malloc(sizeof (HMAC_CTX));
if (!ctx) { if (!ctx) {
free(mctx->mac_ctx); free(mctx->mac_ctx);
return (-1); return (-1);
} }
if (!HMAC_CTX_copy(ctx, mctx->mac_ctx)) { if (!HMAC_CTX_copy(ctx, (HMAC_CTX *)(mctx->mac_ctx))) {
free(ctx); free(ctx);
free(mctx->mac_ctx); free(mctx->mac_ctx);
return (-1); return (-1);
@ -292,7 +292,7 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
mctx->mac_ctx_reinit = ctx; mctx->mac_ctx_reinit = ctx;
} else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) { } else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) {
hashState *ctx = malloc(sizeof (hashState)); hashState *ctx = (hashState *)malloc(sizeof (hashState));
if (!ctx) return (-1); if (!ctx) return (-1);
if (cksum == CKSUM_KECCAK256) { if (cksum == CKSUM_KECCAK256) {
@ -306,7 +306,7 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
return (-1); return (-1);
mctx->mac_ctx = ctx; mctx->mac_ctx = ctx;
ctx = malloc(sizeof (hashState)); ctx = (hashState *)malloc(sizeof (hashState));
if (!ctx) { if (!ctx) {
free(mctx->mac_ctx); free(mctx->mac_ctx);
return (-1); return (-1);
@ -329,12 +329,13 @@ hmac_reinit(mac_ctx_t *mctx)
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) { } else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) { if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_CTX_copy(mctx->mac_ctx, mctx->mac_ctx_reinit); HMAC_CTX_copy((HMAC_CTX *)(mctx->mac_ctx),
(HMAC_CTX *)(mctx->mac_ctx_reinit));
} else { } else {
memcpy(mctx->mac_ctx, mctx->mac_ctx_reinit, sizeof (HMAC_SHA256_Context)); memcpy(mctx->mac_ctx, mctx->mac_ctx_reinit, sizeof (HMAC_SHA256_Context));
} }
} else if (cksum == CKSUM_SHA512) { } else if (cksum == CKSUM_SHA512) {
HMAC_CTX_copy(mctx->mac_ctx, mctx->mac_ctx_reinit); HMAC_CTX_copy((HMAC_CTX *)(mctx->mac_ctx), (HMAC_CTX *)(mctx->mac_ctx_reinit));
} else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) { } else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) {
memcpy(mctx->mac_ctx, mctx->mac_ctx_reinit, sizeof (hashState)); memcpy(mctx->mac_ctx, mctx->mac_ctx_reinit, sizeof (hashState));
@ -350,17 +351,17 @@ hmac_update(mac_ctx_t *mctx, uchar_t *data, uint64_t len)
int cksum = mctx->mac_cksum; int cksum = mctx->mac_cksum;
if (cksum == CKSUM_SKEIN256 || cksum == CKSUM_SKEIN512) { if (cksum == CKSUM_SKEIN256 || cksum == CKSUM_SKEIN512) {
Skein_512_Update(mctx->mac_ctx, data, len); Skein_512_Update((Skein_512_Ctxt_t *)(mctx->mac_ctx), data, len);
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) { } else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) { if (cksum_provider == PROVIDER_OPENSSL) {
if (HMAC_Update(mctx->mac_ctx, data, len) == 0) if (HMAC_Update((HMAC_CTX *)(mctx->mac_ctx), data, len) == 0)
return (-1); return (-1);
} else { } else {
opt_HMAC_SHA256_Update(mctx->mac_ctx, data, len); opt_HMAC_SHA256_Update((HMAC_SHA256_Context *)(mctx->mac_ctx), data, len);
} }
} else if (cksum == CKSUM_SHA512) { } else if (cksum == CKSUM_SHA512) {
if (HMAC_Update(mctx->mac_ctx, data, len) == 0) if (HMAC_Update((HMAC_CTX *)(mctx->mac_ctx), data, len) == 0)
return (-1); return (-1);
} else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) { } else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) {
@ -369,11 +370,11 @@ hmac_update(mac_ctx_t *mctx, uchar_t *data, uint64_t len)
uint64_t blen; uint64_t blen;
blen = KECCAK_MAX_SEG; blen = KECCAK_MAX_SEG;
if (Keccak_Update(mctx->mac_ctx, data, blen << 3) != 0) if (Keccak_Update((hashState *)(mctx->mac_ctx), data, blen << 3) != 0)
return (-1); return (-1);
len -= KECCAK_MAX_SEG; len -= KECCAK_MAX_SEG;
} }
if (Keccak_Update(mctx->mac_ctx, data, len << 3) != 0) if (Keccak_Update((hashState *)(mctx->mac_ctx), data, len << 3) != 0)
return (-1); return (-1);
} else { } else {
return (-1); return (-1);
@ -387,25 +388,25 @@ hmac_final(mac_ctx_t *mctx, uchar_t *hash, unsigned int *len)
int cksum = mctx->mac_cksum; int cksum = mctx->mac_cksum;
if (cksum == CKSUM_SKEIN256) { if (cksum == CKSUM_SKEIN256) {
Skein_512_Final(mctx->mac_ctx, hash); Skein_512_Final((Skein_512_Ctxt_t *)(mctx->mac_ctx), hash);
*len = 32; *len = 32;
} else if (cksum == CKSUM_SKEIN512) { } else if (cksum == CKSUM_SKEIN512) {
Skein_512_Final(mctx->mac_ctx, hash); Skein_512_Final((Skein_512_Ctxt_t *)(mctx->mac_ctx), hash);
*len = 64; *len = 64;
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) { } else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) { if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_Final(mctx->mac_ctx, hash, len); HMAC_Final((HMAC_CTX *)(mctx->mac_ctx), hash, len);
} else { } else {
opt_HMAC_SHA256_Final(mctx->mac_ctx, hash); opt_HMAC_SHA256_Final((HMAC_SHA256_Context *)(mctx->mac_ctx), hash);
*len = 32; *len = 32;
} }
} else if (cksum == CKSUM_SHA512) { } else if (cksum == CKSUM_SHA512) {
HMAC_Final(mctx->mac_ctx, hash, len); HMAC_Final((HMAC_CTX *)(mctx->mac_ctx), hash, len);
} else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) { } else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) {
if (Keccak_Final(mctx->mac_ctx, hash) != 0) if (Keccak_Final((hashState *)(mctx->mac_ctx), hash) != 0)
return (-1); return (-1);
if (cksum == CKSUM_KECCAK256) if (cksum == CKSUM_KECCAK256)
*len = 32; *len = 32;
@ -428,15 +429,15 @@ hmac_cleanup(mac_ctx_t *mctx)
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) { } else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) { if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_CTX_cleanup(mctx->mac_ctx); HMAC_CTX_cleanup((HMAC_CTX *)(mctx->mac_ctx));
HMAC_CTX_cleanup(mctx->mac_ctx_reinit); HMAC_CTX_cleanup((HMAC_CTX *)(mctx->mac_ctx_reinit));
} else { } else {
memset(mctx->mac_ctx, 0, sizeof (HMAC_SHA256_Context)); memset(mctx->mac_ctx, 0, sizeof (HMAC_SHA256_Context));
memset(mctx->mac_ctx_reinit, 0, sizeof (HMAC_SHA256_Context)); memset(mctx->mac_ctx_reinit, 0, sizeof (HMAC_SHA256_Context));
} }
} else if (cksum == CKSUM_SHA512) { } else if (cksum == CKSUM_SHA512) {
HMAC_CTX_cleanup(mctx->mac_ctx); HMAC_CTX_cleanup((HMAC_CTX *)(mctx->mac_ctx));
HMAC_CTX_cleanup(mctx->mac_ctx_reinit); HMAC_CTX_cleanup((HMAC_CTX *)(mctx->mac_ctx_reinit));
} else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) { } else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) {
memset(mctx->mac_ctx, 0, sizeof (hashState)); memset(mctx->mac_ctx, 0, sizeof (hashState));
@ -455,13 +456,13 @@ init_crypto(crypto_ctx_t *cctx, uchar_t *pwd, int pwd_len, int crypto_alg,
uchar_t *salt, int saltlen, uint64_t nonce, int enc_dec) uchar_t *salt, int saltlen, uint64_t nonce, int enc_dec)
{ {
if (crypto_alg == CRYPTO_ALG_AES) { if (crypto_alg == CRYPTO_ALG_AES) {
aes_ctx_t *actx = malloc(sizeof (aes_ctx_t)); aes_ctx_t *actx = (aes_ctx_t *)malloc(sizeof (aes_ctx_t));
if (enc_dec) { if (enc_dec) {
/* /*
* Encryption init. * Encryption init.
*/ */
cctx->salt = malloc(32); cctx->salt = (uchar_t *)malloc(32);
salt = cctx->salt; salt = cctx->salt;
cctx->saltlen = 32; cctx->saltlen = 32;
if (RAND_status() != 1 || RAND_bytes(salt, 32) != 1) { if (RAND_status() != 1 || RAND_bytes(salt, 32) != 1) {
@ -509,7 +510,7 @@ init_crypto(crypto_ctx_t *cctx, uchar_t *pwd, int pwd_len, int crypto_alg,
MAX_SALTLEN); MAX_SALTLEN);
return (-1); return (-1);
} }
cctx->salt = malloc(saltlen); cctx->salt = (uchar_t *)malloc(saltlen);
memcpy(cctx->salt, salt, saltlen); memcpy(cctx->salt, salt, saltlen);
if (aes_init(actx, cctx->salt, saltlen, pwd, pwd_len, nonce, if (aes_init(actx, cctx->salt, saltlen, pwd, pwd_len, nonce,
@ -533,9 +534,9 @@ crypto_buf(crypto_ctx_t *cctx, uchar_t *from, uchar_t *to, int64_t bytes, uint64
{ {
if (cctx->crypto_alg == CRYPTO_ALG_AES) { if (cctx->crypto_alg == CRYPTO_ALG_AES) {
if (cctx->enc_dec == ENCRYPT_FLAG) { if (cctx->enc_dec == ENCRYPT_FLAG) {
return (aes_encrypt(cctx->crypto_ctx, from, to, bytes, id)); return (aes_encrypt((aes_ctx_t *)(cctx->crypto_ctx), from, to, bytes, id));
} else { } else {
return (aes_decrypt(cctx->crypto_ctx, from, to, bytes, id)); return (aes_decrypt((aes_ctx_t *)(cctx->crypto_ctx), from, to, bytes, id));
} }
} else { } else {
fprintf(stderr, "Unrecognized algorithm code: %d\n", cctx->crypto_alg); fprintf(stderr, "Unrecognized algorithm code: %d\n", cctx->crypto_alg);
@ -547,19 +548,19 @@ crypto_buf(crypto_ctx_t *cctx, uchar_t *from, uchar_t *to, int64_t bytes, uint64
uint64_t uint64_t
crypto_nonce(crypto_ctx_t *cctx) crypto_nonce(crypto_ctx_t *cctx)
{ {
return (aes_nonce(cctx->crypto_ctx)); return (aes_nonce((aes_ctx_t *)(cctx->crypto_ctx)));
} }
void void
crypto_clean_pkey(crypto_ctx_t *cctx) crypto_clean_pkey(crypto_ctx_t *cctx)
{ {
aes_clean_pkey(cctx->crypto_ctx); aes_clean_pkey((aes_ctx_t *)(cctx->crypto_ctx));
} }
void void
cleanup_crypto(crypto_ctx_t *cctx) cleanup_crypto(crypto_ctx_t *cctx)
{ {
aes_cleanup(cctx->crypto_ctx); aes_cleanup((aes_ctx_t *)(cctx->crypto_ctx));
memset(cctx->salt, 0, 32); memset(cctx->salt, 0, 32);
free(cctx->salt); free(cctx->salt);
free(cctx); free(cctx);
@ -607,12 +608,12 @@ err0:
} }
int int
get_pw_string(char pw[MAX_PW_LEN], char *prompt, int twice) get_pw_string(uchar_t pw[MAX_PW_LEN], const char *prompt, int twice)
{ {
int fd, len; int fd, len;
FILE *input, *strm; FILE *input, *strm;
struct termios oldt, newt; struct termios oldt, newt;
uchar_t pw1[MAX_PW_LEN], pw2[MAX_PW_LEN], *s; char pw1[MAX_PW_LEN], pw2[MAX_PW_LEN], *s;
// Try TTY first // Try TTY first
fd = open("/dev/tty", O_RDWR | O_NOCTTY); fd = open("/dev/tty", O_RDWR | O_NOCTTY);
@ -667,7 +668,7 @@ get_pw_string(char pw[MAX_PW_LEN], char *prompt, int twice)
len = strlen(pw1); len = strlen(pw1);
pw1[len-1] = '\0'; pw1[len-1] = '\0';
strcpy(pw, pw1); strcpy((char *)pw, (const char *)pw1);
memset(pw1, 0, MAX_PW_LEN); memset(pw1, 0, MAX_PW_LEN);
memset(pw2, 0, MAX_PW_LEN); memset(pw2, 0, MAX_PW_LEN);
return (len); return (len);

5
crypto/crypto_utils.h
View file

@ -75,7 +75,7 @@ typedef struct {
* Generic message digest functions. * Generic message digest functions.
*/ */
int compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, int64_t bytes); int compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, int64_t bytes);
int get_checksum_props(char *name, int *cksum, int *cksum_bytes, int *mac_bytes); int get_checksum_props(const char *name, int *cksum, int *cksum_bytes, int *mac_bytes);
void serialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes); void serialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes);
void deserialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes); void deserialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes);
@ -88,7 +88,7 @@ int crypto_buf(crypto_ctx_t *cctx, uchar_t *from, uchar_t *to, int64_t bytes, ui
uint64_t crypto_nonce(crypto_ctx_t *cctx); uint64_t crypto_nonce(crypto_ctx_t *cctx);
void crypto_clean_pkey(crypto_ctx_t *cctx); void crypto_clean_pkey(crypto_ctx_t *cctx);
void cleanup_crypto(crypto_ctx_t *cctx); void cleanup_crypto(crypto_ctx_t *cctx);
int get_pw_string(char pw[MAX_PW_LEN], char *prompt, int twice); int get_pw_string(uchar_t pw[MAX_PW_LEN], const char *prompt, int twice);
/* /*
* HMAC functions. * HMAC functions.
@ -96,6 +96,7 @@ int get_pw_string(char pw[MAX_PW_LEN], char *prompt, int twice);
int hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx); int hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx);
int hmac_reinit(mac_ctx_t *mctx); int hmac_reinit(mac_ctx_t *mctx);
int hmac_update(mac_ctx_t *mctx, uchar_t *data, uint64_t len); int hmac_update(mac_ctx_t *mctx, uchar_t *data, uint64_t len);
int hmac_final(mac_ctx_t *mctx, uchar_t *hash, unsigned int *len);
int hmac_cleanup(mac_ctx_t *mctx); int hmac_cleanup(mac_ctx_t *mctx);
#ifdef __cplusplus #ifdef __cplusplus

8
crypto/keccak/KeccakNISTInterface.c
View file

@ -43,14 +43,14 @@ HashReturn Keccak_Init(hashState *state, int hashbitlen)
HashReturn Keccak_Update(hashState *state, const BitSequence *data, DataLength databitlen) HashReturn Keccak_Update(hashState *state, const BitSequence *data, DataLength databitlen)
{ {
if ((databitlen % 8) == 0) if ((databitlen % 8) == 0)
return Absorb((spongeState*)state, data, databitlen); return (HashReturn)Absorb((spongeState*)state, data, databitlen);
else { else {
HashReturn ret = Absorb((spongeState*)state, data, databitlen - (databitlen % 8)); HashReturn ret = (HashReturn)Absorb((spongeState*)state, data, databitlen - (databitlen % 8));
if (ret == SUCCESS) { if (ret == SUCCESS) {
unsigned char lastByte; unsigned char lastByte;
// Align the last partial byte to the least significant bits // Align the last partial byte to the least significant bits
lastByte = data[databitlen/8] >> (8 - (databitlen % 8)); lastByte = data[databitlen/8] >> (8 - (databitlen % 8));
return Absorb((spongeState*)state, &lastByte, databitlen % 8); return (HashReturn)Absorb((spongeState*)state, &lastByte, databitlen % 8);
} }
else else
return ret; return ret;
@ -59,7 +59,7 @@ HashReturn Keccak_Update(hashState *state, const BitSequence *data, DataLength d
HashReturn Keccak_Final(hashState *state, BitSequence *hashval) HashReturn Keccak_Final(hashState *state, BitSequence *hashval)
{ {
return Squeeze(state, hashval, state->fixedOutputLength); return (HashReturn)Squeeze(state, hashval, state->fixedOutputLength);
} }
HashReturn Keccak_Hash(int hashbitlen, const BitSequence *data, DataLength databitlen, BitSequence *hashval) HashReturn Keccak_Hash(int hashbitlen, const BitSequence *data, DataLength databitlen, BitSequence *hashval)

50
crypto/keccak/genKAT.c
View file

@ -36,8 +36,8 @@ STATUS_CODES genShortMsgSponge(unsigned int rate, unsigned int capacity, int
STATUS_CODES genDuplexKAT(unsigned int rate, unsigned int capacity, const char *fileName); STATUS_CODES genDuplexKAT(unsigned int rate, unsigned int capacity, const char *fileName);
#endif #endif
int FindMarker(FILE *infile, const char *marker); int FindMarker(FILE *infile, const char *marker);
int ReadHex(FILE *infile, BitSequence *A, int Length, char *str); int ReadHex(FILE *infile, BitSequence *A, int Length, const char *str);
void fprintBstr(FILE *fp, char *S, BitSequence *A, int L); void fprintBstr(FILE *fp, const char *S, BitSequence *A, int L);
STATUS_CODES STATUS_CODES
@ -139,7 +139,7 @@ genShortMsg(int hashbitlen)
printf("ERROR: unable to read 'Msg' from <ShortMsgKAT.txt>\n"); printf("ERROR: unable to read 'Msg' from <ShortMsgKAT.txt>\n");
return KAT_DATA_ERROR; return KAT_DATA_ERROR;
} }
Hash(hashbitlen, Msg, msglen, MD); Keccak_Hash(hashbitlen, Msg, msglen, MD);
fprintf(fp_out, "\nLen = %d\n", msglen); fprintf(fp_out, "\nLen = %d\n", msglen);
fprintBstr(fp_out, "Msg = ", Msg, msgbytelen); fprintBstr(fp_out, "Msg = ", Msg, msgbytelen);
fprintBstr(fp_out, "MD = ", MD, hashbitlen/8); fprintBstr(fp_out, "MD = ", MD, hashbitlen/8);
@ -282,15 +282,15 @@ genLongMsg(int hashbitlen)
} }
#ifdef AllowExtendedFunctions #ifdef AllowExtendedFunctions
if (hashbitlen > 0) if (hashbitlen > 0)
Hash(hashbitlen, Msg, msglen, MD); Keccak_Hash(hashbitlen, Msg, msglen, MD);
else { else {
Init(&state, hashbitlen); Keccak_Init(&state, hashbitlen);
Update(&state, Msg, msglen); Keccak_Update(&state, Msg, msglen);
Final(&state, 0); Keccak_Final(&state, 0);
Squeeze(&state, Squeezed, SqueezingOutputLength); Squeeze(&state, Squeezed, SqueezingOutputLength);
} }
#else #else
Hash(hashbitlen, Msg, msglen, MD); Keccak_Hash(hashbitlen, Msg, msglen, MD);
#endif #endif
fprintf(fp_out, "Len = %d\n", msglen); fprintf(fp_out, "Len = %d\n", msglen);
fprintBstr(fp_out, "Msg = ", Msg, msgbytelen); fprintBstr(fp_out, "Msg = ", Msg, msgbytelen);
@ -368,17 +368,17 @@ genExtremelyLongMsg(int hashbitlen)
// memcpy(Text, "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmno", 64); // memcpy(Text, "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmno", 64);
if ( (retval = Init(&state, hashbitlen)) != KAT_SUCCESS ) { if ( (retval = Keccak_Init(&state, hashbitlen)) != KAT_SUCCESS ) {
printf("Init returned <%d> in genExtremelyLongMsg\n", retval); printf("Keccak_Init returned <%d> in genExtremelyLongMsg\n", retval);
return KAT_HASH_ERROR; return KAT_HASH_ERROR;
} }
for ( i=0; i<repeat; i++ ) for ( i=0; i<repeat; i++ )
if ( (retval = Update(&state, Text, 512)) != KAT_SUCCESS ) { if ( (retval = Keccak_Update(&state, Text, 512)) != KAT_SUCCESS ) {
printf("Update returned <%d> in genExtremelyLongMsg\n", retval); printf("Keccak_Update returned <%d> in genExtremelyLongMsg\n", retval);
return KAT_HASH_ERROR; return KAT_HASH_ERROR;
} }
if ( (retval = Final(&state, MD)) != KAT_SUCCESS ) { if ( (retval = Keccak_Final(&state, MD)) != KAT_SUCCESS ) {
printf("Final returned <%d> in genExtremelyLongMsg\n", retval); printf("Keccak_Final returned <%d> in genExtremelyLongMsg\n", retval);
return KAT_HASH_ERROR; return KAT_HASH_ERROR;
} }
#ifdef AllowExtendedFunctions #ifdef AllowExtendedFunctions
@ -449,7 +449,7 @@ genMonteCarlo(int hashbitlen)
fprintBstr(fp_out, "Seed = ", Seed, 128); fprintBstr(fp_out, "Seed = ", Seed, 128);
for ( j=0; j<100; j++ ) { for ( j=0; j<100; j++ ) {
for ( i=0; i<1000; i++ ) { for ( i=0; i<1000; i++ ) {
Hash(hashbitlen, Msg, 1024, MD); Keccak_Hash(hashbitlen, Msg, 1024, MD);
memcpy(Temp, Msg, 128-bytelen); memcpy(Temp, Msg, 128-bytelen);
memcpy(Msg, MD, bytelen); memcpy(Msg, MD, bytelen);
memcpy(Msg+bytelen, Temp, 128-bytelen); memcpy(Msg+bytelen, Temp, 128-bytelen);
@ -511,22 +511,22 @@ genMonteCarloSqueezing(int hashbitlen)
fprintBstr(fp_out, "Seed = ", Seed, 128); fprintBstr(fp_out, "Seed = ", Seed, 128);
if ( (retval = Init(&state, hashbitlen)) != KAT_SUCCESS ) { if ( (retval = Keccak_Init(&state, hashbitlen)) != KAT_SUCCESS ) {
printf("Init returned <%d> in genMonteCarloSqueezing\n", retval); printf("Keccak_Init returned <%d> in genMonteCarloSqueezing\n", retval);
return KAT_HASH_ERROR; return KAT_HASH_ERROR;
} }
if ( (retval = Update(&state, Seed, 128*8)) != KAT_SUCCESS ) { if ( (retval = Keccak_Update(&state, Seed, 128*8)) != KAT_SUCCESS ) {
printf("Update returned <%d> in genMonteCarloSqueezing\n", retval); printf("Keccak_Update returned <%d> in genMonteCarloSqueezing\n", retval);
return KAT_HASH_ERROR; return KAT_HASH_ERROR;
} }
if ( (retval = Final(&state, 0)) != KAT_SUCCESS ) { if ( (retval = Keccak_Final(&state, 0)) != KAT_SUCCESS ) {
printf("Final returned <%d> in genMonteCarloSqueezing\n", retval); printf("Keccak_Final returned <%d> in genMonteCarloSqueezing\n", retval);
return KAT_HASH_ERROR; return KAT_HASH_ERROR;
} }
bytelen = 64; bytelen = 64;
for ( j=0; j<100; j++ ) { for ( j=0; j<100; j++ ) {
for ( i=0; i<1000; i++ ) { for ( i=0; i<1000; i++ ) {
if ( (retval = Squeeze(&state, MD, bytelen*8)) != KAT_SUCCESS ) { if ( (retval = (HashReturn)Squeeze(&state, MD, bytelen*8)) != KAT_SUCCESS ) {
printf("Squeeze returned <%d> in genMonteCarloSqueezing\n", retval); printf("Squeeze returned <%d> in genMonteCarloSqueezing\n", retval);
return KAT_HASH_ERROR; return KAT_HASH_ERROR;
} }
@ -634,7 +634,7 @@ FindMarker(FILE *infile, const char *marker)
// ALLOW TO READ HEXADECIMAL ENTRY (KEYS, DATA, TEXT, etc.) // ALLOW TO READ HEXADECIMAL ENTRY (KEYS, DATA, TEXT, etc.)
// //
int int
ReadHex(FILE *infile, BitSequence *A, int Length, char *str) ReadHex(FILE *infile, BitSequence *A, int Length, const char *str)
{ {
int i, ch, started; int i, ch, started;
BitSequence ich; BitSequence ich;
@ -664,6 +664,8 @@ ReadHex(FILE *infile, BitSequence *A, int Length, char *str)
ich = ch - 'A' + 10; ich = ch - 'A' + 10;
else if ( (ch >= 'a') && (ch <= 'f') ) else if ( (ch >= 'a') && (ch <= 'f') )
ich = ch - 'a' + 10; ich = ch - 'a' + 10;
else
return 1;
for ( i=0; i<Length-1; i++ ) for ( i=0; i<Length-1; i++ )
A[i] = (A[i] << 4) | (A[i+1] >> 4); A[i] = (A[i] << 4) | (A[i+1] >> 4);
@ -676,7 +678,7 @@ ReadHex(FILE *infile, BitSequence *A, int Length, char *str)
} }
void void
fprintBstr(FILE *fp, char *S, BitSequence *A, int L) fprintBstr(FILE *fp, const char *S, BitSequence *A, int L)
{ {
int i; int i;

2
crypto/scrypt/crypto_aesctr.c
View file

@ -55,7 +55,7 @@ crypto_aesctr_init(AES_KEY * key, uint64_t nonce)
struct crypto_aesctr * stream; struct crypto_aesctr * stream;
/* Allocate memory. */ /* Allocate memory. */
if ((stream = malloc(sizeof(struct crypto_aesctr))) == NULL) if ((stream = (struct crypto_aesctr *)malloc(sizeof(struct crypto_aesctr))) == NULL)
goto err0; goto err0;
/* Initialize values. */ /* Initialize values. */

14
crypto/scrypt/crypto_scrypt-nosse.c
View file

@ -26,6 +26,10 @@
* This file was originally written by Colin Percival as part of the Tarsnap * This file was originally written by Colin Percival as part of the Tarsnap
* online backup system. * online backup system.
*/ */
#define __STDC_LIMIT_MACROS 1
#include <stdint.h>
#include <sys/types.h> #include <sys/types.h>
#include <sys/mman.h> #include <sys/mman.h>
@ -49,8 +53,8 @@ static void smix(uint8_t *, size_t, uint64_t, uint32_t *, uint32_t *);
static void static void
blkcpy(void * dest, void * src, size_t len) blkcpy(void * dest, void * src, size_t len)
{ {
size_t * D = dest; size_t * D = (size_t *)dest;
size_t * S = src; size_t * S = (size_t *)src;
size_t L = len / sizeof(size_t); size_t L = len / sizeof(size_t);
size_t i; size_t i;
@ -61,8 +65,8 @@ blkcpy(void * dest, void * src, size_t len)
static void static void
blkxor(void * dest, void * src, size_t len) blkxor(void * dest, void * src, size_t len)
{ {
size_t * D = dest; size_t * D = (size_t *)dest;
size_t * S = src; size_t * S = (size_t *)src;
size_t L = len / sizeof(size_t); size_t L = len / sizeof(size_t);
size_t i; size_t i;
@ -155,7 +159,7 @@ blockmix_salsa8(uint32_t * Bin, uint32_t * Bout, uint32_t * X, size_t r)
static uint64_t static uint64_t
integerify(void * B, size_t r) integerify(void * B, size_t r)
{ {
uint32_t * X = (void *)((uintptr_t)(B) + (2 * r - 1) * 64); uint32_t * X = (uint32_t *)((uintptr_t)(B) + (2 * r - 1) * 64);
return (((uint64_t)(X[1]) << 32) + X[0]); return (((uint64_t)(X[1]) << 32) + X[0]);
} }

4
crypto/scrypt/sha256.c
View file

@ -252,7 +252,7 @@ SHA256_Update(SHA256_CTX * ctx, const void *in, size_t len)
{ {
uint32_t bitlen[2]; uint32_t bitlen[2];
uint32_t r; uint32_t r;
const unsigned char *src = in; const unsigned char *src = (const unsigned char *)in;
/* Number of bytes left in the buffer from previous updates */ /* Number of bytes left in the buffer from previous updates */
r = (ctx->count[1] >> 3) & 0x3f; r = (ctx->count[1] >> 3) & 0x3f;
@ -313,7 +313,7 @@ HMAC_SHA256_Init(HMAC_SHA256_CTX * ctx, const void * _K, size_t Klen)
{ {
unsigned char pad[64]; unsigned char pad[64];
unsigned char khash[32]; unsigned char khash[32];
const unsigned char * K = _K; const unsigned char * K = (const unsigned char *)_K;
size_t i; size_t i;
/* If Klen > 64, the key is really SHA256(K). */ /* If Klen > 64, the key is really SHA256(K). */

6
crypto/sha2/sha256.c
View file

@ -104,8 +104,6 @@ APS_NAMESPACE(Init_SHA) (processor_info_t *pc)
static void static void
_init (SHA256_Context *sc, const uint32_t iv[SHA256_HASH_WORDS]) _init (SHA256_Context *sc, const uint32_t iv[SHA256_HASH_WORDS])
{ {
int i;
/* /*
* SHA256_HASH_WORDS is 8, must be 8, cannot be anything but 8! * SHA256_HASH_WORDS is 8, must be 8, cannot be anything but 8!
* So we unroll a loop here. * So we unroll a loop here.
@ -132,7 +130,7 @@ APS_NAMESPACE(SHA256_Init) (SHA256_Context *sc)
void void
APS_NAMESPACE(SHA256_Update) (SHA256_Context *sc, const void *vdata, size_t len) APS_NAMESPACE(SHA256_Update) (SHA256_Context *sc, const void *vdata, size_t len)
{ {
const uint8_t *data = vdata; const uint8_t *data = (const uint8_t *)vdata;
uint32_t bufferBytesLeft; uint32_t bufferBytesLeft;
size_t bytesToCopy; size_t bytesToCopy;
int rem; int rem;
@ -215,7 +213,7 @@ APS_NAMESPACE(HMAC_SHA256_Init) (HMAC_SHA256_Context * ctx, const void * _K, siz
{ {
unsigned char pad[64]; unsigned char pad[64];
unsigned char khash[32]; unsigned char khash[32];
const unsigned char * K = _K; const unsigned char * K = (const unsigned char *)_K;
size_t i; size_t i;
/* If Klen > 64, the key is really SHA256(K). */ /* If Klen > 64, the key is really SHA256(K). */

18
crypto/skein/SHA3api_ref.c
View file

@ -25,18 +25,18 @@ HashReturn Init(hashState *state, int hashbitlen)
{ {
Skein_Assert(hashbitlen > 0,BAD_HASHLEN); Skein_Assert(hashbitlen > 0,BAD_HASHLEN);
state->statebits = 64*SKEIN_256_STATE_WORDS; state->statebits = 64*SKEIN_256_STATE_WORDS;
return Skein_256_Init(&state->u.ctx_256,(size_t) hashbitlen); return (HashReturn)Skein_256_Init(&state->u.ctx_256,(size_t) hashbitlen);
} }
#endif #endif
if (hashbitlen <= SKEIN_512_NIST_MAX_HASHBITS) if (hashbitlen <= SKEIN_512_NIST_MAX_HASHBITS)
{ {
state->statebits = 64*SKEIN_512_STATE_WORDS; state->statebits = 64*SKEIN_512_STATE_WORDS;
return Skein_512_Init(&state->u.ctx_512,(size_t) hashbitlen); return (HashReturn)Skein_512_Init(&state->u.ctx_512,(size_t) hashbitlen);
} }
else else
{ {
state->statebits = 64*SKEIN1024_STATE_WORDS; state->statebits = 64*SKEIN1024_STATE_WORDS;
return Skein1024_Init(&state->u.ctx1024,(size_t) hashbitlen); return (HashReturn)Skein1024_Init(&state->u.ctx1024,(size_t) hashbitlen);
} }
} }
@ -52,9 +52,9 @@ HashReturn Update(hashState *state, const BitSequence *data, DataLength databitl
{ {
switch ((state->statebits >> 8) & 3) switch ((state->statebits >> 8) & 3)
{ {
case 2: return Skein_512_Update(&state->u.ctx_512,data,databitlen >> 3); case 2: return (HashReturn)Skein_512_Update(&state->u.ctx_512,data,databitlen >> 3);
case 1: return Skein_256_Update(&state->u.ctx_256,data,databitlen >> 3); case 1: return (HashReturn)Skein_256_Update(&state->u.ctx_256,data,databitlen >> 3);
case 0: return Skein1024_Update(&state->u.ctx1024,data,databitlen >> 3); case 0: return (HashReturn)Skein1024_Update(&state->u.ctx1024,data,databitlen >> 3);
default: return FAIL; default: return FAIL;
} }
} }
@ -92,9 +92,9 @@ HashReturn Final(hashState *state, BitSequence *hashval)
Skein_Assert(state->statebits % 256 == 0 && (state->statebits-256) < 1024,FAIL); Skein_Assert(state->statebits % 256 == 0 && (state->statebits-256) < 1024,FAIL);
switch ((state->statebits >> 8) & 3) switch ((state->statebits >> 8) & 3)
{ {
case 2: return Skein_512_Final(&state->u.ctx_512,hashval); case 2: return (HashReturn)Skein_512_Final(&state->u.ctx_512,hashval);
case 1: return Skein_256_Final(&state->u.ctx_256,hashval); case 1: return (HashReturn)Skein_256_Final(&state->u.ctx_256,hashval);
case 0: return Skein1024_Final(&state->u.ctx1024,hashval); case 0: return (HashReturn)Skein1024_Final(&state->u.ctx1024,hashval);
default: return FAIL; default: return FAIL;
} }
} }

2
delta2/delta2.c
View file

@ -115,7 +115,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int
} else { } else {
uchar_t *srcpos, *dstpos, *lastdst, *lastsrc, *dstend; uchar_t *srcpos, *dstpos, *lastdst, *lastsrc, *dstend;
uint64_t slen, sz, dsz, pending; uint64_t slen, sz, dsz, pending;
int rem, lenc, transp_count, hdr_ovr; int rem, lenc, hdr_ovr;
DEBUG_STAT_EN(double strt, en); DEBUG_STAT_EN(double strt, en);
srcpos = src; srcpos = src;

6
libbsc_compress.c
View file

@ -72,7 +72,7 @@ libbsc_stats(int show)
* when compressing entire file in a single chunk. * when compressing entire file in a single chunk.
*/ */
void void
libbsc_props(algo_props_t *data, int level, int64_t chunksize) { libbsc_props(algo_props_t *data, int level, uint64_t chunksize) {
data->compress_mt_capable = 0; data->compress_mt_capable = 0;
data->decompress_mt_capable = 0; data->decompress_mt_capable = 0;
data->single_chunk_mt_capable = 1; data->single_chunk_mt_capable = 1;
@ -83,14 +83,14 @@ libbsc_props(algo_props_t *data, int level, int64_t chunksize) {
} }
int int
libbsc_init(void **data, int *level, int nthreads, int64_t chunksize, libbsc_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op) int file_version, compress_op_t op)
{ {
struct libbsc_params *bscdat; struct libbsc_params *bscdat;
int rv; int rv;
if (chunksize > BSC_MAX_CHUNK) { if (chunksize > BSC_MAX_CHUNK) {
fprintf(stderr, "Max allowed chunk size for LIBBSC is: %d \n", fprintf(stderr, "Max allowed chunk size for LIBBSC is: %ld \n",
BSC_MAX_CHUNK); BSC_MAX_CHUNK);
return (1); return (1);
} }

8
lz4/lz4hc.c
View file

@ -297,16 +297,16 @@ inline static int LZ4HC_Init (LZ4HC_Data_Structure* hc4, const BYTE* base)
} }
inline static void* LZ4HC_Create (const BYTE* base) inline static LZ4HC_Data_Structure* LZ4HC_Create (const BYTE* base)
{ {
void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure)); LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure *)ALLOCATOR(sizeof(LZ4HC_Data_Structure));
LZ4HC_Init (hc4, base); LZ4HC_Init (hc4, base);
return hc4; return hc4;
} }
inline static int LZ4HC_Free (void** LZ4HC_Data) inline static int LZ4HC_Free (LZ4HC_Data_Structure** LZ4HC_Data)
{ {
FREEMEM(*LZ4HC_Data); FREEMEM(*LZ4HC_Data);
*LZ4HC_Data = NULL; *LZ4HC_Data = NULL;
@ -686,7 +686,7 @@ int LZ4_compressHC(const char* source,
char* dest, char* dest,
int isize) int isize)
{ {
void* ctx = LZ4HC_Create((const BYTE*)source); LZ4HC_Data_Structure* ctx = LZ4HC_Create((const BYTE*)source);
int result = LZ4_compressHCCtx(ctx, source, dest, isize); int result = LZ4_compressHCCtx(ctx, source, dest, isize);
LZ4HC_Free (&ctx); LZ4HC_Free (&ctx);

25
lz4_compress.c
View file

@ -44,7 +44,7 @@ lz4_stats(int show)
} }
int int
lz4_buf_extra(int64_t buflen) lz4_buf_extra(uint64_t buflen)
{ {
if (buflen > LZ4_MAX_CHUNK) if (buflen > LZ4_MAX_CHUNK)
buflen = LZ4_MAX_CHUNK; buflen = LZ4_MAX_CHUNK;
@ -52,7 +52,7 @@ lz4_buf_extra(int64_t buflen)
} }
void void
lz4_props(algo_props_t *data, int level, int64_t chunksize) { lz4_props(algo_props_t *data, int level, uint64_t chunksize) {
data->compress_mt_capable = 0; data->compress_mt_capable = 0;
data->decompress_mt_capable = 0; data->decompress_mt_capable = 0;
data->buf_extra = lz4_buf_extra(chunksize); data->buf_extra = lz4_buf_extra(chunksize);
@ -60,7 +60,7 @@ lz4_props(algo_props_t *data, int level, int64_t chunksize) {
} }
int int
lz4_init(void **data, int *level, int nthreads, int64_t chunksize, lz4_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op) int file_version, compress_op_t op)
{ {
struct lz4_params *lzdat; struct lz4_params *lzdat;
@ -71,7 +71,7 @@ lz4_init(void **data, int *level, int nthreads, int64_t chunksize,
LZ4_MAX_CHUNK); LZ4_MAX_CHUNK);
return (1); return (1);
} }
lzdat = slab_alloc(NULL, sizeof (struct lz4_params)); lzdat = (struct lz4_params *)slab_alloc(NULL, sizeof (struct lz4_params));
lev = *level; lev = *level;
if (lev > 3) lev = 3; if (lev > 3) lev = 3;
@ -104,23 +104,23 @@ lz4_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
uchar_t *dst2; uchar_t *dst2;
if (lzdat->level == 1) { if (lzdat->level == 1) {
rv = LZ4_compress(src, dst, _srclen); rv = LZ4_compress((const char *)src, (char *)dst, _srclen);
} else if (lzdat->level == 2) { } else if (lzdat->level == 2) {
rv = LZ4_compress(src, dst, _srclen); rv = LZ4_compress((const char *)src, (char *)dst, _srclen);
if (rv == 0 || rv > *dstlen) { if (rv == 0 || rv > *dstlen) {
return (-1); return (-1);
} }
dst2 = slab_alloc(NULL, rv + sizeof (int) + LZ4_compressBound(rv)); dst2 = (uchar_t *)slab_alloc(NULL, rv + sizeof (int) + LZ4_compressBound(rv));
*((int *)dst2) = htonl(rv); *((int *)dst2) = htonl(rv);
rv = LZ4_compressHC(dst, dst2 + sizeof (int), rv); rv = LZ4_compressHC((const char *)dst, (char *)(dst2 + sizeof (int)), rv);
if (rv != 0) { if (rv != 0) {
rv += sizeof (int); rv += sizeof (int);
memcpy(dst, dst2, rv); memcpy(dst, dst2, rv);
} }
slab_free(NULL, dst2); slab_free(NULL, dst2);
} else { } else {
rv = LZ4_compressHC(src, dst, _srclen); rv = LZ4_compressHC((const char *)src, (char *)dst, _srclen);
} }
if (rv == 0) { if (rv == 0) {
return (-1); return (-1);
@ -137,10 +137,9 @@ lz4_decompress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
int rv; int rv;
struct lz4_params *lzdat = (struct lz4_params *)data; struct lz4_params *lzdat = (struct lz4_params *)data;
int _dstlen = *dstlen; int _dstlen = *dstlen;
uchar_t *dst2;
if (lzdat->level == 1 || lzdat->level == 3) { if (lzdat->level == 1 || lzdat->level == 3) {
rv = LZ4_uncompress(src, dst, _dstlen); rv = LZ4_uncompress((const char *)src, (char *)dst, _dstlen);
if (rv != srclen) { if (rv != srclen) {
return (-1); return (-1);
} }
@ -149,12 +148,12 @@ lz4_decompress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
int sz1; int sz1;
sz1 = ntohl(*((int *)src)); sz1 = ntohl(*((int *)src));
rv = LZ4_uncompress(src + sizeof (int), dst, sz1); rv = LZ4_uncompress((const char *)src + sizeof (int), (char *)dst, sz1);
if (rv != srclen - sizeof (int)) { if (rv != srclen - sizeof (int)) {
return (-1); return (-1);
} }
memcpy(src, dst, sz1); memcpy(src, dst, sz1);
rv = LZ4_uncompress(src, dst, _dstlen); rv = LZ4_uncompress((const char *)src, (char *)dst, _dstlen);
if (rv != sz1) { if (rv != sz1) {
return (-1); return (-1);
} }

6
lzfx_compress.c
View file

@ -40,12 +40,12 @@ lz_fx_stats(int show)
} }
void void
lz_fx_props(algo_props_t *data, int level, int64_t chunksize) { lz_fx_props(algo_props_t *data, int level, uint64_t chunksize) {
data->delta2_span = 50; data->delta2_span = 50;
} }
int int
lz_fx_init(void **data, int *level, int nthreads, int64_t chunksize, lz_fx_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op) int file_version, compress_op_t op)
{ {
struct lzfx_params *lzdat; struct lzfx_params *lzdat;
@ -55,7 +55,7 @@ lz_fx_init(void **data, int *level, int nthreads, int64_t chunksize,
fprintf(stderr, "Chunk size too big for LZFX.\n"); fprintf(stderr, "Chunk size too big for LZFX.\n");
return (1); return (1);
} }
lzdat = slab_alloc(NULL, sizeof (struct lzfx_params)); lzdat = (struct lzfx_params *)slab_alloc(NULL, sizeof (struct lzfx_params));
lev = *level; lev = *level;
if (lev > 5) lev = 5; if (lev > 5) lev = 5;

1
lzma/LzFind.c
View file

@ -273,7 +273,6 @@ static void MatchFinder_SetLimits(CMatchFinder *p)
void MatchFinder_Init(CMatchFinder *p) void MatchFinder_Init(CMatchFinder *p)
{ {
UInt32 i;
/* /*
* Optimized following loop: * Optimized following loop:
* for (i = 0; i < p->hashSizeSum; i++) * for (i = 0; i < p->hashSizeSum; i++)

3
lzma/LzmaEnc.c
View file

@ -790,7 +790,6 @@ static UInt32 RcTree_ReverseGetPrice(const CLzmaProb *probs, int numBitLevels, U
static void LenEnc_Init(CLenEnc *p) static void LenEnc_Init(CLenEnc *p)
{ {
unsigned i;
UInt64 val; UInt64 val;
val = kProbInitValue; val = kProbInitValue;
val <<= 32; val <<= 32;
@ -1313,7 +1312,6 @@ cont_1:
} }
else else
{ {
UInt32 i;
reps[0] = (pos - LZMA_NUM_REPS); reps[0] = (pos - LZMA_NUM_REPS);
/* Unroll for small iterations. */ /* Unroll for small iterations. */
#if LZMA_NUM_REPS > 4 #if LZMA_NUM_REPS > 4
@ -2127,7 +2125,6 @@ void LzmaEnc_Init(CLzmaEnc *p)
for (i = 0; i < kNumStates; i++) for (i = 0; i < kNumStates; i++)
{ {
UInt32 j;
/* /*
* for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) * for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++)
* { * {

12
lzma_compress.c
View file

@ -47,7 +47,7 @@ lzma_stats(int show)
} }
void void
lzma_mt_props(algo_props_t *data, int level, int64_t chunksize) { lzma_mt_props(algo_props_t *data, int level, uint64_t chunksize) {
data->compress_mt_capable = 1; data->compress_mt_capable = 1;
data->decompress_mt_capable = 0; data->decompress_mt_capable = 0;
data->buf_extra = 0; data->buf_extra = 0;
@ -56,7 +56,7 @@ lzma_mt_props(algo_props_t *data, int level, int64_t chunksize) {
} }
void void
lzma_props(algo_props_t *data, int level, int64_t chunksize) { lzma_props(algo_props_t *data, int level, uint64_t chunksize) {
data->compress_mt_capable = 0; data->compress_mt_capable = 0;
data->decompress_mt_capable = 0; data->decompress_mt_capable = 0;
data->buf_extra = 0; data->buf_extra = 0;
@ -67,7 +67,7 @@ lzma_props(algo_props_t *data, int level, int64_t chunksize) {
* The two functions below are not thread-safe, by design. * The two functions below are not thread-safe, by design.
*/ */
int int
lzma_init(void **data, int *level, int nthreads, int64_t chunksize, lzma_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op) int file_version, compress_op_t op)
{ {
if (!p && op == COMPRESS) { if (!p && op == COMPRESS) {
@ -204,8 +204,8 @@ lzma_compress(void *src, uint64_t srclen, void *dst,
_dst = (Byte *)dst; _dst = (Byte *)dst;
*dstlen -= LZMA_PROPS_SIZE; *dstlen -= LZMA_PROPS_SIZE;
res = LzmaEncode(_dst + LZMA_PROPS_SIZE, dstlen, src, srclen, res = LzmaEncode(_dst + LZMA_PROPS_SIZE, dstlen, (const uchar_t *)src, srclen,
props, _dst, &props_len, 0, NULL, &g_Alloc, &g_Alloc); props, (uchar_t *)_dst, &props_len, 0, NULL, &g_Alloc, &g_Alloc);
if (res != 0) { if (res != 0) {
lzerr(res, 1); lzerr(res, 1);
@ -229,7 +229,7 @@ lzma_decompress(void *src, uint64_t srclen, void *dst,
_src = (uchar_t *)src + LZMA_PROPS_SIZE; _src = (uchar_t *)src + LZMA_PROPS_SIZE;
if ((res = LzmaDecode((uchar_t *)dst, dstlen, _src, &_srclen, if ((res = LzmaDecode((uchar_t *)dst, dstlen, _src, &_srclen,
src, LZMA_PROPS_SIZE, LZMA_FINISH_ANY, (uchar_t *)src, LZMA_PROPS_SIZE, LZMA_FINISH_ANY,
&status, &g_Alloc)) != SZ_OK) { &status, &g_Alloc)) != SZ_OK) {
lzerr(res, 0); lzerr(res, 0);
return (-1); return (-1);

10
lzp/lzp.c
View file

@ -38,6 +38,10 @@ See also the bsc and libbsc web site:
*/ */
#undef LZP_OPENMP #undef LZP_OPENMP
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS 1
#endif
#include <stdlib.h> #include <stdlib.h>
#include <memory.h> #include <memory.h>
#include <string.h> #include <string.h>
@ -74,7 +78,8 @@ int bsc_lzp_encode_block(const unsigned char * input, const unsigned char * inpu
return LZP_NOT_COMPRESSIBLE; return LZP_NOT_COMPRESSIBLE;
} }
if (lookup = (int *)slab_calloc(NULL, (int)(1 << hashSize), sizeof(int))) lookup = (int *)slab_calloc(NULL, (int)(1 << hashSize), sizeof(int));
if (lookup)
{ {
unsigned int mask = (int)(1 << hashSize) - 1; unsigned int mask = (int)(1 << hashSize) - 1;
const unsigned char * inputStart = input; const unsigned char * inputStart = input;
@ -172,7 +177,8 @@ int bsc_lzp_decode_block(const unsigned char * input, const unsigned char * inpu
return LZP_UNEXPECTED_EOB; return LZP_UNEXPECTED_EOB;
} }
if (lookup = (int *)slab_calloc(NULL, (int)(1 << hashSize), sizeof(int))) lookup = (int *)slab_calloc(NULL, (int)(1 << hashSize), sizeof(int));
if (lookup)
{ {
unsigned int mask = (int)(1 << hashSize) - 1; unsigned int mask = (int)(1 << hashSize) - 1;
const unsigned char * outputStart = output; const unsigned char * outputStart = output;

57
main.c
View file

@ -47,6 +47,9 @@
#include <rabin_dedup.h> #include <rabin_dedup.h>
#include <lzp.h> #include <lzp.h>
#include <transpose.h> #include <transpose.h>
#include <delta2/delta2.h>
#include <crypto/crypto_utils.h>
#include <ctype.h>
/* /*
* We use 5MB chunks by default. * We use 5MB chunks by default.
@ -95,7 +98,6 @@ static int do_uncompress = 0;
static int cksum_bytes, mac_bytes; static int cksum_bytes, mac_bytes;
static int cksum = 0, t_errored = 0; static int cksum = 0, t_errored = 0;
static int rab_blk_size = 0; static int rab_blk_size = 0;
static dedupe_context_t *rctx;
static crypto_ctx_t crypto_ctx; static crypto_ctx_t crypto_ctx;
static char *pwd_file = NULL; static char *pwd_file = NULL;
@ -198,7 +200,8 @@ preproc_compress(compress_func_ptr cmp_func, void *src, uint64_t srclen, void *d
uint64_t *dstlen, int level, uchar_t chdr, void *data, algo_props_t *props) uint64_t *dstlen, int level, uchar_t chdr, void *data, algo_props_t *props)
{ {
uchar_t *dest = (uchar_t *)dst, type = 0; uchar_t *dest = (uchar_t *)dst, type = 0;
int64_t result, _dstlen; int64_t result;
uint64_t _dstlen;
DEBUG_STAT_EN(double strt, en); DEBUG_STAT_EN(double strt, en);
_dstlen = *dstlen; _dstlen = *dstlen;
@ -207,7 +210,8 @@ preproc_compress(compress_func_ptr cmp_func, void *src, uint64_t srclen, void *d
type = PREPROC_TYPE_LZP; type = PREPROC_TYPE_LZP;
hashsize = lzp_hash_size(level); hashsize = lzp_hash_size(level);
result = lzp_compress(src, dst, srclen, hashsize, LZP_DEFAULT_LZPMINLEN, 0); result = lzp_compress((const uchar_t *)src, (uchar_t *)dst, srclen,
hashsize, LZP_DEFAULT_LZPMINLEN, 0);
if (result < 0 || result == srclen) { if (result < 0 || result == srclen) {
if (!enable_delta2_encode) if (!enable_delta2_encode)
return (-1); return (-1);
@ -227,7 +231,8 @@ preproc_compress(compress_func_ptr cmp_func, void *src, uint64_t srclen, void *d
if (enable_delta2_encode && props->delta2_span > 0) { if (enable_delta2_encode && props->delta2_span > 0) {
_dstlen = srclen; _dstlen = srclen;
result = delta2_encode(src, srclen, dst, &_dstlen, props->delta2_span); result = delta2_encode((uchar_t *)src, srclen, (uchar_t *)dst,
&_dstlen, props->delta2_span);
if (result != -1) { if (result != -1) {
memcpy(src, dst, _dstlen); memcpy(src, dst, _dstlen);
srclen = _dstlen; srclen = _dstlen;
@ -285,7 +290,7 @@ preproc_decompress(compress_func_ptr dec_func, void *src, uint64_t srclen, void
} }
if (type & PREPROC_TYPE_DELTA2) { if (type & PREPROC_TYPE_DELTA2) {
result = delta2_decode(src, srclen, dst, &_dstlen); result = delta2_decode((uchar_t *)src, srclen, (uchar_t *)dst, &_dstlen);
if (result != -1) { if (result != -1) {
memcpy(src, dst, _dstlen); memcpy(src, dst, _dstlen);
srclen = _dstlen; srclen = _dstlen;
@ -298,7 +303,8 @@ preproc_decompress(compress_func_ptr dec_func, void *src, uint64_t srclen, void
if (type & PREPROC_TYPE_LZP) { if (type & PREPROC_TYPE_LZP) {
int hashsize; int hashsize;
hashsize = lzp_hash_size(level); hashsize = lzp_hash_size(level);
result = lzp_decompress(src, dst, srclen, hashsize, LZP_DEFAULT_LZPMINLEN, 0); result = lzp_decompress((const uchar_t *)src, (uchar_t *)dst, srclen,
hashsize, LZP_DEFAULT_LZPMINLEN, 0);
if (result < 0) { if (result < 0) {
fprintf(stderr, "LZP decompression failed.\n"); fprintf(stderr, "LZP decompression failed.\n");
return (-1); return (-1);
@ -323,9 +329,9 @@ static void *
perform_decompress(void *dat) perform_decompress(void *dat)
{ {
struct cmp_data *tdat = (struct cmp_data *)dat; struct cmp_data *tdat = (struct cmp_data *)dat;
int64_t _chunksize; uint64_t _chunksize;
int64_t dedupe_index_sz, dedupe_data_sz, dedupe_index_sz_cmp, dedupe_data_sz_cmp; uint64_t dedupe_index_sz, dedupe_data_sz, dedupe_index_sz_cmp, dedupe_data_sz_cmp;
int type, rv; int rv = 0;
unsigned int blknum; unsigned int blknum;
uchar_t checksum[CKSUM_MAX_BYTES]; uchar_t checksum[CKSUM_MAX_BYTES];
uchar_t HDR; uchar_t HDR;
@ -388,7 +394,7 @@ redo:
tdat->len_cmp = 0; tdat->len_cmp = 0;
t_errored = 1; t_errored = 1;
sem_post(&tdat->cmp_done_sem); sem_post(&tdat->cmp_done_sem);
return; return (NULL);
} }
/* /*
@ -403,7 +409,7 @@ redo:
main_cancel = 1; main_cancel = 1;
tdat->len_cmp = 0; tdat->len_cmp = 0;
sem_post(&tdat->cmp_done_sem); sem_post(&tdat->cmp_done_sem);
return; return (NULL);
} }
} else if (mac_bytes > 0) { } else if (mac_bytes > 0) {
/* /*
@ -431,7 +437,7 @@ redo:
tdat->len_cmp = 0; tdat->len_cmp = 0;
t_errored = 1; t_errored = 1;
sem_post(&tdat->cmp_done_sem); sem_post(&tdat->cmp_done_sem);
return; return (NULL);
} }
/* /*
@ -603,7 +609,6 @@ cont:
static int static int
start_decompress(const char *filename, const char *to_filename) start_decompress(const char *filename, const char *to_filename)
{ {
char tmpfile[MAXPATHLEN];
char algorithm[ALGO_SZ]; char algorithm[ALGO_SZ];
struct stat sbuf; struct stat sbuf;
struct wdata w; struct wdata w;
@ -752,6 +757,7 @@ start_decompress(const char *filename, const char *to_filename)
* is computed over header and encrypted data. * is computed over header and encrypted data.
*/ */
cksum_bytes = 0; cksum_bytes = 0;
pw_len = -1;
compressed_chunksize += mac_bytes; compressed_chunksize += mac_bytes;
encrypt_type = flags & MASK_CRYPTO_ALG; encrypt_type = flags & MASK_CRYPTO_ALG;
if (Read(compfd, &saltlen, sizeof (saltlen)) < sizeof (saltlen)) { if (Read(compfd, &saltlen, sizeof (saltlen)) < sizeof (saltlen)) {
@ -759,8 +765,8 @@ start_decompress(const char *filename, const char *to_filename)
UNCOMP_BAIL; UNCOMP_BAIL;
} }
saltlen = ntohl(saltlen); saltlen = ntohl(saltlen);
salt1 = malloc(saltlen); salt1 = (uchar_t *)malloc(saltlen);
salt2 = malloc(saltlen); salt2 = (uchar_t *)malloc(saltlen);
if (Read(compfd, salt1, saltlen) < saltlen) { if (Read(compfd, salt1, saltlen) < saltlen) {
free(salt1); free(salt2); free(salt1); free(salt2);
perror("Read: "); perror("Read: ");
@ -867,7 +873,6 @@ start_decompress(const char *filename, const char *to_filename)
} }
} else if (version >= 5) { } else if (version >= 5) {
uint32_t crc1, crc2; uint32_t crc1, crc2;
unsigned int hlen;
unsigned short d1; unsigned short d1;
unsigned int d2; unsigned int d2;
uint64_t ch; uint64_t ch;
@ -1084,7 +1089,6 @@ start_decompress(const char *filename, const char *to_filename)
tdat = dary[p]; tdat = dary[p];
sem_wait(&tdat->write_done_sem); sem_wait(&tdat->write_done_sem);
} }
thread = 0;
} }
uncomp_done: uncomp_done:
if (t_errored) err = t_errored; if (t_errored) err = t_errored;
@ -1151,6 +1155,8 @@ redo:
compressed_chunk = tdat->compressed_chunk + CHUNK_FLAG_SZ; compressed_chunk = tdat->compressed_chunk + CHUNK_FLAG_SZ;
rbytes = tdat->rbytes; rbytes = tdat->rbytes;
dedupe_index_sz = 0;
/* Perform Dedup if enabled. */ /* Perform Dedup if enabled. */
if ((enable_rabin_scan || enable_fixed_scan)) { if ((enable_rabin_scan || enable_fixed_scan)) {
dedupe_context_t *rctx; dedupe_context_t *rctx;
@ -1249,7 +1255,6 @@ plain_index:
update_dedupe_hdr(compressed_chunk, index_size_cmp - RABIN_HDR_SIZE, _chunksize); update_dedupe_hdr(compressed_chunk, index_size_cmp - RABIN_HDR_SIZE, _chunksize);
_chunksize += index_size_cmp; _chunksize += index_size_cmp;
} else { } else {
plain_compress:
_chunksize = tdat->rbytes; _chunksize = tdat->rbytes;
if (lzp_preprocess || enable_delta2_encode) { if (lzp_preprocess || enable_delta2_encode) {
rv = preproc_compress(tdat->compress, rv = preproc_compress(tdat->compress,
@ -1366,7 +1371,6 @@ plain_compress:
* Compute header CRC32 in non-crypto mode. * Compute header CRC32 in non-crypto mode.
*/ */
uchar_t *mac_ptr; uchar_t *mac_ptr;
unsigned int hlen;
uint32_t crc; uint32_t crc;
/* Clean out mac_bytes to 0 for stable CRC32. */ /* Clean out mac_bytes to 0 for stable CRC32. */
@ -1379,7 +1383,6 @@ plain_compress:
*((uint32_t *)mac_ptr) = htonl(crc); *((uint32_t *)mac_ptr) = htonl(crc);
} }
cont:
sem_post(&tdat->cmp_done_sem); sem_post(&tdat->cmp_done_sem);
goto redo; goto redo;
} }
@ -1409,8 +1412,6 @@ repeat:
wbytes = Write(w->wfd, tdat->cmp_seg, tdat->len_cmp); wbytes = Write(w->wfd, tdat->cmp_seg, tdat->len_cmp);
if (unlikely(wbytes != tdat->len_cmp)) { if (unlikely(wbytes != tdat->len_cmp)) {
int i;
perror("Chunk Write: "); perror("Chunk Write: ");
do_cancel: do_cancel:
main_cancel = 1; main_cancel = 1;
@ -1437,8 +1438,8 @@ start_compress(const char *filename, uint64_t chunksize, int level)
struct wdata w; struct wdata w;
char tmpfile1[MAXPATHLEN]; char tmpfile1[MAXPATHLEN];
char to_filename[MAXPATHLEN]; char to_filename[MAXPATHLEN];
int64_t compressed_chunksize; uint64_t compressed_chunksize, n_chunksize;
int64_t n_chunksize, rbytes, rabin_count; int64_t rbytes, rabin_count;
short version, flags; short version, flags;
struct stat sbuf; struct stat sbuf;
int compfd = -1, uncompfd = -1, err; int compfd = -1, uncompfd = -1, err;
@ -1464,6 +1465,7 @@ start_compress(const char *filename, uint64_t chunksize, int level)
*/ */
compressed_chunksize = chunksize + CHUNK_HDR_SZ + zlib_buf_extra(chunksize); compressed_chunksize = chunksize + CHUNK_HDR_SZ + zlib_buf_extra(chunksize);
init_algo_props(&props); init_algo_props(&props);
cread_buf = NULL;
if (_props_func) { if (_props_func) {
_props_func(&props, level, chunksize); _props_func(&props, level, chunksize);
@ -1486,7 +1488,7 @@ start_compress(const char *filename, uint64_t chunksize, int level)
if (encrypt_type) { if (encrypt_type) {
uchar_t pw[MAX_PW_LEN]; uchar_t pw[MAX_PW_LEN];
int pw_len; int pw_len = -1;
compressed_chunksize += mac_bytes; compressed_chunksize += mac_bytes;
if (!pwd_file) { if (!pwd_file) {
@ -1697,7 +1699,7 @@ start_compress(const char *filename, uint64_t chunksize, int level)
*/ */
flags |= cksum; flags |= cksum;
memset(cread_buf, 0, ALGO_SZ); memset(cread_buf, 0, ALGO_SZ);
strncpy(cread_buf, algo, ALGO_SZ); strncpy((char *)cread_buf, algo, ALGO_SZ);
version = htons(VERSION); version = htons(VERSION);
flags = htons(flags); flags = htons(flags);
n_chunksize = htonll(chunksize); n_chunksize = htonll(chunksize);
@ -1991,7 +1993,7 @@ comp_done:
static int static int
init_algo(const char *algo, int bail) init_algo(const char *algo, int bail)
{ {
int rv = 1, i; int rv = 1;
char algorithm[8]; char algorithm[8];
/* Copy given string into known length buffer to avoid memcmp() overruns. */ /* Copy given string into known length buffer to avoid memcmp() overruns. */
@ -2115,6 +2117,7 @@ main(int argc, char *argv[])
exec_name = get_execname(argv[0]); exec_name = get_execname(argv[0]);
level = 6; level = 6;
err = 0;
slab_init(); slab_init();
while ((opt = getopt(argc, argv, "dc:s:l:pt:MCDEew:rLPS:B:F")) != -1) { while ((opt = getopt(argc, argv, "dc:s:l:pt:MCDEew:rLPS:B:F")) != -1) {

4
none_compress.c
View file

@ -37,14 +37,14 @@ none_stats(int show)
} }
int int
none_init(void **data, int *level, int nthreads, int64_t chunksize, none_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op) int file_version, compress_op_t op)
{ {
return (0); return (0);
} }
void void
none_props(algo_props_t *data, int level, int64_t chunksize) { none_props(algo_props_t *data, int level, uint64_t chunksize) {
data->compress_mt_capable = 0; data->compress_mt_capable = 0;
data->decompress_mt_capable = 0; data->decompress_mt_capable = 0;
data->buf_extra = 0; data->buf_extra = 0;

50
pcompress.h
View file

@ -77,8 +77,8 @@ extern "C" {
#define COMPRESS_BSC 4 #define COMPRESS_BSC 4
#define CHDR_ALGO_MASK 7 #define CHDR_ALGO_MASK 7
extern uint32_t zlib_buf_extra(int64_t buflen); extern uint32_t zlib_buf_extra(uint64_t buflen);
extern int lz4_buf_extra(int64_t buflen); extern int lz4_buf_extra(uint64_t buflen);
extern int zlib_compress(void *src, uint64_t srclen, void *dst, extern int zlib_compress(void *src, uint64_t srclen, void *dst,
uint64_t *destlen, int level, uchar_t chdr, void *data); uint64_t *destlen, int level, uchar_t chdr, void *data);
@ -114,34 +114,34 @@ extern int lz4_decompress(void *src, uint64_t srclen, void *dst,
extern int none_decompress(void *src, uint64_t srclen, void *dst, extern int none_decompress(void *src, uint64_t srclen, void *dst,
uint64_t *dstlen, int level, uchar_t chdr, void *data); uint64_t *dstlen, int level, uchar_t chdr, void *data);
extern int adapt_init(void **data, int *level, int nthreads, int64_t chunksize, extern int adapt_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int adapt2_init(void **data, int *level, int nthreads, int64_t chunksize, extern int adapt2_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int lzma_init(void **data, int *level, int nthreads, int64_t chunksize, extern int lzma_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int ppmd_init(void **data, int *level, int nthreads, int64_t chunksize, extern int ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int bzip2_init(void **data, int *level, int nthreads, int64_t chunksize, extern int bzip2_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int zlib_init(void **data, int *level, int nthreads, int64_t chunksize, extern int zlib_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int lz_fx_init(void **data, int *level, int nthreads, int64_t chunksize, extern int lz_fx_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int lz4_init(void **data, int *level, int nthreads, int64_t chunksize, extern int lz4_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern int none_init(void **data, int *level, int nthreads, int64_t chunksize, extern int none_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern void lzma_props(algo_props_t *data, int level, int64_t chunksize); extern void lzma_props(algo_props_t *data, int level, uint64_t chunksize);
extern void lzma_mt_props(algo_props_t *data, int level, int64_t chunksize); extern void lzma_mt_props(algo_props_t *data, int level, uint64_t chunksize);
extern void lz4_props(algo_props_t *data, int level, int64_t chunksize); extern void lz4_props(algo_props_t *data, int level, uint64_t chunksize);
extern void zlib_props(algo_props_t *data, int level, int64_t chunksize); extern void zlib_props(algo_props_t *data, int level, uint64_t chunksize);
extern void ppmd_props(algo_props_t *data, int level, int64_t chunksize); extern void ppmd_props(algo_props_t *data, int level, uint64_t chunksize);
extern void lz_fx_props(algo_props_t *data, int level, int64_t chunksize); extern void lz_fx_props(algo_props_t *data, int level, uint64_t chunksize);
extern void bzip2_props(algo_props_t *data, int level, int64_t chunksize); extern void bzip2_props(algo_props_t *data, int level, uint64_t chunksize);
extern void adapt_props(algo_props_t *data, int level, int64_t chunksize); extern void adapt_props(algo_props_t *data, int level, uint64_t chunksize);
extern void none_props(algo_props_t *data, int level, int64_t chunksize); extern void none_props(algo_props_t *data, int level, uint64_t chunksize);
extern int zlib_deinit(void **data); extern int zlib_deinit(void **data);
extern int adapt_deinit(void **data); extern int adapt_deinit(void **data);
@ -165,9 +165,9 @@ extern int libbsc_compress(void *src, uint64_t srclen, void *dst,
uint64_t *dstlen, int level, uchar_t chdr, void *data); uint64_t *dstlen, int level, uchar_t chdr, void *data);
extern int libbsc_decompress(void *src, uint64_t srclen, void *dst, extern int libbsc_decompress(void *src, uint64_t srclen, void *dst,
uint64_t *dstlen, int level, uchar_t chdr, void *data); uint64_t *dstlen, int level, uchar_t chdr, void *data);
extern int libbsc_init(void **data, int *level, int nthreads, int64_t chunksize, extern int libbsc_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
extern void libbsc_props(algo_props_t *data, int level, int64_t chunksize); extern void libbsc_props(algo_props_t *data, int level, uint64_t chunksize);
extern int libbsc_deinit(void **data); extern int libbsc_deinit(void **data);
extern void libbsc_stats(int show); extern void libbsc_stats(int show);
#endif #endif
@ -180,9 +180,9 @@ struct cmp_data {
uchar_t *compressed_chunk; uchar_t *compressed_chunk;
uchar_t *uncompressed_chunk; uchar_t *uncompressed_chunk;
dedupe_context_t *rctx; dedupe_context_t *rctx;
int64_t rbytes; uint64_t rbytes;
int64_t chunksize; uint64_t chunksize;
int64_t len_cmp, len_cmp_be; uint64_t len_cmp, len_cmp_be;
uchar_t checksum[CKSUM_MAX_BYTES]; uchar_t checksum[CKSUM_MAX_BYTES];
int level; int level;
unsigned int id; unsigned int id;

5
ppmd_compress.c
View file

@ -62,12 +62,12 @@ ppmd_stats(int show)
} }
void void
ppmd_props(algo_props_t *data, int level, int64_t chunksize) { ppmd_props(algo_props_t *data, int level, uint64_t chunksize) {
data->delta2_span = 100; data->delta2_span = 100;
} }
int int
ppmd_init(void **data, int *level, int nthreads, int64_t chunksize, ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op) int file_version, compress_op_t op)
{ {
CPpmd8 *_ppmd; CPpmd8 *_ppmd;
@ -110,7 +110,6 @@ ppmd_compress(void *src, uint64_t srclen, void *dst,
{ {
CPpmd8 *_ppmd = (CPpmd8 *)data; CPpmd8 *_ppmd = (CPpmd8 *)data;
uchar_t *_src = (uchar_t *)src; uchar_t *_src = (uchar_t *)src;
UInt32 i;
Ppmd8_RangeEnc_Init(_ppmd); Ppmd8_RangeEnc_Init(_ppmd);
Ppmd8_Init(_ppmd, _ppmd->Order, PPMD8_RESTORE_METHOD_RESTART); Ppmd8_Init(_ppmd, _ppmd->Order, PPMD8_RESTORE_METHOD_RESTART);

81
rabin/rabin_dedup.c
View file

@ -57,6 +57,10 @@
* *
*/ */
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS 1
#endif
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
@ -64,6 +68,7 @@
#include <utils.h> #include <utils.h>
#include <pthread.h> #include <pthread.h>
#include <heapq.h> #include <heapq.h>
#include <xxhash.h>
#include "rabin_dedup.h" #include "rabin_dedup.h"
@ -78,10 +83,10 @@ extern int lzma_compress(void *src, uint64_t srclen, void *dst,
extern int lzma_decompress(void *src, uint64_t srclen, void *dst, extern int lzma_decompress(void *src, uint64_t srclen, void *dst,
uint64_t *dstlen, int level, uchar_t chdr, void *data); uint64_t *dstlen, int level, uchar_t chdr, void *data);
extern int lzma_deinit(void **data); extern int lzma_deinit(void **data);
extern int bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize, extern int bsdiff(u_char *oldbuf, bsize_t oldsize, u_char *newbuf, bsize_t newsize,
u_char *diff, u_char *scratch, bsize_t scratchsize); u_char *diff, u_char *scratch, bsize_t scratchsize);
extern bsize_t get_bsdiff_sz(u_char *pbuf); extern bsize_t get_bsdiff_sz(u_char *pbuf);
extern int bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, extern int bspatch(u_char *pbuf, u_char *oldbuf, bsize_t oldsize, u_char *newbuf,
bsize_t *_newsize); bsize_t *_newsize);
static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
@ -214,7 +219,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
destroy_dedupe_context(ctx); destroy_dedupe_context(ctx);
return (NULL); return (NULL);
} }
ctx->current_window_data = slab_alloc(NULL, RAB_POLYNOMIAL_WIN_SIZE); ctx->current_window_data = (uchar_t *)slab_alloc(NULL, RAB_POLYNOMIAL_WIN_SIZE);
ctx->blocks = NULL; ctx->blocks = NULL;
if (real_chunksize > 0) { if (real_chunksize > 0) {
ctx->blocks = (rabin_blockentry_t **)slab_calloc(NULL, ctx->blocks = (rabin_blockentry_t **)slab_calloc(NULL,
@ -279,11 +284,11 @@ destroy_dedupe_context(dedupe_context_t *ctx)
* from 4K-128K. * from 4K-128K.
*/ */
uint32_t uint32_t
dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size, int64_t offset, int64_t *rabin_pos) dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t offset, uint64_t *rabin_pos)
{ {
int64_t i, last_offset, j, ary_sz; uint64_t i, last_offset, j, ary_sz;
uint32_t blknum; uint32_t blknum;
char *buf1 = (char *)buf; uchar_t *buf1 = (uchar_t *)buf;
uint32_t length; uint32_t length;
uint64_t cur_roll_checksum, cur_pos_checksum; uint64_t cur_roll_checksum, cur_pos_checksum;
uint32_t *fplist; uint32_t *fplist;
@ -298,6 +303,7 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size, int64_t offs
blknum = 0; blknum = 0;
ctx->valid = 0; ctx->valid = 0;
cur_roll_checksum = 0; cur_roll_checksum = 0;
if (*size < ctx->rabin_poly_avg_block_size) return (0);
DEBUG_STAT_EN(strt = get_wtime_millis()); DEBUG_STAT_EN(strt = get_wtime_millis());
if (ctx->fixed_flag) { if (ctx->fixed_flag) {
@ -377,11 +383,10 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size, int64_t offs
return (0); return (0);
} }
if (*size < ctx->rabin_poly_avg_block_size) return;
j = 0; j = 0;
for (i=offset; i<*size; i++) { for (i=offset; i<*size; i++) {
int64_t pc[4]; uint64_t pc[4];
uchar_t cur_byte = buf1[i]; uchar_t cur_byte = buf1[i];
uint64_t pushed_out = ctx->current_window_data[ctx->window_pos]; uint64_t pushed_out = ctx->current_window_data[ctx->window_pos];
ctx->current_window_data[ctx->window_pos] = cur_byte; ctx->current_window_data[ctx->window_pos] = cur_byte;
@ -439,7 +444,7 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size, int64_t offs
pc[3] = FORTY_PCNT(j); pc[3] = FORTY_PCNT(j);
reset_heap(&heap, pc[ctx->delta_flag]); reset_heap(&heap, pc[ctx->delta_flag]);
ksmallest(fplist, j, &heap); ksmallest((int32_t *)fplist, j, &heap);
ctx->blocks[blknum]->similarity_hash = ctx->blocks[blknum]->similarity_hash =
XXH_fast32((const uchar_t *)fplist, pc[ctx->delta_flag]*4, 0); XXH_fast32((const uchar_t *)fplist, pc[ctx->delta_flag]*4, 0);
memset(fplist, 0, ary_sz); memset(fplist, 0, ary_sz);
@ -462,14 +467,14 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size, int64_t offs
if (ctx->delta_flag) { if (ctx->delta_flag) {
uint64_t cur_sketch; uint64_t cur_sketch;
int64_t pc[3]; uint64_t pc[3];
if (j > 1) { if (j > 1) {
pc[1] = SIXTY_PCNT(j); pc[1] = SIXTY_PCNT(j);
pc[2] = FIFTY_PCNT(j); pc[2] = FIFTY_PCNT(j);
pc[3] = FORTY_PCNT(j); pc[3] = FORTY_PCNT(j);
reset_heap(&heap, pc[ctx->delta_flag]); reset_heap(&heap, pc[ctx->delta_flag]);
ksmallest(fplist, j, &heap); ksmallest((int32_t *)fplist, j, &heap);
cur_sketch = cur_sketch =
XXH_fast32((const uchar_t *)fplist, pc[ctx->delta_flag]*4, 0); XXH_fast32((const uchar_t *)fplist, pc[ctx->delta_flag]*4, 0);
} else { } else {
@ -489,10 +494,10 @@ process_blocks:
DEBUG_STAT_EN(fprintf(stderr, "Original size: %" PRId64 ", blknum: %u\n", *size, blknum)); DEBUG_STAT_EN(fprintf(stderr, "Original size: %" PRId64 ", blknum: %u\n", *size, blknum));
DEBUG_STAT_EN(fprintf(stderr, "Number of maxlen blocks: %u\n", max_count)); DEBUG_STAT_EN(fprintf(stderr, "Number of maxlen blocks: %u\n", max_count));
if (blknum > 2) { if (blknum > 2) {
int64_t pos, matchlen, pos1; uint64_t pos, matchlen, pos1;
int valid = 1; int valid = 1;
uint32_t *dedupe_index; uint32_t *dedupe_index;
int64_t dedupe_index_sz; uint64_t dedupe_index_sz;
rabin_blockentry_t *be; rabin_blockentry_t *be;
DEBUG_STAT_EN(uint32_t delta_calls, delta_fails, merge_count, hash_collisions); DEBUG_STAT_EN(uint32_t delta_calls, delta_fails, merge_count, hash_collisions);
DEBUG_STAT_EN(delta_calls = 0); DEBUG_STAT_EN(delta_calls = 0);
@ -609,11 +614,11 @@ process_blocks:
} }
DEBUG_STAT_EN(fprintf(stderr, "Total Hashtable bucket collisions: %u\n", hash_collisions)); DEBUG_STAT_EN(fprintf(stderr, "Total Hashtable bucket collisions: %u\n", hash_collisions));
dedupe_index_sz = (int64_t)blknum * RABIN_ENTRY_SIZE; dedupe_index_sz = (uint64_t)blknum * RABIN_ENTRY_SIZE;
if (matchlen < dedupe_index_sz) { if (matchlen < dedupe_index_sz) {
DEBUG_STAT_EN(fprintf(stderr, "No Dedupe possible.\n")); DEBUG_STAT_EN(fprintf(stderr, "No Dedupe possible.\n"));
ctx->valid = 0; ctx->valid = 0;
return; return (0);
} }
dedupe_index = (uint32_t *)(ctx->cbuf + RABIN_HDR_SIZE); dedupe_index = (uint32_t *)(ctx->cbuf + RABIN_HDR_SIZE);
@ -648,7 +653,7 @@ process_blocks:
* Final pass update dedupe index and copy data. * Final pass update dedupe index and copy data.
*/ */
blknum = pos; blknum = pos;
dedupe_index_sz = (int64_t)blknum * RABIN_ENTRY_SIZE; dedupe_index_sz = (uint64_t)blknum * RABIN_ENTRY_SIZE;
pos1 = dedupe_index_sz + RABIN_HDR_SIZE; pos1 = dedupe_index_sz + RABIN_HDR_SIZE;
matchlen = ctx->real_chunksize - *size; matchlen = ctx->real_chunksize - *size;
for (i=0; i<blknum; i++) { for (i=0; i<blknum; i++) {
@ -663,20 +668,20 @@ process_blocks:
dedupe_index[i] = htonl((be->other->index | RABIN_INDEX_FLAG) & dedupe_index[i] = htonl((be->other->index | RABIN_INDEX_FLAG) &
CLEAR_SIMILARITY_FLAG); CLEAR_SIMILARITY_FLAG);
} else { } else {
uchar_t *old, *new; uchar_t *oldbuf, *newbuf;
int32_t bsz; int32_t bsz;
/* /*
* Perform bsdiff. * Perform bsdiff.
*/ */
old = buf1 + be->other->offset; oldbuf = buf1 + be->other->offset;
new = buf1 + be->offset; newbuf = buf1 + be->offset;
DEBUG_STAT_EN(delta_calls++); DEBUG_STAT_EN(delta_calls++);
bsz = bsdiff(old, be->other->length, new, be->length, bsz = bsdiff(oldbuf, be->other->length, newbuf, be->length,
ctx->cbuf + pos1, buf1 + *size, matchlen); ctx->cbuf + pos1, buf1 + *size, matchlen);
if (bsz == 0) { if (bsz == 0) {
DEBUG_STAT_EN(delta_fails++); DEBUG_STAT_EN(delta_fails++);
memcpy(ctx->cbuf + pos1, new, be->length); memcpy(ctx->cbuf + pos1, newbuf, be->length);
dedupe_index[i] = htonl(be->length); dedupe_index[i] = htonl(be->length);
pos1 += be->length; pos1 += be->length;
} else { } else {
@ -687,16 +692,16 @@ process_blocks:
} }
} }
} }
cont:
if (valid) { if (valid) {
uchar_t *cbuf = ctx->cbuf; uchar_t *cbuf = ctx->cbuf;
int64_t *entries; uint64_t *entries;
DEBUG_STAT_EN(int64_t sz); DEBUG_STAT_EN(uint64_t sz);
DEBUG_STAT_EN(sz = *size); DEBUG_STAT_EN(sz = *size);
*((uint32_t *)cbuf) = htonl(blknum); *((uint32_t *)cbuf) = htonl(blknum);
cbuf += sizeof (uint32_t); cbuf += sizeof (uint32_t);
entries = (int64_t *)cbuf; entries = (uint64_t *)cbuf;
entries[0] = htonll(*size); entries[0] = htonll(*size);
entries[1] = 0; entries[1] = 0;
entries[2] = htonll(pos1 - dedupe_index_sz - RABIN_HDR_SIZE); entries[2] = htonll(pos1 - dedupe_index_sz - RABIN_HDR_SIZE);
@ -717,41 +722,41 @@ cont:
} }
void void
update_dedupe_hdr(uchar_t *buf, int64_t dedupe_index_sz_cmp, int64_t dedupe_data_sz_cmp) update_dedupe_hdr(uchar_t *buf, uint64_t dedupe_index_sz_cmp, uint64_t dedupe_data_sz_cmp)
{ {
int64_t *entries; uint64_t *entries;
buf += sizeof (uint32_t); buf += sizeof (uint32_t);
entries = (int64_t *)buf; entries = (uint64_t *)buf;
entries[1] = htonll(dedupe_index_sz_cmp); entries[1] = htonll(dedupe_index_sz_cmp);
entries[3] = htonll(dedupe_data_sz_cmp); entries[3] = htonll(dedupe_data_sz_cmp);
} }
void void
parse_dedupe_hdr(uchar_t *buf, uint32_t *blknum, int64_t *dedupe_index_sz, parse_dedupe_hdr(uchar_t *buf, uint32_t *blknum, uint64_t *dedupe_index_sz,
int64_t *dedupe_data_sz, int64_t *dedupe_index_sz_cmp, uint64_t *dedupe_data_sz, uint64_t *dedupe_index_sz_cmp,
int64_t *dedupe_data_sz_cmp, int64_t *deduped_size) uint64_t *dedupe_data_sz_cmp, uint64_t *deduped_size)
{ {
int64_t *entries; uint64_t *entries;
*blknum = ntohl(*((uint32_t *)(buf))); *blknum = ntohl(*((uint32_t *)(buf)));
buf += sizeof (uint32_t); buf += sizeof (uint32_t);
entries = (int64_t *)buf; entries = (uint64_t *)buf;
*dedupe_data_sz = ntohll(entries[0]); *dedupe_data_sz = ntohll(entries[0]);
*dedupe_index_sz = (int64_t)(*blknum) * RABIN_ENTRY_SIZE; *dedupe_index_sz = (uint64_t)(*blknum) * RABIN_ENTRY_SIZE;
*dedupe_index_sz_cmp = ntohll(entries[1]); *dedupe_index_sz_cmp = ntohll(entries[1]);
*deduped_size = ntohll(entries[2]); *deduped_size = ntohll(entries[2]);
*dedupe_data_sz_cmp = ntohll(entries[3]); *dedupe_data_sz_cmp = ntohll(entries[3]);
} }
void void
dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size) dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size)
{ {
uint32_t blknum, blk, oblk, len; uint32_t blknum, blk, oblk, len;
uint32_t *dedupe_index; uint32_t *dedupe_index;
int64_t data_sz, sz, indx_cmp, data_sz_cmp, deduped_sz; uint64_t data_sz, sz, indx_cmp, data_sz_cmp, deduped_sz;
int64_t dedupe_index_sz, pos1, i; uint64_t dedupe_index_sz, pos1;
uchar_t *pos2; uchar_t *pos2;
parse_dedupe_hdr(buf, &blknum, &dedupe_index_sz, &data_sz, &indx_cmp, &data_sz_cmp, &deduped_sz); parse_dedupe_hdr(buf, &blknum, &dedupe_index_sz, &data_sz, &indx_cmp, &data_sz_cmp, &deduped_sz);
@ -844,7 +849,7 @@ dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size)
* TODO: Consolidate rabin dedup and compression/decompression in functions here rather than * TODO: Consolidate rabin dedup and compression/decompression in functions here rather than
* messy code in main program. * messy code in main program.
int int
rabin_compress(dedupe_context_t *ctx, uchar_t *from, int64_t fromlen, uchar_t *to, int64_t *tolen, rabin_compress(dedupe_context_t *ctx, uchar_t *from, uint64_t fromlen, uchar_t *to, uint64_t *tolen,
int level, char chdr, void *data, compress_func_ptr cmp) int level, char chdr, void *data, compress_func_ptr cmp)
{ {
} }

18
rabin/rabin_dedup.h
View file

@ -89,7 +89,7 @@
// Header for a chunk deduped using Rabin // Header for a chunk deduped using Rabin
// Number of rabin blocks, size of original data chunk, size of compressed index, // Number of rabin blocks, size of original data chunk, size of compressed index,
// size of deduped data, size of compressed data // size of deduped data, size of compressed data
#define RABIN_HDR_SIZE (sizeof (unsigned int) + sizeof (int64_t) + sizeof (int64_t) + sizeof (int64_t) + sizeof (int64_t)) #define RABIN_HDR_SIZE (sizeof (unsigned int) + sizeof (uint64_t) + sizeof (uint64_t) + sizeof (uint64_t) + sizeof (uint64_t))
// Maximum number of dedup blocks supported (2^30 - 1) // Maximum number of dedup blocks supported (2^30 - 1)
#define RABIN_MAX_BLOCKS (0x3FFFFFFFUL) #define RABIN_MAX_BLOCKS (0x3FFFFFFFUL)
@ -135,7 +135,7 @@
#define FP_POLY 0xbfe6b8a5bf378d83ULL #define FP_POLY 0xbfe6b8a5bf378d83ULL
typedef struct rab_blockentry { typedef struct rab_blockentry {
int64_t offset; uint64_t offset;
uint32_t similarity_hash; uint32_t similarity_hash;
uint32_t hash; uint32_t hash;
uint32_t index; uint32_t index;
@ -167,13 +167,13 @@ extern dedupe_context_t *create_dedupe_context(uint64_t chunksize, uint64_t real
compress_op_t op); compress_op_t op);
extern void destroy_dedupe_context(dedupe_context_t *ctx); extern void destroy_dedupe_context(dedupe_context_t *ctx);
extern unsigned int dedupe_compress(dedupe_context_t *ctx, unsigned char *buf, extern unsigned int dedupe_compress(dedupe_context_t *ctx, unsigned char *buf,
int64_t *size, int64_t offset, int64_t *rabin_pos); uint64_t *size, uint64_t offset, uint64_t *rabin_pos);
extern void dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size); extern void dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size);
extern void parse_dedupe_hdr(uchar_t *buf, unsigned int *blknum, int64_t *dedupe_index_sz, extern void parse_dedupe_hdr(uchar_t *buf, unsigned int *blknum, uint64_t *dedupe_index_sz,
int64_t *dedupe_data_sz, int64_t *rabin_index_sz_cmp, uint64_t *dedupe_data_sz, uint64_t *rabin_index_sz_cmp,
int64_t *dedupe_data_sz_cmp, int64_t *deduped_size); uint64_t *dedupe_data_sz_cmp, uint64_t *deduped_size);
extern void update_dedupe_hdr(uchar_t *buf, int64_t dedupe_index_sz_cmp, extern void update_dedupe_hdr(uchar_t *buf, uint64_t dedupe_index_sz_cmp,
int64_t dedupe_data_sz_cmp); uint64_t dedupe_data_sz_cmp);
extern void reset_dedupe_context(dedupe_context_t *ctx); extern void reset_dedupe_context(dedupe_context_t *ctx);
extern uint32_t dedupe_buf_extra(uint64_t chunksize, int rab_blk_sz, const char *algo, extern uint32_t dedupe_buf_extra(uint64_t chunksize, int rab_blk_sz, const char *algo,
int delta_flag); int delta_flag);

2
utils/cpuid.h
View file

@ -55,6 +55,8 @@ struct cpu_raw_data_t {
void exec_cpuid(uint32_t *regs); void exec_cpuid(uint32_t *regs);
void cpuid_get_raw_data(struct cpu_raw_data_t* data); void cpuid_get_raw_data(struct cpu_raw_data_t* data);
void cpuid_basic_identify(processor_info_t *pc);
#endif /* __x86_64__ */ #endif /* __x86_64__ */
#endif /* __CPUID_H__ */ #endif /* __CPUID_H__ */

33
utils/utils.c
View file

@ -117,7 +117,7 @@ raise_by_multiplier(int64_t *val, int mult, int power) {
int int
parse_numeric(int64_t *val, const char *str) parse_numeric(int64_t *val, const char *str)
{ {
int i, ovr; int ovr = 2;
char *mult; char *mult;
*val = strtoll(str, &mult, 0); *val = strtoll(str, &mult, 0);
@ -184,7 +184,6 @@ Read(int fd, void *buf, uint64_t count)
{ {
int64_t rcount, rem; int64_t rcount, rem;
uchar_t *cbuf; uchar_t *cbuf;
va_list args;
rem = count; rem = count;
cbuf = (uchar_t *)buf; cbuf = (uchar_t *)buf;
@ -207,23 +206,29 @@ Read(int fd, void *buf, uint64_t count)
int64_t int64_t
Read_Adjusted(int fd, uchar_t *buf, uint64_t count, int64_t *rabin_count, void *ctx) Read_Adjusted(int fd, uchar_t *buf, uint64_t count, int64_t *rabin_count, void *ctx)
{ {
char *buf2; uchar_t *buf2;
int64_t rcount; int64_t rcount;
dedupe_context_t *rctx = (dedupe_context_t *)ctx; dedupe_context_t *rctx = (dedupe_context_t *)ctx;
if (!ctx) return (Read(fd, buf, count)); if (!ctx) return (Read(fd, buf, count));
buf2 = buf; buf2 = buf;
if (*rabin_count) { if (*rabin_count) {
buf2 = (char *)buf + *rabin_count; buf2 = buf + *rabin_count;
count -= *rabin_count; count -= *rabin_count;
} }
rcount = Read(fd, buf2, count); rcount = Read(fd, buf2, count);
if (rcount > 0) { if (rcount > 0) {
rcount += *rabin_count; rcount += *rabin_count;
if (rcount == count) if (rcount == count) {
dedupe_compress(rctx, buf, &rcount, 0, rabin_count); uint64_t rc, rbc;
else rc = rcount;
rbc = *rabin_count;
dedupe_compress(rctx, buf, &rc, 0, &rbc);
rcount = rc;
*rabin_count = rbc;
} else {
*rabin_count = 0; *rabin_count = 0;
}
} else { } else {
if (rcount == 0) rcount = *rabin_count; if (rcount == 0) rcount = *rabin_count;
*rabin_count = 0; *rabin_count = 0;
@ -248,6 +253,20 @@ Write(int fd, const void *buf, uint64_t count)
return (count - rem); return (count - rem);
} }
void
init_algo_props(algo_props_t *props)
{
props->buf_extra = 0;
props->compress_mt_capable = 0;
props->decompress_mt_capable = 0;
props->single_chunk_mt_capable = 0;
props->is_single_chunk = 0;
props->nthreads = 1;
props->c_max_threads = 1;
props->d_max_threads = 1;
props->delta2_span = 0;
}
/* /*
* Thread sizing. We want a balanced combination of chunk threads and compression * Thread sizing. We want a balanced combination of chunk threads and compression
* algorithm threads that best fit the available/allowed number of processors. * algorithm threads that best fit the available/allowed number of processors.

45
utils/utils.h
View file

@ -25,9 +25,15 @@
#define _UTILS_H #define _UTILS_H
#include <arpa/nameser_compat.h> #include <arpa/nameser_compat.h>
#include <arpa/inet.h>
#include <sys/types.h> #include <sys/types.h>
#include <stdint.h>
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS 1
#endif
#include <inttypes.h> #include <inttypes.h>
#include <stdint.h>
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -147,6 +153,7 @@ extern void set_threadcounts(algo_props_t *props, int *nthreads, int nprocs,
extern uint64_t get_total_ram(); extern uint64_t get_total_ram();
extern double get_wtime_millis(void); extern double get_wtime_millis(void);
extern double get_mb_s(uint64_t bytes, double strt, double en); extern double get_mb_s(uint64_t bytes, double strt, double en);
extern void init_algo_props(algo_props_t *props);
/* Pointer type for compress and decompress functions. */ /* Pointer type for compress and decompress functions. */
typedef int (*compress_func_ptr)(void *src, uint64_t srclen, void *dst, typedef int (*compress_func_ptr)(void *src, uint64_t srclen, void *dst,
@ -158,11 +165,11 @@ typedef enum {
} compress_op_t; } compress_op_t;
/* Pointer type for algo specific init/deinit/stats functions. */ /* Pointer type for algo specific init/deinit/stats functions. */
typedef int (*init_func_ptr)(void **data, int *level, int nthreads, int64_t chunksize, typedef int (*init_func_ptr)(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op); int file_version, compress_op_t op);
typedef int (*deinit_func_ptr)(void **data); typedef int (*deinit_func_ptr)(void **data);
typedef void (*stats_func_ptr)(int show); typedef void (*stats_func_ptr)(int show);
typedef void (*props_func_ptr)(algo_props_t *data, int level, int64_t chunksize); typedef void (*props_func_ptr)(algo_props_t *data, int level, uint64_t chunksize);
/* /*
@ -181,38 +188,6 @@ roundup_pow_two(unsigned int v) {
return (v); return (v);
} }
/*
* Hash function for 64Bit pointers/numbers that generates
* a 32Bit hash value.
* Taken from Thomas Wang's Integer hashing paper:
* http://www.cris.com/~Ttwang/tech/inthash.htm
*/
static uint32_t
hash6432shift(uint64_t key)
{
key = (~key) + (key << 18); // key = (key << 18) - key - 1;
key = key ^ (key >> 31);
key = key * 21; // key = (key + (key << 2)) + (key << 4);
key = key ^ (key >> 11);
key = key + (key << 6);
key = key ^ (key >> 22);
return (uint32_t) key;
}
static void
init_algo_props(algo_props_t *props)
{
props->buf_extra = 0;
props->compress_mt_capable = 0;
props->decompress_mt_capable = 0;
props->single_chunk_mt_capable = 0;
props->is_single_chunk = 0;
props->nthreads = 1;
props->c_max_threads = 1;
props->d_max_threads = 1;
props->delta2_span = 0;
}
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

17
zlib_compress.c
View file

@ -46,7 +46,7 @@ slab_alloc_ui(void *p, unsigned int items, unsigned int size) {
} }
uint32_t uint32_t
zlib_buf_extra(int64_t buflen) zlib_buf_extra(uint64_t buflen)
{ {
if (buflen > SINGLE_CALL_MAX) if (buflen > SINGLE_CALL_MAX)
buflen = SINGLE_CALL_MAX; buflen = SINGLE_CALL_MAX;
@ -54,13 +54,13 @@ zlib_buf_extra(int64_t buflen)
} }
int int
zlib_init(void **data, int *level, int nthreads, int64_t chunksize, zlib_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op) int file_version, compress_op_t op)
{ {
z_stream *zs; z_stream *zs;
int ret; int ret;
zs = slab_alloc(NULL, sizeof (z_stream)); zs = (z_stream *)slab_alloc(NULL, sizeof (z_stream));
zs->zalloc = slab_alloc_ui; zs->zalloc = slab_alloc_ui;
zs->zfree = slab_free; zs->zfree = slab_free;
zs->opaque = NULL; zs->opaque = NULL;
@ -90,7 +90,7 @@ zlib_stats(int show)
} }
void void
zlib_props(algo_props_t *data, int level, int64_t chunksize) { zlib_props(algo_props_t *data, int level, uint64_t chunksize) {
data->delta2_span = 100; data->delta2_span = 100;
} }
@ -102,6 +102,7 @@ zlib_deinit(void **data)
deflateEnd(zs); deflateEnd(zs);
slab_free(NULL, *data); slab_free(NULL, *data);
} }
return (0);
} }
static static
@ -144,8 +145,8 @@ zlib_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
unsigned int slen, dlen; unsigned int slen, dlen;
uint64_t _srclen = srclen; uint64_t _srclen = srclen;
uint64_t _dstlen = *dstlen; uint64_t _dstlen = *dstlen;
uchar_t *dst1 = dst; uchar_t *dst1 = (uchar_t *)dst;
uchar_t *src1 = src; uchar_t *src1 = (uchar_t *)src;
z_stream *zs = (z_stream *)data; z_stream *zs = (z_stream *)data;
ending = 0; ending = 0;
@ -207,8 +208,8 @@ zlib_decompress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
unsigned int slen, dlen; unsigned int slen, dlen;
uint64_t _srclen = srclen; uint64_t _srclen = srclen;
uint64_t _dstlen = *dstlen; uint64_t _dstlen = *dstlen;
uchar_t *dst1 = dst; uchar_t *dst1 = (uchar_t *)dst;
uchar_t *src1 = src; uchar_t *src1 = (uchar_t *)src;
z_stream *zs = (z_stream *)data; z_stream *zs = (z_stream *)data;
while (_srclen > 0) { while (_srclen > 0) {