greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Introduce strict compiler flags and fix scores of warnings/issues.

Browse source 
Avoid different optimization flags for Dedupe sources.
Fix liberal mixing of uint64_t and int64_t (should all be uint64_t).
Fix corner case crash when decompressing.
This commit is contained in:
Moinak Ghosh 2012-12-27 23:06:48 +05:30
parent 063624d0d3
commit 26a4f42506
38 changed files with 407 additions and 383 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
Makefile.in
View file

@ -139,7 +139,7 @@ KECCAK_OBJS_ASM = $(KECCAK_SRCS_ASM:.s=.o)
BAKFILES = *~ lzma/*~ lzfx/*~ lz4/*~ rabin/*~ bsdiff/*~ lzp/*~ utils/*~ crypto/sha2/*~ \
crypto/sha2/intel/*~ crypto/aes/*~ crypto/scrypt/*~ crypto/*~ rabin/global/*~ \
delta2/*~ crypto/keccak/*~ transpose/*~
delta2/*~ crypto/keccak/*~ transpose/*~ crypto/skein/*~ crypto/keccak/*.o
RM = rm -f
RM_RF = rm -rf
@ -147,7 +147,8 @@ COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT
-DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32 \
-I./lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I./crypto/sha2 \
-I./crypto/scrypt -I./crypto/aes -I./crypto @KEYLEN@ \
-I./crypto/keccak -I./transpose $(EXTRA_CPPFLAGS)
-I./crypto/keccak -I./transpose $(EXTRA_CPPFLAGS) -pedantic -Wall -Werror -std=gnu99 \
-fno-strict-aliasing -Wno-unused-but-set-variable -Wno-enum-compare
COMMON_VEC_FLAGS = -ftree-vectorize
COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block
LDLIBS = -ldl -L@LIBBZ2_DIR@ -lbz2 -L@LIBZ_DIR@ -lz -lm @LIBBSCLFLAGS@ \
@ -163,7 +164,6 @@ DEBUG_COMPILE_cpp = g++ -m64 -g -msse3 -c
DEBUG_VEC_FLAGS =
DEBUG_LOOP_OPTFLAGS =
DEBUG_GEN_OPT = -O -fno-omit-frame-pointer @LIBBSCGEN_OPT@
DEBUG_RABIN_OPT = -O -fno-omit-frame-pointer
DEBUG_CPPFLAGS = $(COMMON_CPPFLAGS)
DEBUG_FPTR_FLAG =
@ -174,7 +174,6 @@ RELEASE_VEC_FLAGS = $(COMMON_VEC_FLAGS)
RELEASE_LOOP_OPTFLAGS = $(COMMON_LOOP_OPTFLAGS)
RELEASE_CPPFLAGS = $(COMMON_CPPFLAGS) -DNDEBUG
RELEASE_GEN_OPT = -O3 @LIBBSCGEN_OPT@
RELEASE_RABIN_OPT = -O2
RELEASE_FPTR_FLAG = -fomit-frame-pointer
NO_SLAB_CPPFLAGS = -DDEBUG_NO_SLAB
@ -187,7 +186,6 @@ VEC_FLAGS = @VEC_FLAGS@
LOOP_OPTFLAGS = @LOOP_OPTFLAGS@
CPPFLAGS = @CPPFLAGS@ @NO_SLAB_CPPFLAGS@ @DEBUG_STATS_CPPFLAGS@
GEN_OPT = @GEN_OPT@
RABIN_OPT = @RABIN_OPT@
PREFIX=@PREFIX@
SKEIN_FLAGS = $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) @FPTR_FLAG@
@ -206,7 +204,7 @@ $(PPMDOBJS): $(PPMDSRCS) $(PPMDHDRS)
$(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
$(RABINOBJS): $(RABINSRCS) $(RABINHDRS)
$(COMPILE) $(RABIN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
$(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
$(BSDIFFOBJS): $(BSDIFFSRCS) $(BSDIFFHDRS)
$(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@

20
adaptive_compress.c
View file

@ -62,13 +62,13 @@ extern int ppmd_decompress(void *src, uint64_t srclen, void *dst,
extern int libbsc_decompress(void *src, uint64_t srclen, void *dst,
uint64_t *dstlen, int level, uchar_t chdr, void *data);
extern int lzma_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int lzma_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int lzma_deinit(void **data);
extern int ppmd_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int ppmd_deinit(void **data);
extern int libbsc_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int libbsc_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int libbsc_deinit(void **data);
@ -96,17 +96,17 @@ adapt_stats(int show)
}
void
adapt_props(algo_props_t *data, int level, int64_t chunksize)
adapt_props(algo_props_t *data, int level, uint64_t chunksize)
{
data->delta2_span = 200;
}
int
adapt_init(void **data, int *level, int nthreads, int64_t chunksize,
adapt_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
struct adapt_data *adat = (struct adapt_data *)(*data);
int rv;
int rv = 0;
if (!adat) {
adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data));
@ -125,11 +125,11 @@ adapt_init(void **data, int *level, int nthreads, int64_t chunksize,
}
int
adapt2_init(void **data, int *level, int nthreads, int64_t chunksize,
adapt2_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
struct adapt_data *adat = (struct adapt_data *)(*data);
int rv, lv;
int rv = 0, lv;
if (!adat) {
adat = (struct adapt_data *)slab_alloc(NULL, sizeof (struct adapt_data));
@ -160,7 +160,7 @@ int
adapt_deinit(void **data)
{
struct adapt_data *adat = (struct adapt_data *)(*data);
int rv;
int rv = 0;
if (adat) {
rv = ppmd_deinit(&(adat->ppmd_data));
@ -179,7 +179,7 @@ adapt_compress(void *src, uint64_t srclen, void *dst,
struct adapt_data *adat = (struct adapt_data *)(data);
uchar_t *src1 = (uchar_t *)src;
uint64_t i, tot8b, tagcnt;
int rv, tag;
int rv = 0, tag;
/*
* Count number of 8-bit binary bytes and XML tags in source.

27
allocator.c
View file

@ -94,17 +94,33 @@ static struct slabentry slabheads[NUM_SLABS];
static struct bufentry **htable;
static pthread_mutex_t *hbucket_locks;
static pthread_mutex_t htable_lock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t slab_table_lock = PTHREAD_MUTEX_INITIALIZER;
static int inited = 0, bypass = 0;
static uint64_t total_allocs, oversize_allocs, hash_collisions, hash_entries;
/*
* Hash function for 64Bit pointers/numbers that generates
* a 32Bit hash value.
* Taken from Thomas Wang's Integer hashing paper:
* http://www.cris.com/~Ttwang/tech/inthash.htm
*/
static uint32_t
hash6432shift(uint64_t key)
{
key = (~key) + (key << 18); // key = (key << 18) - key - 1;
key = key ^ (key >> 31);
key = key * 21; // key = (key + (key << 2)) + (key << 4);
key = key ^ (key >> 11);
key = key + (key << 6);
key = key ^ (key >> 22);
return (uint32_t) key;
}
void
slab_init()
{
int i;
uint64_t slab_sz;
int nprocs;
/* Check bypass env variable. */
if (getenv("ALLOCATOR_BYPASS") != NULL) {
@ -278,7 +294,7 @@ slab_calloc(void *p, uint64_t items, uint64_t size) {
static unsigned int
find_slot(unsigned int v)
{
unsigned int r, i;
unsigned int r;
/* Round up to nearest power of 2 */
v = roundup_pow_two(v);
@ -304,7 +320,7 @@ find_slot(unsigned int v)
return (r);
}
static void *
static struct slabentry *
try_dynamic_slab(uint64_t size)
{
uint32_t sindx;
@ -361,10 +377,7 @@ slab_cache_add(uint64_t size)
void *
slab_alloc(void *p, uint64_t size)
{
uint64_t slab_sz = SLAB_START_SZ;
int i;
uint64_t div;
void *ptr;
struct slabentry *slab;
if (bypass) return (malloc(size));

12
bsdiff/bscommon.h
View file

@ -64,6 +64,8 @@ BUFOPEN(bufio_t *bio, uchar_t *buf, bsize_t len)
bio->buf = buf; bio->pos = 0; bio->buflen = len;
return (0);
}
#ifndef __IN_BSPATCH__
static bsize_t
BUFWRITE(bufio_t *bio, uchar_t *buf, bsize_t len)
{
@ -75,12 +77,13 @@ BUFWRITE(bufio_t *bio, uchar_t *buf, bsize_t len)
return (-1);
}
}
#endif
#ifndef __IN_BSDIFF__
static bsize_t
BUFREAD(bufio_t *bio, uchar_t *buf, bsize_t len)
{
bsize_t actual;
int i;
actual = len;
if (bio->pos + len > bio->buflen) {
@ -91,7 +94,9 @@ int i;
bio->pos += actual;
return (actual);
}
#endif
#ifndef __IN_BSPATCH__
static bsize_t
BUFTELL(bufio_t *bio)
{
@ -123,9 +128,10 @@ BUFSEEK(bufio_t *bio, bsize_t pos, int typ)
}
return (0);
}
#endif
extern int zero_rle_encode(const void *const ibuf, const unsigned int ilen,
void *obuf, unsigned int *const olen);
extern int zero_rle_encode(const void *ibuf, const unsigned int ilen,
void *obuf, unsigned int *olen);
extern int zero_rle_decode(const void* ibuf, unsigned int ilen,
void* obuf, unsigned int *olen);

70
bsdiff/bsdiff.c
View file

@ -66,6 +66,7 @@ __FBSDID("$FreeBSD: src/usr.bin/bsdiff/bsdiff/bsdiff.c,v 1.1 2005/08/06 01:59:05
#include <emmintrin.h>
#endif
#define __IN_BSDIFF__
#include "bscommon.h"
#define MIN(x,y) (((x)<(y)) ? (x) : (y))
@ -131,7 +132,7 @@ static void split(bsize_t *I,bsize_t *V,bsize_t start,bsize_t len,bsize_t h)
if(start+len>kk) split(I,V,kk,start+len-kk,h);
}
static void qsufsort(bsize_t *I,bsize_t *V,u_char *old,bsize_t oldsize)
static void qsufsort(bsize_t *I,bsize_t *V,u_char *oldbuf,bsize_t oldsize)
{
bsize_t buckets[256];
bsize_t i,h,len;
@ -158,14 +159,14 @@ static void qsufsort(bsize_t *I,bsize_t *V,u_char *old,bsize_t oldsize)
#ifdef __USE_SSE_INTRIN__
}
#endif
for(i=0;i<oldsize;i++) buckets[old[i]]++;
for(i=0;i<oldsize;i++) buckets[oldbuf[i]]++;
for(i=1;i<256;i++) buckets[i]+=buckets[i-1];
for(i=255;i>0;i--) buckets[i]=buckets[i-1];
buckets[0]=0;
for(i=0;i<oldsize;i++) I[++buckets[old[i]]]=i;
for(i=0;i<oldsize;i++) I[++buckets[oldbuf[i]]]=i;
I[0]=oldsize;
for(i=0;i<oldsize;i++) V[i]=buckets[old[i]];
for(i=0;i<oldsize;i++) V[i]=buckets[oldbuf[i]];
V[oldsize]=0;
for(i=1;i<256;i++) if(buckets[i]==buckets[i-1]+1) I[buckets[i]]=-1;
I[0]=-1;
@ -190,24 +191,24 @@ static void qsufsort(bsize_t *I,bsize_t *V,u_char *old,bsize_t oldsize)
for(i=0;i<oldsize+1;i++) I[V[i]]=i;
}
static bsize_t matchlen(u_char *old,bsize_t oldsize,u_char *new,bsize_t newsize)
static bsize_t matchlen(u_char *oldbuf,bsize_t oldsize,u_char *newbuf,bsize_t newsize)
{
bsize_t i;
for(i=0;(i<oldsize)&&(i<newsize);i++)
if(old[i]!=new[i]) break;
if(oldbuf[i]!=newbuf[i]) break;
return i;
}
static bsize_t search(bsize_t *I,u_char *old,bsize_t oldsize,
u_char *new,bsize_t newsize,bsize_t st,bsize_t en,bsize_t *pos)
static bsize_t search(bsize_t *I,u_char *oldbuf,bsize_t oldsize,
u_char *newbuf,bsize_t newsize,bsize_t st,bsize_t en,bsize_t *pos)
{
bsize_t x,y;
if(en-st<2) {
x=matchlen(old+I[st],oldsize-I[st],new,newsize);
y=matchlen(old+I[en],oldsize-I[en],new,newsize);
x=matchlen(oldbuf+I[st],oldsize-I[st],newbuf,newsize);
y=matchlen(oldbuf+I[en],oldsize-I[en],newbuf,newsize);
if(x>y) {
*pos=I[st];
@ -219,19 +220,13 @@ static bsize_t search(bsize_t *I,u_char *old,bsize_t oldsize,
};
x=st+(en-st)/2;
if(memcmp(old+I[x],new,MIN(oldsize-I[x],newsize))<0) {
return search(I,old,oldsize,new,newsize,x,en,pos);
if(memcmp(oldbuf+I[x],newbuf,MIN(oldsize-I[x],newsize))<0) {
return search(I,oldbuf,oldsize,newbuf,newsize,x,en,pos);
} else {
return search(I,old,oldsize,new,newsize,st,x,pos);
return search(I,oldbuf,oldsize,newbuf,newsize,st,x,pos);
};
}
static void
valout(bsize_t x, u_char *buf)
{
*((bsize_t *)buf) = htonll(x);
}
static void
valouti32(bsize_t x, u_char *buf)
{
@ -241,7 +236,7 @@ valouti32(bsize_t x, u_char *buf)
}
bsize_t
bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
bsdiff(u_char *oldbuf, bsize_t oldsize, u_char *newbuf, bsize_t newsize,
u_char *diff, u_char *scratch, bsize_t scratchsize)
{
bsize_t *I,*V;
@ -259,15 +254,15 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
bufio_t pf;
sz = sizeof (bsize_t);
I = slab_alloc(NULL, (oldsize+1)*sz);
V = slab_alloc(NULL, (oldsize+1)*sz);
I = (bsize_t *)slab_alloc(NULL, (oldsize+1)*sz);
V = (bsize_t *)slab_alloc(NULL, (oldsize+1)*sz);
if(I == NULL || V == NULL) return (0);
qsufsort(I,V,old,oldsize);
qsufsort(I,V,oldbuf,oldsize);
slab_free(NULL, V);
if(((db=slab_alloc(NULL, newsize+1))==NULL) ||
((eb=slab_alloc(NULL, newsize+1))==NULL)) {
if(((db=(u_char *)slab_alloc(NULL, newsize+1))==NULL) ||
((eb=(u_char *)slab_alloc(NULL, newsize+1))==NULL)) {
fprintf(stderr, "bsdiff: Memory allocation error.\n");
slab_free(NULL, I);
slab_free(NULL, V);
@ -307,30 +302,31 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
/* Compute the differences, writing ctrl as we go */
scan=0;len=0;
lastscan=0;lastpos=0;lastoffset=0;
pos=0;
while(scan<newsize) {
oldscore=0;
for(scsc=scan+=len;scan<newsize;scan++) {
len=search(I,old,oldsize,new+scan,newsize-scan,
len=search(I,oldbuf,oldsize,newbuf+scan,newsize-scan,
0,oldsize,&pos);
for(;scsc<scan+len;scsc++)
if((scsc+lastoffset<oldsize) &&
(old[scsc+lastoffset] == new[scsc]))
(oldbuf[scsc+lastoffset] == newbuf[scsc]))
oldscore++;
if(((len==oldscore) && (len!=0)) ||
(len>oldscore+sz)) break;
if((scan+lastoffset<oldsize) &&
(old[scan+lastoffset] == new[scan]))
(oldbuf[scan+lastoffset] == newbuf[scan]))
oldscore--;
};
if((len!=oldscore) || (scan==newsize)) {
s=0;Sf=0;lenf=0;
for(i=0;(lastscan+i<scan)&&(lastpos+i<oldsize);) {
if(old[lastpos+i]==new[lastscan+i]) s++;
if(oldbuf[lastpos+i]==newbuf[lastscan+i]) s++;
i++;
if(s*2-i>Sf*2-lenf) { Sf=s; lenf=i; };
};
@ -339,7 +335,7 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
if(scan<newsize) {
s=0;Sb=0;
for(i=1;(scan>=lastscan+i)&&(pos>=i);i++) {
if(old[pos-i]==new[scan-i]) s++;
if(oldbuf[pos-i]==newbuf[scan-i]) s++;
if(s*2-i>Sb*2-lenb) { Sb=s; lenb=i; };
};
};
@ -348,10 +344,10 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
overlap=(lastscan+lenf)-(scan-lenb);
s=0;Ss=0;lens=0;
for(i=0;i<overlap;i++) {
if(new[lastscan+lenf-overlap+i]==
old[lastpos+lenf-overlap+i]) s++;
if(new[scan-lenb+i]==
old[pos-lenb+i]) s--;
if(newbuf[lastscan+lenf-overlap+i]==
oldbuf[lastpos+lenf-overlap+i]) s++;
if(newbuf[scan-lenb+i]==
oldbuf[pos-lenb+i]) s--;
if(s>Ss) { Ss=s; lens=i+1; };
};
@ -360,9 +356,9 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
};
for(i=0;i<lenf;i++)
db[dblen+i]=new[lastscan+i]-old[lastpos+i];
db[dblen+i]=newbuf[lastscan+i]-oldbuf[lastpos+i];
for(i=0;i<(scan-lenb)-(lastscan+lenf);i++)
eb[eblen+i]=new[lastscan+lenf+i];
eb[eblen+i]=newbuf[lastscan+lenf+i];
dblen+=lenf;
eblen+=(scan-lenb)-(lastscan+lenf);
@ -391,7 +387,7 @@ bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
/* If our data can fit in the scratch area use it otherwise alloc. */
if (ulen > scratchsize) {
cb = slab_alloc(NULL, ulen);
cb = (u_char *)slab_alloc(NULL, ulen);
} else {
cb = scratch;
}

24
bsdiff/bspatch.c
View file

@ -37,13 +37,9 @@ __FBSDID("$FreeBSD: src/usr.bin/bsdiff/bspatch/bspatch.c,v 1.1 2005/08/06 01:59:
#include <fcntl.h>
#include <allocator.h>
#include <utils.h>
#include "bscommon.h"
static bsize_t
valin(u_char *buf)
{
return ntohll(*((bsize_t *)buf));
}
#define __IN_BSPATCH__
#include "bscommon.h"
static int32_t
valini32(u_char *buf)
@ -55,7 +51,7 @@ bsize_t
get_bsdiff_sz(u_char *pbuf) {
bsize_t newsize;
bsize_t lzctrllen, ctrllen, lzdatalen, datalen, lzextralen, extralen;
int hdrsz, rv;
int hdrsz;
hdrsz = 4*7;
@ -70,7 +66,7 @@ get_bsdiff_sz(u_char *pbuf) {
}
int
bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsize)
bspatch(u_char *pbuf, u_char *oldbuf, bsize_t oldsize, u_char *newbuf, bsize_t *_newsize)
{
bsize_t newsize;
bsize_t lzctrllen, ctrllen, lzdatalen, datalen, lzextralen, extralen;
@ -123,8 +119,8 @@ bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsi
*_newsize = newsize;
/* Allocate buffers. */
diffdata = slab_alloc(NULL, datalen);
extradata = slab_alloc(NULL, extralen);
diffdata = (u_char *)slab_alloc(NULL, datalen);
extradata = (u_char *)slab_alloc(NULL, extralen);
if (diffdata == NULL || extradata == NULL) {
fprintf(stderr, "bspatch: Out of memory.\n");
if (diffdata) slab_free(NULL, diffdata);
@ -135,7 +131,7 @@ bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsi
/* Decompress ctrldata, diffdata and extradata. */
if (lzctrllen < ctrllen) {
/* Ctrl data will be RLE-d if RLE size is less. */
ctrldata = slab_alloc(NULL, ctrllen);
ctrldata = (u_char *)slab_alloc(NULL, ctrllen);
if (ctrldata == NULL) {
fprintf(stderr, "bspatch: Out of memory.\n");
slab_free(NULL, diffdata);
@ -195,7 +191,7 @@ bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsi
}
/* Read diff string */
lenread = BUFREAD(&dpf, new + newpos, ctrl[0]);
lenread = BUFREAD(&dpf, newbuf + newpos, ctrl[0]);
if (lenread < ctrl[0]) {
fprintf(stderr, "4: Corrupt diff data\n");
rv = 0;
@ -205,7 +201,7 @@ bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsi
/* Add old data to diff string */
for(i=0;i<ctrl[0];i++)
if((oldpos+i>=0) && (oldpos+i<oldsize))
new[newpos+i]+=old[oldpos+i];
newbuf[newpos+i]+=oldbuf[oldpos+i];
/* Adjust pointers */
newpos+=ctrl[0];
@ -219,7 +215,7 @@ bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new, bsize_t *_newsi
}
/* Read extra string */
lenread = BUFREAD(&epf, new + newpos, ctrl[1]);
lenread = BUFREAD(&epf, newbuf + newpos, ctrl[1]);
if (lenread < ctrl[1]) {
fprintf(stderr, "6: Corrupt diff data\n");
rv = 0;

12
bsdiff/rle_encoder.c
View file

@ -30,13 +30,13 @@
#define COUNT_MAX (32767)
int
zero_rle_encode(const void *const ibuf, const unsigned int ilen,
void *obuf, unsigned int *const olen)
zero_rle_encode(const void *ibuf, const unsigned int ilen,
void *obuf, unsigned int *olen)
{
unsigned int pos1, pos2;
unsigned short count;
const uchar_t *const ib = ibuf;
uchar_t *ob = obuf;
const uchar_t *ib = (const uchar_t *)ibuf;
uchar_t *ob = (uchar_t *)obuf;
pos2 = 0;
for (pos1=0; pos1<ilen && pos2<*olen;) {
@ -83,8 +83,8 @@ zero_rle_decode(const void* ibuf, unsigned int ilen,
{
unsigned int pos1, pos2, i;
unsigned short count;
const uchar_t *ib = ibuf;
uchar_t *ob = obuf;
const uchar_t *ib = (const uchar_t *)ibuf;
uchar_t *ob = (uchar_t *)obuf;
pos2 = 0;
pos1 = 0;

12
bzip2_compress.c
View file

@ -49,12 +49,12 @@ bzip2_stats(int show)
}
void
bzip2_props(algo_props_t *data, int level, int64_t chunksize) {
bzip2_props(algo_props_t *data, int level, uint64_t chunksize) {
data->delta2_span = 200;
}
int
bzip2_init(void **data, int *level, int nthreads, int64_t chunksize,
bzip2_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
if (*level > 9) *level = 9;
@ -100,8 +100,8 @@ bzip2_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
unsigned int slen, dlen;
uint64_t _srclen = srclen;
uint64_t _dstlen = *dstlen;
uchar_t *dst1 = dst;
uchar_t *src1 = src;
char *dst1 = (char *)dst;
char *src1 = (char *)src;
bzs.bzalloc = slab_alloc_i;
bzs.bzfree = slab_free;
@ -169,8 +169,8 @@ bzip2_decompress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
unsigned int slen, dlen;
uint64_t _srclen = srclen;
uint64_t _dstlen = *dstlen;
uchar_t *dst1 = dst;
uchar_t *src1 = src;
char *dst1 = (char *)dst;
char *src1 = (char *)src;
bzs.bzalloc = slab_alloc_i;
bzs.bzfree = slab_free;

2
config
View file

@ -383,7 +383,6 @@ vecflagsvar="VEC_FLAGS"
loopoptflagsvar="LOOP_OPTFLAGS"
cppflagsvar="CPPFLAGS"
genoptvar="GEN_OPT"
rabinoptvar="RABIN_OPT"
noslabcppflagsvar="NO_SLAB_CPPFLAGS"
debugstatscppflagsvar="DEBUG_STATS_CPPFLAGS"
prefixvar="PREFIX"
@ -427,7 +426,6 @@ s#@${vecflagsvar}@#\\\$\\(${typ}_${vecflagsvar}\\)#g
s#@${loopoptflagsvar}@#\\\$\\(${typ}_${loopoptflagsvar}\\)#g
s#@${cppflagsvar}@#\\\$\\(${typ}_${cppflagsvar}\\)#g
s#@${genoptvar}@#\\\$\\(${typ}_${genoptvar}\\)#g
s#@${rabinoptvar}@#\\\$\\(${typ}_${rabinoptvar}\\)#g
s#@${fptr_flag_var}@#\\\$\\(${typ}_${fptr_flag_var}\\)#g
s#@${noslabcppflagsvar}@#${noslabcppflagsval}#g
s#@${debugstatscppflagsvar}@#${debugstatscppflagsval}#g

12
crypto/aes/crypto_aes.c
View file

@ -50,6 +50,10 @@
* online backup system.
*/
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS 1
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -106,7 +110,6 @@ int
aes_init(aes_ctx_t *ctx, uchar_t *salt, int saltlen, uchar_t *pwd, int pwd_len,
uint64_t nonce, int enc)
{
int rv;
struct timespec tp;
uint64_t tv;
uchar_t num[25];
@ -141,8 +144,9 @@ aes_init(aes_ctx_t *ctx, uchar_t *salt, int saltlen, uchar_t *pwd, int pwd_len,
} else {
tv = tp.tv_sec * 1000UL + tp.tv_nsec;
}
sprintf(num, "%" PRIu64, tv);
PKCS5_PBKDF2_HMAC(num, strlen(num), salt, saltlen, PBE_ROUNDS, EVP_sha256(), 32, IV);
sprintf((char *)num, "%" PRIu64, tv);
PKCS5_PBKDF2_HMAC((const char *)num, strlen((char *)num), salt,
saltlen, PBE_ROUNDS, EVP_sha256(), 32, IV);
ctx->nonce = lzma_crc64(IV, 32, 0) & 0xffffffff00000000ULL;
// Nullify stack components
memset(num, 0, 25);
@ -177,6 +181,7 @@ aes_encrypt(aes_ctx_t *ctx, uchar_t *plaintext, uchar_t *ciphertext, ssize_t len
crypto_aesctr_stream(strm, plaintext, ciphertext, len);
crypto_aesctr_free(strm);
memset(&key, 0, sizeof (key));
return (0);
}
int
@ -200,6 +205,7 @@ aes_decrypt(aes_ctx_t *ctx, uchar_t *ciphertext, uchar_t *plaintext, ssize_t len
crypto_aesctr_stream(strm, ciphertext, plaintext, len);
crypto_aesctr_free(strm);
memset(&key, 0, sizeof (key));
return (0);
}
uint64_t

95
crypto/crypto_utils.c
View file

@ -39,9 +39,9 @@
#include <sha256.h>
#include <crypto_aes.h>
#include <KeccakNISTInterface.h>
#include <cpuid.h>
#include "crypto_utils.h"
#include "cpuid.h"
#define PROVIDER_OPENSSL 0
#define PROVIDER_X64_OPT 1
@ -53,7 +53,7 @@ static int geturandom_bytes(uchar_t rbytes[32]);
*/
typedef void (*ckinit_func_ptr)(void);
static struct {
char *name;
const char *name;
cksum_t cksum_id;
int bytes, mac_bytes;
ckinit_func_ptr init_func;
@ -67,7 +67,7 @@ static struct {
{"KECCAK512", CKSUM_KECCAK512, 64, 64, NULL}
};
static int cksum_provider = PROVIDER_OPENSSL, ossl_inited = 0;
static int cksum_provider = PROVIDER_OPENSSL;
extern uint64_t lzma_crc64(const uint8_t *buf, uint64_t size, uint64_t crc);
extern uint64_t lzma_crc64_8bchk(const uint8_t *buf, uint64_t size,
@ -153,7 +153,7 @@ init_sha256(void)
* return it's properties.
*/
int
get_checksum_props(char *name, int *cksum, int *cksum_bytes, int *mac_bytes)
get_checksum_props(const char *name, int *cksum, int *cksum_bytes, int *mac_bytes)
{
int i;
@ -212,12 +212,12 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
mctx->mac_cksum = cksum;
if (cksum == CKSUM_SKEIN256) {
Skein_512_Ctxt_t *ctx = malloc(sizeof (Skein_512_Ctxt_t));
Skein_512_Ctxt_t *ctx = (Skein_512_Ctxt_t *)malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) return (-1);
Skein_512_InitExt(ctx, 256, SKEIN_CFG_TREE_INFO_SEQUENTIAL,
actx->pkey, KEYLEN);
mctx->mac_ctx = ctx;
ctx = malloc(sizeof (Skein_512_Ctxt_t));
ctx = (Skein_512_Ctxt_t *)malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) {
free(mctx->mac_ctx);
return (-1);
@ -226,12 +226,12 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
mctx->mac_ctx_reinit = ctx;
} else if (cksum == CKSUM_SKEIN512) {
Skein_512_Ctxt_t *ctx = malloc(sizeof (Skein_512_Ctxt_t));
Skein_512_Ctxt_t *ctx = (Skein_512_Ctxt_t *)malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) return (-1);
Skein_512_InitExt(ctx, 512, SKEIN_CFG_TREE_INFO_SEQUENTIAL,
actx->pkey, KEYLEN);
mctx->mac_ctx = ctx;
ctx = malloc(sizeof (Skein_512_Ctxt_t));
ctx = (Skein_512_Ctxt_t *)malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) {
free(mctx->mac_ctx);
return (-1);
@ -241,30 +241,30 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_CTX *ctx = malloc(sizeof (HMAC_CTX));
HMAC_CTX *ctx = (HMAC_CTX *)malloc(sizeof (HMAC_CTX));
if (!ctx) return (-1);
HMAC_CTX_init(ctx);
HMAC_Init_ex(ctx, actx->pkey, KEYLEN, EVP_sha256(), NULL);
mctx->mac_ctx = ctx;
ctx = malloc(sizeof (HMAC_CTX));
ctx = (HMAC_CTX *)malloc(sizeof (HMAC_CTX));
if (!ctx) {
free(mctx->mac_ctx);
return (-1);
}
if (!HMAC_CTX_copy(ctx, mctx->mac_ctx)) {
if (!HMAC_CTX_copy(ctx, (HMAC_CTX *)(mctx->mac_ctx))) {
free(ctx);
free(mctx->mac_ctx);
return (-1);
}
mctx->mac_ctx_reinit = ctx;
} else {
HMAC_SHA256_Context *ctx = malloc(sizeof (HMAC_SHA256_Context));
HMAC_SHA256_Context *ctx = (HMAC_SHA256_Context *)malloc(sizeof (HMAC_SHA256_Context));
if (!ctx) return (-1);
opt_HMAC_SHA256_Init(ctx, actx->pkey, KEYLEN);
mctx->mac_ctx = ctx;
ctx = malloc(sizeof (HMAC_SHA256_Context));
ctx = (HMAC_SHA256_Context *)malloc(sizeof (HMAC_SHA256_Context));
if (!ctx) {
free(mctx->mac_ctx);
return (-1);
@ -273,18 +273,18 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
mctx->mac_ctx_reinit = ctx;
}
} else if (cksum == CKSUM_SHA512) {
HMAC_CTX *ctx = malloc(sizeof (HMAC_CTX));
HMAC_CTX *ctx = (HMAC_CTX *)malloc(sizeof (HMAC_CTX));
if (!ctx) return (-1);
HMAC_CTX_init(ctx);
HMAC_Init_ex(ctx, actx->pkey, KEYLEN, EVP_sha512(), NULL);
mctx->mac_ctx = ctx;
ctx = malloc(sizeof (HMAC_CTX));
ctx = (HMAC_CTX *)malloc(sizeof (HMAC_CTX));
if (!ctx) {
free(mctx->mac_ctx);
return (-1);
}
if (!HMAC_CTX_copy(ctx, mctx->mac_ctx)) {
if (!HMAC_CTX_copy(ctx, (HMAC_CTX *)(mctx->mac_ctx))) {
free(ctx);
free(mctx->mac_ctx);
return (-1);
@ -292,7 +292,7 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
mctx->mac_ctx_reinit = ctx;
} else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) {
hashState *ctx = malloc(sizeof (hashState));
hashState *ctx = (hashState *)malloc(sizeof (hashState));
if (!ctx) return (-1);
if (cksum == CKSUM_KECCAK256) {
@ -306,7 +306,7 @@ hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
return (-1);
mctx->mac_ctx = ctx;
ctx = malloc(sizeof (hashState));
ctx = (hashState *)malloc(sizeof (hashState));
if (!ctx) {
free(mctx->mac_ctx);
return (-1);
@ -329,12 +329,13 @@ hmac_reinit(mac_ctx_t *mctx)
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_CTX_copy(mctx->mac_ctx, mctx->mac_ctx_reinit);
HMAC_CTX_copy((HMAC_CTX *)(mctx->mac_ctx),
(HMAC_CTX *)(mctx->mac_ctx_reinit));
} else {
memcpy(mctx->mac_ctx, mctx->mac_ctx_reinit, sizeof (HMAC_SHA256_Context));
}
} else if (cksum == CKSUM_SHA512) {
HMAC_CTX_copy(mctx->mac_ctx, mctx->mac_ctx_reinit);
HMAC_CTX_copy((HMAC_CTX *)(mctx->mac_ctx), (HMAC_CTX *)(mctx->mac_ctx_reinit));
} else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) {
memcpy(mctx->mac_ctx, mctx->mac_ctx_reinit, sizeof (hashState));
@ -350,17 +351,17 @@ hmac_update(mac_ctx_t *mctx, uchar_t *data, uint64_t len)
int cksum = mctx->mac_cksum;
if (cksum == CKSUM_SKEIN256 || cksum == CKSUM_SKEIN512) {
Skein_512_Update(mctx->mac_ctx, data, len);
Skein_512_Update((Skein_512_Ctxt_t *)(mctx->mac_ctx), data, len);
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) {
if (HMAC_Update(mctx->mac_ctx, data, len) == 0)
if (HMAC_Update((HMAC_CTX *)(mctx->mac_ctx), data, len) == 0)
return (-1);
} else {
opt_HMAC_SHA256_Update(mctx->mac_ctx, data, len);
opt_HMAC_SHA256_Update((HMAC_SHA256_Context *)(mctx->mac_ctx), data, len);
}
} else if (cksum == CKSUM_SHA512) {
if (HMAC_Update(mctx->mac_ctx, data, len) == 0)
if (HMAC_Update((HMAC_CTX *)(mctx->mac_ctx), data, len) == 0)
return (-1);
} else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) {
@ -369,11 +370,11 @@ hmac_update(mac_ctx_t *mctx, uchar_t *data, uint64_t len)
uint64_t blen;
blen = KECCAK_MAX_SEG;
if (Keccak_Update(mctx->mac_ctx, data, blen << 3) != 0)
if (Keccak_Update((hashState *)(mctx->mac_ctx), data, blen << 3) != 0)
return (-1);
len -= KECCAK_MAX_SEG;
}
if (Keccak_Update(mctx->mac_ctx, data, len << 3) != 0)
if (Keccak_Update((hashState *)(mctx->mac_ctx), data, len << 3) != 0)
return (-1);
} else {
return (-1);
@ -387,25 +388,25 @@ hmac_final(mac_ctx_t *mctx, uchar_t *hash, unsigned int *len)
int cksum = mctx->mac_cksum;
if (cksum == CKSUM_SKEIN256) {
Skein_512_Final(mctx->mac_ctx, hash);
Skein_512_Final((Skein_512_Ctxt_t *)(mctx->mac_ctx), hash);
*len = 32;
} else if (cksum == CKSUM_SKEIN512) {
Skein_512_Final(mctx->mac_ctx, hash);
Skein_512_Final((Skein_512_Ctxt_t *)(mctx->mac_ctx), hash);
*len = 64;
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_Final(mctx->mac_ctx, hash, len);
HMAC_Final((HMAC_CTX *)(mctx->mac_ctx), hash, len);
} else {
opt_HMAC_SHA256_Final(mctx->mac_ctx, hash);
opt_HMAC_SHA256_Final((HMAC_SHA256_Context *)(mctx->mac_ctx), hash);
*len = 32;
}
} else if (cksum == CKSUM_SHA512) {
HMAC_Final(mctx->mac_ctx, hash, len);
HMAC_Final((HMAC_CTX *)(mctx->mac_ctx), hash, len);
} else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) {
if (Keccak_Final(mctx->mac_ctx, hash) != 0)
if (Keccak_Final((hashState *)(mctx->mac_ctx), hash) != 0)
return (-1);
if (cksum == CKSUM_KECCAK256)
*len = 32;
@ -428,15 +429,15 @@ hmac_cleanup(mac_ctx_t *mctx)
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_CTX_cleanup(mctx->mac_ctx);
HMAC_CTX_cleanup(mctx->mac_ctx_reinit);
HMAC_CTX_cleanup((HMAC_CTX *)(mctx->mac_ctx));
HMAC_CTX_cleanup((HMAC_CTX *)(mctx->mac_ctx_reinit));
} else {
memset(mctx->mac_ctx, 0, sizeof (HMAC_SHA256_Context));
memset(mctx->mac_ctx_reinit, 0, sizeof (HMAC_SHA256_Context));
}
} else if (cksum == CKSUM_SHA512) {
HMAC_CTX_cleanup(mctx->mac_ctx);
HMAC_CTX_cleanup(mctx->mac_ctx_reinit);
HMAC_CTX_cleanup((HMAC_CTX *)(mctx->mac_ctx));
HMAC_CTX_cleanup((HMAC_CTX *)(mctx->mac_ctx_reinit));
} else if (cksum == CKSUM_KECCAK256 || cksum == CKSUM_KECCAK512) {
memset(mctx->mac_ctx, 0, sizeof (hashState));
@ -455,13 +456,13 @@ init_crypto(crypto_ctx_t *cctx, uchar_t *pwd, int pwd_len, int crypto_alg,
uchar_t *salt, int saltlen, uint64_t nonce, int enc_dec)
{
if (crypto_alg == CRYPTO_ALG_AES) {
aes_ctx_t *actx = malloc(sizeof (aes_ctx_t));
aes_ctx_t *actx = (aes_ctx_t *)malloc(sizeof (aes_ctx_t));
if (enc_dec) {
/*
* Encryption init.
*/
cctx->salt = malloc(32);
cctx->salt = (uchar_t *)malloc(32);
salt = cctx->salt;
cctx->saltlen = 32;
if (RAND_status() != 1 || RAND_bytes(salt, 32) != 1) {
@ -509,7 +510,7 @@ init_crypto(crypto_ctx_t *cctx, uchar_t *pwd, int pwd_len, int crypto_alg,
MAX_SALTLEN);
return (-1);
}
cctx->salt = malloc(saltlen);
cctx->salt = (uchar_t *)malloc(saltlen);
memcpy(cctx->salt, salt, saltlen);
if (aes_init(actx, cctx->salt, saltlen, pwd, pwd_len, nonce,
@ -533,9 +534,9 @@ crypto_buf(crypto_ctx_t *cctx, uchar_t *from, uchar_t *to, int64_t bytes, uint64
{
if (cctx->crypto_alg == CRYPTO_ALG_AES) {
if (cctx->enc_dec == ENCRYPT_FLAG) {
return (aes_encrypt(cctx->crypto_ctx, from, to, bytes, id));
return (aes_encrypt((aes_ctx_t *)(cctx->crypto_ctx), from, to, bytes, id));
} else {
return (aes_decrypt(cctx->crypto_ctx, from, to, bytes, id));
return (aes_decrypt((aes_ctx_t *)(cctx->crypto_ctx), from, to, bytes, id));
}
} else {
fprintf(stderr, "Unrecognized algorithm code: %d\n", cctx->crypto_alg);
@ -547,19 +548,19 @@ crypto_buf(crypto_ctx_t *cctx, uchar_t *from, uchar_t *to, int64_t bytes, uint64
uint64_t
crypto_nonce(crypto_ctx_t *cctx)
{
return (aes_nonce(cctx->crypto_ctx));
return (aes_nonce((aes_ctx_t *)(cctx->crypto_ctx)));
}
void
crypto_clean_pkey(crypto_ctx_t *cctx)
{
aes_clean_pkey(cctx->crypto_ctx);
aes_clean_pkey((aes_ctx_t *)(cctx->crypto_ctx));
}
void
cleanup_crypto(crypto_ctx_t *cctx)
{
aes_cleanup(cctx->crypto_ctx);
aes_cleanup((aes_ctx_t *)(cctx->crypto_ctx));
memset(cctx->salt, 0, 32);
free(cctx->salt);
free(cctx);
@ -607,12 +608,12 @@ err0:
}
int
get_pw_string(char pw[MAX_PW_LEN], char *prompt, int twice)
get_pw_string(uchar_t pw[MAX_PW_LEN], const char *prompt, int twice)
{
int fd, len;
FILE *input, *strm;
struct termios oldt, newt;
uchar_t pw1[MAX_PW_LEN], pw2[MAX_PW_LEN], *s;
char pw1[MAX_PW_LEN], pw2[MAX_PW_LEN], *s;
// Try TTY first
fd = open("/dev/tty", O_RDWR | O_NOCTTY);
@ -667,7 +668,7 @@ get_pw_string(char pw[MAX_PW_LEN], char *prompt, int twice)
len = strlen(pw1);
pw1[len-1] = '\0';
strcpy(pw, pw1);
strcpy((char *)pw, (const char *)pw1);
memset(pw1, 0, MAX_PW_LEN);
memset(pw2, 0, MAX_PW_LEN);
return (len);

5
crypto/crypto_utils.h
View file

@ -75,7 +75,7 @@ typedef struct {
* Generic message digest functions.
*/
int compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, int64_t bytes);
int get_checksum_props(char *name, int *cksum, int *cksum_bytes, int *mac_bytes);
int get_checksum_props(const char *name, int *cksum, int *cksum_bytes, int *mac_bytes);
void serialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes);
void deserialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes);
@ -88,7 +88,7 @@ int crypto_buf(crypto_ctx_t *cctx, uchar_t *from, uchar_t *to, int64_t bytes, ui
uint64_t crypto_nonce(crypto_ctx_t *cctx);
void crypto_clean_pkey(crypto_ctx_t *cctx);
void cleanup_crypto(crypto_ctx_t *cctx);
int get_pw_string(char pw[MAX_PW_LEN], char *prompt, int twice);
int get_pw_string(uchar_t pw[MAX_PW_LEN], const char *prompt, int twice);
/*
* HMAC functions.
@ -96,6 +96,7 @@ int get_pw_string(char pw[MAX_PW_LEN], char *prompt, int twice);
int hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx);
int hmac_reinit(mac_ctx_t *mctx);
int hmac_update(mac_ctx_t *mctx, uchar_t *data, uint64_t len);
int hmac_final(mac_ctx_t *mctx, uchar_t *hash, unsigned int *len);
int hmac_cleanup(mac_ctx_t *mctx);
#ifdef __cplusplus

8
crypto/keccak/KeccakNISTInterface.c
View file

@ -43,14 +43,14 @@ HashReturn Keccak_Init(hashState *state, int hashbitlen)
HashReturn Keccak_Update(hashState *state, const BitSequence *data, DataLength databitlen)
{
if ((databitlen % 8) == 0)
return Absorb((spongeState*)state, data, databitlen);
return (HashReturn)Absorb((spongeState*)state, data, databitlen);
else {
HashReturn ret = Absorb((spongeState*)state, data, databitlen - (databitlen % 8));
HashReturn ret = (HashReturn)Absorb((spongeState*)state, data, databitlen - (databitlen % 8));
if (ret == SUCCESS) {
unsigned char lastByte;
// Align the last partial byte to the least significant bits
lastByte = data[databitlen/8] >> (8 - (databitlen % 8));
return Absorb((spongeState*)state, &lastByte, databitlen % 8);
return (HashReturn)Absorb((spongeState*)state, &lastByte, databitlen % 8);
}
else
return ret;
@ -59,7 +59,7 @@ HashReturn Keccak_Update(hashState *state, const BitSequence *data, DataLength d
HashReturn Keccak_Final(hashState *state, BitSequence *hashval)
{
return Squeeze(state, hashval, state->fixedOutputLength);
return (HashReturn)Squeeze(state, hashval, state->fixedOutputLength);
}
HashReturn Keccak_Hash(int hashbitlen, const BitSequence *data, DataLength databitlen, BitSequence *hashval)

50
crypto/keccak/genKAT.c
View file

@ -36,8 +36,8 @@ STATUS_CODES genShortMsgSponge(unsigned int rate, unsigned int capacity, int
STATUS_CODES genDuplexKAT(unsigned int rate, unsigned int capacity, const char *fileName);
#endif
int FindMarker(FILE *infile, const char *marker);
int ReadHex(FILE *infile, BitSequence *A, int Length, char *str);
void fprintBstr(FILE *fp, char *S, BitSequence *A, int L);
int ReadHex(FILE *infile, BitSequence *A, int Length, const char *str);
void fprintBstr(FILE *fp, const char *S, BitSequence *A, int L);
STATUS_CODES
@ -139,7 +139,7 @@ genShortMsg(int hashbitlen)
printf("ERROR: unable to read 'Msg' from <ShortMsgKAT.txt>\n");
return KAT_DATA_ERROR;
}
Hash(hashbitlen, Msg, msglen, MD);
Keccak_Hash(hashbitlen, Msg, msglen, MD);
fprintf(fp_out, "\nLen = %d\n", msglen);
fprintBstr(fp_out, "Msg = ", Msg, msgbytelen);
fprintBstr(fp_out, "MD = ", MD, hashbitlen/8);
@ -282,15 +282,15 @@ genLongMsg(int hashbitlen)
}
#ifdef AllowExtendedFunctions
if (hashbitlen > 0)
Hash(hashbitlen, Msg, msglen, MD);
Keccak_Hash(hashbitlen, Msg, msglen, MD);
else {
Init(&state, hashbitlen);
Update(&state, Msg, msglen);
Final(&state, 0);
Keccak_Init(&state, hashbitlen);
Keccak_Update(&state, Msg, msglen);
Keccak_Final(&state, 0);
Squeeze(&state, Squeezed, SqueezingOutputLength);
}
#else
Hash(hashbitlen, Msg, msglen, MD);
Keccak_Hash(hashbitlen, Msg, msglen, MD);
#endif
fprintf(fp_out, "Len = %d\n", msglen);
fprintBstr(fp_out, "Msg = ", Msg, msgbytelen);
@ -368,17 +368,17 @@ genExtremelyLongMsg(int hashbitlen)
// memcpy(Text, "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmno", 64);
if ( (retval = Init(&state, hashbitlen)) != KAT_SUCCESS ) {
printf("Init returned <%d> in genExtremelyLongMsg\n", retval);
if ( (retval = Keccak_Init(&state, hashbitlen)) != KAT_SUCCESS ) {
printf("Keccak_Init returned <%d> in genExtremelyLongMsg\n", retval);
return KAT_HASH_ERROR;
}
for ( i=0; i<repeat; i++ )
if ( (retval = Update(&state, Text, 512)) != KAT_SUCCESS ) {
printf("Update returned <%d> in genExtremelyLongMsg\n", retval);
if ( (retval = Keccak_Update(&state, Text, 512)) != KAT_SUCCESS ) {
printf("Keccak_Update returned <%d> in genExtremelyLongMsg\n", retval);
return KAT_HASH_ERROR;
}
if ( (retval = Final(&state, MD)) != KAT_SUCCESS ) {
printf("Final returned <%d> in genExtremelyLongMsg\n", retval);
if ( (retval = Keccak_Final(&state, MD)) != KAT_SUCCESS ) {
printf("Keccak_Final returned <%d> in genExtremelyLongMsg\n", retval);
return KAT_HASH_ERROR;
}
#ifdef AllowExtendedFunctions
@ -449,7 +449,7 @@ genMonteCarlo(int hashbitlen)
fprintBstr(fp_out, "Seed = ", Seed, 128);
for ( j=0; j<100; j++ ) {
for ( i=0; i<1000; i++ ) {
Hash(hashbitlen, Msg, 1024, MD);
Keccak_Hash(hashbitlen, Msg, 1024, MD);
memcpy(Temp, Msg, 128-bytelen);
memcpy(Msg, MD, bytelen);
memcpy(Msg+bytelen, Temp, 128-bytelen);
@ -511,22 +511,22 @@ genMonteCarloSqueezing(int hashbitlen)
fprintBstr(fp_out, "Seed = ", Seed, 128);
if ( (retval = Init(&state, hashbitlen)) != KAT_SUCCESS ) {
printf("Init returned <%d> in genMonteCarloSqueezing\n", retval);
if ( (retval = Keccak_Init(&state, hashbitlen)) != KAT_SUCCESS ) {
printf("Keccak_Init returned <%d> in genMonteCarloSqueezing\n", retval);
return KAT_HASH_ERROR;
}
if ( (retval = Update(&state, Seed, 128*8)) != KAT_SUCCESS ) {
printf("Update returned <%d> in genMonteCarloSqueezing\n", retval);
if ( (retval = Keccak_Update(&state, Seed, 128*8)) != KAT_SUCCESS ) {
printf("Keccak_Update returned <%d> in genMonteCarloSqueezing\n", retval);
return KAT_HASH_ERROR;
}
if ( (retval = Final(&state, 0)) != KAT_SUCCESS ) {
printf("Final returned <%d> in genMonteCarloSqueezing\n", retval);
if ( (retval = Keccak_Final(&state, 0)) != KAT_SUCCESS ) {
printf("Keccak_Final returned <%d> in genMonteCarloSqueezing\n", retval);
return KAT_HASH_ERROR;
}
bytelen = 64;
for ( j=0; j<100; j++ ) {
for ( i=0; i<1000; i++ ) {
if ( (retval = Squeeze(&state, MD, bytelen*8)) != KAT_SUCCESS ) {
if ( (retval = (HashReturn)Squeeze(&state, MD, bytelen*8)) != KAT_SUCCESS ) {
printf("Squeeze returned <%d> in genMonteCarloSqueezing\n", retval);
return KAT_HASH_ERROR;
}
@ -634,7 +634,7 @@ FindMarker(FILE *infile, const char *marker)
// ALLOW TO READ HEXADECIMAL ENTRY (KEYS, DATA, TEXT, etc.)
//
int
ReadHex(FILE *infile, BitSequence *A, int Length, char *str)
ReadHex(FILE *infile, BitSequence *A, int Length, const char *str)
{
int i, ch, started;
BitSequence ich;
@ -664,6 +664,8 @@ ReadHex(FILE *infile, BitSequence *A, int Length, char *str)
ich = ch - 'A' + 10;
else if ( (ch >= 'a') && (ch <= 'f') )
ich = ch - 'a' + 10;
else
return 1;
for ( i=0; i<Length-1; i++ )
A[i] = (A[i] << 4) | (A[i+1] >> 4);
@ -676,7 +678,7 @@ ReadHex(FILE *infile, BitSequence *A, int Length, char *str)
}
void
fprintBstr(FILE *fp, char *S, BitSequence *A, int L)
fprintBstr(FILE *fp, const char *S, BitSequence *A, int L)
{
int i;

2
crypto/scrypt/crypto_aesctr.c
View file

@ -55,7 +55,7 @@ crypto_aesctr_init(AES_KEY * key, uint64_t nonce)
struct crypto_aesctr * stream;
/* Allocate memory. */
if ((stream = malloc(sizeof(struct crypto_aesctr))) == NULL)
if ((stream = (struct crypto_aesctr *)malloc(sizeof(struct crypto_aesctr))) == NULL)
goto err0;
/* Initialize values. */

14
crypto/scrypt/crypto_scrypt-nosse.c
View file

@ -26,6 +26,10 @@
* This file was originally written by Colin Percival as part of the Tarsnap
* online backup system.
*/
#define __STDC_LIMIT_MACROS 1
#include <stdint.h>
#include <sys/types.h>
#include <sys/mman.h>
@ -49,8 +53,8 @@ static void smix(uint8_t *, size_t, uint64_t, uint32_t *, uint32_t *);
static void
blkcpy(void * dest, void * src, size_t len)
{
size_t * D = dest;
size_t * S = src;
size_t * D = (size_t *)dest;
size_t * S = (size_t *)src;
size_t L = len / sizeof(size_t);
size_t i;
@ -61,8 +65,8 @@ blkcpy(void * dest, void * src, size_t len)
static void
blkxor(void * dest, void * src, size_t len)
{
size_t * D = dest;
size_t * S = src;
size_t * D = (size_t *)dest;
size_t * S = (size_t *)src;
size_t L = len / sizeof(size_t);
size_t i;
@ -155,7 +159,7 @@ blockmix_salsa8(uint32_t * Bin, uint32_t * Bout, uint32_t * X, size_t r)
static uint64_t
integerify(void * B, size_t r)
{
uint32_t * X = (void *)((uintptr_t)(B) + (2 * r - 1) * 64);
uint32_t * X = (uint32_t *)((uintptr_t)(B) + (2 * r - 1) * 64);
return (((uint64_t)(X[1]) << 32) + X[0]);
}

4
crypto/scrypt/sha256.c
View file

@ -252,7 +252,7 @@ SHA256_Update(SHA256_CTX * ctx, const void *in, size_t len)
{
uint32_t bitlen[2];
uint32_t r;
const unsigned char *src = in;
const unsigned char *src = (const unsigned char *)in;
/* Number of bytes left in the buffer from previous updates */
r = (ctx->count[1] >> 3) & 0x3f;
@ -313,7 +313,7 @@ HMAC_SHA256_Init(HMAC_SHA256_CTX * ctx, const void * _K, size_t Klen)
{
unsigned char pad[64];
unsigned char khash[32];
const unsigned char * K = _K;
const unsigned char * K = (const unsigned char *)_K;
size_t i;
/* If Klen > 64, the key is really SHA256(K). */

6
crypto/sha2/sha256.c
View file

@ -104,8 +104,6 @@ APS_NAMESPACE(Init_SHA) (processor_info_t *pc)
static void
_init (SHA256_Context *sc, const uint32_t iv[SHA256_HASH_WORDS])
{
int i;
/*
* SHA256_HASH_WORDS is 8, must be 8, cannot be anything but 8!
* So we unroll a loop here.
@ -132,7 +130,7 @@ APS_NAMESPACE(SHA256_Init) (SHA256_Context *sc)
void
APS_NAMESPACE(SHA256_Update) (SHA256_Context *sc, const void *vdata, size_t len)
{
const uint8_t *data = vdata;
const uint8_t *data = (const uint8_t *)vdata;
uint32_t bufferBytesLeft;
size_t bytesToCopy;
int rem;
@ -215,7 +213,7 @@ APS_NAMESPACE(HMAC_SHA256_Init) (HMAC_SHA256_Context * ctx, const void * _K, siz
{
unsigned char pad[64];
unsigned char khash[32];
const unsigned char * K = _K;
const unsigned char * K = (const unsigned char *)_K;
size_t i;
/* If Klen > 64, the key is really SHA256(K). */

18
crypto/skein/SHA3api_ref.c
View file

@ -25,18 +25,18 @@ HashReturn Init(hashState *state, int hashbitlen)
{
Skein_Assert(hashbitlen > 0,BAD_HASHLEN);
state->statebits = 64*SKEIN_256_STATE_WORDS;
return Skein_256_Init(&state->u.ctx_256,(size_t) hashbitlen);
return (HashReturn)Skein_256_Init(&state->u.ctx_256,(size_t) hashbitlen);
}
#endif
if (hashbitlen <= SKEIN_512_NIST_MAX_HASHBITS)
{
state->statebits = 64*SKEIN_512_STATE_WORDS;
return Skein_512_Init(&state->u.ctx_512,(size_t) hashbitlen);
return (HashReturn)Skein_512_Init(&state->u.ctx_512,(size_t) hashbitlen);
}
else
{
state->statebits = 64*SKEIN1024_STATE_WORDS;
return Skein1024_Init(&state->u.ctx1024,(size_t) hashbitlen);
return (HashReturn)Skein1024_Init(&state->u.ctx1024,(size_t) hashbitlen);
}
}
@ -52,9 +52,9 @@ HashReturn Update(hashState *state, const BitSequence *data, DataLength databitl
{
switch ((state->statebits >> 8) & 3)
{
case 2: return Skein_512_Update(&state->u.ctx_512,data,databitlen >> 3);
case 1: return Skein_256_Update(&state->u.ctx_256,data,databitlen >> 3);
case 0: return Skein1024_Update(&state->u.ctx1024,data,databitlen >> 3);
case 2: return (HashReturn)Skein_512_Update(&state->u.ctx_512,data,databitlen >> 3);
case 1: return (HashReturn)Skein_256_Update(&state->u.ctx_256,data,databitlen >> 3);
case 0: return (HashReturn)Skein1024_Update(&state->u.ctx1024,data,databitlen >> 3);
default: return FAIL;
}
}
@ -92,9 +92,9 @@ HashReturn Final(hashState *state, BitSequence *hashval)
Skein_Assert(state->statebits % 256 == 0 && (state->statebits-256) < 1024,FAIL);
switch ((state->statebits >> 8) & 3)
{
case 2: return Skein_512_Final(&state->u.ctx_512,hashval);
case 1: return Skein_256_Final(&state->u.ctx_256,hashval);
case 0: return Skein1024_Final(&state->u.ctx1024,hashval);
case 2: return (HashReturn)Skein_512_Final(&state->u.ctx_512,hashval);
case 1: return (HashReturn)Skein_256_Final(&state->u.ctx_256,hashval);
case 0: return (HashReturn)Skein1024_Final(&state->u.ctx1024,hashval);
default: return FAIL;
}
}

2
delta2/delta2.c
View file

@ -115,7 +115,7 @@ delta2_encode(uchar_t *src, uint64_t srclen, uchar_t *dst, uint64_t *dstlen, int
} else {
uchar_t *srcpos, *dstpos, *lastdst, *lastsrc, *dstend;
uint64_t slen, sz, dsz, pending;
int rem, lenc, transp_count, hdr_ovr;
int rem, lenc, hdr_ovr;
DEBUG_STAT_EN(double strt, en);
srcpos = src;

6
libbsc_compress.c
View file

@ -72,7 +72,7 @@ libbsc_stats(int show)
* when compressing entire file in a single chunk.
*/
void
libbsc_props(algo_props_t *data, int level, int64_t chunksize) {
libbsc_props(algo_props_t *data, int level, uint64_t chunksize) {
data->compress_mt_capable = 0;
data->decompress_mt_capable = 0;
data->single_chunk_mt_capable = 1;
@ -83,14 +83,14 @@ libbsc_props(algo_props_t *data, int level, int64_t chunksize) {
}
int
libbsc_init(void **data, int *level, int nthreads, int64_t chunksize,
libbsc_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
struct libbsc_params *bscdat;
int rv;
if (chunksize > BSC_MAX_CHUNK) {
fprintf(stderr, "Max allowed chunk size for LIBBSC is: %d \n",
fprintf(stderr, "Max allowed chunk size for LIBBSC is: %ld \n",
BSC_MAX_CHUNK);
return (1);
}

8
lz4/lz4hc.c
View file

@ -297,16 +297,16 @@ inline static int LZ4HC_Init (LZ4HC_Data_Structure* hc4, const BYTE* base)
}
inline static void* LZ4HC_Create (const BYTE* base)
inline static LZ4HC_Data_Structure* LZ4HC_Create (const BYTE* base)
{
void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure));
LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure *)ALLOCATOR(sizeof(LZ4HC_Data_Structure));
LZ4HC_Init (hc4, base);
return hc4;
}
inline static int LZ4HC_Free (void** LZ4HC_Data)
inline static int LZ4HC_Free (LZ4HC_Data_Structure** LZ4HC_Data)
{
FREEMEM(*LZ4HC_Data);
*LZ4HC_Data = NULL;
@ -686,7 +686,7 @@ int LZ4_compressHC(const char* source,
char* dest,
int isize)
{
void* ctx = LZ4HC_Create((const BYTE*)source);
LZ4HC_Data_Structure* ctx = LZ4HC_Create((const BYTE*)source);
int result = LZ4_compressHCCtx(ctx, source, dest, isize);
LZ4HC_Free (&ctx);

25
lz4_compress.c
View file

@ -44,7 +44,7 @@ lz4_stats(int show)
}
int
lz4_buf_extra(int64_t buflen)
lz4_buf_extra(uint64_t buflen)
{
if (buflen > LZ4_MAX_CHUNK)
buflen = LZ4_MAX_CHUNK;
@ -52,7 +52,7 @@ lz4_buf_extra(int64_t buflen)
}
void
lz4_props(algo_props_t *data, int level, int64_t chunksize) {
lz4_props(algo_props_t *data, int level, uint64_t chunksize) {
data->compress_mt_capable = 0;
data->decompress_mt_capable = 0;
data->buf_extra = lz4_buf_extra(chunksize);
@ -60,7 +60,7 @@ lz4_props(algo_props_t *data, int level, int64_t chunksize) {
}
int
lz4_init(void **data, int *level, int nthreads, int64_t chunksize,
lz4_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
struct lz4_params *lzdat;
@ -71,7 +71,7 @@ lz4_init(void **data, int *level, int nthreads, int64_t chunksize,
LZ4_MAX_CHUNK);
return (1);
}
lzdat = slab_alloc(NULL, sizeof (struct lz4_params));
lzdat = (struct lz4_params *)slab_alloc(NULL, sizeof (struct lz4_params));
lev = *level;
if (lev > 3) lev = 3;
@ -104,23 +104,23 @@ lz4_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
uchar_t *dst2;
if (lzdat->level == 1) {
rv = LZ4_compress(src, dst, _srclen);
rv = LZ4_compress((const char *)src, (char *)dst, _srclen);
} else if (lzdat->level == 2) {
rv = LZ4_compress(src, dst, _srclen);
rv = LZ4_compress((const char *)src, (char *)dst, _srclen);
if (rv == 0 || rv > *dstlen) {
return (-1);
}
dst2 = slab_alloc(NULL, rv + sizeof (int) + LZ4_compressBound(rv));
dst2 = (uchar_t *)slab_alloc(NULL, rv + sizeof (int) + LZ4_compressBound(rv));
*((int *)dst2) = htonl(rv);
rv = LZ4_compressHC(dst, dst2 + sizeof (int), rv);
rv = LZ4_compressHC((const char *)dst, (char *)(dst2 + sizeof (int)), rv);
if (rv != 0) {
rv += sizeof (int);
memcpy(dst, dst2, rv);
}
slab_free(NULL, dst2);
} else {
rv = LZ4_compressHC(src, dst, _srclen);
rv = LZ4_compressHC((const char *)src, (char *)dst, _srclen);
}
if (rv == 0) {
return (-1);
@ -137,10 +137,9 @@ lz4_decompress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
int rv;
struct lz4_params *lzdat = (struct lz4_params *)data;
int _dstlen = *dstlen;
uchar_t *dst2;
if (lzdat->level == 1 || lzdat->level == 3) {
rv = LZ4_uncompress(src, dst, _dstlen);
rv = LZ4_uncompress((const char *)src, (char *)dst, _dstlen);
if (rv != srclen) {
return (-1);
}
@ -149,12 +148,12 @@ lz4_decompress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
int sz1;
sz1 = ntohl(*((int *)src));
rv = LZ4_uncompress(src + sizeof (int), dst, sz1);
rv = LZ4_uncompress((const char *)src + sizeof (int), (char *)dst, sz1);
if (rv != srclen - sizeof (int)) {
return (-1);
}
memcpy(src, dst, sz1);
rv = LZ4_uncompress(src, dst, _dstlen);
rv = LZ4_uncompress((const char *)src, (char *)dst, _dstlen);
if (rv != sz1) {
return (-1);
}

6
lzfx_compress.c
View file

@ -40,12 +40,12 @@ lz_fx_stats(int show)
}
void
lz_fx_props(algo_props_t *data, int level, int64_t chunksize) {
lz_fx_props(algo_props_t *data, int level, uint64_t chunksize) {
data->delta2_span = 50;
}
int
lz_fx_init(void **data, int *level, int nthreads, int64_t chunksize,
lz_fx_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
struct lzfx_params *lzdat;
@ -55,7 +55,7 @@ lz_fx_init(void **data, int *level, int nthreads, int64_t chunksize,
fprintf(stderr, "Chunk size too big for LZFX.\n");
return (1);
}
lzdat = slab_alloc(NULL, sizeof (struct lzfx_params));
lzdat = (struct lzfx_params *)slab_alloc(NULL, sizeof (struct lzfx_params));
lev = *level;
if (lev > 5) lev = 5;

1
lzma/LzFind.c
View file

@ -273,7 +273,6 @@ static void MatchFinder_SetLimits(CMatchFinder *p)
void MatchFinder_Init(CMatchFinder *p)
{
UInt32 i;
/*
* Optimized following loop:
* for (i = 0; i < p->hashSizeSum; i++)

3
lzma/LzmaEnc.c
View file

@ -790,7 +790,6 @@ static UInt32 RcTree_ReverseGetPrice(const CLzmaProb *probs, int numBitLevels, U
static void LenEnc_Init(CLenEnc *p)
{
unsigned i;
UInt64 val;
val = kProbInitValue;
val <<= 32;
@ -1313,7 +1312,6 @@ cont_1:
}
else
{
UInt32 i;
reps[0] = (pos - LZMA_NUM_REPS);
/* Unroll for small iterations. */
#if LZMA_NUM_REPS > 4
@ -2127,7 +2125,6 @@ void LzmaEnc_Init(CLzmaEnc *p)
for (i = 0; i < kNumStates; i++)
{
UInt32 j;
/*
* for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++)
* {

12
lzma_compress.c
View file

@ -47,7 +47,7 @@ lzma_stats(int show)
}
void
lzma_mt_props(algo_props_t *data, int level, int64_t chunksize) {
lzma_mt_props(algo_props_t *data, int level, uint64_t chunksize) {
data->compress_mt_capable = 1;
data->decompress_mt_capable = 0;
data->buf_extra = 0;
@ -56,7 +56,7 @@ lzma_mt_props(algo_props_t *data, int level, int64_t chunksize) {
}
void
lzma_props(algo_props_t *data, int level, int64_t chunksize) {
lzma_props(algo_props_t *data, int level, uint64_t chunksize) {
data->compress_mt_capable = 0;
data->decompress_mt_capable = 0;
data->buf_extra = 0;
@ -67,7 +67,7 @@ lzma_props(algo_props_t *data, int level, int64_t chunksize) {
* The two functions below are not thread-safe, by design.
*/
int
lzma_init(void **data, int *level, int nthreads, int64_t chunksize,
lzma_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
if (!p && op == COMPRESS) {
@ -204,8 +204,8 @@ lzma_compress(void *src, uint64_t srclen, void *dst,
_dst = (Byte *)dst;
*dstlen -= LZMA_PROPS_SIZE;
res = LzmaEncode(_dst + LZMA_PROPS_SIZE, dstlen, src, srclen,
props, _dst, &props_len, 0, NULL, &g_Alloc, &g_Alloc);
res = LzmaEncode(_dst + LZMA_PROPS_SIZE, dstlen, (const uchar_t *)src, srclen,
props, (uchar_t *)_dst, &props_len, 0, NULL, &g_Alloc, &g_Alloc);
if (res != 0) {
lzerr(res, 1);
@ -229,7 +229,7 @@ lzma_decompress(void *src, uint64_t srclen, void *dst,
_src = (uchar_t *)src + LZMA_PROPS_SIZE;
if ((res = LzmaDecode((uchar_t *)dst, dstlen, _src, &_srclen,
src, LZMA_PROPS_SIZE, LZMA_FINISH_ANY,
(uchar_t *)src, LZMA_PROPS_SIZE, LZMA_FINISH_ANY,
&status, &g_Alloc)) != SZ_OK) {
lzerr(res, 0);
return (-1);

10
lzp/lzp.c
View file

@ -38,6 +38,10 @@ See also the bsc and libbsc web site:
*/
#undef LZP_OPENMP
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS 1
#endif
#include <stdlib.h>
#include <memory.h>
#include <string.h>
@ -74,7 +78,8 @@ int bsc_lzp_encode_block(const unsigned char * input, const unsigned char * inpu
return LZP_NOT_COMPRESSIBLE;
}
if (lookup = (int *)slab_calloc(NULL, (int)(1 << hashSize), sizeof(int)))
lookup = (int *)slab_calloc(NULL, (int)(1 << hashSize), sizeof(int));
if (lookup)
{
unsigned int mask = (int)(1 << hashSize) - 1;
const unsigned char * inputStart = input;
@ -172,7 +177,8 @@ int bsc_lzp_decode_block(const unsigned char * input, const unsigned char * inpu
return LZP_UNEXPECTED_EOB;
}
if (lookup = (int *)slab_calloc(NULL, (int)(1 << hashSize), sizeof(int)))
lookup = (int *)slab_calloc(NULL, (int)(1 << hashSize), sizeof(int));
if (lookup)
{
unsigned int mask = (int)(1 << hashSize) - 1;
const unsigned char * outputStart = output;

57
main.c
View file

@ -47,6 +47,9 @@
#include <rabin_dedup.h>
#include <lzp.h>
#include <transpose.h>
#include <delta2/delta2.h>
#include <crypto/crypto_utils.h>
#include <ctype.h>
/*
* We use 5MB chunks by default.
@ -95,7 +98,6 @@ static int do_uncompress = 0;
static int cksum_bytes, mac_bytes;
static int cksum = 0, t_errored = 0;
static int rab_blk_size = 0;
static dedupe_context_t *rctx;
static crypto_ctx_t crypto_ctx;
static char *pwd_file = NULL;
@ -198,7 +200,8 @@ preproc_compress(compress_func_ptr cmp_func, void *src, uint64_t srclen, void *d
uint64_t *dstlen, int level, uchar_t chdr, void *data, algo_props_t *props)
{
uchar_t *dest = (uchar_t *)dst, type = 0;
int64_t result, _dstlen;
int64_t result;
uint64_t _dstlen;
DEBUG_STAT_EN(double strt, en);
_dstlen = *dstlen;
@ -207,7 +210,8 @@ preproc_compress(compress_func_ptr cmp_func, void *src, uint64_t srclen, void *d
type = PREPROC_TYPE_LZP;
hashsize = lzp_hash_size(level);
result = lzp_compress(src, dst, srclen, hashsize, LZP_DEFAULT_LZPMINLEN, 0);
result = lzp_compress((const uchar_t *)src, (uchar_t *)dst, srclen,
hashsize, LZP_DEFAULT_LZPMINLEN, 0);
if (result < 0 || result == srclen) {
if (!enable_delta2_encode)
return (-1);
@ -227,7 +231,8 @@ preproc_compress(compress_func_ptr cmp_func, void *src, uint64_t srclen, void *d
if (enable_delta2_encode && props->delta2_span > 0) {
_dstlen = srclen;
result = delta2_encode(src, srclen, dst, &_dstlen, props->delta2_span);
result = delta2_encode((uchar_t *)src, srclen, (uchar_t *)dst,
&_dstlen, props->delta2_span);
if (result != -1) {
memcpy(src, dst, _dstlen);
srclen = _dstlen;
@ -285,7 +290,7 @@ preproc_decompress(compress_func_ptr dec_func, void *src, uint64_t srclen, void
}
if (type & PREPROC_TYPE_DELTA2) {
result = delta2_decode(src, srclen, dst, &_dstlen);
result = delta2_decode((uchar_t *)src, srclen, (uchar_t *)dst, &_dstlen);
if (result != -1) {
memcpy(src, dst, _dstlen);
srclen = _dstlen;
@ -298,7 +303,8 @@ preproc_decompress(compress_func_ptr dec_func, void *src, uint64_t srclen, void
if (type & PREPROC_TYPE_LZP) {
int hashsize;
hashsize = lzp_hash_size(level);
result = lzp_decompress(src, dst, srclen, hashsize, LZP_DEFAULT_LZPMINLEN, 0);
result = lzp_decompress((const uchar_t *)src, (uchar_t *)dst, srclen,
hashsize, LZP_DEFAULT_LZPMINLEN, 0);
if (result < 0) {
fprintf(stderr, "LZP decompression failed.\n");
return (-1);
@ -323,9 +329,9 @@ static void *
perform_decompress(void *dat)
{
struct cmp_data *tdat = (struct cmp_data *)dat;
int64_t _chunksize;
int64_t dedupe_index_sz, dedupe_data_sz, dedupe_index_sz_cmp, dedupe_data_sz_cmp;
int type, rv;
uint64_t _chunksize;
uint64_t dedupe_index_sz, dedupe_data_sz, dedupe_index_sz_cmp, dedupe_data_sz_cmp;
int rv = 0;
unsigned int blknum;
uchar_t checksum[CKSUM_MAX_BYTES];
uchar_t HDR;
@ -388,7 +394,7 @@ redo:
tdat->len_cmp = 0;
t_errored = 1;
sem_post(&tdat->cmp_done_sem);
return;
return (NULL);
}
/*
@ -403,7 +409,7 @@ redo:
main_cancel = 1;
tdat->len_cmp = 0;
sem_post(&tdat->cmp_done_sem);
return;
return (NULL);
}
} else if (mac_bytes > 0) {
/*
@ -431,7 +437,7 @@ redo:
tdat->len_cmp = 0;
t_errored = 1;
sem_post(&tdat->cmp_done_sem);
return;
return (NULL);
}
/*
@ -603,7 +609,6 @@ cont:
static int
start_decompress(const char *filename, const char *to_filename)
{
char tmpfile[MAXPATHLEN];
char algorithm[ALGO_SZ];
struct stat sbuf;
struct wdata w;
@ -752,6 +757,7 @@ start_decompress(const char *filename, const char *to_filename)
* is computed over header and encrypted data.
*/
cksum_bytes = 0;
pw_len = -1;
compressed_chunksize += mac_bytes;
encrypt_type = flags & MASK_CRYPTO_ALG;
if (Read(compfd, &saltlen, sizeof (saltlen)) < sizeof (saltlen)) {
@ -759,8 +765,8 @@ start_decompress(const char *filename, const char *to_filename)
UNCOMP_BAIL;
}
saltlen = ntohl(saltlen);
salt1 = malloc(saltlen);
salt2 = malloc(saltlen);
salt1 = (uchar_t *)malloc(saltlen);
salt2 = (uchar_t *)malloc(saltlen);
if (Read(compfd, salt1, saltlen) < saltlen) {
free(salt1); free(salt2);
perror("Read: ");
@ -867,7 +873,6 @@ start_decompress(const char *filename, const char *to_filename)
}
} else if (version >= 5) {
uint32_t crc1, crc2;
unsigned int hlen;
unsigned short d1;
unsigned int d2;
uint64_t ch;
@ -1084,7 +1089,6 @@ start_decompress(const char *filename, const char *to_filename)
tdat = dary[p];
sem_wait(&tdat->write_done_sem);
}
thread = 0;
}
uncomp_done:
if (t_errored) err = t_errored;
@ -1151,6 +1155,8 @@ redo:
compressed_chunk = tdat->compressed_chunk + CHUNK_FLAG_SZ;
rbytes = tdat->rbytes;
dedupe_index_sz = 0;
/* Perform Dedup if enabled. */
if ((enable_rabin_scan || enable_fixed_scan)) {
dedupe_context_t *rctx;
@ -1249,7 +1255,6 @@ plain_index:
update_dedupe_hdr(compressed_chunk, index_size_cmp - RABIN_HDR_SIZE, _chunksize);
_chunksize += index_size_cmp;
} else {
plain_compress:
_chunksize = tdat->rbytes;
if (lzp_preprocess || enable_delta2_encode) {
rv = preproc_compress(tdat->compress,
@ -1366,7 +1371,6 @@ plain_compress:
* Compute header CRC32 in non-crypto mode.
*/
uchar_t *mac_ptr;
unsigned int hlen;
uint32_t crc;
/* Clean out mac_bytes to 0 for stable CRC32. */
@ -1379,7 +1383,6 @@ plain_compress:
*((uint32_t *)mac_ptr) = htonl(crc);
}
cont:
sem_post(&tdat->cmp_done_sem);
goto redo;
}
@ -1409,8 +1412,6 @@ repeat:
wbytes = Write(w->wfd, tdat->cmp_seg, tdat->len_cmp);
if (unlikely(wbytes != tdat->len_cmp)) {
int i;
perror("Chunk Write: ");
do_cancel:
main_cancel = 1;
@ -1437,8 +1438,8 @@ start_compress(const char *filename, uint64_t chunksize, int level)
struct wdata w;
char tmpfile1[MAXPATHLEN];
char to_filename[MAXPATHLEN];
int64_t compressed_chunksize;
int64_t n_chunksize, rbytes, rabin_count;
uint64_t compressed_chunksize, n_chunksize;
int64_t rbytes, rabin_count;
short version, flags;
struct stat sbuf;
int compfd = -1, uncompfd = -1, err;
@ -1464,6 +1465,7 @@ start_compress(const char *filename, uint64_t chunksize, int level)
*/
compressed_chunksize = chunksize + CHUNK_HDR_SZ + zlib_buf_extra(chunksize);
init_algo_props(&props);
cread_buf = NULL;
if (_props_func) {
_props_func(&props, level, chunksize);
@ -1486,7 +1488,7 @@ start_compress(const char *filename, uint64_t chunksize, int level)
if (encrypt_type) {
uchar_t pw[MAX_PW_LEN];
int pw_len;
int pw_len = -1;
compressed_chunksize += mac_bytes;
if (!pwd_file) {
@ -1697,7 +1699,7 @@ start_compress(const char *filename, uint64_t chunksize, int level)
*/
flags |= cksum;
memset(cread_buf, 0, ALGO_SZ);
strncpy(cread_buf, algo, ALGO_SZ);
strncpy((char *)cread_buf, algo, ALGO_SZ);
version = htons(VERSION);
flags = htons(flags);
n_chunksize = htonll(chunksize);
@ -1991,7 +1993,7 @@ comp_done:
static int
init_algo(const char *algo, int bail)
{
int rv = 1, i;
int rv = 1;
char algorithm[8];
/* Copy given string into known length buffer to avoid memcmp() overruns. */
@ -2115,6 +2117,7 @@ main(int argc, char *argv[])
exec_name = get_execname(argv[0]);
level = 6;
err = 0;
slab_init();
while ((opt = getopt(argc, argv, "dc:s:l:pt:MCDEew:rLPS:B:F")) != -1) {

4
none_compress.c
View file

@ -37,14 +37,14 @@ none_stats(int show)
}
int
none_init(void **data, int *level, int nthreads, int64_t chunksize,
none_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
return (0);
}
void
none_props(algo_props_t *data, int level, int64_t chunksize) {
none_props(algo_props_t *data, int level, uint64_t chunksize) {
data->compress_mt_capable = 0;
data->decompress_mt_capable = 0;
data->buf_extra = 0;

50
pcompress.h
View file

@ -77,8 +77,8 @@ extern "C" {
#define COMPRESS_BSC 4
#define CHDR_ALGO_MASK 7
extern uint32_t zlib_buf_extra(int64_t buflen);
extern int lz4_buf_extra(int64_t buflen);
extern uint32_t zlib_buf_extra(uint64_t buflen);
extern int lz4_buf_extra(uint64_t buflen);
extern int zlib_compress(void *src, uint64_t srclen, void *dst,
uint64_t *destlen, int level, uchar_t chdr, void *data);
@ -114,34 +114,34 @@ extern int lz4_decompress(void *src, uint64_t srclen, void *dst,
extern int none_decompress(void *src, uint64_t srclen, void *dst,
uint64_t *dstlen, int level, uchar_t chdr, void *data);
extern int adapt_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int adapt_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int adapt2_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int adapt2_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int lzma_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int lzma_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int ppmd_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int bzip2_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int bzip2_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int zlib_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int zlib_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int lz_fx_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int lz_fx_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int lz4_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int lz4_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern int none_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int none_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern void lzma_props(algo_props_t *data, int level, int64_t chunksize);
extern void lzma_mt_props(algo_props_t *data, int level, int64_t chunksize);
extern void lz4_props(algo_props_t *data, int level, int64_t chunksize);
extern void zlib_props(algo_props_t *data, int level, int64_t chunksize);
extern void ppmd_props(algo_props_t *data, int level, int64_t chunksize);
extern void lz_fx_props(algo_props_t *data, int level, int64_t chunksize);
extern void bzip2_props(algo_props_t *data, int level, int64_t chunksize);
extern void adapt_props(algo_props_t *data, int level, int64_t chunksize);
extern void none_props(algo_props_t *data, int level, int64_t chunksize);
extern void lzma_props(algo_props_t *data, int level, uint64_t chunksize);
extern void lzma_mt_props(algo_props_t *data, int level, uint64_t chunksize);
extern void lz4_props(algo_props_t *data, int level, uint64_t chunksize);
extern void zlib_props(algo_props_t *data, int level, uint64_t chunksize);
extern void ppmd_props(algo_props_t *data, int level, uint64_t chunksize);
extern void lz_fx_props(algo_props_t *data, int level, uint64_t chunksize);
extern void bzip2_props(algo_props_t *data, int level, uint64_t chunksize);
extern void adapt_props(algo_props_t *data, int level, uint64_t chunksize);
extern void none_props(algo_props_t *data, int level, uint64_t chunksize);
extern int zlib_deinit(void **data);
extern int adapt_deinit(void **data);
@ -165,9 +165,9 @@ extern int libbsc_compress(void *src, uint64_t srclen, void *dst,
uint64_t *dstlen, int level, uchar_t chdr, void *data);
extern int libbsc_decompress(void *src, uint64_t srclen, void *dst,
uint64_t *dstlen, int level, uchar_t chdr, void *data);
extern int libbsc_init(void **data, int *level, int nthreads, int64_t chunksize,
extern int libbsc_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
extern void libbsc_props(algo_props_t *data, int level, int64_t chunksize);
extern void libbsc_props(algo_props_t *data, int level, uint64_t chunksize);
extern int libbsc_deinit(void **data);
extern void libbsc_stats(int show);
#endif
@ -180,9 +180,9 @@ struct cmp_data {
uchar_t *compressed_chunk;
uchar_t *uncompressed_chunk;
dedupe_context_t *rctx;
int64_t rbytes;
int64_t chunksize;
int64_t len_cmp, len_cmp_be;
uint64_t rbytes;
uint64_t chunksize;
uint64_t len_cmp, len_cmp_be;
uchar_t checksum[CKSUM_MAX_BYTES];
int level;
unsigned int id;

5
ppmd_compress.c
View file

@ -62,12 +62,12 @@ ppmd_stats(int show)
}
void
ppmd_props(algo_props_t *data, int level, int64_t chunksize) {
ppmd_props(algo_props_t *data, int level, uint64_t chunksize) {
data->delta2_span = 100;
}
int
ppmd_init(void **data, int *level, int nthreads, int64_t chunksize,
ppmd_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
CPpmd8 *_ppmd;
@ -110,7 +110,6 @@ ppmd_compress(void *src, uint64_t srclen, void *dst,
{
CPpmd8 *_ppmd = (CPpmd8 *)data;
uchar_t *_src = (uchar_t *)src;
UInt32 i;
Ppmd8_RangeEnc_Init(_ppmd);
Ppmd8_Init(_ppmd, _ppmd->Order, PPMD8_RESTORE_METHOD_RESTART);

81
rabin/rabin_dedup.c
View file

@ -57,6 +57,10 @@
*
*/
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS 1
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -64,6 +68,7 @@
#include <utils.h>
#include <pthread.h>
#include <heapq.h>
#include <xxhash.h>
#include "rabin_dedup.h"
@ -78,10 +83,10 @@ extern int lzma_compress(void *src, uint64_t srclen, void *dst,
extern int lzma_decompress(void *src, uint64_t srclen, void *dst,
uint64_t *dstlen, int level, uchar_t chdr, void *data);
extern int lzma_deinit(void **data);
extern int bsdiff(u_char *old, bsize_t oldsize, u_char *new, bsize_t newsize,
extern int bsdiff(u_char *oldbuf, bsize_t oldsize, u_char *newbuf, bsize_t newsize,
u_char *diff, u_char *scratch, bsize_t scratchsize);
extern bsize_t get_bsdiff_sz(u_char *pbuf);
extern int bspatch(u_char *pbuf, u_char *old, bsize_t oldsize, u_char *new,
extern int bspatch(u_char *pbuf, u_char *oldbuf, bsize_t oldsize, u_char *newbuf,
bsize_t *_newsize);
static pthread_mutex_t init_lock = PTHREAD_MUTEX_INITIALIZER;
@ -214,7 +219,7 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
destroy_dedupe_context(ctx);
return (NULL);
}
ctx->current_window_data = slab_alloc(NULL, RAB_POLYNOMIAL_WIN_SIZE);
ctx->current_window_data = (uchar_t *)slab_alloc(NULL, RAB_POLYNOMIAL_WIN_SIZE);
ctx->blocks = NULL;
if (real_chunksize > 0) {
ctx->blocks = (rabin_blockentry_t **)slab_calloc(NULL,
@ -279,11 +284,11 @@ destroy_dedupe_context(dedupe_context_t *ctx)
* from 4K-128K.
*/
uint32_t
dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size, int64_t offset, int64_t *rabin_pos)
dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size, uint64_t offset, uint64_t *rabin_pos)
{
int64_t i, last_offset, j, ary_sz;
uint64_t i, last_offset, j, ary_sz;
uint32_t blknum;
char *buf1 = (char *)buf;
uchar_t *buf1 = (uchar_t *)buf;
uint32_t length;
uint64_t cur_roll_checksum, cur_pos_checksum;
uint32_t *fplist;
@ -298,6 +303,7 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size, int64_t offs
blknum = 0;
ctx->valid = 0;
cur_roll_checksum = 0;
if (*size < ctx->rabin_poly_avg_block_size) return (0);
DEBUG_STAT_EN(strt = get_wtime_millis());
if (ctx->fixed_flag) {
@ -377,11 +383,10 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size, int64_t offs
return (0);
}
if (*size < ctx->rabin_poly_avg_block_size) return;
j = 0;
for (i=offset; i<*size; i++) {
int64_t pc[4];
uint64_t pc[4];
uchar_t cur_byte = buf1[i];
uint64_t pushed_out = ctx->current_window_data[ctx->window_pos];
ctx->current_window_data[ctx->window_pos] = cur_byte;
@ -439,7 +444,7 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size, int64_t offs
pc[3] = FORTY_PCNT(j);
reset_heap(&heap, pc[ctx->delta_flag]);
ksmallest(fplist, j, &heap);
ksmallest((int32_t *)fplist, j, &heap);
ctx->blocks[blknum]->similarity_hash =
XXH_fast32((const uchar_t *)fplist, pc[ctx->delta_flag]*4, 0);
memset(fplist, 0, ary_sz);
@ -462,14 +467,14 @@ dedupe_compress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size, int64_t offs
if (ctx->delta_flag) {
uint64_t cur_sketch;
int64_t pc[3];
uint64_t pc[3];
if (j > 1) {
pc[1] = SIXTY_PCNT(j);
pc[2] = FIFTY_PCNT(j);
pc[3] = FORTY_PCNT(j);
reset_heap(&heap, pc[ctx->delta_flag]);
ksmallest(fplist, j, &heap);
ksmallest((int32_t *)fplist, j, &heap);
cur_sketch =
XXH_fast32((const uchar_t *)fplist, pc[ctx->delta_flag]*4, 0);
} else {
@ -489,10 +494,10 @@ process_blocks:
DEBUG_STAT_EN(fprintf(stderr, "Original size: %" PRId64 ", blknum: %u\n", *size, blknum));
DEBUG_STAT_EN(fprintf(stderr, "Number of maxlen blocks: %u\n", max_count));
if (blknum > 2) {
int64_t pos, matchlen, pos1;
uint64_t pos, matchlen, pos1;
int valid = 1;
uint32_t *dedupe_index;
int64_t dedupe_index_sz;
uint64_t dedupe_index_sz;
rabin_blockentry_t *be;
DEBUG_STAT_EN(uint32_t delta_calls, delta_fails, merge_count, hash_collisions);
DEBUG_STAT_EN(delta_calls = 0);
@ -609,11 +614,11 @@ process_blocks:
}
DEBUG_STAT_EN(fprintf(stderr, "Total Hashtable bucket collisions: %u\n", hash_collisions));
dedupe_index_sz = (int64_t)blknum * RABIN_ENTRY_SIZE;
dedupe_index_sz = (uint64_t)blknum * RABIN_ENTRY_SIZE;
if (matchlen < dedupe_index_sz) {
DEBUG_STAT_EN(fprintf(stderr, "No Dedupe possible.\n"));
ctx->valid = 0;
return;
return (0);
}
dedupe_index = (uint32_t *)(ctx->cbuf + RABIN_HDR_SIZE);
@ -648,7 +653,7 @@ process_blocks:
* Final pass update dedupe index and copy data.
*/
blknum = pos;
dedupe_index_sz = (int64_t)blknum * RABIN_ENTRY_SIZE;
dedupe_index_sz = (uint64_t)blknum * RABIN_ENTRY_SIZE;
pos1 = dedupe_index_sz + RABIN_HDR_SIZE;
matchlen = ctx->real_chunksize - *size;
for (i=0; i<blknum; i++) {
@ -663,20 +668,20 @@ process_blocks:
dedupe_index[i] = htonl((be->other->index | RABIN_INDEX_FLAG) &
CLEAR_SIMILARITY_FLAG);
} else {
uchar_t *old, *new;
uchar_t *oldbuf, *newbuf;
int32_t bsz;
/*
* Perform bsdiff.
*/
old = buf1 + be->other->offset;
new = buf1 + be->offset;
oldbuf = buf1 + be->other->offset;
newbuf = buf1 + be->offset;
DEBUG_STAT_EN(delta_calls++);
bsz = bsdiff(old, be->other->length, new, be->length,
bsz = bsdiff(oldbuf, be->other->length, newbuf, be->length,
ctx->cbuf + pos1, buf1 + *size, matchlen);
if (bsz == 0) {
DEBUG_STAT_EN(delta_fails++);
memcpy(ctx->cbuf + pos1, new, be->length);
memcpy(ctx->cbuf + pos1, newbuf, be->length);
dedupe_index[i] = htonl(be->length);
pos1 += be->length;
} else {
@ -687,16 +692,16 @@ process_blocks:
}
}
}
cont:
if (valid) {
uchar_t *cbuf = ctx->cbuf;
int64_t *entries;
DEBUG_STAT_EN(int64_t sz);
uint64_t *entries;
DEBUG_STAT_EN(uint64_t sz);
DEBUG_STAT_EN(sz = *size);
*((uint32_t *)cbuf) = htonl(blknum);
cbuf += sizeof (uint32_t);
entries = (int64_t *)cbuf;
entries = (uint64_t *)cbuf;
entries[0] = htonll(*size);
entries[1] = 0;
entries[2] = htonll(pos1 - dedupe_index_sz - RABIN_HDR_SIZE);
@ -717,41 +722,41 @@ cont:
}
void
update_dedupe_hdr(uchar_t *buf, int64_t dedupe_index_sz_cmp, int64_t dedupe_data_sz_cmp)
update_dedupe_hdr(uchar_t *buf, uint64_t dedupe_index_sz_cmp, uint64_t dedupe_data_sz_cmp)
{
int64_t *entries;
uint64_t *entries;
buf += sizeof (uint32_t);
entries = (int64_t *)buf;
entries = (uint64_t *)buf;
entries[1] = htonll(dedupe_index_sz_cmp);
entries[3] = htonll(dedupe_data_sz_cmp);
}
void
parse_dedupe_hdr(uchar_t *buf, uint32_t *blknum, int64_t *dedupe_index_sz,
int64_t *dedupe_data_sz, int64_t *dedupe_index_sz_cmp,
int64_t *dedupe_data_sz_cmp, int64_t *deduped_size)
parse_dedupe_hdr(uchar_t *buf, uint32_t *blknum, uint64_t *dedupe_index_sz,
uint64_t *dedupe_data_sz, uint64_t *dedupe_index_sz_cmp,
uint64_t *dedupe_data_sz_cmp, uint64_t *deduped_size)
{
int64_t *entries;
uint64_t *entries;
*blknum = ntohl(*((uint32_t *)(buf)));
buf += sizeof (uint32_t);
entries = (int64_t *)buf;
entries = (uint64_t *)buf;
*dedupe_data_sz = ntohll(entries[0]);
*dedupe_index_sz = (int64_t)(*blknum) * RABIN_ENTRY_SIZE;
*dedupe_index_sz = (uint64_t)(*blknum) * RABIN_ENTRY_SIZE;
*dedupe_index_sz_cmp = ntohll(entries[1]);
*deduped_size = ntohll(entries[2]);
*dedupe_data_sz_cmp = ntohll(entries[3]);
}
void
dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size)
dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size)
{
uint32_t blknum, blk, oblk, len;
uint32_t *dedupe_index;
int64_t data_sz, sz, indx_cmp, data_sz_cmp, deduped_sz;
int64_t dedupe_index_sz, pos1, i;
uint64_t data_sz, sz, indx_cmp, data_sz_cmp, deduped_sz;
uint64_t dedupe_index_sz, pos1;
uchar_t *pos2;
parse_dedupe_hdr(buf, &blknum, &dedupe_index_sz, &data_sz, &indx_cmp, &data_sz_cmp, &deduped_sz);
@ -844,7 +849,7 @@ dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size)
* TODO: Consolidate rabin dedup and compression/decompression in functions here rather than
* messy code in main program.
int
rabin_compress(dedupe_context_t *ctx, uchar_t *from, int64_t fromlen, uchar_t *to, int64_t *tolen,
rabin_compress(dedupe_context_t *ctx, uchar_t *from, uint64_t fromlen, uchar_t *to, uint64_t *tolen,
int level, char chdr, void *data, compress_func_ptr cmp)
{
}

18
rabin/rabin_dedup.h
View file

@ -89,7 +89,7 @@
// Header for a chunk deduped using Rabin
// Number of rabin blocks, size of original data chunk, size of compressed index,
// size of deduped data, size of compressed data
#define RABIN_HDR_SIZE (sizeof (unsigned int) + sizeof (int64_t) + sizeof (int64_t) + sizeof (int64_t) + sizeof (int64_t))
#define RABIN_HDR_SIZE (sizeof (unsigned int) + sizeof (uint64_t) + sizeof (uint64_t) + sizeof (uint64_t) + sizeof (uint64_t))
// Maximum number of dedup blocks supported (2^30 - 1)
#define RABIN_MAX_BLOCKS (0x3FFFFFFFUL)
@ -135,7 +135,7 @@
#define FP_POLY 0xbfe6b8a5bf378d83ULL
typedef struct rab_blockentry {
int64_t offset;
uint64_t offset;
uint32_t similarity_hash;
uint32_t hash;
uint32_t index;
@ -167,13 +167,13 @@ extern dedupe_context_t *create_dedupe_context(uint64_t chunksize, uint64_t real
compress_op_t op);
extern void destroy_dedupe_context(dedupe_context_t *ctx);
extern unsigned int dedupe_compress(dedupe_context_t *ctx, unsigned char *buf,
int64_t *size, int64_t offset, int64_t *rabin_pos);
extern void dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, int64_t *size);
extern void parse_dedupe_hdr(uchar_t *buf, unsigned int *blknum, int64_t *dedupe_index_sz,
int64_t *dedupe_data_sz, int64_t *rabin_index_sz_cmp,
int64_t *dedupe_data_sz_cmp, int64_t *deduped_size);
extern void update_dedupe_hdr(uchar_t *buf, int64_t dedupe_index_sz_cmp,
int64_t dedupe_data_sz_cmp);
uint64_t *size, uint64_t offset, uint64_t *rabin_pos);
extern void dedupe_decompress(dedupe_context_t *ctx, uchar_t *buf, uint64_t *size);
extern void parse_dedupe_hdr(uchar_t *buf, unsigned int *blknum, uint64_t *dedupe_index_sz,
uint64_t *dedupe_data_sz, uint64_t *rabin_index_sz_cmp,
uint64_t *dedupe_data_sz_cmp, uint64_t *deduped_size);
extern void update_dedupe_hdr(uchar_t *buf, uint64_t dedupe_index_sz_cmp,
uint64_t dedupe_data_sz_cmp);
extern void reset_dedupe_context(dedupe_context_t *ctx);
extern uint32_t dedupe_buf_extra(uint64_t chunksize, int rab_blk_sz, const char *algo,
int delta_flag);

2
utils/cpuid.h
View file

@ -55,6 +55,8 @@ struct cpu_raw_data_t {
void exec_cpuid(uint32_t *regs);
void cpuid_get_raw_data(struct cpu_raw_data_t* data);
void cpuid_basic_identify(processor_info_t *pc);
#endif /* __x86_64__ */
#endif /* __CPUID_H__ */

33
utils/utils.c
View file

@ -117,7 +117,7 @@ raise_by_multiplier(int64_t *val, int mult, int power) {
int
parse_numeric(int64_t *val, const char *str)
{
int i, ovr;
int ovr = 2;
char *mult;
*val = strtoll(str, &mult, 0);
@ -184,7 +184,6 @@ Read(int fd, void *buf, uint64_t count)
{
int64_t rcount, rem;
uchar_t *cbuf;
va_list args;
rem = count;
cbuf = (uchar_t *)buf;
@ -207,23 +206,29 @@ Read(int fd, void *buf, uint64_t count)
int64_t
Read_Adjusted(int fd, uchar_t *buf, uint64_t count, int64_t *rabin_count, void *ctx)
{
char *buf2;
uchar_t *buf2;
int64_t rcount;
dedupe_context_t *rctx = (dedupe_context_t *)ctx;
if (!ctx) return (Read(fd, buf, count));
buf2 = buf;
if (*rabin_count) {
buf2 = (char *)buf + *rabin_count;
buf2 = buf + *rabin_count;
count -= *rabin_count;
}
rcount = Read(fd, buf2, count);
if (rcount > 0) {
rcount += *rabin_count;
if (rcount == count)
dedupe_compress(rctx, buf, &rcount, 0, rabin_count);
else
if (rcount == count) {
uint64_t rc, rbc;
rc = rcount;
rbc = *rabin_count;
dedupe_compress(rctx, buf, &rc, 0, &rbc);
rcount = rc;
*rabin_count = rbc;
} else {
*rabin_count = 0;
}
} else {
if (rcount == 0) rcount = *rabin_count;
*rabin_count = 0;
@ -248,6 +253,20 @@ Write(int fd, const void *buf, uint64_t count)
return (count - rem);
}
void
init_algo_props(algo_props_t *props)
{
props->buf_extra = 0;
props->compress_mt_capable = 0;
props->decompress_mt_capable = 0;
props->single_chunk_mt_capable = 0;
props->is_single_chunk = 0;
props->nthreads = 1;
props->c_max_threads = 1;
props->d_max_threads = 1;
props->delta2_span = 0;
}
/*
* Thread sizing. We want a balanced combination of chunk threads and compression
* algorithm threads that best fit the available/allowed number of processors.

45
utils/utils.h
View file

@ -25,9 +25,15 @@
#define _UTILS_H
#include <arpa/nameser_compat.h>
#include <arpa/inet.h>
#include <sys/types.h>
#include <stdint.h>
#ifndef __STDC_FORMAT_MACROS
#define __STDC_FORMAT_MACROS 1
#endif
#include <inttypes.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
@ -147,6 +153,7 @@ extern void set_threadcounts(algo_props_t *props, int *nthreads, int nprocs,
extern uint64_t get_total_ram();
extern double get_wtime_millis(void);
extern double get_mb_s(uint64_t bytes, double strt, double en);
extern void init_algo_props(algo_props_t *props);
/* Pointer type for compress and decompress functions. */
typedef int (*compress_func_ptr)(void *src, uint64_t srclen, void *dst,
@ -158,11 +165,11 @@ typedef enum {
} compress_op_t;
/* Pointer type for algo specific init/deinit/stats functions. */
typedef int (*init_func_ptr)(void **data, int *level, int nthreads, int64_t chunksize,
typedef int (*init_func_ptr)(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op);
typedef int (*deinit_func_ptr)(void **data);
typedef void (*stats_func_ptr)(int show);
typedef void (*props_func_ptr)(algo_props_t *data, int level, int64_t chunksize);
typedef void (*props_func_ptr)(algo_props_t *data, int level, uint64_t chunksize);
/*
@ -181,38 +188,6 @@ roundup_pow_two(unsigned int v) {
return (v);
}
/*
* Hash function for 64Bit pointers/numbers that generates
* a 32Bit hash value.
* Taken from Thomas Wang's Integer hashing paper:
* http://www.cris.com/~Ttwang/tech/inthash.htm
*/
static uint32_t
hash6432shift(uint64_t key)
{
key = (~key) + (key << 18); // key = (key << 18) - key - 1;
key = key ^ (key >> 31);
key = key * 21; // key = (key + (key << 2)) + (key << 4);
key = key ^ (key >> 11);
key = key + (key << 6);
key = key ^ (key >> 22);
return (uint32_t) key;
}
static void
init_algo_props(algo_props_t *props)
{
props->buf_extra = 0;
props->compress_mt_capable = 0;
props->decompress_mt_capable = 0;
props->single_chunk_mt_capable = 0;
props->is_single_chunk = 0;
props->nthreads = 1;
props->c_max_threads = 1;
props->d_max_threads = 1;
props->delta2_span = 0;
}
#ifdef __cplusplus
}
#endif

17
zlib_compress.c
View file

@ -46,7 +46,7 @@ slab_alloc_ui(void *p, unsigned int items, unsigned int size) {
}
uint32_t
zlib_buf_extra(int64_t buflen)
zlib_buf_extra(uint64_t buflen)
{
if (buflen > SINGLE_CALL_MAX)
buflen = SINGLE_CALL_MAX;
@ -54,13 +54,13 @@ zlib_buf_extra(int64_t buflen)
}
int
zlib_init(void **data, int *level, int nthreads, int64_t chunksize,
zlib_init(void **data, int *level, int nthreads, uint64_t chunksize,
int file_version, compress_op_t op)
{
z_stream *zs;
int ret;
zs = slab_alloc(NULL, sizeof (z_stream));
zs = (z_stream *)slab_alloc(NULL, sizeof (z_stream));
zs->zalloc = slab_alloc_ui;
zs->zfree = slab_free;
zs->opaque = NULL;
@ -90,7 +90,7 @@ zlib_stats(int show)
}
void
zlib_props(algo_props_t *data, int level, int64_t chunksize) {
zlib_props(algo_props_t *data, int level, uint64_t chunksize) {
data->delta2_span = 100;
}
@ -102,6 +102,7 @@ zlib_deinit(void **data)
deflateEnd(zs);
slab_free(NULL, *data);
}
return (0);
}
static
@ -144,8 +145,8 @@ zlib_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
unsigned int slen, dlen;
uint64_t _srclen = srclen;
uint64_t _dstlen = *dstlen;
uchar_t *dst1 = dst;
uchar_t *src1 = src;
uchar_t *dst1 = (uchar_t *)dst;
uchar_t *src1 = (uchar_t *)src;
z_stream *zs = (z_stream *)data;
ending = 0;
@ -207,8 +208,8 @@ zlib_decompress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen,
unsigned int slen, dlen;
uint64_t _srclen = srclen;
uint64_t _dstlen = *dstlen;
uchar_t *dst1 = dst;
uchar_t *src1 = src;
uchar_t *dst1 = (uchar_t *)dst;
uchar_t *src1 = (uchar_t *)src;
z_stream *zs = (z_stream *)data;
while (_srclen > 0) {