diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..e73b67b --- /dev/null +++ b/INSTALL @@ -0,0 +1,46 @@ +Copyright (c) 2012 Moinak Ghosh + +Basic Installation +================== +The simplest process to build and install this utility is: + +./config +make +make install + +Strictly speaking the 'make install' step is not required to +run the utility as it is a single stand alone program for now. + +The config script controls various compile time parameters and +generates the actual Makefile. You can get a short description +of all the options by running: + +./config --help + +A more descriptive account is given below: + +./config [] + + can be one of the following: + +--prefix= The installation prefix. + This is set to /usr by default. However alternate + prefix needs to eb used during packaging. + +--enable-debug Enable debug mode compilation. + This reduces the compiler optimization level to basic + and taks out all the loop optimization flags. This is + primary to aid debugging. + +--disable-allocator Disable use of internal memory allocator mechanism. + The internal allocator can be totally disabled by setting + this build time flag. It is also possible to dynamically + disable the allocator by setting the following env variable: + ALLOCATOR_BYPASS=1 + +--enable-debug-stats Enable printing of some verbose debug info. + This at present shows some info related to Dedupe + efficiency. + +--help Display the help message. + diff --git a/Makefile b/Makefile.in similarity index 71% rename from Makefile rename to Makefile.in index cc9c4c9..2f4adba 100644 --- a/Makefile +++ b/Makefile.in @@ -65,39 +65,45 @@ LZPOBJS = $(LZPSRCS:.c=.o) BAKFILES = *~ lzma/*~ lzfx/*~ lz4/*~ rabin/*~ bsdiff/*~ lzp/*~ RM = rm -f -CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT_PROPS \ +COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT_PROPS \ -DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32 \ -I./lzp -VEC_FLAGS = -ftree-vectorize -LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block +COMMON_VEC_FLAGS = -ftree-vectorize +COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block LDLIBS = -ldl -lbz2 $(ZLIB_DIR) -lz -lm OBJS = $(MAINOBJS) $(LZMAOBJS) $(PPMDOBJS) $(LZFXOBJS) $(LZ4OBJS) $(CRCOBJS) \ $(RABINOBJS) $(BSDIFFOBJS) $(LZPOBJS) -ifdef DEBUG -LINK = g++ -m64 -pthread -msse3 -COMPILE = gcc -m64 -g -msse3 -c -COMPILE_cpp = g++ -m64 -g -msse3 -c -VEC_FLAGS = -LOOP_OPTFLAGS = -GEN_OPT = -O -fno-omit-frame-pointer -RABIN_OPT = -O -fno-omit-frame-pointer -else -GEN_OPT = -O3 -RABIN_OPT = -O2 -LINK = g++ -m64 -pthread -msse3 -COMPILE = gcc -m64 -msse3 -c -COMPILE_cpp = g++ -m64 -msse3 -c -CPPFLAGS += -DNDEBUG -endif +DEBUG_LINK = g++ -m64 -pthread -msse3 +DEBUG_COMPILE = gcc -m64 -g -msse3 -c +DEBUG_COMPILE_cpp = g++ -m64 -g -msse3 -c +DEBUG_VEC_FLAGS = +DEBUG_LOOP_OPTFLAGS = +DEBUG_GEN_OPT = -O -fno-omit-frame-pointer +DEBUG_RABIN_OPT = -O -fno-omit-frame-pointer +DEBUG_CPPFLAGS = $(COMMON_CPPFLAGS) -ifdef DEBUG_NO_SLAB -CPPFLAGS += -DDEBUG_NO_SLAB -endif +RELEASE_LINK = g++ -m64 -pthread -msse3 +RELEASE_COMPILE = gcc -m64 -msse3 -c +RELEASE_COMPILE_cpp = g++ -m64 -msse3 -c +RELEASE_VEC_FLAGS = $(COMMON_VEC_FLAGS) +RELEASE_LOOP_OPTFLAGS = $(COMMON_LOOP_OPTFLAGS) +RELEASE_CPPFLAGS = $(COMMON_CPPFLAGS) -DNDEBUG +RELEASE_GEN_OPT = -O3 +RELEASE_RABIN_OPT = -O2 -ifdef DEBUG_STATS -CPPFLAGS += -DDEBUG_STATS -endif +NO_SLAB_CPPFLAGS = -DDEBUG_NO_SLAB +DEBUG_STATS_CPPFLAGS = -DDEBUG_STATS + +LINK = @LINK@ +COMPILE = @COMPILE@ +COMPILE_cpp = @COMPILE_cpp@ +VEC_FLAGS = @VEC_FLAGS@ +LOOP_OPTFLAGS = @LOOP_OPTFLAGS@ +CPPFLAGS = @CPPFLAGS@ @NO_SLAB_CPPFLAGS@ @DEBUG_STATS_CPPFLAGS@ +GEN_OPT = @GEN_OPT@ +RABIN_OPT = @RABIN_OPT@ +PREFIX=@PREFIX@ all: $(PROG) @@ -134,3 +140,17 @@ $(PROG): $(OBJS) clean: $(RM) $(PROG) $(OBJS) $(BAKFILES) +distclean: clean + $(RM) Makefile + +install: $(PROG) + @mkdir -p $(PREFIX)/bin + @chmod 0755 $(PREFIX)/bin + @cp $(PROG) $(PREFIX)/bin + @chmod 0555 $(PREFIX)/bin/$(PROG) + @mkdir -p $(PREFIX)/share/doc/$(PROG) + @chmod 0755 $(PREFIX)/share $(PREFIX)/share/doc $(PREFIX)/share/doc/$(PROG) + @cp README.md $(PREFIX)/share/doc/$(PROG)/README + @chmod 0444 $(PREFIX)/share/doc/$(PROG)/README + + diff --git a/config b/config new file mode 100755 index 0000000..1df1524 --- /dev/null +++ b/config @@ -0,0 +1,82 @@ +#!/bin/sh + +usage() { + prog=$1 + cat << _EOF +${prog} [] + + can be one of the following: + +--prefix= The installation prefix (default: /usr). +--enable-debug Enable debug mode compilation (default: disabled). +--disable-allocator Disable use of internal memory allocator mechanism (default: enabled). +--enable-debug-stats Enable printing of some verbose debug info (default: disabled). +--help Display this help message. + +_EOF +} + +arg1=$1 +debug=0 +allocator=1 +debug_stats=0 +prefix=/usr +while [ "${arg1}" != "" ] +do + case "$arg1" in + --enable-debug) debug=1;; + --disable-allocator) allocator=0;; + --enable-debug-stats) debug_stats=1;; + --prefix=*) + pval=`echo ${arg1} | cut -f2 -d"="` + prefix=$pval + ;; + --help) usage $0;; + *) + echo "Unrecognized option: ${arg1}" + exit 1 + ;; + esac + shift + arg1=$1 +done + +if [ $debug -eq 1 ] +then + typ="DEBUG" +else + typ="RELEASE" +fi + +linkvar="LINK" +compilevar="COMPILE" +compilecppvar="COMPILE_cpp" +vecflagsvar="VEC_FLAGS" +loopoptflagsvar="LOOP_OPTFLAGS" +cppflagsvar="CPPFLAGS" +genoptvar="GEN_OPT" +rabinoptvar="RABIN_OPT" +noslabcppflagsvar="NO_SLAB_CPPFLAGS" +debugstatscppflagsvar="DEBUG_STATS_CPPFLAGS" +prefixvar="PREFIX" + +noslabcppflagsval= +debugstatscppflagsval= + +[ $allocator -eq 0 ] && noslabcppflagsval='\$\(NO_SLAB_CPPFLAGS\)' +[ $debug_stats -eq 1 ] && debugstatscppflagsval='\$\(DEBUG_STATS_CPPFLAGS\)' + +cat Makefile.in | sed " +s#@${linkvar}@#\\\$\\(${typ}_${linkvar}\\)#g +s#@${compilevar}@#\\\$\\(${typ}_${compilevar}\\)#g +s#@${compilecppvar}@#\\\$\\(${typ}_${compilecppvar}\\)#g +s#@${vecflagsvar}@#\\\$\\(${typ}_${vecflagsvar}\\)#g +s#@${loopoptflagsvar}@#\\\$\\(${typ}_${loopoptflagsvar}\\)#g +s#@${cppflagsvar}@#\\\$\\(${typ}_${cppflagsvar}\\)#g +s#@${genoptvar}@#\\\$\\(${typ}_${genoptvar}\\)#g +s#@${rabinoptvar}@#\\\$\\(${typ}_${rabinoptvar}\\)#g +s#@${noslabcppflagsvar}@#${noslabcppflagsval}#g +s#@${debugstatscppflagsvar}@#${debugstatscppflagsval}#g +s#@${prefixvar}@#${prefix}#g +" > Makefile + diff --git a/lzma/LzmaEnc.h b/lzma/LzmaEnc.h index 9ce63af..7049857 100644 --- a/lzma/LzmaEnc.h +++ b/lzma/LzmaEnc.h @@ -1,8 +1,3 @@ -/* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - /* LzmaEnc.h -- LZMA Encoder 2008-10-04 : Igor Pavlov : Public domain */ diff --git a/lzp/lzp.c b/lzp/lzp.c index ecece04..302c0cb 100644 --- a/lzp/lzp.c +++ b/lzp/lzp.c @@ -438,6 +438,11 @@ ssize_t lzp_decompress(const unsigned char * input, unsigned char * output, ssiz return (result == LZP_NO_ERROR) ? dataSize : result; } +/* + * Counter-intuitively we use a larger hash (with better LZP compression) for lower global + * compression levels. So that LZP preprocessing plays along nicely with the primary + * compression algorithm being used and actually provides a benefit. + */ int lzp_hash_size(int level) { if (level > 7) { return (LZP_DEFAULT_LZPHASHSIZE + 2);