diff --git a/INSTALL b/INSTALL index e73b67b..418eaca 100644 --- a/INSTALL +++ b/INSTALL @@ -17,7 +17,14 @@ of all the options by running: ./config --help -A more descriptive account is given below: +NOTE: Basic Installation does not enable support for Libbsc, a + new block-sorting compressor (similar to but better than + Bzip2). See below for details. + +Custom Installation +=================== +The options to the config script are detailed below. Note that this is +not the usual GNU Autoconf script. ./config [] @@ -28,19 +35,43 @@ A more descriptive account is given below: prefix needs to eb used during packaging. --enable-debug Enable debug mode compilation. - This reduces the compiler optimization level to basic - and taks out all the loop optimization flags. This is - primary to aid debugging. + This reduces the compiler optimization level to + basic and taks out all the loop optimization flags. + This is primarily to aid debugging. --disable-allocator Disable use of internal memory allocator mechanism. - The internal allocator can be totally disabled by setting - this build time flag. It is also possible to dynamically - disable the allocator by setting the following env variable: + The internal allocator can be totally disabled by + setting this build time flag. It is also possible + to dynamically disable the allocator by setting the + following env variable: ALLOCATOR_BYPASS=1 --enable-debug-stats Enable printing of some verbose debug info. This at present shows some info related to Dedupe efficiency. +--with-libbsc= + Enable support for libbsc (See: libbsc.com). Full + path to the libbsc source tree must be provided. It + links the library statically. + --help Display the help message. +Steps for building with libbsc support +====================================== +1) Download libbsc source from: http://libbsc.com/ . Click on the + "Download TAR Ball" button. Downloading the 3.1.0 release from the + Github project download page will Not work. + +2) Extract the gzippped tarball. It will create a directory for example: + IlyaGrebnov-libbsc-0b12f29 + +3) Now cd into the pcompress directory and execute the config script + with the full path to the libbsc source directory. For example: + + ./config --with-libbsc=/full/path/to/IlyaGrebnov-libbsc-0b12f29 + +4) Now run make in the pcompress directory. This will also run make in + the libbsc source directory to build it. + + diff --git a/Makefile.in b/Makefile.in index 2f4adba..078cdaf 100644 --- a/Makefile.in +++ b/Makefile.in @@ -62,34 +62,42 @@ LZPSRCS = lzp/lzp.c LZPHDRS = lzp/lzp.h LZPOBJS = $(LZPSRCS:.c=.o) +LIBBSCWRAP = libbsc_compress.c +LIBBSCWRAPOBJ = libbsc_compress.o +LIBBSCDIR = @LIBBSCDIR@ +LIBBSCLFLAGS = -L$(LIBBSCDIR) -lbsc +LIBBSCLIB = @LIBBSCLIB@ +LIBBSCGEN_OPT = -fopenmp +LIBBSCCPPFLAGS = -I$(LIBBSCDIR)/libbsc -DENABLE_PC_LIBBSC + BAKFILES = *~ lzma/*~ lzfx/*~ lz4/*~ rabin/*~ bsdiff/*~ lzp/*~ RM = rm -f COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT_PROPS \ -DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32 \ - -I./lzp + -I./lzp @LIBBSCCPPFLAGS@ COMMON_VEC_FLAGS = -ftree-vectorize COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block -LDLIBS = -ldl -lbz2 $(ZLIB_DIR) -lz -lm +LDLIBS = -ldl -lbz2 $(ZLIB_DIR) -lz -lm @LIBBSCLFLAGS@ OBJS = $(MAINOBJS) $(LZMAOBJS) $(PPMDOBJS) $(LZFXOBJS) $(LZ4OBJS) $(CRCOBJS) \ -$(RABINOBJS) $(BSDIFFOBJS) $(LZPOBJS) +$(RABINOBJS) $(BSDIFFOBJS) $(LZPOBJS) @LIBBSCWRAPOBJ@ -DEBUG_LINK = g++ -m64 -pthread -msse3 +DEBUG_LINK = g++ -m64 -pthread -msse3 @LIBBSCGEN_OPT@ DEBUG_COMPILE = gcc -m64 -g -msse3 -c DEBUG_COMPILE_cpp = g++ -m64 -g -msse3 -c DEBUG_VEC_FLAGS = DEBUG_LOOP_OPTFLAGS = -DEBUG_GEN_OPT = -O -fno-omit-frame-pointer +DEBUG_GEN_OPT = -O -fno-omit-frame-pointer @LIBBSCGEN_OPT@ DEBUG_RABIN_OPT = -O -fno-omit-frame-pointer DEBUG_CPPFLAGS = $(COMMON_CPPFLAGS) -RELEASE_LINK = g++ -m64 -pthread -msse3 +RELEASE_LINK = g++ -m64 -pthread -msse3 @LIBBSCGEN_OPT@ RELEASE_COMPILE = gcc -m64 -msse3 -c RELEASE_COMPILE_cpp = g++ -m64 -msse3 -c RELEASE_VEC_FLAGS = $(COMMON_VEC_FLAGS) RELEASE_LOOP_OPTFLAGS = $(COMMON_LOOP_OPTFLAGS) RELEASE_CPPFLAGS = $(COMMON_CPPFLAGS) -DNDEBUG -RELEASE_GEN_OPT = -O3 +RELEASE_GEN_OPT = -O3 @LIBBSCGEN_OPT@ RELEASE_RABIN_OPT = -O2 NO_SLAB_CPPFLAGS = -DDEBUG_NO_SLAB @@ -131,6 +139,12 @@ $(LZ4OBJS): $(LZ4SRCS) $(LZ4HDRS) $(LZPOBJS): $(LZPSRCS) $(LZPHDRS) $(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@ +$(LIBBSCLIB): + (cd $(LIBBSCDIR); make) + +$(LIBBSCWRAPOBJ): $(LIBBSCWRAP) $(LIBBSCLIB) + $(COMPILE) $(GEN_OPT) $(VEC_FLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@ + $(MAINOBJS): $(MAINSRCS) $(MAINHDRS) $(COMPILE) $(GEN_OPT) $(LOOP_OPTFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@ diff --git a/README.md b/README.md index 8c23e80..4457178 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,11 @@ Usage bzip2 - Bzip2 Algorithm from libbzip2. ppmd - The PPMd algorithm excellent for textual data. PPMd requires at least 64MB X CPUs more memory than the other modes. + + libbsc - A Block Sorting Compressor using the Burrows Wheeler Transform + like Bzip2 but runs faster and gives better compression than + Bzip2 (See: libbsc.com). + adapt - Adaptive mode where ppmd or bzip2 will be used per chunk, depending on which one produces better compression. This mode is obviously fairly slow and requires lots of memory. @@ -58,6 +63,10 @@ Usage - Can be a number from 0 meaning minimum and 14 meaning maximum compression. +NOTE: The option "libbsc" uses Ilya Grebnov's block sorting compression library + from http://libbsc.com/ . It is only available if pcompress in built with + that library. See INSTALL file for details. + To decompress a file compressed using above command: pcompress -d diff --git a/config b/config index 1df1524..4457481 100755 --- a/config +++ b/config @@ -11,6 +11,9 @@ ${prog} [] --enable-debug Enable debug mode compilation (default: disabled). --disable-allocator Disable use of internal memory allocator mechanism (default: enabled). --enable-debug-stats Enable printing of some verbose debug info (default: disabled). +--with-libbsc= + Enable support for libbsc (See: libbsc.com). Full path to the libbsc + source tree must be provided. It links the library statically. --help Display this help message. _EOF @@ -21,6 +24,13 @@ debug=0 allocator=1 debug_stats=0 prefix=/usr +libbsc_dir= +libbsc_lib= +libbsclflags= +libbscwrapobj= +libbscgenopt= +libbsccppflags= + while [ "${arg1}" != "" ] do case "$arg1" in @@ -31,6 +41,20 @@ do pval=`echo ${arg1} | cut -f2 -d"="` prefix=$pval ;; + --with-libbsc=*) + path=`echo ${arg1} | cut -f2 -d"="` + if [ -f ${path}/bsc.cpp -a -f ${path}/libbsc/libbsc.h ] + then + libbsc_dir="${path}" + libbsc_lib="${path}/libbsc.a" + libbsclflags='\$\(LIBBSCLFLAGS\)' + libbscwrapobj='\$\(LIBBSCWRAPOBJ\)' + libbscgenopt='\$\(LIBBSCGEN_OPT\)' + libbsccppflags='\$\(LIBBSCCPPFLAGS\)' + else + echo "Libbsc not found in ${path}, not enabling libbsc support.\n" + fi + ;; --help) usage $0;; *) echo "Unrecognized option: ${arg1}" @@ -60,6 +84,13 @@ noslabcppflagsvar="NO_SLAB_CPPFLAGS" debugstatscppflagsvar="DEBUG_STATS_CPPFLAGS" prefixvar="PREFIX" +libbscdirvar="LIBBSCDIR" +libbsclibvar="LIBBSCLIB" +libbsclflagsvar="LIBBSCLFLAGS" +libbscwrapobjvar="LIBBSCWRAPOBJ" +libbscgenoptvar="LIBBSCGEN_OPT" +libbsccppflagsvar="LIBBSCCPPFLAGS" + noslabcppflagsval= debugstatscppflagsval= @@ -78,5 +109,11 @@ s#@${rabinoptvar}@#\\\$\\(${typ}_${rabinoptvar}\\)#g s#@${noslabcppflagsvar}@#${noslabcppflagsval}#g s#@${debugstatscppflagsvar}@#${debugstatscppflagsval}#g s#@${prefixvar}@#${prefix}#g +s#@${libbscdirvar}@#${libbsc_dir}#g +s#@${libbsclibvar}@#${libbsc_lib}#g +s#@${libbsclflagsvar}@#${libbsclflags}#g +s#@${libbscwrapobjvar}@#${libbscwrapobj}#g +s#@${libbscgenoptvar}@#${libbscgenopt}#g +s#@${libbsccppflagsvar}@#${libbsccppflags}#g " > Makefile diff --git a/libbsc_compress.c b/libbsc_compress.c new file mode 100644 index 0000000..fe7151d --- /dev/null +++ b/libbsc_compress.c @@ -0,0 +1,171 @@ +/* + * This file is a part of Pcompress, a chunked parallel multi- + * algorithm lossless compression and decompression program. + * + * Copyright (C) 2012 Moinak Ghosh. All rights reserved. + * Use is subject to license terms. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * moinakg@belenix.org, http://moinakg.wordpress.com/ + * + * This program includes partly-modified public domain source + * code from the LZMA SDK: http://www.7-zip.org/sdk.html + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +// 1G +#define BSC_MAX_CHUNK 1073741824L + +struct libbsc_params { + int lzpHashSize; + int lzpMinLen; + int bscCoder; + int features; +}; + +static void +libbsc_err(int err) { + switch (err) { + case LIBBSC_BAD_PARAMETER: + fprintf(stderr, "LIBBSC: Bad Parameter.\n"); + break; + case LIBBSC_NOT_ENOUGH_MEMORY: + fprintf(stderr, "LIBBSC: Out of memory.\n"); + break; + case LIBBSC_NOT_SUPPORTED: + fprintf(stderr, "LIBBSC: Using unsupported feature.\n"); + break; + case LIBBSC_UNEXPECTED_EOB: + fprintf(stderr, "LIBBSC: Unexpected end of block.\n"); + break; + case LIBBSC_DATA_CORRUPT: + fprintf(stderr, "LIBBSC: Corrupt data.\n"); + break; + } +} + +void +libbsc_stats(int show) +{ +} + +/* + * BSC uses OpenMP where it is tricky to control thread count + * deterministically. We only use multithread capability in BSC + * when compressing entire file in a single chunk. + */ +void +libbsc_props(algo_props_t *data, int level, ssize_t chunksize) { + data->compress_mt_capable = 0; + data->decompress_mt_capable = 0; + data->single_chunk_mt_capable = 1; + data->buf_extra = 0; + data->c_max_threads = 8; + data->d_max_threads = 8; +} + +int +libbsc_init(void **data, int *level, int nthreads, ssize_t chunksize) +{ + struct libbsc_params *bscdat; + int rv; + + if (chunksize > BSC_MAX_CHUNK) { + fprintf(stderr, "Max allowed chunk size for LIBBSC is: %d \n", + BSC_MAX_CHUNK); + return (1); + } + bscdat = slab_alloc(NULL, sizeof (struct libbsc_params)); + + bscdat->features = LIBBSC_FEATURE_FASTMODE; + if (nthreads > 1) + bscdat->features |= LIBBSC_FEATURE_MULTITHREADING; + + if (*level > 9) *level = 9; + + bscdat->lzpHashSize = LIBBSC_DEFAULT_LZPHASHSIZE + (*level - 1); + bscdat->bscCoder = LIBBSC_CODER_QLFC_STATIC; + if (*level == 0) { + bscdat->lzpMinLen = 32; + + } else if (*level < 3) { + bscdat->lzpMinLen = 64; + + } else if (*level < 5) { + bscdat->lzpMinLen = 128; + bscdat->bscCoder = LIBBSC_CODER_QLFC_ADAPTIVE; + + } else { + bscdat->lzpMinLen = 200; + bscdat->bscCoder = LIBBSC_CODER_QLFC_ADAPTIVE; + } + *data = bscdat; + rv = bsc_init(bscdat->features); + if (rv != LIBBSC_NO_ERROR) { + libbsc_err(rv); + return (-1); + } + + return (0); +} + +int +libbsc_deinit(void **data) +{ + struct libbsc_params *bscdat = (struct libbsc_params *)(*data); + + if (bscdat) { + slab_free(NULL, bscdat); + } + *data = NULL; + return (0); +} + +int +libbsc_compress(void *src, size_t srclen, void *dst, size_t *dstlen, + int level, uchar_t chdr, void *data) +{ + int rv; + struct libbsc_params *bscdat = (struct libbsc_params *)data; + + rv = bsc_compress(src, dst, srclen, bscdat->lzpHashSize, bscdat->lzpMinLen, + LIBBSC_BLOCKSORTER_BWT, bscdat->bscCoder, bscdat->features); + if (rv < 0) { + libbsc_err(rv); + return (-1); + } + *dstlen = rv; + return (0); +} + +int +libbsc_decompress(void *src, size_t srclen, void *dst, size_t *dstlen, + int level, uchar_t chdr, void *data) +{ + int rv; + struct libbsc_params *bscdat = (struct libbsc_params *)data; + + rv = bsc_decompress(src, srclen, dst, *dstlen, bscdat->features); + if (rv != LIBBSC_NO_ERROR) { + libbsc_err(rv); + return (-1); + } + return (0); +} diff --git a/main.c b/main.c index da8101a..fcc48c6 100644 --- a/main.c +++ b/main.c @@ -107,6 +107,11 @@ usage(void) " bzip2 - Bzip2 Algorithm from libbzip2.\n" " ppmd - The PPMd algorithm excellent for textual data. PPMd requires\n" " at least 64MB X CPUs more memory than the other modes.\n" +#ifdef ENABLE_PC_LIBBSC + " libbsc - A Block Sorting Compressor using the Burrows Wheeler Transform\n" + " like Bzip2 but runs faster and gives better compression than\n" + " Bzip2 (See: libbsc.com).\n" +#endif " adapt - Adaptive mode where ppmd or bzip2 will be used per chunk,\n" " depending on which one produces better compression. This mode\n" " is obviously fairly slow and requires lots of memory.\n" @@ -1034,6 +1039,8 @@ start_compress(const char *filename, uint64_t chunksize, int level) enable_rabin_split = 0; // Do not split for whole files. nthreads = 1; single_chunk = 1; + props.is_single_chunk = 1; + flags |= FLAG_SINGLE_CHUNK; } else { if (nthreads == 0 || nthreads > sbuf.st_size / chunksize) { nthreads = sbuf.st_size / chunksize; @@ -1471,6 +1478,17 @@ init_algo(const char *algo, int bail) _stats_func = adapt_stats; adapt_mode = 1; rv = 0; +#ifdef ENABLE_PC_LIBBSC + } else if (memcmp(algorithm, "libbsc", 6) == 0) { + _compress_func = libbsc_compress; + _decompress_func = libbsc_decompress; + _init_func = libbsc_init; + _deinit_func = libbsc_deinit; + _stats_func = libbsc_stats; + _props_func = libbsc_props; + adapt_mode = 1; + rv = 0; +#endif } return (rv); diff --git a/pcompress.h b/pcompress.h index 899f4f4..7d8a490 100644 --- a/pcompress.h +++ b/pcompress.h @@ -39,6 +39,7 @@ extern "C" { #define MIN_CHUNK 2048 #define VERSION 3 #define FLAG_DEDUP 1 +#define FLAG_SINGLE_CHUNK 2 #define UTILITY_VERSION 0.7 #define COMPRESSED 1 @@ -132,6 +133,17 @@ extern void lz_fx_stats(int show); extern void lz4_stats(int show); extern void none_stats(int show); +#ifdef ENABLE_PC_LIBBSC +extern int libbsc_compress(void *src, size_t srclen, void *dst, + size_t *dstlen, int level, uchar_t chdr, void *data); +extern int libbsc_decompress(void *src, size_t srclen, void *dst, + size_t *dstlen, int level, uchar_t chdr, void *data); +extern int libbsc_init(void **data, int *level, int nthreads, ssize_t chunksize); +extern void libbsc_props(algo_props_t *data, int level, ssize_t chunksize); +extern int libbsc_deinit(void **data); +extern void libbsc_stats(int show); +#endif + /* * Per-thread data structure for compression and decompression threads. */ diff --git a/utils.c b/utils.c index a50f244..66dc194 100644 --- a/utils.c +++ b/utils.c @@ -284,5 +284,14 @@ set_threadcounts(algo_props_t *props, int *nthreads, int nprocs, algo_threads_ty } } *nthreads = nthreads1; + + } else if (props->single_chunk_mt_capable && props->is_single_chunk) { + *nthreads = 1; + if (typ == COMPRESS_THREADS) + props->nthreads = props->c_max_threads; + else + props->nthreads = props->d_max_threads; + if (props->nthreads > nprocs) + props->nthreads = nprocs; } } diff --git a/utils.h b/utils.h index 05c4c8e..10347b6 100644 --- a/utils.h +++ b/utils.h @@ -105,6 +105,8 @@ typedef struct { uint32_t buf_extra; int compress_mt_capable; int decompress_mt_capable; + int single_chunk_mt_capable; + int is_single_chunk; int nthreads; int c_max_threads; int d_max_threads; @@ -177,6 +179,8 @@ init_algo_props(algo_props_t *props) props->buf_extra = 0; props->compress_mt_capable = 0; props->decompress_mt_capable = 0; + props->single_chunk_mt_capable = 0; + props->is_single_chunk = 0; props->nthreads = 1; props->c_max_threads = 1; props->d_max_threads = 1;