greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Make Pcompress functionality into a library - initial changes.

Browse source
This commit is contained in:
Moinak Ghosh 2013-06-02 20:54:33 +05:30
parent 8db0bef184
commit c859cf35d5
7 changed files with 3004 additions and 2694 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
Makefile.in
View file

@ -23,13 +23,20 @@
#
PROG= pcompress
MAINSRCS = main.c utils/utils.c allocator.c lzma_compress.c ppmd_compress.c \
LIB=libpcompress.so
LINKLIB=pcompress
LIBVER=1
MAINSRCS = utils/utils.c allocator.c lzma_compress.c ppmd_compress.c \
adaptive_compress.c lzfx_compress.c lz4_compress.c none_compress.c \
utils/xxhash_base.c utils/heap.c utils/cpuid.c
utils/xxhash_base.c utils/heap.c utils/cpuid.c pcompress.c
MAINHDRS = allocator.h pcompress.h utils/utils.h utils/xxhash.h utils/heap.h \
utils/cpuid.h utils/xxhash.h
MAINOBJS = $(MAINSRCS:.c=.o)
PROGSRCS = main.c
PROGHDRS = pcompress.h utils/utils.h
PROGOBJS = $(PROGSRCS:.c=.o)
XSALSA20_STREAM_C = crypto/xsalsa20/stream.c
XSALSA20_STREAM_ASM = crypto/xsalsa20/stream.s
XSALSA20_DEBUG = -DSALSA20_DEBUG
@ -190,25 +197,26 @@ COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT
COMMON_VEC_FLAGS = -ftree-vectorize
COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block
LDLIBS = -ldl -L./buildtmp -Wl,-R@LIBBZ2_DIR@ -lbz2 -L./buildtmp -Wl,-R@LIBZ_DIR@ -lz -lm @LIBBSCLFLAGS@ \
-L./buildtmp -Wl,-R@OPENSSL_LIBDIR@ -lcrypto -lrt $(EXTRA_LDFLAGS)
-L./buildtmp -Wl,-R@OPENSSL_LIBDIR@ -lcrypto -lrt $(EXTRA_LDFLAGS) -Wl,-R/usr/lib,--enable-new-dtags \
-Wl,-R/usr/lib64,--enable-new-dtags -Wl,-R.,--enable-new-dtags
OBJS = $(MAINOBJS) $(LZMAOBJS) $(PPMDOBJS) $(LZFXOBJS) $(LZ4OBJS) $(CRCOBJS) \
$(RABINOBJS) $(BSDIFFOBJS) $(LZPOBJS) $(DELTA2OBJS) @LIBBSCWRAPOBJ@ $(SKEINOBJS) \
$(SKEIN_BLOCK_OBJ) @SHA2ASM_OBJS@ @SHA2_OBJS@ $(KECCAK_OBJS) $(KECCAK_OBJS_ASM) \
$(TRANSP_OBJS) $(CRYPTO_OBJS) $(ZLIB_OBJS) $(BZLIB_OBJS) $(XXHASH_OBJS) $(BLAKE2_OBJS) \
@CRYPTO_COMPAT_OBJS@ $(CRYPTO_ASM_OBJS)
DEBUG_LINK = g++ -pthread @LIBBSCGEN_OPT@ @EXTRA_OPT_FLAGS@ -fopenmp
DEBUG_COMPILE = gcc -g -c @EXTRA_OPT_FLAGS@
DEBUG_COMPILE_cpp = g++ -g -c @EXTRA_OPT_FLAGS@
DEBUG_LINK = g++ -pthread @LIBBSCGEN_OPT@ @EXTRA_OPT_FLAGS@ -fopenmp -fPIC
DEBUG_COMPILE = gcc -g -c @EXTRA_OPT_FLAGS@ -fPIC
DEBUG_COMPILE_cpp = g++ -g -c @EXTRA_OPT_FLAGS@ -fPIC
DEBUG_VEC_FLAGS =
DEBUG_LOOP_OPTFLAGS =
DEBUG_GEN_OPT = -O -fno-omit-frame-pointer @LIBBSCGEN_OPT@ -fopenmp
DEBUG_CPPFLAGS = $(COMMON_CPPFLAGS)
DEBUG_FPTR_FLAG =
RELEASE_LINK = g++ -pthread @LIBBSCGEN_OPT@ @EXTRA_OPT_FLAGS@ -fopenmp
RELEASE_COMPILE = gcc -c @EXTRA_OPT_FLAGS@
RELEASE_COMPILE_cpp = g++ -c @EXTRA_OPT_FLAGS@
RELEASE_LINK = g++ -pthread @LIBBSCGEN_OPT@ @EXTRA_OPT_FLAGS@ -fopenmp -fPIC
RELEASE_COMPILE = gcc -c @EXTRA_OPT_FLAGS@ -fPIC
RELEASE_COMPILE_cpp = g++ -c @EXTRA_OPT_FLAGS@ -fPIC
RELEASE_VEC_FLAGS = $(COMMON_VEC_FLAGS)
RELEASE_LOOP_OPTFLAGS = $(COMMON_LOOP_OPTFLAGS)
RELEASE_CPPFLAGS = $(COMMON_CPPFLAGS) -DNDEBUG
@ -218,7 +226,8 @@ RELEASE_FPTR_FLAG = -fomit-frame-pointer
NO_SLAB_CPPFLAGS = -DDEBUG_NO_SLAB
DEBUG_STATS_CPPFLAGS = -DDEBUG_STATS
LINK = @LINK@
LINK.PROG = @LINK@
LINK.LIB = @LINK@ -shared
COMPILE = @COMPILE@
COMPILE_cpp = @COMPILE_cpp@
VEC_FLAGS = @VEC_FLAGS@
@ -324,14 +333,21 @@ $(BLAKE2_OBJS): $(BLAKE2_SRCS) $(BLAKE2_BASE_SRCS) $(BLAKE2_HDRS)
$(MAINOBJS): $(MAINSRCS) $(MAINHDRS)
$(COMPILE) $(GEN_OPT) $(LOOP_OPTFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
$(PROG): $(OBJS)
$(LINK) -o $@ $(OBJS) $(LDLIBS)
$(PROGOBJS): $(PROGSRCS) $(PROGHDRS)
$(COMPILE) $(GEN_OPT) $(LOOP_OPTFLAGS) $(CPPFLAGS) $(@:.o=.c) -o $@
$(LIB): $(OBJS)
$(LINK.LIB) -Wl,-soname,$(LIB).$(LIBVER) -o $(LIB).$(LIBVER) $(OBJS) $(LDLIBS)
ln -sf $(LIB).$(LIBVER) $(LIB)
$(PROG): $(LIB) $(PROGOBJS)
$(LINK.PROG) -o $@ $(PROGOBJS) $(LDLIBS) -L. -l$(LINKLIB)
test: all
(cd test; ulimit -c unlimited; sh ./run_test.sh $(TESTSUITE) ) 2>&1 | tee test.log
clean:
$(RM) $(PROG) $(OBJS) $(BAKFILES)
$(RM) $(PROG) $(OBJS) $(BAKFILES) $(LIB) $(LIB).$(LIBVER)
$(RM) test.log
$(RM_RF) test/datafiles
$(RM_RF) buildtmp
@ -344,8 +360,15 @@ install: $(PROG)
@chmod 0755 $(DESTDIR)$(PREFIX)/bin
@cp $(PROG) $(DESTDIR)$(PREFIX)/bin
@chmod 0555 $(DESTDIR)$(PREFIX)/bin/$(PROG)
@mkdir -p $(DESTDIR)$(PREFIX)/lib
@chmod 0755 $(DESTDIR)$(PREFIX)/lib
@cp $(LIB).$(LIBVER) $(DESTDIR)$(PREFIX)/lib
@chmod 0555 $(DESTDIR)$(PREFIX)/lib/$(LIB).$(LIBVER)
@ln -sf $(LIB).$(LIBVER) $(DESTDIR)$(PREFIX)/lib/$(LIB)
@mkdir -p $(DESTDIR)$(PREFIX)/share/doc/$(PROG)
@chmod 0755 $(DESTDIR)$(PREFIX)/share $(PREFIX)/share/doc $(PREFIX)/share/doc/$(PROG)
@chmod 0755 $(DESTDIR)$(PREFIX)/share $(DESTDIR)$(PREFIX)/share/doc $(DESTDIR)$(PREFIX)/share/doc/$(PROG)
@cp README.md $(DESTDIR)$(PREFIX)/share/doc/$(PROG)/README
@chmod 0444 $(DESTDIR)$(PREFIX)/share/doc/$(PROG)/README

2
crypto/crypto_utils.c
View file

@ -988,7 +988,7 @@ get_pw_string(uchar_t pw[MAX_PW_LEN], const char *prompt, int twice)
s = fgets(pw1, MAX_PW_LEN, input);
fputs("\n", stderr);
if (s == NULL) {
if (s == NULL || strlen(s) < 2) {
tcsetattr(fd, TCSANOW, &oldt);
fflush(strm);
return (-1);

2693
main.c
View file

File diff suppressed because it is too large Load diff

2885
pcompress.c Normal file
View file

File diff suppressed because it is too large Load diff

52
pcompress.h
View file

@ -66,7 +66,7 @@ extern "C" {
*/
#define COMPRESSED_CHUNKSZ (sizeof (uint64_t))
#define ORIGINAL_CHUNKSZ (sizeof (uint64_t))
#define CHUNK_HDR_SZ (COMPRESSED_CHUNKSZ + cksum_bytes + ORIGINAL_CHUNKSZ + CHUNK_FLAG_SZ)
#define CHUNK_HDR_SZ (COMPRESSED_CHUNKSZ + pctx->cksum_bytes + ORIGINAL_CHUNKSZ + CHUNK_FLAG_SZ)
/*
* lower 3 bits in higher nibble indicate chunk compression algorithm
@ -174,6 +174,44 @@ extern int libbsc_deinit(void **data);
extern void libbsc_stats(int show);
#endif
typedef struct pc_ctx {
compress_func_ptr _compress_func;
compress_func_ptr _decompress_func;
init_func_ptr _init_func;
deinit_func_ptr _deinit_func;
stats_func_ptr _stats_func;
props_func_ptr _props_func;
int inited;
int main_cancel;
int adapt_mode;
int pipe_mode, pipe_out;
int nthreads;
int hide_mem_stats;
int hide_cmp_stats;
int enable_rabin_scan;
int enable_rabin_global;
int enable_delta_encode;
int enable_delta2_encode;
int enable_rabin_split;
int enable_fixed_scan;
int lzp_preprocess;
int encrypt_type;
unsigned int chunk_num;
uint64_t largest_chunk, smallest_chunk, avg_chunk;
uint64_t chunksize;
const char *exec_name, *algo, *filename, *to_filename;
int do_compress, level;
int do_uncompress;
int cksum_bytes, mac_bytes;
int cksum, t_errored;
int rab_blk_size, keylen;
crypto_ctx_t crypto_ctx;
unsigned char *user_pw;
int user_pw_len;
char *pwd_file, *f_name;
} pc_ctx_t;
/*
* Per-thread data structure for compression and decompression threads.
*/
@ -200,8 +238,20 @@ struct cmp_data {
mac_ctx_t chunk_hmac;
algo_props_t *props;
int decompressing;
pc_ctx_t *pctx;
};
void usage(pc_ctx_t *pctx);
pc_ctx_t *create_pc_context(void);
int init_pc_context_argstr(pc_ctx_t *pctx, char *args);
int init_pc_context(pc_ctx_t *pctx, int argc, char *argv[]);
void destroy_pc_context(pc_ctx_t *pctx);
void pc_set_userpw(pc_ctx_t *pctx, unsigned char *pwdata, int pwlen);
int start_pcompress(pc_ctx_t *pctx);
int start_compress(pc_ctx_t *pctx, const char *filename, uint64_t chunksize, int level);
int start_decompress(pc_ctx_t *pctx, const char *filename, const char *to_filename);
#ifdef __cplusplus
}
#endif

14
utils/utils.c
View file

@ -70,6 +70,20 @@ err_exit(int show_errno, const char *format, ...)
exit(1);
}
void
err_print(int show_errno, const char *format, ...)
{
int err = errno;
va_list args;
va_start(args, format);
vfprintf(stderr, format, args);
va_end(args);
if (show_errno)
fprintf(stderr, "\nError: %s\n", strerror(err));
}
/*
* Fetch the command name that started the current process.
* The returned string must be freed by the caller.

1
utils/utils.h
View file

@ -211,6 +211,7 @@ extern processor_info_t proc_info;
#endif
extern void err_exit(int show_errno, const char *format, ...);
extern void err_print(int show_errno, const char *format, ...);
extern const char *get_execname(const char *);
extern int parse_numeric(int64_t *val, const char *str);
extern char *bytes_to_size(uint64_t bytes);