From 2da0d0950be960cb200804f43ba07009368c322e Mon Sep 17 00:00:00 2001 From: Moinak Ghosh Date: Sat, 26 Jan 2013 18:28:13 +0530 Subject: [PATCH] Use BLAKE2 parallel version for single-chunk archives (whole file in one chunk). Set decompression threads correctly for single-chunk archives. --- crypto/crypto_utils.c | 24 +++++++++++++++++------- crypto/crypto_utils.h | 2 +- libbsc_compress.c | 2 +- main.c | 18 +++++++++++++++--- pcompress.h | 2 +- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/crypto/crypto_utils.c b/crypto/crypto_utils.c index b0ade4e..3dcaa03 100644 --- a/crypto/crypto_utils.c +++ b/crypto/crypto_utils.c @@ -164,7 +164,7 @@ PKCS5_PBKDF2_HMAC(const char *pass, int passlen, #endif int -compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, uint64_t bytes) +compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, uint64_t bytes, int mt) { DEBUG_STAT_EN(double strt, en); @@ -174,12 +174,22 @@ compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, uint64_t bytes) *ck = lzma_crc64(buf, bytes, 0); } else if (cksum == CKSUM_BLAKE256) { - if (bdsp.blake2b(cksum_buf, buf, NULL, 32, bytes, 0) != 0) - return (-1); + if (!mt) { + if (bdsp.blake2b(cksum_buf, buf, NULL, 32, bytes, 0) != 0) + return (-1); + } else { + if (bdsp.blake2bp(cksum_buf, buf, NULL, 32, bytes, 0) != 0) + return (-1); + } } else if (cksum == CKSUM_BLAKE512) { - if (bdsp.blake2b(cksum_buf, buf, NULL, 64, bytes, 0) != 0) - return (-1); + if (!mt) { + if (bdsp.blake2b(cksum_buf, buf, NULL, 64, bytes, 0) != 0) + return (-1); + } else { + if (bdsp.blake2bp(cksum_buf, buf, NULL, 64, bytes, 0) != 0) + return (-1); + } } else if (cksum == CKSUM_SKEIN256) { Skein_512_Ctxt_t ctx; @@ -683,10 +693,10 @@ init_crypto(crypto_ctx_t *cctx, uchar_t *pwd, int pwd_len, int crypto_alg, b += 4; *((uint32_t *)&sb[b]) = getpid(); b += 4; - compute_checksum(&sb[b], CKSUM_SHA256, sb, b); + compute_checksum(&sb[b], CKSUM_SHA256, sb, b, 0); b = 8 + 4; *((uint32_t *)&sb[b]) = rand(); - compute_checksum(salt, CKSUM_SHA256, &sb[b], 32 + 4); + compute_checksum(salt, CKSUM_SHA256, &sb[b], 32 + 4, 0); } } diff --git a/crypto/crypto_utils.h b/crypto/crypto_utils.h index 1a0da09..2f73a11 100644 --- a/crypto/crypto_utils.h +++ b/crypto/crypto_utils.h @@ -81,7 +81,7 @@ typedef struct { /* * Generic message digest functions. */ -int compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, uint64_t bytes); +int compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, uint64_t bytes, int mt); void list_checksums(FILE *strm, char *pad); int get_checksum_props(const char *name, int *cksum, int *cksum_bytes, int *mac_bytes, int accept_compatible); diff --git a/libbsc_compress.c b/libbsc_compress.c index 3485ae2..d7a2b76 100644 --- a/libbsc_compress.c +++ b/libbsc_compress.c @@ -67,7 +67,7 @@ libbsc_stats(int show) } /* - * BSC uses OpenMP where it is tricky to control thread count + * BSC uses OpenMP where it does not control thread count * deterministically. We only use multithread capability in BSC * when compressing entire file in a single chunk. */ diff --git a/main.c b/main.c index b450e10..a2e48fc 100644 --- a/main.c +++ b/main.c @@ -572,7 +572,7 @@ redo: * If it does not match we set length of chunk to 0 to indicate * exit to the writer thread. */ - compute_checksum(checksum, cksum, tdat->uncompressed_chunk, _chunksize); + compute_checksum(checksum, cksum, tdat->uncompressed_chunk, _chunksize, tdat->cksum_mt); if (memcmp(checksum, tdat->checksum, cksum_bytes) != 0) { tdat->len_cmp = 0; fprintf(stderr, "ERROR: Chunk %d, checksums do not match.\n", tdat->id); @@ -937,6 +937,8 @@ start_decompress(const char *filename, const char *to_filename) nthreads = nprocs; set_threadcounts(&props, &nthreads, nprocs, DECOMPRESS_THREADS); + if (props.is_single_chunk) + nthreads = 1; fprintf(stderr, "Scaling to %d thread", nthreads * props.nthreads); if (nthreads * props.nthreads > 1) fprintf(stderr, "s"); fprintf(stderr, "\n"); @@ -959,6 +961,10 @@ start_decompress(const char *filename, const char *to_filename) tdat->compress = _compress_func; tdat->decompress = _decompress_func; tdat->cancel = 0; + if (props.is_single_chunk) + tdat->cksum_mt = 1; + else + tdat->cksum_mt = 0; tdat->level = level; tdat->data = NULL; tdat->props = &props; @@ -1196,7 +1202,8 @@ redo: * back into cmp_seg. Avoids an extra memcpy(). */ if (!encrypt_type) - compute_checksum(tdat->checksum, cksum, tdat->cmp_seg, tdat->rbytes); + compute_checksum(tdat->checksum, cksum, tdat->cmp_seg, tdat->rbytes, + tdat->cksum_mt); rctx = tdat->rctx; reset_dedupe_context(tdat->rctx); @@ -1211,7 +1218,8 @@ redo: * Compute checksum of original uncompressed chunk. */ if (!encrypt_type) - compute_checksum(tdat->checksum, cksum, tdat->uncompressed_chunk, tdat->rbytes); + compute_checksum(tdat->checksum, cksum, tdat->uncompressed_chunk, + tdat->rbytes, tdat->cksum_mt); } /* @@ -1692,6 +1700,10 @@ start_compress(const char *filename, uint64_t chunksize, int level) tdat->decompress = _decompress_func; tdat->uncompressed_chunk = (uchar_t *)1; tdat->cancel = 0; + if (single_chunk) + tdat->cksum_mt = 1; + else + tdat->cksum_mt = 0; tdat->level = level; tdat->data = NULL; tdat->props = &props; diff --git a/pcompress.h b/pcompress.h index 042538f..112b1d0 100644 --- a/pcompress.h +++ b/pcompress.h @@ -184,7 +184,7 @@ struct cmp_data { uint64_t chunksize; uint64_t len_cmp, len_cmp_be; uchar_t checksum[CKSUM_MAX_BYTES]; - int level; + int level, cksum_mt; unsigned int id; compress_func_ptr compress; compress_func_ptr decompress;