/* * This file is a part of Pcompress, a chunked parallel multi- * algorithm lossless compression and decompression program. * * Copyright (C) 2012-2014 Moinak Ghosh. All rights reserved. * Use is subject to license terms. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 3 of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this program. * If not, see . * * moinakg@belenix.org, http://moinakg.wordpress.com/ * */ #include #include #include #include #include #include #include #include #include #include #include #include "utils/utils.h" #include "allocator.h" #include "meta_stream.h" extern int lz4_compress(void *src, uint64_t srclen, void *dst, uint64_t *dstlen, int level, uchar_t chdr, int btype, void *data); enum { SRC_CHANNEL = 0, SINK_CHANNEL }; /* * Metadata chunk header format: * 64-bit integer = 1: Compressed length: This indicates that this is a metadata chunk * 64-bit integer: Real Compressed length * 64-bit integer: Uncompressed original length * 1 Byte: Chunk flag * Upto 64-bytes: Checksum, HMAC if encrypting * 32-bit integer: Header CRC32 if not encrypting */ static int compress_and_write(meta_ctx_t *mctx) { pc_ctx_t pctx = mctx->pctx; uchar_t type; uchar_t *comp_chunk, *tobuf; int rv; uint64_t dstlen; int64_t wbytes; /* * Plain checksum if not encrypting. * This place will hold HMAC if encrypting. */ if (!pctx->encrypt_type) { compute_checksum(mctx->checksum, pctx->cksum, mctx->frombuf, mctx->frompos, 0, 1); } type = COMPRESSED; U64_P(mctx->tobuf) = LE64(1); // Indicate metadata chunk tobuf = mctx->tobuf + 8; comp_chunk = mctx->tobuf + METADATA_HDR_SZ; dstlen = METADATA_CHUNK_SIZE; /* * Ok, now compress. */ rv = lz4_compress(mctx->frombuf, mctx->frompos, comp_chunk, &dstlen, mctx->level, 0, TYPE_BINARY, mctx->lz4_dat); if (rv < 0) { type = UNCOMPRESSED; memcpy(comp_chunk, mctx->frombuf, mctx->frompos); } if (pctx->encrypt_type) { rv = crypto_buf(&(pctx->crypto_ctx), comp_chunk, comp_chunk, dstlen, 255); if (rv == -1) { pctx->main_cancel = 1; pctx->t_errored = 1; log_msg(LOG_ERR, 0, "Metadata Encrypion failed") return (0); } } /* * Add header size to the compressed length minus the initial 64-bit value. * That one is a flag value which is read separately during decompression. */ dstlen += METADATA_HDR_SZ - COMPRESSED_CHUNKSIZE; U64_P(mctx->tobuf + 8) = LE64(dstlen); U64_P(mctx->tobuf + 16) = LE64(mctx->frompos); if (!pctx->encrypt_type) serialize_checksum(mctx->checksum, mctx->tobuf + 24, pctx->cksum_bytes); if (pctx->encrypt_type) { uchar_t mac_ptr; uchar_t chash[pctx->mac_bytes]; unsigned int hlen; mac_ptr = mctx->tobuf + 24 + pctx->cksum_bytes; // cksum_bytes will be 0 here but ... memset(mac_ptr, 0, pctx->mac_bytes); hmac_reinit(&mctx->chunk_hmac); hmac_update(&tdat->chunk_hmac, tobuf, dstlen); hmac_final(&tdat->chunk_hmac, chash, &hlen); serialize_checksum(chash, mac_ptr, hlen); } else { uchar_t mac_ptr; uint32_t crc; mac_ptr = mctx->tobuf + 24 + pctx->cksum_bytes; memset(mac_ptr, 0, pctx->mac_bytes); crc = lzma_crc32(tdat->cmp_seg, rbytes, 0); U32_P(mac_ptr) = LE32(crc); } /* * All done. Now grab lock and write. */ dstlen += COMPRESSED_CHUNKSIZE; // The 'full' chunk now pthread_mutex_lock(&pctx->write_mutex); wbytes = Write(mctx->comp_fd, mctx->tobuf, dstlen); pthread_mutex_unlock(&pctx->write_mutex); if (wbytes != dstlen) { log_msg(LOG_ERR, 1, "Metadata Write (expected: %" PRIu64 ", written: %" PRId64 ") : ", dstlen, wbytes); pctx->main_cancel = 1; pctx->t_errored = 1; return (0); } return (1); } void meta_ctx_close_sink_channel(meta_ctx_t *mctx) { if (mctx->pipes[SINK_CHANNEL]) { close(mctx->pipes[SINK_CHANNEL]); mctx->pipes[SINK_CHANNEL] = 0; } } void meta_ctx_close_src_channel(meta_ctx_t *mctx) { if (mctx->pipes[SRC_CHANNEL]) { close(mctx->pipes[SRC_CHANNEL]); mctx->pipes[SRC_CHANNEL] = 0; } } static void * metadata_compress(void *dat) { meta_ctx_t *mctx = (meta_ctx_t *)dat; meta_msg_t msg; int ack; while (Read(mctx->meta_pipes[SINK_CHANNEL], &msg, sizeof (msg)) == sizeof (msg)) { ack = 0; if (mctx->frompos + msg.len > METADATA_CHUNK_SIZE) { if (!compress_and_write(mctx)) { Write(mctx->meta_pipes[SINK_CHANNEL], &ack, sizeof (ack)); return (NULL); } mctx->frompos = 0; memcpy(mctx->frombuf, msg.buf, msg.len); mctx->frompos = msg.len; } else if (mctx->frompos + msg.len == METADATA_CHUNK_SIZE) { memcpy(mctx->frombuf + mctx->frompos, msg.buf, msg.len); if (!compress_and_write(mctx)) { Write(mctx->meta_pipes[SINK_CHANNEL], &ack, sizeof (ack)); return (NULL); } mctx->frompos = 0; } else { memcpy(mctx->frombuf + mctx->frompos, msg.buf, msg.len); mctx->frompos += msg.len; } ack = 1; Write(mctx->meta_pipes[SINK_CHANNEL], &ack, sizeof (ack)) } /* * Flush pending chunk. */ if (mctx->frompos) { if (!compress_and_write(mctx)) { ack = 0; Write(mctx->meta_pipes[SINK_CHANNEL], &ack, sizeof (ack)); return (NULL); } mctx->frompos = 0; } } static void * metadata_decompress(void *dat) { } meta_ctx_t * meta_ctx_create(pc_ctx_t *pctx, int file_version, int comp_fd) { meta_ctx_t *mctx; mctx = (meta_ctx_t *)malloc(sizeof (meta_ctx_t)); if (!mctx) { log_msg(LOG_ERR, 1, "Failed to allocate metadata context."); return (NULL); } if (pctx->encrypt_type) { if (hmac_init(&mctx->chunk_hmac, pctx->cksum, &(pctx->crypto_ctx)) == -1) { (void) free(mctx); log_msg(LOG_ERR, 0, "Cannot initialize metadata hmac."); return (NULL); } } mctx->comp_fd = comp_fd; mctx->frompos = 0; mctx->frombuf = slab_alloc(NULL, METADATA_CHUNK_SIZE + METADATA_HDR_SZ); if (!mctx->frombuf) { (void) free(mctx); log_msg(LOG_ERR, 1, "Failed to allocate metadata buffer."); return (NULL); } mctx->tobuf = slab_alloc(NULL, METADATA_CHUNK_SIZE + METADATA_HDR_SZ); if (!mctx->tobuf) { (void) free(mctx->frombuf); (void) free(mctx); log_msg(LOG_ERR, 1, "Failed to allocate metadata buffer."); return (NULL); } mctx->pctx = pctx; if (socketpair(AF_UNIX, SOCK_STREAM, 0, mctx->meta_pipes) == -1) { (void) free(mctx->frombuf); (void) free(mctx->tobuf); (void) free(mctx); log_msg(LOG_ERR, 1, "Unable to create a metadata ctx channel."); return (NULL); } if (pctx->do_compress) { int rv, level; mctx->comp_level = pctx->level; rv = lz4_init(&mctx->lz4_dat, &mctx->comp_level, 1, METADATA_CHUNK_SIZE, file_version, COMPRESS); if (rv != 0 || pthread_create(&(mctx->meta_thread), NULL, metadata_compress, (void *)mctx) != 0) { (void) close(pctx->meta_pipes[0]); (void) close(pctx->meta_pipes[1]); (void) free(mctx->frombuf); (void) free(mctx->tobuf); (void) free(mctx); if (rv == 0) log_msg(LOG_ERR, 0, "LZ4 init failed."); else log_msg(LOG_ERR, 1, "Unable to create metadata thread."); return (NULL); } } else { int rv, level; mctx->comp_level = pctx->level; rv = lz4_init(&mctx->lz4_dat, &mctx->comp_level, 1, METADATA_CHUNK_SIZE, file_version, DECOMPRESS); if (rv != 0 || pthread_create(&(mctx->meta_thread), NULL, metadata_decompress, (void *)mctx) != 0) { (void) close(pctx->meta_pipes[0]); (void) close(pctx->meta_pipes[1]); (void) free(mctx->frombuf); (void) free(mctx->tobuf); (void) free(mctx); if (rv == 0) log_msg(LOG_ERR, 0, "LZ4 init failed."); else log_msg(LOG_ERR, 1, "Unable to create metadata thread."); return (NULL); } } return (mctx); } int meta_ctx_write(meta_ctx_t *mctx, meta_msg_t *msg) { int ack; meta_msg_t msg; /* * Write the message buffer to the pipe. */ if (Write(pctx->meta_pipes[SRC_CHANNEL], &msg, sizeof (msg)) < sizeof (meta_msg_t)) { log_msg(LOG_ERR, 1, "Meta socket write error."); return (0); } /* * Wait for ACK. */ if (Read(pctx->meta_pipes[SRC_CHANNEL], &ack, sizeof (ack)) < sizeof (ack)) { log_msg(LOG_ERR, 1, "Meta socket read error."); return (0); } if (!ack) { return (-1); } return (1); } int meta_ctx_read(meta_ctx_t *mctx, meta_msg_t *msg) { } int meta_ctx_done(meta_ctx_t *mctx) { }