greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Implement HMAC Functionality.

Browse source 
Refactor crypto utility functions in a separate file.
Add HMAC verification for file header.
Add a few extra input validation checks.
Include space in chunk header for HMAC.
This commit is contained in:
Moinak Ghosh 2012-10-17 23:32:35 +05:30
parent f2d7bea902
commit d13aad830e
11 changed files with 922 additions and 438 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Makefile.in
View file

@ -26,9 +26,10 @@ MAINSRCS = main.c utils/utils.c allocator.c zlib_compress.c bzip2_compress.c \
lzma_compress.c ppmd_compress.c adaptive_compress.c lzfx_compress.c \ lzma_compress.c ppmd_compress.c adaptive_compress.c lzfx_compress.c \
lz4_compress.c none_compress.c utils/xxhash.c utils/heapq.c utils/cpuid.c \ lz4_compress.c none_compress.c utils/xxhash.c utils/heapq.c utils/cpuid.c \
crypto/aes/crypto_aes.c crypto/scrypt/crypto_scrypt-nosse.c \ crypto/aes/crypto_aes.c crypto/scrypt/crypto_scrypt-nosse.c \
crypto/scrypt/sha256.c crypto/scrypt/crypto_aesctr.c crypto/scrypt/sha256.c crypto/scrypt/crypto_aesctr.c crypto/crypto_utils.c
MAINHDRS = allocator.h pcompress.h utils/utils.h utils/xxhash.h utils/heapq.h \ MAINHDRS = allocator.h pcompress.h utils/utils.h utils/xxhash.h utils/heapq.h \
utils/cpuid.h utils/cpuid.h crypto/crypto_utils.h crypto/scrypt/crypto_scrypt.h \
crypto/scrypt/sha256.h crypto/scrypt/crypto_aesctr.h crypto/aes/crypto_aes.h
MAINOBJS = $(MAINSRCS:.c=.o) MAINOBJS = $(MAINSRCS:.c=.o)
RABINSRCS = rabin/rabin_dedup.c RABINSRCS = rabin/rabin_dedup.c
@ -92,13 +93,13 @@ LIBBSCGEN_OPT = -fopenmp
LIBBSCCPPFLAGS = -I$(LIBBSCDIR)/libbsc -DENABLE_PC_LIBBSC LIBBSCCPPFLAGS = -I$(LIBBSCDIR)/libbsc -DENABLE_PC_LIBBSC
BAKFILES = *~ lzma/*~ lzfx/*~ lz4/*~ rabin/*~ bsdiff/*~ lzp/*~ utils/*~ crypto/sha2/*~ \ BAKFILES = *~ lzma/*~ lzfx/*~ lz4/*~ rabin/*~ bsdiff/*~ lzp/*~ utils/*~ crypto/sha2/*~ \
crypto/sha2/intel/*~ crypto/aes/*~ crypto/scrypt/*~ crypto/sha2/intel/*~ crypto/aes/*~ crypto/scrypt/*~ crypto/*~
RM = rm -f RM = rm -f
COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT_PROPS \ COMMON_CPPFLAGS = -I. -I./lzma -I./lzfx -I./lz4 -I./rabin -I./bsdiff -DNODEFAULT_PROPS \
-DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32 \ -DFILE_OFFSET_BITS=64 -D_REENTRANT -D__USE_SSE_INTRIN__ -D_LZMA_PROB32 \
-I./lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I@OPENSSL_INCDIR@ \ -I./lzp @LIBBSCCPPFLAGS@ -I./crypto/skein -I./utils -I@OPENSSL_INCDIR@ \
-I./crypto/sha2 -I./crypto/scrypt -I./crypto/aes @KEYLEN@ -I./crypto/sha2 -I./crypto/scrypt -I./crypto/aes -I./crypto @KEYLEN@
COMMON_VEC_FLAGS = -ftree-vectorize COMMON_VEC_FLAGS = -ftree-vectorize
COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block COMMON_LOOP_OPTFLAGS = $(VEC_FLAGS) -floop-interchange -floop-block
LDLIBS = -ldl -lbz2 $(ZLIB_DIR) -lz -lm @LIBBSCLFLAGS@ -L@OPENSSL_LIBDIR@ -lcrypto -lrt LDLIBS = -ldl -lbz2 $(ZLIB_DIR) -lz -lm @LIBBSCLFLAGS@ -L@OPENSSL_LIBDIR@ -lcrypto -lrt

10
crypto/aes/crypto_aes.c
View file

@ -107,11 +107,11 @@ aes_init(aes_ctx_t *ctx, uchar_t *salt, int saltlen, uchar_t *pwd, int pwd_len,
uint64_t nonce, int enc) uint64_t nonce, int enc)
{ {
int rv; int rv;
uchar_t key[KEYLEN];
struct timespec tp; struct timespec tp;
uint64_t tv; uint64_t tv;
uchar_t num[25]; uchar_t num[25];
uchar_t IV[32]; uchar_t IV[32];
uchar_t *key = ctx->pkey;
#ifndef _USE_PBK #ifndef _USE_PBK
int logN; int logN;
@ -153,8 +153,6 @@ aes_init(aes_ctx_t *ctx, uchar_t *salt, int saltlen, uchar_t *pwd, int pwd_len,
ctx->nonce = nonce; ctx->nonce = nonce;
AES_set_encrypt_key(key, (KEYLEN << 3), &(ctx->key)); AES_set_encrypt_key(key, (KEYLEN << 3), &(ctx->key));
} }
memset(key, 0, KEYLEN);
return (0); return (0);
} }
@ -210,6 +208,12 @@ aes_nonce(aes_ctx_t *ctx)
return (ctx->nonce); return (ctx->nonce);
} }
void
aes_clean_pkey(aes_ctx_t *ctx)
{
memset(ctx->pkey, 0, KEYLEN);
}
void void
aes_cleanup(aes_ctx_t *ctx) aes_cleanup(aes_ctx_t *ctx)
{ {

2
crypto/aes/crypto_aes.h
View file

@ -43,6 +43,7 @@ extern "C" {
typedef struct { typedef struct {
uint64_t nonce; uint64_t nonce;
AES_KEY key; AES_KEY key;
uchar_t pkey[KEYLEN];
} aes_ctx_t; } aes_ctx_t;
int aes_init(aes_ctx_t *ctx, uchar_t *salt, int saltlen, uchar_t *pwd, int pwd_len, int aes_init(aes_ctx_t *ctx, uchar_t *salt, int saltlen, uchar_t *pwd, int pwd_len,
@ -50,6 +51,7 @@ int aes_init(aes_ctx_t *ctx, uchar_t *salt, int saltlen, uchar_t *pwd, int pwd_l
int aes_encrypt(aes_ctx_t *ctx, uchar_t *plaintext, uchar_t *ciphertext, ssize_t len, uint64_t id); int aes_encrypt(aes_ctx_t *ctx, uchar_t *plaintext, uchar_t *ciphertext, ssize_t len, uint64_t id);
int aes_decrypt(aes_ctx_t *ctx, uchar_t *ciphertext, uchar_t *plaintext, ssize_t len, uint64_t id); int aes_decrypt(aes_ctx_t *ctx, uchar_t *ciphertext, uchar_t *plaintext, ssize_t len, uint64_t id);
uint64_t aes_nonce(aes_ctx_t *ctx); uint64_t aes_nonce(aes_ctx_t *ctx);
void aes_clean_pkey(aes_ctx_t *ctx);
void aes_cleanup(aes_ctx_t *ctx); void aes_cleanup(aes_ctx_t *ctx);
#ifdef __cplusplus #ifdef __cplusplus

602
crypto/crypto_utils.c Normal file
View file

@ -0,0 +1,602 @@
/*
* This file is a part of Pcompress, a chunked parallel multi-
* algorithm lossless compression and decompression program.
*
* Copyright (C) 2012 Moinak Ghosh. All rights reserved.
* Use is subject to license terms.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* moinakg@belenix.org, http://moinakg.wordpress.com/
*
* This program includes partly-modified public domain source
* code from the LZMA SDK: http://www.7-zip.org/sdk.html
*/
#include <sys/types.h>
#include <sys/param.h>
#include <fcntl.h>
#include <time.h>
#include <termios.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <skein.h>
#include <openssl/sha.h>
#include <openssl/rand.h>
#include <openssl/evp.h>
#include <openssl/hmac.h>
#include <sha256.h>
#include <crypto_aes.h>
#include "crypto_utils.h"
#include "cpuid.h"
#define PROVIDER_OPENSSL 0
#define PROVIDER_X64_OPT 1
static void init_sha256(void);
static int geturandom_bytes(uchar_t rbytes[32]);
/*
* Checksum properties
*/
typedef void (*ckinit_func_ptr)(void);
static struct {
char *name;
cksum_t cksum_id;
int bytes, mac_bytes;
ckinit_func_ptr init_func;
} cksum_props[] = {
{"CRC64", CKSUM_CRC64, 8, 32, NULL},
{"SKEIN256", CKSUM_SKEIN256, 32, 32, NULL},
{"SKEIN512", CKSUM_SKEIN512, 64, 64, NULL},
{"SHA256", CKSUM_SHA256, 32, 32, init_sha256},
{"SHA512", CKSUM_SHA512, 64, 64, NULL}
};
static int cksum_provider = PROVIDER_OPENSSL, ossl_inited = 0;
extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc);
extern uint64_t lzma_crc64_8bchk(const uint8_t *buf, size_t size,
uint64_t crc, uint64_t *cnt);
int
compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, ssize_t bytes)
{
if (cksum == CKSUM_CRC64) {
uint64_t *ck = (uint64_t *)cksum_buf;
*ck = lzma_crc64(buf, bytes, 0);
} else if (cksum == CKSUM_SKEIN256) {
Skein_512_Ctxt_t ctx;
Skein_512_Init(&ctx, 256);
Skein_512_Update(&ctx, buf, bytes);
Skein_512_Final(&ctx, cksum_buf);
} else if (cksum == CKSUM_SKEIN512) {
Skein_512_Ctxt_t ctx;
Skein_512_Init(&ctx, 512);
Skein_512_Update(&ctx, buf, bytes);
Skein_512_Final(&ctx, cksum_buf);
} else if (cksum == CKSUM_SHA256) {
if (cksum_provider == PROVIDER_OPENSSL) {
SHA256_CTX ctx;
SHA256_Init(&ctx);
SHA256_Update(&ctx, buf, bytes);
SHA256_Final(cksum_buf, &ctx);
} else {
SHA256_Context ctx;
opt_SHA256_Init(&ctx);
opt_SHA256_Update(&ctx, buf, bytes);
opt_SHA256_Final(&ctx, cksum_buf);
}
} else if (cksum == CKSUM_SHA512) {
SHA512_CTX ctx;
SHA512_Init(&ctx);
SHA512_Update(&ctx, buf, bytes);
SHA512_Final(cksum_buf, &ctx);
} else {
return (-1);
}
return (0);
}
static void
init_sha256(void)
{
#ifdef WORDS_BIGENDIAN
cksum_provider = PROVIDER_OPENSSL;
#else
#ifdef __x86_64__
processor_info_t pc;
cksum_provider = PROVIDER_OPENSSL;
cpuid_basic_identify(&pc);
if (pc.proc_type == PROC_X64_INTEL || pc.proc_type == PROC_X64_AMD) {
if (opt_Init_SHA(&pc) == 0) {
cksum_provider = PROVIDER_X64_OPT;
}
}
#endif
#endif
}
/*
* Check is either the given checksum name or id is valid and
* return it's properties.
*/
int
get_checksum_props(char *name, int *cksum, int *cksum_bytes, int *mac_bytes)
{
int i;
for (i=0; i<sizeof (cksum_props); i++) {
if ((name != NULL && strcmp(name, cksum_props[i].name) == 0) ||
(*cksum != 0 && *cksum == cksum_props[i].cksum_id)) {
*cksum = cksum_props[i].cksum_id;
*cksum_bytes = cksum_props[i].bytes;
*mac_bytes = cksum_props[i].mac_bytes;
if (cksum_props[i].init_func)
cksum_props[i].init_func();
return (0);
}
}
return (-1);
}
/*
* Endian independent way of storing the checksum bytes. This is actually
* storing in little endian format and a copy can be avoided in x86 land.
* However unsightly ifdefs are avoided here since this is not so performance
* critical.
*/
void
serialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes)
{
int i,j;
j = 0;
for (i=cksum_bytes; i>0; i--) {
buf[j] = checksum[i-1];
j++;
}
}
void
deserialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes)
{
int i,j;
j = 0;
for (i=cksum_bytes; i>0; i--) {
checksum[i-1] = buf[j];
j++;
}
}
int
hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx)
{
aes_ctx_t *actx = (aes_ctx_t *)(cctx->crypto_ctx);
mctx->mac_cksum = cksum;
if (cksum == CKSUM_SKEIN256) {
Skein_512_Ctxt_t *ctx = malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) return (-1);
Skein_512_InitExt(ctx, 256, SKEIN_CFG_TREE_INFO_SEQUENTIAL,
actx->pkey, KEYLEN);
mctx->mac_ctx = ctx;
ctx = malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) {
free(mctx->mac_ctx);
return (-1);
}
memcpy(ctx, mctx->mac_ctx, sizeof (Skein_512_Ctxt_t));
mctx->mac_ctx_reinit = ctx;
} else if (cksum == CKSUM_SKEIN512) {
Skein_512_Ctxt_t *ctx = malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) return (-1);
Skein_512_InitExt(ctx, 512, SKEIN_CFG_TREE_INFO_SEQUENTIAL,
actx->pkey, KEYLEN);
mctx->mac_ctx = ctx;
ctx = malloc(sizeof (Skein_512_Ctxt_t));
if (!ctx) {
free(mctx->mac_ctx);
return (-1);
}
memcpy(ctx, mctx->mac_ctx, sizeof (Skein_512_Ctxt_t));
mctx->mac_ctx_reinit = ctx;
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_CTX *ctx = malloc(sizeof (HMAC_CTX));
if (!ctx) return (-1);
HMAC_CTX_init(ctx);
HMAC_Init_ex(ctx, actx->pkey, KEYLEN, EVP_sha256(), NULL);
mctx->mac_ctx = ctx;
ctx = malloc(sizeof (HMAC_CTX));
if (!ctx) {
free(mctx->mac_ctx);
return (-1);
}
if (!HMAC_CTX_copy(ctx, mctx->mac_ctx)) {
free(ctx);
free(mctx->mac_ctx);
return (-1);
}
mctx->mac_ctx_reinit = ctx;
} else {
HMAC_SHA256_Context *ctx = malloc(sizeof (HMAC_SHA256_Context));
if (!ctx) return (-1);
opt_HMAC_SHA256_Init(ctx, actx->pkey, KEYLEN);
mctx->mac_ctx = ctx;
ctx = malloc(sizeof (HMAC_SHA256_Context));
if (!ctx) {
free(mctx->mac_ctx);
return (-1);
}
memcpy(ctx, mctx->mac_ctx, sizeof (HMAC_SHA256_Context));
mctx->mac_ctx_reinit = ctx;
}
} else if (cksum == CKSUM_SHA512) {
HMAC_CTX *ctx = malloc(sizeof (HMAC_CTX));
if (!ctx) return (-1);
HMAC_CTX_init(ctx);
HMAC_Init_ex(ctx, actx->pkey, KEYLEN, EVP_sha512(), NULL);
mctx->mac_ctx = ctx;
ctx = malloc(sizeof (HMAC_CTX));
if (!ctx) {
free(mctx->mac_ctx);
return (-1);
}
if (!HMAC_CTX_copy(ctx, mctx->mac_ctx)) {
free(ctx);
free(mctx->mac_ctx);
return (-1);
}
mctx->mac_ctx_reinit = ctx;
} else {
return (-1);
}
return (0);
}
int
hmac_reinit(mac_ctx_t *mctx)
{
int cksum = mctx->mac_cksum;
if (cksum == CKSUM_SKEIN256 || cksum == CKSUM_SKEIN512) {
memcpy(mctx->mac_ctx, mctx->mac_ctx_reinit, sizeof (Skein_512_Ctxt_t));
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_CTX_copy(mctx->mac_ctx, mctx->mac_ctx_reinit);
} else {
memcpy(mctx->mac_ctx, mctx->mac_ctx_reinit, sizeof (HMAC_SHA256_Context));
}
} else if (cksum == CKSUM_SHA512) {
HMAC_CTX_copy(mctx->mac_ctx, mctx->mac_ctx_reinit);
} else {
return (-1);
}
return (0);
}
int
hmac_update(mac_ctx_t *mctx, uchar_t *data, size_t len)
{
int cksum = mctx->mac_cksum;
if (cksum == CKSUM_SKEIN256 || cksum == CKSUM_SKEIN512) {
Skein_512_Update(mctx->mac_ctx, data, len);
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) {
if (HMAC_Update(mctx->mac_ctx, data, len) == 0)
return (-1);
} else {
opt_HMAC_SHA256_Update(mctx->mac_ctx, data, len);
}
} else if (cksum == CKSUM_SHA512) {
if (HMAC_Update(mctx->mac_ctx, data, len) == 0)
return (-1);
} else {
return (-1);
}
return (0);
}
int
hmac_final(mac_ctx_t *mctx, uchar_t *hash, unsigned int *len)
{
int cksum = mctx->mac_cksum;
if (cksum == CKSUM_SKEIN256) {
Skein_512_Final(mctx->mac_ctx, hash);
*len = 32;
} else if (cksum == CKSUM_SKEIN512) {
Skein_512_Final(mctx->mac_ctx, hash);
*len = 64;
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_Final(mctx->mac_ctx, hash, len);
} else {
opt_HMAC_SHA256_Final(mctx->mac_ctx, hash);
*len = 32;
}
} else if (cksum == CKSUM_SHA512) {
HMAC_Final(mctx->mac_ctx, hash, len);
} else {
return (-1);
}
return (0);
}
int
hmac_cleanup(mac_ctx_t *mctx)
{
int cksum = mctx->mac_cksum;
if (cksum == CKSUM_SKEIN256 || cksum == CKSUM_SKEIN512) {
memset(mctx->mac_ctx, 0, sizeof (Skein_512_Ctxt_t));
memset(mctx->mac_ctx_reinit, 0, sizeof (Skein_512_Ctxt_t));
} else if (cksum == CKSUM_SHA256 || cksum == CKSUM_CRC64) {
if (cksum_provider == PROVIDER_OPENSSL) {
HMAC_CTX_cleanup(mctx->mac_ctx);
HMAC_CTX_cleanup(mctx->mac_ctx_reinit);
} else {
memset(mctx->mac_ctx, 0, sizeof (HMAC_SHA256_Context));
memset(mctx->mac_ctx_reinit, 0, sizeof (HMAC_SHA256_Context));
}
} else if (cksum == CKSUM_SHA512) {
HMAC_CTX_cleanup(mctx->mac_ctx);
HMAC_CTX_cleanup(mctx->mac_ctx_reinit);
} else {
return (-1);
}
mctx->mac_cksum = 0;
free(mctx->mac_ctx);
free(mctx->mac_ctx_reinit);
return (0);
}
int
init_crypto(crypto_ctx_t *cctx, uchar_t *pwd, int pwd_len, int crypto_alg,
uchar_t *salt, int saltlen, uint64_t nonce, int enc_dec)
{
if (crypto_alg == CRYPTO_ALG_AES) {
aes_ctx_t *actx = malloc(sizeof (aes_ctx_t));
if (enc_dec) {
/*
* Encryption init.
*/
cctx->salt = malloc(32);
salt = cctx->salt;
cctx->saltlen = 32;
if (RAND_status() != 1 || RAND_bytes(salt, 32) != 1) {
if (geturandom_bytes(salt) != 0) {
uchar_t sb[64];
int b;
struct timespec tp;
b = 0;
/* No good random pool is populated/available. What to do ? */
if (clock_gettime(CLOCK_MONOTONIC, &tp) == -1) {
time((time_t *)&sb[b]);
b += 8;
} else {
uint64_t v;
v = tp.tv_sec * 1000UL + tp.tv_nsec;
*((uint64_t *)&sb[b]) = v;
b += 8;
}
*((uint32_t *)&sb[b]) = rand();
b += 4;
*((uint32_t *)&sb[b]) = getpid();
b += 4;
compute_checksum(&sb[b], CKSUM_SHA256, sb, b);
b = 8 + 4;
*((uint32_t *)&sb[b]) = rand();
compute_checksum(salt, CKSUM_SHA256, &sb[b], 32 + 4);
}
}
/*
* Zero nonce (arg #6) since it will be generated.
*/
if (aes_init(actx, salt, 32, pwd, pwd_len, 0, enc_dec) != 0) {
fprintf(stderr, "Failed to initialize AES context\n");
return (-1);
}
} else {
/*
* Decryption init.
* Pass given nonce and salt.
*/
if (saltlen > MAX_SALTLEN) {
fprintf(stderr, "Salt too long. Max allowed length is %d\n",
MAX_SALTLEN);
return (-1);
}
cctx->salt = malloc(saltlen);
memcpy(cctx->salt, salt, saltlen);
if (aes_init(actx, cctx->salt, saltlen, pwd, pwd_len, nonce,
enc_dec) != 0) {
fprintf(stderr, "Failed to initialize AES context\n");
return (-1);
}
}
cctx->crypto_ctx = actx;
cctx->crypto_alg = crypto_alg;
cctx->enc_dec = enc_dec;
} else {
fprintf(stderr, "Unrecognized algorithm code: %d\n", crypto_alg);
return (-1);
}
return (0);
}
int
crypto_buf(crypto_ctx_t *cctx, uchar_t *from, uchar_t *to, ssize_t bytes, uint64_t id)
{
if (cctx->crypto_alg == CRYPTO_ALG_AES) {
if (cctx->enc_dec == ENCRYPT_FLAG) {
return (aes_encrypt(cctx->crypto_ctx, from, to, bytes, id));
} else {
return (aes_decrypt(cctx->crypto_ctx, from, to, bytes, id));
}
} else {
fprintf(stderr, "Unrecognized algorithm code: %d\n", cctx->crypto_alg);
return (-1);
}
return (0);
}
uint64_t
crypto_nonce(crypto_ctx_t *cctx)
{
return (aes_nonce(cctx->crypto_ctx));
}
void
crypto_clean_pkey(crypto_ctx_t *cctx)
{
aes_clean_pkey(cctx->crypto_ctx);
}
void
cleanup_crypto(crypto_ctx_t *cctx)
{
aes_cleanup(cctx->crypto_ctx);
memset(cctx->salt, 0, 32);
free(cctx->salt);
free(cctx);
}
static int
geturandom_bytes(uchar_t rbytes[32])
{
int fd;
ssize_t lenread;
uchar_t * buf = rbytes;
size_t buflen = 32;
/* Open /dev/urandom. */
if ((fd = open("/dev/urandom", O_RDONLY)) == -1)
goto err0;
/* Read bytes until we have filled the buffer. */
while (buflen > 0) {
if ((lenread = read(fd, buf, buflen)) == -1)
goto err1;
/* The random device should never EOF. */
if (lenread == 0)
goto err1;
/* We're partly done. */
buf += lenread;
buflen -= lenread;
}
/* Close the device. */
while (close(fd) == -1) {
if (errno != EINTR)
goto err0;
}
/* Success! */
return (0);
err1:
close(fd);
err0:
/* Failure! */
return (4);
}
int
get_pw_string(char pw[MAX_PW_LEN], char *prompt)
{
int fd, len;
FILE *input, *strm;
struct termios oldt, newt;
uchar_t pw1[MAX_PW_LEN], pw2[MAX_PW_LEN], *s;
// Try TTY first
fd = open("/dev/tty", O_RDWR | O_NOCTTY);
if (fd != -1) {
input = fdopen(fd, "w+");
strm = input;
} else {
// Fall back to stdin
fd = STDIN_FILENO;
input = stdin;
strm = stderr;
}
tcgetattr(fd, &oldt);
newt = oldt;
newt.c_lflag &= ~ECHO;
tcsetattr(fd, TCSANOW, &newt);
fprintf(stderr, "%s: ", prompt);
fflush(stderr);
s = fgets(pw1, MAX_PW_LEN, input);
fputs("\n", stderr);
if (s == NULL) {
tcsetattr(fd, TCSANOW, &oldt);
fflush(strm);
return (-1);
}
fprintf(stderr, "%s (once more): ", prompt);
fflush(stderr);
s = fgets(pw2, MAX_PW_LEN, input);
tcsetattr(fd, TCSANOW, &oldt);
fflush(strm);
fputs("\n", stderr);
if (s == NULL) {
return (-1);
}
if (strcmp(pw1, pw2) != 0) {
fprintf(stderr, "Passwords do not match!\n");
memset(pw1, 0, MAX_PW_LEN);
memset(pw2, 0, MAX_PW_LEN);
return (-1);
}
len = strlen(pw1);
pw1[len-1] = '\0';
strcpy(pw, pw1);
memset(pw1, 0, MAX_PW_LEN);
memset(pw2, 0, MAX_PW_LEN);
return (len);
}

102
crypto/crypto_utils.h Normal file
View file

@ -0,0 +1,102 @@
/*
* This file is a part of Pcompress, a chunked parallel multi-
* algorithm lossless compression and decompression program.
*
* Copyright (C) 2012 Moinak Ghosh. All rights reserved.
* Use is subject to license terms.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* moinakg@belenix.org, http://moinakg.wordpress.com/
*
* This program includes partly-modified public domain source
* code from the LZMA SDK: http://www.7-zip.org/sdk.html
*/
#ifndef _CRYPTO_UTILS_H
#define _CRYPTO_UTILS_H
#include <arpa/nameser_compat.h>
#include <sys/types.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
#define MAX_PW_LEN 16
#define CKSUM_MASK 0x700
#define CKSUM_MAX_BYTES 64
#define DEFAULT_CKSUM "SKEIN256"
#define ENCRYPT_FLAG 1
#define DECRYPT_FLAG 0
#define CRYPTO_ALG_AES 0x10
#define MAX_SALTLEN 64
/*
* Public checksum properties. CKSUM_MAX_BYTES must be updated if a
* newer larger checksum is added to the list.
*/
typedef enum {
CKSUM_CRC64 = 0x100,
CKSUM_SKEIN256 = 0x200,
CKSUM_SKEIN512 = 0x300,
CKSUM_SHA256 = 0x400,
CKSUM_SHA512 = 0x500
} cksum_t;
typedef struct {
void *crypto_ctx;
int crypto_alg;
int enc_dec;
uchar_t *salt;
int saltlen;
} crypto_ctx_t;
typedef struct {
void *mac_ctx;
void *mac_ctx_reinit;
int mac_cksum;
} mac_ctx_t;
/*
* Generic message digest functions.
*/
int compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, ssize_t bytes);
int get_checksum_props(char *name, int *cksum, int *cksum_bytes, int *mac_bytes);
void serialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes);
void deserialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes);
/*
* Encryption related functions.
*/
int init_crypto(crypto_ctx_t *cctx, uchar_t *pwd, int pwd_len, int crypto_alg,
uchar_t *salt, int saltlen, uint64_t nonce, int enc_dec);
int crypto_buf(crypto_ctx_t *cctx, uchar_t *from, uchar_t *to, ssize_t bytes, uint64_t id);
uint64_t crypto_nonce(crypto_ctx_t *cctx);
void crypto_clean_pkey(crypto_ctx_t *cctx);
void cleanup_crypto(crypto_ctx_t *cctx);
int get_pw_string(char pw[MAX_PW_LEN], char *prompt);
/*
* HMAC functions.
*/
int hmac_init(mac_ctx_t *mctx, int cksum, crypto_ctx_t *cctx);
int hmac_reinit(mac_ctx_t *mctx);
int hmac_update(mac_ctx_t *mctx, uchar_t *data, size_t len);
int hmac_cleanup(mac_ctx_t *mctx);
#ifdef __cplusplus
}
#endif
#endif

87
crypto/sha2/sha256.c
View file

@ -105,7 +105,7 @@ static void
_init (SHA256_Context *sc, const uint32_t iv[SHA256_HASH_WORDS]) _init (SHA256_Context *sc, const uint32_t iv[SHA256_HASH_WORDS])
{ {
int i; int i;
/* /*
* SHA256_HASH_WORDS is 8, must be 8, cannot be anything but 8! * SHA256_HASH_WORDS is 8, must be 8, cannot be anything but 8!
* So we unroll a loop here. * So we unroll a loop here.
@ -118,7 +118,7 @@ _init (SHA256_Context *sc, const uint32_t iv[SHA256_HASH_WORDS])
sc->hash[5] = iv[5]; sc->hash[5] = iv[5];
sc->hash[6] = iv[6]; sc->hash[6] = iv[6];
sc->hash[7] = iv[7]; sc->hash[7] = iv[7];
sc->totalLength = 0LL; sc->totalLength = 0LL;
sc->bufferLength = 0L; sc->bufferLength = 0L;
} }
@ -130,26 +130,26 @@ APS_NAMESPACE(SHA256_Init) (SHA256_Context *sc)
} }
void void
APS_NAMESPACE(SHA256_Update) (SHA256_Context *sc, void *vdata, size_t len) APS_NAMESPACE(SHA256_Update) (SHA256_Context *sc, const void *vdata, size_t len)
{ {
const uint8_t *data = vdata; const uint8_t *data = vdata;
uint32_t bufferBytesLeft; uint32_t bufferBytesLeft;
size_t bytesToCopy; size_t bytesToCopy;
int rem; int rem;
if (sc->bufferLength) { if (sc->bufferLength) {
do { do {
bufferBytesLeft = 64L - sc->bufferLength; bufferBytesLeft = 64L - sc->bufferLength;
bytesToCopy = bufferBytesLeft; bytesToCopy = bufferBytesLeft;
if (bytesToCopy > len) if (bytesToCopy > len)
bytesToCopy = len; bytesToCopy = len;
memcpy (&sc->buffer.bytes[sc->bufferLength], data, bytesToCopy); memcpy (&sc->buffer.bytes[sc->bufferLength], data, bytesToCopy);
sc->totalLength += bytesToCopy * 8L; sc->totalLength += bytesToCopy * 8L;
sc->bufferLength += bytesToCopy; sc->bufferLength += bytesToCopy;
data += bytesToCopy; data += bytesToCopy;
len -= bytesToCopy; len -= bytesToCopy;
if (sc->bufferLength == 64L) { if (sc->bufferLength == 64L) {
sc->blocks = 1; sc->blocks = 1;
sha_update_func(sc->buffer.words, sc->hash, sc->blocks); sha_update_func(sc->buffer.words, sc->hash, sc->blocks);
@ -160,12 +160,12 @@ APS_NAMESPACE(SHA256_Update) (SHA256_Context *sc, void *vdata, size_t len)
} while (len > 0 && len <= 64L); } while (len > 0 && len <= 64L);
if (!len) return; if (!len) return;
} }
sc->blocks = len >> 6; sc->blocks = len >> 6;
rem = len - (sc->blocks << 6); rem = len - (sc->blocks << 6);
len = sc->blocks << 6; len = sc->blocks << 6;
sc->totalLength += rem * 8L; sc->totalLength += rem * 8L;
if (len) { if (len) {
sc->totalLength += len * 8L; sc->totalLength += len * 8L;
sha_update_func((uint32_t *)data, sc->hash, sc->blocks); sha_update_func((uint32_t *)data, sc->hash, sc->blocks);
@ -182,16 +182,16 @@ _final (SHA256_Context *sc, uint8_t *hash, int hashWords)
uint32_t bytesToPad; uint32_t bytesToPad;
uint64_t lengthPad; uint64_t lengthPad;
int i; int i;
bytesToPad = 120L - sc->bufferLength; bytesToPad = 120L - sc->bufferLength;
if (bytesToPad > 64L) if (bytesToPad > 64L)
bytesToPad -= 64L; bytesToPad -= 64L;
lengthPad = BYTESWAP64(sc->totalLength); lengthPad = BYTESWAP64(sc->totalLength);
APS_NAMESPACE(SHA256_Update) (sc, padding, bytesToPad); APS_NAMESPACE(SHA256_Update) (sc, padding, bytesToPad);
APS_NAMESPACE(SHA256_Update) (sc, &lengthPad, 8L); APS_NAMESPACE(SHA256_Update) (sc, &lengthPad, 8L);
if (hash) { if (hash) {
for (i = 0; i < hashWords; i++) { for (i = 0; i < hashWords; i++) {
hash[0] = (uint8_t) (sc->hash[i] >> 24); hash[0] = (uint8_t) (sc->hash[i] >> 24);
@ -208,3 +208,66 @@ APS_NAMESPACE(SHA256_Final) (SHA256_Context *sc, uint8_t hash[SHA256_HASH_SIZE])
{ {
_final (sc, hash, SHA256_HASH_WORDS); _final (sc, hash, SHA256_HASH_WORDS);
} }
/* Initialize an HMAC-SHA256 operation with the given key. */
void
APS_NAMESPACE(HMAC_SHA256_Init) (HMAC_SHA256_Context * ctx, const void * _K, size_t Klen)
{
unsigned char pad[64];
unsigned char khash[32];
const unsigned char * K = _K;
size_t i;
/* If Klen > 64, the key is really SHA256(K). */
if (Klen > 64) {
APS_NAMESPACE(SHA256_Init)(&ctx->ictx);
APS_NAMESPACE(SHA256_Update)(&ctx->ictx, K, Klen);
APS_NAMESPACE(SHA256_Final)(&ctx->ictx, khash);
K = khash;
Klen = 32;
}
/* Inner SHA256 operation is SHA256(K xor [block of 0x36] || data). */
APS_NAMESPACE(SHA256_Init)(&ctx->ictx);
memset(pad, 0x36, 64);
for (i = 0; i < Klen; i++)
pad[i] ^= K[i];
APS_NAMESPACE(SHA256_Update)(&ctx->ictx, pad, 64);
/* Outer SHA256 operation is SHA256(K xor [block of 0x5c] || hash). */
APS_NAMESPACE(SHA256_Init)(&ctx->octx);
memset(pad, 0x5c, 64);
for (i = 0; i < Klen; i++)
pad[i] ^= K[i];
APS_NAMESPACE(SHA256_Update)(&ctx->octx, pad, 64);
/* Clean the stack. */
memset(khash, 0, 32);
}
/* Add bytes to the HMAC-SHA256 operation. */
void
APS_NAMESPACE(HMAC_SHA256_Update) (HMAC_SHA256_Context * ctx, const void *in, size_t len)
{
/* Feed data to the inner SHA256 operation. */
APS_NAMESPACE(SHA256_Update)(&ctx->ictx, in, len);
}
/* Finish an HMAC-SHA256 operation. */
void
APS_NAMESPACE(HMAC_SHA256_Final) (HMAC_SHA256_Context * ctx, unsigned char digest[32])
{
unsigned char ihash[32];
/* Finish the inner SHA256 operation. */
APS_NAMESPACE(SHA256_Final)(&ctx->ictx, ihash);
/* Feed the inner hash to the outer SHA256 operation. */
APS_NAMESPACE(SHA256_Update)(&ctx->octx, ihash, 32);
/* Finish the outer SHA256 operation. */
APS_NAMESPACE(SHA256_Final)(&ctx->octx, digest);
/* Clean the stack. */
memset(ihash, 0, 32);
}

11
crypto/sha2/sha256.h
View file

@ -53,6 +53,11 @@ typedef struct _SHA256_Context {
} buffer; } buffer;
} SHA256_Context; } SHA256_Context;
typedef struct HMAC_SHA256Context {
SHA256_Context ictx;
SHA256_Context octx;
} HMAC_SHA256_Context;
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
@ -62,10 +67,14 @@ extern "C" {
#endif /* !APS_NAMESPACE */ #endif /* !APS_NAMESPACE */
void APS_NAMESPACE(SHA256_Init) (SHA256_Context *sc); void APS_NAMESPACE(SHA256_Init) (SHA256_Context *sc);
void APS_NAMESPACE(SHA256_Update) (SHA256_Context *sc, void *data, size_t len); void APS_NAMESPACE(SHA256_Update) (SHA256_Context *sc, const void *data, size_t len);
void APS_NAMESPACE(SHA256_Final) (SHA256_Context *sc, uint8_t hash[SHA256_HASH_SIZE]); void APS_NAMESPACE(SHA256_Final) (SHA256_Context *sc, uint8_t hash[SHA256_HASH_SIZE]);
int APS_NAMESPACE(Init_SHA) (processor_info_t *pc); int APS_NAMESPACE(Init_SHA) (processor_info_t *pc);
void APS_NAMESPACE(HMAC_SHA256_Init) (HMAC_SHA256_Context * ctx, const void * _K, size_t Klen);
void APS_NAMESPACE(HMAC_SHA256_Update) (HMAC_SHA256_Context * ctx, const void *in, size_t len);
void APS_NAMESPACE(HMAC_SHA256_Final) (HMAC_SHA256_Context * ctx, unsigned char digest[32]);
/* /*
* Intel's optimized SHA256 core routines. These routines are described in an * Intel's optimized SHA256 core routines. These routines are described in an
* Intel White-Paper: * Intel White-Paper:

126
main.c
View file

@ -51,6 +51,7 @@
* We use 5MB chunks by default. * We use 5MB chunks by default.
*/ */
#define DEFAULT_CHUNKSIZE (5 * 1024 * 1024) #define DEFAULT_CHUNKSIZE (5 * 1024 * 1024)
#define EIGHTY_PCT(x) ((x) - ((x)/5))
struct wdata { struct wdata {
struct cmp_data **dary; struct cmp_data **dary;
@ -88,7 +89,7 @@ static const char *exec_name;
static const char *algo = NULL; static const char *algo = NULL;
static int do_compress = 0; static int do_compress = 0;
static int do_uncompress = 0; static int do_uncompress = 0;
static int cksum_bytes; static int cksum_bytes, mac_bytes;
static int cksum = 0; static int cksum = 0;
static int rab_blk_size = 0; static int rab_blk_size = 0;
static dedupe_context_t *rctx; static dedupe_context_t *rctx;
@ -291,7 +292,7 @@ redo:
goto cont; goto cont;
} }
cseg = tdat->compressed_chunk + cksum_bytes; cseg = tdat->compressed_chunk + cksum_bytes + mac_bytes;
_chunksize = tdat->chunksize; _chunksize = tdat->chunksize;
deserialize_checksum(tdat->checksum, tdat->compressed_chunk, cksum_bytes); deserialize_checksum(tdat->checksum, tdat->compressed_chunk, cksum_bytes);
HDR = *cseg; HDR = *cseg;
@ -309,6 +310,8 @@ redo:
* Decrypt compressed data if necessary. * Decrypt compressed data if necessary.
*/ */
if (encrypt_type) { if (encrypt_type) {
uchar_t hmac[CKSUM_MAX_BYTES];
/* /*
* Encryption algorithm should not change the size and * Encryption algorithm should not change the size and
* encryption is in-place. * encryption is in-place.
@ -538,6 +541,25 @@ start_decompress(const char *filename, const char *to_filename)
chunksize = ntohll(chunksize); chunksize = ntohll(chunksize);
level = ntohl(level); level = ntohl(level);
/*
* Check for ridiculous values (malicious tampering or otherwise).
*/
if (version > VERSION) {
fprintf(stderr, "Cannot handle newer archive version %d, capability %d\n",
version, VERSION);
err = 1;
goto uncomp_done;
}
if (chunksize > EIGHTY_PCT(get_total_ram())) {
fprintf(stderr, "Chunk size must not exceed 80%% of total RAM.\n");
err = 1;
goto uncomp_done;
}
if (level > MAX_LEVEL || level < 0) {
fprintf(stderr, "Invalid compression level in header: %d\n", level);
err = 1;
goto uncomp_done;
}
if (version < VERSION-2) { if (version < VERSION-2) {
fprintf(stderr, "Unsupported version: %d\n", version); fprintf(stderr, "Unsupported version: %d\n", version);
err = 1; err = 1;
@ -566,7 +588,7 @@ start_decompress(const char *filename, const char *to_filename)
} }
cksum = flags & CKSUM_MASK; cksum = flags & CKSUM_MASK;
if (get_checksum_props(NULL, &cksum, &cksum_bytes) == -1) { if (get_checksum_props(NULL, &cksum, &cksum_bytes, &mac_bytes) == -1) {
fprintf(stderr, "Invalid checksum algorithm code: %d. File corrupt ?\n", cksum); fprintf(stderr, "Invalid checksum algorithm code: %d. File corrupt ?\n", cksum);
UNCOMP_BAIL; UNCOMP_BAIL;
} }
@ -580,7 +602,13 @@ start_decompress(const char *filename, const char *to_filename)
uint64_t nonce; uint64_t nonce;
uchar_t pw[MAX_PW_LEN]; uchar_t pw[MAX_PW_LEN];
int pw_len; int pw_len;
mac_ctx_t hdr_mac;
uchar_t hdr_hash1[mac_bytes], hdr_hash2[mac_bytes];
unsigned int hlen;
unsigned short d1;
unsigned int d2;
compressed_chunksize += mac_bytes;
encrypt_type = flags & MASK_CRYPTO_ALG; encrypt_type = flags & MASK_CRYPTO_ALG;
if (Read(compfd, &saltlen, sizeof (saltlen)) < sizeof (saltlen)) { if (Read(compfd, &saltlen, sizeof (saltlen)) < sizeof (saltlen)) {
perror("Read: "); perror("Read: ");
@ -606,13 +634,21 @@ start_decompress(const char *filename, const char *to_filename)
} }
nonce = ntohll(nonce); nonce = ntohll(nonce);
if (Read(compfd, hdr_hash1, mac_bytes) < mac_bytes) {
memset(salt2, 0, saltlen);
free(salt2);
perror("Read: ");
UNCOMP_BAIL;
}
deserialize_checksum(hdr_hash2, hdr_hash1, mac_bytes);
if (!pwd_file) { if (!pwd_file) {
pw_len = get_pw_string(pw, pw_len = get_pw_string(pw,
"Please enter encryption password"); "Please enter encryption password");
if (pw_len == -1) { if (pw_len == -1) {
memset(salt2, 0, saltlen); memset(salt2, 0, saltlen);
free(salt2); free(salt2);
err_exit(1, "Failed to get password.\n"); err_exit(0, "Failed to get password.\n");
} }
} else { } else {
int fd, len; int fd, len;
@ -644,21 +680,45 @@ start_decompress(const char *filename, const char *to_filename)
perror(" "); perror(" ");
memset(salt2, 0, saltlen); memset(salt2, 0, saltlen);
free(salt2); free(salt2);
err_exit(1, "Failed to get password.\n"); err_exit(0, "Failed to get password.\n");
} }
close(fd); close(fd);
} }
if (init_crypto(&crypto_ctx, pw, pw_len, encrypt_type, salt2, if (init_crypto(&crypto_ctx, pw, pw_len, encrypt_type, salt2,
saltlen, nonce, DECRYPT_FLAG) == -1) { saltlen, nonce, DECRYPT_FLAG) == -1) {
memset(salt2, 0, saltlen); memset(salt2, 0, saltlen);
free(salt2); free(salt2);
memset(pw, 0, MAX_PW_LEN); memset(pw, 0, MAX_PW_LEN);
err_exit(1, "Failed to initialize crypto\n"); err_exit(0, "Failed to initialize crypto\n");
} }
memset(salt2, 0, saltlen); memset(salt2, 0, saltlen);
free(salt2); free(salt2);
nonce = 0; nonce = 0;
memset(pw, 0, MAX_PW_LEN); memset(pw, 0, MAX_PW_LEN);
/*
* Verify header HMAC.
*/
if (hmac_init(&hdr_mac, cksum, &crypto_ctx) == -1) {
err_exit(0, "Cannot initialize header hmac.\n");
}
hmac_update(&hdr_mac, (uchar_t *)algo, ALGO_SZ);
d1 = htons(version);
hmac_update(&hdr_mac, (uchar_t *)&d1, sizeof (version));
d1 = htons(flags);
hmac_update(&hdr_mac, (uchar_t *)&d1, sizeof (version));
nonce = htonll(chunksize);
hmac_update(&hdr_mac, (uchar_t *)&nonce, sizeof (nonce));
d2 = htonl(level);
hmac_update(&hdr_mac, (uchar_t *)&d2, sizeof (level));
hmac_final(&hdr_mac, hdr_hash1, &hlen);
hmac_cleanup(&hdr_mac);
if (memcmp(hdr_hash2, hdr_hash1, mac_bytes) != 0) {
err_exit(0, "Header verification failed! File tampered.\n");
}
} else {
mac_bytes = 0;
} }
nprocs = sysconf(_SC_NPROCESSORS_ONLN); nprocs = sysconf(_SC_NPROCESSORS_ONLN);
@ -758,6 +818,15 @@ start_decompress(const char *filename, const char *to_filename)
} }
tdat->len_cmp = htonll(tdat->len_cmp); tdat->len_cmp = htonll(tdat->len_cmp);
/*
* Check for ridiculous length.
*/
if (tdat->len_cmp > chunksize + 256) {
fprintf(stderr, "Compressed length too big for chunk: %d\n",
chunk_num);
UNCOMP_BAIL;
}
/* /*
* Zero compressed len means end of file. * Zero compressed len means end of file.
*/ */
@ -799,9 +868,9 @@ start_decompress(const char *filename, const char *to_filename)
* Now read compressed chunk including the checksum. * Now read compressed chunk including the checksum.
*/ */
tdat->rbytes = Read(compfd, tdat->compressed_chunk, tdat->rbytes = Read(compfd, tdat->compressed_chunk,
tdat->len_cmp + cksum_bytes + CHUNK_FLAG_SZ); tdat->len_cmp + cksum_bytes + mac_bytes + CHUNK_FLAG_SZ);
if (main_cancel) break; if (main_cancel) break;
if (tdat->rbytes < tdat->len_cmp + cksum_bytes + CHUNK_FLAG_SZ) { if (tdat->rbytes < tdat->len_cmp + cksum_bytes + mac_bytes + CHUNK_FLAG_SZ) {
if (tdat->rbytes < 0) { if (tdat->rbytes < 0) {
perror("Read: "); perror("Read: ");
UNCOMP_BAIL; UNCOMP_BAIL;
@ -978,6 +1047,7 @@ plain_compress:
compressed_chunk, &_chunksize, tdat->level, 0, tdat->data); compressed_chunk, &_chunksize, tdat->level, 0, tdat->data);
} }
} }
/* /*
* Sanity check to ensure compressed data is lesser than original. * Sanity check to ensure compressed data is lesser than original.
* If at all compression expands/does not shrink data then the chunk * If at all compression expands/does not shrink data then the chunk
@ -1030,7 +1100,7 @@ plain_compress:
serialize_checksum(tdat->checksum, tdat->cmp_seg + sizeof (tdat->len_cmp), cksum_bytes); serialize_checksum(tdat->checksum, tdat->cmp_seg + sizeof (tdat->len_cmp), cksum_bytes);
tdat->len_cmp += CHUNK_FLAG_SZ; tdat->len_cmp += CHUNK_FLAG_SZ;
tdat->len_cmp += sizeof (len_cmp); tdat->len_cmp += sizeof (len_cmp);
tdat->len_cmp += cksum_bytes; tdat->len_cmp += (cksum_bytes + mac_bytes);
if (adapt_mode) if (adapt_mode)
type |= (rv << 4); type |= (rv << 4);
@ -1159,6 +1229,7 @@ start_compress(const char *filename, uint64_t chunksize, int level)
uchar_t pw[MAX_PW_LEN]; uchar_t pw[MAX_PW_LEN];
int pw_len; int pw_len;
compressed_chunksize += mac_bytes;
if (!pwd_file) { if (!pwd_file) {
pw_len = get_pw_string(pw, pw_len = get_pw_string(pw,
"Please enter encryption password"); "Please enter encryption password");
@ -1379,16 +1450,31 @@ start_compress(const char *filename, uint64_t chunksize, int level)
} }
/* /*
* If encryption is enabled, write the salt and nonce. * If encryption is enabled, compute header HMAC. Then
* write the salt, nonce and header hmac in that order.
*/ */
pos = cread_buf;
if (encrypt_type) { if (encrypt_type) {
mac_ctx_t hdr_mac;
uchar_t hdr_hash[mac_bytes];
unsigned int hlen;
if (hmac_init(&hdr_mac, cksum, &crypto_ctx) == -1) {
fprintf(stderr, "Cannot initialize header hmac.\n");
COMP_BAIL;
}
hmac_update(&hdr_mac, cread_buf, pos - cread_buf);
hmac_final(&hdr_mac, hdr_hash, &hlen);
hmac_cleanup(&hdr_mac);
pos = cread_buf;
*((int *)pos) = htonl(crypto_ctx.saltlen); *((int *)pos) = htonl(crypto_ctx.saltlen);
pos += sizeof (int); pos += sizeof (int);
serialize_checksum(crypto_ctx.salt, pos, crypto_ctx.saltlen); serialize_checksum(crypto_ctx.salt, pos, crypto_ctx.saltlen);
pos += crypto_ctx.saltlen; pos += crypto_ctx.saltlen;
*((uint64_t *)pos) = htonll(crypto_nonce(&crypto_ctx)); *((uint64_t *)pos) = htonll(crypto_nonce(&crypto_ctx));
pos += 8; pos += 8;
serialize_checksum(hdr_hash, pos, hlen);
pos += hlen;
if (Write(compfd, cread_buf, pos - cread_buf) != pos - cread_buf) { if (Write(compfd, cread_buf, pos - cread_buf) != pos - cread_buf) {
perror("Write "); perror("Write ");
COMP_BAIL; COMP_BAIL;
@ -1455,7 +1541,8 @@ start_compress(const char *filename, uint64_t chunksize, int level)
tdat->cmp_seg = (uchar_t *)slab_alloc(NULL, tdat->cmp_seg = (uchar_t *)slab_alloc(NULL,
compressed_chunksize); compressed_chunksize);
} }
tdat->compressed_chunk = tdat->cmp_seg + COMPRESSED_CHUNKSZ + cksum_bytes; tdat->compressed_chunk = tdat->cmp_seg + COMPRESSED_CHUNKSZ +
cksum_bytes + mac_bytes;
if (!tdat->cmp_seg || !tdat->uncompressed_chunk) { if (!tdat->cmp_seg || !tdat->uncompressed_chunk) {
fprintf(stderr, "Out of memory\n"); fprintf(stderr, "Out of memory\n");
COMP_BAIL; COMP_BAIL;
@ -1477,7 +1564,8 @@ start_compress(const char *filename, uint64_t chunksize, int level)
tmp = tdat->cmp_seg; tmp = tdat->cmp_seg;
tdat->cmp_seg = cread_buf; tdat->cmp_seg = cread_buf;
cread_buf = tmp; cread_buf = tmp;
tdat->compressed_chunk = tdat->cmp_seg + COMPRESSED_CHUNKSZ + cksum_bytes; tdat->compressed_chunk = tdat->cmp_seg + COMPRESSED_CHUNKSZ +
cksum_bytes + mac_bytes;
/* /*
* If there is data after the last rabin boundary in the chunk, then * If there is data after the last rabin boundary in the chunk, then
@ -1763,11 +1851,14 @@ main(int argc, char *argv[])
if (chunksize < MIN_CHUNK) { if (chunksize < MIN_CHUNK) {
err_exit(0, "Minimum chunk size is %ld\n", MIN_CHUNK); err_exit(0, "Minimum chunk size is %ld\n", MIN_CHUNK);
} }
if (chunksize > EIGHTY_PCT(get_total_ram())) {
err_exit(0, "Chunk size must not exceed 80%% of total RAM.\n");
}
break; break;
case 'l': case 'l':
level = atoi(optarg); level = atoi(optarg);
if (level < 0 || level > 14) if (level < 0 || level > MAX_LEVEL)
err_exit(0, "Compression level should be in range 0 - 14\n"); err_exit(0, "Compression level should be in range 0 - 14\n");
break; break;
@ -1829,7 +1920,7 @@ main(int argc, char *argv[])
break; break;
case 'S': case 'S':
if (get_checksum_props(optarg, &cksum, &cksum_bytes) == -1) { if (get_checksum_props(optarg, &cksum, &cksum_bytes, &mac_bytes) == -1) {
err_exit(0, "Invalid checksum type %s", optarg); err_exit(0, "Invalid checksum type %s", optarg);
} }
break; break;
@ -1873,6 +1964,7 @@ main(int argc, char *argv[])
if (!do_compress && encrypt_type) { if (!do_compress && encrypt_type) {
fprintf(stderr, "Encryption only makes sense when compressing!\n"); fprintf(stderr, "Encryption only makes sense when compressing!\n");
exit(1); exit(1);
} else if (pipe_mode && !pwd_file) { } else if (pipe_mode && !pwd_file) {
fprintf(stderr, "Pipe mode requires password to be provided in a file.\n"); fprintf(stderr, "Pipe mode requires password to be provided in a file.\n");
exit(1); exit(1);
@ -1922,8 +2014,10 @@ main(int argc, char *argv[])
main_cancel = 0; main_cancel = 0;
if (cksum == 0) if (cksum == 0)
get_checksum_props(DEFAULT_CKSUM, &cksum, &cksum_bytes); get_checksum_props(DEFAULT_CKSUM, &cksum, &cksum_bytes, &mac_bytes);
if (!encrypt_type)
mac_bytes = 0;
/* /*
* Start the main routines. * Start the main routines.
*/ */

2
pcompress.h
View file

@ -33,6 +33,7 @@ extern "C" {
#endif #endif
#include <rabin_dedup.h> #include <rabin_dedup.h>
#include <crypto_utils.h>
#define CHUNK_FLAG_SZ 1 #define CHUNK_FLAG_SZ 1
#define ALGO_SZ 8 #define ALGO_SZ 8
@ -43,6 +44,7 @@ extern "C" {
#define FLAG_SINGLE_CHUNK 4 #define FLAG_SINGLE_CHUNK 4
#define UTILITY_VERSION "0.8.6" #define UTILITY_VERSION "0.8.6"
#define MASK_CRYPTO_ALG 0x30 #define MASK_CRYPTO_ALG 0x30
#define MAX_LEVEL 14
#define COMPRESSED 1 #define COMPRESSED 1
#define UNCOMPRESSED 0 #define UNCOMPRESSED 0

368
utils/utils.c
View file

@ -35,44 +35,8 @@
#include <errno.h> #include <errno.h>
#include <link.h> #include <link.h>
#include <rabin_dedup.h> #include <rabin_dedup.h>
#include <skein.h>
#include <openssl/sha.h>
#include <openssl/rand.h>
#include <openssl/evp.h>
#include <sha256.h>
#include <crypto_aes.h>
#include "utils.h" #include "utils.h"
#include "cpuid.h"
#define PROVIDER_OPENSSL 0
#define PROVIDER_X64_OPT 1
static void init_sha256(void);
static int geturandom_bytes(uchar_t rbytes[32]);
/*
* Checksum properties
*/
typedef void (*ckinit_func_ptr)(void);
static struct {
char *name;
cksum_t cksum_id;
int bytes;
ckinit_func_ptr init_func;
} cksum_props[] = {
{"CRC64", CKSUM_CRC64, 8, NULL},
{"SKEIN256", CKSUM_SKEIN256, 32, NULL},
{"SKEIN512", CKSUM_SKEIN512, 64, NULL},
{"SHA256", CKSUM_SHA256, 32, init_sha256},
{"SHA512", CKSUM_SHA512, 64, NULL}
};
static int cksum_provider = PROVIDER_OPENSSL, ossl_inited = 0;
extern uint64_t lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc);
extern uint64_t lzma_crc64_8bchk(const uint8_t *buf, size_t size,
uint64_t crc, uint64_t *cnt);
void void
err_exit(int show_errno, const char *format, ...) err_exit(int show_errno, const char *format, ...)
@ -335,332 +299,12 @@ set_threadcounts(algo_props_t *props, int *nthreads, int nprocs, algo_threads_ty
} }
} }
int
compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, ssize_t bytes)
{
if (cksum == CKSUM_CRC64) {
uint64_t *ck = (uint64_t *)cksum_buf;
*ck = lzma_crc64(buf, bytes, 0);
} else if (cksum == CKSUM_SKEIN256) {
Skein_512_Ctxt_t ctx;
Skein_512_Init(&ctx, 256);
Skein_512_Update(&ctx, buf, bytes);
Skein_512_Final(&ctx, cksum_buf);
} else if (cksum == CKSUM_SKEIN512) {
Skein_512_Ctxt_t ctx;
Skein_512_Init(&ctx, 512);
Skein_512_Update(&ctx, buf, bytes);
Skein_512_Final(&ctx, cksum_buf);
} else if (cksum == CKSUM_SHA256) {
if (cksum_provider == PROVIDER_OPENSSL) {
SHA256_CTX ctx;
SHA256_Init(&ctx);
SHA256_Update(&ctx, buf, bytes);
SHA256_Final(cksum_buf, &ctx);
} else {
SHA256_Context ctx;
opt_SHA256_Init(&ctx);
opt_SHA256_Update(&ctx, buf, bytes);
opt_SHA256_Final(&ctx, cksum_buf);
}
} else if (cksum == CKSUM_SHA512) {
SHA512_CTX ctx;
SHA512_Init(&ctx);
SHA512_Update(&ctx, buf, bytes);
SHA512_Final(cksum_buf, &ctx);
} else {
return (-1);
}
return (0);
}
static void
init_sha256(void)
{
#ifdef WORDS_BIGENDIAN
cksum_provider = PROVIDER_OPENSSL;
#else
#ifdef __x86_64__
processor_info_t pc;
cksum_provider = PROVIDER_OPENSSL;
cpuid_basic_identify(&pc);
if (pc.proc_type == PROC_X64_INTEL || pc.proc_type == PROC_X64_AMD) {
if (opt_Init_SHA(&pc) == 0) {
cksum_provider = PROVIDER_X64_OPT;
}
}
#endif
#endif
}
/*
* Check is either the given checksum name or id is valid and
* return it's properties.
*/
int
get_checksum_props(char *name, int *cksum, int *cksum_bytes)
{
int i;
for (i=0; i<sizeof (cksum_props); i++) {
if ((name != NULL && strcmp(name, cksum_props[i].name) == 0) ||
(*cksum != 0 && *cksum == cksum_props[i].cksum_id)) {
*cksum = cksum_props[i].cksum_id;
*cksum_bytes = cksum_props[i].bytes;
if (cksum_props[i].init_func)
cksum_props[i].init_func();
return (0);
}
}
return (-1);
}
/*
* Endian independent way of storing the checksum bytes. This is actually
* storing in little endian format and a copy can be avoided in x86 land.
* However unsightly ifdefs are avoided here since this is not so performance
* critical.
*/
void
serialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes)
{
int i,j;
j = 0;
for (i=cksum_bytes; i>0; i--) {
buf[j] = checksum[i-1];
j++;
}
}
void
deserialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes)
{
int i,j;
j = 0;
for (i=cksum_bytes; i>0; i--) {
checksum[i-1] = buf[j];
j++;
}
}
int
init_crypto(crypto_ctx_t *cctx, uchar_t *pwd, int pwd_len, int crypto_alg,
uchar_t *salt, int saltlen, uint64_t nonce, int enc_dec)
{
if (crypto_alg == CRYPTO_ALG_AES) {
aes_ctx_t *actx = malloc(sizeof (aes_ctx_t));
if (enc_dec) {
/*
* Encryption init.
*/
cctx->salt = malloc(32);
salt = cctx->salt;
cctx->saltlen = 32;
if (RAND_status() != 1 || RAND_bytes(salt, 32) != 1) {
if (geturandom_bytes(salt) != 0) {
uchar_t sb[64];
int b;
struct timespec tp;
b = 0;
/* No good random pool is populated/available. What to do ? */
if (clock_gettime(CLOCK_MONOTONIC, &tp) == -1) {
time((time_t *)&sb[b]);
b += 8;
} else {
uint64_t v;
v = tp.tv_sec * 1000UL + tp.tv_nsec;
*((uint64_t *)&sb[b]) = v;
b += 8;
}
*((uint32_t *)&sb[b]) = rand();
b += 4;
*((uint32_t *)&sb[b]) = getpid();
b += 4;
compute_checksum(&sb[b], CKSUM_SHA256, sb, b);
b = 8 + 4;
*((uint32_t *)&sb[b]) = rand();
compute_checksum(salt, CKSUM_SHA256, &sb[b], 32 + 4);
}
}
/*
* Zero nonce (arg #6) since it will be generated.
*/
if (aes_init(actx, salt, 32, pwd, pwd_len, 0, enc_dec) != 0) {
fprintf(stderr, "Failed to initialize AES context\n");
return (-1);
}
} else {
/*
* Decryption init.
* Pass given nonce and salt.
*/
if (saltlen > MAX_SALTLEN) {
fprintf(stderr, "Salt too long. Max allowed length is %d\n",
MAX_SALTLEN);
return (-1);
}
cctx->salt = malloc(saltlen);
memcpy(cctx->salt, salt, saltlen);
if (aes_init(actx, cctx->salt, saltlen, pwd, pwd_len, nonce,
enc_dec) != 0) {
fprintf(stderr, "Failed to initialize AES context\n");
return (-1);
}
}
cctx->crypto_ctx = actx;
cctx->crypto_alg = crypto_alg;
cctx->enc_dec = enc_dec;
} else {
fprintf(stderr, "Unrecognized algorithm code: %d\n", crypto_alg);
return (-1);
}
return (0);
}
int
crypto_buf(crypto_ctx_t *cctx, uchar_t *from, uchar_t *to, ssize_t bytes, uint64_t id)
{
if (cctx->crypto_alg == CRYPTO_ALG_AES) {
if (cctx->enc_dec == ENCRYPT_FLAG) {
return (aes_encrypt(cctx->crypto_ctx, from, to, bytes, id));
} else {
return (aes_decrypt(cctx->crypto_ctx, from, to, bytes, id));
}
} else {
fprintf(stderr, "Unrecognized algorithm code: %d\n", cctx->crypto_alg);
return (-1);
}
return (0);
}
uint64_t uint64_t
crypto_nonce(crypto_ctx_t *cctx) get_total_ram()
{ {
return (aes_nonce(cctx->crypto_ctx)); uint64_t phys_pages, page_size;
}
page_size = sysconf(_SC_PAGESIZE);
void phys_pages = sysconf(_SC_PHYS_PAGES);
cleanup_crypto(crypto_ctx_t *cctx) return (phys_pages * page_size);
{
aes_cleanup(cctx->crypto_ctx);
memset(cctx->salt, 0, 32);
free(cctx->salt);
free(cctx);
}
static int
geturandom_bytes(uchar_t rbytes[32])
{
int fd;
ssize_t lenread;
uchar_t * buf = rbytes;
size_t buflen = 32;
/* Open /dev/urandom. */
if ((fd = open("/dev/urandom", O_RDONLY)) == -1)
goto err0;
/* Read bytes until we have filled the buffer. */
while (buflen > 0) {
if ((lenread = read(fd, buf, buflen)) == -1)
goto err1;
/* The random device should never EOF. */
if (lenread == 0)
goto err1;
/* We're partly done. */
buf += lenread;
buflen -= lenread;
}
/* Close the device. */
while (close(fd) == -1) {
if (errno != EINTR)
goto err0;
}
/* Success! */
return (0);
err1:
close(fd);
err0:
/* Failure! */
return (4);
}
int
get_pw_string(char pw[MAX_PW_LEN], char *prompt)
{
int fd, len;
FILE *input, *strm;
struct termios oldt, newt;
uchar_t pw1[MAX_PW_LEN], pw2[MAX_PW_LEN], *s;
// Try TTY first
fd = open("/dev/tty", O_RDWR | O_NOCTTY);
if (fd != -1) {
input = fdopen(fd, "w+");
strm = input;
} else {
// Fall back to stdin
fd = STDIN_FILENO;
input = stdin;
strm = stderr;
}
tcgetattr(fd, &oldt);
newt = oldt;
newt.c_lflag &= ~ECHO;
tcsetattr(fd, TCSANOW, &newt);
fprintf(stderr, "%s: ", prompt);
fflush(stderr);
s = fgets(pw1, MAX_PW_LEN, input);
fputs("\n", stderr);
if (s == NULL) {
tcsetattr(fd, TCSANOW, &oldt);
fflush(strm);
return (-1);
}
fprintf(stderr, "%s (once more): ", prompt);
fflush(stderr);
s = fgets(pw2, MAX_PW_LEN, input);
tcsetattr(fd, TCSANOW, &oldt);
fflush(strm);
fputs("\n", stderr);
if (s == NULL) {
return (-1);
}
if (strcmp(pw1, pw2) != 0) {
fprintf(stderr, "Passwords do not match!\n");
memset(pw1, 0, MAX_PW_LEN);
memset(pw2, 0, MAX_PW_LEN);
return (-1);
}
len = strlen(pw1);
pw1[len-1] = '\0';
strcpy(pw, pw1);
memset(pw1, 0, MAX_PW_LEN);
memset(pw2, 0, MAX_PW_LEN);
return (len);
} }

41
utils/utils.h
View file

@ -34,7 +34,6 @@ extern "C" {
#define DATA_TEXT 1 #define DATA_TEXT 1
#define DATA_BINARY 2 #define DATA_BINARY 2
#define MAX_PW_LEN 16
#if !defined(sun) && !defined(__sun) #if !defined(sun) && !defined(__sun)
#define uchar_t u_char #define uchar_t u_char
@ -102,22 +101,6 @@ typedef ssize_t bsize_t;
#define DEBUG_STAT_EN(...) #define DEBUG_STAT_EN(...)
#endif #endif
/*
* Public checksum properties. CKSUM_MAX_BYTES must be updated if a
* newer larger checksum is added to the list.
*/
typedef enum {
CKSUM_CRC64 = 0x100,
CKSUM_SKEIN256 = 0x200,
CKSUM_SKEIN512 = 0x300,
CKSUM_SHA256 = 0x400,
CKSUM_SHA512 = 0x500
} cksum_t;
#define CKSUM_MASK 0x700
#define CKSUM_MAX_BYTES 64
#define DEFAULT_CKSUM "SKEIN256"
typedef struct { typedef struct {
uint32_t buf_extra; uint32_t buf_extra;
int compress_mt_capable; int compress_mt_capable;
@ -147,19 +130,6 @@ typedef struct {
proc_type_t proc_type; proc_type_t proc_type;
} processor_info_t; } processor_info_t;
#define ENCRYPT_FLAG 1
#define DECRYPT_FLAG 0
#define CRYPTO_ALG_AES 0x10
#define MAX_SALTLEN 64
typedef struct {
void *crypto_ctx;
int crypto_alg;
int enc_dec;
uchar_t *salt;
int saltlen;
} crypto_ctx_t;
extern void err_exit(int show_errno, const char *format, ...); extern void err_exit(int show_errno, const char *format, ...);
extern const char *get_execname(const char *); extern const char *get_execname(const char *);
extern int parse_numeric(ssize_t *val, const char *str); extern int parse_numeric(ssize_t *val, const char *str);
@ -170,16 +140,7 @@ extern ssize_t Read_Adjusted(int fd, uchar_t *buf, size_t count,
extern ssize_t Write(int fd, const void *buf, size_t count); extern ssize_t Write(int fd, const void *buf, size_t count);
extern void set_threadcounts(algo_props_t *props, int *nthreads, int nprocs, extern void set_threadcounts(algo_props_t *props, int *nthreads, int nprocs,
algo_threads_type_t typ); algo_threads_type_t typ);
extern int compute_checksum(uchar_t *cksum_buf, int cksum, uchar_t *buf, ssize_t bytes); extern uint64_t get_total_ram();
extern int get_checksum_props(char *name, int *cksum, int *cksum_bytes);
extern void serialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes);
extern void deserialize_checksum(uchar_t *checksum, uchar_t *buf, int cksum_bytes);
extern int init_crypto(crypto_ctx_t *cctx, uchar_t *pwd, int pwd_len, int crypto_alg,
uchar_t *salt, int saltlen, uint64_t nonce, int enc_dec);
extern int crypto_buf(crypto_ctx_t *cctx, uchar_t *from, uchar_t *to, ssize_t bytes, uint64_t id);
extern uint64_t crypto_nonce(crypto_ctx_t *cctx);
extern void cleanup_crypto(crypto_ctx_t *cctx);
extern int get_pw_string(char pw[MAX_PW_LEN], char *prompt);
/* Pointer type for compress and decompress functions. */ /* Pointer type for compress and decompress functions. */
typedef int (*compress_func_ptr)(void *src, size_t srclen, void *dst, typedef int (*compress_func_ptr)(void *src, size_t srclen, void *dst,