Use diferent average Rabin block sizes depending on compression algorithm.
Misc cleanups.
This commit is contained in:
parent
a13c61e926
commit
1eee08040f
3 changed files with 64 additions and 40 deletions
13
main.c
13
main.c
|
@ -362,6 +362,7 @@ start_decompress(const char *filename, const char *to_filename)
|
||||||
fprintf(stderr, "%s is not a pcompressed file.\n", filename);
|
fprintf(stderr, "%s is not a pcompressed file.\n", filename);
|
||||||
UNCOMP_BAIL;
|
UNCOMP_BAIL;
|
||||||
}
|
}
|
||||||
|
algo = algorithm;
|
||||||
|
|
||||||
if (Read(compfd, &version, sizeof (version)) < sizeof (version) ||
|
if (Read(compfd, &version, sizeof (version)) < sizeof (version) ||
|
||||||
Read(compfd, &flags, sizeof (flags)) < sizeof (flags) ||
|
Read(compfd, &flags, sizeof (flags)) < sizeof (flags) ||
|
||||||
|
@ -433,7 +434,7 @@ start_decompress(const char *filename, const char *to_filename)
|
||||||
if (_init_func)
|
if (_init_func)
|
||||||
_init_func(&(tdat->data), &(tdat->level), chunksize);
|
_init_func(&(tdat->data), &(tdat->level), chunksize);
|
||||||
if (enable_rabin_scan)
|
if (enable_rabin_scan)
|
||||||
tdat->rctx = create_rabin_context(chunksize);
|
tdat->rctx = create_rabin_context(chunksize, algo);
|
||||||
else
|
else
|
||||||
tdat->rctx = NULL;
|
tdat->rctx = NULL;
|
||||||
if (pthread_create(&(tdat->thr), NULL, perform_decompress,
|
if (pthread_create(&(tdat->thr), NULL, perform_decompress,
|
||||||
|
@ -813,9 +814,11 @@ start_compress(const char *filename, uint64_t chunksize, int level)
|
||||||
chunksize = sbuf.st_size;
|
chunksize = sbuf.st_size;
|
||||||
nthreads = 1;
|
nthreads = 1;
|
||||||
} else {
|
} else {
|
||||||
nthreads = sbuf.st_size / chunksize;
|
if (nthreads > sbuf.st_size / chunksize) {
|
||||||
if (sbuf.st_size % chunksize)
|
nthreads = sbuf.st_size / chunksize;
|
||||||
nthreads++;
|
if (sbuf.st_size % chunksize)
|
||||||
|
nthreads++;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
@ -894,7 +897,7 @@ start_compress(const char *filename, uint64_t chunksize, int level)
|
||||||
if (_init_func)
|
if (_init_func)
|
||||||
_init_func(&(tdat->data), &(tdat->level), chunksize);
|
_init_func(&(tdat->data), &(tdat->level), chunksize);
|
||||||
if (enable_rabin_scan)
|
if (enable_rabin_scan)
|
||||||
tdat->rctx = create_rabin_context(chunksize);
|
tdat->rctx = create_rabin_context(chunksize, algo);
|
||||||
else
|
else
|
||||||
tdat->rctx = NULL;
|
tdat->rctx = NULL;
|
||||||
|
|
||||||
|
|
|
@ -83,22 +83,36 @@ extern int lzma_decompress(void *src, size_t srclen, void *dst,
|
||||||
size_t *dstlen, int level, uchar_t chdr, void *data);
|
size_t *dstlen, int level, uchar_t chdr, void *data);
|
||||||
extern int lzma_deinit(void **data);
|
extern int lzma_deinit(void **data);
|
||||||
|
|
||||||
unsigned int rabin_polynomial_max_block_size = RAB_POLYNOMIAL_MAX_BLOCK_SIZE;
|
uint32_t rabin_polynomial_max_block_size = RAB_POLYNOMIAL_MAX_BLOCK_SIZE;
|
||||||
unsigned int rabin_polynomial_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE;
|
|
||||||
unsigned int rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK;
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Initialize the algorithm with the default params.
|
* Initialize the algorithm with the default params.
|
||||||
*/
|
*/
|
||||||
rabin_context_t *
|
rabin_context_t *
|
||||||
create_rabin_context(uint64_t chunksize) {
|
create_rabin_context(uint64_t chunksize, char *algo) {
|
||||||
rabin_context_t *ctx;
|
rabin_context_t *ctx;
|
||||||
unsigned char *current_window_data;
|
unsigned char *current_window_data;
|
||||||
unsigned int blknum, index;
|
uint32_t blknum, index;
|
||||||
int level = 14;
|
int level = 14;
|
||||||
|
|
||||||
blknum = chunksize / rabin_polynomial_min_block_size;
|
/*
|
||||||
if (chunksize % rabin_polynomial_min_block_size)
|
* For LZMA we use 4K minimum Rabin block size. For everything else it
|
||||||
|
* is 1K based on experimentation.
|
||||||
|
*/
|
||||||
|
ctx = (rabin_context_t *)slab_alloc(NULL, sizeof (rabin_context_t));
|
||||||
|
ctx->rabin_poly_max_block_size = RAB_POLYNOMIAL_MAX_BLOCK_SIZE;
|
||||||
|
if (memcmp(algo, "lzma", 4) == 0) {
|
||||||
|
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE;
|
||||||
|
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK;
|
||||||
|
ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE;
|
||||||
|
} else {
|
||||||
|
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE2;
|
||||||
|
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK2;
|
||||||
|
ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE2;
|
||||||
|
}
|
||||||
|
|
||||||
|
blknum = chunksize / ctx->rabin_poly_min_block_size;
|
||||||
|
if (chunksize % ctx->rabin_poly_min_block_size)
|
||||||
blknum++;
|
blknum++;
|
||||||
|
|
||||||
if (blknum > RABIN_MAX_BLOCKS) {
|
if (blknum > RABIN_MAX_BLOCKS) {
|
||||||
|
@ -106,10 +120,9 @@ create_rabin_context(uint64_t chunksize) {
|
||||||
destroy_rabin_context(ctx);
|
destroy_rabin_context(ctx);
|
||||||
return (NULL);
|
return (NULL);
|
||||||
}
|
}
|
||||||
ctx = (rabin_context_t *)slab_alloc(NULL, sizeof (rabin_context_t));
|
|
||||||
current_window_data = slab_alloc(NULL, RAB_POLYNOMIAL_WIN_SIZE);
|
current_window_data = slab_alloc(NULL, RAB_POLYNOMIAL_WIN_SIZE);
|
||||||
ctx->blocks = (rabin_blockentry_t *)slab_alloc(NULL,
|
ctx->blocks = (rabin_blockentry_t *)slab_alloc(NULL,
|
||||||
blknum * rabin_polynomial_min_block_size);
|
blknum * ctx->rabin_poly_min_block_size);
|
||||||
if(ctx == NULL || current_window_data == NULL || ctx->blocks == NULL) {
|
if(ctx == NULL || current_window_data == NULL || ctx->blocks == NULL) {
|
||||||
fprintf(stderr,
|
fprintf(stderr,
|
||||||
"Could not allocate rabin polynomial context, out of memory\n");
|
"Could not allocate rabin polynomial context, out of memory\n");
|
||||||
|
@ -181,13 +194,13 @@ cmpblks(const void *a, const void *b)
|
||||||
* Perform Deduplication based on Rabin Fingerprinting. A 32-byte window is used for
|
* Perform Deduplication based on Rabin Fingerprinting. A 32-byte window is used for
|
||||||
* the rolling checksum and dedup blocks vary in size from 4K-128K.
|
* the rolling checksum and dedup blocks vary in size from 4K-128K.
|
||||||
*/
|
*/
|
||||||
unsigned int
|
uint32_t
|
||||||
rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
|
rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
|
||||||
{
|
{
|
||||||
ssize_t i, last_offset,j;
|
ssize_t i, last_offset,j;
|
||||||
unsigned int blknum;
|
uint32_t blknum;
|
||||||
char *buf1 = (char *)buf;
|
char *buf1 = (char *)buf;
|
||||||
unsigned int length;
|
uint32_t length;
|
||||||
ssize_t rabin_index_sz;
|
ssize_t rabin_index_sz;
|
||||||
|
|
||||||
length = offset;
|
length = offset;
|
||||||
|
@ -195,11 +208,12 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
|
||||||
blknum = 0;
|
blknum = 0;
|
||||||
ctx->valid = 0;
|
ctx->valid = 0;
|
||||||
|
|
||||||
if (*size < RAB_POLYNOMIAL_AVG_BLOCK_SIZE) return;
|
if (*size < ctx->rabin_poly_avg_block_size) return;
|
||||||
for (i=offset; i<*size; i++) {
|
for (i=offset; i<*size; i++) {
|
||||||
char cur_byte = buf1[i];
|
char cur_byte = buf1[i];
|
||||||
uint64_t pushed_out = ctx->current_window_data[ctx->window_pos];
|
uint64_t pushed_out = ctx->current_window_data[ctx->window_pos];
|
||||||
ctx->current_window_data[ctx->window_pos] = cur_byte;
|
ctx->current_window_data[ctx->window_pos] = cur_byte;
|
||||||
|
int msk;
|
||||||
/*
|
/*
|
||||||
* We want to do:
|
* We want to do:
|
||||||
* cur_roll_checksum = cur_roll_checksum * RAB_POLYNOMIAL_CONST + cur_byte;
|
* cur_roll_checksum = cur_roll_checksum * RAB_POLYNOMIAL_CONST + cur_byte;
|
||||||
|
@ -220,10 +234,10 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
|
||||||
if (ctx->window_pos == RAB_POLYNOMIAL_WIN_SIZE) // Loop back around
|
if (ctx->window_pos == RAB_POLYNOMIAL_WIN_SIZE) // Loop back around
|
||||||
ctx->window_pos=0;
|
ctx->window_pos=0;
|
||||||
|
|
||||||
if (length < rabin_polynomial_min_block_size) continue;
|
if (length < ctx->rabin_poly_min_block_size) continue;
|
||||||
|
|
||||||
// If we hit our special value or reached the max block size update block offset
|
// If we hit our special value or reached the max block size update block offset
|
||||||
if ((ctx->cur_roll_checksum & rabin_avg_block_mask) == RAB_POLYNOMIAL_CONST ||
|
if ((ctx->cur_roll_checksum & ctx->rabin_avg_block_mask) == RAB_POLYNOMIAL_CONST ||
|
||||||
length >= rabin_polynomial_max_block_size) {
|
length >= rabin_polynomial_max_block_size) {
|
||||||
ctx->blocks[blknum].offset = last_offset;
|
ctx->blocks[blknum].offset = last_offset;
|
||||||
ctx->blocks[blknum].index = blknum; // Need to store for sorting
|
ctx->blocks[blknum].index = blknum; // Need to store for sorting
|
||||||
|
@ -240,11 +254,11 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
|
||||||
// If we found at least a few chunks, perform dedup.
|
// If we found at least a few chunks, perform dedup.
|
||||||
if (blknum > 2) {
|
if (blknum > 2) {
|
||||||
uint64_t prev_cksum;
|
uint64_t prev_cksum;
|
||||||
unsigned int blk, prev_length;
|
uint32_t blk, prev_length;
|
||||||
ssize_t pos, matches;
|
ssize_t pos, matchlen;
|
||||||
int valid = 1;
|
int valid = 1;
|
||||||
char *tmp, *prev_offset;
|
char *tmp, *prev_offset;
|
||||||
unsigned int *rabin_index, prev_index, prev_blk;
|
uint32_t *rabin_index, prev_index, prev_blk;
|
||||||
|
|
||||||
// Insert the last left-over trailing bytes, if any, into a block.
|
// Insert the last left-over trailing bytes, if any, into a block.
|
||||||
if (last_offset < *size) {
|
if (last_offset < *size) {
|
||||||
|
@ -271,8 +285,8 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
|
||||||
* TODO: Test with a heavily optimized MD5 (from OpenSSL?) later.
|
* TODO: Test with a heavily optimized MD5 (from OpenSSL?) later.
|
||||||
*/
|
*/
|
||||||
qsort(ctx->blocks, blknum, sizeof (rabin_blockentry_t), cmpblks);
|
qsort(ctx->blocks, blknum, sizeof (rabin_blockentry_t), cmpblks);
|
||||||
rabin_index = (unsigned int *)(ctx->cbuf + RABIN_HDR_SIZE);
|
rabin_index = (uint32_t *)(ctx->cbuf + RABIN_HDR_SIZE);
|
||||||
matches = 0;
|
matchlen = 0;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Now make a pass through the sorted block array making identical blocks
|
* Now make a pass through the sorted block array making identical blocks
|
||||||
|
@ -293,7 +307,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
|
||||||
ctx->blocks[blk].length = 0;
|
ctx->blocks[blk].length = 0;
|
||||||
ctx->blocks[blk].index = prev_index;
|
ctx->blocks[blk].index = prev_index;
|
||||||
(ctx->blocks[prev_blk].refcount)++;
|
(ctx->blocks[prev_blk].refcount)++;
|
||||||
matches += prev_length;
|
matchlen += prev_length;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -304,7 +318,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
|
||||||
prev_blk = blk;
|
prev_blk = blk;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (matches < rabin_index_sz) {
|
if (matchlen < rabin_index_sz) {
|
||||||
ctx->valid = 0;
|
ctx->valid = 0;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -367,14 +381,13 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
|
||||||
rabin_index[blk] = htonl(be->index | RABIN_INDEX_FLAG);
|
rabin_index[blk] = htonl(be->index | RABIN_INDEX_FLAG);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
cont:
|
cont:
|
||||||
if (valid) {
|
if (valid) {
|
||||||
uchar_t *cbuf = ctx->cbuf;
|
uchar_t *cbuf = ctx->cbuf;
|
||||||
ssize_t *entries;
|
ssize_t *entries;
|
||||||
|
|
||||||
*((unsigned int *)cbuf) = htonl(blknum);
|
*((uint32_t *)cbuf) = htonl(blknum);
|
||||||
cbuf += sizeof (unsigned int);
|
cbuf += sizeof (uint32_t);
|
||||||
entries = (ssize_t *)cbuf;
|
entries = (ssize_t *)cbuf;
|
||||||
entries[0] = htonll(*size);
|
entries[0] = htonll(*size);
|
||||||
entries[1] = 0;
|
entries[1] = 0;
|
||||||
|
@ -396,21 +409,21 @@ rabin_update_hdr(uchar_t *buf, ssize_t rabin_index_sz_cmp, ssize_t rabin_data_sz
|
||||||
{
|
{
|
||||||
ssize_t *entries;
|
ssize_t *entries;
|
||||||
|
|
||||||
buf += sizeof (unsigned int);
|
buf += sizeof (uint32_t);
|
||||||
entries = (ssize_t *)buf;
|
entries = (ssize_t *)buf;
|
||||||
entries[1] = htonll(rabin_index_sz_cmp);
|
entries[1] = htonll(rabin_index_sz_cmp);
|
||||||
entries[3] = htonll(rabin_data_sz_cmp);
|
entries[3] = htonll(rabin_data_sz_cmp);
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
rabin_parse_hdr(uchar_t *buf, unsigned int *blknum, ssize_t *rabin_index_sz,
|
rabin_parse_hdr(uchar_t *buf, uint32_t *blknum, ssize_t *rabin_index_sz,
|
||||||
ssize_t *rabin_data_sz, ssize_t *rabin_index_sz_cmp,
|
ssize_t *rabin_data_sz, ssize_t *rabin_index_sz_cmp,
|
||||||
ssize_t *rabin_data_sz_cmp, ssize_t *rabin_deduped_size)
|
ssize_t *rabin_data_sz_cmp, ssize_t *rabin_deduped_size)
|
||||||
{
|
{
|
||||||
ssize_t *entries;
|
ssize_t *entries;
|
||||||
|
|
||||||
*blknum = ntohl(*((unsigned int *)(buf)));
|
*blknum = ntohl(*((uint32_t *)(buf)));
|
||||||
buf += sizeof (unsigned int);
|
buf += sizeof (uint32_t);
|
||||||
|
|
||||||
entries = (ssize_t *)buf;
|
entries = (ssize_t *)buf;
|
||||||
*rabin_data_sz = ntohll(entries[0]);
|
*rabin_data_sz = ntohll(entries[0]);
|
||||||
|
@ -423,14 +436,14 @@ rabin_parse_hdr(uchar_t *buf, unsigned int *blknum, ssize_t *rabin_index_sz,
|
||||||
void
|
void
|
||||||
rabin_inverse_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size)
|
rabin_inverse_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size)
|
||||||
{
|
{
|
||||||
unsigned int blknum, blk, oblk, len;
|
uint32_t blknum, blk, oblk, len;
|
||||||
unsigned int *rabin_index;
|
uint32_t *rabin_index;
|
||||||
ssize_t data_sz, sz, indx_cmp, data_sz_cmp, deduped_sz;
|
ssize_t data_sz, sz, indx_cmp, data_sz_cmp, deduped_sz;
|
||||||
ssize_t rabin_index_sz, pos1, i;
|
ssize_t rabin_index_sz, pos1, i;
|
||||||
uchar_t *pos2;
|
uchar_t *pos2;
|
||||||
|
|
||||||
rabin_parse_hdr(buf, &blknum, &rabin_index_sz, &data_sz, &indx_cmp, &data_sz_cmp, &deduped_sz);
|
rabin_parse_hdr(buf, &blknum, &rabin_index_sz, &data_sz, &indx_cmp, &data_sz_cmp, &deduped_sz);
|
||||||
rabin_index = (unsigned int *)(buf + RABIN_HDR_SIZE);
|
rabin_index = (uint32_t *)(buf + RABIN_HDR_SIZE);
|
||||||
pos1 = rabin_index_sz + RABIN_HDR_SIZE;
|
pos1 = rabin_index_sz + RABIN_HDR_SIZE;
|
||||||
pos2 = ctx->cbuf;
|
pos2 = ctx->cbuf;
|
||||||
sz = 0;
|
sz = 0;
|
||||||
|
|
|
@ -73,6 +73,12 @@
|
||||||
#define RAB_POLYNOMIAL_AVG_BLOCK_MASK (RAB_POLYNOMIAL_AVG_BLOCK_SIZE - 1)
|
#define RAB_POLYNOMIAL_AVG_BLOCK_MASK (RAB_POLYNOMIAL_AVG_BLOCK_SIZE - 1)
|
||||||
#define RAB_POLYNOMIAL_MIN_BLOCK_SIZE RAB_POLYNOMIAL_AVG_BLOCK_SIZE
|
#define RAB_POLYNOMIAL_MIN_BLOCK_SIZE RAB_POLYNOMIAL_AVG_BLOCK_SIZE
|
||||||
#define RAB_POLYNOMIAL_MAX_BLOCK_SIZE (128 * 1024)
|
#define RAB_POLYNOMIAL_MAX_BLOCK_SIZE (128 * 1024)
|
||||||
|
|
||||||
|
#define RAB_POLYNOMIAL_AVG_BLOCK_SHIFT2 10
|
||||||
|
#define RAB_POLYNOMIAL_AVG_BLOCK_SIZE2 (1 << RAB_POLYNOMIAL_AVG_BLOCK_SHIFT)
|
||||||
|
#define RAB_POLYNOMIAL_AVG_BLOCK_MASK2 (RAB_POLYNOMIAL_AVG_BLOCK_SIZE - 1)
|
||||||
|
#define RAB_POLYNOMIAL_MIN_BLOCK_SIZE2 RAB_POLYNOMIAL_AVG_BLOCK_SIZE
|
||||||
|
|
||||||
#define RAB_POLYNOMIAL_WIN_SIZE 31
|
#define RAB_POLYNOMIAL_WIN_SIZE 31
|
||||||
#define RAB_POLYNOMIAL_MIN_WIN_SIZE 17
|
#define RAB_POLYNOMIAL_MIN_WIN_SIZE 17
|
||||||
#define RAB_POLYNOMIAL_MAX_WIN_SIZE 63
|
#define RAB_POLYNOMIAL_MAX_WIN_SIZE 63
|
||||||
|
@ -118,18 +124,20 @@ typedef struct {
|
||||||
unsigned char *current_window_data;
|
unsigned char *current_window_data;
|
||||||
rabin_blockentry_t *blocks;
|
rabin_blockentry_t *blocks;
|
||||||
unsigned char *cbuf;
|
unsigned char *cbuf;
|
||||||
unsigned char *buf;
|
|
||||||
int window_pos;
|
int window_pos;
|
||||||
uint64_t cur_roll_checksum;
|
uint64_t cur_roll_checksum;
|
||||||
uint64_t cur_checksum;
|
uint64_t cur_checksum;
|
||||||
uint64_t block_checksum;
|
uint32_t rabin_poly_max_block_size;
|
||||||
|
uint32_t rabin_poly_min_block_size;
|
||||||
|
uint32_t rabin_poly_avg_block_size;
|
||||||
|
uint32_t rabin_avg_block_mask;
|
||||||
int dedup;
|
int dedup;
|
||||||
int valid;
|
int valid;
|
||||||
void *lzma_data;
|
void *lzma_data;
|
||||||
int level;
|
int level;
|
||||||
} rabin_context_t;
|
} rabin_context_t;
|
||||||
|
|
||||||
extern rabin_context_t *create_rabin_context(uint64_t chunksize);
|
extern rabin_context_t *create_rabin_context(uint64_t chunksize, char *algo);
|
||||||
extern void destroy_rabin_context(rabin_context_t *ctx);
|
extern void destroy_rabin_context(rabin_context_t *ctx);
|
||||||
extern unsigned int rabin_dedup(rabin_context_t *ctx, unsigned char *buf,
|
extern unsigned int rabin_dedup(rabin_context_t *ctx, unsigned char *buf,
|
||||||
ssize_t *size, ssize_t offset);
|
ssize_t *size, ssize_t offset);
|
||||||
|
|
Loading…
Reference in a new issue