greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use diferent average Rabin block sizes depending on compression algorithm.

Browse source 
Misc cleanups.
This commit is contained in:
Moinak Ghosh 2012-07-03 22:47:24 +05:30
parent a13c61e926
commit 1eee08040f
3 changed files with 64 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
main.c
View file

@ -362,6 +362,7 @@ start_decompress(const char *filename, const char *to_filename)
fprintf(stderr, "%s is not a pcompressed file.\n", filename); fprintf(stderr, "%s is not a pcompressed file.\n", filename);
UNCOMP_BAIL; UNCOMP_BAIL;
} }
algo = algorithm;
if (Read(compfd, &version, sizeof (version)) < sizeof (version) || if (Read(compfd, &version, sizeof (version)) < sizeof (version) ||
Read(compfd, &flags, sizeof (flags)) < sizeof (flags) || Read(compfd, &flags, sizeof (flags)) < sizeof (flags) ||
@ -433,7 +434,7 @@ start_decompress(const char *filename, const char *to_filename)
if (_init_func) if (_init_func)
_init_func(&(tdat->data), &(tdat->level), chunksize); _init_func(&(tdat->data), &(tdat->level), chunksize);
if (enable_rabin_scan) if (enable_rabin_scan)
tdat->rctx = create_rabin_context(chunksize); tdat->rctx = create_rabin_context(chunksize, algo);
else else
tdat->rctx = NULL; tdat->rctx = NULL;
if (pthread_create(&(tdat->thr), NULL, perform_decompress, if (pthread_create(&(tdat->thr), NULL, perform_decompress,
@ -813,10 +814,12 @@ start_compress(const char *filename, uint64_t chunksize, int level)
chunksize = sbuf.st_size; chunksize = sbuf.st_size;
nthreads = 1; nthreads = 1;
} else { } else {
if (nthreads > sbuf.st_size / chunksize) {
nthreads = sbuf.st_size / chunksize; nthreads = sbuf.st_size / chunksize;
if (sbuf.st_size % chunksize) if (sbuf.st_size % chunksize)
nthreads++; nthreads++;
} }
}
/* /*
* Create a temporary file to hold compressed data which is renamed at * Create a temporary file to hold compressed data which is renamed at
@ -894,7 +897,7 @@ start_compress(const char *filename, uint64_t chunksize, int level)
if (_init_func) if (_init_func)
_init_func(&(tdat->data), &(tdat->level), chunksize); _init_func(&(tdat->data), &(tdat->level), chunksize);
if (enable_rabin_scan) if (enable_rabin_scan)
tdat->rctx = create_rabin_context(chunksize); tdat->rctx = create_rabin_context(chunksize, algo);
else else
tdat->rctx = NULL; tdat->rctx = NULL;

77
rabin/rabin_polynomial.c
View file

@ -83,22 +83,36 @@ extern int lzma_decompress(void *src, size_t srclen, void *dst,
size_t *dstlen, int level, uchar_t chdr, void *data); size_t *dstlen, int level, uchar_t chdr, void *data);
extern int lzma_deinit(void **data); extern int lzma_deinit(void **data);
unsigned int rabin_polynomial_max_block_size = RAB_POLYNOMIAL_MAX_BLOCK_SIZE; uint32_t rabin_polynomial_max_block_size = RAB_POLYNOMIAL_MAX_BLOCK_SIZE;
unsigned int rabin_polynomial_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE;
unsigned int rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK;
/* /*
* Initialize the algorithm with the default params. * Initialize the algorithm with the default params.
*/ */
rabin_context_t * rabin_context_t *
create_rabin_context(uint64_t chunksize) { create_rabin_context(uint64_t chunksize, char *algo) {
rabin_context_t *ctx; rabin_context_t *ctx;
unsigned char *current_window_data; unsigned char *current_window_data;
unsigned int blknum, index; uint32_t blknum, index;
int level = 14; int level = 14;
blknum = chunksize / rabin_polynomial_min_block_size; /*
if (chunksize % rabin_polynomial_min_block_size) * For LZMA we use 4K minimum Rabin block size. For everything else it
* is 1K based on experimentation.
*/
ctx = (rabin_context_t *)slab_alloc(NULL, sizeof (rabin_context_t));
ctx->rabin_poly_max_block_size = RAB_POLYNOMIAL_MAX_BLOCK_SIZE;
if (memcmp(algo, "lzma", 4) == 0) {
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE;
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK;
ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE;
} else {
ctx->rabin_poly_min_block_size = RAB_POLYNOMIAL_MIN_BLOCK_SIZE2;
ctx->rabin_avg_block_mask = RAB_POLYNOMIAL_AVG_BLOCK_MASK2;
ctx->rabin_poly_avg_block_size = RAB_POLYNOMIAL_AVG_BLOCK_SIZE2;
}
blknum = chunksize / ctx->rabin_poly_min_block_size;
if (chunksize % ctx->rabin_poly_min_block_size)
blknum++; blknum++;
if (blknum > RABIN_MAX_BLOCKS) { if (blknum > RABIN_MAX_BLOCKS) {
@ -106,10 +120,9 @@ create_rabin_context(uint64_t chunksize) {
destroy_rabin_context(ctx); destroy_rabin_context(ctx);
return (NULL); return (NULL);
} }
ctx = (rabin_context_t *)slab_alloc(NULL, sizeof (rabin_context_t));
current_window_data = slab_alloc(NULL, RAB_POLYNOMIAL_WIN_SIZE); current_window_data = slab_alloc(NULL, RAB_POLYNOMIAL_WIN_SIZE);
ctx->blocks = (rabin_blockentry_t *)slab_alloc(NULL, ctx->blocks = (rabin_blockentry_t *)slab_alloc(NULL,
blknum * rabin_polynomial_min_block_size); blknum * ctx->rabin_poly_min_block_size);
if(ctx == NULL || current_window_data == NULL || ctx->blocks == NULL) { if(ctx == NULL || current_window_data == NULL || ctx->blocks == NULL) {
fprintf(stderr, fprintf(stderr,
"Could not allocate rabin polynomial context, out of memory\n"); "Could not allocate rabin polynomial context, out of memory\n");
@ -181,13 +194,13 @@ cmpblks(const void *a, const void *b)
* Perform Deduplication based on Rabin Fingerprinting. A 32-byte window is used for * Perform Deduplication based on Rabin Fingerprinting. A 32-byte window is used for
* the rolling checksum and dedup blocks vary in size from 4K-128K. * the rolling checksum and dedup blocks vary in size from 4K-128K.
*/ */
unsigned int uint32_t
rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset) rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
{ {
ssize_t i, last_offset,j; ssize_t i, last_offset,j;
unsigned int blknum; uint32_t blknum;
char *buf1 = (char *)buf; char *buf1 = (char *)buf;
unsigned int length; uint32_t length;
ssize_t rabin_index_sz; ssize_t rabin_index_sz;
length = offset; length = offset;
@ -195,11 +208,12 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
blknum = 0; blknum = 0;
ctx->valid = 0; ctx->valid = 0;
if (*size < RAB_POLYNOMIAL_AVG_BLOCK_SIZE) return; if (*size < ctx->rabin_poly_avg_block_size) return;
for (i=offset; i<*size; i++) { for (i=offset; i<*size; i++) {
char cur_byte = buf1[i]; char cur_byte = buf1[i];
uint64_t pushed_out = ctx->current_window_data[ctx->window_pos]; uint64_t pushed_out = ctx->current_window_data[ctx->window_pos];
ctx->current_window_data[ctx->window_pos] = cur_byte; ctx->current_window_data[ctx->window_pos] = cur_byte;
int msk;
/* /*
* We want to do: * We want to do:
* cur_roll_checksum = cur_roll_checksum * RAB_POLYNOMIAL_CONST + cur_byte; * cur_roll_checksum = cur_roll_checksum * RAB_POLYNOMIAL_CONST + cur_byte;
@ -220,10 +234,10 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
if (ctx->window_pos == RAB_POLYNOMIAL_WIN_SIZE) // Loop back around if (ctx->window_pos == RAB_POLYNOMIAL_WIN_SIZE) // Loop back around
ctx->window_pos=0; ctx->window_pos=0;
if (length < rabin_polynomial_min_block_size) continue; if (length < ctx->rabin_poly_min_block_size) continue;
// If we hit our special value or reached the max block size update block offset // If we hit our special value or reached the max block size update block offset
if ((ctx->cur_roll_checksum & rabin_avg_block_mask) == RAB_POLYNOMIAL_CONST || if ((ctx->cur_roll_checksum & ctx->rabin_avg_block_mask) == RAB_POLYNOMIAL_CONST ||
length >= rabin_polynomial_max_block_size) { length >= rabin_polynomial_max_block_size) {
ctx->blocks[blknum].offset = last_offset; ctx->blocks[blknum].offset = last_offset;
ctx->blocks[blknum].index = blknum; // Need to store for sorting ctx->blocks[blknum].index = blknum; // Need to store for sorting
@ -240,11 +254,11 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
// If we found at least a few chunks, perform dedup. // If we found at least a few chunks, perform dedup.
if (blknum > 2) { if (blknum > 2) {
uint64_t prev_cksum; uint64_t prev_cksum;
unsigned int blk, prev_length; uint32_t blk, prev_length;
ssize_t pos, matches; ssize_t pos, matchlen;
int valid = 1; int valid = 1;
char *tmp, *prev_offset; char *tmp, *prev_offset;
unsigned int *rabin_index, prev_index, prev_blk; uint32_t *rabin_index, prev_index, prev_blk;
// Insert the last left-over trailing bytes, if any, into a block. // Insert the last left-over trailing bytes, if any, into a block.
if (last_offset < *size) { if (last_offset < *size) {
@ -271,8 +285,8 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
* TODO: Test with a heavily optimized MD5 (from OpenSSL?) later. * TODO: Test with a heavily optimized MD5 (from OpenSSL?) later.
*/ */
qsort(ctx->blocks, blknum, sizeof (rabin_blockentry_t), cmpblks); qsort(ctx->blocks, blknum, sizeof (rabin_blockentry_t), cmpblks);
rabin_index = (unsigned int *)(ctx->cbuf + RABIN_HDR_SIZE); rabin_index = (uint32_t *)(ctx->cbuf + RABIN_HDR_SIZE);
matches = 0; matchlen = 0;
/* /*
* Now make a pass through the sorted block array making identical blocks * Now make a pass through the sorted block array making identical blocks
@ -293,7 +307,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
ctx->blocks[blk].length = 0; ctx->blocks[blk].length = 0;
ctx->blocks[blk].index = prev_index; ctx->blocks[blk].index = prev_index;
(ctx->blocks[prev_blk].refcount)++; (ctx->blocks[prev_blk].refcount)++;
matches += prev_length; matchlen += prev_length;
continue; continue;
} }
@ -304,7 +318,7 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
prev_blk = blk; prev_blk = blk;
} }
if (matches < rabin_index_sz) { if (matchlen < rabin_index_sz) {
ctx->valid = 0; ctx->valid = 0;
return; return;
} }
@ -367,14 +381,13 @@ rabin_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size, ssize_t offset)
rabin_index[blk] = htonl(be->index | RABIN_INDEX_FLAG); rabin_index[blk] = htonl(be->index | RABIN_INDEX_FLAG);
} }
} }
cont: cont:
if (valid) { if (valid) {
uchar_t *cbuf = ctx->cbuf; uchar_t *cbuf = ctx->cbuf;
ssize_t *entries; ssize_t *entries;
*((unsigned int *)cbuf) = htonl(blknum); *((uint32_t *)cbuf) = htonl(blknum);
cbuf += sizeof (unsigned int); cbuf += sizeof (uint32_t);
entries = (ssize_t *)cbuf; entries = (ssize_t *)cbuf;
entries[0] = htonll(*size); entries[0] = htonll(*size);
entries[1] = 0; entries[1] = 0;
@ -396,21 +409,21 @@ rabin_update_hdr(uchar_t *buf, ssize_t rabin_index_sz_cmp, ssize_t rabin_data_sz
{ {
ssize_t *entries; ssize_t *entries;
buf += sizeof (unsigned int); buf += sizeof (uint32_t);
entries = (ssize_t *)buf; entries = (ssize_t *)buf;
entries[1] = htonll(rabin_index_sz_cmp); entries[1] = htonll(rabin_index_sz_cmp);
entries[3] = htonll(rabin_data_sz_cmp); entries[3] = htonll(rabin_data_sz_cmp);
} }
void void
rabin_parse_hdr(uchar_t *buf, unsigned int *blknum, ssize_t *rabin_index_sz, rabin_parse_hdr(uchar_t *buf, uint32_t *blknum, ssize_t *rabin_index_sz,
ssize_t *rabin_data_sz, ssize_t *rabin_index_sz_cmp, ssize_t *rabin_data_sz, ssize_t *rabin_index_sz_cmp,
ssize_t *rabin_data_sz_cmp, ssize_t *rabin_deduped_size) ssize_t *rabin_data_sz_cmp, ssize_t *rabin_deduped_size)
{ {
ssize_t *entries; ssize_t *entries;
*blknum = ntohl(*((unsigned int *)(buf))); *blknum = ntohl(*((uint32_t *)(buf)));
buf += sizeof (unsigned int); buf += sizeof (uint32_t);
entries = (ssize_t *)buf; entries = (ssize_t *)buf;
*rabin_data_sz = ntohll(entries[0]); *rabin_data_sz = ntohll(entries[0]);
@ -423,14 +436,14 @@ rabin_parse_hdr(uchar_t *buf, unsigned int *blknum, ssize_t *rabin_index_sz,
void void
rabin_inverse_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size) rabin_inverse_dedup(rabin_context_t *ctx, uchar_t *buf, ssize_t *size)
{ {
unsigned int blknum, blk, oblk, len; uint32_t blknum, blk, oblk, len;
unsigned int *rabin_index; uint32_t *rabin_index;
ssize_t data_sz, sz, indx_cmp, data_sz_cmp, deduped_sz; ssize_t data_sz, sz, indx_cmp, data_sz_cmp, deduped_sz;
ssize_t rabin_index_sz, pos1, i; ssize_t rabin_index_sz, pos1, i;
uchar_t *pos2; uchar_t *pos2;
rabin_parse_hdr(buf, &blknum, &rabin_index_sz, &data_sz, &indx_cmp, &data_sz_cmp, &deduped_sz); rabin_parse_hdr(buf, &blknum, &rabin_index_sz, &data_sz, &indx_cmp, &data_sz_cmp, &deduped_sz);
rabin_index = (unsigned int *)(buf + RABIN_HDR_SIZE); rabin_index = (uint32_t *)(buf + RABIN_HDR_SIZE);
pos1 = rabin_index_sz + RABIN_HDR_SIZE; pos1 = rabin_index_sz + RABIN_HDR_SIZE;
pos2 = ctx->cbuf; pos2 = ctx->cbuf;
sz = 0; sz = 0;

14
rabin/rabin_polynomial.h
View file

@ -73,6 +73,12 @@
#define RAB_POLYNOMIAL_AVG_BLOCK_MASK (RAB_POLYNOMIAL_AVG_BLOCK_SIZE - 1) #define RAB_POLYNOMIAL_AVG_BLOCK_MASK (RAB_POLYNOMIAL_AVG_BLOCK_SIZE - 1)
#define RAB_POLYNOMIAL_MIN_BLOCK_SIZE RAB_POLYNOMIAL_AVG_BLOCK_SIZE #define RAB_POLYNOMIAL_MIN_BLOCK_SIZE RAB_POLYNOMIAL_AVG_BLOCK_SIZE
#define RAB_POLYNOMIAL_MAX_BLOCK_SIZE (128 * 1024) #define RAB_POLYNOMIAL_MAX_BLOCK_SIZE (128 * 1024)
#define RAB_POLYNOMIAL_AVG_BLOCK_SHIFT2 10
#define RAB_POLYNOMIAL_AVG_BLOCK_SIZE2 (1 << RAB_POLYNOMIAL_AVG_BLOCK_SHIFT)
#define RAB_POLYNOMIAL_AVG_BLOCK_MASK2 (RAB_POLYNOMIAL_AVG_BLOCK_SIZE - 1)
#define RAB_POLYNOMIAL_MIN_BLOCK_SIZE2 RAB_POLYNOMIAL_AVG_BLOCK_SIZE
#define RAB_POLYNOMIAL_WIN_SIZE 31 #define RAB_POLYNOMIAL_WIN_SIZE 31
#define RAB_POLYNOMIAL_MIN_WIN_SIZE 17 #define RAB_POLYNOMIAL_MIN_WIN_SIZE 17
#define RAB_POLYNOMIAL_MAX_WIN_SIZE 63 #define RAB_POLYNOMIAL_MAX_WIN_SIZE 63
@ -118,18 +124,20 @@ typedef struct {
unsigned char *current_window_data; unsigned char *current_window_data;
rabin_blockentry_t *blocks; rabin_blockentry_t *blocks;
unsigned char *cbuf; unsigned char *cbuf;
unsigned char *buf;
int window_pos; int window_pos;
uint64_t cur_roll_checksum; uint64_t cur_roll_checksum;
uint64_t cur_checksum; uint64_t cur_checksum;
uint64_t block_checksum; uint32_t rabin_poly_max_block_size;
uint32_t rabin_poly_min_block_size;
uint32_t rabin_poly_avg_block_size;
uint32_t rabin_avg_block_mask;
int dedup; int dedup;
int valid; int valid;
void *lzma_data; void *lzma_data;
int level; int level;
} rabin_context_t; } rabin_context_t;
extern rabin_context_t *create_rabin_context(uint64_t chunksize); extern rabin_context_t *create_rabin_context(uint64_t chunksize, char *algo);
extern void destroy_rabin_context(rabin_context_t *ctx); extern void destroy_rabin_context(rabin_context_t *ctx);
extern unsigned int rabin_dedup(rabin_context_t *ctx, unsigned char *buf, extern unsigned int rabin_dedup(rabin_context_t *ctx, unsigned char *buf,
ssize_t *size, ssize_t offset); ssize_t *size, ssize_t offset);