greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Use SHA256 for Global Dedupe chunk lookup hash by default.

Browse source 
Allow changing Global Dedupe chunk lookup hash via env variable.
This commit is contained in:
Moinak Ghosh 2013-05-02 00:05:05 +05:30
parent 6e4d45b644
commit 120877348c
2 changed files with 22 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
README.md
View file

@ -223,6 +223,10 @@ can be a directory on a Solid State Drive to speed up Global Deduplication. The
space used in this directory is proportional to the size of the dataset being space used in this directory is proportional to the size of the dataset being
processed and is slightly more than 8KB for every 1MB of data. processed and is slightly more than 8KB for every 1MB of data.
The default checksum used for chunk hashes during Global Deduplication is SHA256.
However this can be changed by setting the PCOMPRESS_CHUNK_HASH_GLOBAL environment
variable to one of the other checksum names except CRC64.
Examples Examples
======== ========

20
rabin/rabin_dedup.c
View file

@ -207,7 +207,8 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
*/ */
if (dedupe_flag == RABIN_DEDUPE_FILE_GLOBAL && op == COMPRESS && rab_blk_sz > 0) { if (dedupe_flag == RABIN_DEDUPE_FILE_GLOBAL && op == COMPRESS && rab_blk_sz > 0) {
my_sysinfo msys_info; my_sysinfo msys_info;
int pct_interval; int pct_interval, chunk_cksum, cksum_bytes, mac_bytes;
char *ck;
/* /*
* Get amount of memory to use. The freeram got here is adjusted amount. * Get amount of memory to use. The freeram got here is adjusted amount.
@ -217,8 +218,23 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
if (pipe_mode) if (pipe_mode)
pct_interval = DEFAULT_PCT_INTERVAL; pct_interval = DEFAULT_PCT_INTERVAL;
chunk_cksum = 0;
if ((ck = getenv("PCOMPRESS_CHUNK_HASH_GLOBAL")) != NULL) {
if (get_checksum_props(ck, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0 ||
strcmp(ck, "CRC64") == 0) {
fprintf(stderr, "Invalid PCOMPRESS_CHUNK_HASH_GLOBAL. Reverting to default.\n");
chunk_cksum = -1;
}
}
if (chunk_cksum == 0) {
chunk_cksum = DEFAULT_CHUNK_CKSUM;
if (get_checksum_props(NULL, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0) {
fprintf(stderr, "Invalid default chunk checksum: %d\n", DEFAULT_CHUNK_CKSUM);
return (NULL);
}
}
arc = init_global_db_s(NULL, tmppath, rab_blk_sz, chunksize, pct_interval, arc = init_global_db_s(NULL, tmppath, rab_blk_sz, chunksize, pct_interval,
algo, props->cksum, GLOBAL_SIM_CKSUM, file_size, algo, chunk_cksum, GLOBAL_SIM_CKSUM, file_size,
msys_info.freeram, nthreads); msys_info.freeram, nthreads);
if (arc == NULL) { if (arc == NULL) {
pthread_mutex_unlock(&init_lock); pthread_mutex_unlock(&init_lock);