Use SHA256 for Global Dedupe chunk lookup hash by default.
Allow changing Global Dedupe chunk lookup hash via env variable.
This commit is contained in:
parent
6e4d45b644
commit
120877348c
2 changed files with 22 additions and 2 deletions
|
@ -223,6 +223,10 @@ can be a directory on a Solid State Drive to speed up Global Deduplication. The
|
||||||
space used in this directory is proportional to the size of the dataset being
|
space used in this directory is proportional to the size of the dataset being
|
||||||
processed and is slightly more than 8KB for every 1MB of data.
|
processed and is slightly more than 8KB for every 1MB of data.
|
||||||
|
|
||||||
|
The default checksum used for chunk hashes during Global Deduplication is SHA256.
|
||||||
|
However this can be changed by setting the PCOMPRESS_CHUNK_HASH_GLOBAL environment
|
||||||
|
variable to one of the other checksum names except CRC64.
|
||||||
|
|
||||||
Examples
|
Examples
|
||||||
========
|
========
|
||||||
|
|
||||||
|
|
|
@ -207,7 +207,8 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
|
||||||
*/
|
*/
|
||||||
if (dedupe_flag == RABIN_DEDUPE_FILE_GLOBAL && op == COMPRESS && rab_blk_sz > 0) {
|
if (dedupe_flag == RABIN_DEDUPE_FILE_GLOBAL && op == COMPRESS && rab_blk_sz > 0) {
|
||||||
my_sysinfo msys_info;
|
my_sysinfo msys_info;
|
||||||
int pct_interval;
|
int pct_interval, chunk_cksum, cksum_bytes, mac_bytes;
|
||||||
|
char *ck;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Get amount of memory to use. The freeram got here is adjusted amount.
|
* Get amount of memory to use. The freeram got here is adjusted amount.
|
||||||
|
@ -217,8 +218,23 @@ create_dedupe_context(uint64_t chunksize, uint64_t real_chunksize, int rab_blk_s
|
||||||
if (pipe_mode)
|
if (pipe_mode)
|
||||||
pct_interval = DEFAULT_PCT_INTERVAL;
|
pct_interval = DEFAULT_PCT_INTERVAL;
|
||||||
|
|
||||||
|
chunk_cksum = 0;
|
||||||
|
if ((ck = getenv("PCOMPRESS_CHUNK_HASH_GLOBAL")) != NULL) {
|
||||||
|
if (get_checksum_props(ck, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0 ||
|
||||||
|
strcmp(ck, "CRC64") == 0) {
|
||||||
|
fprintf(stderr, "Invalid PCOMPRESS_CHUNK_HASH_GLOBAL. Reverting to default.\n");
|
||||||
|
chunk_cksum = -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (chunk_cksum == 0) {
|
||||||
|
chunk_cksum = DEFAULT_CHUNK_CKSUM;
|
||||||
|
if (get_checksum_props(NULL, &chunk_cksum, &cksum_bytes, &mac_bytes, 0) != 0) {
|
||||||
|
fprintf(stderr, "Invalid default chunk checksum: %d\n", DEFAULT_CHUNK_CKSUM);
|
||||||
|
return (NULL);
|
||||||
|
}
|
||||||
|
}
|
||||||
arc = init_global_db_s(NULL, tmppath, rab_blk_sz, chunksize, pct_interval,
|
arc = init_global_db_s(NULL, tmppath, rab_blk_sz, chunksize, pct_interval,
|
||||||
algo, props->cksum, GLOBAL_SIM_CKSUM, file_size,
|
algo, chunk_cksum, GLOBAL_SIM_CKSUM, file_size,
|
||||||
msys_info.freeram, nthreads);
|
msys_info.freeram, nthreads);
|
||||||
if (arc == NULL) {
|
if (arc == NULL) {
|
||||||
pthread_mutex_unlock(&init_lock);
|
pthread_mutex_unlock(&init_lock);
|
||||||
|
|
Loading…
Reference in a new issue