diff --git a/test/t4.tst b/test/t4.tst index 85c53d4..a53adb3 100644 --- a/test/t4.tst +++ b/test/t4.tst @@ -45,6 +45,70 @@ do done done +# +# Test Segmented Global Dedupe +# + +echo "#################################################" +echo "# Test Segmented Global Deduplication" +echo "#################################################" + +# +# Select a large file from the list +# +tstf= +tsz=0 +for tf in `cat files.lst` +do + sz=`ls -l ${tf} | awk '{ print $5 }'` + if [ $sz -gt $tsz ] + then + tsz=$sz + tstf="$tf" + fi +done + +# +# Compute minimum index memory needed for segmented dedupe +# sizeof (hash_entry_t) = 20 + sizeof (CRC64) = 28 +# Each hashtable slot ptr = 8bytes +# Total: 28 + 8 = 36 +# Segment size = 833848 +# 25 Similarity indicators per segment, each needing one hash_entry_t +# +nsegs=$((tsz / 833848 + 1)) +nmem=$((nsegs * 25 * 36)) +mem_mb=$((nmem / 1048576 + 1)) + +# +# Now run Global Dedupe with index memory set to force segmented dedupe mechanism +# +export PCOMPRESS_INDEX_MEM=${mem_mb} +cmd="../../pcompress -G -c lz4 -l1 -P -s50m $tstf" +echo "Running $cmd" +eval $cmd +if [ $? -ne 0 ] +then + echo "FATAL: Compression errored." + rm -f ${tstf}.pz + exit +fi +cmd="../../pcompress -d ${tstf}.pz ${tstf}.1" +echo "Running $cmd" +eval $cmd +if [ $? -ne 0 ] +then + echo "FATAL: Decompression errored." + rm -f ${tstf}.pz ${tstf}.1 + exit +fi +diff ${tstf} ${tstf}.1 > /dev/null +if [ $? -ne 0 ] +then + echo "FATAL: Decompression was not correct" +fi +rm -f ${tstf}.pz ${tstf}.1 + echo "#################################################" echo "" diff --git a/test/t9.tst b/test/t9.tst index 3f345db..f21945b 100644 --- a/test/t9.tst +++ b/test/t9.tst @@ -21,6 +21,17 @@ do fi done +export PCOMPRESS_CHUNK_HASH_GLOBAL=CRC64 +cmd="../../pcompress -G -c none -l1 -s10m $tstf" +echo "Running $cmd" +eval $cmd +if [ $? -eq 0 ] +then + echo "FATAL: Compression DID NOT ERROR where expected" + rm -f ${tstf}.pz +fi +unset PCOMPRESS_CHUNK_HASH_GLOBAL + for feat in "-L" "-L -D" "-L -D -E" "-L -B5" "-L -D -E -B2" "-F" "-F -L" do cmd="../../pcompress -c dummy -l4 -s1m $feat $tstf" @@ -30,9 +41,9 @@ do then echo "FATAL: Compression DID NOT ERROR where expected" rm -f ${tstf}.pz + break fi rm -f ${tstf}.pz - break done for feat in "-B8 -s2m -l1" "-B0 -s2m -l1" "-D -s10k -l1" "-D -F -s2m -l1" "-p -e AES -s2m -l1" "-s2m -l15" "-e AES -k64" "-e SALSA20 -k8" "-e AES -k8" "-e SALSA20 -k64" @@ -46,9 +57,9 @@ do then echo "FATAL: Compression DID NOT ERROR where expected" rm -f ${tstf}.pz + break fi rm -f ${tstf}.pz - break done done