Add more tests covering Segmented Global Dedupe.
Fix some tests.
This commit is contained in:
parent
2740a00c76
commit
a755d59dff
2 changed files with 77 additions and 2 deletions
64
test/t4.tst
64
test/t4.tst
|
@ -45,6 +45,70 @@ do
|
|||
done
|
||||
done
|
||||
|
||||
#
|
||||
# Test Segmented Global Dedupe
|
||||
#
|
||||
|
||||
echo "#################################################"
|
||||
echo "# Test Segmented Global Deduplication"
|
||||
echo "#################################################"
|
||||
|
||||
#
|
||||
# Select a large file from the list
|
||||
#
|
||||
tstf=
|
||||
tsz=0
|
||||
for tf in `cat files.lst`
|
||||
do
|
||||
sz=`ls -l ${tf} | awk '{ print $5 }'`
|
||||
if [ $sz -gt $tsz ]
|
||||
then
|
||||
tsz=$sz
|
||||
tstf="$tf"
|
||||
fi
|
||||
done
|
||||
|
||||
#
|
||||
# Compute minimum index memory needed for segmented dedupe
|
||||
# sizeof (hash_entry_t) = 20 + sizeof (CRC64) = 28
|
||||
# Each hashtable slot ptr = 8bytes
|
||||
# Total: 28 + 8 = 36
|
||||
# Segment size = 833848
|
||||
# 25 Similarity indicators per segment, each needing one hash_entry_t
|
||||
#
|
||||
nsegs=$((tsz / 833848 + 1))
|
||||
nmem=$((nsegs * 25 * 36))
|
||||
mem_mb=$((nmem / 1048576 + 1))
|
||||
|
||||
#
|
||||
# Now run Global Dedupe with index memory set to force segmented dedupe mechanism
|
||||
#
|
||||
export PCOMPRESS_INDEX_MEM=${mem_mb}
|
||||
cmd="../../pcompress -G -c lz4 -l1 -P -s50m $tstf"
|
||||
echo "Running $cmd"
|
||||
eval $cmd
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo "FATAL: Compression errored."
|
||||
rm -f ${tstf}.pz
|
||||
exit
|
||||
fi
|
||||
cmd="../../pcompress -d ${tstf}.pz ${tstf}.1"
|
||||
echo "Running $cmd"
|
||||
eval $cmd
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo "FATAL: Decompression errored."
|
||||
rm -f ${tstf}.pz ${tstf}.1
|
||||
exit
|
||||
fi
|
||||
diff ${tstf} ${tstf}.1 > /dev/null
|
||||
if [ $? -ne 0 ]
|
||||
then
|
||||
echo "FATAL: Decompression was not correct"
|
||||
fi
|
||||
rm -f ${tstf}.pz ${tstf}.1
|
||||
|
||||
echo "#################################################"
|
||||
echo ""
|
||||
|
||||
|
|
15
test/t9.tst
15
test/t9.tst
|
@ -21,6 +21,17 @@ do
|
|||
fi
|
||||
done
|
||||
|
||||
export PCOMPRESS_CHUNK_HASH_GLOBAL=CRC64
|
||||
cmd="../../pcompress -G -c none -l1 -s10m $tstf"
|
||||
echo "Running $cmd"
|
||||
eval $cmd
|
||||
if [ $? -eq 0 ]
|
||||
then
|
||||
echo "FATAL: Compression DID NOT ERROR where expected"
|
||||
rm -f ${tstf}.pz
|
||||
fi
|
||||
unset PCOMPRESS_CHUNK_HASH_GLOBAL
|
||||
|
||||
for feat in "-L" "-L -D" "-L -D -E" "-L -B5" "-L -D -E -B2" "-F" "-F -L"
|
||||
do
|
||||
cmd="../../pcompress -c dummy -l4 -s1m $feat $tstf"
|
||||
|
@ -30,9 +41,9 @@ do
|
|||
then
|
||||
echo "FATAL: Compression DID NOT ERROR where expected"
|
||||
rm -f ${tstf}.pz
|
||||
break
|
||||
fi
|
||||
rm -f ${tstf}.pz
|
||||
break
|
||||
done
|
||||
|
||||
for feat in "-B8 -s2m -l1" "-B0 -s2m -l1" "-D -s10k -l1" "-D -F -s2m -l1" "-p -e AES -s2m -l1" "-s2m -l15" "-e AES -k64" "-e SALSA20 -k8" "-e AES -k8" "-e SALSA20 -k64"
|
||||
|
@ -46,9 +57,9 @@ do
|
|||
then
|
||||
echo "FATAL: Compression DID NOT ERROR where expected"
|
||||
rm -f ${tstf}.pz
|
||||
break
|
||||
fi
|
||||
rm -f ${tstf}.pz
|
||||
break
|
||||
done
|
||||
done
|
||||
|
||||
|
|
Loading…
Reference in a new issue