greg/pcompress
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add more tests covering Segmented Global Dedupe.

Browse source 
Fix some tests.
This commit is contained in:
Moinak Ghosh 2013-05-07 22:30:36 +05:30
parent 2740a00c76
commit a755d59dff
2 changed files with 77 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
test/t4.tst
View file

@ -45,6 +45,70 @@ do
done done
done done
#
# Test Segmented Global Dedupe
#
echo "#################################################"
echo "# Test Segmented Global Deduplication"
echo "#################################################"
#
# Select a large file from the list
#
tstf=
tsz=0
for tf in `cat files.lst`
do
sz=`ls -l ${tf} | awk '{ print $5 }'`
if [ $sz -gt $tsz ]
then
tsz=$sz
tstf="$tf"
fi
done
#
# Compute minimum index memory needed for segmented dedupe
# sizeof (hash_entry_t) = 20 + sizeof (CRC64) = 28
# Each hashtable slot ptr = 8bytes
# Total: 28 + 8 = 36
# Segment size = 833848
# 25 Similarity indicators per segment, each needing one hash_entry_t
#
nsegs=$((tsz / 833848 + 1))
nmem=$((nsegs * 25 * 36))
mem_mb=$((nmem / 1048576 + 1))
#
# Now run Global Dedupe with index memory set to force segmented dedupe mechanism
#
export PCOMPRESS_INDEX_MEM=${mem_mb}
cmd="../../pcompress -G -c lz4 -l1 -P -s50m $tstf"
echo "Running $cmd"
eval $cmd
if [ $? -ne 0 ]
then
echo "FATAL: Compression errored."
rm -f ${tstf}.pz
exit
fi
cmd="../../pcompress -d ${tstf}.pz ${tstf}.1"
echo "Running $cmd"
eval $cmd
if [ $? -ne 0 ]
then
echo "FATAL: Decompression errored."
rm -f ${tstf}.pz ${tstf}.1
exit
fi
diff ${tstf} ${tstf}.1 > /dev/null
if [ $? -ne 0 ]
then
echo "FATAL: Decompression was not correct"
fi
rm -f ${tstf}.pz ${tstf}.1
echo "#################################################" echo "#################################################"
echo "" echo ""

15
test/t9.tst
View file

@ -21,6 +21,17 @@ do
fi fi
done done
export PCOMPRESS_CHUNK_HASH_GLOBAL=CRC64
cmd="../../pcompress -G -c none -l1 -s10m $tstf"
echo "Running $cmd"
eval $cmd
if [ $? -eq 0 ]
then
echo "FATAL: Compression DID NOT ERROR where expected"
rm -f ${tstf}.pz
fi
unset PCOMPRESS_CHUNK_HASH_GLOBAL
for feat in "-L" "-L -D" "-L -D -E" "-L -B5" "-L -D -E -B2" "-F" "-F -L" for feat in "-L" "-L -D" "-L -D -E" "-L -B5" "-L -D -E -B2" "-F" "-F -L"
do do
cmd="../../pcompress -c dummy -l4 -s1m $feat $tstf" cmd="../../pcompress -c dummy -l4 -s1m $feat $tstf"
@ -30,9 +41,9 @@ do
then then
echo "FATAL: Compression DID NOT ERROR where expected" echo "FATAL: Compression DID NOT ERROR where expected"
rm -f ${tstf}.pz rm -f ${tstf}.pz
break
fi fi
rm -f ${tstf}.pz rm -f ${tstf}.pz
break
done done
for feat in "-B8 -s2m -l1" "-B0 -s2m -l1" "-D -s10k -l1" "-D -F -s2m -l1" "-p -e AES -s2m -l1" "-s2m -l15" "-e AES -k64" "-e SALSA20 -k8" "-e AES -k8" "-e SALSA20 -k64" for feat in "-B8 -s2m -l1" "-B0 -s2m -l1" "-D -s10k -l1" "-D -F -s2m -l1" "-p -e AES -s2m -l1" "-s2m -l15" "-e AES -k64" "-e SALSA20 -k8" "-e AES -k8" "-e SALSA20 -k64"
@ -46,9 +57,9 @@ do
then then
echo "FATAL: Compression DID NOT ERROR where expected" echo "FATAL: Compression DID NOT ERROR where expected"
rm -f ${tstf}.pz rm -f ${tstf}.pz
break
fi fi
rm -f ${tstf}.pz rm -f ${tstf}.pz
break
done done
done done