Add more tests covering Segmented Global Dedupe.
Fix some tests.
This commit is contained in:
parent
2740a00c76
commit
a755d59dff
2 changed files with 77 additions and 2 deletions
64
test/t4.tst
64
test/t4.tst
|
@ -45,6 +45,70 @@ do
|
||||||
done
|
done
|
||||||
done
|
done
|
||||||
|
|
||||||
|
#
|
||||||
|
# Test Segmented Global Dedupe
|
||||||
|
#
|
||||||
|
|
||||||
|
echo "#################################################"
|
||||||
|
echo "# Test Segmented Global Deduplication"
|
||||||
|
echo "#################################################"
|
||||||
|
|
||||||
|
#
|
||||||
|
# Select a large file from the list
|
||||||
|
#
|
||||||
|
tstf=
|
||||||
|
tsz=0
|
||||||
|
for tf in `cat files.lst`
|
||||||
|
do
|
||||||
|
sz=`ls -l ${tf} | awk '{ print $5 }'`
|
||||||
|
if [ $sz -gt $tsz ]
|
||||||
|
then
|
||||||
|
tsz=$sz
|
||||||
|
tstf="$tf"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
#
|
||||||
|
# Compute minimum index memory needed for segmented dedupe
|
||||||
|
# sizeof (hash_entry_t) = 20 + sizeof (CRC64) = 28
|
||||||
|
# Each hashtable slot ptr = 8bytes
|
||||||
|
# Total: 28 + 8 = 36
|
||||||
|
# Segment size = 833848
|
||||||
|
# 25 Similarity indicators per segment, each needing one hash_entry_t
|
||||||
|
#
|
||||||
|
nsegs=$((tsz / 833848 + 1))
|
||||||
|
nmem=$((nsegs * 25 * 36))
|
||||||
|
mem_mb=$((nmem / 1048576 + 1))
|
||||||
|
|
||||||
|
#
|
||||||
|
# Now run Global Dedupe with index memory set to force segmented dedupe mechanism
|
||||||
|
#
|
||||||
|
export PCOMPRESS_INDEX_MEM=${mem_mb}
|
||||||
|
cmd="../../pcompress -G -c lz4 -l1 -P -s50m $tstf"
|
||||||
|
echo "Running $cmd"
|
||||||
|
eval $cmd
|
||||||
|
if [ $? -ne 0 ]
|
||||||
|
then
|
||||||
|
echo "FATAL: Compression errored."
|
||||||
|
rm -f ${tstf}.pz
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
cmd="../../pcompress -d ${tstf}.pz ${tstf}.1"
|
||||||
|
echo "Running $cmd"
|
||||||
|
eval $cmd
|
||||||
|
if [ $? -ne 0 ]
|
||||||
|
then
|
||||||
|
echo "FATAL: Decompression errored."
|
||||||
|
rm -f ${tstf}.pz ${tstf}.1
|
||||||
|
exit
|
||||||
|
fi
|
||||||
|
diff ${tstf} ${tstf}.1 > /dev/null
|
||||||
|
if [ $? -ne 0 ]
|
||||||
|
then
|
||||||
|
echo "FATAL: Decompression was not correct"
|
||||||
|
fi
|
||||||
|
rm -f ${tstf}.pz ${tstf}.1
|
||||||
|
|
||||||
echo "#################################################"
|
echo "#################################################"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
|
|
15
test/t9.tst
15
test/t9.tst
|
@ -21,6 +21,17 @@ do
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
export PCOMPRESS_CHUNK_HASH_GLOBAL=CRC64
|
||||||
|
cmd="../../pcompress -G -c none -l1 -s10m $tstf"
|
||||||
|
echo "Running $cmd"
|
||||||
|
eval $cmd
|
||||||
|
if [ $? -eq 0 ]
|
||||||
|
then
|
||||||
|
echo "FATAL: Compression DID NOT ERROR where expected"
|
||||||
|
rm -f ${tstf}.pz
|
||||||
|
fi
|
||||||
|
unset PCOMPRESS_CHUNK_HASH_GLOBAL
|
||||||
|
|
||||||
for feat in "-L" "-L -D" "-L -D -E" "-L -B5" "-L -D -E -B2" "-F" "-F -L"
|
for feat in "-L" "-L -D" "-L -D -E" "-L -B5" "-L -D -E -B2" "-F" "-F -L"
|
||||||
do
|
do
|
||||||
cmd="../../pcompress -c dummy -l4 -s1m $feat $tstf"
|
cmd="../../pcompress -c dummy -l4 -s1m $feat $tstf"
|
||||||
|
@ -30,9 +41,9 @@ do
|
||||||
then
|
then
|
||||||
echo "FATAL: Compression DID NOT ERROR where expected"
|
echo "FATAL: Compression DID NOT ERROR where expected"
|
||||||
rm -f ${tstf}.pz
|
rm -f ${tstf}.pz
|
||||||
|
break
|
||||||
fi
|
fi
|
||||||
rm -f ${tstf}.pz
|
rm -f ${tstf}.pz
|
||||||
break
|
|
||||||
done
|
done
|
||||||
|
|
||||||
for feat in "-B8 -s2m -l1" "-B0 -s2m -l1" "-D -s10k -l1" "-D -F -s2m -l1" "-p -e AES -s2m -l1" "-s2m -l15" "-e AES -k64" "-e SALSA20 -k8" "-e AES -k8" "-e SALSA20 -k64"
|
for feat in "-B8 -s2m -l1" "-B0 -s2m -l1" "-D -s10k -l1" "-D -F -s2m -l1" "-p -e AES -s2m -l1" "-s2m -l15" "-e AES -k64" "-e SALSA20 -k8" "-e AES -k8" "-e SALSA20 -k64"
|
||||||
|
@ -46,9 +57,9 @@ do
|
||||||
then
|
then
|
||||||
echo "FATAL: Compression DID NOT ERROR where expected"
|
echo "FATAL: Compression DID NOT ERROR where expected"
|
||||||
rm -f ${tstf}.pz
|
rm -f ${tstf}.pz
|
||||||
|
break
|
||||||
fi
|
fi
|
||||||
rm -f ${tstf}.pz
|
rm -f ${tstf}.pz
|
||||||
break
|
|
||||||
done
|
done
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue