nightly benchmark graphing infrastructure.

benchmarks/nightly/Makefile

@@ -1,4 +1,12 @@
 
+## .SECONDARY means that the files will be deleted if their generation
+## is interrupted, but will not be automatically deleted when make
+## exits normally.
+
+.SECONDARY :
+# %.ci-raw %.def
+
+
 # This lets us keep the .def files in their own directory, but treat
 # them as though they're in the current (per run) directory.
 #VPATH=../defs/
@@ -31,8 +39,11 @@ all-ps-graphs  : $(patsubst %, %.ps,  $(GRAPHS))
 	$(MAKE) $(MAKEFLAGS) `grep Data-Series\: $< | \
               perl -ne 's/[\"\s]+/\t/g;print;print"\n"' | \
               cut -f2 | \
-              perl -ne 'chomp;print"$*-";print;print".dat\n"'` \
-	| grep -v 'is up to date.' || true
+              perl -ne 'chomp;print"$*-";print;print".dat\n"'`
+	touch $*.results
+
+# \
+#	| grep -v 'is up to date.' || true
 
 %.png: %.results %.def
 	../plot --format png $* | R $(RFLAGS) > /dev/null
@@ -44,9 +55,13 @@ all-ps-graphs  : $(patsubst %, %.ps,  $(GRAPHS))
 %.def : defs/%.def 
 	ln -s $< .
 
-%.dat : ${all-binaries} ${all-defs}
+
+%.ci-raw : ${all-binaries} ${all-defs} 
+#	echo "Foo" > $*.ci-raw
 	../generateGraphData $*
-	cp defs/$@ $@
+
+%.dat: %.ci-raw ${all-defs}
+	../ci-parser $*.def < $*.ci-raw > $*.dat
 
 %.R : %.def
 	plotting.pl $* > $*.R
@@ -54,7 +69,7 @@ all-ps-graphs  : $(patsubst %, %.ps,  $(GRAPHS))
 clean :
 	rm -f *.def *.R *.png *.pdf *.ps
 veryclean : clean
-	rm -f *.exe *.dat *.results binaries
+	rm -f *.exe *.dat *.ci-raw *.results binaries
 
 binaries:
 	touch binaries

benchmarks/nightly/ci-parser

@@ -0,0 +1,71 @@
+#! /usr/bin/perl  -w 
+
+use strict;
+
+#my $usage = qq(
+#Usage: $0 [--force] 'expression to calculate x value' 'expression to calculate y value'
+#
+#For example:  cat FOO.out | $0 "\\\$arg[0]/\\\$arg[1]" "\\\$time"
+#
+#will parse lines like this, dividing the first argument to the command
+#by the second.
+#
+#CI mean was: 26.3276666666667   ../linearHashNTAThreaded        1   1000 1
+#
+#Which would produce this output:
+#0.001\t26.3276666666667
+#
+#);
+
+sub getConfig {
+    my $basename = shift;
+    my $key = shift;
+    my $value = `grep ^$key: $basename.def`;
+    $value =~ s/^$key\:\s+//;
+    chomp $value;
+    return $value
+}
+
+
+my $usage = qq(Usage: $0 foo.def\n);
+
+my $ci_failed = qq(
+The input contains a failed confidence interval.  (--force will allow the script to continue)
+);
+
+my $def = $ARGV[0];
+
+$def =~ s/\-.+$//g;
+my $defFile = "$def.def";
+
+(-f $defFile)|| die $usage;
+
+my $force = getConfig($def, "Force-Calc");
+defined($force) || die "Required Force-Calc line missing in $defFile\n";
+my $eval_x = getConfig($def, "X-Calc") 
+    || die "Required X-Calc line missing in $defFile\n";
+my $eval_y = getConfig($def, "Y-Calc") 
+    || die "Required X-Calc line missing in $defFile\n";
+
+my $line;
+
+while($line = <STDIN>) {
+    if($line =~ /^CI/) { 
+	if ($line =~ /failed/i) {
+	    if(!$force) {
+		die $ci_failed;
+	    } else {
+		warn "Detected failed CI data point. Line was $line";
+	    }
+	} 
+	## Remove annotation from line.
+	$line =~ s/^.+\:\s+//;
+	
+	my @tok  = split /\s+/, $line;
+	my $time = shift @tok;
+	my $cmd  = shift @tok;
+	my @arg  = @tok;
+	
+	print ((eval $eval_x) . "\t" . (eval $eval_y) . "\n");
+    }
+}

benchmarks/nightly/defs/BULK_LOAD.def

@@ -2,7 +2,53 @@ Title: Bulk Load Time - Single Transaction
 Plot-Type: 2D Line
 X-Label: Insertions
 Y-Label: Seconds
-X-Range: qw(10 50 100 500 1000)
-Data-Series: "LINEAR_HASH_NTA" linearHashNTA 1 $x
-Data-Series: "Linear_Hash"     logicalHash   1 $x
-Data-Series: "Berkeley_DB"     bdbHash       1 $x
+
+### Example X-Range definitions.  
+###
+### Explicitly list each value that should be run:
+### 10, 50, 100, 500, 1000
+### X-Range: 10, 50, 100, 500, 1000
+###
+### List a range of values, incremented by 1:
+### 1, 2, ... 10
+### X-Range: 1..10
+###
+### List a range of values, then pass each value 
+### into a perl expression:
+### 100, 200 ... 1000
+
+X-Range: 10, 100, 1000, 10000
+
+### For each command, the first argument is the number of 
+### transactions (fixed), while the second is the number 
+### of operations per transaction.  $x will be replaced 
+### accordingly.
+
+Data-Series: Linear_Hash_NTA linearHashNTA 1 $x
+Data-Series: Linear_Hash     logicalHash   1 $x
+Data-Series: Berkeley_DB     bdbHash       1 $x
+
+###
+### These are passed into ci-parser.  They are both perl expressions, 
+### and have access some predefied values. $time contains the $time 
+### (raw Y value) returned by an invocation of a command.  @arg 
+### contains the arguments passed into the command.
+###
+### The example below graphs time against the number of hashtable 
+### operations (transactions * ops/transaction) in this running 
+### example.
+###
+
+X-Calc: $arg[0] * $arg[1]
+Y-Calc: $time
+
+### Another way of presenting the data:
+### time / operation:
+#X-Calc: \$time / (\$arg[0] * \$arg[1])
+### number of operations:
+#Y-Calc: (\$arg[0](\$arg[1]))
+
+### If true, then a graph will be produced even if some confidence
+### intervals failed.
+
+Force-Calc: 1

benchmarks/nightly/defs/LLADD_HASH_TPS.def

@@ -1,3 +1,12 @@
 Title:   Concurrent Transactions Per Second
 X-Label: Number of concurrent requests
 Y-Label: TPS
+
+X-Range: 1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100
+
+Data-Series: Linear_Hash_NTA linearHashNTAThreaded ??
+Data-Series: Berkeley_DB bdbHashThreaded ??
+
+X-Calc: ??
+Y-Calc: $time
+Force-Calc: 1

benchmarks/nightly/generateGraphData

@@ -2,6 +2,15 @@
 
 use strict;
 
+sub getConfig {
+    my $basename = shift;
+    my $key = shift;
+    my $value = `grep ^$key: $basename.def`;
+    $value =~ s/^$key\:\s+//;
+    chomp $value;
+    return $value
+}
+
 my $dataSeries = shift @ARGV;
 
 my @tok = split /\-/, $dataSeries;
@@ -10,8 +19,39 @@ my $usage = "Usage $0 GRAPH_NAME-SERIES_NAME\n";
 
 @tok == 2 || die $usage;
 #(-r $tok[0]) || (-r $tok[0]) || die ("Couldn't open ."$tok[0]."\n");
-my $seriesLine = `grep Data-Series\: $tok[0].def | grep $tok[1]` 
+my $seriesLine = `grep Data-Series\: $tok[0].def | grep '$tok[1] '` 
     || die "No Data-Series entry\n";
-my $range      = `grep X-Range\: $tok[0].def` || die "No X-Range entry\n";
 
-warn "Series line:\t$seriesLine\rRange:$range";
+$seriesLine =~ /^\S+\:\s+(\S+)\s+(.+)$/ || die "Bad data series line\n ($seriesLine)\n";
+my $seriesName = $1;
+my $command = $2;
+
+## Change command so that the first token ends in .exe, and starts
+## with ./ (ie:  "foo bar" -> "./foo.exe bar")
+
+$command =~ s/^(\S+)(.+)$/.\/$1.exe$2/; 
+
+
+my $range = getConfig($tok[0], "X-Range");
+defined($range) || die "No X-Range entry";
+
+warn "-------- GENERATE GRAPH DATA ----------\n";
+
+#warn "Series line:\t$seriesLine\rRange:$range";
+
+my @a = eval "(".$range.")";
+
+if(-e "$dataSeries.ci-raw") {
+    my $i = 0;
+    while(-e "$dataSeries.$i.ci-raw") { 
+	$i++;
+    }
+    `mv $dataSeries.ci-raw $dataSeries.$i.ci-raw`;
+}
+foreach my $x (@a) { 
+    # The eval lets the $x in command be expanded.
+    eval "print \"echo $command | ../timer >> $dataSeries.ci-raw\n\";"; 
+    eval "`echo $command | ../timer >> $dataSeries.ci-raw`"; 
+}
+
+warn "----- END GENERATE GRAPH DATA ---------\n";