Fixed some parsing / tokenization problems (added support for more characters / whitespace within tuples.

Implemented perl driver (and sample .cgi) for toplevel.c's network server.

src/apps/referential/Referential.pm

@@ -0,0 +1,117 @@
+#!/usr/bin/perl -w
+
+package Referential;
+
+use strict;
+
+use Expect;
+use IO::Socket::INET;
+use URI::Escape;
+
+sub new {
+    my $class = shift;
+
+    my %self;
+
+    $self{port} = shift;
+    $self{host} = shift || "localhost";
+    $self{log} = 1;
+    $self{socket} = IO::Socket::INET->new(PeerAddr => $self{host},
+					   PeerPort => $self{port},
+					   Proto    => "tcp",
+					   Type     => SOCK_STREAM)
+	or die "Couldn't connect to $self{host}:$self{port} : $!\n";
+
+    $self{exp} = Expect->exp_init($self{socket});
+    $self{exp}->raw_pty(1);
+
+    my @prompt;
+    push @prompt, '-re', '.*\>\s*';
+
+    my $patidx = $self{exp}->expect(10, @prompt);
+
+    if(!defined($patidx)) {
+	die "Didn't get prompt\n";
+    }
+    return bless \%self, $class;
+}
+
+sub escape {
+    my $self = shift;
+    my $s = shift;
+    return uri_escape($s, "^A-Za-z0-9\-_.!~'");
+}
+sub unescape {
+    my $self = shift;
+    my $s = shift;
+    return uri_unescape($s);
+}
+
+sub query {
+    my $self = shift;
+    my $q = shift;
+
+    #warn "REFERENTIAL QUERY: query $q\n";
+    $self->{exp}->send("query $q\n");
+    my @pat;
+
+    my @tpls;
+    push @tpls, '-re', '^[A-Za-z0-9\-_.!~\'%.,\ ]+[^\r\n]';
+    push @tpls, '-re', '.*\>\s*';
+    my @tups;
+    while(my $idx = $self->{exp}->expect(10, @tpls)) {
+	if($idx==1) {
+	    my $match =  $self->{exp}->exp_match();
+	    chomp $match;
+	    my @tok = split ",", $match;
+	    my @tok2;
+	    for(my $i = 0; $i < @tok; $i++) {
+		push @tok2, $self->unescape($tok[$i]);
+	    }
+	    push @tups, \@tok2;
+	} elsif($idx == 2) {
+	    last;
+	}
+    }
+    return \@tups;
+}
+
+sub _cmd {
+    my $self = shift;
+    my $q = shift;
+    my $m = shift;
+
+    if($self->{log}) {
+	my $d = `date`;
+	chomp $d;
+	warn "\n[$d] REFERENTIAL_CMD: $m $q\n";
+    }
+
+    $self->{exp}->send("$m $q\n");
+    my @pats;
+    push @pats, '-re', '.*\>\s*';
+    my $idx = $self->{exp}->expect(10, @pats);
+    if($idx != 1) {
+	warn "No prompt?!?\n";
+	return 0;
+    }
+    return 1;
+}
+
+sub insert {
+    my $self = shift;
+    my $q = shift;
+    return _cmd($self, $q, "insert");
+}
+sub delete {
+    my $self = shift;
+    my $q = shift;
+    return _cmd($self, $q, "delete");
+}
+
+sub close {
+    my $self = shift;
+    $self->{exp}->send("!exit\n");
+    $self->{exp}->soft_close();
+}
+1;

src/apps/referential/algebra.c

@@ -40,13 +40,14 @@ static void ts_releaseLock(int xid, void *it) {
 char ** split(char * in, char ** freeme, int* count, char * delim) {
   *freeme = strdup(in);
   *count = 0;
-  char * tok = strtok(*freeme, delim);
+  char * strtoks;
+  char * tok = strtok_r(*freeme, delim,&strtoks);
   char ** ret = 0;
   while(tok) {
     (*count)++;
     ret = realloc(ret, sizeof(char*) * *count);
     ret[(*count)-1] = tok;
-    tok = strtok(NULL,delim);
+    tok = strtok_r(NULL,delim,&strtoks);
   }
   ret = realloc(ret, sizeof(char*) * ((*count)+1));
   ret[*count]=0;
@@ -95,6 +96,48 @@ void tplFree(char ** tup) {
   free(tup);
 }
 
+int isWhitelistChar(char c) {
+  return(c == ';'
+	 ||
+	 c == '/'
+	 ||
+	 c == '?'
+	 ||
+	 c == ':'
+	 ||
+	 c == '@'
+	 ||
+	 c == '&'
+	 ||
+	 c == '='
+	 ||
+	 c == '+'
+	 ||
+	 c == '$'
+	 ||
+	 c == '['
+	 ||
+	 c == ']'
+	 ||
+	 c == '-'
+	 ||
+	 c == '_'
+	 ||
+	 c == '.'
+	 ||
+	 c == '!'
+	 ||
+	 c == '~'
+	 ||
+	 c == '*'
+	 ||
+	 c == '\''
+	 ||
+	 c == '%'
+	 ||
+	 c == '\\');
+}
+
 lladdIterator_t* ReferentialAlgebra_OpenTableScanner(int xid, recordid catalog,
 						    char * tablename) {
   char * table;
@@ -222,6 +265,41 @@ lladdIterator_t* ReferentialAlgebra_Select(int xid, lladdIterator_t * it, char *
   return new_it;
 }
 
+char * strtokempt(char *str, const char * delim, char ** saveptr, int * extra) {
+  char * ret;
+  if(str) {
+    *saveptr = str;
+  }
+  ret = *saveptr;
+
+  if(!(**saveptr)) {
+    if(*extra) {
+      *extra = 0;
+      return ret;
+    } else {
+      return 0;
+    }
+  }
+  do {
+    for(int i = 0; delim[i]; i++) {
+      if(**saveptr == delim[i]) {
+	// terminate returned string
+	**saveptr = 0;
+	// consume deliminator character from last time.
+	(*saveptr)++;
+	//printf("extra: %d\n",*extra);
+	*extra = 1;
+	return ret;
+      }
+    }
+    // consume character we just looked at.
+    (*saveptr)++;
+  } while(**saveptr);
+  //  printf("extra: %d\n",*extra);
+  *extra = 0;
+  return ret;
+}
+
 static int matchPredicate(const char const * tup, char ** pred) {
   char * tupcpy = strdup(tup);
   int colcount = 0;
@@ -231,7 +309,9 @@ static int matchPredicate(const char const * tup, char ** pred) {
 
   char * ths;
   const char const * DELIM = ",";
-  if((ths = strtok(tupcpy, DELIM))) {
+  char * strtoks;
+  int extra = 0;
+  if((ths = strtokempt(tupcpy, DELIM,&strtoks,&extra))) {
     colcount++;
     if(colcount > predcount) {
       free(tupcpy);
@@ -241,7 +321,7 @@ static int matchPredicate(const char const * tup, char ** pred) {
       tok[colcount-1] = ths;
     }
   }
-  while((ths = strtok(NULL, DELIM))) {
+  while((ths = strtokempt(NULL, DELIM,&strtoks,&extra))) {
     colcount++;
     if(colcount > predcount) {
       free(tupcpy);
@@ -438,7 +518,7 @@ static int matchComparator(char ** tup1,
   while(pred[col] && match) {
     char * lhs_start = pred[col];
     char * lhs_end = lhs_start;
-    while(isalnum(*lhs_end)) { lhs_end++; }
+    while(isWhitelistChar(*lhs_end)||isalnum(*lhs_end)) { lhs_end++; }
     int lhs_len = lhs_end - lhs_start;
 
     char * lhs = calloc(lhs_len+1,sizeof(char));
@@ -447,7 +527,7 @@ static int matchComparator(char ** tup1,
     char * op_start = lhs_end;
     while(isblank(*op_start)) { op_start++; }
     char * op_end = op_start;
-    while(!(isblank(*op_end) || isalnum(*op_end))) { op_end++; }
+    while(!(isblank(*op_end) || isWhitelistChar(*lhs_end)||isalnum(*op_end))) { op_end++; }
     int op_len = op_end - op_start;
 
     char * op = calloc(op_len+1,sizeof(char));
@@ -456,7 +536,7 @@ static int matchComparator(char ** tup1,
     char * rhs_start = op_end;
     while(isblank(*rhs_start)) { rhs_start++; }
     char * rhs_end = rhs_start;
-    while(isalnum(*rhs_end)) { rhs_end++; }
+    while(isWhitelistChar(*lhs_end)||isalnum(*rhs_end)) { rhs_end++; }
     int rhs_len = rhs_end - rhs_start;
 
     char * rhs = calloc(rhs_len+1,sizeof(char));
@@ -644,26 +724,29 @@ void ReferentialAlgebra_init() {
   @return one of the above.  If returns STRING, set *tok to be the new
   token. (*tok should be freed by caller in this case)
  */
-int nextToken(char ** head, char ** tok);
+int nextToken(char ** head, char ** tok, int breakOnSpace);
 
 char** parseTuple(char ** head) {
   char **tok = calloc(1,sizeof(char*));;
   char * mytok;
-  char ret = nextToken(head, &mytok);
+  char ret = nextToken(head, &mytok,0);
   assert(ret == LBRACKET);
   int count = 0;
   while(1) {
-    ret = nextToken(head, &mytok);
+    ret = nextToken(head, &mytok,0);
     if(ret == RBRACKET) {
       break;
     }
-    assert(ret == STRING);
+    if(ret == COMMA) {
+      tok = realloc(tok, sizeof(char*)*(count+1));
+      tok[count] = 0;
+      tok[count-1] = calloc(1,sizeof(char));
+    } else if(ret == STRING) {
       count++;
       tok = realloc(tok, sizeof(char*)*(count+1));
       tok[count] = 0;
       tok[count-1] = mytok;
-
+      ret = nextToken(head, &mytok,0);
-    ret = nextToken(head, &mytok);
       if(ret == STRING) { free(mytok); }
       if(ret == RBRACKET) {
 	break;
@@ -672,6 +755,10 @@ char** parseTuple(char ** head) {
 	tplFree(tok);
 	return 0;
       }
+    } else {
+      tplFree(tok);
+      return 0;
+    }
   }
   return tok;
 }
@@ -719,12 +806,17 @@ lladdIterator_t * parseExpression(int xid, recordid catalog,
       return 0;
     }
     char * foo;
-    char ret = nextToken(head, &foo);
+    char ret = nextToken(head, &foo,0);
-    assert(ret == RPAREN);
+
+    if(ret != RPAREN) {
+      Titerator_close(xid,it);
+      return 0;
+    } else {
       return it;
+    }
   } else {
     char * tablename;
-    char ret = nextToken(head, &tablename);
+    char ret = nextToken(head, &tablename,1);
     assert(ret == STRING);
     lladdIterator_t * it2 =
       ReferentialAlgebra_OpenTableScanner(xid, catalog, tablename);
@@ -740,7 +832,7 @@ lladdIterator_t * parseExpression(int xid, recordid catalog,
   abort();
 }
 
-int nextToken(char ** head, char ** tok) {
+int nextToken(char ** head, char ** tok, int breakOnSpace) {
   while(isblank(**head) && **head) { (*head)++; }
   switch(**head) {
   case LPAREN: {
@@ -770,10 +862,25 @@ int nextToken(char ** head, char ** tok) {
   default: {
     if(!**head) { return EOS; };
     char * first = *head;
-    while(isalnum(**head)||**head=='*'||**head=='=') { (*head)++; }
+    while(isalnum(**head)
+	  ||isWhitelistChar(**head)
+	  ||(**head==' '&&!breakOnSpace)) {
+      (*head)++;
+    }
     char * last = *head;
     *tok = calloc(1 + last - first, sizeof(char));
-    strncpy(*tok, first, last - first);  // The remaining byte is the null terminator.
+    // The remaining byte is the null terminator
+    strncpy(*tok, first, last - first);
+    int i = (last-first)-1;
+    int firstloop = 1;
+    while((*tok)[i] == ' ') {
+      (*tok)[i] = '\0';
+      i++;
+      if(firstloop) {
+	(*head)--;
+	firstloop = 0;
+      }
+    }
     return STRING;
   } break;
   }

src/apps/referential/dml.c

@@ -6,15 +6,16 @@
 #include "algebra.h"
 int executeInsert(int xid, recordid tables, char * insert) {
   char * linecopy = strdup(insert+strlen("insert"));
-  char * tbl = strtok(linecopy," ");
+  char * strtoks;
-  char * tup = strtok(NULL,"\r\n");
+  char * tbl = strtok_r(linecopy," ",&strtoks);
+  char * tup = strtok_r(NULL,"\r\n",&strtoks);
   if((!tbl) || (!tup)) {
     printf("parse error\n");
     return 0;
   }
   char * tupcopy = strdup(tup);
-  char * key = strtok(tupcopy,",");
+  char * key = strtok_r(tupcopy,",",&strtoks);
-  char * trunctup = strtok(NULL,"\r\n");
+  char * trunctup = strtok_r(NULL,"\r\n",&strtoks);
   char * table;
   if(!trunctup) {
     trunctup = "";
@@ -35,15 +36,16 @@ int executeInsert(int xid, recordid tables, char * insert) {
 }
 int executeDelete(int xid, recordid tables, char * delete) {
   char * linecopy = strdup(delete+strlen("delete"));
-  char * tbl = strtok(linecopy," ");
+  char * strtoks;
-  char * tup = strtok(NULL,"\r\n");
+  char * tbl = strtok_r(linecopy," ",&strtoks);
+  char * tup = strtok_r(NULL,"\r\n",&strtoks);
   if((!tbl) || (!tup)) {
     printf("parse error\n");
     return 0;
   }
   char * table;
   char * tupcopy = strdup(tup);
-  char * key = strtok(tupcopy,",");
+  char * key = strtok_r(tupcopy,",",&strtoks);
   int sz = ThashLookup(xid, tables, (byte*)tbl, strlen(tbl)+1, (byte**)&table);
   if(sz == -1) {
     printf("Unknown table %s\n", tbl);

src/apps/referential/survey.cgi

@@ -0,0 +1,82 @@
+#!/usr/bin/perl -w
+use strict;
+
+use CGI qw/:standard/;
+use Referential;
+
+sub printResponse {
+    my $t = shift;
+
+    print "<p>$$t[0]: ".($$t[1]?'Yes':'No')."</p>\n";
+    if(defined ($$t[2]) && $$t[2] =~ /\S/) {
+	print "<b>Comment:</b><pre>\n".$$t[2]."</pre>";
+    }
+}
+
+my $ref = new Referential("6667");
+
+my $cgi = new CGI;
+
+if(param('name')) {
+
+    my $name = $ref->escape(param('name'));
+    my $vote = $ref->escape(param('vote'));
+    my $comments = $ref->escape(param('comments'));
+    print header;
+    print "\n\n<html><head><title>Survey repsonse completed</title></head></html>\n";
+    print "<body><h1>Thanks!</h1>\n";
+    my $tups = $ref->query("{s ($name,*,*) peeps}");
+
+    if(defined $$tups[0][0]) {
+	print "<h2>Found and deleted old response:</h2>";
+	printResponse($$tups[0]);
+    }
+
+    $ref->insert("peeps $name,$vote,$comments");
+
+    $tups = $ref->query("{s ($name,*,*) peeps}");
+
+    if(defined $$tups[0][0]) {
+	print "<h2>Recorded your response:</h2>";
+	printResponse($$tups[0]);
+    } else {
+	print "query failed!\n";
+    }
+
+    print "</body></html>";
+} elsif(param('secret') eq 'wellkept') {
+    my $format = param("format") || "html";
+    my $tups = $ref->query("peeps");
+    if($format ne 'html') {
+	print header('text/plain')."\n\nVote tabulation\n";
+	foreach my $t (@{$tups}) {
+	    print "$$t[1],$$t[0]\n";
+	}
+	print "\n";
+    } else {
+	print header."\n\n<html><head><title>Responses</title></head><body><h1>Responses</h1>";
+	foreach my $t (@{$tups}) {
+	    printResponse($t);
+	}
+	print "</body></html>";
+    }
+} else {
+    print header."\n\n".qq(
+<html><head><title>Survey</title></head><body>
+<h1>Survey</h1>
+<form>
+Name: <input type='text' name='name'/>
+<br>
+<p>Will you attend?</p>
+<input type="radio" name="vote" value="1">Yes</input>
+<br>
+<input type="radio" name="vote" value="0">No</input>
+<p>Comments:</p>
+<textarea name="comments" cols="72" rows="10">
+</textarea><br>
+<input type="submit" value="Submit">
+<br>
+</form>
+</body>
+</html>);
+}

src/apps/referential/test.pl

@@ -0,0 +1,34 @@
+#!/usr/bin/perl -w
+use strict;
+
+use Referential;
+
+my $r = new Referential($ARGV[0]);
+
+print "Connected...\n";
+
+my $tups = $r->query("{s (foo,*,*,*) TABLES}");
+my @tups = @{$tups};
+
+if(@tups != 1) {
+    die("Expected one foo table, found ".(@tups+0));
+}
+
+#print "test $tups[0][3]\n";
+$r->insert("foo a,b,c");
+$r->insert("foo b,c,d");
+$tups = $r->query("foo");
+
+foreach my $t (@{$tups}) {
+    print ((join ",",@{$t})."\n");
+}
+
+$r->delete("foo b");
+$tups = $r->query("foo");
+
+foreach my $t (@{$tups}) {
+    print ((join ",",@{$t})."\n");
+}
+
+$r->close();
+

src/apps/referential/toplevel.c

@@ -134,7 +134,8 @@ int openInterpreter(FILE * in, FILE * out, recordid hash) {
       if(!strncmp(line,"create",strlen("create"))) {
 
 	char * linecopy = strdup(line+strlen("create"));
-	char * tablename = strtok(linecopy, " \r\n");
+	char * strtoks;
+	char * tablename = strtok_r(linecopy, " \r\n",&strtoks);
 	size_t sz;
 	byte* delme;