Working, non-concurrent linear hash and linked list implementations.

The idea is to implement the non-concurrent versions, and then convert
them to concurrent versions using nested top actions.

lladd.pws

@@ -75,15 +75,11 @@ filter.dir.ignore.hidden=0
 0=0
 1=0:5
 2=0:6
-3=0:6:2
+3=0:9
-4=0:9
+4=0:10
-5=0:10
+5=0:11
-6=0:10:0
+6=0:11:4
-7=0:10:2
+7=0:11:5
-8=0:10:3
-9=0:11
-10=0:11:4
-11=0:11:5
 
 [executer args]
 0=check_linearHash

lladd/constants.h

@@ -76,7 +76,7 @@ terms specified in this license.
 #define PAGE_SIZE 4096
 
 /* #define MAX_BUFFER_SIZE 100003 */
-/*#define MAX_BUFFER_SIZE 20029*/
+/*#define MAX_BUFFER_SIZE 20029 */
 /*#define MAX_BUFFER_SIZE 10007 */
 #define MAX_BUFFER_SIZE 5003
 /*#define MAX_BUFFER_SIZE 2003 */

lladd/operations.h

@@ -151,6 +151,9 @@ typedef struct {
 #include "operations/linearHash.h"
 #include "operations/naiveLinearHash.h"
 #include "operations/nestedTopActions.h"
+#include "operations/linkedListNTA.h"
+#include "operations/linearHashNTA.h"
+
 
 extern Operation operationsTable[]; /* [MAX_OPERATIONS];  memset somewhere */
 

lladd/operations/linearHash.h

@@ -25,6 +25,21 @@ void TlogicalHashUpdate(int xid, recordid hashRid, void * key, int keySize, void
 void TlogicalHashInsert(int xid, recordid hashRid, void * key, int keySize, void * val, int valSize);
 int  TlogicalHashDelete(int xid, recordid hashRid, void * key, int keySize, void * val, int valSize);
 int TlogicalHashLookup(int xid, recordid hashRid, void * key, int keySize, void * buf, int valSize);
+typedef struct {
+  long current_hashBucket;
+  recordid current_rid;
+} linearHash_iterator;
+typedef struct {
+  byte * key;
+  byte * value;
+} linearHash_iteratorPair;
+
+linearHash_iterator * TlogicalHashIterator(int xid, recordid hashRid);
+void TlogicalHashIteratorFree(linearHash_iterator * it);
+linearHash_iteratorPair TlogicalHashIteratorNext(int xid, recordid hashRid, linearHash_iterator * it, int keySize, int valSize);
+
+
+Operation getLinearInsert();
 Operation getLinearInsert();
 Operation getLinearDelete();
 Operation getUndoLinearInsert();

lladd/operations/linearHashNTA.h

@@ -0,0 +1,30 @@
+#ifndef __LINEAR_HASH_NTA_H
+#define __LINEAR_HASH_NTA_H
+
+typedef struct {
+  long current_hashBucket;
+  recordid current_recordid;
+} lladd_hash_iterator;
+
+/** Implementaiton of a linear hash table that makes use of nested top actions. */
+
+recordid ThashCreate(int xid, int keySize, int valSize);
+void ThashDelete(int xid, recordid hash);
+/* @return 1 if the key was defined, 0 otherwise. */
+int ThashInsert(int xid, recordid hash, const byte* key, int keySize, const byte* value, int valueSize);
+/* @return 1 if the key was defined, 0 otherwise. */
+int ThashRemove(int xid, recordid hash, const byte* key, int keySize);
+
+/** @return size of the value associated with key, or -1 if key not found. 
+                   (a return value of zero means the key is associated with an
+		   empty value.) */
+int ThashLookup(int xid, recordid hash, const byte* key, int keySize, byte ** value);
+lladd_hash_iterator * ThashIterator(int xid, recordid hash);
+int ThashIteratorNext(int xid, recordid hash, lladd_hash_iterator * it, byte ** key, byte** value);
+void ThashIteratorFree(int xid, lladd_hash_iterator * it);
+
+//Support 16 entries by default.
+#define HASH_INIT_BITS 4
+#define HASH_FILL_FACTOR 0.7
+
+#endif // __LINEAR_HASH_NTA_H

lladd/operations/linkedListNTA.h

@@ -0,0 +1,36 @@
+#ifndef __LINKED_LIST_NTA_H
+#define __LINKED_LIST_NTA_H
+typedef struct {
+  recordid next;
+} lladd_linkedList_entry;
+typedef struct {
+  int keySize;
+  int valueSize;
+  recordid next;
+  /** The implementation of TlinkedListRemove always preserves 
+      the location of the head of the linked list.  Therefore, 
+      if the first entry is removed, *and* the iterator just returned
+      the head of the list, then the iterator needs to reset itself. */
+  int first;
+  recordid listRoot;
+} lladd_linkedList_iterator;
+
+int TlinkedListInsert(int xid, recordid list, const byte * key, int keySize, const byte * value, int valueSize);
+int TlinkedListFind(int xid, recordid list, const byte * key, int keySize, byte ** value);
+int TlinkedListRemove(int xid, recordid list, const byte * key, int keySize);
+int TlinkedListMove(int xid, recordid start_list, recordid end_list, const byte *key, int keySize);
+/** The linked list iterator can tolerate the concurrent removal of values that 
+    it has already returned.  In the presence of such removals, the iterator 
+    will return the keys and values present in the list as it existed when next()
+    was first called.
+
+    @return a new iterator initialized to the head of the list.  */
+lladd_linkedList_iterator * TlinkedListIterator(int xid, recordid list, int keySize, int valueSize);
+/** @return 1 if there was another entry to be iterated over. 0 otherwise.  
+     If this function returns 1, the caller must free() the malloced memory 
+     returned via the key and value arguments.*/
+int TlinkedListNext(int xid, lladd_linkedList_iterator * it, byte ** key, int * keySize, byte ** value, int * valueSize);
+recordid TlinkedListCreate(int xid, int keySize, int ValueSize);
+void TlinkedListDelete(int xid, recordid list);
+
+#endif //__LINKED_LIST_NTA_H

lladd/operations/naiveLinearHash.h

@@ -12,7 +12,6 @@
 */
 
 
-
 recordid ThashAlloc(int xid, int keySize, int valSize) ;
 
 void TnaiveHashInsert(int xid, recordid hashRid, 

src/lladd/Makefile.am

@@ -11,5 +11,5 @@ liblladd_a_SOURCES=crc32.c common.c stats.c io.c bufferManager.c linkedlist.c op
 		   operations/alloc.c operations/noop.c operations/instantSet.c \
 		   page/slotted.c operations/lladdhash.c page/header.c page/fixed.c \
 		   operations/arrayList.c hash.c operations/linearHash.c operations/naiveLinearHash.c \
-		   operations/nestedTopActions.c
+		   operations/nestedTopActions.c operations/linearHashNTA.c operations/linkedListNTA.c
 AM_CFLAGS= -g -Wall -pedantic -std=gnu99

src/lladd/logger/logWriter.c

@@ -144,7 +144,7 @@ int openLogWriter() {
 
   assert(!posix_memalign((void*)&(buffer), PAGE_SIZE, BUFSIZE));
 
-  int logFD = open (LOG_FILE, O_CREAT | O_RDWR | O_APPEND | O_SYNC, S_IRWXU | S_IRWXG | S_IRWXO);
+  int logFD = open (LOG_FILE, O_CREAT | O_RDWR | O_APPEND /*| O_SYNC*/, S_IRWXU | S_IRWXG | S_IRWXO);
   if(logFD == -1) {
     perror("Couldn't open log file (A)");
     abort();
@@ -347,10 +347,10 @@ void syncLog() {
   // Since we open the logfile with O_SYNC, fflush suffices.
 #ifdef HAVE_FDATASYNC
   /* Should be available in linux >= 2.4 */
-  /*  fdatasync(fileno(log));  */
+  fdatasync(fileno(log));  
 #else
   /* Slow - forces fs implementation to sync the file metadata to disk */
-  /* fsync(fileno(log));   */
+  fsync(fileno(log));   
 #endif
 
   writelock(flushedLSN_lock, 0);

src/lladd/operations/linearHash.c

@@ -44,7 +44,8 @@ static int operateUndoInsert(int xid, Page * p, lsn_t lsn, recordid rid, const v
   int keySize = rid.size;
   int valSize = rid.slot;
   rid.slot = 0;
-  rid.size = sizeof(recordid);
+//  rid.size = sizeof(recordid);
+  rid.slot = sizeof(hashEntry) + keySize + valSize;
 
   if(!pblHtLookup(openHashes, &rid.page, sizeof(int))) {
     abort();
@@ -158,6 +159,7 @@ int TlogicalHashDelete(int xid, recordid hashRid, void * key, int keySize, void
     hashRid.size = sizeof(undoDeleteArg) + keySize + valSize;
 
     Tupdate(xid, hashRid, arg, OPERATION_LINEAR_DELETE);
+    hashRid.size = sizeof(hashEntry) + keySize + valSize;
     free(arg);
     /*    hashRid.size = sizeof(recordid); */
     ThashInstantDelete(xid, hashRid, key, keySize, valSize);
@@ -189,7 +191,6 @@ void instant_expand (int xid, recordid hash, int next_split, int i, int keySize,
   /*  static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
       static pthread_mutex_t slow_mutex = PTHREAD_MUTEX_INITIALIZER; */
 
-
 /*  static int count = 4096 * .25;
 
   pthread_mutex_lock(&exp_mutex);
@@ -445,14 +446,16 @@ void instant_rehash(int xid, recordid hashRid, int next_split, int i, int keySiz
     if(D_contents->next.size != -1) {
       D = D_contents->next;
       TreadUnlocked(xid, D, D_contents);
+    } else {
+      abort(); // Got here?  We're starting a new bucket, but found that it is already -1 terminated...
     }
-  }
+  } 
 
   int old_hash;
   int new_hash = hash(A_contents+1, keySize, i,   ULONG_MAX) + 2;
 
   while(new_hash != next_split) {
-    // Need a record in A that belongs in the first bucket... 
+    // Move things into the new bucket until we find something that belongs in the first bucket... 
     
     recordid oldANext = A_contents->next;
 
@@ -487,6 +490,7 @@ void instant_rehash(int xid, recordid hashRid, int next_split, int i, int keySiz
     assert(A.size == sizeof(hashEntry) + keySize + valSize);
     if(oldANext.size == -1) {
       memset(A_contents, 0, sizeof(hashEntry) + keySize + valSize);
+      A_contents->next.size = -1; // added
       //      assert(memcmp(&A_contents->next, &A, sizeof(recordid)));
       TinstantSet(xid, A, A_contents);
       free(D_contents);
@@ -813,14 +817,7 @@ int TlogicalHashLookup(int xid, recordid hashRid, void * key, int keySize, void
   return ret;
 }
 
-typedef struct {
+
-  long current_hashBucket;
-  recordid current_rid;
-} linearHash_iterator;
-typedef struct {
-  byte * key;
-  byte * value;
-} linearHash_iteratorPair;
 
 linearHash_iterator * TlogicalHashIterator(int xid, recordid hashRid) {
   recordid NULLRID; NULLRID.page = 0; NULLRID.slot=2; NULLRID.size = -1;
@@ -843,23 +840,32 @@ linearHash_iteratorPair TlogicalHashIteratorNext(int xid, recordid hashRid, line
   //next.size == 0 -> empty bucket.  == -1 -> end of list.
   int inBucket = 0;
   //while(!memcmp(&(it->current_rid), &(NULLRID), sizeof(recordid)) 
-  while(it->current_rid.size == -1
+  printf("--- %d %d %d\n", it->current_rid.size, it->current_hashBucket, max_bucket(headerHashBits, headerNextSplit)); fflush(NULL);	 
-	&& it->current_hashBucket <= max_bucket(headerHashBits, headerNextSplit)) {
+  int found = 0;
+  while(/*it->current_rid.size == -1
+	&& */it->current_hashBucket <= max_bucket(headerHashBits, headerNextSplit)) {
     hashRid.slot = it->current_hashBucket;
     Tread(xid, hashRid, e);
-    if(e->next.size == -1) {
+    it->current_rid = hashRid;
-      it->current_rid = hashRid;
-      inBucket = 1;
-    } // else, it stays NULLRID.
     it->current_hashBucket++;
+    if(e->next.size == 0) {
+//      printf("aaa {%d, %d, %d} {%d, %d, %d} %d %d\n", e->next.page, e->next.slot, e->next.size, it->current_rid.page, it->current_rid.slot, it->current_rid.size, it->current_hashBucket, max_bucket(headerHashBits, headerNextSplit)); fflush(NULL);	 
+      inBucket = 1;
+    } else {
+      found = 1;
+      printf("bbb {%d, %d, %d} {%d, %d, %d} %d %d\n", e->next.page, e->next.slot, e->next.size, it->current_rid.page, it->current_rid.slot, it->current_rid.size, it->current_hashBucket, max_bucket(headerHashBits, headerNextSplit)); fflush(NULL);	 
+      break;
+    }      // else, it stays NULLRID.
   }
-  if(! it->current_hashBucket <= max_bucket(headerHashBits, headerNextSplit)) {
+  if(it->current_hashBucket > max_bucket(headerHashBits, headerNextSplit)) {
       p.key   = NULL;
       p.value = NULL;
-      memcpy(&(it->current_rid), &(NULLRID), sizeof(recordid));
+    it->current_rid = NULLRID;
+//      memcpy(&(it->current_rid), &(NULLRID), sizeof(recordid));
       it->current_hashBucket = 0;
   } else {
-      p.key   = memcpy(malloc(keySize), (e+1), keySize);
+//      Tread(xid, e->next, e);
+      p.key   = memcpy(malloc(keySize), e+1, keySize);
       p.value = memcpy(malloc(valSize), ((byte*)(e+1))+keySize, valSize);
       it->current_rid = e->next;
   }

src/lladd/operations/linearHashNTA.c

@@ -0,0 +1,139 @@
+#include <lladd/transactional.h>
+#include <lladd/hash.h>
+#include <stdlib.h>
+#include <assert.h>
+
+
+typedef struct {
+  recordid buckets;
+  int keySize;
+  int valueSize;
+  long nextSplit;
+  int bits;
+  long numEntries;
+} lladd_hash_header;
+
+
+
+/* private methods... */
+static void ThashSplitBucket(int xid, recordid hashHeader, lladd_hash_header * lhh);
+
+
+#define HASH_INIT_ARRAY_LIST_COUNT twoToThe(HASH_INIT_BITS)
+#define HASH_INIT_ARRAY_LIST_MULT    2
+
+recordid ThashCreate(int xid, int keySize, int valueSize) {
+  recordid hashHeader = Talloc(xid, sizeof(lladd_hash_header));
+  lladd_hash_header lhh;
+  lhh.buckets = TarrayListAlloc(xid, HASH_INIT_ARRAY_LIST_COUNT, HASH_INIT_ARRAY_LIST_MULT, sizeof(lladd_linkedList_entry) + keySize + valueSize);
+  TarrayListExtend(xid, lhh.buckets, HASH_INIT_ARRAY_LIST_COUNT);
+  int i;
+  byte * entry = calloc(1, sizeof(lhh.buckets));
+  recordid bucket = lhh.buckets;
+  for(i = 0; i < HASH_INIT_ARRAY_LIST_COUNT; i++) {
+    bucket.slot = i;
+    Tset(xid, bucket, entry);
+  }
+  free (entry);
+  lhh.keySize = keySize;
+  lhh.valueSize = valueSize;
+  lhh.nextSplit = 0;
+  lhh.bits = HASH_INIT_BITS;
+  lhh.numEntries = 0;
+  
+  Tset(xid, hashHeader, &lhh);
+  return hashHeader;
+}
+  
+void ThashDelete(int xid, recordid hash) {
+  abort();
+}
+
+
+int ThashInsert(int xid, recordid hashHeader, const byte* key, int keySize, const byte* value, int valueSize) {
+  lladd_hash_header lhh;
+  Tread(xid, hashHeader, &lhh);
+
+  lhh.numEntries ++;
+  
+  if(lhh.numEntries > (int)((double)(lhh.nextSplit + twoToThe(lhh.bits-1)) * HASH_FILL_FACTOR)) {
+  	ThashSplitBucket(xid, hashHeader, &lhh);
+  }
+  
+  Tset(xid, hashHeader, &lhh);
+  
+  assert(lhh.keySize == keySize); assert(lhh.valueSize == valueSize);
+  
+  recordid bucket = lhh.buckets;
+  bucket.slot = hash(key, keySize, lhh.bits, lhh.nextSplit);
+  
+  int ret = TlinkedListInsert(xid, bucket, key, keySize, value, valueSize);
+
+  return ret;
+}
+
+int ThashRemove(int xid, recordid hashHeader, const byte * key, int keySize) {
+  lladd_hash_header lhh;
+  Tread(xid, hashHeader, &lhh);
+
+  lhh.numEntries--;
+  
+  Tset(xid, hashHeader, &lhh);
+  
+  assert(lhh.keySize == keySize);
+  
+  recordid bucket = lhh.buckets;
+  bucket.slot = hash(key, keySize, lhh.bits, lhh.nextSplit);
+  
+  int ret = TlinkedListRemove(xid, bucket, key, keySize);
+  
+  return ret;
+}
+
+int ThashLookup(int xid, recordid hashHeader, const byte * key, int keySize, byte ** value) {
+  lladd_hash_header lhh;
+  Tread(xid, hashHeader, &lhh);
+  
+  assert(lhh.keySize == keySize);
+  
+  recordid bucket = lhh.buckets;
+  bucket.slot = hash(key, keySize, lhh.bits, lhh.nextSplit);
+  
+  int ret = TlinkedListFind(xid, bucket, key, keySize, value);
+  
+  return ret;
+}
+/** @todo Write ThashSplitBucket */
+static void ThashSplitBucket(int xid, recordid hashHeader, lladd_hash_header * lhh) {
+  long old_bucket = lhh->nextSplit;
+  long new_bucket = old_bucket + twoToThe(lhh->bits-1);
+  recordid old_bucket_rid = lhh->buckets;
+  recordid new_bucket_rid = lhh->buckets;
+  old_bucket_rid.slot = old_bucket;
+  new_bucket_rid.slot = new_bucket;
+
+  TarrayListExtend(xid, lhh->buckets, 1);
+  byte * entry = calloc(1, lhh->buckets.size);
+  Tset(xid, new_bucket_rid, entry);
+  free(entry);  
+  if(lhh->nextSplit < twoToThe(lhh->bits-1)-1) {
+    lhh->nextSplit++;
+  } else {
+    lhh->nextSplit = 0;
+    lhh->bits++;    
+  }
+  lladd_linkedList_iterator * it = TlinkedListIterator(xid, old_bucket_rid, lhh->keySize, lhh->valueSize);
+  byte * key, *value;
+  int keySize, valueSize;
+  while(TlinkedListNext(xid, it, &key, &keySize, &value, &valueSize)) {
+    assert(valueSize == lhh->valueSize);
+    assert(keySize == lhh->keySize);
+    if(hash(key, keySize, lhh->bits, lhh->nextSplit) != old_bucket) {
+      TlinkedListRemove(xid, old_bucket_rid, key, keySize);
+      TlinkedListInsert(xid, new_bucket_rid, key, keySize, value, valueSize);
+    }
+    free(key);
+    free(value);
+  }
+  return;
+}

src/lladd/operations/linkedListNTA.c

@@ -0,0 +1,233 @@
+#include <lladd/transactional.h>
+#include <lladd/hash.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+/** A quick note on the format of linked lists.  Each entry consists 
+     of a struct with some variable length data appended to it.
+
+  To access an entry's contents:
+  
+  lladd_linkedList_entry * entry; 
+  ...
+  if(entry->size) {
+  	key = (byte*)(entry + 1);
+  	value = ((byte*)(entry+1)) + keySize;
+  } else {
+    entry->size must be nonzero if the entry is defined.  It will be 
+      zero if the entry is uniniailized (this can be the case if the 
+      list has not yet been initialized. The end of the list is marked 
+      by a next field with size -1.
+  }
+  
+  To get the successor in the list:
+  
+  lladd_linkedList_entry next = entry->next;
+  
+  
+  
+	@file
+*/
+int TlinkedListInsert(int xid, recordid list, const byte * key, int keySize, const byte * value, int valueSize) {
+  int ret = TlinkedListRemove(xid, list, key, keySize);
+  lladd_linkedList_entry * entry = malloc(sizeof(lladd_linkedList_entry) + keySize + valueSize);
+  Tread(xid, list, entry);
+  if(!entry->next.size) {
+    memcpy(entry+1, key, keySize);
+    memcpy(((byte*)(entry+1))+keySize, value, valueSize);
+    entry->next.page = 0;
+    entry->next.slot = 0;
+    entry->next.size = -1;
+    Tset(xid, list, entry);
+  } else {
+    lladd_linkedList_entry * newEntry = malloc(sizeof(lladd_linkedList_entry) + keySize + valueSize);
+    memcpy(newEntry + 1, key, keySize);
+    memcpy(((byte*)(newEntry+1))+keySize, value, valueSize);
+    newEntry->next = entry->next;
+    recordid newRid = Talloc(xid, sizeof(lladd_linkedList_entry) + keySize + valueSize);
+    Tset(xid, newRid, newEntry);
+    entry->next = newRid;
+    Tset(xid, list, entry);
+    free(newEntry);
+  }
+  free(entry);
+  return ret;
+}
+
+int TlinkedListFind(int xid, recordid list, const byte * key, int keySize, byte ** value) {
+  lladd_linkedList_entry * entry = malloc(list.size);
+  Tread(xid, list, entry);
+
+  if(!entry->next.size) {
+    free(entry);    
+    return -1; // empty list 
+  }
+  while(1) {
+    if(!memcmp(entry + 1, key, keySize)) { 
+      // Bucket contains the entry of interest.
+      int valueSize = list.size - (sizeof(lladd_linkedList_entry) + keySize);
+     *value  = malloc(valueSize);
+      memcpy(*value, ((byte*)(entry+1))+keySize, valueSize);
+      free(entry);
+      return valueSize;
+    }
+    if(entry->next.size != -1) {
+       assert(entry->next.size == list.size);  // Don't handle lists with variable length records for now
+       Tread(xid, entry->next, entry);
+    } else {
+       break;
+    }
+  }
+  free(entry);
+  return -1;
+}
+int TlinkedListRemove(int xid, recordid list, const byte * key, int keySize) {
+  lladd_linkedList_entry * entry = malloc(list.size);
+
+  Tread(xid, list, entry);
+  if(entry->next.size == 0) {
+    //Empty List.
+    free(entry);
+    return 0;
+  }
+  int listRoot = 1;
+  recordid lastRead = list;
+  recordid oldLastRead;
+  oldLastRead.size = -2;
+  while(1) {
+    if(!memcmp(entry + 1, key, keySize)) { 
+      // Bucket contains the entry of interest.
+      if(listRoot) {
+	if(entry->next.size == -1) {
+	  memset(entry, 0, list.size);
+	  Tset(xid, lastRead, entry);
+	} else {
+	  assert(entry->next.size == list.size);  // Otherwise, sometihng strange is happening, or the list contains entries with variable sizes.
+	  lladd_linkedList_entry * entry2 = malloc(list.size);
+	  Tread(xid, entry->next, entry2);
+	  Tdealloc(xid, entry->next); // could break iterator, since it writes one entry ahead.
+	  Tset(xid, lastRead, entry2);
+	  free(entry2);
+	  }
+      } else {
+	lladd_linkedList_entry * entry2 = malloc(list.size);
+	assert(oldLastRead.size != -2);
+	Tread(xid, oldLastRead, entry2);
+	memcpy(&(entry2->next), &(entry->next), sizeof(recordid));
+	Tset(xid, oldLastRead, entry2);
+	Tdealloc(xid, lastRead);
+	free (entry2);
+      }
+      free(entry);
+      return 1;
+    } else { // Entry doesn't match the key we're looking for.
+      if(entry->next.size != -1) {
+	 assert(entry->next.size == list.size);  // Don't handle lists with variable length records for now
+	 oldLastRead = lastRead;
+	 lastRead = entry->next;
+	 Tread(xid, entry->next, entry);
+	 listRoot = 0;
+      } else {
+	 break;
+      }
+    }
+  } 
+  free(entry);
+  return 0;
+}
+/*** @todo TlinkedListMove could be much faster, but this is good enough for a first pass */
+int TlinkedListMove(int xid, recordid start_list, recordid end_list, const byte *key, int keySize) {
+  byte * value;
+  int valueSize = TlinkedListFind(xid, start_list, key, keySize, &value);
+  if(valueSize == -1) {
+    return 0;
+  } else {
+    TlinkedListRemove(xid, start_list, key, keySize);
+    TlinkedListInsert(xid, end_list, key, keySize, value, valueSize);
+    return 1;
+  }
+}
+recordid TlinkedListCreate(int xid, int keySize, int valueSize) {
+  recordid ret = Talloc(xid, sizeof(lladd_linkedList_entry) + keySize + valueSize);
+  byte * cleared = calloc(sizeof(lladd_linkedList_entry) + keySize + valueSize, sizeof(byte));
+  Tset(xid, ret, cleared);
+  free(cleared);
+  return ret;
+}
+void TlinkedListDelete(int xid, recordid list) {
+  lladd_linkedList_entry * entry = malloc(list.size);
+  
+  Tread(xid, list, entry);
+  Tdealloc(xid, list);
+  
+  if(entry->next.size == 0) {
+    return;
+  }
+  
+  while(entry->next.size != -1) {
+    recordid nextEntry;
+    Tread(xid, nextEntry, entry);
+    assert(!memcmp(&nextEntry, &(entry->next), sizeof(recordid)));
+    Tdealloc(xid, nextEntry);
+  }
+  
+  free(entry);
+}
+
+lladd_linkedList_iterator * TlinkedListIterator(int xid, recordid list, int keySize, int valueSize) {
+  lladd_linkedList_iterator * it = malloc(sizeof(lladd_linkedList_iterator));
+  it->keySize = keySize;
+  it->valueSize = valueSize;
+  it->next = list;
+  it->first = -1;
+  it->listRoot = list;
+  return it;
+}
+
+int TlinkedListNext(int xid, lladd_linkedList_iterator * it, byte ** key, int * keySize, byte **value, int * valueSize) {
+
+  if(it->next.size == -1)  { free(it); return 0; }
+
+  if(it->first == -1) {
+    it->first = 1;
+  } else if(it->first) {
+    lladd_linkedList_entry * entry = malloc(it->next.size);
+    Tread(xid, it->listRoot, entry);
+    int listTouched;
+    listTouched = memcmp(&(entry->next), &(it->next), sizeof(recordid));
+    free(entry);
+    if(listTouched) {
+      //The root entry was removed.  Reset the iterator.
+      it->first = -1;
+      it->next = it->listRoot;
+      return TlinkedListNext(xid, it, key, keySize, value, valueSize);
+    } else {
+      //continue as normal.
+      it->first = 0;
+    }
+  }
+  
+  assert(it->keySize + it->valueSize + sizeof(lladd_linkedList_entry) == it->next.size);
+
+  lladd_linkedList_entry * entry = malloc(it->next.size);
+  Tread(xid, it->next, entry);
+  if(entry->next.size) {
+    *keySize = it->keySize;
+    *valueSize = it->valueSize;
+    *key = malloc(*keySize);
+    *value = malloc(*valueSize);
+    
+    it->next = entry->next;
+  
+    memcpy(*key, entry+1, *keySize);
+    memcpy(*value, ((byte*)(entry + 1))+*keySize, *valueSize);
+    
+    free(entry);  
+    return 1;
+  } else {
+    // This entry was empty (this case occurs with empty lists)
+    free(it); 
+    free(entry);
+    return 0;
+  }
+}

test/lladd/Makefile.am

@@ -1,12 +1,11 @@
 INCLUDES = @CHECK_CFLAGS@
 if HAVE_CHECK
 ## Had to disable check_lht because lht needs to be rewritten.
-TESTS = check_logEntry check_logWriter check_page check_operations check_transactional2 check_recovery check_blobRecovery check_bufferManager check_indirect check_lladdhash check_pageOperations check_linearHash  check_logicalLinearHash check_header
+TESTS = check_logEntry check_logWriter check_page check_operations check_transactional2 check_recovery check_blobRecovery check_bufferManager check_indirect check_lladdhash check_pageOperations check_linearHash  check_logicalLinearHash check_header check_linkedListNTA check_linearHashNTA
 else
 TESTS = 
 endif
 noinst_PROGRAMS = $(TESTS)
 LDADD = @CHECK_LIBS@ $(top_builddir)/src/lladd/liblladd.a $(top_builddir)/src/pbl/libpbl.a  $(top_builddir)/src/libdfa/librw.a #-lefence
-CLEANFILES = check_lht.log check_logEntry.log storefile.txt logfile.txt blob0_file.txt blob1_file.txt check_blobRecovery.log check_logWriter.log check_operations.log check_recovery.log check_transactional2.log check_page.log check_bufferManager.log check_indirect.log check_bufferMananger.log check_lladdhash.log check_pageOperations.log check_linearhash.log
+CLEANFILES = check_lht.log check_logEntry.log storefile.txt logfile.txt blob0_file.txt blob1_file.txt check_blobRecovery.log check_logWriter.log check_operations.log check_recovery.log check_transactional2.log check_page.log check_bufferManager.log check_indirect.log check_bufferMananger.log check_lladdhash.log check_pageOperations.log check_linearhash.log check_linkedListNTA.log check_linearHashNTA.log
 AM_CFLAGS= -g -Wall -pedantic -std=gnu99
-

test/lladd/check_linearHash.c

@@ -57,7 +57,7 @@ terms specified in this license.
     executes each of the insert / remove / lookup operations a few times.
 */
 //#define NUM_ENTRIES 100000   
-#define NUM_ENTRIES 2001
+#define NUM_ENTRIES 10000
 /* #define NUM_ENTRIES 1000  */
 /*#define NUM_ENTRIES 100  */
 
@@ -278,6 +278,8 @@ START_TEST(transactionalLinearHashTest)
 	
 } END_TEST
 
+
+
 Suite * check_suite(void) {
   Suite *s = suite_create("linearHash");
   /* Begin a new test */

test/lladd/check_linearHashNTA.c

@@ -0,0 +1,130 @@
+/*---
+This software is copyrighted by the Regents of the University of
+California, and other parties. The following terms apply to all files
+associated with the software unless explicitly disclaimed in
+individual files.
+
+The authors hereby grant permission to use, copy, modify, distribute,
+and license this software and its documentation for any purpose,
+provided that existing copyright notices are retained in all copies
+and that this notice is included verbatim in any distributions. No
+written agreement, license, or royalty fee is required for any of the
+authorized uses. Modifications to this software may be copyrighted by
+their authors and need not follow the licensing terms described here,
+provided that the new terms are clearly indicated on the first page of
+each file where they apply.
+
+IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
+FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
+ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
+DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
+NON-INFRINGEMENT. THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, AND
+THE AUTHORS AND DISTRIBUTORS HAVE NO OBLIGATION TO PROVIDE
+MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+
+GOVERNMENT USE: If you are acquiring this software on behalf of the
+U.S. government, the Government shall have only "Restricted Rights" in
+the software and related documentation as defined in the Federal
+Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you are
+acquiring the software on behalf of the Department of Defense, the
+software shall be classified as "Commercial Computer Software" and the
+Government shall have only "Restricted Rights" as defined in Clause
+252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the
+authors grant the U.S. Government and others acting in its behalf
+permission to use and distribute the software in accordance with the
+terms specified in this license.
+---*/
+
+#include <config.h>
+#include <check.h>
+#include "../check_includes.h"
+
+#include <lladd/transactional.h>
+
+#include <assert.h>
+#include <limits.h>
+#include <math.h>
+
+#define LOG_NAME   "check_linearHashNTA.log"
+#define NUM_ENTRIES 100000
+START_TEST(linearHashNTAtest)
+{
+  Tinit();
+  
+  int xid = Tbegin();
+  recordid val;
+  recordid hashHeader = ThashCreate(xid, sizeof(int), sizeof(recordid));
+  recordid * val2;
+  int i;
+  printf("\n"); fflush(stdout);
+  for(i = 0; i < NUM_ENTRIES; i++) {
+    if(!(i % (NUM_ENTRIES/10))) {
+      printf("."); fflush(stdout);
+    }
+    val.page = i * NUM_ENTRIES;
+    val.slot = val.page * NUM_ENTRIES;
+    val.size = val.slot * NUM_ENTRIES;
+    assert(-1 == ThashLookup(xid, hashHeader, (byte*)&i, sizeof(int), (byte**)&val2));
+    ThashInsert(xid, hashHeader, (byte*)&i, sizeof(int), (byte*)&val, sizeof(recordid));
+    assert(sizeof(recordid) == ThashLookup(xid, hashHeader, (byte*)&i, sizeof(int), (byte**)&val2));
+    assert(val2->page == i * NUM_ENTRIES);
+    assert(val2->slot == val2->page * NUM_ENTRIES);
+    assert(val2->size == val2->slot * NUM_ENTRIES);
+    free(val2);
+  }
+  
+  Tcommit(xid);
+  printf("\n"); fflush(stdout);
+
+  xid = Tbegin();
+  for(i = 0; i < NUM_ENTRIES; i+=10){
+    if(!(i % (NUM_ENTRIES/10))) {
+      printf("-"); fflush(stdout);
+    }
+    assert(sizeof(recordid) == ThashLookup(xid, hashHeader, (byte*)&i, sizeof(int), (byte**)&val2));
+    free(val2);
+    assert(ThashRemove(xid, hashHeader, (byte*)&i, sizeof(int)));
+    assert(-1==ThashLookup(xid, hashHeader, (byte*)&i, sizeof(int), (byte**)&val2));
+    assert(!ThashRemove(xid, hashHeader, (byte*)&i, sizeof(int)));
+  }
+  printf("\nabort()\n"); fflush(stdout);
+  Tabort(xid);
+  xid = Tbegin();
+  for(i = 0; i < NUM_ENTRIES; i++) {
+    if(!(i % (NUM_ENTRIES/10))) {
+      printf("+"); fflush(stdout);
+    }
+    assert(sizeof(recordid) == ThashLookup(xid, hashHeader, (byte*)&i, sizeof(int), (byte**)&val2));
+    assert(val2->page == i * NUM_ENTRIES);
+    assert(val2->slot == val2->page * NUM_ENTRIES);
+    assert(val2->size == val2->slot * NUM_ENTRIES);
+    free(val2);
+  }
+  Tcommit(xid);
+  Tdeinit();
+} END_TEST
+
+Suite * check_suite(void) {
+  Suite *s = suite_create("linearHashNTA");
+  /* Begin a new test */
+  TCase *tc = tcase_create("simple");
+
+
+  /* Sub tests are added, one per line, here */
+
+  tcase_add_test(tc, linearHashNTAtest);
+  
+  /* --------------------------------------------- */
+  
+  tcase_add_checked_fixture(tc, setup, teardown);
+
+  suite_add_tcase(s, tc);
+  return s;
+}
+
+#include "../check_setup.h"

test/lladd/check_linkedListNTA.c

@@ -0,0 +1,123 @@
+/*---
+This software is copyrighted by the Regents of the University of
+California, and other parties. The following terms apply to all files
+associated with the software unless explicitly disclaimed in
+individual files.
+
+The authors hereby grant permission to use, copy, modify, distribute,
+and license this software and its documentation for any purpose,
+provided that existing copyright notices are retained in all copies
+and that this notice is included verbatim in any distributions. No
+written agreement, license, or royalty fee is required for any of the
+authorized uses. Modifications to this software may be copyrighted by
+their authors and need not follow the licensing terms described here,
+provided that the new terms are clearly indicated on the first page of
+each file where they apply.
+
+IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
+FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
+ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
+DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND
+NON-INFRINGEMENT. THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, AND
+THE AUTHORS AND DISTRIBUTORS HAVE NO OBLIGATION TO PROVIDE
+MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+
+GOVERNMENT USE: If you are acquiring this software on behalf of the
+U.S. government, the Government shall have only "Restricted Rights" in
+the software and related documentation as defined in the Federal
+Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you are
+acquiring the software on behalf of the Department of Defense, the
+software shall be classified as "Commercial Computer Software" and the
+Government shall have only "Restricted Rights" as defined in Clause
+252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the
+authors grant the U.S. Government and others acting in its behalf
+permission to use and distribute the software in accordance with the
+terms specified in this license.
+---*/
+
+#include <config.h>
+#include <check.h>
+#include "../check_includes.h"
+
+#include <lladd/transactional.h>
+
+#include <assert.h>
+#include <limits.h>
+#include <math.h>
+
+#define LOG_NAME   "check_linkedListNTA.log"
+START_TEST(linkedListNTAtest)
+{
+  Tinit();
+
+  int xid = Tbegin();
+  
+  recordid linkedList = TlinkedListCreate(xid, sizeof(int), sizeof(recordid));
+  int i;
+  for(i = 0; i < 1000; i++) {
+    recordid val;
+    recordid * val2 = NULL;
+    val.page = i * 1000;
+    val.slot = val.page * 1000;
+    val.size = val.slot * 1000;
+    assert(-1==TlinkedListFind(xid, linkedList, (byte*)(&i), sizeof(int), (byte**)&val2));
+    TlinkedListInsert(xid, linkedList, (byte*)&i, sizeof(int), (byte*)&val, sizeof(recordid));
+    assert(sizeof(recordid)==TlinkedListFind(xid, linkedList, (byte*)(&i), sizeof(int), (byte**)&val2));
+    assert(!memcmp(&val, val2, sizeof(recordid)));
+    free(val2);
+  }
+  
+  Tcommit(xid);
+  
+  xid = Tbegin();
+  for(i = 0; i < 1000; i+=10) {
+    recordid * val2 = NULL;
+    assert(sizeof(recordid)==TlinkedListFind(xid, linkedList, (byte*)(&i), sizeof(int), (byte**)&val2));
+    assert(val2->page == i * 1000);
+    assert(val2->slot == i * 1000 * 1000);
+    assert(val2->size == i * 1000 * 1000 * 1000);
+    free(val2);
+    assert(TlinkedListRemove(xid, linkedList, (byte*)&i, sizeof(int)));
+    assert(-1==TlinkedListFind(xid, linkedList, (byte*)(&i), sizeof(int), (byte**)&val2));
+    assert(!TlinkedListRemove(xid, linkedList, (byte*)&i, sizeof(int)));
+  }
+  Tabort(xid);
+  xid = Tbegin();
+  for(i = 0; i < 1000; i++) {
+    recordid * val2;
+    assert(sizeof(recordid)==TlinkedListFind(xid, linkedList, (byte*)(&i), sizeof(int), (byte**)&val2));
+    assert(val2->page == i * 1000);
+    assert(val2->slot == i * 1000 * 1000);
+    assert(val2->size == i * 1000 * 1000 * 1000);
+    free(val2);
+  }
+  Tcommit(xid);
+  Tdeinit();
+} END_TEST
+
+
+
+Suite * check_suite(void) {
+  Suite *s = suite_create("linkedListNTA");
+  /* Begin a new test */
+  TCase *tc = tcase_create("simple");
+
+
+  /* Sub tests are added, one per line, here */
+
+  tcase_add_test(tc, linkedListNTAtest);
+  
+  /* --------------------------------------------- */
+  
+  tcase_add_checked_fixture(tc, setup, teardown);
+
+  suite_add_tcase(s, tc);
+  return s;
+}
+
+#include "../check_setup.h"

test/lladd/check_logicalLinearHash.c

@@ -57,8 +57,8 @@ terms specified in this license.
     executes each of the insert / remove / lookup operations a few times.
 */
 //#define NUM_ENTRIES 100000   
-#define NUM_ENTRIES 2001  
+/*#define NUM_ENTRIES 10000*/
-/* #define NUM_ENTRIES 1000  */
+#define NUM_ENTRIES 1000  
 /*#define NUM_ENTRIES 100  */
 
 /**
@@ -200,7 +200,48 @@ START_TEST(simpleLinearHashTest)
 
 }
 END_TEST
+#define NUM_ITERATOR_ENTRIES 2000
 
+START_TEST(check_linearHashIterator) {
+  Tinit();
+  
+  int xid = Tbegin();
+  
+  recordid rid = ThashAlloc(xid, sizeof(int), sizeof(int));
+  
+  printf("Testing iterator.\n");  
+  int key;
+  int val;
+  
+  int * keySeen = calloc(NUM_ITERATOR_ENTRIES, sizeof(int));
+  for(int i = 0; i < NUM_ITERATOR_ENTRIES; i++) {
+    key = i;
+    val = NUM_ITERATOR_ENTRIES * key;
+    TnaiveHashInsert(xid, rid, &key, sizeof(int), &val, sizeof(int));
+  }
+  Tcommit(xid);
+  
+  xid = Tbegin();
+  for(int i = 0; i < NUM_ITERATOR_ENTRIES; i++) {
+    key = i;
+    TlogicalHashLookup(xid, rid, &key, sizeof(int), &val, sizeof(int));
+    assert(key == i);
+    assert(val == NUM_ITERATOR_ENTRIES * key);
+  }
+  Tcommit(xid);
+  xid = Tbegin();
+  linearHash_iterator * it = TlogicalHashIterator(xid, rid);
+  
+  linearHash_iteratorPair next = TlogicalHashIteratorNext(xid,rid, it, sizeof(int), sizeof(int));
+  assert(next.key );
+  while(next.key != NULL) {
+    	printf("%d -> %d\n", *(next.key), *(next.value));
+	next = TlogicalHashIteratorNext(xid, rid, it, sizeof(int), sizeof(int));
+  }
+  TlogicalHashIteratorFree(it);
+  Tcommit(xid);
+  Tdeinit();
+} END_TEST
 Suite * check_suite(void) {
   Suite *s = suite_create("linearHash");
   /* Begin a new test */
@@ -211,7 +252,7 @@ Suite * check_suite(void) {
 
   /*  tcase_add_test(tc, checkHashFcn); */
   tcase_add_test(tc, simpleLinearHashTest);
-
+  tcase_add_test(tc, check_linearHashIterator);
   /* --------------------------------------------- */
   
   tcase_add_checked_fixture(tc, setup, teardown);