From 360f0d15e2117f90aca18bd035fe57e64c97f998 Mon Sep 17 00:00:00 2001 From: Sears Russell Date: Fri, 14 Jan 2005 01:52:53 +0000 Subject: [PATCH] Working, non-concurrent linear hash and linked list implementations. The idea is to implement the non-concurrent versions, and then convert them to concurrent versions using nested top actions. --- lladd.pws | 14 +- lladd/constants.h | 2 +- lladd/operations.h | 3 + lladd/operations/linearHash.h | 15 ++ lladd/operations/linearHashNTA.h | 30 ++++ lladd/operations/linkedListNTA.h | 36 +++++ lladd/operations/naiveLinearHash.h | 1 - src/lladd/Makefile.am | 2 +- src/lladd/logger/logWriter.c | 6 +- src/lladd/operations/linearHash.c | 48 +++--- src/lladd/operations/linearHashNTA.c | 139 ++++++++++++++++ src/lladd/operations/linkedListNTA.c | 233 +++++++++++++++++++++++++++ test/lladd/Makefile.am | 5 +- test/lladd/check_linearHash.c | 4 +- test/lladd/check_linearHashNTA.c | 130 +++++++++++++++ test/lladd/check_linkedListNTA.c | 123 ++++++++++++++ test/lladd/check_logicalLinearHash.c | 47 +++++- 17 files changed, 795 insertions(+), 43 deletions(-) create mode 100644 lladd/operations/linearHashNTA.h create mode 100644 lladd/operations/linkedListNTA.h create mode 100644 src/lladd/operations/linearHashNTA.c create mode 100644 src/lladd/operations/linkedListNTA.c create mode 100644 test/lladd/check_linearHashNTA.c create mode 100644 test/lladd/check_linkedListNTA.c diff --git a/lladd.pws b/lladd.pws index a82ef49..f9c084c 100644 --- a/lladd.pws +++ b/lladd.pws @@ -75,15 +75,11 @@ filter.dir.ignore.hidden=0 0=0 1=0:5 2=0:6 -3=0:6:2 -4=0:9 -5=0:10 -6=0:10:0 -7=0:10:2 -8=0:10:3 -9=0:11 -10=0:11:4 -11=0:11:5 +3=0:9 +4=0:10 +5=0:11 +6=0:11:4 +7=0:11:5 [executer args] 0=check_linearHash diff --git a/lladd/constants.h b/lladd/constants.h index 4b1469c..3798598 100644 --- a/lladd/constants.h +++ b/lladd/constants.h @@ -76,7 +76,7 @@ terms specified in this license. #define PAGE_SIZE 4096 /* #define MAX_BUFFER_SIZE 100003 */ -/*#define MAX_BUFFER_SIZE 20029*/ +/*#define MAX_BUFFER_SIZE 20029 */ /*#define MAX_BUFFER_SIZE 10007 */ #define MAX_BUFFER_SIZE 5003 /*#define MAX_BUFFER_SIZE 2003 */ diff --git a/lladd/operations.h b/lladd/operations.h index 7b9c961..db114e2 100644 --- a/lladd/operations.h +++ b/lladd/operations.h @@ -151,6 +151,9 @@ typedef struct { #include "operations/linearHash.h" #include "operations/naiveLinearHash.h" #include "operations/nestedTopActions.h" +#include "operations/linkedListNTA.h" +#include "operations/linearHashNTA.h" + extern Operation operationsTable[]; /* [MAX_OPERATIONS]; memset somewhere */ diff --git a/lladd/operations/linearHash.h b/lladd/operations/linearHash.h index c3d2b3b..3eda00c 100644 --- a/lladd/operations/linearHash.h +++ b/lladd/operations/linearHash.h @@ -25,6 +25,21 @@ void TlogicalHashUpdate(int xid, recordid hashRid, void * key, int keySize, void void TlogicalHashInsert(int xid, recordid hashRid, void * key, int keySize, void * val, int valSize); int TlogicalHashDelete(int xid, recordid hashRid, void * key, int keySize, void * val, int valSize); int TlogicalHashLookup(int xid, recordid hashRid, void * key, int keySize, void * buf, int valSize); +typedef struct { + long current_hashBucket; + recordid current_rid; +} linearHash_iterator; +typedef struct { + byte * key; + byte * value; +} linearHash_iteratorPair; + +linearHash_iterator * TlogicalHashIterator(int xid, recordid hashRid); +void TlogicalHashIteratorFree(linearHash_iterator * it); +linearHash_iteratorPair TlogicalHashIteratorNext(int xid, recordid hashRid, linearHash_iterator * it, int keySize, int valSize); + + +Operation getLinearInsert(); Operation getLinearInsert(); Operation getLinearDelete(); Operation getUndoLinearInsert(); diff --git a/lladd/operations/linearHashNTA.h b/lladd/operations/linearHashNTA.h new file mode 100644 index 0000000..0601ce8 --- /dev/null +++ b/lladd/operations/linearHashNTA.h @@ -0,0 +1,30 @@ +#ifndef __LINEAR_HASH_NTA_H +#define __LINEAR_HASH_NTA_H + +typedef struct { + long current_hashBucket; + recordid current_recordid; +} lladd_hash_iterator; + +/** Implementaiton of a linear hash table that makes use of nested top actions. */ + +recordid ThashCreate(int xid, int keySize, int valSize); +void ThashDelete(int xid, recordid hash); +/* @return 1 if the key was defined, 0 otherwise. */ +int ThashInsert(int xid, recordid hash, const byte* key, int keySize, const byte* value, int valueSize); +/* @return 1 if the key was defined, 0 otherwise. */ +int ThashRemove(int xid, recordid hash, const byte* key, int keySize); + +/** @return size of the value associated with key, or -1 if key not found. + (a return value of zero means the key is associated with an + empty value.) */ +int ThashLookup(int xid, recordid hash, const byte* key, int keySize, byte ** value); +lladd_hash_iterator * ThashIterator(int xid, recordid hash); +int ThashIteratorNext(int xid, recordid hash, lladd_hash_iterator * it, byte ** key, byte** value); +void ThashIteratorFree(int xid, lladd_hash_iterator * it); + +//Support 16 entries by default. +#define HASH_INIT_BITS 4 +#define HASH_FILL_FACTOR 0.7 + +#endif // __LINEAR_HASH_NTA_H diff --git a/lladd/operations/linkedListNTA.h b/lladd/operations/linkedListNTA.h new file mode 100644 index 0000000..d7ac5de --- /dev/null +++ b/lladd/operations/linkedListNTA.h @@ -0,0 +1,36 @@ +#ifndef __LINKED_LIST_NTA_H +#define __LINKED_LIST_NTA_H +typedef struct { + recordid next; +} lladd_linkedList_entry; +typedef struct { + int keySize; + int valueSize; + recordid next; + /** The implementation of TlinkedListRemove always preserves + the location of the head of the linked list. Therefore, + if the first entry is removed, *and* the iterator just returned + the head of the list, then the iterator needs to reset itself. */ + int first; + recordid listRoot; +} lladd_linkedList_iterator; + +int TlinkedListInsert(int xid, recordid list, const byte * key, int keySize, const byte * value, int valueSize); +int TlinkedListFind(int xid, recordid list, const byte * key, int keySize, byte ** value); +int TlinkedListRemove(int xid, recordid list, const byte * key, int keySize); +int TlinkedListMove(int xid, recordid start_list, recordid end_list, const byte *key, int keySize); +/** The linked list iterator can tolerate the concurrent removal of values that + it has already returned. In the presence of such removals, the iterator + will return the keys and values present in the list as it existed when next() + was first called. + + @return a new iterator initialized to the head of the list. */ +lladd_linkedList_iterator * TlinkedListIterator(int xid, recordid list, int keySize, int valueSize); +/** @return 1 if there was another entry to be iterated over. 0 otherwise. + If this function returns 1, the caller must free() the malloced memory + returned via the key and value arguments.*/ +int TlinkedListNext(int xid, lladd_linkedList_iterator * it, byte ** key, int * keySize, byte ** value, int * valueSize); +recordid TlinkedListCreate(int xid, int keySize, int ValueSize); +void TlinkedListDelete(int xid, recordid list); + +#endif //__LINKED_LIST_NTA_H diff --git a/lladd/operations/naiveLinearHash.h b/lladd/operations/naiveLinearHash.h index 939406a..7d21c24 100644 --- a/lladd/operations/naiveLinearHash.h +++ b/lladd/operations/naiveLinearHash.h @@ -12,7 +12,6 @@ */ - recordid ThashAlloc(int xid, int keySize, int valSize) ; void TnaiveHashInsert(int xid, recordid hashRid, diff --git a/src/lladd/Makefile.am b/src/lladd/Makefile.am index 831ac41..11def94 100644 --- a/src/lladd/Makefile.am +++ b/src/lladd/Makefile.am @@ -11,5 +11,5 @@ liblladd_a_SOURCES=crc32.c common.c stats.c io.c bufferManager.c linkedlist.c op operations/alloc.c operations/noop.c operations/instantSet.c \ page/slotted.c operations/lladdhash.c page/header.c page/fixed.c \ operations/arrayList.c hash.c operations/linearHash.c operations/naiveLinearHash.c \ - operations/nestedTopActions.c + operations/nestedTopActions.c operations/linearHashNTA.c operations/linkedListNTA.c AM_CFLAGS= -g -Wall -pedantic -std=gnu99 diff --git a/src/lladd/logger/logWriter.c b/src/lladd/logger/logWriter.c index a3869b0..7ea8346 100644 --- a/src/lladd/logger/logWriter.c +++ b/src/lladd/logger/logWriter.c @@ -144,7 +144,7 @@ int openLogWriter() { assert(!posix_memalign((void*)&(buffer), PAGE_SIZE, BUFSIZE)); - int logFD = open (LOG_FILE, O_CREAT | O_RDWR | O_APPEND | O_SYNC, S_IRWXU | S_IRWXG | S_IRWXO); + int logFD = open (LOG_FILE, O_CREAT | O_RDWR | O_APPEND /*| O_SYNC*/, S_IRWXU | S_IRWXG | S_IRWXO); if(logFD == -1) { perror("Couldn't open log file (A)"); abort(); @@ -347,10 +347,10 @@ void syncLog() { // Since we open the logfile with O_SYNC, fflush suffices. #ifdef HAVE_FDATASYNC /* Should be available in linux >= 2.4 */ - /* fdatasync(fileno(log)); */ + fdatasync(fileno(log)); #else /* Slow - forces fs implementation to sync the file metadata to disk */ - /* fsync(fileno(log)); */ + fsync(fileno(log)); #endif writelock(flushedLSN_lock, 0); diff --git a/src/lladd/operations/linearHash.c b/src/lladd/operations/linearHash.c index dd69d14..598b439 100644 --- a/src/lladd/operations/linearHash.c +++ b/src/lladd/operations/linearHash.c @@ -44,7 +44,8 @@ static int operateUndoInsert(int xid, Page * p, lsn_t lsn, recordid rid, const v int keySize = rid.size; int valSize = rid.slot; rid.slot = 0; - rid.size = sizeof(recordid); +// rid.size = sizeof(recordid); + rid.slot = sizeof(hashEntry) + keySize + valSize; if(!pblHtLookup(openHashes, &rid.page, sizeof(int))) { abort(); @@ -158,6 +159,7 @@ int TlogicalHashDelete(int xid, recordid hashRid, void * key, int keySize, void hashRid.size = sizeof(undoDeleteArg) + keySize + valSize; Tupdate(xid, hashRid, arg, OPERATION_LINEAR_DELETE); + hashRid.size = sizeof(hashEntry) + keySize + valSize; free(arg); /* hashRid.size = sizeof(recordid); */ ThashInstantDelete(xid, hashRid, key, keySize, valSize); @@ -189,7 +191,6 @@ void instant_expand (int xid, recordid hash, int next_split, int i, int keySize, /* static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t slow_mutex = PTHREAD_MUTEX_INITIALIZER; */ - /* static int count = 4096 * .25; pthread_mutex_lock(&exp_mutex); @@ -445,14 +446,16 @@ void instant_rehash(int xid, recordid hashRid, int next_split, int i, int keySiz if(D_contents->next.size != -1) { D = D_contents->next; TreadUnlocked(xid, D, D_contents); + } else { + abort(); // Got here? We're starting a new bucket, but found that it is already -1 terminated... } - } + } int old_hash; int new_hash = hash(A_contents+1, keySize, i, ULONG_MAX) + 2; while(new_hash != next_split) { - // Need a record in A that belongs in the first bucket... + // Move things into the new bucket until we find something that belongs in the first bucket... recordid oldANext = A_contents->next; @@ -487,6 +490,7 @@ void instant_rehash(int xid, recordid hashRid, int next_split, int i, int keySiz assert(A.size == sizeof(hashEntry) + keySize + valSize); if(oldANext.size == -1) { memset(A_contents, 0, sizeof(hashEntry) + keySize + valSize); + A_contents->next.size = -1; // added // assert(memcmp(&A_contents->next, &A, sizeof(recordid))); TinstantSet(xid, A, A_contents); free(D_contents); @@ -813,14 +817,7 @@ int TlogicalHashLookup(int xid, recordid hashRid, void * key, int keySize, void return ret; } -typedef struct { - long current_hashBucket; - recordid current_rid; -} linearHash_iterator; -typedef struct { - byte * key; - byte * value; -} linearHash_iteratorPair; + linearHash_iterator * TlogicalHashIterator(int xid, recordid hashRid) { recordid NULLRID; NULLRID.page = 0; NULLRID.slot=2; NULLRID.size = -1; @@ -843,23 +840,32 @@ linearHash_iteratorPair TlogicalHashIteratorNext(int xid, recordid hashRid, line //next.size == 0 -> empty bucket. == -1 -> end of list. int inBucket = 0; //while(!memcmp(&(it->current_rid), &(NULLRID), sizeof(recordid)) - while(it->current_rid.size == -1 - && it->current_hashBucket <= max_bucket(headerHashBits, headerNextSplit)) { + printf("--- %d %d %d\n", it->current_rid.size, it->current_hashBucket, max_bucket(headerHashBits, headerNextSplit)); fflush(NULL); + int found = 0; + while(/*it->current_rid.size == -1 + && */it->current_hashBucket <= max_bucket(headerHashBits, headerNextSplit)) { hashRid.slot = it->current_hashBucket; Tread(xid, hashRid, e); - if(e->next.size == -1) { - it->current_rid = hashRid; - inBucket = 1; - } // else, it stays NULLRID. + it->current_rid = hashRid; it->current_hashBucket++; + if(e->next.size == 0) { +// printf("aaa {%d, %d, %d} {%d, %d, %d} %d %d\n", e->next.page, e->next.slot, e->next.size, it->current_rid.page, it->current_rid.slot, it->current_rid.size, it->current_hashBucket, max_bucket(headerHashBits, headerNextSplit)); fflush(NULL); + inBucket = 1; + } else { + found = 1; + printf("bbb {%d, %d, %d} {%d, %d, %d} %d %d\n", e->next.page, e->next.slot, e->next.size, it->current_rid.page, it->current_rid.slot, it->current_rid.size, it->current_hashBucket, max_bucket(headerHashBits, headerNextSplit)); fflush(NULL); + break; + } // else, it stays NULLRID. } - if(! it->current_hashBucket <= max_bucket(headerHashBits, headerNextSplit)) { + if(it->current_hashBucket > max_bucket(headerHashBits, headerNextSplit)) { p.key = NULL; p.value = NULL; - memcpy(&(it->current_rid), &(NULLRID), sizeof(recordid)); + it->current_rid = NULLRID; +// memcpy(&(it->current_rid), &(NULLRID), sizeof(recordid)); it->current_hashBucket = 0; } else { - p.key = memcpy(malloc(keySize), (e+1), keySize); +// Tread(xid, e->next, e); + p.key = memcpy(malloc(keySize), e+1, keySize); p.value = memcpy(malloc(valSize), ((byte*)(e+1))+keySize, valSize); it->current_rid = e->next; } diff --git a/src/lladd/operations/linearHashNTA.c b/src/lladd/operations/linearHashNTA.c new file mode 100644 index 0000000..8f0d893 --- /dev/null +++ b/src/lladd/operations/linearHashNTA.c @@ -0,0 +1,139 @@ +#include +#include +#include +#include + + +typedef struct { + recordid buckets; + int keySize; + int valueSize; + long nextSplit; + int bits; + long numEntries; +} lladd_hash_header; + + + +/* private methods... */ +static void ThashSplitBucket(int xid, recordid hashHeader, lladd_hash_header * lhh); + + +#define HASH_INIT_ARRAY_LIST_COUNT twoToThe(HASH_INIT_BITS) +#define HASH_INIT_ARRAY_LIST_MULT 2 + +recordid ThashCreate(int xid, int keySize, int valueSize) { + recordid hashHeader = Talloc(xid, sizeof(lladd_hash_header)); + lladd_hash_header lhh; + lhh.buckets = TarrayListAlloc(xid, HASH_INIT_ARRAY_LIST_COUNT, HASH_INIT_ARRAY_LIST_MULT, sizeof(lladd_linkedList_entry) + keySize + valueSize); + TarrayListExtend(xid, lhh.buckets, HASH_INIT_ARRAY_LIST_COUNT); + int i; + byte * entry = calloc(1, sizeof(lhh.buckets)); + recordid bucket = lhh.buckets; + for(i = 0; i < HASH_INIT_ARRAY_LIST_COUNT; i++) { + bucket.slot = i; + Tset(xid, bucket, entry); + } + free (entry); + lhh.keySize = keySize; + lhh.valueSize = valueSize; + lhh.nextSplit = 0; + lhh.bits = HASH_INIT_BITS; + lhh.numEntries = 0; + + Tset(xid, hashHeader, &lhh); + return hashHeader; +} + +void ThashDelete(int xid, recordid hash) { + abort(); +} + + +int ThashInsert(int xid, recordid hashHeader, const byte* key, int keySize, const byte* value, int valueSize) { + lladd_hash_header lhh; + Tread(xid, hashHeader, &lhh); + + lhh.numEntries ++; + + if(lhh.numEntries > (int)((double)(lhh.nextSplit + twoToThe(lhh.bits-1)) * HASH_FILL_FACTOR)) { + ThashSplitBucket(xid, hashHeader, &lhh); + } + + Tset(xid, hashHeader, &lhh); + + assert(lhh.keySize == keySize); assert(lhh.valueSize == valueSize); + + recordid bucket = lhh.buckets; + bucket.slot = hash(key, keySize, lhh.bits, lhh.nextSplit); + + int ret = TlinkedListInsert(xid, bucket, key, keySize, value, valueSize); + + return ret; +} + +int ThashRemove(int xid, recordid hashHeader, const byte * key, int keySize) { + lladd_hash_header lhh; + Tread(xid, hashHeader, &lhh); + + lhh.numEntries--; + + Tset(xid, hashHeader, &lhh); + + assert(lhh.keySize == keySize); + + recordid bucket = lhh.buckets; + bucket.slot = hash(key, keySize, lhh.bits, lhh.nextSplit); + + int ret = TlinkedListRemove(xid, bucket, key, keySize); + + return ret; +} + +int ThashLookup(int xid, recordid hashHeader, const byte * key, int keySize, byte ** value) { + lladd_hash_header lhh; + Tread(xid, hashHeader, &lhh); + + assert(lhh.keySize == keySize); + + recordid bucket = lhh.buckets; + bucket.slot = hash(key, keySize, lhh.bits, lhh.nextSplit); + + int ret = TlinkedListFind(xid, bucket, key, keySize, value); + + return ret; +} +/** @todo Write ThashSplitBucket */ +static void ThashSplitBucket(int xid, recordid hashHeader, lladd_hash_header * lhh) { + long old_bucket = lhh->nextSplit; + long new_bucket = old_bucket + twoToThe(lhh->bits-1); + recordid old_bucket_rid = lhh->buckets; + recordid new_bucket_rid = lhh->buckets; + old_bucket_rid.slot = old_bucket; + new_bucket_rid.slot = new_bucket; + + TarrayListExtend(xid, lhh->buckets, 1); + byte * entry = calloc(1, lhh->buckets.size); + Tset(xid, new_bucket_rid, entry); + free(entry); + if(lhh->nextSplit < twoToThe(lhh->bits-1)-1) { + lhh->nextSplit++; + } else { + lhh->nextSplit = 0; + lhh->bits++; + } + lladd_linkedList_iterator * it = TlinkedListIterator(xid, old_bucket_rid, lhh->keySize, lhh->valueSize); + byte * key, *value; + int keySize, valueSize; + while(TlinkedListNext(xid, it, &key, &keySize, &value, &valueSize)) { + assert(valueSize == lhh->valueSize); + assert(keySize == lhh->keySize); + if(hash(key, keySize, lhh->bits, lhh->nextSplit) != old_bucket) { + TlinkedListRemove(xid, old_bucket_rid, key, keySize); + TlinkedListInsert(xid, new_bucket_rid, key, keySize, value, valueSize); + } + free(key); + free(value); + } + return; +} diff --git a/src/lladd/operations/linkedListNTA.c b/src/lladd/operations/linkedListNTA.c new file mode 100644 index 0000000..1eb3476 --- /dev/null +++ b/src/lladd/operations/linkedListNTA.c @@ -0,0 +1,233 @@ +#include +#include +#include +#include +#include +/** A quick note on the format of linked lists. Each entry consists + of a struct with some variable length data appended to it. + + To access an entry's contents: + + lladd_linkedList_entry * entry; + ... + if(entry->size) { + key = (byte*)(entry + 1); + value = ((byte*)(entry+1)) + keySize; + } else { + entry->size must be nonzero if the entry is defined. It will be + zero if the entry is uniniailized (this can be the case if the + list has not yet been initialized. The end of the list is marked + by a next field with size -1. + } + + To get the successor in the list: + + lladd_linkedList_entry next = entry->next; + + + + @file +*/ +int TlinkedListInsert(int xid, recordid list, const byte * key, int keySize, const byte * value, int valueSize) { + int ret = TlinkedListRemove(xid, list, key, keySize); + lladd_linkedList_entry * entry = malloc(sizeof(lladd_linkedList_entry) + keySize + valueSize); + Tread(xid, list, entry); + if(!entry->next.size) { + memcpy(entry+1, key, keySize); + memcpy(((byte*)(entry+1))+keySize, value, valueSize); + entry->next.page = 0; + entry->next.slot = 0; + entry->next.size = -1; + Tset(xid, list, entry); + } else { + lladd_linkedList_entry * newEntry = malloc(sizeof(lladd_linkedList_entry) + keySize + valueSize); + memcpy(newEntry + 1, key, keySize); + memcpy(((byte*)(newEntry+1))+keySize, value, valueSize); + newEntry->next = entry->next; + recordid newRid = Talloc(xid, sizeof(lladd_linkedList_entry) + keySize + valueSize); + Tset(xid, newRid, newEntry); + entry->next = newRid; + Tset(xid, list, entry); + free(newEntry); + } + free(entry); + return ret; +} + +int TlinkedListFind(int xid, recordid list, const byte * key, int keySize, byte ** value) { + lladd_linkedList_entry * entry = malloc(list.size); + Tread(xid, list, entry); + + if(!entry->next.size) { + free(entry); + return -1; // empty list + } + while(1) { + if(!memcmp(entry + 1, key, keySize)) { + // Bucket contains the entry of interest. + int valueSize = list.size - (sizeof(lladd_linkedList_entry) + keySize); + *value = malloc(valueSize); + memcpy(*value, ((byte*)(entry+1))+keySize, valueSize); + free(entry); + return valueSize; + } + if(entry->next.size != -1) { + assert(entry->next.size == list.size); // Don't handle lists with variable length records for now + Tread(xid, entry->next, entry); + } else { + break; + } + } + free(entry); + return -1; +} +int TlinkedListRemove(int xid, recordid list, const byte * key, int keySize) { + lladd_linkedList_entry * entry = malloc(list.size); + + Tread(xid, list, entry); + if(entry->next.size == 0) { + //Empty List. + free(entry); + return 0; + } + int listRoot = 1; + recordid lastRead = list; + recordid oldLastRead; + oldLastRead.size = -2; + while(1) { + if(!memcmp(entry + 1, key, keySize)) { + // Bucket contains the entry of interest. + if(listRoot) { + if(entry->next.size == -1) { + memset(entry, 0, list.size); + Tset(xid, lastRead, entry); + } else { + assert(entry->next.size == list.size); // Otherwise, sometihng strange is happening, or the list contains entries with variable sizes. + lladd_linkedList_entry * entry2 = malloc(list.size); + Tread(xid, entry->next, entry2); + Tdealloc(xid, entry->next); // could break iterator, since it writes one entry ahead. + Tset(xid, lastRead, entry2); + free(entry2); + } + } else { + lladd_linkedList_entry * entry2 = malloc(list.size); + assert(oldLastRead.size != -2); + Tread(xid, oldLastRead, entry2); + memcpy(&(entry2->next), &(entry->next), sizeof(recordid)); + Tset(xid, oldLastRead, entry2); + Tdealloc(xid, lastRead); + free (entry2); + } + free(entry); + return 1; + } else { // Entry doesn't match the key we're looking for. + if(entry->next.size != -1) { + assert(entry->next.size == list.size); // Don't handle lists with variable length records for now + oldLastRead = lastRead; + lastRead = entry->next; + Tread(xid, entry->next, entry); + listRoot = 0; + } else { + break; + } + } + } + free(entry); + return 0; +} +/*** @todo TlinkedListMove could be much faster, but this is good enough for a first pass */ +int TlinkedListMove(int xid, recordid start_list, recordid end_list, const byte *key, int keySize) { + byte * value; + int valueSize = TlinkedListFind(xid, start_list, key, keySize, &value); + if(valueSize == -1) { + return 0; + } else { + TlinkedListRemove(xid, start_list, key, keySize); + TlinkedListInsert(xid, end_list, key, keySize, value, valueSize); + return 1; + } +} +recordid TlinkedListCreate(int xid, int keySize, int valueSize) { + recordid ret = Talloc(xid, sizeof(lladd_linkedList_entry) + keySize + valueSize); + byte * cleared = calloc(sizeof(lladd_linkedList_entry) + keySize + valueSize, sizeof(byte)); + Tset(xid, ret, cleared); + free(cleared); + return ret; +} +void TlinkedListDelete(int xid, recordid list) { + lladd_linkedList_entry * entry = malloc(list.size); + + Tread(xid, list, entry); + Tdealloc(xid, list); + + if(entry->next.size == 0) { + return; + } + + while(entry->next.size != -1) { + recordid nextEntry; + Tread(xid, nextEntry, entry); + assert(!memcmp(&nextEntry, &(entry->next), sizeof(recordid))); + Tdealloc(xid, nextEntry); + } + + free(entry); +} + +lladd_linkedList_iterator * TlinkedListIterator(int xid, recordid list, int keySize, int valueSize) { + lladd_linkedList_iterator * it = malloc(sizeof(lladd_linkedList_iterator)); + it->keySize = keySize; + it->valueSize = valueSize; + it->next = list; + it->first = -1; + it->listRoot = list; + return it; +} + +int TlinkedListNext(int xid, lladd_linkedList_iterator * it, byte ** key, int * keySize, byte **value, int * valueSize) { + + if(it->next.size == -1) { free(it); return 0; } + + if(it->first == -1) { + it->first = 1; + } else if(it->first) { + lladd_linkedList_entry * entry = malloc(it->next.size); + Tread(xid, it->listRoot, entry); + int listTouched; + listTouched = memcmp(&(entry->next), &(it->next), sizeof(recordid)); + free(entry); + if(listTouched) { + //The root entry was removed. Reset the iterator. + it->first = -1; + it->next = it->listRoot; + return TlinkedListNext(xid, it, key, keySize, value, valueSize); + } else { + //continue as normal. + it->first = 0; + } + } + + assert(it->keySize + it->valueSize + sizeof(lladd_linkedList_entry) == it->next.size); + + lladd_linkedList_entry * entry = malloc(it->next.size); + Tread(xid, it->next, entry); + if(entry->next.size) { + *keySize = it->keySize; + *valueSize = it->valueSize; + *key = malloc(*keySize); + *value = malloc(*valueSize); + + it->next = entry->next; + + memcpy(*key, entry+1, *keySize); + memcpy(*value, ((byte*)(entry + 1))+*keySize, *valueSize); + + free(entry); + return 1; + } else { + // This entry was empty (this case occurs with empty lists) + free(it); + free(entry); + return 0; + } +} diff --git a/test/lladd/Makefile.am b/test/lladd/Makefile.am index b9983cb..61afc6b 100644 --- a/test/lladd/Makefile.am +++ b/test/lladd/Makefile.am @@ -1,12 +1,11 @@ INCLUDES = @CHECK_CFLAGS@ if HAVE_CHECK ## Had to disable check_lht because lht needs to be rewritten. -TESTS = check_logEntry check_logWriter check_page check_operations check_transactional2 check_recovery check_blobRecovery check_bufferManager check_indirect check_lladdhash check_pageOperations check_linearHash check_logicalLinearHash check_header +TESTS = check_logEntry check_logWriter check_page check_operations check_transactional2 check_recovery check_blobRecovery check_bufferManager check_indirect check_lladdhash check_pageOperations check_linearHash check_logicalLinearHash check_header check_linkedListNTA check_linearHashNTA else TESTS = endif noinst_PROGRAMS = $(TESTS) LDADD = @CHECK_LIBS@ $(top_builddir)/src/lladd/liblladd.a $(top_builddir)/src/pbl/libpbl.a $(top_builddir)/src/libdfa/librw.a #-lefence -CLEANFILES = check_lht.log check_logEntry.log storefile.txt logfile.txt blob0_file.txt blob1_file.txt check_blobRecovery.log check_logWriter.log check_operations.log check_recovery.log check_transactional2.log check_page.log check_bufferManager.log check_indirect.log check_bufferMananger.log check_lladdhash.log check_pageOperations.log check_linearhash.log +CLEANFILES = check_lht.log check_logEntry.log storefile.txt logfile.txt blob0_file.txt blob1_file.txt check_blobRecovery.log check_logWriter.log check_operations.log check_recovery.log check_transactional2.log check_page.log check_bufferManager.log check_indirect.log check_bufferMananger.log check_lladdhash.log check_pageOperations.log check_linearhash.log check_linkedListNTA.log check_linearHashNTA.log AM_CFLAGS= -g -Wall -pedantic -std=gnu99 - diff --git a/test/lladd/check_linearHash.c b/test/lladd/check_linearHash.c index f8af65e..a67abc7 100644 --- a/test/lladd/check_linearHash.c +++ b/test/lladd/check_linearHash.c @@ -57,7 +57,7 @@ terms specified in this license. executes each of the insert / remove / lookup operations a few times. */ //#define NUM_ENTRIES 100000 -#define NUM_ENTRIES 2001 +#define NUM_ENTRIES 10000 /* #define NUM_ENTRIES 1000 */ /*#define NUM_ENTRIES 100 */ @@ -278,6 +278,8 @@ START_TEST(transactionalLinearHashTest) } END_TEST + + Suite * check_suite(void) { Suite *s = suite_create("linearHash"); /* Begin a new test */ diff --git a/test/lladd/check_linearHashNTA.c b/test/lladd/check_linearHashNTA.c new file mode 100644 index 0000000..0343638 --- /dev/null +++ b/test/lladd/check_linearHashNTA.c @@ -0,0 +1,130 @@ +/*--- +This software is copyrighted by the Regents of the University of +California, and other parties. The following terms apply to all files +associated with the software unless explicitly disclaimed in +individual files. + +The authors hereby grant permission to use, copy, modify, distribute, +and license this software and its documentation for any purpose, +provided that existing copyright notices are retained in all copies +and that this notice is included verbatim in any distributions. No +written agreement, license, or royalty fee is required for any of the +authorized uses. Modifications to this software may be copyrighted by +their authors and need not follow the licensing terms described here, +provided that the new terms are clearly indicated on the first page of +each file where they apply. + +IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY +FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES +ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY +DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +NON-INFRINGEMENT. THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, AND +THE AUTHORS AND DISTRIBUTORS HAVE NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +GOVERNMENT USE: If you are acquiring this software on behalf of the +U.S. government, the Government shall have only "Restricted Rights" in +the software and related documentation as defined in the Federal +Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you are +acquiring the software on behalf of the Department of Defense, the +software shall be classified as "Commercial Computer Software" and the +Government shall have only "Restricted Rights" as defined in Clause +252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the +authors grant the U.S. Government and others acting in its behalf +permission to use and distribute the software in accordance with the +terms specified in this license. +---*/ + +#include +#include +#include "../check_includes.h" + +#include + +#include +#include +#include + +#define LOG_NAME "check_linearHashNTA.log" +#define NUM_ENTRIES 100000 +START_TEST(linearHashNTAtest) +{ + Tinit(); + + int xid = Tbegin(); + recordid val; + recordid hashHeader = ThashCreate(xid, sizeof(int), sizeof(recordid)); + recordid * val2; + int i; + printf("\n"); fflush(stdout); + for(i = 0; i < NUM_ENTRIES; i++) { + if(!(i % (NUM_ENTRIES/10))) { + printf("."); fflush(stdout); + } + val.page = i * NUM_ENTRIES; + val.slot = val.page * NUM_ENTRIES; + val.size = val.slot * NUM_ENTRIES; + assert(-1 == ThashLookup(xid, hashHeader, (byte*)&i, sizeof(int), (byte**)&val2)); + ThashInsert(xid, hashHeader, (byte*)&i, sizeof(int), (byte*)&val, sizeof(recordid)); + assert(sizeof(recordid) == ThashLookup(xid, hashHeader, (byte*)&i, sizeof(int), (byte**)&val2)); + assert(val2->page == i * NUM_ENTRIES); + assert(val2->slot == val2->page * NUM_ENTRIES); + assert(val2->size == val2->slot * NUM_ENTRIES); + free(val2); + } + + Tcommit(xid); + printf("\n"); fflush(stdout); + + xid = Tbegin(); + for(i = 0; i < NUM_ENTRIES; i+=10){ + if(!(i % (NUM_ENTRIES/10))) { + printf("-"); fflush(stdout); + } + assert(sizeof(recordid) == ThashLookup(xid, hashHeader, (byte*)&i, sizeof(int), (byte**)&val2)); + free(val2); + assert(ThashRemove(xid, hashHeader, (byte*)&i, sizeof(int))); + assert(-1==ThashLookup(xid, hashHeader, (byte*)&i, sizeof(int), (byte**)&val2)); + assert(!ThashRemove(xid, hashHeader, (byte*)&i, sizeof(int))); + } + printf("\nabort()\n"); fflush(stdout); + Tabort(xid); + xid = Tbegin(); + for(i = 0; i < NUM_ENTRIES; i++) { + if(!(i % (NUM_ENTRIES/10))) { + printf("+"); fflush(stdout); + } + assert(sizeof(recordid) == ThashLookup(xid, hashHeader, (byte*)&i, sizeof(int), (byte**)&val2)); + assert(val2->page == i * NUM_ENTRIES); + assert(val2->slot == val2->page * NUM_ENTRIES); + assert(val2->size == val2->slot * NUM_ENTRIES); + free(val2); + } + Tcommit(xid); + Tdeinit(); +} END_TEST + +Suite * check_suite(void) { + Suite *s = suite_create("linearHashNTA"); + /* Begin a new test */ + TCase *tc = tcase_create("simple"); + + + /* Sub tests are added, one per line, here */ + + tcase_add_test(tc, linearHashNTAtest); + + /* --------------------------------------------- */ + + tcase_add_checked_fixture(tc, setup, teardown); + + suite_add_tcase(s, tc); + return s; +} + +#include "../check_setup.h" diff --git a/test/lladd/check_linkedListNTA.c b/test/lladd/check_linkedListNTA.c new file mode 100644 index 0000000..c91e2fe --- /dev/null +++ b/test/lladd/check_linkedListNTA.c @@ -0,0 +1,123 @@ +/*--- +This software is copyrighted by the Regents of the University of +California, and other parties. The following terms apply to all files +associated with the software unless explicitly disclaimed in +individual files. + +The authors hereby grant permission to use, copy, modify, distribute, +and license this software and its documentation for any purpose, +provided that existing copyright notices are retained in all copies +and that this notice is included verbatim in any distributions. No +written agreement, license, or royalty fee is required for any of the +authorized uses. Modifications to this software may be copyrighted by +their authors and need not follow the licensing terms described here, +provided that the new terms are clearly indicated on the first page of +each file where they apply. + +IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY +FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES +ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY +DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +NON-INFRINGEMENT. THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, AND +THE AUTHORS AND DISTRIBUTORS HAVE NO OBLIGATION TO PROVIDE +MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +GOVERNMENT USE: If you are acquiring this software on behalf of the +U.S. government, the Government shall have only "Restricted Rights" in +the software and related documentation as defined in the Federal +Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you are +acquiring the software on behalf of the Department of Defense, the +software shall be classified as "Commercial Computer Software" and the +Government shall have only "Restricted Rights" as defined in Clause +252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the +authors grant the U.S. Government and others acting in its behalf +permission to use and distribute the software in accordance with the +terms specified in this license. +---*/ + +#include +#include +#include "../check_includes.h" + +#include + +#include +#include +#include + +#define LOG_NAME "check_linkedListNTA.log" +START_TEST(linkedListNTAtest) +{ + Tinit(); + + int xid = Tbegin(); + + recordid linkedList = TlinkedListCreate(xid, sizeof(int), sizeof(recordid)); + int i; + for(i = 0; i < 1000; i++) { + recordid val; + recordid * val2 = NULL; + val.page = i * 1000; + val.slot = val.page * 1000; + val.size = val.slot * 1000; + assert(-1==TlinkedListFind(xid, linkedList, (byte*)(&i), sizeof(int), (byte**)&val2)); + TlinkedListInsert(xid, linkedList, (byte*)&i, sizeof(int), (byte*)&val, sizeof(recordid)); + assert(sizeof(recordid)==TlinkedListFind(xid, linkedList, (byte*)(&i), sizeof(int), (byte**)&val2)); + assert(!memcmp(&val, val2, sizeof(recordid))); + free(val2); + } + + Tcommit(xid); + + xid = Tbegin(); + for(i = 0; i < 1000; i+=10) { + recordid * val2 = NULL; + assert(sizeof(recordid)==TlinkedListFind(xid, linkedList, (byte*)(&i), sizeof(int), (byte**)&val2)); + assert(val2->page == i * 1000); + assert(val2->slot == i * 1000 * 1000); + assert(val2->size == i * 1000 * 1000 * 1000); + free(val2); + assert(TlinkedListRemove(xid, linkedList, (byte*)&i, sizeof(int))); + assert(-1==TlinkedListFind(xid, linkedList, (byte*)(&i), sizeof(int), (byte**)&val2)); + assert(!TlinkedListRemove(xid, linkedList, (byte*)&i, sizeof(int))); + } + Tabort(xid); + xid = Tbegin(); + for(i = 0; i < 1000; i++) { + recordid * val2; + assert(sizeof(recordid)==TlinkedListFind(xid, linkedList, (byte*)(&i), sizeof(int), (byte**)&val2)); + assert(val2->page == i * 1000); + assert(val2->slot == i * 1000 * 1000); + assert(val2->size == i * 1000 * 1000 * 1000); + free(val2); + } + Tcommit(xid); + Tdeinit(); +} END_TEST + + + +Suite * check_suite(void) { + Suite *s = suite_create("linkedListNTA"); + /* Begin a new test */ + TCase *tc = tcase_create("simple"); + + + /* Sub tests are added, one per line, here */ + + tcase_add_test(tc, linkedListNTAtest); + + /* --------------------------------------------- */ + + tcase_add_checked_fixture(tc, setup, teardown); + + suite_add_tcase(s, tc); + return s; +} + +#include "../check_setup.h" diff --git a/test/lladd/check_logicalLinearHash.c b/test/lladd/check_logicalLinearHash.c index 3ec53cd..59ad2e2 100644 --- a/test/lladd/check_logicalLinearHash.c +++ b/test/lladd/check_logicalLinearHash.c @@ -57,8 +57,8 @@ terms specified in this license. executes each of the insert / remove / lookup operations a few times. */ //#define NUM_ENTRIES 100000 -#define NUM_ENTRIES 2001 -/* #define NUM_ENTRIES 1000 */ +/*#define NUM_ENTRIES 10000*/ +#define NUM_ENTRIES 1000 /*#define NUM_ENTRIES 100 */ /** @@ -200,7 +200,48 @@ START_TEST(simpleLinearHashTest) } END_TEST +#define NUM_ITERATOR_ENTRIES 2000 +START_TEST(check_linearHashIterator) { + Tinit(); + + int xid = Tbegin(); + + recordid rid = ThashAlloc(xid, sizeof(int), sizeof(int)); + + printf("Testing iterator.\n"); + int key; + int val; + + int * keySeen = calloc(NUM_ITERATOR_ENTRIES, sizeof(int)); + for(int i = 0; i < NUM_ITERATOR_ENTRIES; i++) { + key = i; + val = NUM_ITERATOR_ENTRIES * key; + TnaiveHashInsert(xid, rid, &key, sizeof(int), &val, sizeof(int)); + } + Tcommit(xid); + + xid = Tbegin(); + for(int i = 0; i < NUM_ITERATOR_ENTRIES; i++) { + key = i; + TlogicalHashLookup(xid, rid, &key, sizeof(int), &val, sizeof(int)); + assert(key == i); + assert(val == NUM_ITERATOR_ENTRIES * key); + } + Tcommit(xid); + xid = Tbegin(); + linearHash_iterator * it = TlogicalHashIterator(xid, rid); + + linearHash_iteratorPair next = TlogicalHashIteratorNext(xid,rid, it, sizeof(int), sizeof(int)); + assert(next.key ); + while(next.key != NULL) { + printf("%d -> %d\n", *(next.key), *(next.value)); + next = TlogicalHashIteratorNext(xid, rid, it, sizeof(int), sizeof(int)); + } + TlogicalHashIteratorFree(it); + Tcommit(xid); + Tdeinit(); +} END_TEST Suite * check_suite(void) { Suite *s = suite_create("linearHash"); /* Begin a new test */ @@ -211,7 +252,7 @@ Suite * check_suite(void) { /* tcase_add_test(tc, checkHashFcn); */ tcase_add_test(tc, simpleLinearHashTest); - + tcase_add_test(tc, check_linearHashIterator); /* --------------------------------------------- */ tcase_add_checked_fixture(tc, setup, teardown);