From b26105e76fda062bfe2b4b2a57a95098d44d7e48 Mon Sep 17 00:00:00 2001 From: Sears Russell Date: Mon, 21 Mar 2005 08:03:45 +0000 Subject: [PATCH] Benchmark debugging output / scalability fixes. --- ChangeLog | 145 ++++++++++++++++++ benchmarks/berkeleyDB/bdbHashThreaded.c | 2 + benchmarks/berkeleyDB/bdbHashWriteRequests.c | 29 +++- benchmarks/berkeleyDB/genericBerkeleyDBCode.c | 48 +++--- benchmarks/linearHashNTAThreaded.c | 9 +- benchmarks/linearHashNTAWriteRequests.c | 20 ++- 6 files changed, 219 insertions(+), 34 deletions(-) diff --git a/ChangeLog b/ChangeLog index ba55a86..a667299 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,148 @@ +2005-03-14 sears@euglenoid + + * linearHashNTA.h: + This commit breaks the build so that Jimmy and I can sync. Sorry. :( + +2005-03-10 sears@euglenoid + + * linearHashNTA.h: + Fixed unit tests, added iterator interface, and generic (almost) iterator test harness. + +2005-03-02 sears@euglenoid + + * linearHashNTA.h, linkedListNTA.h: Compiles on FreeBSD! + +2005-03-01 sears@euglenoid + + * noop.h: Bugfixes for OASYS. + +2005-02-24 sears@euglenoid + + * alloc.h, pageOrientedListNTA.h: + Fixeds cht test / finished error checking within LLADD, but haven't tested error handling yet. + +2005-02-22 sears@euglenoid + + * linkedListNTA.h: Error handling for linked list. + + * linearHashNTA.h, pageOperations.h, pageOrientedListNTA.h, arrayList.h: + More error handling... + +2005-02-16 sears@euglenoid + + * linearHashNTA.h, set.h: Started adding error handling code to LLADD. + +2005-01-31 sears@euglenoid + + * pageOrientedListNTA.h: Bugfix for pageOrientedList. + +2005-01-29 sears@euglenoid + + * pageOrientedListNTA.h: + Re-implemented pageOrientedListNTA from scratch. It's now more flexible, but relies on features that are not provided by Talloc/Tdealloc, so performance isn't as good as it could be. + +2005-01-28 sears@euglenoid + + * linearHashNTA.h, pageOrientedListNTA.h: + Working, but slow and non-concurrent variable length key/value support for linear hashtable. + + * alloc.h, pageOrientedListNTA.h: + Implemented pageOrientedList, also did some bug fixing, and added some support functions to alloc.c + +2005-01-21 sears@euglenoid + + * linearHashNTA.h, prepare.h, alloc.h: More documentation updates. + +2005-01-20 sears@euglenoid + + * noop.h, pageOperations.h, prepare.h, set.h, increment.h, instantSet.h, linearHash.h, linearHashNTA.h, linkedListNTA.h, lladdhash.h, naiveLinearHash.h, nestedTopActions.h, alloc.h, arrayList.h, decrement.h: + Documentation update. + +2005-01-15 sears@euglenoid + + * linearHashNTA.h, nestedTopActions.h: + - linkedListNTA is now re-entrant. + - linearHashNTA now uses nested top actions, is re-entrant, and supports a + non-re-entrant iterator + - 'recursive' nested top actions allow nested top actions to be nested within each + other. + +2005-01-14 sears@euglenoid + + * linkedListNTA.h, nestedTopActions.h: + The linked list implementation now uses nested top actions. (Also, many bugfixes regarding nested top actions were introduced.) + + * linearHash.h, linearHashNTA.h, linkedListNTA.h, naiveLinearHash.h: + Working, non-concurrent linear hash and linked list implementations. + The idea is to implement the non-concurrent versions, and then convert + them to concurrent versions using nested top actions. + +2005-01-10 sears@euglenoid + + * naiveLinearHash.h: + Starting to rip the old jbhash implementation out, and replace it with a + linear hash implementation. In particular, I've implemented (sort of) an + iterator for linear hash, and (sort of) ported libdfa to linearhash.c + + I say 'sort of' since this functionality is broken with this commit. On the other hand, CVS should build now, and the tests under lladd will pass. libdfa's new tests don't pass at the moment. + +2004-12-03 sears@euglenoid + + * set.h: Implemented TsetRange, fixed bug in linear hash. + +2004-12-01 sears@euglenoid + + * alloc.h: Implemented TrecordType() + +2004-11-29 sears@euglenoid + + * nestedTopActions.h: + Support for nested top actions, with simple unit test. (Haven't tested for thread safety yet.) + + * naiveLinearHash.h: + Implemented recover_split(), which is needed by the linear hash to recover properly from crashes. It hasn't been tested at all, and is not currently called by anything. + +2004-11-28 sears@euglenoid + + * naiveLinearHash.h: Did some misc. cleanup work on linearHash + +2004-10-20 sears@euglenoid + + * linearHash.h: + Starting work on the actual graph-generation portion of the benchmarks. :) + +2004-10-19 sears@euglenoid + + * arrayList.h, linearHash.h, naiveLinearHash.h: + Lots of optimizations; re-did on-disk layout for the linear hash table. + +2004-10-18 sears@euglenoid + + * arrayList.h, linearHash.h, naiveLinearHash.h: + Preliminary version of logical logging linear hash. (No latching yet, and there are some bugs re-opening a closed hash.) + +2004-10-17 sears@euglenoid + + * linearHash.h: + Optimized the 'naive' linear hash table, and moved to sourceforge. :) + +2004-10-12 sears@euglenoid + + * linearHash.h: + Implemented a linear hash from scratch. Concurrency hasn't been done, but it's probably correct in the single thread case. + +2004-10-06 sears@euglenoid + + * arrayList.h: + Implemented pages that store fixed sized records efficiently, and a java-style ArrayList data structure that efficiently supports (relatively) clustered, O(1) access time expandable arrays. (This will be used for the hash implementation...) + +2004-10-02 sears@euglenoid + + * alloc.h, instantSet.h, noop.h, prepare.h: + Added test cases for Tprepare(), implemented some redo-only operations, and started to clean up record allocation/deallocation. + + Also, numerous bugfixes. + 2004-10-02 sears@euglenoid * alloc.h, instantSet.h, noop.h, prepare.h: diff --git a/benchmarks/berkeleyDB/bdbHashThreaded.c b/benchmarks/berkeleyDB/bdbHashThreaded.c index 72fb543..3fec15f 100644 --- a/benchmarks/berkeleyDB/bdbHashThreaded.c +++ b/benchmarks/berkeleyDB/bdbHashThreaded.c @@ -107,6 +107,8 @@ main(int argc, char *argv[]) db->close(db, 0); dbenv->close(dbenv, 0); + printf("committed %d times, put %d times\n", commitCount, putCount); + return (0); } diff --git a/benchmarks/berkeleyDB/bdbHashWriteRequests.c b/benchmarks/berkeleyDB/bdbHashWriteRequests.c index 81728e4..fdb3d5f 100644 --- a/benchmarks/berkeleyDB/bdbHashWriteRequests.c +++ b/benchmarks/berkeleyDB/bdbHashWriteRequests.c @@ -31,6 +31,20 @@ int max_active = 0; pthread_cond_t never; pthread_mutex_t mutex; +void addTimespec(struct timespec * ts, long nsec) { + ts->tv_nsec += nsec; + + // 0123456789 + if(ts->tv_nsec > 1000000000) { + ts->tv_nsec -= 1000000000; + ts->tv_sec ++; + } +} + + +double thread_requests_per_sec = 10.0; +int alwaysCommit; + int num_xact; int insert_per_xact; void * runThread(void * arg); @@ -43,7 +57,7 @@ main(int argc, char *argv[]) assert(argc == 3 || argc == 4); - int alwaysCommit = (argc == 4); + alwaysCommit = (argc == 4); /* threads have static thread sizes. Ughh. */ pthread_attr_t attr; @@ -65,11 +79,15 @@ main(int argc, char *argv[]) int r; int num_threads = atoi(argv[1]); + thread_requests_per_sec = (double) atoi(argv[2]); + + printf("%d %f\n", num_threads, thread_requests_per_sec); if(alwaysCommit) { - num_xact = atoi(argv[2]); + num_xact = 10.0 * thread_requests_per_sec;//atoi(argv[2]); insert_per_xact = 1; } else { + abort(); num_xact = 1; insert_per_xact = atoi(argv[2]); } @@ -99,10 +117,12 @@ main(int argc, char *argv[]) int k; double log_multiplier = (COUNTER_RESOLUTION / log(MAX_SECONDS * 1000000000.0)); + int total = 0; for(k = 0; k < COUNTER_RESOLUTION; k++) { printf("%3.4f\t%d\n", exp(((double)k)/log_multiplier)/1000000000.0, buckets[k]); + total += buckets[k]; } - + printf("Total requests: %d\n", total); db->close(db, 0); dbenv->close(dbenv, 0); @@ -170,8 +190,9 @@ void * runThread(void * arg) { if(bucket >= COUNTER_RESOLUTION) { bucket = COUNTER_RESOLUTION - 1; } - timeout.tv_sec++; + addTimespec(&timeout, 1000000000.0 / thread_requests_per_sec); pthread_mutex_lock(&mutex); + // timeout.tv_sec++; buckets[bucket]++; pthread_cond_timedwait(&never, &mutex, &timeout); pthread_mutex_unlock(&mutex); diff --git a/benchmarks/berkeleyDB/genericBerkeleyDBCode.c b/benchmarks/berkeleyDB/genericBerkeleyDBCode.c index 28df1d3..8dbe119 100644 --- a/benchmarks/berkeleyDB/genericBerkeleyDBCode.c +++ b/benchmarks/berkeleyDB/genericBerkeleyDBCode.c @@ -2,6 +2,8 @@ DB_ENV *dbenv; DB *db_cats; +int commitCount = 0; +int putCount = 0; void env_dir_create() { @@ -271,36 +273,38 @@ retry: /* Begin the transaction. */ // pthread_mutex_lock(&hack); switch (ret = db->put(db, tid, &key, &data, 0)) { - case 0: - break; - case DB_LOCK_DEADLOCK: - // va_end(ap); - abort(); // Locking should be disabled! - /* Deadlock: retry the operation. */ - if ((ret = dbc->c_close(dbc)) != 0) { - dbenv->err( - dbenv, ret, "dbc->c_close"); - exit (1); + case 0: + break; + case DB_LOCK_DEADLOCK: + // va_end(ap); + abort(); // Locking should be disabled! + /* Deadlock: retry the operation. */ + if ((ret = dbc->c_close(dbc)) != 0) { + dbenv->err( + dbenv, ret, "dbc->c_close"); + exit (1); + } + if ((ret = tid->abort(tid)) != 0) { + dbenv->err(dbenv, ret, "DB_TXN->abort"); + exit (1); } - if ((ret = tid->abort(tid)) != 0) { - dbenv->err(dbenv, ret, "DB_TXN->abort"); - exit (1); - } - goto retry; - default: - /* Error: run recovery. */ - dbenv->err(dbenv, ret, "dbc->put: %d/%d", q, q); - abort(); // Error invalidates benchmark! - exit (1); - } + goto retry; + default: + /* Error: run recovery. */ + dbenv->err(dbenv, ret, "dbc->put: %d/%d", q, q); + abort(); // Error invalidates benchmark! + exit (1); + } // pthread_mutex_unlock(&hack); - + putCount++; } + if ((ret = tid->commit(tid, 0)) != 0) { dbenv->err(dbenv, ret, "DB_TXN->commit"); exit (1); } + commitCount++; #ifdef DEBUG_BDB printf("Called commit\n"); #endif diff --git a/benchmarks/linearHashNTAThreaded.c b/benchmarks/linearHashNTAThreaded.c index 273ecec..69ca409 100644 --- a/benchmarks/linearHashNTAThreaded.c +++ b/benchmarks/linearHashNTAThreaded.c @@ -21,6 +21,10 @@ int activeThreads = 0; int max_active = 0; int alwaysCommit; + +int commitCount = 0; +int putCount = 0; + /* double avg_var = 0; double max_var = 0; @@ -57,7 +61,7 @@ static void * go (void * arg_ptr) { // gettimeofday(&start, NULL); ThashInsert(xid, hash, (byte*)&j, sizeof(int), (byte*)&j, sizeof(int)); - + putCount++; /* gettimeofday(&endtime, NULL); double microSecondsPassed = 1000000 * (endtime.tv_sec - start.tv_sec); @@ -72,6 +76,7 @@ static void * go (void * arg_ptr) { if(alwaysCommit) { // printf("Commit"); + commitCount++; Tcommit(xid); xid = Tbegin(); @@ -161,4 +166,6 @@ int main(int argc, char** argv) { } Tdeinit(); + + printf("Committed %d times, put %d times.\n", commitCount, putCount); } diff --git a/benchmarks/linearHashNTAWriteRequests.c b/benchmarks/linearHashNTAWriteRequests.c index 242d28e..78aad6c 100644 --- a/benchmarks/linearHashNTAWriteRequests.c +++ b/benchmarks/linearHashNTAWriteRequests.c @@ -11,7 +11,7 @@ pthread_cond_t never; pthread_mutex_t mutex; -double thread_requests_per_sec = 1.0; +double thread_requests_per_sec = 10.0; int alwaysCommit; #define MAX_SECONDS 100 @@ -190,15 +190,19 @@ int main(int argc, char** argv) { assert(argc == 4); - int thread_count = atoi(argv[1]); - count = atoi(argv[2]); + int thread_count = atoi(argv[1]) ; + // count = atoi(argv[2]); + thread_requests_per_sec = (double)(atoi(argv[2])); + count = 10.0 * thread_requests_per_sec; alwaysCommit = atoi(argv[3]); - unlink("storefile.txt"); + printf("%d %f\n", thread_count, thread_requests_per_sec); + + /* unlink("storefile.txt"); unlink("logfile.txt"); unlink("blob0_file.txt"); - unlink("blob1_file.txt"); + unlink("blob1_file.txt"); */ int l; @@ -243,12 +247,14 @@ int main(int argc, char** argv) { double log_multiplier = (COUNTER_RESOLUTION / log(MAX_SECONDS * 1000000000.0)); - + int total = 0; for(k = 0; k < COUNTER_RESOLUTION; k++) { printf("%3.4f\t%d\n", exp(((double)k)/log_multiplier)/1000000000.0, buckets[k]); + total += buckets[k]; } - + + printf("Total requests %d\n", total); /* printf("mean: (max, avg) %f, %f\n", max_mean, avg_mean / (double)thread_count); printf("variance: (max, avg) %f, %f\n", max_var, avg_var / (double)thread_count); */