From 6a2d6a1cd407a73a6b0295cdd5e116c7ffebdf9e Mon Sep 17 00:00:00 2001 From: sears Date: Tue, 23 Feb 2010 17:05:47 +0000 Subject: [PATCH] Generalized comparator, fixed some bugs in record length handling. git-svn-id: svn+ssh://svn.corp.yahoo.com/yahoo/yrl/labs/pnuts/code/logstore@613 8dad8b1f-cf64-0410-95b6-bcf113ffbcfe --- datapage.cpp | 2 +- datatuple.h | 25 +++++++++++++++++++------ diskTreeComponent.cpp | 5 +++-- logserver.cpp | 14 ++++++++++---- merger.cpp | 4 ++-- 5 files changed, 35 insertions(+), 15 deletions(-) diff --git a/datapage.cpp b/datapage.cpp index 65c0e99..ad6056c 100644 --- a/datapage.cpp +++ b/datapage.cpp @@ -276,7 +276,7 @@ bool DataPage::recordRead(typename TUPLE::key_t key, size_t keySize, TUP int match = -1; while((*buf=itr.getnext()) != 0) { - match = TUPLE::compare((*buf)->key(), key); + match = TUPLE::compare((*buf)->key(), (*buf)->keylen(), key, keySize); if(match<0) //keep searching { diff --git a/datatuple.h b/datatuple.h index 8817e77..ee1b8ec 100644 --- a/datatuple.h +++ b/datatuple.h @@ -49,7 +49,7 @@ public: //this is used by the stl set bool operator() (const datatuple* lhs, const datatuple* rhs) const { - return compare(lhs->key(), rhs->key()) < 0; //strcmp((char*)lhs.key(),(char*)rhs.key()) < 0; + return compare(lhs->key(), lhs->keylen(), rhs->key(), rhs->keylen()) < 0; //strcmp((char*)lhs.key(),(char*)rhs.key()) < 0; } /** @@ -57,14 +57,27 @@ public: * 0 if k1 == k2 * 1 of k1 > k2 **/ - static int compare(const byte* k1,const byte* k2) { - // XXX string comparison is probably not the right approach. - //for char* ending with \0 - return strcmp((char*)k1,(char*)k2); + static int compare(const byte* k1,size_t k1l, const byte* k2, size_t k2l) { + // This function handles ASCII and UTF-8 correctly. + // It also handles a Sherpa LSM-Tree specific encoding, where multiple utf-8 strings + // are concatenated with \254., and the \254 is replaced with a \255 for 'max value in range'. + + size_t min_l = k1l < k2l ? k1l : k2l; + + int ret = memcmp(k1,k2, min_l); + if(ret) return ret; + if(k1l < k2l) return -1; + if(k1l == k2l) return 0; + return 1; + + //for testing with char* ending with \0 + /*assert(strlen((char*)k1) == k1l - 1); + assert(strlen((char*)k2) == k2l - 1); + return strcmp((char*)k1,(char*)k2); */ } static int compare_obj(const datatuple * a, const datatuple* b) { - return compare(a->key(), b->key()); + return compare(a->key(), a->keylen(), b->key(), b->keylen()); } inline void setDelete() { diff --git a/diskTreeComponent.cpp b/diskTreeComponent.cpp index e1da97e..10b3f9c 100644 --- a/diskTreeComponent.cpp +++ b/diskTreeComponent.cpp @@ -726,7 +726,8 @@ recordid diskTreeComponent::lookup(int xid, for(int i = FIRST_SLOT+1; i < *stasis_page_slotted_numslots_ptr(node); i++) { rec = (const indexnode_rec*)readRecord(xid,node,i,0); - int cmpval = datatuple::compare((datatuple::key_t) (rec+1),(datatuple::key_t) key); + int cmpval = datatuple::compare((datatuple::key_t) (rec+1), *stasis_page_slotted_slot_length_ptr(node, i)-sizeof(*rec), + (datatuple::key_t) key, keySize); if(cmpval>0) //changed it from > break; match = i; @@ -738,7 +739,7 @@ recordid diskTreeComponent::lookup(int xid, pageid_t child_id = ((const indexnode_rec*)readRecord(xid,node,match,0))->ptr; Page* child_page = loadPage(xid, child_id); readlock(child_page->rwlatch,0); - recordid ret = lookup(xid,child_page,depth-1,key,0); + recordid ret = lookup(xid,child_page,depth-1,key,keySize); unlock(child_page->rwlatch); releasePage(child_page); return ret; diff --git a/logserver.cpp b/logserver.cpp index 7294716..c89f25e 100644 --- a/logserver.cpp +++ b/logserver.cpp @@ -263,6 +263,7 @@ void logserver::eventLoop() //wake up the thread to do work pthread_mutex_lock(idle_th.data->th_mut); //set the job of the idle thread + assert(currsock != -1); *(idle_th.data->workitem) = currsock; pthread_cond_signal(idle_th.data->th_cond); pthread_mutex_unlock(idle_th.data->th_mut); @@ -362,9 +363,10 @@ void *serverLoop(void *args) //insert the given element to the ready queue sdata->ready_queue->push(newsockfd); - if(sdata->ready_queue->size() == 1) //signal the event loop + /* if(sdata->ready_queue->size() == 1) //signal the event loop pthread_cond_signal(sdata->selcond); - + else */ if(sdata->ready_queue->size()) //in case multiple events were received in race. + pthread_cond_broadcast(sdata->selcond); pthread_mutex_unlock(sdata->qlock); } } @@ -445,6 +447,10 @@ void * thread_work_fn( void * args) if(!err) { tuple2 = readtuplefromsocket(*(item->data->workitem), &err); } //step 3: process the tuple + if(tuple) { + printf("Tuple req = %d key = >%s<\n", opcode, tuple->key()); + } + if(opcode == OP_INSERT) { //insert/update/delete @@ -464,7 +470,7 @@ void * thread_work_fn( void * args) } else if( dt->datalen() != 1024) { DEBUG("data len for\t%s:\t%d\n", datatuple::key_to_str(tuple.key()).c_str(), dt->datalen); - if(datatuple::compare(tuple->key(), dt->key()) != 0) { + if(datatuple::compare(tuple->key(), tuple->keylen(), dt->key(), dt->keylen()) != 0) { DEBUG("key not equal:\t%s\t%s\n", datatuple::key_to_str(tuple.key()).c_str(), datatuple::key_to_str(dt->key).c_str()); } @@ -481,7 +487,7 @@ void * thread_work_fn( void * args) } else { dt_needs_free = true; } - + fprintf(stderr, "find result: %s\n", dt->isDelete() ? "not found" : "found"); //send the reply code int err = writeoptosocket(*(item->data->workitem), LOGSTORE_RESPONSE_SENDING_TUPLES); if(!err) { diff --git a/merger.cpp b/merger.cpp index 22af8a5..8485305 100644 --- a/merger.cpp +++ b/merger.cpp @@ -488,7 +488,7 @@ int64_t merge_iterators(int xid, DEBUG("tuple\t%lld: keylen %d datalen %d\n", ntuples, *(t2->keylen),*(t2->datalen) ); - while(t1 != 0 && datatuple::compare(t1->key(), t2->key()) < 0) // t1 is less than t2 + while(t1 != 0 && datatuple::compare(t1->key(), t1->keylen(), t2->key(), t2->keylen()) < 0) // t1 is less than t2 { //insert t1 dp = insertTuple(xid, dp, t1, ltable, scratch_tree, @@ -500,7 +500,7 @@ int64_t merge_iterators(int xid, t1 = itrA->getnext(); } - if(t1 != 0 && datatuple::compare(t1->key(), t2->key()) == 0) + if(t1 != 0 && datatuple::compare(t1->key(), t1->keylen(), t2->key(), t2->keylen()) == 0) { datatuple *mtuple = ltable->gettuplemerger()->merge(t1,t2);