greg/stasis-bLSM
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

remove ltable->tsize and ltable->tree_bytes fields

Browse source 
git-svn-id: svn+ssh://svn.corp.yahoo.com/yahoo/yrl/labs/pnuts/code/logstore@1507 8dad8b1f-cf64-0410-95b6-bcf113ffbcfe
This commit is contained in:
sears 2010-12-14 23:06:21 +00:00
parent 7e2d12f6ba
commit cd27f50baf
6 changed files with 16 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
logstore.cpp
View file

@ -48,9 +48,6 @@ logtable<TUPLE>::logtable(pageid_t max_c0_size, pageid_t internal_region_size, p
pthread_cond_init(&c1_needed, 0); pthread_cond_init(&c1_needed, 0);
pthread_cond_init(&c1_ready, 0); pthread_cond_init(&c1_ready, 0);
tsize = 0;
tree_bytes = 0;
epoch = 0; epoch = 0;
this->internal_region_size = internal_region_size; this->internal_region_size = internal_region_size;
@ -88,8 +85,8 @@ void logtable<TUPLE>::init_stasis() {
DataPage<datatuple>::register_stasis_page_impl(); DataPage<datatuple>::register_stasis_page_impl();
//stasis_buffer_manager_size = 768 * 1024; // 4GB = 2^10 pages: //stasis_buffer_manager_size = 768 * 1024; // 4GB = 2^10 pages:
// XXX Workaround Stasis' (still broken) default concurrent buffer manager // XXX Workaround Stasis' (still broken) default concurrent buffer manager
// stasis_buffer_manager_factory = stasis_buffer_manager_hash_factory; stasis_buffer_manager_factory = stasis_buffer_manager_hash_factory;
// stasis_buffer_manager_hint_writes_are_sequential = 0; stasis_buffer_manager_hint_writes_are_sequential = 0;
Tinit(); Tinit();
} }
@ -191,8 +188,7 @@ void logtable<TUPLE>::flushTable()
merge_count ++; merge_count ++;
merge_mgr->get_merge_stats(0)->starting_merge(); merge_mgr->get_merge_stats(0)->starting_merge();
tsize = 0; merge_mgr->get_merge_stats(0)->current_size = 0;
tree_bytes = 0;
if(blocked && stop - start > 1.0) { if(blocked && stop - start > 1.0) {
if(first) if(first)
@ -497,7 +493,7 @@ datatuple * logtable<TUPLE>::insertTupleHelper(datatuple *tuple)
tree_c0->insert(new_t); //insert the new tuple tree_c0->insert(new_t); //insert the new tuple
//update the tree size (+ new_t size - pre_t size) //update the tree size (+ new_t size - pre_t size)
tree_bytes += ((int64_t)new_t->byte_length() - (int64_t)pre_t->byte_length()); merge_mgr->get_merge_stats(0)->current_size += ((int64_t)new_t->byte_length() - (int64_t)pre_t->byte_length());
} }
else //no tuple with same key exists in mem-tree else //no tuple with same key exists in mem-tree
@ -508,8 +504,7 @@ datatuple * logtable<TUPLE>::insertTupleHelper(datatuple *tuple)
//insert tuple into the rbtree //insert tuple into the rbtree
tree_c0->insert(t); tree_c0->insert(t);
tsize++; merge_mgr->get_merge_stats(0)->current_size += t->byte_length();// + RB_TREE_OVERHEAD;
tree_bytes += t->byte_length();// + RB_TREE_OVERHEAD;
} }
merge_mgr->wrote_tuple(0, t); // needs to be here; doesn't grab a mutex. merge_mgr->wrote_tuple(0, t); // needs to be here; doesn't grab a mutex.

2
logstore.h
View file

@ -143,9 +143,7 @@ private:
memTreeComponent<datatuple>::rbtree_ptr_t tree_c0_mergeable; // in-mem red black tree: ready to be merged with c1. memTreeComponent<datatuple>::rbtree_ptr_t tree_c0_mergeable; // in-mem red black tree: ready to be merged with c1.
bool c0_is_merging; bool c0_is_merging;
int tsize; //number of tuples
public: public:
int64_t tree_bytes; //number of bytes
bool flushing; bool flushing;
//DATA PAGE SETTINGS //DATA PAGE SETTINGS

4
memTreeComponent.h
View file

@ -186,7 +186,7 @@ public:
} }
public: public:
batchedRevalidatingIterator( rbtree_t *s, int64_t* cur_size, int64_t target_size, bool * flushing, int batch_size, pthread_mutex_t * rb_mut ) : s_(s), cur_size_(cur_size), target_size_(target_size), flushing_(flushing), batch_size_(batch_size), num_batched_(batch_size), cur_off_(batch_size), mut_(rb_mut) { batchedRevalidatingIterator( rbtree_t *s, pageid_t* cur_size, int64_t target_size, bool * flushing, int batch_size, pthread_mutex_t * rb_mut ) : s_(s), cur_size_(cur_size), target_size_(target_size), flushing_(flushing), batch_size_(batch_size), num_batched_(batch_size), cur_off_(batch_size), mut_(rb_mut) {
next_ret_ = (TUPLE**)malloc(sizeof(next_ret_[0]) * batch_size_); next_ret_ = (TUPLE**)malloc(sizeof(next_ret_[0]) * batch_size_);
populate_next_ret(); populate_next_ret();
} }
@ -217,7 +217,7 @@ public:
rbtree_t *s_; rbtree_t *s_;
TUPLE ** next_ret_; TUPLE ** next_ret_;
int64_t* cur_size_; // a pointer to the current size of the red-black tree, in bytes. pageid_t* cur_size_; // a pointer to the current size of the red-black tree, in bytes.
int64_t target_size_; // the low-water size for the tree. If cur_size_ is not null, and *cur_size_ < C * target_size_, we sleep. int64_t target_size_; // the low-water size for the tree. If cur_size_ is not null, and *cur_size_ < C * target_size_, we sleep.
bool* flushing_; // never block if *flushing is true. bool* flushing_; // never block if *flushing is true.
int batch_size_; int batch_size_;

13
mergeManager.cpp
View file

@ -77,7 +77,7 @@ void mergeManager::update_progress(mergeStats * s, int delta) {
} }
if(s->merge_level == 0) { if(s->merge_level == 0) {
s->current_size = ltable->tree_bytes; // we need to track the number of bytes consumed by the merger; this data is not present in s, so fall back on ltable's aggregate. // ltable manages c0's current size directly.
} else { } else {
s->current_size = s->base_size + s->bytes_out - s->bytes_in_large; s->current_size = s->base_size + s->bytes_out - s->bytes_in_large;
} }
@ -163,16 +163,15 @@ void mergeManager::tick(mergeStats * s) {
// Simple backpressure algorithm based on how full C0 is. // Simple backpressure algorithm based on how full C0 is.
// Is C0 bigger than is allowed? // Is C0 bigger than is allowed?
while(ltable->tree_bytes > ltable->max_c0_size) { // can't use s->current_size, since this is the thread that maintains that number... while(c0->current_size > ltable->max_c0_size) { // can't use s->current_size, since this is the thread that maintains that number...
printf("\nMEMORY OVERRUN!!!! SLEEP!!!!\n"); printf("\nMEMORY OVERRUN!!!! SLEEP!!!!\n");
struct timespec ts; struct timespec ts;
double_to_ts(&ts, 0.1); double_to_ts(&ts, 0.1);
nanosleep(&ts, 0); nanosleep(&ts, 0);
} }
// Linear backpressure model // Linear backpressure model
s->current_size = ltable->tree_bytes; s->out_progress = ((double)c0->current_size)/((double)ltable->max_c0_size);
s->out_progress = ((double)ltable->tree_bytes)/((double)ltable->max_c0_size); double delta = ((double)c0->current_size)/(0.9*(double)ltable->max_c0_size); // 0 <= delta <= 1.111...
double delta = ((double)ltable->tree_bytes)/(0.9*(double)ltable->max_c0_size); // 0 <= delta <= 1.111...
delta -= 1.0; delta -= 1.0;
if(delta > 0.00005) { if(delta > 0.00005) {
double slp = 0.001 + 5.0 * delta; //0.0015 < slp < 1.112111.. double slp = 0.001 + 5.0 * delta; //0.0015 < slp < 1.112111..
@ -325,8 +324,8 @@ void mergeManager::pretty_print(FILE * out) {
have_c2 = NULL != lt->get_tree_c2(); have_c2 = NULL != lt->get_tree_c2();
} }
pageid_t mb = 1024 * 1024; pageid_t mb = 1024 * 1024;
fprintf(out,"[merge progress MB/s window (lifetime)]: app [%s %6lldMB ~ %3.0f%%/%3.0f%% %6.1fsec %4.1f (%4.1f)] %s %s [%s %3.0f%% ~ %3.0f%% %4.1f (%4.1f)] %s %s [%s %3.0f%% %4.1f (%4.1f)] %s ", fprintf(out,"[merge progress MB/s window (lifetime)]: app [%s %6lldMB tot %6lldMB cur ~ %3.0f%%/%3.0f%% %6.1fsec %4.1f (%4.1f)] %s %s [%s %3.0f%% ~ %3.0f%% %4.1f (%4.1f)] %s %s [%s %3.0f%% %4.1f (%4.1f)] %s ",
c0->active ? "RUN" : "---", (long long)(c0->stats_lifetime_consumed / mb), 100.0 * c0->out_progress, 100.0 * ((double)ltable->tree_bytes)/(double)ltable->max_c0_size, c0->stats_lifetime_elapsed, c0->stats_bps/((double)mb), c0->stats_lifetime_consumed/(((double)mb)*c0->stats_lifetime_elapsed), c0->active ? "RUN" : "---", (long long)(c0->stats_lifetime_consumed / mb), (long long)(c0->current_size / mb), 100.0 * c0->out_progress, 100.0 * ((double)c0->current_size)/(double)ltable->max_c0_size, c0->stats_lifetime_elapsed, c0->stats_bps/((double)mb), c0->stats_lifetime_consumed/(((double)mb)*c0->stats_lifetime_elapsed),
have_c0 ? "C0" : "..", have_c0 ? "C0" : "..",
have_c0m ? "C0'" : "...", have_c0m ? "C0'" : "...",
c1->active ? "RUN" : "---", 100.0 * c1->in_progress, 100.0 * c1->out_progress, c1->stats_bps/((double)mb), c1->stats_lifetime_consumed/(((double)mb)*c1->stats_lifetime_elapsed), c1->active ? "RUN" : "---", 100.0 * c1->in_progress, 100.0 * c1->out_progress, c1->stats_bps/((double)mb), c1->stats_lifetime_consumed/(((double)mb)*c1->stats_lifetime_elapsed),

2
mergeStats.h
View file

@ -164,8 +164,8 @@ class mergeStats {
pageid_t mergeable_size; // protected by mutex. pageid_t mergeable_size; // protected by mutex.
public: public:
pageid_t target_size; pageid_t target_size;
protected:
pageid_t current_size; pageid_t current_size;
protected:
pageid_t bytes_out; // How many bytes worth of tuples did we write? pageid_t bytes_out; // How many bytes worth of tuples did we write?
public: public:
pageid_t bytes_in_small; // How many bytes from the small input tree (for C0, we ignore tree overheads)? pageid_t bytes_in_small; // How many bytes from the small input tree (for C0, we ignore tree overheads)?

4
merger.cpp
View file

@ -103,7 +103,7 @@ void * merge_scheduler::memMergeThread() {
// needs to be past the rwlc_unlock... // needs to be past the rwlc_unlock...
memTreeComponent<datatuple>::batchedRevalidatingIterator *itrB = memTreeComponent<datatuple>::batchedRevalidatingIterator *itrB =
new memTreeComponent<datatuple>::batchedRevalidatingIterator(ltable_->get_tree_c0(), &ltable_->tree_bytes, ltable_->max_c0_size, &ltable_->flushing, 100, &ltable_->rb_mut); new memTreeComponent<datatuple>::batchedRevalidatingIterator(ltable_->get_tree_c0(), &ltable_->merge_mgr->get_merge_stats(0)->current_size, ltable_->max_c0_size, &ltable_->flushing, 100, &ltable_->rb_mut);
//: do the merge //: do the merge
DEBUG("mmt:\tMerging:\n"); DEBUG("mmt:\tMerging:\n");
@ -339,7 +339,7 @@ static int garbage_collect(logtable<datatuple> * ltable_, datatuple ** garbage,
} // close rbitr before touching the tree. } // close rbitr before touching the tree.
if(t2tmp) { if(t2tmp) {
ltable_->get_tree_c0()->erase(garbage[i]); ltable_->get_tree_c0()->erase(garbage[i]);
ltable_->tree_bytes -= garbage[i]->byte_length(); ltable_->merge_mgr->get_merge_stats(0)->current_size -= garbage[i]->byte_length();
datatuple::freetuple(t2tmp); datatuple::freetuple(t2tmp);
} }
datatuple::freetuple(garbage[i]); datatuple::freetuple(garbage[i]);