Static dispatch, multiple column types per page.

benchmarks/roseTable.cpp

@@ -1,17 +1,17 @@
 #include <stdio.h>
+#include <stdlib.h>
 #include <errno.h>
 #include "stasis/operations/lsmTable.h"
 
 #include "stasis/transactional.h"
 
 #include "stasis/page/compression/multicolumn-impl.h"
+#include "stasis/page/compression/staticMulticolumn.h"
 #include "stasis/page/compression/for-impl.h"
 #include "stasis/page/compression/rle-impl.h"
 #include "stasis/page/compression/staticTuple.h"
 #include "stasis/page/compression/pageLayout.h"
 
-typedef int32_t val_t; // XXX want multiple types!
-
 namespace rose {
   template<class PAGELAYOUT>
   int main(int argc, char **argv) {
@@ -22,7 +22,7 @@ namespace rose {
     unlink("logfile.txt");
 
     sync();
-    stasis_page_impl_register(Multicolumn<typename PAGELAYOUT::FMT::TUP >::impl());
+    stasis_page_impl_register(PAGELAYOUT::FMT::impl());
     bufferManagerNonBlockingSlowHandleType = IO_HANDLE_PFILE;
 
     Tinit();
@@ -37,8 +37,12 @@ namespace rose {
 
     typename PAGELAYOUT::FMT::TUP t;
 
-
+    long INSERTS; 
-  static const long INSERTS = 10000000;
+    if(argc == 2) {
+      INSERTS = atoll(argv[1]);
+    } else {
+      INSERTS = 10 * 1000 * 1000;
+    }
     //    static const long INSERTS = 10000000;
 //  static const long INSERTS = 100000;
     static const long COUNT = INSERTS / 100;
@@ -51,13 +55,24 @@ namespace rose {
     start = rose::tv_to_double(start_tv);
     last_start = start;
 
-    printf("tuple 'size'%d\n", PAGELAYOUT::FMT::TUP::sizeofBytes());
+    printf("tuple 'size'%d ; %d\n", PAGELAYOUT::FMT::TUP::sizeofBytes(), sizeof(typename PAGELAYOUT::FMT::TUP));
 
     for(long int i = 0; i < INSERTS; i++) {
-      t.set0(&i);
+      typename PAGELAYOUT::FMT::TUP::TYP0 m = i;// % 65536;
-      t.set1(&i);
+      typename PAGELAYOUT::FMT::TUP::TYP1 j = 0 / 65536;
-      t.set2(&i);
+      typename PAGELAYOUT::FMT::TUP::TYP2 k = 0 / 12514500;
-      t.set3(&i);
+      typename PAGELAYOUT::FMT::TUP::TYP3 l = 0 / 10000000;
+
+      t.set0(&m);
+      t.set1(&l);
+      t.set2(&l);
+      t.set3(&l);
+      t.set4(&l);
+      t.set5(&l);
+      t.set6(&l);
+      t.set7(&l);
+      t.set8(&l);
+      t.set9(&l);
       TlsmTableInsert(h,t);
       count --;
       if(!count) {
@@ -85,12 +100,43 @@ namespace rose {
 }
 
 int main(int argc, char **argv) {
-  //  typedef rose::StaticTuple<4,int64_t,int32_t,int16_t,int8_t> tup;
+
-  typedef rose::StaticTuple<4,int64_t,int64_t,int64_t,int64_t> tup;
+  typedef int64_t typ0;
-  // XXX multicolumn is deprecated; want static dispatch!
+  typedef int64_t typ1;
-  return rose::main
+  typedef int64_t typ2;
+  typedef int64_t typ3;
+  typedef int64_t typ4;
+  typedef int64_t typ5;
+  typedef int64_t typ6;
+  typedef int64_t typ7;
+  typedef int64_t typ8;
+  typedef int64_t typ9;
+
+  #define COLS 10
+
+  typedef rose::StaticTuple<COLS,typ0,typ1,typ2,typ3,typ4,typ5,typ6,typ7,typ8,typ9> tup;
+  using rose::For;
+  using rose::Rle;
+
+  // multicolumn is deprecated; want static dispatch!
+
+  /*  return rose::main
     <rose::SingleColumnTypePageLayout
       <rose::Multicolumn<tup>,rose::For<int64_t> > >
+      (argc,argv); */
+
+  return rose::main
+    <rose::MultiColumnTypePageLayout
+    <COLS,
+    rose::StaticMulticolumn<COLS,tup,
+    For<typ0>,Rle<typ1>,
+    Rle<typ2>,Rle<typ3>,
+    Rle<typ4>,Rle<typ5>,
+    Rle<typ6>,Rle<typ7>,
+    Rle<typ8>,Rle<typ9> >
+    >
+    >
     (argc,argv);
+
   return 0;
 }

src/stasis/operations/lsmTable.h

@@ -328,8 +328,8 @@ namespace rose {
   static const int RB_TREE_OVERHEAD = 450;
   static const pageid_t MEM_SIZE = 800 * 1000 * 1000;
   // How many pages should we try to fill with the first C1 merge?
-  static const pageid_t START_SIZE = 10 * 1000;
+  static const pageid_t START_SIZE = /*10 **/ 1000;
-  static const int R = 40;
+  static const int R = 10; // XXX set this as low as possible (for dynamic setting.  = sqrt(C2 size / C0 size))
 
   template<class PAGELAYOUT>
     lsmTableHandle <PAGELAYOUT> * TlsmTableStart(recordid& tree) {
@@ -444,7 +444,7 @@ namespace rose {
 	ret->still_open,
 	block0_size,
 	block1_size,
-	(R * MEM_SIZE) / (PAGE_SIZE * 4),  // 4 = estimated compression ratio
+	(R * MEM_SIZE) / (PAGE_SIZE * 4),  // XXX 4 = estimated compression ratio
 	R,
 	new typename LSM_ITER::treeIteratorHandle(NULLRID),
 	block0_scratch,

src/stasis/page/compression/multicolumn-impl.h

@@ -36,11 +36,6 @@ Multicolumn<TUPLE>::Multicolumn(int xid, Page *p, column_number_t column_count,
   p->impl = this;
 }
 
-/**
-   XXX this eagerly unpacks the page at load; that's a waste of
-   processor time and RAM, as read-only pages don't need to be
-   unpacked.
-*/
 template<class TUPLE>
 Multicolumn<TUPLE>::Multicolumn(Page * p) :
     p_(p),
@@ -68,7 +63,7 @@ Multicolumn<TUPLE>::Multicolumn(Page * p) :
 
   bytes_left_ = *exceptions_offset_ptr() - first_free;
 
-  assert(*stasis_page_type_ptr(p) == Multicolumn<TUPLE>::plugin_id());
+  assert(*stasis_page_type_ptr(p) == (Multicolumn<TUPLE>::plugin_id()));
 }
 
 template <class TUPLE>

src/stasis/page/compression/multicolumn.h

@@ -6,7 +6,8 @@
 #include <stasis/page.h>
 #include <stasis/constants.h>
 
-#include "pstar.h"  // for typedefs + consts (XXX add new header?)
+#include "compression.h"
+//#include "pstar.h"  // for typedefs + consts (XXX add new header?)
 #include "tuple.h" // XXX rename tuple.hx
 #include "pluginDispatcher.h"
 // Copyright 2007 Google Inc. All Rights Reserved.

src/stasis/page/compression/pageLayout.h

@@ -59,5 +59,89 @@ namespace rose {
   template <class PAGELAYOUT>
     recordid TlsmTableAlloc();
 
+  //// --- multicolumn static page layout
+
+  template <int N, class FORMAT>
+    class MultiColumnTypePageLayout {
+  public:
+    typedef FORMAT FMT;
+    static inline void initPageLayout() {
+      stasis_page_impl_register(FMT::impl());
+
+      // XXX these should register template instantiations of worker
+      // threads that are statically compiled to deal with the tree
+      // we're instantiating.
+      lsmTreeRegisterComparator(cmp_num, FMT::TUP::cmp);
+      lsmTreeRegisterPageInitializer
+	(init_num, (lsm_page_initializer_t)initPage);
+      my_cmp_num = cmp_num;
+      cmp_num++;
+      my_init_num = init_num;
+      init_num++;
+    }
+    static inline FORMAT * initPage(Page *p, const typename FORMAT::TUP * t) {
+      const column_number_t column_count = t->column_count();
+
+      //plugin_id_t pluginid = plugin_id<FORMAT, COMPRESSOR, typename COMPRESSOR::TYP>();
+
+      plugin_id_t plugins[N];
+      if(0 < N) plugins[0] = plugin_id<FORMAT, typename FORMAT::CMP0, typename FORMAT::CMP0::TYP>();
+      if(1 < N) plugins[1] = plugin_id<FORMAT, typename FORMAT::CMP1, typename FORMAT::CMP1::TYP>();
+      if(2 < N) plugins[2] = plugin_id<FORMAT, typename FORMAT::CMP2, typename FORMAT::CMP2::TYP>();
+      if(3 < N) plugins[3] = plugin_id<FORMAT, typename FORMAT::CMP3, typename FORMAT::CMP3::TYP>();
+      if(4 < N) plugins[4] = plugin_id<FORMAT, typename FORMAT::CMP4, typename FORMAT::CMP4::TYP>();
+      if(5 < N) plugins[5] = plugin_id<FORMAT, typename FORMAT::CMP5, typename FORMAT::CMP5::TYP>();
+      if(6 < N) plugins[6] = plugin_id<FORMAT, typename FORMAT::CMP6, typename FORMAT::CMP6::TYP>();
+      if(7 < N) plugins[7] = plugin_id<FORMAT, typename FORMAT::CMP7, typename FORMAT::CMP7::TYP>();
+      if(8 < N) plugins[8] = plugin_id<FORMAT, typename FORMAT::CMP8, typename FORMAT::CMP8::TYP>();
+      if(9 < N) plugins[9] = plugin_id<FORMAT, typename FORMAT::CMP9, typename FORMAT::CMP9::TYP>();
+
+      FORMAT * f = new FORMAT(-1,p);//N,plugins);
+
+      //plugin_id_t * plugins = (plugin_id_t*)malloc(column_count * sizeof(plugin_id_t));
+      //for(column_number_t c = 0; c < column_count; c++) {
+      //plugins[c] = pluginid;
+      //}
+      //      FORMAT * f = new FORMAT(-1,p,column_count,plugins);
+      
+      if(0 < N) f->compressor0()->offset(*t->get0());
+      if(1 < N) f->compressor1()->offset(*t->get1());
+      if(2 < N) f->compressor2()->offset(*t->get2());
+      if(3 < N) f->compressor3()->offset(*t->get3());
+      if(4 < N) f->compressor4()->offset(*t->get4());
+      if(5 < N) f->compressor5()->offset(*t->get5());
+      if(6 < N) f->compressor6()->offset(*t->get6());
+      if(7 < N) f->compressor7()->offset(*t->get7());
+      if(8 < N) f->compressor8()->offset(*t->get8());
+      if(9 < N) f->compressor9()->offset(*t->get9());
+
+      /*for(column_number_t c = 0; c < column_count; c++) {
+	COMPRESSOR* com = (COMPRESSOR*) f->compressor(c);
+	typename COMPRESSOR::TYP val = *(typename COMPRESSOR::TYP*)(t->get(c));
+	com->offset(val);
+	}*/
+      //      free(plugins);
+      return f;
+    }
+    static inline int cmp_id() {
+      return my_cmp_num;
+    }
+    static inline int init_id() {
+      return my_init_num;
+    }
+  private:
+    static int my_cmp_num;
+    static int my_init_num;
+  };
+  template <int N, class FORMAT>
+    int MultiColumnTypePageLayout<N,FORMAT>::my_cmp_num = -1;
+  template <int N, class FORMAT>
+    int MultiColumnTypePageLayout<N,FORMAT>::my_init_num = -1;
+
+  template <class PAGELAYOUT>
+    recordid TlsmTableAlloc();
+
+
+
 }
 #endif  // _ROSE_COMPRESSION_PAGELAYOUT_H__

src/stasis/page/compression/staticMulticolumn.h

@@ -0,0 +1,455 @@
+#ifndef _ROSE_COMPRESSION_STATIC_MULTICOLUMN_H__
+#define _ROSE_COMPRESSION_STATIC_MULTICOLUMN_H__
+
+#include <limits.h>
+
+#include <stasis/common.h>
+#include <stasis/page.h>
+#include <stasis/constants.h>
+
+#include "compression.h"
+
+//#include "pstar.h"  // for typedefs + consts (XXX add new header?)
+#include "tuple.h" // XXX rename tuple.hx
+//#include "pluginDispatcher.h"
+// Copyright 2007 Google Inc. All Rights Reserved.
+// Author: sears@google.com (Rusty Sears)
+
+/**
+  @file Page implementation for multi-column, compressed data
+
+ STRUCTURE OF A MULTICOLUMN PAGE
+
+<pre>
+ +----------------------------------------------------------------------+
+ | col #0 compressed data (opaque) | col #1 compressed data (opaque)    |
+ +-----+---------------------------+-----+------------------------------|
+ | ... | col #N compressed data (opaque) |                              |
+ +-----+---------------------------------+                              |
+ |  Free space                                                          |
+ |                                                                      |
+ |                                                                      |
+ |                             +----------------------------------------+
+ |                             | Exceptions:                            |
+ +-----------------------------+ Includes data from multiple cols       |
+ |                                                                      |
+ | Exception data is managed (bytes are copied in and out of this       |
+ | region) by the column implementations.  Multicolumn mediates between |
+ | the columns, by recording the length and offset of this region.      |
+ |                                                                      |
+ |                                      +---------------+---------------+
+ |                                  ... | exception # 1 | exception # 0 |
+ +-----------------------+--------------------+----+--------------------+
+ |  first header byte -> | col #N off, plugin | .. | col #1 off, plugin |
+ +--------------------+--+-------------+------+----+----+-----------+---+
+ | col #0 off, plugin | exceptions len | exceptions off | # of cols | ? |
+ +--------------------+----------------+----------------+-----------+---+
+</pre>
+
+ Notes:
+
+   The 'exceptions' portion of the page grows down from
+   first_header_byte, while the column data portion grows up from byte
+   zero...  This was an arbitrary decision, and complicated the
+   implementation somewhat...
+
+   Functions whose names end in "_ptr" return pointers to bytes in the
+   page.  That memory is persistant; and will eventually be written
+   back to the page file.
+
+*/
+
+namespace rose {
+
+/**
+ * A "pageLoaded()" callback function for Stasis' buffer manager.
+ */
+template <int N, class TUPLE,
+  class COMP0, class COMP1, class COMP2, class COMP3, class COMP4,
+  class COMP5, class COMP6, class COMP7, class COMP8, class COMP9>
+void staticMulticolumnLoaded(Page *p);
+
+template <int N, class TUPLE,
+  class COMP0, class COMP1 = For<bool>, class COMP2 = For<bool>,
+  class COMP3=For<bool>, class COMP4=For<bool>, class COMP5=For<bool>,
+  class COMP6=For<bool>, class COMP7=For<bool>, class COMP8=For<bool>,
+  class COMP9=For<bool> >
+class StaticMulticolumn {
+ public:
+  static page_impl impl();
+  static const plugin_id_t PAGE_FORMAT_ID = 1;
+  typedef TUPLE TUP;
+
+ typedef COMP0 CMP0;
+ typedef COMP1 CMP1;
+ typedef COMP2 CMP2;
+ typedef COMP3 CMP3;
+ typedef COMP4 CMP4;
+ typedef COMP5 CMP5;
+ typedef COMP6 CMP6;
+ typedef COMP7 CMP7;
+ typedef COMP8 CMP8;
+ typedef COMP9 CMP9;
+
+ StaticMulticolumn(int xid, Page *p) :
+    p_(p),
+    first_exception_byte_(USABLE_SIZE_OF_PAGE),
+    exceptions_(new byte[USABLE_SIZE_OF_PAGE]),
+    unpacked_(1)
+{
+
+  *column_count_ptr() = N;
+
+  bytes_left_ = first_header_byte_ptr()- p->memAddr;
+
+  /*  for(int i = 0; i < N; i++) {
+    *column_plugin_id_ptr(i) = plugins[i];
+    dispatcher_.set_plugin(columns_[i],i,plugins[i]);
+    dispatcher_.init_mem(columns_[i],i);
+    bytes_left_ -= dispatcher_.bytes_used(i);
+    } */
+
+#define STATIC_MC_INIT(i,typ,cmp)			   \
+  if(i < N) {						   \
+    *column_plugin_id_ptr(i) = cmp->PLUGIN_ID;		   \
+    columns_[i] = new byte[USABLE_SIZE_OF_PAGE];           \
+    /*    if(plugin0) delete plugin0; */		   \
+    cmp = new typ(xid,columns_[i]);			   \
+    cmp->init_mem(columns_[i]);	 			   \
+    bytes_left_ -= cmp->bytes_used();			   \
+  }
+
+  STATIC_MC_INIT(0, CMP0, plugin0) STATIC_MC_INIT(1, CMP1, plugin1) STATIC_MC_INIT(2, CMP2, plugin2) ;
+  STATIC_MC_INIT(3, CMP3, plugin3) STATIC_MC_INIT(4, CMP4, plugin4) STATIC_MC_INIT(5, CMP5, plugin5) ;
+  STATIC_MC_INIT(6, CMP6, plugin6) STATIC_MC_INIT(7, CMP7, plugin7) STATIC_MC_INIT(8, CMP8, plugin8) ;
+  STATIC_MC_INIT(9, CMP9, plugin9);
+
+#undef STATIC_MC_INIT
+
+  *stasis_page_type_ptr(p) = plugin_id();
+  p->impl = this;
+}
+
+ ~StaticMulticolumn() {
+  byte_off_t first_free = 0;
+  byte_off_t last_free  = (intptr_t)(first_header_byte_ptr() - p_->memAddr);
+  if(unpacked_) {
+    *exceptions_len_ptr() = USABLE_SIZE_OF_PAGE - first_exception_byte_;
+    last_free -= *exceptions_len_ptr();
+
+    *exceptions_offset_ptr() = last_free;
+    memcpy(&(p_->memAddr[*exceptions_offset_ptr()]),
+	   exceptions_ + first_exception_byte_, *exceptions_len_ptr());
+
+#define STATIC_MC_DEINIT(i,plug)		            \
+    if(i < N) {						    \
+      *column_offset_ptr(i) = first_free;		    \
+      byte_off_t bytes_used = plug->bytes_used();	    \
+      memcpy(column_base_ptr(i), columns_[i], bytes_used);  \
+      first_free += bytes_used;				    \
+      assert(first_free <= last_free);			    \
+      delete [] columns_[i];				    \
+    }
+
+    STATIC_MC_DEINIT(0,plugin0);
+    STATIC_MC_DEINIT(1,plugin1);
+    STATIC_MC_DEINIT(2,plugin2);
+    STATIC_MC_DEINIT(3,plugin3);
+    STATIC_MC_DEINIT(4,plugin4);
+    STATIC_MC_DEINIT(5,plugin5);
+    STATIC_MC_DEINIT(6,plugin6);
+    STATIC_MC_DEINIT(7,plugin7);
+    STATIC_MC_DEINIT(8,plugin8);
+    STATIC_MC_DEINIT(9,plugin9);
+
+    delete [] exceptions_;
+  }
+ }
+
+  /**
+     @return the compressor used for a column.  The nature of the
+     mapping between table region and compressor instance is
+     implementation defined, but there will never be more than one
+     compressor per-column, per-page.
+
+     @param col The column whose compressor should be returned.
+     @return A pointer to a compressor.  This pointer is guaranteed to
+     be valid until the next call to this Multicolumn object.  After
+     that, the pointer returned here is invalid.
+  */
+  //  void* compressor(column_number_t col) {
+    // XXX return dispatcher_.compressor(col);
+  //  }
+  COMP0 * compressor0() const { return plugin0; }
+  COMP1 * compressor1() const { return plugin1; }
+  COMP2 * compressor2() const { return plugin2; }
+  COMP3 * compressor3() const { return plugin3; }
+  COMP4 * compressor4() const { return plugin4; }
+  COMP5 * compressor5() const { return plugin5; }
+  COMP6 * compressor6() const { return plugin6; }
+  COMP7 * compressor7() const { return plugin7; }
+  COMP8 * compressor8() const { return plugin8; }
+  COMP9 * compressor9() const { return plugin9; }
+
+  inline slot_index_t append(int xid, TUPLE const & dat) {
+    slot_index_t ret = 0;
+    if(0 < N) ret = plugin0->append(xid, *dat.get0(),&first_exception_byte_,
+				   exceptions_, &bytes_left_);
+    if(1 < N) ret = plugin1->append(xid, *dat.get1(),&first_exception_byte_,
+				   exceptions_, &bytes_left_);
+    if(2 < N) ret = plugin2->append(xid, *dat.get2(),&first_exception_byte_,
+				   exceptions_, &bytes_left_);
+    if(3 < N) ret = plugin3->append(xid, *dat.get3(),&first_exception_byte_,
+				   exceptions_, &bytes_left_);
+    if(4 < N) ret = plugin4->append(xid, *dat.get4(),&first_exception_byte_,
+				   exceptions_, &bytes_left_);
+    if(5 < N) ret = plugin5->append(xid, *dat.get5(),&first_exception_byte_,
+				   exceptions_, &bytes_left_);
+    if(6 < N) ret = plugin6->append(xid, *dat.get6(),&first_exception_byte_,
+				   exceptions_, &bytes_left_);
+    if(7 < N) ret = plugin7->append(xid, *dat.get7(),&first_exception_byte_,
+				   exceptions_, &bytes_left_);
+    if(8 < N) ret = plugin8->append(xid, *dat.get8(),&first_exception_byte_,
+				   exceptions_, &bytes_left_);
+    if(9 < N) ret = plugin9->append(xid, *dat.get9(),&first_exception_byte_,
+				   exceptions_, &bytes_left_);
+    return bytes_left_ < 0 ? NOSPACE : ret;
+  }
+ inline TUPLE * recordRead(int xid, slot_index_t slot, TUPLE * buf) {
+   bool ret = 1;
+   if(0 < N) ret = plugin0->recordRead(xid,slot,exceptions_,const_cast<typename TUP::TYP0*>(buf->get0())) ? ret : 0;
+   if(1 < N) ret = plugin1->recordRead(xid,slot,exceptions_,const_cast<typename TUP::TYP1*>(buf->get1())) ? ret : 0;
+   if(2 < N) ret = plugin2->recordRead(xid,slot,exceptions_,const_cast<typename TUP::TYP2*>(buf->get2())) ? ret : 0;
+   if(3 < N) ret = plugin3->recordRead(xid,slot,exceptions_,const_cast<typename TUP::TYP3*>(buf->get3())) ? ret : 0;
+   if(4 < N) ret = plugin4->recordRead(xid,slot,exceptions_,const_cast<typename TUP::TYP4*>(buf->get4())) ? ret : 0;
+   if(5 < N) ret = plugin5->recordRead(xid,slot,exceptions_,const_cast<typename TUP::TYP5*>(buf->get5())) ? ret : 0;
+   if(6 < N) ret = plugin6->recordRead(xid,slot,exceptions_,const_cast<typename TUP::TYP6*>(buf->get6())) ? ret : 0;
+   if(7 < N) ret = plugin7->recordRead(xid,slot,exceptions_,const_cast<typename TUP::TYP7*>(buf->get7())) ? ret : 0;
+   if(8 < N) ret = plugin8->recordRead(xid,slot,exceptions_,const_cast<typename TUP::TYP8*>(buf->get8())) ? ret : 0;
+   if(9 < N) ret = plugin9->recordRead(xid,slot,exceptions_,const_cast<typename TUP::TYP9*>(buf->get9())) ? ret : 0;
+   return ret ? buf : 0;
+ }
+  inline void pack() {
+    byte_off_t first_free = 0;
+    byte_off_t last_free  = (intptr_t)(first_header_byte_ptr() - p_->memAddr);
+    if(unpacked_) {
+      *exceptions_len_ptr() = USABLE_SIZE_OF_PAGE - first_exception_byte_;
+      last_free -= *exceptions_len_ptr();
+
+      *exceptions_offset_ptr() = last_free;
+      memcpy(&(p_->memAddr[*exceptions_offset_ptr()]),
+	     exceptions_ + first_exception_byte_, *exceptions_len_ptr());
+
+#define STATIC_MC_PACK(i,comp)				              \
+      if(i < N) {						      \
+	*column_offset_ptr(i) = first_free;			      \
+	byte_off_t bytes_used = comp->bytes_used();		      \
+	memcpy(column_base_ptr(i), columns_[i], bytes_used);	      \
+	first_free += bytes_used;				      \
+	assert(first_free <= last_free);			      \
+	delete [] columns_[i];					      \
+	columns_[i] = column_base_ptr(i);			      \
+	comp->mem(columns_[i]);					      \
+      }
+
+      STATIC_MC_PACK(0,plugin0) STATIC_MC_PACK(1,plugin1) ;
+      STATIC_MC_PACK(2,plugin2) STATIC_MC_PACK(3,plugin3) ;
+      STATIC_MC_PACK(4,plugin4) STATIC_MC_PACK(5,plugin5) ;
+      STATIC_MC_PACK(6,plugin6) STATIC_MC_PACK(7,plugin7) ;
+      STATIC_MC_PACK(8,plugin8) STATIC_MC_PACK(9,plugin9) ;
+
+#undef STATIC_MC_PACK
+
+      delete [] exceptions_;
+      exceptions_ = p_->memAddr + *exceptions_offset_ptr();
+      unpacked_ = 0;
+    }
+  }
+ private:
+  COMP0* plugin0; COMP1* plugin1; COMP2* plugin2; COMP3* plugin3;
+  COMP4* plugin4; COMP5* plugin5; COMP6* plugin6; COMP7* plugin7;
+  COMP8* plugin8; COMP9* plugin9;
+
+  typedef struct column_header {
+    byte_off_t off;
+    plugin_id_t plugin_id;
+  } column_header;
+
+  /**
+     Load an existing multicolumn Page
+  */
+ StaticMulticolumn(Page * p) :
+    p_(p),
+    first_exception_byte_(USABLE_SIZE_OF_PAGE - *exceptions_len_ptr()),
+    exceptions_(p_->memAddr + *exceptions_offset_ptr()), 
+    unpacked_(0)  {
+      byte_off_t first_free = 0;
+      assert(N == *column_count_ptr());
+
+#define STATIC_MC_INIT(i,plug,cmp)						\
+      if(i < N) {							\
+	/*byte * page_column_ptr = p_->memAddr + *column_offset_ptr(i);*/ \
+	columns_[i] = p_->memAddr + *column_offset_ptr(i);		\
+	plug = new cmp((void*)columns_[i]);			\
+	first_free = *column_offset_ptr(i) + plug->bytes_used();	\
+      }
+
+      STATIC_MC_INIT(0, plugin0,COMP0) ;
+      STATIC_MC_INIT(1, plugin1,COMP1) ;
+      STATIC_MC_INIT(2, plugin2,COMP2) ;
+      STATIC_MC_INIT(3, plugin3,COMP3) ;
+      STATIC_MC_INIT(4, plugin4,COMP4) ;
+      STATIC_MC_INIT(5, plugin5,COMP5) ;
+      STATIC_MC_INIT(6, plugin6,COMP6) ;
+      STATIC_MC_INIT(7, plugin7,COMP7) ;
+      STATIC_MC_INIT(8, plugin8,COMP8) ;
+      STATIC_MC_INIT(9, plugin9,COMP9) ;
+
+#undef STATIC_MC_INIT
+
+      assert(first_free <= *exceptions_offset_ptr());
+      assert(first_exception_byte_ <= USABLE_SIZE_OF_PAGE);
+
+      bytes_left_ = *exceptions_offset_ptr() - first_free;
+
+      assert(*stasis_page_type_ptr(p) == (plugin_id()));
+    }
+
+  /**
+     The following functions perform pointer arithmetic.  This code is
+     performance critical.  These short, inlined functions mostly
+     perform simple arithmetic expression involving constants.  g++'s
+     optimizer seems to combine and simplify these expressions for us.
+
+     See the page layout diagram at the top of this file for an
+     explanation of where these pointers are stored
+   */
+
+  inline column_number_t * column_count_ptr() {
+    return reinterpret_cast<column_number_t*>(p_->memAddr+USABLE_SIZE_OF_PAGE)-1;
+  }
+  inline byte_off_t * exceptions_offset_ptr() {
+    return reinterpret_cast<byte_off_t*>(column_count_ptr())-1;
+  }
+  inline byte_off_t * exceptions_len_ptr() {
+    return exceptions_offset_ptr()-1;;
+  }
+  inline column_header * column_header_ptr(column_number_t column_number) {
+    return reinterpret_cast<column_header*>(exceptions_len_ptr())-(1+column_number);
+  }
+  inline byte_off_t * column_offset_ptr(column_number_t column_number) {
+    return &(column_header_ptr(column_number)->off);
+  }
+  /**
+     This stores the plugin_id associated with this page's compressor.
+
+     @see rose::plugin_id()
+  */
+  inline plugin_id_t * column_plugin_id_ptr(column_number_t column_number) {
+    return &(column_header_ptr(column_number)->plugin_id);
+  }
+  /**
+     The first byte that contains data for this column.
+
+     The length of the column data can be determined by calling
+     COMPRESSOR's bytes_used() member function.  (PluginDispatcher
+     can handle this).
+  */
+  inline byte * column_base_ptr(column_number_t column_number) {
+    return *column_offset_ptr(column_number) + p_->memAddr;
+  }
+  inline byte * first_header_byte_ptr() {
+    return reinterpret_cast<byte*>(column_header_ptr((*column_count_ptr())-1));
+  }
+
+  static inline plugin_id_t plugin_id() {
+    // XXX collides with multicolumn.h
+    return USER_DEFINED_PAGE(0) + 32 + TUPLE::TUPLE_ID;
+  }
+  Page * p_;
+  byte * columns_[N];
+  byte_off_t first_exception_byte_;
+  byte * exceptions_;
+  int bytes_left_;
+  int unpacked_;
+  friend void staticMulticolumnLoaded<N,TUPLE,COMP0,COMP1,COMP2,COMP3,COMP4,COMP5,COMP6,COMP7,COMP8,COMP9>(Page *p);
+};
+
+
+/// End performance-critical code ---------------------------------------------
+
+/// Stuff below this line interfaces with Stasis' buffer manager --------------
+
+/**
+   Basic page_impl for multicolumn pages
+
+   @see stasis/page.h and pstar-impl.h
+
+*/
+static const page_impl static_multicolumn_impl = {
+  -1,
+  0,  // multicolumnRead,
+  0,       // multicolumnWrite,
+  0,  // multicolumnReadDone,
+  0,       // multicolumnWriteDone,
+  0,  // multicolumnGetType,
+  0,  // multicolumnSetType,
+  0,  // multicolumnGetLength,
+  0,  // multicolumnFirst,
+  0,  // multicolumnNext,
+  0,  // multicolumnIsBlockSupported,
+  0,  // multicolumnBlockFirst,
+  0,  // multicolumnBlockNext,
+  0,  // multicolumnBlockDone,
+  0,  // multicolumnFreespace,
+  0,       // multicolumnCompact,
+  0,       // multicolumnPreRalloc,
+  0,       // multicolumnPostRalloc,
+  0,       // multicolumnFree,
+  0,       // dereference_identity,
+  0,  // multicolumnLoaded,
+  0,  // multicolumnFlushed
+  0,  // multicolumnCleanup
+};
+
+template <int N, class TUPLE,
+  class COMP0, class COMP1, class COMP2, class COMP3, class COMP4,
+  class COMP5, class COMP6, class COMP7, class COMP8, class COMP9>
+void staticMulticolumnLoaded(Page *p) {
+  p->LSN = *stasis_page_lsn_ptr(p);
+  assert(*stasis_page_type_ptr(p) == (StaticMulticolumn<N,TUPLE,COMP0,COMP1,COMP2,COMP3,COMP4,COMP5,COMP6,COMP7,COMP8,COMP9>::plugin_id()));
+  p->impl = new StaticMulticolumn<N,TUPLE,COMP0,COMP1,COMP2,COMP3,COMP4,COMP5,COMP6,COMP7,COMP8,COMP9>(p);
+}
+template <int N, class TUPLE,
+  class COMP0, class COMP1, class COMP2, class COMP3, class COMP4,
+  class COMP5, class COMP6, class COMP7, class COMP8, class COMP9>
+static void staticMulticolumnFlushed(Page *p) {
+  *stasis_page_lsn_ptr(p) = p->LSN;
+  ((StaticMulticolumn<N,TUPLE,COMP0,COMP1,COMP2,COMP3,COMP4,COMP5,COMP6,COMP7,COMP8,COMP9>*)(p->impl))->pack();
+}
+template <int N, class TUPLE,
+  class COMP0, class COMP1, class COMP2, class COMP3, class COMP4,
+  class COMP5, class COMP6, class COMP7, class COMP8, class COMP9>
+static void staticMulticolumnCleanup(Page *p) {
+  delete (StaticMulticolumn<N,TUPLE,COMP0,COMP1,COMP2,COMP3,COMP4,COMP5,COMP6,COMP7,COMP8,COMP9>*)p->impl;
+  p->impl = 0;
+}
+
+template <int N, class TUPLE,
+  class COMP0, class COMP1, class COMP2, class COMP3, class COMP4,
+  class COMP5, class COMP6, class COMP7, class COMP8, class COMP9>
+page_impl StaticMulticolumn<N,TUPLE,COMP0,COMP1,COMP2,COMP3,COMP4,COMP5,COMP6,COMP7,COMP8,COMP9>::impl() {
+  page_impl ret = static_multicolumn_impl;
+  ret.page_type = StaticMulticolumn<N,TUPLE,COMP0,COMP1,COMP2,COMP3,COMP4,COMP5,COMP6,COMP7,COMP8,COMP9>::plugin_id();
+  ret.pageLoaded = staticMulticolumnLoaded<N,TUPLE,COMP0,COMP1,COMP2,COMP3,COMP4,COMP5,COMP6,COMP7,COMP8,COMP9>;
+  ret.pageFlushed = staticMulticolumnFlushed<N,TUPLE,COMP0,COMP1,COMP2,COMP3,COMP4,COMP5,COMP6,COMP7,COMP8,COMP9>;
+  ret.pageCleanup = staticMulticolumnCleanup<N,TUPLE,COMP0,COMP1,COMP2,COMP3,COMP4,COMP5,COMP6,COMP7,COMP8,COMP9>;
+  return ret;
+}
+
+
+
+}  // namespace rose
+
+
+#endif // _ROSE_COMPRESSION_STATIC_MULTICOLUMN_H__

src/stasis/page/compression/staticTuple.h

@@ -2,7 +2,6 @@
 #define _ROSE_COMPRESSION_STATICTUPLE_H__
 
 namespace rose {
-
   template<int N, class TYPE0,
     class TYPE1=bool, class TYPE2=bool, class TYPE3=bool, class TYPE4=bool,
     class TYPE5=bool, class TYPE6=bool, class TYPE7=bool, class TYPE8=bool,
@@ -13,6 +12,18 @@ namespace rose {
   static const char TOMBSTONE = 1;
   static const int TUPLE_ID = 1;
 
+  typedef TYPE0 TYP0;
+  typedef TYPE1 TYP1;
+  typedef TYPE2 TYP2;
+  typedef TYPE3 TYP3;
+  typedef TYPE4 TYP4;
+  typedef TYPE5 TYP5;
+  typedef TYPE6 TYP6;
+  typedef TYPE7 TYP7;
+  typedef TYPE8 TYP8;
+  typedef TYPE9 TYP9;
+
+
   explicit inline StaticTuple() {
     s.flag_ = NORMAL;
     initializePointers();
@@ -50,8 +61,8 @@ namespace rose {
   }
 
   inline void* set(column_number_t col, void* val) {
-    memcpy(cols_[col],val,size_[col]);
+    memcpy(((byte*)&s)+cols_[col],val,size_[col]);
-    return(cols_[col]);
+    return(((byte*)&s)+cols_[col]);
   }
 
   inline TYPE0 * set0(TYPE0* val) { s.cols0_=*val; }
@@ -65,9 +76,20 @@ namespace rose {
   inline TYPE8 * set8(TYPE8* val) { s.cols8_=*val; }
   inline TYPE9 * set9(TYPE9* val) { s.cols9_=*val; }
 
-  inline void* get(column_number_t col) const {
+  inline const TYPE0 * get0() const { return &s.cols0_; }
-    return cols_[col];
+  inline const TYPE1 * get1() const { return &s.cols1_; }
-  }
+  inline const TYPE2 * get2() const { return &s.cols2_; }
+  inline const TYPE3 * get3() const { return &s.cols3_; }
+  inline const TYPE4 * get4() const { return &s.cols4_; }
+  inline const TYPE5 * get5() const { return &s.cols5_; }
+  inline const TYPE6 * get6() const { return &s.cols6_; }
+  inline const TYPE7 * get7() const { return &s.cols7_; }
+  inline const TYPE8 * get8() const { return &s.cols8_; }
+  inline const TYPE9 * get9() const { return &s.cols9_; }
+
+  /*  inline void* get(column_number_t col) const {
+    return ((byte*)&s) + cols_[col];
+    } */
   inline column_number_t column_count() const { return N; }
 
   inline byte_off_t column_len(column_number_t col) const {
@@ -203,12 +225,12 @@ namespace rose {
     StaticTuple scratch_;
   };
   private:
-
+  static bool first_;
-  void * cols_[N];
+  static short cols_[N];
-  size_t size_[N];
+  static byte_off_t size_[N];
   typedef char flag_t;
   typedef unsigned int epoch_t;
-  struct {
+  typedef struct {
     TYPE0 cols0_;
     TYPE1 cols1_;
     TYPE2 cols2_;
@@ -219,35 +241,61 @@ namespace rose {
     TYPE7 cols7_;
     TYPE8 cols8_;
     TYPE9 cols9_;
-    flag_t flag_;
+    flag_t flag_ : 1;
-    epoch_t epoch_;
+    epoch_t epoch_ : 31;
-  } s;
+  } st;
+
+  st s;
 
   inline void initializePointers() {
-    if(0 < N) cols_[0] = &s.cols0_;
+    if(first_) {
-    if(1 < N) cols_[1] = &s.cols1_;
+      st str;
-    if(2 < N) cols_[2] = &s.cols2_;
+      if(0 < N) cols_[0] = (byte*)&str.cols0_ - (byte*)&str;
-    if(3 < N) cols_[3] = &s.cols3_;
+      if(1 < N) cols_[1] = (byte*)&str.cols1_ - (byte*)&str;
-    if(4 < N) cols_[4] = &s.cols4_;
+      if(2 < N) cols_[2] = (byte*)&str.cols2_ - (byte*)&str;
-    if(5 < N) cols_[5] = &s.cols5_;
+      if(3 < N) cols_[3] = (byte*)&str.cols3_ - (byte*)&str;
-    if(6 < N) cols_[6] = &s.cols6_;
+      if(4 < N) cols_[4] = (byte*)&str.cols4_ - (byte*)&str;
-    if(7 < N) cols_[7] = &s.cols7_;
+      if(5 < N) cols_[5] = (byte*)&str.cols5_ - (byte*)&str;
-    if(8 < N) cols_[8] = &s.cols8_;
+      if(6 < N) cols_[6] = (byte*)&str.cols6_ - (byte*)&str;
-    if(9 < N) cols_[9] = &s.cols9_;
+      if(7 < N) cols_[7] = (byte*)&str.cols7_ - (byte*)&str;
+      if(8 < N) cols_[8] = (byte*)&str.cols8_ - (byte*)&str;
+      if(9 < N) cols_[9] = (byte*)&str.cols9_ - (byte*)&str;
 
-    if(0 < N) size_[0] = sizeof(s.cols0_);
+      if(0 < N) size_[0] = sizeof(str.cols0_);
-    if(1 < N) size_[1] = sizeof(s.cols1_);
+      if(1 < N) size_[1] = sizeof(str.cols1_);
-    if(2 < N) size_[2] = sizeof(s.cols2_);
+      if(2 < N) size_[2] = sizeof(str.cols2_);
-    if(3 < N) size_[3] = sizeof(s.cols3_);
+      if(3 < N) size_[3] = sizeof(str.cols3_);
-    if(4 < N) size_[4] = sizeof(s.cols4_);
+      if(4 < N) size_[4] = sizeof(str.cols4_);
-    if(5 < N) size_[5] = sizeof(s.cols5_);
+      if(5 < N) size_[5] = sizeof(str.cols5_);
-    if(6 < N) size_[6] = sizeof(s.cols6_);
+      if(6 < N) size_[6] = sizeof(str.cols6_);
-    if(7 < N) size_[7] = sizeof(s.cols7_);
+      if(7 < N) size_[7] = sizeof(str.cols7_);
-    if(8 < N) size_[8] = sizeof(s.cols8_);
+      if(8 < N) size_[8] = sizeof(str.cols8_);
-    if(9 < N) size_[9] = sizeof(s.cols9_);
+      if(9 < N) size_[9] = sizeof(str.cols9_);
+
+      first_ = 0;
+    }
   }
+
   };
 
-}
+  template<int N, class TYPE0,
+    class TYPE1, class TYPE2, class TYPE3, class TYPE4,
+    class TYPE5, class TYPE6, class TYPE7, class TYPE8,
+    class TYPE9>
+  short StaticTuple<N,TYPE0,TYPE1,TYPE2,TYPE3,TYPE4,
+    TYPE5,TYPE6,TYPE7,TYPE8,TYPE9>::cols_[N];
+  template<int N, class TYPE0,
+    class TYPE1, class TYPE2, class TYPE3, class TYPE4,
+    class TYPE5, class TYPE6, class TYPE7, class TYPE8,
+    class TYPE9>
+  byte_off_t StaticTuple<N,TYPE0,TYPE1,TYPE2,TYPE3,TYPE4,
+    TYPE5,TYPE6,TYPE7,TYPE8,TYPE9>::size_[N];
+  template<int N, class TYPE0,
+    class TYPE1, class TYPE2, class TYPE3, class TYPE4,
+    class TYPE5, class TYPE6, class TYPE7, class TYPE8,
+    class TYPE9>
+  bool StaticTuple<N,TYPE0,TYPE1,TYPE2,TYPE3,TYPE4,
+    TYPE5,TYPE6,TYPE7,TYPE8,TYPE9>::first_ = true;
 
+}
 #endif  // _ROSE_COMPRESSION_STATICTUPLE_H__