Merge pull request #11 from basho-labs/gsb-2.0-fixes

Changes related to Riak 2.0 and an issue with how statistics were gathered from the backend

.gitignore

@@ -7,6 +7,8 @@ c_src/*.o
 c_src/bzip2-1.0.6
 c_src/snappy-1.0.4
 deps/
-priv/
+priv/wt
+priv/*.so*
+priv/*.dylib*
 log/
 *~

Makefile

@@ -52,19 +52,17 @@ endif
 .PHONY: all compile doc clean test dialyzer typer shell distclean pdf \
   update-deps clean-common-test-data rebuild
 
-all: deps compile test
+all: deps compile
 
 # =============================================================================
 # Rules to build the system
 # =============================================================================
 
 deps:
-	c_src/build_deps.sh get-deps
 	$(REBAR) get-deps
 	$(REBAR) compile
 
 update-deps:
-	c_src/build_deps.sh update-deps
 	$(REBAR) update-deps
 	$(REBAR) compile
 

c_src/build_deps.sh

@@ -11,10 +11,10 @@ unset POSIX_SHELL # clear it so if we invoke other scripts, they run as ksh as w
 set -e
 
 WT_REPO=http://github.com/wiredtiger/wiredtiger.git
-#WT_BRANCH=develop
+WT_BRANCH=develop
-#WT_DIR=wiredtiger-`basename $WT_BRANCH`
+WT_DIR=wiredtiger-`basename $WT_BRANCH`
-WT_REF="tags/1.6.4"
+#WT_REF="tags/1.6.4"
-WT_DIR=wiredtiger-`basename $WT_REF`
+#WT_DIR=wiredtiger-`basename $WT_REF`
 
 SNAPPY_VSN="1.0.4"
 SNAPPY_DIR=snappy-$SNAPPY_VSN
@@ -26,8 +26,7 @@ export BASEDIR="$PWD"
 which gmake 1>/dev/null 2>/dev/null && MAKE=gmake
 MAKE=${MAKE:-make}
 
-export CFLAGS="$CFLAGS -I $BASEDIR/system/include"
+export CPPFLAGS="$CPPLAGS -I $BASEDIR/system/include -O3 -mtune=native -march=native"
-export CXXFLAGS="$CXXFLAGS -I $BASEDIR/system/include"
 export LDFLAGS="$LDFLAGS -L$BASEDIR/system/lib"
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$BASEDIR/system/lib:$LD_LIBRARY_PATH"
 
@@ -58,7 +57,7 @@ get_wt ()
 wt_configure ()
 {
     (cd $BASEDIR/$WT_DIR/build_posix
-        CFLAGS+=-g ../configure --with-pic \
+        CFLAGS+=-g $BASEDIR/$WT_DIR/configure --with-pic \
                      --enable-snappy \
                      --prefix=${BASEDIR}/system || exit 1)
 }
@@ -107,11 +106,6 @@ build_snappy ()
     )
 }
 
-install_shared_lib ()
-{
-    cp -p -P $1 ${BASEDIR}/../priv/`basename ${1%.*}.so`
-}
-
 case "$1" in
     clean)
         [ -e $BASEDIR/$WT_DIR/build_posix/Makefile ] && \
@@ -147,12 +141,12 @@ case "$1" in
         # Build WiredTiger
         [ -d $WT_DIR ] || get_wt;
         [ -d $BASEDIR/$WT_DIR ] || (echo "Missing WiredTiger source directory" && exit 1)
-        test -f  $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].$SUFFIXES -a \
+        test -f  $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].${SUFFIXES} -a \
-	    -f $BASEDIR/system/lib/libwiredtiger_snappy.$SUFFIXES || build_wt;
+	    -f $BASEDIR/system/lib/libwiredtiger_snappy.${SUFFIXES} || build_wt;
         [ -d $BASEDIR/../priv ] || mkdir ${BASEDIR}/../priv
-        cp -p -P $BASEDIR/system/bin/wt ${BASEDIR}/../priv
+	cp -p -P $BASEDIR/system/bin/wt ${BASEDIR}/../priv
-        install_shared_lib ${BASEDIR}/system/lib/libwiredtiger-[0-9].[0-9].[0-9].$SUFFIXES
+	cp -p -P ${BASEDIR}/system/lib/libwiredtiger-[0-9].[0-9].[0-9].${SUFFIXES} ${BASEDIR}/../priv
-        install_shared_lib ${BASEDIR}/system/lib/libwiredtiger_snappy.$SUFFIXES*
+	cp -p -P ${BASEDIR}/system/lib/libwiredtiger_snappy.${SUFFIXES} ${BASEDIR}/../priv
-        install_shared_lib ${BASEDIR}/system/lib/libsnappy.$SUFFIXES*
+	cp -p -P ${BASEDIR}/system/lib/libsnappy.${SUFFIXES}* ${BASEDIR}/../priv
         ;;
 esac

c_src/wiredtiger-build.patch

@@ -1,106 +1,12 @@
 diff --git a/ext/compressors/snappy/Makefile.am b/ext/compressors/snappy/Makefile.am
-index 6d78823..2122cf8 100644
+index 6d78823..c423590 100644
 --- a/ext/compressors/snappy/Makefile.am
 +++ b/ext/compressors/snappy/Makefile.am
 @@ -2,5 +2,6 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
-
+ 
  lib_LTLIBRARIES = libwiredtiger_snappy.la
  libwiredtiger_snappy_la_SOURCES = snappy_compress.c
 -libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module
 +libwiredtiger_snappy_la_CFLAGS = -I$(abs_top_builddir)/../../system/include
 +libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(abs_top_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv
  libwiredtiger_snappy_la_LIBADD = -lsnappy
-diff --git a/src/support/cksum.c b/src/support/cksum.c
-index 7e9befe..b924db7 100644
---- a/src/support/cksum.c
-+++ b/src/support/cksum.c
-@@ -27,6 +27,13 @@
-
- #include "wt_internal.h"
-
-+#if defined(__amd64) || defined(__x86_64)
-+#define USE_HARDWARE_CRC32 1
-+#else
-+#undef USE_HARDWARE_CRC32
-+#endif
-+
-+#ifdef USE_HARDWARE_CRC32
- static const uint32_t g_crc_slicing[8][256] = {
- #ifdef WORDS_BIGENDIAN
- 	/*
-@@ -1078,6 +1085,7 @@ static const uint32_t g_crc_slicing[8][256] = {
- 	}
- #endif
- };
-+#endif /* USE_HARDWARE_CRC32 */
-
- /*
-  * __wt_cksum --
-@@ -1106,15 +1114,29 @@ __wt_cksum(const void *chunk, size_t len)
- 	/* Checksum one byte at a time to the first 4B boundary. */
- 	for (p = chunk;
- 	    ((uintptr_t)p & (sizeof(uint32_t) - 1)) != 0 &&
--	    len > 0; ++p, --len)
-+	    len > 0; ++p, --len) {
-+#ifdef USE_HARDWARE_CRC32
-+	    __asm__ __volatile__(
-+		".byte 0xF2, 0x0F, 0x38, 0xF0, 0xF1"
-+		: "=S" (crc)
-+		: "0" (crc), "c" (*p));
-+#else
- #ifdef WORDS_BIGENDIAN
- 		crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8);
- #else
- 		crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8);
- #endif
-+#endif
-+	}
-
- 	/* Checksum in 8B chunks. */
- 	for (nqwords = len / sizeof(uint64_t); nqwords; nqwords--) {
-+#ifdef USE_HARDWARE_CRC32
-+	    __asm__ __volatile__ (
-+		".byte 0xf2, 0x48, 0x0f, 0x38, 0xf0, 0xf1;"
-+		: "=S"(crc)
-+		: "S"(crc), "c"(*p));
-+#else
- 		crc ^= *(uint32_t *)p;
- 		p += sizeof(uint32_t);
- 		next = *(uint32_t *)p;
-@@ -1139,22 +1161,32 @@ __wt_cksum(const void *chunk, size_t len)
- 			g_crc_slicing[1][(next >> 16) & 0xFF] ^
- 			g_crc_slicing[0][(next >> 24)];
- #endif
-+#endif
- 	}
-
- 	/* Checksum trailing bytes one byte at a time. */
-+	for (len &= 0x7; len > 0; ++p, len--) {
-+#ifdef USE_HARDWARE_CRC32
-+	    __asm__ __volatile__(
-+		".byte 0xF2, 0x0F, 0x38, 0xF0, 0xF1"
-+		: "=S" (crc)
-+		: "0" (crc), "c" (*p));
-+#else
- #ifdef WORDS_BIGENDIAN
--	for (len &= 0x7; len > 0; ++p, len--)
- 		crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8);
-+#else
-+		crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8);
-+#endif
-+#endif
-+	}
-
-+#ifdef WORDS_BIGENDIAN
- 	/* Do final byte swap to produce a result identical to little endian */
- 	crc =
- 		((crc << 24) & 0xFF000000) |
- 		((crc <<  8) & 0x00FF0000) |
- 		((crc >>  8) & 0x0000FF00) |
- 		((crc >> 24) & 0x000000FF);
--#else
--	for (len &= 0x7; len > 0; ++p, len--)
--		crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8);
- #endif
- 	return (~crc);
- }

priv/wterl.schema

@@ -0,0 +1,6 @@
+%%%% This is the WiredTiger section
+
+%% @doc wiredtiger data_root
+{mapping, "wiredtiger.data_root", "wterl.data_root", [
+  {default, "{{platform_data_dir}}/wiredtiger"}
+]}.

rebar.config

@@ -38,8 +38,8 @@
 {port_specs, [{"priv/wterl.so", ["c_src/*.c"]}]}.
 
 {port_env, [
-            {"DRV_CFLAGS",  "$DRV_CFLAGS -fPIC -Wall -Wextra -Werror -I c_src/system/include"},
+            {"DRV_CFLAGS",  "$DRV_CFLAGS  -O3 -mtune=native -march=native -fPIC -Wall -Wextra -Werror -I c_src/system/include"},
-            {"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath,lib/wterl/priv:priv -Lc_src/system/lib -lwiredtiger"}
+            {"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath,lib/wterl/priv:lib/wterl-0.9.0/priv:priv -Lc_src/system/lib -lwiredtiger"}
            ]}.
 
 {pre_hooks, [{compile, "c_src/build_deps.sh compile"}]}.

src/riak_kv_wterl_backend.erl

@@ -120,9 +120,6 @@ start(Partition, Config) ->
                         [{internal_page_max, "128K"},
                          {leaf_page_max, "16K"},
                          {lsm_chunk_size, "100MB"},
-                         {lsm_merge_threads, 2},
-                         {prefix_compression, true},
-                         {lsm_bloom_newest, true},
                          {lsm_bloom_oldest, true} ,
                          {lsm_bloom_bit_count, 28},
                          {lsm_bloom_hash_count, 19},
@@ -341,22 +338,23 @@ is_empty(#state{connection=Connection, table=Table}) ->
 %% @doc Get the status information for this wterl backend
 -spec status(state()) -> [{atom(), term()}].
 status(#state{connection=Connection, table=Table}) ->
-    case wterl:cursor_open(Connection, Table) of
+    [].
-        {ok, Cursor} ->
+%%     case wterl:cursor_open(Connection, "statistics:" ++ Table, [{statistics_fast, true}]) of
-	    TheStats =
+%%         {ok, Cursor} ->
-		case fetch_status(Cursor) of
+%% 	    TheStats =
-		    {ok, Stats} ->
+%% 		case fetch_status(Cursor) of
-			Stats;
+%% 		    {ok, Stats} ->
-		    {error, {eperm, _}} -> % TODO: review/fix this logic
+%% 			Stats;
-			{ok, []};
+%% 		    {error, {eperm, _}} -> % TODO: review/fix this logic
-		    _ ->
+%% 			{ok, []};
-			{ok, []}
+%% 		    _ ->
-		end,
+%% 			{ok, []}
-	    wterl:cursor_close(Cursor),
+%% 		end,
-	    TheStats;
+%% 	    wterl:cursor_close(Cursor),
-        {error, Reason2} ->
+%% 	    TheStats;
-            {error, Reason2}
+%%         {error, Reason2} ->
-    end.
+%%             {error, Reason2}
+%%    end.
 
 %% @doc Register an asynchronous callback
 -spec callback(reference(), any(), state()) -> {ok, state()}.
@@ -411,7 +409,7 @@ establish_connection(Config, Type) ->
             ConnectionOpts =
                 orddict:from_list(
                   [ wterl:config_value(create, Config, true),
-                    wterl:config_value(sync, Config, false),
+                    wterl:config_value(checkpoint_sync, Config, false),
                     wterl:config_value(session_max, Config, max_sessions(Config)),
                     wterl:config_value(cache_size, Config, size_cache(RequestedCacheSize)),
                     wterl:config_value(statistics_log, Config, [{wait, 600}]), % in seconds
@@ -543,15 +541,15 @@ from_index_key(LKey) ->
 
 %% @private
 %% Return all status from wterl statistics cursor
-fetch_status(Cursor) ->
+%% fetch_status(Cursor) ->
-   {ok, fetch_status(Cursor, wterl:cursor_next_value(Cursor), [])}.
+%%    {ok, fetch_status(Cursor, wterl:cursor_next_value(Cursor), [])}.
-fetch_status(_Cursor, {error, _}, Acc) ->
+%% fetch_status(_Cursor, {error, _}, Acc) ->
-    lists:reverse(Acc);
+%%     lists:reverse(Acc);
-fetch_status(_Cursor, not_found, Acc) ->
+%% fetch_status(_Cursor, not_found, Acc) ->
-    lists:reverse(Acc);
+%%     lists:reverse(Acc);
-fetch_status(Cursor, {ok, Stat}, Acc) ->
+%% fetch_status(Cursor, {ok, Stat}, Acc) ->
-    [What,Val|_] = [binary_to_list(B) || B <- binary:split(Stat, [<<0>>], [global])],
+%%     [What,Val|_] = [binary_to_list(B) || B <- binary:split(Stat, [<<0>>], [global])],
-    fetch_status(Cursor, wterl:cursor_next_value(Cursor), [{What,Val}|Acc]).
+%%     fetch_status(Cursor, wterl:cursor_next_value(Cursor), [{What,Val}|Acc]).
 
 size_cache(RequestedSize) ->
     Size =

src/wterl.erl

@@ -96,8 +96,8 @@ nif_stub_error(Line) ->
 -spec init() -> ok | {error, any()}.
 init() ->
     erlang:load_nif(filename:join([priv_dir(), atom_to_list(?MODULE)]),
-           [{wterl_vsn, "53307e8"},
+           [{wterl_vsn, "942e51b"},
-	    {wiredtiger_vsn, "1.6.2-0-g07cb0a5"}]).
+	    {wiredtiger_vsn, "1.6.4-275-g9c44420"}]). %% TODO automate these
 
 -spec connection_open(string(), config_list()) -> {ok, connection()} | {error, term()}.
 -spec connection_open(string(), config_list(), config_list()) -> {ok, connection()} | {error, term()}.
@@ -456,6 +456,7 @@ config_to_bin([{Key, Value} | Rest], Acc) ->
     [{block_compressor, {string, quoted}},
      {cache_size, string},
      {checkpoint, config},
+     {checkpoint_sync, bool},
      {checksum, string},
      {create, bool},
      {direct_io, list},
@@ -489,7 +490,6 @@ config_to_bin([{Key, Value} | Rest], Acc) ->
      {raw, bool},
      {session_max, integer},
      {statistics_log, config},
-     {sync, bool},
      {target, {list, quoted}},
      {to, string},
      {transactional, bool},