Change from static linking to dynamic. Enable snappy and bzip2 compression.

c_src/build_deps.sh

@@ -26,7 +26,11 @@ case "$1" in
         fi
         (cd wiredtiger/build_posix && \
             ../configure --with-pic \
+                         --enable-snappy \
+                         --enable-bzip2 \
                          --prefix=$BASEDIR/system && \
-            make -j 8 && make install)
+            make -j && make install)
+	cp system/bin/wt ../priv
+	cp system/lib/*.so ../priv
         ;;
 esac

c_src/wterl.c

@@ -90,7 +90,7 @@ static ERL_NIF_TERM wterl_session_verify(ErlNifEnv* env, int argc, const ERL_NIF
 
 static ErlNifFunc nif_funcs[] =
 {
-    {"conn_close", 1, wterl_conn_close},
+    {"connection_close", 1, wterl_conn_close},
     {"conn_open", 2, wterl_conn_open},
     {"cursor_close", 1, wterl_cursor_close},
     {"cursor_insert", 3, wterl_cursor_insert},

rebar.config

@@ -37,7 +37,7 @@
 
 {port_env, [
             {"DRV_CFLAGS",  "$DRV_CFLAGS -Werror -I c_src/system/include"},
-            {"DRV_LDFLAGS", "$DRV_LDFLAGS c_src/system/lib/libwiredtiger.a"}
+            {"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath -Wl,priv -Lpriv -lwiredtiger"}
            ]}.
 
 {pre_hooks, [{compile, "c_src/build_deps.sh"}]}.

src/riak_kv_wterl_backend.erl

@@ -85,7 +85,6 @@ capabilities(_, _) ->
 %% @doc Start the wterl backend
 -spec start(integer(), config()) -> {ok, state()} | {error, term()}.
 start(Partition, Config) ->
-    %lager:start(),
     AppStart =
         case application:start(wterl) of
             ok ->
@@ -355,13 +354,13 @@ establish_connection(Config) ->
 		    wterl:config_value(session_max, Config, max_sessions(Config)),
 		    wterl:config_value(cache_size, Config, size_cache(RequestedCacheSize)),
 		    wterl:config_value(statistics_log, Config, [{wait, 30}]), % sec
-		    wterl:config_value(checkpoint, Config, [{wait, 10}]), % sec
+		    %% NOTE: LSM auto-checkpoints, so we don't have too.
+		    %% wterl:config_value(checkpoint, Config, [{wait, 10}]), % sec
 		    wterl:config_value(verbose, Config, [ 
 			  %"ckpt" "block", "shared_cache", "evictserver", "fileops",
 			  %"hazard", "mutex", "read", "readserver", "reconcile",
 			  %"salvage", "verify", "write", "evict", "lsm"
 		    ]) ] ++ proplists:get_value(wterl, Config, [])), % sec
-            %lager:info("WiredTiger connection:open(~s, ~s)", [DataRoot, wterl:config_to_bin(Opts)]),
             case wterl_conn:open(DataRoot, Opts) of
                 {ok, Connection} ->
                     {ok, Connection};
@@ -384,13 +383,15 @@ establish_session(Connection, Table) ->
     case wterl:session_open(Connection, wterl:config_to_bin([{isolation, "snapshot"}])) of
         {ok, Session} ->
             SessionOpts =
-                [%TODO {block_compressor, "snappy"},
+                [{block_compressor, "snappy"},
                  {internal_page_max, "128K"},
                  {leaf_page_max, "128K"},
-                 {lsm_chunk_size, "200MB"},
+                 {lsm_chunk_size, "25MB"},
 		 {lsm_bloom_newest, true},
 		 {lsm_bloom_oldest, true} ,
-                 {lsm_bloom_config, [{leaf_page_max, "10MB"}]} ],
+		 {lsm_bloom_bit_count, 128},
+		 {lsm_bloom_hash_count, 64},
+                 {lsm_bloom_config, [{leaf_page_max, "8MB"}]} ],
             case wterl:session_create(Session, Table, wterl:config_to_bin(SessionOpts)) of
                 ok ->
 		    {ok, Session};
@@ -548,7 +549,6 @@ size_cache(RequestedSize) ->
                             "1GB"
                     end,
                 application:set_env(wterl, cache_size, FinalGuess),
-		%lager:info("Using cache size of ~p for WiredTiger storage backend.", [FinalGuess]),
                 FinalGuess;
             Value when is_list(Value) ->
                 Value;

src/wterl.erl

@@ -20,8 +20,8 @@
 %%
 %% -------------------------------------------------------------------
 -module(wterl).
--export([conn_open/2,
+-export([connection_open/2,
-         conn_close/1,
+         connection_close/1,
          cursor_close/1,
          cursor_insert/3,
          cursor_next/1,
@@ -60,6 +60,7 @@
          session_verify/3,
          config_value/3,
          config_to_bin/1,
+	 priv_dir/0,
          fold_keys/3,
          fold/3]).
 
@@ -92,22 +93,23 @@ nif_stub_error(Line) ->
 
 -spec init() -> ok | {error, any()}.
 init() ->
-    PrivDir = case code:priv_dir(?MODULE) of
+    erlang:load_nif(filename:join(priv_dir(), atom_to_list(?MODULE)), 0).
-                  {error, bad_name} ->
+
-                      EbinDir = filename:dirname(code:which(?MODULE)),
+-spec connection_open(string(), config()) -> {ok, connection()} | {error, term()}.
-                      AppPath = filename:dirname(EbinDir),
+connection_open(HomeDir, Config) ->
-                      filename:join(AppPath, "priv");
+    PrivDir = wterl:priv_dir(),
-                  Path ->
+    Bin = config_to_bin([{extensions,
-                      Path
+			  [filename:join(PrivDir, "libwiredtiger_snappy.so"),
-              end,
+			   filename:join(PrivDir, "libwiredtiger_bzip2.so")]}],
-    erlang:load_nif(filename:join(PrivDir, atom_to_list(?MODULE)), 0).
+			[<<",">>, Config]),
+    conn_open(HomeDir, Bin).
 
 -spec conn_open(string(), config()) -> {ok, connection()} | {error, term()}.
 conn_open(_HomeDir, _Config) ->
     ?nif_stub.
 
--spec conn_close(connection()) -> ok | {error, term()}.
+-spec connection_close(connection()) -> ok | {error, term()}.
-conn_close(_ConnRef) ->
+connection_close(_ConnRef) ->
     ?nif_stub.
 
 -spec session_open(connection()) -> {ok, session()} | {error, term()}.
@@ -266,11 +268,21 @@ fold(_Cursor, _Fun, Acc, not_found) ->
 fold(Cursor, Fun, Acc, {ok, Key, Value}) ->
     fold(Cursor, Fun, Fun({Key, Value}, Acc), cursor_next(Cursor)).
 
+priv_dir() ->
+    case code:priv_dir(?MODULE) of
+	{error, bad_name} ->
+	    EbinDir = filename:dirname(code:which(?MODULE)),
+	    AppPath = filename:dirname(EbinDir),
+	    filename:join(AppPath, "priv");
+	Path ->
+	    Path
+    end.
+
 %%
 %% Configuration type information.
 %%
 config_types() ->
-    [{block_compressor, string},
+    [{block_compressor, {string, quoted}},
      {cache_size, string},
      {checkpoint, config},
      {create, bool},
@@ -279,7 +291,7 @@ config_types() ->
      {error_prefix, string},
      {eviction_target, integer},
      {eviction_trigger, integer},
-     {extensions, string},
+     {extensions, {list, quoted}},
      {force, bool},
      {hazard_max, integer},
      {home_environment, bool},
@@ -320,8 +332,13 @@ config_encode(config, Value) ->
     list_to_binary(["(", config_to_bin(Value, []), ")"]);
 config_encode(list, Value) ->
     list_to_binary(["(", string:join(Value, ","), ")"]);
+config_encode({list, quoted}, Value) ->
+    Values = lists:map(fun(S) -> "\"" ++ S ++ "\"" end, Value),
+    list_to_binary(["(", string:join(Values, ","), ")"]);
 config_encode(string, Value) when is_list(Value) ->
     list_to_binary(Value);
+config_encode({string, quoted}, Value) when is_list(Value) ->
+    list_to_binary("\"" ++ Value ++ "\"");
 config_encode(string, Value) when is_number(Value) ->
     list_to_binary(integer_to_list(Value));
 config_encode(bool, true) ->
@@ -373,19 +390,19 @@ config_to_bin([{Key, Value} | Rest], Acc) ->
 open_test_conn(DataDir) ->
     ?assertCmd("rm -rf "++DataDir),
     ?assertMatch(ok, filelib:ensure_dir(filename:join(DataDir, "x"))),
-    OpenConfig = config_to_bin([{create,true},{cache_size,"100MB"}]),
+    OpenConfig = config_to_bin([{create,true}, {cache_size,"100MB"}]),
-    {ok, ConnRef} = conn_open(DataDir, OpenConfig),
+    {ok, ConnRef} = connection_open(DataDir, OpenConfig),
     ConnRef.
 
 open_test_session(ConnRef) ->
     {ok, SRef} = session_open(ConnRef),
     ?assertMatch(ok, session_drop(SRef, "table:test", config_to_bin([{force,true}]))),
-    ?assertMatch(ok, session_create(SRef, "table:test")),
+    ?assertMatch(ok, session_create(SRef, "table:test", config_to_bin([{block_compressor, "snappy"}]))),
     SRef.
 
 conn_test() ->
     ConnRef = open_test_conn(?TEST_DATA_DIR),
-    ?assertMatch(ok, conn_close(ConnRef)).
+    ?assertMatch(ok, connection_close(ConnRef)).
 
 session_test_() ->
     {setup,
@@ -393,7 +410,7 @@ session_test_() ->
              open_test_conn(?TEST_DATA_DIR)
      end,
      fun(ConnRef) ->
-             ok = conn_close(ConnRef)
+             ok = connection_close(ConnRef)
      end,
      fun(ConnRef) ->
              {inorder,
@@ -418,7 +435,7 @@ insert_delete_test() ->
     ?assertMatch(ok,  session_delete(SRef, "table:test", <<"a">>)),
     ?assertMatch(not_found,  session_get(SRef, "table:test", <<"a">>)),
     ok = session_close(SRef),
-    ok = conn_close(ConnRef).
+    ok = connection_close(ConnRef).
 
 init_test_table() ->
     ConnRef = open_test_conn(?TEST_DATA_DIR),
@@ -432,7 +449,7 @@ init_test_table() ->
 
 stop_test_table({ConnRef, SRef}) ->
     ?assertMatch(ok, session_close(SRef)),
-    ?assertMatch(ok, conn_close(ConnRef)).
+    ?assertMatch(ok, connection_close(ConnRef)).
 
 various_session_test_() ->
     {setup,
@@ -642,7 +659,7 @@ prop_put_delete() ->
                      ?cmd("rm -rf "++DataDir),
                      ok = filelib:ensure_dir(filename:join(DataDir, "x")),
                      Cfg = wterl:config_to_bin([{create,true}]),
-                     {ok, Conn} = wterl:conn_open(DataDir, Cfg),
+                     {ok, Conn} = wterl:connection_open(DataDir, Cfg),
                      {ok, SRef} = wterl:session_open(Conn),
                      try
                          wterl:session_create(SRef, Table),
@@ -658,7 +675,7 @@ prop_put_delete() ->
                          true
                      after
                          wterl:session_close(SRef),
-                         wterl:conn_close(Conn)
+                         wterl:connection_close(Conn)
                      end
                  end)).
 

src/wterl_conn.erl

@@ -82,7 +82,7 @@ init([]) ->
 
 handle_call({open, Dir, Config, Caller}, _From, #state{conn=undefined}=State) ->
     {Reply, NState} =
-	case wterl:conn_open(Dir, wterl:config_to_bin(Config)) of
+	case wterl:connection_open(Dir, wterl:config_to_bin(Config)) of
 	    {ok, ConnRef}=OK ->
 		Monitor = erlang:monitor(process, Caller),
 		true = ets:insert(wterl_ets, {Monitor, Caller}),
@@ -164,7 +164,7 @@ code_change(_OldVsn, State, _Extra) ->
 do_close(undefined) ->
     ok;
 do_close(ConnRef) ->
-    wterl:conn_close(ConnRef).
+    wterl:connection_close(ConnRef).
 
 
 -ifdef(TEST).