From 3ef48e0ac02df1df3b56e6090e91b0a4219bc092 Mon Sep 17 00:00:00 2001 From: Gregory Burd Date: Thu, 28 Mar 2013 10:24:26 -0400 Subject: [PATCH] Change from static linking to dynamic. Enable snappy and bzip2 compression. --- c_src/build_deps.sh | 6 +++- c_src/wterl.c | 2 +- rebar.config | 2 +- src/riak_kv_wterl_backend.erl | 14 ++++---- src/wterl.erl | 65 ++++++++++++++++++++++------------- src/wterl_conn.erl | 4 +-- 6 files changed, 57 insertions(+), 36 deletions(-) diff --git a/c_src/build_deps.sh b/c_src/build_deps.sh index 371c99f..cab9e4a 100755 --- a/c_src/build_deps.sh +++ b/c_src/build_deps.sh @@ -26,7 +26,11 @@ case "$1" in fi (cd wiredtiger/build_posix && \ ../configure --with-pic \ + --enable-snappy \ + --enable-bzip2 \ --prefix=$BASEDIR/system && \ - make -j 8 && make install) + make -j && make install) + cp system/bin/wt ../priv + cp system/lib/*.so ../priv ;; esac diff --git a/c_src/wterl.c b/c_src/wterl.c index 5e75998..2be9161 100644 --- a/c_src/wterl.c +++ b/c_src/wterl.c @@ -90,7 +90,7 @@ static ERL_NIF_TERM wterl_session_verify(ErlNifEnv* env, int argc, const ERL_NIF static ErlNifFunc nif_funcs[] = { - {"conn_close", 1, wterl_conn_close}, + {"connection_close", 1, wterl_conn_close}, {"conn_open", 2, wterl_conn_open}, {"cursor_close", 1, wterl_cursor_close}, {"cursor_insert", 3, wterl_cursor_insert}, diff --git a/rebar.config b/rebar.config index dc2f7eb..701571a 100644 --- a/rebar.config +++ b/rebar.config @@ -37,7 +37,7 @@ {port_env, [ {"DRV_CFLAGS", "$DRV_CFLAGS -Werror -I c_src/system/include"}, - {"DRV_LDFLAGS", "$DRV_LDFLAGS c_src/system/lib/libwiredtiger.a"} + {"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath -Wl,priv -Lpriv -lwiredtiger"} ]}. {pre_hooks, [{compile, "c_src/build_deps.sh"}]}. diff --git a/src/riak_kv_wterl_backend.erl b/src/riak_kv_wterl_backend.erl index dc37c02..2dc429c 100644 --- a/src/riak_kv_wterl_backend.erl +++ b/src/riak_kv_wterl_backend.erl @@ -85,7 +85,6 @@ capabilities(_, _) -> %% @doc Start the wterl backend -spec start(integer(), config()) -> {ok, state()} | {error, term()}. start(Partition, Config) -> - %lager:start(), AppStart = case application:start(wterl) of ok -> @@ -355,13 +354,13 @@ establish_connection(Config) -> wterl:config_value(session_max, Config, max_sessions(Config)), wterl:config_value(cache_size, Config, size_cache(RequestedCacheSize)), wterl:config_value(statistics_log, Config, [{wait, 30}]), % sec - wterl:config_value(checkpoint, Config, [{wait, 10}]), % sec + %% NOTE: LSM auto-checkpoints, so we don't have too. + %% wterl:config_value(checkpoint, Config, [{wait, 10}]), % sec wterl:config_value(verbose, Config, [ %"ckpt" "block", "shared_cache", "evictserver", "fileops", %"hazard", "mutex", "read", "readserver", "reconcile", %"salvage", "verify", "write", "evict", "lsm" ]) ] ++ proplists:get_value(wterl, Config, [])), % sec - %lager:info("WiredTiger connection:open(~s, ~s)", [DataRoot, wterl:config_to_bin(Opts)]), case wterl_conn:open(DataRoot, Opts) of {ok, Connection} -> {ok, Connection}; @@ -384,13 +383,15 @@ establish_session(Connection, Table) -> case wterl:session_open(Connection, wterl:config_to_bin([{isolation, "snapshot"}])) of {ok, Session} -> SessionOpts = - [%TODO {block_compressor, "snappy"}, + [{block_compressor, "snappy"}, {internal_page_max, "128K"}, {leaf_page_max, "128K"}, - {lsm_chunk_size, "200MB"}, + {lsm_chunk_size, "25MB"}, {lsm_bloom_newest, true}, {lsm_bloom_oldest, true} , - {lsm_bloom_config, [{leaf_page_max, "10MB"}]} ], + {lsm_bloom_bit_count, 128}, + {lsm_bloom_hash_count, 64}, + {lsm_bloom_config, [{leaf_page_max, "8MB"}]} ], case wterl:session_create(Session, Table, wterl:config_to_bin(SessionOpts)) of ok -> {ok, Session}; @@ -548,7 +549,6 @@ size_cache(RequestedSize) -> "1GB" end, application:set_env(wterl, cache_size, FinalGuess), - %lager:info("Using cache size of ~p for WiredTiger storage backend.", [FinalGuess]), FinalGuess; Value when is_list(Value) -> Value; diff --git a/src/wterl.erl b/src/wterl.erl index dc19fd5..a717176 100644 --- a/src/wterl.erl +++ b/src/wterl.erl @@ -20,8 +20,8 @@ %% %% ------------------------------------------------------------------- -module(wterl). --export([conn_open/2, - conn_close/1, +-export([connection_open/2, + connection_close/1, cursor_close/1, cursor_insert/3, cursor_next/1, @@ -60,6 +60,7 @@ session_verify/3, config_value/3, config_to_bin/1, + priv_dir/0, fold_keys/3, fold/3]). @@ -92,22 +93,23 @@ nif_stub_error(Line) -> -spec init() -> ok | {error, any()}. init() -> - PrivDir = case code:priv_dir(?MODULE) of - {error, bad_name} -> - EbinDir = filename:dirname(code:which(?MODULE)), - AppPath = filename:dirname(EbinDir), - filename:join(AppPath, "priv"); - Path -> - Path - end, - erlang:load_nif(filename:join(PrivDir, atom_to_list(?MODULE)), 0). + erlang:load_nif(filename:join(priv_dir(), atom_to_list(?MODULE)), 0). + +-spec connection_open(string(), config()) -> {ok, connection()} | {error, term()}. +connection_open(HomeDir, Config) -> + PrivDir = wterl:priv_dir(), + Bin = config_to_bin([{extensions, + [filename:join(PrivDir, "libwiredtiger_snappy.so"), + filename:join(PrivDir, "libwiredtiger_bzip2.so")]}], + [<<",">>, Config]), + conn_open(HomeDir, Bin). -spec conn_open(string(), config()) -> {ok, connection()} | {error, term()}. conn_open(_HomeDir, _Config) -> ?nif_stub. --spec conn_close(connection()) -> ok | {error, term()}. -conn_close(_ConnRef) -> +-spec connection_close(connection()) -> ok | {error, term()}. +connection_close(_ConnRef) -> ?nif_stub. -spec session_open(connection()) -> {ok, session()} | {error, term()}. @@ -266,11 +268,21 @@ fold(_Cursor, _Fun, Acc, not_found) -> fold(Cursor, Fun, Acc, {ok, Key, Value}) -> fold(Cursor, Fun, Fun({Key, Value}, Acc), cursor_next(Cursor)). +priv_dir() -> + case code:priv_dir(?MODULE) of + {error, bad_name} -> + EbinDir = filename:dirname(code:which(?MODULE)), + AppPath = filename:dirname(EbinDir), + filename:join(AppPath, "priv"); + Path -> + Path + end. + %% %% Configuration type information. %% config_types() -> - [{block_compressor, string}, + [{block_compressor, {string, quoted}}, {cache_size, string}, {checkpoint, config}, {create, bool}, @@ -279,7 +291,7 @@ config_types() -> {error_prefix, string}, {eviction_target, integer}, {eviction_trigger, integer}, - {extensions, string}, + {extensions, {list, quoted}}, {force, bool}, {hazard_max, integer}, {home_environment, bool}, @@ -320,8 +332,13 @@ config_encode(config, Value) -> list_to_binary(["(", config_to_bin(Value, []), ")"]); config_encode(list, Value) -> list_to_binary(["(", string:join(Value, ","), ")"]); +config_encode({list, quoted}, Value) -> + Values = lists:map(fun(S) -> "\"" ++ S ++ "\"" end, Value), + list_to_binary(["(", string:join(Values, ","), ")"]); config_encode(string, Value) when is_list(Value) -> list_to_binary(Value); +config_encode({string, quoted}, Value) when is_list(Value) -> + list_to_binary("\"" ++ Value ++ "\""); config_encode(string, Value) when is_number(Value) -> list_to_binary(integer_to_list(Value)); config_encode(bool, true) -> @@ -373,19 +390,19 @@ config_to_bin([{Key, Value} | Rest], Acc) -> open_test_conn(DataDir) -> ?assertCmd("rm -rf "++DataDir), ?assertMatch(ok, filelib:ensure_dir(filename:join(DataDir, "x"))), - OpenConfig = config_to_bin([{create,true},{cache_size,"100MB"}]), - {ok, ConnRef} = conn_open(DataDir, OpenConfig), + OpenConfig = config_to_bin([{create,true}, {cache_size,"100MB"}]), + {ok, ConnRef} = connection_open(DataDir, OpenConfig), ConnRef. open_test_session(ConnRef) -> {ok, SRef} = session_open(ConnRef), ?assertMatch(ok, session_drop(SRef, "table:test", config_to_bin([{force,true}]))), - ?assertMatch(ok, session_create(SRef, "table:test")), + ?assertMatch(ok, session_create(SRef, "table:test", config_to_bin([{block_compressor, "snappy"}]))), SRef. conn_test() -> ConnRef = open_test_conn(?TEST_DATA_DIR), - ?assertMatch(ok, conn_close(ConnRef)). + ?assertMatch(ok, connection_close(ConnRef)). session_test_() -> {setup, @@ -393,7 +410,7 @@ session_test_() -> open_test_conn(?TEST_DATA_DIR) end, fun(ConnRef) -> - ok = conn_close(ConnRef) + ok = connection_close(ConnRef) end, fun(ConnRef) -> {inorder, @@ -418,7 +435,7 @@ insert_delete_test() -> ?assertMatch(ok, session_delete(SRef, "table:test", <<"a">>)), ?assertMatch(not_found, session_get(SRef, "table:test", <<"a">>)), ok = session_close(SRef), - ok = conn_close(ConnRef). + ok = connection_close(ConnRef). init_test_table() -> ConnRef = open_test_conn(?TEST_DATA_DIR), @@ -432,7 +449,7 @@ init_test_table() -> stop_test_table({ConnRef, SRef}) -> ?assertMatch(ok, session_close(SRef)), - ?assertMatch(ok, conn_close(ConnRef)). + ?assertMatch(ok, connection_close(ConnRef)). various_session_test_() -> {setup, @@ -642,7 +659,7 @@ prop_put_delete() -> ?cmd("rm -rf "++DataDir), ok = filelib:ensure_dir(filename:join(DataDir, "x")), Cfg = wterl:config_to_bin([{create,true}]), - {ok, Conn} = wterl:conn_open(DataDir, Cfg), + {ok, Conn} = wterl:connection_open(DataDir, Cfg), {ok, SRef} = wterl:session_open(Conn), try wterl:session_create(SRef, Table), @@ -658,7 +675,7 @@ prop_put_delete() -> true after wterl:session_close(SRef), - wterl:conn_close(Conn) + wterl:connection_close(Conn) end end)). diff --git a/src/wterl_conn.erl b/src/wterl_conn.erl index afe91ee..4a680d0 100644 --- a/src/wterl_conn.erl +++ b/src/wterl_conn.erl @@ -82,7 +82,7 @@ init([]) -> handle_call({open, Dir, Config, Caller}, _From, #state{conn=undefined}=State) -> {Reply, NState} = - case wterl:conn_open(Dir, wterl:config_to_bin(Config)) of + case wterl:connection_open(Dir, wterl:config_to_bin(Config)) of {ok, ConnRef}=OK -> Monitor = erlang:monitor(process, Caller), true = ets:insert(wterl_ets, {Monitor, Caller}), @@ -164,7 +164,7 @@ code_change(_OldVsn, State, _Extra) -> do_close(undefined) -> ok; do_close(ConnRef) -> - wterl:conn_close(ConnRef). + wterl:connection_close(ConnRef). -ifdef(TEST).