Start with a more rational default configuration. (Before you ask... the
answer is 'no'). So far there has been no effort to validate that these settings are in fact the best for Riak/KV or CS data access patterns. These particular settings are, at best, an educated guess based on past experience, the docs and reading about the benchmark the WiredTiger team published here: https://github.com/wiredtiger/wiredtiger/wiki/YCSB-Mapkeeper-benchmark
This commit is contained in:
parent
a8bc4bf6c0
commit
137088ff55
2 changed files with 50 additions and 9 deletions
|
@ -99,11 +99,30 @@ start(Partition, Config) ->
|
||||||
case AppStart of
|
case AppStart of
|
||||||
ok ->
|
ok ->
|
||||||
ok = filelib:ensure_dir(filename:join(DataRoot, "x")),
|
ok = filelib:ensure_dir(filename:join(DataRoot, "x")),
|
||||||
case wterl_conn:open(DataRoot, Config) of
|
ConnectionOpts = [Config,
|
||||||
|
{create, true},
|
||||||
|
{loging, true},
|
||||||
|
{transactional, true},
|
||||||
|
{direct_io, ["data", "log"]},
|
||||||
|
{session_max, 128},
|
||||||
|
{cache_size, "2GB"},
|
||||||
|
{sync, false},
|
||||||
|
{verbose,
|
||||||
|
["block", "shared_cache", "ckpt", "evict",
|
||||||
|
"evictserver", "fileops", "hazard", "lsm",
|
||||||
|
"mutex", "read", "readserver", "reconcile",
|
||||||
|
"salvage", "verify", "write"]}],
|
||||||
|
case wterl_conn:open(DataRoot, ConnectionOpts) of
|
||||||
{ok, ConnRef} ->
|
{ok, ConnRef} ->
|
||||||
Table = "table:wt" ++ integer_to_list(Partition),
|
Table = "table:wt" ++ integer_to_list(Partition),
|
||||||
{ok, SRef} = wterl:session_open(ConnRef),
|
SessionOpenOpts = [{isolation, "snapshot"}],
|
||||||
ok = wterl:session_create(SRef, Table),
|
{ok, SRef} = wterl:session_open(ConnRef, wterl:config_to_bin(SessionOpenOpts)),
|
||||||
|
SessionOpts = [%TODO {block_compressor, "snappy"},
|
||||||
|
{internal_page_max, "128K"},
|
||||||
|
{leaf_page_max, "128K"},
|
||||||
|
{lsm_chunk_size, "200MB"},
|
||||||
|
{lsm_bloom_config, [{leaf_page_max, "10MB"}]} ],
|
||||||
|
ok = wterl:session_create(SRef, Table, wterl:config_to_bin(SessionOpts)),
|
||||||
{ok, #state{conn=ConnRef,
|
{ok, #state{conn=ConnRef,
|
||||||
table=Table,
|
table=Table,
|
||||||
session=SRef,
|
session=SRef,
|
||||||
|
|
|
@ -46,6 +46,7 @@
|
||||||
session_drop/3,
|
session_drop/3,
|
||||||
session_get/3,
|
session_get/3,
|
||||||
session_open/1,
|
session_open/1,
|
||||||
|
session_open/2,
|
||||||
session_put/4,
|
session_put/4,
|
||||||
session_rename/3,
|
session_rename/3,
|
||||||
session_rename/4,
|
session_rename/4,
|
||||||
|
@ -57,6 +58,7 @@
|
||||||
session_upgrade/3,
|
session_upgrade/3,
|
||||||
session_verify/2,
|
session_verify/2,
|
||||||
session_verify/3,
|
session_verify/3,
|
||||||
|
config_value/3,
|
||||||
config_to_bin/1,
|
config_to_bin/1,
|
||||||
fold_keys/3,
|
fold_keys/3,
|
||||||
fold/3]).
|
fold/3]).
|
||||||
|
@ -78,7 +80,7 @@
|
||||||
-type key() :: binary().
|
-type key() :: binary().
|
||||||
-type value() :: binary().
|
-type value() :: binary().
|
||||||
|
|
||||||
-export_type([connection/0, session/0]).
|
-export_type([connection/0, session/0, cursor/0]).
|
||||||
|
|
||||||
-on_load(init/0).
|
-on_load(init/0).
|
||||||
|
|
||||||
|
@ -109,7 +111,11 @@ conn_close(_ConnRef) ->
|
||||||
?nif_stub.
|
?nif_stub.
|
||||||
|
|
||||||
-spec session_open(connection()) -> {ok, session()} | {error, term()}.
|
-spec session_open(connection()) -> {ok, session()} | {error, term()}.
|
||||||
session_open(_ConnRef) ->
|
session_open(ConnRef) ->
|
||||||
|
session_open(ConnRef, ?EMPTY_CONFIG).
|
||||||
|
|
||||||
|
-spec session_open(connection(), config()) -> {ok, session()} | {error, term()}.
|
||||||
|
session_open(_ConnRef, _Config) ->
|
||||||
?nif_stub.
|
?nif_stub.
|
||||||
|
|
||||||
-spec session_close(session()) -> ok | {error, term()}.
|
-spec session_close(session()) -> ok | {error, term()}.
|
||||||
|
@ -264,8 +270,10 @@ fold(Cursor, Fun, Acc, {ok, Key, Value}) ->
|
||||||
%% Configuration type information.
|
%% Configuration type information.
|
||||||
%%
|
%%
|
||||||
config_types() ->
|
config_types() ->
|
||||||
[{cache_size, string},
|
[{block_compressor, string},
|
||||||
|
{cache_size, string},
|
||||||
{create, bool},
|
{create, bool},
|
||||||
|
{direct_io, map},
|
||||||
{drop, list},
|
{drop, list},
|
||||||
{error_prefix, string},
|
{error_prefix, string},
|
||||||
{eviction_target, integer},
|
{eviction_target, integer},
|
||||||
|
@ -275,13 +283,23 @@ config_types() ->
|
||||||
{hazard_max, integer},
|
{hazard_max, integer},
|
||||||
{home_environment, bool},
|
{home_environment, bool},
|
||||||
{home_environment_priv, bool},
|
{home_environment_priv, bool},
|
||||||
|
{internal_page_max, string},
|
||||||
|
{isolation, string},
|
||||||
|
{key_type, string},
|
||||||
|
{leaf_page_max, string},
|
||||||
{logging, bool},
|
{logging, bool},
|
||||||
|
{lsm_bloom_config, config},
|
||||||
|
{lsm_chunk_size, string},
|
||||||
{multiprocess, bool},
|
{multiprocess, bool},
|
||||||
{name, string},
|
{name, string},
|
||||||
{session_max, integer},
|
{session_max, integer},
|
||||||
|
{sync, bool},
|
||||||
{target, list},
|
{target, list},
|
||||||
{transactional, bool},
|
{transactional, bool},
|
||||||
{verbose, string}].
|
{verbose, map}].
|
||||||
|
|
||||||
|
config_value(Key, Config, Default) ->
|
||||||
|
{Key, app_helper:get_prop_or_env(Key, Config, wterl, Default)}.
|
||||||
|
|
||||||
config_encode(integer, Value) ->
|
config_encode(integer, Value) ->
|
||||||
try
|
try
|
||||||
|
@ -290,8 +308,12 @@ config_encode(integer, Value) ->
|
||||||
_:_ ->
|
_:_ ->
|
||||||
invalid
|
invalid
|
||||||
end;
|
end;
|
||||||
|
config_encode(config, Value) ->
|
||||||
|
list_to_binary(["(", config_to_bin(Value, []), ")"]);
|
||||||
config_encode(list, Value) ->
|
config_encode(list, Value) ->
|
||||||
list_to_binary(["(", string:join(Value, ","), ")"]);
|
list_to_binary(["(", string:join(Value, ","), ")"]);
|
||||||
|
config_encode(map, Value) ->
|
||||||
|
list_to_binary(["[", string:join(Value, ","), "]"]);
|
||||||
config_encode(string, Value) ->
|
config_encode(string, Value) ->
|
||||||
list_to_binary(Value);
|
list_to_binary(Value);
|
||||||
config_encode(bool, true) ->
|
config_encode(bool, true) ->
|
||||||
|
@ -303,9 +325,9 @@ config_encode(_Type, _Value) ->
|
||||||
|
|
||||||
-spec config_to_bin(config_list()) -> config().
|
-spec config_to_bin(config_list()) -> config().
|
||||||
config_to_bin(Opts) ->
|
config_to_bin(Opts) ->
|
||||||
config_to_bin(Opts, []).
|
iolist_to_binary([config_to_bin(Opts, []), <<"\0">>]).
|
||||||
config_to_bin([], Acc) ->
|
config_to_bin([], Acc) ->
|
||||||
iolist_to_binary([Acc, ?EMPTY_CONFIG]);
|
iolist_to_binary(Acc);
|
||||||
config_to_bin([{Key, Value} | Rest], Acc) ->
|
config_to_bin([{Key, Value} | Rest], Acc) ->
|
||||||
case lists:keysearch(Key, 1, config_types()) of
|
case lists:keysearch(Key, 1, config_types()) of
|
||||||
{value, {Key, Type}} ->
|
{value, {Key, Type}} ->
|
||||||
|
|
Loading…
Reference in a new issue