Add new mmap config option.

Changes related to Riak 2.0 and an issue with how statistics were gathered from the backend

c_src/build_deps.sh

@@ -13,7 +13,7 @@ set -e
 WT_REPO=http://github.com/wiredtiger/wiredtiger.git
 WT_BRANCH=develop
 WT_DIR=wiredtiger-`basename $WT_BRANCH`
-#WT_REF="tags/1.6.4"
+#WT_REF="tags/1.6.6"
 #WT_DIR=wiredtiger-`basename $WT_REF`
 
 SNAPPY_VSN="1.0.4"

c_src/wterl.c

@@ -440,6 +440,7 @@ __close_cursors_on(WterlConnHandle *conn_handle, const char *uri)
     return;
 }
 
+
 /**
  * Callback to handle error messages.
  *
@@ -454,13 +455,15 @@ __close_cursors_on(WterlConnHandle *conn_handle, const char *uri)
  *          operation or library failure.
  */
 int
-__wterl_error_handler(WT_EVENT_HANDLER *handler, int error, const char *message)
+__wterl_error_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session,
+    int error, const char *message)
 {
     struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
     ErlNifEnv *msg_env;
     ErlNifPid *to_pid;
     int rc = 0;
 
+    UNUSED(session);
     enif_mutex_lock(eh->error_mutex);
     msg_env = eh->msg_env_error;
     to_pid = &eh->to_pid;
@@ -492,13 +495,14 @@ __wterl_error_handler(WT_EVENT_HANDLER *handler, int error, const char *message)
  *          operation or library failure.
  */
 int
-__wterl_message_handler(WT_EVENT_HANDLER *handler, const char *message)
+__wterl_message_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message)
 {
     struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
     ErlNifEnv *msg_env;
     ErlNifPid *to_pid;
     int rc = 0;
 
+    UNUSED(session);
     enif_mutex_lock(eh->message_mutex);
     msg_env = eh->msg_env_message;
     to_pid = &eh->to_pid;
@@ -529,13 +533,14 @@ __wterl_message_handler(WT_EVENT_HANDLER *handler, const char *message)
  *          operation or library failure.
  */
 int
-__wterl_progress_handler(WT_EVENT_HANDLER *handler, const char *operation, uint64_t counter)
+__wterl_progress_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *operation, uint64_t counter)
 {
     struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
     ErlNifEnv *msg_env;
     ErlNifPid *to_pid;
     int rc = 0;
 
+    UNUSED(session);
     enif_mutex_lock(eh->progress_mutex);
     msg_env = eh->msg_env_progress;
     to_pid = &eh->to_pid;
@@ -2303,7 +2308,7 @@ on_load(ErlNifEnv *env, void **priv_data, ERL_NIF_TERM load_info)
 
     char msg[1024];
     snprintf(msg, 1024, "NIF on_load complete (wterl version: %s, wiredtiger version: %s)", priv->wterl_vsn, priv->wiredtiger_vsn);
-    __wterl_message_handler((WT_EVENT_HANDLER *)&priv->eh, msg);
+    __wterl_message_handler((WT_EVENT_HANDLER *)&priv->eh, NULL, msg);
     return 0;
 }
 

src/riak_kv_wterl_backend.erl

@@ -22,6 +22,7 @@
 
 -module(riak_kv_wterl_backend).
 -behavior(temp_riak_kv_backend).
+-compile([{parse_transform, lager_transform}]).
 
 %% KV Backend API
 -export([api_version/0,
@@ -42,7 +43,7 @@
 
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
--compiel(export_all).
+-compile(export_all).
 -endif.
 
 -define(API_VERSION, 1).
@@ -119,11 +120,14 @@ start(Partition, Config) ->
                     "lsm" ->
                         [{internal_page_max, "128K"},
                          {leaf_page_max, "16K"},
-                         {lsm_chunk_size, "100MB"},
+                         {lsm, [
-                         {lsm_bloom_oldest, true} ,
+                             {bloom_config, [{leaf_page_max, "8MB"}]},
-                         {lsm_bloom_bit_count, 28},
+                             {bloom_bit_count, 28},
-                         {lsm_bloom_hash_count, 19},
+                             {bloom_hash_count, 19},
-                         {lsm_bloom_config, [{leaf_page_max, "8MB"}]}
+                             {bloom_oldest, true},
+                             {chunk_size, "100MB"},
+                             {merge_threads, 2}
+                         ]}
                         ] ++ Compressor;
                     "table" ->
                         Compressor
@@ -397,30 +401,41 @@ establish_connection(Config, Type) ->
             ok = filelib:ensure_dir(filename:join(DataRoot, "x")),
 
             %% WT Connection Options:
-            %% NOTE: LSM auto-checkpoints, so we don't have too.
+	    LogSetting = app_helper:get_prop_or_env(log, Config, wterl, false),
             CheckpointSetting =
                 case Type =:= "lsm" of
                     true ->
-                        [];
+			case LogSetting of
+			    true ->
+				%% Turn checkpoints on if logging is on, checkpoints enable log archival.
+				app_helper:get_prop_or_env(checkpoint, Config, wterl, [{wait, 30}]);  % in seconds
+			    _ ->
+				[]
+			end;
                     false ->
-                        app_helper:get_prop_or_env(checkpoint, Config, wterl, [{wait, 10}])
+                        app_helper:get_prop_or_env(checkpoint, Config, wterl, [{wait, 30}])
                 end,
             RequestedCacheSize = app_helper:get_prop_or_env(cache_size, Config, wterl),
             ConnectionOpts =
                 orddict:from_list(
                   [ wterl:config_value(create, Config, true),
                     wterl:config_value(checkpoint_sync, Config, false),
+                    wterl:config_value(transaction_sync, Config, "none"),
+                    wterl:config_value(log, Config, [{enabled, LogSetting}]),
+                    wterl:config_value(mmap, Config, false),
+                    wterl:config_value(checkpoint, Config, CheckpointSetting),
                     wterl:config_value(session_max, Config, max_sessions(Config)),
                     wterl:config_value(cache_size, Config, size_cache(RequestedCacheSize)),
+                    wterl:config_value(statistics, Config, [ "fast", "clear"]),
                     wterl:config_value(statistics_log, Config, [{wait, 600}]), % in seconds
                     wterl:config_value(verbose, Config, [ "salvage", "verify"
                          % Note: for some unknown reason, if you add these additional
                          % verbose flags Erlang SEGV's "size_object: bad tag for 0x80"
-                         % no idea why... yet... you've been warned.
+                         % no idea why... you've been warned.
                          %"block", "shared_cache", "reconcile", "evict", "lsm",
                          %"fileops", "read", "write", "readserver", "evictserver",
                          %"hazard", "mutex", "ckpt"
-                         ]) ] ++ CheckpointSetting ++ proplists:get_value(wterl, Config, [])), % sec
+                         ]) ] ++ proplists:get_value(wterl, Config, [])), % sec
 
             %% WT Session Options:
             SessionOpts = [{isolation, "snapshot"}],

src/wterl.erl

@@ -454,18 +454,26 @@ config_to_bin([], Acc) ->
 config_to_bin([{Key, Value} | Rest], Acc) ->
     ConfigTypes =
     [{block_compressor, {string, quoted}},
+     {bloom_bit_count, integer},
+     {bloom_config, config},
+     {bloom_hash_count, integer},
+     {bloom_newest, bool},
+     {bloom_oldest, bool},
      {cache_size, string},
      {checkpoint, config},
      {checkpoint_sync, bool},
      {checksum, string},
+     {chunk_size, string},
      {create, bool},
      {direct_io, list},
      {drop, list},
+     {enabled, bool},
      {error_prefix, string},
      {eviction_target, integer},
      {eviction_trigger, integer},
      {extensions, {list, quoted}},
      {statistics_fast, bool},
+     {file_max, string},
      {force, bool},
      {from, string},
      {hazard_max, integer},
@@ -475,23 +483,21 @@ config_to_bin([{Key, Value} | Rest], Acc) ->
      {isolation, string},
      {key_type, string},
      {leaf_page_max, string},
-     {logging, bool},
+     {log, config},
-     {lsm_bloom_bit_count, integer},
+     {lsm, config},
-     {lsm_bloom_config, config},
+     {mmap, bool},
-     {lsm_bloom_hash_count, integer},
+     {merge_threads, integer},
-     {lsm_bloom_newest, bool},
-     {lsm_bloom_oldest, bool},
-     {lsm_chunk_size, string},
-     {prefix_compression, bool},
-     {lsm_merge_threads, integer},
      {multiprocess, bool},
      {name, string},
      {overwrite, bool},
+     {prefix_compression, bool},
      {raw, bool},
      {session_max, integer},
+     {statistics, list},
      {statistics_log, config},
      {target, {list, quoted}},
      {to, string},
+     {transaction_sync, string},
      {transactional, bool},
      {verbose, list},
      {wait, integer}],

tools/wterl-b_b.config

@@ -25,7 +25,7 @@
 
 {mode, max}.
 {duration, 10}.
-{concurrent, 4}.
+{concurrent, 16}.
 {report_interval, 1}.
 {pb_timeout_general, 1000}. % ms
 %{pb_timeout_read, ?}.
@@ -43,7 +43,9 @@
 {wterl, [
         {connection, [
                       {create, true},
-                      {sync, false},
+                      {session_sync, false},
+		      {transaction_sync, "none"},
+		      {log, [{enabled, false}]},
                       {session_max, 1024},
                       {cache_size, 4294967296},
                       {verbose, []},
@@ -58,11 +60,11 @@
                      ]},
         {session, [ {isolation, "snapshot"} ]},
         {table_uri, "lsm:test"},
+	{lsm_merge_threads, 2},
         {table, [
                  {internal_page_max, "128K"},
                  {leaf_page_max, "128K"},
                  {lsm_chunk_size, "25MB"},
-                 {prefix_compression, false},
                  {lsm_bloom_newest, true},
                  {lsm_bloom_oldest, true} ,
                  {lsm_bloom_bit_count, 128},
@@ -76,7 +78,9 @@
 {wterl_, [
         {connection, [
                       {create, true},
-                      {sync, false},
+                      {session_sync, false},
+		      {transaction_sync, "none"},
+		      {log, [{enabled, false}]},
                       {session_max, 1024},
                       {cache_size, 4294967296},
                       {verbose, []},
@@ -93,7 +97,6 @@
         {session, [ {isolation, "snapshot"} ]},
         {table_uri, "table:test"},
         {table, [
-                 {prefix_compression, false},
                  {block_compressor, "snappy"} % bzip2
                 ]}
         ]}.