From 17585a99b14b656fb2373b49b390e2ad69386da3 Mon Sep 17 00:00:00 2001 From: Gregory Burd Date: Wed, 2 Oct 2013 14:38:41 -0400 Subject: [PATCH 1/6] priv now has the schema file in it, so be more specific with what we ignore in that dir --- .gitignore | 4 +++- priv/wterl.schema | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 priv/wterl.schema diff --git a/.gitignore b/.gitignore index b312112..38c0cdc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ c_src/*.o c_src/bzip2-1.0.6 c_src/snappy-1.0.4 deps/ -priv/ +priv/wt +priv/*.so* +priv/*.dylib* log/ *~ diff --git a/priv/wterl.schema b/priv/wterl.schema new file mode 100644 index 0000000..0d93e58 --- /dev/null +++ b/priv/wterl.schema @@ -0,0 +1,6 @@ +%%%% This is the WiredTiger section + +%% @doc wiredtiger data_root +{mapping, "wiredtiger.data_root", "wterl.data_root", [ + {default, "{{platform_data_dir}}/wiredtiger"} +]}. From 9d2896016bce60efc91005abe90f663201dc4a1c Mon Sep 17 00:00:00 2001 From: Gregory Burd Date: Wed, 2 Oct 2013 14:41:06 -0400 Subject: [PATCH 2/6] A few build automation changes/fixes. --- Makefile | 4 +--- c_src/build_deps.sh | 30 ++++++++++++------------------ rebar.config | 4 ++-- 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index 8e86444..83bd55b 100644 --- a/Makefile +++ b/Makefile @@ -52,19 +52,17 @@ endif .PHONY: all compile doc clean test dialyzer typer shell distclean pdf \ update-deps clean-common-test-data rebuild -all: deps compile test +all: deps compile # ============================================================================= # Rules to build the system # ============================================================================= deps: - c_src/build_deps.sh get-deps $(REBAR) get-deps $(REBAR) compile update-deps: - c_src/build_deps.sh update-deps $(REBAR) update-deps $(REBAR) compile diff --git a/c_src/build_deps.sh b/c_src/build_deps.sh index a435913..505967d 100755 --- a/c_src/build_deps.sh +++ b/c_src/build_deps.sh @@ -11,10 +11,10 @@ unset POSIX_SHELL # clear it so if we invoke other scripts, they run as ksh as w set -e WT_REPO=http://github.com/wiredtiger/wiredtiger.git -#WT_BRANCH=develop -#WT_DIR=wiredtiger-`basename $WT_BRANCH` -WT_REF="tags/1.6.4" -WT_DIR=wiredtiger-`basename $WT_REF` +WT_BRANCH=develop +WT_DIR=wiredtiger-`basename $WT_BRANCH` +#WT_REF="tags/1.6.4" +#WT_DIR=wiredtiger-`basename $WT_REF` SNAPPY_VSN="1.0.4" SNAPPY_DIR=snappy-$SNAPPY_VSN @@ -26,8 +26,7 @@ export BASEDIR="$PWD" which gmake 1>/dev/null 2>/dev/null && MAKE=gmake MAKE=${MAKE:-make} -export CFLAGS="$CFLAGS -I $BASEDIR/system/include" -export CXXFLAGS="$CXXFLAGS -I $BASEDIR/system/include" +export CPPFLAGS="$CPPLAGS -I $BASEDIR/system/include -O3 -mtune=native -march=native" export LDFLAGS="$LDFLAGS -L$BASEDIR/system/lib" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$BASEDIR/system/lib:$LD_LIBRARY_PATH" @@ -58,7 +57,7 @@ get_wt () wt_configure () { (cd $BASEDIR/$WT_DIR/build_posix - CFLAGS+=-g ../configure --with-pic \ + CFLAGS+=-g $BASEDIR/$WT_DIR/configure --with-pic \ --enable-snappy \ --prefix=${BASEDIR}/system || exit 1) } @@ -107,11 +106,6 @@ build_snappy () ) } -install_shared_lib () -{ - cp -p -P $1 ${BASEDIR}/../priv/`basename ${1%.*}.so` -} - case "$1" in clean) [ -e $BASEDIR/$WT_DIR/build_posix/Makefile ] && \ @@ -147,12 +141,12 @@ case "$1" in # Build WiredTiger [ -d $WT_DIR ] || get_wt; [ -d $BASEDIR/$WT_DIR ] || (echo "Missing WiredTiger source directory" && exit 1) - test -f $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].$SUFFIXES -a \ - -f $BASEDIR/system/lib/libwiredtiger_snappy.$SUFFIXES || build_wt; + test -f $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].${SUFFIXES} -a \ + -f $BASEDIR/system/lib/libwiredtiger_snappy.${SUFFIXES} || build_wt; [ -d $BASEDIR/../priv ] || mkdir ${BASEDIR}/../priv - cp -p -P $BASEDIR/system/bin/wt ${BASEDIR}/../priv - install_shared_lib ${BASEDIR}/system/lib/libwiredtiger-[0-9].[0-9].[0-9].$SUFFIXES - install_shared_lib ${BASEDIR}/system/lib/libwiredtiger_snappy.$SUFFIXES* - install_shared_lib ${BASEDIR}/system/lib/libsnappy.$SUFFIXES* + cp -p -P $BASEDIR/system/bin/wt ${BASEDIR}/../priv + cp -p -P ${BASEDIR}/system/lib/libwiredtiger-[0-9].[0-9].[0-9].${SUFFIXES} ${BASEDIR}/../priv + cp -p -P ${BASEDIR}/system/lib/libwiredtiger_snappy.${SUFFIXES} ${BASEDIR}/../priv + cp -p -P ${BASEDIR}/system/lib/libsnappy.${SUFFIXES}* ${BASEDIR}/../priv ;; esac diff --git a/rebar.config b/rebar.config index 52c1c94..ecdc6ea 100644 --- a/rebar.config +++ b/rebar.config @@ -38,8 +38,8 @@ {port_specs, [{"priv/wterl.so", ["c_src/*.c"]}]}. {port_env, [ - {"DRV_CFLAGS", "$DRV_CFLAGS -fPIC -Wall -Wextra -Werror -I c_src/system/include"}, - {"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath,lib/wterl/priv:priv -Lc_src/system/lib -lwiredtiger"} + {"DRV_CFLAGS", "$DRV_CFLAGS -fPIC -Wall -Wextra -Werror -O3 -march=native -mtune=native -I c_src/system/include"}, + {"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath,lib/wterl/priv:lib/wterl-0.9.0/priv:priv -Lc_src/system/lib -lwiredtiger"} ]}. {pre_hooks, [{compile, "c_src/build_deps.sh compile"}]}. From 84a85bbe38c238ee2cb19046a40730fd05be0ef1 Mon Sep 17 00:00:00 2001 From: Gregory Burd Date: Wed, 2 Oct 2013 14:41:36 -0400 Subject: [PATCH 3/6] Open a *statistics* cursor when gathering statistics. --- src/riak_kv_wterl_backend.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/riak_kv_wterl_backend.erl b/src/riak_kv_wterl_backend.erl index c075317..e1a2e3d 100644 --- a/src/riak_kv_wterl_backend.erl +++ b/src/riak_kv_wterl_backend.erl @@ -341,7 +341,7 @@ is_empty(#state{connection=Connection, table=Table}) -> %% @doc Get the status information for this wterl backend -spec status(state()) -> [{atom(), term()}]. status(#state{connection=Connection, table=Table}) -> - case wterl:cursor_open(Connection, Table) of + case wterl:cursor_open(Connection, "statistics:" ++ Table, [{statistics_fast, true}]) of {ok, Cursor} -> TheStats = case fetch_status(Cursor) of From 7d0ad2dce10b4d35f8e734c1dd6e8c133e358d89 Mon Sep 17 00:00:00 2001 From: Gregory Burd Date: Wed, 2 Oct 2013 14:42:26 -0400 Subject: [PATCH 4/6] Update the version strings and a few config values which changed names. --- src/riak_kv_wterl_backend.erl | 2 +- src/wterl.erl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/riak_kv_wterl_backend.erl b/src/riak_kv_wterl_backend.erl index e1a2e3d..82682b9 100644 --- a/src/riak_kv_wterl_backend.erl +++ b/src/riak_kv_wterl_backend.erl @@ -411,7 +411,7 @@ establish_connection(Config, Type) -> ConnectionOpts = orddict:from_list( [ wterl:config_value(create, Config, true), - wterl:config_value(sync, Config, false), + wterl:config_value(checkpoint_sync, Config, false), wterl:config_value(session_max, Config, max_sessions(Config)), wterl:config_value(cache_size, Config, size_cache(RequestedCacheSize)), wterl:config_value(statistics_log, Config, [{wait, 600}]), % in seconds diff --git a/src/wterl.erl b/src/wterl.erl index 952bb61..bec41ed 100644 --- a/src/wterl.erl +++ b/src/wterl.erl @@ -96,8 +96,8 @@ nif_stub_error(Line) -> -spec init() -> ok | {error, any()}. init() -> erlang:load_nif(filename:join([priv_dir(), atom_to_list(?MODULE)]), - [{wterl_vsn, "53307e8"}, - {wiredtiger_vsn, "1.6.2-0-g07cb0a5"}]). + [{wterl_vsn, "942e51b"}, + {wiredtiger_vsn, "1.6.4-275-g9c44420"}]). %% TODO automate these -spec connection_open(string(), config_list()) -> {ok, connection()} | {error, term()}. -spec connection_open(string(), config_list(), config_list()) -> {ok, connection()} | {error, term()}. @@ -456,6 +456,7 @@ config_to_bin([{Key, Value} | Rest], Acc) -> [{block_compressor, {string, quoted}}, {cache_size, string}, {checkpoint, config}, + {checkpoint_sync, bool}, {checksum, string}, {create, bool}, {direct_io, list}, @@ -489,7 +490,6 @@ config_to_bin([{Key, Value} | Rest], Acc) -> {raw, bool}, {session_max, integer}, {statistics_log, config}, - {sync, bool}, {target, {list, quoted}}, {to, string}, {transactional, bool}, From 75305dae941469fc2d581f15098c262d9b2f6228 Mon Sep 17 00:00:00 2001 From: Gregory Burd Date: Sat, 12 Oct 2013 21:48:05 -0400 Subject: [PATCH 5/6] Minor updates. --- c_src/wiredtiger-build.patch | 98 +----------------------------------- rebar.config | 2 +- 2 files changed, 3 insertions(+), 97 deletions(-) diff --git a/c_src/wiredtiger-build.patch b/c_src/wiredtiger-build.patch index 22628e6..8932c20 100644 --- a/c_src/wiredtiger-build.patch +++ b/c_src/wiredtiger-build.patch @@ -1,106 +1,12 @@ diff --git a/ext/compressors/snappy/Makefile.am b/ext/compressors/snappy/Makefile.am -index 6d78823..2122cf8 100644 +index 6d78823..c423590 100644 --- a/ext/compressors/snappy/Makefile.am +++ b/ext/compressors/snappy/Makefile.am @@ -2,5 +2,6 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include - + lib_LTLIBRARIES = libwiredtiger_snappy.la libwiredtiger_snappy_la_SOURCES = snappy_compress.c -libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module +libwiredtiger_snappy_la_CFLAGS = -I$(abs_top_builddir)/../../system/include +libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(abs_top_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv libwiredtiger_snappy_la_LIBADD = -lsnappy -diff --git a/src/support/cksum.c b/src/support/cksum.c -index 7e9befe..b924db7 100644 ---- a/src/support/cksum.c -+++ b/src/support/cksum.c -@@ -27,6 +27,13 @@ - - #include "wt_internal.h" - -+#if defined(__amd64) || defined(__x86_64) -+#define USE_HARDWARE_CRC32 1 -+#else -+#undef USE_HARDWARE_CRC32 -+#endif -+ -+#ifdef USE_HARDWARE_CRC32 - static const uint32_t g_crc_slicing[8][256] = { - #ifdef WORDS_BIGENDIAN - /* -@@ -1078,6 +1085,7 @@ static const uint32_t g_crc_slicing[8][256] = { - } - #endif - }; -+#endif /* USE_HARDWARE_CRC32 */ - - /* - * __wt_cksum -- -@@ -1106,15 +1114,29 @@ __wt_cksum(const void *chunk, size_t len) - /* Checksum one byte at a time to the first 4B boundary. */ - for (p = chunk; - ((uintptr_t)p & (sizeof(uint32_t) - 1)) != 0 && -- len > 0; ++p, --len) -+ len > 0; ++p, --len) { -+#ifdef USE_HARDWARE_CRC32 -+ __asm__ __volatile__( -+ ".byte 0xF2, 0x0F, 0x38, 0xF0, 0xF1" -+ : "=S" (crc) -+ : "0" (crc), "c" (*p)); -+#else - #ifdef WORDS_BIGENDIAN - crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8); - #else - crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8); - #endif -+#endif -+ } - - /* Checksum in 8B chunks. */ - for (nqwords = len / sizeof(uint64_t); nqwords; nqwords--) { -+#ifdef USE_HARDWARE_CRC32 -+ __asm__ __volatile__ ( -+ ".byte 0xf2, 0x48, 0x0f, 0x38, 0xf0, 0xf1;" -+ : "=S"(crc) -+ : "S"(crc), "c"(*p)); -+#else - crc ^= *(uint32_t *)p; - p += sizeof(uint32_t); - next = *(uint32_t *)p; -@@ -1139,22 +1161,32 @@ __wt_cksum(const void *chunk, size_t len) - g_crc_slicing[1][(next >> 16) & 0xFF] ^ - g_crc_slicing[0][(next >> 24)]; - #endif -+#endif - } - - /* Checksum trailing bytes one byte at a time. */ -+ for (len &= 0x7; len > 0; ++p, len--) { -+#ifdef USE_HARDWARE_CRC32 -+ __asm__ __volatile__( -+ ".byte 0xF2, 0x0F, 0x38, 0xF0, 0xF1" -+ : "=S" (crc) -+ : "0" (crc), "c" (*p)); -+#else - #ifdef WORDS_BIGENDIAN -- for (len &= 0x7; len > 0; ++p, len--) - crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8); -+#else -+ crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8); -+#endif -+#endif -+ } - -+#ifdef WORDS_BIGENDIAN - /* Do final byte swap to produce a result identical to little endian */ - crc = - ((crc << 24) & 0xFF000000) | - ((crc << 8) & 0x00FF0000) | - ((crc >> 8) & 0x0000FF00) | - ((crc >> 24) & 0x000000FF); --#else -- for (len &= 0x7; len > 0; ++p, len--) -- crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8); - #endif - return (~crc); - } diff --git a/rebar.config b/rebar.config index ecdc6ea..d1c7022 100644 --- a/rebar.config +++ b/rebar.config @@ -38,7 +38,7 @@ {port_specs, [{"priv/wterl.so", ["c_src/*.c"]}]}. {port_env, [ - {"DRV_CFLAGS", "$DRV_CFLAGS -fPIC -Wall -Wextra -Werror -O3 -march=native -mtune=native -I c_src/system/include"}, + {"DRV_CFLAGS", "$DRV_CFLAGS -O3 -mtune=native -march=native -fPIC -Wall -Wextra -Werror -I c_src/system/include"}, {"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath,lib/wterl/priv:lib/wterl-0.9.0/priv:priv -Lc_src/system/lib -lwiredtiger"} ]}. From ac2c5caeff1003fd6ab7c4f011a498451491c00d Mon Sep 17 00:00:00 2001 From: Gregory Burd Date: Wed, 30 Oct 2013 11:50:20 -0400 Subject: [PATCH 6/6] Change a few default configs and comment out the stats gathering for now. --- src/riak_kv_wterl_backend.erl | 54 +++++++++++++++++------------------ 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/src/riak_kv_wterl_backend.erl b/src/riak_kv_wterl_backend.erl index 82682b9..9d9c3eb 100644 --- a/src/riak_kv_wterl_backend.erl +++ b/src/riak_kv_wterl_backend.erl @@ -120,9 +120,6 @@ start(Partition, Config) -> [{internal_page_max, "128K"}, {leaf_page_max, "16K"}, {lsm_chunk_size, "100MB"}, - {lsm_merge_threads, 2}, - {prefix_compression, true}, - {lsm_bloom_newest, true}, {lsm_bloom_oldest, true} , {lsm_bloom_bit_count, 28}, {lsm_bloom_hash_count, 19}, @@ -341,22 +338,23 @@ is_empty(#state{connection=Connection, table=Table}) -> %% @doc Get the status information for this wterl backend -spec status(state()) -> [{atom(), term()}]. status(#state{connection=Connection, table=Table}) -> - case wterl:cursor_open(Connection, "statistics:" ++ Table, [{statistics_fast, true}]) of - {ok, Cursor} -> - TheStats = - case fetch_status(Cursor) of - {ok, Stats} -> - Stats; - {error, {eperm, _}} -> % TODO: review/fix this logic - {ok, []}; - _ -> - {ok, []} - end, - wterl:cursor_close(Cursor), - TheStats; - {error, Reason2} -> - {error, Reason2} - end. + []. +%% case wterl:cursor_open(Connection, "statistics:" ++ Table, [{statistics_fast, true}]) of +%% {ok, Cursor} -> +%% TheStats = +%% case fetch_status(Cursor) of +%% {ok, Stats} -> +%% Stats; +%% {error, {eperm, _}} -> % TODO: review/fix this logic +%% {ok, []}; +%% _ -> +%% {ok, []} +%% end, +%% wterl:cursor_close(Cursor), +%% TheStats; +%% {error, Reason2} -> +%% {error, Reason2} +%% end. %% @doc Register an asynchronous callback -spec callback(reference(), any(), state()) -> {ok, state()}. @@ -543,15 +541,15 @@ from_index_key(LKey) -> %% @private %% Return all status from wterl statistics cursor -fetch_status(Cursor) -> - {ok, fetch_status(Cursor, wterl:cursor_next_value(Cursor), [])}. -fetch_status(_Cursor, {error, _}, Acc) -> - lists:reverse(Acc); -fetch_status(_Cursor, not_found, Acc) -> - lists:reverse(Acc); -fetch_status(Cursor, {ok, Stat}, Acc) -> - [What,Val|_] = [binary_to_list(B) || B <- binary:split(Stat, [<<0>>], [global])], - fetch_status(Cursor, wterl:cursor_next_value(Cursor), [{What,Val}|Acc]). +%% fetch_status(Cursor) -> +%% {ok, fetch_status(Cursor, wterl:cursor_next_value(Cursor), [])}. +%% fetch_status(_Cursor, {error, _}, Acc) -> +%% lists:reverse(Acc); +%% fetch_status(_Cursor, not_found, Acc) -> +%% lists:reverse(Acc); +%% fetch_status(Cursor, {ok, Stat}, Acc) -> +%% [What,Val|_] = [binary_to_list(B) || B <- binary:split(Stat, [<<0>>], [global])], +%% fetch_status(Cursor, wterl:cursor_next_value(Cursor), [{What,Val}|Acc]). size_cache(RequestedSize) -> Size =