Compare commits
20 commits
Author | SHA1 | Date | |
---|---|---|---|
|
cc807a97d6 | ||
|
ea99493ea3 | ||
|
08b2d18463 | ||
|
3302ab26ed | ||
|
db2daf99b2 | ||
|
68d9ed942b | ||
|
36faa4e713 | ||
|
e560185420 | ||
|
448c0b555c | ||
|
634bcd188a | ||
|
1664fdcf8c | ||
|
95515f111c | ||
|
ac2c5caeff | ||
|
75305dae94 | ||
|
7d0ad2dce1 | ||
|
84a85bbe38 | ||
|
9d2896016b | ||
|
17585a99b1 | ||
|
942e51b753 | ||
|
c60fa22422 |
11 changed files with 124 additions and 181 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
@ -7,6 +7,8 @@ c_src/*.o
|
||||||
c_src/bzip2-1.0.6
|
c_src/bzip2-1.0.6
|
||||||
c_src/snappy-1.0.4
|
c_src/snappy-1.0.4
|
||||||
deps/
|
deps/
|
||||||
priv/
|
priv/wt
|
||||||
|
priv/*.so*
|
||||||
|
priv/*.dylib*
|
||||||
log/
|
log/
|
||||||
*~
|
*~
|
||||||
|
|
4
Makefile
4
Makefile
|
@ -52,19 +52,17 @@ endif
|
||||||
.PHONY: all compile doc clean test dialyzer typer shell distclean pdf \
|
.PHONY: all compile doc clean test dialyzer typer shell distclean pdf \
|
||||||
update-deps clean-common-test-data rebuild
|
update-deps clean-common-test-data rebuild
|
||||||
|
|
||||||
all: deps compile test
|
all: deps compile
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Rules to build the system
|
# Rules to build the system
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
||||||
deps:
|
deps:
|
||||||
c_src/build_deps.sh get-deps
|
|
||||||
$(REBAR) get-deps
|
$(REBAR) get-deps
|
||||||
$(REBAR) compile
|
$(REBAR) compile
|
||||||
|
|
||||||
update-deps:
|
update-deps:
|
||||||
c_src/build_deps.sh update-deps
|
|
||||||
$(REBAR) update-deps
|
$(REBAR) update-deps
|
||||||
$(REBAR) compile
|
$(REBAR) compile
|
||||||
|
|
||||||
|
|
|
@ -11,10 +11,10 @@ unset POSIX_SHELL # clear it so if we invoke other scripts, they run as ksh as w
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
WT_REPO=http://github.com/wiredtiger/wiredtiger.git
|
WT_REPO=http://github.com/wiredtiger/wiredtiger.git
|
||||||
#WT_BRANCH=develop
|
WT_BRANCH=develop
|
||||||
#WT_DIR=wiredtiger-`basename $WT_BRANCH`
|
WT_DIR=wiredtiger-`basename $WT_BRANCH`
|
||||||
WT_REF="tags/1.6.4"
|
#WT_REF="tags/1.6.6"
|
||||||
WT_DIR=wiredtiger-`basename $WT_REF`
|
#WT_DIR=wiredtiger-`basename $WT_REF`
|
||||||
|
|
||||||
SNAPPY_VSN="1.0.4"
|
SNAPPY_VSN="1.0.4"
|
||||||
SNAPPY_DIR=snappy-$SNAPPY_VSN
|
SNAPPY_DIR=snappy-$SNAPPY_VSN
|
||||||
|
@ -26,8 +26,7 @@ export BASEDIR="$PWD"
|
||||||
which gmake 1>/dev/null 2>/dev/null && MAKE=gmake
|
which gmake 1>/dev/null 2>/dev/null && MAKE=gmake
|
||||||
MAKE=${MAKE:-make}
|
MAKE=${MAKE:-make}
|
||||||
|
|
||||||
export CFLAGS="$CFLAGS -I $BASEDIR/system/include"
|
export CPPFLAGS="$CPPLAGS -I $BASEDIR/system/include -O3 -mtune=native -march=native"
|
||||||
export CXXFLAGS="$CXXFLAGS -I $BASEDIR/system/include"
|
|
||||||
export LDFLAGS="$LDFLAGS -L$BASEDIR/system/lib"
|
export LDFLAGS="$LDFLAGS -L$BASEDIR/system/lib"
|
||||||
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$BASEDIR/system/lib:$LD_LIBRARY_PATH"
|
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$BASEDIR/system/lib:$LD_LIBRARY_PATH"
|
||||||
|
|
||||||
|
@ -58,7 +57,7 @@ get_wt ()
|
||||||
wt_configure ()
|
wt_configure ()
|
||||||
{
|
{
|
||||||
(cd $BASEDIR/$WT_DIR/build_posix
|
(cd $BASEDIR/$WT_DIR/build_posix
|
||||||
CFLAGS+=-g ../configure --with-pic \
|
CFLAGS+=-g $BASEDIR/$WT_DIR/configure --with-pic \
|
||||||
--enable-snappy \
|
--enable-snappy \
|
||||||
--prefix=${BASEDIR}/system || exit 1)
|
--prefix=${BASEDIR}/system || exit 1)
|
||||||
}
|
}
|
||||||
|
@ -131,21 +130,23 @@ case "$1" in
|
||||||
;;
|
;;
|
||||||
|
|
||||||
*)
|
*)
|
||||||
|
shopt -s extglob
|
||||||
|
SUFFIXES='@(so|dylib)'
|
||||||
|
|
||||||
# Build Snappy
|
# Build Snappy
|
||||||
[ -d $SNAPPY_DIR ] || get_snappy;
|
[ -d $SNAPPY_DIR ] || get_snappy;
|
||||||
[ -d $BASEDIR/$SNAPPY_DIR ] || (echo "Missing Snappy source directory" && exit 1)
|
[ -d $BASEDIR/$SNAPPY_DIR ] || (echo "Missing Snappy source directory" && exit 1)
|
||||||
test -f $BASEDIR/system/lib/libsnappy.so.[0-9].[0-9].[0-9] || build_snappy;
|
test -f $BASEDIR/system/lib/libsnappy.so.[0-9].[0-9].[0-9].* || build_snappy;
|
||||||
|
|
||||||
# Build WiredTiger
|
# Build WiredTiger
|
||||||
[ -d $WT_DIR ] || get_wt;
|
[ -d $WT_DIR ] || get_wt;
|
||||||
[ -d $BASEDIR/$WT_DIR ] || (echo "Missing WiredTiger source directory" && exit 1)
|
[ -d $BASEDIR/$WT_DIR ] || (echo "Missing WiredTiger source directory" && exit 1)
|
||||||
test -f $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].so \
|
test -f $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].${SUFFIXES} -a \
|
||||||
-a -f $BASEDIR/system/lib/libwiredtiger_snappy.so || build_wt;
|
-f $BASEDIR/system/lib/libwiredtiger_snappy.${SUFFIXES} || build_wt;
|
||||||
|
|
||||||
[ -d $BASEDIR/../priv ] || mkdir ${BASEDIR}/../priv
|
[ -d $BASEDIR/../priv ] || mkdir ${BASEDIR}/../priv
|
||||||
cp -p -P $BASEDIR/system/bin/wt ${BASEDIR}/../priv
|
cp -p -P $BASEDIR/system/bin/wt ${BASEDIR}/../priv
|
||||||
cp -p -P $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].so ${BASEDIR}/../priv
|
cp -p -P ${BASEDIR}/system/lib/libwiredtiger-[0-9].[0-9].[0-9].${SUFFIXES} ${BASEDIR}/../priv
|
||||||
cp -p -P $BASEDIR/system/lib/libwiredtiger_snappy.so* ${BASEDIR}/../priv
|
cp -p -P ${BASEDIR}/system/lib/libwiredtiger_snappy.${SUFFIXES} ${BASEDIR}/../priv
|
||||||
cp -p -P $BASEDIR/system/lib/libsnappy.so* ${BASEDIR}/../priv
|
cp -p -P ${BASEDIR}/system/lib/libsnappy.${SUFFIXES}* ${BASEDIR}/../priv
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
|
@ -1,106 +1,12 @@
|
||||||
diff --git a/ext/compressors/snappy/Makefile.am b/ext/compressors/snappy/Makefile.am
|
diff --git a/ext/compressors/snappy/Makefile.am b/ext/compressors/snappy/Makefile.am
|
||||||
index 6d78823..2122cf8 100644
|
index 6d78823..c423590 100644
|
||||||
--- a/ext/compressors/snappy/Makefile.am
|
--- a/ext/compressors/snappy/Makefile.am
|
||||||
+++ b/ext/compressors/snappy/Makefile.am
|
+++ b/ext/compressors/snappy/Makefile.am
|
||||||
@@ -2,5 +2,6 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
|
@@ -2,5 +2,6 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
|
||||||
|
|
||||||
lib_LTLIBRARIES = libwiredtiger_snappy.la
|
lib_LTLIBRARIES = libwiredtiger_snappy.la
|
||||||
libwiredtiger_snappy_la_SOURCES = snappy_compress.c
|
libwiredtiger_snappy_la_SOURCES = snappy_compress.c
|
||||||
-libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module
|
-libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module
|
||||||
+libwiredtiger_snappy_la_CFLAGS = -I$(abs_top_builddir)/../../system/include
|
+libwiredtiger_snappy_la_CFLAGS = -I$(abs_top_builddir)/../../system/include
|
||||||
+libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(abs_top_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv
|
+libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(abs_top_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv
|
||||||
libwiredtiger_snappy_la_LIBADD = -lsnappy
|
libwiredtiger_snappy_la_LIBADD = -lsnappy
|
||||||
diff --git a/src/support/cksum.c b/src/support/cksum.c
|
|
||||||
index 7e9befe..b924db7 100644
|
|
||||||
--- a/src/support/cksum.c
|
|
||||||
+++ b/src/support/cksum.c
|
|
||||||
@@ -27,6 +27,13 @@
|
|
||||||
|
|
||||||
#include "wt_internal.h"
|
|
||||||
|
|
||||||
+#if defined(__amd64) || defined(__x86_64)
|
|
||||||
+#define USE_HARDWARE_CRC32 1
|
|
||||||
+#else
|
|
||||||
+#undef USE_HARDWARE_CRC32
|
|
||||||
+#endif
|
|
||||||
+
|
|
||||||
+#ifdef USE_HARDWARE_CRC32
|
|
||||||
static const uint32_t g_crc_slicing[8][256] = {
|
|
||||||
#ifdef WORDS_BIGENDIAN
|
|
||||||
/*
|
|
||||||
@@ -1078,6 +1085,7 @@ static const uint32_t g_crc_slicing[8][256] = {
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
};
|
|
||||||
+#endif /* USE_HARDWARE_CRC32 */
|
|
||||||
|
|
||||||
/*
|
|
||||||
* __wt_cksum --
|
|
||||||
@@ -1106,15 +1114,29 @@ __wt_cksum(const void *chunk, size_t len)
|
|
||||||
/* Checksum one byte at a time to the first 4B boundary. */
|
|
||||||
for (p = chunk;
|
|
||||||
((uintptr_t)p & (sizeof(uint32_t) - 1)) != 0 &&
|
|
||||||
- len > 0; ++p, --len)
|
|
||||||
+ len > 0; ++p, --len) {
|
|
||||||
+#ifdef USE_HARDWARE_CRC32
|
|
||||||
+ __asm__ __volatile__(
|
|
||||||
+ ".byte 0xF2, 0x0F, 0x38, 0xF0, 0xF1"
|
|
||||||
+ : "=S" (crc)
|
|
||||||
+ : "0" (crc), "c" (*p));
|
|
||||||
+#else
|
|
||||||
#ifdef WORDS_BIGENDIAN
|
|
||||||
crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8);
|
|
||||||
#else
|
|
||||||
crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8);
|
|
||||||
#endif
|
|
||||||
+#endif
|
|
||||||
+ }
|
|
||||||
|
|
||||||
/* Checksum in 8B chunks. */
|
|
||||||
for (nqwords = len / sizeof(uint64_t); nqwords; nqwords--) {
|
|
||||||
+#ifdef USE_HARDWARE_CRC32
|
|
||||||
+ __asm__ __volatile__ (
|
|
||||||
+ ".byte 0xf2, 0x48, 0x0f, 0x38, 0xf0, 0xf1;"
|
|
||||||
+ : "=S"(crc)
|
|
||||||
+ : "S"(crc), "c"(*p));
|
|
||||||
+#else
|
|
||||||
crc ^= *(uint32_t *)p;
|
|
||||||
p += sizeof(uint32_t);
|
|
||||||
next = *(uint32_t *)p;
|
|
||||||
@@ -1139,22 +1161,32 @@ __wt_cksum(const void *chunk, size_t len)
|
|
||||||
g_crc_slicing[1][(next >> 16) & 0xFF] ^
|
|
||||||
g_crc_slicing[0][(next >> 24)];
|
|
||||||
#endif
|
|
||||||
+#endif
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Checksum trailing bytes one byte at a time. */
|
|
||||||
+ for (len &= 0x7; len > 0; ++p, len--) {
|
|
||||||
+#ifdef USE_HARDWARE_CRC32
|
|
||||||
+ __asm__ __volatile__(
|
|
||||||
+ ".byte 0xF2, 0x0F, 0x38, 0xF0, 0xF1"
|
|
||||||
+ : "=S" (crc)
|
|
||||||
+ : "0" (crc), "c" (*p));
|
|
||||||
+#else
|
|
||||||
#ifdef WORDS_BIGENDIAN
|
|
||||||
- for (len &= 0x7; len > 0; ++p, len--)
|
|
||||||
crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8);
|
|
||||||
+#else
|
|
||||||
+ crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8);
|
|
||||||
+#endif
|
|
||||||
+#endif
|
|
||||||
+ }
|
|
||||||
|
|
||||||
+#ifdef WORDS_BIGENDIAN
|
|
||||||
/* Do final byte swap to produce a result identical to little endian */
|
|
||||||
crc =
|
|
||||||
((crc << 24) & 0xFF000000) |
|
|
||||||
((crc << 8) & 0x00FF0000) |
|
|
||||||
((crc >> 8) & 0x0000FF00) |
|
|
||||||
((crc >> 24) & 0x000000FF);
|
|
||||||
-#else
|
|
||||||
- for (len &= 0x7; len > 0; ++p, len--)
|
|
||||||
- crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8);
|
|
||||||
#endif
|
|
||||||
return (~crc);
|
|
||||||
}
|
|
||||||
|
|
|
@ -440,6 +440,7 @@ __close_cursors_on(WterlConnHandle *conn_handle, const char *uri)
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Callback to handle error messages.
|
* Callback to handle error messages.
|
||||||
*
|
*
|
||||||
|
@ -454,13 +455,15 @@ __close_cursors_on(WterlConnHandle *conn_handle, const char *uri)
|
||||||
* operation or library failure.
|
* operation or library failure.
|
||||||
*/
|
*/
|
||||||
int
|
int
|
||||||
__wterl_error_handler(WT_EVENT_HANDLER *handler, int error, const char *message)
|
__wterl_error_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session,
|
||||||
|
int error, const char *message)
|
||||||
{
|
{
|
||||||
struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
|
struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
|
||||||
ErlNifEnv *msg_env;
|
ErlNifEnv *msg_env;
|
||||||
ErlNifPid *to_pid;
|
ErlNifPid *to_pid;
|
||||||
int rc = 0;
|
int rc = 0;
|
||||||
|
|
||||||
|
UNUSED(session);
|
||||||
enif_mutex_lock(eh->error_mutex);
|
enif_mutex_lock(eh->error_mutex);
|
||||||
msg_env = eh->msg_env_error;
|
msg_env = eh->msg_env_error;
|
||||||
to_pid = &eh->to_pid;
|
to_pid = &eh->to_pid;
|
||||||
|
@ -492,13 +495,14 @@ __wterl_error_handler(WT_EVENT_HANDLER *handler, int error, const char *message)
|
||||||
* operation or library failure.
|
* operation or library failure.
|
||||||
*/
|
*/
|
||||||
int
|
int
|
||||||
__wterl_message_handler(WT_EVENT_HANDLER *handler, const char *message)
|
__wterl_message_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message)
|
||||||
{
|
{
|
||||||
struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
|
struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
|
||||||
ErlNifEnv *msg_env;
|
ErlNifEnv *msg_env;
|
||||||
ErlNifPid *to_pid;
|
ErlNifPid *to_pid;
|
||||||
int rc = 0;
|
int rc = 0;
|
||||||
|
|
||||||
|
UNUSED(session);
|
||||||
enif_mutex_lock(eh->message_mutex);
|
enif_mutex_lock(eh->message_mutex);
|
||||||
msg_env = eh->msg_env_message;
|
msg_env = eh->msg_env_message;
|
||||||
to_pid = &eh->to_pid;
|
to_pid = &eh->to_pid;
|
||||||
|
@ -529,13 +533,14 @@ __wterl_message_handler(WT_EVENT_HANDLER *handler, const char *message)
|
||||||
* operation or library failure.
|
* operation or library failure.
|
||||||
*/
|
*/
|
||||||
int
|
int
|
||||||
__wterl_progress_handler(WT_EVENT_HANDLER *handler, const char *operation, uint64_t counter)
|
__wterl_progress_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *operation, uint64_t counter)
|
||||||
{
|
{
|
||||||
struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
|
struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
|
||||||
ErlNifEnv *msg_env;
|
ErlNifEnv *msg_env;
|
||||||
ErlNifPid *to_pid;
|
ErlNifPid *to_pid;
|
||||||
int rc = 0;
|
int rc = 0;
|
||||||
|
|
||||||
|
UNUSED(session);
|
||||||
enif_mutex_lock(eh->progress_mutex);
|
enif_mutex_lock(eh->progress_mutex);
|
||||||
msg_env = eh->msg_env_progress;
|
msg_env = eh->msg_env_progress;
|
||||||
to_pid = &eh->to_pid;
|
to_pid = &eh->to_pid;
|
||||||
|
@ -2303,7 +2308,7 @@ on_load(ErlNifEnv *env, void **priv_data, ERL_NIF_TERM load_info)
|
||||||
|
|
||||||
char msg[1024];
|
char msg[1024];
|
||||||
snprintf(msg, 1024, "NIF on_load complete (wterl version: %s, wiredtiger version: %s)", priv->wterl_vsn, priv->wiredtiger_vsn);
|
snprintf(msg, 1024, "NIF on_load complete (wterl version: %s, wiredtiger version: %s)", priv->wterl_vsn, priv->wiredtiger_vsn);
|
||||||
__wterl_message_handler((WT_EVENT_HANDLER *)&priv->eh, msg);
|
__wterl_message_handler((WT_EVENT_HANDLER *)&priv->eh, NULL, msg);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
6
priv/wterl.schema
Normal file
6
priv/wterl.schema
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
%%%% This is the WiredTiger section
|
||||||
|
|
||||||
|
%% @doc wiredtiger data_root
|
||||||
|
{mapping, "wiredtiger.data_root", "wterl.data_root", [
|
||||||
|
{default, "{{platform_data_dir}}/wiredtiger"}
|
||||||
|
]}.
|
|
@ -38,8 +38,8 @@
|
||||||
{port_specs, [{"priv/wterl.so", ["c_src/*.c"]}]}.
|
{port_specs, [{"priv/wterl.so", ["c_src/*.c"]}]}.
|
||||||
|
|
||||||
{port_env, [
|
{port_env, [
|
||||||
{"DRV_CFLAGS", "$DRV_CFLAGS -fPIC -Wall -Wextra -Werror -I c_src/system/include"},
|
{"DRV_CFLAGS", "$DRV_CFLAGS -O3 -mtune=native -march=native -fPIC -Wall -Wextra -Werror -I c_src/system/include"},
|
||||||
{"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath,lib/wterl/priv:priv -Lc_src/system/lib -lwiredtiger"}
|
{"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath,lib/wterl/priv:lib/wterl-0.9.0/priv:priv -Lc_src/system/lib -lwiredtiger"}
|
||||||
]}.
|
]}.
|
||||||
|
|
||||||
{pre_hooks, [{compile, "c_src/build_deps.sh compile"}]}.
|
{pre_hooks, [{compile, "c_src/build_deps.sh compile"}]}.
|
||||||
|
|
|
@ -22,7 +22,7 @@
|
||||||
%% -------------------------------------------------------------------
|
%% -------------------------------------------------------------------
|
||||||
|
|
||||||
-define(ASYNC_NIF_CALL(Fun, Args),
|
-define(ASYNC_NIF_CALL(Fun, Args),
|
||||||
F = fun(F) ->
|
F = fun(F, T) ->
|
||||||
R = erlang:make_ref(),
|
R = erlang:make_ref(),
|
||||||
case erlang:apply(Fun, [R|Args]) of
|
case erlang:apply(Fun, [R|Args]) of
|
||||||
{ok, {enqueued, PctBusy}} ->
|
{ok, {enqueued, PctBusy}} ->
|
||||||
|
@ -43,9 +43,12 @@
|
||||||
Reply
|
Reply
|
||||||
end;
|
end;
|
||||||
{error, eagain} ->
|
{error, eagain} ->
|
||||||
F(F);
|
case T of
|
||||||
|
3 -> not_found;
|
||||||
|
_ -> F(F, T + 1)
|
||||||
|
end;
|
||||||
Other ->
|
Other ->
|
||||||
Other
|
Other
|
||||||
end
|
end
|
||||||
end,
|
end,
|
||||||
F(F)).
|
F(F, 1)).
|
||||||
|
|
|
@ -22,6 +22,7 @@
|
||||||
|
|
||||||
-module(riak_kv_wterl_backend).
|
-module(riak_kv_wterl_backend).
|
||||||
-behavior(temp_riak_kv_backend).
|
-behavior(temp_riak_kv_backend).
|
||||||
|
-compile([{parse_transform, lager_transform}]).
|
||||||
|
|
||||||
%% KV Backend API
|
%% KV Backend API
|
||||||
-export([api_version/0,
|
-export([api_version/0,
|
||||||
|
@ -42,7 +43,7 @@
|
||||||
|
|
||||||
-ifdef(TEST).
|
-ifdef(TEST).
|
||||||
-include_lib("eunit/include/eunit.hrl").
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
-compiel(export_all).
|
-compile(export_all).
|
||||||
-endif.
|
-endif.
|
||||||
|
|
||||||
-define(API_VERSION, 1).
|
-define(API_VERSION, 1).
|
||||||
|
@ -119,14 +120,14 @@ start(Partition, Config) ->
|
||||||
"lsm" ->
|
"lsm" ->
|
||||||
[{internal_page_max, "128K"},
|
[{internal_page_max, "128K"},
|
||||||
{leaf_page_max, "16K"},
|
{leaf_page_max, "16K"},
|
||||||
{lsm_chunk_size, "100MB"},
|
{lsm, [
|
||||||
{lsm_merge_threads, 2},
|
{bloom_config, [{leaf_page_max, "8MB"}]},
|
||||||
{prefix_compression, true},
|
{bloom_bit_count, 28},
|
||||||
{lsm_bloom_newest, true},
|
{bloom_hash_count, 19},
|
||||||
{lsm_bloom_oldest, true} ,
|
{bloom_oldest, true},
|
||||||
{lsm_bloom_bit_count, 28},
|
{chunk_size, "100MB"},
|
||||||
{lsm_bloom_hash_count, 19},
|
{merge_threads, 2}
|
||||||
{lsm_bloom_config, [{leaf_page_max, "8MB"}]}
|
]}
|
||||||
] ++ Compressor;
|
] ++ Compressor;
|
||||||
"table" ->
|
"table" ->
|
||||||
Compressor
|
Compressor
|
||||||
|
@ -341,22 +342,23 @@ is_empty(#state{connection=Connection, table=Table}) ->
|
||||||
%% @doc Get the status information for this wterl backend
|
%% @doc Get the status information for this wterl backend
|
||||||
-spec status(state()) -> [{atom(), term()}].
|
-spec status(state()) -> [{atom(), term()}].
|
||||||
status(#state{connection=Connection, table=Table}) ->
|
status(#state{connection=Connection, table=Table}) ->
|
||||||
case wterl:cursor_open(Connection, Table) of
|
[].
|
||||||
{ok, Cursor} ->
|
%% case wterl:cursor_open(Connection, "statistics:" ++ Table, [{statistics_fast, true}]) of
|
||||||
TheStats =
|
%% {ok, Cursor} ->
|
||||||
case fetch_status(Cursor) of
|
%% TheStats =
|
||||||
{ok, Stats} ->
|
%% case fetch_status(Cursor) of
|
||||||
Stats;
|
%% {ok, Stats} ->
|
||||||
{error, {eperm, _}} -> % TODO: review/fix this logic
|
%% Stats;
|
||||||
{ok, []};
|
%% {error, {eperm, _}} -> % TODO: review/fix this logic
|
||||||
_ ->
|
%% {ok, []};
|
||||||
{ok, []}
|
%% _ ->
|
||||||
end,
|
%% {ok, []}
|
||||||
wterl:cursor_close(Cursor),
|
%% end,
|
||||||
TheStats;
|
%% wterl:cursor_close(Cursor),
|
||||||
{error, Reason2} ->
|
%% TheStats;
|
||||||
{error, Reason2}
|
%% {error, Reason2} ->
|
||||||
end.
|
%% {error, Reason2}
|
||||||
|
%% end.
|
||||||
|
|
||||||
%% @doc Register an asynchronous callback
|
%% @doc Register an asynchronous callback
|
||||||
-spec callback(reference(), any(), state()) -> {ok, state()}.
|
-spec callback(reference(), any(), state()) -> {ok, state()}.
|
||||||
|
@ -399,30 +401,41 @@ establish_connection(Config, Type) ->
|
||||||
ok = filelib:ensure_dir(filename:join(DataRoot, "x")),
|
ok = filelib:ensure_dir(filename:join(DataRoot, "x")),
|
||||||
|
|
||||||
%% WT Connection Options:
|
%% WT Connection Options:
|
||||||
%% NOTE: LSM auto-checkpoints, so we don't have too.
|
LogSetting = app_helper:get_prop_or_env(log, Config, wterl, false),
|
||||||
CheckpointSetting =
|
CheckpointSetting =
|
||||||
case Type =:= "lsm" of
|
case Type =:= "lsm" of
|
||||||
true ->
|
true ->
|
||||||
[];
|
case LogSetting of
|
||||||
|
true ->
|
||||||
|
%% Turn checkpoints on if logging is on, checkpoints enable log archival.
|
||||||
|
app_helper:get_prop_or_env(checkpoint, Config, wterl, [{wait, 30}]); % in seconds
|
||||||
|
_ ->
|
||||||
|
[]
|
||||||
|
end;
|
||||||
false ->
|
false ->
|
||||||
app_helper:get_prop_or_env(checkpoint, Config, wterl, [{wait, 10}])
|
app_helper:get_prop_or_env(checkpoint, Config, wterl, [{wait, 30}])
|
||||||
end,
|
end,
|
||||||
RequestedCacheSize = app_helper:get_prop_or_env(cache_size, Config, wterl),
|
RequestedCacheSize = app_helper:get_prop_or_env(cache_size, Config, wterl),
|
||||||
ConnectionOpts =
|
ConnectionOpts =
|
||||||
orddict:from_list(
|
orddict:from_list(
|
||||||
[ wterl:config_value(create, Config, true),
|
[ wterl:config_value(create, Config, true),
|
||||||
wterl:config_value(sync, Config, false),
|
wterl:config_value(checkpoint_sync, Config, false),
|
||||||
|
wterl:config_value(transaction_sync, Config, "none"),
|
||||||
|
wterl:config_value(log, Config, [{enabled, LogSetting}]),
|
||||||
|
wterl:config_value(mmap, Config, false),
|
||||||
|
wterl:config_value(checkpoint, Config, CheckpointSetting),
|
||||||
wterl:config_value(session_max, Config, max_sessions(Config)),
|
wterl:config_value(session_max, Config, max_sessions(Config)),
|
||||||
wterl:config_value(cache_size, Config, size_cache(RequestedCacheSize)),
|
wterl:config_value(cache_size, Config, size_cache(RequestedCacheSize)),
|
||||||
|
wterl:config_value(statistics, Config, [ "fast", "clear"]),
|
||||||
wterl:config_value(statistics_log, Config, [{wait, 600}]), % in seconds
|
wterl:config_value(statistics_log, Config, [{wait, 600}]), % in seconds
|
||||||
wterl:config_value(verbose, Config, [ "salvage", "verify"
|
wterl:config_value(verbose, Config, [ "salvage", "verify"
|
||||||
% Note: for some unknown reason, if you add these additional
|
% Note: for some unknown reason, if you add these additional
|
||||||
% verbose flags Erlang SEGV's "size_object: bad tag for 0x80"
|
% verbose flags Erlang SEGV's "size_object: bad tag for 0x80"
|
||||||
% no idea why... yet... you've been warned.
|
% no idea why... you've been warned.
|
||||||
%"block", "shared_cache", "reconcile", "evict", "lsm",
|
%"block", "shared_cache", "reconcile", "evict", "lsm",
|
||||||
%"fileops", "read", "write", "readserver", "evictserver",
|
%"fileops", "read", "write", "readserver", "evictserver",
|
||||||
%"hazard", "mutex", "ckpt"
|
%"hazard", "mutex", "ckpt"
|
||||||
]) ] ++ CheckpointSetting ++ proplists:get_value(wterl, Config, [])), % sec
|
]) ] ++ proplists:get_value(wterl, Config, [])), % sec
|
||||||
|
|
||||||
%% WT Session Options:
|
%% WT Session Options:
|
||||||
SessionOpts = [{isolation, "snapshot"}],
|
SessionOpts = [{isolation, "snapshot"}],
|
||||||
|
@ -543,15 +556,15 @@ from_index_key(LKey) ->
|
||||||
|
|
||||||
%% @private
|
%% @private
|
||||||
%% Return all status from wterl statistics cursor
|
%% Return all status from wterl statistics cursor
|
||||||
fetch_status(Cursor) ->
|
%% fetch_status(Cursor) ->
|
||||||
{ok, fetch_status(Cursor, wterl:cursor_next_value(Cursor), [])}.
|
%% {ok, fetch_status(Cursor, wterl:cursor_next_value(Cursor), [])}.
|
||||||
fetch_status(_Cursor, {error, _}, Acc) ->
|
%% fetch_status(_Cursor, {error, _}, Acc) ->
|
||||||
lists:reverse(Acc);
|
%% lists:reverse(Acc);
|
||||||
fetch_status(_Cursor, not_found, Acc) ->
|
%% fetch_status(_Cursor, not_found, Acc) ->
|
||||||
lists:reverse(Acc);
|
%% lists:reverse(Acc);
|
||||||
fetch_status(Cursor, {ok, Stat}, Acc) ->
|
%% fetch_status(Cursor, {ok, Stat}, Acc) ->
|
||||||
[What,Val|_] = [binary_to_list(B) || B <- binary:split(Stat, [<<0>>], [global])],
|
%% [What,Val|_] = [binary_to_list(B) || B <- binary:split(Stat, [<<0>>], [global])],
|
||||||
fetch_status(Cursor, wterl:cursor_next_value(Cursor), [{What,Val}|Acc]).
|
%% fetch_status(Cursor, wterl:cursor_next_value(Cursor), [{What,Val}|Acc]).
|
||||||
|
|
||||||
size_cache(RequestedSize) ->
|
size_cache(RequestedSize) ->
|
||||||
Size =
|
Size =
|
||||||
|
|
|
@ -96,8 +96,8 @@ nif_stub_error(Line) ->
|
||||||
-spec init() -> ok | {error, any()}.
|
-spec init() -> ok | {error, any()}.
|
||||||
init() ->
|
init() ->
|
||||||
erlang:load_nif(filename:join([priv_dir(), atom_to_list(?MODULE)]),
|
erlang:load_nif(filename:join([priv_dir(), atom_to_list(?MODULE)]),
|
||||||
[{wterl_vsn, "53307e8"},
|
[{wterl_vsn, "942e51b"},
|
||||||
{wiredtiger_vsn, "1.6.2-0-g07cb0a5"}]).
|
{wiredtiger_vsn, "1.6.4-275-g9c44420"}]). %% TODO automate these
|
||||||
|
|
||||||
-spec connection_open(string(), config_list()) -> {ok, connection()} | {error, term()}.
|
-spec connection_open(string(), config_list()) -> {ok, connection()} | {error, term()}.
|
||||||
-spec connection_open(string(), config_list(), config_list()) -> {ok, connection()} | {error, term()}.
|
-spec connection_open(string(), config_list(), config_list()) -> {ok, connection()} | {error, term()}.
|
||||||
|
@ -454,17 +454,26 @@ config_to_bin([], Acc) ->
|
||||||
config_to_bin([{Key, Value} | Rest], Acc) ->
|
config_to_bin([{Key, Value} | Rest], Acc) ->
|
||||||
ConfigTypes =
|
ConfigTypes =
|
||||||
[{block_compressor, {string, quoted}},
|
[{block_compressor, {string, quoted}},
|
||||||
|
{bloom_bit_count, integer},
|
||||||
|
{bloom_config, config},
|
||||||
|
{bloom_hash_count, integer},
|
||||||
|
{bloom_newest, bool},
|
||||||
|
{bloom_oldest, bool},
|
||||||
{cache_size, string},
|
{cache_size, string},
|
||||||
{checkpoint, config},
|
{checkpoint, config},
|
||||||
|
{checkpoint_sync, bool},
|
||||||
{checksum, string},
|
{checksum, string},
|
||||||
|
{chunk_size, string},
|
||||||
{create, bool},
|
{create, bool},
|
||||||
{direct_io, list},
|
{direct_io, list},
|
||||||
{drop, list},
|
{drop, list},
|
||||||
|
{enabled, bool},
|
||||||
{error_prefix, string},
|
{error_prefix, string},
|
||||||
{eviction_target, integer},
|
{eviction_target, integer},
|
||||||
{eviction_trigger, integer},
|
{eviction_trigger, integer},
|
||||||
{extensions, {list, quoted}},
|
{extensions, {list, quoted}},
|
||||||
{statistics_fast, bool},
|
{statistics_fast, bool},
|
||||||
|
{file_max, string},
|
||||||
{force, bool},
|
{force, bool},
|
||||||
{from, string},
|
{from, string},
|
||||||
{hazard_max, integer},
|
{hazard_max, integer},
|
||||||
|
@ -474,24 +483,21 @@ config_to_bin([{Key, Value} | Rest], Acc) ->
|
||||||
{isolation, string},
|
{isolation, string},
|
||||||
{key_type, string},
|
{key_type, string},
|
||||||
{leaf_page_max, string},
|
{leaf_page_max, string},
|
||||||
{logging, bool},
|
{log, config},
|
||||||
{lsm_bloom_bit_count, integer},
|
{lsm, config},
|
||||||
{lsm_bloom_config, config},
|
{mmap, bool},
|
||||||
{lsm_bloom_hash_count, integer},
|
{merge_threads, integer},
|
||||||
{lsm_bloom_newest, bool},
|
|
||||||
{lsm_bloom_oldest, bool},
|
|
||||||
{lsm_chunk_size, string},
|
|
||||||
{prefix_compression, bool},
|
|
||||||
{lsm_merge_threads, integer},
|
|
||||||
{multiprocess, bool},
|
{multiprocess, bool},
|
||||||
{name, string},
|
{name, string},
|
||||||
{overwrite, bool},
|
{overwrite, bool},
|
||||||
|
{prefix_compression, bool},
|
||||||
{raw, bool},
|
{raw, bool},
|
||||||
{session_max, integer},
|
{session_max, integer},
|
||||||
|
{statistics, list},
|
||||||
{statistics_log, config},
|
{statistics_log, config},
|
||||||
{sync, bool},
|
|
||||||
{target, {list, quoted}},
|
{target, {list, quoted}},
|
||||||
{to, string},
|
{to, string},
|
||||||
|
{transaction_sync, string},
|
||||||
{transactional, bool},
|
{transactional, bool},
|
||||||
{verbose, list},
|
{verbose, list},
|
||||||
{wait, integer}],
|
{wait, integer}],
|
||||||
|
|
|
@ -25,7 +25,7 @@
|
||||||
|
|
||||||
{mode, max}.
|
{mode, max}.
|
||||||
{duration, 10}.
|
{duration, 10}.
|
||||||
{concurrent, 4}.
|
{concurrent, 16}.
|
||||||
{report_interval, 1}.
|
{report_interval, 1}.
|
||||||
{pb_timeout_general, 1000}. % ms
|
{pb_timeout_general, 1000}. % ms
|
||||||
%{pb_timeout_read, ?}.
|
%{pb_timeout_read, ?}.
|
||||||
|
@ -43,7 +43,9 @@
|
||||||
{wterl, [
|
{wterl, [
|
||||||
{connection, [
|
{connection, [
|
||||||
{create, true},
|
{create, true},
|
||||||
{sync, false},
|
{session_sync, false},
|
||||||
|
{transaction_sync, "none"},
|
||||||
|
{log, [{enabled, false}]},
|
||||||
{session_max, 1024},
|
{session_max, 1024},
|
||||||
{cache_size, 4294967296},
|
{cache_size, 4294967296},
|
||||||
{verbose, []},
|
{verbose, []},
|
||||||
|
@ -58,11 +60,11 @@
|
||||||
]},
|
]},
|
||||||
{session, [ {isolation, "snapshot"} ]},
|
{session, [ {isolation, "snapshot"} ]},
|
||||||
{table_uri, "lsm:test"},
|
{table_uri, "lsm:test"},
|
||||||
|
{lsm_merge_threads, 2},
|
||||||
{table, [
|
{table, [
|
||||||
{internal_page_max, "128K"},
|
{internal_page_max, "128K"},
|
||||||
{leaf_page_max, "128K"},
|
{leaf_page_max, "128K"},
|
||||||
{lsm_chunk_size, "25MB"},
|
{lsm_chunk_size, "25MB"},
|
||||||
{prefix_compression, false},
|
|
||||||
{lsm_bloom_newest, true},
|
{lsm_bloom_newest, true},
|
||||||
{lsm_bloom_oldest, true} ,
|
{lsm_bloom_oldest, true} ,
|
||||||
{lsm_bloom_bit_count, 128},
|
{lsm_bloom_bit_count, 128},
|
||||||
|
@ -76,7 +78,9 @@
|
||||||
{wterl_, [
|
{wterl_, [
|
||||||
{connection, [
|
{connection, [
|
||||||
{create, true},
|
{create, true},
|
||||||
{sync, false},
|
{session_sync, false},
|
||||||
|
{transaction_sync, "none"},
|
||||||
|
{log, [{enabled, false}]},
|
||||||
{session_max, 1024},
|
{session_max, 1024},
|
||||||
{cache_size, 4294967296},
|
{cache_size, 4294967296},
|
||||||
{verbose, []},
|
{verbose, []},
|
||||||
|
@ -93,7 +97,6 @@
|
||||||
{session, [ {isolation, "snapshot"} ]},
|
{session, [ {isolation, "snapshot"} ]},
|
||||||
{table_uri, "table:test"},
|
{table_uri, "table:test"},
|
||||||
{table, [
|
{table, [
|
||||||
{prefix_compression, false},
|
|
||||||
{block_compressor, "snappy"} % bzip2
|
{block_compressor, "snappy"} % bzip2
|
||||||
]}
|
]}
|
||||||
]}.
|
]}.
|
||||||
|
|
Loading…
Reference in a new issue