diff --git a/.gdbinit b/.gdbinit new file mode 100644 index 0000000..1f3ca93 --- /dev/null +++ b/.gdbinit @@ -0,0 +1,4 @@ +handle SIGPIPE nostop noprint pass +#b erl_nif.c:1203 +#b sys/unix/erl_unix_sys_ddll.c:234 + diff --git a/.gitignore b/.gitignore index fa6ba18..b312112 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,12 @@ *.beam .eunit ebin -priv/*.so c_src/system c_src/wiredtiger*/ c_src/*.o +c_src/bzip2-1.0.6 +c_src/snappy-1.0.4 +deps/ +priv/ +log/ *~ diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..9eabdbf --- /dev/null +++ b/AUTHORS @@ -0,0 +1,2 @@ +-author('Steve Vinoski '). +-author('Gregory Burd '). % greg@burd.me @gregburd diff --git a/Makefile b/Makefile index af34362..14d9196 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ TARGET= wterl REBAR= ./rebar #REBAR= /usr/bin/env rebar ERL= /usr/bin/env erl +ERLEXEC= ${ERL_ROOTDIR}/lib/erlang/erts-5.9.1/bin/erlexec DIALYZER= /usr/bin/env dialyzer @@ -13,12 +14,14 @@ all: compile deps: get-deps get-deps: + c_src/build_deps.sh get-deps @$(REBAR) get-deps update-deps: + c_src/build_deps.sh update-deps @$(REBAR) update-deps -c_src/wterl.o: +c_src/wterl.o: c_src/async_nif.h touch c_src/wterl.c ebin/app_helper.beam: @@ -38,22 +41,41 @@ eunit: compile @$(REBAR) eunit skip_deps=true eunit_console: - @$(ERL) -pa .eunit deps/*/ebin + @$(ERL) -pa .eunit deps/lager/ebin plt: compile - @$(DIALYZER) --build_plt --output_plt .$(TARGET).plt -pa deps/*/ebin --apps kernel stdlib + @$(DIALYZER) --build_plt --output_plt .$(TARGET).plt -pa deps/lager/ebin --apps kernel stdlib analyze: compile - $(DIALYZER) --plt .$(TARGET).plt -pa deps/*/ebin ebin + $(DIALYZER) --plt .$(TARGET).plt -pa deps/lager/ebin ebin repl: - $(ERL) -pz deps/*/ebin -pa ebin + $(ERL) -pz deps/lager/ebin -pa ebin gdb-repl: - USE_GDB=1 $(ERL) -pz deps/*/ebin -pa ebin + USE_GDB=1 $(ERL) -pz deps/lager/ebin -pa ebin eunit-repl: - $(ERL) -pa .eunit -pz deps/*/ebin -pz ebin -exec 'cd(".eunit").' + $(ERL) -pz deps/lager/ebin -pa ebin -pa .eunit gdb-eunit-repl: - USE_GDB=1 $(ERL) -pa .eunit -pz deps/*/ebin -pz ebin -exec 'cd(".eunit").' + USE_GDB=1 $(ERL) -pa .eunit -pz deps/lager/ebin -pz ebin -exec 'cd(".eunit").' + + +ERL_TOP= /home/gburd/eng/otp_R15B01 +CERL= ${ERL_TOP}/bin/cerl +VALGRIND_MISC_FLAGS= "--verbose --leak-check=full --show-reachable=yes --trace-children=yes --track-origins=yes --suppressions=${ERL_TOP}/erts/emulator/valgrind/suppress.standard --show-possibly-lost=no --malloc-fill=AB --free-fill=CD" + +helgrind: + valgrind --verbose --tool=helgrind \ + --leak-check=full + --show-reachable=yes \ + --trace-children=yes \ + --track-origins=yes \ + --suppressions=${ERL_TOP}/erts/emulator/valgrind/suppress.standard \ + --show-possibly-lost=no \ + --malloc-fill=AB \ + --free-fill=CD ${ERLEXEC} -pz deps/lager/ebin -pa ebin -pa .eunit + +valgrind: + ${CERL} -valgrind ${VALGRIND_FLAGS} --log-file=${ROOTDIR}/valgrind_log-beam.smp.%p -- -pz deps/lager/ebin -pa ebin -pa .eunit -exec 'eunit:test(wterl).' diff --git a/c_src/async_nif.h b/c_src/async_nif.h new file mode 100644 index 0000000..a89262a --- /dev/null +++ b/c_src/async_nif.h @@ -0,0 +1,411 @@ +/* + * async_nif: An async thread-pool layer for Erlang's NIF API + * + * Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. + * Author: Gregory Burd + * + * This file is provided to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef __ASYNC_NIF_H__ +#define __ASYNC_NIF_H__ + +#if defined(__cplusplus) +extern "C" { +#endif + +#include +#include "fifo_q.h" +#ifdef ASYNC_NIF_STATS +#include "stats.h" // TODO: measure, measure... measure again +#endif + +#define ASYNC_NIF_MAX_WORKERS 128 +#define ASYNC_NIF_WORKER_QUEUE_SIZE 500 + +DECL_FIFO_QUEUE(reqs, struct async_nif_req_entry); + +struct async_nif_work_queue { + ErlNifMutex *reqs_mutex; + ErlNifCond *reqs_cnd; + FIFO_QUEUE_TYPE(reqs) reqs; +}; + +struct async_nif_worker_entry { + ErlNifTid tid; + unsigned int worker_id; + struct async_nif_state *async_nif; + struct async_nif_work_queue *q; +}; + +struct async_nif_state { + unsigned int shutdown; + unsigned int num_workers; + struct async_nif_worker_entry worker_entries[ASYNC_NIF_MAX_WORKERS]; + unsigned int num_queues; + unsigned int next_q; + struct async_nif_work_queue queues[]; +}; + +#define ASYNC_NIF_DECL(decl, frame, pre_block, work_block, post_block) \ + struct decl ## _args frame; \ + static void fn_work_ ## decl (ErlNifEnv *env, ERL_NIF_TERM ref, ErlNifPid *pid, unsigned int worker_id, struct decl ## _args *args) work_block \ + static void fn_post_ ## decl (struct decl ## _args *args) { \ + do post_block while(0); \ + } \ + static ERL_NIF_TERM decl(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv_in[]) { \ + struct decl ## _args on_stack_args; \ + struct decl ## _args *args = &on_stack_args; \ + struct decl ## _args *copy_of_args; \ + struct async_nif_req_entry *req = NULL; \ + const char *affinity = NULL; \ + ErlNifEnv *new_env = NULL; \ + /* argv[0] is a ref used for selective recv */ \ + const ERL_NIF_TERM *argv = argv_in + 1; \ + argc -= 1; \ + /* Note: !!! this assumes that the first element of priv_data is ours */ \ + struct async_nif_state *async_nif = *(struct async_nif_state**)enif_priv_data(env); \ + if (async_nif->shutdown) \ + return enif_make_tuple2(env, enif_make_atom(env, "error"), \ + enif_make_atom(env, "shutdown")); \ + if (!(new_env = enif_alloc_env())) { \ + return enif_make_tuple2(env, enif_make_atom(env, "error"), \ + enif_make_atom(env, "enomem")); \ + } \ + do pre_block while(0); \ + req = (struct async_nif_req_entry*)enif_alloc(sizeof(struct async_nif_req_entry)); \ + if (!req) { \ + fn_post_ ## decl (args); \ + enif_free_env(new_env); \ + return enif_make_tuple2(env, enif_make_atom(env, "error"), \ + enif_make_atom(env, "enomem")); \ + } \ + memset(req, 0, sizeof(struct async_nif_req_entry)); \ + copy_of_args = (struct decl ## _args *)enif_alloc(sizeof(struct decl ## _args)); \ + if (!copy_of_args) { \ + fn_post_ ## decl (args); \ + enif_free(req); \ + enif_free_env(new_env); \ + return enif_make_tuple2(env, enif_make_atom(env, "error"), \ + enif_make_atom(env, "enomem")); \ + } \ + memcpy(copy_of_args, args, sizeof(struct decl ## _args)); \ + req->env = new_env; \ + req->ref = enif_make_copy(new_env, argv_in[0]); \ + enif_self(env, &req->pid); \ + req->args = (void*)copy_of_args; \ + req->fn_work = (void (*)(ErlNifEnv *, ERL_NIF_TERM, ErlNifPid*, unsigned int, void *))fn_work_ ## decl ; \ + req->fn_post = (void (*)(void *))fn_post_ ## decl; \ + int h = -1; \ + if (affinity) \ + h = async_nif_str_hash_func(affinity) % async_nif->num_queues; \ + ERL_NIF_TERM reply = async_nif_enqueue_req(async_nif, req, h); \ + if (!reply) { \ + fn_post_ ## decl (args); \ + enif_free(req); \ + enif_free_env(new_env); \ + enif_free(copy_of_args); \ + return enif_make_tuple2(env, enif_make_atom(env, "error"), \ + enif_make_atom(env, "shutdown")); \ + } \ + return reply; \ + } + +#define ASYNC_NIF_INIT(name) \ + static ErlNifMutex *name##_async_nif_coord = NULL; + +#define ASYNC_NIF_LOAD(name, priv) do { \ + if (!name##_async_nif_coord) \ + name##_async_nif_coord = enif_mutex_create(NULL); \ + enif_mutex_lock(name##_async_nif_coord); \ + priv = async_nif_load(); \ + enif_mutex_unlock(name##_async_nif_coord); \ + } while(0); +#define ASYNC_NIF_UNLOAD(name, env, priv) do { \ + if (!name##_async_nif_coord) \ + name##_async_nif_coord = enif_mutex_create(NULL); \ + enif_mutex_lock(name##_async_nif_coord); \ + async_nif_unload(env, priv); \ + enif_mutex_unlock(name##_async_nif_coord); \ + enif_mutex_destroy(name##_async_nif_coord); \ + name##_async_nif_coord = NULL; \ + } while(0); +#define ASYNC_NIF_UPGRADE(name, env) do { \ + if (!name##_async_nif_coord) \ + name##_async_nif_coord = enif_mutex_create(NULL); \ + enif_mutex_lock(name##_async_nif_coord); \ + async_nif_upgrade(env); \ + enif_mutex_unlock(name##_async_nif_coord); \ + } while(0); + +#define ASYNC_NIF_RETURN_BADARG() return enif_make_badarg(env); +#define ASYNC_NIF_WORK_ENV new_env + +#define ASYNC_NIF_REPLY(msg) enif_send(NULL, pid, env, enif_make_tuple2(env, ref, msg)) + +/** + * TODO: + */ +static inline unsigned int async_nif_str_hash_func(const char *s) +{ + unsigned int h = (unsigned int)*s; + if (h) for (++s ; *s; ++s) h = (h << 5) - h + (unsigned int)*s; + return h; +} + +/** + * TODO: + */ +static ERL_NIF_TERM +async_nif_enqueue_req(struct async_nif_state* async_nif, struct async_nif_req_entry *req, int hint) +{ + /* Identify the most appropriate worker for this request. */ + unsigned int qid = (hint != -1) ? hint : async_nif->next_q; + struct async_nif_work_queue *q = NULL; + do { + q = &async_nif->queues[qid]; + enif_mutex_lock(q->reqs_mutex); + + /* Now that we hold the lock, check for shutdown. As long as we + hold this lock either a) we're shutting down so exit now or + b) this queue will be valid until we release the lock. */ + if (async_nif->shutdown) { + enif_mutex_unlock(q->reqs_mutex); + return 0; + } + + if (fifo_q_full(reqs, q->reqs)) { // TODO: || (q->avg_latency > median_latency) + enif_mutex_unlock(q->reqs_mutex); + qid = (qid + 1) % async_nif->num_queues; + q = &async_nif->queues[qid]; + } else { + break; + } + } while(1); + + /* We hold the queue's lock, and we've seletect a reasonable queue for this + new request so add the request. */ + fifo_q_put(reqs, q->reqs, req); + + /* Build the term before releasing the lock so as not to race on the use of + the req pointer (which will soon become invalid in another thread + performing the request). */ + ERL_NIF_TERM reply = enif_make_tuple2(req->env, enif_make_atom(req->env, "ok"), + enif_make_atom(req->env, "enqueued")); + enif_mutex_unlock(q->reqs_mutex); + enif_cond_signal(q->reqs_cnd); + return reply; +} + +static void * +async_nif_worker_fn(void *arg) +{ + struct async_nif_worker_entry *we = (struct async_nif_worker_entry *)arg; + unsigned int worker_id = we->worker_id; + struct async_nif_state *async_nif = we->async_nif; + struct async_nif_work_queue *q = we->q; + struct async_nif_req_entry *req = NULL; + + for(;;) { + /* Examine the request queue, are there things to be done? */ + enif_mutex_lock(q->reqs_mutex); + check_again_for_work: + if (async_nif->shutdown) { + enif_mutex_unlock(q->reqs_mutex); + break; + } + if (fifo_q_empty(reqs, q->reqs)) { + /* Queue is empty, wait for work */ + enif_cond_wait(q->reqs_cnd, q->reqs_mutex); + goto check_again_for_work; + } else { + assert(fifo_q_size(reqs, q->reqs) > 0); + assert(fifo_q_size(reqs, q->reqs) < fifo_q_capacity(reqs, q->reqs)); + /* At this point the next req is ours to process and we hold the + reqs_mutex lock. Take the request off the queue. */ + req = fifo_q_get(reqs, q->reqs); + enif_mutex_unlock(q->reqs_mutex); + + /* Ensure that there is at least one other worker thread watching this + queue. */ + enif_cond_signal(q->reqs_cnd); + + /* Perform the work. */ + req->fn_work(req->env, req->ref, &req->pid, worker_id, req->args); + + /* Now call the post-work cleanup function. */ + req->fn_post(req->args); + + /* Free resources allocated for this async request. */ + enif_free_env(req->env); + enif_free(req->args); + enif_free(req); + req = NULL; + } + } + enif_thread_exit(0); + return 0; +} + +static void +async_nif_unload(ErlNifEnv *env, struct async_nif_state *async_nif) +{ + unsigned int i; + unsigned int num_queues = async_nif->num_queues; + struct async_nif_work_queue *q = NULL; + + /* Signal the worker threads, stop what you're doing and exit. To + ensure that we don't race with the enqueue() process we first + lock all the worker queues, then set shutdown to true, then + unlock. The enqueue function will take the queue mutex, then + test for shutdown condition, then enqueue only if not shutting + down. */ + for (i = 0; i < num_queues; i++) { + q = &async_nif->queues[i]; + enif_mutex_lock(q->reqs_mutex); + } + async_nif->shutdown = 1; + for (i = 0; i < num_queues; i++) { + q = &async_nif->queues[i]; + enif_cond_broadcast(q->reqs_cnd); + enif_mutex_unlock(q->reqs_mutex); + } + + /* Join for the now exiting worker threads. */ + for (i = 0; i < async_nif->num_workers; ++i) { + void *exit_value = 0; /* We ignore the thread_join's exit value. */ + enif_thread_join(async_nif->worker_entries[i].tid, &exit_value); + } + + /* Cleanup requests, mutexes and conditions in each work queue. */ + for (i = 0; i < num_queues; i++) { + q = &async_nif->queues[i]; + + /* Worker threads are stopped, now toss anything left in the queue. */ + struct async_nif_req_entry *req = NULL; + fifo_q_foreach(reqs, q->reqs, req, { + enif_send(NULL, &req->pid, req->env, + enif_make_tuple2(req->env, enif_make_atom(req->env, "error"), + enif_make_atom(req->env, "shutdown"))); + req->fn_post(req->args); + enif_free_env(req->env); + enif_free(req->args); + enif_free(req); + }); + fifo_q_free(reqs, q->reqs); + enif_mutex_destroy(q->reqs_mutex); + enif_cond_destroy(q->reqs_cnd); + } + memset(async_nif, 0, sizeof(struct async_nif_state) + (sizeof(struct async_nif_work_queue) * async_nif->num_queues)); + enif_free(async_nif); +} + +static void * +async_nif_load() +{ + static int has_init = 0; + unsigned int i, j, num_queues; + ErlNifSysInfo info; + struct async_nif_state *async_nif; + + /* Don't init more than once. */ + if (has_init) return 0; + else has_init = 1; + + /* Find out how many schedulers there are. */ + enif_system_info(&info, sizeof(ErlNifSysInfo)); + + /* Size the number of work queues according to schedulers. */ + if (info.scheduler_threads > ASYNC_NIF_MAX_WORKERS / 2) { + num_queues = ASYNC_NIF_MAX_WORKERS / 2; + } else { + int remainder = ASYNC_NIF_MAX_WORKERS % info.scheduler_threads; + if (remainder != 0) + num_queues = info.scheduler_threads - remainder; + else + num_queues = info.scheduler_threads; + if (num_queues < 2) + num_queues = 2; + } + + /* Init our portion of priv_data's module-specific state. */ + async_nif = enif_alloc(sizeof(struct async_nif_state) + + sizeof(struct async_nif_work_queue) * num_queues); + if (!async_nif) + return NULL; + memset(async_nif, 0, sizeof(struct async_nif_state) + + sizeof(struct async_nif_work_queue) * num_queues); + + async_nif->num_queues = num_queues; + async_nif->num_workers = ASYNC_NIF_MAX_WORKERS; // TODO: start with 2 per queue, then grow if needed + async_nif->next_q = 0; + async_nif->shutdown = 0; + + for (i = 0; i < async_nif->num_queues; i++) { + struct async_nif_work_queue *q = &async_nif->queues[i]; + q->reqs = fifo_q_new(reqs, ASYNC_NIF_WORKER_QUEUE_SIZE); + q->reqs_mutex = enif_mutex_create(NULL); + q->reqs_cnd = enif_cond_create(NULL); + } + + /* Setup the thread pool management. */ + memset(async_nif->worker_entries, 0, sizeof(struct async_nif_worker_entry) * ASYNC_NIF_MAX_WORKERS); + + /* Start the worker threads. */ + for (i = 0; i < async_nif->num_workers; i++) { + struct async_nif_worker_entry *we = &async_nif->worker_entries[i]; + we->async_nif = async_nif; + we->worker_id = i; + we->q = &async_nif->queues[i % async_nif->num_queues]; + if (enif_thread_create(NULL, &async_nif->worker_entries[i].tid, + &async_nif_worker_fn, (void*)we, NULL) != 0) { + async_nif->shutdown = 1; + + for (j = 0; j < async_nif->num_queues; j++) { + struct async_nif_work_queue *q = &async_nif->queues[j]; + enif_cond_broadcast(q->reqs_cnd); + } + + while(i-- > 0) { + void *exit_value = 0; /* Ignore this. */ + enif_thread_join(async_nif->worker_entries[i].tid, &exit_value); + } + + for (j = 0; j < async_nif->num_queues; j++) { + struct async_nif_work_queue *q = &async_nif->queues[j]; + enif_mutex_destroy(q->reqs_mutex); + enif_cond_destroy(q->reqs_cnd); + } + + memset(async_nif->worker_entries, 0, sizeof(struct async_nif_worker_entry) * ASYNC_NIF_MAX_WORKERS); + enif_free(async_nif); + return NULL; + } + } + return async_nif; +} + +static void +async_nif_upgrade(ErlNifEnv *env) +{ + // TODO: +} + + +#if defined(__cplusplus) +} +#endif + +#endif // __ASYNC_NIF_H__ diff --git a/c_src/build_deps.sh b/c_src/build_deps.sh index 6ecdc50..70163f5 100755 --- a/c_src/build_deps.sh +++ b/c_src/build_deps.sh @@ -1,40 +1,185 @@ #!/bin/bash +# /bin/sh on Solaris is not a POSIX compatible shell, but /usr/bin/ksh is. +if [ `uname -s` = 'SunOS' -a "${POSIX_SHELL}" != "true" ]; then + POSIX_SHELL="true" + export POSIX_SHELL + exec /usr/bin/ksh $0 $@ +fi +unset POSIX_SHELL # clear it so if we invoke other scripts, they run as ksh as well + set -e +WT_REPO=http://github.com/wiredtiger/wiredtiger.git WT_BRANCH=basho -WT_REMOTE_REPO=http://github.com/wiredtiger/wiredtiger.git +WT_VSN="" +WT_DIR=wiredtiger-$WT_BRANCH + +SNAPPY_VSN="1.0.4" +SNAPPY_DIR=snappy-$SNAPPY_VSN + +BZIP2_VSN="1.0.6" +BZIP2_DIR=bzip2-$BZIP2_VSN [ `basename $PWD` != "c_src" ] && cd c_src -BASEDIR="$PWD" +export BASEDIR="$PWD" + +which gmake 1>/dev/null 2>/dev/null && MAKE=gmake +MAKE=${MAKE:-make} + +export CFLAGS="$CFLAGS -g -I $BASEDIR/system/include" +export CXXFLAGS="$CXXFLAGS -I $BASEDIR/system/include" +export LDFLAGS="$LDFLAGS -L$BASEDIR/system/lib" +export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$BASEDIR/system/lib:$LD_LIBRARY_PATH" + +get_wt () +{ + if [ -d $BASEDIR/$WT_DIR/.git ]; then + (cd $BASEDIR/$WT_DIR && git pull -u) || exit 1 + else + if [ "X$WT_VSN" == "X" ]; then + git clone ${WT_REPO} && \ + (cd $BASEDIR/wiredtiger && git checkout $WT_VSN || exit 1) + else + git clone -b ${WT_BRANCH} --single-branch ${WT_REPO} && \ + (cd $BASEDIR/wiredtiger && git checkout -b $WT_BRANCH origin/$WT_BRANCH || exit 1) + fi + mv wiredtiger $WT_DIR || exit 1 + fi + [ -d $BASEDIR/$WT_DIR ] || (echo "Missing WiredTiger source directory" && exit 1) + (cd $BASEDIR/$WT_DIR && git cherry-pick a3c8c2a13758ae9c44edabcc1a780984a7882904 || exit 1) + (cd $BASEDIR/$WT_DIR + [ -e $BASEDIR/wiredtiger-build.patch ] && \ + (patch -p1 --forward < $BASEDIR/wiredtiger-build.patch || exit 1 ) + ./autogen.sh || exit 1 + cd ./build_posix || exit 1 + [ -e Makefile ] && $MAKE distclean + ../configure --with-pic \ + --enable-snappy \ + --enable-bzip2 \ + --prefix=${BASEDIR}/system || exit 1 + ) +} + +get_snappy () +{ + [ -e snappy-$SNAPPY_VSN.tar.gz ] || (echo "Missing Snappy ($SNAPPY_VSN) source package" && exit 1) + [ -d $BASEDIR/$SNAPPY_DIR ] || tar -xzf snappy-$SNAPPY_VSN.tar.gz + [ -e $BASEDIR/snappy-build.patch ] && \ + (cd $BASEDIR/$SNAPPY_DIR + patch -p1 --forward < $BASEDIR/snappy-build.patch || exit 1) + (cd $BASEDIR/$SNAPPY_DIR + ./configure --with-pic --prefix=$BASEDIR/system || exit 1) +} + +get_bzip2 () +{ + [ -e bzip2-$BZIP2_VSN.tar.gz ] || (echo "Missing bzip2 ($BZIP2_VSN) source package" && exit 1) + [ -d $BASEDIR/$BZIP2_DIR ] || tar -xzf bzip2-$BZIP2_VSN.tar.gz + [ -e $BASEDIR/bzip2-build.patch ] && \ + (cd $BASEDIR/$BZIP2_DIR + patch -p1 --forward < $BASEDIR/bzip2-build.patch || exit 1) +} + +get_deps () +{ + get_wt; + get_snappy; + get_bzip2; +} + +update_deps () +{ + if [ -d $BASEDIR/$WT_DIR/.git ]; then + (cd $BASEDIR/$WT_DIR + if [ "X$WT_VSN" == "X" ]; then + git pull -u || exit 1 + else + git checkout $WT_VSN || exit 1 + fi + ) + fi +} + +build_wt () +{ + (cd $BASEDIR/$WT_DIR/build_posix && \ + $MAKE -j && $MAKE install) +} + +build_snappy () +{ + (cd $BASEDIR/$SNAPPY_DIR && \ + $MAKE -j && \ + $MAKE install + ) +} + +build_bzip2 () +{ + (cd $BASEDIR/$BZIP2_DIR && \ + $MAKE -j -f Makefile-libbz2_so && \ + mkdir -p $BASEDIR/system/lib && \ + cp -f bzlib.h $BASEDIR/system/include && \ + cp -f libbz2.so.1.0.6 $BASEDIR/system/lib && \ + ln -s $BASEDIR/system/lib/libbz2.so.1.0.6 $BASEDIR/system/lib/libbz2.so && \ + ln -s $BASEDIR/system/lib/libbz2.so.1.0.6 $BASEDIR/system/lib/libbz2-1.so && \ + ln -s $BASEDIR/system/lib/libbz2.so.1.0.6 $BASEDIR/system/lib/libbz2-1.0.so + ) +} case "$1" in clean) - rm -rf system wiredtiger + rm -rf system $WT_DIR $SNAPPY_DIR $BZIP2_DIR + rm -f ${BASEDIR}/../priv/wt + rm -f ${BASEDIR}/../priv/libwiredtiger-*.so + rm -f ${BASEDIR}/../priv/libwiredtiger_*.so + rm -f ${BASEDIR}/../priv/libbz2.so.* + rm -f ${BASEDIR}/../priv/libsnappy.so.* + ;; + + test) + (cd $BASEDIR/$WT_DIR && $MAKE -j test) + ;; + + update-deps) + update-deps; + ;; + + get-deps) + get_deps; ;; *) - test -f system/lib/libwiredtiger.a && exit 0 + [ -d $WT_DIR ] || get_wt; + [ -d $SNAPPY_DIR ] || get_snappy; + [ -d $BZIP2_DIR ] || get_bzip2; - if [ -d wiredtiger/.git ]; then - (cd wiredtiger && \ - git fetch && \ - git merge origin/$WT_BRANCH) - else - git clone -b $WT_BRANCH --single-branch $WT_REMOTE_REPO && \ - (cd wiredtiger && \ - patch -p1 < ../wiredtiger-extension-link.patch && \ - ./autogen.sh) - fi - (cd wiredtiger/build_posix && \ - ../configure --with-pic \ - --enable-snappy \ - --enable-bzip2 \ - --prefix=$BASEDIR/system && \ - make -j && make install) - [ -d $BASEDIR/../priv ] || mkdir $BASEDIR/../priv - cp $BASEDIR/system/bin/wt $BASEDIR/../priv - cp $BASEDIR/system/lib/*.so $BASEDIR/../priv + # Build Snappy + [ -d $BASEDIR/$SNAPPY_DIR ] || (echo "Missing Snappy source directory" && exit 1) + test -f $BASEDIR/system/lib/libsnappy.so.[0-9].[0-9].[0-9] || build_snappy; + + # Build BZIP2 + [ -d $BASEDIR/$BZIP2_DIR ] || (echo "Missing BZip2 source directory" && exit 1) + test -f $BASEDIR/system/lib/libbz2.so.[0-9].[0-9].[0-9] || build_bzip2; + + # Build WiredTiger + [ -d $BASEDIR/$WT_DIR ] || (echo "Missing WiredTiger source directory" && exit 1) + test -f $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].so \ + -a -f $BASEDIR/system/lib/libwiredtiger_snappy.so \ + -a -f $BASEDIR/system/lib/libwiredtiger_bzip2.so.[0-9].[0-9].[0-9] || build_wt; + + [ -d $BASEDIR/../priv ] || mkdir ${BASEDIR}/../priv + cp -p -P $BASEDIR/system/bin/wt ${BASEDIR}/../priv + cp -p -P $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].so ${BASEDIR}/../priv + cp -p -P $BASEDIR/system/lib/libwiredtiger_snappy.so ${BASEDIR}/../priv + cp -p -P $BASEDIR/system/lib/libwiredtiger_bzip2.so* ${BASEDIR}/../priv + cp -p -P $BASEDIR/system/lib/libbz2.so.[0-9].[0-9].[0-9] ${BASEDIR}/../priv + (cd ${BASEDIR}/../priv + [ -L libbz2.so ] || ln -s libbz2.so.1.0.6 libbz2.so + [ -L libbz2.so.1 ] || ln -s libbz2.so.1.0.6 libbz2.so.1 + [ -L libbz2.so.1.0 ] || ln -s libbz2.so.1.0.6 libbz2.so.1.0) + cp -p -P $BASEDIR/system/lib/libsnappy.so* ${BASEDIR}/../priv ;; esac diff --git a/c_src/bzip2-1.0.6.tar.gz b/c_src/bzip2-1.0.6.tar.gz new file mode 100644 index 0000000..e47e903 Binary files /dev/null and b/c_src/bzip2-1.0.6.tar.gz differ diff --git a/c_src/bzip2-build.patch b/c_src/bzip2-build.patch new file mode 100644 index 0000000..e69de29 diff --git a/c_src/fifo_q.h b/c_src/fifo_q.h new file mode 100644 index 0000000..b9291f2 --- /dev/null +++ b/c_src/fifo_q.h @@ -0,0 +1,102 @@ +/* + * fifo_q: a macro-based implementation of a FIFO Queue + * + * Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. + * Author: Gregory Burd + * + * This file is provided to you under the Apache License, + * Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef __FIFO_Q_H__ +#define __FIFO_Q_H__ + +#if defined(__cplusplus) +extern "C" { +#endif + +#define FIFO_QUEUE_TYPE(name) \ + struct fifo_q__ ## name * +#define DECL_FIFO_QUEUE(name, type) \ + struct fifo_q__ ## name { \ + unsigned int h, t, s; \ + type *items[]; \ + }; \ + static struct fifo_q__ ## name *fifo_q_ ## name ## _new(unsigned int n) { \ + int sz = sizeof(struct fifo_q__ ## name) + ((n+1) * sizeof(type *));\ + struct fifo_q__ ## name *q = enif_alloc(sz); \ + if (!q) \ + return 0; \ + memset(q, 0, sz); \ + q->s = n + 1; \ + return q; \ + } \ + static inline void fifo_q_ ## name ## _free(struct fifo_q__ ## name *q) { \ + memset(q, 0, sizeof(struct fifo_q__ ## name) + (q->s * sizeof(type *))); \ + enif_free(q); \ + } \ + static inline type *fifo_q_ ## name ## _put(struct fifo_q__ ## name *q, type *n) { \ + q->items[q->h] = n; \ + q->h = (q->h + 1) % q->s; \ + return n; \ + } \ + static inline type *fifo_q_ ## name ## _get(struct fifo_q__ ## name *q) { \ + type *n = q->items[q->t]; \ + q->items[q->t] = 0; \ + q->t = (q->t + 1) % q->s; \ + return n; \ + } \ + static inline unsigned int fifo_q_ ## name ## _size(struct fifo_q__ ## name *q) { \ + return (q->h - q->t + q->s) % q->s; \ + } \ + static inline unsigned int fifo_q_ ## name ## _capacity(struct fifo_q__ ## name *q) { \ + return q->s - 1; \ + } \ + static inline int fifo_q_ ## name ## _empty(struct fifo_q__ ## name *q) { \ + return (q->t == q->h); \ + } \ + static inline int fifo_q_ ## name ## _full(struct fifo_q__ ## name *q) { \ + return ((q->h + 1) % q->s) == q->t; \ + } + +#define fifo_q_new(name, size) fifo_q_ ## name ## _new(size) +#define fifo_q_free(name, queue) fifo_q_ ## name ## _free(queue) +#define fifo_q_get(name, queue) fifo_q_ ## name ## _get(queue) +#define fifo_q_put(name, queue, item) fifo_q_ ## name ## _put(queue, item) +#define fifo_q_size(name, queue) fifo_q_ ## name ## _size(queue) +#define fifo_q_capacity(name, queue) fifo_q_ ## name ## _capacity(queue) +#define fifo_q_empty(name, queue) fifo_q_ ## name ## _empty(queue) +#define fifo_q_full(name, queue) fifo_q_ ## name ## _full(queue) +#define fifo_q_foreach(name, queue, item, task) do { \ + while(!fifo_q_ ## name ## _empty(queue)) { \ + item = fifo_q_ ## name ## _get(queue); \ + do task while(0); \ + } \ + } while(0); + +struct async_nif_req_entry { + ERL_NIF_TERM ref; + ErlNifEnv *env; + ErlNifPid pid; + void *args; + void (*fn_work)(ErlNifEnv*, ERL_NIF_TERM, ErlNifPid*, unsigned int, void *); + void (*fn_post)(void *); +}; + + +#if defined(__cplusplus) +} +#endif + +#endif // __FIFO_Q_H__ diff --git a/c_src/khash.h b/c_src/khash.h new file mode 100644 index 0000000..ab157b1 --- /dev/null +++ b/c_src/khash.h @@ -0,0 +1,643 @@ +/* The MIT License + + Copyright (c) 2008, 2009, 2011 by Attractive Chaos + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +/* + An example: + + #include "khash.h" + KHASH_MAP_INIT_INT(32, char) + int main() { + int ret, is_missing; + khiter_t k; + khash_t(32) *h = kh_init(32); + k = kh_put(32, h, 5, &ret); + kh_value(h, k) = 10; + k = kh_get(32, h, 10); + is_missing = (k == kh_end(h)); + k = kh_get(32, h, 5); + kh_del(32, h, k); + for (k = kh_begin(h); k != kh_end(h); ++k) + if (kh_exist(h, k)) kh_value(h, k) = 1; + kh_destroy(32, h); + return 0; + } +*/ + +/* + 2011-12-29 (0.2.7): + + * Minor code clean up; no actual effect. + + 2011-09-16 (0.2.6): + + * The capacity is a power of 2. This seems to dramatically improve the + speed for simple keys. Thank Zilong Tan for the suggestion. Reference: + + - http://code.google.com/p/ulib/ + - http://nothings.org/computer/judy/ + + * Allow to optionally use linear probing which usually has better + performance for random input. Double hashing is still the default as it + is more robust to certain non-random input. + + * Added Wang's integer hash function (not used by default). This hash + function is more robust to certain non-random input. + + 2011-02-14 (0.2.5): + + * Allow to declare global functions. + + 2009-09-26 (0.2.4): + + * Improve portability + + 2008-09-19 (0.2.3): + + * Corrected the example + * Improved interfaces + + 2008-09-11 (0.2.2): + + * Improved speed a little in kh_put() + + 2008-09-10 (0.2.1): + + * Added kh_clear() + * Fixed a compiling error + + 2008-09-02 (0.2.0): + + * Changed to token concatenation which increases flexibility. + + 2008-08-31 (0.1.2): + + * Fixed a bug in kh_get(), which has not been tested previously. + + 2008-08-31 (0.1.1): + + * Added destructor + */ + + +#ifndef __AC_KHASH_H +#define __AC_KHASH_H + +/*! + @header + + Generic hash table library. +*/ + +#define AC_VERSION_KHASH_H "0.2.6" + +#include +#include +#include + +/* compiler specific configuration */ + +#if UINT_MAX == 0xffffffffu +typedef unsigned int khint32_t; +#elif ULONG_MAX == 0xffffffffu +typedef unsigned long khint32_t; +#endif + +#if ULONG_MAX == ULLONG_MAX +typedef unsigned long khint64_t; +#else +typedef unsigned long long khint64_t; +#endif + +#ifdef _MSC_VER +#define kh_inline __inline +#else +#define kh_inline inline +#endif + +typedef khint32_t khint_t; +typedef khint_t khiter_t; + +#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2) +#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1) +#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3) +#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1))) +#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1))) +#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1))) +#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1)) + +#ifdef KHASH_LINEAR +#define __ac_inc(k, m) 1 +#else +#define __ac_inc(k, m) (((k)>>3 ^ (k)<<3) | 1) & (m) +#endif + +#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4) + +#ifndef kroundup32 +#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x)) +#endif + +#ifndef kcalloc +#define kcalloc(N,Z) calloc(N,Z) +#endif +#ifndef kmalloc +#define kmalloc(Z) malloc(Z) +#endif +#ifndef krealloc +#define krealloc(P,Z) realloc(P,Z) +#endif +#ifndef kfree +#define kfree(P) free(P) +#endif + +static const double __ac_HASH_UPPER = 0.77; + +#define __KHASH_TYPE(name, khkey_t, khval_t) \ + typedef struct { \ + khint_t n_buckets, size, n_occupied, upper_bound; \ + khint32_t *flags; \ + khkey_t *keys; \ + khval_t *vals; \ + } kh_##name##_t; + +#define __KHASH_PROTOTYPES(name, khkey_t, khval_t) \ + extern kh_##name##_t *kh_init_##name(void); \ + extern void kh_destroy_##name(kh_##name##_t *h); \ + extern void kh_clear_##name(kh_##name##_t *h); \ + extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \ + extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \ + extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \ + extern void kh_del_##name(kh_##name##_t *h, khint_t x); + +#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ + SCOPE kh_##name##_t *kh_init_##name(void) { \ + return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t)); \ + } \ + SCOPE void kh_destroy_##name(kh_##name##_t *h) \ + { \ + if (h) { \ + kfree((void *)h->keys); kfree(h->flags); \ + kfree((void *)h->vals); \ + kfree(h); \ + } \ + } \ + SCOPE void kh_clear_##name(kh_##name##_t *h) \ + { \ + if (h && h->flags) { \ + memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \ + h->size = h->n_occupied = 0; \ + } \ + } \ + SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \ + { \ + if (h->n_buckets) { \ + khint_t inc, k, i, last, mask; \ + mask = h->n_buckets - 1; \ + k = __hash_func(key); i = k & mask; \ + inc = __ac_inc(k, mask); last = i; /* inc==1 for linear probing */ \ + while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ + i = (i + inc) & mask; \ + if (i == last) return h->n_buckets; \ + } \ + return __ac_iseither(h->flags, i)? h->n_buckets : i; \ + } else return 0; \ + } \ + SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \ + { /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \ + khint32_t *new_flags = 0; \ + khint_t j = 1; \ + { \ + kroundup32(new_n_buckets); \ + if (new_n_buckets < 4) new_n_buckets = 4; \ + if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; /* requested size is too small */ \ + else { /* hash table size to be changed (shrink or expand); rehash */ \ + new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t)); \ + if (!new_flags) return -1; \ + memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \ + if (h->n_buckets < new_n_buckets) { /* expand */ \ + khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \ + if (!new_keys) return -1; \ + h->keys = new_keys; \ + if (kh_is_map) { \ + khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \ + if (!new_vals) return -1; \ + h->vals = new_vals; \ + } \ + } /* otherwise shrink */ \ + } \ + } \ + if (j) { /* rehashing is needed */ \ + for (j = 0; j != h->n_buckets; ++j) { \ + if (__ac_iseither(h->flags, j) == 0) { \ + khkey_t key = h->keys[j]; \ + khval_t val; \ + khint_t new_mask; \ + new_mask = new_n_buckets - 1; \ + if (kh_is_map) val = h->vals[j]; \ + __ac_set_isdel_true(h->flags, j); \ + while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \ + khint_t inc, k, i; \ + k = __hash_func(key); \ + i = k & new_mask; \ + inc = __ac_inc(k, new_mask); \ + while (!__ac_isempty(new_flags, i)) i = (i + inc) & new_mask; \ + __ac_set_isempty_false(new_flags, i); \ + if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \ + { khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \ + if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \ + __ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \ + } else { /* write the element and jump out of the loop */ \ + h->keys[i] = key; \ + if (kh_is_map) h->vals[i] = val; \ + break; \ + } \ + } \ + } \ + } \ + if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \ + h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \ + if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \ + } \ + kfree(h->flags); /* free the working space */ \ + h->flags = new_flags; \ + h->n_buckets = new_n_buckets; \ + h->n_occupied = h->size; \ + h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \ + } \ + return 0; \ + } \ + SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \ + { \ + khint_t x; \ + if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \ + if (h->n_buckets > (h->size<<1)) { \ + if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \ + *ret = -1; return h->n_buckets; \ + } \ + } else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \ + *ret = -1; return h->n_buckets; \ + } \ + } /* TODO: to implement automatically shrinking; resize() already support shrinking */ \ + { \ + khint_t inc, k, i, site, last, mask = h->n_buckets - 1; \ + x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \ + if (__ac_isempty(h->flags, i)) x = i; /* for speed up */ \ + else { \ + inc = __ac_inc(k, mask); last = i; \ + while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \ + if (__ac_isdel(h->flags, i)) site = i; \ + i = (i + inc) & mask; \ + if (i == last) { x = site; break; } \ + } \ + if (x == h->n_buckets) { \ + if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \ + else x = i; \ + } \ + } \ + } \ + if (__ac_isempty(h->flags, x)) { /* not present at all */ \ + h->keys[x] = key; \ + __ac_set_isboth_false(h->flags, x); \ + ++h->size; ++h->n_occupied; \ + *ret = 1; \ + } else if (__ac_isdel(h->flags, x)) { /* deleted */ \ + h->keys[x] = key; \ + __ac_set_isboth_false(h->flags, x); \ + ++h->size; \ + *ret = 2; \ + } else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \ + return x; \ + } \ + SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x) \ + { \ + if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \ + __ac_set_isdel_true(h->flags, x); \ + --h->size; \ + } \ + } + +#define KHASH_DECLARE(name, khkey_t, khval_t) \ + __KHASH_TYPE(name, khkey_t, khval_t) \ + __KHASH_PROTOTYPES(name, khkey_t, khval_t) + +#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ + __KHASH_TYPE(name, khkey_t, khval_t) \ + __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) + +#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \ + KHASH_INIT2(name, static kh_inline, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) + +/* --- BEGIN OF HASH FUNCTIONS --- */ + +/*! @function + @abstract Integer hash function + @param key The integer [khint32_t] + @return The hash value [khint_t] +*/ +#define kh_int_hash_func(key) (khint32_t)(key) +/*! @function + @abstract Integer comparison function +*/ +#define kh_int_hash_equal(a, b) ((a) == (b)) +/*! @function + @abstract 64-bit integer hash function + @param key The integer [khint64_t] + @return The hash value [khint_t] +*/ +#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11) +/*! @function + @abstract 64-bit integer comparison function +*/ +#define kh_int64_hash_equal(a, b) ((a) == (b)) +/*! @function + @abstract Pointer hash function + @param key The integer void * + @return The hash value [khint_t] +*/ +#define kh_ptr_hash_func(key) (khint32_t)(key) +/*! @function + @abstract Pointer comparison function +*/ +#define kh_ptr_hash_equal(a, b) ((a) == (b)) +/*! @function + @abstract 64-bit pointer hash function + @param key The integer void * + @return The hash value [khint_t] +*/ +#define kh_ptr64_hash_func(key) (khint32_t)(((khint64_t)key)>>33^((khint64_t)key)^((khint64_t)key)<<11) +/*! @function + @abstract 64-bit pointer comparison function +*/ +#define kh_ptr64_hash_equal(a, b) ((a) == (b)) +/*! @function + @abstract const char* hash function + @param s Pointer to a null terminated string + @return The hash value +*/ +static kh_inline khint_t __ac_X31_hash_string(const char *s) +{ + khint_t h = (khint_t)*s; + if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s; + return h; +} +/*! @function + @abstract Another interface to const char* hash function + @param key Pointer to a null terminated string [const char*] + @return The hash value [khint_t] +*/ +#define kh_str_hash_func(key) __ac_X31_hash_string(key) +/*! @function + @abstract Const char* comparison function +*/ +#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0) + +static kh_inline khint_t __ac_Wang_hash(khint_t key) +{ + key += ~(key << 15); + key ^= (key >> 10); + key += (key << 3); + key ^= (key >> 6); + key += ~(key << 11); + key ^= (key >> 16); + return key; +} +#define kh_int_hash_func2(k) __ac_Wang_hash((khint_t)key) + +/* --- END OF HASH FUNCTIONS --- */ + +/* Other convenient macros... */ + +/*! + @abstract Type of the hash table. + @param name Name of the hash table [symbol] +*/ +#define khash_t(name) kh_##name##_t + +/*! @function + @abstract Initiate a hash table. + @param name Name of the hash table [symbol] + @return Pointer to the hash table [khash_t(name)*] +*/ +#define kh_init(name) kh_init_##name() + +/*! @function + @abstract Destroy a hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] +*/ +#define kh_destroy(name, h) kh_destroy_##name(h) + +/*! @function + @abstract Reset a hash table without deallocating memory. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] +*/ +#define kh_clear(name, h) kh_clear_##name(h) + +/*! @function + @abstract Resize a hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param s New size [khint_t] +*/ +#define kh_resize(name, h, s) kh_resize_##name(h, s) + +/*! @function + @abstract Insert a key to the hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param k Key [type of keys] + @param r Extra return code: 0 if the key is present in the hash table; + 1 if the bucket is empty (never used); 2 if the element in + the bucket has been deleted [int*] + @return Iterator to the inserted element [khint_t] +*/ +#define kh_put(name, h, k, r) kh_put_##name(h, k, r) + +/*! @function + @abstract Retrieve a key from the hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param k Key [type of keys] + @return Iterator to the found element, or kh_end(h) if the element is absent [khint_t] +*/ +#define kh_get(name, h, k) kh_get_##name(h, k) + +/*! @function + @abstract Remove a key from the hash table. + @param name Name of the hash table [symbol] + @param h Pointer to the hash table [khash_t(name)*] + @param k Iterator to the element to be deleted [khint_t] +*/ +#define kh_del(name, h, k) kh_del_##name(h, k) + +/*! @function + @abstract Test whether a bucket contains data. + @param h Pointer to the hash table [khash_t(name)*] + @param x Iterator to the bucket [khint_t] + @return 1 if containing data; 0 otherwise [int] +*/ +#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x))) + +/*! @function + @abstract Get key given an iterator + @param h Pointer to the hash table [khash_t(name)*] + @param x Iterator to the bucket [khint_t] + @return Key [type of keys] +*/ +#define kh_key(h, x) ((h)->keys[x]) + +/*! @function + @abstract Get value given an iterator + @param h Pointer to the hash table [khash_t(name)*] + @param x Iterator to the bucket [khint_t] + @return Value [type of values] + @discussion For hash sets, calling this results in segfault. +*/ +#define kh_val(h, x) ((h)->vals[x]) + +/*! @function + @abstract Alias of kh_val() +*/ +#define kh_value(h, x) ((h)->vals[x]) + +/*! @function + @abstract Get the start iterator + @param h Pointer to the hash table [khash_t(name)*] + @return The start iterator [khint_t] +*/ +#define kh_begin(h) (khint_t)(0) + +/*! @function + @abstract Get the end iterator + @param h Pointer to the hash table [khash_t(name)*] + @return The end iterator [khint_t] +*/ +#define kh_end(h) ((h)->n_buckets) + +/*! @function + @abstract Get the number of elements in the hash table + @param h Pointer to the hash table [khash_t(name)*] + @return Number of elements in the hash table [khint_t] +*/ +#define kh_size(h) ((h)->size) + +/*! @function + @abstract Get the number of buckets in the hash table + @param h Pointer to the hash table [khash_t(name)*] + @return Number of buckets in the hash table [khint_t] +*/ +#define kh_n_buckets(h) ((h)->n_buckets) + +/*! @function + @abstract Iterate over the entries in the hash table + @param h Pointer to the hash table [khash_t(name)*] + @param kvar Variable to which key will be assigned + @param vvar Variable to which value will be assigned + @param code Block of code to execute +*/ +#define kh_foreach(h, kvar, vvar, code) { khint_t __i; \ + for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \ + if (!kh_exist(h,__i)) continue; \ + (kvar) = kh_key(h,__i); \ + (vvar) = kh_val(h,__i); \ + code; \ + } } + +/*! @function + @abstract Iterate over the values in the hash table + @param h Pointer to the hash table [khash_t(name)*] + @param vvar Variable to which value will be assigned + @param code Block of code to execute +*/ +#define kh_foreach_value(h, vvar, code) { khint_t __i; \ + for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \ + if (!kh_exist(h,__i)) continue; \ + (vvar) = kh_val(h,__i); \ + code; \ + } } + +/* More conenient interfaces */ + +/*! @function + @abstract Instantiate a hash map containing (void *) keys + @param name Name of the hash table [symbol] + @param khval_t Type of values [type] +*/ +#ifdef __x86_64__ +#define KHASH_MAP_INIT_PTR(name, khval_t) \ + KHASH_INIT(name, void*, khval_t, 1, kh_ptr64_hash_func, kh_ptr64_hash_equal) +#else +#define KHASH_MAP_INIT_PTR(name, khval_t) \ + KHASH_INIT(name, void*, khval_t, 1, kh_ptr_hash_func, kh_ptr_hash_equal) +#endif + +/*! @function + @abstract Instantiate a hash set containing integer keys + @param name Name of the hash table [symbol] +*/ +#define KHASH_SET_INIT_INT(name) \ + KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing integer keys + @param name Name of the hash table [symbol] + @param khval_t Type of values [type] +*/ +#define KHASH_MAP_INIT_INT(name, khval_t) \ + KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing 64-bit integer keys + @param name Name of the hash table [symbol] +*/ +#define KHASH_SET_INIT_INT64(name) \ + KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing 64-bit integer keys + @param name Name of the hash table [symbol] + @param khval_t Type of values [type] +*/ +#define KHASH_MAP_INIT_INT64(name, khval_t) \ + KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal) + +typedef const char *kh_cstr_t; +/*! @function + @abstract Instantiate a hash map containing const char* keys + @param name Name of the hash table [symbol] +*/ +#define KHASH_SET_INIT_STR(name) \ + KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal) + +/*! @function + @abstract Instantiate a hash map containing const char* keys + @param name Name of the hash table [symbol] + @param khval_t Type of values [type] +*/ +#define KHASH_MAP_INIT_STR(name, khval_t) \ + KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal) + +#endif /* __AC_KHASH_H */ diff --git a/c_src/snappy-1.0.4.tar.gz b/c_src/snappy-1.0.4.tar.gz new file mode 100644 index 0000000..a34852f Binary files /dev/null and b/c_src/snappy-1.0.4.tar.gz differ diff --git a/c_src/wiredtiger-extension-link.patch b/c_src/wiredtiger-build.patch similarity index 54% rename from c_src/wiredtiger-extension-link.patch rename to c_src/wiredtiger-build.patch index 46beb29..54ab0ea 100644 --- a/c_src/wiredtiger-extension-link.patch +++ b/c_src/wiredtiger-build.patch @@ -1,22 +1,24 @@ diff --git a/ext/compressors/bzip2/Makefile.am b/ext/compressors/bzip2/Makefile.am -index 0aedc2e..1cc4cf6 100644 +index 0aedc2e..a70ae2e 100644 --- a/ext/compressors/bzip2/Makefile.am +++ b/ext/compressors/bzip2/Makefile.am -@@ -2,5 +2,5 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include +@@ -2,5 +2,6 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include lib_LTLIBRARIES = libwiredtiger_bzip2.la libwiredtiger_bzip2_la_SOURCES = bzip2_compress.c -libwiredtiger_bzip2_la_LDFLAGS = -avoid-version -module -+libwiredtiger_bzip2_la_LDFLAGS = -avoid-version -module -Wl,-rpath,lib/wterl/priv:priv:/usr/local/lib ++libwiredtiger_snappy_la_CFLAGS = -I$(src_builddir)/../../system/include ++libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(src_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv libwiredtiger_bzip2_la_LIBADD = -lbz2 diff --git a/ext/compressors/snappy/Makefile.am b/ext/compressors/snappy/Makefile.am -index 6d78823..7d35777 100644 +index 6d78823..2122cf8 100644 --- a/ext/compressors/snappy/Makefile.am +++ b/ext/compressors/snappy/Makefile.am -@@ -2,5 +2,5 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include +@@ -2,5 +2,6 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include lib_LTLIBRARIES = libwiredtiger_snappy.la libwiredtiger_snappy_la_SOURCES = snappy_compress.c -libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -+libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -Wl,-rpath,lib/wterl/priv:priv:/usr/local/lib ++libwiredtiger_snappy_la_CFLAGS = -I$(src_builddir)/../../system/include ++libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(src_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv libwiredtiger_snappy_la_LIBADD = -lsnappy diff --git a/c_src/wterl.c b/c_src/wterl.c index 2be9161..3eab666 100644 --- a/c_src/wterl.c +++ b/c_src/wterl.c @@ -1,677 +1,2046 @@ -// ------------------------------------------------------------------- -// -// wterl: Erlang Wrapper for WiredTiger -// -// Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. -// -// This file is provided to you under the Apache License, -// Version 2.0 (the "License"); you may not use this file -// except in compliance with the License. You may obtain -// a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// -// ------------------------------------------------------------------- +/* + * wterl: an Erlang NIF for WiredTiger + * + * Copyright (c) 2012-2013 Basho Technologies, Inc. All Rights Reserved. + * + * This file is provided to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + */ #include "erl_nif.h" #include "erl_driver.h" #include #include +#include + +#ifdef DEBUG +#include +#include +#define dprint(s, ...) do { \ + fprintf(stderr, s, ##__VA_ARGS__); \ + fprintf(stderr, "\r\n"); \ + fflush(stderr); \ + } while(0); +#else +# define dprint(s, ...) {} +#endif #include "wiredtiger.h" +#include "async_nif.h" +#include "khash.h" -static ErlNifResourceType* wterl_conn_RESOURCE; -static ErlNifResourceType* wterl_session_RESOURCE; -static ErlNifResourceType* wterl_cursor_RESOURCE; +#ifdef WTERL_STATS +#include "stats.h" +#endif + +static ErlNifResourceType *wterl_conn_RESOURCE; +static ErlNifResourceType *wterl_cursor_RESOURCE; + +/* Generators for 'cursors' a named, type-specific hash table functions. */ +KHASH_MAP_INIT_STR(cursors, WT_CURSOR*); + +/** + * We will have exactly one (1) WterlCtx for each async worker thread. As + * requests arrive we will reuse the same WterlConnHandle->contexts[worker_id] + * WterlCtx in the work block ensuring that each async worker thread a) has + * a separate WT_SESSION (because they are not thread safe) and b) when + * possible we avoid opening new cursors by first looking for one in the + * cursors hash table. In practice this means we could have (num_workers + * * num_tables) of cursors open which we need to account for when setting + * session_max in the configuration of WiredTiger so that it creates enough + * hazard pointers for this extreme case. + * + * Note: We don't protect access to this struct with a mutex because it will + * only be accessed by the same worker thread. + */ +typedef struct { + WT_SESSION *session; + khash_t(cursors) *cursors; +} WterlCtx; typedef struct { - WT_CONNECTION* conn; + WT_CONNECTION *conn; + const char *session_config; + ErlNifMutex *contexts_mutex; + WterlCtx contexts[ASYNC_NIF_MAX_WORKERS]; } WterlConnHandle; typedef struct { - WT_SESSION* session; -} WterlSessionHandle; - -typedef struct { - WT_CURSOR* cursor; + WT_CURSOR *cursor; + WT_SESSION *session; } WterlCursorHandle; -typedef char Uri[128]; // object names +/* WiredTiger object names*/ +typedef char Uri[128]; -// Atoms (initialized in on_load) +/* Atoms (initialized in on_load) */ static ERL_NIF_TERM ATOM_ERROR; static ERL_NIF_TERM ATOM_OK; +static ERL_NIF_TERM ATOM_NOT_FOUND; +static ERL_NIF_TERM ATOM_FIRST; +static ERL_NIF_TERM ATOM_LAST; -typedef ERL_NIF_TERM (*CursorRetFun)(ErlNifEnv* env, WT_CURSOR* cursor, int rc); +/* Generators for 'conns' a named, type-specific hash table functions. */ +KHASH_MAP_INIT_PTR(conns, WterlConnHandle*); -// Prototypes -static ERL_NIF_TERM wterl_conn_close(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_conn_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_close(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_insert(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_key_ret(ErlNifEnv* env, WT_CURSOR *cursor, int rc); -static ERL_NIF_TERM wterl_cursor_kv_ret(ErlNifEnv* env, WT_CURSOR *cursor, int rc); -static ERL_NIF_TERM wterl_cursor_next(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_next_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_next_value(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_np_worker(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[], - CursorRetFun cursor_ret_fun, int next); -static ERL_NIF_TERM wterl_cursor_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_prev(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_prev_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_prev_value(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_remove(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_reset(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_search(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_search_near(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_search_worker(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[], int near); -static ERL_NIF_TERM wterl_cursor_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_cursor_value_ret(ErlNifEnv* env, WT_CURSOR *cursor, int rc); -static ERL_NIF_TERM wterl_session_checkpoint(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_close(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_create(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_delete(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_drop(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_get(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_put(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_rename(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_salvage(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_truncate(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_upgrade(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); -static ERL_NIF_TERM wterl_session_verify(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - -static ErlNifFunc nif_funcs[] = -{ - {"connection_close", 1, wterl_conn_close}, - {"conn_open", 2, wterl_conn_open}, - {"cursor_close", 1, wterl_cursor_close}, - {"cursor_insert", 3, wterl_cursor_insert}, - {"cursor_next", 1, wterl_cursor_next}, - {"cursor_next_key", 1, wterl_cursor_next_key}, - {"cursor_next_value", 1, wterl_cursor_next_value}, - {"cursor_open", 2, wterl_cursor_open}, - {"cursor_prev", 1, wterl_cursor_prev}, - {"cursor_prev_key", 1, wterl_cursor_prev_key}, - {"cursor_prev_value", 1, wterl_cursor_prev_value}, - {"cursor_remove", 2, wterl_cursor_remove}, - {"cursor_reset", 1, wterl_cursor_reset}, - {"cursor_search", 2, wterl_cursor_search}, - {"cursor_search_near", 2, wterl_cursor_search_near}, - {"cursor_update", 3, wterl_cursor_update}, - {"session_checkpoint", 2, wterl_session_checkpoint}, - {"session_close", 1, wterl_session_close}, - {"session_create", 3, wterl_session_create}, - {"session_delete", 3, wterl_session_delete}, - {"session_drop", 3, wterl_session_drop}, - {"session_get", 3, wterl_session_get}, - {"session_open", 2, wterl_session_open}, - {"session_put", 4, wterl_session_put}, - {"session_rename", 4, wterl_session_rename}, - {"session_salvage", 3, wterl_session_salvage}, - {"session_truncate", 3, wterl_session_truncate}, - {"session_upgrade", 3, wterl_session_upgrade}, - {"session_verify", 3, wterl_session_verify}, +struct wterl_priv_data { + void *async_nif_priv; // Note: must be first element in struct + ErlNifMutex *conns_mutex; + khash_t(conns) *conns; }; -static inline ERL_NIF_TERM wterl_strerror(ErlNifEnv* env, int rc) -{ - return rc == WT_NOTFOUND ? - enif_make_atom(env, "not_found") : - enif_make_tuple2(env, ATOM_ERROR, - enif_make_string(env, wiredtiger_strerror(rc), ERL_NIF_LATIN1)); -} +/* Global init for async_nif. */ +ASYNC_NIF_INIT(wterl); -static ERL_NIF_TERM wterl_conn_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) + +/** + * Open a WT_SESSION for the thread context 'ctx' to use, also init the + * shared cursor hash table. + * + * Note: always call within enif_mutex_lock/unlock(conn_handle->contexts_mutex) + */ +static int +__init_session_and_cursor_cache(WterlConnHandle *conn_handle, WterlCtx *ctx) { - ErlNifBinary config; - char homedir[4096]; - if (enif_get_string(env, argv[0], homedir, sizeof homedir, ERL_NIF_LATIN1) && - enif_inspect_binary(env, argv[1], &config)) - { - WT_CONNECTION* conn; - int rc = wiredtiger_open(homedir, NULL, (const char*)config.data, &conn); - if (rc == 0) - { - WterlConnHandle* conn_handle = enif_alloc_resource(wterl_conn_RESOURCE, sizeof(WterlConnHandle)); - conn_handle->conn = conn; - ERL_NIF_TERM result = enif_make_resource(env, conn_handle); - enif_release_resource(conn_handle); - return enif_make_tuple2(env, ATOM_OK, result); - } - else - { - return wterl_strerror(env, rc); - } + /* Create a context for this worker thread to reuse. */ + WT_CONNECTION *conn = conn_handle->conn; + int rc = conn->open_session(conn, NULL, conn_handle->session_config, &ctx->session); + if (rc != 0) { + ctx->session = NULL; + return rc; } - return enif_make_badarg(env); -} + ctx->cursors = kh_init(cursors); -static ERL_NIF_TERM wterl_conn_close(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - WterlConnHandle* conn_handle; - if (enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&conn_handle)) - { - WT_CONNECTION* conn = conn_handle->conn; - int rc = conn->close(conn, NULL); - return rc == 0 ? ATOM_OK : wterl_strerror(env, rc); - } - return enif_make_badarg(env); -} - -#define WTERL_OP_CREATE 1 -#define WTERL_OP_DROP 2 -#define WTERL_OP_SALVAGE 3 -#define WTERL_OP_TRUNCATE 4 -#define WTERL_OP_UPGRADE 5 -#define WTERL_OP_VERIFY 6 - -static inline ERL_NIF_TERM wterl_session_worker(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[], int op) -{ - WterlSessionHandle* session_handle; - if (enif_get_resource(env, argv[0], wterl_session_RESOURCE, (void**)&session_handle)) - { - WT_SESSION* session = session_handle->session; - int rc; - Uri uri; - ErlNifBinary config; - if (enif_get_string(env, argv[1], uri, sizeof uri, ERL_NIF_LATIN1) && - enif_inspect_binary(env, argv[2], &config)) - { - switch (op) - { - case WTERL_OP_CREATE: - rc = session->create(session, uri, (const char*)config.data); - break; - case WTERL_OP_DROP: - rc = session->drop(session, uri, (const char*)config.data); - break; - case WTERL_OP_SALVAGE: - rc = session->salvage(session, uri, (const char*)config.data); - break; - case WTERL_OP_TRUNCATE: - // Ignore the cursor start/stop form of truncation for now, - // support only the full file truncation. - rc = session->truncate(session, uri, NULL, NULL, (const char*)config.data); - break; - case WTERL_OP_UPGRADE: - rc = session->upgrade(session, uri, (const char*)config.data); - break; - case WTERL_OP_VERIFY: - default: - rc = session->verify(session, uri, (const char*)config.data); - break; - } - return rc == 0 ? ATOM_OK : wterl_strerror(env, rc); - } - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_session_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - WterlConnHandle* conn_handle; - ErlNifBinary config; - if (enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&conn_handle) && - enif_inspect_binary(env, argv[1], &config)) - { - WT_CONNECTION* conn = conn_handle->conn; - WT_SESSION* session; - int rc = conn->open_session(conn, NULL, (const char *)config.data, &session); - if (rc == 0) - { - WterlSessionHandle* session_handle = - enif_alloc_resource(wterl_session_RESOURCE, sizeof(WterlSessionHandle)); - session_handle->session = session; - ERL_NIF_TERM result = enif_make_resource(env, session_handle); - enif_keep_resource(conn_handle); - enif_release_resource(session_handle); - return enif_make_tuple2(env, ATOM_OK, result); - } - else - { - return wterl_strerror(env, rc); - } - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_session_close(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - WterlSessionHandle* session_handle; - if (enif_get_resource(env, argv[0], wterl_session_RESOURCE, (void**)&session_handle)) - { - WT_SESSION* session = session_handle->session; - int rc = session->close(session, NULL); - return rc == 0 ? ATOM_OK : wterl_strerror(env, rc); - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_session_create(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_session_worker(env, argc, argv, WTERL_OP_CREATE); -} - -static ERL_NIF_TERM wterl_session_drop(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_session_worker(env, argc, argv, WTERL_OP_DROP); -} - -static ERL_NIF_TERM wterl_session_rename(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - WterlSessionHandle* session_handle; - if (enif_get_resource(env, argv[0], wterl_session_RESOURCE, (void**)&session_handle)) - { - ErlNifBinary config; - Uri oldname, newname; - if (enif_get_string(env, argv[1], oldname, sizeof oldname, ERL_NIF_LATIN1) && - enif_get_string(env, argv[2], newname, sizeof newname, ERL_NIF_LATIN1) && - enif_inspect_binary(env, argv[3], &config)) - { - WT_SESSION* session = session_handle->session; - int rc = session->rename(session, oldname, newname, (const char*)config.data); - return rc == 0 ? ATOM_OK : wterl_strerror(env, rc); - } - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_session_salvage(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_session_worker(env, argc, argv, WTERL_OP_SALVAGE); -} - -static ERL_NIF_TERM wterl_session_checkpoint(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - WterlSessionHandle* session_handle; - ErlNifBinary config; - if (enif_get_resource(env, argv[0], wterl_session_RESOURCE, (void**)&session_handle) && - enif_inspect_binary(env, argv[1], &config)) - { - WT_SESSION* session = session_handle->session; - int rc = session->checkpoint(session, (const char*)config.data); - return rc == 0 ? ATOM_OK : wterl_strerror(env, rc); - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_session_truncate(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_session_worker(env, argc, argv, WTERL_OP_TRUNCATE); -} - -static ERL_NIF_TERM wterl_session_upgrade(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_session_worker(env, argc, argv, WTERL_OP_UPGRADE); -} - -static ERL_NIF_TERM wterl_session_verify(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_session_worker(env, argc, argv, WTERL_OP_VERIFY); -} - -static ERL_NIF_TERM wterl_session_delete(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - WterlSessionHandle* session_handle; - if (enif_get_resource(env, argv[0], wterl_session_RESOURCE, (void**)&session_handle)) - { - Uri uri; - ErlNifBinary key; - if (enif_get_string(env, argv[1], uri, sizeof uri, ERL_NIF_LATIN1) && - enif_inspect_binary(env, argv[2], &key)) - { - WT_SESSION* session = session_handle->session; - WT_CURSOR* cursor; - int rc = session->open_cursor(session, uri, NULL, "raw", &cursor); - if (rc != 0) - { - return wterl_strerror(env, rc); - } - WT_ITEM raw_key; - raw_key.data = key.data; - raw_key.size = key.size; - cursor->set_key(cursor, &raw_key); - rc = cursor->remove(cursor); - cursor->close(cursor); - return rc == 0 ? ATOM_OK : wterl_strerror(env, rc); - } - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_session_get(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - WterlSessionHandle* session_handle; - if (enif_get_resource(env, argv[0], wterl_session_RESOURCE, (void**)&session_handle)) - { - Uri uri; - ErlNifBinary key; - if (enif_get_string(env, argv[1], uri, sizeof uri, ERL_NIF_LATIN1) && - enif_inspect_binary(env, argv[2], &key)) - { - WT_SESSION* session = session_handle->session; - WT_CURSOR* cursor; - int rc = session->open_cursor(session, uri, NULL, "overwrite,raw", &cursor); - if (rc != 0) - { - return wterl_strerror(env, rc); - } - WT_ITEM raw_key, raw_value; - raw_key.data = key.data; - raw_key.size = key.size; - cursor->set_key(cursor, &raw_key); - rc = cursor->search(cursor); - if (rc == 0) - { - rc = cursor->get_value(cursor, &raw_value); - if (rc == 0) - { - ERL_NIF_TERM value; - unsigned char* bin = enif_make_new_binary(env, raw_value.size, &value); - memcpy(bin, raw_value.data, raw_value.size); - cursor->close(cursor); - return enif_make_tuple2(env, ATOM_OK, value); - } - } - cursor->close(cursor); - return wterl_strerror(env, rc); - } - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_session_put(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - WterlSessionHandle* session_handle; - if (enif_get_resource(env, argv[0], wterl_session_RESOURCE, (void**)&session_handle)) - { - Uri uri; - ErlNifBinary key, value; - if (enif_get_string(env, argv[1], uri, sizeof uri, ERL_NIF_LATIN1) && - enif_inspect_binary(env, argv[2], &key) && - enif_inspect_binary(env, argv[3], &value)) - { - WT_SESSION* session = session_handle->session; - WT_CURSOR* cursor; - int rc = session->open_cursor(session, uri, NULL, "overwrite,raw", &cursor); - if (rc != 0) - { - return wterl_strerror(env, rc); - } - WT_ITEM raw_key, raw_value; - raw_key.data = key.data; - raw_key.size = key.size; - cursor->set_key(cursor, &raw_key); - raw_value.data = value.data; - raw_value.size = value.size; - cursor->set_value(cursor, &raw_value); - rc = cursor->insert(cursor); - cursor->close(cursor); - return rc == 0 ? ATOM_OK : wterl_strerror(env, rc); - } - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_cursor_open(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - WterlSessionHandle* session_handle; - if (enif_get_resource(env, argv[0], wterl_session_RESOURCE, (void**)&session_handle)) - { - WT_CURSOR* cursor; - Uri uri; - if (enif_get_string(env, argv[1], uri, sizeof uri, ERL_NIF_LATIN1)) - { - WT_SESSION* session = session_handle->session; - int rc = session->open_cursor(session, uri, NULL, "overwrite,raw", &cursor); - if (rc == 0) - { - WterlCursorHandle* cursor_handle = - enif_alloc_resource(wterl_cursor_RESOURCE, sizeof(WterlCursorHandle)); - cursor_handle->cursor = cursor; - ERL_NIF_TERM result = enif_make_resource(env, cursor_handle); - enif_keep_resource(session_handle); - enif_release_resource(cursor_handle); - return enif_make_tuple2(env, ATOM_OK, result); - } - else - { - return wterl_strerror(env, rc); - } - } - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_cursor_close(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - WterlCursorHandle *cursor_handle; - if (enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&cursor_handle)) - { - WT_CURSOR* cursor = cursor_handle->cursor; - int rc = cursor->close(cursor); - return rc == 0 ? ATOM_OK : wterl_strerror(env, rc); - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_cursor_key_ret(ErlNifEnv* env, WT_CURSOR *cursor, int rc) -{ - if (rc == 0) - { - WT_ITEM raw_key; - rc = cursor->get_key(cursor, &raw_key); - if (rc == 0) - { - ERL_NIF_TERM key; - memcpy(enif_make_new_binary(env, raw_key.size, &key), raw_key.data, raw_key.size); - return enif_make_tuple2(env, ATOM_OK, key); - } - } - return wterl_strerror(env, rc); -} - -static ERL_NIF_TERM wterl_cursor_kv_ret(ErlNifEnv* env, WT_CURSOR *cursor, int rc) -{ - if (rc == 0) - { - WT_ITEM raw_key, raw_value; - rc = cursor->get_key(cursor, &raw_key); - if (rc == 0) - { - rc = cursor->get_value(cursor, &raw_value); - if (rc == 0) - { - ERL_NIF_TERM key, value; - memcpy(enif_make_new_binary(env, raw_key.size, &key), raw_key.data, raw_key.size); - memcpy(enif_make_new_binary(env, raw_value.size, &value), raw_value.data, raw_value.size); - return enif_make_tuple3(env, ATOM_OK, key, value); - } - } - } - return wterl_strerror(env, rc); -} - -static ERL_NIF_TERM wterl_cursor_value_ret(ErlNifEnv* env, WT_CURSOR *cursor, int rc) -{ - if (rc == 0) - { - WT_ITEM raw_value; - rc = cursor->get_value(cursor, &raw_value); - if (rc == 0) - { - ERL_NIF_TERM value; - memcpy(enif_make_new_binary(env, raw_value.size, &value), raw_value.data, raw_value.size); - return enif_make_tuple2(env, ATOM_OK, value); - } - } - return wterl_strerror(env, rc); -} - -static ERL_NIF_TERM wterl_cursor_np_worker(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[], - CursorRetFun cursor_ret, int prev) -{ - WterlCursorHandle *cursor_handle; - if (enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&cursor_handle)) - { - WT_CURSOR* cursor = cursor_handle->cursor; - return cursor_ret(env, cursor, prev == 0 ? cursor->next(cursor) : cursor->prev(cursor)); - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_cursor_next(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_cursor_np_worker(env, argc, argv, wterl_cursor_kv_ret, 0); -} - -static ERL_NIF_TERM wterl_cursor_next_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_cursor_np_worker(env, argc, argv, wterl_cursor_key_ret, 0); -} - -static ERL_NIF_TERM wterl_cursor_next_value(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_cursor_np_worker(env, argc, argv, wterl_cursor_value_ret, 0); -} - -static ERL_NIF_TERM wterl_cursor_prev(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_cursor_np_worker(env, argc, argv, wterl_cursor_kv_ret, 1); -} - -static ERL_NIF_TERM wterl_cursor_prev_key(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_cursor_np_worker(env, argc, argv, wterl_cursor_key_ret, 1); -} - -static ERL_NIF_TERM wterl_cursor_prev_value(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_cursor_np_worker(env, argc, argv, wterl_cursor_value_ret, 1); -} - -static ERL_NIF_TERM wterl_cursor_search_worker(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[], int near) -{ - WterlCursorHandle *cursor_handle; - ErlNifBinary key; - if (enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&cursor_handle) && - enif_inspect_binary(env, argv[1], &key)) - { - WT_CURSOR* cursor = cursor_handle->cursor; - WT_ITEM raw_key; - int exact; - raw_key.data = key.data; - raw_key.size = key.size; - cursor->set_key(cursor, &raw_key); - - // We currently ignore the less-than, greater-than or equals-to return information - // from the cursor.search_near method. - return wterl_cursor_value_ret(env, cursor, - near == 1 ? - cursor->search_near(cursor, &exact) : cursor->search(cursor)); - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_cursor_search(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_cursor_search_worker(env, argc, argv, 0); -} - -static ERL_NIF_TERM wterl_cursor_search_near(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_cursor_search_worker(env, argc, argv, 1); -} - -static ERL_NIF_TERM wterl_cursor_reset(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - WterlCursorHandle *cursor_handle; - if (enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&cursor_handle)) - { - WT_CURSOR* cursor = cursor_handle->cursor; - int rc = cursor->reset(cursor); - return rc == 0 ? ATOM_OK : wterl_strerror(env, rc); - } - return enif_make_badarg(env); -} - -#define WTERL_OP_CURSOR_INSERT 1 -#define WTERL_OP_CURSOR_UPDATE 2 -#define WTERL_OP_CURSOR_REMOVE 3 - -static inline ERL_NIF_TERM wterl_cursor_data_op(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[], int op) -{ - WterlCursorHandle *cursor_handle; - if (enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&cursor_handle)) - { - ErlNifBinary key, value; - int rc; - - if (enif_inspect_binary(env, argv[1], &key) && - (op == WTERL_OP_CURSOR_REMOVE ? 1 : enif_inspect_binary(env, argv[2], &value))) - { - WT_CURSOR* cursor = cursor_handle->cursor; - WT_ITEM raw_key, raw_value; - raw_key.data = key.data; - raw_key.size = key.size; - cursor->set_key(cursor, &raw_key); - if (op != WTERL_OP_CURSOR_REMOVE) - { - raw_value.data = value.data; - raw_value.size = value.size; - cursor->set_value(cursor, &raw_value); - } - switch (op) - { - case WTERL_OP_CURSOR_INSERT: - rc = cursor->insert(cursor); - break; - case WTERL_OP_CURSOR_UPDATE: - rc = cursor->update(cursor); - break; - case WTERL_OP_CURSOR_REMOVE: - default: - rc = cursor->remove(cursor); - break; - } - return rc == 0 ? ATOM_OK : wterl_strerror(env, rc); - } - } - return enif_make_badarg(env); -} - -static ERL_NIF_TERM wterl_cursor_insert(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_cursor_data_op(env, argc, argv, WTERL_OP_CURSOR_INSERT); -} - -static ERL_NIF_TERM wterl_cursor_update(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_cursor_data_op(env, argc, argv, WTERL_OP_CURSOR_UPDATE); -} - -static ERL_NIF_TERM wterl_cursor_remove(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - return wterl_cursor_data_op(env, argc, argv, WTERL_OP_CURSOR_REMOVE); -} - -static int on_load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) -{ - ErlNifResourceFlags flags = ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER; - wterl_conn_RESOURCE = enif_open_resource_type(env, NULL, "wterl_conn_resource", NULL, flags, NULL); - wterl_session_RESOURCE = enif_open_resource_type(env, NULL, "wterl_session_resource", NULL, flags, NULL); - wterl_cursor_RESOURCE = enif_open_resource_type(env, NULL, "wterl_cursor_resource", NULL, flags, NULL); - ATOM_ERROR = enif_make_atom(env, "error"); - ATOM_OK = enif_make_atom(env, "ok"); return 0; } -ERL_NIF_INIT(wterl, nif_funcs, &on_load, NULL, NULL, NULL); +/** + * Get the per-worker reusable WT_SESSION for a worker_id. + */ +static int +__session_for(WterlConnHandle *conn_handle, unsigned int worker_id, WT_SESSION **session) +{ + WterlCtx *ctx = &conn_handle->contexts[worker_id]; + int rc = 0; + + if (ctx->session == NULL) { + enif_mutex_lock(conn_handle->contexts_mutex); + rc = __init_session_and_cursor_cache(conn_handle, ctx); + enif_mutex_unlock(conn_handle->contexts_mutex); + } + *session = ctx->session; + return rc; +} + +/** + * Close all sessions and all cursors open on any objects. + * + * Note: always call within enif_mutex_lock/unlock(conn_handle->contexts_mutex) + */ +void +__close_all_sessions(WterlConnHandle *conn_handle) +{ + int i; + + for (i = 0; i < ASYNC_NIF_MAX_WORKERS; i++) { + WterlCtx *ctx = &conn_handle->contexts[i]; + if (ctx->session != NULL) { + WT_SESSION *session = ctx->session; + khash_t(cursors) *h = ctx->cursors; + khiter_t itr; + for (itr = kh_begin(h); itr != kh_end(h); ++itr) { + if (kh_exist(h, itr)) { + WT_CURSOR *cursor = kh_val(h, itr); + char *key = (char *)kh_key(h, itr); + cursor->close(cursor); + kh_del(cursors, h, itr); + enif_free(key); + kh_value(h, itr) = NULL; + } + } + kh_destroy(cursors, h); + session->close(session, NULL); + ctx->session = NULL; + } + } +} + +/** + * Close cursors open on 'uri' object. + * + * Note: always call within enif_mutex_lock/unlock(conn_handle->contexts_mutex) + */ +void +__close_cursors_on(WterlConnHandle *conn_handle, const char *uri) +{ + int i; + + for (i = 0; i < ASYNC_NIF_MAX_WORKERS; i++) { + WterlCtx *ctx = &conn_handle->contexts[i]; + if (ctx->session != NULL) { + khash_t(cursors) *h = ctx->cursors; + khiter_t itr = kh_get(cursors, h, (char *)uri); + if (itr != kh_end(h)) { + WT_CURSOR *cursor = kh_value(h, itr); + char *key = (char *)kh_key(h, itr); + cursor->close(cursor); + kh_del(cursors, h, itr); + enif_free(key); + kh_value(h, itr) = NULL; + } + } + } +} + +/** + * Get a reusable cursor that was opened for a particular worker within its + * session. + */ +static int +__retain_cursor(WterlConnHandle *conn_handle, unsigned int worker_id, const char *uri, WT_CURSOR **cursor) +{ + /* Check to see if we have a cursor open for this uri and if so reuse it. */ + WterlCtx *ctx = &conn_handle->contexts[worker_id]; + khash_t(cursors) *h = NULL; + khiter_t itr; + int rc; + + if (ctx->session == NULL) { + enif_mutex_lock(conn_handle->contexts_mutex); + rc = __init_session_and_cursor_cache(conn_handle, ctx); + enif_mutex_unlock(conn_handle->contexts_mutex); + if (rc != 0) + return rc; + } + + h = ctx->cursors; + itr = kh_get(cursors, h, (char *)uri); + if (itr != kh_end(h)) { + // key exists in hash table, retrieve it + *cursor = (WT_CURSOR*)kh_value(h, itr); + } else { + // key does not exist in hash table, create and insert one + enif_mutex_lock(conn_handle->contexts_mutex); + WT_SESSION *session = conn_handle->contexts[worker_id].session; + rc = session->open_cursor(session, uri, NULL, "overwrite,raw", cursor); + if (rc != 0) { + enif_mutex_unlock(conn_handle->contexts_mutex); + return rc; + } + + char *key = enif_alloc(sizeof(Uri)); + if (!key) { + session->close(session, NULL); + enif_mutex_unlock(conn_handle->contexts_mutex); + return ENOMEM; + } + memcpy(key, uri, 128); + int itr_status; + itr = kh_put(cursors, h, key, &itr_status); + kh_value(h, itr) = *cursor; + enif_mutex_unlock(conn_handle->contexts_mutex); + } + return 0; +} + +static void +__release_cursor(WterlConnHandle *conn_handle, unsigned int worker_id, const char *uri, WT_CURSOR *cursor) +{ + cursor->reset(cursor); +} + +/** + * Convenience function to generate {error, {errno, Reason}} or 'not_found' + * Erlang terms to return to callers. + * + * env NIF environment + * rc code returned by WiredTiger + */ +static ERL_NIF_TERM +__strerror_term(ErlNifEnv* env, int rc) +{ + if (rc == WT_NOTFOUND) { + return ATOM_NOT_FOUND; + } else { + /* We return the errno value as well as the message here because the + error message provided by strerror() for differ across platforms + and/or may be localized to any given language (i18n). Use the errno + atom rather than the message when matching in Erlang. You've been + warned. */ + return enif_make_tuple2(env, ATOM_ERROR, + enif_make_tuple2(env, + enif_make_atom(env, erl_errno_id(rc)), + enif_make_string(env, wiredtiger_strerror(rc), ERL_NIF_LATIN1))); + } +} + +/** + * Opens a WiredTiger WT_CONNECTION object. + * + * argv[0] path to directory for the database files + * argv[1] WiredTiger connection config string as an Erlang binary + * argv[2] WiredTiger session config string as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_conn_open, + { // struct + + ERL_NIF_TERM config; + ERL_NIF_TERM session_config; + char homedir[4096]; + struct wterl_priv_data *priv; + }, + { // pre + + if (!(argc == 3 && + (enif_get_string(env, argv[0], args->homedir, sizeof args->homedir, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[1]) && + enif_is_binary(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->config = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[1]); + args->session_config = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + + args->priv = (struct wterl_priv_data *)enif_priv_data(env); + }, + { // work + + WT_CONNECTION *conn; + ErlNifBinary config; + ErlNifBinary session_config; + + if (!enif_inspect_binary(env, args->config, &config)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + if (!enif_inspect_binary(env, args->session_config, &session_config)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + int rc = wiredtiger_open(args->homedir, NULL, config.data[0] != 0 ? (const char*)config.data : NULL, &conn); + if (rc == 0) { + WterlConnHandle *conn_handle = enif_alloc_resource(wterl_conn_RESOURCE, sizeof(WterlConnHandle)); + if (!conn_handle) { + ASYNC_NIF_REPLY(__strerror_term(env, ENOMEM)); + return; + } + if (session_config.size > 1) { + char *sc = enif_alloc(session_config.size); + if (!sc) { + enif_release_resource(conn_handle); + ASYNC_NIF_REPLY(__strerror_term(env, ENOMEM)); + return; + } + memcpy(sc, session_config.data, session_config.size); + + conn_handle->session_config = (const char *)sc; + } else { + conn_handle->session_config = NULL; + } + conn_handle->contexts_mutex = enif_mutex_create(NULL); + enif_mutex_lock(conn_handle->contexts_mutex); + conn_handle->conn = conn; + memset(conn_handle->contexts, 0, sizeof(WterlCtx) * ASYNC_NIF_MAX_WORKERS); + ERL_NIF_TERM result = enif_make_resource(env, conn_handle); + + /* Keep track of open connections so as to free when unload/reload/etc. + are called. */ + khash_t(conns) *h; + enif_mutex_lock(args->priv->conns_mutex); + h = args->priv->conns; + int itr_status = 0; + khiter_t itr = kh_put(conns, h, conn, &itr_status); + if (itr_status != 0) // 0 indicates the key exists already + kh_value(h, itr) = conn_handle; + enif_mutex_unlock(args->priv->conns_mutex); + + enif_release_resource(conn_handle); + enif_mutex_unlock(conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(enif_make_tuple2(env, ATOM_OK, result)); + } + else + { + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + } + }, + { // post + + }); + +/** + * Closes a WiredTiger WT_CONNECTION object. + * + * argv[0] WterlConnHandle resource + */ +ASYNC_NIF_DECL( + wterl_conn_close, + { // struct + + WterlConnHandle* conn_handle; + struct wterl_priv_data *priv; + }, + { // pre + + if (!(argc == 1 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle))) { + ASYNC_NIF_RETURN_BADARG(); + } + enif_keep_resource((void*)args->conn_handle); + + args->priv = (struct wterl_priv_data *)enif_priv_data(env); + }, + { // work + + /* Free up the shared sessions and cursors. */ + enif_mutex_lock(args->conn_handle->contexts_mutex); + __close_all_sessions(args->conn_handle); + if (args->conn_handle->session_config) { + enif_free((char *)args->conn_handle->session_config); + args->conn_handle->session_config = NULL; + } + WT_CONNECTION* conn = args->conn_handle->conn; + int rc = conn->close(conn, NULL); + + /* Connection is closed, remove it so we don't free on unload/reload/etc. */ + khash_t(conns) *h; + enif_mutex_lock(args->priv->conns_mutex); + h = args->priv->conns; + khiter_t itr; + itr = kh_get(conns, h, conn); + if (itr != kh_end(h)) { + /* key exists in table (as expected) delete it */ + kh_del(conns, h, itr); + kh_value(h, itr) = NULL; + } + enif_mutex_unlock(args->priv->conns_mutex); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + enif_mutex_destroy(args->conn_handle->contexts_mutex); + memset(args->conn_handle, 0, sizeof(WterlConnHandle)); + + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Create a WiredTiger table, column group, index or file. + * + * We create, use and discard a WT_SESSION here because table creation is not + * too performance sensitive. + * + * argv[0] WterlConnHandle resource + * argv[1] object name URI string + * argv[2] config string as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_create, + { // struct + + WterlConnHandle *conn_handle; + Uri uri; + ERL_NIF_TERM config; + }, + { // pre + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + (enif_get_string(env, argv[1], args->uri, sizeof args->uri, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->config = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + enif_keep_resource((void*)args->conn_handle); + }, + { // work + + ErlNifBinary config; + if (!enif_inspect_binary(env, args->config, &config)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + /* We create, use and discard a WT_SESSION here because a) we don't need a + cursor and b) we don't anticipate doing this operation frequently enough + to impact performance. */ + WT_CONNECTION *conn = args->conn_handle->conn; + WT_SESSION *session = NULL; + int rc = conn->open_session(conn, NULL, args->conn_handle->session_config, &session); + if (rc != 0) { + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + rc = session->create(session, args->uri, (const char*)config.data); + (void)session->close(session, NULL); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Drop (remove) a WiredTiger table, column group, index or file. + * + * argv[0] WterlConnHandle resource + * argv[1] object name URI string + * argv[2] config string as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_drop, + { // struct + + Uri uri; + ERL_NIF_TERM config; + WterlConnHandle *conn_handle; + }, + { // pre + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + (enif_get_string(env, argv[1], args->uri, sizeof args->uri, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->config = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + enif_keep_resource((void*)args->conn_handle); + }, + { // work + + /* This call requires that there be no open cursors referencing the object. */ + enif_mutex_lock(args->conn_handle->contexts_mutex); + __close_cursors_on(args->conn_handle, args->uri); + + ErlNifBinary config; + if (!enif_inspect_binary(env, args->config, &config)) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + /* We create, use and discard a WT_SESSION here because a) we don't need a + cursor and b) we don't anticipate doing this operation frequently enough + to impact performance. */ + WT_CONNECTION *conn = args->conn_handle->conn; + WT_SESSION *session = NULL; + int rc = conn->open_session(conn, NULL, args->conn_handle->session_config, &session); + if (rc != 0) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + /* Note: we locked the context mutex and called __close_cursors_on() + earlier so that we are sure that before we call into WiredTiger we have + first closed all open cursors referencing this object. Failure to do + this will result in EBUSY(16) "Device or resource busy". */ + rc = session->drop(session, args->uri, (const char*)config.data); + (void)session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Rename a WiredTiger table, column group, index or file. + * + * argv[0] WterlConnHandle resource + * argv[1] old object name URI string + * argv[2] new object name URI string + * argv[3] config string as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_rename, + { // struct + + Uri oldname; + Uri newname; + ERL_NIF_TERM config; + WterlConnHandle *conn_handle; + }, + { // pre + + if (!(argc == 4 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + (enif_get_string(env, argv[1], args->oldname, sizeof args->oldname, ERL_NIF_LATIN1) > 0) && + (enif_get_string(env, argv[2], args->newname, sizeof args->newname, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[3]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->config = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[3]); + enif_keep_resource((void*)args->conn_handle); + }, + { // work + + /* This call requires that there be no open cursors referencing the object. */ + enif_mutex_lock(args->conn_handle->contexts_mutex); + __close_cursors_on(args->conn_handle, args->oldname); + + ErlNifBinary config; + if (!enif_inspect_binary(env, args->config, &config)) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + /* We create, use and discard a WT_SESSION here because a) we don't need a + cursor and b) we don't anticipate doing this operation frequently enough + to impact performance. */ + WT_CONNECTION *conn = args->conn_handle->conn; + WT_SESSION *session = NULL; + int rc = conn->open_session(conn, NULL, args->conn_handle->session_config, &session); + if (rc != 0) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + /* Note: we locked the context mutex and called __close_cursors_on() + earlier so that we are sure that before we call into WiredTiger we have + first closed all open cursors referencing this object. Failure to do + this will result in EBUSY(16) "Device or resource busy". */ + rc = session->rename(session, args->oldname, args->newname, (const char*)config.data); + (void)session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + + +/** + * Salvage rebuilds the file, or files of which a table is comprised, + * discarding any corrupted file blocks. Use this as a fallback if + * an attempt to open a WiredTiger database object fails. + * + * argv[0] WterlConnHandle resource + * argv[1] object name URI string + * argv[2] config string as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_salvage, + { // struct + + WterlConnHandle *conn_handle; + Uri uri; + ERL_NIF_TERM config; + }, + { // pre + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + (enif_get_string(env, argv[1], args->uri, sizeof args->uri, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->config = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + enif_keep_resource((void*)args->conn_handle); + }, + { // work + + /* This call requires that there be no open cursors referencing the object. */ + enif_mutex_lock(args->conn_handle->contexts_mutex); + __close_cursors_on(args->conn_handle, args->uri); + + ErlNifBinary config; + if (!enif_inspect_binary(env, args->config, &config)) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + /* We create, use and discard a WT_SESSION here because a) we don't need a + cursor and b) we don't anticipate doing this operation frequently enough + to impact performance. */ + WT_CONNECTION *conn = args->conn_handle->conn; + WT_SESSION *session = NULL; + int rc = conn->open_session(conn, NULL, args->conn_handle->session_config, &session); + if (rc != 0) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + rc = session->salvage(session, args->uri, (const char*)config.data); + (void)session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Checkpoint writes a transactionally consistent snapshot of a database or set + * of objects specified. + * + * argv[0] WterlConnHandle resource + * argv[1] object name URI string + * argv[2] config string as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_checkpoint, + { // struct + + WterlConnHandle *conn_handle; + ERL_NIF_TERM config; + }, + { // pre + + if (!(argc == 2 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + enif_is_binary(env, argv[1]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->config = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[1]); + enif_keep_resource((void*)args->conn_handle); + }, + { // work + + ErlNifBinary config; + if (!enif_inspect_binary(env, args->config, &config)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + WT_SESSION *session = NULL; + int rc = __session_for(args->conn_handle, worker_id, &session); + if (rc != 0) { + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + rc = session->checkpoint(session, (const char*)config.data); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Truncate a file, table or cursor range. + * + * argv[0] WterlConnHandle resource + * argv[1] object name URI string + * argv[2] start key as an Erlang binary + * argv[3] stop key as an Erlang binary + * argv[4] config string as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_truncate, + { // struct + + WterlConnHandle *conn_handle; + Uri uri; + int from_first; + int to_last; + ERL_NIF_TERM start; + ERL_NIF_TERM stop; + ERL_NIF_TERM config; + }, + { // pre + + if (!(argc == 5 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + (enif_get_string(env, argv[1], args->uri, sizeof args->uri, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[4]))) { + ASYNC_NIF_RETURN_BADARG(); + } + if (enif_is_binary(env, argv[2])) { + args->from_first = 0; + args->start = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + } else if (enif_is_atom(env, argv[2])) { // TODO && argv[2] == ATOM_FIRST) { + args->from_first = 1; + args->start = 0; + } else { + ASYNC_NIF_RETURN_BADARG(); + } + if (enif_is_binary(env, argv[3])) { + args->to_last = 0; + args->stop = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[3]); + } else if (enif_is_atom(env, argv[3])) { // TODO && argv[3] == ATOM_LAST) { + args->to_last = 1; + args->stop = 0; + } else { + ASYNC_NIF_RETURN_BADARG(); + } + args->config = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[4]); + enif_keep_resource((void*)args->conn_handle); + affinity = args->uri; + }, + { // work + + /* This call requires that there be no open cursors referencing the object. */ + enif_mutex_lock(args->conn_handle->contexts_mutex); + __close_cursors_on(args->conn_handle, args->uri); + + ErlNifBinary config; + if (!enif_inspect_binary(env, args->config, &config)) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + /* Note: we locked the context mutex and called __close_cursors_on() + earlier so that we are sure that before we call into WiredTiger we have + first closed all open cursors referencing this object. Failure to do + this will result in EBUSY(16) "Device or resource busy". */ + WT_CONNECTION *conn = args->conn_handle->conn; + WT_SESSION *session = NULL; + int rc = conn->open_session(conn, NULL, args->conn_handle->session_config, &session); + if (rc != 0) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + ErlNifBinary start_key; + ErlNifBinary stop_key; + WT_CURSOR *start = NULL; + WT_CURSOR *stop = NULL; + + /* The truncate method should be passed either a URI or start/stop cursors, + but not both. So we simply open cursors no matter what to avoid the + mess. */ + if (!args->from_first) { + if (!enif_inspect_binary(env, args->start, &start_key)) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + } + rc = session->open_cursor(session, args->uri, NULL, "raw", &start); + if (rc != 0) { + session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + /* Position the start cursor at the first record or the specified record. */ + if (args->from_first) { + rc = start->next(start); + if (rc != 0) { + start->close(start); + session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + } else { + WT_ITEM item_start; + item_start.data = start_key.data; + item_start.size = start_key.size; + start->set_key(start, item_start); + rc = start->search(start); + if (rc != 0) { + start->close(start); + session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + } + + if (!args->to_last) { + if (!enif_inspect_binary(env, args->stop, &stop_key)) { + start->close(start); + session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + } + rc = session->open_cursor(session, args->uri, NULL, "raw", &stop); + if (rc != 0) { + start->close(start); + session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + /* Position the stop cursor at the last record or the specified record. */ + if (args->to_last) { + rc = stop->prev(stop); + if (rc != 0) { + start->close(start); + stop->close(stop); + session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + } else { + WT_ITEM item_stop; + item_stop.data = stop_key.data; + item_stop.size = stop_key.size; + stop->set_key(stop, item_stop); + rc = stop->search(stop); + if (rc != 0) { + start->close(start); + stop->close(stop); + session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + } + + /* Always pass NULL for URI here because we always specify the range with the + start and stop cursors which were opened referencing that URI. */ + rc = session->truncate(session, NULL, start, stop, (const char*)config.data); + + start->close(start); + stop->close(stop); + session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Upgrade upgrades a file or table, if upgrade is required. + * + * argv[0] WterlConnHandle resource + * argv[1] object name URI string + * argv[2] config string as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_upgrade, + { // struct + + WterlConnHandle *conn_handle; + Uri uri; + ERL_NIF_TERM config; + }, + { // pre + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + (enif_get_string(env, argv[1], args->uri, sizeof args->uri, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->config = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + enif_keep_resource((void*)args->conn_handle); + }, + { // work + + /* This call requires that there be no open cursors referencing the object. */ + enif_mutex_lock(args->conn_handle->contexts_mutex); + __close_cursors_on(args->conn_handle, args->uri); + + ErlNifBinary config; + if (!enif_inspect_binary(env, args->config, &config)) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + /* We create, use and discard a WT_SESSION here because a) we don't need a + cursor and b) we don't anticipate doing this operation frequently enough + to impact performance. */ + WT_CONNECTION *conn = args->conn_handle->conn; + WT_SESSION *session = NULL; + int rc = conn->open_session(conn, NULL, args->conn_handle->session_config, &session); + if (rc != 0) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + rc = session->upgrade(session, args->uri, (const char*)config.data); + (void)session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Verify reports if a file, or the files of which a table is comprised, have + * been corrupted. + * + * argv[0] WterlConnHandle resource + * argv[1] object name URI string + * argv[2] config string as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_verify, + { // struct + + WterlConnHandle *conn_handle; + Uri uri; + ERL_NIF_TERM config; + }, + { // pre + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + (enif_get_string(env, argv[1], args->uri, sizeof args->uri, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->config = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + enif_keep_resource((void*)args->conn_handle); + }, + { // work + + /* This call requires that there be no open cursors referencing the object. */ + enif_mutex_lock(args->conn_handle->contexts_mutex); + __close_all_sessions(args->conn_handle); + + ErlNifBinary config; + if (!enif_inspect_binary(env, args->config, &config)) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + /* We create, use and discard a WT_SESSION here because a) we don't need a + cursor and b) we don't anticipate doing this operation frequently enough + to impact performance. */ + WT_CONNECTION *conn = args->conn_handle->conn; + WT_SESSION *session = NULL; + int rc = conn->open_session(conn, NULL, args->conn_handle->session_config, &session); + if (rc != 0) { + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + rc = session->verify(session, args->uri, (const char*)config.data); + (void)session->close(session, NULL); + enif_mutex_unlock(args->conn_handle->contexts_mutex); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Delete a key's value from the specified table or index. + * + * argv[0] WterlConnHandle resource + * argv[1] object name URI string + * argv[2] key as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_delete, + { // struct + + WterlConnHandle *conn_handle; + Uri uri; + ERL_NIF_TERM key; + }, + { // pre + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + (enif_get_string(env, argv[1], args->uri, sizeof args->uri, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + enif_keep_resource((void*)args->conn_handle); + affinity = args->uri; + }, + { // work + + ErlNifBinary key; + if (!enif_inspect_binary(env, args->key, &key)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + WT_CURSOR *cursor = NULL; + int rc = __retain_cursor(args->conn_handle, worker_id, args->uri, &cursor); + if (rc != 0) { + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + WT_ITEM item_key; + item_key.data = key.data; + item_key.size = key.size; + cursor->set_key(cursor, &item_key); + rc = cursor->remove(cursor); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + __release_cursor(args->conn_handle, worker_id, args->uri, cursor); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Get the value for the key's value from the specified table or index. + * + * argv[0] WterlConnHandle resource + * argv[1] object name URI string + * argv[2] key as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_get, + { // struct + + WterlConnHandle *conn_handle; + Uri uri; + ERL_NIF_TERM key; + }, + { // pre + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + (enif_get_string(env, argv[1], args->uri, sizeof args->uri, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + enif_keep_resource((void*)args->conn_handle); + affinity = args->uri; + }, + { // work + + ErlNifBinary key; + if (!enif_inspect_binary(env, args->key, &key)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + WT_CURSOR *cursor = NULL; + int rc = __retain_cursor(args->conn_handle, worker_id, args->uri, &cursor); + if (rc != 0) { + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + WT_ITEM item_key; + WT_ITEM item_value; + item_key.data = key.data; + item_key.size = key.size; + cursor->set_key(cursor, &item_key); + rc = cursor->search(cursor); + if (rc != 0) { + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + rc = cursor->get_value(cursor, &item_value); + if (rc != 0) { + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + ERL_NIF_TERM value; + unsigned char *bin = enif_make_new_binary(env, item_value.size, &value); + memcpy(bin, item_value.data, item_value.size); + ASYNC_NIF_REPLY(enif_make_tuple2(env, ATOM_OK, value)); + __release_cursor(args->conn_handle, worker_id, args->uri, cursor); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Store a value for the key's value from the specified table or index. + * + * argv[0] WterlConnHandle resource + * argv[1] object name URI string + * argv[2] key as an Erlang binary + * argv[3] value as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_put, + { // struct + + WterlConnHandle *conn_handle; + Uri uri; + ERL_NIF_TERM key; + ERL_NIF_TERM value; + }, + { // pre + + if (!(argc == 4 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + (enif_get_string(env, argv[1], args->uri, sizeof args->uri, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[2]) && + enif_is_binary(env, argv[3]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + args->value = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[3]); + enif_keep_resource((void*)args->conn_handle); + affinity = args->uri; + }, + { // work + + ErlNifBinary key; + ErlNifBinary value; + if (!enif_inspect_binary(env, args->key, &key)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + if (!enif_inspect_binary(env, args->value, &value)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + WT_CURSOR *cursor = NULL; + int rc = __retain_cursor(args->conn_handle, worker_id, args->uri, &cursor); + if (rc != 0) { + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + WT_ITEM item_key; + WT_ITEM item_value; + item_key.data = key.data; + item_key.size = key.size; + cursor->set_key(cursor, &item_key); + item_value.data = value.data; + item_value.size = value.size; + cursor->set_value(cursor, &item_value); + rc = cursor->insert(cursor); + __release_cursor(args->conn_handle, worker_id, args->uri, cursor); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Open a cursor on a table or index. + * + * argv[0] WterlConnHandle resource + * argv[1] object name URI string + * argv[2] config string as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_cursor_open, + { // struct + + WterlConnHandle *conn_handle; + Uri uri; + ERL_NIF_TERM config; + }, + { // pre + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_conn_RESOURCE, (void**)&args->conn_handle) && + (enif_get_string(env, argv[1], args->uri, sizeof args->uri, ERL_NIF_LATIN1) > 0) && + enif_is_binary(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->config = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + enif_keep_resource((void*)args->conn_handle); + affinity = args->uri; + }, + { // work + + ErlNifBinary config; + if (!enif_inspect_binary(env, args->config, &config)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + /* We create a separate session here to ensure that operations are thread safe. */ + WT_CONNECTION *conn = args->conn_handle->conn; + WT_SESSION *session = NULL; + int rc = conn->open_session(conn, NULL, args->conn_handle->session_config, &session); + if (rc != 0) { + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + WT_CURSOR* cursor; + rc = session->open_cursor(session, args->uri, NULL, (config.data[0] != 0) ? (char *)config.data : "overwrite,raw", &cursor); + if (rc != 0) { + session->close(session, NULL); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + WterlCursorHandle* cursor_handle = enif_alloc_resource(wterl_cursor_RESOURCE, sizeof(WterlCursorHandle)); + if (!cursor_handle) { + cursor->close(cursor); + session->close(session, NULL); + ASYNC_NIF_REPLY(__strerror_term(env, ENOMEM)); + return; + } + cursor_handle->session = session; + cursor_handle->cursor = cursor; + ERL_NIF_TERM result = enif_make_resource(env, cursor_handle); + enif_release_resource(cursor_handle); + ASYNC_NIF_REPLY(enif_make_tuple2(env, ATOM_OK, result)); + }, + { // post + + enif_release_resource((void*)args->conn_handle); + }); + +/** + * Close a cursor releasing resources it held. + * + * argv[0] WterlCursorHandle resource + */ +ASYNC_NIF_DECL( + wterl_cursor_close, + { // struct + + WterlCursorHandle *cursor_handle; + }, + { // pre + + if (!(argc == 1 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle))) { + ASYNC_NIF_RETURN_BADARG(); + } + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + /* Note: session->close() will cause all open cursors in the session to be + closed first, so we don't have explicitly to do that here. + + WT_CURSOR *cursor = args->cursor_handle->cursor; + rc = cursor->close(cursor); + */ + WT_SESSION *session = args->cursor_handle->session; + int rc = session->close(session, NULL); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +static ERL_NIF_TERM +__cursor_key_ret(ErlNifEnv *env, WT_CURSOR *cursor, int rc) +{ + if (rc == 0) { + WT_ITEM item_key; + rc = cursor->get_key(cursor, &item_key); + if (rc == 0) { + ERL_NIF_TERM key; + memcpy(enif_make_new_binary(env, item_key.size, &key), item_key.data, item_key.size); + return enif_make_tuple2(env, ATOM_OK, key); + } + } + return __strerror_term(env, rc); +} + +static ERL_NIF_TERM +__cursor_kv_ret(ErlNifEnv *env, WT_CURSOR *cursor, int rc) +{ + if (rc == 0) { + WT_ITEM item_key, item_value; + rc = cursor->get_key(cursor, &item_key); + if (rc == 0) { + rc = cursor->get_value(cursor, &item_value); + if (rc == 0) { + ERL_NIF_TERM key, value; + memcpy(enif_make_new_binary(env, item_key.size, &key), item_key.data, item_key.size); + memcpy(enif_make_new_binary(env, item_value.size, &value), item_value.data, item_value.size); + return enif_make_tuple3(env, ATOM_OK, key, value); + } + } + } + return __strerror_term(env, rc); +} + +static ERL_NIF_TERM +__cursor_value_ret(ErlNifEnv* env, WT_CURSOR *cursor, int rc) +{ + if (rc == 0) { + WT_ITEM item_value; + rc = cursor->get_value(cursor, &item_value); + if (rc == 0) { + ERL_NIF_TERM value; + memcpy(enif_make_new_binary(env, item_value.size, &value), item_value.data, item_value.size); + return enif_make_tuple2(env, ATOM_OK, value); + } + } + return __strerror_term(env, rc); +} + +/** + * Use a cursor to fetch the next key/value pair from the table or index. + * + * argv[0] WterlCursorHandle resource + */ +ASYNC_NIF_DECL( + wterl_cursor_next, + { // struct + + WterlCursorHandle *cursor_handle; + }, + { // pre + + if (!(argc == 1 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle))) { + ASYNC_NIF_RETURN_BADARG(); + } + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + ASYNC_NIF_REPLY(__cursor_kv_ret(env, cursor, cursor->next(cursor))); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Use a cursor to fetch the next key from the table or index. + * + * argv[0] WterlCursorHandle resource + */ +ASYNC_NIF_DECL( + wterl_cursor_next_key, + { // struct + + WterlCursorHandle *cursor_handle; + }, + { // pre + + if (!(enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle))) { + ASYNC_NIF_RETURN_BADARG(); + } + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + ASYNC_NIF_REPLY(__cursor_key_ret(env, cursor, cursor->next(cursor))); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Use a cursor to fetch the next value from the table or index. + * + * argv[0] WterlCursorHandle resource + */ +ASYNC_NIF_DECL( + wterl_cursor_next_value, + { // struct + + WterlCursorHandle *cursor_handle; + }, + { // pre + + if (!(argc == 1 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle))) { + ASYNC_NIF_RETURN_BADARG(); + } + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + ASYNC_NIF_REPLY(__cursor_value_ret(env, cursor, cursor->next(cursor))); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Use a cursor to fetch the previous key/value pair from the table or index. + * + * argv[0] WterlCursorHandle resource + */ +ASYNC_NIF_DECL( + wterl_cursor_prev, + { // struct + + WterlCursorHandle *cursor_handle; + }, + { // pre + + if (!(argc == 1 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle))) { + ASYNC_NIF_RETURN_BADARG(); + } + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + ASYNC_NIF_REPLY(__cursor_kv_ret(env, cursor, cursor->prev(cursor))); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Use a cursor to fetch the previous key from the table or index. + * + * argv[0] WterlCursorHandle resource + */ +ASYNC_NIF_DECL( + wterl_cursor_prev_key, + { // struct + + WterlCursorHandle *cursor_handle; + }, + { // pre + + if (!(argc == 1 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle))) { + ASYNC_NIF_RETURN_BADARG(); + } + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + ASYNC_NIF_REPLY(__cursor_key_ret(env, cursor, cursor->prev(cursor))); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Use a cursor to fetch the previous value from the table or index. + * + * argv[0] WterlCursorHandle resource + */ +ASYNC_NIF_DECL( + wterl_cursor_prev_value, + { // struct + + WterlCursorHandle *cursor_handle; + }, + { // pre + + if (!(argc == 1 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle))) { + ASYNC_NIF_RETURN_BADARG(); + } + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + ASYNC_NIF_REPLY(__cursor_value_ret(env, cursor, cursor->prev(cursor))); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Position the cursor at the record matching the key. + * + * argv[0] WterlCursorHandle resource + * argv[1] key as an Erlang binary + * argv[2] boolean, when false the cursor will be reset + */ +ASYNC_NIF_DECL( + wterl_cursor_search, + { // struct + + WterlCursorHandle *cursor_handle; + ERL_NIF_TERM key; + int scanning; + }, + { // pre + + static ERL_NIF_TERM ATOM_TRUE = 0; + if (ATOM_TRUE == 0) + enif_make_atom(env, "true"); + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle) && + enif_is_binary(env, argv[1]) && + enif_is_atom(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[1]); + args->scanning = (enif_is_identical(argv[2], ATOM_TRUE)) ? 1 : 0; + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + ErlNifBinary key; + if (!enif_inspect_binary(env, args->key, &key)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + + WT_ITEM item_key; + item_key.data = key.data; + item_key.size = key.size; + cursor->set_key(cursor, &item_key); + + ERL_NIF_TERM reply = __cursor_value_ret(env, cursor, cursor->search(cursor)); + if (!args->scanning) + (void)cursor->reset(cursor); + ASYNC_NIF_REPLY(reply); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Position the cursor at the record matching the key if it exists, or a record + * that would be adjacent. + * + * argv[0] WterlCursorHandle resource + * argv[1] key as an Erlang binary + * argv[2] boolean, when false the cursor will be reset + */ +ASYNC_NIF_DECL( + wterl_cursor_search_near, + { // struct + + WterlCursorHandle *cursor_handle; + ERL_NIF_TERM key; + int scanning; + }, + { // pre + + static ERL_NIF_TERM ATOM_TRUE = 0; + if (ATOM_TRUE == 0) + enif_make_atom(env, "true"); + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle) && + enif_is_binary(env, argv[1]) && + enif_is_atom(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[1]); + args->scanning = (enif_is_identical(argv[2], ATOM_TRUE)) ? 1 : 0; + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + ErlNifBinary key; + if (!enif_inspect_binary(env, args->key, &key)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + WT_ITEM item_key; + int exact = 0; + + item_key.data = key.data; + item_key.size = key.size; + cursor->set_key(cursor, &item_key); + + int rc = cursor->search_near(cursor, &exact); + if (rc != 0) { + (void)cursor->reset(cursor); + ASYNC_NIF_REPLY(__strerror_term(env, rc)); + return; + } + + if (!args->scanning) + (void)cursor->reset(cursor); + + if (exact == 0) { + /* an exact match */ + ASYNC_NIF_REPLY(enif_make_tuple2(env, ATOM_OK, enif_make_atom(env, "match"))); + } else if (exact < 0) { + /* cursor now positioned at the next smaller key */ + ASYNC_NIF_REPLY(enif_make_tuple2(env, ATOM_OK, enif_make_atom(env, "lt"))); + } else if (exact > 0) { + /* cursor now positioned at the next larger key */ + ASYNC_NIF_REPLY(enif_make_tuple2(env, ATOM_OK, enif_make_atom(env, "gt"))); + } + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Reset the position of the cursor. + * + * Any resources held by the cursor are released, and the cursor's key and + * position are no longer valid. A subsequent iteration with wterl_cursor_next + * will move to the first record, or with wterl_cursor_prev will move to the + * last record. + * + * argv[0] WterlCursorHandle resource + */ +ASYNC_NIF_DECL( + wterl_cursor_reset, + { // struct + + WterlCursorHandle *cursor_handle; + }, + { // pre + + if (!(argc == 1 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle))) { + ASYNC_NIF_RETURN_BADARG(); + } + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + int rc = cursor->reset(cursor); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Insert, or overwrite, a record using a cursor. + * + * argv[0] WterlCursorHandle resource + * argv[1] key as an Erlang binary + * argv[2] value as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_cursor_insert, + { // struct + + WterlCursorHandle *cursor_handle; + ERL_NIF_TERM key; + ERL_NIF_TERM value; + int rc; + }, + { // pre + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle) && + enif_is_binary(env, argv[1]) && + enif_is_binary(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[1]); + args->value = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + ErlNifBinary key; + ErlNifBinary value; + if (!enif_inspect_binary(env, args->key, &key)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + if (!enif_inspect_binary(env, args->value, &value)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + WT_ITEM item_key; + WT_ITEM item_value; + + item_key.data = key.data; + item_key.size = key.size; + cursor->set_key(cursor, &item_key); + item_value.data = value.data; + item_value.size = value.size; + cursor->set_value(cursor, &item_value); + int rc = cursor->insert(cursor); + if (rc == 0) + rc = cursor->reset(cursor); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Update an existing record using a cursor. + * + * argv[0] WterlCursorHandle resource + * argv[1] key as an Erlang binary + * argv[2] value as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_cursor_update, + { // struct + + WterlCursorHandle *cursor_handle; + ERL_NIF_TERM key; + ERL_NIF_TERM value; + int rc; + }, + { // pre + + if (!(argc == 3 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle) && + enif_is_binary(env, argv[1]) && + enif_is_binary(env, argv[2]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[1]); + args->value = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[2]); + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + ErlNifBinary key; + ErlNifBinary value; + if (!enif_inspect_binary(env, args->key, &key)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + if (!enif_inspect_binary(env, args->value, &value)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + WT_ITEM item_key; + WT_ITEM item_value; + + item_key.data = key.data; + item_key.size = key.size; + cursor->set_key(cursor, &item_key); + item_value.data = value.data; + item_value.size = value.size; + cursor->set_value(cursor, &item_value); + int rc = cursor->update(cursor); + if (rc == 0) + rc = cursor->reset(cursor); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Remove a record using a cursor. + * + * argv[0] WterlCursorHandle resource + * argv[1] key as an Erlang binary + */ +ASYNC_NIF_DECL( + wterl_cursor_remove, + { // struct + + WterlCursorHandle *cursor_handle; + ERL_NIF_TERM key; + int rc; + }, + { // pre + + if (!(argc == 2 && + enif_get_resource(env, argv[0], wterl_cursor_RESOURCE, (void**)&args->cursor_handle) && + enif_is_binary(env, argv[1]))) { + ASYNC_NIF_RETURN_BADARG(); + } + args->key = enif_make_copy(ASYNC_NIF_WORK_ENV, argv[1]); + enif_keep_resource((void*)args->cursor_handle); + }, + { // work + + WT_CURSOR* cursor = args->cursor_handle->cursor; + ErlNifBinary key; + if (!enif_inspect_binary(env, args->key, &key)) { + ASYNC_NIF_REPLY(enif_make_badarg(env)); + return; + } + WT_ITEM item_key; + + item_key.data = key.data; + item_key.size = key.size; + cursor->set_key(cursor, &item_key); + int rc = cursor->remove(cursor); + if (rc == 0) + rc = cursor->reset(cursor); + ASYNC_NIF_REPLY(rc == 0 ? ATOM_OK : __strerror_term(env, rc)); + }, + { // post + + enif_release_resource((void*)args->cursor_handle); + }); + +/** + * Called as this driver is loaded by the Erlang BEAM runtime triggered by the + * module's on_load directive. + * + * env the NIF environment + * priv_data used to hold the state for this NIF rather than global variables + * load_info an Erlang term, in this case a list with two two-tuples of the + * form: [{wterl, ""}, {wiredtiger, ""}] where the strings contain + * the git hash of the last commit for this code. This is used + * to determin if a rolling upgrade is possible or not. + */ +static int +on_load(ErlNifEnv *env, void **priv_data, ERL_NIF_TERM load_info) +{ + ErlNifResourceFlags flags = ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER; + wterl_conn_RESOURCE = enif_open_resource_type(env, NULL, "wterl_conn_resource", + NULL, flags, NULL); + wterl_cursor_RESOURCE = enif_open_resource_type(env, NULL, "wterl_cursor_resource", + NULL, flags, NULL); + + ATOM_ERROR = enif_make_atom(env, "error"); + ATOM_OK = enif_make_atom(env, "ok"); + ATOM_NOT_FOUND = enif_make_atom(env, "not_found"); + ATOM_FIRST = enif_make_atom(env, "first"); + ATOM_LAST = enif_make_atom(env, "last"); + + struct wterl_priv_data *priv = enif_alloc(sizeof(struct wterl_priv_data)); + if (!priv) + return ENOMEM; + memset(priv, 0, sizeof(struct wterl_priv_data)); + + /* Note: !!! the first element of our priv_data struct *must* be the + pointer to the async_nif's private data which we set here. */ + ASYNC_NIF_LOAD(wterl, priv->async_nif_priv); + + priv->conns_mutex = enif_mutex_create(NULL); + priv->conns = kh_init(conns); + *priv_data = priv; + return *priv_data ? 0 : ENOMEM; +} + +static int +on_reload(ErlNifEnv *env, void **priv_data, ERL_NIF_TERM load_info) +{ + return 0; // TODO: implement +} + +static void +on_unload(ErlNifEnv *env, void *priv_data) +{ + unsigned int i; + struct wterl_priv_data *priv = (struct wterl_priv_data *)priv_data; + khash_t(conns) *h; + khiter_t itr_conns; + WterlConnHandle *conn_handle; + + enif_mutex_lock(priv->conns_mutex); + h = priv->conns; + + for (itr_conns = kh_begin(h); itr_conns != kh_end(h); ++itr_conns) { + if (kh_exist(h, itr_conns)) { + conn_handle = kh_val(h, itr_conns); + if (conn_handle) { + enif_mutex_lock(conn_handle->contexts_mutex); + enif_free((void*)conn_handle->session_config); + for (i = 0; i < ASYNC_NIF_MAX_WORKERS; i++) { + WterlCtx *ctx = &conn_handle->contexts[i]; + if (ctx->session != NULL) { + WT_SESSION *session = ctx->session; + khash_t(cursors) *h = ctx->cursors; + khiter_t itr_cursors; + for (itr_cursors = kh_begin(h); itr_cursors != kh_end(h); ++itr_cursors) { + if (kh_exist(h, itr_cursors)) { + WT_CURSOR *cursor = kh_val(h, itr_cursors); + char *key = (char *)kh_key(h, itr_cursors); + cursor->close(cursor); + kh_del(cursors, h, itr_cursors); + enif_free(key); + kh_value(h, itr_cursors) = NULL; + } + } + kh_destroy(cursors, h); + session->close(session, NULL); + } + } + } + + /* This would have closed all cursors and sessions for us + but we do that explicitly above. */ + conn_handle->conn->close(conn_handle->conn, NULL); + } + } + + /* Continue to hold the context mutex while unloading the async_nif + to prevent new work from coming in while shutting down. */ + ASYNC_NIF_UNLOAD(wterl, env, priv->async_nif_priv); + + for (itr_conns = kh_begin(h); itr_conns != kh_end(h); ++itr_conns) { + if (kh_exist(h, itr_conns)) { + conn_handle = kh_val(h, itr_conns); + if (conn_handle) { + enif_mutex_unlock(conn_handle->contexts_mutex); + enif_mutex_destroy(conn_handle->contexts_mutex); + } + } + } + + kh_destroy(conns, h); + enif_mutex_unlock(priv->conns_mutex); + enif_mutex_destroy(priv->conns_mutex); + enif_free(priv); +} + +static int +on_upgrade(ErlNifEnv *env, void **priv_data, void **old_priv_data, ERL_NIF_TERM load_info) +{ + ASYNC_NIF_UPGRADE(wterl, env); // TODO: implement + return 0; +} + +static ErlNifFunc nif_funcs[] = +{ + {"checkpoint_nif", 3, wterl_checkpoint}, + {"conn_close_nif", 2, wterl_conn_close}, + {"conn_open_nif", 4, wterl_conn_open}, + {"create_nif", 4, wterl_create}, + {"delete_nif", 4, wterl_delete}, + {"drop_nif", 4, wterl_drop}, + {"get_nif", 4, wterl_get}, + {"put_nif", 5, wterl_put}, + {"rename_nif", 5, wterl_rename}, + {"salvage_nif", 4, wterl_salvage}, + // TODO: {"txn_begin", 3, wterl_txn_begin}, + // TODO: {"txn_commit", 3, wterl_txn_commit}, + // TODO: {"txn_abort", 3, wterl_txn_abort}, + {"truncate_nif", 6, wterl_truncate}, + {"upgrade_nif", 4, wterl_upgrade}, + {"verify_nif", 4, wterl_verify}, + {"cursor_close_nif", 2, wterl_cursor_close}, + {"cursor_insert_nif", 4, wterl_cursor_insert}, + {"cursor_next_key_nif", 2, wterl_cursor_next_key}, + {"cursor_next_nif", 2, wterl_cursor_next}, + {"cursor_next_value_nif", 2, wterl_cursor_next_value}, + {"cursor_open_nif", 4, wterl_cursor_open}, + {"cursor_prev_key_nif", 2, wterl_cursor_prev_key}, + {"cursor_prev_nif", 2, wterl_cursor_prev}, + {"cursor_prev_value_nif", 2, wterl_cursor_prev_value}, + {"cursor_remove_nif", 3, wterl_cursor_remove}, + {"cursor_reset_nif", 2, wterl_cursor_reset}, + {"cursor_search_near_nif", 4, wterl_cursor_search_near}, + {"cursor_search_nif", 4, wterl_cursor_search}, + {"cursor_update_nif", 4, wterl_cursor_update}, +}; + +ERL_NIF_INIT(wterl, nif_funcs, &on_load, &on_reload, &on_upgrade, &on_unload); diff --git a/rebar.config b/rebar.config index 52d6171..9a8dc93 100644 --- a/rebar.config +++ b/rebar.config @@ -37,9 +37,8 @@ {port_env, [ {"DRV_CFLAGS", "$DRV_CFLAGS -Werror -I c_src/system/include"}, - {"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath,lib/wterl/priv:priv -Lpriv -lwiredtiger"} + {"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath,lib/wterl/priv:priv -Lc_src/system/lib -lwiredtiger"} ]}. -{pre_hooks, [{compile, "c_src/build_deps.sh"}]}. - +{pre_hooks, [{compile, "c_src/build_deps.sh compile"}]}. {post_hooks, [{clean, "c_src/build_deps.sh clean"}]}. diff --git a/src/async_nif.hrl b/src/async_nif.hrl new file mode 100644 index 0000000..5110fa2 --- /dev/null +++ b/src/async_nif.hrl @@ -0,0 +1,42 @@ +%% ------------------------------------------------------------------- +%% +%% async_nif: An async thread-pool layer for Erlang's NIF API +%% +%% Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved. +%% Author: Gregory Burd +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-define(ASYNC_NIF_CALL(Fun, Args), + begin + NIFRef = erlang:make_ref(), + case erlang:apply(Fun, [NIFRef|Args]) of + {ok, enqueued} -> + receive + {NIFRef, {error, shutdown}=Error} -> + %% Work unit was queued, but not executed. + Error; + {NIFRef, {error, _Reason}=Error} -> + %% Work unit returned an error. + Error; + {NIFRef, Reply} -> + Reply + end; + Other -> + Other + end + end). diff --git a/src/riak_kv_wterl_backend.erl b/src/riak_kv_wterl_backend.erl index fa4282b..f9f08ba 100644 --- a/src/riak_kv_wterl_backend.erl +++ b/src/riak_kv_wterl_backend.erl @@ -22,7 +22,6 @@ -module(riak_kv_wterl_backend). -behavior(temp_riak_kv_backend). --author('Steve Vinoski '). %% KV Backend API -export([api_version/0, @@ -51,13 +50,11 @@ %%-define(CAPABILITIES, [async_fold, indexes]). -define(CAPABILITIES, [async_fold]). --record(pass, {session :: wterl:session(), - cursor :: wterl:cursor()}). --type pass() :: #pass{}. - -record(state, {table :: string(), + type :: string(), connection :: wterl:connection(), - passes :: [pass()]}). + is_empty_cursor :: wterl:cursor(), + status_cursor :: wterl:cursor()}). -type state() :: #state{}. -type config() :: [{atom(), term()}]. @@ -91,39 +88,80 @@ start(Partition, Config) -> ok; {error, {already_started, _}} -> ok; - {error, Reason1} -> - lager:error("Failed to start wterl: ~p", [Reason1]), - {error, Reason1} + {error, Reason1} -> + lager:error("Failed to start wterl: ~p", [Reason1]), + {error, Reason1} end, case AppStart of ok -> - Table = "lsm:wt" ++ integer_to_list(Partition), - {ok, Connection} = establish_connection(Config), - Passes = establish_passes(erlang:system_info(schedulers), Connection, Table), - {ok, #state{table=Table, connection=Connection, passes=Passes}}; - {error, Reason2} -> - {error, Reason2} + Type = + case wterl:config_value(type, Config, "lsm") of + {type, "lsm"} -> "lsm"; + {type, "table"} -> "table"; + {type, "btree"} -> "table"; + {type, BadType} -> + lager:info("wterl:start ignoring unknown type ~p, using lsm instead", [BadType]), + "lsm"; + _ -> + lager:info("wterl:start ignoring mistaken setting defaulting to lsm"), + "lsm" + end, + {ok, Connection} = establish_connection(Config, Type), + Table = Type ++ ":" ++ integer_to_list(Partition), + Compressor = + case wterl:config_value(block_compressor, Config, "snappy") of + {block_compressor, "snappy"}=C -> [C]; + {block_compressor, "bzip2"}=C -> [C]; + {block_compressor, "none"} -> []; + {block_compressor, none} -> []; + {block_compressor, _} -> [{block_compressor, "snappy"}]; + _ -> [{block_compressor, "snappy"}] + end, + TableOpts = + case Type of + "lsm" -> + [{internal_page_max, "128K"}, + {leaf_page_max, "128K"}, + {lsm_chunk_size, "25MB"}, + {prefix_compression, false}, + {lsm_bloom_newest, true}, + {lsm_bloom_oldest, true} , + {lsm_bloom_bit_count, 128}, + {lsm_bloom_hash_count, 64}, + {lsm_bloom_config, [{leaf_page_max, "8MB"}]} + ] ++ Compressor; + "table" -> + Compressor + end, + case wterl:create(Connection, Table, TableOpts) of + ok -> + case establish_utility_cursors(Connection, Table) of + {ok, IsEmptyCursor, StatusCursor} -> + {ok, #state{table=Table, type=Type, + connection=Connection, + is_empty_cursor=IsEmptyCursor, + status_cursor=StatusCursor}}; + {error, Reason2} -> + {error, Reason2} + end; + {error, Reason3} -> + {error, Reason3} + end end. %% @doc Stop the wterl backend -spec stop(state()) -> ok. -stop(#state{passes=Passes}) -> - lists:foreach(fun(Elem) -> - {Session, Cursor} = Elem, - ok = wterl:cursor_close(Cursor), - ok = wterl:session_close(Session) - end, Passes), - ok. +stop(_State) -> + ok. %% The connection is closed by wterl_conn:stop() %% @doc Retrieve an object from the wterl backend -spec get(riak_object:bucket(), riak_object:key(), state()) -> {ok, any(), state()} | {ok, not_found, state()} | {error, term(), state()}. -get(Bucket, Key, #state{passes=Passes}=State) -> +get(Bucket, Key, #state{connection=Connection, table=Table}=State) -> WTKey = to_object_key(Bucket, Key), - {_Session, Cursor} = lists:nth(erlang:system_info(scheduler_id), Passes), - case wterl:cursor_search(Cursor, WTKey) of + case wterl:get(Connection, Table, WTKey) of {ok, Value} -> {ok, Value, State}; not_found -> @@ -140,10 +178,8 @@ get(Bucket, Key, #state{passes=Passes}=State) -> -spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> {ok, state()} | {error, term(), state()}. -put(Bucket, PrimaryKey, _IndexSpecs, Val, #state{passes=Passes}=State) -> - WTKey = to_object_key(Bucket, PrimaryKey), - {_Session, Cursor} = lists:nth(erlang:system_info(scheduler_id), Passes), - case wterl:cursor_insert(Cursor, WTKey, Val) of +put(Bucket, PrimaryKey, _IndexSpecs, Val, #state{connection=Connection, table=Table}=State) -> + case wterl:put(Connection, Table, to_object_key(Bucket, PrimaryKey), Val) of ok -> {ok, State}; {error, Reason} -> @@ -157,10 +193,8 @@ put(Bucket, PrimaryKey, _IndexSpecs, Val, #state{passes=Passes}=State) -> -spec delete(riak_object:bucket(), riak_object:key(), [index_spec()], state()) -> {ok, state()} | {error, term(), state()}. -delete(Bucket, Key, _IndexSpecs, #state{passes=Passes}=State) -> - WTKey = to_object_key(Bucket, Key), - {_Session, Cursor} = lists:nth(erlang:system_info(scheduler_id), Passes), - case wterl:cursor_remove(Cursor, WTKey) of +delete(Bucket, Key, _IndexSpecs, #state{connection=Connection, table=Table}=State) -> + case wterl:delete(Connection, Table, to_object_key(Bucket, Key)) of ok -> {ok, State}; {error, Reason} -> @@ -172,14 +206,12 @@ delete(Bucket, Key, _IndexSpecs, #state{passes=Passes}=State) -> any(), [], state()) -> {ok, any()} | {async, fun()}. -fold_buckets(FoldBucketsFun, Acc, Opts, #state{table=Table}) -> - {ok, Connection} = wterl_conn:get(), +fold_buckets(FoldBucketsFun, Acc, Opts, #state{connection=Connection, table=Table}) -> FoldFun = fold_buckets_fun(FoldBucketsFun), BucketFolder = fun() -> - {ok, Session} = wterl:session_open(Connection), - case wterl:cursor_open(Session, Table) of - {error, "No such file or directory"} -> + case wterl:cursor_open(Connection, Table) of + {error, {enoent, _Message}} -> Acc; {ok, Cursor} -> try @@ -190,8 +222,7 @@ fold_buckets(FoldBucketsFun, Acc, Opts, #state{table=Table}) -> {break, AccFinal} -> AccFinal after - ok = wterl:cursor_close(Cursor), - ok = wterl:session_close(Session) + ok = wterl:cursor_close(Cursor) end end end, @@ -207,7 +238,7 @@ fold_buckets(FoldBucketsFun, Acc, Opts, #state{table=Table}) -> any(), [{atom(), term()}], state()) -> {ok, term()} | {async, fun()}. -fold_keys(FoldKeysFun, Acc, Opts, #state{table=Table}) -> +fold_keys(FoldKeysFun, Acc, Opts, #state{connection=Connection, table=Table}) -> %% Figure out how we should limit the fold: by bucket, by %% secondary index, or neither (fold across everything.) Bucket = lists:keyfind(bucket, 1, Opts), @@ -220,15 +251,12 @@ fold_keys(FoldKeysFun, Acc, Opts, #state{table=Table}) -> true -> undefined end, - {ok, Connection} = wterl_conn:get(), - %% Set up the fold... FoldFun = fold_keys_fun(FoldKeysFun, Limiter), KeyFolder = fun() -> - {ok, Session} = wterl:session_open(Connection), - case wterl:cursor_open(Session, Table) of - {error, "No such file or directory"} -> + case wterl:cursor_open(Connection, Table) of + {error, {enoent, _Message}} -> Acc; {ok, Cursor} -> try @@ -237,8 +265,7 @@ fold_keys(FoldKeysFun, Acc, Opts, #state{table=Table}) -> {break, AccFinal} -> AccFinal after - ok = wterl:cursor_close(Cursor), - ok = wterl:session_close(Session) + ok = wterl:cursor_close(Cursor) end end end, @@ -254,15 +281,13 @@ fold_keys(FoldKeysFun, Acc, Opts, #state{table=Table}) -> any(), [{atom(), term()}], state()) -> {ok, any()} | {async, fun()}. -fold_objects(FoldObjectsFun, Acc, Opts, #state{table=Table}) -> - {ok, Connection} = wterl_conn:get(), +fold_objects(FoldObjectsFun, Acc, Opts, #state{connection=Connection, table=Table}) -> Bucket = proplists:get_value(bucket, Opts), FoldFun = fold_objects_fun(FoldObjectsFun, Bucket), ObjectFolder = fun() -> - {ok, Session} = wterl:session_open(Connection), - case wterl:cursor_open(Session, Table) of - {error, "No such file or directory"} -> + case wterl:cursor_open(Connection, Table) of + {error, {enoent, _Message}} -> Acc; {ok, Cursor} -> try @@ -271,8 +296,14 @@ fold_objects(FoldObjectsFun, Acc, Opts, #state{table=Table}) -> {break, AccFinal} -> AccFinal after - ok = wterl:cursor_close(Cursor), - ok = wterl:session_close(Session) + case wterl:cursor_close(Cursor) of + ok -> + ok; + {error, {eperm, _}} -> %% TODO: review/fix + ok; + {error, _}=E -> + E + end end end end, @@ -285,11 +316,12 @@ fold_objects(FoldObjectsFun, Acc, Opts, #state{table=Table}) -> %% @doc Delete all objects from this wterl backend -spec drop(state()) -> {ok, state()} | {error, term(), state()}. -drop(#state{passes=Passes, table=Table}=State) -> - {Session, _Cursor} = lists:nth(erlang:system_info(scheduler_id), Passes), - case wterl:session_truncate(Session, Table) of +drop(#state{connection=Connection, table=Table}=State) -> + case wterl:drop(Connection, Table) of ok -> {ok, State}; + {error, {ebusy, _}} -> %% TODO: review/fix + {ok, State}; Error -> {error, Error, State} end. @@ -297,24 +329,25 @@ drop(#state{passes=Passes, table=Table}=State) -> %% @doc Returns true if this wterl backend contains any %% non-tombstone values; otherwise returns false. -spec is_empty(state()) -> boolean(). -is_empty(#state{passes=Passes}) -> - {_Session, Cursor} = lists:nth(erlang:system_info(scheduler_id), Passes), +is_empty(#state{is_empty_cursor=Cursor}) -> wterl:cursor_reset(Cursor), - try - not_found =:= wterl:cursor_next(Cursor) - after - ok = wterl:cursor_close(Cursor) + case wterl:cursor_next(Cursor) of + not_found -> true; + {error, {eperm, _}} -> false; % TODO: review/fix this logic + _ -> false end. %% @doc Get the status information for this wterl backend -spec status(state()) -> [{atom(), term()}]. -status(#state{passes=Passes}) -> - {_Session, Cursor} = lists:nth(erlang:system_info(scheduler_id), Passes), - try - Stats = fetch_status(Cursor), - [{stats, Stats}] - after - ok = wterl:cursor_close(Cursor) +status(#state{status_cursor=Cursor}) -> + wterl:cursor_reset(Cursor), + case fetch_status(Cursor) of + {ok, Stats} -> + Stats; + {error, {eperm, _}} -> % TODO: review/fix this logic + {ok, []}; + _ -> + {ok, []} end. %% @doc Register an asynchronous callback @@ -327,16 +360,35 @@ callback(_Ref, _Msg, State) -> %% Internal functions %% =================================================================== +%% @private max_sessions(Config) -> RingSize = case app_helper:get_prop_or_env(ring_creation_size, Config, riak_core) of undefined -> 1024; Size -> Size end, - 2 * (RingSize * erlang:system_info(schedulers)). + Est = 100 * (RingSize * erlang:system_info(schedulers)), % TODO: review/fix this logic + case Est > 1000000000 of % Note: WiredTiger uses a signed int for this + true -> 1000000000; + false -> Est + end. %% @private -establish_connection(Config) -> +establish_utility_cursors(Connection, Table) -> + case wterl:cursor_open(Connection, Table) of + {ok, IsEmptyCursor} -> + case wterl:cursor_open(Connection, "statistics:" ++ Table, [{statistics_fast, true}]) of + {ok, StatusCursor} -> + {ok, IsEmptyCursor, StatusCursor}; + {error, Reason1} -> + {error, Reason1} + end; + {error, Reason2} -> + {error, Reason2} + end. + +%% @private +establish_connection(Config, Type) -> %% Get the data root directory case app_helper:get_prop_or_env(data_root, Config, wterl) of undefined -> @@ -344,8 +396,18 @@ establish_connection(Config) -> {error, data_root_unset}; DataRoot -> ok = filelib:ensure_dir(filename:join(DataRoot, "x")), + + %% WT Connection Options: + %% NOTE: LSM auto-checkpoints, so we don't have too. + CheckpointSetting = + case Type =:= "lsm" of + true -> + []; + false -> + app_helper:get_prop_or_env(checkpoint, Config, wterl, [{wait, 10}]) + end, RequestedCacheSize = app_helper:get_prop_or_env(cache_size, Config, wterl), - Opts = + ConnectionOpts = orddict:from_list( [ wterl:config_value(create, Config, true), wterl:config_value(sync, Config, false), @@ -354,14 +416,18 @@ establish_connection(Config) -> wterl:config_value(session_max, Config, max_sessions(Config)), wterl:config_value(cache_size, Config, size_cache(RequestedCacheSize)), wterl:config_value(statistics_log, Config, [{wait, 30}]), % sec - %% NOTE: LSM auto-checkpoints, so we don't have too. - %% wterl:config_value(checkpoint, Config, [{wait, 10}]), % sec wterl:config_value(verbose, Config, [ - %"ckpt" "block", "shared_cache", "evictserver", "fileops", - %"hazard", "mutex", "read", "readserver", "reconcile", - %"salvage", "verify", "write", "evict", "lsm" - ]) ] ++ proplists:get_value(wterl, Config, [])), % sec - case wterl_conn:open(DataRoot, Opts) of + %"ckpt" "block", "shared_cache", "evictserver", "fileops", + %"hazard", "mutex", "read", "readserver", "reconcile", + %"salvage", "verify", "write", "evict", "lsm" + ]) ] ++ CheckpointSetting ++ proplists:get_value(wterl, Config, [])), % sec + + + + %% WT Session Options: + SessionOpts = [{isolation, "snapshot"}], + + case wterl_conn:open(DataRoot, ConnectionOpts, SessionOpts) of {ok, Connection} -> {ok, Connection}; {error, Reason2} -> @@ -370,40 +436,6 @@ establish_connection(Config) -> end end. -establish_passes(Count, Connection, Table) - when is_number(Count), Count > 0 -> - lists:map(fun(_Elem) -> - {ok, Session} = establish_session(Connection, Table), - {ok, Cursor} = wterl:cursor_open(Session, Table), - {Session, Cursor} - end, lists:seq(1, Count)). - -%% @private -establish_session(Connection, Table) -> - case wterl:session_open(Connection, wterl:config_to_bin([{isolation, "snapshot"}])) of - {ok, Session} -> - SessionOpts = - [{block_compressor, "snappy"}, - {internal_page_max, "128K"}, - {leaf_page_max, "128K"}, - {lsm_chunk_size, "25MB"}, - {lsm_bloom_newest, true}, - {lsm_bloom_oldest, true} , - {lsm_bloom_bit_count, 128}, - {lsm_bloom_hash_count, 64}, - {lsm_bloom_config, [{leaf_page_max, "8MB"}]} ], - case wterl:session_create(Session, Table, wterl:config_to_bin(SessionOpts)) of - ok -> - {ok, Session}; - {error, Reason} -> - lager:error("Failed to start wterl backend: ~p\n", [Reason]), - {error, Reason} - end; - {error, Reason} -> - lager:error("Failed to open a WiredTiger session: ~p\n", [Reason]), - {error, Reason} - end. - %% @private %% Return a function to fold over the buckets on this backend fold_buckets_fun(FoldBucketsFun) -> @@ -512,7 +544,9 @@ from_index_key(LKey) -> %% @private %% Return all status from wterl statistics cursor fetch_status(Cursor) -> - fetch_status(Cursor, wterl:cursor_next_value(Cursor), []). + {ok, fetch_status(Cursor, wterl:cursor_next_value(Cursor), [])}. +fetch_status(_Cursor, {error, _}, Acc) -> + lists:reverse(Acc); fetch_status(_Cursor, not_found, Acc) -> lists:reverse(Acc); fetch_status(Cursor, {ok, Stat}, Acc) -> @@ -563,12 +597,14 @@ size_cache(RequestedSize) -> -ifdef(TEST). simple_test_() -> - ?assertCmd("rm -rf test/wterl-backend"), + {ok, CWD} = file:get_cwd(), + rmdir:path(filename:join([CWD, "test/wterl-backend"])), %?assertCmd("rm -rf test/wterl-backend"), application:set_env(wterl, data_root, "test/wterl-backend"), temp_riak_kv_backend:standard_test(?MODULE, []). custom_config_test_() -> - ?assertCmd("rm -rf test/wterl-backend"), + {ok, CWD} = file:get_cwd(), + rmdir:path(filename:join([CWD, "test/wterl-backend"])), %?assertCmd("rm -rf test/wterl-backend"), application:set_env(wterl, data_root, ""), temp_riak_kv_backend:standard_test(?MODULE, [{data_root, "test/wterl-backend"}]). diff --git a/src/rmdir.erl b/src/rmdir.erl new file mode 100644 index 0000000..d26a129 --- /dev/null +++ b/src/rmdir.erl @@ -0,0 +1,26 @@ +-module(rmdir). + +-export([path/1]). + +-include_lib("kernel/include/file.hrl"). + +path(Dir) -> + remove_all_files(".", [Dir]). + +remove_all_files(Dir, Files) -> + lists:foreach(fun(File) -> + FilePath = filename:join([Dir, File]), + case file:read_file_info(FilePath) of + {ok, FileInfo} -> + case FileInfo#file_info.type of + directory -> + {ok, DirFiles} = file:list_dir(FilePath), + remove_all_files(FilePath, DirFiles), + file:del_dir(FilePath); + _ -> + file:delete(FilePath) + end; + {error, _Reason} -> + ok + end + end, Files). diff --git a/src/temp_riak_kv_backend.erl b/src/temp_riak_kv_backend.erl index 5c27461..c41a38d 100644 --- a/src/temp_riak_kv_backend.erl +++ b/src/temp_riak_kv_backend.erl @@ -272,13 +272,16 @@ empty_check({Backend, State}) -> }. setup({BackendMod, Config}) -> - lager:start(), + application:start(lager), application:start(sasl), application:start(os_mon), {ok, S} = BackendMod:start(42, Config), {BackendMod, S}. cleanup({BackendMod, S}) -> - ok = BackendMod:stop(S). + ok = BackendMod:stop(S), + application:stop(lager), + application:stop(sasl), + application:stop(os_mon). -endif. % TEST diff --git a/src/wterl.app.src b/src/wterl.app.src index 534c8f0..b235371 100644 --- a/src/wterl.app.src +++ b/src/wterl.app.src @@ -1,6 +1,6 @@ {application, wterl, [ - {description, "Erlang Wrapper for WiredTiger"}, + {description, "Erlang NIF Wrapper for WiredTiger"}, {vsn, "0.9.0"}, {registered, []}, {applications, [ diff --git a/src/wterl.erl b/src/wterl.erl index f55de4d..ee672a9 100644 --- a/src/wterl.erl +++ b/src/wterl.erl @@ -20,7 +20,9 @@ %% %% ------------------------------------------------------------------- -module(wterl). + -export([connection_open/2, + connection_open/3, connection_close/1, cursor_close/1, cursor_insert/3, @@ -28,47 +30,49 @@ cursor_next_key/1, cursor_next_value/1, cursor_open/2, + cursor_open/3, cursor_prev/1, cursor_prev_key/1, cursor_prev_value/1, cursor_remove/2, cursor_reset/1, cursor_search/2, + cursor_search/3, cursor_search_near/2, + cursor_search_near/3, cursor_update/3, - session_checkpoint/1, - session_checkpoint/2, - session_close/1, - session_create/2, - session_create/3, - session_delete/3, - session_drop/2, - session_drop/3, - session_get/3, - session_open/1, - session_open/2, - session_put/4, - session_rename/3, - session_rename/4, - session_salvage/2, - session_salvage/3, - session_truncate/2, - session_truncate/3, - session_upgrade/2, - session_upgrade/3, - session_verify/2, - session_verify/3, + checkpoint/1, + checkpoint/2, + create/2, + create/3, + delete/3, + drop/2, + drop/3, + get/3, + put/4, + rename/3, + rename/4, + salvage/2, + salvage/3, + truncate/2, + truncate/3, + truncate/4, + truncate/5, + upgrade/2, + upgrade/3, + verify/2, + verify/3, config_value/3, - config_to_bin/1, - priv_dir/0, + priv_dir/0, fold_keys/3, fold/3]). +-include("async_nif.hrl"). + -ifdef(TEST). -ifdef(EQC). -include_lib("eqc/include/eqc.hrl"). --define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). +-define(QC_OUT(P), eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). -endif. -include_lib("eunit/include/eunit.hrl"). -endif. @@ -76,12 +80,11 @@ -type config() :: binary(). -type config_list() :: [{atom(), any()}]. -opaque connection() :: reference(). --opaque session() :: reference(). -opaque cursor() :: reference(). -type key() :: binary(). -type value() :: binary(). --export_type([connection/0, session/0, cursor/0]). +-export_type([connection/0, cursor/0]). -on_load(init/0). @@ -89,169 +92,284 @@ nif_stub_error(Line) -> erlang:nif_error({nif_not_loaded,module,?MODULE,line,Line}). --define(EMPTY_CONFIG, <<"\0">>). - -spec init() -> ok | {error, any()}. init() -> - erlang:load_nif(filename:join(priv_dir(), atom_to_list(?MODULE)), 0). + erlang:load_nif(filename:join([priv_dir(), atom_to_list(?MODULE)]), + [{wterl, "07061ed6e8252543c2f06b81a646eca6945cc558"}, + {wiredtiger, "6f7a4b961c744bfb21f0c21d4c28c2d162400f1b"}]). --spec connection_open(string(), config()) -> {ok, connection()} | {error, term()}. -connection_open(HomeDir, Config) -> +-spec connection_open(string(), config_list()) -> {ok, connection()} | {error, term()}. +-spec connection_open(string(), config_list(), config_list()) -> {ok, connection()} | {error, term()}. +connection_open(HomeDir, ConnectionConfig) -> + connection_open(HomeDir, ConnectionConfig, []). +connection_open(HomeDir, ConnectionConfig, SessionConfig) -> PrivDir = wterl:priv_dir(), {ok, PrivFiles} = file:list_dir(PrivDir), SoFiles = - lists:filter(fun(Elem) -> - case re:run(Elem, "^libwiredtiger_.*\.so$") of - {match, _} -> true; - nomatch -> false - end - end, PrivFiles), - SoPaths = lists:map(fun(Elem) -> filename:join(PrivDir, Elem) end, SoFiles), - Bin = config_to_bin([{extensions, SoPaths}], [<<",">>, Config]), - conn_open(HomeDir, Bin). + lists:filter(fun(Elem) -> + case re:run(Elem, "^libwiredtiger_.*\.so$") of + {match, _} -> true; + nomatch -> false + end + end, PrivFiles), + SoPaths = lists:map(fun(Elem) -> filename:join([PrivDir, Elem]) end, SoFiles), + conn_open(HomeDir, [{extensions, SoPaths}] ++ ConnectionConfig, SessionConfig). --spec conn_open(string(), config()) -> {ok, connection()} | {error, term()}. -conn_open(_HomeDir, _Config) -> +-spec conn_open(string(), config_list(), config_list()) -> {ok, connection()} | {error, term()}. +conn_open(HomeDir, ConnectionConfig, SessionConfig) -> + ?ASYNC_NIF_CALL(fun conn_open_nif/4, [HomeDir, + config_to_bin(ConnectionConfig), + config_to_bin(SessionConfig)]). + +-spec conn_open_nif(reference(), string(), config(), config()) -> {ok, connection()} | {error, term()}. +conn_open_nif(_AsyncRef, _HomeDir, _ConnectionConfig, _SessionConfig) -> ?nif_stub. -spec connection_close(connection()) -> ok | {error, term()}. -connection_close(_ConnRef) -> +connection_close(ConnRef) -> + ?ASYNC_NIF_CALL(fun conn_close_nif/2, [ConnRef]). + +-spec conn_close_nif(reference(), connection()) -> ok | {error, term()}. +conn_close_nif(_AsyncRef, _ConnRef) -> ?nif_stub. --spec session_open(connection()) -> {ok, session()} | {error, term()}. -session_open(ConnRef) -> - session_open(ConnRef, ?EMPTY_CONFIG). +-spec create(connection(), string()) -> ok | {error, term()}. +-spec create(connection(), string(), config_list()) -> ok | {error, term()}. +create(Ref, Name) -> + create(Ref, Name, []). +create(Ref, Name, Config) -> + ?ASYNC_NIF_CALL(fun create_nif/4, [Ref, Name, config_to_bin(Config)]). --spec session_open(connection(), config()) -> {ok, session()} | {error, term()}. -session_open(_ConnRef, _Config) -> +-spec create_nif(reference(), connection(), string(), config()) -> ok | {error, term()}. +create_nif(_AsyncNif, _Ref, _Name, _Config) -> ?nif_stub. --spec session_close(session()) -> ok | {error, term()}. -session_close(_Ref) -> +-spec drop(connection(), string()) -> ok | {error, term()}. +-spec drop(connection(), string(), config_list()) -> ok | {error, term()}. +drop(Ref, Name) -> + drop(Ref, Name, [{force, true}]). +drop(Ref, Name, Config) -> + ?ASYNC_NIF_CALL(fun drop_nif/4, [Ref, Name, config_to_bin(Config)]). + +-spec drop_nif(reference(), connection(), string(), config()) -> ok | {error, term()}. +drop_nif(_AsyncRef, _Ref, _Name, _Config) -> ?nif_stub. --spec session_create(session(), string()) -> ok | {error, term()}. --spec session_create(session(), string(), config()) -> ok | {error, term()}. -session_create(Ref, Name) -> - session_create(Ref, Name, ?EMPTY_CONFIG). -session_create(_Ref, _Name, _Config) -> +-spec delete(connection(), string(), key()) -> ok | {error, term()}. +delete(Ref, Table, Key) -> + ?ASYNC_NIF_CALL(fun delete_nif/4, [Ref, Table, Key]). + +-spec delete_nif(reference(), connection(), string(), key()) -> ok | {error, term()}. +delete_nif(_AsyncRef, _Ref, _Table, _Key) -> ?nif_stub. --spec session_drop(session(), string()) -> ok | {error, term()}. --spec session_drop(session(), string(), config()) -> ok | {error, term()}. -session_drop(Ref, Name) -> - session_drop(Ref, Name, ?EMPTY_CONFIG). -session_drop(_Ref, _Name, _Config) -> +-spec get(connection(), string(), key()) -> {ok, value()} | not_found | {error, term()}. +get(Ref, Table, Key) -> + ?ASYNC_NIF_CALL(fun get_nif/4, [Ref, Table, Key]). + +-spec get_nif(reference(), connection(), string(), key()) -> {ok, value()} | not_found | {error, term()}. +get_nif(_AsyncRef, _Ref, _Table, _Key) -> ?nif_stub. --spec session_delete(session(), string(), key()) -> ok | {error, term()}. -session_delete(_Ref, _Table, _Key) -> +-spec put(connection(), string(), key(), value()) -> ok | {error, term()}. +put(Ref, Table, Key, Value) -> + ?ASYNC_NIF_CALL(fun put_nif/5, [Ref, Table, Key, Value]). + +-spec put_nif(reference(), connection(), string(), key(), value()) -> ok | {error, term()}. +put_nif(_AsyncRef, _Ref, _Table, _Key, _Value) -> ?nif_stub. --spec session_get(session(), string(), key()) -> {ok, value()} | not_found | {error, term()}. -session_get(_Ref, _Table, _Key) -> +-spec rename(connection(), string(), string()) -> ok | {error, term()}. +-spec rename(connection(), string(), string(), config_list()) -> ok | {error, term()}. +rename(Ref, OldName, NewName) -> + rename(Ref, OldName, NewName, []). +rename(Ref, OldName, NewName, Config) -> + ?ASYNC_NIF_CALL(fun rename_nif/5, [Ref, OldName, NewName, config_to_bin(Config)]). + +-spec rename_nif(reference(), connection(), string(), string(), config()) -> ok | {error, term()}. +rename_nif(_AsyncRef, _Ref, _OldName, _NewName, _Config) -> ?nif_stub. --spec session_put(session(), string(), key(), value()) -> ok | {error, term()}. -session_put(_Ref, _Table, _Key, _Value) -> +-spec salvage(connection(), string()) -> ok | {error, term()}. +-spec salvage(connection(), string(), config_list()) -> ok | {error, term()}. +salvage(Ref, Name) -> + salvage(Ref, Name, []). +salvage(Ref, Name, Config) -> + ?ASYNC_NIF_CALL(fun salvage_nif/4, [Ref, Name, config_to_bin(Config)]). + +-spec salvage_nif(reference(), connection(), string(), config()) -> ok | {error, term()}. +salvage_nif(_AsyncRef, _Ref, _Name, _Config) -> ?nif_stub. --spec session_rename(session(), string(), string()) -> ok | {error, term()}. --spec session_rename(session(), string(), string(), config()) -> ok | {error, term()}. -session_rename(Ref, OldName, NewName) -> - session_rename(Ref, OldName, NewName, ?EMPTY_CONFIG). -session_rename(_Ref, _OldName, _NewName, _Config) -> +-spec checkpoint(connection()) -> ok | {error, term()}. +-spec checkpoint(connection(), config_list()) -> ok | {error, term()}. +checkpoint(_Ref) -> + checkpoint(_Ref, []). +checkpoint(Ref, Config) -> + ?ASYNC_NIF_CALL(fun checkpoint_nif/3, [Ref, config_to_bin(Config)]). + +-spec checkpoint_nif(reference(), connection(), config()) -> ok | {error, term()}. +checkpoint_nif(_AsyncRef, _Ref, _Config) -> ?nif_stub. --spec session_salvage(session(), string()) -> ok | {error, term()}. --spec session_salvage(session(), string(), config()) -> ok | {error, term()}. -session_salvage(Ref, Name) -> - session_salvage(Ref, Name, ?EMPTY_CONFIG). -session_salvage(_Ref, _Name, _Config) -> +-spec truncate(connection(), string()) -> ok | {error, term()}. +-spec truncate(connection(), string(), config_list()) -> ok | {error, term()}. +-spec truncate(connection(), string(), binary() | first, binary() | last) -> ok | {error, term()}. +-spec truncate(connection(), string(), binary() | first, binary() | last, config()) -> ok | {error, term()}. +truncate(Ref, Name) -> + truncate(Ref, Name, first, last, []). +truncate(Ref, Name, Config) -> + truncate(Ref, Name, first, last, Config). +truncate(Ref, Name, Start, Stop) -> + truncate(Ref, Name, Start, Stop, []). +truncate(Ref, Name, Start, Stop, Config) -> + ?ASYNC_NIF_CALL(fun truncate_nif/6, [Ref, Name, Start, Stop, config_to_bin(Config)]). + +-spec truncate_nif(reference(), connection(), string(), cursor() | first, cursor() | last, config()) -> ok | {error, term()}. +truncate_nif(_AsyncRef, _Ref, _Name, _Start, _Stop, _Config) -> ?nif_stub. --spec session_checkpoint(session()) -> ok | {error, term()}. --spec session_checkpoint(session(), config()) -> ok | {error, term()}. -session_checkpoint(_Ref) -> - session_checkpoint(_Ref, ?EMPTY_CONFIG). -session_checkpoint(_Ref, _Config) -> +-spec upgrade(connection(), string()) -> ok | {error, term()}. +-spec upgrade(connection(), string(), config_list()) -> ok | {error, term()}. +upgrade(Ref, Name) -> + upgrade(Ref, Name, []). +upgrade(Ref, Name, Config) -> + ?ASYNC_NIF_CALL(fun upgrade_nif/4, [Ref, Name, config_to_bin(Config)]). + +-spec upgrade_nif(reference(), connection(), string(), config()) -> ok | {error, term()}. +upgrade_nif(_AsyncRef, _Ref, _Name, _Config) -> ?nif_stub. --spec session_truncate(session(), string()) -> ok | {error, term()}. --spec session_truncate(session(), string(), config()) -> ok | {error, term()}. -session_truncate(Ref, Name) -> - session_truncate(Ref, Name, ?EMPTY_CONFIG). -session_truncate(_Ref, _Name, _Config) -> +-spec verify(connection(), string()) -> ok | {error, term()}. +-spec verify(connection(), string(), config_list()) -> ok | {error, term()}. +verify(Ref, Name) -> + verify(Ref, Name, []). +verify(Ref, Name, Config) -> + ?ASYNC_NIF_CALL(fun verify_nif/4, [Ref, Name, config_to_bin(Config)]). + +-spec verify_nif(reference(), connection(), string(), config()) -> ok | {error, term()}. +verify_nif(_AsyncRef, _Ref, _Name, _Config) -> ?nif_stub. --spec session_upgrade(session(), string()) -> ok | {error, term()}. --spec session_upgrade(session(), string(), config()) -> ok | {error, term()}. -session_upgrade(Ref, Name) -> - session_upgrade(Ref, Name, ?EMPTY_CONFIG). -session_upgrade(_Ref, _Name, _Config) -> - ?nif_stub. +-spec cursor_open(connection(), string()) -> {ok, cursor()} | {error, term()}. +-spec cursor_open(connection(), string(), config_list()) -> {ok, cursor()} | {error, term()}. +cursor_open(Ref, Table) -> + cursor_open(Ref, Table, []). +cursor_open(Ref, Table, Config) -> + ?ASYNC_NIF_CALL(fun cursor_open_nif/4, [Ref, Table, config_to_bin(Config)]). --spec session_verify(session(), string()) -> ok | {error, term()}. --spec session_verify(session(), string(), config()) -> ok | {error, term()}. -session_verify(Ref, Name) -> - session_verify(Ref, Name, ?EMPTY_CONFIG). -session_verify(_Ref, _Name, _Config) -> - ?nif_stub. - --spec cursor_open(session(), string()) -> {ok, cursor()} | {error, term()}. -cursor_open(_Ref, _Table) -> +-spec cursor_open_nif(reference(), connection(), string(), config()) -> {ok, cursor()} | {error, term()}. +cursor_open_nif(_AsyncRef, _Ref, _Table, _Config) -> ?nif_stub. -spec cursor_close(cursor()) -> ok | {error, term()}. -cursor_close(_Cursor) -> +cursor_close(Cursor) -> + ?ASYNC_NIF_CALL(fun cursor_close_nif/2, [Cursor]). + +-spec cursor_close_nif(reference(), cursor()) -> ok | {error, term()}. +cursor_close_nif(_AsyncRef, _Cursor) -> ?nif_stub. -spec cursor_next(cursor()) -> {ok, key(), value()} | not_found | {error, term()}. -cursor_next(_Cursor) -> +cursor_next(Cursor) -> + ?ASYNC_NIF_CALL(fun cursor_next_nif/2, [Cursor]). + +-spec cursor_next_nif(reference(), cursor()) -> {ok, key(), value()} | not_found | {error, term()}. +cursor_next_nif(_AsyncRef, _Cursor) -> ?nif_stub. -spec cursor_next_key(cursor()) -> {ok, key()} | not_found | {error, term()}. -cursor_next_key(_Cursor) -> +cursor_next_key(Cursor) -> + ?ASYNC_NIF_CALL(fun cursor_next_key_nif/2, [Cursor]). + +-spec cursor_next_key_nif(reference(), cursor()) -> {ok, key()} | not_found | {error, term()}. +cursor_next_key_nif(_AsyncRef, _Cursor) -> ?nif_stub. -spec cursor_next_value(cursor()) -> {ok, value()} | not_found | {error, term()}. -cursor_next_value(_Cursor) -> +cursor_next_value(Cursor) -> + ?ASYNC_NIF_CALL(fun cursor_next_value_nif/2, [Cursor]). + +-spec cursor_next_value_nif(reference(), cursor()) -> {ok, value()} | not_found | {error, term()}. +cursor_next_value_nif(_AsyncRef, _Cursor) -> ?nif_stub. -spec cursor_prev(cursor()) -> {ok, key(), value()} | not_found | {error, term()}. -cursor_prev(_Cursor) -> +cursor_prev(Cursor) -> + ?ASYNC_NIF_CALL(fun cursor_prev_nif/2, [Cursor]). + +-spec cursor_prev_nif(reference(), cursor()) -> {ok, key(), value()} | not_found | {error, term()}. +cursor_prev_nif(_AsyncRef, _Cursor) -> ?nif_stub. -spec cursor_prev_key(cursor()) -> {ok, key()} | not_found | {error, term()}. -cursor_prev_key(_Cursor) -> +cursor_prev_key(Cursor) -> + ?ASYNC_NIF_CALL(fun cursor_prev_key_nif/2, [Cursor]). + +-spec cursor_prev_key_nif(reference(), cursor()) -> {ok, key()} | not_found | {error, term()}. +cursor_prev_key_nif(_AsyncRef, _Cursor) -> ?nif_stub. -spec cursor_prev_value(cursor()) -> {ok, value()} | not_found | {error, term()}. -cursor_prev_value(_Cursor) -> +cursor_prev_value(Cursor) -> + ?ASYNC_NIF_CALL(fun cursor_prev_value_nif/2, [Cursor]). + +-spec cursor_prev_value_nif(reference(), cursor()) -> {ok, value()} | not_found | {error, term()}. +cursor_prev_value_nif(_AsyncRef, _Cursor) -> ?nif_stub. -spec cursor_search(cursor(), key()) -> {ok, value()} | {error, term()}. -cursor_search(_Cursor, _Key) -> +-spec cursor_search(cursor(), key(), boolean()) -> {ok, value()} | {error, term()}. +cursor_search(Cursor, Key) -> + ?ASYNC_NIF_CALL(fun cursor_search_nif/4, [Cursor, Key, false]). +cursor_search(Cursor, Key, Scanning) when is_boolean(Scanning) -> + ?ASYNC_NIF_CALL(fun cursor_search_nif/4, [Cursor, Key, Scanning]). + +-spec cursor_search_nif(reference(), cursor(), key(), boolean()) -> {ok, value()} | {error, term()}. +cursor_search_nif(_AsyncRef, _Cursor, _Key, _Scanning) -> ?nif_stub. -spec cursor_search_near(cursor(), key()) -> {ok, value()} | {error, term()}. -cursor_search_near(_Cursor, _Key) -> +-spec cursor_search_near(cursor(), key(), boolean()) -> {ok, value()} | {error, term()}. +cursor_search_near(Cursor, Key) -> + ?ASYNC_NIF_CALL(fun cursor_search_near_nif/4, [Cursor, Key, false]). +cursor_search_near(Cursor, Key, Scanning) when is_boolean(Scanning) -> + ?ASYNC_NIF_CALL(fun cursor_search_near_nif/4, [Cursor, Key, Scanning]). + +-spec cursor_search_near_nif(reference(), cursor(), key(), boolean()) -> {ok, value()} | {error, term()}. +cursor_search_near_nif(_AsyncRef, _Cursor, _Key, _Scanning) -> ?nif_stub. -spec cursor_reset(cursor()) -> ok | {error, term()}. -cursor_reset(_Cursor) -> +cursor_reset(Cursor) -> + ?ASYNC_NIF_CALL(fun cursor_reset_nif/2, [Cursor]). + +-spec cursor_reset_nif(reference(), cursor()) -> ok | {error, term()}. +cursor_reset_nif(_AsyncRef, _Cursor) -> ?nif_stub. -spec cursor_insert(cursor(), key(), value()) -> ok | {error, term()}. -cursor_insert(_Cursor, _Key, _Value) -> +cursor_insert(Cursor, Key, Value) -> + ?ASYNC_NIF_CALL(fun cursor_insert_nif/4, [Cursor, Key, Value]). + +-spec cursor_insert_nif(reference(), cursor(), key(), value()) -> ok | {error, term()}. +cursor_insert_nif(_AsyncRef, _Cursor, _Key, _Value) -> ?nif_stub. -spec cursor_update(cursor(), key(), value()) -> ok | {error, term()}. -cursor_update(_Cursor, _Key, _Value) -> +cursor_update(Cursor, Key, Value) -> + ?ASYNC_NIF_CALL(fun cursor_update_nif/4, [Cursor, Key, Value]). + +-spec cursor_update_nif(reference(), cursor(), key(), value()) -> ok | {error, term()}. +cursor_update_nif(_AsyncRef, _Cursor, _Key, _Value) -> ?nif_stub. -spec cursor_remove(cursor(), key()) -> ok | {error, term()}. -cursor_remove(_Cursor, _Key) -> +cursor_remove(Cursor, Key) -> + ?ASYNC_NIF_CALL(fun cursor_remove_nif/3, [Cursor, Key]). + +-spec cursor_remove_nif(reference(), cursor(), key()) -> ok | {error, term()}. +cursor_remove_nif(_AsyncRef, _Cursor, _Key) -> ?nif_stub. -type fold_keys_fun() :: fun((Key::binary(), any()) -> any()). @@ -276,53 +394,17 @@ fold(Cursor, Fun, Acc, {ok, Key, Value}) -> priv_dir() -> case code:priv_dir(?MODULE) of - {error, bad_name} -> - EbinDir = filename:dirname(code:which(?MODULE)), - AppPath = filename:dirname(EbinDir), - filename:join(AppPath, "priv"); - Path -> - Path + {error, bad_name} -> + EbinDir = filename:dirname(code:which(?MODULE)), + AppPath = filename:dirname(EbinDir), + filename:join([AppPath, "priv"]); + Path -> + Path end. %% -%% Configuration type information. +%% Configuration information. %% -config_types() -> - [{block_compressor, {string, quoted}}, - {cache_size, string}, - {checkpoint, config}, - {create, bool}, - {direct_io, list}, - {drop, list}, - {error_prefix, string}, - {eviction_target, integer}, - {eviction_trigger, integer}, - {extensions, {list, quoted}}, - {force, bool}, - {hazard_max, integer}, - {home_environment, bool}, - {home_environment_priv, bool}, - {internal_page_max, string}, - {isolation, string}, - {key_type, string}, - {leaf_page_max, string}, - {logging, bool}, - {lsm_bloom_bit_count, integer}, - {lsm_bloom_config, config}, - {lsm_bloom_hash_count, integer}, - {lsm_bloom_newest, bool}, - {lsm_bloom_oldest, bool}, - {lsm_chunk_size, string}, - {lsm_merge_threads, integer}, - {multiprocess, bool}, - {name, string}, - {session_max, integer}, - {statistics_log, config}, - {sync, bool}, - {target, list}, - {transactional, bool}, - {verbose, list}, - {wait, integer}]. config_value(Key, Config, Default) -> {Key, app_helper:get_prop_or_env(Key, Config, wterl, Default)}. @@ -368,7 +450,48 @@ config_to_bin(Opts) -> config_to_bin([], Acc) -> iolist_to_binary(Acc); config_to_bin([{Key, Value} | Rest], Acc) -> - case lists:keysearch(Key, 1, config_types()) of + ConfigTypes = + [{block_compressor, {string, quoted}}, + {cache_size, string}, + {checkpoint, config}, + {create, bool}, + {direct_io, list}, + {drop, list}, + {error_prefix, string}, + {eviction_target, integer}, + {eviction_trigger, integer}, + {extensions, {list, quoted}}, + {statistics_fast, bool}, + {force, bool}, + {from, string}, + {hazard_max, integer}, + {home_environment, bool}, + {home_environment_priv, bool}, + {internal_page_max, string}, + {isolation, string}, + {key_type, string}, + {leaf_page_max, string}, + {logging, bool}, + {lsm_bloom_bit_count, integer}, + {lsm_bloom_config, config}, + {lsm_bloom_hash_count, integer}, + {lsm_bloom_newest, bool}, + {lsm_bloom_oldest, bool}, + {lsm_chunk_size, string}, + {prefix_compression, bool}, + {lsm_merge_threads, integer}, + {multiprocess, bool}, + {name, string}, + {session_max, integer}, + {statistics_log, config}, + {sync, bool}, + {target, {list, quoted}}, + {to, string}, + {transactional, bool}, + {verbose, list}, + {wait, integer}], + + case lists:keysearch(Key, 1, ConfigTypes) of {value, {Key, Type}} -> Acc2 = case config_encode(Type, Value) of invalid -> @@ -394,23 +517,25 @@ config_to_bin([{Key, Value} | Rest], Acc) -> -define(TEST_DATA_DIR, "test/wterl.basic"). open_test_conn(DataDir) -> - ?assertCmd("rm -rf "++DataDir), - ?assertMatch(ok, filelib:ensure_dir(filename:join(DataDir, "x"))), - OpenConfig = config_to_bin([{create,true}, {cache_size,"100MB"}]), - {ok, ConnRef} = connection_open(DataDir, OpenConfig), + open_test_conn(DataDir, [{create,true},{cache_size,"100MB"},{session_max, 8192}]). +open_test_conn(DataDir, OpenConfig) -> + {ok, CWD} = file:get_cwd(), + rmdir:path(filename:join([CWD, DataDir])), %?cmd("rm -rf " ++ filename:join([CWD, DataDir])), + ?assertMatch(ok, filelib:ensure_dir(filename:join([DataDir, "x"]))), + {ok, ConnRef} = connection_open(filename:join([CWD, DataDir]), OpenConfig), ConnRef. -open_test_session(ConnRef) -> - {ok, SRef} = session_open(ConnRef), - ?assertMatch(ok, session_drop(SRef, "table:test", config_to_bin([{force,true}]))), - ?assertMatch(ok, session_create(SRef, "table:test", config_to_bin([{block_compressor, "snappy"}]))), - SRef. +open_test_table(ConnRef) -> + open_test_table(ConnRef, "table", []). -conn_test() -> - ConnRef = open_test_conn(?TEST_DATA_DIR), - ?assertMatch(ok, connection_close(ConnRef)). +open_test_table(ConnRef, Type) -> + open_test_table(ConnRef, Type, []). -session_test_() -> +open_test_table(ConnRef, Type, Opts) -> + ?assertMatch(ok, create(ConnRef, Type ++ ":test", Opts)), + ConnRef. + +conn_test_() -> {setup, fun() -> open_test_conn(?TEST_DATA_DIR) @@ -420,169 +545,252 @@ session_test_() -> end, fun(ConnRef) -> {inorder, - [{"open/close a session", + [{"open and close a connection", fun() -> - {ok, SRef} = session_open(ConnRef), - ?assertMatch(ok, session_close(SRef)) + ConnRef = open_test_table(ConnRef) end}, - {"create and drop a table", + {"create, verify, drop a table(btree)", fun() -> - SRef = open_test_session(ConnRef), - ?assertMatch(ok, session_drop(SRef, "table:test")), - ?assertMatch(ok, session_close(SRef)) - end}]} + wterl:create(ConnRef, "table:test", []), + ?assertMatch(ok, verify(ConnRef, "table:test")), + ?assertMatch(ok, drop(ConnRef, "table:test")) + end}, + {"create, test verify, drop a table(lsm)", + fun() -> + ConnRef = open_test_table(ConnRef, "lsm"), + ?assertMatch(ok, verify(ConnRef, "lsm:test")), + ?assertMatch(ok, drop(ConnRef, "lsm:test")) + end}, + {"create, verify, drop a table(btree, snappy)", + fun() -> + ConnRef = open_test_table(ConnRef, "table", [{block_compressor, "snappy"}]), + ?assertMatch(ok, verify(ConnRef, "table:test")), + ?assertMatch(ok, drop(ConnRef, "table:test")) + end}, + {"create, verify, drop a table(btree, bzip2)", + fun() -> + ConnRef = open_test_table(ConnRef, "table", [{block_compressor, "bzip2"}]), + ?assertMatch(ok, verify(ConnRef, "table:test")), + ?assertMatch(ok, drop(ConnRef, "table:test")) + end} + ]} end}. insert_delete_test() -> ConnRef = open_test_conn(?TEST_DATA_DIR), - SRef = open_test_session(ConnRef), - ?assertMatch(ok, session_put(SRef, "table:test", <<"a">>, <<"apple">>)), - ?assertMatch({ok, <<"apple">>}, session_get(SRef, "table:test", <<"a">>)), - ?assertMatch(ok, session_delete(SRef, "table:test", <<"a">>)), - ?assertMatch(not_found, session_get(SRef, "table:test", <<"a">>)), - ok = session_close(SRef), + ConnRef = open_test_table(ConnRef), + ?assertMatch(ok, put(ConnRef, "table:test", <<"a">>, <<"apple">>)), + ?assertMatch({ok, <<"apple">>}, get(ConnRef, "table:test", <<"a">>)), + ?assertMatch(ok, delete(ConnRef, "table:test", <<"a">>)), + ?assertMatch(not_found, get(ConnRef, "table:test", <<"a">>)), ok = connection_close(ConnRef). init_test_table() -> ConnRef = open_test_conn(?TEST_DATA_DIR), - SRef = open_test_session(ConnRef), - ?assertMatch(ok, session_put(SRef, "table:test", <<"a">>, <<"apple">>)), - ?assertMatch(ok, session_put(SRef, "table:test", <<"b">>, <<"banana">>)), - ?assertMatch(ok, session_put(SRef, "table:test", <<"c">>, <<"cherry">>)), - ?assertMatch(ok, session_put(SRef, "table:test", <<"d">>, <<"date">>)), - ?assertMatch(ok, session_put(SRef, "table:test", <<"g">>, <<"gooseberry">>)), - {ConnRef, SRef}. + ConnRef = open_test_table(ConnRef), + populate_test_table(ConnRef). -stop_test_table({ConnRef, SRef}) -> - ?assertMatch(ok, session_close(SRef)), +populate_test_table(ConnRef) -> + ?assertMatch(ok, put(ConnRef, "table:test", <<"a">>, <<"apple">>)), + ?assertMatch(ok, put(ConnRef, "table:test", <<"b">>, <<"banana">>)), + ?assertMatch(ok, put(ConnRef, "table:test", <<"c">>, <<"cherry">>)), + ?assertMatch(ok, put(ConnRef, "table:test", <<"d">>, <<"date">>)), + ?assertMatch(ok, put(ConnRef, "table:test", <<"e">>, <<"elephant">>)), + ?assertMatch(ok, put(ConnRef, "table:test", <<"f">>, <<"forest">>)), + ?assertMatch(ok, put(ConnRef, "table:test", <<"g">>, <<"gooseberry">>)), + ConnRef. + +stop_test_table(ConnRef) -> ?assertMatch(ok, connection_close(ConnRef)). -various_session_test_() -> +various_online_test_() -> {setup, fun init_test_table/0, fun stop_test_table/1, - fun({_, SRef}) -> + fun(ConnRef) -> {inorder, - [{"session verify", + [ + {"checkpoint", fun() -> - ?assertMatch(ok, session_verify(SRef, "table:test")), - ?assertMatch({ok, <<"apple">>}, - session_get(SRef, "table:test", <<"a">>)) + ?assertMatch(ok, checkpoint(ConnRef, [{target, ["table:test"]}])), + ?assertMatch({ok, <<"apple">>}, get(ConnRef, "table:test", <<"a">>)) end}, - {"session checkpoint", + {"truncate entire table", fun() -> - Cfg = wterl:config_to_bin([{target, ["\"table:test\""]}]), - ?assertMatch(ok, session_checkpoint(SRef, Cfg)), - ?assertMatch({ok, <<"apple">>}, - session_get(SRef, "table:test", <<"a">>)) + ?assertMatch(ok, truncate(ConnRef, "table:test")), + ?assertMatch(not_found, get(ConnRef, "table:test", <<"a">>)) end}, - {"session salvage", + %% {"truncate range [<>..last], ensure value outside range is found after", + %% fun() -> + %% ?assertMatch(ok, truncate(ConnRef, "table:test", <<"b">>, last)), + %% ?assertMatch({ok, <<"apple">>}, get(ConnRef, "table:test", <<"a">>)) + %% end}, + %% {"truncate range [first..<>], ensure value inside range is not_found after", + %% fun() -> + %% ?assertMatch(ok, truncate(ConnRef, "table:test", first, <<"b">>)), + %% ?assertMatch(not_found, get(ConnRef, "table:test", <<"a">>)) + %% end}, + %% {"truncate range [first..not_found] with a key that doesn't exist", + %% fun() -> + %% ?assertMatch(not_found, truncate(ConnRef, "table:test", first, <<"z">>)) + %% end}, + %% {"truncate range [not_found..last] with a key that doesn't exist", + %% fun() -> + %% ?assertMatch(not_found, truncate(ConnRef, "table:test", <<"0">>, last)) + %% end}, + %% {"truncate range [not_found..not_found] with keys that don't exist", + %% fun() -> + %% ?assertMatch(not_found, truncate(ConnRef, "table:test", <<"0">>, <<"0">>)) + %% end}, + %% {"truncate range [<...<>], ensure value before & after range still exist", + %% fun() -> + %% ?assertMatch(ok, truncate(ConnRef, "table:test", <<"b">>, <<"f">>)), + %% ?assertMatch({ok, <<"apple">>}, get(ConnRef, "table:test", <<"a">>)), + %% ?assertMatch(not_found, get(ConnRef, "table:test", <<"b">>)), + %% ?assertMatch(not_found, get(ConnRef, "table:test", <<"c">>)), + %% ?assertMatch(not_found, get(ConnRef, "table:test", <<"d">>)), + %% ?assertMatch(not_found, get(ConnRef, "table:test", <<"e">>)), + %% ?assertMatch(not_found, get(ConnRef, "table:test", <<"f">>)), + %% ?assertMatch({ok, <<"gooseberry">>}, get(ConnRef, "table:test", <<"g">>)) + %% end}, + {"drop table", fun() -> - ok = session_salvage(SRef, "table:test"), - {ok, <<"apple">>} = session_get(SRef, "table:test", <<"a">>) + ?assertMatch(ok, drop(ConnRef, "table:test")) + end} + ]} + end}. + +various_maintenance_test_() -> + {setup, + fun () -> + {ok, CWD} = file:get_cwd(), + ?assertMatch(ok, filelib:ensure_dir(filename:join([?TEST_DATA_DIR, "x"]))), + {ok, ConnRef} = connection_open(filename:join([CWD, ?TEST_DATA_DIR]), []), + ConnRef + end, + fun (ConnRef) -> + ?assertMatch(ok, connection_close(ConnRef)) + end, + fun(ConnRef) -> + {inorder, + [ + {"drop table", + fun() -> + ?assertMatch(ok, create(ConnRef, "table:test")), + ?assertMatch(ok, drop(ConnRef, "table:test")), + ?assertMatch(ok, create(ConnRef, "table:test")) end}, - {"session upgrade", + {"salvage", fun() -> - ?assertMatch(ok, session_upgrade(SRef, "table:test")), - ?assertMatch({ok, <<"apple">>}, - session_get(SRef, "table:test", <<"a">>)) + ?assertMatch(ok, salvage(ConnRef, "table:test")) end}, - {"session rename", + {"upgrade", fun() -> - ?assertMatch(ok, - session_rename(SRef, "table:test", "table:new")), - ?assertMatch({ok, <<"apple">>}, - session_get(SRef, "table:new", <<"a">>)), - ?assertMatch(ok, - session_rename(SRef, "table:new", "table:test")), - ?assertMatch({ok, <<"apple">>}, - session_get(SRef, "table:test", <<"a">>)) + ?assertMatch(ok, upgrade(ConnRef, "table:test")) end}, - {"session truncate", + {"rename", fun() -> - ?assertMatch(ok, session_truncate(SRef, "table:test")), - ?assertMatch(not_found, session_get(SRef, "table:test", <<"a">>)) - end}]} + ?assertMatch(ok, rename(ConnRef, "table:test", "table:new")), + ?assertMatch(ok, rename(ConnRef, "table:new", "table:test")) + end} + ]} end}. cursor_open_close_test() -> - {ConnRef, SRef} = init_test_table(), - {ok, Cursor1} = cursor_open(SRef, "table:test"), + ConnRef = init_test_table(), + {ok, Cursor1} = cursor_open(ConnRef, "table:test"), ?assertMatch({ok, <<"a">>, <<"apple">>}, cursor_next(Cursor1)), ?assertMatch(ok, cursor_close(Cursor1)), - {ok, Cursor2} = cursor_open(SRef, "table:test"), + {ok, Cursor2} = cursor_open(ConnRef, "table:test"), ?assertMatch({ok, <<"g">>, <<"gooseberry">>}, cursor_prev(Cursor2)), ?assertMatch(ok, cursor_close(Cursor2)), - stop_test_table({ConnRef, SRef}). + stop_test_table(ConnRef). various_cursor_test_() -> {setup, fun init_test_table/0, fun stop_test_table/1, - fun({_, SRef}) -> + fun(ConnRef) -> {inorder, [{"move a cursor back and forth, getting key", fun() -> - {ok, Cursor} = cursor_open(SRef, "table:test"), + {ok, Cursor} = cursor_open(ConnRef, "table:test"), ?assertMatch({ok, <<"a">>}, cursor_next_key(Cursor)), ?assertMatch({ok, <<"b">>}, cursor_next_key(Cursor)), ?assertMatch({ok, <<"c">>}, cursor_next_key(Cursor)), ?assertMatch({ok, <<"d">>}, cursor_next_key(Cursor)), ?assertMatch({ok, <<"c">>}, cursor_prev_key(Cursor)), ?assertMatch({ok, <<"d">>}, cursor_next_key(Cursor)), + ?assertMatch({ok, <<"e">>}, cursor_next_key(Cursor)), + ?assertMatch({ok, <<"f">>}, cursor_next_key(Cursor)), ?assertMatch({ok, <<"g">>}, cursor_next_key(Cursor)), ?assertMatch(not_found, cursor_next_key(Cursor)), ?assertMatch(ok, cursor_close(Cursor)) end}, {"move a cursor back and forth, getting value", fun() -> - {ok, Cursor} = cursor_open(SRef, "table:test"), + {ok, Cursor} = cursor_open(ConnRef, "table:test"), ?assertMatch({ok, <<"apple">>}, cursor_next_value(Cursor)), ?assertMatch({ok, <<"banana">>}, cursor_next_value(Cursor)), ?assertMatch({ok, <<"cherry">>}, cursor_next_value(Cursor)), ?assertMatch({ok, <<"date">>}, cursor_next_value(Cursor)), ?assertMatch({ok, <<"cherry">>}, cursor_prev_value(Cursor)), ?assertMatch({ok, <<"date">>}, cursor_next_value(Cursor)), + ?assertMatch({ok, <<"elephant">>}, cursor_next_value(Cursor)), + ?assertMatch({ok, <<"forest">>}, cursor_next_value(Cursor)), ?assertMatch({ok, <<"gooseberry">>}, cursor_next_value(Cursor)), ?assertMatch(not_found, cursor_next_value(Cursor)), ?assertMatch(ok, cursor_close(Cursor)) end}, {"move a cursor back and forth, getting key and value", fun() -> - {ok, Cursor} = cursor_open(SRef, "table:test"), + {ok, Cursor} = cursor_open(ConnRef, "table:test"), ?assertMatch({ok, <<"a">>, <<"apple">>}, cursor_next(Cursor)), ?assertMatch({ok, <<"b">>, <<"banana">>}, cursor_next(Cursor)), ?assertMatch({ok, <<"c">>, <<"cherry">>}, cursor_next(Cursor)), ?assertMatch({ok, <<"d">>, <<"date">>}, cursor_next(Cursor)), ?assertMatch({ok, <<"c">>, <<"cherry">>}, cursor_prev(Cursor)), ?assertMatch({ok, <<"d">>, <<"date">>}, cursor_next(Cursor)), + ?assertMatch({ok, <<"e">>, <<"elephant">>}, cursor_next(Cursor)), + ?assertMatch({ok, <<"f">>, <<"forest">>}, cursor_next(Cursor)), ?assertMatch({ok, <<"g">>, <<"gooseberry">>}, cursor_next(Cursor)), ?assertMatch(not_found, cursor_next(Cursor)), ?assertMatch(ok, cursor_close(Cursor)) end}, {"fold keys", fun() -> - {ok, Cursor} = cursor_open(SRef, "table:test"), - ?assertMatch([<<"g">>, <<"d">>, <<"c">>, <<"b">>, <<"a">>], + {ok, Cursor} = cursor_open(ConnRef, "table:test"), + ?assertMatch([<<"g">>, <<"f">>, <<"e">>, <<"d">>, <<"c">>, <<"b">>, <<"a">>], fold_keys(Cursor, fun(Key, Acc) -> [Key | Acc] end, [])), ?assertMatch(ok, cursor_close(Cursor)) end}, {"search for an item", fun() -> - {ok, Cursor} = cursor_open(SRef, "table:test"), + {ok, Cursor} = cursor_open(ConnRef, "table:test"), ?assertMatch({ok, <<"banana">>}, cursor_search(Cursor, <<"b">>)), ?assertMatch(ok, cursor_close(Cursor)) end}, - {"range search for an item", + {"proxmity search for an item, and find it", fun() -> - {ok, Cursor} = cursor_open(SRef, "table:test"), - ?assertMatch({ok, <<"gooseberry">>}, - cursor_search_near(Cursor, <<"z">>)), + {ok, Cursor} = cursor_open(ConnRef, "table:test"), + ?assertMatch({ok, match}, cursor_search_near(Cursor, <<"e">>)), + ?assertMatch(ok, cursor_close(Cursor)) + end}, + {"proxmity search for an item, find next smallest key", + fun() -> + {ok, Cursor} = cursor_open(ConnRef, "table:test"), + ?assertMatch({ok, lt}, cursor_search_near(Cursor, <<"z">>)), + ?assertMatch(ok, cursor_close(Cursor)) + end}, + {"proxmity search for an item, find next largest key", + fun() -> + {ok, Cursor} = cursor_open(ConnRef, "table:test"), + ?assertMatch({ok, gt}, cursor_search_near(Cursor, <<"0">>)), ?assertMatch(ok, cursor_close(Cursor)) end}, {"check cursor reset", fun() -> - {ok, Cursor} = cursor_open(SRef, "table:test"), + {ok, Cursor} = cursor_open(ConnRef, "table:test"), ?assertMatch({ok, <<"apple">>}, cursor_next_value(Cursor)), ?assertMatch(ok, cursor_reset(Cursor)), ?assertMatch({ok, <<"apple">>}, cursor_next_value(Cursor)), @@ -590,39 +798,30 @@ various_cursor_test_() -> end}, {"insert/overwrite an item using a cursor", fun() -> - {ok, Cursor} = cursor_open(SRef, "table:test"), - ?assertMatch(ok, - cursor_insert(Cursor, <<"h">>, <<"huckleberry">>)), - ?assertMatch({ok, <<"huckleberry">>}, - cursor_search(Cursor, <<"h">>)), - ?assertMatch(ok, - cursor_insert(Cursor, <<"g">>, <<"grapefruit">>)), - ?assertMatch({ok, <<"grapefruit">>}, - cursor_search(Cursor, <<"g">>)), + {ok, Cursor} = cursor_open(ConnRef, "table:test"), + ?assertMatch(ok, cursor_insert(Cursor, <<"h">>, <<"huckleberry">>)), + ?assertMatch({ok, <<"huckleberry">>}, cursor_search(Cursor, <<"h">>)), + ?assertMatch(ok, cursor_insert(Cursor, <<"g">>, <<"grapefruit">>)), + ?assertMatch({ok, <<"grapefruit">>}, cursor_search(Cursor, <<"g">>)), ?assertMatch(ok, cursor_close(Cursor)), - ?assertMatch({ok, <<"grapefruit">>}, - session_get(SRef, "table:test", <<"g">>)), - ?assertMatch({ok, <<"huckleberry">>}, - session_get(SRef, "table:test", <<"h">>)) + ?assertMatch({ok, <<"grapefruit">>}, get(ConnRef, "table:test", <<"g">>)), + ?assertMatch({ok, <<"huckleberry">>}, get(ConnRef, "table:test", <<"h">>)) end}, {"update an item using a cursor", fun() -> - {ok, Cursor} = cursor_open(SRef, "table:test"), - ?assertMatch(ok, - cursor_update(Cursor, <<"g">>, <<"goji berries">>)), - ?assertMatch(not_found, - cursor_update(Cursor, <<"k">>, <<"kumquat">>)), + {ok, Cursor} = cursor_open(ConnRef, "table:test"), + ?assertMatch(ok, cursor_update(Cursor, <<"g">>, <<"goji berries">>)), + ?assertMatch(not_found, cursor_update(Cursor, <<"k">>, <<"kumquat">>)), ?assertMatch(ok, cursor_close(Cursor)), - ?assertMatch({ok, <<"goji berries">>}, - session_get(SRef, "table:test", <<"g">>)) + ?assertMatch({ok, <<"goji berries">>}, get(ConnRef, "table:test", <<"g">>)) end}, {"remove an item using a cursor", fun() -> - {ok, Cursor} = cursor_open(SRef, "table:test"), + {ok, Cursor} = cursor_open(ConnRef, "table:test"), ?assertMatch(ok, cursor_remove(Cursor, <<"g">>)), ?assertMatch(not_found, cursor_remove(Cursor, <<"l">>)), ?assertMatch(ok, cursor_close(Cursor)), - ?assertMatch(not_found, session_get(SRef, "table:test", <<"g">>)) + ?assertMatch(not_found, get(ConnRef, "table:test", <<"g">>)) end}]} end}. @@ -640,13 +839,13 @@ values() -> ops(Keys, Values) -> {oneof([put, delete]), oneof(Keys), oneof(Values)}. -apply_kv_ops([], _SRef, _Tbl, Acc0) -> +apply_kv_ops([], _ConnRef, _Tbl, Acc0) -> Acc0; -apply_kv_ops([{put, K, V} | Rest], SRef, Tbl, Acc0) -> - ok = wterl:session_put(SRef, Tbl, K, V), - apply_kv_ops(Rest, SRef, Tbl, orddict:store(K, V, Acc0)); -apply_kv_ops([{delete, K, _} | Rest], SRef, Tbl, Acc0) -> - ok = case wterl:session_delete(SRef, Tbl, K) of +apply_kv_ops([{put, K, V} | Rest], ConnRef, Tbl, Acc0) -> + ok = wterl:put(ConnRef, Tbl, K, V), + apply_kv_ops(Rest, ConnRef, Tbl, orddict:store(K, V, Acc0)); +apply_kv_ops([{delete, K, _} | Rest], ConnRef, Tbl, Acc0) -> + ok = case wterl:delete(ConnRef, Tbl, K) of ok -> ok; not_found -> @@ -654,34 +853,32 @@ apply_kv_ops([{delete, K, _} | Rest], SRef, Tbl, Acc0) -> Else -> Else end, - apply_kv_ops(Rest, SRef, Tbl, orddict:store(K, deleted, Acc0)). + apply_kv_ops(Rest, ConnRef, Tbl, orddict:store(K, deleted, Acc0)). prop_put_delete() -> ?LET({Keys, Values}, {keys(), values()}, ?FORALL(Ops, eqc_gen:non_empty(list(ops(Keys, Values))), begin - DataDir = "/tmp/wterl.putdelete.qc", + DataDir = "test/wterl.putdelete.qc", Table = "table:eqc", + {ok, CWD} = file:get_cwd(), ?cmd("rm -rf "++DataDir), - ok = filelib:ensure_dir(filename:join(DataDir, "x")), - Cfg = wterl:config_to_bin([{create,true}]), - {ok, Conn} = wterl:connection_open(DataDir, Cfg), - {ok, SRef} = wterl:session_open(Conn), + ok = filelib:ensure_dir(filename:join([DataDir, "x"])), + {ok, ConnRef} = wterl:connection_open(DataDir, [{create,true}]), try - wterl:session_create(SRef, Table), - Model = apply_kv_ops(Ops, SRef, Table, []), + wterl:create(ConnRef, Table), + Model = apply_kv_ops(Ops, ConnRef, Table, []), %% Validate that all deleted values return not_found F = fun({K, deleted}) -> - ?assertEqual(not_found, wterl:session_get(SRef, Table, K)); + ?assertEqual(not_found, wterl:get(ConnRef, Table, K)); ({K, V}) -> - ?assertEqual({ok, V}, wterl:session_get(SRef, Table, K)) + ?assertEqual({ok, V}, wterl:get(ConnRef, Table, K)) end, lists:map(F, Model), true after - wterl:session_close(SRef), - wterl:connection_close(Conn) + wterl:connection_close(ConnRef) end end)). diff --git a/src/wterl_conn.erl b/src/wterl_conn.erl index 4a680d0..b7ead8b 100644 --- a/src/wterl_conn.erl +++ b/src/wterl_conn.erl @@ -30,7 +30,7 @@ %% API -export([start_link/0, stop/0, - open/1, open/2, is_open/0, get/0, close/1]). + open/1, open/2, open/3, is_open/0, get/0, close/1]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -53,12 +53,14 @@ stop() -> gen_server:cast(?MODULE, stop). -spec open(string()) -> {ok, wterl:connection()} | {error, term()}. -open(Dir) -> - open(Dir, []). - -spec open(string(), config_list()) -> {ok, wterl:connection()} | {error, term()}. -open(Dir, Config) -> - gen_server:call(?MODULE, {open, Dir, Config, self()}, infinity). +-spec open(string(), config_list(), config_list()) -> {ok, wterl:connection()} | {error, term()}. +open(Dir) -> + open(Dir, [], []). +open(Dir, ConnectionConfig) -> + gen_server:call(?MODULE, {open, Dir, ConnectionConfig, [], self()}, infinity). +open(Dir, ConnectionConfig, SessionConfig) -> + gen_server:call(?MODULE, {open, Dir, ConnectionConfig, SessionConfig, self()}, infinity). -spec is_open() -> boolean(). is_open() -> @@ -80,9 +82,9 @@ init([]) -> true = wterl_ets:table_ready(), {ok, #state{}}. -handle_call({open, Dir, Config, Caller}, _From, #state{conn=undefined}=State) -> +handle_call({open, Dir, ConnectionConfig, SessionConfig, Caller}, _From, #state{conn=undefined}=State) -> {Reply, NState} = - case wterl:connection_open(Dir, wterl:config_to_bin(Config)) of + case wterl:connection_open(Dir, ConnectionConfig, SessionConfig) of {ok, ConnRef}=OK -> Monitor = erlang:monitor(process, Caller), true = ets:insert(wterl_ets, {Monitor, Caller}), @@ -91,7 +93,7 @@ handle_call({open, Dir, Config, Caller}, _From, #state{conn=undefined}=State) -> {Error, State} end, {reply, Reply, NState}; -handle_call({open, _Dir, _Config, Caller}, _From,#state{conn=ConnRef}=State) -> +handle_call({open, _Dir, _ConnectionConfig, _SessionConfig, Caller}, _From, #state{conn=ConnRef}=State) -> Monitor = erlang:monitor(process, Caller), true = ets:insert(wterl_ets, {Monitor, Caller}), {reply, {ok, ConnRef}, State}; diff --git a/update-version.sh b/update-version.sh new file mode 100755 index 0000000..459fc0c --- /dev/null +++ b/update-version.sh @@ -0,0 +1,11 @@ +#!/bin/sh - + +# Note: also, remember to update version numbers in rpath specs so that shared libs can be found at runtime!!! + +wterl=`git log -n 1 --pretty=format:"%H"` +wiredtiger0=`(cd c_src/wiredtiger && git log -n 1 --pretty=format:"%H")` +wiredtiger=`echo $wiredtiger0 | awk '{print $2}'` + +echo $wterl +echo $wiredtiger +