diff --git a/.gdbinit b/.gdbinit new file mode 100644 index 0000000..f4ddf19 --- /dev/null +++ b/.gdbinit @@ -0,0 +1 @@ +handle SIGPIPE nostop noprint pass diff --git a/Makefile b/Makefile index c68cfbb..9e61fc4 100644 --- a/Makefile +++ b/Makefile @@ -40,22 +40,16 @@ eunit: compile @$(REBAR) eunit skip_deps=true eunit_console: - @$(ERL) -pa .eunit deps/*/ebin + @$(ERL) -pa .eunit deps/lager/ebin plt: compile - @$(DIALYZER) --build_plt --output_plt .$(TARGET).plt -pa deps/*/ebin --apps kernel stdlib + @$(DIALYZER) --build_plt --output_plt .$(TARGET).plt -pa deps/lager/ebin --apps kernel stdlib analyze: compile - $(DIALYZER) --plt .$(TARGET).plt -pa deps/*/ebin ebin + $(DIALYZER) --plt .$(TARGET).plt -pa deps/lager/ebin ebin repl: - $(ERL) -pz deps/*/ebin -pa ebin - -gdb-repl: - USE_GDB=1 $(ERL) -pz deps/*/ebin -pa ebin + $(ERL) -pz deps/lager/ebin -pa ebin eunit-repl: - $(ERL) -pa .eunit -pz deps/*/ebin -pz ebin -exec 'cd(".eunit").' - -gdb-eunit-repl: - USE_GDB=1 $(ERL) -pa .eunit -pz deps/*/ebin -pz ebin -exec 'cd(".eunit").' + $(ERL) -pz deps/lager/ebin -pa ebin -pa .eunit diff --git a/c_src/async_nif.h b/c_src/async_nif.h index eb4715a..9d63c52 100644 --- a/c_src/async_nif.h +++ b/c_src/async_nif.h @@ -278,10 +278,9 @@ async_nif_worker_fn(void *arg) unsigned int worker_id = we->worker_id; struct async_nif_state *async_nif = we->async_nif; struct async_nif_work_queue *q = we->q; + struct async_nif_req_entry *req = NULL; for(;;) { - struct async_nif_req_entry *req = NULL; - /* Examine the request queue, are there things to be done? */ enif_mutex_lock(q->reqs_mutex); check_again_for_work: @@ -295,34 +294,22 @@ async_nif_worker_fn(void *arg) goto check_again_for_work; } else { /* At this point the next req is ours to process and we hold the - reqs_mutex lock. */ + reqs_mutex lock. Take the request off the queue. */ + req = fifo_q_get(reqs, q->reqs); + enif_mutex_unlock(q->reqs_mutex); - do { - /* Take the request off the queue. */ - req = fifo_q_get(reqs, q->reqs); - enif_mutex_unlock(q->reqs_mutex); + /* Wake up another thread working on this queue. */ + enif_cond_signal(q->reqs_cnd); - /* Wake up another thread working on this queue. */ - enif_cond_signal(q->reqs_cnd); - - /* Finally, do the work. */ - req->fn_work(req->env, req->ref, &req->pid, worker_id, req->args); - req->fn_post(req->args); - /* Note: we don't call enif_free_env(req->env) because it has called - enif_send() which invalidates it (free'ing it for us). If a work - block doesn't call ASYNC_NIF_REPLY() at some point then it must - call ASYNC_NIF_NOREPLY() to free this env. */ - enif_free(req->args); - enif_free(req); - - /* Continue working if more requests are in the queue, otherwise wait - for new work to arrive. */ - if (fifo_q_empty(reqs, q->reqs)) - req = NULL; - else - enif_mutex_lock(q->reqs_mutex); - - } while(req); + /* Finally, do the work. */ + req->fn_work(req->env, req->ref, &req->pid, worker_id, req->args); + req->fn_post(req->args); + /* Note: we don't call enif_free_env(req->env) because it has called + enif_send() which invalidates it (free'ing it for us). If a work + block doesn't call ASYNC_NIF_REPLY() at some point then it must + call ASYNC_NIF_NOREPLY() to free this env. */ + enif_free(req->args); + enif_free(req); } } enif_thread_exit(0); diff --git a/c_src/build_deps.sh b/c_src/build_deps.sh index 601e917..c07f10b 100755 --- a/c_src/build_deps.sh +++ b/c_src/build_deps.sh @@ -54,11 +54,10 @@ get_wt () ./autogen.sh || exit 1 cd ./build_posix || exit 1 [ -e Makefile ] && $MAKE distclean - LDFLAGS="-L$BASEDIR/system/lib" CFLAGS="-I$BASEDIR/system/include" \ - ../configure --with-pic \ - --enable-snappy \ - --enable-bzip2 \ - --prefix=${BASEDIR}/system || exit 1 + ../configure --with-pic \ + --enable-snappy \ + --enable-bzip2 \ + --prefix=${BASEDIR}/system || exit 1 ) } diff --git a/c_src/wiredtiger-build.patch b/c_src/wiredtiger-build.patch index f689da0..54ab0ea 100644 --- a/c_src/wiredtiger-build.patch +++ b/c_src/wiredtiger-build.patch @@ -1,24 +1,24 @@ diff --git a/ext/compressors/bzip2/Makefile.am b/ext/compressors/bzip2/Makefile.am -index 0aedc2e..2137b8c 100644 +index 0aedc2e..a70ae2e 100644 --- a/ext/compressors/bzip2/Makefile.am +++ b/ext/compressors/bzip2/Makefile.am -@@ -2,5 +2,5 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include +@@ -2,5 +2,6 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include lib_LTLIBRARIES = libwiredtiger_bzip2.la libwiredtiger_bzip2_la_SOURCES = bzip2_compress.c -libwiredtiger_bzip2_la_LDFLAGS = -avoid-version -module --libwiredtiger_bzip2_la_LIBADD = -lbz2 -+libwiredtiger_bzip2_la_LDFLAGS = -avoid-version -module -Wl,-rpath,.:lib/wterl-0.9.0/priv:lib/wterl/priv:priv -+libwiredtiger_bzip2_la_LIBADD = -lbz2 ++libwiredtiger_snappy_la_CFLAGS = -I$(src_builddir)/../../system/include ++libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(src_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv + libwiredtiger_bzip2_la_LIBADD = -lbz2 diff --git a/ext/compressors/snappy/Makefile.am b/ext/compressors/snappy/Makefile.am -index 6d78823..2f2567e 100644 +index 6d78823..2122cf8 100644 --- a/ext/compressors/snappy/Makefile.am +++ b/ext/compressors/snappy/Makefile.am -@@ -2,5 +2,5 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include +@@ -2,5 +2,6 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include lib_LTLIBRARIES = libwiredtiger_snappy.la libwiredtiger_snappy_la_SOURCES = snappy_compress.c -libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module --libwiredtiger_snappy_la_LIBADD = -lsnappy -+libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -Wl,-rpath,.:lib/wterl-0.9.0/priv:lib/wterl/priv:priv -+libwiredtiger_snappy_la_LIBADD = -lsnappy ++libwiredtiger_snappy_la_CFLAGS = -I$(src_builddir)/../../system/include ++libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(src_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv + libwiredtiger_snappy_la_LIBADD = -lsnappy diff --git a/src/riak_kv_wterl_backend.erl b/src/riak_kv_wterl_backend.erl index 4c345c0..6322882 100644 --- a/src/riak_kv_wterl_backend.erl +++ b/src/riak_kv_wterl_backend.erl @@ -94,25 +94,43 @@ start(Partition, Config) -> end, case AppStart of ok -> - {type, Type} = wterl:config_value(type, Config, "lsm"), + Type = + case wterl:config_value(type, Config, "lsm") of + {type, "lsm"} -> "lsm"; + {type, "table"} -> "table"; + {type, "btree"} -> "table"; + {type, BadType} -> + lager:info("wterl:start ignoring unknown type ~p, using lsm instead", [BadType]), + "lsm"; + _ -> + lager:info("wterl:start ignoring mistaken setting defaulting to lsm"), + "lsm" + end, {ok, Connection} = establish_connection(Config, Type), Table = Type ++ ":wt" ++ integer_to_list(Partition), + Compressor = + case wterl:config_value(block_compressor, Config, "snappy") of + {block_compressor, "snappy"}=C -> [C]; + {block_compressor, "bzip2"}=C -> [C]; + {block_compressor, "none"} -> []; + {block_compressor, none} -> []; + {block_compressor, _} -> [{block_compressor, "snappy"}]; + _ -> [{block_compressor, "snappy"}] + end, TableOpts = case Type of "lsm" -> - [{block_compressor, "snappy"}, - {internal_page_max, "128K"}, + [{internal_page_max, "128K"}, {leaf_page_max, "128K"}, {lsm_chunk_size, "25MB"}, {lsm_bloom_newest, true}, {lsm_bloom_oldest, true} , {lsm_bloom_bit_count, 128}, {lsm_bloom_hash_count, 64}, - {lsm_bloom_config, [{leaf_page_max, "8MB"}]} ]; + {lsm_bloom_config, [{leaf_page_max, "8MB"}]} + ] ++ Compressor; "table" -> - [{block_compressor, "snappy"}]; - _ -> - [] + Compressor end, case wterl:create(Connection, Table, TableOpts) of ok -> @@ -561,14 +579,12 @@ size_cache(RequestedSize) -> simple_test_() -> {ok, CWD} = file:get_cwd(), - ?assertMatch(true, lists:suffix("wterl/.eunit", CWD)), rmdir:path(filename:join([CWD, "test/wterl-backend"])), %?assertCmd("rm -rf test/wterl-backend"), application:set_env(wterl, data_root, "test/wterl-backend"), temp_riak_kv_backend:standard_test(?MODULE, []). custom_config_test_() -> {ok, CWD} = file:get_cwd(), - ?assertMatch(true, lists:suffix("wterl/.eunit", CWD)), rmdir:path(filename:join([CWD, "test/wterl-backend"])), %?assertCmd("rm -rf test/wterl-backend"), application:set_env(wterl, data_root, ""), temp_riak_kv_backend:standard_test(?MODULE, [{data_root, "test/wterl-backend"}]). diff --git a/src/wterl.erl b/src/wterl.erl index 0c28d1a..10e51dd 100644 --- a/src/wterl.erl +++ b/src/wterl.erl @@ -518,7 +518,6 @@ open_test_conn(DataDir) -> open_test_conn(DataDir, [{create,true},{cache_size,"100MB"}]). open_test_conn(DataDir, OpenConfig) -> {ok, CWD} = file:get_cwd(), - ?assertMatch(true, lists:suffix("wterl/.eunit", CWD)), rmdir:path(filename:join([CWD, DataDir])), %?cmd("rm -rf " ++ filename:join([CWD, DataDir])), ?assertMatch(ok, filelib:ensure_dir(filename:join([DataDir, "x"]))), {ok, ConnRef} = connection_open(filename:join([CWD, DataDir]), OpenConfig), @@ -660,7 +659,6 @@ various_maintenance_test_() -> {setup, fun () -> {ok, CWD} = file:get_cwd(), - ?assertMatch(true, lists:suffix("wterl/.eunit", CWD)), ?assertMatch(ok, filelib:ensure_dir(filename:join([?TEST_DATA_DIR, "x"]))), {ok, ConnRef} = connection_open(filename:join([CWD, ?TEST_DATA_DIR]), []), ConnRef @@ -859,7 +857,6 @@ prop_put_delete() -> DataDir = "test/wterl.putdelete.qc", Table = "table:eqc", {ok, CWD} = file:get_cwd(), - ?assertMatch(true, lists:suffix("wterl/.eunit", CWD)), ?cmd("rm -rf "++DataDir), ok = filelib:ensure_dir(filename:join([DataDir, "x"])), {ok, ConnRef} = wterl:connection_open(DataDir, [{create,true}]),