greg/wterl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Simplified the worker look function. Added ability to pick block

Browse source 
compressor in config, default is snappy, off is {block_compressor, none}.
This commit is contained in:
Gregory Burd 2013-04-17 13:19:06 -04:00
parent 87f70d75a1
commit 123dfa600e
7 changed files with 60 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gdbinit Normal file
View file

@ -0,0 +1 @@
handle SIGPIPE nostop noprint pass

16
Makefile
View file

@ -40,22 +40,16 @@ eunit: compile
@$(REBAR) eunit skip_deps=true
eunit_console:
@$(ERL) -pa .eunit deps/*/ebin
@$(ERL) -pa .eunit deps/lager/ebin
plt: compile
@$(DIALYZER) --build_plt --output_plt .$(TARGET).plt -pa deps/*/ebin --apps kernel stdlib
@$(DIALYZER) --build_plt --output_plt .$(TARGET).plt -pa deps/lager/ebin --apps kernel stdlib
analyze: compile
$(DIALYZER) --plt .$(TARGET).plt -pa deps/*/ebin ebin
$(DIALYZER) --plt .$(TARGET).plt -pa deps/lager/ebin ebin
repl:
$(ERL) -pz deps/*/ebin -pa ebin
gdb-repl:
USE_GDB=1 $(ERL) -pz deps/*/ebin -pa ebin
$(ERL) -pz deps/lager/ebin -pa ebin
eunit-repl:
$(ERL) -pa .eunit -pz deps/*/ebin -pz ebin -exec 'cd(".eunit").'
gdb-eunit-repl:
USE_GDB=1 $(ERL) -pa .eunit -pz deps/*/ebin -pz ebin -exec 'cd(".eunit").'
$(ERL) -pz deps/lager/ebin -pa ebin -pa .eunit

43
c_src/async_nif.h
View file

@ -278,10 +278,9 @@ async_nif_worker_fn(void *arg)
unsigned int worker_id = we->worker_id;
struct async_nif_state *async_nif = we->async_nif;
struct async_nif_work_queue *q = we->q;
struct async_nif_req_entry *req = NULL;
for(;;) {
struct async_nif_req_entry *req = NULL;
/* Examine the request queue, are there things to be done? */
enif_mutex_lock(q->reqs_mutex);
check_again_for_work:
@ -295,34 +294,22 @@ async_nif_worker_fn(void *arg)
goto check_again_for_work;
} else {
/* At this point the next req is ours to process and we hold the
reqs_mutex lock. */
reqs_mutex lock. Take the request off the queue. */
req = fifo_q_get(reqs, q->reqs);
enif_mutex_unlock(q->reqs_mutex);
do {
/* Take the request off the queue. */
req = fifo_q_get(reqs, q->reqs);
enif_mutex_unlock(q->reqs_mutex);
/* Wake up another thread working on this queue. */
enif_cond_signal(q->reqs_cnd);
/* Wake up another thread working on this queue. */
enif_cond_signal(q->reqs_cnd);
/* Finally, do the work. */
req->fn_work(req->env, req->ref, &req->pid, worker_id, req->args);
req->fn_post(req->args);
/* Note: we don't call enif_free_env(req->env) because it has called
enif_send() which invalidates it (free'ing it for us). If a work
block doesn't call ASYNC_NIF_REPLY() at some point then it must
call ASYNC_NIF_NOREPLY() to free this env. */
enif_free(req->args);
enif_free(req);
/* Continue working if more requests are in the queue, otherwise wait
for new work to arrive. */
if (fifo_q_empty(reqs, q->reqs))
req = NULL;
else
enif_mutex_lock(q->reqs_mutex);
} while(req);
/* Finally, do the work. */
req->fn_work(req->env, req->ref, &req->pid, worker_id, req->args);
req->fn_post(req->args);
/* Note: we don't call enif_free_env(req->env) because it has called
enif_send() which invalidates it (free'ing it for us). If a work
block doesn't call ASYNC_NIF_REPLY() at some point then it must
call ASYNC_NIF_NOREPLY() to free this env. */
enif_free(req->args);
enif_free(req);
}
}
enif_thread_exit(0);

9
c_src/build_deps.sh
View file

@ -54,11 +54,10 @@ get_wt ()
./autogen.sh || exit 1
cd ./build_posix || exit 1
[ -e Makefile ] && $MAKE distclean
LDFLAGS="-L$BASEDIR/system/lib" CFLAGS="-I$BASEDIR/system/include" \
../configure --with-pic \
--enable-snappy \
--enable-bzip2 \
--prefix=${BASEDIR}/system || exit 1
../configure --with-pic \
--enable-snappy \
--enable-bzip2 \
--prefix=${BASEDIR}/system || exit 1
)
}

20
c_src/wiredtiger-build.patch
View file

@ -1,24 +1,24 @@
diff --git a/ext/compressors/bzip2/Makefile.am b/ext/compressors/bzip2/Makefile.am
index 0aedc2e..2137b8c 100644
index 0aedc2e..a70ae2e 100644
--- a/ext/compressors/bzip2/Makefile.am
+++ b/ext/compressors/bzip2/Makefile.am
@@ -2,5 +2,5 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
@@ -2,5 +2,6 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
lib_LTLIBRARIES = libwiredtiger_bzip2.la
libwiredtiger_bzip2_la_SOURCES = bzip2_compress.c
-libwiredtiger_bzip2_la_LDFLAGS = -avoid-version -module
-libwiredtiger_bzip2_la_LIBADD = -lbz2
+libwiredtiger_bzip2_la_LDFLAGS = -avoid-version -module -Wl,-rpath,.:lib/wterl-0.9.0/priv:lib/wterl/priv:priv
+libwiredtiger_bzip2_la_LIBADD = -lbz2
+libwiredtiger_snappy_la_CFLAGS = -I$(src_builddir)/../../system/include
+libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(src_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv
libwiredtiger_bzip2_la_LIBADD = -lbz2
diff --git a/ext/compressors/snappy/Makefile.am b/ext/compressors/snappy/Makefile.am
index 6d78823..2f2567e 100644
index 6d78823..2122cf8 100644
--- a/ext/compressors/snappy/Makefile.am
+++ b/ext/compressors/snappy/Makefile.am
@@ -2,5 +2,5 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
@@ -2,5 +2,6 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
lib_LTLIBRARIES = libwiredtiger_snappy.la
libwiredtiger_snappy_la_SOURCES = snappy_compress.c
-libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module
-libwiredtiger_snappy_la_LIBADD = -lsnappy
+libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -Wl,-rpath,.:lib/wterl-0.9.0/priv:lib/wterl/priv:priv
+libwiredtiger_snappy_la_LIBADD = -lsnappy
+libwiredtiger_snappy_la_CFLAGS = -I$(src_builddir)/../../system/include
+libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(src_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv
libwiredtiger_snappy_la_LIBADD = -lsnappy

34
src/riak_kv_wterl_backend.erl
View file

@ -94,25 +94,43 @@ start(Partition, Config) ->
end,
case AppStart of
ok ->
{type, Type} = wterl:config_value(type, Config, "lsm"),
Type =
case wterl:config_value(type, Config, "lsm") of
{type, "lsm"} -> "lsm";
{type, "table"} -> "table";
{type, "btree"} -> "table";
{type, BadType} ->
lager:info("wterl:start ignoring unknown type ~p, using lsm instead", [BadType]),
"lsm";
_ ->
lager:info("wterl:start ignoring mistaken setting defaulting to lsm"),
"lsm"
end,
{ok, Connection} = establish_connection(Config, Type),
Table = Type ++ ":wt" ++ integer_to_list(Partition),
Compressor =
case wterl:config_value(block_compressor, Config, "snappy") of
{block_compressor, "snappy"}=C -> [C];
{block_compressor, "bzip2"}=C -> [C];
{block_compressor, "none"} -> [];
{block_compressor, none} -> [];
{block_compressor, _} -> [{block_compressor, "snappy"}];
_ -> [{block_compressor, "snappy"}]
end,
TableOpts =
case Type of
"lsm" ->
[{block_compressor, "snappy"},
{internal_page_max, "128K"},
[{internal_page_max, "128K"},
{leaf_page_max, "128K"},
{lsm_chunk_size, "25MB"},
{lsm_bloom_newest, true},
{lsm_bloom_oldest, true} ,
{lsm_bloom_bit_count, 128},
{lsm_bloom_hash_count, 64},
{lsm_bloom_config, [{leaf_page_max, "8MB"}]} ];
{lsm_bloom_config, [{leaf_page_max, "8MB"}]}
] ++ Compressor;
"table" ->
[{block_compressor, "snappy"}];
_ ->
[]
Compressor
end,
case wterl:create(Connection, Table, TableOpts) of
ok ->
@ -561,14 +579,12 @@ size_cache(RequestedSize) ->
simple_test_() ->
{ok, CWD} = file:get_cwd(),
?assertMatch(true, lists:suffix("wterl/.eunit", CWD)),
rmdir:path(filename:join([CWD, "test/wterl-backend"])), %?assertCmd("rm -rf test/wterl-backend"),
application:set_env(wterl, data_root, "test/wterl-backend"),
temp_riak_kv_backend:standard_test(?MODULE, []).
custom_config_test_() ->
{ok, CWD} = file:get_cwd(),
?assertMatch(true, lists:suffix("wterl/.eunit", CWD)),
rmdir:path(filename:join([CWD, "test/wterl-backend"])), %?assertCmd("rm -rf test/wterl-backend"),
application:set_env(wterl, data_root, ""),
temp_riak_kv_backend:standard_test(?MODULE, [{data_root, "test/wterl-backend"}]).

3
src/wterl.erl
View file

@ -518,7 +518,6 @@ open_test_conn(DataDir) ->
open_test_conn(DataDir, [{create,true},{cache_size,"100MB"}]).
open_test_conn(DataDir, OpenConfig) ->
{ok, CWD} = file:get_cwd(),
?assertMatch(true, lists:suffix("wterl/.eunit", CWD)),
rmdir:path(filename:join([CWD, DataDir])), %?cmd("rm -rf " ++ filename:join([CWD, DataDir])),
?assertMatch(ok, filelib:ensure_dir(filename:join([DataDir, "x"]))),
{ok, ConnRef} = connection_open(filename:join([CWD, DataDir]), OpenConfig),
@ -660,7 +659,6 @@ various_maintenance_test_() ->
{setup,
fun () ->
{ok, CWD} = file:get_cwd(),
?assertMatch(true, lists:suffix("wterl/.eunit", CWD)),
?assertMatch(ok, filelib:ensure_dir(filename:join([?TEST_DATA_DIR, "x"]))),
{ok, ConnRef} = connection_open(filename:join([CWD, ?TEST_DATA_DIR]), []),
ConnRef
@ -859,7 +857,6 @@ prop_put_delete() ->
DataDir = "test/wterl.putdelete.qc",
Table = "table:eqc",
{ok, CWD} = file:get_cwd(),
?assertMatch(true, lists:suffix("wterl/.eunit", CWD)),
?cmd("rm -rf "++DataDir),
ok = filelib:ensure_dir(filename:join([DataDir, "x"])),
{ok, ConnRef} = wterl:connection_open(DataDir, [{create,true}]),