Compare commits
No commits in common. "master" and "gsb-ctx-cache" have entirely different histories.
master
...
gsb-ctx-ca
14 changed files with 1317 additions and 465 deletions
4
.gitignore
vendored
4
.gitignore
vendored
|
@ -7,8 +7,6 @@ c_src/*.o
|
|||
c_src/bzip2-1.0.6
|
||||
c_src/snappy-1.0.4
|
||||
deps/
|
||||
priv/wt
|
||||
priv/*.so*
|
||||
priv/*.dylib*
|
||||
priv/
|
||||
log/
|
||||
*~
|
||||
|
|
187
Makefile
187
Makefile
|
@ -1,70 +1,31 @@
|
|||
# Copyright 2012 Erlware, LLC. All Rights Reserved.
|
||||
#
|
||||
# This file is provided to you under the Apache License,
|
||||
# Version 2.0 (the "License"); you may not use this file
|
||||
# except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
TARGET= wterl
|
||||
|
||||
# Source: https://gist.github.com/ericbmerritt/5706091
|
||||
REBAR= ./rebar
|
||||
#REBAR= /usr/bin/env rebar
|
||||
ERL= /usr/bin/env erl
|
||||
ERLEXEC= ${ERL_ROOTDIR}/lib/erlang/erts-5.9.1/bin/erlexec
|
||||
DIALYZER= /usr/bin/env dialyzer
|
||||
ARCHIVETAG?= $(shell git describe --always --long --tags)
|
||||
ARCHIVE?= $(shell basename $(CURDIR))-$(ARCHIVETAG)
|
||||
WT_ARCHIVETAG?= $(shell cd c_src/wiredtiger-basho; git describe --always --long --tags)
|
||||
|
||||
.PHONY: plt analyze all deps compile get-deps clean
|
||||
|
||||
ERLFLAGS= -pa $(CURDIR)/.eunit -pa $(CURDIR)/ebin -pa $(CURDIR)/deps/*/ebin
|
||||
all: compile
|
||||
|
||||
DEPS_PLT=$(CURDIR)/.deps_plt
|
||||
DEPS=erts kernel stdlib
|
||||
archive:
|
||||
@rm -f $(ARCHIVE).tar.gz
|
||||
git archive --format=tar --prefix=$(ARCHIVE)/ $(ARCHIVETAG) | gzip >$(ARCHIVE).tar.gz
|
||||
|
||||
# =============================================================================
|
||||
# Verify that the programs we need to run are installed on this system
|
||||
# =============================================================================
|
||||
ERL = $(shell which erl)
|
||||
deps: get-deps
|
||||
|
||||
ifeq ($(ERL),)
|
||||
$(error "Erlang not available on this system")
|
||||
endif
|
||||
|
||||
REBAR=$(shell which rebar)
|
||||
|
||||
ifeq ($(REBAR),)
|
||||
$(error "Rebar not available on this system")
|
||||
endif
|
||||
|
||||
DIALYZER=$(shell which dialyzer)
|
||||
|
||||
ifeq ($(DIALYZER),)
|
||||
$(error "Dialyzer not available on this system")
|
||||
endif
|
||||
|
||||
TYPER=$(shell which typer)
|
||||
|
||||
ifeq ($(TYPER),)
|
||||
$(error "Typer not available on this system")
|
||||
endif
|
||||
|
||||
.PHONY: all compile doc clean test dialyzer typer shell distclean pdf \
|
||||
update-deps clean-common-test-data rebuild
|
||||
|
||||
all: deps compile
|
||||
|
||||
# =============================================================================
|
||||
# Rules to build the system
|
||||
# =============================================================================
|
||||
|
||||
deps:
|
||||
$(REBAR) get-deps
|
||||
$(REBAR) compile
|
||||
get-deps:
|
||||
c_src/build_deps.sh get-deps
|
||||
@$(REBAR) get-deps
|
||||
|
||||
update-deps:
|
||||
$(REBAR) update-deps
|
||||
$(REBAR) compile
|
||||
c_src/build_deps.sh update-deps
|
||||
@$(REBAR) update-deps
|
||||
|
||||
c_src/wterl.o: c_src/async_nif.h
|
||||
touch c_src/wterl.c
|
||||
|
@ -75,52 +36,70 @@ ebin/app_helper.beam:
|
|||
@/bin/false
|
||||
|
||||
compile: c_src/wterl.o ebin/app_helper.beam
|
||||
$(REBAR) skip_deps=true compile
|
||||
|
||||
doc:
|
||||
$(REBAR) skip_deps=true doc
|
||||
|
||||
eunit: compile clean-common-test-data
|
||||
$(REBAR) skip_deps=true eunit
|
||||
|
||||
test: compile eunit
|
||||
|
||||
$(DEPS_PLT):
|
||||
@echo Building local plt at $(DEPS_PLT)
|
||||
@echo
|
||||
dialyzer --output_plt $(DEPS_PLT) --build_plt \
|
||||
--apps $(DEPS) -r deps
|
||||
|
||||
dialyzer: $(DEPS_PLT)
|
||||
$(DIALYZER) --fullpath --plt $(DEPS_PLT) -Wrace_conditions -r ./ebin
|
||||
|
||||
typer:
|
||||
$(TYPER) --plt $(DEPS_PLT) -r ./src
|
||||
|
||||
xref:
|
||||
$(REBAR) xref skip_deps=true
|
||||
|
||||
# You often want *rebuilt* rebar tests to be available to the shell you have to
|
||||
# call eunit (to get the tests rebuilt). However, eunit runs the tests, which
|
||||
# probably fails (thats probably why You want them in the shell). This
|
||||
# (prefixing the command with "-") runs eunit but tells make to ignore the
|
||||
# result.
|
||||
shell: deps compile
|
||||
- @$(REBAR) skip_deps=true eunit
|
||||
@$(ERL) $(ERLFLAGS)
|
||||
|
||||
pdf:
|
||||
pandoc README.md -o README.pdf
|
||||
@$(REBAR) compile
|
||||
|
||||
clean:
|
||||
- c_src/build_deps.sh clean
|
||||
- rm -rf $(CURDIR)/test/*.beam
|
||||
- rm -rf $(CURDIR)/logs
|
||||
- rm -rf $(CURDIR)/ebin
|
||||
$(REBAR) skip_deps=true clean
|
||||
@rm -f $(ARCHIVE).tar.gz
|
||||
@$(REBAR) clean
|
||||
|
||||
distclean: clean
|
||||
- rm -rf $(DEPS_PLT)
|
||||
- rm -rvf $(CURDIR)/deps
|
||||
xref:
|
||||
@$(REBAR) xref skip_deps=true
|
||||
|
||||
rebuild: distclean deps compile escript dialyzer test
|
||||
test: eunit
|
||||
|
||||
eunit: compile-for-eunit
|
||||
@$(REBAR) eunit skip_deps=true
|
||||
|
||||
eqc: compile-for-eqc
|
||||
@$(REBAR) eqc skip_deps=true
|
||||
|
||||
proper: compile-for-proper
|
||||
@echo "rebar does not implement a 'proper' command" && false
|
||||
|
||||
triq: compile-for-triq
|
||||
@$(REBAR) triq skip_deps=true
|
||||
|
||||
compile-for-eunit:
|
||||
@$(REBAR) compile eunit compile_only=true
|
||||
|
||||
compile-for-eqc:
|
||||
@$(REBAR) -D QC -D QC_EQC compile eqc compile_only=true
|
||||
|
||||
compile-for-eqcmini:
|
||||
@$(REBAR) -D QC -D QC_EQCMINI compile eqc compile_only=true
|
||||
|
||||
compile-for-proper:
|
||||
@$(REBAR) -D QC -D QC_PROPER compile eqc compile_only=true
|
||||
|
||||
compile-for-triq:
|
||||
@$(REBAR) -D QC -D QC_TRIQ compile triq compile_only=true
|
||||
|
||||
plt: compile
|
||||
@$(DIALYZER) --build_plt --output_plt .$(TARGET).plt -pa deps/lager/ebin --apps kernel stdlib
|
||||
|
||||
analyze: compile
|
||||
@$(DIALYZER) --plt .$(TARGET).plt -pa deps/lager/ebin ebin
|
||||
|
||||
repl:
|
||||
@$(ERL) -pa ebin -pz deps/lager/ebin
|
||||
|
||||
eunit-repl:
|
||||
@$(ERL) erl -pa .eunit -pz deps/lager/ebin
|
||||
|
||||
ERL_TOP= /home/gburd/repos/otp_R15B01
|
||||
CERL= ${ERL_TOP}/bin/cerl
|
||||
VALGRIND_MISC_FLAGS= "--verbose --leak-check=full --show-reachable=yes --trace-children=yes --track-origins=yes --suppressions=${ERL_TOP}/erts/emulator/valgrind/suppress.standard --show-possibly-lost=no --malloc-fill=AB --free-fill=CD"
|
||||
|
||||
helgrind:
|
||||
valgrind --verbose --tool=helgrind \
|
||||
--leak-check=full
|
||||
--show-reachable=yes \
|
||||
--trace-children=yes \
|
||||
--track-origins=yes \
|
||||
--suppressions=${ERL_TOP}/erts/emulator/valgrind/suppress.standard \
|
||||
--show-possibly-lost=no \
|
||||
--malloc-fill=AB \
|
||||
--free-fill=CD ${ERLEXEC} -pz deps/lager/ebin -pa ebin -pa .eunit
|
||||
|
||||
valgrind:
|
||||
${CERL} -valgrind ${VALGRIND_FLAGS} --log-file=${ROOTDIR}/valgrind_log-beam.smp.%p -- -pz deps/lager/ebin -pa ebin -pa .eunit -exec 'eunit:test(wterl).'
|
||||
|
|
|
@ -34,18 +34,9 @@ extern "C" {
|
|||
|
||||
#define ASYNC_NIF_MAX_WORKERS 1024
|
||||
#define ASYNC_NIF_MIN_WORKERS 2
|
||||
#define ASYNC_NIF_WORKER_QUEUE_SIZE 8192
|
||||
#define ASYNC_NIF_WORKER_QUEUE_SIZE 100
|
||||
#define ASYNC_NIF_MAX_QUEUED_REQS ASYNC_NIF_WORKER_QUEUE_SIZE * ASYNC_NIF_MAX_WORKERS
|
||||
|
||||
/* Atoms (initialized in on_load) */
|
||||
static ERL_NIF_TERM ATOM_EAGAIN;
|
||||
static ERL_NIF_TERM ATOM_ENOMEM;
|
||||
static ERL_NIF_TERM ATOM_ENQUEUED;
|
||||
static ERL_NIF_TERM ATOM_ERROR;
|
||||
static ERL_NIF_TERM ATOM_OK;
|
||||
static ERL_NIF_TERM ATOM_SHUTDOWN;
|
||||
|
||||
|
||||
struct async_nif_req_entry {
|
||||
ERL_NIF_TERM ref;
|
||||
ErlNifEnv *env;
|
||||
|
@ -62,7 +53,6 @@ struct async_nif_work_queue {
|
|||
unsigned int depth;
|
||||
ErlNifMutex *reqs_mutex;
|
||||
ErlNifCond *reqs_cnd;
|
||||
struct async_nif_work_queue *next;
|
||||
STAILQ_HEAD(reqs, async_nif_req_entry) reqs;
|
||||
};
|
||||
|
||||
|
@ -113,20 +103,25 @@ struct async_nif_state {
|
|||
argc -= 1; \
|
||||
/* Note: !!! this assumes that the first element of priv_data is ours */ \
|
||||
struct async_nif_state *async_nif = *(struct async_nif_state**)enif_priv_data(env); \
|
||||
if (async_nif->shutdown) \
|
||||
return enif_make_tuple2(env, ATOM_ERROR, ATOM_SHUTDOWN); \
|
||||
if (async_nif->shutdown) { \
|
||||
return enif_make_tuple2(env, enif_make_atom(env, "error"), \
|
||||
enif_make_atom(env, "shutdown")); \
|
||||
} \
|
||||
req = async_nif_reuse_req(async_nif); \
|
||||
if (!req) \
|
||||
return enif_make_tuple2(env, ATOM_ERROR, ATOM_ENOMEM); \
|
||||
if (!req) { \
|
||||
return enif_make_tuple2(env, enif_make_atom(env, "error"), \
|
||||
enif_make_atom(env, "eagain")); \
|
||||
} \
|
||||
new_env = req->env; \
|
||||
DPRINTF("async_nif: calling \"%s\"", __func__); \
|
||||
do pre_block while(0); \
|
||||
DPRINTF("async_nif: returned from \"%s\"", __func__); \
|
||||
copy_of_args = (struct decl ## _args *)malloc(sizeof(struct decl ## _args)); \
|
||||
copy_of_args = (struct decl ## _args *)enif_alloc(sizeof(struct decl ## _args)); \
|
||||
if (!copy_of_args) { \
|
||||
fn_post_ ## decl (args); \
|
||||
async_nif_recycle_req(req, async_nif); \
|
||||
return enif_make_tuple2(env, ATOM_ERROR, ATOM_ENOMEM); \
|
||||
return enif_make_tuple2(env, enif_make_atom(env, "error"), \
|
||||
enif_make_atom(env, "enomem")); \
|
||||
} \
|
||||
memcpy(copy_of_args, args, sizeof(struct decl ## _args)); \
|
||||
req->ref = enif_make_copy(new_env, argv_in[0]); \
|
||||
|
@ -141,8 +136,9 @@ struct async_nif_state {
|
|||
if (!reply) { \
|
||||
fn_post_ ## decl (args); \
|
||||
async_nif_recycle_req(req, async_nif); \
|
||||
free(copy_of_args); \
|
||||
return enif_make_tuple2(env, ATOM_ERROR, ATOM_EAGAIN); \
|
||||
enif_free(copy_of_args); \
|
||||
return enif_make_tuple2(env, enif_make_atom(env, "error"), \
|
||||
enif_make_atom(env, "eagain")); \
|
||||
} \
|
||||
return reply; \
|
||||
}
|
||||
|
@ -150,16 +146,16 @@ struct async_nif_state {
|
|||
#define ASYNC_NIF_INIT(name) \
|
||||
static ErlNifMutex *name##_async_nif_coord = NULL;
|
||||
|
||||
#define ASYNC_NIF_LOAD(name, env, priv) do { \
|
||||
#define ASYNC_NIF_LOAD(name, priv) do { \
|
||||
if (!name##_async_nif_coord) \
|
||||
name##_async_nif_coord = enif_mutex_create("nif_coord load"); \
|
||||
name##_async_nif_coord = enif_mutex_create(NULL); \
|
||||
enif_mutex_lock(name##_async_nif_coord); \
|
||||
priv = async_nif_load(env); \
|
||||
priv = async_nif_load(); \
|
||||
enif_mutex_unlock(name##_async_nif_coord); \
|
||||
} while(0);
|
||||
#define ASYNC_NIF_UNLOAD(name, env, priv) do { \
|
||||
if (!name##_async_nif_coord) \
|
||||
name##_async_nif_coord = enif_mutex_create("nif_coord unload"); \
|
||||
name##_async_nif_coord = enif_mutex_create(NULL); \
|
||||
enif_mutex_lock(name##_async_nif_coord); \
|
||||
async_nif_unload(env, priv); \
|
||||
enif_mutex_unlock(name##_async_nif_coord); \
|
||||
|
@ -168,7 +164,7 @@ struct async_nif_state {
|
|||
} while(0);
|
||||
#define ASYNC_NIF_UPGRADE(name, env) do { \
|
||||
if (!name##_async_nif_coord) \
|
||||
name##_async_nif_coord = enif_mutex_create("nif_coord upgrade"); \
|
||||
name##_async_nif_coord = enif_mutex_create(NULL); \
|
||||
enif_mutex_lock(name##_async_nif_coord); \
|
||||
async_nif_upgrade(env); \
|
||||
enif_mutex_unlock(name##_async_nif_coord); \
|
||||
|
@ -195,15 +191,15 @@ async_nif_reuse_req(struct async_nif_state *async_nif)
|
|||
enif_mutex_lock(async_nif->recycled_req_mutex);
|
||||
if (STAILQ_EMPTY(&async_nif->recycled_reqs)) {
|
||||
if (async_nif->num_reqs < ASYNC_NIF_MAX_QUEUED_REQS) {
|
||||
req = malloc(sizeof(struct async_nif_req_entry));
|
||||
req = enif_alloc(sizeof(struct async_nif_req_entry));
|
||||
if (req) {
|
||||
memset(req, 0, sizeof(struct async_nif_req_entry));
|
||||
env = enif_alloc_env();
|
||||
if (env) {
|
||||
req->env = env;
|
||||
__sync_fetch_and_add(&async_nif->num_reqs, 1);
|
||||
async_nif->num_reqs++;
|
||||
} else {
|
||||
free(req);
|
||||
enif_free(req);
|
||||
req = NULL;
|
||||
}
|
||||
}
|
||||
|
@ -257,7 +253,7 @@ async_nif_start_worker(struct async_nif_state *async_nif, struct async_nif_work_
|
|||
SLIST_REMOVE(&async_nif->we_joining, we, async_nif_worker_entry, entries);
|
||||
void *exit_value = 0; /* We ignore the thread_join's exit value. */
|
||||
enif_thread_join(we->tid, &exit_value);
|
||||
free(we);
|
||||
enif_free(we);
|
||||
async_nif->we_active--;
|
||||
we = n;
|
||||
}
|
||||
|
@ -267,7 +263,7 @@ async_nif_start_worker(struct async_nif_state *async_nif, struct async_nif_work_
|
|||
return EAGAIN;
|
||||
}
|
||||
|
||||
we = malloc(sizeof(struct async_nif_worker_entry));
|
||||
we = enif_alloc(sizeof(struct async_nif_worker_entry));
|
||||
if (!we) {
|
||||
enif_mutex_unlock(async_nif->we_mutex);
|
||||
return ENOMEM;
|
||||
|
@ -291,9 +287,9 @@ static ERL_NIF_TERM
|
|||
async_nif_enqueue_req(struct async_nif_state* async_nif, struct async_nif_req_entry *req, int hint)
|
||||
{
|
||||
/* Identify the most appropriate worker for this request. */
|
||||
unsigned int i, last_qid, qid = 0;
|
||||
unsigned int i, qid = 0;
|
||||
struct async_nif_work_queue *q = NULL;
|
||||
double avg_depth = 0.0;
|
||||
double avg_depth;
|
||||
|
||||
/* Either we're choosing a queue based on some affinity/hinted value or we
|
||||
need to select the next queue in the rotation and atomically update that
|
||||
|
@ -301,10 +297,9 @@ async_nif_enqueue_req(struct async_nif_state* async_nif, struct async_nif_req_en
|
|||
if (hint >= 0) {
|
||||
qid = (unsigned int)hint;
|
||||
} else {
|
||||
do {
|
||||
last_qid = __sync_fetch_and_add(&async_nif->next_q, 0);
|
||||
qid = (last_qid + 1) % async_nif->num_queues;
|
||||
} while (!__sync_bool_compare_and_swap(&async_nif->next_q, last_qid, qid));
|
||||
qid = async_nif->next_q;
|
||||
qid = (qid + 1) % async_nif->num_queues;
|
||||
async_nif->next_q = qid;
|
||||
}
|
||||
|
||||
/* Now we inspect and interate across the set of queues trying to select one
|
||||
|
@ -319,13 +314,18 @@ async_nif_enqueue_req(struct async_nif_state* async_nif, struct async_nif_req_en
|
|||
avg_depth += async_nif->queues[j].depth;
|
||||
}
|
||||
}
|
||||
if (avg_depth) avg_depth /= n;
|
||||
if (avg_depth != 0)
|
||||
avg_depth /= n;
|
||||
|
||||
/* Lock this queue under consideration, then check for shutdown. While
|
||||
we hold this lock either a) we're shutting down so exit now or b) this
|
||||
queue will be valid until we release the lock. */
|
||||
q = &async_nif->queues[qid];
|
||||
enif_mutex_lock(q->reqs_mutex);
|
||||
if (async_nif->shutdown) {
|
||||
enif_mutex_unlock(q->reqs_mutex);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Try not to enqueue a request into a queue that isn't keeping up with
|
||||
the request volume. */
|
||||
|
@ -337,31 +337,26 @@ async_nif_enqueue_req(struct async_nif_state* async_nif, struct async_nif_req_en
|
|||
}
|
||||
|
||||
/* If the for loop finished then we didn't find a suitable queue for this
|
||||
request, meaning we're backed up so trigger eagain. Note that if we left
|
||||
the loop in this way we hold no lock. */
|
||||
if (i == async_nif->num_queues) return 0;
|
||||
request, meaning we're backed up so trigger eagain. */
|
||||
if (i == async_nif->num_queues) {
|
||||
enif_mutex_unlock(q->reqs_mutex);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Add the request to the queue. */
|
||||
STAILQ_INSERT_TAIL(&q->reqs, req, entries);
|
||||
__sync_fetch_and_add(&q->depth, 1);
|
||||
q->depth++;
|
||||
|
||||
/* We've selected a queue for this new request now check to make sure there are
|
||||
enough workers actively processing requests on this queue. */
|
||||
while (q->depth > q->num_workers) {
|
||||
switch(async_nif_start_worker(async_nif, q)) {
|
||||
case EINVAL: case ENOMEM: default: return 0;
|
||||
case EAGAIN: continue;
|
||||
case 0: __sync_fetch_and_add(&q->num_workers, 1); goto done;
|
||||
}
|
||||
}done:;
|
||||
if (q->depth > q->num_workers)
|
||||
if (async_nif_start_worker(async_nif, q) == 0) q->num_workers++;
|
||||
|
||||
/* Build the term before releasing the lock so as not to race on the use of
|
||||
the req pointer (which will soon become invalid in another thread
|
||||
performing the request). */
|
||||
double pct_full = (double)avg_depth / (double)ASYNC_NIF_WORKER_QUEUE_SIZE;
|
||||
ERL_NIF_TERM reply = enif_make_tuple2(req->env, ATOM_OK,
|
||||
enif_make_tuple2(req->env, ATOM_ENQUEUED,
|
||||
enif_make_double(req->env, pct_full)));
|
||||
ERL_NIF_TERM reply = enif_make_tuple2(req->env, enif_make_atom(req->env, "ok"),
|
||||
enif_make_atom(req->env, "enqueued"));
|
||||
enif_cond_signal(q->reqs_cnd);
|
||||
enif_mutex_unlock(q->reqs_mutex);
|
||||
return reply;
|
||||
|
@ -380,7 +375,6 @@ async_nif_worker_fn(void *arg)
|
|||
struct async_nif_state *async_nif = we->async_nif;
|
||||
struct async_nif_work_queue *q = we->q;
|
||||
struct async_nif_req_entry *req = NULL;
|
||||
unsigned int tries = async_nif->num_queues;
|
||||
|
||||
for(;;) {
|
||||
/* Examine the request queue, are there things to be done? */
|
||||
|
@ -392,33 +386,22 @@ async_nif_worker_fn(void *arg)
|
|||
}
|
||||
if (STAILQ_EMPTY(&q->reqs)) {
|
||||
/* Queue is empty so we wait for more work to arrive. */
|
||||
enif_mutex_unlock(q->reqs_mutex);
|
||||
if (tries == 0 && q == we->q) {
|
||||
if (q->num_workers > ASYNC_NIF_MIN_WORKERS) {
|
||||
/* At this point we've tried to find/execute work on all queues
|
||||
* and there are at least MIN_WORKERS on this queue so we
|
||||
* leaving this loop (break) which leads to a thread exit/join. */
|
||||
break;
|
||||
} else {
|
||||
enif_mutex_lock(q->reqs_mutex);
|
||||
enif_cond_wait(q->reqs_cnd, q->reqs_mutex);
|
||||
goto check_again_for_work;
|
||||
}
|
||||
} else {
|
||||
tries--;
|
||||
__sync_fetch_and_add(&q->num_workers, -1);
|
||||
q = q->next;
|
||||
__sync_fetch_and_add(&q->num_workers, 1);
|
||||
continue; // try next queue
|
||||
}
|
||||
if (q->num_workers > ASYNC_NIF_MIN_WORKERS) {
|
||||
enif_mutex_unlock(q->reqs_mutex);
|
||||
break;
|
||||
} else {
|
||||
enif_cond_wait(q->reqs_cnd, q->reqs_mutex);
|
||||
goto check_again_for_work;
|
||||
}
|
||||
} else {
|
||||
/* At this point the next req is ours to process and we hold the
|
||||
reqs_mutex lock. Take the request off the queue. */
|
||||
req = STAILQ_FIRST(&q->reqs);
|
||||
STAILQ_REMOVE(&q->reqs, req, async_nif_req_entry, entries);
|
||||
__sync_fetch_and_add(&q->depth, -1);
|
||||
q->depth--;
|
||||
|
||||
/* Wake up other worker thread watching this queue to help process work. */
|
||||
/* Ensure that there is at least one other worker thread watching this
|
||||
queue. */
|
||||
enif_cond_signal(q->reqs_cnd);
|
||||
enif_mutex_unlock(q->reqs_mutex);
|
||||
|
||||
|
@ -432,7 +415,7 @@ async_nif_worker_fn(void *arg)
|
|||
req->ref = 0;
|
||||
req->fn_work = 0;
|
||||
req->fn_post = 0;
|
||||
free(req->args);
|
||||
enif_free(req->args);
|
||||
req->args = NULL;
|
||||
async_nif_recycle_req(req, async_nif);
|
||||
req = NULL;
|
||||
|
@ -441,7 +424,7 @@ async_nif_worker_fn(void *arg)
|
|||
enif_mutex_lock(async_nif->we_mutex);
|
||||
SLIST_INSERT_HEAD(&async_nif->we_joining, we, entries);
|
||||
enif_mutex_unlock(async_nif->we_mutex);
|
||||
__sync_fetch_and_add(&q->num_workers, -1);
|
||||
q->num_workers--;
|
||||
enif_thread_exit(0);
|
||||
return 0;
|
||||
}
|
||||
|
@ -484,7 +467,7 @@ async_nif_unload(ErlNifEnv *env, struct async_nif_state *async_nif)
|
|||
SLIST_REMOVE(&async_nif->we_joining, we, async_nif_worker_entry, entries);
|
||||
void *exit_value = 0; /* We ignore the thread_join's exit value. */
|
||||
enif_thread_join(we->tid, &exit_value);
|
||||
free(we);
|
||||
enif_free(we);
|
||||
async_nif->we_active--;
|
||||
we = n;
|
||||
}
|
||||
|
@ -503,11 +486,12 @@ async_nif_unload(ErlNifEnv *env, struct async_nif_state *async_nif)
|
|||
struct async_nif_req_entry *n = STAILQ_NEXT(req, entries);
|
||||
enif_clear_env(req->env);
|
||||
enif_send(NULL, &req->pid, req->env,
|
||||
enif_make_tuple2(req->env, ATOM_ERROR, ATOM_SHUTDOWN));
|
||||
enif_make_tuple2(req->env, enif_make_atom(req->env, "error"),
|
||||
enif_make_atom(req->env, "shutdown")));
|
||||
req->fn_post(req->args);
|
||||
enif_free_env(req->env);
|
||||
free(req->args);
|
||||
free(req);
|
||||
enif_free(req->args);
|
||||
enif_free(req);
|
||||
req = n;
|
||||
}
|
||||
enif_mutex_destroy(q->reqs_mutex);
|
||||
|
@ -521,18 +505,18 @@ async_nif_unload(ErlNifEnv *env, struct async_nif_state *async_nif)
|
|||
while(req != NULL) {
|
||||
struct async_nif_req_entry *n = STAILQ_NEXT(req, entries);
|
||||
enif_free_env(req->env);
|
||||
free(req);
|
||||
enif_free(req);
|
||||
req = n;
|
||||
}
|
||||
|
||||
enif_mutex_unlock(async_nif->recycled_req_mutex);
|
||||
enif_mutex_destroy(async_nif->recycled_req_mutex);
|
||||
memset(async_nif, 0, sizeof(struct async_nif_state) + (sizeof(struct async_nif_work_queue) * async_nif->num_queues));
|
||||
free(async_nif);
|
||||
enif_free(async_nif);
|
||||
}
|
||||
|
||||
static void *
|
||||
async_nif_load(ErlNifEnv *env)
|
||||
async_nif_load()
|
||||
{
|
||||
static int has_init = 0;
|
||||
unsigned int i, num_queues;
|
||||
|
@ -543,14 +527,6 @@ async_nif_load(ErlNifEnv *env)
|
|||
if (has_init) return 0;
|
||||
else has_init = 1;
|
||||
|
||||
/* Init some static references to commonly used atoms. */
|
||||
ATOM_EAGAIN = enif_make_atom(env, "eagain");
|
||||
ATOM_ENOMEM = enif_make_atom(env, "enomem");
|
||||
ATOM_ENQUEUED = enif_make_atom(env, "enqueued");
|
||||
ATOM_ERROR = enif_make_atom(env, "error");
|
||||
ATOM_OK = enif_make_atom(env, "ok");
|
||||
ATOM_SHUTDOWN = enif_make_atom(env, "shutdown");
|
||||
|
||||
/* Find out how many schedulers there are. */
|
||||
enif_system_info(&info, sizeof(ErlNifSysInfo));
|
||||
|
||||
|
@ -568,8 +544,8 @@ async_nif_load(ErlNifEnv *env)
|
|||
}
|
||||
|
||||
/* Init our portion of priv_data's module-specific state. */
|
||||
async_nif = malloc(sizeof(struct async_nif_state) +
|
||||
sizeof(struct async_nif_work_queue) * num_queues);
|
||||
async_nif = enif_alloc(sizeof(struct async_nif_state) +
|
||||
sizeof(struct async_nif_work_queue) * num_queues);
|
||||
if (!async_nif)
|
||||
return NULL;
|
||||
memset(async_nif, 0, sizeof(struct async_nif_state) +
|
||||
|
@ -580,16 +556,15 @@ async_nif_load(ErlNifEnv *env)
|
|||
async_nif->next_q = 0;
|
||||
async_nif->shutdown = 0;
|
||||
STAILQ_INIT(&async_nif->recycled_reqs);
|
||||
async_nif->recycled_req_mutex = enif_mutex_create("recycled_req");
|
||||
async_nif->we_mutex = enif_mutex_create("we");
|
||||
async_nif->recycled_req_mutex = enif_mutex_create(NULL);
|
||||
async_nif->we_mutex = enif_mutex_create(NULL);
|
||||
SLIST_INIT(&async_nif->we_joining);
|
||||
|
||||
for (i = 0; i < async_nif->num_queues; i++) {
|
||||
struct async_nif_work_queue *q = &async_nif->queues[i];
|
||||
STAILQ_INIT(&q->reqs);
|
||||
q->reqs_mutex = enif_mutex_create("reqs");
|
||||
q->reqs_cnd = enif_cond_create("reqs");
|
||||
q->next = &async_nif->queues[(i + 1) % num_queues];
|
||||
q->reqs_mutex = enif_mutex_create(NULL);
|
||||
q->reqs_cnd = enif_cond_create(NULL);
|
||||
}
|
||||
return async_nif;
|
||||
}
|
||||
|
|
|
@ -11,10 +11,9 @@ unset POSIX_SHELL # clear it so if we invoke other scripts, they run as ksh as w
|
|||
set -e
|
||||
|
||||
WT_REPO=http://github.com/wiredtiger/wiredtiger.git
|
||||
WT_BRANCH=develop
|
||||
WT_DIR=wiredtiger-`basename $WT_BRANCH`
|
||||
#WT_REF="tags/1.6.6"
|
||||
#WT_DIR=wiredtiger-`basename $WT_REF`
|
||||
WT_BRANCH=
|
||||
WT_REF="tags/1.6.2"
|
||||
WT_DIR=wiredtiger-`basename $WT_REF`
|
||||
|
||||
SNAPPY_VSN="1.0.4"
|
||||
SNAPPY_DIR=snappy-$SNAPPY_VSN
|
||||
|
@ -26,7 +25,8 @@ export BASEDIR="$PWD"
|
|||
which gmake 1>/dev/null 2>/dev/null && MAKE=gmake
|
||||
MAKE=${MAKE:-make}
|
||||
|
||||
export CPPFLAGS="$CPPLAGS -I $BASEDIR/system/include -O3 -mtune=native -march=native"
|
||||
export CFLAGS="$CFLAGS -I $BASEDIR/system/include"
|
||||
export CXXFLAGS="$CXXFLAGS -I $BASEDIR/system/include"
|
||||
export LDFLAGS="$LDFLAGS -L$BASEDIR/system/lib"
|
||||
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$BASEDIR/system/lib:$LD_LIBRARY_PATH"
|
||||
|
||||
|
@ -57,7 +57,7 @@ get_wt ()
|
|||
wt_configure ()
|
||||
{
|
||||
(cd $BASEDIR/$WT_DIR/build_posix
|
||||
CFLAGS+=-g $BASEDIR/$WT_DIR/configure --with-pic \
|
||||
../configure --with-pic \
|
||||
--enable-snappy \
|
||||
--prefix=${BASEDIR}/system || exit 1)
|
||||
}
|
||||
|
@ -75,8 +75,8 @@ get_snappy ()
|
|||
|
||||
get_deps ()
|
||||
{
|
||||
get_snappy;
|
||||
get_wt;
|
||||
get_snappy;
|
||||
}
|
||||
|
||||
update_deps ()
|
||||
|
@ -109,7 +109,7 @@ build_snappy ()
|
|||
case "$1" in
|
||||
clean)
|
||||
[ -e $BASEDIR/$WT_DIR/build_posix/Makefile ] && \
|
||||
(cd $BASEDIR/$WT_DIR/build_posix && $MAKE clean)
|
||||
(cd $BASEDIR/$WT_DIR/build_posix && $MAKE distclean)
|
||||
rm -rf system $SNAPPY_DIR
|
||||
rm -f ${BASEDIR}/../priv/wt
|
||||
rm -f ${BASEDIR}/../priv/libwiredtiger-*.so
|
||||
|
@ -130,23 +130,22 @@ case "$1" in
|
|||
;;
|
||||
|
||||
*)
|
||||
shopt -s extglob
|
||||
SUFFIXES='@(so|dylib)'
|
||||
[ -d $WT_DIR ] || get_wt;
|
||||
[ -d $SNAPPY_DIR ] || get_snappy;
|
||||
|
||||
# Build Snappy
|
||||
[ -d $SNAPPY_DIR ] || get_snappy;
|
||||
[ -d $BASEDIR/$SNAPPY_DIR ] || (echo "Missing Snappy source directory" && exit 1)
|
||||
test -f $BASEDIR/system/lib/libsnappy.so.[0-9].[0-9].[0-9].* || build_snappy;
|
||||
test -f $BASEDIR/system/lib/libsnappy.so.[0-9].[0-9].[0-9] || build_snappy;
|
||||
|
||||
# Build WiredTiger
|
||||
[ -d $WT_DIR ] || get_wt;
|
||||
[ -d $BASEDIR/$WT_DIR ] || (echo "Missing WiredTiger source directory" && exit 1)
|
||||
test -f $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].${SUFFIXES} -a \
|
||||
-f $BASEDIR/system/lib/libwiredtiger_snappy.${SUFFIXES} || build_wt;
|
||||
test -f $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].so \
|
||||
-a -f $BASEDIR/system/lib/libwiredtiger_snappy.so || build_wt;
|
||||
|
||||
[ -d $BASEDIR/../priv ] || mkdir ${BASEDIR}/../priv
|
||||
cp -p -P $BASEDIR/system/bin/wt ${BASEDIR}/../priv
|
||||
cp -p -P ${BASEDIR}/system/lib/libwiredtiger-[0-9].[0-9].[0-9].${SUFFIXES} ${BASEDIR}/../priv
|
||||
cp -p -P ${BASEDIR}/system/lib/libwiredtiger_snappy.${SUFFIXES} ${BASEDIR}/../priv
|
||||
cp -p -P ${BASEDIR}/system/lib/libsnappy.${SUFFIXES}* ${BASEDIR}/../priv
|
||||
cp -p -P $BASEDIR/system/bin/wt ${BASEDIR}/../priv
|
||||
cp -p -P $BASEDIR/system/lib/libwiredtiger-[0-9].[0-9].[0-9].so ${BASEDIR}/../priv
|
||||
cp -p -P $BASEDIR/system/lib/libwiredtiger_snappy.so* ${BASEDIR}/../priv
|
||||
cp -p -P $BASEDIR/system/lib/libsnappy.so* ${BASEDIR}/../priv
|
||||
;;
|
||||
esac
|
||||
|
|
159
c_src/cas.h
Normal file
159
c_src/cas.h
Normal file
|
@ -0,0 +1,159 @@
|
|||
/*
|
||||
* wterl: an Erlang NIF for WiredTiger
|
||||
*
|
||||
* Copyright (c) 2012-2013 Basho Technologies, Inc. All Rights Reserved.
|
||||
*
|
||||
* This file is provided to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at:
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*
|
||||
*/
|
||||
|
||||
/*
|
||||
* Most of the following source code is copied directly from: "The Lock-Free
|
||||
* Library" (http://www.cl.cam.ac.uk/research/srg/netos/lock-free/) reused and
|
||||
* redistrubuted in accordance with their license:
|
||||
*
|
||||
* Copyright (c) 2002-2003 K A Fraser, All Rights Reserved.
|
||||
*
|
||||
* * Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are
|
||||
* met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright notice,
|
||||
* this list of conditions and the following disclaimer.
|
||||
*
|
||||
* * Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* * The name of the author may not be used to endorse or promote products
|
||||
* derived from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
|
||||
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
||||
* EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
||||
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
||||
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef __CAS_H_
|
||||
#define __CAS_H_
|
||||
|
||||
#define CACHE_LINE_SIZE 64
|
||||
|
||||
#define ATOMIC_INCR(_v,_newval) \
|
||||
do { \
|
||||
__typeof(_v) __val = (_v); \
|
||||
while ( (_newval = CASIO(&(_v),__val,__val+1)) != __val ) \
|
||||
__val = _newval; \
|
||||
} while ( 0 )
|
||||
#define ATOMIC_ADD_TO(_v,_x) \
|
||||
do { \
|
||||
__typeof(_v) __val = (_v), __newval; \
|
||||
while ( (__newval = CASIO(&(_v),__val,__val+(_x))) != __val ) \
|
||||
__val = __newval; \
|
||||
} while ( 0 )
|
||||
|
||||
#define ATOMIC_SET_TO(_v,_x) \
|
||||
do { \
|
||||
int __val = (_v), __newval; \
|
||||
while ( (__newval = CASIO(&(_v),__val,__val=(_x))) != __val ) \
|
||||
__val = __newval; \
|
||||
} while ( 0 )
|
||||
|
||||
#define CACHE_ALIGNED_SIZEOF(_s) \
|
||||
((sizeof(_s)) + CACHE_LINE_SIZE*2) + \
|
||||
CACHE_LINE_SIZE - 1) & ~(CACHE_LINE_SIZE-1))) \
|
||||
|
||||
/*
|
||||
* I. Compare-and-swap.
|
||||
*/
|
||||
|
||||
/*
|
||||
* This is a strong barrier! Reads cannot be delayed beyond a later store.
|
||||
* Reads cannot be hoisted beyond a LOCK prefix. Stores always in-order.
|
||||
*/
|
||||
#define CAS(_a, _o, _n) \
|
||||
({ __typeof__(_o) __o = _o; \
|
||||
__asm__ __volatile__( \
|
||||
"lock cmpxchg %3,%1" \
|
||||
: "=a" (__o), "=m" (*(volatile unsigned int *)(_a)) \
|
||||
: "0" (__o), "r" (_n) ); \
|
||||
__o; \
|
||||
})
|
||||
|
||||
#define FAS(_a, _n) \
|
||||
({ __typeof__(_n) __o; \
|
||||
__asm__ __volatile__( \
|
||||
"lock xchg %0,%1" \
|
||||
: "=r" (__o), "=m" (*(volatile unsigned int *)(_a)) \
|
||||
: "0" (_n) ); \
|
||||
__o; \
|
||||
})
|
||||
|
||||
#define CAS64(_a, _o, _n) \
|
||||
({ __typeof__(_o) __o = _o; \
|
||||
__asm__ __volatile__( \
|
||||
"movl %3, %%ecx;" \
|
||||
"movl %4, %%ebx;" \
|
||||
"lock cmpxchg8b %1" \
|
||||
: "=A" (__o), "=m" (*(volatile unsigned long long *)(_a)) \
|
||||
: "0" (__o), "m" (_n >> 32), "m" (_n) \
|
||||
: "ebx", "ecx" ); \
|
||||
__o; \
|
||||
})
|
||||
|
||||
/* Update Integer location, return Old value. */
|
||||
#define CASIO CAS
|
||||
#define FASIO FAS
|
||||
/* Update Pointer location, return Old value. */
|
||||
#define CASPO CAS
|
||||
#define FASPO FAS
|
||||
/* Update 32/64-bit location, return Old value. */
|
||||
#define CAS32O CAS
|
||||
#define CAS64O CAS64
|
||||
|
||||
/*
|
||||
* II. Memory barriers.
|
||||
* WMB(): All preceding write operations must commit before any later writes.
|
||||
* RMB(): All preceding read operations must commit before any later reads.
|
||||
* MB(): All preceding memory accesses must commit before any later accesses.
|
||||
*
|
||||
* If the compiler does not observe these barriers (but any sane compiler
|
||||
* will!), then VOLATILE should be defined as 'volatile'.
|
||||
*/
|
||||
|
||||
#define MB() __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory")
|
||||
#define WMB() __asm__ __volatile__ ("" : : : "memory")
|
||||
#define RMB() MB()
|
||||
#define VOLATILE /*volatile*/
|
||||
|
||||
/* On Intel, CAS is a strong barrier, but not a compile barrier. */
|
||||
#define RMB_NEAR_CAS() WMB()
|
||||
#define WMB_NEAR_CAS() WMB()
|
||||
#define MB_NEAR_CAS() WMB()
|
||||
|
||||
|
||||
/*
|
||||
* III. Cycle counter access.
|
||||
*/
|
||||
|
||||
typedef unsigned long long tick_t;
|
||||
#define RDTICK() \
|
||||
({ tick_t __t; __asm__ __volatile__ ("rdtsc" : "=A" (__t)); __t; })
|
||||
|
||||
#endif /* __CAS_H_ */
|
643
c_src/khash.h
Normal file
643
c_src/khash.h
Normal file
|
@ -0,0 +1,643 @@
|
|||
/* The MIT License
|
||||
|
||||
Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor@live.co.uk>
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining
|
||||
a copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, sublicense, and/or sell copies of the Software, and to
|
||||
permit persons to whom the Software is furnished to do so, subject to
|
||||
the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be
|
||||
included in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
||||
BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
||||
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
*/
|
||||
|
||||
/*
|
||||
An example:
|
||||
|
||||
#include "khash.h"
|
||||
KHASH_MAP_INIT_INT(32, char)
|
||||
int main() {
|
||||
int ret, is_missing;
|
||||
khiter_t k;
|
||||
khash_t(32) *h = kh_init(32);
|
||||
k = kh_put(32, h, 5, &ret);
|
||||
kh_value(h, k) = 10;
|
||||
k = kh_get(32, h, 10);
|
||||
is_missing = (k == kh_end(h));
|
||||
k = kh_get(32, h, 5);
|
||||
kh_del(32, h, k);
|
||||
for (k = kh_begin(h); k != kh_end(h); ++k)
|
||||
if (kh_exist(h, k)) kh_value(h, k) = 1;
|
||||
kh_destroy(32, h);
|
||||
return 0;
|
||||
}
|
||||
*/
|
||||
|
||||
/*
|
||||
2011-12-29 (0.2.7):
|
||||
|
||||
* Minor code clean up; no actual effect.
|
||||
|
||||
2011-09-16 (0.2.6):
|
||||
|
||||
* The capacity is a power of 2. This seems to dramatically improve the
|
||||
speed for simple keys. Thank Zilong Tan for the suggestion. Reference:
|
||||
|
||||
- http://code.google.com/p/ulib/
|
||||
- http://nothings.org/computer/judy/
|
||||
|
||||
* Allow to optionally use linear probing which usually has better
|
||||
performance for random input. Double hashing is still the default as it
|
||||
is more robust to certain non-random input.
|
||||
|
||||
* Added Wang's integer hash function (not used by default). This hash
|
||||
function is more robust to certain non-random input.
|
||||
|
||||
2011-02-14 (0.2.5):
|
||||
|
||||
* Allow to declare global functions.
|
||||
|
||||
2009-09-26 (0.2.4):
|
||||
|
||||
* Improve portability
|
||||
|
||||
2008-09-19 (0.2.3):
|
||||
|
||||
* Corrected the example
|
||||
* Improved interfaces
|
||||
|
||||
2008-09-11 (0.2.2):
|
||||
|
||||
* Improved speed a little in kh_put()
|
||||
|
||||
2008-09-10 (0.2.1):
|
||||
|
||||
* Added kh_clear()
|
||||
* Fixed a compiling error
|
||||
|
||||
2008-09-02 (0.2.0):
|
||||
|
||||
* Changed to token concatenation which increases flexibility.
|
||||
|
||||
2008-08-31 (0.1.2):
|
||||
|
||||
* Fixed a bug in kh_get(), which has not been tested previously.
|
||||
|
||||
2008-08-31 (0.1.1):
|
||||
|
||||
* Added destructor
|
||||
*/
|
||||
|
||||
|
||||
#ifndef __AC_KHASH_H
|
||||
#define __AC_KHASH_H
|
||||
|
||||
/*!
|
||||
@header
|
||||
|
||||
Generic hash table library.
|
||||
*/
|
||||
|
||||
#define AC_VERSION_KHASH_H "0.2.6"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <limits.h>
|
||||
|
||||
/* compiler specific configuration */
|
||||
|
||||
#if UINT_MAX == 0xffffffffu
|
||||
typedef unsigned int khint32_t;
|
||||
#elif ULONG_MAX == 0xffffffffu
|
||||
typedef unsigned long khint32_t;
|
||||
#endif
|
||||
|
||||
#if ULONG_MAX == ULLONG_MAX
|
||||
typedef unsigned long khint64_t;
|
||||
#else
|
||||
typedef unsigned long long khint64_t;
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#define kh_inline __inline
|
||||
#else
|
||||
#define kh_inline inline
|
||||
#endif
|
||||
|
||||
typedef khint32_t khint_t;
|
||||
typedef khint_t khiter_t;
|
||||
|
||||
#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
|
||||
#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
|
||||
#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
|
||||
#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
|
||||
#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
|
||||
#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
|
||||
#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
|
||||
|
||||
#ifdef KHASH_LINEAR
|
||||
#define __ac_inc(k, m) 1
|
||||
#else
|
||||
#define __ac_inc(k, m) (((k)>>3 ^ (k)<<3) | 1) & (m)
|
||||
#endif
|
||||
|
||||
#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4)
|
||||
|
||||
#ifndef kroundup32
|
||||
#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
|
||||
#endif
|
||||
|
||||
#ifndef kcalloc
|
||||
#define kcalloc(N,Z) calloc(N,Z)
|
||||
#endif
|
||||
#ifndef kmalloc
|
||||
#define kmalloc(Z) malloc(Z)
|
||||
#endif
|
||||
#ifndef krealloc
|
||||
#define krealloc(P,Z) realloc(P,Z)
|
||||
#endif
|
||||
#ifndef kfree
|
||||
#define kfree(P) free(P)
|
||||
#endif
|
||||
|
||||
static const double __ac_HASH_UPPER = 0.77;
|
||||
|
||||
#define __KHASH_TYPE(name, khkey_t, khval_t) \
|
||||
typedef struct { \
|
||||
khint_t n_buckets, size, n_occupied, upper_bound; \
|
||||
khint32_t *flags; \
|
||||
khkey_t *keys; \
|
||||
khval_t *vals; \
|
||||
} kh_##name##_t;
|
||||
|
||||
#define __KHASH_PROTOTYPES(name, khkey_t, khval_t) \
|
||||
extern kh_##name##_t *kh_init_##name(void); \
|
||||
extern void kh_destroy_##name(kh_##name##_t *h); \
|
||||
extern void kh_clear_##name(kh_##name##_t *h); \
|
||||
extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \
|
||||
extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
|
||||
extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
|
||||
extern void kh_del_##name(kh_##name##_t *h, khint_t x);
|
||||
|
||||
#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
|
||||
SCOPE kh_##name##_t *kh_init_##name(void) { \
|
||||
return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t)); \
|
||||
} \
|
||||
SCOPE void kh_destroy_##name(kh_##name##_t *h) \
|
||||
{ \
|
||||
if (h) { \
|
||||
kfree((void *)h->keys); kfree(h->flags); \
|
||||
kfree((void *)h->vals); \
|
||||
kfree(h); \
|
||||
} \
|
||||
} \
|
||||
SCOPE void kh_clear_##name(kh_##name##_t *h) \
|
||||
{ \
|
||||
if (h && h->flags) { \
|
||||
memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \
|
||||
h->size = h->n_occupied = 0; \
|
||||
} \
|
||||
} \
|
||||
SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \
|
||||
{ \
|
||||
if (h->n_buckets) { \
|
||||
khint_t inc, k, i, last, mask; \
|
||||
mask = h->n_buckets - 1; \
|
||||
k = __hash_func(key); i = k & mask; \
|
||||
inc = __ac_inc(k, mask); last = i; /* inc==1 for linear probing */ \
|
||||
while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
|
||||
i = (i + inc) & mask; \
|
||||
if (i == last) return h->n_buckets; \
|
||||
} \
|
||||
return __ac_iseither(h->flags, i)? h->n_buckets : i; \
|
||||
} else return 0; \
|
||||
} \
|
||||
SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
|
||||
{ /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \
|
||||
khint32_t *new_flags = 0; \
|
||||
khint_t j = 1; \
|
||||
{ \
|
||||
kroundup32(new_n_buckets); \
|
||||
if (new_n_buckets < 4) new_n_buckets = 4; \
|
||||
if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; /* requested size is too small */ \
|
||||
else { /* hash table size to be changed (shrink or expand); rehash */ \
|
||||
new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
|
||||
if (!new_flags) return -1; \
|
||||
memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
|
||||
if (h->n_buckets < new_n_buckets) { /* expand */ \
|
||||
khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
|
||||
if (!new_keys) return -1; \
|
||||
h->keys = new_keys; \
|
||||
if (kh_is_map) { \
|
||||
khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
|
||||
if (!new_vals) return -1; \
|
||||
h->vals = new_vals; \
|
||||
} \
|
||||
} /* otherwise shrink */ \
|
||||
} \
|
||||
} \
|
||||
if (j) { /* rehashing is needed */ \
|
||||
for (j = 0; j != h->n_buckets; ++j) { \
|
||||
if (__ac_iseither(h->flags, j) == 0) { \
|
||||
khkey_t key = h->keys[j]; \
|
||||
khval_t val; \
|
||||
khint_t new_mask; \
|
||||
new_mask = new_n_buckets - 1; \
|
||||
if (kh_is_map) val = h->vals[j]; \
|
||||
__ac_set_isdel_true(h->flags, j); \
|
||||
while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \
|
||||
khint_t inc, k, i; \
|
||||
k = __hash_func(key); \
|
||||
i = k & new_mask; \
|
||||
inc = __ac_inc(k, new_mask); \
|
||||
while (!__ac_isempty(new_flags, i)) i = (i + inc) & new_mask; \
|
||||
__ac_set_isempty_false(new_flags, i); \
|
||||
if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \
|
||||
{ khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
|
||||
if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
|
||||
__ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \
|
||||
} else { /* write the element and jump out of the loop */ \
|
||||
h->keys[i] = key; \
|
||||
if (kh_is_map) h->vals[i] = val; \
|
||||
break; \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \
|
||||
h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
|
||||
if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
|
||||
} \
|
||||
kfree(h->flags); /* free the working space */ \
|
||||
h->flags = new_flags; \
|
||||
h->n_buckets = new_n_buckets; \
|
||||
h->n_occupied = h->size; \
|
||||
h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
|
||||
} \
|
||||
return 0; \
|
||||
} \
|
||||
SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
|
||||
{ \
|
||||
khint_t x; \
|
||||
if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \
|
||||
if (h->n_buckets > (h->size<<1)) { \
|
||||
if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \
|
||||
*ret = -1; return h->n_buckets; \
|
||||
} \
|
||||
} else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \
|
||||
*ret = -1; return h->n_buckets; \
|
||||
} \
|
||||
} /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
|
||||
{ \
|
||||
khint_t inc, k, i, site, last, mask = h->n_buckets - 1; \
|
||||
x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \
|
||||
if (__ac_isempty(h->flags, i)) x = i; /* for speed up */ \
|
||||
else { \
|
||||
inc = __ac_inc(k, mask); last = i; \
|
||||
while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
|
||||
if (__ac_isdel(h->flags, i)) site = i; \
|
||||
i = (i + inc) & mask; \
|
||||
if (i == last) { x = site; break; } \
|
||||
} \
|
||||
if (x == h->n_buckets) { \
|
||||
if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
|
||||
else x = i; \
|
||||
} \
|
||||
} \
|
||||
} \
|
||||
if (__ac_isempty(h->flags, x)) { /* not present at all */ \
|
||||
h->keys[x] = key; \
|
||||
__ac_set_isboth_false(h->flags, x); \
|
||||
++h->size; ++h->n_occupied; \
|
||||
*ret = 1; \
|
||||
} else if (__ac_isdel(h->flags, x)) { /* deleted */ \
|
||||
h->keys[x] = key; \
|
||||
__ac_set_isboth_false(h->flags, x); \
|
||||
++h->size; \
|
||||
*ret = 2; \
|
||||
} else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \
|
||||
return x; \
|
||||
} \
|
||||
SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x) \
|
||||
{ \
|
||||
if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \
|
||||
__ac_set_isdel_true(h->flags, x); \
|
||||
--h->size; \
|
||||
} \
|
||||
}
|
||||
|
||||
#define KHASH_DECLARE(name, khkey_t, khval_t) \
|
||||
__KHASH_TYPE(name, khkey_t, khval_t) \
|
||||
__KHASH_PROTOTYPES(name, khkey_t, khval_t)
|
||||
|
||||
#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
|
||||
__KHASH_TYPE(name, khkey_t, khval_t) \
|
||||
__KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
|
||||
|
||||
#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
|
||||
KHASH_INIT2(name, static kh_inline, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
|
||||
|
||||
/* --- BEGIN OF HASH FUNCTIONS --- */
|
||||
|
||||
/*! @function
|
||||
@abstract Integer hash function
|
||||
@param key The integer [khint32_t]
|
||||
@return The hash value [khint_t]
|
||||
*/
|
||||
#define kh_int_hash_func(key) (khint32_t)(key)
|
||||
/*! @function
|
||||
@abstract Integer comparison function
|
||||
*/
|
||||
#define kh_int_hash_equal(a, b) ((a) == (b))
|
||||
/*! @function
|
||||
@abstract 64-bit integer hash function
|
||||
@param key The integer [khint64_t]
|
||||
@return The hash value [khint_t]
|
||||
*/
|
||||
#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11)
|
||||
/*! @function
|
||||
@abstract 64-bit integer comparison function
|
||||
*/
|
||||
#define kh_int64_hash_equal(a, b) ((a) == (b))
|
||||
/*! @function
|
||||
@abstract Pointer hash function
|
||||
@param key The integer void *
|
||||
@return The hash value [khint_t]
|
||||
*/
|
||||
#define kh_ptr_hash_func(key) (khint32_t)(key)
|
||||
/*! @function
|
||||
@abstract Pointer comparison function
|
||||
*/
|
||||
#define kh_ptr_hash_equal(a, b) ((a) == (b))
|
||||
/*! @function
|
||||
@abstract 64-bit pointer hash function
|
||||
@param key The integer void *
|
||||
@return The hash value [khint_t]
|
||||
*/
|
||||
#define kh_ptr64_hash_func(key) (khint32_t)(((khint64_t)key)>>33^((khint64_t)key)^((khint64_t)key)<<11)
|
||||
/*! @function
|
||||
@abstract 64-bit pointer comparison function
|
||||
*/
|
||||
#define kh_ptr64_hash_equal(a, b) ((a) == (b))
|
||||
/*! @function
|
||||
@abstract const char* hash function
|
||||
@param s Pointer to a null terminated string
|
||||
@return The hash value
|
||||
*/
|
||||
static kh_inline khint_t __ac_X31_hash_string(const char *s)
|
||||
{
|
||||
khint_t h = (khint_t)*s;
|
||||
if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s;
|
||||
return h;
|
||||
}
|
||||
/*! @function
|
||||
@abstract Another interface to const char* hash function
|
||||
@param key Pointer to a null terminated string [const char*]
|
||||
@return The hash value [khint_t]
|
||||
*/
|
||||
#define kh_str_hash_func(key) __ac_X31_hash_string(key)
|
||||
/*! @function
|
||||
@abstract Const char* comparison function
|
||||
*/
|
||||
#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
|
||||
|
||||
static kh_inline khint_t __ac_Wang_hash(khint_t key)
|
||||
{
|
||||
key += ~(key << 15);
|
||||
key ^= (key >> 10);
|
||||
key += (key << 3);
|
||||
key ^= (key >> 6);
|
||||
key += ~(key << 11);
|
||||
key ^= (key >> 16);
|
||||
return key;
|
||||
}
|
||||
#define kh_int_hash_func2(k) __ac_Wang_hash((khint_t)key)
|
||||
|
||||
/* --- END OF HASH FUNCTIONS --- */
|
||||
|
||||
/* Other convenient macros... */
|
||||
|
||||
/*!
|
||||
@abstract Type of the hash table.
|
||||
@param name Name of the hash table [symbol]
|
||||
*/
|
||||
#define khash_t(name) kh_##name##_t
|
||||
|
||||
/*! @function
|
||||
@abstract Initiate a hash table.
|
||||
@param name Name of the hash table [symbol]
|
||||
@return Pointer to the hash table [khash_t(name)*]
|
||||
*/
|
||||
#define kh_init(name) kh_init_##name()
|
||||
|
||||
/*! @function
|
||||
@abstract Destroy a hash table.
|
||||
@param name Name of the hash table [symbol]
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
*/
|
||||
#define kh_destroy(name, h) kh_destroy_##name(h)
|
||||
|
||||
/*! @function
|
||||
@abstract Reset a hash table without deallocating memory.
|
||||
@param name Name of the hash table [symbol]
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
*/
|
||||
#define kh_clear(name, h) kh_clear_##name(h)
|
||||
|
||||
/*! @function
|
||||
@abstract Resize a hash table.
|
||||
@param name Name of the hash table [symbol]
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@param s New size [khint_t]
|
||||
*/
|
||||
#define kh_resize(name, h, s) kh_resize_##name(h, s)
|
||||
|
||||
/*! @function
|
||||
@abstract Insert a key to the hash table.
|
||||
@param name Name of the hash table [symbol]
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@param k Key [type of keys]
|
||||
@param r Extra return code: 0 if the key is present in the hash table;
|
||||
1 if the bucket is empty (never used); 2 if the element in
|
||||
the bucket has been deleted [int*]
|
||||
@return Iterator to the inserted element [khint_t]
|
||||
*/
|
||||
#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
|
||||
|
||||
/*! @function
|
||||
@abstract Retrieve a key from the hash table.
|
||||
@param name Name of the hash table [symbol]
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@param k Key [type of keys]
|
||||
@return Iterator to the found element, or kh_end(h) if the element is absent [khint_t]
|
||||
*/
|
||||
#define kh_get(name, h, k) kh_get_##name(h, k)
|
||||
|
||||
/*! @function
|
||||
@abstract Remove a key from the hash table.
|
||||
@param name Name of the hash table [symbol]
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@param k Iterator to the element to be deleted [khint_t]
|
||||
*/
|
||||
#define kh_del(name, h, k) kh_del_##name(h, k)
|
||||
|
||||
/*! @function
|
||||
@abstract Test whether a bucket contains data.
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@param x Iterator to the bucket [khint_t]
|
||||
@return 1 if containing data; 0 otherwise [int]
|
||||
*/
|
||||
#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
|
||||
|
||||
/*! @function
|
||||
@abstract Get key given an iterator
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@param x Iterator to the bucket [khint_t]
|
||||
@return Key [type of keys]
|
||||
*/
|
||||
#define kh_key(h, x) ((h)->keys[x])
|
||||
|
||||
/*! @function
|
||||
@abstract Get value given an iterator
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@param x Iterator to the bucket [khint_t]
|
||||
@return Value [type of values]
|
||||
@discussion For hash sets, calling this results in segfault.
|
||||
*/
|
||||
#define kh_val(h, x) ((h)->vals[x])
|
||||
|
||||
/*! @function
|
||||
@abstract Alias of kh_val()
|
||||
*/
|
||||
#define kh_value(h, x) ((h)->vals[x])
|
||||
|
||||
/*! @function
|
||||
@abstract Get the start iterator
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@return The start iterator [khint_t]
|
||||
*/
|
||||
#define kh_begin(h) (khint_t)(0)
|
||||
|
||||
/*! @function
|
||||
@abstract Get the end iterator
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@return The end iterator [khint_t]
|
||||
*/
|
||||
#define kh_end(h) ((h)->n_buckets)
|
||||
|
||||
/*! @function
|
||||
@abstract Get the number of elements in the hash table
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@return Number of elements in the hash table [khint_t]
|
||||
*/
|
||||
#define kh_size(h) ((h)->size)
|
||||
|
||||
/*! @function
|
||||
@abstract Get the number of buckets in the hash table
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@return Number of buckets in the hash table [khint_t]
|
||||
*/
|
||||
#define kh_n_buckets(h) ((h)->n_buckets)
|
||||
|
||||
/*! @function
|
||||
@abstract Iterate over the entries in the hash table
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@param kvar Variable to which key will be assigned
|
||||
@param vvar Variable to which value will be assigned
|
||||
@param code Block of code to execute
|
||||
*/
|
||||
#define kh_foreach(h, kvar, vvar, code) { khint_t __i; \
|
||||
for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
|
||||
if (!kh_exist(h,__i)) continue; \
|
||||
(kvar) = kh_key(h,__i); \
|
||||
(vvar) = kh_val(h,__i); \
|
||||
code; \
|
||||
} }
|
||||
|
||||
/*! @function
|
||||
@abstract Iterate over the values in the hash table
|
||||
@param h Pointer to the hash table [khash_t(name)*]
|
||||
@param vvar Variable to which value will be assigned
|
||||
@param code Block of code to execute
|
||||
*/
|
||||
#define kh_foreach_value(h, vvar, code) { khint_t __i; \
|
||||
for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
|
||||
if (!kh_exist(h,__i)) continue; \
|
||||
(vvar) = kh_val(h,__i); \
|
||||
code; \
|
||||
} }
|
||||
|
||||
/* More conenient interfaces */
|
||||
|
||||
/*! @function
|
||||
@abstract Instantiate a hash map containing (void *) keys
|
||||
@param name Name of the hash table [symbol]
|
||||
@param khval_t Type of values [type]
|
||||
*/
|
||||
#ifdef __x86_64__
|
||||
#define KHASH_MAP_INIT_PTR(name, khval_t) \
|
||||
KHASH_INIT(name, void*, khval_t, 1, kh_ptr64_hash_func, kh_ptr64_hash_equal)
|
||||
#else
|
||||
#define KHASH_MAP_INIT_PTR(name, khval_t) \
|
||||
KHASH_INIT(name, void*, khval_t, 1, kh_ptr_hash_func, kh_ptr_hash_equal)
|
||||
#endif
|
||||
|
||||
/*! @function
|
||||
@abstract Instantiate a hash set containing integer keys
|
||||
@param name Name of the hash table [symbol]
|
||||
*/
|
||||
#define KHASH_SET_INIT_INT(name) \
|
||||
KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
|
||||
|
||||
/*! @function
|
||||
@abstract Instantiate a hash map containing integer keys
|
||||
@param name Name of the hash table [symbol]
|
||||
@param khval_t Type of values [type]
|
||||
*/
|
||||
#define KHASH_MAP_INIT_INT(name, khval_t) \
|
||||
KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
|
||||
|
||||
/*! @function
|
||||
@abstract Instantiate a hash map containing 64-bit integer keys
|
||||
@param name Name of the hash table [symbol]
|
||||
*/
|
||||
#define KHASH_SET_INIT_INT64(name) \
|
||||
KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
|
||||
|
||||
/*! @function
|
||||
@abstract Instantiate a hash map containing 64-bit integer keys
|
||||
@param name Name of the hash table [symbol]
|
||||
@param khval_t Type of values [type]
|
||||
*/
|
||||
#define KHASH_MAP_INIT_INT64(name, khval_t) \
|
||||
KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
|
||||
|
||||
typedef const char *kh_cstr_t;
|
||||
/*! @function
|
||||
@abstract Instantiate a hash map containing const char* keys
|
||||
@param name Name of the hash table [symbol]
|
||||
*/
|
||||
#define KHASH_SET_INIT_STR(name) \
|
||||
KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
|
||||
|
||||
/*! @function
|
||||
@abstract Instantiate a hash map containing const char* keys
|
||||
@param name Name of the hash table [symbol]
|
||||
@param khval_t Type of values [type]
|
||||
*/
|
||||
#define KHASH_MAP_INIT_STR(name, khval_t) \
|
||||
KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
|
||||
|
||||
#endif /* __AC_KHASH_H */
|
|
@ -1,5 +1,5 @@
|
|||
diff --git a/ext/compressors/snappy/Makefile.am b/ext/compressors/snappy/Makefile.am
|
||||
index 6d78823..c423590 100644
|
||||
index 6d78823..2122cf8 100644
|
||||
--- a/ext/compressors/snappy/Makefile.am
|
||||
+++ b/ext/compressors/snappy/Makefile.am
|
||||
@@ -2,5 +2,6 @@ AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
|
||||
|
@ -7,6 +7,100 @@ index 6d78823..c423590 100644
|
|||
lib_LTLIBRARIES = libwiredtiger_snappy.la
|
||||
libwiredtiger_snappy_la_SOURCES = snappy_compress.c
|
||||
-libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module
|
||||
+libwiredtiger_snappy_la_CFLAGS = -I$(abs_top_builddir)/../../system/include
|
||||
+libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(abs_top_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv
|
||||
+libwiredtiger_snappy_la_CFLAGS = -I$(src_builddir)/../../system/include
|
||||
+libwiredtiger_snappy_la_LDFLAGS = -avoid-version -module -L$(src_builddir)/../../system/lib -Wl,-rpath,lib/wterl-0.9.0/priv:lib/wterl/priv:priv
|
||||
libwiredtiger_snappy_la_LIBADD = -lsnappy
|
||||
diff --git a/src/support/cksum.c b/src/support/cksum.c
|
||||
index 7e9befe..b924db7 100644
|
||||
--- a/src/support/cksum.c
|
||||
+++ b/src/support/cksum.c
|
||||
@@ -27,6 +27,13 @@
|
||||
|
||||
#include "wt_internal.h"
|
||||
|
||||
+#if defined(__amd64) || defined(__x86_64)
|
||||
+#define USE_HARDWARE_CRC32 1
|
||||
+#else
|
||||
+#undef USE_HARDWARE_CRC32
|
||||
+#endif
|
||||
+
|
||||
+#ifdef USE_HARDWARE_CRC32
|
||||
static const uint32_t g_crc_slicing[8][256] = {
|
||||
#ifdef WORDS_BIGENDIAN
|
||||
/*
|
||||
@@ -1078,6 +1085,7 @@ static const uint32_t g_crc_slicing[8][256] = {
|
||||
}
|
||||
#endif
|
||||
};
|
||||
+#endif /* USE_HARDWARE_CRC32 */
|
||||
|
||||
/*
|
||||
* __wt_cksum --
|
||||
@@ -1106,15 +1114,29 @@ __wt_cksum(const void *chunk, size_t len)
|
||||
/* Checksum one byte at a time to the first 4B boundary. */
|
||||
for (p = chunk;
|
||||
((uintptr_t)p & (sizeof(uint32_t) - 1)) != 0 &&
|
||||
- len > 0; ++p, --len)
|
||||
+ len > 0; ++p, --len) {
|
||||
+#ifdef USE_HARDWARE_CRC32
|
||||
+ __asm__ __volatile__(
|
||||
+ ".byte 0xF2, 0x0F, 0x38, 0xF0, 0xF1"
|
||||
+ : "=S" (crc)
|
||||
+ : "0" (crc), "c" (*p));
|
||||
+#else
|
||||
#ifdef WORDS_BIGENDIAN
|
||||
crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8);
|
||||
#else
|
||||
crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8);
|
||||
#endif
|
||||
+#endif
|
||||
+ }
|
||||
|
||||
/* Checksum in 8B chunks. */
|
||||
for (nqwords = len / sizeof(uint64_t); nqwords; nqwords--) {
|
||||
+#ifdef USE_HARDWARE_CRC32
|
||||
+ __asm__ __volatile__ (
|
||||
+ ".byte 0xf2, 0x48, 0x0f, 0x38, 0xf0, 0xf1;"
|
||||
+ : "=S"(crc)
|
||||
+ : "S"(crc), "c"(*p));
|
||||
+#else
|
||||
crc ^= *(uint32_t *)p;
|
||||
p += sizeof(uint32_t);
|
||||
next = *(uint32_t *)p;
|
||||
@@ -1139,22 +1161,32 @@ __wt_cksum(const void *chunk, size_t len)
|
||||
g_crc_slicing[1][(next >> 16) & 0xFF] ^
|
||||
g_crc_slicing[0][(next >> 24)];
|
||||
#endif
|
||||
+#endif
|
||||
}
|
||||
|
||||
/* Checksum trailing bytes one byte at a time. */
|
||||
+ for (len &= 0x7; len > 0; ++p, len--) {
|
||||
+#ifdef USE_HARDWARE_CRC32
|
||||
+ __asm__ __volatile__(
|
||||
+ ".byte 0xF2, 0x0F, 0x38, 0xF0, 0xF1"
|
||||
+ : "=S" (crc)
|
||||
+ : "0" (crc), "c" (*p));
|
||||
+#else
|
||||
#ifdef WORDS_BIGENDIAN
|
||||
- for (len &= 0x7; len > 0; ++p, len--)
|
||||
crc = g_crc_slicing[0][((crc >> 24) ^ *p) & 0xFF] ^ (crc << 8);
|
||||
+#else
|
||||
+ crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8);
|
||||
+#endif
|
||||
+#endif
|
||||
+ }
|
||||
|
||||
+#ifdef WORDS_BIGENDIAN
|
||||
/* Do final byte swap to produce a result identical to little endian */
|
||||
crc =
|
||||
((crc << 24) & 0xFF000000) |
|
||||
((crc << 8) & 0x00FF0000) |
|
||||
((crc >> 8) & 0x0000FF00) |
|
||||
((crc >> 24) & 0x000000FF);
|
||||
-#else
|
||||
- for (len &= 0x7; len > 0; ++p, len--)
|
||||
- crc = g_crc_slicing[0][(crc ^ *p) & 0xFF] ^ (crc >> 8);
|
||||
#endif
|
||||
return (~crc);
|
||||
}
|
||||
|
|
219
c_src/wterl.c
219
c_src/wterl.c
|
@ -31,6 +31,7 @@
|
|||
#include "common.h"
|
||||
#include "async_nif.h"
|
||||
#include "queue.h"
|
||||
#include "cas.h"
|
||||
|
||||
#define MAX_CACHE_SIZE ASYNC_NIF_MAX_WORKERS
|
||||
|
||||
|
@ -59,6 +60,7 @@ typedef struct wterl_conn {
|
|||
STAILQ_HEAD(ctxs, wterl_ctx) cache;
|
||||
ErlNifMutex *cache_mutex;
|
||||
uint32_t cache_size;
|
||||
struct wterl_ctx *mru_ctx[ASYNC_NIF_MAX_WORKERS];
|
||||
} WterlConnHandle;
|
||||
|
||||
typedef struct {
|
||||
|
@ -204,7 +206,7 @@ __ctx_cache_evict(WterlConnHandle *conn_handle)
|
|||
STAILQ_REMOVE(&conn_handle->cache, c, wterl_ctx, entries);
|
||||
if (c->session)
|
||||
c->session->close(c->session, NULL);
|
||||
free(c);
|
||||
enif_free(c);
|
||||
num_evicted++;
|
||||
}
|
||||
}
|
||||
|
@ -286,7 +288,6 @@ __retain_ctx(WterlConnHandle *conn_handle, uint32_t worker_id,
|
|||
struct wterl_ctx **ctx,
|
||||
int count, const char *session_config, ...)
|
||||
{
|
||||
UNUSED(worker_id);
|
||||
int i = 0;
|
||||
uint32_t hash = 0;
|
||||
uint32_t crc = 0;
|
||||
|
@ -323,50 +324,74 @@ __retain_ctx(WterlConnHandle *conn_handle, uint32_t worker_id,
|
|||
DPRINTF("sig %llu [%u:%u]", PRIuint64(sig), crc, hash);
|
||||
va_end(ap);
|
||||
|
||||
// check the cache
|
||||
c = __ctx_cache_find(conn_handle, sig);
|
||||
if (c == NULL) {
|
||||
// cache miss:
|
||||
DPRINTF("[%.4u] cache miss: %llu [cache size: %d]", worker_id, PRIuint64(sig), conn_handle->cache_size);
|
||||
WT_CONNECTION *conn = conn_handle->conn;
|
||||
WT_SESSION *session = NULL;
|
||||
int rc = conn->open_session(conn, NULL, session_config, &session);
|
||||
if (rc != 0) return rc;
|
||||
size_t s = sizeof(struct wterl_ctx) + (count * sizeof(struct cursor_info)) + sig_len;
|
||||
c = malloc(s); // TODO: enif_alloc_resource()
|
||||
if (c == NULL) {
|
||||
session->close(session, NULL);
|
||||
return ENOMEM;
|
||||
}
|
||||
memset(c, 0, s);
|
||||
c->sig = sig;
|
||||
c->session = session;
|
||||
c->sig_len = sig_len;
|
||||
char *p = (char *)c + (s - sig_len);
|
||||
c->session_config = __copy_str_into(&p, session_config);
|
||||
c->num_cursors = count;
|
||||
session_config = arg;
|
||||
va_start(ap, session_config);
|
||||
for (i = 0; i < count; i++) {
|
||||
const char *uri = va_arg(ap, const char *);
|
||||
const char *config = va_arg(ap, const char *);
|
||||
// TODO: what to do (if anything) when uri or config is NULL?
|
||||
c->ci[i].uri = __copy_str_into(&p, uri);
|
||||
c->ci[i].config = __copy_str_into(&p, config);
|
||||
rc = session->open_cursor(session, uri, NULL, config, &c->ci[i].cursor);
|
||||
if (rc != 0) {
|
||||
free(c);
|
||||
session->close(session, NULL); // this will free the cursors too
|
||||
va_end(ap);
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
va_end(ap);
|
||||
} else {
|
||||
// cache hit:
|
||||
DPRINTF("[%.4u] cache hit: %llu [cache size: %d]", worker_id, PRIuint64(sig), conn_handle->cache_size);
|
||||
*ctx = NULL;
|
||||
|
||||
c = conn_handle->mru_ctx[worker_id];
|
||||
if (CASPO(&conn_handle->mru_ctx[worker_id], c, 0) == c) {
|
||||
if (c == 0) {
|
||||
// mru miss:
|
||||
DPRINTF("[%.4u] mru miss, empty", worker_id);
|
||||
*ctx = NULL;
|
||||
} else {
|
||||
if (c->sig == sig) {
|
||||
// mru hit:
|
||||
DPRINTF("[%.4u] mru hit: %llu found", worker_id, PRIuint64(sig));
|
||||
*ctx = c;
|
||||
} else {
|
||||
// mru mismatch:
|
||||
DPRINTF("[%.4u] mru miss: %llu != %llu", worker_id, PRIuint64(sig), PRIuint64(c->sig));
|
||||
__ctx_cache_add(conn_handle, c);
|
||||
*ctx = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (*ctx == NULL) {
|
||||
// check the cache
|
||||
(*ctx) = __ctx_cache_find(conn_handle, sig);
|
||||
if ((*ctx) == NULL) {
|
||||
// cache miss:
|
||||
DPRINTF("[%.4u] cache miss: %llu [cache size: %d]", worker_id, PRIuint64(sig), conn_handle->cache_size);
|
||||
WT_CONNECTION *conn = conn_handle->conn;
|
||||
WT_SESSION *session = NULL;
|
||||
int rc = conn->open_session(conn, NULL, session_config, &session);
|
||||
if (rc != 0) {
|
||||
return rc;
|
||||
}
|
||||
size_t s = sizeof(struct wterl_ctx) + (count * sizeof(struct cursor_info)) + sig_len;
|
||||
*ctx = enif_alloc(s); // TODO: enif_alloc_resource()
|
||||
if (*ctx == NULL) {
|
||||
session->close(session, NULL);
|
||||
return ENOMEM;
|
||||
}
|
||||
memset(*ctx, 0, s);
|
||||
(*ctx)->sig = sig;
|
||||
(*ctx)->session = session;
|
||||
(*ctx)->sig_len = sig_len;
|
||||
char *p = (char *)(*ctx) + (s - sig_len);
|
||||
(*ctx)->session_config = __copy_str_into(&p, session_config);
|
||||
(*ctx)->num_cursors = count;
|
||||
session_config = arg;
|
||||
va_start(ap, session_config);
|
||||
for (i = 0; i < count; i++) {
|
||||
const char *uri = va_arg(ap, const char *);
|
||||
const char *config = va_arg(ap, const char *);
|
||||
// TODO: what to do (if anything) when uri or config is NULL?
|
||||
(*ctx)->ci[i].uri = __copy_str_into(&p, uri);
|
||||
(*ctx)->ci[i].config = __copy_str_into(&p, config);
|
||||
rc = session->open_cursor(session, uri, NULL, config, &(*ctx)->ci[i].cursor);
|
||||
if (rc != 0) {
|
||||
enif_free(*ctx);
|
||||
session->close(session, NULL); // this will free the cursors too
|
||||
return rc;
|
||||
}
|
||||
}
|
||||
va_end(ap);
|
||||
} else {
|
||||
// cache hit:
|
||||
DPRINTF("[%.4u] cache hit: %llu [cache size: %d]", worker_id, PRIuint64(sig), conn_handle->cache_size);
|
||||
}
|
||||
}
|
||||
*ctx = c;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -376,16 +401,27 @@ __retain_ctx(WterlConnHandle *conn_handle, uint32_t worker_id,
|
|||
static void
|
||||
__release_ctx(WterlConnHandle *conn_handle, uint32_t worker_id, struct wterl_ctx *ctx)
|
||||
{
|
||||
UNUSED(worker_id);
|
||||
uint32_t i;
|
||||
WT_CURSOR *cursor;
|
||||
struct wterl_ctx *c = NULL;
|
||||
|
||||
for (i = 0; i < ctx->num_cursors; i++) {
|
||||
cursor = ctx->ci[i].cursor;
|
||||
cursor->reset(cursor);
|
||||
}
|
||||
__ctx_cache_add(conn_handle, ctx);
|
||||
DPRINTF("[%.4u] reset %d cursors, returnd ctx to cache", worker_id, ctx->num_cursors);
|
||||
|
||||
c = conn_handle->mru_ctx[worker_id];
|
||||
if (CASPO(&conn_handle->mru_ctx[worker_id], c, ctx) != c) {
|
||||
__ctx_cache_add(conn_handle, ctx);
|
||||
DPRINTF("[%.4u] reset %d cursors, returnd ctx to cache", worker_id, ctx->num_cursors);
|
||||
} else {
|
||||
if (c != NULL) {
|
||||
__ctx_cache_add(conn_handle, c);
|
||||
DPRINTF("[%.4u] reset %d cursors, returned ctx to cache", worker_id, ctx->num_cursors);
|
||||
} else {
|
||||
DPRINTF("[%.4u] reset %d cursors, returned ctx to mru", worker_id, ctx->num_cursors);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -397,6 +433,19 @@ void
|
|||
__close_all_sessions(WterlConnHandle *conn_handle)
|
||||
{
|
||||
struct wterl_ctx *c, *n;
|
||||
int worker_id;
|
||||
|
||||
// clear out the mru
|
||||
for (worker_id = 0; worker_id < ASYNC_NIF_MAX_WORKERS; worker_id++) {
|
||||
do {
|
||||
c = conn_handle->mru_ctx[worker_id];
|
||||
} while(CASPO(&conn_handle->mru_ctx[worker_id], c, 0) != c);
|
||||
|
||||
if (c != 0) {
|
||||
c->session->close(c->session, NULL);
|
||||
enif_free(c);
|
||||
}
|
||||
}
|
||||
|
||||
// clear out the cache
|
||||
c = STAILQ_FIRST(&conn_handle->cache);
|
||||
|
@ -405,7 +454,7 @@ __close_all_sessions(WterlConnHandle *conn_handle)
|
|||
STAILQ_REMOVE(&conn_handle->cache, c, wterl_ctx, entries);
|
||||
conn_handle->cache_size -= 1;
|
||||
c->session->close(c->session, NULL);
|
||||
free(c);
|
||||
enif_free(c);
|
||||
c = n;
|
||||
}
|
||||
}
|
||||
|
@ -419,9 +468,28 @@ void
|
|||
__close_cursors_on(WterlConnHandle *conn_handle, const char *uri)
|
||||
{
|
||||
struct wterl_ctx *c, *n;
|
||||
int idx, cnt;
|
||||
int worker_id, idx, cnt;
|
||||
|
||||
// walk the entries in the cache, look for open cursors on matching uri
|
||||
// walk the mru first, look for open cursors on matching uri
|
||||
for (worker_id = 0; worker_id < ASYNC_NIF_MAX_WORKERS; worker_id++) {
|
||||
c = conn_handle->mru_ctx[worker_id];
|
||||
if (CASPO(&conn_handle->mru_ctx[worker_id], c, 0) == c && c != 0) {
|
||||
cnt = c->num_cursors;
|
||||
for(idx = 0; idx < cnt; idx++) {
|
||||
if (!strcmp(c->ci[idx].uri, uri)) {
|
||||
c->session->close(c->session, NULL);
|
||||
enif_free(c);
|
||||
break;
|
||||
} else {
|
||||
if (CASPO(&conn_handle->mru_ctx[worker_id], 0, c) != 0) {
|
||||
__ctx_cache_add(conn_handle, c);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// next we walk the cache, look for open cursors on matching uri
|
||||
c = STAILQ_FIRST(&conn_handle->cache);
|
||||
while (c != NULL) {
|
||||
n = STAILQ_NEXT(c, entries);
|
||||
|
@ -431,7 +499,7 @@ __close_cursors_on(WterlConnHandle *conn_handle, const char *uri)
|
|||
STAILQ_REMOVE(&conn_handle->cache, c, wterl_ctx, entries);
|
||||
conn_handle->cache_size -= 1;
|
||||
c->session->close(c->session, NULL);
|
||||
free(c);
|
||||
enif_free(c);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
@ -440,7 +508,6 @@ __close_cursors_on(WterlConnHandle *conn_handle, const char *uri)
|
|||
return;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Callback to handle error messages.
|
||||
*
|
||||
|
@ -455,15 +522,13 @@ __close_cursors_on(WterlConnHandle *conn_handle, const char *uri)
|
|||
* operation or library failure.
|
||||
*/
|
||||
int
|
||||
__wterl_error_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session,
|
||||
int error, const char *message)
|
||||
__wterl_error_handler(WT_EVENT_HANDLER *handler, int error, const char *message)
|
||||
{
|
||||
struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
|
||||
ErlNifEnv *msg_env;
|
||||
ErlNifPid *to_pid;
|
||||
int rc = 0;
|
||||
|
||||
UNUSED(session);
|
||||
enif_mutex_lock(eh->error_mutex);
|
||||
msg_env = eh->msg_env_error;
|
||||
to_pid = &eh->to_pid;
|
||||
|
@ -495,14 +560,13 @@ __wterl_error_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session,
|
|||
* operation or library failure.
|
||||
*/
|
||||
int
|
||||
__wterl_message_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *message)
|
||||
__wterl_message_handler(WT_EVENT_HANDLER *handler, const char *message)
|
||||
{
|
||||
struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
|
||||
ErlNifEnv *msg_env;
|
||||
ErlNifPid *to_pid;
|
||||
int rc = 0;
|
||||
|
||||
UNUSED(session);
|
||||
enif_mutex_lock(eh->message_mutex);
|
||||
msg_env = eh->msg_env_message;
|
||||
to_pid = &eh->to_pid;
|
||||
|
@ -533,14 +597,13 @@ __wterl_message_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session, const ch
|
|||
* operation or library failure.
|
||||
*/
|
||||
int
|
||||
__wterl_progress_handler(WT_EVENT_HANDLER *handler, WT_SESSION *session, const char *operation, uint64_t counter)
|
||||
__wterl_progress_handler(WT_EVENT_HANDLER *handler, const char *operation, uint64_t counter)
|
||||
{
|
||||
struct wterl_event_handlers *eh = (struct wterl_event_handlers *)handler;
|
||||
ErlNifEnv *msg_env;
|
||||
ErlNifPid *to_pid;
|
||||
int rc = 0;
|
||||
|
||||
UNUSED(session);
|
||||
enif_mutex_lock(eh->progress_mutex);
|
||||
msg_env = eh->msg_env_progress;
|
||||
to_pid = &eh->to_pid;
|
||||
|
@ -642,7 +705,7 @@ ASYNC_NIF_DECL(
|
|||
return;
|
||||
}
|
||||
if (session_config.size > 1) {
|
||||
char *sc = malloc(session_config.size);
|
||||
char *sc = enif_alloc(session_config.size);
|
||||
if (!sc) {
|
||||
enif_release_resource(conn_handle);
|
||||
ASYNC_NIF_REPLY(__strerror_term(env, ENOMEM));
|
||||
|
@ -653,7 +716,7 @@ ASYNC_NIF_DECL(
|
|||
} else {
|
||||
conn_handle->session_config = NULL;
|
||||
}
|
||||
conn_handle->cache_mutex = enif_mutex_create("conn_handle");
|
||||
conn_handle->cache_mutex = enif_mutex_create(NULL);
|
||||
enif_mutex_lock(conn_handle->cache_mutex);
|
||||
conn_handle->conn = conn;
|
||||
ERL_NIF_TERM result = enif_make_resource(env, conn_handle);
|
||||
|
@ -703,7 +766,7 @@ ASYNC_NIF_DECL(
|
|||
enif_mutex_lock(args->conn_handle->cache_mutex);
|
||||
__close_all_sessions(args->conn_handle);
|
||||
if (args->conn_handle->session_config) {
|
||||
free((char *)args->conn_handle->session_config);
|
||||
enif_free((char *)args->conn_handle->session_config);
|
||||
args->conn_handle->session_config = NULL;
|
||||
}
|
||||
WT_CONNECTION* conn = args->conn_handle->conn;
|
||||
|
@ -1341,10 +1404,6 @@ ASYNC_NIF_DECL(
|
|||
ASYNC_NIF_REPLY(enif_make_badarg(env));
|
||||
return;
|
||||
}
|
||||
if (key.size == 0) {
|
||||
ASYNC_NIF_REPLY(enif_make_badarg(env));
|
||||
return;
|
||||
}
|
||||
|
||||
struct wterl_ctx *ctx = NULL;
|
||||
WT_CURSOR *cursor = NULL;
|
||||
|
@ -1404,10 +1463,6 @@ ASYNC_NIF_DECL(
|
|||
ASYNC_NIF_REPLY(enif_make_badarg(env));
|
||||
return;
|
||||
}
|
||||
if (key.size == 0) {
|
||||
ASYNC_NIF_REPLY(enif_make_badarg(env));
|
||||
return;
|
||||
}
|
||||
|
||||
struct wterl_ctx *ctx = NULL;
|
||||
WT_CURSOR *cursor = NULL;
|
||||
|
@ -1493,10 +1548,6 @@ ASYNC_NIF_DECL(
|
|||
ASYNC_NIF_REPLY(enif_make_badarg(env));
|
||||
return;
|
||||
}
|
||||
if (key.size == 0 || value.size == 0) {
|
||||
ASYNC_NIF_REPLY(enif_make_badarg(env));
|
||||
return;
|
||||
}
|
||||
|
||||
struct wterl_ctx *ctx = NULL;
|
||||
WT_CURSOR *cursor = NULL;
|
||||
|
@ -1570,7 +1621,7 @@ ASYNC_NIF_DECL(
|
|||
}
|
||||
|
||||
WT_CURSOR* cursor;
|
||||
rc = session->open_cursor(session, args->uri, NULL, (config.data[0] != 0) ? (char *)config.data : "raw", &cursor);
|
||||
rc = session->open_cursor(session, args->uri, NULL, (config.data[0] != 0) ? (char *)config.data : "overwrite,raw", &cursor);
|
||||
if (rc != 0) {
|
||||
session->close(session, NULL);
|
||||
ASYNC_NIF_REPLY(__strerror_term(env, rc));
|
||||
|
@ -2270,15 +2321,15 @@ on_load(ErlNifEnv *env, void **priv_data, ERL_NIF_TERM load_info)
|
|||
ATOM_WIREDTIGER_VSN = enif_make_atom(env, "wiredtiger_vsn");
|
||||
ATOM_MSG_PID = enif_make_atom(env, "message_pid");
|
||||
|
||||
struct wterl_priv_data *priv = malloc(sizeof(struct wterl_priv_data));
|
||||
struct wterl_priv_data *priv = enif_alloc(sizeof(struct wterl_priv_data));
|
||||
if (!priv)
|
||||
return ENOMEM;
|
||||
memset(priv, 0, sizeof(struct wterl_priv_data));
|
||||
|
||||
struct wterl_event_handlers *eh = &priv->eh;
|
||||
eh->error_mutex = enif_mutex_create("error_mutex");
|
||||
eh->message_mutex = enif_mutex_create("message_mutex");
|
||||
eh->progress_mutex = enif_mutex_create("progress_mutex");
|
||||
eh->error_mutex = enif_mutex_create(NULL);
|
||||
eh->message_mutex = enif_mutex_create(NULL);
|
||||
eh->progress_mutex = enif_mutex_create(NULL);
|
||||
|
||||
/* Process the load_info array of tuples, we expect:
|
||||
[{wterl_vsn, "a version string"},
|
||||
|
@ -2298,17 +2349,17 @@ on_load(ErlNifEnv *env, void **priv_data, ERL_NIF_TERM load_info)
|
|||
|
||||
/* Note: !!! the first element of our priv_data struct *must* be the
|
||||
pointer to the async_nif's private data which we set here. */
|
||||
ASYNC_NIF_LOAD(wterl, env, priv->async_nif_priv);
|
||||
ASYNC_NIF_LOAD(wterl, priv->async_nif_priv);
|
||||
if (!priv->async_nif_priv) {
|
||||
memset(priv, 0, sizeof(struct wterl_priv_data));
|
||||
free(priv);
|
||||
enif_free(priv);
|
||||
return ENOMEM;
|
||||
}
|
||||
*priv_data = priv;
|
||||
|
||||
char msg[1024];
|
||||
snprintf(msg, 1024, "NIF on_load complete (wterl version: %s, wiredtiger version: %s)", priv->wterl_vsn, priv->wiredtiger_vsn);
|
||||
__wterl_message_handler((WT_EVENT_HANDLER *)&priv->eh, NULL, msg);
|
||||
__wterl_message_handler((WT_EVENT_HANDLER *)&priv->eh, msg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -2347,7 +2398,7 @@ on_unload(ErlNifEnv *env, void *priv_data)
|
|||
enif_free_env(eh->msg_env_progress);
|
||||
|
||||
memset(priv, 0, sizeof(struct wterl_priv_data));
|
||||
free(priv);
|
||||
enif_free(priv);
|
||||
|
||||
priv_data = NULL;
|
||||
}
|
||||
|
|
|
@ -1,6 +0,0 @@
|
|||
%%%% This is the WiredTiger section
|
||||
|
||||
%% @doc wiredtiger data_root
|
||||
{mapping, "wiredtiger.data_root", "wterl.data_root", [
|
||||
{default, "{{platform_data_dir}}/wiredtiger"}
|
||||
]}.
|
16
rebar.config
16
rebar.config
|
@ -5,15 +5,13 @@
|
|||
|
||||
{cover_enabled, true}.
|
||||
|
||||
%{eunit_opts, [verbose, {report, {eunit_surefire, [{dir, "."}]}}]}.
|
||||
{eunit_opts, [verbose, {report, {eunit_surefire, [{dir, "."}]}}]}.
|
||||
|
||||
{erl_opts, [
|
||||
%native, {hipe, [o3,verbose]}, inline, {inline_size, 1024},
|
||||
{parse_transform, lager_transform},
|
||||
debug_info,
|
||||
{d,'DEBUG',true},
|
||||
strict_validation,
|
||||
fail_on_warning,
|
||||
debug_info, %{d,'DEBUG',true},
|
||||
%strict_validation,
|
||||
%fail_on_warning,
|
||||
%warn_missing_spec,
|
||||
warn_bif_clash,
|
||||
warn_deprecated_function,
|
||||
|
@ -24,7 +22,7 @@
|
|||
warn_shadow_vars,
|
||||
warn_untyped_record,
|
||||
warn_unused_function,
|
||||
warn_unused_import,
|
||||
%warn_unused_import,
|
||||
warn_unused_record,
|
||||
warn_unused_vars
|
||||
]}.
|
||||
|
@ -38,8 +36,8 @@
|
|||
{port_specs, [{"priv/wterl.so", ["c_src/*.c"]}]}.
|
||||
|
||||
{port_env, [
|
||||
{"DRV_CFLAGS", "$DRV_CFLAGS -O3 -mtune=native -march=native -fPIC -Wall -Wextra -Werror -I c_src/system/include"},
|
||||
{"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath,lib/wterl/priv:lib/wterl-0.9.0/priv:priv -Lc_src/system/lib -lwiredtiger"}
|
||||
{"DRV_CFLAGS", "$DRV_CFLAGS -fPIC -Wall -Wextra -Werror -I c_src/system/include"},
|
||||
{"DRV_LDFLAGS", "$DRV_LDFLAGS -Wl,-rpath,lib/wterl/priv:priv -Lc_src/system/lib -lwiredtiger"}
|
||||
]}.
|
||||
|
||||
{pre_hooks, [{compile, "c_src/build_deps.sh compile"}]}.
|
||||
|
|
|
@ -21,34 +21,27 @@
|
|||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-define(ASYNC_NIF_CALL(Fun, Args),
|
||||
F = fun(F, T) ->
|
||||
R = erlang:make_ref(),
|
||||
case erlang:apply(Fun, [R|Args]) of
|
||||
{ok, {enqueued, PctBusy}} ->
|
||||
if
|
||||
PctBusy > 0.25 andalso PctBusy =< 1.0 ->
|
||||
erlang:bump_reductions(erlang:trunc(2000 * PctBusy));
|
||||
true ->
|
||||
ok
|
||||
end,
|
||||
receive
|
||||
{R, {error, shutdown}=Error} ->
|
||||
%% Work unit was queued, but not executed.
|
||||
Error;
|
||||
{R, {error, _Reason}=Error} ->
|
||||
%% Work unit returned an error.
|
||||
Error;
|
||||
{R, Reply} ->
|
||||
Reply
|
||||
end;
|
||||
{error, eagain} ->
|
||||
case T of
|
||||
3 -> not_found;
|
||||
_ -> F(F, T + 1)
|
||||
end;
|
||||
Other ->
|
||||
Other
|
||||
end
|
||||
end,
|
||||
F(F, 1)).
|
||||
-spec async_nif_enqueue(reference(), function(), [term()]) -> term() | {error, term()}.
|
||||
async_nif_enqueue(R, F, A) ->
|
||||
case erlang:apply(F, [R|A]) of
|
||||
{ok, enqueued} ->
|
||||
receive
|
||||
{R, {error, shutdown}=Error} ->
|
||||
%% Work unit was queued, but not executed.
|
||||
Error;
|
||||
{R, {error, _Reason}=Error} ->
|
||||
%% Work unit returned an error.
|
||||
Error;
|
||||
{R, Reply} ->
|
||||
Reply
|
||||
end;
|
||||
{error, eagain} ->
|
||||
%% Work unit was not queued, try again.
|
||||
async_nif_enqueue(R, F, A);
|
||||
%{error, enomem} ->
|
||||
%{error, shutdown} ->
|
||||
Other ->
|
||||
Other
|
||||
end.
|
||||
|
||||
-define(ASYNC_NIF_CALL(Fun, Args), async_nif_enqueue(erlang:make_ref(), Fun, Args)).
|
||||
|
|
|
@ -22,7 +22,6 @@
|
|||
|
||||
-module(riak_kv_wterl_backend).
|
||||
-behavior(temp_riak_kv_backend).
|
||||
-compile([{parse_transform, lager_transform}]).
|
||||
|
||||
%% KV Backend API
|
||||
-export([api_version/0,
|
||||
|
@ -43,7 +42,7 @@
|
|||
|
||||
-ifdef(TEST).
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-compile(export_all).
|
||||
-compiel(export_all).
|
||||
-endif.
|
||||
|
||||
-define(API_VERSION, 1).
|
||||
|
@ -110,24 +109,24 @@ start(Partition, Config) ->
|
|||
Compressor =
|
||||
case wterl:config_value(block_compressor, Config, "snappy") of
|
||||
{block_compressor, "snappy"}=C -> [C];
|
||||
{block_compressor, "none"} -> [];
|
||||
{block_compressor, none} -> [];
|
||||
{block_compressor, _} -> [{block_compressor, "snappy"}];
|
||||
_ -> [{block_compressor, "snappy"}]
|
||||
{block_compressor, "none"} -> [];
|
||||
{block_compressor, none} -> [];
|
||||
{block_compressor, _} -> [{block_compressor, "snappy"}];
|
||||
_ -> [{block_compressor, "snappy"}]
|
||||
end,
|
||||
TableOpts =
|
||||
case Type of
|
||||
"lsm" ->
|
||||
[{internal_page_max, "128K"},
|
||||
{leaf_page_max, "16K"},
|
||||
{lsm, [
|
||||
{bloom_config, [{leaf_page_max, "8MB"}]},
|
||||
{bloom_bit_count, 28},
|
||||
{bloom_hash_count, 19},
|
||||
{bloom_oldest, true},
|
||||
{chunk_size, "100MB"},
|
||||
{merge_threads, 2}
|
||||
]}
|
||||
{leaf_page_max, "128K"},
|
||||
{lsm_chunk_size, "100MB"},
|
||||
{lsm_merge_threads, 2},
|
||||
{prefix_compression, false},
|
||||
{lsm_bloom_newest, true},
|
||||
{lsm_bloom_oldest, true} ,
|
||||
{lsm_bloom_bit_count, 128},
|
||||
{lsm_bloom_hash_count, 64},
|
||||
{lsm_bloom_config, [{leaf_page_max, "8MB"}]}
|
||||
] ++ Compressor;
|
||||
"table" ->
|
||||
Compressor
|
||||
|
@ -342,23 +341,22 @@ is_empty(#state{connection=Connection, table=Table}) ->
|
|||
%% @doc Get the status information for this wterl backend
|
||||
-spec status(state()) -> [{atom(), term()}].
|
||||
status(#state{connection=Connection, table=Table}) ->
|
||||
[].
|
||||
%% case wterl:cursor_open(Connection, "statistics:" ++ Table, [{statistics_fast, true}]) of
|
||||
%% {ok, Cursor} ->
|
||||
%% TheStats =
|
||||
%% case fetch_status(Cursor) of
|
||||
%% {ok, Stats} ->
|
||||
%% Stats;
|
||||
%% {error, {eperm, _}} -> % TODO: review/fix this logic
|
||||
%% {ok, []};
|
||||
%% _ ->
|
||||
%% {ok, []}
|
||||
%% end,
|
||||
%% wterl:cursor_close(Cursor),
|
||||
%% TheStats;
|
||||
%% {error, Reason2} ->
|
||||
%% {error, Reason2}
|
||||
%% end.
|
||||
case wterl:cursor_open(Connection, Table) of
|
||||
{ok, Cursor} ->
|
||||
TheStats =
|
||||
case fetch_status(Cursor) of
|
||||
{ok, Stats} ->
|
||||
Stats;
|
||||
{error, {eperm, _}} -> % TODO: review/fix this logic
|
||||
{ok, []};
|
||||
_ ->
|
||||
{ok, []}
|
||||
end,
|
||||
wterl:cursor_close(Cursor),
|
||||
TheStats;
|
||||
{error, Reason2} ->
|
||||
{error, Reason2}
|
||||
end.
|
||||
|
||||
%% @doc Register an asynchronous callback
|
||||
-spec callback(reference(), any(), state()) -> {ok, state()}.
|
||||
|
@ -377,17 +375,10 @@ max_sessions(Config) ->
|
|||
undefined -> 1024;
|
||||
Size -> Size
|
||||
end,
|
||||
Est = RingSize * erlang:system_info(schedulers),
|
||||
case Est > 8192 of
|
||||
true ->
|
||||
8192;
|
||||
false ->
|
||||
case Est < 1024 of
|
||||
true ->
|
||||
1024;
|
||||
false ->
|
||||
Est
|
||||
end
|
||||
Est = 100 * (RingSize * erlang:system_info(schedulers)), % TODO: review/fix this logic
|
||||
case Est > 1000000000 of % Note: WiredTiger uses a signed int for this
|
||||
true -> 1000000000;
|
||||
false -> Est
|
||||
end.
|
||||
|
||||
%% @private
|
||||
|
@ -401,41 +392,33 @@ establish_connection(Config, Type) ->
|
|||
ok = filelib:ensure_dir(filename:join(DataRoot, "x")),
|
||||
|
||||
%% WT Connection Options:
|
||||
LogSetting = app_helper:get_prop_or_env(log, Config, wterl, false),
|
||||
%% NOTE: LSM auto-checkpoints, so we don't have too.
|
||||
CheckpointSetting =
|
||||
case Type =:= "lsm" of
|
||||
true ->
|
||||
case LogSetting of
|
||||
true ->
|
||||
%% Turn checkpoints on if logging is on, checkpoints enable log archival.
|
||||
app_helper:get_prop_or_env(checkpoint, Config, wterl, [{wait, 30}]); % in seconds
|
||||
_ ->
|
||||
[]
|
||||
end;
|
||||
[];
|
||||
false ->
|
||||
app_helper:get_prop_or_env(checkpoint, Config, wterl, [{wait, 30}])
|
||||
app_helper:get_prop_or_env(checkpoint, Config, wterl, [{wait, 10}])
|
||||
end,
|
||||
RequestedCacheSize = app_helper:get_prop_or_env(cache_size, Config, wterl),
|
||||
ConnectionOpts =
|
||||
orddict:from_list(
|
||||
[ wterl:config_value(create, Config, true),
|
||||
wterl:config_value(checkpoint_sync, Config, false),
|
||||
wterl:config_value(transaction_sync, Config, "none"),
|
||||
wterl:config_value(log, Config, [{enabled, LogSetting}]),
|
||||
wterl:config_value(mmap, Config, false),
|
||||
wterl:config_value(checkpoint, Config, CheckpointSetting),
|
||||
wterl:config_value(sync, Config, false),
|
||||
wterl:config_value(logging, Config, true),
|
||||
wterl:config_value(transactional, Config, true),
|
||||
wterl:config_value(session_max, Config, max_sessions(Config)),
|
||||
wterl:config_value(cache_size, Config, size_cache(RequestedCacheSize)),
|
||||
wterl:config_value(statistics, Config, [ "fast", "clear"]),
|
||||
wterl:config_value(statistics_log, Config, [{wait, 600}]), % in seconds
|
||||
wterl:config_value(statistics_log, Config, [{wait, 300}]), % sec
|
||||
wterl:config_value(verbose, Config, [ "salvage", "verify"
|
||||
% Note: for some unknown reason, if you add these additional
|
||||
% verbose flags Erlang SEGV's "size_object: bad tag for 0x80"
|
||||
% no idea why... you've been warned.
|
||||
% no idea why... yet... you've been warned.
|
||||
|
||||
%"block", "shared_cache", "reconcile", "evict", "lsm",
|
||||
%"fileops", "read", "write", "readserver", "evictserver",
|
||||
%"hazard", "mutex", "ckpt"
|
||||
]) ] ++ proplists:get_value(wterl, Config, [])), % sec
|
||||
]) ] ++ CheckpointSetting ++ proplists:get_value(wterl, Config, [])), % sec
|
||||
|
||||
%% WT Session Options:
|
||||
SessionOpts = [{isolation, "snapshot"}],
|
||||
|
@ -556,15 +539,15 @@ from_index_key(LKey) ->
|
|||
|
||||
%% @private
|
||||
%% Return all status from wterl statistics cursor
|
||||
%% fetch_status(Cursor) ->
|
||||
%% {ok, fetch_status(Cursor, wterl:cursor_next_value(Cursor), [])}.
|
||||
%% fetch_status(_Cursor, {error, _}, Acc) ->
|
||||
%% lists:reverse(Acc);
|
||||
%% fetch_status(_Cursor, not_found, Acc) ->
|
||||
%% lists:reverse(Acc);
|
||||
%% fetch_status(Cursor, {ok, Stat}, Acc) ->
|
||||
%% [What,Val|_] = [binary_to_list(B) || B <- binary:split(Stat, [<<0>>], [global])],
|
||||
%% fetch_status(Cursor, wterl:cursor_next_value(Cursor), [{What,Val}|Acc]).
|
||||
fetch_status(Cursor) ->
|
||||
{ok, fetch_status(Cursor, wterl:cursor_next_value(Cursor), [])}.
|
||||
fetch_status(_Cursor, {error, _}, Acc) ->
|
||||
lists:reverse(Acc);
|
||||
fetch_status(_Cursor, not_found, Acc) ->
|
||||
lists:reverse(Acc);
|
||||
fetch_status(Cursor, {ok, Stat}, Acc) ->
|
||||
[What,Val|_] = [binary_to_list(B) || B <- binary:split(Stat, [<<0>>], [global])],
|
||||
fetch_status(Cursor, wterl:cursor_next_value(Cursor), [{What,Val}|Acc]).
|
||||
|
||||
size_cache(RequestedSize) ->
|
||||
Size =
|
||||
|
@ -579,7 +562,7 @@ size_cache(RequestedSize) ->
|
|||
TotalRAM = proplists:get_value(system_total_memory, Memory),
|
||||
FreeRAM = proplists:get_value(free_memory, Memory),
|
||||
UsedByBeam = proplists:get_value(total, erlang:memory()),
|
||||
Target = ((TotalRAM - UsedByBeam) div 3),
|
||||
Target = ((TotalRAM - UsedByBeam) div 4),
|
||||
FirstGuess = (Target - (Target rem (1024 * 1024))),
|
||||
SecondGuess =
|
||||
case FirstGuess > FreeRAM of
|
||||
|
|
|
@ -96,8 +96,8 @@ nif_stub_error(Line) ->
|
|||
-spec init() -> ok | {error, any()}.
|
||||
init() ->
|
||||
erlang:load_nif(filename:join([priv_dir(), atom_to_list(?MODULE)]),
|
||||
[{wterl_vsn, "942e51b"},
|
||||
{wiredtiger_vsn, "1.6.4-275-g9c44420"}]). %% TODO automate these
|
||||
[{wterl_vsn, "53307e8"},
|
||||
{wiredtiger_vsn, "1.6.2-0-g07cb0a5"}]).
|
||||
|
||||
-spec connection_open(string(), config_list()) -> {ok, connection()} | {error, term()}.
|
||||
-spec connection_open(string(), config_list(), config_list()) -> {ok, connection()} | {error, term()}.
|
||||
|
@ -256,7 +256,6 @@ verify_nif(_AsyncRef, _Ref, _Name, _Config) ->
|
|||
-spec cursor_open(connection(), string(), config_list()) -> {ok, cursor()} | {error, term()}.
|
||||
cursor_open(Ref, Table) ->
|
||||
cursor_open(Ref, Table, []).
|
||||
|
||||
cursor_open(Ref, Table, Config) ->
|
||||
?ASYNC_NIF_CALL(fun cursor_open_nif/4, [Ref, Table, config_to_bin(Config)]).
|
||||
|
||||
|
@ -454,26 +453,17 @@ config_to_bin([], Acc) ->
|
|||
config_to_bin([{Key, Value} | Rest], Acc) ->
|
||||
ConfigTypes =
|
||||
[{block_compressor, {string, quoted}},
|
||||
{bloom_bit_count, integer},
|
||||
{bloom_config, config},
|
||||
{bloom_hash_count, integer},
|
||||
{bloom_newest, bool},
|
||||
{bloom_oldest, bool},
|
||||
{cache_size, string},
|
||||
{checkpoint, config},
|
||||
{checkpoint_sync, bool},
|
||||
{checksum, string},
|
||||
{chunk_size, string},
|
||||
{create, bool},
|
||||
{direct_io, list},
|
||||
{drop, list},
|
||||
{enabled, bool},
|
||||
{error_prefix, string},
|
||||
{eviction_target, integer},
|
||||
{eviction_trigger, integer},
|
||||
{extensions, {list, quoted}},
|
||||
{statistics_fast, bool},
|
||||
{file_max, string},
|
||||
{force, bool},
|
||||
{from, string},
|
||||
{hazard_max, integer},
|
||||
|
@ -483,21 +473,22 @@ config_to_bin([{Key, Value} | Rest], Acc) ->
|
|||
{isolation, string},
|
||||
{key_type, string},
|
||||
{leaf_page_max, string},
|
||||
{log, config},
|
||||
{lsm, config},
|
||||
{mmap, bool},
|
||||
{merge_threads, integer},
|
||||
{logging, bool},
|
||||
{lsm_bloom_bit_count, integer},
|
||||
{lsm_bloom_config, config},
|
||||
{lsm_bloom_hash_count, integer},
|
||||
{lsm_bloom_newest, bool},
|
||||
{lsm_bloom_oldest, bool},
|
||||
{lsm_chunk_size, string},
|
||||
{prefix_compression, bool},
|
||||
{lsm_merge_threads, integer},
|
||||
{multiprocess, bool},
|
||||
{name, string},
|
||||
{overwrite, bool},
|
||||
{prefix_compression, bool},
|
||||
{raw, bool},
|
||||
{session_max, integer},
|
||||
{statistics, list},
|
||||
{statistics_log, config},
|
||||
{sync, bool},
|
||||
{target, {list, quoted}},
|
||||
{to, string},
|
||||
{transaction_sync, string},
|
||||
{transactional, bool},
|
||||
{verbose, list},
|
||||
{wait, integer}],
|
||||
|
@ -619,7 +610,7 @@ many_open_tables_test_() ->
|
|||
DataDir = ?TEST_DATA_DIR,
|
||||
KeyGen =
|
||||
fun(X) ->
|
||||
crypto:hash(sha, <<X>>)
|
||||
crypto:sha(<<X>>)
|
||||
end,
|
||||
ValGen =
|
||||
fun() ->
|
||||
|
@ -881,7 +872,7 @@ various_cursor_test_() ->
|
|||
end},
|
||||
{"update an item using a cursor",
|
||||
fun() ->
|
||||
{ok, Cursor} = cursor_open(ConnRef, "table:test", [{overwrite, false}, {raw,true}]),
|
||||
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
|
||||
?assertMatch(ok, cursor_update(Cursor, <<"g">>, <<"goji berries">>)),
|
||||
?assertMatch(not_found, cursor_update(Cursor, <<"k">>, <<"kumquat">>)),
|
||||
?assertMatch(ok, cursor_close(Cursor)),
|
||||
|
@ -889,7 +880,7 @@ various_cursor_test_() ->
|
|||
end},
|
||||
{"remove an item using a cursor",
|
||||
fun() ->
|
||||
{ok, Cursor} = cursor_open(ConnRef, "table:test", [{overwrite, false}, {raw,true}]),
|
||||
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
|
||||
?assertMatch(ok, cursor_remove(Cursor, <<"g">>)),
|
||||
?assertMatch(not_found, cursor_remove(Cursor, <<"l">>)),
|
||||
?assertMatch(ok, cursor_close(Cursor)),
|
||||
|
|
|
@ -25,13 +25,7 @@
|
|||
|
||||
{mode, max}.
|
||||
{duration, 10}.
|
||||
{concurrent, 16}.
|
||||
{report_interval, 1}.
|
||||
{pb_timeout_general, 1000}. % ms
|
||||
%{pb_timeout_read, ?}.
|
||||
%{pb_timeout_write, ?}.
|
||||
%{pb_timeout_listkeys, ?}.
|
||||
%{pb_timeout_mapreduce, ?}.
|
||||
{concurrent, 4}.
|
||||
{driver, basho_bench_driver_wterl}.
|
||||
{key_generator, {int_to_bin_littleendian,{uniform_int, 5000000}}}.
|
||||
{value_generator, {fixed_bin, 10000}}.
|
||||
|
@ -43,9 +37,9 @@
|
|||
{wterl, [
|
||||
{connection, [
|
||||
{create, true},
|
||||
{session_sync, false},
|
||||
{transaction_sync, "none"},
|
||||
{log, [{enabled, false}]},
|
||||
{sync, false},
|
||||
{logging, true},
|
||||
{transactional, true},
|
||||
{session_max, 1024},
|
||||
{cache_size, 4294967296},
|
||||
{verbose, []},
|
||||
|
@ -60,11 +54,11 @@
|
|||
]},
|
||||
{session, [ {isolation, "snapshot"} ]},
|
||||
{table_uri, "lsm:test"},
|
||||
{lsm_merge_threads, 2},
|
||||
{table, [
|
||||
{internal_page_max, "128K"},
|
||||
{leaf_page_max, "128K"},
|
||||
{lsm_chunk_size, "25MB"},
|
||||
{prefix_compression, false},
|
||||
{lsm_bloom_newest, true},
|
||||
{lsm_bloom_oldest, true} ,
|
||||
{lsm_bloom_bit_count, 128},
|
||||
|
@ -78,9 +72,9 @@
|
|||
{wterl_, [
|
||||
{connection, [
|
||||
{create, true},
|
||||
{session_sync, false},
|
||||
{transaction_sync, "none"},
|
||||
{log, [{enabled, false}]},
|
||||
{sync, false},
|
||||
{logging, true},
|
||||
{transactional, true},
|
||||
{session_max, 1024},
|
||||
{cache_size, 4294967296},
|
||||
{verbose, []},
|
||||
|
@ -97,6 +91,7 @@
|
|||
{session, [ {isolation, "snapshot"} ]},
|
||||
{table_uri, "table:test"},
|
||||
{table, [
|
||||
{prefix_compression, false},
|
||||
{block_compressor, "snappy"} % bzip2
|
||||
]}
|
||||
]}.
|
||||
|
|
Loading…
Reference in a new issue