greg/wterl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

WIP-- Not Yet Compiling or Functional -- WIP

Browse source 
* No longer expose WT_SESSION into Erlang at all as WT's model is to
maintain one WT_SESSION per-thread and we don't know anything about
threads in Erlang.

* async_nif worker threads don't pull both mutexes on every loop
when processing requests, only one

* async_nif provides a worker_id (int, 0 - MAX_WORKERS) within the
work block scope which we use to find our per-worker WT_SESSIONs

* async_nif maintained a number of globals which I'm moving into
the NIF's priv_data so that on upgrade/reload we have a fighting
chance to "Do the Right Thing(TM)".

* NIF Upgrades/Reloads started to plumb this in.

* Use a khash to manage the cache of URI->WT_CURSORs per WT_SESSION.

* Added start/stop positions into truncate call to allow for truncating
sub-ranges data.

* Lots of other details I'm sure I've forgotten and more left undone.
Search for "TODO:" or try to compile to see what's left, and then
there is a need for a lot more tests given all this new complexity.
This commit is contained in:
Gregory Burd 2013-04-05 18:09:54 -04:00
parent 163a5073cb
commit 19268b7c77
6 changed files with 1843 additions and 800 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

250
c_src/async_nif.h
View file

@ -1,5 +1,4 @@
/*
*
* async_nif: An async thread-pool layer for Erlang's NIF API
*
* Copyright (c) 2012 Basho Technologies, Inc. All Rights Reserved.
@ -18,7 +17,6 @@
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
#ifndef __ASYNC_NIF_H__
@ -28,49 +26,42 @@
extern "C" {
#endif
/* Redefine this in your NIF implementation before including this file to
change the thread pool size. The maximum number of threads might be
bounded on your OS. For instance, to allow 1,000,000 threads on a Linux
system you must do the following before launching the process.
echo 1000000 > /proc/sys/kernel/threads-max
and for all UNIX systems there will be ulimit maximums. */
#ifndef ASYNC_NIF_MAX_WORKERS
#define ASYNC_NIF_MAX_WORKERS 16
#endif
#include "queue.h"
#define ASYNC_NIF_MAX_WORKERS 32
struct async_nif_req_entry {
ERL_NIF_TERM ref, *argv;
ErlNifEnv *env;
ErlNifPid pid;
void *args;
void *priv_data;
void (*fn_work)(ErlNifEnv*, ERL_NIF_TERM, void *, ErlNifPid*, void *);
void (*fn_work)(ErlNifEnv*, ERL_NIF_TERM, ErlNifPid*, unsigned int, void *);
void (*fn_post)(void *);
STAILQ_ENTRY(async_nif_req_entry) entries;
};
STAILQ_HEAD(reqs, async_nif_req_entry) async_nif_reqs = STAILQ_HEAD_INITIALIZER(async_nif_reqs);
struct async_nif_worker_entry {
ErlNifTid tid;
LIST_ENTRY(async_nif_worker_entry) entries;
};
LIST_HEAD(idle_workers, async_nif_worker_entry) async_nif_idle_workers = LIST_HEAD_INITIALIZER(async_nif_worker);
static volatile unsigned int async_nif_req_count = 0;
static volatile unsigned int async_nif_shutdown = 0;
static ErlNifMutex *async_nif_req_mutex = NULL;
static ErlNifMutex *async_nif_worker_mutex = NULL;
static ErlNifCond *async_nif_cnd = NULL;
static struct async_nif_worker_entry async_nif_worker_entries[ASYNC_NIF_MAX_WORKERS];
struct async_nif_state {
volatile unsigned int req_count;
volatile unsigned int shutdown;
ErlNifMutex *req_mutex;
ErlNifMutex *worker_mutex;
ErlNifCond *cnd;
STAILQ_HEAD(reqs, async_nif_req_entry) reqs;
LIST_HEAD(workers, async_nif_worker_entry) workers;
unsigned int num_workers;
struct async_nif_worker_entry worker_entries[ASYNC_NIF_MAX_WORKERS];
};
#define ASYNC_NIF_DECL(decl, frame, pre_block, work_block, post_block) \
struct decl ## _args frame; \
static void fn_work_ ## decl (ErlNifEnv *env, ERL_NIF_TERM ref, void *priv_data, ErlNifPid *pid, struct decl ## _args *args) work_block \
static void fn_work_ ## decl (ErlNifEnv *env, ERL_NIF_TERM ref, ErlNifPid *pid, unsigned int worker_id, struct decl ## _args *args) work_block \
static void fn_post_ ## decl (struct decl ## _args *args) { \
do post_block while(0); \
do post_block while(0); \
} \
static ERL_NIF_TERM decl(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv_in[]) { \
struct decl ## _args on_stack_args; \
@ -81,7 +72,8 @@ static struct async_nif_worker_entry async_nif_worker_entries[ASYNC_NIF_MAX_WORK
/* argv[0] is a ref used for selective recv */ \
const ERL_NIF_TERM *argv = argv_in + 1; \
argc--; \
if (async_nif_shutdown) \
async_nif = (struct async_nif_state*)enif_priv_data(env); \
if (async_nif->shutdown) \
return enif_make_tuple2(env, enif_make_atom(env, "error"), \
enif_make_atom(env, "shutdown")); \
if (!(new_env = enif_alloc_env())) { \
@ -108,17 +100,14 @@ static struct async_nif_worker_entry async_nif_worker_entries[ASYNC_NIF_MAX_WORK
req->ref = enif_make_copy(new_env, argv_in[0]); \
enif_self(env, &req->pid); \
req->args = (void*)copy_of_args; \
req->priv_data = enif_priv_data(env); \
req->fn_work = (void (*)(ErlNifEnv *, ERL_NIF_TERM, void*, ErlNifPid*, void *))fn_work_ ## decl ; \
req->fn_work = (void (*)(ErlNifEnv *, ERL_NIF_TERM, ErlNifPid*, unsigned int, void *))fn_work_ ## decl ; \
req->fn_post = (void (*)(void *))fn_post_ ## decl; \
async_nif_enqueue_req(req); \
return enif_make_tuple2(env, enif_make_atom(env, "ok"), \
enif_make_tuple2(env, enif_make_atom(env, "enqueued"), \
enif_make_int(env, async_nif_req_count))); \
return async_nif_enqueue_req(async_nif, req); \
}
#define ASYNC_NIF_LOAD() if (async_nif_init() != 0) return -1;
#define ASYNC_NIF_LOAD() async_nif_load();
#define ASYNC_NIF_UNLOAD() async_nif_unload();
//define ASYNC_NIF_RELOAD()
#define ASYNC_NIF_UPGRADE() async_nif_unload();
#define ASYNC_NIF_RETURN_BADARG() return enif_make_badarg(env);
@ -130,14 +119,25 @@ static struct async_nif_worker_entry async_nif_worker_entries[ASYNC_NIF_MAX_WORK
#define ASYNC_NIF_REPLY(msg) PULSE_SEND(NULL, pid, env, enif_make_tuple2(env, ref, msg))
#endif
static void async_nif_enqueue_req(struct async_nif_req_entry *r)
static ERL_NIF_TERM
async_nif_enqueue_req(struct async_nif_state* async_nif, struct async_nif_req_entry *req)
{
/* Add the request to the work queue. */
enif_mutex_lock(async_nif_req_mutex);
STAILQ_INSERT_TAIL(&async_nif_reqs, r, entries);
async_nif_req_count++;
enif_mutex_unlock(async_nif_req_mutex);
enif_cond_broadcast(async_nif_cnd);
/* If we're shutting down return an error term and ignore the request. */
if (async_nif->shutdown) {
return enif_make_tuple2(req->env, enif_make_atom(req->env, "error"),
enif_make_atom(req->env, "shutdown")));
}
/* Otherwise, add the request to the work queue. */
enif_mutex_lock(async_nif->req_mutex);
STAILQ_INSERT_TAIL(&async_nif->reqs, req, entries);
async_nif->req_count++;
enif_mutex_unlock(async_nif->req_mutex);
enif_cond_broadcast(async_nif->cnd);
return enif_make_tuple2(env, enif_make_atom(env, "ok"),
enif_make_tuple2(env, enif_make_atom(env, "enqueued"),
enif_make_int(env, async_nif->req_count))); \
}
static void *async_nif_worker_fn(void *arg)
@ -151,128 +151,164 @@ static void *async_nif_worker_fn(void *arg)
*/
for(;;) {
/* Examine the request queue, are there things to be done? */
enif_mutex_lock(async_nif_req_mutex);
enif_mutex_lock(async_nif_worker_mutex);
LIST_INSERT_HEAD(&async_nif_idle_workers, worker, entries);
enif_mutex_unlock(async_nif_worker_mutex);
enif_mutex_lock(async_nif->req_mutex);
enif_mutex_lock(async_nif->worker_mutex);
LIST_INSERT_HEAD(&async_nif->workers, worker, entries);
enif_mutex_unlock(async_nif->worker_mutex);
check_again_for_work:
if (async_nif_shutdown) { enif_mutex_unlock(async_nif_req_mutex); break; }
if ((req = STAILQ_FIRST(&async_nif_reqs)) == NULL) {
if (async_nif->shutdown) { enif_mutex_unlock(async_nif->req_mutex); break; }
if ((req = STAILQ_FIRST(&async_nif->reqs)) == NULL) {
/* Queue is empty, join the list of idle workers and wait for work */
enif_cond_wait(async_nif_cnd, async_nif_req_mutex);
enif_cond_wait(async_nif->cnd, async_nif->req_mutex);
goto check_again_for_work;
} else {
/* `req` is our work request and we hold the lock. */
enif_cond_broadcast(async_nif_cnd);
/* `req` is our work request and we hold the req_mutex lock. */
// TODO: do we need this? enif_cond_broadcast(async_nif->cnd);
/* Take the request off the queue. */
STAILQ_REMOVE(&async_nif_reqs, req, async_nif_req_entry, entries); async_nif_req_count--;
/* Now we need to remove this thread from the list of idle threads. */
enif_mutex_lock(async_nif_worker_mutex);
/* Remove this thread from the list of idle threads. */
enif_mutex_lock(async_nif->worker_mutex);
LIST_REMOVE(worker, entries);
enif_mutex_unlock(async_nif->worker_mutex);
/* Release the locks in reverse order that we acquired them,
so as not to self-deadlock. */
enif_mutex_unlock(async_nif_worker_mutex);
enif_mutex_unlock(async_nif_req_mutex);
do {
/* Take the request off the queue. */
STAILQ_REMOVE(&async_nif->reqs, req, async_nif->req_entry, entries);
async_nif->req_count--;
enif_mutex_unlock(async_nif->req_mutex);
/* Finally, let's do the work! :) */
req->fn_work(req->env, req->ref, req->priv_data, &req->pid, req->args);
req->fn_post(req->args);
enif_free(req->args);
enif_free_env(req->env);
enif_free(req);
/* Finally, do the work. */
unsigned int worker_id = (unsigned int)(worker - worker_entries);
req->fn_work(req->env, req->ref, &req->pid, worker_id, req->args);
req->fn_post(req->args);
enif_free(req->args);
enif_free_env(req->env);
enif_free(req);
/* Finally, check the request queue for more work before switching
into idle mode. */
enif_mutex_lock(async_nif->req_mutex);
if ((req = STAILQ_FIRST(&async_nif->reqs)) == NULL) {
enif_mutex_unlock(async_nif->req_mutex);
}
/* Take a second to see if we need to adjust the number of active
worker threads up or down. */
// TODO: if queue_depth > last_depth && num_workers < MAX, start one up
} while(req);
}
}
enif_thread_exit(0);
return 0;
}
static void async_nif_unload(void)
static void async_nif_unload(ErlNifEnv *env)
{
unsigned int i;
struct_nif_state *async_nif = (struct async_nif_state*)enif_priv_data(env);
/* Signal the worker threads, stop what you're doing and exit. */
enif_mutex_lock(async_nif_req_mutex);
async_nif_shutdown = 1;
enif_cond_broadcast(async_nif_cnd);
enif_mutex_unlock(async_nif_req_mutex);
enif_mutex_lock(async_nif->req_mutex);
async_nif->shutdown = 1;
enif_cond_broadcast(async_nif->cnd);
enif_mutex_unlock(async_nif->req_mutex);
/* Join for the now exiting worker threads. */
for (i = 0; i < ASYNC_NIF_MAX_WORKERS; ++i) {
void *exit_value = 0; /* Ignore this. */
enif_thread_join(async_nif_worker_entries[i].tid, &exit_value);
enif_thread_join(async_nif->worker_entries[i].tid, &exit_value);
}
/* We won't get here until all threads have exited.
Patch things up, and carry on. */
enif_mutex_lock(async_nif_req_mutex);
enif_mutex_lock(async_nif->req_mutex);
/* Worker threads are stopped, now toss anything left in the queue. */
struct async_nif_req_entry *req = NULL;
STAILQ_FOREACH(req, &async_nif_reqs, entries) {
STAILQ_REMOVE(&async_nif_reqs, STAILQ_LAST(&async_nif_reqs, async_nif_req_entry, entries),
STAILQ_FOREACH(req, &async_nif->reqs, entries) {
STAILQ_REMOVE(&async_nif->reqs, STAILQ_LAST(&async_nif->reqs, async_nif_req_entry, entries),
async_nif_req_entry, entries);
#ifdef PULSE
PULSE_SEND(NULL, &req->pid, req->env,
enif_make_tuple2(req->env, enif_make_atom(req->env, "error"),
enif_make_atom(req->env, "shutdown")));
#else
enif_send(NULL, &req->pid, req->env,
enif_make_tuple2(req->env, enif_make_atom(req->env, "error"),
enif_make_atom(req->env, "shutdown")));
#endif
req->fn_post(req->args);
enif_free(req->args);
enif_free(req);
async_nif_req_count--;
async_nif->req_count--;
}
enif_mutex_unlock(async_nif_req_mutex);
enif_mutex_unlock(async_nif->req_mutex);
memset(async_nif_worker_entries, sizeof(struct async_nif_worker_entry) * ASYNC_NIF_MAX_WORKERS, 0);
enif_cond_destroy(async_nif_cnd); async_nif_cnd = NULL;
enif_mutex_destroy(async_nif_req_mutex); async_nif_req_mutex = NULL;
enif_mutex_destroy(async_nif_worker_mutex); async_nif_worker_mutex = NULL;
bzero(async_nif->worker_entries, sizeof(struct async_nif_worker_entry) * ASYNC_NIF_MAX_WORKERS);
enif_cond_destroy(async_nif->cnd); async_nif->cnd = NULL;
enif_mutex_destroy(async_nif->req_mutex); async_nif->req_mutex = NULL;
enif_mutex_destroy(async_nif->worker_mutex); async_nif->worker_mutex = NULL;
bzero(async_nif, sizeof(struct async_nif_state));
free(async_nif);
}
static int async_nif_init(void)
static void *
async_nif_load(void)
{
int i;
int i, num_schedulers;
ErlDrvSysInfo info;
struct async_nif_state *async_nif;
/* Don't init more than once. */
if (async_nif_req_mutex) return 0;
async_nif_req_mutex = enif_mutex_create(NULL);
async_nif_worker_mutex = enif_mutex_create(NULL);
async_nif_cnd = enif_cond_create(NULL);
/* Find out how many schedulers there are. */
erl_drv_sys_info(&info, sizeof(ErlDrvSysInfo));
num_schedulers = info->scheduler_threads;
/* Init our portion of priv_data's module-specific state. */
async_nif = malloc(sizeof(struct async_nif_state));
if (!async_nif)
return NULL;
STAILQ_INIT(async_nif->reqs);
LIST_INIT(async_nif->workers);
async_nif->shutdown = 0;
async_nif->req_mutex = enif_mutex_create(NULL);
async_nif->worker_mutex = enif_mutex_create(NULL);
async_nif->cnd = enif_cond_create(NULL);
/* Setup the requests management. */
async_nif_req_count = 0;
async_nif->req_count = 0;
/* Setup the thread pool management. */
enif_mutex_lock(async_nif_worker_mutex);
memset(async_nif_worker_entries, sizeof(struct async_nif_worker_entry) * ASYNC_NIF_MAX_WORKERS, 0);
enif_mutex_lock(async_nif->worker_mutex);
bzero(async_nif->worker_entries, sizeof(struct async_nif_worker_entry) * ASYNC_NIF_MAX_WORKERS);
for (i = 0; i < ASYNC_NIF_MAX_WORKERS; i++) {
if (enif_thread_create(NULL, &async_nif_worker_entries[i].tid,
&async_nif_worker_fn, (void*)&async_nif_worker_entries[i], NULL) != 0) {
async_nif_shutdown = 1;
enif_cond_broadcast(async_nif_cnd);
enif_mutex_unlock(async_nif_worker_mutex);
/* Start the minimum of max workers allowed or number of scheduler threads running. */
unsigned int num_worker_threads = ASYNC_NIF_MAX_WORKERS;
if (num_schedulers < ASYNC_NIF_MAX_WORKERS)
num_worker_threads = num_schedulers;
if (num_worker_threads < 1)
num_worker_threads = 1;
for (i = 0; i < num_worker_threads; i++) {
if (enif_thread_create(NULL, &async_nif->worker_entries[i].tid,
&async_nif_worker_fn, (void*)&async_nif->worker_entries[i], NULL) != 0) {
async_nif->shutdown = 1;
enif_cond_broadcast(async_nif->cnd);
enif_mutex_unlock(async_nif->worker_mutex);
while(i-- > 0) {
void *exit_value = 0; /* Ignore this. */
enif_thread_join(async_nif_worker_entries[i].tid, &exit_value);
enif_thread_join(async_nif->worker_entries[i].tid, &exit_value);
}
memset(async_nif_worker_entries, sizeof(struct async_nif_worker_entry) * ASYNC_NIF_MAX_WORKERS, 0);
enif_cond_destroy(async_nif_cnd); async_nif_cnd = NULL;
enif_mutex_destroy(async_nif_req_mutex); async_nif_req_mutex = NULL;
enif_mutex_destroy(async_nif_worker_mutex); async_nif_worker_mutex = NULL;
return -1;
bzero(async_nif->worker_entries, sizeof(struct async_nif_worker_entry) * ASYNC_NIF_MAX_WORKERS);
enif_cond_destroy(async_nif->cnd);
async_nif->cnd = NULL;
enif_mutex_destroy(async_nif->req_mutex);
async_nif->req_mutex = NULL;
enif_mutex_destroy(async_nif->worker_mutex);
async_nif->worker_mutex = NULL;
return NULL;
}
}
enif_mutex_unlock(async_nif_worker_mutex);
return 0;
async_nif->num_workers = num_worker_threads;
enif_mutex_unlock(async_nif->worker_mutex);
return async_nif;
}
#if defined(__cplusplus)

610
c_src/khash.h Normal file
View file

@ -0,0 +1,610 @@
/* The MIT License
Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor@live.co.uk>
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
/*
An example:
#include "khash.h"
KHASH_MAP_INIT_INT(32, char)
int main() {
int ret, is_missing;
khiter_t k;
khash_t(32) *h = kh_init(32);
k = kh_put(32, h, 5, &ret);
kh_value(h, k) = 10;
k = kh_get(32, h, 10);
is_missing = (k == kh_end(h));
k = kh_get(32, h, 5);
kh_del(32, h, k);
for (k = kh_begin(h); k != kh_end(h); ++k)
if (kh_exist(h, k)) kh_value(h, k) = 1;
kh_destroy(32, h);
return 0;
}
*/
/*
2011-12-29 (0.2.7):
* Minor code clean up; no actual effect.
2011-09-16 (0.2.6):
* The capacity is a power of 2. This seems to dramatically improve the
speed for simple keys. Thank Zilong Tan for the suggestion. Reference:
- http://code.google.com/p/ulib/
- http://nothings.org/computer/judy/
* Allow to optionally use linear probing which usually has better
performance for random input. Double hashing is still the default as it
is more robust to certain non-random input.
* Added Wang's integer hash function (not used by default). This hash
function is more robust to certain non-random input.
2011-02-14 (0.2.5):
* Allow to declare global functions.
2009-09-26 (0.2.4):
* Improve portability
2008-09-19 (0.2.3):
* Corrected the example
* Improved interfaces
2008-09-11 (0.2.2):
* Improved speed a little in kh_put()
2008-09-10 (0.2.1):
* Added kh_clear()
* Fixed a compiling error
2008-09-02 (0.2.0):
* Changed to token concatenation which increases flexibility.
2008-08-31 (0.1.2):
* Fixed a bug in kh_get(), which has not been tested previously.
2008-08-31 (0.1.1):
* Added destructor
*/
#ifndef __AC_KHASH_H
#define __AC_KHASH_H
/*!
@header
Generic hash table library.
*/
#define AC_VERSION_KHASH_H "0.2.6"
#include <stdlib.h>
#include <string.h>
#include <limits.h>
/* compiler specific configuration */
#if UINT_MAX == 0xffffffffu
typedef unsigned int khint32_t;
#elif ULONG_MAX == 0xffffffffu
typedef unsigned long khint32_t;
#endif
#if ULONG_MAX == ULLONG_MAX
typedef unsigned long khint64_t;
#else
typedef unsigned long long khint64_t;
#endif
#ifdef _MSC_VER
#define kh_inline __inline
#else
#define kh_inline inline
#endif
typedef khint32_t khint_t;
typedef khint_t khiter_t;
#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
#ifdef KHASH_LINEAR
#define __ac_inc(k, m) 1
#else
#define __ac_inc(k, m) (((k)>>3 ^ (k)<<3) | 1) & (m)
#endif
#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4)
#ifndef kroundup32
#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
#endif
#ifndef kcalloc
#define kcalloc(N,Z) calloc(N,Z)
#endif
#ifndef kmalloc
#define kmalloc(Z) malloc(Z)
#endif
#ifndef krealloc
#define krealloc(P,Z) realloc(P,Z)
#endif
#ifndef kfree
#define kfree(P) free(P)
#endif
static const double __ac_HASH_UPPER = 0.77;
#define __KHASH_TYPE(name, khkey_t, khval_t) \
typedef struct { \
khint_t n_buckets, size, n_occupied, upper_bound; \
khint32_t *flags; \
khkey_t *keys; \
khval_t *vals; \
} kh_##name##_t;
#define __KHASH_PROTOTYPES(name, khkey_t, khval_t) \
extern kh_##name##_t *kh_init_##name(void); \
extern void kh_destroy_##name(kh_##name##_t *h); \
extern void kh_clear_##name(kh_##name##_t *h); \
extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); \
extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
extern void kh_del_##name(kh_##name##_t *h, khint_t x);
#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
SCOPE kh_##name##_t *kh_init_##name(void) { \
return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t)); \
} \
SCOPE void kh_destroy_##name(kh_##name##_t *h) \
{ \
if (h) { \
kfree((void *)h->keys); kfree(h->flags); \
kfree((void *)h->vals); \
kfree(h); \
} \
} \
SCOPE void kh_clear_##name(kh_##name##_t *h) \
{ \
if (h && h->flags) { \
memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \
h->size = h->n_occupied = 0; \
} \
} \
SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) \
{ \
if (h->n_buckets) { \
khint_t inc, k, i, last, mask; \
mask = h->n_buckets - 1; \
k = __hash_func(key); i = k & mask; \
inc = __ac_inc(k, mask); last = i; /* inc==1 for linear probing */ \
while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
i = (i + inc) & mask; \
if (i == last) return h->n_buckets; \
} \
return __ac_iseither(h->flags, i)? h->n_buckets : i; \
} else return 0; \
} \
SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
{ /* This function uses 0.25*n_buckets bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \
khint32_t *new_flags = 0; \
khint_t j = 1; \
{ \
kroundup32(new_n_buckets); \
if (new_n_buckets < 4) new_n_buckets = 4; \
if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0; /* requested size is too small */ \
else { /* hash table size to be changed (shrink or expand); rehash */ \
new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
if (!new_flags) return -1; \
memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
if (h->n_buckets < new_n_buckets) { /* expand */ \
khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
if (!new_keys) return -1; \
h->keys = new_keys; \
if (kh_is_map) { \
khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
if (!new_vals) return -1; \
h->vals = new_vals; \
} \
} /* otherwise shrink */ \
} \
} \
if (j) { /* rehashing is needed */ \
for (j = 0; j != h->n_buckets; ++j) { \
if (__ac_iseither(h->flags, j) == 0) { \
khkey_t key = h->keys[j]; \
khval_t val; \
khint_t new_mask; \
new_mask = new_n_buckets - 1; \
if (kh_is_map) val = h->vals[j]; \
__ac_set_isdel_true(h->flags, j); \
while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \
khint_t inc, k, i; \
k = __hash_func(key); \
i = k & new_mask; \
inc = __ac_inc(k, new_mask); \
while (!__ac_isempty(new_flags, i)) i = (i + inc) & new_mask; \
__ac_set_isempty_false(new_flags, i); \
if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \
{ khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
__ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \
} else { /* write the element and jump out of the loop */ \
h->keys[i] = key; \
if (kh_is_map) h->vals[i] = val; \
break; \
} \
} \
} \
} \
if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \
h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
} \
kfree(h->flags); /* free the working space */ \
h->flags = new_flags; \
h->n_buckets = new_n_buckets; \
h->n_occupied = h->size; \
h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
} \
return 0; \
} \
SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
{ \
khint_t x; \
if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \
if (h->n_buckets > (h->size<<1)) { \
if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \
*ret = -1; return h->n_buckets; \
} \
} else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \
*ret = -1; return h->n_buckets; \
} \
} /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
{ \
khint_t inc, k, i, site, last, mask = h->n_buckets - 1; \
x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \
if (__ac_isempty(h->flags, i)) x = i; /* for speed up */ \
else { \
inc = __ac_inc(k, mask); last = i; \
while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
if (__ac_isdel(h->flags, i)) site = i; \
i = (i + inc) & mask; \
if (i == last) { x = site; break; } \
} \
if (x == h->n_buckets) { \
if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
else x = i; \
} \
} \
} \
if (__ac_isempty(h->flags, x)) { /* not present at all */ \
h->keys[x] = key; \
__ac_set_isboth_false(h->flags, x); \
++h->size; ++h->n_occupied; \
*ret = 1; \
} else if (__ac_isdel(h->flags, x)) { /* deleted */ \
h->keys[x] = key; \
__ac_set_isboth_false(h->flags, x); \
++h->size; \
*ret = 2; \
} else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \
return x; \
} \
SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x) \
{ \
if (x != h->n_buckets && !__ac_iseither(h->flags, x)) { \
__ac_set_isdel_true(h->flags, x); \
--h->size; \
} \
}
#define KHASH_DECLARE(name, khkey_t, khval_t) \
__KHASH_TYPE(name, khkey_t, khval_t) \
__KHASH_PROTOTYPES(name, khkey_t, khval_t)
#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
__KHASH_TYPE(name, khkey_t, khval_t) \
__KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
KHASH_INIT2(name, static kh_inline, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
/* --- BEGIN OF HASH FUNCTIONS --- */
/*! @function
@abstract Integer hash function
@param key The integer [khint32_t]
@return The hash value [khint_t]
*/
#define kh_int_hash_func(key) (khint32_t)(key)
/*! @function
@abstract Integer comparison function
*/
#define kh_int_hash_equal(a, b) ((a) == (b))
/*! @function
@abstract 64-bit integer hash function
@param key The integer [khint64_t]
@return The hash value [khint_t]
*/
#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11)
/*! @function
@abstract 64-bit integer comparison function
*/
#define kh_int64_hash_equal(a, b) ((a) == (b))
/*! @function
@abstract const char* hash function
@param s Pointer to a null terminated string
@return The hash value
*/
static kh_inline khint_t __ac_X31_hash_string(const char *s)
{
khint_t h = (khint_t)*s;
if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s;
return h;
}
/*! @function
@abstract Another interface to const char* hash function
@param key Pointer to a null terminated string [const char*]
@return The hash value [khint_t]
*/
#define kh_str_hash_func(key) __ac_X31_hash_string(key)
/*! @function
@abstract Const char* comparison function
*/
#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
static kh_inline khint_t __ac_Wang_hash(khint_t key)
{
key += ~(key << 15);
key ^= (key >> 10);
key += (key << 3);
key ^= (key >> 6);
key += ~(key << 11);
key ^= (key >> 16);
return key;
}
#define kh_int_hash_func2(k) __ac_Wang_hash((khint_t)key)
/* --- END OF HASH FUNCTIONS --- */
/* Other convenient macros... */
/*!
@abstract Type of the hash table.
@param name Name of the hash table [symbol]
*/
#define khash_t(name) kh_##name##_t
/*! @function
@abstract Initiate a hash table.
@param name Name of the hash table [symbol]
@return Pointer to the hash table [khash_t(name)*]
*/
#define kh_init(name) kh_init_##name()
/*! @function
@abstract Destroy a hash table.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
*/
#define kh_destroy(name, h) kh_destroy_##name(h)
/*! @function
@abstract Reset a hash table without deallocating memory.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
*/
#define kh_clear(name, h) kh_clear_##name(h)
/*! @function
@abstract Resize a hash table.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
@param s New size [khint_t]
*/
#define kh_resize(name, h, s) kh_resize_##name(h, s)
/*! @function
@abstract Insert a key to the hash table.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
@param k Key [type of keys]
@param r Extra return code: 0 if the key is present in the hash table;
1 if the bucket is empty (never used); 2 if the element in
the bucket has been deleted [int*]
@return Iterator to the inserted element [khint_t]
*/
#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
/*! @function
@abstract Retrieve a key from the hash table.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
@param k Key [type of keys]
@return Iterator to the found element, or kh_end(h) if the element is absent [khint_t]
*/
#define kh_get(name, h, k) kh_get_##name(h, k)
/*! @function
@abstract Remove a key from the hash table.
@param name Name of the hash table [symbol]
@param h Pointer to the hash table [khash_t(name)*]
@param k Iterator to the element to be deleted [khint_t]
*/
#define kh_del(name, h, k) kh_del_##name(h, k)
/*! @function
@abstract Test whether a bucket contains data.
@param h Pointer to the hash table [khash_t(name)*]
@param x Iterator to the bucket [khint_t]
@return 1 if containing data; 0 otherwise [int]
*/
#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
/*! @function
@abstract Get key given an iterator
@param h Pointer to the hash table [khash_t(name)*]
@param x Iterator to the bucket [khint_t]
@return Key [type of keys]
*/
#define kh_key(h, x) ((h)->keys[x])
/*! @function
@abstract Get value given an iterator
@param h Pointer to the hash table [khash_t(name)*]
@param x Iterator to the bucket [khint_t]
@return Value [type of values]
@discussion For hash sets, calling this results in segfault.
*/
#define kh_val(h, x) ((h)->vals[x])
/*! @function
@abstract Alias of kh_val()
*/
#define kh_value(h, x) ((h)->vals[x])
/*! @function
@abstract Get the start iterator
@param h Pointer to the hash table [khash_t(name)*]
@return The start iterator [khint_t]
*/
#define kh_begin(h) (khint_t)(0)
/*! @function
@abstract Get the end iterator
@param h Pointer to the hash table [khash_t(name)*]
@return The end iterator [khint_t]
*/
#define kh_end(h) ((h)->n_buckets)
/*! @function
@abstract Get the number of elements in the hash table
@param h Pointer to the hash table [khash_t(name)*]
@return Number of elements in the hash table [khint_t]
*/
#define kh_size(h) ((h)->size)
/*! @function
@abstract Get the number of buckets in the hash table
@param h Pointer to the hash table [khash_t(name)*]
@return Number of buckets in the hash table [khint_t]
*/
#define kh_n_buckets(h) ((h)->n_buckets)
/*! @function
@abstract Iterate over the entries in the hash table
@param h Pointer to the hash table [khash_t(name)*]
@param kvar Variable to which key will be assigned
@param vvar Variable to which value will be assigned
@param code Block of code to execute
*/
#define kh_foreach(h, kvar, vvar, code) { khint_t __i; \
for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
if (!kh_exist(h,__i)) continue; \
(kvar) = kh_key(h,__i); \
(vvar) = kh_val(h,__i); \
code; \
} }
/*! @function
@abstract Iterate over the values in the hash table
@param h Pointer to the hash table [khash_t(name)*]
@param vvar Variable to which value will be assigned
@param code Block of code to execute
*/
#define kh_foreach_value(h, vvar, code) { khint_t __i; \
for (__i = kh_begin(h); __i != kh_end(h); ++__i) { \
if (!kh_exist(h,__i)) continue; \
(vvar) = kh_val(h,__i); \
code; \
} }
/* More conenient interfaces */
/*! @function
@abstract Instantiate a hash set containing integer keys
@param name Name of the hash table [symbol]
*/
#define KHASH_SET_INIT_INT(name) \
KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
/*! @function
@abstract Instantiate a hash map containing integer keys
@param name Name of the hash table [symbol]
@param khval_t Type of values [type]
*/
#define KHASH_MAP_INIT_INT(name, khval_t) \
KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
/*! @function
@abstract Instantiate a hash map containing 64-bit integer keys
@param name Name of the hash table [symbol]
*/
#define KHASH_SET_INIT_INT64(name) \
KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
/*! @function
@abstract Instantiate a hash map containing 64-bit integer keys
@param name Name of the hash table [symbol]
@param khval_t Type of values [type]
*/
#define KHASH_MAP_INIT_INT64(name, khval_t) \
KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
typedef const char *kh_cstr_t;
/*! @function
@abstract Instantiate a hash map containing const char* keys
@param name Name of the hash table [symbol]
*/
#define KHASH_SET_INIT_STR(name) \
KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
/*! @function
@abstract Instantiate a hash map containing const char* keys
@param name Name of the hash table [symbol]
@param khval_t Type of values [type]
*/
#define KHASH_MAP_INIT_STR(name, khval_t) \
KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
#endif /* __AC_KHASH_H */

1390
c_src/wterl.c
View file

File diff suppressed because it is too large Load diff

2
src/riak_kv_wterl_backend.erl
View file

@ -97,6 +97,8 @@ start(Partition, Config) ->
end,
case AppStart of
ok ->
%% TODO: on failure to open a table try to verify, and then salvage it
%% if the cluster size > the n value
Table = "lsm:wt" ++ integer_to_list(Partition),
{ok, Connection} = establish_connection(Config),
Passes = establish_passes(erlang:system_info(schedulers), Connection, Table),

382
src/wterl.erl
View file

@ -28,6 +28,7 @@
cursor_next_key/1,
cursor_next_value/1,
cursor_open/2,
cursor_open/3,
cursor_prev/1,
cursor_prev_key/1,
cursor_prev_value/1,
@ -36,28 +37,26 @@
cursor_search/2,
cursor_search_near/2,
cursor_update/3,
session_checkpoint/1,
session_checkpoint/2,
session_close/1,
session_create/2,
session_create/3,
session_delete/3,
session_drop/2,
session_drop/3,
session_get/3,
session_open/1,
session_open/2,
session_put/4,
session_rename/3,
session_rename/4,
session_salvage/2,
session_salvage/3,
session_truncate/2,
session_truncate/3,
session_upgrade/2,
session_upgrade/3,
session_verify/2,
session_verify/3,
checkpoint/1,
checkpoint/2,
create/2,
create/4,
delete/3,
drop/2,
drop/3,
get/3,
put/4,
rename/3,
rename/4,
salvage/2,
salvage/3,
truncate/2,
truncate/3,
truncate/5,
upgrade/2,
upgrade/3,
verify/2,
verify/3,
config_value/3,
config_to_bin/1,
priv_dir/0,
@ -77,12 +76,11 @@
-type config() :: binary().
-type config_list() :: [{atom(), any()}].
-opaque connection() :: reference().
-opaque session() :: reference().
-opaque cursor() :: reference().
-type key() :: binary().
-type value() :: binary().
-export_type([connection/0, session/0, cursor/0]).
-export_type([connection/0, cursor/0]).
-on_load(init/0).
@ -94,7 +92,9 @@ nif_stub_error(Line) ->
-spec init() -> ok | {error, any()}.
init() ->
erlang:load_nif(filename:join(priv_dir(), atom_to_list(?MODULE)), 0).
erlang:load_nif(filename:join(priv_dir(), atom_to_list(?MODULE)),
[{wterl, "163a5073cb85db2a270ebe904e788bd8d478ea1c"},
{wiredtiger, "e9a607b1b78ffa528631519b5cb6ac944468991e"}]).
-spec connection_open(string(), config()) -> {ok, connection()} | {error, term()}.
connection_open(HomeDir, Config) ->
@ -127,143 +127,130 @@ connection_close(ConnRef) ->
conn_close_nif(_AsyncRef, _ConnRef) ->
?nif_stub.
-spec session_open(connection()) -> {ok, session()} | {error, term()}.
-spec session_open(connection(), config()) -> {ok, session()} | {error, term()}.
session_open(ConnRef) ->
session_open(ConnRef, ?EMPTY_CONFIG).
session_open(ConnRef, Config) ->
?ASYNC_NIF_CALL(fun session_open_nif/3, [ConnRef, Config]).
-spec create(connection(), string()) -> ok | {error, term()}.
-spec create(connection(), string(), config(), config()) -> ok | {error, term()}.
create(Ref, Name) ->
create(Ref, Name, ?EMPTY_CONFIG, ?EMPTY_CONFIG).
create(Ref, Name, Config, SessionConfig) ->
?ASYNC_NIF_CALL(fun create_nif/5, [Ref, Name, Config, SessionConfig]).
-spec session_open_nif(reference(), connection(), config()) -> {ok, session()} | {error, term()}.
session_open_nif(_AsyncRef, _ConnRef, _Config) ->
-spec create_nif(reference(), connection(), string(), config(), config()) -> ok | {error, term()}.
create_nif(_AsyncNif, _Ref, _Name, _Config, _SessionConfig) ->
?nif_stub.
-spec session_close(session()) -> ok | {error, term()}.
session_close(Ref) ->
?ASYNC_NIF_CALL(fun session_close_nif/2, [Ref]).
-spec drop(connection(), string()) -> ok | {error, term()}.
-spec drop(connection(), string(), config()) -> ok | {error, term()}.
drop(Ref, Name) ->
drop(Ref, Name, ?EMPTY_CONFIG).
drop(Ref, Name, Config) ->
?ASYNC_NIF_CALL(fun drop_nif/4, [Ref, Name, Config]).
-spec session_close_nif(reference(), session()) -> ok | {error, term()}.
session_close_nif(_AsyncRef, _Ref) ->
-spec drop_nif(reference(), connection(), string(), config()) -> ok | {error, term()}.
drop_nif(_AsyncRef, _Ref, _Name, _Config) ->
?nif_stub.
-spec session_create(session(), string()) -> ok | {error, term()}.
-spec session_create(session(), string(), config()) -> ok | {error, term()}.
session_create(Ref, Name) ->
session_create(Ref, Name, ?EMPTY_CONFIG).
session_create(Ref, Name, Config) ->
?ASYNC_NIF_CALL(fun session_create_nif/4, [Ref, Name, Config]).
-spec delete(connection(), string(), key()) -> ok | {error, term()}.
delete(Ref, Table, Key) ->
?ASYNC_NIF_CALL(fun delete_nif/4, [Ref, Table, Key]).
-spec session_create_nif(reference(), session(), string(), config()) -> ok | {error, term()}.
session_create_nif(_AsyncNif, _Ref, _Name, _Config) ->
-spec delete_nif(reference(), connection(), string(), key()) -> ok | {error, term()}.
delete_nif(_AsyncRef, _Ref, _Table, _Key) ->
?nif_stub.
-spec session_drop(session(), string()) -> ok | {error, term()}.
-spec session_drop(session(), string(), config()) -> ok | {error, term()}.
session_drop(Ref, Name) ->
session_drop(Ref, Name, ?EMPTY_CONFIG).
session_drop(Ref, Name, Config) ->
?ASYNC_NIF_CALL(fun session_drop_nif/4, [Ref, Name, Config]).
-spec get(connection(), string(), key()) -> {ok, value()} | not_found | {error, term()}.
get(Ref, Table, Key) ->
?ASYNC_NIF_CALL(fun get_nif/4, [Ref, Table, Key]).
-spec session_drop_nif(reference(), session(), string(), config()) -> ok | {error, term()}.
session_drop_nif(_AsyncRef, _Ref, _Name, _Config) ->
-spec get_nif(reference(), connection(), string(), key()) -> {ok, value()} | not_found | {error, term()}.
get_nif(_AsyncRef, _Ref, _Table, _Key) ->
?nif_stub.
-spec session_delete(session(), string(), key()) -> ok | {error, term()}.
session_delete(Ref, Table, Key) ->
?ASYNC_NIF_CALL(fun session_delete_nif/4, [Ref, Table, Key]).
-spec put(connection(), string(), key(), value()) -> ok | {error, term()}.
put(Ref, Table, Key, Value) ->
?ASYNC_NIF_CALL(fun put_nif/5, [Ref, Table, Key, Value]).
-spec session_delete_nif(reference(), session(), string(), key()) -> ok | {error, term()}.
session_delete_nif(_AsyncRef, _Ref, _Table, _Key) ->
-spec put_nif(reference(), connection(), string(), key(), value()) -> ok | {error, term()}.
put_nif(_AsyncRef, _Ref, _Table, _Key, _Value) ->
?nif_stub.
-spec session_get(session(), string(), key()) -> {ok, value()} | not_found | {error, term()}.
session_get(Ref, Table, Key) ->
?ASYNC_NIF_CALL(fun session_get_nif/4, [Ref, Table, Key]).
-spec rename(connection(), string(), string()) -> ok | {error, term()}.
-spec rename(connection(), string(), string(), config()) -> ok | {error, term()}.
rename(Ref, OldName, NewName) ->
rename(Ref, OldName, NewName, ?EMPTY_CONFIG).
rename(Ref, OldName, NewName, Config) ->
?ASYNC_NIF_CALL(fun rename_nif/5, [Ref, OldName, NewName, Config]).
-spec session_get_nif(reference(), session(), string(), key()) -> {ok, value()} | not_found | {error, term()}.
session_get_nif(_AsyncRef, _Ref, _Table, _Key) ->
-spec rename_nif(reference(), connection(), string(), string(), config()) -> ok | {error, term()}.
rename_nif(_AsyncRef, _Ref, _OldName, _NewName, _Config) ->
?nif_stub.
-spec session_put(session(), string(), key(), value()) -> ok | {error, term()}.
session_put(Ref, Table, Key, Value) ->
?ASYNC_NIF_CALL(fun session_put_nif/5, [Ref, Table, Key, Value]).
-spec salvage(connection(), string()) -> ok | {error, term()}.
-spec salvage(connection(), string(), config()) -> ok | {error, term()}.
salvage(Ref, Name) ->
salvage(Ref, Name, ?EMPTY_CONFIG).
salvage(Ref, Name, Config) ->
?ASYNC_NIF_CALL(fun salvage_nif/4, [Ref, Name, Config]).
-spec session_put_nif(reference(), session(), string(), key(), value()) -> ok | {error, term()}.
session_put_nif(_AsyncRef, _Ref, _Table, _Key, _Value) ->
-spec salvage_nif(reference(), connection(), string(), config()) -> ok | {error, term()}.
salvage_nif(_AsyncRef, _Ref, _Name, _Config) ->
?nif_stub.
-spec session_rename(session(), string(), string()) -> ok | {error, term()}.
-spec session_rename(session(), string(), string(), config()) -> ok | {error, term()}.
session_rename(Ref, OldName, NewName) ->
session_rename(Ref, OldName, NewName, ?EMPTY_CONFIG).
session_rename(Ref, OldName, NewName, Config) ->
?ASYNC_NIF_CALL(fun session_rename_nif/5, [Ref, OldName, NewName, Config]).
-spec checkpoint(connection()) -> ok | {error, term()}.
-spec checkpoint(connection(), config()) -> ok | {error, term()}.
checkpoint(_Ref) ->
checkpoint(_Ref, ?EMPTY_CONFIG).
checkpoint(Ref, Config) ->
?ASYNC_NIF_CALL(fun checkpoint_nif/3, [Ref, Config]).
-spec session_rename_nif(reference(), session(), string(), string(), config()) -> ok | {error, term()}.
session_rename_nif(_AsyncRef, _Ref, _OldName, _NewName, _Config) ->
-spec checkpoint_nif(reference(), connection(), config()) -> ok | {error, term()}.
checkpoint_nif(_AsyncRef, _Ref, _Config) ->
?nif_stub.
-spec session_salvage(session(), string()) -> ok | {error, term()}.
-spec session_salvage(session(), string(), config()) -> ok | {error, term()}.
session_salvage(Ref, Name) ->
session_salvage(Ref, Name, ?EMPTY_CONFIG).
session_salvage(Ref, Name, Config) ->
?ASYNC_NIF_CALL(fun session_salvage_nif/4, [Ref, Name, Config]).
-spec truncate(connection(), string()) -> ok | {error, term()}.
-spec truncate(connection(), string(), config()) -> ok | {error, term()}.
truncate(Ref, Name, Config) ->
truncate(Ref, Name, 0, 0, Config).
-spec truncate(connection(), string(), cursor() | 0, cursor() | 0, config()) -> ok | {error, term()}.
truncate(Ref, Name) ->
truncate(Ref, Name, 0, 0, ?EMPTY_CONFIG).
truncate(Ref, Name, Start, Stop, Config) ->
?ASYNC_NIF_CALL(fun truncate_nif/6, [Ref, Name, Start, Stop, Config]).
-spec session_salvage_nif(reference(), session(), string(), config()) -> ok | {error, term()}.
session_salvage_nif(_AsyncRef, _Ref, _Name, _Config) ->
-spec truncate_nif(reference(), connection(), string(), cursor() | 0, cursor() | 0, config()) -> ok | {error, term()}.
truncate_nif(_AsyncRef, _Ref, _Name, _Start, _Stop, _Config) ->
?nif_stub.
-spec session_checkpoint(session()) -> ok | {error, term()}.
-spec session_checkpoint(session(), config()) -> ok | {error, term()}.
session_checkpoint(_Ref) ->
session_checkpoint(_Ref, ?EMPTY_CONFIG).
session_checkpoint(Ref, Config) ->
?ASYNC_NIF_CALL(fun session_checkpoint_nif/3, [Ref, Config]).
-spec upgrade(connection(), string()) -> ok | {error, term()}.
-spec upgrade(connection(), string(), config()) -> ok | {error, term()}.
upgrade(Ref, Name) ->
upgrade(Ref, Name, ?EMPTY_CONFIG).
upgrade(Ref, Name, Config) ->
?ASYNC_NIF_CALL(fun upgrade_nif/4, [Ref, Name, Config]).
-spec session_checkpoint_nif(reference(), session(), config()) -> ok | {error, term()}.
session_checkpoint_nif(_AsyncRef, _Ref, _Config) ->
-spec upgrade_nif(reference(), connection(), string(), config()) -> ok | {error, term()}.
upgrade_nif(_AsyncRef, _Ref, _Name, _Config) ->
?nif_stub.
-spec session_truncate(session(), string()) -> ok | {error, term()}.
-spec session_truncate(session(), string(), config()) -> ok | {error, term()}.
session_truncate(Ref, Name) ->
session_truncate(Ref, Name, ?EMPTY_CONFIG).
session_truncate(Ref, Name, Config) ->
?ASYNC_NIF_CALL(fun session_truncate_nif/4, [Ref, Name, Config]).
-spec verify(connection(), string()) -> ok | {error, term()}.
-spec verify(connection(), string(), config()) -> ok | {error, term()}.
verify(Ref, Name) ->
verify(Ref, Name, ?EMPTY_CONFIG).
verify(Ref, Name, Config) ->
?ASYNC_NIF_CALL(fun verify_nif/4, [Ref, Name, Config]).
-spec session_truncate_nif(reference(), session(), string(), config()) -> ok | {error, term()}.
session_truncate_nif(_AsyncRef, _Ref, _Name, _Config) ->
-spec verify_nif(reference(), connection(), string(), config()) -> ok | {error, term()}.
verify_nif(_AsyncRef, _Ref, _Name, _Config) ->
?nif_stub.
-spec session_upgrade(session(), string()) -> ok | {error, term()}.
-spec session_upgrade(session(), string(), config()) -> ok | {error, term()}.
session_upgrade(Ref, Name) ->
session_upgrade(Ref, Name, ?EMPTY_CONFIG).
session_upgrade(Ref, Name, Config) ->
?ASYNC_NIF_CALL(fun session_upgrade_nif/4, [Ref, Name, Config]).
-spec session_upgrade_nif(reference(), session(), string(), config()) -> ok | {error, term()}.
session_upgrade_nif(_AsyncRef, _Ref, _Name, _Config) ->
?nif_stub.
-spec session_verify(session(), string()) -> ok | {error, term()}.
-spec session_verify(session(), string(), config()) -> ok | {error, term()}.
session_verify(Ref, Name) ->
session_verify(Ref, Name, ?EMPTY_CONFIG).
session_verify(Ref, Name, Config) ->
?ASYNC_NIF_CALL(fun session_verify_nif/4, [Ref, Name, Config]).
-spec session_verify_nif(reference(), session(), string(), config()) -> ok | {error, term()}.
session_verify_nif(_AsyncRef, _Ref, _Name, _Config) ->
?nif_stub.
-spec cursor_open(session(), string()) -> {ok, cursor()} | {error, term()}.
-spec cursor_open(connection(), string()) -> {ok, cursor()} | {error, term()}.
-spec cursor_open(connection(), string(), config() | 0) -> {ok, cursor()} | {error, term()}.
cursor_open(Ref, Table) ->
?ASYNC_NIF_CALL(fun cursor_open_nif/3, [Ref, Table]).
cursor_open(Ref, Table, 0).
cursor_open(Ref, Table, Config) ->
?ASYNC_NIF_CALL(fun cursor_open_nif/4, [Ref, Table, Config]).
-spec cursor_open_nif(reference(), session(), string()) -> {ok, cursor()} | {error, term()}.
cursor_open_nif(_AsyncRef, _Ref, _Table) ->
-spec cursor_open_nif(reference(), connection(), string(), config() | 0) -> {ok, cursor()} | {error, term()}.
cursor_open_nif(_AsyncRef, _Ref, _Table, _Config) ->
?nif_stub.
-spec cursor_close(cursor()) -> ok | {error, term()}.
@ -518,11 +505,10 @@ open_test_conn(DataDir) ->
{ok, ConnRef} = connection_open(DataDir, OpenConfig),
ConnRef.
open_test_session(ConnRef) ->
{ok, SRef} = session_open(ConnRef),
?assertMatch(ok, session_drop(SRef, "table:test", config_to_bin([{force,true}]))),
?assertMatch(ok, session_create(SRef, "table:test", config_to_bin([{block_compressor, "snappy"}]))),
SRef.
open_test_table(ConnRef) ->
?assertMatch(ok, drop(ConnRef, "table:test", config_to_bin([{force,true}]))),
?assertMatch(ok, create(ConnRef, "table:test", config_to_bin([{block_compressor, "snappy"}]))),
ConnRef.
conn_test() ->
ConnRef = open_test_conn(?TEST_DATA_DIR),
@ -538,110 +524,102 @@ session_test_() ->
end,
fun(ConnRef) ->
{inorder,
[{"open/close a session",
[{"create and drop a table",
fun() ->
{ok, SRef} = session_open(ConnRef),
?assertMatch(ok, session_close(SRef))
end},
{"create and drop a table",
fun() ->
SRef = open_test_session(ConnRef),
?assertMatch(ok, session_drop(SRef, "table:test")),
?assertMatch(ok, session_close(SRef))
ConnRef = open_test_table(ConnRef),
?assertMatch(ok, drop(ConnRef, "table:test"))
end}]}
end}.
insert_delete_test() ->
ConnRef = open_test_conn(?TEST_DATA_DIR),
SRef = open_test_session(ConnRef),
?assertMatch(ok, session_put(SRef, "table:test", <<"a">>, <<"apple">>)),
?assertMatch({ok, <<"apple">>}, session_get(SRef, "table:test", <<"a">>)),
?assertMatch(ok, session_delete(SRef, "table:test", <<"a">>)),
?assertMatch(not_found, session_get(SRef, "table:test", <<"a">>)),
ok = session_close(SRef),
ConnRef = open_test_table(ConnRef),
?assertMatch(ok, put(ConnRef, "table:test", <<"a">>, <<"apple">>)),
?assertMatch({ok, <<"apple">>}, get(ConnRef, "table:test", <<"a">>)),
?assertMatch(ok, delete(ConnRef, "table:test", <<"a">>)),
?assertMatch(not_found, get(ConnRef, "table:test", <<"a">>)),
ok = connection_close(ConnRef).
init_test_table() ->
ConnRef = open_test_conn(?TEST_DATA_DIR),
SRef = open_test_session(ConnRef),
?assertMatch(ok, session_put(SRef, "table:test", <<"a">>, <<"apple">>)),
?assertMatch(ok, session_put(SRef, "table:test", <<"b">>, <<"banana">>)),
?assertMatch(ok, session_put(SRef, "table:test", <<"c">>, <<"cherry">>)),
?assertMatch(ok, session_put(SRef, "table:test", <<"d">>, <<"date">>)),
?assertMatch(ok, session_put(SRef, "table:test", <<"g">>, <<"gooseberry">>)),
{ConnRef, SRef}.
ConnRef = open_test_table(ConnRef),
?assertMatch(ok, put(ConnRef, "table:test", <<"a">>, <<"apple">>)),
?assertMatch(ok, put(ConnRef, "table:test", <<"b">>, <<"banana">>)),
?assertMatch(ok, put(ConnRef, "table:test", <<"c">>, <<"cherry">>)),
?assertMatch(ok, put(ConnRef, "table:test", <<"d">>, <<"date">>)),
?assertMatch(ok, put(ConnRef, "table:test", <<"g">>, <<"gooseberry">>)),
ConnRef.
stop_test_table({ConnRef, SRef}) ->
?assertMatch(ok, session_close(SRef)),
stop_test_table(ConnRef) ->
?assertMatch(ok, connection_close(ConnRef)).
various_session_test_() ->
{setup,
fun init_test_table/0,
fun stop_test_table/1,
fun({_, SRef}) ->
fun(ConnRef) ->
{inorder,
[{"session verify",
fun() ->
?assertMatch(ok, session_verify(SRef, "table:test")),
?assertMatch(ok, verify(ConnRef, "table:test")),
?assertMatch({ok, <<"apple">>},
session_get(SRef, "table:test", <<"a">>))
get(ConnRef, "table:test", <<"a">>))
end},
{"session checkpoint",
fun() ->
Cfg = wterl:config_to_bin([{target, ["\"table:test\""]}]),
?assertMatch(ok, session_checkpoint(SRef, Cfg)),
?assertMatch(ok, checkpoint(ConnRef, Cfg)),
?assertMatch({ok, <<"apple">>},
session_get(SRef, "table:test", <<"a">>))
get(ConnRef, "table:test", <<"a">>))
end},
{"session salvage",
fun() ->
ok = session_salvage(SRef, "table:test"),
{ok, <<"apple">>} = session_get(SRef, "table:test", <<"a">>)
ok = salvage(ConnRef, "table:test"),
{ok, <<"apple">>} = get(ConnRef, "table:test", <<"a">>)
end},
{"session upgrade",
fun() ->
?assertMatch(ok, session_upgrade(SRef, "table:test")),
?assertMatch(ok, upgrade(ConnRef, "table:test")),
?assertMatch({ok, <<"apple">>},
session_get(SRef, "table:test", <<"a">>))
get(ConnRef, "table:test", <<"a">>))
end},
{"session rename",
fun() ->
?assertMatch(ok,
session_rename(SRef, "table:test", "table:new")),
rename(ConnRef, "table:test", "table:new")),
?assertMatch({ok, <<"apple">>},
session_get(SRef, "table:new", <<"a">>)),
get(ConnRef, "table:new", <<"a">>)),
?assertMatch(ok,
session_rename(SRef, "table:new", "table:test")),
rename(ConnRef, "table:new", "table:test")),
?assertMatch({ok, <<"apple">>},
session_get(SRef, "table:test", <<"a">>))
get(ConnRef, "table:test", <<"a">>))
end},
{"session truncate",
fun() ->
?assertMatch(ok, session_truncate(SRef, "table:test")),
?assertMatch(not_found, session_get(SRef, "table:test", <<"a">>))
?assertMatch(ok, truncate(ConnRef, "table:test")),
?assertMatch(not_found, get(ConnRef, "table:test", <<"a">>))
end}]}
end}.
cursor_open_close_test() ->
{ConnRef, SRef} = init_test_table(),
{ok, Cursor1} = cursor_open(SRef, "table:test"),
ConnRef = init_test_table(),
{ok, Cursor1} = cursor_open(ConnRef, "table:test"),
?assertMatch({ok, <<"a">>, <<"apple">>}, cursor_next(Cursor1)),
?assertMatch(ok, cursor_close(Cursor1)),
{ok, Cursor2} = cursor_open(SRef, "table:test"),
{ok, Cursor2} = cursor_open(ConnRef, "table:test"),
?assertMatch({ok, <<"g">>, <<"gooseberry">>}, cursor_prev(Cursor2)),
?assertMatch(ok, cursor_close(Cursor2)),
stop_test_table({ConnRef, SRef}).
stop_test_table(ConnRef).
various_cursor_test_() ->
{setup,
fun init_test_table/0,
fun stop_test_table/1,
fun({_, SRef}) ->
fun(ConnRef) ->
{inorder,
[{"move a cursor back and forth, getting key",
fun() ->
{ok, Cursor} = cursor_open(SRef, "table:test"),
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
?assertMatch({ok, <<"a">>}, cursor_next_key(Cursor)),
?assertMatch({ok, <<"b">>}, cursor_next_key(Cursor)),
?assertMatch({ok, <<"c">>}, cursor_next_key(Cursor)),
@ -654,7 +632,7 @@ various_cursor_test_() ->
end},
{"move a cursor back and forth, getting value",
fun() ->
{ok, Cursor} = cursor_open(SRef, "table:test"),
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
?assertMatch({ok, <<"apple">>}, cursor_next_value(Cursor)),
?assertMatch({ok, <<"banana">>}, cursor_next_value(Cursor)),
?assertMatch({ok, <<"cherry">>}, cursor_next_value(Cursor)),
@ -667,7 +645,7 @@ various_cursor_test_() ->
end},
{"move a cursor back and forth, getting key and value",
fun() ->
{ok, Cursor} = cursor_open(SRef, "table:test"),
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
?assertMatch({ok, <<"a">>, <<"apple">>}, cursor_next(Cursor)),
?assertMatch({ok, <<"b">>, <<"banana">>}, cursor_next(Cursor)),
?assertMatch({ok, <<"c">>, <<"cherry">>}, cursor_next(Cursor)),
@ -680,27 +658,27 @@ various_cursor_test_() ->
end},
{"fold keys",
fun() ->
{ok, Cursor} = cursor_open(SRef, "table:test"),
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
?assertMatch([<<"g">>, <<"d">>, <<"c">>, <<"b">>, <<"a">>],
fold_keys(Cursor, fun(Key, Acc) -> [Key | Acc] end, [])),
?assertMatch(ok, cursor_close(Cursor))
end},
{"search for an item",
fun() ->
{ok, Cursor} = cursor_open(SRef, "table:test"),
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
?assertMatch({ok, <<"banana">>}, cursor_search(Cursor, <<"b">>)),
?assertMatch(ok, cursor_close(Cursor))
end},
{"range search for an item",
fun() ->
{ok, Cursor} = cursor_open(SRef, "table:test"),
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
?assertMatch({ok, <<"gooseberry">>},
cursor_search_near(Cursor, <<"z">>)),
?assertMatch(ok, cursor_close(Cursor))
end},
{"check cursor reset",
fun() ->
{ok, Cursor} = cursor_open(SRef, "table:test"),
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
?assertMatch({ok, <<"apple">>}, cursor_next_value(Cursor)),
?assertMatch(ok, cursor_reset(Cursor)),
?assertMatch({ok, <<"apple">>}, cursor_next_value(Cursor)),
@ -708,7 +686,7 @@ various_cursor_test_() ->
end},
{"insert/overwrite an item using a cursor",
fun() ->
{ok, Cursor} = cursor_open(SRef, "table:test"),
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
?assertMatch(ok,
cursor_insert(Cursor, <<"h">>, <<"huckleberry">>)),
?assertMatch({ok, <<"huckleberry">>},
@ -719,28 +697,28 @@ various_cursor_test_() ->
cursor_search(Cursor, <<"g">>)),
?assertMatch(ok, cursor_close(Cursor)),
?assertMatch({ok, <<"grapefruit">>},
session_get(SRef, "table:test", <<"g">>)),
get(ConnRef, "table:test", <<"g">>)),
?assertMatch({ok, <<"huckleberry">>},
session_get(SRef, "table:test", <<"h">>))
get(ConnRef, "table:test", <<"h">>))
end},
{"update an item using a cursor",
fun() ->
{ok, Cursor} = cursor_open(SRef, "table:test"),
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
?assertMatch(ok,
cursor_update(Cursor, <<"g">>, <<"goji berries">>)),
?assertMatch(not_found,
cursor_update(Cursor, <<"k">>, <<"kumquat">>)),
?assertMatch(ok, cursor_close(Cursor)),
?assertMatch({ok, <<"goji berries">>},
session_get(SRef, "table:test", <<"g">>))
get(ConnRef, "table:test", <<"g">>))
end},
{"remove an item using a cursor",
fun() ->
{ok, Cursor} = cursor_open(SRef, "table:test"),
{ok, Cursor} = cursor_open(ConnRef, "table:test"),
?assertMatch(ok, cursor_remove(Cursor, <<"g">>)),
?assertMatch(not_found, cursor_remove(Cursor, <<"l">>)),
?assertMatch(ok, cursor_close(Cursor)),
?assertMatch(not_found, session_get(SRef, "table:test", <<"g">>))
?assertMatch(not_found, get(ConnRef, "table:test", <<"g">>))
end}]}
end}.
@ -758,13 +736,13 @@ values() ->
ops(Keys, Values) ->
{oneof([put, delete]), oneof(Keys), oneof(Values)}.
apply_kv_ops([], _SRef, _Tbl, Acc0) ->
apply_kv_ops([], _ConnRef, _Tbl, Acc0) ->
Acc0;
apply_kv_ops([{put, K, V} | Rest], SRef, Tbl, Acc0) ->
ok = wterl:session_put(SRef, Tbl, K, V),
apply_kv_ops(Rest, SRef, Tbl, orddict:store(K, V, Acc0));
apply_kv_ops([{delete, K, _} | Rest], SRef, Tbl, Acc0) ->
ok = case wterl:session_delete(SRef, Tbl, K) of
apply_kv_ops([{put, K, V} | Rest], ConnRef, Tbl, Acc0) ->
ok = wterl:put(ConnRef, Tbl, K, V),
apply_kv_ops(Rest, ConnRef, Tbl, orddict:store(K, V, Acc0));
apply_kv_ops([{delete, K, _} | Rest], ConnRef, Tbl, Acc0) ->
ok = case wterl:delete(ConnRef, Tbl, K) of
ok ->
ok;
not_found ->
@ -772,7 +750,7 @@ apply_kv_ops([{delete, K, _} | Rest], SRef, Tbl, Acc0) ->
Else ->
Else
end,
apply_kv_ops(Rest, SRef, Tbl, orddict:store(K, deleted, Acc0)).
apply_kv_ops(Rest, ConnRef, Tbl, orddict:store(K, deleted, Acc0)).
prop_put_delete() ->
?LET({Keys, Values}, {keys(), values()},
@ -786,21 +764,19 @@ prop_put_delete() ->
ok = filelib:ensure_dir(filename:join(DataDir, "x")),
Cfg = wterl:config_to_bin([{create,true}]),
{ok, Conn} = wterl:connection_open(DataDir, Cfg),
{ok, SRef} = wterl:session_open(Conn),
try
wterl:session_create(SRef, Table),
Model = apply_kv_ops(Ops, SRef, Table, []),
wterl:create(ConnRef, Table),
Model = apply_kv_ops(Ops, ConnRef, Table, []),
%% Validate that all deleted values return not_found
F = fun({K, deleted}) ->
?assertEqual(not_found, wterl:session_get(SRef, Table, K));
?assertEqual(not_found, wterl:get(ConnRef, Table, K));
({K, V}) ->
?assertEqual({ok, V}, wterl:session_get(SRef, Table, K))
?assertEqual({ok, V}, wterl:get(ConnRef, Table, K))
end,
lists:map(F, Model),
true
after
wterl:session_close(SRef),
wterl:connection_close(Conn)
end
end)).

9
update-version.sh Executable file
View file

@ -0,0 +1,9 @@
#!/bin/sh -
wterl=`git log -n 1 --pretty=format:"%H"`
wiredtiger0=`(cd c_src/wiredtiger && git log -n 1 --pretty=format:"%H")`
wiredtiger=`echo $wiredtiger0 | awk '{print $2}'`
echo $wterl
echo $wiredtiger