diff --git a/README.md b/README.md index 0a49751..2e00629 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,12 @@ But this module could also be used as a general key-value store to replace: * Erlang R14B04+ * GCC 4.2+ or MS VisualStudio 2010+ Build ----- $ make API --- The following functions were implemented: -* `open/1`: creates a new MDB database. This call also re-open an already existing one. +* `open/1`: equivalent to `emdb:open(DirName, 10485760)`. +* `open/1`: equivalent to `emdb:open(DirName, 10485760, 0)`. +* `open/3`: creates a new MDB database. This call also re-open an already existing one. Arguments are: + * DirName: database directory name + * MapSize: database map size (see [map.hrl](http://gitorious.org/mdb/mdb/blobs/master/libraries/libmdb/mdb.h)) + * EnvFlags: database environment flags (see [map.hrl](http://gitorious.org/mdb/mdb/blobs/master/libraries/libmdb/mdb.h)). The possible values are defined in **emdb.hrl**. * `close/2`: closes the database * `put/2`: inserts Key with value Val into the database. Assumes that the key is not present, 'key_exit' is returned otherwise. * `get/1`: retrieves the value stored with Key in the database. @@ -58,9 +63,15 @@ $ ./start.sh %% close the database 16> ok = Handle:close(). + ... + 17> q(). +####Note: +This code below create a new database with **80GB** MapSize, **avoid fsync** +after each commit (for max speed) and use the experimental **MDB_FIXEDMAP**. {ok, Handle} = emdb:open("/tmp/emdb2", 85899345920, ?MDB_NOSYNC bor ?MDB_FIXEDMAP). + Performance ----------- For maximum speed, this library use only binaries for both keys and values. See the impressive [microbench](http://highlandsun.com/hyc/mdb/microbench/) against: diff --git a/c_src/emdb_drv.c b/c_src/emdb_drv.c index 410a8b0..fcfb73a 100644 --- a/c_src/emdb_drv.c +++ b/c_src/emdb_drv.c @@ -65,6 +65,7 @@ static struct emdb_map_t * emdb_map = NULL; /* emdb errors */ #define EMDB_MALLOC_ERR "error_malloc" +#define EMDB_MAKE_BINARY_ERR "error_make_binary" #define EMDB_CREATE_ERR "error_create" #define EMDB_MAPSIZE_ERR "error_mapsize" #define EMDB_OPEN_ERR "error_open" @@ -100,20 +101,28 @@ static ERL_NIF_TERM emdb_open_nif (ErlNifEnv * env, struct emdb_map_t * node; MDB_txn * txn; char * err; + ErlNifUInt64 mapsize; + ErlNifUInt64 envflags; if (enif_get_string(env, argv[0], dirname, MAXPATHLEN, ERL_NIF_LATIN1) <= 0) return enif_make_badarg(env); - + if(! (node = calloc(1, sizeof(struct emdb_map_t)))) FAIL_FAST(EMDB_MALLOC_ERR, err3); if (mdb_env_create(& (node -> env))) FAIL_FAST(EMDB_CREATE_ERR, err2); - if (mdb_env_set_mapsize(node -> env, 10485760)) + if (! enif_get_uint64(env, argv[1], & mapsize)) + return enif_make_badarg(env); + + if (mdb_env_set_mapsize(node -> env, mapsize)) FAIL_FAST(EMDB_MAPSIZE_ERR, err2); - if (mdb_env_open(node -> env, dirname, MDB_FIXEDMAP, 0664)) + if (! enif_get_uint64(env, argv[2], & envflags)) + return enif_make_badarg(env); + + if (mdb_env_open(node -> env, dirname, envflags, 0664)) FAIL_FAST(EMDB_OPEN_ERR, err2); if (mdb_txn_begin(node -> env, NULL, 0, & txn)) @@ -209,7 +218,7 @@ static ERL_NIF_TERM emdb_put_nif (ErlNifEnv * env, ret = mdb_put(txn, node -> dbi, & mkey, & mdata, MDB_NOOVERWRITE); if (MDB_KEYEXIST == ret) FAIL_FAST(EMDB_RET_KEY_EXIST, err1); - if (ret) + if (ret) FAIL_FAST(EMDB_PUT_ERR, err1); if (mdb_txn_commit(txn)) @@ -229,6 +238,7 @@ static ERL_NIF_TERM emdb_get_nif (ErlNifEnv * env, { ErlNifBinary key; ErlNifBinary val; +/* ERL_NIF_TERM term; */ MDB_val mkey; MDB_val mdata; @@ -265,7 +275,7 @@ static ERL_NIF_TERM emdb_get_nif (ErlNifEnv * env, } if (! enif_alloc_binary(mdata.mv_size, & val)) - FAIL_FAST(EMDB_MALLOC_ERR, err2); + FAIL_FAST(EMDB_MALLOC_ERR, err2); memcpy(val.data, mdata.mv_data, mdata.mv_size); @@ -275,6 +285,16 @@ static ERL_NIF_TERM emdb_get_nif (ErlNifEnv * env, atom_ok, enif_make_binary(env, & val)); +/* val.size = mdata.mv_size; */ +/* val.data = mdata.mv_data; */ + +/* mdb_txn_abort(txn); */ + +/* return enif_make_tuple(env, 2, */ +/* atom_ok, */ +/* term); */ + + err2: mdb_txn_abort(txn); err1: @@ -451,7 +471,7 @@ static void emdb_unload(ErlNifEnv* env, void* priv) static ErlNifFunc nif_funcs [] = { - {"open", 1, emdb_open_nif}, + {"open", 3, emdb_open_nif}, {"close", 1, emdb_close_nif}, {"put", 3, emdb_put_nif}, {"get", 2, emdb_get_nif}, diff --git a/include/emdb.hrl b/include/emdb.hrl new file mode 100644 index 0000000..17863cb --- /dev/null +++ b/include/emdb.hrl @@ -0,0 +1,36 @@ +%%------------------------------------------------------------------- +%% This file is part of EMDB - Erlang MDB API +%% +%% Copyright (c) 2012 by Aleph Archives. All rights reserved. +%% +%%------------------------------------------------------------------- +%% Redistribution and use in source and binary forms, with or without +%% modification, are permitted only as authorized by the OpenLDAP +%% Public License. +%% +%% A copy of this license is available in the file LICENSE in the +%% top-level directory of the distribution or, alternatively, at +%% . +%% +%% Permission to use, copy, modify, and distribute this software for any +%% purpose with or without fee is hereby granted, provided that the above +%% copyright notice and this permission notice appear in all copies. +%% +%% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +%% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +%% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +%% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +%% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +%% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +%% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +%%------------------------------------------------------------------- + + +%% Environment Flags +-define(MDB_FIXEDMAP, 16#01). +-define(MDB_NOSUBDIR, 16#02). +-define(MDB_NOSYNC, 16#10000). +-define(MDB_RDONLY, 16#20000). +-define(MDB_NOMETASYNC, 16#40000). +-define(MDB_WRITEMAP, 16#80000). +-define(MDB_MAPASYNC, 16#100000). diff --git a/include/emdb.hrl~ b/include/emdb.hrl~ new file mode 100644 index 0000000..bb42f7b --- /dev/null +++ b/include/emdb.hrl~ @@ -0,0 +1,95 @@ +%% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. + +-define(LEVELS, + [debug, info, notice, warning, error, critical, alert, emergency, none]). + +-define(DEBUG, 7). +-define(INFO, 6). +-define(NOTICE, 5). +-define(WARNING, 4). +-define(ERROR, 3). +-define(CRITICAL, 2). +-define(ALERT, 1). +-define(EMERGENCY, 0). +-define(LOG_NONE, -1). + +-define(LEVEL2NUM(Level), + case Level of + debug -> ?DEBUG; + info -> ?INFO; + notice -> ?NOTICE; + warning -> ?WARNING; + error -> ?ERROR; + critical -> ?CRITICAL; + alert -> ?ALERT; + emergency -> ?EMERGENCY + end). + +-define(NUM2LEVEL(Num), + case Num of + ?DEBUG -> debug; + ?INFO -> info; + ?NOTICE -> notice; + ?WARNING -> warning; + ?ERROR -> error; + ?CRITICAL -> critical; + ?ALERT -> alert; + ?EMERGENCY -> emergency + end). + +-define(SHOULD_LOG(Level), + lager_util:level_to_num(Level) =< element(1, lager_mochiglobal:get(loglevel, {?LOG_NONE, []}))). + +-define(NOTIFY(Level, Pid, Format, Args), + gen_event:notify(lager_event, {log, lager_util:level_to_num(Level), + lager_util:format_time(), [io_lib:format("[~p] ", [Level]), + io_lib:format("~p ", [Pid]), io_lib:format(Format, Args)]})). + +%% FOR INTERNAL USE ONLY +%% internal non-blocking logging call +%% there's some special handing for when we try to log (usually errors) while +%% lager is still starting. +-ifdef(TEST). +-define(INT_LOG(Level, Format, Args), + case ?SHOULD_LOG(Level) of + true -> + ?NOTIFY(Level, self(), Format, Args); + _ -> + ok + end). +-else. +-define(INT_LOG(Level, Format, Args), + Self = self(), + %% do this in a spawn so we don't cause a deadlock calling gen_event:which_handlers + %% from a gen_event handler + spawn(fun() -> + case catch(gen_event:which_handlers(lager_event)) of + X when X == []; X == {'EXIT', noproc} -> + %% there's no handlers yet or lager isn't running, try again + %% in half a second. + timer:sleep(500), + ?NOTIFY(Level, Self, Format, Args); + _ -> + case ?SHOULD_LOG(Level) of + true -> + ?NOTIFY(Level, Self, Format, Args); + _ -> + ok + end + end + end)). +-endif. diff --git a/src/emdb.erl b/src/emdb.erl index 8b0a844..45117ce 100644 --- a/src/emdb.erl +++ b/src/emdb.erl @@ -32,19 +32,23 @@ %% EXPORTS %%==================================================================== -export([ - open/1 + open/1, + open/2, + open/3 ]). %%==================================================================== -%% Types +%% Includes %%==================================================================== --record(emdb_oop, { - handle :: non_neg_integer() - }). +-include("emdb.hrl"). - +%%==================================================================== +%% Macros +%%==================================================================== +-define(MDB_MAP_SIZE, 10485760). %% 10MB + %%==================================================================== %% PUBLIC API %%==================================================================== @@ -53,12 +57,14 @@ %% @doc Create a new MDB database %% @end %%-------------------------------------------------------------------- --spec open(file:name()) -> #emdb_oop{}. open(DirName) -> + open(DirName, ?MDB_MAP_SIZE). +open(DirName, MapSize) when is_integer(MapSize) andalso MapSize > 0 -> + open(DirName, MapSize, 0). +open(DirName, MapSize, EnvFlags) when is_integer(MapSize) andalso MapSize > 0 andalso is_integer(EnvFlags) andalso EnvFlags >= 0 -> %% ensure directory exists ok = filelib:ensure_dir(DirName ++ "/"), - decorate(emdb_drv:open(DirName)). - + decorate(emdb_drv:open(DirName, MapSize, EnvFlags)). %%==================================================================== %% PRIVATE API diff --git a/src/emdb_drv.erl b/src/emdb_drv.erl index 6315a5a..e9a1e0e 100644 --- a/src/emdb_drv.erl +++ b/src/emdb_drv.erl @@ -31,7 +31,7 @@ %% EXPORTS %%==================================================================== -export([ - open/1, + open/3, close/1, put/3, @@ -62,7 +62,7 @@ %% @doc %% @end %%-------------------------------------------------------------------- -open(_DirName) -> +open(_DirName, _MapSize, _EnvFlags) -> ?NOT_LOADED. %%--------------------------------------------------------------------