add open/2 and open/3

2012-09-30 22:16:46 +02:00 · 2012-09-30 22:16:46 +02:00 · ebb69cd53d
commit ebb69cd53d
parent 1a33be0432
6 changed files with 186 additions and 18 deletions
--- a/README.md
+++ b/README.md
@ -12,7 +12,12 @@ But this module could also be used as a general key-value store to replace:
 * Erlang R14B04+
* GCC 4.2+ or MS VisualStudio 2010+


Build
-----
$ make


API
---
 
The following functions were implemented:

-* `open/1`: creates a new MDB database. This call also re-open an already existing one.
+* `open/1`: equivalent to `emdb:open(DirName, 10485760)`.
+* `open/1`: equivalent to `emdb:open(DirName, 10485760, 0)`.
+* `open/3`: creates a new MDB database. This call also re-open an already existing one. Arguments are:
+	* DirName: database directory name
+	* MapSize: database map size (see [map.hrl](http://gitorious.org/mdb/mdb/blobs/master/libraries/libmdb/mdb.h))
+	* EnvFlags: database environment flags (see [map.hrl](http://gitorious.org/mdb/mdb/blobs/master/libraries/libmdb/mdb.h)). The possible values are defined in **emdb.hrl**.
 * `close/2`: closes the database
 * `put/2`: inserts Key with value Val into the database. Assumes that the key is not present, 'key_exit' is returned otherwise.
 * `get/1`: retrieves the value stored with Key in the database.
@ -58,9 +63,15 @@ $ ./start.sh

 	%% close the database
	16> ok = Handle:close().

+	...
+
 	17> q().  
  

+####Note:
+This code below create a new database with **80GB** MapSize, **avoid fsync**
+after each commit (for max speed) and use the experimental **MDB_FIXEDMAP**.

	{ok, Handle} = emdb:open("/tmp/emdb2", 85899345920, ?MDB_NOSYNC bor ?MDB_FIXEDMAP).
+
	
 
Performance
-----------

For maximum speed, this library use only binaries for both keys and values.
 
See the impressive [microbench](http://highlandsun.com/hyc/mdb/microbench/) against:

--- a/c_src/emdb_drv.c
+++ b/c_src/emdb_drv.c
@ -65,6 +65,7 @@ static struct emdb_map_t * emdb_map = NULL;

 /* emdb errors */
 #define EMDB_MALLOC_ERR              "error_malloc"
+#define EMDB_MAKE_BINARY_ERR         "error_make_binary"
 #define EMDB_CREATE_ERR              "error_create"
 #define EMDB_MAPSIZE_ERR             "error_mapsize"
 #define EMDB_OPEN_ERR                "error_open"
@ -100,20 +101,28 @@ static ERL_NIF_TERM emdb_open_nif (ErlNifEnv * env,
  struct emdb_map_t * node;
  MDB_txn * txn;
  char * err;
+  ErlNifUInt64 mapsize;
+  ErlNifUInt64 envflags;

  if (enif_get_string(env, argv[0], dirname, MAXPATHLEN, ERL_NIF_LATIN1) <= 0)
    return enif_make_badarg(env);
-
+  
  if(! (node = calloc(1, sizeof(struct emdb_map_t))))
    FAIL_FAST(EMDB_MALLOC_ERR, err3);
  
  if (mdb_env_create(& (node -> env)))
    FAIL_FAST(EMDB_CREATE_ERR, err2);
  
-  if (mdb_env_set_mapsize(node -> env, 10485760))
+  if (! enif_get_uint64(env, argv[1], & mapsize))
+    return enif_make_badarg(env);
+
+  if (mdb_env_set_mapsize(node -> env, mapsize))
    FAIL_FAST(EMDB_MAPSIZE_ERR, err2);
      
-  if (mdb_env_open(node -> env, dirname, MDB_FIXEDMAP, 0664))
+  if (! enif_get_uint64(env, argv[2], & envflags))
+    return enif_make_badarg(env);
+
+  if (mdb_env_open(node -> env, dirname, envflags, 0664))
    FAIL_FAST(EMDB_OPEN_ERR, err2);

  if (mdb_txn_begin(node -> env, NULL, 0, & txn))
@ -209,7 +218,7 @@ static ERL_NIF_TERM emdb_put_nif (ErlNifEnv * env,
  ret = mdb_put(txn, node -> dbi, & mkey, & mdata, MDB_NOOVERWRITE);
  if (MDB_KEYEXIST == ret)
    FAIL_FAST(EMDB_RET_KEY_EXIST, err1);
-   if (ret)
+  if (ret)
    FAIL_FAST(EMDB_PUT_ERR, err1);

  if (mdb_txn_commit(txn))
@ -229,6 +238,7 @@ static ERL_NIF_TERM emdb_get_nif (ErlNifEnv * env,
 {
  ErlNifBinary key;
  ErlNifBinary val;
+/*   ERL_NIF_TERM term; */

  MDB_val mkey;
  MDB_val mdata;
@ -265,7 +275,7 @@ static ERL_NIF_TERM emdb_get_nif (ErlNifEnv * env,
    }

  if (! enif_alloc_binary(mdata.mv_size, & val))
-    FAIL_FAST(EMDB_MALLOC_ERR, err2);
+      FAIL_FAST(EMDB_MALLOC_ERR, err2);

  memcpy(val.data, mdata.mv_data, mdata.mv_size);

@ -275,6 +285,16 @@ static ERL_NIF_TERM emdb_get_nif (ErlNifEnv * env,
                         atom_ok,
                         enif_make_binary(env, & val));

+/*   val.size = mdata.mv_size; */
+/*   val.data = mdata.mv_data; */
+  
+/*   mdb_txn_abort(txn); */
+
+/*   return enif_make_tuple(env, 2, */
+/*                          atom_ok, */
+/*                          term); */
+
+
 err2:
  mdb_txn_abort(txn);
 err1:
@ -451,7 +471,7 @@ static void emdb_unload(ErlNifEnv* env, void* priv)


 static ErlNifFunc nif_funcs [] = {
-  {"open",        1, emdb_open_nif},
+  {"open",        3, emdb_open_nif},
  {"close",       1, emdb_close_nif},
  {"put",         3, emdb_put_nif},
  {"get",         2, emdb_get_nif},
--- a/include/emdb.hrl
+++ b/include/emdb.hrl
@ -0,0 +1,36 @@
+%%-------------------------------------------------------------------
+%% This file is part of EMDB - Erlang MDB API
+%%
+%% Copyright (c) 2012 by Aleph Archives. All rights reserved.
+%%
+%%-------------------------------------------------------------------
+%% Redistribution and use in source and binary forms, with or without
+%% modification, are permitted only as authorized by the OpenLDAP
+%% Public License.
+%%
+%% A copy of this license is available in the file LICENSE in the
+%% top-level directory of the distribution or, alternatively, at
+%% <http://www.OpenLDAP.org/license.html>.
+%%
+%% Permission to use, copy, modify, and distribute this software for any
+%% purpose with or without fee is hereby granted, provided that the above
+%% copyright notice and this permission notice appear in all copies.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+%% WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+%% MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+%% ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+%% WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+%% ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+%% OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+%%-------------------------------------------------------------------
+
+
+%% Environment Flags
+-define(MDB_FIXEDMAP,   16#01).
+-define(MDB_NOSUBDIR,   16#02).
+-define(MDB_NOSYNC,     16#10000).
+-define(MDB_RDONLY,     16#20000).
+-define(MDB_NOMETASYNC, 16#40000).
+-define(MDB_WRITEMAP,   16#80000).
+-define(MDB_MAPASYNC,   16#100000).
--- a/include/emdb.hrl~
+++ b/include/emdb.hrl~
@ -0,0 +1,95 @@
+%% Copyright (c) 2011 Basho Technologies, Inc.  All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+
+-define(LEVELS,
+    [debug, info, notice, warning, error, critical, alert, emergency, none]).
+
+-define(DEBUG, 7).
+-define(INFO, 6).
+-define(NOTICE, 5).
+-define(WARNING, 4).
+-define(ERROR, 3).
+-define(CRITICAL, 2).
+-define(ALERT, 1).
+-define(EMERGENCY, 0).
+-define(LOG_NONE, -1).
+
+-define(LEVEL2NUM(Level),
+    case Level of
+        debug -> ?DEBUG;
+        info -> ?INFO;
+        notice -> ?NOTICE;
+        warning -> ?WARNING;
+        error -> ?ERROR;
+        critical -> ?CRITICAL;
+        alert -> ?ALERT;
+        emergency -> ?EMERGENCY
+    end).
+
+-define(NUM2LEVEL(Num),
+    case Num of
+        ?DEBUG -> debug;
+        ?INFO -> info;
+        ?NOTICE -> notice;
+        ?WARNING -> warning;
+        ?ERROR -> error;
+        ?CRITICAL -> critical;
+        ?ALERT -> alert;
+        ?EMERGENCY -> emergency
+    end).
+
+-define(SHOULD_LOG(Level),
+    lager_util:level_to_num(Level) =< element(1, lager_mochiglobal:get(loglevel, {?LOG_NONE, []}))).
+
+-define(NOTIFY(Level, Pid, Format, Args),
+    gen_event:notify(lager_event, {log, lager_util:level_to_num(Level),
+            lager_util:format_time(), [io_lib:format("[~p] ", [Level]),
+                io_lib:format("~p ", [Pid]), io_lib:format(Format, Args)]})).
+
+%% FOR INTERNAL USE ONLY
+%% internal non-blocking logging call
+%% there's some special handing for when we try to log (usually errors) while
+%% lager is still starting.
+-ifdef(TEST).
+-define(INT_LOG(Level, Format, Args),
+    case ?SHOULD_LOG(Level) of
+        true ->
+            ?NOTIFY(Level, self(), Format, Args);
+        _ ->
+            ok
+    end).
+-else.
+-define(INT_LOG(Level, Format, Args),
+    Self = self(),
+    %% do this in a spawn so we don't cause a deadlock calling gen_event:which_handlers
+    %% from a gen_event handler
+    spawn(fun() ->
+            case catch(gen_event:which_handlers(lager_event)) of
+                X when X == []; X == {'EXIT', noproc} ->
+                    %% there's no handlers yet or lager isn't running, try again
+                    %% in half a second.
+                    timer:sleep(500),
+                    ?NOTIFY(Level, Self, Format, Args);
+                _ ->
+                    case ?SHOULD_LOG(Level) of
+                        true ->
+                            ?NOTIFY(Level, Self, Format, Args);
+                        _ ->
+                            ok
+                    end
+            end
+    end)).
+-endif.
--- a/src/emdb.erl
+++ b/src/emdb.erl
@ -32,19 +32,23 @@
 %% EXPORTS
 %%====================================================================
 -export([
-         open/1
+         open/1,
+         open/2,
+         open/3
        ]).


 %%====================================================================
-%% Types
+%% Includes
 %%====================================================================
-record(emdb_oop, {
-          handle :: non_neg_integer()
-         }).
+-include("emdb.hrl").


-    
+%%====================================================================
+%% Macros
+%%====================================================================
+-define(MDB_MAP_SIZE, 10485760). %% 10MB
+
 %%====================================================================
 %% PUBLIC API
 %%====================================================================
@ -53,12 +57,14 @@
 %% @doc Create a new MDB database
 %% @end
 %%--------------------------------------------------------------------
-spec open(file:name()) -> #emdb_oop{}.
 open(DirName) ->
+    open(DirName, ?MDB_MAP_SIZE).
+open(DirName, MapSize) when is_integer(MapSize) andalso MapSize > 0 ->
+    open(DirName, MapSize, 0).
+open(DirName, MapSize, EnvFlags) when is_integer(MapSize) andalso MapSize > 0 andalso is_integer(EnvFlags) andalso EnvFlags >= 0 ->
    %% ensure directory exists
    ok = filelib:ensure_dir(DirName ++ "/"),
-    decorate(emdb_drv:open(DirName)).
-
+    decorate(emdb_drv:open(DirName, MapSize, EnvFlags)).

 %%====================================================================
 %% PRIVATE API
--- a/src/emdb_drv.erl
+++ b/src/emdb_drv.erl
@ -31,7 +31,7 @@
 %% EXPORTS
 %%====================================================================
 -export([
-         open/1,
+         open/3,
         close/1,

         put/3,
@ -62,7 +62,7 @@
 %% @doc 
 %% @end
 %%--------------------------------------------------------------------
-open(_DirName) ->
+open(_DirName, _MapSize, _EnvFlags) ->
    ?NOT_LOADED.

 %%--------------------------------------------------------------------