From 4a2b2d6ac3fcc9ee8f7b4125ff98613cae22bee2 Mon Sep 17 00:00:00 2001 From: Joseph Wayne Norton Date: Wed, 26 Oct 2011 00:12:25 +0900 Subject: [PATCH] Initial commit --- .gitignore | 10 + LICENSE | 21 + README.md | 648 ++++++++++++++++++++++++++ c_src/build_deps.sh | 91 ++++ c_src/lets_drv.cc | 751 ++++++++++++++++++++++++++++++ c_src/lets_drv.h | 48 ++ c_src/lets_drv_lib.cc | 455 ++++++++++++++++++ c_src/lets_drv_lib.h | 100 ++++ c_src/lets_nif.cc | 578 +++++++++++++++++++++++ c_src/lets_nif.h | 68 +++ c_src/lets_nif_lib.cc | 300 ++++++++++++ c_src/lets_nif_lib.h | 119 +++++ doc/README.md | 648 ++++++++++++++++++++++++++ doc/edoc-info | 3 + doc/lets.md | 796 ++++++++++++++++++++++++++++++++ doc/lets_drv.md | 147 ++++++ doc/lets_ets.md | 147 ++++++ doc/lets_nif.md | 147 ++++++ doc/overview.edoc | 318 +++++++++++++ rebar.config | 72 +++ src/lets.app.src | 33 ++ src/lets.erl | 510 ++++++++++++++++++++ src/lets.hrl | 40 ++ src/lets_drv.erl | 278 +++++++++++ src/lets_ets.erl | 99 ++++ src/lets_nif.erl | 219 +++++++++ test/qc/qc_lets_proxy.erl | 197 ++++++++ test/qc/qc_lets_raw.erl | 116 +++++ test/qc/qc_lets_slave_proxy.erl | 118 +++++ test/qc/qc_leveldb.erl | 281 +++++++++++ test/qc/qc_statem_lets.erl | 446 ++++++++++++++++++ test/qc/qc_statemc_lets.erl | 237 ++++++++++ 32 files changed, 8041 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100755 c_src/build_deps.sh create mode 100644 c_src/lets_drv.cc create mode 100644 c_src/lets_drv.h create mode 100644 c_src/lets_drv_lib.cc create mode 100644 c_src/lets_drv_lib.h create mode 100644 c_src/lets_nif.cc create mode 100644 c_src/lets_nif.h create mode 100644 c_src/lets_nif_lib.cc create mode 100644 c_src/lets_nif_lib.h create mode 100644 doc/README.md create mode 100644 doc/edoc-info create mode 100644 doc/lets.md create mode 100644 doc/lets_drv.md create mode 100644 doc/lets_ets.md create mode 100644 doc/lets_nif.md create mode 100644 doc/overview.edoc create mode 100644 rebar.config create mode 100644 src/lets.app.src create mode 100644 src/lets.erl create mode 100644 src/lets.hrl create mode 100644 src/lets_drv.erl create mode 100644 src/lets_ets.erl create mode 100644 src/lets_nif.erl create mode 100644 test/qc/qc_lets_proxy.erl create mode 100644 test/qc/qc_lets_raw.erl create mode 100644 test/qc/qc_lets_slave_proxy.erl create mode 100644 test/qc/qc_leveldb.erl create mode 100644 test/qc/qc_statem_lets.erl create mode 100644 test/qc/qc_statemc_lets.erl diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1d97097 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +*.o +.eunit/ +c_src/perftools-*/ +c_src/perftools/ +c_src/snappy-*/ +c_src/snappy/ +c_src/leveldb-*/ +c_src/leveldb/ +ebin/ +priv/lib/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..927c590 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (C) 2011 by Joseph Wayne Norton + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..98b2ba7 --- /dev/null +++ b/README.md @@ -0,0 +1,648 @@ + + +#LETS - LevelDB-based Erlang Term Storage# + + +Copyright (c) 2011 by Joseph Wayne Norton + +__Authors:__ Joseph Wayne Norton ([`norton@alum.mit.edu`](mailto:norton@alum.mit.edu)).

LETS is an alternative Erlang Term Storage using LevelDB as the +storage implementation. LETS tries to address some bad properties of +ETS and DETS. ETS is limited by physical memory. DETS is limited by +a 2 GB file size limitation and does not implement ordered sets. LETS +has neither of these limitations.

+

For testing and comparison purposes, LETS supports three +implementations:

+ +

LETS is not intended to be an exact clone of ETS. The currently +supported APIs are:

+ +

This repository is experimental in nature - use at your own risk and +please contribute if you find LETS useful.

+ + +##Quick Start Recipe## + + +

To download and build the lets application in one shot, please follow +this recipe:

+ + +
$ mkdir working-directory-name
+$ cd working-directory-name
+$ git clone git://github.com/norton/snappy.git snappy
+$ git clone git://github.com/norton/leveldb.git leveldb
+$ git clone git://github.com/norton/lets.git lets
+$ cd lets
+$ ./rebar get-deps
+$ ./rebar clean
+$ ./rebar compile
+ +

For an alternative recipe with other "features" albeit more complex, +please read further.

+ + + +##Documentation## + + +

Where should I start?

+

This README is the only bit of documentation right now.

+

The QC (a.k.a. QuickCheck, Proper, etc.) tests underneath the +"tests/qc" directory should be helpful for understanding the +specification and behavior of ETS and LETS. These QC tests also +illustrate several strategies for testing Erlang Driver-based and +NIF-based implementations.

+

What is ETS and DETS?

+

ETS and DETS are Erlang/OTP's standard library modules for Erlang +term storage. ETS is a memory-based implementation. DETS is a +disk-based implementation.

+

See http://www.erlang.org/doc/man/ets.html and +http://www.erlang.org/doc/man/dets.html for further details.

+

What is LevelDB?

+

LevelDB is a fast key-value storage library written at Google that +provides an ordered mapping from string keys to string values.

+

See http://code.google.com/p/leveldb/ for further details.

+ + + +##To download## + + +
    +
  1. +

    +Configure your e-mail and name for Git +

    + + +
    $ git config \--global user.email "you@example.com"
+$ git config \--global user.name "Your Name"
    + +
    2. +
  2. +

    +Install Repo +

    + + +
    $ mkdir -p ~/bin
+$ wget -O - https://github.com/android/tools_repo/raw/master/repo > ~/bin/repo
+$ perl -i.bak -pe 's!git://android.git.kernel.org/tools/repo.git!git://github.com/android/tools_repo.git!;' ~/bin/repo
+$ chmod a+x ~/bin/repo
    + + + + + +
    +Caution +Since access to kernel.org has been shutdown due to hackers, +fetch and replace repo tool with android's GitHub repository mirror.
    + +
    3. +
  3. +

    +Create working directory +

    + + +
    $ mkdir working-directory-name
+$ cd working-directory-name
+$ repo init -u git://github.com/norton/manifests.git -m lets-default.xml
    + + + + + +
    +Note +Your "Git" identity is needed during the init step. Please +enter the name and email of your GitHub account if you have one. Team +members having read-write access are recommended to use "repo init -u +git@github.com:norton/manifests.git -m lets-default-rw.xml".
    + + + + + +
    +Tip +If you want to checkout the latest development version, please +append " -b dev" to the repo init command.
    + +
    4. +
  4. +

    +Download Git repositories +

    + + +
    $ cd working-directory-name
+$ repo sync
    + +
    5. +
+

For futher information and help for related tools, please refer to the +following links:

+ + + + +##To build - basic recipe## + + +
    +
  1. +

    +Get and install an erlang system http://www.erlang.org +

    +
    2. +
  2. +

    +Build +

    + + +
    $ cd working-directory-name/src
+$ make compile
    + +
    3. +
+ + + +##To build - optional features## + + +
    +
  1. +

    +Dialyzer Testing basic recipe +

    +
      +
    1. +

      +Build Dialyzer's PLT (required once) +

      + + +
      $ cd working-directory-name/src
+$ make build-plt
      + + + + + +
      +Tip +Check Makefile and dialyzer's documentation for further +information.
      + +
      2. +
    2. +

      +Dialyze with specs +

      + + +
      $ cd working-directory-name/src
+$ make dialyze
      + + + + + +
      +Caution +If you manually run dialyzer with the "-r" option, execute +"make clean compile" first to avoid finding duplicate beam files +underneath rebar's .eunit directory. Check Makefile for further +information.
      + +
      3. +
    3. +

      +Dialyze without specs +

      + + +
      $ cd working-directory-name/src
+$ make dialyze-nospec
      + +
      4. +
    +
    2. +
+ + + +##To test - QuickCheck## + + +
    +
  1. +

    +Make sure QuickCheck is in your Erlang code path. One simple way + to accomplish this is by adding the code path to your ~/.erlang + resource file. +

    + + +
    true = code:add_pathz(os:getenv("HOME")++"/.erlang.d/lib/quviq/eqc-X.Y.Z/ebin").
    + +
    2. +
  2. +

    +Compile for QuickCheck +

    + + +
    $ cd working-directory-name/src
+$ make clean
+$ make compile-eqc eqc-compile
    + +
    3. +
  3. +

    +Run 5,000 QuickCheck tests +

    + + +
    $ cd working-directory-name/src/lib/lets/.eunit
+$ erl -smp +A 5 -pz ../../sext/ebin -pz ../../qc/ebin
+
+1> qc_statem_lets:run(5000).
+.......
    + + + + + +
    +Tip +For testing LevelDB directly using the C bindings, try + qc_statemc_lets:run(5000).
    + +
    4. +
+ + + +##To test - Proper## + + +
    +
  1. +

    +Make sure Proper is in your Erlang code path. One simple way to + accomplish this is by adding the code path to your ~/.erlang + resource file. +

    + + +
    true = code:add_pathz(os:getenv("HOME")++"/.erlang.d/lib/proper/ebin").
    + +
    2. +
  2. +

    +Compile for Proper +

    + + +
    $ cd working-directory-name/src
+$ make clean
+$ make compile-proper proper-compile
    + +
    3. +
  3. +

    +Run 5,000 Proper tests +

    + + +
    $ cd working-directory-name/src/lib/lets/.eunit
+$ erl -smp +A 5 -pz ../../sext/ebin -pz ../../qc/ebin
+
+1> qc_statem_lets:run(5000).
+.......
    + +
    4. +
+ + + +##Roadmap## + + + + + +
Sanjay Ghemawat
+View profile
+ More options Sep 30, 1:04 am
+On Thu, Sep 29, 2011 at 8:30 AM, Joseph Wayne Norton wrote:
+> Hans -
+> Thanks.  Is is correct to assume that it is the caller's responsibility to
+> ensure this does not happen?
+
+leveldb guarantees that it will catch when two distinct processes
+try to open the db concurrently. However it doesn't guarantee what happens
+if the same process tries to do so and therefore it is the caller's
+responsibility
+to check for concurrent opens from the same process.
+This is ugly, but the unix file locking primitives are very annoying in
+this regard. I'll think about whether or not we should clean up the spec
+by doing extra checks inside the leveldb implementation.
+ + + + + +##Modules## + + + + + + +
lets
lets_drv
lets_ets
lets_nif
+ diff --git a/c_src/build_deps.sh b/c_src/build_deps.sh new file mode 100755 index 0000000..d5a6628 --- /dev/null +++ b/c_src/build_deps.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +# The MIT License +# +# Copyright (C) 2011 by Joseph Wayne Norton +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +set -e +set -o pipefail + +SNAPPY_VSN=HEAD +LEVELDB_VSN=HEAD + +if [ `basename $PWD` != "c_src" ]; then + pushd c_src +fi + +BASEDIR="$PWD" + +case "$1" in + clean) + rm -rf snappy snappy-$SNAPPY_VSN + rm -rf leveldb leveldb-$LEVELDB_VSN + ;; + + *) + # snappy + if [ ! -f $BASEDIR/snappy/lib/libsnappy.a ]; then + LIBTOOLIZE=libtoolize + ($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { + LIBTOOLIZE=glibtoolize + ($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have libtool (& friends) installed to compile Judy." + echo + exit -1 + } + } + + (cd ../../snappy && git archive --format=tar --prefix=snappy-$SNAPPY_VSN/ $SNAPPY_VSN) \ + | tar xf - + (cd snappy-$SNAPPY_VSN && \ + sed -ibak '/^AC_ARG_WITH.*$/, /^fi$/d' configure.ac + ) + (cd snappy-$SNAPPY_VSN && \ + rm -rf autom4te.cache && \ + aclocal -I m4 && \ + autoheader && \ + $LIBTOOLIZE --copy && \ + automake --add-missing --copy && \ + autoconf) + (cd snappy-$SNAPPY_VSN && \ + ./configure $CONFFLAGS \ + --enable-static \ + --disable-shared \ + --with-pic \ + --prefix=$BASEDIR/snappy && \ + make install) + fi + # leveldb + if [ ! -f $BASEDIR/leveldb/lib/libleveldb.a ]; then + (cd ../../leveldb && git archive --format=tar --prefix=leveldb-$LEVELDB_VSN/ $LEVELDB_VSN) \ + | tar xf - + (cd leveldb-$LEVELDB_VSN && \ + echo "echo \"PLATFORM_CFLAGS+=-fPIC -I$BASEDIR/snappy/include\" >> build_config.mk" >> build_detect_platform && + echo "echo \"PLATFORM_LDFLAGS+=-L $BASEDIR/snappy/lib -lsnappy\" >> build_config.mk" >> build_detect_platform && + make SNAPPY=1 && \ + mkdir -p $BASEDIR/leveldb/include/leveldb && \ + install include/leveldb/*.h $BASEDIR/leveldb/include/leveldb && \ + mkdir -p $BASEDIR/leveldb/lib && \ + install libleveldb.a $BASEDIR/leveldb/lib) + fi + ;; +esac diff --git a/c_src/lets_drv.cc b/c_src/lets_drv.cc new file mode 100644 index 0000000..9659acb --- /dev/null +++ b/c_src/lets_drv.cc @@ -0,0 +1,751 @@ +// The MIT License +// +// Copyright (C) 2011 by Joseph Wayne Norton +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "lets_drv.h" +#include "lets_drv_lib.h" + +#include + + +#if 0 +#define GOTOBADARG { fprintf(stderr, "GOTOBADARG %s:%d\n", __FILE__, __LINE__); goto badarg; } +#else +#define GOTOBADARG { goto badarg; } +#endif + + +#define LETS_BADARG 0x00 +#define LETS_TRUE 0x01 +#define LETS_END_OF_TABLE 0x02 + +#define LETS_OPEN6 0x00 +#define LETS_DESTROY6 0x01 +#define LETS_REPAIR6 0x02 +#define LETS_INSERT2 0x03 +#define LETS_INSERT3 0x04 +#define LETS_INSERT_NEW2 0x05 +#define LETS_INSERT_NEW3 0x06 +#define LETS_DELETE1 0x07 +#define LETS_DELETE2 0x08 +#define LETS_DELETE_ALL_OBJECTS1 0x09 +#define LETS_LOOKUP2 0x0A +#define LETS_FIRST1 0x0B +#define LETS_NEXT2 0x0C +#define LETS_INFO_MEMORY1 0x0D +#define LETS_INFO_SIZE1 0x0E + +// DrvData +typedef struct { + ErlDrvPort port; + lets_impl impl; +} DrvData; + +struct DrvAsync { + DrvData* drvdata; + ErlDrvTermData caller; + int command; + + leveldb::Slice skey; + leveldb::WriteBatch batch; + leveldb::Status status; + + DrvAsync(DrvData* d, ErlDrvTermData c, int cmd) : + drvdata(d), caller(c), command(cmd) { + } + DrvAsync(DrvData* d, ErlDrvTermData c, int cmd, char* key, int keylen) : + drvdata(d), caller(c), command(cmd), skey((const char*) key, keylen) { + } + + void put(char* key, int keylen, char* blob, int bloblen) { + leveldb::Slice skey((const char*) key, keylen); + leveldb::Slice sblob((const char*) blob, bloblen); + batch.Put(skey, sblob); + } + + void del(char* key, int keylen) { + leveldb::Slice skey((const char*) key, keylen); + batch.Delete(skey); + } +}; + +static void lets_output_open6(DrvData* d, char* buf, int len, int* index, int items); +static void lets_output_destroy6(DrvData* d, char* buf, int len, int* index, int items); +static void lets_output_repair6(DrvData* d, char* buf, int len, int* index, int items); +static void lets_output_insert2(DrvData* d, char* buf, int len, int* index, int items); +static void lets_output_insert3(DrvData* d, char* buf, int len, int* index, int items); +// static void lets_output_insert_new2(DrvData* d, char* buf, int len, int* index, int items); +// static void lets_output_insert_new3(DrvData* d, char* buf, int len, int* index, int items); +static void lets_output_delete1(DrvData* d, char* buf, int len, int* index, int items); +static void lets_output_delete2(DrvData* d, char* buf, int len, int* index, int items); +// static void lets_output_delete_all_objects1(DrvData* d, char* buf, int len, int* index, int items); +static void lets_output_lookup2(DrvData* d, char* buf, int len, int* index, int items); +static void lets_output_first1(DrvData* d, char* buf, int len, int* index, int items); +static void lets_output_next2(DrvData* d, char* buf, int len, int* index, int items); +// static void lets_output_info_memory1(DrvData* d, char* buf, int len, int* index, int items); +// static void lets_output_info_size1(DrvData* d, char* buf, int len, int* index, int items); + +static void +driver_send_int(DrvData* d, const int i) +{ + ErlDrvTermData caller = driver_caller(d->port); + ErlDrvTermData spec[] = { + ERL_DRV_PORT, driver_mk_port(d->port), + ERL_DRV_INT, i, + ERL_DRV_TUPLE, 2, + }; + driver_send_term(d->port, caller, spec, sizeof(spec) / sizeof(spec[0])); +} + +static void +driver_send_binary(DrvData* d, const char *buf, const ErlDrvUInt len) +{ + ErlDrvTermData caller = driver_caller(d->port); + ErlDrvTermData spec[] = { + ERL_DRV_PORT, driver_mk_port(d->port), + ERL_DRV_INT, LETS_TRUE, + ERL_DRV_BUF2BINARY, (ErlDrvTermData) buf, len, + ERL_DRV_TUPLE, 3, + }; + driver_send_term(d->port, caller, spec, sizeof(spec) / sizeof(spec[0])); +} + + +// +// Callbacks +// +static ErlDrvEntry drv_driver_entry = { + drv_init, + drv_start, + drv_stop, + drv_output, + NULL, + NULL, + (char*) "lets_drv", + NULL, + NULL, + NULL, + NULL, + NULL, + drv_ready_async, + NULL, + NULL, + NULL, + ERL_DRV_EXTENDED_MARKER, + ERL_DRV_EXTENDED_MAJOR_VERSION, + ERL_DRV_EXTENDED_MINOR_VERSION, + ERL_DRV_FLAG_USE_PORT_LOCKING, + NULL, + NULL, + NULL +}; + +DRIVER_INIT (lets_drv) // must match name in driver_entry +{ + return &drv_driver_entry; +} + +int +drv_init() +{ + if (!lets_drv_lib_init()) { + return -1; + } + + return 0; +} + +ErlDrvData +drv_start(ErlDrvPort port, char* command) +{ + DrvData* d; + + if (port == NULL) { + return ERL_DRV_ERROR_GENERAL; + } + + if ((d = (DrvData*) driver_alloc(sizeof(DrvData))) == NULL) { + errno = ENOMEM; + return ERL_DRV_ERROR_ERRNO; + } else { + memset(d, 0, sizeof(DrvData)); + } + + // port + d->port = port; + + return (ErlDrvData) d; +} + +void +drv_stop(ErlDrvData handle) +{ + DrvData* d = (DrvData*) handle; + + // alive + d->impl.alive = 0; + + // db + delete d->impl.db; + + // db_block_cache + delete d->impl.db_block_cache; + + // name + delete d->impl.name; + + driver_free(handle); +} + +void +drv_output(ErlDrvData handle, char* buf, int len) +{ + DrvData* d = (DrvData*) handle; + int ng, index, version, items; + char command; + +#if 0 + { + int term_type, size_needed; + + index = 0; + ng = ei_decode_version(buf, &index, &version); + if (ng) GOTOBADARG; + + ng = ei_get_type(buf, &index, &term_type, &size_needed); + if (ng) GOTOBADARG; + fprintf(stderr, "DEBUG %c %d: ", term_type, (int) size_needed); + + index = 0; + ng = ei_decode_version(buf, &index, &version); + if (ng) GOTOBADARG; + + ei_print_term(stderr, buf, &index); + fprintf(stderr, "\n"); + } +#endif + + index = 0; + ng = ei_decode_version(buf, &index, &version); + if (ng) GOTOBADARG; + + ng = ei_decode_tuple_header(buf, &index, &items); + if (ng) GOTOBADARG; + ng = (items < 1); + if (ng) GOTOBADARG; + + ng = ei_decode_char(buf, &index, &command); + if (ng) GOTOBADARG; + + items--; + switch (command) { + case LETS_OPEN6: + ng = (items != 6); + if (ng) GOTOBADARG; + lets_output_open6(d, buf, len, &index, items); + break; + case LETS_DESTROY6: + ng = (items != 6); + if (ng) GOTOBADARG; + lets_output_destroy6(d, buf, len, &index, items); + break; + case LETS_REPAIR6: + ng = (items != 6); + if (ng) GOTOBADARG; + lets_output_repair6(d, buf, len, &index, items); + break; + case LETS_INSERT2: + ng = (items != 1); + if (ng) GOTOBADARG; + lets_output_insert2(d, buf, len, &index, items); + break; + case LETS_INSERT3: + ng = (items != 2); + if (ng) GOTOBADARG; + lets_output_insert3(d, buf, len, &index, items); + break; + case LETS_INSERT_NEW2: + ng = (items != 1); + if (ng) GOTOBADARG; + GOTOBADARG; + break; + case LETS_INSERT_NEW3: + ng = (items != 2); + if (ng) GOTOBADARG; + GOTOBADARG; + break; + case LETS_DELETE1: + ng = (items != 0); + if (ng) GOTOBADARG; + lets_output_delete1(d, buf, len, &index, items); + break; + case LETS_DELETE2: + ng = (items != 1); + if (ng) GOTOBADARG; + lets_output_delete2(d, buf, len, &index, items); + break; + case LETS_DELETE_ALL_OBJECTS1: + ng = (items != 0); + if (ng) GOTOBADARG; + GOTOBADARG; + break; + case LETS_LOOKUP2: + ng = (items != 1); + if (ng) GOTOBADARG; + lets_output_lookup2(d, buf, len, &index, items); + break; + case LETS_FIRST1: + ng = (items != 0); + if (ng) GOTOBADARG; + lets_output_first1(d, buf, len, &index, items); + break; + case LETS_NEXT2: + ng = (items != 1); + if (ng) GOTOBADARG; + lets_output_next2(d, buf, len, &index, items); + break; + case LETS_INFO_MEMORY1: + ng = (items != 0); + if (ng) GOTOBADARG; + GOTOBADARG; + break; + case LETS_INFO_SIZE1: + ng = (items != 0); + if (ng) GOTOBADARG; + GOTOBADARG; + break; + default: + GOTOBADARG; + } + + return; + + badarg: + driver_send_int(d, LETS_BADARG); + return; +} + +void +drv_ready_async(ErlDrvData handle, ErlDrvThreadData async_data) +{ + DrvData* d = (DrvData*) handle; + DrvAsync* a = (DrvAsync*) async_data; + + (void) d; + (void) a; +} + +void +lets_output_do(void* async_data) +{ + DrvAsync* a = (DrvAsync*) async_data; + + (void) a; +} + + + +// +// Commands +// + +static void +lets_output_create6(const char op, DrvData* d, char* buf, int len, int* index, int items) +{ + char type; + char privacy; + char *name; + long namelen; + char* options; + int options_len; + char* read_options; + int read_options_len; + char* write_options; + int write_options_len; + + int ng, term_type, size_needed; + + if (!ei_inspect_atom(buf, index, (char*) "set")) { + type = SET; + } else if (!ei_inspect_atom(buf, index, (char*) "ordered_set")) { + type = ORDERED_SET; + } else { + GOTOBADARG; + } + + if (!ei_inspect_atom(buf, index, (char*) "private")) { + privacy = PRIVATE; + } else if (!ei_inspect_atom(buf, index, (char*) "protected")) { + privacy = PROTECTED; + } else if (!ei_inspect_atom(buf, index, (char*) "public")) { + privacy = PUBLIC; + } else { + GOTOBADARG; + } + + ng = ei_inspect_binary(buf, index, (void**) &name, &namelen); + if (ng) GOTOBADARG; + if (!namelen) GOTOBADARG; + + ng = ei_get_type(buf, index, &term_type, &size_needed); + if (ng) GOTOBADARG; + if (!(term_type == ERL_LIST_EXT || term_type == ERL_NIL_EXT)) GOTOBADARG; + options = buf + *index; + options_len = size_needed; + ng = ei_skip_term(buf, index); + if (ng) GOTOBADARG; + + ng = ei_get_type(buf, index, &term_type, &size_needed); + if (ng) GOTOBADARG; + if (!(term_type == ERL_LIST_EXT || term_type == ERL_NIL_EXT)) GOTOBADARG; + read_options = buf + *index; + read_options_len = size_needed; + ng = ei_skip_term(buf, index); + if (ng) GOTOBADARG; + + ng = ei_get_type(buf, index, &term_type, &size_needed); + if (ng) GOTOBADARG; + if (!(term_type == ERL_LIST_EXT || term_type == ERL_NIL_EXT)) GOTOBADARG; + write_options = buf + *index; + write_options_len = size_needed; + ng = ei_skip_term(buf, index); + if (ng) GOTOBADARG; + + + if (!lets_init(d->impl, type, privacy, name, namelen)) { + GOTOBADARG; + } + + if (!lets_parse_options(d->impl, options, options_len)) { + GOTOBADARG; + } + if (!lets_parse_read_options(d->impl, read_options, read_options_len)) { + GOTOBADARG; + } + if (!lets_parse_write_options(d->impl, write_options, write_options_len)) { + GOTOBADARG; + } + + if (!lets_create(d->impl, op)) { + GOTOBADARG; + } + + driver_send_int(d, LETS_TRUE); + return; + + badarg: + driver_send_int(d, LETS_BADARG); + return; +} + +static void +lets_output_open6(DrvData* d, char* buf, int len, int* index, int items) +{ + lets_output_create6(OPEN, d, buf, len, index, items); +} + +static void +lets_output_destroy6(DrvData* d, char* buf, int len, int* index, int items) +{ + lets_output_create6(DESTROY, d, buf, len, index, items); +} + +static void +lets_output_repair6(DrvData* d, char* buf, int len, int* index, int items) +{ + lets_output_create6(REPAIR, d, buf, len, index, items); +} + +static void +lets_output_insert2(DrvData* d, char* buf, int len, int* index, int items) +{ + int ng, arity; + char *key; + long keylen; + char *blob; + long bloblen; + leveldb::WriteBatch batch; + leveldb::Status status; + + ng = ei_decode_list_header(buf, index, &items); + if (ng) GOTOBADARG; + + if (!items) { + driver_send_int(d, LETS_TRUE); + return; + } + + while (items) { + ng = ei_decode_tuple_header(buf, index, &arity); + if (ng) GOTOBADARG; + ng = (arity != 2); + if (ng) GOTOBADARG; + ng = ei_inspect_binary(buf, index, (void**) &key, &keylen); + if (ng) GOTOBADARG; + ng = ei_inspect_binary(buf, index, (void**) &blob, &bloblen); + if (ng) GOTOBADARG; + + leveldb::Slice skey((const char*) key, keylen); + leveldb::Slice sblob((const char*) blob, bloblen); + batch.Put(skey, sblob); + items--; + } + + ng = ei_decode_list_header(buf, index, &items); + if (ng) GOTOBADARG; + ng = (items != 0); + if (ng) GOTOBADARG; + + if (!d->impl.alive) { + GOTOBADARG; + } + + status = d->impl.db->Write(d->impl.db_write_options, &batch); + if (!status.ok()) { + GOTOBADARG; + } + + driver_send_int(d, LETS_TRUE); + return; + + badarg: + driver_send_int(d, LETS_BADARG); + return; +} + +static void +lets_output_insert3(DrvData* d, char* buf, int len, int* index, int items) +{ + int ng; + char *key; + long keylen; + char *blob; + long bloblen; + leveldb::WriteBatch batch; + leveldb::Status status; + + ng = ei_inspect_binary(buf, index, (void**) &key, &keylen); + if (ng) GOTOBADARG; + ng = ei_inspect_binary(buf, index, (void**) &blob, &bloblen); + if (ng) GOTOBADARG; + + { + leveldb::Slice skey((const char*) key, keylen); + leveldb::Slice sblob((const char*) blob, bloblen); + batch.Put(skey, sblob); + } + + if (!d->impl.alive) { + GOTOBADARG; + } + + status = d->impl.db->Write(d->impl.db_write_options, &batch); + if (!status.ok()) { + GOTOBADARG; + } + + driver_send_int(d, LETS_TRUE); + return; + + badarg: + driver_send_int(d, LETS_BADARG); + return; +} + +static void +lets_output_delete1(DrvData* d, char* buf, int len, int* index, int items) +{ + leveldb::WriteOptions db_write_options; + leveldb::WriteBatch batch; + leveldb::Status status; + + if (!d->impl.alive) { + GOTOBADARG; + } + + // alive + d->impl.alive = 0; + + db_write_options.sync = true; + status = d->impl.db->Write(db_write_options, &batch); + if (!status.ok()) { + GOTOBADARG; + } + + // @TBD This is quite risky ... need to re-consider. + // delete d->impl.db; + // d->impl.db = NULL; + + driver_send_int(d, LETS_TRUE); + return; + + badarg: + driver_send_int(d, LETS_BADARG); + return; +} + +static void +lets_output_delete2(DrvData* d, char* buf, int len, int* index, int items) +{ + int ng; + char *key; + long keylen; + leveldb::WriteBatch batch; + leveldb::Status status; + + ng = ei_inspect_binary(buf, index, (void**) &key, &keylen); + if (ng) GOTOBADARG; + + { + leveldb::Slice skey((const char*) key, keylen); + batch.Delete(skey); + } + + if (!d->impl.alive) { + GOTOBADARG; + } + + status = d->impl.db->Write(d->impl.db_write_options, &batch); + if (!status.ok()) { + GOTOBADARG; + } + + driver_send_int(d, LETS_TRUE); + return; + + badarg: + driver_send_int(d, LETS_BADARG); + return; +} + +static void +lets_output_lookup2(DrvData* d, char* buf, int len, int* index, int items) +{ + int ng; + char *key; + long keylen; + + ng = ei_inspect_binary(buf, index, (void**) &key, &keylen); + if (ng) GOTOBADARG; + + if (!d->impl.alive) { + GOTOBADARG; + } + + { + leveldb::Iterator* it = d->impl.db->NewIterator(d->impl.db_read_options); + if (!it) { + GOTOBADARG; + } + + leveldb::Slice skey((const char*) key, keylen); + it->Seek(skey); + if (!it->Valid() || it->key().compare(skey) != 0) { + driver_send_int(d, LETS_TRUE); + delete it; + return; + } + + driver_send_binary(d, it->value().data(), it->value().size()); + delete it; + } + return; + + badarg: + driver_send_int(d, LETS_BADARG); + return; +} + +static void +lets_output_first1(DrvData* d, char* buf, int len, int* index, int items) +{ + if (!d->impl.alive) { + GOTOBADARG; + } + + { + leveldb::Iterator* it = d->impl.db->NewIterator(d->impl.db_read_options); + if (!it) { + GOTOBADARG; + } + + it->SeekToFirst(); + if (!it->Valid()) { + driver_send_int(d, LETS_END_OF_TABLE); + delete it; + return; + } + + driver_send_binary(d, it->key().data(), it->key().size()); + delete it; + } + return; + + badarg: + driver_send_int(d, LETS_BADARG); + return; +} + +static void +lets_output_next2(DrvData* d, char* buf, int len, int* index, int items) +{ + int ng; + char *key; + long keylen; + + ng = ei_inspect_binary(buf, index, (void**) &key, &keylen); + if (ng) GOTOBADARG; + + if (!d->impl.alive) { + GOTOBADARG; + } + + { + leveldb::Iterator* it = d->impl.db->NewIterator(d->impl.db_read_options); + if (!it) { + GOTOBADARG; + } + + leveldb::Slice skey((const char*) key, keylen); + it->Seek(skey); + if (!it->Valid()) { + driver_send_int(d, LETS_END_OF_TABLE); + delete it; + return; + } + + if (it->key().compare(skey) == 0) { + it->Next(); + if (!it->Valid()) { + driver_send_int(d, LETS_END_OF_TABLE); + delete it; + return; + } + } + + driver_send_binary(d, it->key().data(), it->key().size()); + delete it; + } + return; + + badarg: + driver_send_int(d, LETS_BADARG); +} diff --git a/c_src/lets_drv.h b/c_src/lets_drv.h new file mode 100644 index 0000000..9e8b329 --- /dev/null +++ b/c_src/lets_drv.h @@ -0,0 +1,48 @@ +// The MIT License +// +// Copyright (C) 2011 by Joseph Wayne Norton +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef LETS_DRV_H +#define LETS_DRV_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + + // @doc driver init + DRIVER_INIT(lib_lets_drv); + + // @doc driver callbacks + int drv_init(void); + + ErlDrvData drv_start(ErlDrvPort port, char* command); + void drv_stop(ErlDrvData); + void drv_output(ErlDrvData drv_data, char* buf, int len); + void drv_ready_async(ErlDrvData, ErlDrvThreadData); + +#ifdef __cplusplus +} +#endif + +#endif /* LETS_DRV_H */ diff --git a/c_src/lets_drv_lib.cc b/c_src/lets_drv_lib.cc new file mode 100644 index 0000000..15895d9 --- /dev/null +++ b/c_src/lets_drv_lib.cc @@ -0,0 +1,455 @@ +// The MIT License +// +// Copyright (C) 2011 by Joseph Wayne Norton +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "lets_drv_lib.h" + +#include + + +#if 0 +static bool +return_false(unsigned line) { + fprintf(stderr, "FALSE %s:%d\n", __FILE__, line); + return false; +} +#define FALSE return_false(__LINE__) +#else +#define FALSE false +#endif + + +bool +lets_drv_lib_init() +{ + return true; +} + +bool +lets_init(lets_impl& impl, + const char type, const char privacy, const char* name, const size_t namelen) +{ + impl.type = type; + impl.privacy = privacy; + impl.name = new std::string(name, namelen); + if (!impl.name) { + return FALSE; + } + + impl.db_options = leveldb::Options(); + impl.db_read_options = leveldb::ReadOptions(); + impl.db_write_options = leveldb::WriteOptions(); + + return true; +} + +bool +lets_create(lets_impl& impl, + const char op) +{ + leveldb::Status status; + + // db + switch (op) { + case OPEN: + status = leveldb::DB::Open(impl.db_options, impl.name->c_str(), &(impl.db)); + if (!status.ok()) { + return FALSE; + } else { + // alive + impl.alive = 1; + } + break; + case DESTROY: + status = DestroyDB(impl.name->c_str(), impl.db_options); + if (!status.ok()) { + return FALSE; + } + break; + case REPAIR: + status = RepairDB(impl.name->c_str(), impl.db_options); + if (!status.ok()) { + return FALSE; + } + break; + default: + return FALSE; + } + + return true; +} + +bool +lets_parse_options(lets_impl& impl, + const char* buf, int len) +{ + int index, ng, items, arity, term_type, size_needed; + char atom[MAXATOMLEN]; + + index = 0; + ng = ei_decode_list_header(buf, &index, &items); + if (ng) return FALSE; + ng = (items < 0); + if (ng) return FALSE; + + if (!items) return true; + + while (items) { + ng = ei_get_type(buf, &index, &term_type, &size_needed); + if (ng) return FALSE; + + switch (term_type) { + case ERL_ATOM_EXT: + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "create_if_missing") == 0) { + impl.db_options.create_if_missing = true; + } else if (strcmp(atom, "error_if_exists") == 0) { + impl.db_options.error_if_exists = true; + } else if (strcmp(atom, "paranoid_checks") == 0) { + impl.db_options.paranoid_checks = true; + } else { + return FALSE; + } + break; + case ERL_SMALL_TUPLE_EXT: + ng = ei_decode_tuple_header(buf, &index, &arity); + if (ng) return FALSE; + ng = (arity != 2); + if (ng) return FALSE; + unsigned long val; + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "create_if_missing") == 0) { + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "true") == 0) { + impl.db_options.create_if_missing = true; + } else if (strcmp(atom, "false") == 0) { + impl.db_options.create_if_missing = false; + } else { + return FALSE; + } + } else if (strcmp(atom, "error_if_exists") == 0) { + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "true") == 0) { + impl.db_options.error_if_exists = true; + } else if (strcmp(atom, "false") == 0) { + impl.db_options.error_if_exists = false; + } else { + return FALSE; + } + } else if (strcmp(atom, "paranoid_checks") == 0) { + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "true") == 0) { + impl.db_options.paranoid_checks = true; + } else if (strcmp(atom, "false") == 0) { + impl.db_options.paranoid_checks = false; + } else { + return FALSE; + } + } else if (strcmp(atom, "write_buffer_size") == 0) { + ng = ei_decode_ulong(buf, &index, &val); + if (ng) return FALSE; + impl.db_options.write_buffer_size = val; + } else if (strcmp(atom, "max_open_files") == 0) { + ng = ei_decode_ulong(buf, &index, &val); + if (ng) return FALSE; + impl.db_options.max_open_files = val; + } else if (strcmp(atom, "block_cache_size") == 0) { + ng = ei_decode_ulong(buf, &index, &val); + if (ng) return FALSE; + impl.db_block_cache_size = val; + impl.db_block_cache = leveldb::NewLRUCache(impl.db_block_cache_size); + impl.db_options.block_cache = impl.db_block_cache; + if (!impl.db_options.block_cache) { + return FALSE; + } + } else if (strcmp(atom, "block_size") == 0) { + ng = ei_decode_ulong(buf, &index, &val); + if (ng) return FALSE; + impl.db_options.block_size = val; + } else if (strcmp(atom, "block_restart_interval") == 0) { + ng = ei_decode_ulong(buf, &index, &val); + if (ng) return FALSE; + impl.db_options.block_restart_interval = val; + } else if (strcmp(atom, "compression") == 0) { + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "no") == 0) { + impl.db_options.compression = leveldb::kNoCompression; + } else if (strcmp(atom, "snappy") == 0) { + impl.db_options.compression = leveldb::kSnappyCompression; + } else { + return FALSE; + } + } else { + return FALSE; + } + break; + default: + return FALSE; + } + items--; + } + + ng = ei_decode_list_header(buf, &index, &items); + if (ng) return FALSE; + ng = (items != 0); + if (ng) return FALSE; + + return true; +} + +bool +lets_parse_read_options(lets_impl& impl, + const char* buf, int len) +{ + int index, ng, items, arity, term_type, size_needed; + char atom[MAXATOMLEN]; + + index = 0; + ng = ei_decode_list_header(buf, &index, &items); + if (ng) return FALSE; + ng = (items < 0); + if (ng) return FALSE; + + if (!items) return true; + + while (items) { + ng = ei_get_type(buf, &index, &term_type, &size_needed); + if (ng) return FALSE; + + switch (term_type) { + case ERL_ATOM_EXT: + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "verify_checksums") == 0) { + impl.db_read_options.verify_checksums = true; + } else if (strcmp(atom, "fill_cache") == 0) { + impl.db_read_options.fill_cache = true; + } else { + return FALSE; + } + break; + case ERL_SMALL_TUPLE_EXT: + ng = ei_decode_tuple_header(buf, &index, &arity); + if (ng) return FALSE; + ng = (arity != 2); + if (ng) return FALSE; + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "verify_checksums") == 0) { + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "true") == 0) { + impl.db_read_options.verify_checksums = true; + } else if (strcmp(atom, "false") == 0) { + impl.db_read_options.verify_checksums = false; + } else { + return FALSE; + } + } else if (strcmp(atom, "fill_cache") == 0) { + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "true") == 0) { + impl.db_read_options.fill_cache = true; + } else if (strcmp(atom, "false") == 0) { + impl.db_read_options.fill_cache = false; + } else { + return FALSE; + } + } else { + return FALSE; + } + break; + default: + return FALSE; + } + items--; + } + + ng = ei_decode_list_header(buf, &index, &items); + if (ng) return FALSE; + ng = (items != 0); + if (ng) return FALSE; + + return true; +} + +bool +lets_parse_write_options(lets_impl& impl, + const char* buf, int len) +{ + int index, ng, items, arity, term_type, size_needed; + char atom[MAXATOMLEN]; + + index = 0; + ng = ei_decode_list_header(buf, &index, &items); + if (ng) return FALSE; + ng = (items < 0); + if (ng) return FALSE; + + if (!items) return true; + + while (items) { + ng = ei_get_type(buf, &index, &term_type, &size_needed); + if (ng) return FALSE; + + switch (term_type) { + case ERL_ATOM_EXT: + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "sync") == 0) { + impl.db_write_options.sync = true; + } else { + return FALSE; + } + break; + case ERL_SMALL_TUPLE_EXT: + ng = ei_decode_tuple_header(buf, &index, &arity); + if (ng) return FALSE; + ng = (arity != 2); + if (ng) return FALSE; + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "sync") == 0) { + ng = ei_decode_atom(buf, &index, atom); + if (ng) return FALSE; + if (strcmp(atom, "true") == 0) { + impl.db_write_options.sync = true; + } else if (strcmp(atom, "false") == 0) { + impl.db_write_options.sync = false; + } else { + return FALSE; + } + } else { + return FALSE; + } + break; + default: + return FALSE; + } + items--; + } + + ng = ei_decode_list_header(buf, &index, &items); + if (ng) return FALSE; + ng = (items != 0); + if (ng) return FALSE; + + return true; +} + +// +// KEEP AS PLACEHOLDER UNTIL PATCHES ARE ACCEPTED BY Erlang/OTP TEAM +// + +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + * + + */ + +#define get8(s) \ + ((s) += 1, \ + ((unsigned char *)(s))[-1] & 0xff) + +#define get16be(s) \ + ((s) += 2, \ + (((((unsigned char *)(s))[-2] << 8) | \ + ((unsigned char *)(s))[-1])) & 0xffff) + +#define get32be(s) \ + ((s) += 4, \ + ((((unsigned char *)(s))[-4] << 24) | \ + (((unsigned char *)(s))[-3] << 16) | \ + (((unsigned char *)(s))[-2] << 8) | \ + ((unsigned char *)(s))[-1])) + +// This function inspects an atom from the binary format. The p +// parameter is the name of the atom and the name should be +// zero-terminated. If the name is equal to the atom in binary +// format, returns 0. Otherwise, return -1. If name is NULL, no +// comparison is done and returns 0. + +int +ei_inspect_atom(const char *buf, int *index, char *p) +{ + const char *s = buf + *index; + const char *s0 = s; + int len; + + if (get8(s) != ERL_ATOM_EXT) return -1; + + len = get16be(s); + + if (len > MAXATOMLEN) return -1; + + if (p) { + if (len != (int) strlen(p)) return -1; + if (memcmp(p, s, len)) return -1; + } + s += len; + *index += s-s0; + + return 0; +} + +// This function inspects a binary from the binary format. The p +// parameter is set to the address of the binary. The len parameter +// is set to the actual size of the binary. + +int +ei_inspect_binary(const char *buf, int *index, void **p, long *lenp) +{ + const char *s = buf + *index; + const char *s0 = s; + long len; + + if (get8(s) != ERL_BINARY_EXT) return -1; + + len = get32be(s); + if (p) *p = (void*) s; + s += len; + + if (lenp) *lenp = len; + *index += s-s0; + + return 0; +} diff --git a/c_src/lets_drv_lib.h b/c_src/lets_drv_lib.h new file mode 100644 index 0000000..bff8def --- /dev/null +++ b/c_src/lets_drv_lib.h @@ -0,0 +1,100 @@ +// The MIT License +// +// Copyright (C) 2011 by Joseph Wayne Norton +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef LETS_DRV_LIB_H +#define LETS_DRV_LIB_H + +#include + +#include "leveldb/db.h" +#include "leveldb/cache.h" +#include "leveldb/slice.h" +#include "leveldb/write_batch.h" + +#include "erl_driver.h" + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct { + long buflen; + char buf[1]; + } Binary; + + enum Type { + SET = 0x0, + ORDERED_SET = 0x1 + }; + + enum PrivacyType { + PRIVATE = 0x0, + PROTECTED = 0x1, + PUBLIC = 0x2 + }; + + enum DBOpType { + OPEN = 0x0, + DESTROY = 0x1, + REPAIR = 0x2 + }; + + typedef struct + { + char alive; + char type; + char privacy; + std::string* name; + leveldb::Options db_options; + leveldb::ReadOptions db_read_options; + leveldb::WriteOptions db_write_options; + size_t db_block_cache_size; + leveldb::Cache* db_block_cache; + leveldb::DB* db; + ErlDrvUInt64 db_memory; + ErlDrvUInt64 db_size; + } lets_impl; + + // prototypes + extern bool lets_drv_lib_init(); + + extern bool lets_init(lets_impl& impl, + const char type, const char privacy, const char* name, const size_t namelen); + extern bool lets_create(lets_impl& impl, + const char op); + + extern bool lets_parse_options(lets_impl& impl, + const char* buf, int len); + extern bool lets_parse_read_options(lets_impl& impl, + const char* buf, int len); + extern bool lets_parse_write_options(lets_impl& impl, + const char* buf, int len); + + // helpers + extern int ei_inspect_atom(const char *buf, int *index, char *p); + extern int ei_inspect_binary(const char *buf, int *index, void **p, long *lenp); + +#ifdef __cplusplus +} +#endif + +#endif /* LETS_DRV_LIB_H */ diff --git a/c_src/lets_nif.cc b/c_src/lets_nif.cc new file mode 100644 index 0000000..f3c4e65 --- /dev/null +++ b/c_src/lets_nif.cc @@ -0,0 +1,578 @@ +// The MIT License +// +// Copyright (C) 2011 by Joseph Wayne Norton +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "lets_nif.h" +#include "lets_nif_lib.h" + +#include + +static inline ERL_NIF_TERM +make_badarg(ErlNifEnv* env, const leveldb::Status& status, unsigned line=0) { + if (line) { + fprintf(stderr, "MAKEBADARG %s:%d %s\n", __FILE__, line, status.ToString().c_str()); + } + return enif_make_badarg(env); +} + +#if 0 +#define MAKEBADARG(env, status) make_badarg(env, status, __LINE__) +#else +#define MAKEBADARG(env, status) make_badarg(env, status) +#endif + +// NifHandle +typedef struct +{ + lets_impl impl; +} lets_nif_handle; + +static ErlNifResourceType* lets_nif_RESOURCE = NULL; +static unsigned lets_nif_RESOURCE_SIZE = sizeof(lets_nif_handle); +static ErlNifFunc nif_funcs[] = + { + {"impl_open", 6, lets_nif_open6}, + {"impl_destroy", 6, lets_nif_destroy6}, + {"impl_repair", 6, lets_nif_repair6}, + {"impl_insert", 2, lets_nif_insert2}, + {"impl_insert", 3, lets_nif_insert3}, + {"impl_insert_new", 2, lets_nif_insert_new2}, + {"impl_insert_new", 3, lets_nif_insert_new3}, + {"impl_delete", 1, lets_nif_delete1}, + {"impl_delete", 2, lets_nif_delete2}, + {"impl_delete_all_objects", 1, lets_nif_delete_all_objects1}, + {"impl_lookup", 2, lets_nif_lookup2}, + {"impl_first", 1, lets_nif_first1}, + {"impl_next", 2, lets_nif_next2}, + {"impl_info_memory", 1, lets_nif_info_memory1}, + {"impl_info_size", 1, lets_nif_info_size1}, + }; + +static void +lets_nif_resource_dtor(ErlNifEnv* env, void* arg) +{ + lets_nif_handle* h = (lets_nif_handle*) arg; + + // alive + h->impl.alive = 0; + + // db + delete h->impl.db; + + // db_block_cache + delete h->impl.db_block_cache; + + // name + delete h->impl.name; +} + +static int +on_load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + if (!lets_nif_lib_init(env)) { + return -1; + } + + ErlNifResourceFlags flags = (ErlNifResourceFlags) (ERL_NIF_RT_CREATE | ERL_NIF_RT_TAKEOVER); + lets_nif_RESOURCE = enif_open_resource_type(env, NULL, + "lets_nif_resource", + &lets_nif_resource_dtor, + flags, NULL); + if (lets_nif_RESOURCE == NULL) { + return -1; + } + + return 0; +} + +ERL_NIF_INIT(lets_nif, nif_funcs, &on_load, NULL, NULL, NULL); + +static ERL_NIF_TERM +db_create6(const char op, ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[], lets_nif_handle** reth) +{ + char type; + char privacy; + ErlNifBinary path; + ERL_NIF_TERM options; + unsigned options_len; + ERL_NIF_TERM read_options; + unsigned read_options_len; + ERL_NIF_TERM write_options; + unsigned write_options_len; + leveldb::Status status; + + if (enif_is_identical(argv[0], lets_atom_set)) { + type = SET; + } else if (enif_is_identical(argv[0], lets_atom_ordered_set)) { + type = ORDERED_SET; + } else { + return MAKEBADARG(env, status); + } + + if (enif_is_identical(argv[1], lets_atom_private)) { + privacy = PRIVATE; + } else if (enif_is_identical(argv[1], lets_atom_protected)) { + privacy = PROTECTED; + } else if (enif_is_identical(argv[1], lets_atom_public)) { + privacy = PUBLIC; + } else { + return MAKEBADARG(env, status); + } + + if (!enif_inspect_binary(env, argv[2], &path) || !path.size) { + return MAKEBADARG(env, status); + } + + if (!enif_get_list_length(env, argv[3], &options_len)) { + return MAKEBADARG(env, status); + } + options = argv[3]; + + if (!enif_get_list_length(env, argv[4], &read_options_len)) { + return MAKEBADARG(env, status); + } + read_options = argv[4]; + + if (!enif_get_list_length(env, argv[5], &write_options_len)) { + return MAKEBADARG(env, status); + } + write_options = argv[5]; + + lets_nif_handle* h = (lets_nif_handle*) enif_alloc_resource(lets_nif_RESOURCE, lets_nif_RESOURCE_SIZE); + if (!h) { + return MAKEBADARG(env, status); + } + memset(h, 0, lets_nif_RESOURCE_SIZE); + + if (!lets_init(h->impl, type, privacy, (const char*) path.data, path.size)) { + enif_release_resource(h); + return MAKEBADARG(env, status); + } + + if (!lets_parse_options(env, h->impl, options, options_len)) { + enif_release_resource(h); + return MAKEBADARG(env, status); + } + if (!lets_parse_read_options(env, h->impl, read_options, read_options_len)) { + enif_release_resource(h); + return MAKEBADARG(env, status); + } + if (!lets_parse_write_options(env, h->impl, write_options, write_options_len)) { + enif_release_resource(h); + return MAKEBADARG(env, status); + } + + if (!lets_create(h->impl, op)) { + enif_release_resource(h); + return MAKEBADARG(env, status); + } + + *reth = h; + return 0; +} + +ERL_NIF_TERM +lets_nif_open6(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h = NULL; + ERL_NIF_TERM badarg = db_create6(OPEN, env, argc, argv, &h); + + if (!h) { + return badarg; + } + + ERL_NIF_TERM result = enif_make_resource(env, h); + enif_release_resource(h); + return result; +} + +ERL_NIF_TERM +lets_nif_destroy6(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h = NULL; + ERL_NIF_TERM badarg = db_create6(DESTROY, env, argc, argv, &h); + + if (!h) { + return badarg; + } + + enif_release_resource(h); + return lets_atom_true; +} + +ERL_NIF_TERM +lets_nif_repair6(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h = NULL; + ERL_NIF_TERM badarg = db_create6(REPAIR, env, argc, argv, &h); + + if (!h) { + return badarg; + } + + enif_release_resource(h); + return lets_atom_true; +} + +ERL_NIF_TERM +lets_nif_insert2(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h; + ERL_NIF_TERM list; + unsigned list_len; + ERL_NIF_TERM head, tail; + int arity; + const ERL_NIF_TERM* tuple; + ErlNifBinary key; + ErlNifBinary blob; + leveldb::WriteBatch batch; + leveldb::Status status; + + if (!enif_get_resource(env, argv[0], lets_nif_RESOURCE, (void**)&h)) { + return MAKEBADARG(env, status); + } + if (!enif_get_list_length(env, argv[1], &list_len)) { + return MAKEBADARG(env, status); + } + list = argv[1]; + + if (!h->impl.alive) { + return MAKEBADARG(env, status); + } + + while (enif_get_list_cell(env, list, &head, &tail)) { + if (enif_get_tuple(env, head, &arity, &tuple) && arity == 2) { + if (!enif_inspect_binary(env, tuple[0], &key)) { + return MAKEBADARG(env, status); + } + if (!enif_inspect_binary(env, tuple[1], &blob)) { + return MAKEBADARG(env, status); + } + + leveldb::Slice skey((const char*) key.data, key.size); + leveldb::Slice sblob((const char*) blob.data, blob.size); + batch.Put(skey, sblob); + } else { + return MAKEBADARG(env, status); + } + list = tail; + } + + status = h->impl.db->Write(h->impl.db_write_options, &batch); + if (!status.ok()) { + return MAKEBADARG(env, status); + } + + return lets_atom_true; +} + +ERL_NIF_TERM +lets_nif_insert3(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h; + ErlNifBinary key; + ErlNifBinary blob; + leveldb::WriteBatch batch; + leveldb::Status status; + + if (!enif_get_resource(env, argv[0], lets_nif_RESOURCE, (void**)&h)) { + return MAKEBADARG(env, status); + } + if (!enif_inspect_binary(env, argv[1], &key)) { + return MAKEBADARG(env, status); + } + if (!enif_inspect_binary(env, argv[2], &blob)) { + return MAKEBADARG(env, status); + } + + if (!h->impl.alive) { + return MAKEBADARG(env, status); + } + + leveldb::Slice skey((const char*) key.data, key.size); + leveldb::Slice sblob((const char*) blob.data, blob.size); + batch.Put(skey, sblob); + + status = h->impl.db->Write(h->impl.db_write_options, &batch); + if (!status.ok()) { + return MAKEBADARG(env, status); + } + + return lets_atom_true; +} + +ERL_NIF_TERM +lets_nif_insert_new2(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + leveldb::Status status; + // @TODO not supported by leveldb + return MAKEBADARG(env, status); +} + +ERL_NIF_TERM +lets_nif_insert_new3(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + leveldb::Status status; + // @TODO not supported by leveldb + return MAKEBADARG(env, status); +} + +ERL_NIF_TERM +lets_nif_delete1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h; + leveldb::WriteOptions db_write_options; + leveldb::WriteBatch batch; + leveldb::Status status; + + if (!enif_get_resource(env, argv[0], lets_nif_RESOURCE, (void**)&h)) { + return MAKEBADARG(env, status); + } + + // alive + h->impl.alive = 0; + + db_write_options.sync = true; + status = h->impl.db->Write(db_write_options, &batch); + if (!status.ok()) { + return MAKEBADARG(env, status); + } + + // @TBD This is quite risky ... need to re-consider. + // delete h->impl.db; + // h->impl.db = NULL; + + return lets_atom_true; +} + +ERL_NIF_TERM +lets_nif_delete2(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h; + ErlNifBinary key; + leveldb::WriteBatch batch; + leveldb::Status status; + + if (!enif_get_resource(env, argv[0], lets_nif_RESOURCE, (void**)&h)) { + return MAKEBADARG(env, status); + } + if (!enif_inspect_binary(env, argv[1], &key)) { + return MAKEBADARG(env, status); + } + + if (!h->impl.alive) { + return MAKEBADARG(env, status); + } + + leveldb::Slice skey((const char*) key.data, key.size); + batch.Delete(skey); + + status = h->impl.db->Write(h->impl.db_write_options, &batch); + if (!status.ok()) { + return MAKEBADARG(env, status); + } + + return lets_atom_true; +} + +ERL_NIF_TERM +lets_nif_delete_all_objects1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + leveldb::Status status; + // @TODO not supported by leveldb + return MAKEBADARG(env, status); +} + +ERL_NIF_TERM +lets_nif_lookup2(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h; + ErlNifBinary key; + ERL_NIF_TERM blob = 0; + leveldb::Status status; + + if (!enif_get_resource(env, argv[0], lets_nif_RESOURCE, (void**)&h)) { + return MAKEBADARG(env, status); + } + if (!enif_inspect_binary(env, argv[1], &key)) { + return MAKEBADARG(env, status); + } + + if (!h->impl.alive) { + return MAKEBADARG(env, status); + } + + leveldb::Iterator* it = h->impl.db->NewIterator(h->impl.db_read_options); + if (!it) { + return MAKEBADARG(env, status); + } + + leveldb::Slice skey((const char*) key.data, key.size); + it->Seek(skey); + if (!it->Valid() || it->key().compare(skey) != 0) { + delete it; + return lets_atom_true; + } + + size_t size = it->value().size(); + unsigned char* b = enif_make_new_binary(env, size, &blob); + if (!b) { + delete it; + return MAKEBADARG(env, status); + } + + memcpy(b, it->value().data(), size); + delete it; + return blob; +} + +ERL_NIF_TERM +lets_nif_first1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h; + ERL_NIF_TERM first_key = 0; + leveldb::Status status; + + if (!enif_get_resource(env, argv[0], lets_nif_RESOURCE, (void**)&h)) { + return MAKEBADARG(env, status); + } + + if (!h->impl.alive) { + return MAKEBADARG(env, status); + } + + leveldb::Iterator* it = h->impl.db->NewIterator(h->impl.db_read_options); + if (!it) { + return MAKEBADARG(env, status); + } + + it->SeekToFirst(); + if (!it->Valid()) { + delete it; + return lets_atom_end_of_table; + } + + size_t size = it->key().size(); + unsigned char* k = enif_make_new_binary(env, size, &first_key); + if (!k) { + delete it; + return MAKEBADARG(env, status); + } + + memcpy(k, it->key().data(), size); + delete it; + return first_key; +} + +ERL_NIF_TERM +lets_nif_next2(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h; + ErlNifBinary key; + ERL_NIF_TERM next_key = 0; + leveldb::Status status; + + if (!enif_get_resource(env, argv[0], lets_nif_RESOURCE, (void**)&h)) { + return MAKEBADARG(env, status); + } + if (!enif_inspect_binary(env, argv[1], &key)) { + return MAKEBADARG(env, status); + } + + if (!h->impl.alive) { + return MAKEBADARG(env, status); + } + + leveldb::Iterator* it = h->impl.db->NewIterator(h->impl.db_read_options); + if (!it) { + return MAKEBADARG(env, status); + } + + leveldb::Slice skey((const char*) key.data, key.size); + it->Seek(skey); + if (!it->Valid()) { + delete it; + return lets_atom_end_of_table; + } + + if (it->key().compare(skey) == 0) { + it->Next(); + if (!it->Valid()) { + delete it; + return lets_atom_end_of_table; + } + } + + size_t size = it->key().size(); + unsigned char* k = enif_make_new_binary(env, size, &next_key); + if (!k) { + delete it; + return MAKEBADARG(env, status); + } + + memcpy(k, it->key().data(), size); + delete it; + return next_key; +} + +ERL_NIF_TERM +lets_nif_info_memory1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h; + // ERL_NIF_TERM info; + leveldb::Status status; + + if (!enif_get_resource(env, argv[0], lets_nif_RESOURCE, (void**)&h)) { + return MAKEBADARG(env, status); + } + + if (!h->impl.alive) { + return MAKEBADARG(env, status); + } + + // @TODO implementation + // info = enif_make_uint64(env, h->impl.db_memory); + + // return info; + return MAKEBADARG(env, status); +} + +ERL_NIF_TERM +lets_nif_info_size1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + lets_nif_handle* h; + // ERL_NIF_TERM info; + leveldb::Status status; + + if (!enif_get_resource(env, argv[0], lets_nif_RESOURCE, (void**)&h)) { + return MAKEBADARG(env, status); + } + + if (!h->impl.alive) { + return MAKEBADARG(env, status); + } + + // @TODO implementation + // info = enif_make_uint64(env, h->impl.db_size); + + // return info; + return MAKEBADARG(env, status); +} diff --git a/c_src/lets_nif.h b/c_src/lets_nif.h new file mode 100644 index 0000000..483d07b --- /dev/null +++ b/c_src/lets_nif.h @@ -0,0 +1,68 @@ +// The MIT License +// +// Copyright (C) 2011 by Joseph Wayne Norton +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef LETS_NIF_H +#define LETS_NIF_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + + // Prototypes + ERL_NIF_TERM + lets_nif_open6(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_destroy6(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_repair6(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_insert2(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_insert3(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_insert_new2(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_insert_new3(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_delete1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_delete2(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_delete_all_objects1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_lookup2(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_first1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_next2(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_info_memory1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + ERL_NIF_TERM + lets_nif_info_size1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +#ifdef __cplusplus +} +#endif + +#endif /* LETS_NIF_H */ diff --git a/c_src/lets_nif_lib.cc b/c_src/lets_nif_lib.cc new file mode 100644 index 0000000..31adfb8 --- /dev/null +++ b/c_src/lets_nif_lib.cc @@ -0,0 +1,300 @@ +// The MIT License +// +// Copyright (C) 2011 by Joseph Wayne Norton +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#include "lets_nif_lib.h" + + +#if 0 +static bool +return_false(unsigned line) { + fprintf(stderr, "FALSE %s:%d\n", __FILE__, line); + return false; +} +#define FALSE return_false(__LINE__) +#else +#define FALSE false +#endif + + +ERL_NIF_TERM lets_atom_true = 0; +ERL_NIF_TERM lets_atom_false = 0; +ERL_NIF_TERM lets_atom_set = 0; +ERL_NIF_TERM lets_atom_ordered_set = 0; +ERL_NIF_TERM lets_atom_private = 0; +ERL_NIF_TERM lets_atom_protected = 0; +ERL_NIF_TERM lets_atom_public = 0; +ERL_NIF_TERM lets_atom_create_if_missing = 0; +ERL_NIF_TERM lets_atom_error_if_exists = 0; +ERL_NIF_TERM lets_atom_paranoid_checks = 0; +ERL_NIF_TERM lets_atom_write_buffer_size = 0; +ERL_NIF_TERM lets_atom_max_open_files = 0; +ERL_NIF_TERM lets_atom_block_cache_size = 0; +ERL_NIF_TERM lets_atom_block_size = 0; +ERL_NIF_TERM lets_atom_compression = 0; +ERL_NIF_TERM lets_atom_no = 0; +ERL_NIF_TERM lets_atom_snappy = 0; +ERL_NIF_TERM lets_atom_block_restart_interval = 0; +ERL_NIF_TERM lets_atom_verify_checksums = 0; +ERL_NIF_TERM lets_atom_fill_cache = 0; +ERL_NIF_TERM lets_atom_sync = 0; +ERL_NIF_TERM lets_atom_end_of_table = 0; + +bool +lets_nif_lib_init(ErlNifEnv* env) +{ + lets_atom_true = enif_make_atom(env, "true"); + lets_atom_false = enif_make_atom(env, "false"); + lets_atom_set = enif_make_atom(env, "set"); + lets_atom_ordered_set = enif_make_atom(env, "ordered_set"); + lets_atom_private = enif_make_atom(env, "private"); + lets_atom_protected = enif_make_atom(env, "protected"); + lets_atom_public = enif_make_atom(env, "public"); + lets_atom_create_if_missing = enif_make_atom(env, "create_if_missing"); + lets_atom_error_if_exists = enif_make_atom(env, "error_if_exists"); + lets_atom_paranoid_checks = enif_make_atom(env, "paranoid_checks"); + lets_atom_write_buffer_size = enif_make_atom(env, "write_buffer_size"); + lets_atom_max_open_files = enif_make_atom(env, "max_open_files"); + lets_atom_block_cache_size = enif_make_atom(env, "block_cache_size"); + lets_atom_block_size = enif_make_atom(env, "block_size"); + lets_atom_compression = enif_make_atom(env, "compression"); + lets_atom_no = enif_make_atom(env, "no"); + lets_atom_snappy = enif_make_atom(env, "snappy"); + lets_atom_block_restart_interval = enif_make_atom(env, "block_restart_interval"); + lets_atom_verify_checksums = enif_make_atom(env, "verify_checksums"); + lets_atom_fill_cache = enif_make_atom(env, "fill_cache"); + lets_atom_sync = enif_make_atom(env, "sync"); + lets_atom_end_of_table = enif_make_atom(env, "$end_of_table"); + + return true; +} + +bool +lets_init(lets_impl& impl, + const char type, const char privacy, const char* name, const size_t namelen) +{ + impl.type = type; + impl.privacy = privacy; + impl.name = new std::string(name, namelen); + if (!impl.name) { + return FALSE; + } + + impl.db_options = leveldb::Options(); + impl.db_read_options = leveldb::ReadOptions(); + impl.db_write_options = leveldb::WriteOptions(); + + return true; +} + +bool +lets_create(lets_impl& impl, + const char op) +{ + leveldb::Status status; + + // db + switch (op) { + case OPEN: + status = leveldb::DB::Open(impl.db_options, impl.name->c_str(), &(impl.db)); + if (!status.ok()) { + return FALSE; + } else { + // alive + impl.alive = 1; + } + break; + case DESTROY: + status = DestroyDB(impl.name->c_str(), impl.db_options); + if (!status.ok()) { + return FALSE; + } + break; + case REPAIR: + status = RepairDB(impl.name->c_str(), impl.db_options); + if (!status.ok()) { + return FALSE; + } + break; + default: + return FALSE; + } + + return true; +} + +bool +lets_parse_options(ErlNifEnv* env, lets_impl& impl, + ERL_NIF_TERM& options, const ERL_NIF_TERM& options_len) +{ + ERL_NIF_TERM head, tail; + int arity; + const ERL_NIF_TERM* tuple; + + while (enif_get_list_cell(env, options, &head, &tail)) { + if (enif_is_identical(head, lets_atom_create_if_missing)) { + impl.db_options.create_if_missing = true; + } else if (enif_is_identical(head, lets_atom_error_if_exists)) { + impl.db_options.error_if_exists = true; + } else if (enif_is_identical(head, lets_atom_paranoid_checks)) { + impl.db_options.paranoid_checks = true; + } else if (enif_get_tuple(env, head, &arity, &tuple) && arity == 2) { + unsigned int val; + if (enif_is_identical(tuple[0], lets_atom_create_if_missing)) { + if (enif_is_identical(tuple[1], lets_atom_true)) { + impl.db_options.create_if_missing = true; + } else if (enif_is_identical(tuple[1], lets_atom_false)) { + impl.db_options.create_if_missing = false; + } else { + return FALSE; + } + } else if (enif_is_identical(tuple[0], lets_atom_error_if_exists)) { + if (enif_is_identical(tuple[1], lets_atom_true)) { + impl.db_options.error_if_exists = true; + } else if (enif_is_identical(tuple[1], lets_atom_false)) { + impl.db_options.error_if_exists = false; + } else { + return FALSE; + } + } else if (enif_is_identical(tuple[0], lets_atom_paranoid_checks)) { + impl.db_options.paranoid_checks = true; + if (enif_is_identical(tuple[1], lets_atom_true)) { + impl.db_options.paranoid_checks = true; + } else if (enif_is_identical(tuple[1], lets_atom_false)) { + impl.db_options.paranoid_checks = false; + } else { + return FALSE; + } + } else if (enif_is_identical(tuple[0], lets_atom_write_buffer_size) && + enif_get_uint(env, tuple[1], &val)) { + impl.db_options.write_buffer_size = val; + } else if (enif_is_identical(tuple[0], lets_atom_max_open_files) && + enif_get_uint(env, tuple[1], &val)) { + impl.db_options.max_open_files = val; + } else if (enif_is_identical(tuple[0], lets_atom_block_cache_size) && + enif_get_uint(env, tuple[1], &val)) { + impl.db_block_cache_size = val; + impl.db_block_cache = leveldb::NewLRUCache(impl.db_block_cache_size); + impl.db_options.block_cache = impl.db_block_cache; + if (!impl.db_options.block_cache) { + return FALSE; + } + } else if (enif_is_identical(tuple[0], lets_atom_block_size) && + enif_get_uint(env, tuple[1], &val)) { + impl.db_options.block_size = val; + } else if (enif_is_identical(tuple[0], lets_atom_block_restart_interval) && + enif_get_uint(env, tuple[1], &val)) { + impl.db_options.block_restart_interval = val; + } else if (enif_is_identical(tuple[0], lets_atom_compression)) { + if (enif_is_identical(tuple[1], lets_atom_no)) { + impl.db_options.compression = leveldb::kNoCompression; + } else if (enif_is_identical(tuple[1], lets_atom_snappy)) { + impl.db_options.compression = leveldb::kSnappyCompression; + } else { + return FALSE; + } + } else { + return FALSE; + } + } else { + return FALSE; + } + options = tail; + } + + return true; +} + +bool +lets_parse_read_options(ErlNifEnv* env, lets_impl& impl, + ERL_NIF_TERM& options, const ERL_NIF_TERM& options_len) +{ + ERL_NIF_TERM head, tail; + int arity; + const ERL_NIF_TERM* tuple; + + // TODO: snapshot + while (enif_get_list_cell(env, options, &head, &tail)) { + if (enif_is_identical(head, lets_atom_verify_checksums)) { + impl.db_read_options.verify_checksums = true; + } else if (enif_is_identical(head, lets_atom_fill_cache)) { + impl.db_read_options.fill_cache = true; + } else if (enif_get_tuple(env, head, &arity, &tuple) && arity == 2) { + if (enif_is_identical(tuple[0], lets_atom_verify_checksums)) { + if (enif_is_identical(tuple[1], lets_atom_true)) { + impl.db_read_options.verify_checksums = true; + } else if (enif_is_identical(tuple[1], lets_atom_false)) { + impl.db_read_options.verify_checksums = false; + } else { + return FALSE; + } + } else if (enif_is_identical(tuple[0], lets_atom_fill_cache)) { + if (enif_is_identical(tuple[1], lets_atom_true)) { + impl.db_read_options.fill_cache = true; + } else if (enif_is_identical(tuple[1], lets_atom_false)) { + impl.db_read_options.fill_cache = false; + } else { + return FALSE; + } + } else { + return FALSE; + } + } else { + return FALSE; + } + options = tail; + } + + return true; +} + +bool +lets_parse_write_options(ErlNifEnv* env, lets_impl& impl, + ERL_NIF_TERM& options, const ERL_NIF_TERM& options_len) +{ + ERL_NIF_TERM head, tail; + int arity; + const ERL_NIF_TERM* tuple; + + // TODO: snapshot + while (enif_get_list_cell(env, options, &head, &tail)) { + if (enif_is_identical(head, lets_atom_sync)) { + impl.db_write_options.sync = true; + } else if (enif_get_tuple(env, head, &arity, &tuple) && arity == 2) { + if (enif_is_identical(tuple[0], lets_atom_sync)) { + if (enif_is_identical(tuple[1], lets_atom_true)) { + impl.db_write_options.sync = true; + } else if (enif_is_identical(tuple[1], lets_atom_false)) { + impl.db_write_options.sync = false; + } else { + return FALSE; + } + } else { + return FALSE; + } + } else { + return FALSE; + } + options = tail; + } + + return true; +} diff --git a/c_src/lets_nif_lib.h b/c_src/lets_nif_lib.h new file mode 100644 index 0000000..4413663 --- /dev/null +++ b/c_src/lets_nif_lib.h @@ -0,0 +1,119 @@ +// The MIT License +// +// Copyright (C) 2011 by Joseph Wayne Norton +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef LETS_NIF_LIB_H +#define LETS_NIF_LIB_H + +#include + +#include "leveldb/db.h" +#include "leveldb/cache.h" +#include "leveldb/slice.h" +#include "leveldb/write_batch.h" + +#include "erl_nif.h" + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct { + long buflen; + char buf[1]; + } Binary; + + enum Type { + SET = 0x0, + ORDERED_SET = 0x1 + }; + + enum PrivacyType { + PRIVATE = 0x0, + PROTECTED = 0x1, + PUBLIC = 0x2 + }; + + enum DBOpType { + OPEN = 0x0, + DESTROY = 0x1, + REPAIR = 0x2 + }; + + typedef struct + { + char alive; + char type; + char privacy; + std::string* name; + leveldb::Options db_options; + leveldb::ReadOptions db_read_options; + leveldb::WriteOptions db_write_options; + size_t db_block_cache_size; + leveldb::Cache* db_block_cache; + leveldb::DB* db; + ErlNifUInt64 db_memory; + ErlNifUInt64 db_size; + } lets_impl; + + extern ERL_NIF_TERM lets_atom_true; + extern ERL_NIF_TERM lets_atom_false; + extern ERL_NIF_TERM lets_atom_set; + extern ERL_NIF_TERM lets_atom_ordered_set; + extern ERL_NIF_TERM lets_atom_private; + extern ERL_NIF_TERM lets_atom_protected; + extern ERL_NIF_TERM lets_atom_public; + extern ERL_NIF_TERM lets_atom_create_if_missing; + extern ERL_NIF_TERM lets_atom_error_if_exists; + extern ERL_NIF_TERM lets_atom_paranoid_checks; + extern ERL_NIF_TERM lets_atom_write_buffer_size; + extern ERL_NIF_TERM lets_atom_max_open_files; + extern ERL_NIF_TERM lets_atom_block_cache_size; + extern ERL_NIF_TERM lets_atom_block_size; + extern ERL_NIF_TERM lets_atom_compression; + extern ERL_NIF_TERM lets_atom_no; + extern ERL_NIF_TERM lets_atom_snappy; + extern ERL_NIF_TERM lets_atom_block_restart_interval; + extern ERL_NIF_TERM lets_atom_verify_checksums; + extern ERL_NIF_TERM lets_atom_fill_cache; + extern ERL_NIF_TERM lets_atom_sync; + extern ERL_NIF_TERM lets_atom_end_of_table; + + // prototypes + extern bool lets_nif_lib_init(ErlNifEnv* env); + + extern bool lets_init(lets_impl& impl, + const char type, const char privacy, const char* name, const size_t namelen); + extern bool lets_create(lets_impl& impl, + const char op); + + extern bool lets_parse_options(ErlNifEnv* env, lets_impl& impl, + ERL_NIF_TERM& options, const ERL_NIF_TERM& options_len); + extern bool lets_parse_read_options(ErlNifEnv* env, lets_impl& impl, + ERL_NIF_TERM& options, const ERL_NIF_TERM& options_len); + extern bool lets_parse_write_options(ErlNifEnv* env, lets_impl& impl, + ERL_NIF_TERM& options, const ERL_NIF_TERM& options_len); + +#ifdef __cplusplus +} +#endif + +#endif /* LETS_NIF_LIB_H */ diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..fb857c8 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,648 @@ + + +#LETS - LevelDB-based Erlang Term Storage# + + +Copyright (c) 2011 by Joseph Wayne Norton + +__Authors:__ Joseph Wayne Norton ([`norton@alum.mit.edu`](mailto:norton@alum.mit.edu)).

LETS is an alternative Erlang Term Storage using LevelDB as the +storage implementation. LETS tries to address some bad properties of +ETS and DETS. ETS is limited by physical memory. DETS is limited by +a 2 GB file size limitation and does not implement ordered sets. LETS +has neither of these limitations.

+

For testing and comparison purposes, LETS supports three +implementations:

+
    +
  • +

    +drv C+\+ Driver with LevelDB backend (default) +

    +
    • +
  • +

    +nif C+\+ NIF with LevelDB backend +

    +
    • +
  • +

    +ets Erlang ETS backend +

    +
    • +
+

LETS is not intended to be an exact clone of ETS. The currently +supported APIs are:

+
    +
  • +

    +new/2 +

    +
    • +
  • +

    +destroy/2 only driver and nif implementations +

    +
    • +
  • +

    +repair/2 only driver and nif implementations +

    +
    • +
  • +

    +insert/2 +

    +
    • +
  • +

    +insert_new/2 only the ets implementation +

    +
    • +
  • +

    +delete/1 +

    +
    • +
  • +

    +delete/2 +

    +
    • +
  • +

    +delete_all_objects/1 only the ets implementation +

    +
    • +
  • +

    +lookup/2 +

    +
    • +
  • +

    +first/1 +

    +
    • +
  • +

    +next/2 +

    +
    • +
  • +

    +info/2 only a subset of items +

    +
    • +
  • +

    +tab2list/1 +

    +
    • +
+

This repository is experimental in nature - use at your own risk and +please contribute if you find LETS useful.

+ + +##Quick Start Recipe## + + +

To download and build the lets application in one shot, please follow +this recipe:

+ + +
$ mkdir working-directory-name
+$ cd working-directory-name
+$ git clone git://github.com/norton/snappy.git snappy
+$ git clone git://github.com/norton/leveldb.git leveldb
+$ git clone git://github.com/norton/lets.git lets
+$ cd lets
+$ ./rebar get-deps
+$ ./rebar clean
+$ ./rebar compile
+ +

For an alternative recipe with other "features" albeit more complex, +please read further.

+ + + +##Documentation## + + +

Where should I start?

+

This README is the only bit of documentation right now.

+

The QC (a.k.a. QuickCheck, Proper, etc.) tests underneath the +"tests/qc" directory should be helpful for understanding the +specification and behavior of ETS and LETS. These QC tests also +illustrate several strategies for testing Erlang Driver-based and +NIF-based implementations.

+

What is ETS and DETS?

+

ETS and DETS are Erlang/OTP's standard library modules for Erlang +term storage. ETS is a memory-based implementation. DETS is a +disk-based implementation.

+

See http://www.erlang.org/doc/man/ets.html and +http://www.erlang.org/doc/man/dets.html for further details.

+

What is LevelDB?

+

LevelDB is a fast key-value storage library written at Google that +provides an ordered mapping from string keys to string values.

+

See http://code.google.com/p/leveldb/ for further details.

+ + + +##To download## + + +
    +
  1. +

    +Configure your e-mail and name for Git +

    + + +
    $ git config \--global user.email "you@example.com"
+$ git config \--global user.name "Your Name"
    + +
    2. +
  2. +

    +Install Repo +

    + + +
    $ mkdir -p ~/bin
+$ wget -O - https://github.com/android/tools_repo/raw/master/repo > ~/bin/repo
+$ perl -i.bak -pe 's!git://android.git.kernel.org/tools/repo.git!git://github.com/android/tools_repo.git!;' ~/bin/repo
+$ chmod a+x ~/bin/repo
    + + + + + +
    +Caution +Since access to kernel.org has been shutdown due to hackers, +fetch and replace repo tool with android's GitHub repository mirror.
    + +
    3. +
  3. +

    +Create working directory +

    + + +
    $ mkdir working-directory-name
+$ cd working-directory-name
+$ repo init -u git://github.com/norton/manifests.git -m lets-default.xml
    + + + + + +
    +Note +Your "Git" identity is needed during the init step. Please +enter the name and email of your GitHub account if you have one. Team +members having read-write access are recommended to use "repo init -u +git@github.com:norton/manifests.git -m lets-default-rw.xml".
    + + + + + +
    +Tip +If you want to checkout the latest development version, please +append " -b dev" to the repo init command.
    + +
    4. +
  4. +

    +Download Git repositories +

    + + +
    $ cd working-directory-name
+$ repo sync
    + +
    5. +
+

For futher information and help for related tools, please refer to the +following links:

+ + + + +##To build - basic recipe## + + +
    +
  1. +

    +Get and install an erlang system http://www.erlang.org +

    +
    2. +
  2. +

    +Build +

    + + +
    $ cd working-directory-name/src
+$ make compile
    + +
    3. +
+ + + +##To build - optional features## + + +
    +
  1. +

    +Dialyzer Testing basic recipe +

    +
      +
    1. +

      +Build Dialyzer's PLT (required once) +

      + + +
      $ cd working-directory-name/src
+$ make build-plt
      + + + + + +
      +Tip +Check Makefile and dialyzer's documentation for further +information.
      + +
      2. +
    2. +

      +Dialyze with specs +

      + + +
      $ cd working-directory-name/src
+$ make dialyze
      + + + + + +
      +Caution +If you manually run dialyzer with the "-r" option, execute +"make clean compile" first to avoid finding duplicate beam files +underneath rebar's .eunit directory. Check Makefile for further +information.
      + +
      3. +
    3. +

      +Dialyze without specs +

      + + +
      $ cd working-directory-name/src
+$ make dialyze-nospec
      + +
      4. +
    +
    2. +
+ + + +##To test - QuickCheck## + + +
    +
  1. +

    +Make sure QuickCheck is in your Erlang code path. One simple way + to accomplish this is by adding the code path to your ~/.erlang + resource file. +

    + + +
    true = code:add_pathz(os:getenv("HOME")++"/.erlang.d/lib/quviq/eqc-X.Y.Z/ebin").
    + +
    2. +
  2. +

    +Compile for QuickCheck +

    + + +
    $ cd working-directory-name/src
+$ make clean
+$ make compile-eqc eqc-compile
    + +
    3. +
  3. +

    +Run 5,000 QuickCheck tests +

    + + +
    $ cd working-directory-name/src/lib/lets/.eunit
+$ erl -smp +A 5 -pz ../../sext/ebin -pz ../../qc/ebin
+
+1> qc_statem_lets:run(5000).
+.......
    + + + + + +
    +Tip +For testing LevelDB directly using the C bindings, try + qc_statemc_lets:run(5000).
    + +
    4. +
+ + + +##To test - Proper## + + +
    +
  1. +

    +Make sure Proper is in your Erlang code path. One simple way to + accomplish this is by adding the code path to your ~/.erlang + resource file. +

    + + +
    true = code:add_pathz(os:getenv("HOME")++"/.erlang.d/lib/proper/ebin").
    + +
    2. +
  2. +

    +Compile for Proper +

    + + +
    $ cd working-directory-name/src
+$ make clean
+$ make compile-proper proper-compile
    + +
    3. +
  3. +

    +Run 5,000 Proper tests +

    + + +
    $ cd working-directory-name/src/lib/lets/.eunit
+$ erl -smp +A 5 -pz ../../sext/ebin -pz ../../qc/ebin
+
+1> qc_statem_lets:run(5000).
+.......
    + +
    4. +
+ + + +##Roadmap## + + +
    +
  • +

    +Documentation +

    +
      +
    • +

      +Explain how to run QuickCheck/Proper tests using a new rebar + plugin. +

      +
      • +
    • +

      +Explain how to build and to run lets with valgrind enabled + OTP/Erlang virtual machine +

      +
      • +
    +
    • +
  • +

    +Bugs +

    + +
    • +
  • +

    +Performance +

    +
      +
    • +

      +Update driver implementation to use Erlang's asynchronous driver + thread pool for all LevelDB operations. +

      +
      • +
    +
    • +
  • +

    +Testing +

    +
      +
    • +

      +Functional +

      +
        +
      • +

        +Update test model to include LevelDB's database, read, and + write options. These options have not undergone any explicit + testing. +

        +
        • +
      +
      • +
    • +

      +Performance (TBD) +

      +
      • +
    • +

      +Stability (TBD) +

      +
      • +
    +
    • +
  • +

    +New APIs (TBD) +

    + +
    • +
  • +

    +Existing APIs (TBD) +

    +
      +
    • +

      +new/2 - automatically detect and prevent multiple callers of the + same Erlang virtual machine from simultaneously opening the same + LevelDB. Excerpt from the Google leveldb mailing list: +
      +

      +
      • +
    +
    • +
+ + +
Sanjay Ghemawat
+View profile
+ More options Sep 30, 1:04 am
+On Thu, Sep 29, 2011 at 8:30 AM, Joseph Wayne Norton wrote:
+> Hans -
+> Thanks.  Is is correct to assume that it is the caller's responsibility to
+> ensure this does not happen?
+
+leveldb guarantees that it will catch when two distinct processes
+try to open the db concurrently. However it doesn't guarantee what happens
+if the same process tries to do so and therefore it is the caller's
+responsibility
+to check for concurrent opens from the same process.
+This is ugly, but the unix file locking primitives are very annoying in
+this regard. I'll think about whether or not we should clean up the spec
+by doing extra checks inside the leveldb implementation.
+ +
    +
  • +

    +new/2 - investigate if LevelDB's snapshot feature is useful (or + not) for LETS +

    +
    • +
  • +

    +info/2 - investigate if LevelDB's implementation can (easily) + support size and memory info items +

    +
    • +
  • +

    +consider adding explicit read_options and write_options for LET's + operations (rather than just new/2, destroy/2, and repair/2 + operations). +

    +
    • +
+ + + +##Modules## + + + + + + +
lets
lets_drv
lets_ets
lets_nif
+ diff --git a/doc/edoc-info b/doc/edoc-info new file mode 100644 index 0000000..e3ec996 --- /dev/null +++ b/doc/edoc-info @@ -0,0 +1,3 @@ +{application,lets}. +{packages,[]}. +{modules,[lets,lets_drv,lets_ets,lets_nif]}. diff --git a/doc/lets.md b/doc/lets.md new file mode 100644 index 0000000..4abfe57 --- /dev/null +++ b/doc/lets.md @@ -0,0 +1,796 @@ + + +#Module lets# +* [Data Types](#types) +* [Function Index](#index) +* [Function Details](#functions) + + + + + + + +##Data Types## + + + + +###db_opts()## + + + +
db_opts() = {db, [{path, [file:filename()](file.md#type-filename)} | create_if_missing | {create_if_missing, boolean()} | error_if_exists | {error_if_exists, boolean()} | paranoid_checks | {paranoid_checks, boolean()} | {write_buffer_size, pos_integer()} | {max_open_files, pos_integer()} | {block_cache_size, pos_integer()} | {block_size, pos_integer()} | {block_restart_interval, pos_integer()}]}
+ + + +###db_read_opts()## + + + +
db_read_opts() = {db_read, [verify_checksums | {verify_checksums, boolean()} | fill_cache | {fill_cache, boolean()}]}
+ + + +###db_write_opts()## + + + +
db_write_opts() = {db_write, [sync | {sync, boolean()}]}
+ + + +###ets_opt()## + + + +
ets_opt() = set | ordered_set | named_table | {key_pos, pos_integer()} | public | protected | private | compressed
+ + + +###impl_opt()## + + + +
impl_opt() = drv | nif | ets
+ + + +###key()## + + + +
key() = binary()
+ + + +###object()## + + + +
object() = term()
+ + + +###opts()## + + + +
opts() = [[ets_opt()](#type-ets_opt) | [impl_opt()](#type-impl_opt) | [db_opts()](#type-db_opts) | [db_read_opts()](#type-db_read_opts) | [db_write_opts()](#type-db_write_opts)]
+ + + +###tab()## + + + +__abstract datatype__: `tab()` + + +##Function Index## + + +
delete/1

Deletes the entire table Tab.

.
delete/2

Deletes all objects with the key Key from the table Tab.

.
delete_all_objects/1

Delete all objects in the table Tab. The operation is +guaranteed to be atomic and isolated. This function only applies +to the ets implementation.

.
destroy/2

Destroy the contents of the specified table. This function +only applies to driver and nif implementations.

.
first/1

Returns the first key Key in the table Tab. If the table +is empty, $end_of_table will be returned.

.
info/2

Returns information about the table Tab as a list of {Item, + Value} tuples.

+ + +
Valid +Item+ options are:
+ +
    +
  • +

    +owner +

    +
    • +
  • +

    +name +

    +
    • +
  • +

    +named_table only the ets implementation +

    +
    • +
  • +

    +type +

    +
    • +
  • +

    +keypos +

    +
    • +
  • +

    +protection +

    +
    • +
  • +

    +compressed +

    +
    • +
  • +

    +memory only the ets implementation +

    +
    • +
  • +

    +size only the ets implementation +

    +
    • +
.
insert/2

Inserts the object or all of the objects in the list +ObjectOrObjects into the table Tab.

.
insert_new/2

This function works exactly like insert/2, with the +exception that instead of overwriting objects with the same key, it +simply returns false. This function only applies to the ets +implementation.

.
lookup/2

Returns a list of all objects with the key Key in the table +Tab.

.
new/2

Creates a new table and returns a table identifier which can +be used in subsequent operations. The table identifier can be sent +to other processes so that a table can be shared between different +processes within a node.

+ + +
Valid LETS properties for +Options+ are:
+ +
    +
  • +

    +set The table is a set table - one key, one object, no order +among objects. This is the default table type. +

    +
    • +
  • +

    +ordered_set The table is an ordered_set table - one key, one +object, ordered in Erlang term order, which is the order implied +by the < and > operators. +

    +
    • +
  • +

    +named_table If this option is present, the name Name is +associated with the table identifier. only the ets +implementation +

    +
    • +
  • +

    +{key_pos,pos_integer()} Specfies which element in the stored +tuples should be used as key. By default, it is the first +element, i.e. Pos=1. +

    +
    • +
  • +

    +public Any process may read or write to the table. +

    +
    • +
  • +

    +protected The owner process can read and write to the table. +Other processes can only read the table. This is the default +setting for the access rights. +

    +
    • +
  • +

    +private Only the owner process can read or write to the table. +

    +
    • +
  • +

    +compressed If this option is present, the table data will be +stored in a compressed format. +

    +
    • +
  • +

    +drv If this option is present, the table data will be stored +with LevelDB backend via an Erlang Driver. This is the default +setting for the table implementation. +

    +
    • +
  • +

    +nif If this option is present, the table data will be stored +with LevelDB backend via an Erlang NIF. +

    +
    • +
  • +

    +ets If this option is present, the table data will be stored +with ETS as the backend. +

    +
    • +
  • +

    +{db, [db_opts()]} LevelDB database options. +

    +
    • +
  • +

    +{db_read, [db_read_opts()]} LevelDB read options. +

    +
    • +
  • +

    +{db_write, [db_write_opts()]} LevelDB write options. +

    + + +
    Valid LevelDB database properties for +db_opts()+ are:
    + +
    • +
  • +

    +{path, file:filename()} Open the database with the specified +path. The default is Name. +

    +
    • +
  • +

    +create_if_missing | {create_if_missing, boolean()} If true, +the database will be created if it is missing. The default is +false. +

    +
    • +
  • +

    +error_if_exists | {error_if_exists, boolean()} If true, an +error is raised if the database already exists. The default is +false. +

    +
    • +
  • +

    +paranoid_checks | {paranoid_checks, boolean()} If true, the +implementation will do aggressive checking of the data it is +processing and will stop early if it detects any errors. The +default is false. +

    +
    • +
  • +

    +{write_buffer_size, pos_integer()} The default is 4MB. +

    +
    • +
  • +

    +{max_open_files, pos_integer()} The default is 1000. +

    +
    • +
  • +

    +{block_cache_size, pos_integer()} The default is 8MB. +

    +
    • +
  • +

    +{block_size, pos_integer()} The default is 4K. +

    +
    • +
  • +

    +{block_restart_interval, pos_integer()} The default is 16. +

    + + +
    Valid LevelDB read properties for +db_read_opts()+ are:
    + +
    • +
  • +

    +verify_checksums | {verify_checksums, boolean()} If true, all +data read from underlying storage will be verified against +corresponding checksums. The default is false. +

    +
    • +
  • +

    +fill_cache | {fill_cache, boolean()} If true, the data read +should be cached in memory. The default is true. +

    + + +
    Valid LevelDB write properties for +db_write_opts()+ are:
    + +
    • +
  • +

    +sync | {sync, boolean()} If true, the write will be flushed +from the operating system buffer cache before the write is +considered complete. The default is false. +

    +
    • +
.
next/2

Returns the next key Key2, following the key Key1 in the +table Tab. If there is no next key, $end_of_table is +returned.

.
repair/2

If a table cannot be opened, you may attempt to call this +method to resurrect as much of the contents of the table as +possible. Some data may be lost, so be careful when calling this +function on a table that contains important information. This +function only applies to driver and nif implementations.

.
tab2list/1

Returns a list of all objects in the table Tab. The +operation is not guaranteed to be atomic and isolated.

.
+ + + + +##Function Details## + + + +###delete/1## + + + + +
delete(Tab::[tab()](#type-tab)) -> true
+

+ + + + +

Deletes the entire table Tab.

+ + +__See also:__ [ets:delete/1](ets.md#delete-1). + +###delete/2## + + + + +
delete(Tab::[tab()](#type-tab), Key::[key()](#type-key)) -> true
+

+ + + + +

Deletes all objects with the key Key from the table Tab.

+ + +__See also:__ [ets:delete/2](ets.md#delete-2). + +###delete_all_objects/1## + + + + +
delete_all_objects(Tab::[tab()](#type-tab)) -> true
+

+ + + + +

Delete all objects in the table Tab. The operation is +guaranteed to be atomic and isolated. This function only applies +to the ets implementation.

+ + +__See also:__ [ets:delete_all_objects/1](ets.md#delete_all_objects-1). + +###destroy/2## + + + + +
destroy(Name::atom(), Options::[opts()](#type-opts)) -> true
+

+ + + + +

Destroy the contents of the specified table. This function +only applies to driver and nif implementations.

+ + +###first/1## + + + + +
first(Tab::[tab()](#type-tab)) -> [key()](#type-key) | '$end_of_table'
+

+ + + + +

Returns the first key Key in the table Tab. If the table +is empty, $end_of_table will be returned.

+ + +__See also:__ [ets:first/1](ets.md#first-1). + +###info/2## + + + + +
info(Tab::[tab()](#type-tab), Item::atom()) -> term()
+

+ + + + +

Returns information about the table Tab as a list of {Item, + Value} tuples.

+ + +
Valid +Item+ options are:
+ +
    +
  • +

    +owner +

    +
    • +
  • +

    +name +

    +
    • +
  • +

    +named_table only the ets implementation +

    +
    • +
  • +

    +type +

    +
    • +
  • +

    +keypos +

    +
    • +
  • +

    +protection +

    +
    • +
  • +

    +compressed +

    +
    • +
  • +

    +memory only the ets implementation +

    +
    • +
  • +

    +size only the ets implementation +

    +
    • +
+ + +__See also:__ [ets:info/2](ets.md#info-2). + +###insert/2## + + + + +
insert(Tab::[tab()](#type-tab), ObjectOrObjects::[object()](#type-object) | [[object()](#type-object)]) -> true
+

+ + + + +

Inserts the object or all of the objects in the list +ObjectOrObjects into the table Tab.

+ + +__See also:__ [ets:insert/2](ets.md#insert-2). + +###insert_new/2## + + + + +
insert_new(Tab::[tab()](#type-tab), ObjectOrObjects::[object()](#type-object) | [[object()](#type-object)]) -> true
+

+ + + + +

This function works exactly like insert/2, with the +exception that instead of overwriting objects with the same key, it +simply returns false. This function only applies to the ets +implementation.

+ + +__See also:__ [ets:insert_new/2](ets.md#insert_new-2). + +###lookup/2## + + + + +
lookup(Tab::[tab()](#type-tab), Key::[key()](#type-key)) -> [[object()](#type-object)]
+

+ + + + +

Returns a list of all objects with the key Key in the table +Tab.

+ + +__See also:__ [ets:lookup/2](ets.md#lookup-2). + +###new/2## + + + + +
new(Name::atom(), Options::[opts()](#type-opts)) -> [tab()](#type-tab)
+

+ + + + +

Creates a new table and returns a table identifier which can +be used in subsequent operations. The table identifier can be sent +to other processes so that a table can be shared between different +processes within a node.

+ + +
Valid LETS properties for +Options+ are:
+ +
    +
  • +

    +set The table is a set table - one key, one object, no order +among objects. This is the default table type. +

    +
    • +
  • +

    +ordered_set The table is an ordered_set table - one key, one +object, ordered in Erlang term order, which is the order implied +by the < and > operators. +

    +
    • +
  • +

    +named_table If this option is present, the name Name is +associated with the table identifier. only the ets +implementation +

    +
    • +
  • +

    +{key_pos,pos_integer()} Specfies which element in the stored +tuples should be used as key. By default, it is the first +element, i.e. Pos=1. +

    +
    • +
  • +

    +public Any process may read or write to the table. +

    +
    • +
  • +

    +protected The owner process can read and write to the table. +Other processes can only read the table. This is the default +setting for the access rights. +

    +
    • +
  • +

    +private Only the owner process can read or write to the table. +

    +
    • +
  • +

    +compressed If this option is present, the table data will be +stored in a compressed format. +

    +
    • +
  • +

    +drv If this option is present, the table data will be stored +with LevelDB backend via an Erlang Driver. This is the default +setting for the table implementation. +

    +
    • +
  • +

    +nif If this option is present, the table data will be stored +with LevelDB backend via an Erlang NIF. +

    +
    • +
  • +

    +ets If this option is present, the table data will be stored +with ETS as the backend. +

    +
    • +
  • +

    +{db, [db_opts()]} LevelDB database options. +

    +
    • +
  • +

    +{db_read, [db_read_opts()]} LevelDB read options. +

    +
    • +
  • +

    +{db_write, [db_write_opts()]} LevelDB write options. +

    + + +
    Valid LevelDB database properties for +db_opts()+ are:
    + +
    • +
  • +

    +{path, file:filename()} Open the database with the specified +path. The default is Name. +

    +
    • +
  • +

    +create_if_missing | {create_if_missing, boolean()} If true, +the database will be created if it is missing. The default is +false. +

    +
    • +
  • +

    +error_if_exists | {error_if_exists, boolean()} If true, an +error is raised if the database already exists. The default is +false. +

    +
    • +
  • +

    +paranoid_checks | {paranoid_checks, boolean()} If true, the +implementation will do aggressive checking of the data it is +processing and will stop early if it detects any errors. The +default is false. +

    +
    • +
  • +

    +{write_buffer_size, pos_integer()} The default is 4MB. +

    +
    • +
  • +

    +{max_open_files, pos_integer()} The default is 1000. +

    +
    • +
  • +

    +{block_cache_size, pos_integer()} The default is 8MB. +

    +
    • +
  • +

    +{block_size, pos_integer()} The default is 4K. +

    +
    • +
  • +

    +{block_restart_interval, pos_integer()} The default is 16. +

    + + +
    Valid LevelDB read properties for +db_read_opts()+ are:
    + +
    • +
  • +

    +verify_checksums | {verify_checksums, boolean()} If true, all +data read from underlying storage will be verified against +corresponding checksums. The default is false. +

    +
    • +
  • +

    +fill_cache | {fill_cache, boolean()} If true, the data read +should be cached in memory. The default is true. +

    + + +
    Valid LevelDB write properties for +db_write_opts()+ are:
    + +
    • +
  • +

    +sync | {sync, boolean()} If true, the write will be flushed +from the operating system buffer cache before the write is +considered complete. The default is false. +

    +
    • +
+ + +__See also:__ [ets:new/2](ets.md#new-2). + +###next/2## + + + + +
next(Tab::[tab()](#type-tab), Key::[key()](#type-key)) -> [key()](#type-key) | '$end_of_table'
+

+ + + + +

Returns the next key Key2, following the key Key1 in the +table Tab. If there is no next key, $end_of_table is +returned.

+ + +__See also:__ [ets:next/2](ets.md#next-2). + +###repair/2## + + + + +
repair(Name::atom(), Options::[opts()](#type-opts)) -> true
+

+ + + + +

If a table cannot be opened, you may attempt to call this +method to resurrect as much of the contents of the table as +possible. Some data may be lost, so be careful when calling this +function on a table that contains important information. This +function only applies to driver and nif implementations.

+ + +###tab2list/1## + + + + +
tab2list(Tab::[tab()](#type-tab)) -> [[object()](#type-object)]
+

+ + + + +

Returns a list of all objects in the table Tab. The +operation is not guaranteed to be atomic and isolated.

+ + +__See also:__ [ets:tab2list/1](ets.md#tab2list-1). \ No newline at end of file diff --git a/doc/lets_drv.md b/doc/lets_drv.md new file mode 100644 index 0000000..4ccb273 --- /dev/null +++ b/doc/lets_drv.md @@ -0,0 +1,147 @@ + + +#Module lets_drv# +* [Function Index](#index) +* [Function Details](#functions) + + + + + + +##Function Index## + + +
delete/2
delete/3
delete_all_objects/2
destroy/4
first/2
info_memory/2
info_size/2
insert/3
insert_new/3
lookup/3
next/3
open/4
repair/4
tab2list/2
+ + + + +##Function Details## + + + +###delete/2## + + + + +`delete(Tab, Drv) -> any()` + + + +###delete/3## + + + + +`delete(Tab, Drv, Key) -> any()` + + + +###delete_all_objects/2## + + + + +`delete_all_objects(Tab, Drv) -> any()` + + + +###destroy/4## + + + + +`destroy(Tab, Options, ReadOptions, WriteOptions) -> any()` + + + +###first/2## + + + + +`first(Tab, Drv) -> any()` + + + +###info_memory/2## + + + + +`info_memory(Tab, Drv) -> any()` + + + +###info_size/2## + + + + +`info_size(Tab, Drv) -> any()` + + + +###insert/3## + + + + +`insert(Tab, Drv, Object) -> any()` + + + +###insert_new/3## + + + + +`insert_new(Tab, Drv, Object) -> any()` + + + +###lookup/3## + + + + +`lookup(Tab, Drv, Key) -> any()` + + + +###next/3## + + + + +`next(Tab, Drv, Key) -> any()` + + + +###open/4## + + + + +`open(Tab, Options, ReadOptions, WriteOptions) -> any()` + + + +###repair/4## + + + + +`repair(Tab, Options, ReadOptions, WriteOptions) -> any()` + + + +###tab2list/2## + + + + +`tab2list(Tab, Drv) -> any()` + diff --git a/doc/lets_ets.md b/doc/lets_ets.md new file mode 100644 index 0000000..ad37f21 --- /dev/null +++ b/doc/lets_ets.md @@ -0,0 +1,147 @@ + + +#Module lets_ets# +* [Function Index](#index) +* [Function Details](#functions) + + + + + + +##Function Index## + + +
delete/2
delete/3
delete_all_objects/2
destroy/1
first/2
info_memory/2
info_size/2
insert/3
insert_new/3
lookup/3
next/3
open/1
repair/1
tab2list/2
+ + + + +##Function Details## + + + +###delete/2## + + + + +`delete(Tab, Ets) -> any()` + + + +###delete/3## + + + + +`delete(Tab, Ets, Key) -> any()` + + + +###delete_all_objects/2## + + + + +`delete_all_objects(Tab, Ets) -> any()` + + + +###destroy/1## + + + + +`destroy(Tab) -> any()` + + + +###first/2## + + + + +`first(Tab, Ets) -> any()` + + + +###info_memory/2## + + + + +`info_memory(Tab, Ets) -> any()` + + + +###info_size/2## + + + + +`info_size(Tab, Ets) -> any()` + + + +###insert/3## + + + + +`insert(Tab, Ets, ObjectOrObjects) -> any()` + + + +###insert_new/3## + + + + +`insert_new(Tab, Ets, ObjectOrObjects) -> any()` + + + +###lookup/3## + + + + +`lookup(Tab, Ets, Key) -> any()` + + + +###next/3## + + + + +`next(Tab, Ets, Key) -> any()` + + + +###open/1## + + + + +`open(Tab) -> any()` + + + +###repair/1## + + + + +`repair(Tab) -> any()` + + + +###tab2list/2## + + + + +`tab2list(Tab, Ets) -> any()` + diff --git a/doc/lets_nif.md b/doc/lets_nif.md new file mode 100644 index 0000000..f7b4a42 --- /dev/null +++ b/doc/lets_nif.md @@ -0,0 +1,147 @@ + + +#Module lets_nif# +* [Function Index](#index) +* [Function Details](#functions) + + + + + + +##Function Index## + + +
delete/2
delete/3
delete_all_objects/2
destroy/4
first/2
info_memory/2
info_size/2
insert/3
insert_new/3
lookup/3
next/3
open/4
repair/4
tab2list/2
+ + + + +##Function Details## + + + +###delete/2## + + + + +`delete(Tab, Nif) -> any()` + + + +###delete/3## + + + + +`delete(Tab, Nif, Key) -> any()` + + + +###delete_all_objects/2## + + + + +`delete_all_objects(Tab, Nif) -> any()` + + + +###destroy/4## + + + + +`destroy(Tab, Options, ReadOptions, WriteOptions) -> any()` + + + +###first/2## + + + + +`first(Tab, Nif) -> any()` + + + +###info_memory/2## + + + + +`info_memory(Tab, Nif) -> any()` + + + +###info_size/2## + + + + +`info_size(Tab, Nif) -> any()` + + + +###insert/3## + + + + +`insert(Tab, Nif, Object) -> any()` + + + +###insert_new/3## + + + + +`insert_new(Tab, Nif, Object) -> any()` + + + +###lookup/3## + + + + +`lookup(Tab, Nif, Key) -> any()` + + + +###next/3## + + + + +`next(Tab, Nif, Key) -> any()` + + + +###open/4## + + + + +`open(Tab, Options, ReadOptions, WriteOptions) -> any()` + + + +###repair/4## + + + + +`repair(Tab, Options, ReadOptions, WriteOptions) -> any()` + + + +###tab2list/2## + + + + +`tab2list(Tab, Nif) -> any()` + diff --git a/doc/overview.edoc b/doc/overview.edoc new file mode 100644 index 0000000..96fe7b9 --- /dev/null +++ b/doc/overview.edoc @@ -0,0 +1,318 @@ +%% -*- Doc -*- +%% vim: set syntax=asciidoc: +@author Joseph Wayne Norton +@copyright 2011 by Joseph Wayne Norton +@title LETS - LevelDB-based Erlang Term Storage +@doc +LETS is an alternative Erlang Term Storage using LevelDB as the +storage implementation. LETS tries to address some bad properties of +ETS and DETS. ETS is limited by physical memory. DETS is limited by +a 2 GB file size limitation and does not implement ordered sets. LETS +has neither of these limitations. + +For testing and comparison purposes, LETS supports three +implementations: + +- +drv+ C\+\+ Driver with LevelDB backend _(default)_ +- +nif+ C\+\+ NIF with LevelDB backend +- +ets+ Erlang ETS backend + +LETS is not intended to be an exact clone of ETS. The currently +supported APIs are: + +- +new/2+ +- +destroy/2+ _only driver and nif implementations_ +- +repair/2+ _only driver and nif implementations_ +- +insert/2+ +- +insert_new/2+ _only the ets implementation_ +- +delete/1+ +- +delete/2+ +- +delete_all_objects/1+ _only the ets implementation_ +- +lookup/2+ +- +first/1+ +- +next/2+ +- +info/2+ _only a subset of items_ +- +tab2list/1+ + +_This repository is experimental in nature - use at your own risk and +please contribute if you find LETS useful._ + +== Quick Start Recipe + +To download and build the lets application in one shot, please follow +this recipe: + +------ +$ mkdir working-directory-name +$ cd working-directory-name +$ git clone git://github.com/norton/snappy.git snappy +$ git clone git://github.com/norton/leveldb.git leveldb +$ git clone git://github.com/norton/lets.git lets +$ cd lets +$ ./rebar get-deps +$ ./rebar clean +$ ./rebar compile +------ + +For an alternative recipe with other "features" albeit more complex, +please read further. + +== Documentation + +=== Where should I start? + +This README is the only bit of documentation right now. + +The QC (a.k.a. QuickCheck, Proper, etc.) tests underneath the +"tests/qc" directory should be helpful for understanding the +specification and behavior of ETS and LETS. These QC tests also +illustrate several strategies for testing Erlang Driver-based and +NIF-based implementations. + +=== What is ETS and DETS? + +ETS and DETS are Erlang/OTP\'s standard library modules for Erlang +term storage. ETS is a memory-based implementation. DETS is a +disk-based implementation. + +See http://www.erlang.org/doc/man/ets.html and +http://www.erlang.org/doc/man/dets.html for further details. + +=== What is LevelDB? + +LevelDB is a fast key-value storage library written at Google that +provides an ordered mapping from string keys to string values. + +See http://code.google.com/p/leveldb/ for further details. + +== To download + +1. Configure your e-mail and name for Git ++ +------ +$ git config \--global user.email "you@example.com" +$ git config \--global user.name "Your Name" +------ + +2. Install Repo ++ +------ +$ mkdir -p ~/bin +$ wget -O - https://github.com/android/tools_repo/raw/master/repo > ~/bin/repo +$ perl -i.bak -pe 's!git://android.git.kernel.org/tools/repo.git!git://github.com/android/tools_repo.git!;' ~/bin/repo +$ chmod a+x ~/bin/repo +------ ++ +CAUTION: Since access to kernel.org has been shutdown due to hackers, +fetch and replace repo tool with android\'s GitHub repository mirror. + +3. Create working directory ++ +------ +$ mkdir working-directory-name +$ cd working-directory-name +$ repo init -u git://github.com/norton/manifests.git -m lets-default.xml +------ ++ +NOTE: Your "Git" identity is needed during the init step. Please +enter the name and email of your GitHub account if you have one. Team +members having read-write access are recommended to use "repo init -u +git@github.com:norton/manifests.git -m lets-default-rw.xml". ++ +TIP: If you want to checkout the latest development version, please +append " -b dev" to the repo init command. + +4. Download Git repositories ++ +------ +$ cd working-directory-name +$ repo sync +------ + +For futher information and help for related tools, please refer to the +following links: + +- Erlang - http://www.erlang.org/ + * *R14B03 or newer, R14B04 has been tested most recently* +- Git - http://git-scm.com/ + * *Git 1.5.4 or newer, Git 1.7.7 has been tested recently* + * _required for Repo and GitHub_ +- GitHub - https://github.com +- Python - http://www.python.org + * *Python 2.4 or newer, Python 2.7.1 has been tested most recently + (CAUTION: Python 3.x might be too new)* +* _required for Repo_ + - Rebar - https://github.com/basho/rebar/wiki + - Repo - http://source.android.com/source/git-repo.html + +== To build - basic recipe + +1. Get and install an erlang system http://www.erlang.org + +2. Build ++ +------ +$ cd working-directory-name/src +$ make compile +------ + +== To build - optional features + +A. Dialyzer Testing _basic recipe_ + 1. Build Dialyzer\'s PLT _(required once)_ ++ +------ +$ cd working-directory-name/src +$ make build-plt +------ ++ +TIP: Check Makefile and dialyzer\'s documentation for further +information. + + 2. Dialyze with specs ++ +------ +$ cd working-directory-name/src +$ make dialyze +------ ++ +CAUTION: If you manually run dialyzer with the "-r" option, execute +"make clean compile" first to avoid finding duplicate beam files +underneath rebar\'s .eunit directory. Check Makefile for further +information. + + 3. Dialyze without specs ++ +------ +$ cd working-directory-name/src +$ make dialyze-nospec +------ + +== To test - QuickCheck + +1. Make sure QuickCheck is in your Erlang code path. One simple way + to accomplish this is by adding the code path to your +~/.erlang+ + resource file. ++ +------ +true = code:add_pathz(os:getenv("HOME")++"/.erlang.d/lib/quviq/eqc-X.Y.Z/ebin"). +------ + +2. Compile for QuickCheck ++ +------ +$ cd working-directory-name/src +$ make clean +$ make compile-eqc eqc-compile +------ + +3. Run 5,000 QuickCheck tests ++ +------ +$ cd working-directory-name/src/lib/lets/.eunit +$ erl -smp +A 5 -pz ../../sext/ebin -pz ../../qc/ebin + +1> qc_statem_lets:run(5000). +....... +------ ++ +TIP: For testing LevelDB directly using the C bindings, try + +qc_statemc_lets:run(5000)+. + +== To test - Proper + +1. Make sure Proper is in your Erlang code path. One simple way to + accomplish this is by adding the code path to your +~/.erlang+ + resource file. ++ +------ +true = code:add_pathz(os:getenv("HOME")++"/.erlang.d/lib/proper/ebin"). +------ + +2. Compile for Proper ++ +------ +$ cd working-directory-name/src +$ make clean +$ make compile-proper proper-compile +------ + +3. Run 5,000 Proper tests ++ +------ +$ cd working-directory-name/src/lib/lets/.eunit +$ erl -smp +A 5 -pz ../../sext/ebin -pz ../../qc/ebin + +1> qc_statem_lets:run(5000). +....... +------ + +== Roadmap + +- Documentation + * Explain how to run QuickCheck/Proper tests using a new rebar + plugin. + * Explain how to build and to run lets with valgrind enabled + OTP/Erlang virtual machine + +- Bugs + * LevelDB - Reappearing "ghost" key after 17 steps + (http://code.google.com/p/leveldb/issues/detail?id=44) + + + NOTE: LET\'s QC tests are hard-coded not to close and then to + reopen a LevelDB database until this bug has been fixed. + +- Performance + * Update driver implementation to use Erlang\'s asynchronous driver + thread pool for all LevelDB operations. + +- Testing + * Functional + ** Update test model to include LevelDB\'s database, read, and + write options. These options have not undergone any explicit + testing. + * Performance (TBD) + * Stability (TBD) + +- New APIs (TBD) + * +insert_new/2+ + (http://code.google.com/p/leveldb/issues/detail?id=42) + + * +delete_all_objects/1+ + (http://code.google.com/p/leveldb/issues/detail?id=43) + +- Existing APIs (TBD) + * +new/2+ - automatically detect and prevent multiple callers of the + same Erlang virtual machine from simultaneously opening the same + LevelDB. Excerpt from the Google leveldb mailing list: + + +------ +Sanjay Ghemawat +View profile + More options Sep 30, 1:04 am +On Thu, Sep 29, 2011 at 8:30 AM, Joseph Wayne Norton wrote: +> Hans - +> Thanks. Is is correct to assume that it is the caller's responsibility to +> ensure this does not happen? + +leveldb guarantees that it will catch when two distinct processes +try to open the db concurrently. However it doesn't guarantee what happens +if the same process tries to do so and therefore it is the caller's +responsibility +to check for concurrent opens from the same process. +This is ugly, but the unix file locking primitives are very annoying in +this regard. I'll think about whether or not we should clean up the spec +by doing extra checks inside the leveldb implementation. +------ + + * +new/2+ - investigate if LevelDB\'s snapshot feature is useful (or + not) for LETS + + * +info/2+ - investigate if LevelDB\'s implementation can (easily) + support size and memory info items + + * consider adding explicit read_options and write_options for LET\'s + operations (rather than just +new/2+, +destroy/2+, and +repair/2+ + operations). + +@end diff --git a/rebar.config b/rebar.config new file mode 100644 index 0000000..e392608 --- /dev/null +++ b/rebar.config @@ -0,0 +1,72 @@ +%%% -*- mode: erlang -*- + +%% Require OTP version R13B04 or R14 +{require_otp_vsn, "R13B04|R14"}. + +%% Depends +{deps_dir, "../"}. +{deps, [{meck, "", {git, "git://github.com/norton/meck.git"}} + , {edown, "", {git, "git://github.com/norton/edown.git"}} + , {asciiedoc, "", {git, "git://github.com/norton/asciiedoc.git"}} + , {qc, "", {git, "git://github.com/norton/qc.git"}} + , {sext, "", {git, "git://github.com/norton/sext.git"}} + ]}. + +%% Erlang compiler options +{erl_opts, [debug_info, warnings_as_errors + ]}. +%% Erlang edoc options for asciiedown_doclet +{edoc_opts, [{doclet, asciiedown_doclet} + , {app_default, "http://www.erlang.org/doc/man"} + , {new, true} + , {packages, false} + , {stylesheet, ""} % don't copy stylesheet.css + , {image, ""} % don't copy erlang.png + , {top_level_readme, {"./README.md", "https://github.com/norton/lets"}} + ]}. + +%% Erlang edoc options for asciiedoc_doclet +%% {edoc_opts, [{doclet, asciiedoc_doclet} +%% , {app_default, "http://www.erlang.org/doc/man"} +%% , {new, true} +%% , {packages, false} +%% ]}. + +%% Eunit compiler options +{eunit_compile_opts, [ + %% {d, 'QC', true} + %% For Proper, comment QC_EQC, uncomment + %% QC_PROPER, and re-compile + %% , {d, 'QC_PROPER', true} + %% For QuickCheck, comment QC_PROPER, uncomment + %% QC_EQC, and re-compile + %% , {d, 'QC_EQC', true} + ]}. + +%% EUnit options +{eunit_opts, [verbose]}. + +%% Port compilation environment variables. +{port_envs, [ + %% DRV_CFLAGS + {"DRV_CFLAGS", "$DRV_CFLAGS -fPIC -g -Werror"}, + {"DRV_CFLAGS", "$DRV_CFLAGS -I c_src/leveldb/include"}, + %% DRV_LDFLAGS + {"DRV_LDFLAGS", "$DRV_LDFLAGS c_src/leveldb/lib/libleveldb.a c_src/snappy/lib/libsnappy.a -lstdc++"} + ]}. + +%% Pre command hooks +{pre_hooks, [{clean, "c_src/build_deps.sh clean"}, + {compile, "c_src/build_deps.sh c_src/leveldb/lib/libleveldb.a"}]}. + +%% port_sources +{port_sources, ["c_src/lets_drv_lib.cc" + , "c_src/lets_drv.cc" + , "c_src/lets_nif_lib.cc" + , "c_src/lets_nif.cc" + ]}. + +%% so_specs +{so_specs, [{"priv/lib/lets_drv.so", ["c_src/lets_drv_lib.o", "c_src/lets_drv.o"]} + , {"priv/lib/lets_nif.so", ["c_src/lets_nif_lib.o", "c_src/lets_nif.o"]} + ]}. diff --git a/src/lets.app.src b/src/lets.app.src new file mode 100644 index 0000000..ca04535 --- /dev/null +++ b/src/lets.app.src @@ -0,0 +1,33 @@ +%%% -*- mode: erlang -*- + +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +{application, lets, + [ + {description, "LevelDB ETS"}, + {vsn, git}, + {registered, []}, + {applications, [kernel, stdlib, sasl]}, + {modules, [lets, lets_drv, lets_ets, lets_nif]} + ] +}. diff --git a/src/lets.erl b/src/lets.erl new file mode 100644 index 0000000..63f51f8 --- /dev/null +++ b/src/lets.erl @@ -0,0 +1,510 @@ +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +-module(lets). + +-include("lets.hrl"). + +%% External exports +-export([new/2 + , destroy/2 + , repair/2 + , insert/2 + , insert_new/2 + , delete/1 + , delete/2 + , delete_all_objects/1 + , lookup/2 + , first/1 + , next/2 + , info/2 + , tab2list/1 + ]). + +-export_type([tab/0]). + + +%%%---------------------------------------------------------------------- +%%% Types/Specs/Records +%%%---------------------------------------------------------------------- + +-opaque tab() :: #tab{}. + +-type opts() :: [ets_opt() | impl_opt() | db_opts() | db_read_opts() | db_write_opts()]. +-type ets_opt() :: set | ordered_set | named_table | {key_pos,pos_integer()} | public | protected | private | compressed. +-type impl_opt() :: drv | nif | ets. + +-type db_opts() :: {db, [{path,file:filename()} | create_if_missing | {create_if_missing,boolean()} | error_if_exists | {error_if_exists,boolean()} | paranoid_checks | {paranoid_checks,boolean()} | {write_buffer_size,pos_integer()} | {max_open_files,pos_integer()} | {block_cache_size,pos_integer()} | {block_size,pos_integer()} | {block_restart_interval,pos_integer()}]}. +-type db_read_opts() :: {db_read, [verify_checksums | {verify_checksums,boolean()} | fill_cache | {fill_cache,boolean()}]}. +-type db_write_opts() :: {db_write, [sync | {sync,boolean()}]}. + +-type key() :: binary(). +-type object() :: term(). + + +%%%---------------------------------------------------------------------- +%%% API +%%%---------------------------------------------------------------------- + +%% @doc Creates a new table and returns a table identifier which can +%% be used in subsequent operations. The table identifier can be sent +%% to other processes so that a table can be shared between different +%% processes within a node. +%% +%% Valid LETS properties for +Options+ are: +%% +%% - +set+ The table is a set table - one key, one object, no order +%% among objects. This is the default table type. +%% +%% - +ordered_set+ The table is an ordered_set table - one key, one +%% object, ordered in Erlang term order, which is the order implied +%% by the +<+ and +>+ operators. +%% +%% - +named_table+ If this option is present, the name +Name+ is +%% associated with the table identifier. _only the ets +%% implementation_ +%% +%% - +{key_pos,pos_integer()}+ Specfies which element in the stored +%% tuples should be used as key. By default, it is the first +%% element, i.e. +Pos=1+. +%% +%% - +public+ Any process may read or write to the table. +%% +%% - +protected+ The owner process can read and write to the table. +%% Other processes can only read the table. This is the default +%% setting for the access rights. +%% +%% - +private+ Only the owner process can read or write to the table. +%% +%% - +compressed+ If this option is present, the table data will be +%% stored in a compressed format. +%% +%% - +drv+ If this option is present, the table data will be stored +%% with LevelDB backend via an Erlang Driver. This is the default +%% setting for the table implementation. +%% +%% - +nif+ If this option is present, the table data will be stored +%% with LevelDB backend via an Erlang NIF. +%% +%% - +ets+ If this option is present, the table data will be stored +%% with ETS as the backend. +%% +%% - +{db, [db_opts()]}+ LevelDB database options. +%% +%% - +{db_read, [db_read_opts()]}+ LevelDB read options. +%% +%% - +{db_write, [db_write_opts()]}+ LevelDB write options. +%% +%% Valid LevelDB database properties for +db_opts()+ are: +%% +%% - +{path, file:filename()}+ Open the database with the specified +%% path. The default is +Name+. +%% +%% - +create_if_missing | {create_if_missing, boolean()}+ If +true+, +%% the database will be created if it is missing. The default is +%% +false+. +%% +%% - +error_if_exists | {error_if_exists, boolean()}+ If +true+, an +%% error is raised if the database already exists. The default is +%% +false+. +%% +%% - +paranoid_checks | {paranoid_checks, boolean()}+ If +true+, the +%% implementation will do aggressive checking of the data it is +%% processing and will stop early if it detects any errors. The +%% default is +false+. +%% +%% - +{write_buffer_size, pos_integer()}+ The default is 4MB. +%% +%% - +{max_open_files, pos_integer()}+ The default is 1000. +%% +%% - +{block_cache_size, pos_integer()}+ The default is 8MB. +%% +%% - +{block_size, pos_integer()}+ The default is 4K. +%% +%% - +{block_restart_interval, pos_integer()}+ The default is 16. +%% +%% Valid LevelDB read properties for +db_read_opts()+ are: +%% +%% - +verify_checksums | {verify_checksums, boolean()}+ If +true+, all +%% data read from underlying storage will be verified against +%% corresponding checksums. The default is +false+. +%% +%% - +fill_cache | {fill_cache, boolean()}+ If +true+, the data read +%% should be cached in memory. The default is +true+. +%% +%% Valid LevelDB write properties for +db_write_opts()+ are: +%% +%% - +sync | {sync, boolean()}+ If +true+, the write will be flushed +%% from the operating system buffer cache before the write is +%% considered complete. The default is +false+. +%% +%% @end +%% @see ets:new/2 + +-spec new(Name::atom(), Options::opts()) -> tab(). +new(Name, Opts) -> + create(open, Name, Opts). + +%% @doc Destroy the contents of the specified table. This function +%% only applies to +driver+ and +nif+ implementations. +%% @end + +-spec destroy(Name::atom(), Options::opts()) -> true. +destroy(Name, Opts) -> + create(destroy, Name, Opts). + + +%% @doc If a table cannot be opened, you may attempt to call this +%% method to resurrect as much of the contents of the table as +%% possible. Some data may be lost, so be careful when calling this +%% function on a table that contains important information. This +%% function only applies to +driver+ and +nif+ implementations. +%% @end + +-spec repair(Name::atom(), Options::opts()) -> true. +repair(Name, Opts) -> + create(repair, Name, Opts). + +%% @doc Inserts the object or all of the objects in the list +%% +ObjectOrObjects+ into the table +Tab+. +%% @end +%% @see ets:insert/2 + +-spec insert(Tab::tab(), ObjectOrObjects::object() | [object()]) -> true. +insert(Tab, ObjectOrObjects) -> + case check_access(Tab) of + undefined -> + erlang:error(badarg, [Tab]); + {Mod, Impl} -> + Mod:insert(Tab, Impl, ObjectOrObjects) + end. + +%% @doc This function works exactly like +insert/2+, with the +%% exception that instead of overwriting objects with the same key, it +%% simply returns false. This function only applies to the +ets+ +%% implementation. +%% @end +%% @see ets:insert_new/2 + +-spec insert_new(tab(), object() | [object()]) -> true. +insert_new(Tab, ObjectOrObjects) -> + case check_access(Tab) of + undefined -> + erlang:error(badarg, [Tab]); + {Mod, Impl} -> + Mod:insert_new(Tab, Impl, ObjectOrObjects) + end. + +%% @doc Deletes the entire table +Tab+. +%% @end +%% @see ets:delete/1 + +-spec delete(tab()) -> true. +delete(Tab) -> + case check_access(Tab) of + undefined -> + erlang:error(badarg, [Tab]); + {Mod, Impl} -> + Mod:delete(Tab, Impl) + end. + +%% @doc Deletes all objects with the key +Key+ from the table +Tab+. +%% @end +%% @see ets:delete/2 + +-spec delete(tab(), key()) -> true. +delete(Tab, Key) -> + case check_access(Tab) of + undefined -> + erlang:error(badarg, [Tab]); + {Mod, Impl} -> + Mod:delete(Tab, Impl, Key) + end. + +%% @doc Delete all objects in the table +Tab+. The operation is +%% guaranteed to be atomic and isolated. This function only applies +%% to the +ets+ implementation. +%% @end +%% @see ets:delete_all_objects/1 + +-spec delete_all_objects(tab()) -> true. +delete_all_objects(Tab) -> + case check_access(Tab) of + undefined -> + erlang:error(badarg, [Tab]); + {Mod, Impl} -> + Mod:delete_all_objects(Tab, Impl) + end. + +%% @doc Returns a list of all objects with the key +Key+ in the table +%% +Tab+. +%% @end +%% @see ets:lookup/2 + +-spec lookup(tab(), key()) -> [object()]. +lookup(Tab, Key) -> + case check_access(Tab) of + undefined -> + erlang:error(badarg, [Tab]); + {Mod, Impl} -> + Mod:lookup(Tab, Impl, Key) + end. + +%% @doc Returns the first key +Key+ in the table +Tab+. If the table +%% is empty, +'$end_of_table'+ will be returned. +%% @end +%% @see ets:first/1 + +-spec first(tab()) -> key() | '$end_of_table'. +first(Tab) -> + case check_access(Tab) of + undefined -> + erlang:error(badarg, [Tab]); + {Mod, Impl} -> + Mod:first(Tab, Impl) + end. + +%% @doc Returns the next key +Key2+, following the key +Key1+ in the +%% table +Tab+. If there is no next key, +'$end_of_table'+ is +%% returned. +%% @end +%% @see ets:next/2 + +-spec next(tab(), key()) -> key() | '$end_of_table'. +next(Tab, Key) -> + case check_access(Tab) of + undefined -> + erlang:error(badarg, [Tab]); + {Mod, Impl} -> + Mod:next(Tab, Impl, Key) + end. + +%% @doc Returns information about the table +Tab+ as a list of +{Item, +%% Value}+ tuples. +%% +%% Valid +Item+ options are: +%% +%% - +owner+ +%% - +name+ +%% - +named_table+ _only the ets implementation_ +%% - +type+ +%% - +keypos+ +%% - +protection+ +%% - +compressed+ +%% - +memory+ _only the ets implementation_ +%% - +size+ _only the ets implementation_ +%% +%% @end +%% @see ets:info/2 + +-spec info(tab(), atom()) -> term(). +info(Tab, Item) -> + case check_access(Tab) of + undefined -> + erlang:error(badarg, [Tab]); + {Mod, Impl} -> + case Item of + owner -> + Tab#tab.owner; + name -> + Tab#tab.name; + named_table -> + Tab#tab.named_table; + type -> + Tab#tab.type; + keypos -> + Tab#tab.keypos; + protection -> + Tab#tab.protection; + compressed -> + Tab#tab.compressed; + memory -> + Mod:info_memory(Tab, Impl); + size -> + Mod:info_size(Tab, Impl); + _ -> + erlang:error(badarg, [Tab, Item]) + end + end. + +%% @doc Returns a list of all objects in the table +Tab+. The +%% operation is *not* guaranteed to be atomic and isolated. +%% @end +%% @see ets:tab2list/1 + +-spec tab2list(tab()) -> [object()]. +tab2list(Tab) -> + case check_access(Tab) of + undefined -> + erlang:error(badarg, [Tab]); + {Mod, Impl} -> + Mod:tab2list(Tab, Impl) + end. + + +%%%---------------------------------------------------------------------- +%%% Internal functions +%%%---------------------------------------------------------------------- + + +check_access(#tab{protection=Protection, owner=Owner, drv=Drv, nif=undefined, ets=undefined}) + when Protection==public orelse Owner==self() -> + {lets_drv, Drv}; +check_access(#tab{protection=Protection, owner=Owner, drv=undefined, nif=Nif, ets=undefined}) + when Protection==public orelse Owner==self() -> + {lets_nif, Nif}; +check_access(#tab{protection=Protection, owner=Owner, drv=undefined, nif=undefined, ets=Ets}) + when Protection==public orelse Owner==self() -> + {lets_ets, Ets}; +check_access(_Tab) -> + undefined. + +create(Op, Name, Opts) -> + case options(Opts) of + {POpts, []} -> + POpts; + {POpts, BadArgs} -> + erlang:error(badarg, [Name, BadArgs]) + end, + + Owner = self(), + NamedTable = proplists:get_bool(named_table, POpts), + Type = + case proplists:get_bool(ordered_set, POpts) of + true -> + ordered_set; + false -> + set + end, + KeyPos = proplists:get_value(keypos, POpts, 1), + Protection = + case proplists:get_bool(private, POpts) of + true -> + private; + false -> + case proplists:get_bool(protected, POpts) of + true -> + protected; + false -> + case proplists:get_bool(public, POpts) of + true -> + public; + false -> + protected + end + end + end, + Compressed = proplists:get_bool(compressed, POpts), + Drv = proplists:get_bool(drv, POpts), + Nif = proplists:get_bool(nif, POpts), + Ets = proplists:get_bool(ets, POpts), + + Tab = #tab{owner=Owner, + name=Name, + named_table=NamedTable, + type=Type, + keypos=KeyPos, + protection=Protection, + compressed=Compressed}, + + DBOptions = fix_db_options(Tab, proplists:get_value(db, POpts, [])), + DBReadOptions = proplists:get_value(db_read, POpts, []), + DBWriteOptions = proplists:get_value(db_write, POpts, []), + + if Drv -> + lets_drv:Op(Tab, DBOptions, DBReadOptions, DBWriteOptions); + Nif -> + lets_nif:Op(Tab, DBOptions, DBReadOptions, DBWriteOptions); + Ets -> + lets_ets:Op(Tab); + true -> + lets_drv:Op(Tab, DBOptions, DBReadOptions, DBWriteOptions) + end. + +fix_db_options(#tab{name=Name, compressed=Compressed}, Options) -> + fix_db_options_compression(Compressed, fix_db_options_path(Name, Options)). + +fix_db_options_path(Name, Options) -> + case proplists:lookup(path, Options) of + none -> + [{path, binify(Name)}|Options]; + {path, Path} -> + [{path, binify(Path)}|proplists:delete(path, Options)] + end. + +fix_db_options_compression(false, Options) -> + [{compression, no}|Options]; +fix_db_options_compression(true, Options) -> + [{compression, snappy}|Options]. + +binify(X) when is_atom(X) -> + list_to_binary(atom_to_list(X)); +binify(X) when is_list(X) -> + list_to_binary(X); +binify(X) when is_binary(X) -> + X. + +options(Options) -> + Keys = [set, ordered_set, named_table, keypos, public, protected, private, compressed, drv, nif, ets, db, db_read, db_write], + options(Options, Keys). + +options(Options, Keys) when is_list(Options) -> + options(Options, Keys, []); +options(Option, Keys) -> + options([Option], Keys, []). + +options(Options, [Key|Keys], L) when is_list(Options) -> + case proplists:lookup(Key, Options) of + none -> + options(Options, Keys, L); + {Key, Value} -> + case Key of + Key when Key == db; Key == db_read; Key == db_write -> + sub_options(Key, Value, Options, Keys, L); + _ -> + NewOptions = proplists:delete(Key, Options), + options(NewOptions, Keys, [{Key,Value}|L]) + end + end; +options(Options, [], L) -> + {lists:reverse(L), Options}. + +sub_options(db=Key, Value, Options, Keys, L) -> + SubKeys = [path, create_if_missing, error_if_exists, paranoid_checks, write_buffer_size, max_open_files, block_cache_size, block_size, block_restart_interval], + sub_options(Key, Value, Options, Keys, L, SubKeys); +sub_options(db_read=Key, Value, Options, Keys, L) -> + %% @TODO snapshot + SubKeys = [verify_checksums, fill_cache], + sub_options(Key, Value, Options, Keys, L, SubKeys); +sub_options(db_write=Key, Value, Options, Keys, L) -> + %% @TODO snapshot + SubKeys = [sync], + sub_options(Key, Value, Options, Keys, L, SubKeys). + +sub_options(Key, Value, Options, Keys, L, SubKeys) -> + case options(Value, SubKeys) of + {NewValue, []} -> + NewOptions = proplists:delete(Key, Options), + options(NewOptions, Keys, [{Key,NewValue}|L]); + {_NewValue, _} -> + options(Options, Keys, L) + end. diff --git a/src/lets.hrl b/src/lets.hrl new file mode 100644 index 0000000..00b1479 --- /dev/null +++ b/src/lets.hrl @@ -0,0 +1,40 @@ +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +-ifndef(lets). +-define(lets, true). + +-opaque nif() :: term(). + +-record(tab, {owner :: pid(), + name :: atom(), + named_table=false :: boolean(), + type=set :: set|ordered_set, + keypos=1 :: pos_integer(), + protection=protected :: public|protected|private, + compressed=false :: boolean(), + drv :: port() | undefined, + nif :: nif() | undefined, + ets :: ets:tab() | undefined + }). + +-endif. % -ifndef(lets). diff --git a/src/lets_drv.erl b/src/lets_drv.erl new file mode 100644 index 0000000..f4bfdd7 --- /dev/null +++ b/src/lets_drv.erl @@ -0,0 +1,278 @@ +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +-module(lets_drv). + +-include("lets.hrl"). + +%% External exports +-export([open/4 + , destroy/4 + , repair/4 + , insert/3 + , insert_new/3 + , delete/2 + , delete/3 + , delete_all_objects/2 + , lookup/3 + , first/2 + , next/3 + , info_memory/2 + , info_size/2 + , tab2list/2 + ]). + + +%%%---------------------------------------------------------------------- +%%% Types/Specs/Records +%%%---------------------------------------------------------------------- + +-define(LETS_BADARG, 16#00). +-define(LETS_TRUE, 16#01). +-define(LETS_END_OF_TABLE, 16#02). + +-define(LETS_OPEN6, 16#00). +-define(LETS_DESTROY6, 16#01). +-define(LETS_REPAIR6, 16#02). +-define(LETS_INSERT2, 16#03). +-define(LETS_INSERT3, 16#04). +-define(LETS_INSERT_NEW2, 16#05). +-define(LETS_INSERT_NEW3, 16#06). +-define(LETS_DELETE1, 16#07). +-define(LETS_DELETE2, 16#08). +-define(LETS_DELETE_ALL_OBJECTS1, 16#09). +-define(LETS_LOOKUP2, 16#0A). +-define(LETS_FIRST1, 16#0B). +-define(LETS_NEXT2, 16#0C). +-define(LETS_INFO_MEMORY1, 16#0D). +-define(LETS_INFO_SIZE1, 16#0E). + + +%%%---------------------------------------------------------------------- +%%% API +%%%---------------------------------------------------------------------- + +init() -> + Path = + case code:priv_dir(lets) of + {error, bad_name} -> + "../priv/lib"; + Dir -> + filename:join([Dir, "lib"]) + end, + case erl_ddll:load_driver(Path, lets_drv) of + ok -> ok; + {error, already_loaded} -> ok; + {error, permanent} -> ok; + {error, {open_error, _}=Err} -> + FormattedErr = erl_ddll:format_error(Err), + error_logger:error_msg("Failed to load the driver library lets_drv. " + ++ "Error: ~p, Path: ~p~n", + [FormattedErr, + filename:join(Path, lets_drv) + ]), + erlang:exit({Err, FormattedErr}) + end, + open_port({spawn, "lets_drv"}, [binary]). + +open(#tab{name=_Name, named_table=_Named, type=Type, protection=Protection}=Tab, Options, ReadOptions, WriteOptions) -> + {value, {path,Path}, NewOptions} = lists:keytake(path, 1, Options), + Drv = impl_open(Type, Protection, Path, NewOptions, ReadOptions, WriteOptions), + %% @TODO implement named Drv (of sorts) + Tab#tab{drv=Drv}. + +destroy(#tab{type=Type, protection=Protection}, Options, ReadOptions, WriteOptions) -> + {value, {path,Path}, NewOptions} = lists:keytake(path, 1, Options), + impl_destroy(Type, Protection, Path, NewOptions, ReadOptions, WriteOptions). + +repair(#tab{type=Type, protection=Protection}, Options, ReadOptions, WriteOptions) -> + {value, {path,Path}, NewOptions} = lists:keytake(path, 1, Options), + impl_repair(Type, Protection, Path, NewOptions, ReadOptions, WriteOptions). + +insert(#tab{keypos=KeyPos, type=Type}, Drv, Object) when is_tuple(Object) -> + Key = element(KeyPos,Object), + Val = Object, + impl_insert(Drv, encode(Type, Key), encode(Type, Val)); +insert(#tab{keypos=KeyPos, type=Type}, Drv, Objects) when is_list(Objects) -> + List = [{encode(Type, element(KeyPos,Object)), encode(Type, Object)} || Object <- Objects ], + impl_insert(Drv, List). + +insert_new(#tab{keypos=KeyPos, type=Type}, Drv, Object) when is_tuple(Object) -> + Key = element(KeyPos,Object), + Val = Object, + impl_insert_new(Drv, encode(Type, Key), encode(Type, Val)); +insert_new(#tab{keypos=KeyPos, type=Type}, Drv, Objects) when is_list(Objects) -> + List = [{encode(Type, element(KeyPos,Object)), encode(Type, Object)} || Object <- Objects ], + impl_insert_new(Drv, List). + +delete(_Tab, Drv) -> + impl_delete(Drv). + +delete(#tab{type=Type}, Drv, Key) -> + impl_delete(Drv, encode(Type, Key)). + +delete_all_objects(_Tab, Drv) -> + impl_delete_all_objects(Drv). + +lookup(#tab{type=Type}, Drv, Key) -> + case impl_lookup(Drv, encode(Type, Key)) of + true -> + []; + Object when is_binary(Object) -> + [decode(Type, Object)] + end. + +first(#tab{type=Type}, Drv) -> + case impl_first(Drv) of + '$end_of_table' -> + '$end_of_table'; + Key -> + decode(Type, Key) + end. + +next(#tab{type=Type}, Drv, Key) -> + case impl_next(Drv, encode(Type, Key)) of + '$end_of_table' -> + '$end_of_table'; + Next -> + decode(Type, Next) + end. + +info_memory(_Tab, Drv) -> + case impl_info_memory(Drv) of + Memory when is_integer(Memory) -> + erlang:round(Memory / erlang:system_info(wordsize)); + Else -> + Else + end. + +info_size(_Tab, Drv) -> + impl_info_size(Drv). + +tab2list(Tab, Drv) -> + tab2list(Tab, Drv, impl_first(Drv), []). + +tab2list(_Tab, _Drv, '$end_of_table', Acc) -> + lists:reverse(Acc); +tab2list(#tab{type=Type}=Tab, Drv, Key, Acc) -> + NewAcc = + case impl_lookup(Drv, Key) of + true -> + %% @NOTE This is not an atomic operation + Acc; + Object when is_binary(Object) -> + [decode(Type, Object)|Acc] + end, + tab2list(Tab, Drv, impl_next(Drv, Key), NewAcc). + + +%%%---------------------------------------------------------------------- +%%% Internal functions +%%%---------------------------------------------------------------------- + +encode(set, Term) -> + term_to_binary(Term); +encode(ordered_set, Term) -> + sext:encode(Term). + +decode(set, Term) -> + binary_to_term(Term); +decode(ordered_set, Term) -> + sext:decode(Term). + +impl_open(Type, Protection, Path, Options, ReadOptions, WriteOptions) -> + Drv = init(), + true = call(Drv, {?LETS_OPEN6, Type, Protection, Path, Options, ReadOptions, WriteOptions}), + Drv. + +impl_destroy(Type, Protection, Path, Options, ReadOptions, WriteOptions) -> + Drv = init(), + true = call(Drv, {?LETS_OPEN6, Type, Protection, Path, Options, ReadOptions, WriteOptions}), + _ = port_close(Drv), + _ = erl_ddll:unload(lets_drv), + true. + +impl_repair(Type, Protection, Path, Options, ReadOptions, WriteOptions) -> + Drv = init(), + true = call(Drv, {?LETS_REPAIR6, Type, Protection, Path, Options, ReadOptions, WriteOptions}), + _ = port_close(Drv), + _ = erl_ddll:unload(lets_drv), + true. + +impl_insert(Drv, Key, Object) -> + call(Drv, {?LETS_INSERT3, Key, Object}). + +impl_insert(Drv, List) -> + call(Drv, {?LETS_INSERT2, List}). + +impl_insert_new(Drv, Key, Object) -> + call(Drv, {?LETS_INSERT_NEW3, Key, Object}). + +impl_insert_new(Drv, List) -> + call(Drv, {?LETS_INSERT_NEW2, List}). + +impl_delete(Drv) -> + Res = call(Drv, {?LETS_DELETE1}), + _ = port_close(Drv), + _ = erl_ddll:unload(lets_drv), + Res. + +impl_delete(Drv, Key) -> + call(Drv, {?LETS_DELETE2, Key}). + +impl_delete_all_objects(Drv) -> + call(Drv, {?LETS_DELETE_ALL_OBJECTS1}). + +impl_lookup(Drv, Key) -> + call(Drv, {?LETS_LOOKUP2, Key}). + +impl_first(Drv) -> + call(Drv, {?LETS_FIRST1}). + +impl_next(Drv, Key) -> + call(Drv, {?LETS_NEXT2, Key}). + +impl_info_memory(Drv) -> + call(Drv, {?LETS_INFO_MEMORY1}). + +impl_info_size(Drv) -> + call(Drv, {?LETS_INFO_SIZE1}). + +call(Drv, Tuple) -> + Data = term_to_binary(Tuple), + port_command(Drv, Data), + receive + {Drv, ?LETS_TRUE, Reply} -> + Reply; + {Drv, ?LETS_TRUE} -> + true; + {Drv, ?LETS_END_OF_TABLE} -> + '$end_of_table'; + {Drv, ?LETS_BADARG} -> + erlang:error(badarg, [Drv]) + %% after 1000 -> + %% receive X -> + %% erlang:error(timeout, [Drv, X]) + %% after 0 -> + %% erlang:error(timeout, [Drv]) + %% end + end. diff --git a/src/lets_ets.erl b/src/lets_ets.erl new file mode 100644 index 0000000..91b7215 --- /dev/null +++ b/src/lets_ets.erl @@ -0,0 +1,99 @@ +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +-module(lets_ets). + +-include("lets.hrl"). + +%% External exports +-export([open/1 + , destroy/1 + , repair/1 + , insert/3 + , insert_new/3 + , delete/2 + , delete/3 + , delete_all_objects/2 + , lookup/3 + , first/2 + , next/3 + , info_memory/2 + , info_size/2 + , tab2list/2 + ]). + + +%%%---------------------------------------------------------------------- +%%% Types/Specs/Records +%%%---------------------------------------------------------------------- + + +%%%---------------------------------------------------------------------- +%%% API +%%%---------------------------------------------------------------------- + +open(#tab{name=Name, named_table=NamedTable, type=Type, keypos=KeyPos, protection=Protection, compressed=Compressed}=Tab) -> + Opts = + [Type, {keypos,KeyPos}, Protection] ++ + [named_table || NamedTable ] ++ + [compressed || Compressed ], + Ets = ets:new(Name, Opts), + Tab#tab{ets=Ets}. + +destroy(#tab{}) -> + true. + +repair(#tab{}) -> + true. + +insert(_Tab, Ets, ObjectOrObjects) -> + ets:insert(Ets, ObjectOrObjects). + +insert_new(_Tab, Ets, ObjectOrObjects) -> + ets:insert_new(Ets, ObjectOrObjects). + +delete(_Tab, Ets) -> + ets:delete(Ets). + +delete(_Tab, Ets, Key) -> + ets:delete(Ets, Key). + +delete_all_objects(_Tab, Ets) -> + ets:delete_all_objects(Ets). + +lookup(_Tab, Ets, Key) -> + ets:lookup(Ets, Key). + +first(_Tab, Ets) -> + ets:first(Ets). + +next(_Tab, Ets, Key) -> + ets:next(Ets, Key). + +info_memory(_Tab, Ets) -> + ets:info(Ets, memory). + +info_size(_Tab, Ets) -> + ets:info(Ets, size). + +tab2list(_Tab, Ets) -> + ets:tab2list(Ets). diff --git a/src/lets_nif.erl b/src/lets_nif.erl new file mode 100644 index 0000000..2fd7ddd --- /dev/null +++ b/src/lets_nif.erl @@ -0,0 +1,219 @@ +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +-module(lets_nif). + +-include("lets.hrl"). + +%% External exports +-export([open/4 + , destroy/4 + , repair/4 + , insert/3 + , insert_new/3 + , delete/2 + , delete/3 + , delete_all_objects/2 + , lookup/3 + , first/2 + , next/3 + , info_memory/2 + , info_size/2 + , tab2list/2 + ]). + +-on_load(init/0). + + +%%%---------------------------------------------------------------------- +%%% Types/Specs/Records +%%%---------------------------------------------------------------------- + +-define(NIF_STUB, nif_stub_error(?LINE)). + + +%%%---------------------------------------------------------------------- +%%% API +%%%---------------------------------------------------------------------- + +init() -> + Path = + case code:priv_dir(lets) of + {error, bad_name} -> + "../priv/lib"; + Dir -> + filename:join([Dir, "lib"]) + end, + erlang:load_nif(filename:join(Path, "lets_nif"), 0). + +open(#tab{name=_Name, named_table=_Named, type=Type, protection=Protection}=Tab, Options, ReadOptions, WriteOptions) -> + {value, {path,Path}, NewOptions} = lists:keytake(path, 1, Options), + Nif = impl_open(Type, Protection, Path, NewOptions, ReadOptions, WriteOptions), + %% @TODO implement named Nif (of sorts) + Tab#tab{nif=Nif}. + +destroy(#tab{type=Type, protection=Protection}, Options, ReadOptions, WriteOptions) -> + {value, {path,Path}, NewOptions} = lists:keytake(path, 1, Options), + impl_destroy(Type, Protection, Path, NewOptions, ReadOptions, WriteOptions). + +repair(#tab{type=Type, protection=Protection}, Options, ReadOptions, WriteOptions) -> + {value, {path,Path}, NewOptions} = lists:keytake(path, 1, Options), + impl_repair(Type, Protection, Path, NewOptions, ReadOptions, WriteOptions). + +insert(#tab{keypos=KeyPos, type=Type}, Nif, Object) when is_tuple(Object) -> + Key = element(KeyPos,Object), + Val = Object, + impl_insert(Nif, encode(Type, Key), encode(Type, Val)); +insert(#tab{keypos=KeyPos, type=Type}, Nif, Objects) when is_list(Objects) -> + List = [{encode(Type, element(KeyPos,Object)), encode(Type, Object)} || Object <- Objects ], + impl_insert(Nif, List). + +insert_new(#tab{keypos=KeyPos, type=Type}, Nif, Object) when is_tuple(Object) -> + Key = element(KeyPos,Object), + Val = Object, + impl_insert_new(Nif, encode(Type, Key), encode(Type, Val)); +insert_new(#tab{keypos=KeyPos, type=Type}, Nif, Objects) when is_list(Objects) -> + List = [{encode(Type, element(KeyPos,Object)), encode(Type, Object)} || Object <- Objects ], + impl_insert_new(Nif, List). + +delete(_Tab, Nif) -> + impl_delete(Nif). + +delete(#tab{type=Type}, Nif, Key) -> + impl_delete(Nif, encode(Type, Key)). + +delete_all_objects(_Tab, Nif) -> + impl_delete_all_objects(Nif). + +lookup(#tab{type=Type}, Nif, Key) -> + case impl_lookup(Nif, encode(Type, Key)) of + true -> + []; + Object when is_binary(Object) -> + [decode(Type, Object)] + end. + +first(#tab{type=Type}, Nif) -> + case impl_first(Nif) of + '$end_of_table' -> + '$end_of_table'; + Key -> + decode(Type, Key) + end. + +next(#tab{type=Type}, Nif, Key) -> + case impl_next(Nif, encode(Type, Key)) of + '$end_of_table' -> + '$end_of_table'; + Next -> + decode(Type, Next) + end. + +info_memory(_Tab, Nif) -> + case impl_info_memory(Nif) of + Memory when is_integer(Memory) -> + erlang:round(Memory / erlang:system_info(wordsize)); + Else -> + Else + end. + +info_size(_Tab, Nif) -> + impl_info_size(Nif). + +tab2list(Tab, Nif) -> + tab2list(Tab, Nif, impl_first(Nif), []). + +tab2list(_Tab, _Nif, '$end_of_table', Acc) -> + lists:reverse(Acc); +tab2list(#tab{type=Type}=Tab, Nif, Key, Acc) -> + NewAcc = + case impl_lookup(Nif, Key) of + true -> + %% @NOTE This is not an atomic operation + Acc; + Object when is_binary(Object) -> + [decode(Type, Object)|Acc] + end, + tab2list(Tab, Nif, impl_next(Nif, Key), NewAcc). + + +%%%---------------------------------------------------------------------- +%%% Internal functions +%%%---------------------------------------------------------------------- + +encode(set, Term) -> + term_to_binary(Term); +encode(ordered_set, Term) -> + sext:encode(Term). + +decode(set, Term) -> + binary_to_term(Term); +decode(ordered_set, Term) -> + sext:decode(Term). + +nif_stub_error(Line) -> + erlang:nif_error({nif_not_loaded,module,?MODULE,line,Line}). + +impl_open(_Type, _Protection, _Path, _Options, _ReadOptions, _WriteOptions) -> + ?NIF_STUB. + +impl_destroy(_Type, _Protection, _Path, _Options, _ReadOptions, _WriteOptions) -> + ?NIF_STUB. + +impl_repair(_Type, _Protection, _Path, _Options, _ReadOptions, _WriteOptions) -> + ?NIF_STUB. + +impl_insert(_Nif, _Key, _Object) -> + ?NIF_STUB. + +impl_insert(_Nif, _List) -> + ?NIF_STUB. + +impl_insert_new(_Nif, _Key, _Object) -> + ?NIF_STUB. + +impl_insert_new(_Nif, _List) -> + ?NIF_STUB. + +impl_delete(_Nif) -> + ?NIF_STUB. + +impl_delete(_Nif, _Key) -> + ?NIF_STUB. + +impl_delete_all_objects(_Nif) -> + ?NIF_STUB. + +impl_lookup(_Nif, _Key) -> + ?NIF_STUB. + +impl_first(_Nif) -> + ?NIF_STUB. + +impl_next(_Nif, _Key) -> + ?NIF_STUB. + +impl_info_memory(_Nif) -> + ?NIF_STUB. + +impl_info_size(_Nif) -> + ?NIF_STUB. diff --git a/test/qc/qc_lets_proxy.erl b/test/qc/qc_lets_proxy.erl new file mode 100644 index 0000000..4d2a3b0 --- /dev/null +++ b/test/qc/qc_lets_proxy.erl @@ -0,0 +1,197 @@ +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +-module(qc_lets_proxy). + +-behaviour(gen_server). + +%% API +-export([%% test + teardown/1 + , is_table/1 + %% lets + , new/2 + , new/3 + , destroy/3 + , repair/3 + , insert/2 + , insert_new/2 + , delete/1 + , delete/2 + , delete_all_objects/1 + , lookup/2 + , first/1 + , next/2 + , info/2 + , tab2list/1 + ]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-record(state, {tab}). + + +%%%=================================================================== +%%% API +%%%=================================================================== + +teardown(Name) -> + qc_lets_raw:teardown(Name). + +is_table(Tab) -> + is_pid(Tab). + +new(Name, Options) -> + {ok, Pid} = gen_server:start({local, Name}, ?MODULE, [Name, Options], []), + Pid. + +new(_Tab, Name, Options) -> + %% _Tab is to help control generators and shrinking + new(Name, Options). + +destroy(Tab, Name, Options) -> + qc_lets_raw:destroy(Tab, Name, Options). + +repair(Tab, Name, Options) -> + qc_lets_raw:repair(Tab, Name, Options). + +insert(Tab, ObjOrObjs) -> + gen_server:call(Tab, {insert, ObjOrObjs}). + +insert_new(Tab, ObjOrObjs) -> + gen_server:call(Tab, {insert_new, ObjOrObjs}). + +delete(Tab) -> + gen_server:call(Tab, delete). + +delete(Tab, Key) -> + gen_server:call(Tab, {delete, Key}). + +delete_all_objects(Tab) -> + gen_server:call(Tab, delete_all_objects). + +lookup(Tab, Key) -> + gen_server:call(Tab, {lookup, Key}). + +first(Tab) -> + gen_server:call(Tab, first). + +next(Tab, Key) -> + gen_server:call(Tab, {next, Key}). + +info(Tab, Item) -> + gen_server:call(Tab, {info, Item}). + +tab2list(Tab) -> + gen_server:call(Tab, tab2list). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Initializes the server +%%-------------------------------------------------------------------- +init([Name, Options]) -> + Tab = qc_lets_raw:new(Name, Options), + {ok, #state{tab=Tab}}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling call messages +%%-------------------------------------------------------------------- +handle_call({insert, ObjOrObjs}, _From, #state{tab=Tab}=State) -> + Reply = qc_lets_raw:insert(Tab, ObjOrObjs), + {reply, Reply, State}; +handle_call({insert_new, ObjOrObjs}, _From, #state{tab=Tab}=State) -> + Reply = qc_lets_raw:insert_new(Tab, ObjOrObjs), + {reply, Reply, State}; +handle_call(delete, _From, #state{tab=Tab}=State) -> + Reply = qc_lets_raw:delete(Tab), + {stop, normal, Reply, State#state{tab=undefined}}; +handle_call({delete, Key}, _From, #state{tab=Tab}=State) -> + Reply = qc_lets_raw:delete(Tab, Key), + {reply, Reply, State}; +handle_call(delete_all_objects, _From, #state{tab=Tab}=State) -> + Reply = qc_lets_raw:delete_all_objects(Tab), + {reply, Reply, State}; +handle_call({lookup, Key}, _From, #state{tab=Tab}=State) -> + Reply = qc_lets_raw:lookup(Tab, Key), + {reply, Reply, State}; +handle_call(first, _From, #state{tab=Tab}=State) -> + Reply = qc_lets_raw:first(Tab), + {reply, Reply, State}; +handle_call({next, Key}, _From, #state{tab=Tab}=State) -> + Reply = qc_lets_raw:next(Tab, Key), + {reply, Reply, State}; +handle_call({info, Item}, _From, #state{tab=Tab}=State) -> + Reply = qc_lets_raw:info(Tab, Item), + {reply, Reply, State}; +handle_call(tab2list, _From, #state{tab=Tab}=State) -> + Reply = qc_lets_raw:tab2list(Tab), + {reply, Reply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling cast messages +%%-------------------------------------------------------------------- +handle_cast(_Msg, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Handling all non call/cast messages +%%-------------------------------------------------------------------- +handle_info(_Info, State) -> + {noreply, State}. + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% This function is called by a gen_server when it is about to +%% terminate. It should be the opposite of Module:init/1 and do any +%% necessary cleaning up. When it returns, the gen_server terminates +%% with Reason. The return value is ignored. +%%-------------------------------------------------------------------- +terminate(_Reason, #state{tab=undefined}) -> + ok; +terminate(_Reason, #state{tab=Tab}) -> + qc_lets_raw:delete(Tab). + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Convert process state when code is changed +%%-------------------------------------------------------------------- +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== diff --git a/test/qc/qc_lets_raw.erl b/test/qc/qc_lets_raw.erl new file mode 100644 index 0000000..7ee9456 --- /dev/null +++ b/test/qc/qc_lets_raw.erl @@ -0,0 +1,116 @@ +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +-module(qc_lets_raw). + +-include("../../src/lets.hrl"). + +%% API +-export([%% test + teardown/1 + , is_table/1 + %% lets + , new/2 + , new/3 + , destroy/3 + , repair/3 + , insert/2 + , insert_new/2 + , delete/1 + , delete/2 + , delete_all_objects/1 + , lookup/2 + , first/1 + , next/2 + , info/2 + , tab2list/1 + ]). + + +%%%=================================================================== +%%% API +%%%=================================================================== + +teardown(Name) -> + %% @TODO make this more robust + catch ets:delete(Name), + catch exit(whereis(Name), kill), + os:cmd("rm -rf " ++ ?MODULE_STRING). + +is_table(Res) -> + is_record(Res, tab). + +new(Name, Options) -> + ok = filelib:ensure_dir(?MODULE_STRING), + catch lets:new(Name, filter_options(Options)). + +new(_Tab, Name, Options) -> + %% _Tab is to help control generators and shrinking + new(Name, Options). + +destroy(_Tab, Name, Options) -> + %% _Tab is to help control generators and shrinking + catch lets:destroy(Name, filter_options(Options)). + +repair(_Tab, Name, Options) -> + %% _Tab is to help control generators and shrinking + catch lets:repair(Name, filter_options(Options)). + +insert(Tab, ObjOrObjs) -> + catch lets:insert(Tab, ObjOrObjs). + +insert_new(Tab, ObjOrObjs) -> + catch lets:insert_new(Tab, ObjOrObjs). + +delete(Tab) -> + catch lets:delete(Tab). + +delete(Tab, Key) -> + catch lets:delete(Tab, Key). + +delete_all_objects(Tab) -> + catch lets:delete_all_objects(Tab). + +lookup(Tab, Key) -> + catch lets:lookup(Tab, Key). + +first(Tab) -> + catch lets:first(Tab). + +next(Tab, Key) -> + catch lets:next(Tab, Key). + +info(Tab, Item) -> + catch lets:info(Tab, Item). + +tab2list(Tab) -> + catch lets:tab2list(Tab). + + +%%%=================================================================== +%%% Internal +%%%=================================================================== + +filter_options(Options) -> + X = proplists:get_value(db, Options, []), + Y = [{path, ?MODULE_STRING}|proplists:delete(path, X)], + [{db, Y}|proplists:delete(db, Options)]. diff --git a/test/qc/qc_lets_slave_proxy.erl b/test/qc/qc_lets_slave_proxy.erl new file mode 100644 index 0000000..027a86b --- /dev/null +++ b/test/qc/qc_lets_slave_proxy.erl @@ -0,0 +1,118 @@ +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +-module(qc_lets_slave_proxy). + +%% API +-export([%% test + teardown/1 + , is_table/1 + %% lets + , new/2 + , new/3 + , destroy/3 + , repair/3 + , insert/2 + , insert_new/2 + , delete/1 + , delete/2 + , delete_all_objects/1 + , lookup/2 + , first/1 + , next/2 + , info/2 + , tab2list/1 + ]). + + +%%%=================================================================== +%%% API +%%%=================================================================== + +teardown(Name) -> + stop_slave(), + qc_lets_proxy:teardown(Name). + +is_table(Tab) -> + qc_lets_proxy:is_table(Tab). + +new(Name, Options) -> + call_slave(new, [Name, Options]). + +new(_Tab, Name, Options) -> + %% _Tab is to help control generators and shrinking + new(Name, Options). + +destroy(Tab, Name, Options) -> + qc_lets_raw:destroy(Tab, Name, Options). + +repair(Tab, Name, Options) -> + qc_lets_raw:repair(Tab, Name, Options). + +insert(Tab, ObjOrObjs) -> + qc_lets_proxy:insert(Tab, ObjOrObjs). + +insert_new(Tab, ObjOrObjs) -> + qc_lets_proxy:insert_new(Tab, ObjOrObjs). + +delete(Tab) -> + qc_lets_proxy:delete(Tab). + +delete(Tab, Key) -> + qc_lets_proxy:delete(Tab, Key). + +delete_all_objects(Tab) -> + qc_lets_proxy:delete_all_objects(Tab). + +lookup(Tab, Key) -> + qc_lets_proxy:lookup(Tab, Key). + +first(Tab) -> + qc_lets_proxy:first(Tab). + +next(Tab, Key) -> + qc_lets_proxy:next(Tab, Key). + +info(Tab, Item) -> + qc_lets_proxy:info(Tab, Item). + +tab2list(Tab) -> + qc_lets_proxy:tab2list(Tab). + + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +stop_slave() -> + qc_slave:stop_slave(?MODULE). + +call_slave(Function, Args) -> + Slave = qc_slave:restart_slave(?MODULE), + case rpc:call(Slave, qc_lets_proxy, Function, Args) of + {badrpc, {'EXIT', _}=Exit} -> + Exit; + {badrpc, _}=BadRpc -> + BadRpc; + Res -> + Res + end. diff --git a/test/qc/qc_leveldb.erl b/test/qc/qc_leveldb.erl new file mode 100644 index 0000000..7d74a72 --- /dev/null +++ b/test/qc/qc_leveldb.erl @@ -0,0 +1,281 @@ +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +-module(qc_leveldb). + +%% API +-export([%% test + setup/0 + , teardown/0 + , is_db/1 + %% lets + , open/0, open/1 + , reopen/0, reopen/1 + , close/1 + , put/2, put/3 + , delete/2, delete/3 + , get/2, get/3 + , first/1, first/2 + , last/1, last/2 + , next/2, next/3 + ]). + + +%%%=================================================================== +%%% API +%%%=================================================================== + +setup() -> + Options = [{c_src,"../c_src/leveldb/include/leveldb/c.h"}, + {additional_files, ["../c_src/leveldb/lib/libleveldb.a", "../c_src/snappy/lib/libsnappy.a"]}, + {cflags, "-lstdc++ -lpthread"}], + eqc_c:start(leveldb, Options). + +teardown() -> + os:cmd("rm -rf " ++ ?MODULE_STRING). + +is_db({ptr, {struct, leveldb_t}, _}) -> + true; +is_db(_) -> + false. + +open() -> + Options = leveldb:leveldb_options_create(), + try + leveldb:leveldb_options_set_create_if_missing(Options, 1), + leveldb:leveldb_options_set_error_if_exists(Options, 1), + open(Options) + after + leveldb:leveldb_options_destroy(Options) + end. + +open(Options) -> + ErrPtr = errptr(), + try + case leveldb:leveldb_open(Options, ?MODULE_STRING, ErrPtr) of + {ptr, {struct, leveldb_t}, 0} -> + read_errptr(ErrPtr); + {ptr, {struct, leveldb_t}, _}=Db -> + Db + end + after + free_ptr(ErrPtr) + end. + +reopen() -> + Options = leveldb:leveldb_options_create(), + try + reopen(Options) + after + leveldb:leveldb_options_destroy(Options) + end. + +reopen(Options) -> + ErrPtr = errptr(), + try + case leveldb:leveldb_open(Options, ?MODULE_STRING, ErrPtr) of + {ptr, {struct, leveldb_t}, 0} -> + read_errptr(ErrPtr); + {ptr, {struct, leveldb_t}, _}=Db -> + Db + end + after + free_ptr(ErrPtr) + end. + +close(Db) -> + ok == leveldb:leveldb_close(Db). + +put(Db, Obj) -> + Options = leveldb:leveldb_writeoptions_create(), + try + put(Db, Options, Obj) + after + leveldb:leveldb_writeoptions_destroy(Options) + end. + +put(Db, Options, {obj,Key,Val}) -> + ErrPtr = errptr(), + try + leveldb:leveldb_put(Db, Options, binary_to_list(Key), byte_size(Key), binary_to_list(Val), byte_size(Val), ErrPtr), + read_errptr(ErrPtr) + after + free_ptr(ErrPtr) + end. + +delete(Db, Key) -> + Options = leveldb:leveldb_writeoptions_create(), + try + delete(Db, Options, Key) + after + leveldb:leveldb_writeoptions_destroy(Options) + end. + +delete(Db, Options, Key) -> + ErrPtr = errptr(), + try + leveldb:leveldb_delete(Db, Options, binary_to_list(Key), byte_size(Key), ErrPtr), + read_errptr(ErrPtr) + after + free_ptr(ErrPtr) + end. + +get(Db, Key) -> + Options = leveldb:leveldb_readoptions_create(), + try + get(Db, Options, Key) + after + leveldb:leveldb_readoptions_destroy(Options) + end. + +get(Db, Options, Key) -> + ErrPtr = errptr(), + LenPtr = lenptr(), + try + case leveldb:leveldb_get(Db, Options, binary_to_list(Key), byte_size(Key), LenPtr, ErrPtr) of + 0 -> + read_errptr(ErrPtr); + ValPtr -> + read_binary(ValPtr, LenPtr) + end + after + free_ptr(ErrPtr), + free_ptr(LenPtr) + end. + +first(Db) -> + Options = leveldb:leveldb_readoptions_create(), + try + first(Db, Options) + after + leveldb:leveldb_readoptions_destroy(Options) + end. + +first(Db, Options) -> + Iter = leveldb:leveldb_create_iterator(Db, Options), + LenPtr = lenptr(), + try + leveldb:leveldb_iter_seek_to_first(Iter), + case leveldb:leveldb_iter_valid(Iter) of + 0 -> + true; + 1 -> + KeyPtr = leveldb:leveldb_iter_key(Iter, LenPtr), + read_binary(KeyPtr, LenPtr) + end + after + leveldb:leveldb_iter_destroy(Iter), + free_ptr(LenPtr) + end. + +last(Db) -> + Options = leveldb:leveldb_readoptions_create(), + try + last(Db, Options) + after + leveldb:leveldb_readoptions_destroy(Options) + end. + +last(Db, Options) -> + Iter = leveldb:leveldb_create_iterator(Db, Options), + LenPtr = lenptr(), + try + leveldb:leveldb_iter_seek_to_last(Iter), + case leveldb:leveldb_iter_valid(Iter) of + 0 -> + true; + 1 -> + KeyPtr = leveldb:leveldb_iter_key(Iter, LenPtr), + read_binary(KeyPtr, LenPtr) + end + after + leveldb:leveldb_iter_destroy(Iter), + free_ptr(LenPtr) + end. + +next(Db, Key) -> + Options = leveldb:leveldb_readoptions_create(), + try + next(Db, Key, Options) + after + leveldb:leveldb_readoptions_destroy(Options) + end. + +next(Db, Key, Options) -> + Iter = leveldb:leveldb_create_iterator(Db, Options), + LenPtr = lenptr(), + LenPtr1 = lenptr(), + try + leveldb:leveldb_iter_seek(Iter, binary_to_list(Key), byte_size(Key)), + case leveldb:leveldb_iter_valid(Iter) of + 0 -> + true; + 1 -> + KeyPtr = leveldb:leveldb_iter_key(Iter, LenPtr), + K = read_binary(KeyPtr, LenPtr), + if K =/= Key -> + K; + true -> + leveldb:leveldb_iter_next(Iter), + case leveldb:leveldb_iter_valid(Iter) of + 0 -> + true; + 1 -> + KeyPtr1 = leveldb:leveldb_iter_key(Iter, LenPtr1), + read_binary(KeyPtr1, LenPtr1) + end + end + end + after + leveldb:leveldb_iter_destroy(Iter), + free_ptr(LenPtr), + free_ptr(LenPtr1) + end. + +%%%=================================================================== +%%% Internal +%%%=================================================================== + +errptr() -> + eqc_c:alloc({ptr, char}, [0]). + +lenptr() -> + eqc_c:alloc(unsigned_long, 0). + +free_ptr(Ptr) -> + eqc_c:free(Ptr). + +read_errptr(ErrPtr) -> + case eqc_c:read_string(eqc_c:deref(ErrPtr)) of + [] -> + true; + Err -> + Err + end. + +read_lenptr(LenPtr) -> + eqc_c:deref(LenPtr). + +read_binary({ptr, char, 0}, _LenPtr) -> + true; +read_binary(ValPtr, LenPtr) -> + list_to_binary(eqc_c:read_array(ValPtr, read_lenptr(LenPtr))). diff --git a/test/qc/qc_statem_lets.erl b/test/qc/qc_statem_lets.erl new file mode 100644 index 0000000..378a31b --- /dev/null +++ b/test/qc/qc_statem_lets.erl @@ -0,0 +1,446 @@ +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +-module(qc_statem_lets). + +-ifdef(QC). + +%% qc_statem Callbacks +-behaviour(qc_statem). +-export([command_gen/2]). +-export([initial_state/0, state_is_sane/1, next_state/3, precondition/2, postcondition/3]). +-export([commands_setup/1, commands_teardown/1, commands_teardown/2]). + +%% @NOTE For boilerplate exports, see "qc_statem.hrl" +-include_lib("qc/include/qc_statem.hrl"). + + +%%%---------------------------------------------------------------------- +%%% defines, types, records +%%%---------------------------------------------------------------------- + +%%-define(IMPL, qc_lets_raw). +%%-define(IMPL, qc_lets_proxy). +-define(IMPL, qc_lets_slave_proxy). + +-define(TAB, ?MODULE). +-define(INT_KEYS, lists:seq(0,10)). +-define(FLOAT_KEYS, [float(Key) || Key <- ?INT_KEYS]). +-define(BINARY_KEYS, [term_to_binary(Key) || Key <- ?INT_KEYS]). + +-record(obj, {key :: integer() | float() | binary(), val :: integer() | float() | binary()}). + +-type obj() :: #obj{}. +-type ets_type() :: set | ordered_set. %% default is set +-type ets_impl() :: drv | nif | ets. %% default is drv + +-record(state, { + parallel=false :: boolean(), + type=undefined :: undefined | ets_type(), + impl=undefined :: undefined | ets_impl(), + exists=false :: boolean(), + tab=undefined :: undefined | tuple(), + objs=[] :: [obj()] + }). + + +%%%---------------------------------------------------------------------- +%%% qc_statem Callbacks +%%%---------------------------------------------------------------------- + +command_gen(Mod,#state{parallel=false}=S) -> + serial_command_gen(Mod,S); +command_gen(Mod,#state{parallel=true}=S) -> + parallel_command_gen(Mod,S). + +serial_command_gen(_Mod,#state{tab=undefined, type=undefined, impl=undefined}=S) -> + {call,?IMPL,new,[?TAB,gen_options(new,S)]}; +serial_command_gen(_Mod,#state{tab=undefined}=S) -> + {call,?IMPL,new,[undefined,?TAB,gen_options(new,S)]}; +serial_command_gen(_Mod,#state{tab=Tab, type=Type}=S) -> + %% @TODO insert/3, insert_new/3, delete/3, delete_all_objs/2 write_gen_options + %% @TODO lookup/3 read_gen_options + oneof([{call,?IMPL,insert,[Tab,oneof([gen_obj(S),gen_objs(S)])]}] + ++ [{call,?IMPL,insert_new,[Tab,oneof([gen_obj(S),gen_objs(S)])]} || Type == ets] + %% @TODO ++ [{call,?IMPL,delete,[Tab]}] + ++ [{call,?IMPL,delete,[Tab,gen_key(S)]}] + ++ [{call,?IMPL,delete_all_objs,[Tab]} || Type == ets] + ++ [{call,?IMPL,lookup,[Tab,gen_key(S)]}] + ++ [{call,?IMPL,first,[Tab]}] + ++ [{call,?IMPL,next,[Tab,gen_key(S)]}] + %% @TODO info + ++ [{call,?IMPL,tab2list,[Tab]}] + ). + +parallel_command_gen(_Mod,#state{tab=undefined, type=undefined, impl=undefined}=S) -> + {call,?IMPL,new,[?TAB,gen_options(new,S)]}; +parallel_command_gen(_Mod,#state{tab=Tab, type=Type}=S) -> + %% @TODO insert/3, insert_new/3, delete_all_objs/2 write_gen_options + %% @TODO lookup/3 read_gen_options + oneof([{call,?IMPL,insert,[Tab,oneof([gen_obj(S),gen_objs(S)])]}] + ++ [{call,?IMPL,insert_new,[Tab,oneof([gen_obj(S),gen_objs(S)])]} || Type == ets] + ++ [{call,?IMPL,delete,[Tab,gen_key(S)]}] + ++ [{call,?IMPL,delete_all_objs,[Tab]} || Type == ets] + ++ [{call,?IMPL,lookup,[Tab,gen_key(S)]}] + ++ [{call,?IMPL,first,[Tab]}] + ++ [{call,?IMPL,next,[Tab,gen_key(S)]}] + ). + +-spec initial_state() -> #state{}. +initial_state() -> + ?LET(Parallel,parameter(parallel,false), + #state{parallel=Parallel}). + +-spec state_is_sane(#state{}) -> boolean(). +state_is_sane(_S) -> + %% @TODO + true. + +-spec next_state(#state{}, term(), tuple()) -> #state{}. +next_state(#state{tab=undefined, type=undefined, impl=undefined}=S, V, {call,_,new,[?TAB,Options]}) -> + %% @TODO Options + case [proplists:get_bool(X, Options) || X <- [set, ordered_set]] of + [_, false] -> + Type = set; + [false, true] -> + Type = ordered_set + end, + case [proplists:get_bool(X, Options) || X <- [drv, nif, ets]] of + [_, false, false] -> + Impl = drv; + [false, true, _] -> + Impl = nif; + [false, false, true] -> + Impl = ets + end, + S#state{type=Type, impl=Impl, exists=true, tab=V}; +next_state(#state{tab=undefined}=S, V, {call,_,new,[_Tab,?TAB,Options]}) -> + %% @TODO Options + case [proplists:get_bool(X, Options) || X <- [set, ordered_set]] of + [_, false] -> + Type = set; + [false, true] -> + Type = ordered_set + end, + case [proplists:get_bool(X, Options) || X <- [drv, nif, ets]] of + [_, false, false] -> + Impl = drv; + [false, true, _] -> + Impl = nif; + [false, false, true] -> + Impl = ets + end, + S#state{type=Type, impl=Impl, exists=true, tab=V}; +next_state(#state{impl=Impl}=S, _V, {call,_,destroy,[_Tab,?TAB,_Options]}) + when Impl /= ets -> + S#state{tab=undefined, exists=false, objs=[]}; +next_state(S, _V, {call,_,insert,[_Tab,Objs]}) when is_list(Objs) -> + insert_objs(S, Objs); +next_state(S, _V, {call,_,insert,[_Tab,Obj]}) -> + insert_objs(S, [Obj]); +next_state(#state{impl=ets}=S, _V, {call,_,insert_new,[_Tab,Objs]}) when is_list(Objs) -> + insert_new_objs(S, Objs); +next_state(#state{impl=ets}=S, _V, {call,_,insert_new,[_Tab,Obj]}) -> + insert_new_objs(S, [Obj]); +next_state(S, _V, {call,_,insert_new,[_Tab,_ObjOrObjs]}) -> + S; +next_state(#state{impl=ets}=S, _V, {call,_,delete,[_Tab]}) -> + S#state{tab=undefined, exists=false, objs=[]}; +next_state(#state{exists=Exists}=S, _V, {call,_,delete,[_Tab]}) -> + S#state{tab=undefined, exists=Exists}; +next_state(S, _V, {call,_,delete,[_Tab,Key]}) -> + S#state{objs=keydelete(Key, S)}; +next_state(#state{impl=ets}=S, _V, {call,_,delete_all_objs,[_Tab]}) -> + S#state{objs=[]}; +next_state(S, _V, {call,_,delete_all_objs,[_Tab]}) -> + S; +next_state(S, _V, {call,_,_,_}) -> + S. + +-spec precondition(#state{}, tuple()) -> boolean(). +precondition(#state{tab=undefined, type=undefined, impl=undefined}, {call,_,new,[?TAB,Options]}) -> + L = proplists:get_value(db, Options, []), + proplists:get_bool(create_if_missing, L) andalso proplists:get_bool(error_if_exists, L); +precondition(#state{tab=undefined, type=undefined, impl=undefined}, {call,_,new,[_Tab,?TAB,Options]}) -> + L = proplists:get_value(db, Options, []), + proplists:get_bool(create_if_missing, L) andalso proplists:get_bool(error_if_exists, L); +precondition(#state{tab=Tab}, {call,_,new,[?TAB,_Options]}) when Tab /= undefined -> + false; +precondition(#state{tab=Tab}, {call,_,new,[_Tab,?TAB,_Options]}) when Tab /= undefined -> + false; +precondition(_S, {call,_,_,_}) -> + true. + +-spec postcondition(#state{}, tuple(), term()) -> boolean(). +postcondition(#state{tab=undefined}, {call,_,new,[?TAB,_Options]}, Res) -> + ?IMPL:is_table(Res); +postcondition(_S, {call,_,new,[?TAB,_Options]}, Res) -> + case Res of + {'EXIT', {badarg, _}} -> + true; + _ -> + false + end; +postcondition(#state{tab=undefined}, {call,_,new,[_Tab,?TAB,_Options]}, Res) -> + ?IMPL:is_table(Res); +postcondition(_S, {call,_,destroy,[_Tab,?TAB,_Options]}, Res) -> + Res =:= true; +postcondition(_S, {call,_,repair,[_Tab,?TAB,_Options]}, Res) -> + Res =:= true; +postcondition(_S, {call,_,insert,[_Tab,_ObjOrObjs]}, Res) -> + Res =:= true; +postcondition(#state{impl=ets}=S, {call,_,insert_new,[_Tab,Objs]}, Res) when is_list(Objs) -> + Res =:= has_insert_new_objs(S, Objs); +postcondition(#state{impl=ets}=S, {call,_,insert_new,[_Tab,Obj]}, Res) -> + Res =:= has_insert_new_objs(S, [Obj]); +postcondition(_S, {call,_,insert_new,[_Tab,_ObjOrObjs]}, {'EXIT',{badarg,_}}) -> + true; +postcondition(_S, {call,_,delete,[_Tab]}, Res) -> + Res =:= true; +postcondition(_S, {call,_,delete,[_Tab,_Key]}, Res) -> + Res =:= true; +postcondition(#state{impl=ets}=_S, {call,_,delete_all_objs,[_Tab]}, Res) -> + Res =:= true; +postcondition(_S, {call,_,delete_all_objs,[_Tab]}, {'EXIT',{badarg,_}}) -> + true; +postcondition(S, {call,_,lookup,[_Tab,Key]}, Res) -> + Res =:= keyfind(Key, S); +postcondition(#state{objs=[]}, {call,_,first,[_Tab]}, Res) -> + Res =:= '$end_of_table'; +postcondition(#state{type=set}=S, {call,_,first,[_Tab]}, Res) -> + keymember(Res, S); +postcondition(#state{type=ordered_set}=S, {call,_,first,[_Tab]}, Res) -> + #obj{key=K} = hd(sort(S)), + Res =:= K; +postcondition(#state{impl=ets, type=set}=S, {call,_,next,[_Tab, Key]}, {'EXIT',{badarg,_}}) -> + not keymember(Key, S); +postcondition(#state{impl=ets, type=set}=S, {call,_,next,[_Tab, Key]}, '$end_of_table') -> + keymember(Key, S); +postcondition(#state{impl=ets, type=set}=S, {call,_,next,[_Tab, Key]}, Res) -> + keymember(Key, S) andalso keymember(Res, S); +postcondition(#state{type=set}, {call,_,next,[_Tab, _Key]}, '$end_of_table') -> + true; +postcondition(#state{type=set}=S, {call,_,next,[_Tab, _Key]}, Res) -> + keymember(Res, S); +postcondition(#state{type=ordered_set, objs=[]}, {call,_,next,[_Tab, _Key]}, Res) -> + Res =:= '$end_of_table'; +postcondition(#state{type=ordered_set}=S, {call,_,next,[_Tab, Key]}, Res) -> + case lists:dropwhile(fun(#obj{key=X}) -> lteq(X, Key, S) end, sort(S)) of + [] -> + Res =:= '$end_of_table'; + [#obj{key=K}|_] -> + Res =:= K + end; +postcondition(#state{type=set}=S, {call,_,tab2list,[_Tab]}, Res) -> + [] == (S#state.objs -- Res); +postcondition(#state{type=ordered_set}=S, {call,_,tab2list,[_Tab]}, Res) -> + sort(S) =:= Res; +postcondition(_S, {call,_,_,_}, _Res) -> + false. + +-spec commands_setup(boolean()) -> {ok, term()}. +commands_setup(_Hard) -> + ?IMPL:teardown(?TAB), + {ok, unused}. + +-spec commands_teardown(term()) -> ok. +commands_teardown(unused) -> + ?IMPL:teardown(?TAB), + ok. + +-spec commands_teardown(term(), #state{}) -> ok. +commands_teardown(Ref, _State) -> + commands_teardown(Ref). + + +%%%---------------------------------------------------------------------- +%%% Internal +%%%---------------------------------------------------------------------- + +gen_options(Op,#state{tab=undefined, type=undefined, impl=undefined}=S) -> + ?LET({Type,Impl}, {gen_ets_type(), gen_ets_impl()}, + gen_options(Op,S#state{type=Type, impl=Impl})); +gen_options(Op,#state{type=Type, impl=drv=Impl}=S) -> + [Type, public, named_table, {keypos,#obj.key}, Impl] + ++ gen_leveldb_options(Op,S); +gen_options(Op,#state{type=Type, impl=nif=Impl}=S) -> + [Type, public, named_table, {keypos,#obj.key}, Impl] + ++ gen_leveldb_options(Op,S); +gen_options(_Op,#state{type=Type, impl=ets=Impl}) -> + [Type, public, named_table, {keypos,#obj.key}, Impl]. + +gen_leveldb_options(Op,S) -> + [gen_db_options(Op,S), gen_db_read_options(Op,S), gen_db_write_options(Op,S)]. + +gen_db_options(new,#state{exists=Exists}) -> + ExistsOptions = if Exists -> []; true -> [create_if_missing, error_if_exists] end, + ?LET(Options, ulist(gen_db_options()), {db, Options ++ ExistsOptions}); +gen_db_options(_Op,_S) -> + ?LET(Options, ulist(gen_db_options()), {db, Options}). + +gen_db_read_options(_Op,_S) -> + ?LET(Options, ulist(gen_db_read_options()), {db_read, Options}). + +gen_db_write_options(_Op,_S) -> + ?LET(Options, ulist(gen_db_write_options()), {db_write, Options}). + +gen_db_options() -> + oneof([paranoid_checks, {paranoid_checks,gen_boolean()}, {write_buffer_size,gen_pos_integer()}, {max_open_files,gen_pos_integer()}, {block_cache_size,gen_pos_integer()}, {block_size,gen_pos_integer()}, {block_restart_interval,gen_pos_integer()}]). + +gen_db_read_options() -> + oneof([verify_checksums, {verify_checksums,gen_boolean()}, fill_cache, {fill_cache,gen_boolean()}]). + +gen_db_write_options() -> + oneof([sync, {sync,gen_boolean()}]). + +gen_boolean() -> + oneof([true, false]). + +gen_pos_integer() -> + nat(). + +gen_ets_type() -> + noshrink(oneof([set, ordered_set])). + +gen_ets_impl() -> + %% @NOTE Remove one or two of these to restrict to a particular + %% implementation. + noshrink(oneof([drv,nif,ets])). + +gen_integer_key() -> + oneof(?INT_KEYS). + +gen_float_key() -> + oneof(?FLOAT_KEYS). + +gen_binary_key() -> + oneof(?BINARY_KEYS). + +gen_key() -> + frequency([{5, gen_integer_key()}, {1, gen_float_key()}, {1, gen_binary_key()}]). + +gen_key(#state{objs=[]}) -> + gen_key(); +gen_key(#state{objs=Objs}) -> + oneof([?LET(Obj, oneof(Objs), Obj#obj.key), gen_key()]). + +gen_int_or_float_or_bin() -> + frequency([{5, int()}, {1, real()}, {1, binary()}]). + +gen_val() -> + gen_int_or_float_or_bin(). + +gen_obj() -> + #obj{key=gen_key(), val=gen_val()}. + +gen_obj(#state{objs=[]}) -> + gen_obj(); +gen_obj(#state{objs=Objs}) -> + oneof([oneof(Objs), gen_obj()]). + +gen_objs(S) -> + frequency([{9, non_empty(list(gen_obj(S)))}, {1, list(gen_obj(S))}]). + +insert_objs(S, []) -> + S; +insert_objs(S, [#obj{key=K}=Obj|T]) -> + case keymember(K, S) of + false -> + insert_objs(S#state{objs=[Obj|S#state.objs]}, T); + true -> + insert_objs(S#state{objs=keyreplace(K, Obj, S)}, T) + end. + +insert_new_objs(S, L) -> + insert_new_objs(S, lists:reverse(L), S). + +insert_new_objs(S, [], _S0) -> + S; +insert_new_objs(S, [#obj{key=K}=Obj|T], S0) -> + case keymember(K, S) of + false -> + NewT = keydelete(K, T, S), + insert_new_objs(S#state{objs=[Obj|S#state.objs]}, NewT, S0); + true -> + S0 + end. + +has_insert_new_objs(S, L) -> + has_insert_new_objs(S, lists:reverse(L), true). + +has_insert_new_objs(_S, [], Bool) -> + Bool; +has_insert_new_objs(S, [#obj{key=K}=Obj|T], _Bool) -> + case keymember(K, S) of + false -> + NewT = keydelete(K, T, S), + has_insert_new_objs(S#state{objs=[Obj|S#state.objs]}, NewT, true); + true -> + false + end. + +keydelete(X, #state{objs=L}=S) -> + keydelete(X, L, S). + +keydelete(X, L, S) -> + lists:filter(fun(#obj{key=K}) -> neq(X, K, S) end, L). + +keyreplace(X, Y, #state{objs=L}=S) -> + lists:map(fun(Z=#obj{key=K}) -> case eq(X, K, S) of true -> Y; false -> Z end end, L). + +keyfind(X, #state{objs=L}=S) -> + lists:filter(fun(#obj{key=K}) -> eq(X, K, S) end, L). + +keymember(X, S) -> + [] /= keyfind(X, S). + +eq(X, Y, #state{type=set, impl=ets}) -> + X =:= Y; +eq(X, Y, #state{type=ordered_set, impl=ets}) -> + X == Y; +eq(X, Y, #state{type=set}) -> + term_to_binary(X) == term_to_binary(Y); +eq(X, Y, #state{type=ordered_set}) -> + sext:encode(X) == sext:encode(Y). + +neq(X, Y, #state{type=set, impl=ets}) -> + X =/= Y; +neq(X, Y, #state{type=ordered_set, impl=ets}) -> + X /= Y; +neq(X, Y, #state{type=set}) -> + term_to_binary(X) /= term_to_binary(Y); +neq(X, Y, #state{type=ordered_set}) -> + sext:encode(X) /= sext:encode(Y). + +lteq(X, Y, #state{impl=ets}) -> + X =< Y; +lteq(X, Y, #state{type=set}) -> + term_to_binary(X) =< term_to_binary(Y); +lteq(X, Y, #state{type=ordered_set}) -> + sext:encode(X) =< sext:encode(Y). + +sort(#state{impl=ets, objs=L}) -> + lists:sort(L); +sort(#state{objs=L}) -> + [ sext:decode(X) || X <- lists:sort([ sext:encode(Y) || Y <- L ]) ]. + +-endif. %% -ifdef(QC). diff --git a/test/qc/qc_statemc_lets.erl b/test/qc/qc_statemc_lets.erl new file mode 100644 index 0000000..f2b6397 --- /dev/null +++ b/test/qc/qc_statemc_lets.erl @@ -0,0 +1,237 @@ +%%% The MIT License +%%% +%%% Copyright (C) 2011 by Joseph Wayne Norton +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. + +-module(qc_statemc_lets). + +-ifdef(QC). +-ifdef(EQC). + +%% qc_statem Callbacks +-behaviour(qc_statem). +-export([command_gen/2]). +-export([initial_state/0, state_is_sane/1, next_state/3, precondition/2, postcondition/3]). +-export([commands_setup/1, commands_teardown/1, commands_teardown/2]). + +%% @NOTE For boilerplate exports, see "qc_statem.hrl" +-include_lib("eqc/include/eqc_c.hrl"). +-include_lib("qc/include/qc_statem.hrl"). + + +%%%---------------------------------------------------------------------- +%%% defines, types, records +%%%---------------------------------------------------------------------- + +-define(IMPL, qc_leveldb). + +-record(obj, {key :: binary(), val :: binary()}). + +-type obj() :: #obj{}. + +-record(state, { + parallel=false :: boolean(), + exists=false :: boolean(), + db=undefined :: undefined | term(), + objs=[] :: [obj()] + }). + + +%%%---------------------------------------------------------------------- +%%% qc_statem Callbacks +%%%---------------------------------------------------------------------- + +command_gen(Mod,#state{parallel=false}=S) -> + serial_command_gen(Mod,S); +command_gen(Mod,#state{parallel=true}=S) -> + parallel_command_gen(Mod,S). + +serial_command_gen(_Mod,#state{db=undefined, exists=false}) -> + {call,?IMPL,open,[]}; +serial_command_gen(_Mod,#state{db=undefined, exists=true}) -> + {call,?IMPL,reopen,[]}; +serial_command_gen(_Mod,#state{db=Db}=S) -> + oneof([{call,?IMPL,close,[Db]}, + {call,?IMPL,put,[Db,gen_obj(S)]}, + {call,?IMPL,delete,[Db,gen_key(S)]}, + {call,?IMPL,get,[Db,gen_key(S)]}, + {call,?IMPL,first,[Db]}, + {call,?IMPL,last,[Db]}, + {call,?IMPL,next,[Db,gen_key(S)]} + ]). + +parallel_command_gen(_Mod,#state{db=undefined}) -> + {call,?IMPL,open,[]}; +parallel_command_gen(_Mod,#state{db=Db}=S) -> + oneof([{call,?IMPL,put,[Db,gen_obj(S)]}, + {call,?IMPL,delete,[Db,gen_key(S)]}, + {call,?IMPL,get,[Db,gen_key(S)]} + ]). + +-spec initial_state() -> #state{}. +initial_state() -> + ?LET(Parallel,parameter(parallel,false), + #state{parallel=Parallel}). + +-spec state_is_sane(#state{}) -> boolean(). +state_is_sane(_S) -> + %% @TODO + true. + +-spec next_state(#state{}, term(), tuple()) -> #state{}. +next_state(#state{db=undefined, exists=false}=S, V, {call,_,open,[]}) -> + S#state{db=V, exists=true}; +next_state(#state{db=undefined, exists=true}=S, V, {call,_,reopen,[]}) -> + S#state{db=V, exists=true}; +next_state(#state{db=Db}=S, _V, {call,_,close,[Db]}) when Db /= undefined -> + S#state{db=undefined}; +next_state(S, _V, {call,_,put,[_Db,Obj]}) -> + insert_obj(S, Obj); +next_state(S, _V, {call,_,delete,[_Db,Key]}) -> + delete_obj(S, Key); +next_state(S, _V, {call,_,_,_}) -> + S. + +-spec precondition(#state{}, tuple()) -> boolean(). +precondition(#state{exists=true}, {call,_,open,[]}) -> + false; +precondition(#state{exists=false}, {call,_,reopen,[]}) -> + false; +precondition(#state{db=Db}, {call,_,open,[]}) when Db /= undefined-> + false; +precondition(#state{db=Db}, {call,_,reopen,[]}) when Db /= undefined-> + false; +precondition(_S, {call,_,_,_}) -> + true. + +-spec postcondition(#state{}, tuple(), term()) -> boolean(). +postcondition(#state{exists=false}, {call,_,open,[]}, Res) -> + ?IMPL:is_db(Res); +postcondition(#state{exists=true}, {call,_,reopen,[]}, Res) -> + ?IMPL:is_db(Res); +postcondition(#state{db=Db}, {call,_,close,[_Db]}, Res) -> + Res andalso Db /= undefined; +postcondition(_S, {call,_,put,[_Db,_]}, Res) -> + Res; +postcondition(_S, {call,_,delete,[_Db,_]}, Res) -> + Res; +postcondition(S, {call,_,get,[_Db,Key]}, Res) -> + Res =:= get_val(S, Key); +postcondition(#state{objs=[]}, {call,_,first,[_Db]}, Res) -> + Res; +postcondition(S, {call,_,first,[_Db]}, Res) -> + #obj{key=K} = hd(sort_objs(S)), + Res =:= K; +postcondition(#state{objs=[]}, {call,_,last,[_Db]}, Res) -> + Res; +postcondition(S, {call,_,last,[_Db]}, Res) -> + #obj{key=K} = hd(lists:reverse(sort_objs(S))), + Res =:= K; +postcondition(S, {call,_,next,[_Db,Key]}, Res) -> + case lists:dropwhile(fun(#obj{key=X}) -> X =< Key end, sort_objs(S)) of + [] -> + Res; + [#obj{key=K}|_] -> + Res =:= K + end; +postcondition(_S, {call,_,_,_}, _Res) -> + false. + +-spec commands_setup(boolean()) -> {ok, term()}. +commands_setup(_Hard) -> + ?IMPL:setup(), + teardown(), + {ok, unused}. + +-spec commands_teardown(term()) -> ok. +commands_teardown(unused) -> + teardown(), + ok. + +-spec commands_teardown(term(), #state{}) -> ok. +commands_teardown(Ref, _State) -> + commands_teardown(Ref). + + +%%%---------------------------------------------------------------------- +%%% Internal +%%%---------------------------------------------------------------------- + +teardown() -> + ?IMPL:teardown(). + +gen_bytes() -> + ?LET(B, list(choose(0,127)), list_to_binary(B)). + +gen_key() -> + gen_bytes(). + +gen_val() -> + gen_bytes(). + +gen_obj() -> + #obj{key=gen_key(), val=gen_val()}. + +gen_key(#state{objs=[]}) -> + gen_key(); +gen_key(#state{objs=Objs}) -> + oneof([?LET(Obj, oneof(Objs), Obj#obj.key), gen_key()]). + +gen_obj(#state{objs=[]}) -> + gen_obj(); +gen_obj(#state{objs=Objs}) -> + oneof([oneof(Objs), gen_obj()]). + +insert_obj(S, #obj{key=K}=Obj) -> + case keymember(K, S) of + false -> + S#state{objs=[Obj|S#state.objs]}; + true -> + S#state{objs=keyreplace(K, Obj, S)} + end. + +delete_obj(S, K) -> + S#state{objs=keydelete(K, S)}. + +get_val(S, K) -> + case keyfind(K, S) of + [] -> + true; + [#obj{val=Val}] -> + Val + end. + +sort_objs(#state{objs=Objs}) -> + lists:sort(Objs). + +keydelete(X, #state{objs=L}) -> + lists:filter(fun(#obj{key=K}) -> K =/= X end, L). + +keyreplace(X, Y, #state{objs=L}) -> + lists:map(fun(Z=#obj{key=K}) -> case K =:= X of true -> Y; false -> Z end end, L). + +keyfind(X, #state{objs=L}) -> + lists:filter(fun(#obj{key=K}) -> K =:= X end, L). + +keymember(X, S) -> + [] /= keyfind(X, S). + +-endif. %% -ifdef(EQC). +-endif. %% -ifdef(QC).