From bfdb63094f0ab97564635f312e8181b83bdebdfa Mon Sep 17 00:00:00 2001 From: Gregory Burd Date: Tue, 19 Jun 2012 10:13:24 +0100 Subject: [PATCH] Remove Riak-specific integration code. Deliver it later as a separate repo. --- enable-hanoidb | 60 --- src/hanoidb_bbench.config | 31 -- src/hanoidb_temp_riak_kv_backend.erl | 280 -------------- src/riak_kv_hanoidb_backend.erl | 546 --------------------------- test/backend_eqc.erl | 372 ------------------ 5 files changed, 1289 deletions(-) delete mode 100755 enable-hanoidb delete mode 100644 src/hanoidb_bbench.config delete mode 100644 src/hanoidb_temp_riak_kv_backend.erl delete mode 100644 src/riak_kv_hanoidb_backend.erl delete mode 100644 test/backend_eqc.erl diff --git a/enable-hanoidb b/enable-hanoidb deleted file mode 100755 index b83514d..0000000 --- a/enable-hanoidb +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh - -# This script adds hanoidb to a riak github repo. Run it in the riak repo -# directory. -# -# First it adds hanoidb, then runs "make all devrel" and then enables the -# hanoidb storage backend in the resulting dev nodes. -# -# This script is intended to be temporary. Once hanoidb is made into a proper -# riak citizen, this script will no longer be needed. - -set -e - -wd=`pwd` -if [ `basename $wd` != riak ]; then - echo "This doesn't appear to be a riak repo directory. Exiting." - exit 1 -fi - -if [ -d dev ]; then - echo - echo 'NOTE: THIS SCRIPT WILL DELETE YOUR EXISTING DEV DIRECTORY!' - while [ 1 ]; do - printf '\n%s' ' Do you wish to proceed? [no] ' - read answer - answer=${answer:-n} - case $answer in - [Nn]*) exit 0 ;; - [Yy]*) break ;; - *) echo 'Please answer y or n.' ;; - esac - done -fi - -./rebar get-deps - -file=./deps/riak_kv/src/riak_kv.app.src -if ! grep -q hanoidb $file ; then - echo - echo "Modifying $file, saving the original as ${file}.orig ..." - perl -i.orig -pe '/\bos_mon,/ && print qq( hanoidb,\n)' $file -fi - -file=./deps/riak_kv/rebar.config -if ! grep -q hanoidb $file ; then - echo - echo "Modifying $file, saving the original as ${file}.orig ..." - perl -i.orig -pe '/\bsext\b/ && print qq( {hanoidb, ".*", {git, "git\@github.com:basho/hanoidb.git", "master"}},\n)' $file -fi - -./rebar get-deps - -rm -rf dev -make all devrel - -echo -echo 'Modifying all dev/dev*/etc/app.config files, saving originals with .orig suffix...' -perl -i.orig -ne 'if (/\bstorage_backend,/) { s/(storage_backend, )[^\}]+/\1riak_kv_hanoidb_backend/; print } elsif (/\{eleveldb,/) { $eleveldb++; print } elsif ($eleveldb && /^\s+\]\},/) { $eleveldb = 0; print; print qq(\n {hanoidb, [\n {data_root, "./data/hanoidb"}\n ]},\n\n) } else { print }' dev/dev*/etc/app.config - -exit 0 diff --git a/src/hanoidb_bbench.config b/src/hanoidb_bbench.config deleted file mode 100644 index 5978fc0..0000000 --- a/src/hanoidb_bbench.config +++ /dev/null @@ -1,31 +0,0 @@ - -{mode, max}. - -{duration, 20}. - -{concurrent, 1}. - -{driver, basho_bench_driver_hanoidb}. - -{key_generator, {int_to_bin,{uniform_int, 5000000}}}. - -{value_generator, {fixed_bin, 10000}}. - -{operations, [{get, 1}, {put, 1}]}. - -%% the second element in the list below (e.g., "../../public/bitcask") must point to -%% the relevant directory of a hanoi installation -{code_paths, ["deps/stats", - "../hanoidb/ebin", - "../hanoidb/deps/plain_fsm/ebin", - "../hanoidb/deps/ebloom/ebin" - ]}. - -% {hanoidb_dir, "/tmp/hanoidb.bench"}. - -{hanoidb_flags, [{expiry_secs, 0}, - {sync_strategy, none}, %% |sync|{seconds, N} - {compress, none}, %% |gzip - {page_size, 8192}, % - {merge_strategy, fast} %% |predictable - ]}. diff --git a/src/hanoidb_temp_riak_kv_backend.erl b/src/hanoidb_temp_riak_kv_backend.erl deleted file mode 100644 index 7724d43..0000000 --- a/src/hanoidb_temp_riak_kv_backend.erl +++ /dev/null @@ -1,280 +0,0 @@ -%% ---------------------------------------------------------------------------- -%% -%% riak_kv_backend: Riak backend behaviour -%% -%% Copyright (c) 2007-2012 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, Version 2.0 (the -%% "License"); you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -%% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -%% License for the specific language governing permissions and limitations -%% under the License. -%% -%% ---------------------------------------------------------------------------- - -%%% NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE -%%% -%%% This is a temporary copy of riak_kv_backend, just here to keep hanoidb -%%% development private for now. When riak_kv_hanoidb_backend is moved to -%%% riak_kv, delete this file. -%%% -%%% NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE NOTE - - --module(hanoidb_temp_riak_kv_backend). - --export([behaviour_info/1]). --export([callback_after/3]). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --compile(export_all). --export([standard_test/2]). --endif. - --type fold_buckets_fun() :: fun((binary(), any()) -> any() | no_return()). --type fold_keys_fun() :: fun((binary(), binary(), any()) -> any() | - no_return()). --type fold_objects_fun() :: fun((binary(), binary(), term(), any()) -> - any() | - no_return()). --export_type([fold_buckets_fun/0, - fold_keys_fun/0, - fold_objects_fun/0]). - --spec behaviour_info(atom()) -> 'undefined' | [{atom(), arity()}]. -behaviour_info(callbacks) -> - [ - {api_version,0}, - {capabilities, 1}, % (State) - {capabilities, 2}, % (Bucket, State) - {start,2}, % (Partition, Config) - {stop,1}, % (State) - {get,3}, % (Bucket, Key, State) - {put,5}, % (Bucket, Key, IndexSpecs, Val, State) - {delete,4}, % (Bucket, Key, IndexSpecs, State) - {drop,1}, % (State) - {fold_buckets,4}, % (FoldBucketsFun, Acc, Opts, State), - % FoldBucketsFun(Bucket, Acc) - {fold_keys,4}, % (FoldKeysFun, Acc, Opts, State), - % FoldKeysFun(Bucket, Key, Acc) - {fold_objects,4}, % (FoldObjectsFun, Acc, Opts, State), - % FoldObjectsFun(Bucket, Key, Object, Acc) - {is_empty,1}, % (State) - {status,1}, % (State) - {callback,3}]; % (Ref, Msg, State) -> -behaviour_info(_Other) -> - undefined. - -%% Queue a callback for the backend after Time ms. --spec callback_after(integer(), reference(), term()) -> reference(). -callback_after(Time, Ref, Msg) when is_integer(Time), is_reference(Ref) -> - riak_core_vnode:send_command_after(Time, {backend_callback, Ref, Msg}). - --ifdef(TEST). - -standard_test(BackendMod, Config) -> - {spawn, - [ - {setup, - fun() -> ?MODULE:setup({BackendMod, Config}) end, - fun ?MODULE:cleanup/1, - fun(X) -> - [?MODULE:basic_store_and_fetch(X), - ?MODULE:fold_buckets(X), - ?MODULE:fold_keys(X), - ?MODULE:delete_object(X), - ?MODULE:fold_objects(X), - ?MODULE:empty_check(X) - ] - end - }]}. - -basic_store_and_fetch({Backend, State}) -> - {"basic store and fetch test", - fun() -> - [ - ?_assertMatch({ok, _}, - Backend:put(<<"b1">>, <<"k1">>, [], <<"v1">>, State)), - ?_assertMatch({ok, _}, - Backend:put(<<"b2">>, <<"k2">>, [], <<"v2">>, State)), - ?_assertMatch({ok,<<"v2">>, _}, - Backend:get(<<"b2">>, <<"k2">>, State)), - ?_assertMatch({error, not_found, _}, - Backend:get(<<"b1">>, <<"k3">>, State)) - ] - end - }. - -fold_buckets({Backend, State}) -> - {"bucket folding test", - fun() -> - FoldBucketsFun = - fun(Bucket, Acc) -> - [Bucket | Acc] - end, - - ?_assertEqual([<<"b1">>, <<"b2">>], - begin - {ok, Buckets1} = - Backend:fold_buckets(FoldBucketsFun, - [], - [], - State), - lists:sort(Buckets1) - end) - end - }. - -fold_keys({Backend, State}) -> - {"key folding test", - fun() -> - FoldKeysFun = - fun(Bucket, Key, Acc) -> - [{Bucket, Key} | Acc] - end, - FoldKeysFun1 = - fun(_Bucket, Key, Acc) -> - [Key | Acc] - end, - FoldKeysFun2 = - fun(Bucket, Key, Acc) -> - case Bucket =:= <<"b1">> of - true -> - [Key | Acc]; - false -> - Acc - end - end, - FoldKeysFun3 = - fun(Bucket, Key, Acc) -> - case Bucket =:= <<"b1">> of - true -> - Acc; - false -> - [Key | Acc] - end - end, - [ - ?_assertEqual([{<<"b1">>, <<"k1">>}, {<<"b2">>, <<"k2">>}], - begin - {ok, Keys1} = - Backend:fold_keys(FoldKeysFun, - [], - [], - State), - lists:sort(Keys1) - end), - ?_assertEqual({ok, [<<"k1">>]}, - Backend:fold_keys(FoldKeysFun1, - [], - [{bucket, <<"b1">>}], - State)), - ?_assertEqual([<<"k2">>], - Backend:fold_keys(FoldKeysFun1, - [], - [{bucket, <<"b2">>}], - State)), - ?_assertEqual({ok, [<<"k1">>]}, - Backend:fold_keys(FoldKeysFun2, [], [], State)), - ?_assertEqual({ok, [<<"k1">>]}, - Backend:fold_keys(FoldKeysFun2, - [], - [{bucket, <<"b1">>}], - State)), - ?_assertEqual({ok, [<<"k2">>]}, - Backend:fold_keys(FoldKeysFun3, [], [], State)), - ?_assertEqual({ok, []}, - Backend:fold_keys(FoldKeysFun3, - [], - [{bucket, <<"b1">>}], - State)) - ] - end - }. - -delete_object({Backend, State}) -> - {"object deletion test", - fun() -> - [ - ?_assertMatch({ok, _}, Backend:delete(<<"b2">>, <<"k2">>, State)), - ?_assertMatch({error, not_found, _}, - Backend:get(<<"b2">>, <<"k2">>, State)) - ] - end - }. - -fold_objects({Backend, State}) -> - {"object folding test", - fun() -> - FoldKeysFun = - fun(Bucket, Key, Acc) -> - [{Bucket, Key} | Acc] - end, - FoldObjectsFun = - fun(Bucket, Key, Value, Acc) -> - [{{Bucket, Key}, Value} | Acc] - end, - [ - ?_assertEqual([{<<"b1">>, <<"k1">>}], - begin - {ok, Keys} = - Backend:fold_keys(FoldKeysFun, - [], - [], - State), - lists:sort(Keys) - end), - - ?_assertEqual([{{<<"b1">>,<<"k1">>}, <<"v1">>}], - begin - {ok, Objects1} = - Backend:fold_objects(FoldObjectsFun, - [], - [], - State), - lists:sort(Objects1) - end), - ?_assertMatch({ok, _}, - Backend:put(<<"b3">>, <<"k3">>, [], <<"v3">>, State)), - ?_assertEqual([{{<<"b1">>,<<"k1">>},<<"v1">>}, - {{<<"b3">>,<<"k3">>},<<"v3">>}], - begin - {ok, Objects} = - Backend:fold_objects(FoldObjectsFun, - [], - [], - State), - lists:sort(Objects) - end) - ] - end - }. - -empty_check({Backend, State}) -> - {"is_empty test", - fun() -> - [ - ?_assertEqual(false, Backend:is_empty(State)), - ?_assertMatch({ok, _}, Backend:delete(<<"b1">>,<<"k1">>, State)), - ?_assertMatch({ok, _}, Backend:delete(<<"b3">>,<<"k3">>, State)), - ?_assertEqual(true, Backend:is_empty(State)) - ] - end - }. - -setup({BackendMod, Config}) -> - %% Start the backend - {ok, S} = BackendMod:start(42, Config), - {BackendMod, S}. - -cleanup({BackendMod, S}) -> - ok = BackendMod:stop(S). - --endif. % TEST diff --git a/src/riak_kv_hanoidb_backend.erl b/src/riak_kv_hanoidb_backend.erl deleted file mode 100644 index eb2590f..0000000 --- a/src/riak_kv_hanoidb_backend.erl +++ /dev/null @@ -1,546 +0,0 @@ -%% ---------------------------------------------------------------------------- -%% -%% hanoidb: LSM-trees (Log-Structured Merge Trees) Indexed Storage -%% -%% Copyright 2012 (c) Basho Technologies, Inc. All Rights Reserved. -%% http://basho.com/ info@basho.com -%% -%% This file is provided to you under the Apache License, Version 2.0 (the -%% "License"); you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -%% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -%% License for the specific language governing permissions and limitations -%% under the License. -%% -%% ---------------------------------------------------------------------------- - --module(riak_kv_hanoidb_backend). --behavior(hanoidb_temp_riak_kv_backend). --author('Steve Vinoski '). --author('Greg Burd '). - -%% KV Backend API --export([api_version/0, - capabilities/1, - capabilities/2, - start/2, - stop/1, - get/3, - put/5, - delete/4, - drop/1, - fold_buckets/4, - fold_keys/4, - fold_objects/4, - is_empty/1, - status/1, - callback/3]). - - --define(log(Fmt,Args),ok). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --export([to_index_key/4,from_index_key/1, - to_object_key/2,from_object_key/1, - to_key_range/1]). --endif. - --include("include/hanoidb.hrl"). - --define(API_VERSION, 1). -%% TODO: for when this backend supports 2i --define(CAPABILITIES, [async_fold, indexes]). -%-define(CAPABILITIES, [async_fold]). - --record(state, {tree, - partition :: integer(), - config :: config() }). - --type state() :: #state{}. --type config_option() :: {data_root, string()} | hanoidb:config_option(). --type config() :: [config_option()]. - -%% =================================================================== -%% Public API -%% =================================================================== - -%% @doc Return the major version of the -%% current API. --spec api_version() -> {ok, integer()}. -api_version() -> - {ok, ?API_VERSION}. - -%% @doc Return the capabilities of the backend. --spec capabilities(state()) -> {ok, [atom()]}. -capabilities(_) -> - {ok, ?CAPABILITIES}. - -%% @doc Return the capabilities of the backend. --spec capabilities(riak_object:bucket(), state()) -> {ok, [atom()]}. -capabilities(_, _) -> - {ok, ?CAPABILITIES}. - -%% @doc Start the hanoidb backend --spec start(integer(), config()) -> {ok, state()} | {error, term()}. -start(Partition, Config) -> - %% Get the data root directory - case app_helper:get_prop_or_env(data_root, Config, hanoidb) of - undefined -> - lager:error("Failed to create hanoidb dir: data_root is not set"), - {error, data_root_unset}; - DataRoot -> - AppStart = case application:start(hanoidb) of - ok -> - ok; - {error, {already_started, _}} -> - ok; - {error, StartReason} -> - lager:error("Failed to init the hanoidb backend: ~p", [StartReason]), - {error, StartReason} - end, - case AppStart of - ok -> - case get_data_dir(DataRoot, integer_to_list(Partition)) of - {ok, DataDir} -> - case hanoidb:open(DataDir, Config) of - {ok, Tree} -> - {ok, #state{tree=Tree, partition=Partition, config=Config }}; - {error, OpenReason}=OpenError -> - lager:error("Failed to open hanoidb: ~p\n", [OpenReason]), - OpenError - end; - {error, Reason} -> - lager:error("Failed to start hanoidb backend: ~p\n", [Reason]), - {error, Reason} - end; - Error -> - Error - end - end. - -%% @doc Stop the hanoidb backend --spec stop(state()) -> ok. -stop(#state{tree=Tree}) -> - ok = hanoidb:close(Tree). - -%% @doc Retrieve an object from the hanoidb backend --spec get(riak_object:bucket(), riak_object:key(), state()) -> - {ok, any(), state()} | - {ok, not_found, state()} | - {error, term(), state()}. -get(Bucket, Key, #state{tree=Tree}=State) -> - BKey = to_object_key(Bucket, Key), - case hanoidb:get(Tree, BKey) of - {ok, Value} -> - {ok, Value, State}; - not_found -> - {error, not_found, State}; - {error, Reason} -> - {error, Reason, State} - end. - -%% @doc Insert an object into the hanoidb backend. --type index_spec() :: {add, Index, SecondaryKey} | {remove, Index, SecondaryKey}. --spec put(riak_object:bucket(), riak_object:key(), [index_spec()], binary(), state()) -> - {ok, state()} | - {error, term(), state()}. -put(Bucket, PrimaryKey, IndexSpecs, Val, #state{tree=Tree}=State) -> - %% Create the KV update... - StorageKey = to_object_key(Bucket, PrimaryKey), - Updates1 = [{put, StorageKey, Val}], - - %% Convert IndexSpecs to index updates... - F = fun({add, Field, Value}) -> - {put, to_index_key(Bucket, PrimaryKey, Field, Value), <<>>}; - ({remove, Field, Value}) -> - {delete, to_index_key(Bucket, PrimaryKey, Field, Value)} - end, - Updates2 = [F(X) || X <- IndexSpecs], - - ok = hanoidb:transact(Tree, Updates1 ++ Updates2), - {ok, State}. - -%% @doc Delete an object from the hanoidb backend --spec delete(riak_object:bucket(), riak_object:key(), [index_spec()], state()) -> - {ok, state()} | - {error, term(), state()}. -delete(Bucket, PrimaryKey, IndexSpecs, #state{tree=Tree}=State) -> - - %% Create the KV delete... - StorageKey = to_object_key(Bucket, PrimaryKey), - Updates1 = [{delete, StorageKey}], - - %% Convert IndexSpecs to index deletes... - F = fun({remove, Field, Value}) -> - {delete, to_index_key(Bucket, PrimaryKey, Field, Value)} - end, - Updates2 = [F(X) || X <- IndexSpecs], - - case hanoidb:transact(Tree, Updates1 ++ Updates2) of - ok -> - {ok, State}; - {error, Reason} -> - {error, Reason, State} - end. - -%% @doc Fold over all the buckets --spec fold_buckets(riak_kv_backend:fold_buckets_fun(), - any(), - [], - state()) -> {ok, any()} | {async, fun()}. -fold_buckets(FoldBucketsFun, Acc, Opts, #state{tree=Tree}) -> - BucketFolder = - fun() -> - fold_list_buckets(undefined, Tree, FoldBucketsFun, Acc) - end, - case proplists:get_bool(async_fold, Opts) of - true -> - {async, BucketFolder}; - false -> - {ok, BucketFolder()} - end. - - -fold_list_buckets(PrevBucket, Tree, FoldBucketsFun, Acc) -> - ?log("fold_list_buckets prev=~p~n", [PrevBucket]), - case PrevBucket of - undefined -> - RangeStart = to_object_key(<<>>, '_'); - _ -> - RangeStart = to_object_key(<>, '_') - end, - - Range = #key_range{ from_key=RangeStart, from_inclusive=true, - to_key=undefined, to_inclusive=undefined, - limit=1 }, - - %% grab next bucket, it's a limit=1 range query :-) - case hanoidb:fold_range(Tree, - fun(BucketKey,_Value,none) -> - ?log( "IN_FOLDER ~p~n", [BucketKey]), - case from_object_key(BucketKey) of - {Bucket, _Key} -> - [Bucket]; - _ -> - none - end - end, - none, - Range) - of - none -> - ?log( "NO_MORE_BUCKETS~n", []), - Acc; - [Bucket] -> - ?log( "NEXT_BUCKET ~p~n", [Bucket]), - fold_list_buckets(Bucket, Tree, FoldBucketsFun, FoldBucketsFun(Bucket, Acc)) - end. - - -%% @doc Fold over all the keys for one or all buckets. --spec fold_keys(riak_kv_backend:fold_keys_fun(), - any(), - [{atom(), term()}], - state()) -> {ok, term()} | {async, fun()}. -fold_keys(FoldKeysFun, Acc, Opts, #state{tree=Tree}) -> - %% Figure out how we should limit the fold: by bucket, by - %% secondary index, or neither (fold across everything.) - Bucket = lists:keyfind(bucket, 1, Opts), - Index = lists:keyfind(index, 1, Opts), - - %% Multiple limiters may exist. Take the most specific limiter. - Limiter = - if Index /= false -> Index; - Bucket /= false -> Bucket; - true -> undefined - end, - - %% Set up the fold... - FoldFun = fold_keys_fun(FoldKeysFun, Limiter), - Range = to_key_range(Limiter), - case proplists:get_bool(async_fold, Opts) of - true -> - {async, fun() -> hanoidb:fold_range(Tree, FoldFun, Acc, Range) end}; - false -> - {ok, hanoidb:fold_range(Tree, FoldFun, Acc, Range)} - end. - -%% @doc Fold over all the objects for one or all buckets. --spec fold_objects(riak_kv_backend:fold_objects_fun(), - any(), - [{atom(), term()}], - state()) -> {ok, any()} | {async, fun()}. -fold_objects(FoldObjectsFun, Acc, Opts, #state{tree=Tree}) -> - Bucket = proplists:get_value(bucket, Opts), - FoldFun = fold_objects_fun(FoldObjectsFun, Bucket), - ObjectFolder = - fun() -> -% io:format(user, "starting fold_objects in ~p~n", [self()]), - Result = hanoidb:fold_range(Tree, FoldFun, Acc, to_key_range(Bucket)), -% io:format(user, "ended fold_objects in ~p => ~P~n", [self(),Result,20]), - Result - end, - case proplists:get_bool(async_fold, Opts) of - true -> - {async, ObjectFolder}; - false -> - {ok, ObjectFolder()} - end. - -%% @doc Delete all objects from this hanoidb backend --spec drop(state()) -> {ok, state()} | {error, term(), state()}. -drop(#state{ tree=Tree, partition=Partition, config=Config }=State) -> - case hanoidb:destroy(Tree) of - ok -> - start(Partition, Config); - {error, Term} -> - {error, Term, State} - end. - -%% @doc Returns true if this hanoidb backend contains any -%% non-tombstone values; otherwise returns false. --spec is_empty(state()) -> boolean(). -is_empty(#state{tree=Tree}) -> - FoldFun = fun(K, _V, Acc) -> [K|Acc] end, - try - Range = to_key_range(undefined), - [] =:= hanoidb:fold_range(Tree, FoldFun, [], Range#key_range{ limit=1 }) - catch - _:ok -> - false - end. - -%% @doc Get the status information for this hanoidb backend --spec status(state()) -> [{atom(), term()}]. -status(#state{}) -> - %% TODO: not yet implemented - []. - -%% @doc Register an asynchronous callback --spec callback(reference(), any(), state()) -> {ok, state()}. -callback(_Ref, _Msg, State) -> - {ok, State}. - - -%% =================================================================== -%% Internal functions -%% =================================================================== - -%% @private -%% Create the directory for this partition's LSM-BTree files -get_data_dir(DataRoot, Partition) -> - PartitionDir = filename:join([DataRoot, Partition]), - case filelib:ensure_dir(filename:join([filename:absname(DataRoot), Partition, "x"])) of - ok -> - {ok, PartitionDir}; - {error, Reason} -> - lager:error("Failed to create hanoidb dir ~s: ~p", [PartitionDir, Reason]), - {error, Reason} - end. - -%% @private -%% Return a function to fold over keys on this backend -fold_keys_fun(FoldKeysFun, undefined) -> - %% Fold across everything... - fun(K, _V, Acc) -> - case from_object_key(K) of - {Bucket, Key} -> - FoldKeysFun(Bucket, Key, Acc) - end - end; -fold_keys_fun(FoldKeysFun, {bucket, FilterBucket}) -> - %% Fold across a specific bucket... - fun(K, _V, Acc) -> - case from_object_key(K) of - {Bucket, Key} when Bucket == FilterBucket -> - FoldKeysFun(Bucket, Key, Acc) - end - end; -fold_keys_fun(FoldKeysFun, {index, FilterBucket, {eq, <<"$bucket">>, _}}) -> - %% 2I exact match query on special $bucket field... - fold_keys_fun(FoldKeysFun, {bucket, FilterBucket}); -fold_keys_fun(FoldKeysFun, {index, FilterBucket, {eq, FilterField, FilterTerm}}) -> - %% Rewrite 2I exact match query as a range... - NewQuery = {range, FilterField, FilterTerm, FilterTerm}, - fold_keys_fun(FoldKeysFun, {index, FilterBucket, NewQuery}); -fold_keys_fun(FoldKeysFun, {index, FilterBucket, {range, <<"$key">>, StartKey, EndKey}}) -> - %% 2I range query on special $key field... - fun(StorageKey, Acc) -> - case from_object_key(StorageKey) of - {Bucket, Key} when FilterBucket == Bucket, - StartKey =< Key, - EndKey >= Key -> - FoldKeysFun(Bucket, Key, Acc) - end - end; -fold_keys_fun(FoldKeysFun, {index, FilterBucket, {range, FilterField, StartTerm, EndTerm}}) -> - %% 2I range query... - fun(StorageKey, Acc) -> - case from_index_key(StorageKey) of - {Bucket, Key, Field, Term} when FilterBucket == Bucket, - FilterField == Field, - StartTerm =< Term, - EndTerm >= Term -> - FoldKeysFun(Bucket, Key, Acc) - end - end; -fold_keys_fun(_FoldKeysFun, Other) -> - throw({unknown_limiter, Other}). - -%% @private -%% Return a function to fold over the objects on this backend -fold_objects_fun(FoldObjectsFun, FilterBucket) -> - fun(StorageKey, Value, Acc) -> - ?log( "OFOLD: ~p, filter=~p~n", [sext:decode(StorageKey), FilterBucket]), - case from_object_key(StorageKey) of - {Bucket, Key} when FilterBucket == undefined; - Bucket == FilterBucket -> - FoldObjectsFun(Bucket, Key, Value, Acc) - end - end. - - -%% This is guaranteed larger than any object key --define(MAX_OBJECT_KEY, <<16,0,0,0,4>>). - -%% This is guaranteed larger than any index key --define(MAX_INDEX_KEY, <<16,0,0,0,6>>). - -to_key_range(undefined) -> - #key_range{ from_key = to_object_key(<<>>, <<>>), - from_inclusive = true, - to_key = ?MAX_OBJECT_KEY, - to_inclusive = false - }; -to_key_range({bucket, Bucket}) -> - #key_range{ from_key = to_object_key(Bucket, <<>>), - from_inclusive = true, - to_key = to_object_key(<>, <<>>), - to_inclusive = false }; -to_key_range({index, Bucket, {eq, <<"$bucket">>, _Term}}) -> - to_key_range(Bucket); -to_key_range({index, Bucket, {eq, Field, Term}}) -> - to_key_range({index, Bucket, {range, Field, Term, Term}}); -to_key_range({index, Bucket, {range, <<"$key">>, StartTerm, EndTerm}}) -> - #key_range{ from_key = to_object_key(Bucket, StartTerm), - from_inclusive = true, - to_key = to_object_key(Bucket, EndTerm), - to_inclusive = true }; -to_key_range({index, Bucket, {range, Field, StartTerm, EndTerm}}) -> - #key_range{ from_key = to_index_key(Bucket, <<>>, Field, StartTerm), - from_inclusive = true, - to_key = to_index_key(Bucket, <<16#ff,16#ff,16#ff,16#ff, - 16#ff,16#ff,16#ff,16#ff, - 16#ff,16#ff,16#ff,16#ff, - 16#ff,16#ff,16#ff,16#ff, - 16#ff,16#ff,16#ff,16#ff, - 16#ff,16#ff,16#ff,16#ff, - 16#ff,16#ff,16#ff,16#ff, - 16#ff,16#ff,16#ff,16#ff >>, Field, EndTerm), - to_inclusive = false }; -to_key_range(Other) -> - erlang:throw({unknown_limiter, Other}). - - - - -to_object_key(Bucket, Key) -> - sext:encode({o, Bucket, Key}). - -from_object_key(LKey) -> - case sext:decode(LKey) of - {o, Bucket, Key} -> - {Bucket, Key}; - _ -> - undefined - end. - -to_index_key(Bucket, Key, Field, Term) -> - sext:encode({i, Bucket, Field, Term, Key}). - -from_index_key(LKey) -> - case sext:decode(LKey) of - {i, Bucket, Field, Term, Key} -> - {Bucket, Key, Field, Term}; - _ -> - undefined - end. - -%% =================================================================== -%% EUnit tests -%% =================================================================== --ifdef(TEST). - --include("src/hanoidb.hrl"). - -key_range_test() -> - Range = to_key_range({bucket, <<"a">>}), - - ?assertEqual(true, ?KEY_IN_RANGE( to_object_key(<<"a">>, <<>>) , Range)), - ?assertEqual(true, ?KEY_IN_RANGE( to_object_key(<<"a">>, <<16#ff,16#ff,16#ff,16#ff>>), Range )), - ?assertEqual(false, ?KEY_IN_RANGE( to_object_key(<<>>, <<>>), Range )), - ?assertEqual(false, ?KEY_IN_RANGE( to_object_key(<<"a",0>>, <<>>), Range )). - -index_range_test() -> - Range = to_key_range({index, <<"idx">>, {range, <<"f">>, <<6>>, <<7,3>>}}), - - ?assertEqual(false, ?KEY_IN_RANGE( to_index_key(<<"idx">>, <<"key1">>, <<"f">>, <<5>>) , Range)), - ?assertEqual(true, ?KEY_IN_RANGE( to_index_key(<<"idx">>, <<"key1">>, <<"f">>, <<6>>) , Range)), - ?assertEqual(true, ?KEY_IN_RANGE( to_index_key(<<"idx">>, <<"key1">>, <<"f">>, <<7>>) , Range)), - ?assertEqual(false, ?KEY_IN_RANGE( to_index_key(<<"idx">>, <<"key1">>, <<"f">>, <<7,4>>) , Range)), - ?assertEqual(false, ?KEY_IN_RANGE( to_index_key(<<"idx">>, <<"key1">>, <<"f">>, <<9>>) , Range)). - - -simple_test_() -> - ?assertCmd("rm -rf test/hanoidb-backend"), - application:set_env(hanoidb, data_root, "test/hanoidbd-backend"), - hanoidb_temp_riak_kv_backend:standard_test(?MODULE, []). - -custom_config_test_() -> - ?assertCmd("rm -rf test/hanoidb-backend"), - application:set_env(hanoidb, data_root, ""), - hanoidb_temp_riak_kv_backend:standard_test(?MODULE, [{data_root, "test/hanoidb-backend"}]). - --ifdef(PROPER). - -eqc_test_() -> - {spawn, - [{inorder, - [{setup, - fun setup/0, - fun cleanup/1, - [ - {timeout, 60, - [?_assertEqual(true, - backend_eqc:test(?MODULE, false, - [{data_root, - "test/hanoidbdb-backend"}, - {async_fold, false}]))]}, - {timeout, 60, - [?_assertEqual(true, - backend_eqc:test(?MODULE, false, - [{data_root, - "test/hanoidbdb-backend"}]))]} - ]}]}]}. - -setup() -> - application:load(sasl), - application:set_env(sasl, sasl_error_logger, {file, "riak_kv_hanoidbdb_backend_eqc_sasl.log"}), - error_logger:tty(false), - error_logger:logfile({open, "riak_kv_hanoidbdb_backend_eqc.log"}), - - ok. - -cleanup(_) -> - ?_assertCmd("rm -rf test/hanoidbdb-backend"). - --endif. % EQC - - --endif. diff --git a/test/backend_eqc.erl b/test/backend_eqc.erl deleted file mode 100644 index b75621f..0000000 --- a/test/backend_eqc.erl +++ /dev/null @@ -1,372 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% backend_eqc: Quickcheck testing for the backend api. -%% -%% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(backend_eqc). - --define(log(Fmt,Args),ok). - --ifdef(TRIQ). - --include_lib("triq/include/triq.hrl"). --include_lib("eunit/include/eunit.hrl"). - -%% Public API --compile(export_all). --export([test/1, - test/2, - test/3, - test/4, - test/5]). - -%% eqc_fsm callbacks --export([initial_state/0, - initial_state_data/0, - next_state_data/5, - precondition/4, - postcondition/5]). - -%% eqc property --export([prop_backend/4]). - -%% States --export([stopped/1, - running/1]). - -%% Helpers --export([drop/2, - init_backend/3]). - --define(TEST_ITERATIONS, 50). - --record(qcst, {backend, % Backend module under test - volatile, % Indicates if backend is volatile - c, % Backend config - s, % Module state returned by Backend:start - olds=sets:new(), % Old states after a stop - d=[]}).% Orddict of values stored - -%% ==================================================================== -%% Public API -%% ==================================================================== - -test(Backend) -> - test(Backend, false). - -test(Backend, Volatile) -> - test(Backend, Volatile, []). - -test(Backend, Volatile, Config) -> - test(Backend, Volatile, Config, fun(_BeState,_Olds) -> ok end). - -test(Backend, Volatile, Config, Cleanup) -> - test(Backend, Volatile, Config, Cleanup, ?TEST_ITERATIONS). - -test(Backend, Volatile, Config, Cleanup, NumTests) -> - triq:check(%triq:numtests(NumTests, - prop_backend(Backend, Volatile, Config, Cleanup) -% ) -). - -%% ==================================================================== -%% eqc property -%% ==================================================================== - -prop_backend(Backend, Volatile, Config, Cleanup) -> - ?FORALL(Cmds, - triq_fsm:commands(?MODULE, - {stopped, - initial_state_data(Backend, Volatile, Config)}), - begin - {H,{_F,S},Res} = triq_fsm:run_commands(?MODULE, Cmds), - Cleanup(S#qcst.s, sets:to_list(S#qcst.olds)), -% aggregate(zip(triq_fsm:state_names(H), command_names(Cmds)), - ?WHENFAIL( - begin - ?debugFmt("Cmds: ~p~n", - [triq_statem:zip(triq_fsm:state_names(H), - triq_statem:command_names(Cmds))]), - ?debugFmt("Result: ~p~n", [Res]), - ?debugFmt("History: ~p~n", [H]), - ?debugFmt("BE Config: ~p~nBE State: ~p~nD: ~p~n", - [S#qcst.c, - S#qcst.s, - orddict:to_list(S#qcst.d)]) - end, - begin - ?log( "!!!!!!!!!!!!!!!!!!! Res: ~p~n", [Res]), - case Res of - ok -> true; - true -> true; - _ -> false - end - end - ) -% ) - end - ). - -%%==================================================================== -%% Generators -%%==================================================================== - -bucket() -> - elements([<<"b1">>,<<"b2">>,<<"b3">>,<<"b4">>]). - -key() -> - elements([<<"k1">>,<<"k2">>,<<"k3">>,<<"k4">>]). - -val() -> - %% The creation of the riak object and the call - %% to term_to_binary are to facilitate testing - %% of riak_kv_index_backend. It does not matter - %% that the bucket and key in the object may - %% differ since at this point in the processing - %% pipeline the information has already been - %% extracted. - term_to_binary(riak_object:new(<<"b1">>, <<"k1">>, binary())). - -g_opts() -> - frequency([{5, [async_fold]}, {2, []}]). - -%%==================================================================== -%% Helpers -%%==================================================================== - -fold_buckets_fun() -> - fun(Bucket, Acc) -> - riak_kv_fold_buffer:add(Bucket, Acc) - end. - -fold_keys_fun() -> - fun(Bucket, Key, Acc) -> - riak_kv_fold_buffer:add({Bucket, Key}, Acc) - end. - -fold_objects_fun() -> - fun(Bucket, Key, Value, Acc) -> - riak_kv_fold_buffer:add({{Bucket, Key}, Value}, Acc) - end. - -get_partition() -> - {MegaSecs, Secs, MicroSecs} = erlang:now(), - Partition = integer_to_list(MegaSecs) ++ - integer_to_list(Secs) ++ - integer_to_list(MicroSecs), - case erlang:get(Partition) of - undefined -> - erlang:put(Partition, ok), - list_to_integer(Partition); - _ -> - get_partition() - end. - -%% @TODO Volatile is unused now so remove it. Will require -%% updating each backend module as well. -init_backend(Backend, _Volatile, Config) -> - Partition = get_partition(), - %% Start an async worker pool - {ok, PoolPid} = - riak_core_vnode_worker_pool:start_link(riak_kv_worker, - 2, - Partition, - [], - worker_props), - %% Shutdown any previous running worker pool - case erlang:get(worker_pool) of - undefined -> - ok; - OldPoolPid -> - riak_core_vnode_worker_pool:stop(OldPoolPid, normal) - end, - %% Store the info about the worker pool - erlang:put(worker_pool, PoolPid), - %% Start the backend - {ok, S} = Backend:start(Partition, Config), - S. - -drop(Backend, State) -> - case Backend:drop(State) of - {ok, NewState} -> - NewState; - {error, _, NewState} -> - NewState - end. - -get_fold_buffer() -> - riak_kv_fold_buffer:new(100, - get_fold_buffer_fun({raw, foldid, self()})). - -get_fold_buffer_fun(From) -> - fun(Results) -> - riak_core_vnode:reply(From, - Results) - end. - -%% @private -finish_fun(Sender) -> - fun(Buffer) -> - finish_fold(Buffer, Sender) - end. - -%% @private -finish_fold(Buffer, Sender) -> - ?log( "finish_fold: ~p,~p~n", [Buffer, Sender]), - riak_kv_fold_buffer:flush(Buffer), - riak_core_vnode:reply(Sender, done). - -receive_fold_results(Acc) -> - receive - {_, done} -> - Acc; - {error, Error} -> - ?debugFmt("Error occurred: ~p~n", [Error]), - []; - {_, Results} -> - receive_fold_results(Acc++Results) - after 1000 -> - receive MSG -> exit({bad, MSG}) end - end. - -%%==================================================================== -%% eqc_fsm callbacks -%%==================================================================== - -initial_state() -> - {stopped, true}. - -initial_state_data() -> - #qcst{d = orddict:new()}. - -initial_state_data(Backend, Volatile, Config) -> - #qcst{backend=Backend, - c=Config, - d=orddict:new(), - volatile=Volatile}. - -next_state_data(running, stopped, S, _R, - {call, _M, stop, _}) -> - S#qcst{d=orddict:new(), - olds = sets:add_element(S#qcst.s, S#qcst.olds)}; -next_state_data(stopped, running, S, R, {call, _M, init_backend, _}) -> - S#qcst{s=R}; -next_state_data(_From, _To, S, _R, {call, _M, put, [Bucket, Key, [], Val, _]}) -> - S#qcst{d = orddict:store({Bucket, Key}, Val, S#qcst.d)}; -next_state_data(_From, _To, S, _R, {call, _M, delete, [Bucket, Key, [], _]}) -> - S#qcst{d = orddict:erase({Bucket, Key}, S#qcst.d)}; -next_state_data(_From, _To, S, R, {call, ?MODULE, drop, _}) -> - S#qcst{d=orddict:new(), s=R}; -next_state_data(_From, _To, S, _R, _C) -> - S. - -stopped(#qcst{backend=Backend, - c=Config, - volatile=Volatile}) -> - [{running, - {call, ?MODULE, init_backend, [Backend, Volatile, Config]}}]. - -running(#qcst{backend=Backend, - s=State}) -> - [ - {history, {call, Backend, put, [bucket(), key(), [], val(), State]}}, - {history, {call, Backend, get, [bucket(), key(), State]}}, - {history, {call, Backend, delete, [bucket(), key(), [], State]}}, - {history, {call, Backend, fold_buckets, [fold_buckets_fun(), get_fold_buffer(), g_opts(), State]}}, - {history, {call, Backend, fold_keys, [fold_keys_fun(), get_fold_buffer(), g_opts(), State]}}, - {history, {call, Backend, fold_objects, [fold_objects_fun(), get_fold_buffer(), g_opts(), State]}}, - {history, {call, Backend, is_empty, [State]}}, - {history, {call, ?MODULE, drop, [Backend, State]}}, - {stopped, {call, Backend, stop, [State]}} - ]. - -precondition(_From,_To,_S,_C) -> - true. - -postcondition(_From, _To, S, _C={call, _M, get, [Bucket, Key, _BeState]}, R) -> - case R of - {error, not_found, _} -> - not orddict:is_key({Bucket, Key}, S#qcst.d); - {ok, Val, _} -> - Res = orddict:find({Bucket, Key}, S#qcst.d), - {ok, Val} =:= Res - end; -postcondition(_From, _To, _S, - {call, _M, put, [_Bucket, _Key, _IndexEntries, _Val, _BeState]}, {R, _RState}) -> - R =:= ok orelse R =:= already_exists; -postcondition(_From, _To, _S, - {call, _M, delete,[_Bucket, _Key, _IndexEntries, _BeState]}, {R, _RState}) -> - R =:= ok; -postcondition(_From, _To, S, - {call, _M, fold_buckets, [_FoldFun, _Acc, _Opts, _BeState]}, FoldRes) -> - ExpectedEntries = orddict:to_list(S#qcst.d), - Buckets = [Bucket || {{Bucket, _}, _} <- ExpectedEntries], - From = {raw, foldid, self()}, - case FoldRes of - {async, Work} -> - Pool = erlang:get(worker_pool), - ?log( "pool: ~p~n", [Pool]), - FinishFun = finish_fun(From), - riak_core_vnode_worker_pool:handle_work(Pool, {fold, Work, FinishFun}, From); - {ok, Buffer} -> - ?log( "got: ~p~n", [Buffer]), - finish_fold(Buffer, From) - end, - R = receive_fold_results([]), - lists:usort(Buckets) =:= lists:sort(R); -postcondition(_From, _To, S, - {call, _M, fold_keys, [_FoldFun, _Acc, _Opts, _BeState]}, FoldRes) -> - ExpectedEntries = orddict:to_list(S#qcst.d), - Keys = [{Bucket, Key} || {{Bucket, Key}, _} <- ExpectedEntries], - From = {raw, foldid, self()}, - case FoldRes of - {async, Work} -> - Pool = erlang:get(worker_pool), - ?log( "pool: ~p~n", [Pool]), - FinishFun = finish_fun(From), - riak_core_vnode_worker_pool:handle_work(Pool, {fold, Work, FinishFun}, From); - {ok, Buffer} -> - ?log( "got: ~p~n", [Buffer]), - finish_fold(Buffer, From) - end, - R = receive_fold_results([]), - lists:sort(Keys) =:= lists:sort(R); -postcondition(_From, _To, S, - {call, _M, fold_objects, [_FoldFun, _Acc, _Opts, _BeState]}, FoldRes) -> - ExpectedEntries = orddict:to_list(S#qcst.d), - Objects = [Object || Object <- ExpectedEntries], - From = {raw, foldid, self()}, - case FoldRes of - {async, Work} -> - Pool = erlang:get(worker_pool), - FinishFun = finish_fun(From), - riak_core_vnode_worker_pool:handle_work(Pool, {fold, Work, FinishFun}, From); - {ok, Buffer} -> - finish_fold(Buffer, From) - end, - R = receive_fold_results([]), - ?log( "POST: fold_objects ~p =:= ~p~n", [Objects, R]), - lists:sort(Objects) =:= lists:sort(R); -postcondition(_From, _To, S,{call, _M, is_empty, [_BeState]}, R) -> - R =:= (orddict:size(S#qcst.d) =:= 0); -postcondition(_From, _To, _S, _C, _R) -> - true. - --endif.