From de75b5ed3d3952333a5b5530b92ce456e629218d Mon Sep 17 00:00:00 2001 From: Kresten Krab Thorup Date: Wed, 2 Apr 2014 23:51:23 +0200 Subject: [PATCH] Improve types/specs This patch also removes array representations from the bloom implementation, as it is unused. --- Makefile | 54 +++++++++++++++++++++++++++++++++++ rebar.config | 2 +- src/hanoidb_bloom.erl | 55 +++++++++++++----------------------- src/hanoidb_dense_bitmap.erl | 15 ++++++++-- src/hanoidb_reader.erl | 1 + 5 files changed, 88 insertions(+), 39 deletions(-) diff --git a/Makefile b/Makefile index 79771c0..60b6ee4 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,43 @@ REBAR= rebar DIALYZER= dialyzer +DEPS = $(CURDIR)/deps + +DIALYZER_OPTS = # -Wunderspecs + +# List dependencies that should be included in a cached dialyzer PLT file. +DIALYZER_DEPS = deps/lz4/ebin \ + deps/snappy/ebin \ + deps/plain_fsm/ebin + +DEPS_PLT = hanoi.plt + +ERLANG_DIALYZER_APPS = asn1 \ + compiler \ + crypto \ + edoc \ + edoc \ + erts \ + eunit \ + eunit \ + gs \ + hipe \ + inets \ + kernel \ + mnesia \ + mnesia \ + observer \ + public_key \ + runtime_tools \ + runtime_tools \ + ssl \ + stdlib \ + syntax_tools \ + syntax_tools \ + tools \ + webtool \ + xmerl + .PHONY: plt analyze all deps compile get-deps clean @@ -47,5 +84,22 @@ analyze-nospec: compile --no_spec \ ebin +# Only include local PLT if we have deps that we are going to analyze +ifeq ($(strip $(DIALYZER_DEPS)),) +dialyzer: ~/.dialyzer_plt compile + @dialyzer $(DIALYZER_OPTS) -r ebin +else +dialyzer: ~/.dialyzer_plt $(DEPS_PLT) compile + @dialyzer $(DIALYZER_OPTS) --plts ~/.dialyzer_plt $(DEPS_PLT) -r ebin + +$(DEPS_PLT): + @dialyzer --build_plt $(DIALYZER_DEPS) --output_plt $(DEPS_PLT) +endif + +~/.dialyzer_plt: + @echo "ERROR: Missing ~/.dialyzer_plt. Please wait while a new PLT is compiled." + dialyzer --build_plt --apps $(ERLANG_DIALYZER_APPS) + @echo "now try your build again" + repl: erl -pz deps/*/ebin -pa ebin diff --git a/rebar.config b/rebar.config index 26bb07a..80b9275 100644 --- a/rebar.config +++ b/rebar.config @@ -26,7 +26,7 @@ {deps, [ {sext, ".*", {git, "git://github.com/esl/sext", {branch, "master"}}} , {lager, ".*", {git, "git://github.com/basho/lager", {branch, "master"}}} - , {snappy, "1.0.*", {git, "git://github.com/fdmanana/snappy-erlang-nif.git", {branch, "master"}}} + , {snappy, "1.1.*", {git, "git://github.com/fdmanana/snappy-erlang-nif.git", {branch, "master"}}} , {plain_fsm, "1.1.*", {git, "git://github.com/gburd/plain_fsm", {branch, "master"}}} % , {basho_bench, ".*", {git, "git://github.com/basho/basho_bench", {branch, "master"}}} , {lz4, ".*", {git, "git://github.com/krestenkrab/erlang-lz4.git", {branch, "master"}}} diff --git a/src/hanoidb_bloom.erl b/src/hanoidb_bloom.erl index 25120eb..7013763 100644 --- a/src/hanoidb_bloom.erl +++ b/src/hanoidb_bloom.erl @@ -41,7 +41,9 @@ -define(W, 27). --type bitmask() :: array() | any(). +-type bitmask() :: {sparse_bitmap, non_neg_integer(), list()} + | hanoidb_dense_bitmap:bitmap() + . -record(bloom, { e :: float(), % error probability @@ -113,16 +115,19 @@ sbf(N, E, S, R) when is_number(N), N > 0, %% Returns number of elements %% +-spec size( #bloom{} | #sbf{} ) -> non_neg_integer(). size(#bloom{size=Size}) -> Size; size(#sbf{size=Size}) -> Size. %% Returns capacity %% +-spec capacity( #bloom{} | #sbf{} ) -> non_neg_integer() | infinity. capacity(#bloom{n=N}) -> N; capacity(#sbf{}) -> infinity. %% Test for membership %% +-spec member( any(), #bloom{} | #sbf{} ) -> boolean(). member(Elem, #bloom{mb=Mb}=B) -> Hashes = make_hashes(Mb, Elem), hash_member(Hashes, B); @@ -189,62 +194,42 @@ bitmask_new(LogN) -> hanoidb_dense_bitmap:new(1 bsl LogN) end. + bitmask_set(I, BM) -> case element(1,BM) of - array -> bitarray_set(I, as_array(BM)); +% array -> bitarray_set(I, as_array(BM)); sparse_bitmap -> hanoidb_sparse_bitmap:set(I, BM); dense_bitmap_ets -> hanoidb_dense_bitmap:set(I, BM); - dense_bitmap -> + dense_bitmap_term -> %% Surprise - we need to mutate a built representation: hanoidb_dense_bitmap:set(I, hanoidb_dense_bitmap:unbuild(BM)) end. %%% Convert to external form. +-spec bitmask_build( bitmask() ) -> {sparse_bitmap, non_neg_integer(), list()} + | {dense_bitmap_term, tuple()}. + bitmask_build(BM) -> case element(1,BM) of - array -> BM; - sparse_bitmap -> BM; - dense_bitmap_ets -> hanoidb_dense_bitmap:build(BM) + dense_bitmap_ets -> hanoidb_dense_bitmap:build(BM); + _ -> BM end. +-spec bitmask_get( non_neg_integer(), bitmask() ) -> boolean(). bitmask_get(I, BM) -> case element(1,BM) of - array -> bitarray_get(I, as_array(BM)); - sparse_bitmap -> hanoidb_sparse_bitmap:member(I, BM); - dense_bitmap_ets -> hanoidb_dense_bitmap:member(I, BM); - dense_bitmap -> hanoidb_dense_bitmap:member(I, BM) + sparse_bitmap -> hanoidb_sparse_bitmap:member(I, BM); + dense_bitmap_ets -> hanoidb_dense_bitmap:member(I, BM); + dense_bitmap_term -> hanoidb_dense_bitmap:member(I, BM) end. --spec as_array(bitmask()) -> array(). -as_array(BM) -> - case array:is_array(BM) of - true -> BM - end. - -%%%========== Bitarray representation - suitable for sparse arrays ========== -bitarray_new(N) -> array:new((N-1) div ?W + 1, {default, 0}). - --spec bitarray_set( non_neg_integer(), array() ) -> array(). -bitarray_set(I, A1) -> - A = as_array(A1), - AI = I div ?W, - V = array:get(AI, A), - V1 = V bor (1 bsl (I rem ?W)), - if V =:= V1 -> A; % The bit is already set - true -> array:set(AI, V1, A) - end. - --spec bitarray_get( non_neg_integer(), array() ) -> boolean(). -bitarray_get(I, A) -> - AI = I div ?W, - V = array:get(AI, A), - (V band (1 bsl (I rem ?W))) =/= 0. - %%%^^^^^^^^^^ Bitarray representation - suitable for sparse arrays ^^^^^^^^^^ +-spec encode( #bloom{} | #sbf{} ) -> binary(). encode(Bloom) -> zlib:gzip(term_to_binary(bloom_build(Bloom))). +-spec decode( binary() ) -> #bloom{} | #sbf{}. decode(Bin) -> binary_to_term(zlib:gunzip(Bin)). diff --git a/src/hanoidb_dense_bitmap.erl b/src/hanoidb_dense_bitmap.erl index 1f45218..ec01e5d 100644 --- a/src/hanoidb_dense_bitmap.erl +++ b/src/hanoidb_dense_bitmap.erl @@ -5,6 +5,11 @@ -define(REPR_NAME, dense_bitmap). +-type bitmap() :: {dense_bitmap_ets, non_neg_integer()|undefined, non_neg_integer()|undefined, ets:tid()} + | {dense_bitmap_term, tuple() }. +-export_type([ bitmap/0 ]). + +-spec new( non_neg_integer() ) -> bitmap(). new(N) -> Tab = ets:new(dense_bitmap, [private, set]), Width = 1 + (N-1) div ?BITS_PER_CELL, @@ -28,14 +33,18 @@ set(I, {dense_bitmap_ets, _,_, Tab}=DBM) -> build({dense_bitmap_ets, _, _, Tab}) -> [Row] = ets:lookup(Tab, ?REPR_NAME), ets:delete(Tab), - Row. + {dense_bitmap_term, Row}; +build({dense_bitmap_term, _}=Value) -> + Value. -unbuild(Row) when element(1,Row)==?REPR_NAME -> +-spec unbuild( {dense_bitmap_term, any()} ) -> bitmap(). +unbuild({dense_bitmap_term, Row}) when element(1,Row)==?REPR_NAME -> Tab = ets:new(dense_bitmap, [private, set]), ets:insert(Tab, Row), {dense_bitmap_ets, undefined, undefined, Tab}. -member(I, Row) when element(1,Row)==?REPR_NAME -> +-spec member( non_neg_integer(), bitmap() ) -> boolean(). +member(I, {dense_bitmap_term, Row}) when element(1,Row)==?REPR_NAME -> Cell = 2 + I div ?BITS_PER_CELL, BitInCell = I rem ?BITS_PER_CELL, CellValue = element(Cell, Row), diff --git a/src/hanoidb_reader.erl b/src/hanoidb_reader.erl index 932c6d8..dd222a0 100644 --- a/src/hanoidb_reader.erl +++ b/src/hanoidb_reader.erl @@ -46,6 +46,7 @@ config=[] :: term() }). -type read_file() :: #index{}. +-export_type([read_file/0]). -spec open(Name::string()) -> {ok, read_file()} | {error, any()}. open(Name) ->