diff --git a/src/fractal_btree_reader.erl b/src/fractal_btree_reader.erl index 29315f7..00c5d1e 100644 --- a/src/fractal_btree_reader.erl +++ b/src/fractal_btree_reader.erl @@ -37,7 +37,7 @@ fold0(File,Fun,_InnerNode,Acc0) -> fold1(File,Fun,Acc0). fold1(File,Fun,Acc0) -> - case read_node(File) of + case read_leaf_node(File) of eof -> Acc0; {ok, Node} -> @@ -50,12 +50,12 @@ first_node(#index{file=File}) -> {node, Members} end. -next_node(#index{file=File}=Index) -> - case read_node(File) of +next_node(#index{file=File}=_Index) -> + case read_leaf_node(File) of {ok, #node{level=0, members=Members}} -> {node, Members}; - {ok, #node{level=N}} when N>0 -> - next_node(Index); +% {ok, #node{level=N}} when N>0 -> +% next_node(Index); eof -> end_of_data end. @@ -82,8 +82,8 @@ lookup_in_node(_File,#node{level=0,members=Members},Key) -> lookup_in_node(File,#node{members=Members},Key) -> case find(Key, Members) of - {ok, Pos} -> - {ok, Node} = read_node(File, Pos), + {ok, {Pos,Size}} -> + {ok, Node} = read_node(File, {Pos,Size}), lookup_in_node(File, Node, Key); notfound -> notfound @@ -100,21 +100,37 @@ find(_, _) -> notfound. +read_node(File,{Pos,Size}) -> + {ok, <<_:32, Level:16/unsigned, Data/binary>>} = file:pread(File, Pos, Size), + fractal_btree_util:decode_index_node(Level, Data); + read_node(File,Pos) -> {ok, Pos} = file:position(File, Pos), Result = read_node(File), -% error_logger:info_msg("decoded ~p ~p~n", [Pos, Result]), +% error_logger:info_msg("decoded ~p ~p~n", [Pos, Result]), Result. read_node(File) -> - {ok, <>} = file:read(File, 4), + {ok, <>} = file:read(File, 6), case Len of 0 -> eof; _ -> - {ok, Data} = file:read(File, Len), - {ok, Node} = fractal_btree_util:decode_index_node(Data), + {ok, Data} = file:read(File, Len-2), + {ok, Node} = fractal_btree_util:decode_index_node(Level, Data), {ok, Node} end. +read_leaf_node(File) -> + case file:read(File, 6) of + {ok, <<0:32, _:16>>} -> + eof; + {ok, <>} -> + {ok, Data} = file:read(File, Len-2), + fractal_btree_util:decode_index_node(0, Data); + {ok, <>} -> + {ok, _} = file:position(File, {cur,Len-2}), + read_leaf_node(File) + end. + diff --git a/src/fractal_btree_util.erl b/src/fractal_btree_util.erl index 0c5cfd5..661432e 100644 --- a/src/fractal_btree_util.erl +++ b/src/fractal_btree_util.erl @@ -15,17 +15,19 @@ estimate_node_size_increment(_KVList,Key,Value) -> is_binary(Value) -> 5 + byte_size(Value); is_atom(Value) -> - 8 + 8; + is_tuple(Value) -> + 13 end. encode_index_node(Level, KVList) -> Data = %zlib:zip( - erlang:term_to_binary({Level, KVList}) + erlang:term_to_binary(KVList) % ) , - Size = byte_size(Data), - {ok, Size+4, [ <> | Data ] }. + Size = byte_size(Data)+2, + {ok, Size+4, [ <> | Data ] }. -decode_index_node(Data) -> - {Level,KVList} = erlang:binary_to_term(Data), %zlib:unzip(Data)), +decode_index_node(Level, <>) -> + KVList = erlang:binary_to_term(Data), %zlib:unzip(Data)), {ok, {node, Level, KVList}}. diff --git a/src/fractal_btree_writer.erl b/src/fractal_btree_writer.erl index 06f6be0..d654a06 100644 --- a/src/fractal_btree_writer.erl +++ b/src/fractal_btree_writer.erl @@ -22,6 +22,7 @@ index_file_pos, last_node_pos :: pos_integer(), + last_node_size :: pos_integer(), nodes = [] :: [ #node{} ], @@ -97,12 +98,12 @@ code_change(_OldVsn, State, _Extra) -> -flush_nodes(#state{ nodes=[], last_node_pos=LastNodePos, bloom=Ref }=State) -> +flush_nodes(#state{ nodes=[], last_node_pos=LastNodePos, last_node_size=LastNodeSize, bloom=Ref }=State) -> Bloom = zlib:zip(ebloom:serialize(Ref)), BloomSize = byte_size(Bloom), - Trailer = << 0:32, Bloom/binary, BloomSize:32/unsigned, LastNodePos:64/unsigned >>, + Trailer = << 0:32, Bloom/binary, BloomSize:32/unsigned, LastNodePos:64/unsigned >>, IdxFile = State#state.index_file, ok = file:write(IdxFile, Trailer), @@ -155,7 +156,9 @@ close_node(#state{nodes=[#node{ level=Level, members=NodeMembers }|RestNodes]} = ok = file:write(State#state.index_file, Data), {FirstKey, _} = hd(OrderedMembers), - add_record(Level+1, FirstKey, NodePos, + add_record(Level+1, FirstKey, {NodePos, DataSize}, State#state{ nodes = RestNodes, index_file_pos = NodePos + DataSize, - last_node_pos = NodePos}). + last_node_pos = NodePos, + last_node_size = DataSize + }).