From b21e25332426655656a504a18d5cf1d8137656bc Mon Sep 17 00:00:00 2001
From: Kresten Krab Thorup <krab@trifork.com>
Date: Fri, 6 Jan 2012 00:29:05 +0100
Subject: [PATCH] Store child-refs as {Pos,Size} so we can pread

This allows us to use file:pread to read a
child-node, rather than two separate reads
(one for node block size, and then one for
the node block itself).

Also, encode the level# in node header, so that
scanning leafs doesn't need to decode the
node contents for inner nodes.
---
 src/fractal_btree_reader.erl | 38 +++++++++++++++++++++++++-----------
 src/fractal_btree_util.erl   | 14 +++++++------
 src/fractal_btree_writer.erl | 11 +++++++----
 3 files changed, 42 insertions(+), 21 deletions(-)

diff --git a/src/fractal_btree_reader.erl b/src/fractal_btree_reader.erl
index 29315f7..00c5d1e 100644
--- a/src/fractal_btree_reader.erl
+++ b/src/fractal_btree_reader.erl
@@ -37,7 +37,7 @@ fold0(File,Fun,_InnerNode,Acc0) ->
     fold1(File,Fun,Acc0).
 
 fold1(File,Fun,Acc0) ->
-    case read_node(File) of
+    case read_leaf_node(File) of
         eof ->
             Acc0;
         {ok, Node} ->
@@ -50,12 +50,12 @@ first_node(#index{file=File}) ->
             {node, Members}
     end.
 
-next_node(#index{file=File}=Index) ->
-    case read_node(File) of
+next_node(#index{file=File}=_Index) ->
+    case read_leaf_node(File) of
         {ok, #node{level=0, members=Members}} ->
             {node, Members};
-        {ok, #node{level=N}} when N>0 ->
-            next_node(Index);
+%        {ok, #node{level=N}} when N>0 ->
+%            next_node(Index);
         eof ->
             end_of_data
     end.
@@ -82,8 +82,8 @@ lookup_in_node(_File,#node{level=0,members=Members},Key) ->
 
 lookup_in_node(File,#node{members=Members},Key) ->
     case find(Key, Members) of
-        {ok, Pos} ->
-            {ok, Node} = read_node(File, Pos),
+        {ok, {Pos,Size}} ->
+            {ok, Node} = read_node(File, {Pos,Size}),
             lookup_in_node(File, Node, Key);
         notfound ->
             notfound
@@ -100,21 +100,37 @@ find(_, _) ->
     notfound.
 
 
+read_node(File,{Pos,Size}) ->
+    {ok, <<_:32, Level:16/unsigned, Data/binary>>} = file:pread(File, Pos, Size),
+    fractal_btree_util:decode_index_node(Level, Data);
+
 read_node(File,Pos) ->
 
     {ok, Pos} = file:position(File, Pos),
     Result = read_node(File),
-%    error_logger:info_msg("decoded ~p ~p~n", [Pos, Result]),
+%   error_logger:info_msg("decoded ~p ~p~n", [Pos, Result]),
     Result.
 
 read_node(File) ->
-    {ok, <<Len:32>>} = file:read(File, 4),
+    {ok, <<Len:32, Level:16/unsigned>>} = file:read(File, 6),
     case Len of
         0 -> eof;
         _ ->
-            {ok, Data} = file:read(File, Len),
-            {ok, Node} = fractal_btree_util:decode_index_node(Data),
+            {ok, Data} = file:read(File, Len-2),
+            {ok, Node} = fractal_btree_util:decode_index_node(Level, Data),
             {ok, Node}
     end.
 
 
+read_leaf_node(File) ->
+    case file:read(File, 6) of
+        {ok, <<0:32, _:16>>} ->
+            eof;
+        {ok, <<Len:32, 0:16>>} ->
+            {ok, Data} = file:read(File, Len-2),
+            fractal_btree_util:decode_index_node(0, Data);
+        {ok, <<Len:32, _:16>>} ->
+            {ok, _} = file:position(File, {cur,Len-2}),
+            read_leaf_node(File)
+    end.
+
diff --git a/src/fractal_btree_util.erl b/src/fractal_btree_util.erl
index 0c5cfd5..661432e 100644
--- a/src/fractal_btree_util.erl
+++ b/src/fractal_btree_util.erl
@@ -15,17 +15,19 @@ estimate_node_size_increment(_KVList,Key,Value) ->
               is_binary(Value) ->
                   5 + byte_size(Value);
               is_atom(Value) ->
-                  8
+                  8;
+              is_tuple(Value) ->
+                  13
           end.
 
 encode_index_node(Level, KVList) ->
     Data = %zlib:zip(
-             erlang:term_to_binary({Level, KVList})
+             erlang:term_to_binary(KVList)
            % )
         ,
-    Size = byte_size(Data),
-    {ok, Size+4, [ <<Size:32>> | Data ] }.
+    Size = byte_size(Data)+2,
+    {ok, Size+4, [ <<Size:32/unsigned, Level:16/unsigned>> | Data ] }.
 
-decode_index_node(Data) ->
-    {Level,KVList} = erlang:binary_to_term(Data), %zlib:unzip(Data)),
+decode_index_node(Level, <<Data/binary>>) ->
+    KVList = erlang:binary_to_term(Data), %zlib:unzip(Data)),
     {ok, {node, Level, KVList}}.
diff --git a/src/fractal_btree_writer.erl b/src/fractal_btree_writer.erl
index 06f6be0..d654a06 100644
--- a/src/fractal_btree_writer.erl
+++ b/src/fractal_btree_writer.erl
@@ -22,6 +22,7 @@
                  index_file_pos,
 
                  last_node_pos :: pos_integer(),
+                 last_node_size :: pos_integer(),
 
                  nodes = [] :: [ #node{} ],
 
@@ -97,12 +98,12 @@ code_change(_OldVsn, State, _Extra) ->
 
 
 
-flush_nodes(#state{ nodes=[], last_node_pos=LastNodePos, bloom=Ref }=State) ->
+flush_nodes(#state{ nodes=[], last_node_pos=LastNodePos, last_node_size=LastNodeSize, bloom=Ref }=State) ->
 
     Bloom = zlib:zip(ebloom:serialize(Ref)),
     BloomSize = byte_size(Bloom),
 
-    Trailer = << 0:32, Bloom/binary, BloomSize:32/unsigned, LastNodePos:64/unsigned >>,
+    Trailer = << 0:32, Bloom/binary, BloomSize:32/unsigned,  LastNodePos:64/unsigned >>,
     IdxFile = State#state.index_file,
 
     ok = file:write(IdxFile, Trailer),
@@ -155,7 +156,9 @@ close_node(#state{nodes=[#node{ level=Level, members=NodeMembers }|RestNodes]} =
     ok = file:write(State#state.index_file, Data),
 
     {FirstKey, _} = hd(OrderedMembers),
-    add_record(Level+1, FirstKey, NodePos,
+    add_record(Level+1, FirstKey, {NodePos, DataSize},
                State#state{ nodes          = RestNodes,
                             index_file_pos = NodePos + DataSize,
-                            last_node_pos  = NodePos}).
+                            last_node_pos  = NodePos,
+                            last_node_size = DataSize
+                          }).