Add magic "HAN1" to btree file format

BREAKING CHANGE!  This change provides for future
file format changes, but also breaks backwards
compatibility.

Also describe the file format in design_document

src/hanoidb.hrl

@@ -26,6 +26,7 @@
 %% smallest levels are 256 entries
 -define(TOP_LEVEL, 8).
 -define(BTREE_SIZE(Level), (1 bsl (Level))).
+-define(FIRST_BLOCK_POS, 4).
 
 -define(TOMBSTONE, 'deleted').
 

src/hanoidb_reader.erl

@@ -65,6 +65,9 @@ open(Name, Config) ->
 
             {ok, FileInfo} = file:read_file_info(Name),
 
+            %% read and validate magic tag
+            {ok, <<"HAN1">>} = file:pread(File, 0, 4),
+
             %% read root position
             {ok, <<RootPos:64/unsigned>>} = file:pread(File, FileInfo#file_info.size-8, 8),
             {ok, <<BloomSize:32/unsigned>>} = file:pread(File, FileInfo#file_info.size-12, 4),
@@ -95,7 +98,7 @@ deserialize({seq_read_file, Index, Position}) ->
 
 
 fold(Fun, Acc0, #index{file=File}) ->
-    {ok, Node} = read_node(File,0),
+    {ok, Node} = read_node(File,?FIRST_BLOCK_POS),
     fold0(File,fun({K,V},Acc) -> Fun(K,V,Acc) end,Node,Acc0).
 
 fold0(File,Fun,#node{level=0,members=List},Acc0) ->
@@ -113,7 +116,7 @@ fold1(File,Fun,Acc0) ->
     end.
 
 range_fold(Fun, Acc0, #index{file=File,root=Root}, Range) ->
-    case lookup_node(File,Range#key_range.from_key,Root,0) of
+    case lookup_node(File,Range#key_range.from_key,Root,?FIRST_BLOCK_POS) of
         {ok, {Pos,_}} ->
             file:position(File, Pos),
             do_range_fold(Fun, Acc0, File, Range, Range#key_range.limit);
@@ -200,7 +203,7 @@ lookup_node(File,FromKey,#node{members=Members,level=N},_) ->
 
 
 first_node(#index{file=File}) ->
-    case read_node(File, 0) of
+    case read_node(File, ?FIRST_BLOCK_POS) of
         {ok, #node{level=0, members=Members}} ->
             {node, Members}
     end.

src/hanoidb_util.erl

@@ -77,7 +77,7 @@ estimate_node_size_increment(_KVList,Key,Value) ->
 
 encode_index_node(KVList, Compress) ->
 
-    TermData = [ ?CRC_ENCODED |
+    TermData = [ ?TAG_END |
                  lists:map(fun ({Key,Value}) ->
                                    crc_encapsulate_kv_entry(Key, Value)
                            end,
@@ -144,6 +144,9 @@ crc_encapsulate(Blob) ->
     Size = erlang:iolist_size(Blob),
     [ << (Size):32/unsigned, CRC:32/unsigned >>, Blob, ?TAG_END ].
 
+decode_kv_list(<<?TAG_END, Custom/binary>>) ->
+    decode_crc_data(Custom, [], []);
+
 decode_kv_list(<<?ERLANG_ENCODED, _/binary>>=TermData) ->
     {ok, erlang:term_to_binary(TermData)};
 

src/hanoidb_writer.erl

@@ -86,10 +86,11 @@ init([Name,Options]) ->
 %    io:format("got name: ~p~n", [Name]),
     case do_open(Name, Options, [exclusive]) of
         {ok, IdxFile} ->
+            file:write(IdxFile, <<"HAN1">>),
             {ok, BloomFilter} = ebloom:new(erlang:min(Size,16#ffffffff), 0.01, 123),
             BlockSize = hanoidb:get_opt(block_size, Options, ?NODE_SIZE),
             {ok, #state{ name=Name,
-                         index_file_pos=0, index_file=IdxFile,
+                         index_file_pos=?FIRST_BLOCK_POS, index_file=IdxFile,
                          bloom = BloomFilter,
                          block_size = BlockSize,
                          compress = hanoidb:get_opt(compress, Options, none),