Add magic "HAN1" to btree file format

BREAKING CHANGE!  This change provides for future
file format changes, but also breaks backwards
compatibility.

Also describe the file format in design_document
This commit is contained in:
Kresten Krab Thorup 2012-05-08 17:27:05 +02:00
parent f7681da1db
commit 245d815e4c
6 changed files with 11321 additions and 8039 deletions

File diff suppressed because it is too large Load diff

Binary file not shown.

View file

@ -26,6 +26,7 @@
%% smallest levels are 256 entries %% smallest levels are 256 entries
-define(TOP_LEVEL, 8). -define(TOP_LEVEL, 8).
-define(BTREE_SIZE(Level), (1 bsl (Level))). -define(BTREE_SIZE(Level), (1 bsl (Level))).
-define(FIRST_BLOCK_POS, 4).
-define(TOMBSTONE, 'deleted'). -define(TOMBSTONE, 'deleted').

View file

@ -65,6 +65,9 @@ open(Name, Config) ->
{ok, FileInfo} = file:read_file_info(Name), {ok, FileInfo} = file:read_file_info(Name),
%% read and validate magic tag
{ok, <<"HAN1">>} = file:pread(File, 0, 4),
%% read root position %% read root position
{ok, <<RootPos:64/unsigned>>} = file:pread(File, FileInfo#file_info.size-8, 8), {ok, <<RootPos:64/unsigned>>} = file:pread(File, FileInfo#file_info.size-8, 8),
{ok, <<BloomSize:32/unsigned>>} = file:pread(File, FileInfo#file_info.size-12, 4), {ok, <<BloomSize:32/unsigned>>} = file:pread(File, FileInfo#file_info.size-12, 4),
@ -95,7 +98,7 @@ deserialize({seq_read_file, Index, Position}) ->
fold(Fun, Acc0, #index{file=File}) -> fold(Fun, Acc0, #index{file=File}) ->
{ok, Node} = read_node(File,0), {ok, Node} = read_node(File,?FIRST_BLOCK_POS),
fold0(File,fun({K,V},Acc) -> Fun(K,V,Acc) end,Node,Acc0). fold0(File,fun({K,V},Acc) -> Fun(K,V,Acc) end,Node,Acc0).
fold0(File,Fun,#node{level=0,members=List},Acc0) -> fold0(File,Fun,#node{level=0,members=List},Acc0) ->
@ -113,7 +116,7 @@ fold1(File,Fun,Acc0) ->
end. end.
range_fold(Fun, Acc0, #index{file=File,root=Root}, Range) -> range_fold(Fun, Acc0, #index{file=File,root=Root}, Range) ->
case lookup_node(File,Range#key_range.from_key,Root,0) of case lookup_node(File,Range#key_range.from_key,Root,?FIRST_BLOCK_POS) of
{ok, {Pos,_}} -> {ok, {Pos,_}} ->
file:position(File, Pos), file:position(File, Pos),
do_range_fold(Fun, Acc0, File, Range, Range#key_range.limit); do_range_fold(Fun, Acc0, File, Range, Range#key_range.limit);
@ -200,7 +203,7 @@ lookup_node(File,FromKey,#node{members=Members,level=N},_) ->
first_node(#index{file=File}) -> first_node(#index{file=File}) ->
case read_node(File, 0) of case read_node(File, ?FIRST_BLOCK_POS) of
{ok, #node{level=0, members=Members}} -> {ok, #node{level=0, members=Members}} ->
{node, Members} {node, Members}
end. end.

View file

@ -77,7 +77,7 @@ estimate_node_size_increment(_KVList,Key,Value) ->
encode_index_node(KVList, Compress) -> encode_index_node(KVList, Compress) ->
TermData = [ ?CRC_ENCODED | TermData = [ ?TAG_END |
lists:map(fun ({Key,Value}) -> lists:map(fun ({Key,Value}) ->
crc_encapsulate_kv_entry(Key, Value) crc_encapsulate_kv_entry(Key, Value)
end, end,
@ -144,6 +144,9 @@ crc_encapsulate(Blob) ->
Size = erlang:iolist_size(Blob), Size = erlang:iolist_size(Blob),
[ << (Size):32/unsigned, CRC:32/unsigned >>, Blob, ?TAG_END ]. [ << (Size):32/unsigned, CRC:32/unsigned >>, Blob, ?TAG_END ].
decode_kv_list(<<?TAG_END, Custom/binary>>) ->
decode_crc_data(Custom, [], []);
decode_kv_list(<<?ERLANG_ENCODED, _/binary>>=TermData) -> decode_kv_list(<<?ERLANG_ENCODED, _/binary>>=TermData) ->
{ok, erlang:term_to_binary(TermData)}; {ok, erlang:term_to_binary(TermData)};

View file

@ -86,10 +86,11 @@ init([Name,Options]) ->
% io:format("got name: ~p~n", [Name]), % io:format("got name: ~p~n", [Name]),
case do_open(Name, Options, [exclusive]) of case do_open(Name, Options, [exclusive]) of
{ok, IdxFile} -> {ok, IdxFile} ->
file:write(IdxFile, <<"HAN1">>),
{ok, BloomFilter} = ebloom:new(erlang:min(Size,16#ffffffff), 0.01, 123), {ok, BloomFilter} = ebloom:new(erlang:min(Size,16#ffffffff), 0.01, 123),
BlockSize = hanoidb:get_opt(block_size, Options, ?NODE_SIZE), BlockSize = hanoidb:get_opt(block_size, Options, ?NODE_SIZE),
{ok, #state{ name=Name, {ok, #state{ name=Name,
index_file_pos=0, index_file=IdxFile, index_file_pos=?FIRST_BLOCK_POS, index_file=IdxFile,
bloom = BloomFilter, bloom = BloomFilter,
block_size = BlockSize, block_size = BlockSize,
compress = hanoidb:get_opt(compress, Options, none), compress = hanoidb:get_opt(compress, Options, none),