greg/hanoidb
1
0
Fork 0
Code Issues 10 Pull requests 3 Projects Releases Packages Wiki Activity

Implement hibernation in merge processes

Browse source 
Analysis seems to indicate that merge processes
(from high-numbered levels) tend to be activated
quite infrequent. Thus, we term-to-bin/gzip the
merge process state, and invoke explicit gc
before waiting for a {step, …} message again.
This commit is contained in:
Kresten Krab Thorup 2012-04-29 00:17:43 +02:00
parent 801817cf70
commit 15fc05634a
1 changed files with 22 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
src/hanoi_merger.erl
View file

@ -53,17 +53,15 @@ merge(A,B,C, Size, IsLastLevel, Options) ->
{node, AKVs} = hanoi_reader:first_node(BT1), {node, AKVs} = hanoi_reader:first_node(BT1),
{node, BKVs} = hanoi_reader:first_node(BT2), {node, BKVs} = hanoi_reader:first_node(BT2),
{ok, Count, Out2} = scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, 0, {0, none}), scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, 0, {0, none}).
%% finish stream tree terminate(Count, Out) ->
ok = hanoi_reader:close(BT1),
ok = hanoi_reader:close(BT2),
case ?LOCAL_WRITER of case ?LOCAL_WRITER of
true -> true ->
{stop, normal, ok, _} = hanoi_writer:handle_call(close, self(), Out2); {stop, normal, ok, _} = hanoi_writer:handle_call(close, self(), Out);
false -> false ->
ok = hanoi_writer:close(Out2) ok = hanoi_writer:close(Out)
end, end,
{ok, Count}. {ok, Count}.
@ -71,6 +69,16 @@ merge(A,B,C, Size, IsLastLevel, Options) ->
step({N, From}) -> step({N, From}) ->
{N-1, From}. {N-1, From}.
hibernate_scan(Keep) ->
error_logger:info_msg("hibernating ~p~n", [self()]),
erlang:garbage_collect(),
receive
{step, From, HowMany} ->
error_logger:info_msg("waking up ~p~n", [self()]),
{BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count} = erlang:binary_to_term( zlib:gunzip( Keep ) ),
scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count, {HowMany, From})
end.
scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count, {0, FromPID}) -> scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count, {0, FromPID}) ->
case FromPID of case FromPID of
none -> none ->
@ -82,6 +90,10 @@ scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count, {0, FromPID}) ->
receive receive
{step, From, HowMany} -> {step, From, HowMany} ->
scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count, {HowMany, From}) scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count, {HowMany, From})
after 10000 ->
Args = {BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count},
Keep = zlib:gzip ( erlang:term_to_binary( Args ) ),
hibernate_scan(Keep)
end; end;
scan(BT1, BT2, Out, IsLastLevel, [], BKVs, Count, Step) -> scan(BT1, BT2, Out, IsLastLevel, [], BKVs, Count, Step) ->
@ -89,6 +101,7 @@ scan(BT1, BT2, Out, IsLastLevel, [], BKVs, Count, Step) ->
{node, AKVs} -> {node, AKVs} ->
scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count, Step); scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count, Step);
end_of_data -> end_of_data ->
hanoi_reader:close(BT1),
scan_only(BT2, Out, IsLastLevel, BKVs, Count, Step) scan_only(BT2, Out, IsLastLevel, BKVs, Count, Step)
end; end;
@ -97,6 +110,7 @@ scan(BT1, BT2, Out, IsLastLevel, AKVs, [], Count, Step) ->
{node, BKVs} -> {node, BKVs} ->
scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count, Step); scan(BT1, BT2, Out, IsLastLevel, AKVs, BKVs, Count, Step);
end_of_data -> end_of_data ->
hanoi_reader:close(BT2),
scan_only(BT1, Out, IsLastLevel, AKVs, Count, Step) scan_only(BT1, Out, IsLastLevel, AKVs, Count, Step)
end; end;
@ -144,7 +158,8 @@ scan_only(BT, Out, IsLastLevel, [], Count, {_, FromPID}=Step) ->
{PID, Ref} -> {PID, Ref} ->
PID ! {Ref, step_done} PID ! {Ref, step_done}
end, end,
{ok, Count, Out} hanoi_reader:close(BT),
terminate(Count, Out)
end; end;
scan_only(BT, Out, true, [{_,?TOMBSTONE}|Rest], Count, Step) -> scan_only(BT, Out, true, [{_,?TOMBSTONE}|Rest], Count, Step) ->