greg/machi
1
0
Fork 0
Code Issues 11 Pull requests 1 Projects Releases Packages Wiki Activity Actions

Avoid io:format() calls that can consume huge amounts of time

Browse source 
Example CLI session:

    f(Leaves).
    Leaves = [{{Pos*65,65}, begin {A,B,C} = now(), crypto:hash(sha, <<A:32, B:32, C:32>>) end} || Pos <- lists:seq(1,1*1000*1000)].
    timer:tc(fun() -> catch machi_merkle_tree_mgr:build_tree(x, {mt, "/tmp/foo", true, undefined, [], [], [], Leaves}) end).

On my MPB, I see about 550000 - 650000 msec, depending on the run. I've
made no attempt yet to be more rigorous about making measurements.

A result looks like this.  If we ignore the timer:tc() tuple wrapper and
time report, then the output list is [Root, Lvl3, Lvl2, Lvl1].

    Root: <<250,83,19,148,149,195,105,253,231,116,144,49,227,168,224,51,76,63,162,
            70>>
    {551957,
     [<<250,83,19,148,149,195,105,253,231,116,144,49,227,168,
        224,51,76,63,162,70>>,
      [<<102,159,171,226,30,76,65,38,81,168,229,143,240,112,
         208,13,169,216,57,35>>],
      [<<67,189,3,58,14,201,74,206,240,202,163,83,182,232,130,
         14,107,29,84,181>>],
      [<<210,57,128,48,28,28,87,185,240,55,129,158,119,77,172,0,
         183,166,205,157>>,
       <<78,154,155,117,1,220,48,116,74,66,172,146,29,39,7,7,247,
         188,142,72>>,
       <<4,50,223,192,38,141,54,249,168,79,213,90,94,32,91,117,
         36,186,41,139>>,
       <<20,181,105,237,50,97,98,91,178,26,127,20,134,174,146,96,
         133,208,245,...>>,
       <<227,25,30,162,182,226,172,29,132,240,125,248,65,109,82,
         6,116,102,...>>,
       <<229,221,31,166,106,212,120,146,96,229,224,85,237,206,
         160,38,137,...>>,
       <<124,195,225,167,238,245,75,161,71,53,43,154,103,188,
         180,171,...>>]]}
This commit is contained in:
Scott Lystig Fritchie 2015-10-29 12:58:03 +09:00
parent 4480f47ee6
commit f741a8a413
1 changed files with 7 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
src/machi_merkle_tree_mgr.erl
View file

@ -33,6 +33,7 @@
-module(machi_merkle_tree_mgr).
-behaviour(gen_server).
-compile(export_all).
-include("machi.hrl").
@ -301,18 +302,18 @@ find(Tid, Filename) ->
build_tree(Tid, MT = #mt{ leaves = D }) ->
Leaves = lists:map(fun map_dict/1, orddict:to_list(D)),
io:format(user, "Leaves: ~p~n", [Leaves]),
%io:format(user, "Leaves: ~p~n", [Leaves]),
Lvl1s = build_level_1(?CHUNK_SIZE, Leaves, 1, [ crypto:hash_init(?H) ]),
io:format(user, "Lvl1: ~p~n", [Lvl1s]),
%io:format(user, "Lvl1: ~p~n", [Lvl1s]),
Mod2 = length(Lvl1s) div ?LEVEL_SIZE,
Lvl2s = build_int_level(Mod2, Lvl1s, 1, [ crypto:hash_init(?H) ]),
io:format(user, "Lvl2: ~p~n", [Lvl2s]),
%io:format(user, "Lvl2: ~p~n", [Lvl2s]),
Mod3 = length(Lvl2s) div 2,
Lvl3s = build_int_level(Mod3, Lvl2s, 1, [ crypto:hash_init(?H) ]),
io:format(user, "Lvl3: ~p~n", [Lvl3s]),
%io:format(user, "Lvl3: ~p~n", [Lvl3s]),
Root = build_root(Lvl3s, crypto:hash_init(?H)),
io:format(user, "Root: ~p~n", [Root]),
ets:insert(Tid, MT#mt{ root = Root, lvl1 = Lvl1s, lvl2 = Lvl2s, lvl3 = Lvl3s, recalc = false }),
%%% ets:insert(Tid, MT#mt{ root = Root, lvl1 = Lvl1s, lvl2 = Lvl2s, lvl3 = Lvl3s, recalc = false }),
[Root, Lvl3s, Lvl2s, Lvl1s].
build_root([], Ctx) ->
@ -338,6 +339,6 @@ build_level_1(Size, [{Pos, Hash}|T], Multiple, [ Ctx | Rest ]) when Pos > ( Size
build_level_1(Size, T, Multiple+1,
[ crypto:hash_update(NewCtx, Hash), crypto:hash_final(Ctx) | Rest ]);
build_level_1(Size, [{Pos, Hash}|T], Multiple, [ Ctx | Rest ]) when Pos =< ( Size * Multiple ) ->
io:format(user, "Size: ~p, Pos: ~p, Multiple: ~p~n", [Size, Pos, Multiple]),
%io:format(user, "Size: ~p, Pos: ~p, Multiple: ~p~n", [Size, Pos, Multiple]),
build_level_1(Size, T, Multiple, [ crypto:hash_update(Ctx, Hash) | Rest ]).