greg/machi
1
0
Fork 0
Code Issues 11 Pull requests 1 Projects Releases Packages Wiki Activity Actions

Round 1 of cleanup

Browse source
This commit is contained in:
Scott Lystig Fritchie 2015-06-02 18:10:45 +09:00
parent 000d687588
commit 67019493aa
3 changed files with 122 additions and 180 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
TODO-shortterm.org
View file

@ -41,7 +41,7 @@ func, and pattern match Erlang style in that func.
** DONE Add major comment sections to the CR-impl client ** DONE Add major comment sections to the CR-impl client
** DONE Simple basho_bench driver, put some unscientific chalk on the benchtop ** DONE Simple basho_bench driver, put some unscientific chalk on the benchtop
** TODO Create parallel PULSE test for basic API plus chain manager repair ** TODO Create parallel PULSE test for basic API plus chain manager repair
** TODO Add client-side vs. server-side checksum type, expand client API? ** DONE Add client-side vs. server-side checksum type, expand client API?
** TODO Add gproc and get rid of registered name rendezvous ** TODO Add gproc and get rid of registered name rendezvous
*** TODO Fixes the atom table leak *** TODO Fixes the atom table leak
*** TODO Fixes the problem of having active sequencer for the same prefix *** TODO Fixes the problem of having active sequencer for the same prefix

250
test/machi_chain_manager1_converge_demo.erl
View file

@ -199,16 +199,9 @@ convergence_demo_testfun(NumFLUs) ->
S_min, S_max_rand, S_min, S_max_rand,
M_name, All_list), M_name, All_list),
_ = ?MGR:test_react_to_env(MMM), _ = ?MGR:test_react_to_env(MMM),
%% if M_name == d ->
%% [_ = ?MGR:test_react_to_env(MMM) ||
%% _ <- lists:seq(1,3)],
%% superunfair;
%% true ->
%% ok
%% end,
%% Be more unfair by not %% Be more unfair by not
%% sleeping here. %% sleeping here.
%% timer:sleep(S_max - Elapsed), % timer:sleep(S_max - Elapsed),
Elapsed Elapsed
end || _ <- lists:seq(1, Iters)], end || _ <- lists:seq(1, Iters)],
Parent ! done Parent ! done
@ -221,100 +214,26 @@ convergence_demo_testfun(NumFLUs) ->
end || _ <- Pids] end || _ <- Pids]
end, end,
_XandYs1 = [[{X,Y}] || X <- All_list, Y <- All_list, X /= Y],
_XandYs2 = [[{X,Y}, {A,B}] || X <- All_list, Y <- All_list, X /= Y,
A <- All_list, B <- All_list, A /= B,
X /= A],
_XandYs3 = [[{X,Y}, {A,B}, {C,D}] || X <- All_list, Y <- All_list, X /= Y,
A <- All_list, B <- All_list, A /= B,
C <- All_list, D <- All_list, C /= D,
X /= A, X /= C, A /= C],
%% AllPartitionCombinations = _XandYs1 ++ _XandYs2,
%% AllPartitionCombinations = _XandYs3,
AllPartitionCombinations = _XandYs1 ++ _XandYs2 ++ _XandYs3,
?D({?LINE, length(AllPartitionCombinations)}),
machi_partition_simulator:reset_thresholds(10, 50), machi_partition_simulator:reset_thresholds(10, 50),
io:format(user, "\nLet loose the dogs of war!\n", []), io:format(user, "\nLet loose the dogs of war!\n", []),
DoIt(30, 0, 0), DoIt(30, 0, 0),
[begin [begin
%% io:format(user, "\nSET partitions = ~w.\n", [ [] ]),machi_partition_simulator:no_partitions(),
%% [DoIt(50, 10, 100) || _ <- [1,2,3]],
io:format(user, "\nLet loose the dogs of war!\n", []),
DoIt(30, 0, 0),
io:format(user, "\nSET partitions = ~w.\n", [ [] ]),machi_partition_simulator:no_partitions(),
[DoIt(10, 10, 100) || _ <- [1]],
%% machi_partition_simulator:reset_thresholds(10, 50),
%% io:format(user, "\nLet loose the dogs of war!\n", []),
%% DoIt(30, 0, 0),
machi_partition_simulator:always_these_partitions(Partition), machi_partition_simulator:always_these_partitions(Partition),
io:format(user, "\nSET partitions = ~w.\n", [Partition]), io:format(user, "\nSET partitions = ~w.\n", [Partition]),
[DoIt(50, 10, 100) || _ <- [1,2,3,4] ], [DoIt(40, 10, 50) || _ <- [1,2,3,4,5,6] ],
_PPP =
[begin
{ok, PPPallPubs} = ?FLU_PC:list_all_projections(FLU,public),
[begin
{ok, Pr} = todo_why_does_this_crash_sometimes(
FLUName, FLU, PPPepoch),
{Pr#projection_v1.epoch_number, FLUName, Pr}
end || PPPepoch <- PPPallPubs]
end || {FLUName, FLU} <- Namez],
%% io:format(user, "PPP ~p\n", [lists:sort(lists:append(_PPP))]),
%%%%%%%% {stable,true} = {stable,private_projections_are_stable(Namez, DoIt)}, {stable,true} = {stable,private_projections_are_stable(Namez, DoIt)},
{hosed_ok,true} = {hosed_ok,all_hosed_lists_are_identical(Namez, Partition)}, io:format(user, "\nSweet, private projections are stable\n", []),
io:format(user, "\nSweet, all_hosed are identical-or-islands-inconclusive.\n", []), timer:sleep(1250),
timer:sleep(1000),
ok ok
%% end || Partition <- AllPartitionCombinations end || Partition <- make_partition_list(All_list)
%% end || Partition <- [ [{a,b},{b,d},{c,b}],
%% [{a,b},{b,d},{c,b}, {a,b},{b,a},{a,c},{c,a},{a,d},{d,a}],
%% %% [{a,b},{b,d},{c,b}, {b,a},{a,b},{b,c},{c,b},{b,d},{d,b}],
%% [{a,b},{b,d},{c,b}, {c,a},{a,c},{c,b},{b,c},{c,d},{d,c}],
%% [{a,b},{b,d},{c,b}, {d,a},{a,d},{d,b},{b,d},{d,c},{c,d}] ]
%% end || Partition <- [ [{a,b}, {b,c}],
%% [{a,b}, {c,b}] ]
%% end || Partition <- [ [{a,b}, {b,c}] ] %% hosed-not-equal @ 3 FLUs
%% end || Partition <- [ [{b,d}] ]
%% end || Partition <- [ [{a,b}, {b,a}] ]
%% end || Partition <- [ [{a,b}, {b,a}, {a,c},{c,a}] ]
end || Partition <- [ [{a,b}],
[{b,a}] ]
%% end || Partition <- [ [{a,b}, {c,b}],
%% [{a,b}, {b,c}] ]
%% end || Partition <- [ [{a,b}, {b,c}, {c,d}],
%% [{a,b}, {b,c},{b,d}, {c,d}],
%% [{b,a}, {b,c}, {c,d}],
%% [{a,b}, {c,b}, {c,d}],
%% [{a,b}, {b,c}, {d,c}] ]
%% end || Partition <- [ [{a,b}, {b,c}, {c,d}, {d,e}],
%% [{b,a}, {b,c}, {c,d}, {d,e}],
%% [{a,b}, {c,b}, {c,d}, {d,e}],
%% [{a,b}, {b,c}, {d,c}, {d,e}],
%% [{a,b}, {b,c}, {c,d}, {e,d}] ]
%% end || Partition <- [ [{c,a}] ]
%% end || Partition <- [ [{c,a}], [{c,b}, {a, b}] ]
%% end || Partition <- [ [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}],
%% [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}, {b,c}],
%% [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}, {c,d}] ]
%% end || Partition <- [ [{a,b}],
%% [{a,b}, {a,b},{b,a},{a,c},{c,a},{a,d},{d,a}],
%% [{a,b}, {b,a},{a,b},{b,c},{c,b},{b,d},{d,b}],
%% [{a,b}, {c,a},{a,c},{c,b},{b,c},{c,d},{d,c}],
%% [{a,b}, {d,a},{a,d},{d,b},{b,d},{d,c},{c,d}] ]
], ],
%% exit(end_experiment), %% exit(end_experiment),
io:format(user, "\nSET partitions = []\n", []), io:format(user, "\nSET partitions = []\n", []),
io:format(user, "We should see convergence to 1 correct chain.\n", []), io:format(user, "We should see convergence to 1 correct chain.\n", []),
machi_partition_simulator:no_partitions(), machi_partition_simulator:no_partitions(),
[DoIt(50, 10, 100) || _ <- [1]], [DoIt(50, 10, 50) || _ <- [1]],
io:format(user, "Sweet, finishing early\n", []), exit(yoyoyo_testing_hack_finishing_early),
%% WARNING: In asymmetric partitions, private_projections_are_stable()
%% will never be true; code beyond this point on the -exp3
%% branch is bit-rotted, sorry!
true = private_projections_are_stable(Namez, DoIt), true = private_projections_are_stable(Namez, DoIt),
io:format(user, "~s\n", [os:cmd("date")]), io:format(user, "~s\n", [os:cmd("date")]),
@ -333,6 +252,17 @@ convergence_demo_testfun(NumFLUs) ->
%% Given the report, we flip it around so that we observe the %% Given the report, we flip it around so that we observe the
%% sets of chain transitions relative to each FLU. %% sets of chain transitions relative to each FLU.
%% R_Chains's type is: list({RelativeFlu, list({Epoch, UPI, Repairing})})
%% For example:
%% [{a,[{3,[a],[b,c]},
%% {7,[a,b],[c]},
%% {14,[a,b,c],[]},
%% {83,[a],[]},
%% {164,[c],[a,b]},
%% {218,[a],[]},
%% {226,[a],[b,c]},
%% {228,[a],[b,c]},
%% ....
R_Chains = [machi_chain_manager1_test:extract_chains_relative_to_flu( R_Chains = [machi_chain_manager1_test:extract_chains_relative_to_flu(
FLU, Report) || FLU <- All_list], FLU, Report) || FLU <- All_list],
%% ?D(R_Chains), %% ?D(R_Chains),
@ -366,6 +296,71 @@ convergence_demo_testfun(NumFLUs) ->
ok = machi_partition_simulator:stop() ok = machi_partition_simulator:stop()
end. end.
%% Many of the static partition lists below have been problematic at one
%% time or another.....
%%
%% Uncomment *one* of the following make_partition_list() bodies.
make_partition_list(All_list) ->
_X_Ys1 = [[{X,Y}] || X <- All_list, Y <- All_list, X /= Y],
_X_Ys2 = [[{X,Y}, {A,B}] || X <- All_list, Y <- All_list, X /= Y,
A <- All_list, B <- All_list, A /= B,
X /= A],
_X_Ys3 = [[{X,Y}, {A,B}, {C,D}] || X <- All_list, Y <- All_list, X /= Y,
A <- All_list, B <- All_list, A /= B,
C <- All_list, D <- All_list, C /= D,
X /= A, X /= C, A /= C],
%% _X_Ys1 ++ _X_Ys2.
_X_Ys3.
%% _X_Ys1 ++ _X_Ys2 ++ _X_Ys3.
%% [ [{a,b},{b,d},{c,b}],
%% [{a,b},{b,d},{c,b}, {a,b},{b,a},{a,c},{c,a},{a,d},{d,a}],
%% %% [{a,b},{b,d},{c,b}, {b,a},{a,b},{b,c},{c,b},{b,d},{d,b}],
%% [{a,b},{b,d},{c,b}, {c,a},{a,c},{c,b},{b,c},{c,d},{d,c}],
%% [{a,b},{b,d},{c,b}, {d,a},{a,d},{d,b},{b,d},{d,c},{c,d}] ].
%% [ [{a,b}, {b,c}],
%% [{a,b}, {c,b}] ].
%% [{a,b}, {b,c}] ]. %% hosed-not-equal @ 3 FLUs
%% [{b,d}] ].
%% [ [{a,b}, {b,a}] ].
%% [ [{a,b},{b,c},{c,a}],
%% [{a,b}, {b,a}, {a,c},{c,a}] ].
%% [{a,b}, {c,b}],
%% [{a,b}, {b,c}] ].
%% [ [{a,b}, {b,c}, {c,d}],
%% [{a,b}, {b,c},{b,d}, {c,d}],
%% [{b,a}, {b,c}, {c,d}],
%% [{a,b}, {c,b}, {c,d}],
%% [{a,b}, {b,c}, {d,c}] ].
%% [ [{a,b}, {b,c}, {c,d}, {d,e}],
%% [{b,a}, {b,c}, {c,d}, {d,e}],
%% [{a,b}, {c,b}, {c,d}, {d,e}],
%% [{a,b}, {b,c}, {d,c}, {d,e}],
%% [{a,b}, {b,c}, {c,d}, {e,d}] ].
%% [ [{c,a}] ].
%% [ [{c,a}], [{c,b}, {a, b}] ].
%% [ [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}],
%% [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}, {b,c}],
%% [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}, {c,d}] ].
%% [ [{a,b}],
%% [{a,b}, {a,b},{b,a},{a,c},{c,a},{a,d},{d,a}],
%% [{a,b}, {b,a},{a,b},{b,c},{c,b},{b,d},{d,b}],
%% [{a,b}, {c,a},{a,c},{c,b},{b,c},{c,d},{d,c}],
%% [{a,b}, {d,a},{a,d},{d,b},{b,d},{d,c},{c,d}] ].
todo_why_does_this_crash_sometimes(FLUName, FLU, PPPepoch) -> todo_why_does_this_crash_sometimes(FLUName, FLU, PPPepoch) ->
try try
{ok, _}=Res = ?FLU_PC:read_projection(FLU, public, PPPepoch), {ok, _}=Res = ?FLU_PC:read_projection(FLU, public, PPPepoch),
@ -374,75 +369,26 @@ todo_why_does_this_crash_sometimes(FLUName, FLU, PPPepoch) ->
io:format(user, "QQQ Whoa, it crashed this time for ~p at epoch ~p\n", io:format(user, "QQQ Whoa, it crashed this time for ~p at epoch ~p\n",
[FLUName, PPPepoch]), [FLUName, PPPepoch]),
timer:sleep(1000), timer:sleep(1000),
exit(still_a_problem),
?FLU_PC:read_projection(FLU, public, PPPepoch) ?FLU_PC:read_projection(FLU, public, PPPepoch)
end. end.
private_projections_are_stable(Namez, PollFunc) -> private_projections_are_stable(Namez, PollFunc) ->
Private1 = [?FLU_PC:get_latest_epochid(FLU, private) || Private1 = [get_latest_inner_proj_summ(FLU) || {_Name, FLU} <- Namez],
{_Name, FLU} <- Namez],
PollFunc(5, 1, 10), PollFunc(5, 1, 10),
Private2 = [?FLU_PC:get_latest_epochid(FLU, private) || Private2 = [get_latest_inner_proj_summ(FLU) || {_Name, FLU} <- Namez],
{_Name, FLU} <- Namez], if Private1 == Private2 ->
ok;
true ->
io:format(user, "Oops: Private1: ~p\n", [Private1]),
io:format(user, "Oops: Private2: ~p\n", [Private2])
end,
true = (Private1 == Private2). true = (Private1 == Private2).
all_hosed_lists_are_identical(Namez, Partition0) -> get_latest_inner_proj_summ(FLU) ->
Partition = lists:usort(Partition0), {ok, Proj} = ?FLU_PC:read_latest_projection(FLU, private),
Ps = [element(2,?FLU_PC:read_latest_projection(FLU, private)) || #projection_v1{epoch_number=E, upi=UPI, repairing=Repairing, down=Down} =
{_Name, FLU} <- Namez], machi_chain_manager1:inner_projection_or_self(Proj),
UniqueAllHoseds = lists:usort([machi_chain_manager1:get_all_hosed(P) || {E, UPI, Repairing, Down}.
{ok, P} <- Ps]),
Members = [M || {M, _Pid} <- Namez],
Islands = machi_partition_simulator:partitions2num_islands(
Members, Partition),
%% io:format(user, "all_hosed_lists_are_identical:\n", []),
%% io:format(user, " Uniques = ~p Islands ~p\n Partition ~p\n",
%% [Uniques, Islands, Partition]),
case length(UniqueAllHoseds) of
1 ->
true;
%% TODO: With the addition of the digraph stuff below, the clause
%% below probably isn't necessary anymore, since the
%% digraph calculation should catch complete partition islands?
_ when Islands == 'many' ->
%% There are at least two partitions, so yes, it's quite
%% possible that the all_hosed lists may differ.
%% TODO Fix this up to be smarter about fully-isolated
%% islands of partition.
true;
_ ->
DG = digraph:new(),
Connection = machi_partition_simulator:partition2connection(
Members, Partition),
[digraph:add_vertex(DG, X) || X <- Members],
[digraph:add_edge(DG, X, Y) || {X,Y} <- Connection],
Any =
lists:any(
fun(X) ->
NotX = Members -- [X],
lists:any(
fun(Y) ->
%% There must be a shortest path of length
%% two in both directions, otherwise
%% the read projection call will fail.
%% And it's that failure that we're
%% interested in here.
XtoY = digraph:get_short_path(DG, X, Y),
YtoX = digraph:get_short_path(DG, Y, X),
(XtoY == false orelse
length(XtoY) > 2)
orelse
(YtoX == false orelse
length(YtoX) > 2)
end, NotX)
end, Members),
digraph:delete(DG),
if Any == true ->
%% There's a missing path of length 2 between some
%% two FLUs, so yes, there's going to be
%% non-identical all_hosed lists.
true;
true ->
false % There's no excuse, buddy
end
end.
-endif. % TEST -endif. % TEST

50
test/machi_chain_manager1_test.erl
View file

@ -45,6 +45,17 @@
-include_lib("eunit/include/eunit.hrl"). -include_lib("eunit/include/eunit.hrl").
-compile(export_all). -compile(export_all).
%% Example:
%% [{1,{ok_disjoint,[{agreed_membership,{[a],[b,c]}}]}},
%% {3,{ok_disjoint,[{agreed_membership,{[a],[b,c]}}]}},
%% {8,
%% {ok_disjoint,[{not_agreed,{[a],[b,c]},
%% [{b,not_in_this_epoch},
%% <<65,159,66,113,232,15,156,244,197,
%% 210,39,82,229,84,192,19,27,45,161,38>>]}]}},
%% {10,{ok_disjoint,[{agreed_membership,{[c],[]}}]}},
%% ...]
unanimous_report(Namez) -> unanimous_report(Namez) ->
UniquePrivateEs = UniquePrivateEs =
lists:usort(lists:flatten( lists:usort(lists:flatten(
@ -53,10 +64,13 @@ unanimous_report(Namez) ->
[unanimous_report(Epoch, Namez) || Epoch <- UniquePrivateEs]. [unanimous_report(Epoch, Namez) || Epoch <- UniquePrivateEs].
unanimous_report(Epoch, Namez) -> unanimous_report(Epoch, Namez) ->
Projs = [{FLUName, case ?FLU_PC:read_projection(FLU, private, Epoch) of Projs = [{FLUName,
{ok, T} -> T; case ?FLU_PC:read_projection(FLU, private, Epoch) of
_Else -> not_in_this_epoch {ok, T} ->
end} || {FLUName, FLU} <- Namez], machi_chain_manager1:inner_projection_or_self(T);
_Else ->
{FLUName, not_in_this_epoch}
end} || {FLUName, FLU} <- Namez],
UPI_R_Sums = [{Proj#projection_v1.upi, Proj#projection_v1.repairing, UPI_R_Sums = [{Proj#projection_v1.upi, Proj#projection_v1.repairing,
Proj#projection_v1.epoch_csum} || Proj#projection_v1.epoch_csum} ||
{_FLUname, Proj} <- Projs, {_FLUname, Proj} <- Projs,
@ -71,31 +85,12 @@ unanimous_report(Epoch, Namez) ->
%% that all FLUs are in agreement. %% that all FLUs are in agreement.
{UPI, Repairing, _CSum} = {UPI, Repairing, _CSum} =
lists:keyfind(UPI, 1, UPI_R_Sums), lists:keyfind(UPI, 1, UPI_R_Sums),
%% TODO: make certain that this subtlety doesn't get
%% last in later implementations.
%% So, this is a bit of a tricky thing. If we're at
%% upi=[c] and repairing=[a,b], then the transition
%% (eventually!) to upi=[c,a] does not currently depend
%% on b being an active participant in the repair.
%%
%% Yes, b's state is very important for making certain
%% that all repair operations succeed both to a & b.
%% However, in this simulation, we only consider that
%% the head(Repairing) is sane. Therefore, we use only
%% the "HeadOfRepairing" in our considerations here.
HeadOfRepairing = case Repairing of
[H_Rep|_] ->
[H_Rep];
_ ->
[]
end,
Tmp = [{FLU, case proplists:get_value(FLU, Projs) of Tmp = [{FLU, case proplists:get_value(FLU, Projs) of
P when is_record(P, projection_v1) -> P when is_record(P, projection_v1) ->
P#projection_v1.epoch_csum; P#projection_v1.epoch_csum;
Else -> Else ->
Else Else
end} || FLU <- UPI ++ HeadOfRepairing], end} || FLU <- UPI ++ Repairing],
case lists:usort([CSum || {_FLU, CSum} <- Tmp]) of case lists:usort([CSum || {_FLU, CSum} <- Tmp]) of
[_] -> [_] ->
{agreed_membership, {UPI, Repairing}}; {agreed_membership, {UPI, Repairing}};
@ -103,7 +98,7 @@ unanimous_report(Epoch, Namez) ->
{not_agreed, {UPI, Repairing}, Else2} {not_agreed, {UPI, Repairing}, Else2}
end; end;
_Else -> _Else ->
{UPI, not_unique, Epoch, _Else} exit({UPI, not_unique, Epoch, _Else})
end end
end || UPI <- UniqueUPIs], end || UPI <- UniqueUPIs],
AgreedResUPI_Rs = [UPI++Repairing || AgreedResUPI_Rs = [UPI++Repairing ||
@ -128,8 +123,9 @@ extract_chains_relative_to_flu(FLU, Report) ->
lists:member(FLU, UPI) orelse lists:member(FLU, Repairing)]}. lists:member(FLU, UPI) orelse lists:member(FLU, Repairing)]}.
chain_to_projection(MyName, Epoch, UPI_list, Repairing_list, All_list) -> chain_to_projection(MyName, Epoch, UPI_list, Repairing_list, All_list) ->
exit({todo_broken_fixme,?MODULE,?LINE}), MemberDict = orddict:from_list([{FLU, #p_srvr{name=FLU}} ||
machi_projection:new(Epoch, MyName, All_list, FLU <- All_list]),
machi_projection:new(Epoch, MyName, MemberDict,
All_list -- (UPI_list ++ Repairing_list), All_list -- (UPI_list ++ Repairing_list),
UPI_list, Repairing_list, []). UPI_list, Repairing_list, []).