From 064b637d81e81856d184b7fc1c1834f9b49bcaa6 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 31 Oct 2014 20:10:56 +0900 Subject: [PATCH] Remove docs/machi/flowchart-machi-chain-mgmt1.jpg --- .../poc-machi/src/machi_chain_manager.erl | 191 ------- .../test/machi_chain_manager_test.erl | 468 ------------------ .../test/machi_chain_manager_test0.erl | 468 ------------------ 3 files changed, 1127 deletions(-) delete mode 100644 prototype/poc-machi/src/machi_chain_manager.erl delete mode 100644 prototype/poc-machi/test/machi_chain_manager_test.erl delete mode 100644 prototype/poc-machi/test/machi_chain_manager_test0.erl diff --git a/prototype/poc-machi/src/machi_chain_manager.erl b/prototype/poc-machi/src/machi_chain_manager.erl deleted file mode 100644 index 99c2362..0000000 --- a/prototype/poc-machi/src/machi_chain_manager.erl +++ /dev/null @@ -1,191 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Machi: a small village of replicated files -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- --module(machi_chain_manager). - --include("machi.hrl"). - --export([make_initial_state/3, calc_projection/3, make_projection_summary/1]). - --ifdef(TEST). --compile(export_all). --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). --endif. --ifdef(PULSE). --compile({parse_transform, pulse_instrument}). --endif. - --endif. %TEST - -make_initial_state(MyName, All_list, Seed) -> - RunEnv = [{seed, Seed}, - {network_partitions, []}], - #ch_mgr{name=MyName, - proj=make_initial_projection(MyName, All_list, All_list, [], []), - runenv=RunEnv}. - -make_initial_projection(MyName, All_list, UPI_list, Repairing_list, Ps) -> - make_projection(1, 0, <<>>, - MyName, All_list, [], UPI_list, Repairing_list, Ps). - -make_projection(EpochNum, PrevEpochNum, PrevEpochCSum, - MyName, All_list, Down_list, UPI_list, Repairing_list, Ps) -> - P = #projection{epoch_number=EpochNum, - epoch_csum= <<>>, - prev_epoch_num=PrevEpochNum, - prev_epoch_csum=PrevEpochCSum, - creation_time=now(), - author_server=MyName, - all_members=All_list, - down=Down_list, - upi=UPI_list, - repairing=Repairing_list, - dbg=Ps}, - CSum = crypto:hash(sha, term_to_binary(P)), - P#projection{epoch_csum=CSum}. - -%% OldThreshold: Percent chance of using the old/previous network partition list -%% NoPartitionThreshold: If the network partition changes, what are the odds -%% that there are no partitions at all? - -calc_projection(OldThreshold, NoPartitionThreshold, - #ch_mgr{name=MyName, proj=LastProj, runenv=RunEnv1} = S) -> - #projection{epoch_number=OldEpochNum, - epoch_csum=OldEpochCsum, - all_members=All_list, - upi=OldUPI_list, - repairing=OldRepairing_list - } = LastProj, - LastUp = lists:usort(OldUPI_list ++ OldRepairing_list), - AllMembers = (S#ch_mgr.proj)#projection.all_members, - {Up, RunEnv2} = calc_up_nodes(MyName, OldThreshold, NoPartitionThreshold, - AllMembers, RunEnv1), - NewUp = Up -- LastUp, - Down = AllMembers -- Up, - - NewUPI_list = [X || X <- OldUPI_list, lists:member(X, Up)], - Repairing_list2 = [X || X <- OldRepairing_list, lists:member(X, Up)], - {NewUPI_list3, Repairing_list3, RunEnv3} = - case {NewUp, Repairing_list2} of - {[], []} -> - {NewUPI_list, [], RunEnv2}; - {[], [H|T]} -> - {Prob, RunEnvX} = roll_dice(100, RunEnv2), - if Prob =< 50 -> - {NewUPI_list ++ [H], T, RunEnvX}; - true -> - {NewUPI_list, OldRepairing_list, RunEnvX} - end; - {_, _} -> - {NewUPI_list, OldRepairing_list, RunEnv2} - end, - Repairing_list4 = case NewUp of - [] -> Repairing_list3; - NewUp -> Repairing_list3 ++ NewUp - end, - Repairing_list5 = Repairing_list4 -- Down, - P = make_projection(OldEpochNum + 1, OldEpochNum, OldEpochCsum, - MyName, All_list, Down, NewUPI_list3, Repairing_list5, - [goo]), - {P, S#ch_mgr{runenv=RunEnv3}}. - -calc_up_nodes(MyName, OldThreshold, NoPartitionThreshold, - AllMembers, RunEnv1) -> - Seed1 = proplists:get_value(seed, RunEnv1), - Partitions1 = proplists:get_value(network_partitions, RunEnv1), - {Seed2, Partitions2} = - calc_network_partitions(AllMembers, Seed1, Partitions1, - OldThreshold, NoPartitionThreshold), - UpNodes = lists:sort( - [Node || Node <- AllMembers, - not lists:member({MyName, Node}, Partitions2), - not lists:member({Node, MyName}, Partitions2)]), - RunEnv2 = replace(RunEnv1, - [{seed, Seed2}, {network_partitions, Partitions2}]), - {UpNodes, RunEnv2}. - - -calc_network_partitions(Nodes, Seed1, OldPartition, - OldThreshold, NoPartitionThreshold) -> - {Cutoff2, Seed2} = random:uniform_s(100, Seed1), - if Cutoff2 < OldThreshold -> - {Seed2, OldPartition}; - true -> - {Cutoff3, Seed3} = random:uniform_s(100, Seed1), - if Cutoff3 < NoPartitionThreshold -> - {Seed3, []}; - %% Cutoff3 rem 10 < 5 -> - %% %% case get(goofus) of undefined -> put(goofus, true), io:format(user, "~w", [Cutoff3 rem 10]); _ -> ok end, - %% OldSeed = case random:seed(Seed3) of undefined -> now(); - %% Else -> Else - %% end, - %% {Down, _} = lists:partition(fun(X) -> X /= 'a' andalso random:uniform() < 0.5 end, Nodes), - %% %% case get(goofus) of undefined -> put(goofus, true), io:format(user, "~w", [Down]); _ -> ok end, - %% Partitions = [{X, Y} || X <- Nodes, Y <- Down], - %% random:seed(OldSeed), - %% {Seed3, Partitions}; - true -> - make_network_partition_locations(Nodes, Seed3) - end - end. - -replace(PropList, Items) -> - lists:foldl(fun({Key, Val}, Ps) -> - lists:keyreplace(Key, 1, Ps, {Key,Val}) - end, PropList, Items). - -make_projection_summary(#projection{epoch_number=EpochNum, - all_members=_All_list, - down=Down_list, - upi=UPI_list, - repairing=Repairing_list}) -> - [{epoch,EpochNum}, - {upi,UPI_list},{repair,Repairing_list},{down,Down_list}]. - -roll_dice(N, RunEnv) -> - Seed1 = proplists:get_value(seed, RunEnv), - {Val, Seed2} = random:uniform_s(N, Seed1), - {Val, replace(RunEnv, [{seed, Seed2}])}. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -make_network_partition_locations(Nodes, Seed1) -> - Pairs = make_all_pairs(Nodes), - Num = length(Pairs), - {Seed2, Weights} = lists:foldl( - fun(_, {Seeda, Acc}) -> - {Cutoff, Seedb} = random:uniform_s(100, Seeda), - {Seedb, [Cutoff|Acc]} - end, {Seed1, []}, lists:seq(1, Num)), - {Cutoff3, Seed3} = random:uniform_s(100, Seed2), - {Seed3, [X || {Weight, X} <- lists:zip(Weights, Pairs), - Weight < Cutoff3]}. - -make_all_pairs(L) -> - lists:flatten(make_all_pairs2(lists:usort(L))). - -make_all_pairs2([]) -> - []; -make_all_pairs2([_]) -> - []; -make_all_pairs2([H1|T]) -> - [[{H1, X}, {X, H1}] || X <- T] ++ make_all_pairs(T). diff --git a/prototype/poc-machi/test/machi_chain_manager_test.erl b/prototype/poc-machi/test/machi_chain_manager_test.erl deleted file mode 100644 index 3539d7f..0000000 --- a/prototype/poc-machi/test/machi_chain_manager_test.erl +++ /dev/null @@ -1,468 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Machi: a small village of replicated files -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- --module(machi_chain_manager_test). - --include("machi.hrl"). - --define(MGR, machi_chain_manager). --define(D(X), io:format(user, "~s ~p\n", [??X, X])). - --export([]). - --ifdef(TEST). - --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). -%% -include_lib("eqc/include/eqc_statem.hrl"). --define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). --endif. - --include_lib("eunit/include/eunit.hrl"). --compile(export_all). - -smoke0_test() -> - S0 = ?MGR:make_initial_state(a, [a,b,c,d], {4,5,6}), - lists:foldl(fun(_, S) -> - {P1, S1} = ?MGR:calc_projection(20, 1, S), - %% io:format(user, "~p\n", [?MGR:make_projection_summary(P1)]), - S1#ch_mgr{proj=P1} - end, S0, lists:seq(1,10)). - --ifdef(CRUFT_DELETE_ME_MAYBE). - --record(s, { - step = 0 :: non_neg_integer(), - seed :: {integer(), integer(), integer()} - }). - -gen_all_nodes() -> - [a, b, c]. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -initial_state() -> - #s{}. - -command(#s{step=0}) -> - L = gen_all_nodes(), - {call, ?MODULE, init_run, [gen_rand_seed(), L]}; -command(_S) -> - foo. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -prop_m() -> - ?FORALL({_Cmds, _PulseSeed}, {commands(?MODULE), pulse:seed()}, - begin - true - end). - --endif. % CRUFT_DELETE_ME_MAYBE - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -gen_rand_seed() -> - noshrink({gen_num(), gen_num(), gen_num()}). - -gen_num() -> - choose(1, 50000). - -prop_calc_projection() -> - prop_calc_projection([a,b,c], false). - -prop_calc_projection(Nodes, TrackUseTab) -> - ?FORALL( - {Seed, OldThreshold, NoPartitionThreshold, Steps, HeadNode}, - {gen_rand_seed(), - oneof([0,15,35,55,75,85,100]), - oneof([0,15,35,55,75,85,100]), - 5000, - oneof(Nodes)}, - begin - erase(goofus), - {RandN, _} = random:uniform_s(length(Nodes), Seed), - NodesShuffle = lists:nth(RandN, perms(Nodes)), - S0 = ?MGR:make_initial_state(HeadNode, NodesShuffle, Seed), - F = fun(_, {S, Acc}) -> - {P1, S1} = ?MGR:calc_projection( - OldThreshold, NoPartitionThreshold, S), - %%io:format(user, "~p\n", [make_projection_summary(P1)]), - {S1#ch_mgr{proj=P1}, [P1|Acc]} - end, - {_, Projs0} = lists:foldl(F, {S0, []}, lists:seq(1,Steps)), - Projs = lists:reverse(Projs0), - if TrackUseTab == false -> - ok; - true -> - Transitions = extract_upi_transitions(Projs), - %% io:format(user, "\n~P\n", [lists:usort(Transitions), 10]), - [ets:update_counter(TrackUseTab, Trans, 1) || - Trans <- Transitions] - end, - true = projection_transitions_are_sane(Projs) - end). - -extract_upi_transitions([]) -> - []; -extract_upi_transitions([_]) -> - []; -extract_upi_transitions([P1, P2|T]) -> - Trans = {P1#projection.upi, P2#projection.upi}, - [Trans|extract_upi_transitions([P2|T])]. - -find_common_prefix([], _) -> - []; -find_common_prefix(_, []) -> - []; -find_common_prefix([H|L1], [H|L2]) -> - [H|find_common_prefix(L1, L2)]; -find_common_prefix(_, _) -> - []. - -calc_projection_test_() -> - Runtime = 15, %% Runtime = 60*60, - {timeout, Runtime * 500, - fun() -> - Nodes = [a,b,c], - Cs = combinations(Nodes), - Combos = lists:sort([{UPI1, UPI2} || UPI1 <- Cs, UPI2 <- Cs]), - put(hack, 0), - timer:sleep(500), - io:format(user, "\n", []), - Rs = [begin - Hack = get(hack),if Hack rem 10000 == 0 -> ?D({time(), Hack}); true -> ok end,put(hack, Hack + 1), - P1 = ?MGR:make_projection(2, 1, <<>>, HdNd, Nodes, - [], PsUPI1, Nodes -- PsUPI1, []), - P2 = ?MGR:make_projection(3, 2, <<>>, HdNd, Nodes, - [], PsUPI2, Nodes -- PsUPI2, []), - Res = case projection_transition_is_sane(P1, P2) of - true -> true; - _ -> false - end, - {Res, PsUPI1, PsUPI2} - end || HdNd <- Nodes, - {PsUPI1,PsUPI2} <- Combos, - %% We assume that the author appears in any UPI list - PsUPI1 /= [], - PsUPI2 /= [], - %% HdNd is the author for all of these - %% tests, so it must be present in UPI1 & UPI2 - lists:member(HdNd, PsUPI1), - lists:member(HdNd, PsUPI2)], - OKs = [begin - {UPI1,UPI2} - end || {true, UPI1, UPI2} <- Rs], - %% io:format(user, "OKs = ~p\n", [lists:usort(OKs)]), - Tab = ets:new(count, [public, set, {keypos, 1}]), - [ets:insert(Tab, {Transition, 0}) || Transition <- OKs], - - true = eqc:quickcheck( - eqc:testing_time(Runtime, - ?QC_OUT(prop_calc_projection(Nodes, Tab)))), - - NotCounted = lists:sort([Transition || {Transition, 0} <- ets:tab2list(Tab)]), - Counted = [X || {_, N}=X <- ets:tab2list(Tab), - N > 0], - CountedX = lists:sort(fun(X, Y) -> - element(2, X) < element(2, Y) - end, Counted), - timer:sleep(100), - File = io_lib:format("/tmp/manager-test.~w.~w.~w", - tuple_to_list(time())), - {ok, FH} = file:open(File, [write]), - io:format(FH, "% NotCounted =\n~p.\n", [NotCounted]), - file:close(FH), - %% io:format(user, "\tNotCounted = ~p\n", [NotCounted]), - io:format(user, "\n\tNotCounted list was written to ~s\n", [File]), - io:format(user, "\tOKs length = ~p\n", [length(OKs)]), - io:format(user, "\tTransitions hit = ~p\n", - [length(OKs) - length(NotCounted)]), - %% io:format(user, "\tTransitions = ~p\n", [lists:sort(_Counted)]), - io:format(user, "\tNotCounted length = ~p\n", - [length(NotCounted)]), - io:format(user, "\tLeast-counted transition = ~p\n", - [hd(CountedX)]), - io:format(user, "\tMost-counted transition = ~p\n", - [lists:last(CountedX)]), - ok - end}. - -projection_transitions_are_sane([]) -> - true; -projection_transitions_are_sane([_]) -> - true; -projection_transitions_are_sane([P1, P2|T]) -> - case projection_transition_is_sane(P1, P2) of - true -> - projection_transitions_are_sane([P2|T]); - Else -> - Else - end. - -projection_transition_is_sane( - #projection{epoch_number=Epoch1, - epoch_csum=CSum1, - prev_epoch_num=PrevEpoch1, - prev_epoch_csum=PrevCSum1, - creation_time=CreationTime1, - author_server=AuthorServer1, - all_members=All_list1, - down=Down_list1, - upi=UPI_list1, - repairing=Repairing_list1, - dbg=Dbg1} = P1, - #projection{epoch_number=Epoch2, - epoch_csum=CSum2, - prev_epoch_num=PrevEpoch2, - prev_epoch_csum=PrevCSum2, - creation_time=CreationTime2, - author_server=AuthorServer2, - all_members=All_list2, - down=Down_list2, - upi=UPI_list2, - repairing=Repairing_list2, - dbg=Dbg2} = P2) -> - try - true = is_integer(Epoch1) andalso is_integer(Epoch2), - true = is_binary(CSum1) andalso is_binary(CSum2), - true = is_integer(PrevEpoch1) andalso is_integer(PrevEpoch2), - true = is_binary(PrevCSum1) andalso is_binary(PrevCSum2), - {_,_,_} = CreationTime1, - {_,_,_} = CreationTime2, - true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2), % todo will probably change - true = is_list(All_list1) andalso is_list(All_list2), - true = is_list(Down_list1) andalso is_list(Down_list2), - true = is_list(UPI_list1) andalso is_list(UPI_list2), - true = is_list(Repairing_list1) andalso is_list(Repairing_list2), - true = is_list(Dbg1) andalso is_list(Dbg2), - - true = Epoch2 > Epoch1, - true = PrevEpoch2 > PrevEpoch1, - All_list1 = All_list2, % todo will probably change - - %% No duplicates - true = lists:sort(Down_list2) == lists:usort(Down_list2), - true = lists:sort(UPI_list2) == lists:usort(UPI_list2), - true = lists:sort(Repairing_list2) == lists:usort(Repairing_list2), - - %% Disjoint-ness - true = lists:sort(All_list2) == lists:sort(Down_list2 ++ UPI_list2 ++ - Repairing_list2), - [] = [X || X <- Down_list2, not lists:member(X, All_list2)], - [] = [X || X <- UPI_list2, not lists:member(X, All_list2)], - [] = [X || X <- Repairing_list2, not lists:member(X, All_list2)], - DownS2 = sets:from_list(Down_list2), - UPIS2 = sets:from_list(UPI_list2), - RepairingS2 = sets:from_list(Repairing_list2), - true = sets:is_disjoint(DownS2, UPIS2), - true = sets:is_disjoint(DownS2, RepairingS2), - true = sets:is_disjoint(UPIS2, RepairingS2), - - %% The author must not be down. - false = lists:member(AuthorServer1, Down_list1), - false = lists:member(AuthorServer2, Down_list2), - %% The author must be in either the UPI or repairing list. - true = lists:member(AuthorServer1, UPI_list1 ++ Repairing_list1), - true = lists:member(AuthorServer2, UPI_list2 ++ Repairing_list2), - - %% Additions to the UPI chain may only be at the tail - UPI_common_prefix = find_common_prefix(UPI_list1, UPI_list2), - if UPI_common_prefix == [] -> - if UPI_list1 == [] orelse UPI_list2 == [] -> - %% If the common prefix is empty, then one of the - %% inputs must be empty. - true; - true -> - %% Otherwise, we have a case of UPI changing from - %% one of these two situations: - %% - %% UPI_list1 -> UPI_list2 - %% [d,c,b,a] -> [c,a] - %% [d,c,b,a] -> [c,a,repair_finished_added_to_tail]. - NotUPI2 = (Down_list2 ++ Repairing_list2), - true = lists:prefix(UPI_list1 -- NotUPI2, - UPI_list2) - end; - true -> - true - end, - true = lists:prefix(UPI_common_prefix, UPI_list1), - true = lists:prefix(UPI_common_prefix, UPI_list2), - UPI_1_suffix = UPI_list1 -- UPI_common_prefix, - UPI_2_suffix = UPI_list2 -- UPI_common_prefix, - - %% Where did elements in UPI_2_suffix come from? - %% Only two sources are permitted. - [true = lists:member(X, Repairing_list1) % X added after repair done - orelse - lists:member(X, UPI_list1) % X in UPI_list1 after common pref - || X <- UPI_2_suffix], - - %% The UPI_2_suffix must exactly be equal to: ordered items from - %% UPI_list1 concat'ed with ordered items from Repairing_list1. - %% Both temp vars below preserve relative order! - UPI_2_suffix_from_UPI1 = [X || X <- UPI_1_suffix, - lists:member(X, UPI_list2)], - UPI_2_suffix_from_Repairing1 = [X || X <- UPI_2_suffix, - lists:member(X, Repairing_list1)], - %% true? - UPI_2_suffix = UPI_2_suffix_from_UPI1 ++ UPI_2_suffix_from_Repairing1, - - true - catch - _Type:_Err -> - S1 = ?MGR:make_projection_summary(P1), - S2 = ?MGR:make_projection_summary(P2), - Trace = erlang:get_stacktrace(), - {err, from, S1, to, S2, stack, Trace} - end. - -pass1_smoke_test() -> - P2 = ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d], - [], [d,c,b,a], [], []), - P3 = ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d], - [b,c,d], [a], [], []), - true = projection_transition_is_sane(P2, P3), - - P2b= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d], - [], [d,c,b,a], [], []), - P3b= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d], - [b,d], [c,a], [], []), - true = projection_transition_is_sane(P2b, P3b), - - P2c= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d,e], - [], [e,d,c,b], [a], []), - P3c= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d,e], - [e,c], [d,b,a], [], []), - true = projection_transition_is_sane(P2c, P3c), - - true. - -fail1_smoke_test() -> - P15 = ?MGR:make_projection(15, 14, <<>>, a, [a,b,c,d], - [b,d], [a], [c], []), - P16 = ?MGR:make_projection(16, 15, <<>>, a, [a,b,c,d], - [b,d], [c,a], [], []), - true = projection_transition_is_sane(P15, P16) /= true, - - P2d= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d,e], - [], [e,d,c,b], [a], []), - P3d= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d,e], - [e,c], [d,a,b], [], []), - true = projection_transition_is_sane(P2d, P3d) /= true, - - P2e= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d,e], - [], [e,d,c,b], [a], []), - P3e= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d,e], - [e,c], [a,d,b], [], []), - true = projection_transition_is_sane(P2e, P3e) /= true, - - true. - -fail2_smoke_test() -> - AB = [a,b], - BadSets = [{1, [b,a],[a,b]}, - {2, [b],[a,b]}, - %% {3, [a],[b]}, % weird but valid: [a] means b is repairing; - % so if a fails and b finishes repairing, - % swapping both at the same time is "ok". - {4, [a,b],[b,a]}, - %% {5, [b],[a]}, % weird but valid: see above - {6, [a],[b,a]} - ], - [begin - P2f= ?MGR:make_projection(2, 1, <<>>, a, AB, - [], Two, AB -- Two, []), - P3f= ?MGR:make_projection(3, 2, <<>>, a, AB, - [], Three, AB -- Three, []), - {Label, true} = - {Label, (projection_transition_is_sane(P2f, P3f) /= true)} - end || {Label, Two, Three} <- BadSets], - - true. - -fail3_smoke_test() -> - Nodes = [a,b], - UPI1 = [a,b], - UPI2 = [b,a], - [begin - PsUPI1 = UPI1, - PsUPI2 = UPI2, - P1 = ?MGR:make_projection(2, 1, <<>>, HdNd, Nodes, - [], PsUPI1, Nodes -- PsUPI1, []), - P2 = ?MGR:make_projection(3, 2, <<>>, HdNd, Nodes, - [], PsUPI2, Nodes -- PsUPI2, []), - Res = case projection_transition_is_sane(P1, P2) of - true -> true; - _ -> false - end, - {Res, HdNd, PsUPI1, PsUPI2} = {false, HdNd, PsUPI1, PsUPI2} - end || HdNd <- Nodes], - - true. - -fail4_smoke_test() -> - Nodes = [a,b,c], - Two = [c,b,a], - Three = [c,a,b], - P2a= ?MGR:make_projection(2, 1, <<>>, a, Nodes, - [], Two, Nodes -- Two, []), - P3a= ?MGR:make_projection(3, 2, <<>>, a, Nodes, - [], Three, Nodes -- Three, []), - true = (projection_transition_is_sane(P2a, P3a) /= true), - - true. - - -%% aaa_smoke_test() -> -%% L = [a,b,c,d], -%% Cs = combinations(L), -%% Combos = lists:sort([{X, Y} || X <- Cs, Y <- Cs]), -%% timer:sleep(500), -%% io:format(user, "\n", []), -%% Rs = [begin -%% P1 = ?MGR:make_projection(2, 1, <<>>, a, L, -%% [], X, L -- X, []), -%% P2 = ?MGR:make_projection(3, 2, <<>>, a, L, -%% [], Y, L -- Y, []), -%% Res = case projection_transition_is_sane(P1, P2) of -%% true -> true; -%% _ -> false -%% end, -%% {Res, P1, P2} -%% end || {X,Y} <- Combos], -%% OKs = [{X,Y} || {true, X, Y} <- Rs], -%% io:format(user, "Cs are ~p\n", [length(Cs)]), -%% io:format(user, "OKs are ~p\n", [length(OKs)]), -%% Bads = [{X,Y} || {false, X, Y} <- Rs], -%% io:format(user, "Bads are ~p\n", [length(Bads)]), -%% %% [io:format(user, "~p -> ~p: ~p\n", [X, Y, Res]); -%% ok. - -combinations(L) -> - lists:usort(perms(L) ++ lists:append([ combinations(L -- [X]) || X <- L])). - -perms([]) -> [[]]; -perms(L) -> [[H|T] || H <- L, T <- perms(L--[H])]. - --endif. diff --git a/prototype/poc-machi/test/machi_chain_manager_test0.erl b/prototype/poc-machi/test/machi_chain_manager_test0.erl deleted file mode 100644 index f5e243f..0000000 --- a/prototype/poc-machi/test/machi_chain_manager_test0.erl +++ /dev/null @@ -1,468 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Machi: a small village of replicated files -%% -%% Copyright (c) 2014 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- --module(machi_chain_manager_test0). - --include("machi.hrl"). - --define(MGR, machi_chain_manager0). --define(D(X), io:format(user, "~s ~p\n", [??X, X])). - --export([]). - --ifdef(TEST). - --ifdef(EQC). --include_lib("eqc/include/eqc.hrl"). -%% -include_lib("eqc/include/eqc_statem.hrl"). --define(QC_OUT(P), - eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). --endif. - --include_lib("eunit/include/eunit.hrl"). --compile(export_all). - -smoke0_test() -> - S0 = ?MGR:make_initial_state(a, [a,b,c,d], {4,5,6}), - lists:foldl(fun(_, S) -> - {P1, S1} = ?MGR:calc_projection(20, 1, S), - %% io:format(user, "~p\n", [?MGR:make_projection_summary(P1)]), - S1#ch_mgr{proj=P1} - end, S0, lists:seq(1,10)). - --ifdef(CRUFT_DELETE_ME_MAYBE). - --record(s, { - step = 0 :: non_neg_integer(), - seed :: {integer(), integer(), integer()} - }). - -gen_all_nodes() -> - [a, b, c]. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -initial_state() -> - #s{}. - -command(#s{step=0}) -> - L = gen_all_nodes(), - {call, ?MODULE, init_run, [gen_rand_seed(), L]}; -command(_S) -> - foo. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -prop_m() -> - ?FORALL({_Cmds, _PulseSeed}, {commands(?MODULE), pulse:seed()}, - begin - true - end). - --endif. % CRUFT_DELETE_ME_MAYBE - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -gen_rand_seed() -> - noshrink({gen_num(), gen_num(), gen_num()}). - -gen_num() -> - choose(1, 50000). - -prop_calc_projection() -> - prop_calc_projection([a,b,c], false). - -prop_calc_projection(Nodes, TrackUseTab) -> - ?FORALL( - {Seed, OldThreshold, NoPartitionThreshold, Steps, HeadNode}, - {gen_rand_seed(), - oneof([0,15,35,55,75,85,100]), - oneof([0,15,35,55,75,85,100]), - 5000, - oneof(Nodes)}, - begin - erase(goofus), - {RandN, _} = random:uniform_s(length(Nodes), Seed), - NodesShuffle = lists:nth(RandN, perms(Nodes)), - S0 = ?MGR:make_initial_state(HeadNode, NodesShuffle, Seed), - F = fun(_, {S, Acc}) -> - {P1, S1} = ?MGR:calc_projection( - OldThreshold, NoPartitionThreshold, S), - %%io:format(user, "~p\n", [make_projection_summary(P1)]), - {S1#ch_mgr{proj=P1}, [P1|Acc]} - end, - {_, Projs0} = lists:foldl(F, {S0, []}, lists:seq(1,Steps)), - Projs = lists:reverse(Projs0), - if TrackUseTab == false -> - ok; - true -> - Transitions = extract_upi_transitions(Projs), - %% io:format(user, "\n~P\n", [lists:usort(Transitions), 10]), - [ets:update_counter(TrackUseTab, Trans, 1) || - Trans <- Transitions] - end, - true = projection_transitions_are_sane(Projs) - end). - -extract_upi_transitions([]) -> - []; -extract_upi_transitions([_]) -> - []; -extract_upi_transitions([P1, P2|T]) -> - Trans = {P1#projection.upi, P2#projection.upi}, - [Trans|extract_upi_transitions([P2|T])]. - -find_common_prefix([], _) -> - []; -find_common_prefix(_, []) -> - []; -find_common_prefix([H|L1], [H|L2]) -> - [H|find_common_prefix(L1, L2)]; -find_common_prefix(_, _) -> - []. - -calc_projection_test_() -> - Runtime = 15, %% Runtime = 60*60, - {timeout, Runtime * 500, - fun() -> - Nodes = [a,b,c], - Cs = combinations(Nodes), - Combos = lists:sort([{UPI1, UPI2} || UPI1 <- Cs, UPI2 <- Cs]), - put(hack, 0), - timer:sleep(500), - io:format(user, "\n", []), - Rs = [begin - Hack = get(hack),if Hack rem 10000 == 0 -> ?D({time(), Hack}); true -> ok end,put(hack, Hack + 1), - P1 = ?MGR:make_projection(2, 1, <<>>, HdNd, Nodes, - [], PsUPI1, Nodes -- PsUPI1, []), - P2 = ?MGR:make_projection(3, 2, <<>>, HdNd, Nodes, - [], PsUPI2, Nodes -- PsUPI2, []), - Res = case projection_transition_is_sane(P1, P2) of - true -> true; - _ -> false - end, - {Res, PsUPI1, PsUPI2} - end || HdNd <- Nodes, - {PsUPI1,PsUPI2} <- Combos, - %% We assume that the author appears in any UPI list - PsUPI1 /= [], - PsUPI2 /= [], - %% HdNd is the author for all of these - %% tests, so it must be present in UPI1 & UPI2 - lists:member(HdNd, PsUPI1), - lists:member(HdNd, PsUPI2)], - OKs = [begin - {UPI1,UPI2} - end || {true, UPI1, UPI2} <- Rs], - %% io:format(user, "OKs = ~p\n", [lists:usort(OKs)]), - Tab = ets:new(count, [public, set, {keypos, 1}]), - [ets:insert(Tab, {Transition, 0}) || Transition <- OKs], - - true = eqc:quickcheck( - eqc:testing_time(Runtime, - ?QC_OUT(prop_calc_projection(Nodes, Tab)))), - - NotCounted = lists:sort([Transition || {Transition, 0} <- ets:tab2list(Tab)]), - Counted = [X || {_, N}=X <- ets:tab2list(Tab), - N > 0], - CountedX = lists:sort(fun(X, Y) -> - element(2, X) < element(2, Y) - end, Counted), - timer:sleep(100), - File = io_lib:format("/tmp/manager-test.~w.~w.~w", - tuple_to_list(time())), - {ok, FH} = file:open(File, [write]), - io:format(FH, "% NotCounted =\n~p.\n", [NotCounted]), - file:close(FH), - %% io:format(user, "\tNotCounted = ~p\n", [NotCounted]), - io:format(user, "\n\tNotCounted list was written to ~s\n", [File]), - io:format(user, "\tOKs length = ~p\n", [length(OKs)]), - io:format(user, "\tTransitions hit = ~p\n", - [length(OKs) - length(NotCounted)]), - %% io:format(user, "\tTransitions = ~p\n", [lists:sort(_Counted)]), - io:format(user, "\tNotCounted length = ~p\n", - [length(NotCounted)]), - io:format(user, "\tLeast-counted transition = ~p\n", - [hd(CountedX)]), - io:format(user, "\tMost-counted transition = ~p\n", - [lists:last(CountedX)]), - ok - end}. - -projection_transitions_are_sane([]) -> - true; -projection_transitions_are_sane([_]) -> - true; -projection_transitions_are_sane([P1, P2|T]) -> - case projection_transition_is_sane(P1, P2) of - true -> - projection_transitions_are_sane([P2|T]); - Else -> - Else - end. - -projection_transition_is_sane( - #projection{epoch_number=Epoch1, - epoch_csum=CSum1, - prev_epoch_num=PrevEpoch1, - prev_epoch_csum=PrevCSum1, - creation_time=CreationTime1, - author_server=AuthorServer1, - all_members=All_list1, - down=Down_list1, - upi=UPI_list1, - repairing=Repairing_list1, - dbg=Dbg1} = P1, - #projection{epoch_number=Epoch2, - epoch_csum=CSum2, - prev_epoch_num=PrevEpoch2, - prev_epoch_csum=PrevCSum2, - creation_time=CreationTime2, - author_server=AuthorServer2, - all_members=All_list2, - down=Down_list2, - upi=UPI_list2, - repairing=Repairing_list2, - dbg=Dbg2} = P2) -> - try - true = is_integer(Epoch1) andalso is_integer(Epoch2), - true = is_binary(CSum1) andalso is_binary(CSum2), - true = is_integer(PrevEpoch1) andalso is_integer(PrevEpoch2), - true = is_binary(PrevCSum1) andalso is_binary(PrevCSum2), - {_,_,_} = CreationTime1, - {_,_,_} = CreationTime2, - true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2), % todo will probably change - true = is_list(All_list1) andalso is_list(All_list2), - true = is_list(Down_list1) andalso is_list(Down_list2), - true = is_list(UPI_list1) andalso is_list(UPI_list2), - true = is_list(Repairing_list1) andalso is_list(Repairing_list2), - true = is_list(Dbg1) andalso is_list(Dbg2), - - true = Epoch2 > Epoch1, - true = PrevEpoch2 > PrevEpoch1, - All_list1 = All_list2, % todo will probably change - - %% No duplicates - true = lists:sort(Down_list2) == lists:usort(Down_list2), - true = lists:sort(UPI_list2) == lists:usort(UPI_list2), - true = lists:sort(Repairing_list2) == lists:usort(Repairing_list2), - - %% Disjoint-ness - true = lists:sort(All_list2) == lists:sort(Down_list2 ++ UPI_list2 ++ - Repairing_list2), - [] = [X || X <- Down_list2, not lists:member(X, All_list2)], - [] = [X || X <- UPI_list2, not lists:member(X, All_list2)], - [] = [X || X <- Repairing_list2, not lists:member(X, All_list2)], - DownS2 = sets:from_list(Down_list2), - UPIS2 = sets:from_list(UPI_list2), - RepairingS2 = sets:from_list(Repairing_list2), - true = sets:is_disjoint(DownS2, UPIS2), - true = sets:is_disjoint(DownS2, RepairingS2), - true = sets:is_disjoint(UPIS2, RepairingS2), - - %% The author must not be down. - false = lists:member(AuthorServer1, Down_list1), - false = lists:member(AuthorServer2, Down_list2), - %% The author must be in either the UPI or repairing list. - true = lists:member(AuthorServer1, UPI_list1 ++ Repairing_list1), - true = lists:member(AuthorServer2, UPI_list2 ++ Repairing_list2), - - %% Additions to the UPI chain may only be at the tail - UPI_common_prefix = find_common_prefix(UPI_list1, UPI_list2), - if UPI_common_prefix == [] -> - if UPI_list1 == [] orelse UPI_list2 == [] -> - %% If the common prefix is empty, then one of the - %% inputs must be empty. - true; - true -> - %% Otherwise, we have a case of UPI changing from - %% one of these two situations: - %% - %% UPI_list1 -> UPI_list2 - %% [d,c,b,a] -> [c,a] - %% [d,c,b,a] -> [c,a,repair_finished_added_to_tail]. - NotUPI2 = (Down_list2 ++ Repairing_list2), - true = lists:prefix(UPI_list1 -- NotUPI2, - UPI_list2) - end; - true -> - true - end, - true = lists:prefix(UPI_common_prefix, UPI_list1), - true = lists:prefix(UPI_common_prefix, UPI_list2), - UPI_1_suffix = UPI_list1 -- UPI_common_prefix, - UPI_2_suffix = UPI_list2 -- UPI_common_prefix, - - %% Where did elements in UPI_2_suffix come from? - %% Only two sources are permitted. - [true = lists:member(X, Repairing_list1) % X added after repair done - orelse - lists:member(X, UPI_list1) % X in UPI_list1 after common pref - || X <- UPI_2_suffix], - - %% The UPI_2_suffix must exactly be equal to: ordered items from - %% UPI_list1 concat'ed with ordered items from Repairing_list1. - %% Both temp vars below preserve relative order! - UPI_2_suffix_from_UPI1 = [X || X <- UPI_1_suffix, - lists:member(X, UPI_list2)], - UPI_2_suffix_from_Repairing1 = [X || X <- UPI_2_suffix, - lists:member(X, Repairing_list1)], - %% true? - UPI_2_suffix = UPI_2_suffix_from_UPI1 ++ UPI_2_suffix_from_Repairing1, - - true - catch - _Type:_Err -> - S1 = ?MGR:make_projection_summary(P1), - S2 = ?MGR:make_projection_summary(P2), - Trace = erlang:get_stacktrace(), - {err, from, S1, to, S2, stack, Trace} - end. - -pass1_smoke_test() -> - P2 = ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d], - [], [d,c,b,a], [], []), - P3 = ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d], - [b,c,d], [a], [], []), - true = projection_transition_is_sane(P2, P3), - - P2b= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d], - [], [d,c,b,a], [], []), - P3b= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d], - [b,d], [c,a], [], []), - true = projection_transition_is_sane(P2b, P3b), - - P2c= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d,e], - [], [e,d,c,b], [a], []), - P3c= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d,e], - [e,c], [d,b,a], [], []), - true = projection_transition_is_sane(P2c, P3c), - - true. - -fail1_smoke_test() -> - P15 = ?MGR:make_projection(15, 14, <<>>, a, [a,b,c,d], - [b,d], [a], [c], []), - P16 = ?MGR:make_projection(16, 15, <<>>, a, [a,b,c,d], - [b,d], [c,a], [], []), - true = projection_transition_is_sane(P15, P16) /= true, - - P2d= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d,e], - [], [e,d,c,b], [a], []), - P3d= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d,e], - [e,c], [d,a,b], [], []), - true = projection_transition_is_sane(P2d, P3d) /= true, - - P2e= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d,e], - [], [e,d,c,b], [a], []), - P3e= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d,e], - [e,c], [a,d,b], [], []), - true = projection_transition_is_sane(P2e, P3e) /= true, - - true. - -fail2_smoke_test() -> - AB = [a,b], - BadSets = [{1, [b,a],[a,b]}, - {2, [b],[a,b]}, - %% {3, [a],[b]}, % weird but valid: [a] means b is repairing; - % so if a fails and b finishes repairing, - % swapping both at the same time is "ok". - {4, [a,b],[b,a]}, - %% {5, [b],[a]}, % weird but valid: see above - {6, [a],[b,a]} - ], - [begin - P2f= ?MGR:make_projection(2, 1, <<>>, a, AB, - [], Two, AB -- Two, []), - P3f= ?MGR:make_projection(3, 2, <<>>, a, AB, - [], Three, AB -- Three, []), - {Label, true} = - {Label, (projection_transition_is_sane(P2f, P3f) /= true)} - end || {Label, Two, Three} <- BadSets], - - true. - -fail3_smoke_test() -> - Nodes = [a,b], - UPI1 = [a,b], - UPI2 = [b,a], - [begin - PsUPI1 = UPI1, - PsUPI2 = UPI2, - P1 = ?MGR:make_projection(2, 1, <<>>, HdNd, Nodes, - [], PsUPI1, Nodes -- PsUPI1, []), - P2 = ?MGR:make_projection(3, 2, <<>>, HdNd, Nodes, - [], PsUPI2, Nodes -- PsUPI2, []), - Res = case projection_transition_is_sane(P1, P2) of - true -> true; - _ -> false - end, - {Res, HdNd, PsUPI1, PsUPI2} = {false, HdNd, PsUPI1, PsUPI2} - end || HdNd <- Nodes], - - true. - -fail4_smoke_test() -> - Nodes = [a,b,c], - Two = [c,b,a], - Three = [c,a,b], - P2a= ?MGR:make_projection(2, 1, <<>>, a, Nodes, - [], Two, Nodes -- Two, []), - P3a= ?MGR:make_projection(3, 2, <<>>, a, Nodes, - [], Three, Nodes -- Three, []), - true = (projection_transition_is_sane(P2a, P3a) /= true), - - true. - - -%% aaa_smoke_test() -> -%% L = [a,b,c,d], -%% Cs = combinations(L), -%% Combos = lists:sort([{X, Y} || X <- Cs, Y <- Cs]), -%% timer:sleep(500), -%% io:format(user, "\n", []), -%% Rs = [begin -%% P1 = ?MGR:make_projection(2, 1, <<>>, a, L, -%% [], X, L -- X, []), -%% P2 = ?MGR:make_projection(3, 2, <<>>, a, L, -%% [], Y, L -- Y, []), -%% Res = case projection_transition_is_sane(P1, P2) of -%% true -> true; -%% _ -> false -%% end, -%% {Res, P1, P2} -%% end || {X,Y} <- Combos], -%% OKs = [{X,Y} || {true, X, Y} <- Rs], -%% io:format(user, "Cs are ~p\n", [length(Cs)]), -%% io:format(user, "OKs are ~p\n", [length(OKs)]), -%% Bads = [{X,Y} || {false, X, Y} <- Rs], -%% io:format(user, "Bads are ~p\n", [length(Bads)]), -%% %% [io:format(user, "~p -> ~p: ~p\n", [X, Y, Res]); -%% ok. - -combinations(L) -> - lists:usort(perms(L) ++ lists:append([ combinations(L -- [X]) || X <- L])). - -perms([]) -> [[]]; -perms(L) -> [[H|T] || H <- L, T <- perms(L--[H])]. - --endif.