From 5aeaf872d97f857ba2cf786c5238c66853005c11 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Mon, 7 Dec 2015 14:41:56 +0900 Subject: [PATCH] WIP: machi_chain_manager1:set_chain_members() API change, all tests pass, yay --- include/machi_projection.hrl | 4 +- src/machi.proto | 23 +++--- src/machi_chain_bootstrap.erl | 7 +- src/machi_chain_manager1.erl | 81 +++++++++++++-------- src/machi_pb_translate.erl | 4 + test/machi_ap_repair_eqc.erl | 4 +- test/machi_chain_manager1_converge_demo.erl | 21 +++--- test/machi_chain_manager1_test.erl | 4 +- test/machi_cr_client_test.erl | 12 ++- 9 files changed, 100 insertions(+), 60 deletions(-) diff --git a/include/machi_projection.hrl b/include/machi_projection.hrl index ce6fe4a..5ad4e54 100644 --- a/include/machi_projection.hrl +++ b/include/machi_projection.hrl @@ -22,10 +22,11 @@ -define(MACHI_PROJECTION_HRL, true). -type pv1_consistency_mode() :: 'ap_mode' | 'cp_mode'. +-type pv1_chain_name():: atom(). -type pv1_csum() :: binary(). -type pv1_epoch() :: {pv1_epoch_n(), pv1_csum()}. -type pv1_epoch_n() :: non_neg_integer(). --type pv1_server() :: atom() | binary(). +-type pv1_server() :: atom(). -type pv1_timestamp() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}. -record(p_srvr, { @@ -55,6 +56,7 @@ epoch_number :: pv1_epoch_n() | ?SPAM_PROJ_EPOCH, epoch_csum :: pv1_csum(), author_server :: pv1_server(), + chain_name = ch_not_def_yet :: pv1_chain_name(), all_members :: [pv1_server()], witnesses = [] :: [pv1_server()], creation_time :: pv1_timestamp(), diff --git a/src/machi.proto b/src/machi.proto index c9251cb..2fa8657 100644 --- a/src/machi.proto +++ b/src/machi.proto @@ -330,18 +330,17 @@ message Mpb_ProjectionV1 { required uint32 epoch_number = 1; required bytes epoch_csum = 2; required string author_server = 3; - repeated string all_members = 4; - repeated string witnesses = 5; - required Mpb_Now creation_time = 6; - required Mpb_Mode mode = 7; - repeated string upi = 8; - repeated string repairing = 9; - repeated string down = 10; - optional bytes opaque_flap = 11; - optional bytes opaque_inner = 12; - required bytes opaque_dbg = 13; - required bytes opaque_dbg2 = 14; - repeated Mpb_MembersDictEntry members_dict = 15; + required string chain_name = 4; + repeated string all_members = 5; + repeated string witnesses = 6; + required Mpb_Now creation_time = 7; + required Mpb_Mode mode = 8; + repeated string upi = 9; + repeated string repairing = 10; + repeated string down = 11; + required bytes opaque_dbg = 12; + required bytes opaque_dbg2 = 13; + repeated Mpb_MembersDictEntry members_dict = 14; } ////////////////////////////////////////// diff --git a/src/machi_chain_bootstrap.erl b/src/machi_chain_bootstrap.erl index 306defc..2406401 100644 --- a/src/machi_chain_bootstrap.erl +++ b/src/machi_chain_bootstrap.erl @@ -131,7 +131,10 @@ perhaps_bootstrap_chains([CD|ChainDefs], FLUs) -> io:format(user, "TODO: no local flus in ~P\n", [CD, 10]), ok; [FLU1|_] -> - io:format(user, "TODO: config ~p as bootstrap member of ~p\n", [FLU1, CD]), - yoyo + bootstrap_chain(CD, FLU1) end, perhaps_bootstrap_chains(ChainDefs, FLUs). + +bootstrap_chain(CD, FLU) -> + io:format(user, "TODO: config ~p as bootstrap member of ~p\n", [FLU, CD]), + todo. diff --git a/src/machi_chain_manager1.erl b/src/machi_chain_manager1.erl index 157ac5b..895a3d4 100644 --- a/src/machi_chain_manager1.erl +++ b/src/machi_chain_manager1.erl @@ -108,7 +108,7 @@ %% API -export([start_link/2, start_link/3, stop/1, ping/1, - set_chain_members/2, set_chain_members/3, set_active/2, + set_chain_members/2, set_chain_members/6, set_active/2, trigger_react_to_env/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, format_status/2, code_change/3]). @@ -168,13 +168,22 @@ ping(Pid) -> %% with lowest rank, i.e. name z* first, name a* last. set_chain_members(Pid, MembersDict) -> - set_chain_members(Pid, MembersDict, []). + set_chain_members(Pid, ch0_name, 0, ap_mode, MembersDict, []). -set_chain_members(Pid, MembersDict, Witness_list) -> - case lists:all(fun(Witness) -> orddict:is_key(Witness, MembersDict) end, - Witness_list) of +set_chain_members(Pid, ChainName, OldEpoch, CMode, MembersDict, Witness_list) + when is_atom(ChainName) andalso + is_integer(OldEpoch) andalso OldEpoch >= 0 andalso + (CMode == ap_mode orelse CMode == cp_mode) andalso + is_list(MembersDict) andalso + is_list(Witness_list) -> + case lists:all(fun({X, #p_srvr{name=X}}) -> true; + (_) -> false + end, MembersDict) + andalso + lists:all(fun(Witness) -> orddict:is_key(Witness, MembersDict) end, + Witness_list) of true -> - Cmd = {set_chain_members, MembersDict, Witness_list}, + Cmd = {set_chain_members, ChainName, OldEpoch, CMode, MembersDict, Witness_list}, gen_server:call(Pid, Cmd, infinity); false -> {error, bad_arg} @@ -291,7 +300,8 @@ init({MyName, InitMembersDict, MgrOpts}) -> handle_call({ping}, _From, S) -> {reply, pong, S}; -handle_call({set_chain_members, MembersDict, Witness_list}, _From, +handle_call({set_chain_members, ChainName, OldEpoch, CMode, + MembersDict, Witness_list}, _From, #ch_mgr{name=MyName, proj=#projection_v1{all_members=OldAll_list, epoch_number=OldEpoch, @@ -310,10 +320,10 @@ handle_call({set_chain_members, MembersDict, Witness_list}, _From, {NUPI, All_list -- NUPI} end, NewEpoch = OldEpoch + ?SET_CHAIN_MEMBERS_EPOCH_SKIP, - CMode = calc_consistency_mode(Witness_list), ok = set_consistency_mode(machi_flu_psup:make_proj_supname(MyName), CMode), NewProj = machi_projection:update_checksum( OldProj#projection_v1{author_server=MyName, + chain_name=ChainName, creation_time=now(), mode=CMode, epoch_number=NewEpoch, @@ -601,6 +611,8 @@ rank_and_sort_projections_with_extra(All_queried_list, FLUsRs, ProjectionType, Witness_list = CurrentProj#projection_v1.witnesses, NoneProj = make_none_projection(0, MyName, [], Witness_list, orddict:new()), + ChainName = CurrentProj#projection_v1.chain_name, + NoneProj2 = NoneProj#projection_v1{chain_name=ChainName}, Extra2 = [{all_members_replied, true}, {all_queried_list, All_queried_list}, {flus_rs, FLUsRs}, @@ -723,13 +735,14 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, runenv=RunEnv1, repair_final_status=RepairFS}=S) -> #projection_v1{epoch_number=OldEpochNum, + chain_name=ChainName, members_dict=MembersDict, witnesses=OldWitness_list, upi=OldUPI_list, repairing=OldRepairing_list } = LastProj, LastUp = lists:usort(OldUPI_list ++ OldRepairing_list), - AllMembers = (S#ch_mgr.proj)#projection_v1.all_members, + AllMembers = CurrentProj#projection_v1.all_members, {Up0, Partitions, RunEnv2} = calc_up_nodes(MyName, AllMembers, RunEnv1), Up = Up0 -- AllHosed, @@ -821,10 +834,11 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, end, ?REACT({calc,?LINE,[{new_upi, NewUPI},{new_rep, NewRepairing}]}), - P = machi_projection:new(OldEpochNum + 1, - MyName, MembersDict, Down, NewUPI, NewRepairing, - D_foo ++ - Dbg ++ [{ps, Partitions},{nodes_up, Up}]), + P0 = machi_projection:new(OldEpochNum + 1, + MyName, MembersDict, Down, NewUPI, NewRepairing, + D_foo ++ + Dbg ++ [{ps, Partitions},{nodes_up, Up}]), + P1 = P0#projection_v1{chain_name=ChainName}, P2 = if CMode == cp_mode -> UpWitnesses = [W || W <- Up, lists:member(W, OldWitness_list)], Majority = full_majority_size(AllMembers), @@ -833,7 +847,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, SoFar = length(NewUPI ++ NewRepairing), if SoFar >= Majority -> ?REACT({calc,?LINE,[]}), - P; + P1; true -> Need = Majority - SoFar, UpWitnesses = [W || W <- Up, @@ -842,7 +856,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, Ws = lists:sublist(UpWitnesses, Need), ?REACT({calc,?LINE,[{ws, Ws}]}), machi_projection:update_checksum( - P#projection_v1{upi=Ws++NewUPI}); + P1#projection_v1{upi=Ws++NewUPI}); true -> ?REACT({calc,?LINE,[]}), P_none0 = make_none_projection( @@ -855,6 +869,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, "Not enough witnesses are available now" end, P_none1 = P_none0#projection_v1{ + chain_name=ChainName, %% Stable creation time! creation_time={1,2,3}, dbg=[{none_projection,true}, @@ -875,7 +890,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, end; CMode == ap_mode -> ?REACT({calc,?LINE,[]}), - P + P1 end, P3 = machi_projection:update_checksum( P2#projection_v1{mode=CMode, witnesses=OldWitness_list}), @@ -1045,13 +1060,13 @@ do_react_to_env(#ch_mgr{name=MyName, {{empty_members_dict, [], Epoch}, S}; true -> {_, S2} = do_set_chain_members_dict(NewMembersDict, S), - CMode = calc_consistency_mode(NewProj#projection_v1.witnesses), + CMode = NewProj#projection_v1.mode, {{empty_members_dict, [], Epoch}, set_proj(S2#ch_mgr{members_dict=NewMembersDict, consistency_mode=CMode}, NewProj)} end; do_react_to_env(S) -> -put(ttt, [?LINE]), + put(ttt, [?LINE]), %% The not_sanes manager counting dictionary is not strictly %% limited to flapping scenarios. (Though the mechanism first %% started as a way to deal with rare flapping scenarios.) @@ -1287,7 +1302,8 @@ react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra, end. react_to_env_A30(Retries, P_latest, LatestUnanimousP, P_current_calc, - #ch_mgr{name=MyName, consistency_mode=CMode} = S) -> + #ch_mgr{name=MyName, proj=P_current, + consistency_mode=CMode} = S) -> V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end, if V -> io:format(user, "A30: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end, ?REACT(a30), @@ -1307,15 +1323,17 @@ react_to_env_A30(Retries, P_latest, LatestUnanimousP, P_current_calc, P = #projection_v1{down=Down} = make_none_projection(Epoch + 1, MyName, All_list, Witness_list, MembersDict), + ChainName = P_current#projection_v1.chain_name, + P1 = P#projection_v1{chain_name=ChainName}, P_newprop = if CMode == ap_mode -> %% Not really none proj: just myself, AP style machi_projection:update_checksum( - P#projection_v1{upi=[MyName], + P1#projection_v1{upi=[MyName], down=Down -- [MyName], dbg=[{hosed_list,AllHosed}]}); CMode == cp_mode -> machi_projection:update_checksum( - P#projection_v1{dbg=[{hosed_list,AllHosed}]}) + P1#projection_v1{dbg=[{hosed_list,AllHosed}]}) end, react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP, P_current_calc, true, S); @@ -1850,7 +1868,9 @@ react_to_env_C103(#projection_v1{epoch_number=_Epoch_newprop} = _P_newprop, members_dict=MembersDict} = P_current, P_none0 = make_none_projection(Epoch_latest, MyName, All_list, Witness_list, MembersDict), - P_none1 = P_none0#projection_v1{dbg=[{none_projection,true}]}, + ChainName = P_current#projection_v1.chain_name, + P_none1 = P_none0#projection_v1{chain_name=ChainName, + dbg=[{none_projection,true}]}, P_none = machi_projection:update_checksum(P_none1), ?REACT({c103, ?LINE, [{current_epoch, P_current#projection_v1.epoch_number}, @@ -2206,6 +2226,7 @@ projection_transition_is_sane_except_si_epoch( creation_time=CreationTime1, mode=CMode1, author_server=AuthorServer1, + chain_name=ChainName1, all_members=All_list1, witnesses=Witness_list1, down=Down_list1, @@ -2217,6 +2238,7 @@ projection_transition_is_sane_except_si_epoch( creation_time=CreationTime2, mode=CMode2, author_server=AuthorServer2, + chain_name=ChainName2, all_members=All_list2, witnesses=Witness_list2, down=Down_list2, @@ -2237,7 +2259,8 @@ projection_transition_is_sane_except_si_epoch( true = is_binary(CSum1) andalso is_binary(CSum2), {_,_,_} = CreationTime1, {_,_,_} = CreationTime2, - true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2), % todo type may change? + true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2), + true = is_atom(ChainName1) andalso is_atom(ChainName2), true = is_list(All_list1) andalso is_list(All_list2), true = is_list(Witness_list1) andalso is_list(Witness_list2), true = is_list(Down_list1) andalso is_list(Down_list2), @@ -2249,6 +2272,9 @@ projection_transition_is_sane_except_si_epoch( %% projection_transition_is_sane_with_si_epoch(). true = Epoch2 >= Epoch1, + %% Don't change chain names in the middle of the stream. + true = (ChainName1 == ChainName2), + %% No duplicates true = lists:sort(Witness_list2) == lists:usort(Witness_list2), true = lists:sort(Down_list2) == lists:usort(Down_list2), @@ -2772,6 +2798,7 @@ full_majority_size(L) when is_list(L) -> full_majority_size(length(L)). make_zerf(#projection_v1{epoch_number=OldEpochNum, + chain_name=ChainName, all_members=AllMembers, members_dict=MembersDict, witnesses=OldWitness_list @@ -2794,7 +2821,8 @@ make_zerf(#projection_v1{epoch_number=OldEpochNum, MyName, AllMembers, OldWitness_list, MembersDict), machi_projection:update_checksum( - P#projection_v1{mode=cp_mode, + P#projection_v1{chain_name=ChainName, + mode=cp_mode, dbg2=[zerf_none,{up,Up},{maj,MajoritySize}]}); true -> make_zerf2(OldEpochNum, Up, MajoritySize, MyName, @@ -2916,11 +2944,6 @@ perhaps_verbose_c111(P_latest2, S) -> ok end. -calc_consistency_mode(_Witness_list = []) -> - ap_mode; -calc_consistency_mode(_Witness_list) -> - cp_mode. - set_proj(S, Proj) -> S#ch_mgr{proj=Proj, proj_unanimous=false}. diff --git a/src/machi_pb_translate.erl b/src/machi_pb_translate.erl index 0b49908..676c5a1 100644 --- a/src/machi_pb_translate.erl +++ b/src/machi_pb_translate.erl @@ -811,6 +811,7 @@ conv_to_epoch_id(#mpb_epochid{epoch_number=Epoch, conv_to_projection_v1(#mpb_projectionv1{epoch_number=Epoch, epoch_csum=CSum, author_server=Author, + chain_name=ChainName, all_members=AllMembers, witnesses=Witnesses, creation_time=CTime, @@ -824,6 +825,7 @@ conv_to_projection_v1(#mpb_projectionv1{epoch_number=Epoch, #projection_v1{epoch_number=Epoch, epoch_csum=CSum, author_server=to_atom(Author), + chain_name=to_atom(ChainName), all_members=[to_atom(X) || X <- AllMembers], witnesses=[to_atom(X) || X <- Witnesses], creation_time=conv_to_now(CTime), @@ -971,6 +973,7 @@ conv_from_boolean(true) -> conv_from_projection_v1(#projection_v1{epoch_number=Epoch, epoch_csum=CSum, author_server=Author, + chain_name=ChainName, all_members=AllMembers, witnesses=Witnesses, creation_time=CTime, @@ -984,6 +987,7 @@ conv_from_projection_v1(#projection_v1{epoch_number=Epoch, #mpb_projectionv1{epoch_number=Epoch, epoch_csum=CSum, author_server=to_list(Author), + chain_name=to_list(ChainName), all_members=[to_list(X) || X <- AllMembers], witnesses=[to_list(X) || X <- Witnesses], creation_time=conv_from_now(CTime), diff --git a/test/machi_ap_repair_eqc.erl b/test/machi_ap_repair_eqc.erl index a42d7cc..0c9f349 100644 --- a/test/machi_ap_repair_eqc.erl +++ b/test/machi_ap_repair_eqc.erl @@ -407,8 +407,8 @@ stabilize(0, _T) -> stabilize(_CmdsLen, #target{flu_names=FLUNames, mgr_names=MgrNames, verbose=Verbose}) -> machi_partition_simulator:no_partitions(), - wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames, - 100, Verbose), + true = wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames, + 100, Verbose), ok. chain_state_all_ok(FLUNames) -> diff --git a/test/machi_chain_manager1_converge_demo.erl b/test/machi_chain_manager1_converge_demo.erl index 9303701..cee7a78 100644 --- a/test/machi_chain_manager1_converge_demo.erl +++ b/test/machi_chain_manager1_converge_demo.erl @@ -187,15 +187,18 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) -> end || #p_srvr{name=Name}=P <- Ps], MembersDict = machi_projection:make_members_dict(Ps), Witnesses = proplists:get_value(witnesses, MgrOpts, []), + CMode = case {Witnesses, proplists:get_value(consistency_mode, MgrOpts, + ap_mode)} of + {[_|_], _} -> cp_mode; + {_, cp_mode} -> cp_mode; + {_, ap_mode} -> ap_mode + end, MgrNamez = [begin MgrName = machi_flu_psup:make_mgr_supname(Name), - ok = ?MGR:set_chain_members(MgrName,MembersDict,Witnesses), + ok = ?MGR:set_chain_members(MgrName, ch_demo, 0, CMode, + MembersDict,Witnesses), {Name, MgrName} end || #p_srvr{name=Name} <- Ps], - CpApMode = case Witnesses /= [] of - true -> cp_mode; - false -> ap_mode - end, try [{_, Ma}|_] = MgrNamez, @@ -303,9 +306,9 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) -> [{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane_retrospective(Psx, FLU)} || {FLU, Psx} <- PrivProjs] catch - _Err:_What when CpApMode == cp_mode -> + _Err:_What when CMode == cp_mode -> io:format(user, "none proj skip detected, TODO? ", []); - _Err:_What when CpApMode == ap_mode -> + _Err:_What when CMode == ap_mode -> io:format(user, "PrivProjs ~p\n", [PrivProjs]), exit({line, ?LINE, _Err, _What}) end, @@ -371,9 +374,9 @@ timer:sleep(1234), {FLU, Psx} <- PrivProjs], io:format(user, "\nAll sanity checks pass, hooray!\n", []) catch - _Err:_What when CpApMode == cp_mode -> + _Err:_What when CMode == cp_mode -> io:format(user, "none proj skip detected, TODO? ", []); - _Err:_What when CpApMode == ap_mode -> + _Err:_What when CMode == ap_mode -> io:format(user, "Report ~p\n", [Report]), io:format(user, "PrivProjs ~p\n", [PrivProjs]), exit({line, ?LINE, _Err, _What}) diff --git a/test/machi_chain_manager1_test.erl b/test/machi_chain_manager1_test.erl index e13193f..27e40ec 100644 --- a/test/machi_chain_manager1_test.erl +++ b/test/machi_chain_manager1_test.erl @@ -349,8 +349,8 @@ nonunanimous_setup_and_fix_test() -> [element(2,?FLU_PC:start_link(P)) || P <- P_s], MembersDict = machi_projection:make_members_dict(P_s), [Ma,Mb] = [a_chmgr, b_chmgr], - ok = machi_chain_manager1:set_chain_members(Ma, MembersDict, []), - ok = machi_chain_manager1:set_chain_members(Mb, MembersDict, []), + ok = machi_chain_manager1:set_chain_members(Ma, MembersDict), + ok = machi_chain_manager1:set_chain_members(Mb, MembersDict), try {ok, P1} = ?MGR:test_calc_projection(Ma, false), diff --git a/test/machi_cr_client_test.erl b/test/machi_cr_client_test.erl index f5d3513..299a78a 100644 --- a/test/machi_cr_client_test.erl +++ b/test/machi_cr_client_test.erl @@ -58,9 +58,15 @@ setup_smoke_test(Host, PortBase, Os, Witness_list) -> %% 4. Wait until all others are using epoch id from #3. %% %% Damn, this is a pain to make 100% deterministic, bleh. - ok = machi_chain_manager1:set_chain_members(a_chmgr, D, Witness_list), - ok = machi_chain_manager1:set_chain_members(b_chmgr, D, Witness_list), - ok = machi_chain_manager1:set_chain_members(c_chmgr, D, Witness_list), + CMode = if Witness_list == [] -> ap_mode; + Witness_list /= [] -> cp_mode + end, + ok = machi_chain_manager1:set_chain_members(a_chmgr, ch0, 0, CMode, + D, Witness_list), + ok = machi_chain_manager1:set_chain_members(b_chmgr, ch0, 0, CMode, + D, Witness_list), + ok = machi_chain_manager1:set_chain_members(c_chmgr, ch0, 0, CMode, + D, Witness_list), run_ticks([a_chmgr,b_chmgr,c_chmgr]), %% Everyone is settled on the same damn epoch id. {ok, EpochID} = machi_flu1_client:get_latest_epochid(Host, PortBase+0,