Bug fixes: model and real bugs, thanks PULSE and converge_demo both!

src/machi_chain_manager1.erl

@@ -1352,6 +1352,7 @@ react_to_env_C100(P_newprop, P_latest,
 react_to_env_C110(P_latest, #ch_mgr{name=MyName} = S) ->
     ?REACT(c110),
     Extra_todo = [],
+    %% Extra_todo = [{hee, lists:reverse(get(react))}],
     P_latest2 = machi_projection:update_dbg2(P_latest, Extra_todo),
 
     MyNamePid = proxy_pid(MyName, S),
@@ -1446,6 +1447,7 @@ react_to_env_C310(P_newprop, S) ->
     ?REACT({c310, ?LINE,
             [{newprop, machi_projection:make_summary(P_newprop)},
             {write_result, WriteRes}]}),
+%%    io:format(user, "HEE310 ~w ~w ~w\n", [S#ch_mgr.name, self(), lists:reverse(get(react))]),
     react_to_env_A10(S2).
 
 calculate_flaps(P_newprop, _P_current, _FlapLimit,
@@ -1721,6 +1723,35 @@ projection_transition_is_sane(
         not (lists:member(RelativeToServer, Down_list2) orelse
              lists:member(RelativeToServer, Repairing_list2)),
     
+    UPIs_are_disjointP = ordsets:is_disjoint(ordsets:from_list(UPI_list1),
+                                             ordsets:from_list(UPI_list2)),
+    case UPI_2_suffix -- UPI_list1 of
+        [] ->
+            true;
+        [_|_] = _Added_by_2 ->
+            if RetrospectiveP ->
+                    %% Any servers added to the UPI must be added from the
+                    %% repairing list ... but in retrospective mode (where
+                    %% we're checking only the transitions where all
+                    %% UPI+repairing participants have unanimous private
+                    %% projections!), and if we're under asymmetric
+                    %% partition/churn, then we may not see the repairing
+                    %% list.  So we will not check that condition here.
+                    true;
+               not RetrospectiveP ->
+                    %% We're not retrospective.  So, if some server was
+                    %% added by to the UPI, then that means that it was
+                    %% added by repair.  And repair is coordinated by the
+                    %% UPI tail/last.
+%io:format(user, "g: UPI_list1=~w, UPI_list2=~w, UPI_2_suffix=~w, ",
+%          [UPI_list1, UPI_list2, UPI_2_suffix]),
+%io:format(user, "g", []),
+                    true = UPI_list1 == [] orelse
+                           UPIs_are_disjointP orelse
+                           (lists:last(UPI_list1) == AuthorServer2)
+            end
+    end,
+
     if not MoreCheckingP ->
             ok;
         MoreCheckingP ->
@@ -1788,16 +1819,26 @@ projection_transition_is_sane(
                             %% The retrospective view by
                             %% machi_chain_manager1_pulse.erl just can't
                             %% reason correctly about this situation.  We
-                            %% will instead rely on the non-introspective
+                            %% will instead rely on the non-retrospective
                             %% sanity checking that each FLU does before it
                             %% writes to its private projection store and
                             %% then adopts that projection (and unwedges
                             %% itself, etc etc).
 
-                            exit({todo, revisit, ?MODULE, ?LINE}),
+                            if UPIs_are_disjointP ->
+                                    true;
+                               true ->
+                                    exit({todo, revisit, ?MODULE, ?LINE,
+                                          [
+                                           {oops_check_UPI_2_suffix, Oops_check_UPI_2_suffix},
+                                           {upi_2_suffix, UPI_2_suffix},
+                                           {upi_2_concat, UPI_2_concat},
+                                           {retrospectivep, RetrospectiveP}
+                                          ]}),
                                     io:format(user, "|~p,~p TODO revisit|",
                                               [?MODULE, ?LINE]),
-                            ok;
+                                    ok
+                            end;
                        true ->
                             %% The following is OK: We're shifting from a
                             %% normal projection to an inner one.  The old
@@ -1843,7 +1884,13 @@ projection_transition_is_sane(
                                     true;
                                SecondCase_p ->
                                     true;
-                               not RetrospectiveP ->
+                               UPIs_are_disjointP ->
+                                    %% If there's no overlap at all between
+                                    %% UPI_list1 & UPI_list2, then we're OK
+                                    %% here.
+                                    io:format(user, "QQQ Oops_check_UPI_2_suffix=~w, ", [Oops_check_UPI_2_suffix]),
+                                    true;
+                               true ->
                                     exit({upi_2_suffix_error, UPI_2_suffix})
                             end
                     end

test/machi_chain_manager1_converge_demo.erl

@@ -251,35 +251,21 @@ convergence_demo_testfun(NumFLUs) ->
       %% unique chains are disjoint.
       true = machi_chain_manager1_test:all_reports_are_disjoint(Report),
 
-      %% Given the report, we flip it around so that we observe the
-      %% sets of chain transitions relative to each FLU.
-      %% R_Chains's type is: list({RelativeFlu, list({Epoch, UPI, Repairing})})
-      %% For example:
-      %%          [{a,[{3,[a],[b,c]},
-      %%               {7,[a,b],[c]},
-      %%               {14,[a,b,c],[]},
-      %%               {83,[a],[]},
-      %%               {164,[c],[a,b]},
-      %%               {218,[a],[]},
-      %%               {226,[a],[b,c]},
-      %%               {228,[a],[b,c]},
-      %%               ....
-      R_Chains = [machi_chain_manager1_test:extract_chains_relative_to_flu(
-                    FLU, Report) || FLU <- All_list],
-      %% ?D(R_Chains),
-      R_Projs = [{FLU, [machi_chain_manager1_test:chain_to_projection(
-                          FLU, Epoch, UPI, Repairing, All_list) ||
-                           {Epoch, UPI, Repairing} <- E_Chains]} ||
-                    {FLU, E_Chains} <- R_Chains],
-
       %% For each chain transition experienced by a particular FLU,
       %% confirm that each state transition is OK.
+      PrivProjs = [{Name, begin
+                              {ok, Ps9} = ?FLU_PC:get_all_projections(FLU,
+                                                                      private),
+                              [P || P <- Ps9,
+                                    P#projection_v1.epoch_number /= 0]
+                          end} || {Name, FLU} <- Namez],
       try
-          [{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane(Psx, FLU)} ||
+          [{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane_retrospective(Psx, FLU)} ||
-              {FLU, Psx} <- R_Projs],
+              {FLU, Psx} <- PrivProjs],
           io:format(user, "\nAll sanity checks pass, hooray!\n", [])
       catch _Err:_What ->
               io:format(user, "Report ~p\n", [Report]),
+              io:format(user, "PrivProjs ~p\n", [PrivProjs]),
               exit({line, ?LINE, _Err, _What})
       end,
       %% ?D(R_Projs),
@@ -311,9 +297,10 @@ make_partition_list(All_list) ->
                                        A <- All_list, B <- All_list, A /= B,
                                        C <- All_list, D <- All_list, C /= D,
                                        X /= A, X /= C, A /= C],
-    %% _X_Ys1 ++ _X_Ys2.
+    %% Concat = _X_Ys1 ++ _X_Ys2.
-    %% _X_Ys3.
+    %% Concat = _X_Ys3.
-    _X_Ys1 ++ _X_Ys2 ++ _X_Ys3.
+    Concat = _X_Ys1 ++ _X_Ys2 ++ _X_Ys3,
+    lists:usort([lists:sort(L) || L <- Concat]).
 
     %% [ [{a,b},{b,d},{c,b}],
     %%   [{a,b},{b,d},{c,b}, {a,b},{b,a},{a,c},{c,a},{a,d},{d,a}],

test/machi_chain_manager1_pulse.erl

@@ -110,10 +110,10 @@ all_list_extra() ->
                 props=[{chmgr, b_chmgr}]}, "./data.pulse.b"}
      , {#p_srvr{name=c, address="localhost", port=7402,
                 props=[{chmgr, c_chmgr}]}, "./data.pulse.c"}
-     , {#p_srvr{name=d, address="localhost", port=7403,
+     %% , {#p_srvr{name=d, address="localhost", port=7403,
-                props=[{chmgr, d_chmgr}]}, "./data.pulse.d"}
+     %%            props=[{chmgr, d_chmgr}]}, "./data.pulse.d"}
-     , {#p_srvr{name=e, address="localhost", port=7404,
+     %% , {#p_srvr{name=e, address="localhost", port=7404,
-                props=[{chmgr, e_chmgr}]}, "./data.pulse.e"}
+     %%            props=[{chmgr, e_chmgr}]}, "./data.pulse.e"}
     ].
 
 all_list() ->
@@ -261,11 +261,21 @@ dump_state() ->
     %%           || {FLUName, _FLU} <- Namez]
     %%          || Epoch <- UniquePrivateEs],
 
+    PrivProjs = [{Name, begin
+                            {ok, Ps} = ?FLU_PC:get_all_projections(Proxy,
+                                                                   private),
+                            [P || P <- Ps,
+                                  P#projection_v1.epoch_number /= 0]
+                        end} || {Name, Proxy} <- ProxiesDict],
+
     ?QC_FMT(")", []),
     Diag1 = Diag2 = "skip_diags",
-    {Report, lists:flatten([Diag1, Diag2])}
+    {Report, PrivProjs, lists:flatten([Diag1, Diag2])}
   catch XX:YY ->
-        ?QC_FMT("OUCH: ~p ~p @ ~p\n", [XX, YY, erlang:get_stacktrace()])
+        ?QC_FMT("OUCH: ~p ~p @ ~p\n", [XX, YY, erlang:get_stacktrace()]),
+        ?QC_FMT("Exiting now to move to manual post-mortem....\n", []),
+        erlang:halt(0),
+        false
   end.
 
 prop_pulse() ->
@@ -296,8 +306,6 @@ prop_pulse() ->
                                    {_H, _S, _R} = run_commands(?MODULE, Cmds)
                            end, [{seed, Seed},
                                  {strategy, unfair}]),
-        ok = shutdown_hard(),
-
         %% ?QC_FMT("S2 ~p\n", [S2]),
         case S2#state.dump_state of
             undefined ->
@@ -305,7 +313,7 @@ prop_pulse() ->
             _ ->
                 ok
         end,
-        {Report, Diag} = S2#state.dump_state,
+        {Report, PrivProjs, Diag} = S2#state.dump_state,
 
         %% Report is ordered by Epoch.  For each private projection
         %% written during any given epoch, confirm that all chain
@@ -313,21 +321,12 @@ prop_pulse() ->
         %% unique chains are disjoint.
         AllDisjointP = ?MGRTEST:all_reports_are_disjoint(Report),
 
-        %% Given the report, we flip it around so that we observe the
-        %% sets of chain transitions relative to each FLU.
-        R_Chains = [?MGRTEST:extract_chains_relative_to_flu(FLU, Report) ||
-                       FLU <- all_list()],
-        R_Projs = [{FLU, [?MGRTEST:chain_to_projection(
-                             FLU, Epoch, UPI, Repairing, all_list()) ||
-                             {Epoch, UPI, Repairing} <- E_Chains]} ||
-                      {FLU, E_Chains} <- R_Chains],
-
         %% For each chain transition experienced by a particular FLU,
         %% confirm that each state transition is OK.
         Sane =
           [{FLU,_SaneRes} = {FLU,?MGR:projection_transitions_are_sane_retrospective(
                                     Ps, FLU)} ||
-              {FLU, Ps} <- R_Projs],
+              {FLU, Ps} <- PrivProjs],
         SaneP = lists:all(fun({_FLU, SaneRes}) -> SaneRes == true end, Sane),
 
         %% The final report item should say that all are agreed_membership.
@@ -342,12 +341,14 @@ prop_pulse() ->
                                       LastRepXs
                               end,
 
+        ok = shutdown_hard(),
         ?WHENFAIL(
         begin
             ?QC_FMT("Cmds = ~p\n", [Cmds]),
             ?QC_FMT("Res = ~p\n", [Res]),
             ?QC_FMT("Diag = ~s\n", [Diag]),
             ?QC_FMT("Report = ~p\n", [Report]),
+            ?QC_FMT("PrivProjs = ~p\n", [PrivProjs]),
             ?QC_FMT("Sane = ~p\n", [Sane]),
             ?QC_FMT("SingleChainNoRepair failure =\n    ~p\n", [SingleChainNoRepair])
 ,erlang:halt(0)

test/machi_chain_manager1_test.erl

@@ -99,7 +99,7 @@ unanimous_report(Epoch, Namez) ->
                              {not_agreed, {UPI, Repairing}, Else2}
                      end;
                  _Else ->
-                     exit({UPI, not_unique, Epoch, _Else})
+                     {not_agreed, {undefined, undefined}, Projs}
              end
          end || UPI <- UniqueUPIs],
     AgreedResUPI_Rs = [UPI++Repairing ||
@@ -128,7 +128,7 @@ chain_to_projection(MyName, Epoch, UPI_list, Repairing_list, All_list) ->
                                        FLU <- All_list]),
     machi_projection:new(Epoch, MyName, MemberDict,
                          All_list -- (UPI_list ++ Repairing_list),
-                         UPI_list, Repairing_list, []).
+                         UPI_list, Repairing_list, [{artificial_by, ?MODULE}]).
 
 -ifndef(PULSE).