End-to-end chain state checking is still broken (more)

If we use verbose output from:

    machi_chain_manager1_converge_demo:t(3, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a]}]).

And use:

    tail -f typescript_file | egrep --line-buffered 'SET|attempted|CONFIRM'

... then we can clearly see a chain safety violation when moving from
epoch 81 -> 83.  I need to add more smarts to the safety checking,
both at the individual transition sanity check and at the converge_demo
overall rolling sanity check.

Key to output: CONFIRM by epoch {num} {csum} at {UPI} {Repairing}

    SET # of FLUs = 3 members [a,b,c]).
    CONFIRM by epoch 1 <<96,161,96,...>> at [a,b] [c]
    CONFIRM by epoch 5 <<134,243,175,...>> at [b,c] []
    CONFIRM by epoch 7 <<207,93,225,...>> at [b,c] []
    CONFIRM by epoch 47 <<60,142,248,...>> at [b,c] []
    SET partitions = [{c,b},{c,a}] (1 of 2) at {22,3,34}
    CONFIRM by epoch 81 <<223,58,184,...>> at [a,b] []
    SET partitions = [{b,c},{b,a}] (2 of 2) at {22,3,38}
    CONFIRM by epoch 83 <<33,208,224,...>> at [a,c] []
    SET partitions = []
    CONFIRM by epoch 85 <<173,179,149,...>> at [a,c] [b]

src/machi_chain_manager1.erl

@@ -1765,6 +1765,7 @@ react_to_env_C120(P_latest, FinalProps, #ch_mgr{proj_history=H,
     %% HH = [if is_atom(X) -> X; is_tuple(X) -> {element(1,X), element(2,X)} end || X <- get(react), is_atom(X) orelse size(X) == 3],
     %% ?V("HEE120 ~w ~w ~w\n", [S#ch_mgr.name, self(), lists:reverse(HH)]),
 
+    diversion_c120_verbose_goop(P_latest, S),
     ?REACT({c120, [{latest, machi_projection:make_summary(P_latest)}]}),
     {{now_using, FinalProps, P_latest#projection_v1.epoch_number},
      S#ch_mgr{proj=P_latest, proj_history=H3, sane_transitions=Xtns + 1}}.
@@ -2681,6 +2682,41 @@ my_lists_split(N, L) ->
             {L, []}
     end.
 
+diversion_c120_verbose_goop(#projection_v1{upi=[], repairing=[]}, S) ->
+    ok;
+diversion_c120_verbose_goop(Proj, S) ->
+    case proplists:get_value(private_write_verbose, S#ch_mgr.opts) of
+        true ->
+            diversion_c120_verbose_goop2(Proj, S);
+        _ ->
+            ok
+    end.
+
+diversion_c120_verbose_goop2(#projection_v1{epoch_number=Epoch, epoch_csum=CSum,
+                                       upi=UPI, repairing=Repairing}=P_latest0,
+                        S) ->
+    P_latest = machi_projection:update_checksum(P_latest0#projection_v1{dbg2=[]}),
+    UPI_Rs = UPI ++ Repairing,
+    R = [try
+             true = (UPI_Rs /= []),
+             Proxy = proxy_pid(FLU, S),
+             {ok, P} = ?FLU_PC:read_projection(Proxy, private, Epoch),
+             case machi_projection:update_checksum(P#projection_v1{dbg2=[]}) of
+                 X when X == P_latest ->
+                     FLU;
+                 _ ->
+                     nope
+             end
+         catch _:_ ->
+                 definitely_not
+         end || FLU <- UPI_Rs],
+    if R == UPI_Rs ->
+            io:format(user, "\nCONFIRM by epoch ~p ~W at ~p ~p\n",
+                      [Epoch, CSum, 4, UPI, Repairing]);
+       true ->
+            ok
+    end.
+
 perhaps_verbose_c110(P_latest2, S) ->
     case proplists:get_value(private_write_verbose, S#ch_mgr.opts) of
         true ->

test/machi_chain_manager1_converge_demo.erl

@@ -394,15 +394,20 @@ make_partition_list(All_list) ->
     %% [ [{a,b}, {b,a}] ].
 
     [
-      [{a,b}], [],
+      [{c,b}, {c,a}],
-      [{b,a}, {b,c}], [],
+      [{b,c}, {b,a}]
-      [{c,b}, {c,a}, {d,c}]
+    ].
+
+    %% [
+    %%   [{a,b}], [],
+    %%   [{b,a}, {b,c}], [],
+    %%   [{c,b}, {c,a}, {d,c}], [],
     %%   [{c,b}, {c,a}], [],
 
     %%   [{b,a}, {c,a}], [],
     %%   [{a,b}, {c,b}], [],
     %%   [{b,c}, {a,c}]
-    ].
+    %% ].
 
     %% [ [{a,b},{b,c},{c,a}],
     %%   [{a,b}, {b,a}, {a,c},{c,a}] ].