Verbose debugging cruft

src/machi_chain_manager1.erl

@@ -81,7 +81,8 @@
 -define(TO, (2*1000)).                          % default timeout
 
 %% Keep a history of our flowchart execution in the process dictionary.
--define(REACT(T), put(react, [T|get(react)])).
+-define(REACT(T), begin put(ttt, [?LINE|get(ttt)]), put(react, [T|get(react)]) end).
+-define(TTT(), begin put(ttt, [?LINE|get(ttt)]) end).
 
 %% Define the period of private projection stability before we'll
 %% start repair.
@@ -224,6 +225,7 @@ test_read_latest_public_projection(Pid, ReadRepairP) ->
 %% local projection store.
 
 init({MyName, InitMembersDict, MgrOpts}) ->
+put(ttt, [?LINE]),
     random:seed(now()),
     init_remember_down_list(),
     Opt = fun(Key, Default) -> proplists:get_value(Key, MgrOpts, Default) end,
@@ -1017,6 +1019,7 @@ do_react_to_env(#ch_mgr{name=MyName,
                                 consistency_mode=CMode}, NewProj)}
     end;
 do_react_to_env(S) ->
+put(ttt, [?LINE]),
     %% The not_sanes manager counting dictionary is not strictly
     %% limited to flapping scenarios.  (Though the mechanism first
     %% started as a way to deal with rare flapping scenarios.)
@@ -1067,12 +1070,14 @@ do_react_to_env(S) ->
                      S
              end,
         %% Perhaps tell the fitness server to spam everyone.
+?TTT(),
         case random:uniform(100) of
             N when N < 5 ->
-                machi_fitness:send_spam_to_everyone(S#ch_mgr.fitness_svr);
+                machi_fitness:send_spam_to_everyone(S#ch_mgr.fitness_svr),?TTT();
             _ ->
                 ok
         end,
+?TTT(),
         %% NOTE: If we use the fitness server's unfit list at the start, then
         %% we would need to add some kind of poll/check for down members to
         %% check if they are now up.  Instead, our lazy attempt to read from
@@ -1084,10 +1089,14 @@ do_react_to_env(S) ->
         %% jitter smoother by only talking to servers that we believe are fit.
         %% But we will defer such work because it may never be necessary.
         {Res, S3} = react_to_env_A10(S2),
+?TTT(),
         S4 = manage_last_down_list(S3),
         %% When in CP mode, we call the poll function twice: once at the start
         %% of reacting (in state A10) & once after.  This call is the 2nd.
-        {Res, poll_private_proj_is_upi_unanimous(S4)}
+?TTT(),
+        Sx = poll_private_proj_is_upi_unanimous(S4),
+?TTT(),
+        {Res, Sx}
     catch
         throw:{zerf,_}=_Throw ->
             Proj = S#ch_mgr.proj,

test/machi_chain_manager1_converge_demo.erl

@@ -216,7 +216,7 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
       DoIt = fun(Iters, S_min, S_max) ->
                      %% io:format(user, "\nDoIt: top\n\n", []),
                      io:format(user, "DoIt, ", []),
-                     Pids = [spawn(fun() ->
+                     Pids = [{spawn(fun() ->
                                            random:seed(now()),
                                            [begin
                                                 erlang:yield(),
@@ -233,14 +233,26 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
                                                 % timer:sleep(S_max - Elapsed),
                                                 Elapsed
                                             end || _ <- lists:seq(1, Iters)],
-                                           Parent ! done
+                                           Parent ! {done, self()}
-                                   end) || {M_name, MMM} <- MgrNamez ],
+                                   end), M_name} || {M_name, MMM} <- MgrNamez ],
                      [receive
-                          done ->
+                          {done, ThePid} ->
                               ok
                       after 120*1000 ->
-                              exit(icky_timeout)
+                              [begin
-                      end || _ <- Pids]
+                                   case whereis(XX) of
+                                       undefined -> ok;
+                                       XXPid -> {_, XXbin} = process_info(XXPid, backtrace),
+                                                {_, XXdict} = process_info(XXPid, dictionary),
+                                                TTT = proplists:get_value(ttt, XXdict),
+                                                io:format(user, "BACK ~w: ttt=~w\n~s\n", [XX, TTT, XXbin])
+                                   end
+                               end || XX <- [file_server_2] ++
+                                          [a_chmgr,b_chmgr,c_chmgr,d_chmgr,e_chmgr,f_chmgr,g_chmgr,h_chmgr,i_chmgr,j_chmgr] ++
+                                          [a_pstore,b_pstore,c_pstore,d_pstore,e_pstore,f_pstore,g_pstore,h_pstore,i_pstore,j_pstore] ++
+                                          [a_fitness,b_fitness,c_fitness,d_fitness,e_fitness,f_fitness,g_fitness,h_fitness,i_fitness,j_fitness] ],
+                              exit({icky_timeout, M_name})
+                      end || {ThePid,M_name} <- Pids]
              end,
 
       %% machi_partition_simulator:reset_thresholds(10, 50),