WIP: added trim_proj_with_all_hosed(). Convergence fails

This is the first effort for trying to modify the projection
to take into account the all_hosed list.  It doesn't work,
but there are a couple things here that need to move together.

1. The implementation & use of trim_proj_with_all_hosed() itself.

2. Any additional compensation that needs to take place, e.g.
   in react_to_env_A20().  I added some hack experiment there,
   but more work in A20 is needed at least, I think.

prototype/chain-manager/Makefile

@@ -23,7 +23,7 @@ eunit:
 
 pulse: compile
 	env USE_PULSE=1 $(REBAR_BIN) skip_deps=true clean compile
-	env USE_PULSE=1 $(REBAR_BIN) skip_deps=true -D PULSE eunit
+	env USE_PULSE=1 $(REBAR_BIN) -v skip_deps=true -D PULSE eunit
 
 CONC_ARGS = --pz ./.eunit --treat_as_normal shutdown --after_timeout 1000
 

prototype/chain-manager/RUNLOG

@@ -0,0 +1,431 @@
+/Users/fritchie/b/src/rebar/rebar get-deps
+==> goldrush (get-deps)
+==> lager (get-deps)
+==> chain-manager (get-deps)
+/Users/fritchie/b/src/rebar/rebar compile
+==> goldrush (compile)
+Compiled src/gre.erl
+Compiled src/gr_param_sup.erl
+Compiled src/gr_sup.erl
+Compiled src/gr_manager_sup.erl
+Compiled src/gr_counter_sup.erl
+Compiled src/gr_manager.erl
+Compiled src/gr_context.erl
+Compiled src/gr_app.erl
+Compiled src/gr_param.erl
+Compiled src/glc_ops.erl
+Compiled src/gr_counter.erl
+Compiled src/glc.erl
+Compiled src/glc_lib.erl
+Compiled src/glc_code.erl
+==> lager (compile)
+Compiled src/lager_util.erl
+Compiled src/lager_transform.erl
+Compiled src/lager_sup.erl
+Compiled src/lager_msg.erl
+Compiled src/lager_handler_watcher_sup.erl
+Compiled src/lager_handler_watcher.erl
+Compiled src/lager_stdlib.erl
+Compiled src/lager_trunc_io.erl
+Compiled src/lager_default_formatter.erl
+Compiled src/lager_format.erl
+Compiled src/lager_crash_log.erl
+Compiled src/lager_console_backend.erl
+Compiled src/lager_file_backend.erl
+Compiled src/lager_config.erl
+Compiled src/lager_backend_throttle.erl
+Compiled src/lager_app.erl
+Compiled src/lager.erl
+Compiled src/error_logger_lager_h.erl
+==> chain-manager (compile)
+Compiled src/machi_util.erl
+Compiled src/machi_flu0.erl
+Compiled src/machi_chain_manager0.erl
+Compiled src/machi_chain_manager1.erl
+/Users/fritchie/b/src/rebar/rebar -v skip_deps=true eunit
+INFO:  Looking for lager-2.0.1 ; found lager-2.0.1 at /Users/fritchie/b/src/machi/prototype/chain-manager/deps/lager
+INFO:  Looking for lager-2.0.1 ; found lager-2.0.1 at /Users/fritchie/b/src/machi/prototype/chain-manager/deps/lager
+INFO:  Looking for goldrush-.* ; found goldrush-0.1.5 at /Users/fritchie/b/src/machi/prototype/chain-manager/deps/goldrush
+INFO:  Looking for goldrush-.* ; found goldrush-0.1.5 at /Users/fritchie/b/src/machi/prototype/chain-manager/deps/goldrush
+==> chain-manager (eunit)
+INFO:  sh info:
+	cwd: "/Users/fritchie/b/src/machi/prototype/chain-manager"
+	cmd: cp -R src/machi_util.erl src/machi_flu0.erl src/machi_chain_manager1.erl src/machi_chain_manager0.erl test/pulse_util/lamport_clock.erl test/pulse_util/handle_errors.erl test/pulse_util/event_logger.erl test/machi_util_test.erl test/machi_partition_simulator.erl test/machi_flu0_test.erl test/machi_chain_manager1_test.erl test/machi_chain_manager1_pulse.erl test/machi_chain_manager0_test.erl ".eunit"
+Compiled src/machi_flu0.erl
+Compiled src/machi_util.erl
+Compiled test/pulse_util/lamport_clock.erl
+Compiled test/pulse_util/handle_errors.erl
+Compiled src/machi_chain_manager0.erl
+Compiled test/pulse_util/event_logger.erl
+Compiled test/machi_partition_simulator.erl
+Compiled test/machi_util_test.erl
+Compiled test/machi_flu0_test.erl
+Compiled test/machi_chain_manager1_pulse.erl
+Compiled test/machi_chain_manager1_test.erl
+Compiled src/machi_chain_manager1.erl
+Compiled test/machi_chain_manager0_test.erl
+======================== EUnit ========================
+machi_util_test: repair_merge_test_ (module 'machi_util_test')...............................................................................................................................................................................................................................................................................................................
+OK, passed 300 tests
+[1.733 s] ok
+module 'machi_util'
+module 'machi_partition_simulator'
+module 'machi_flu0_test'
+  machi_flu0_test: repair_status_test...[0.002 s] ok
+  machi_flu0_test: concuerror1_test...ok
+  machi_flu0_test: concuerror2_test...ok
+  machi_flu0_test: concuerror3_test...ok
+  machi_flu0_test: concuerror4_test...ok
+  machi_flu0_test: proj_store_test...ok
+  machi_flu0_test: wedge_test...ok
+  machi_flu0_test: proj0_test...[0.001 s] ok
+  [done in 0.026 s]
+module 'machi_flu0'
+module 'machi_chain_manager1_test'
+  machi_chain_manager1_test: smoke0_test...[{epoch,1},{author,a},{upi,[a,b,c]},{repair,[]},{down,[]},{d,[{author_proc,call},{ps,[]},{nodes_up,[a,b,c]}]},{d2,[]}]
+[{epoch,1},{author,a},{upi,[a]},{repair,[]},{down,[b,c]},{d,[{author_proc,call},{ps,[{a,b},{c,a},{b,c},{c,b}]},{nodes_up,[a]}]},{d2,[]}]
+[{epoch,1},{author,a},{upi,[a,b]},{repair,[]},{down,[c]},{d,[{author_proc,call},{ps,[{c,a},{b,c}]},{nodes_up,[a,b]}]},{d2,[]}]
+[{epoch,1},{author,a},{upi,[a,b]},{repair,[]},{down,[c]},{d,[{author_proc,call},{ps,[{c,a},{b,c}]},{nodes_up,[a,b]}]},{d2,[]}]
+[{epoch,1},{author,a},{upi,[a]},{repair,[]},{down,[b,c]},{d,[{author_proc,call},{ps,[{a,b},{b,a},{a,c},{c,a},{b,c}]},{nodes_up,[a]}]},{d2,[]}]
+[0.003 s] ok
+  machi_chain_manager1_test: smoke1_test...[0.001 s] ok
+  machi_chain_manager1_test: nonunanimous_setup_and_fix_test...x x
+_XX {not_unanimous,{projection,1,<<110,253,187,8,20,172,231,72,56,72,97,162,22,0,234,37,105,166,11,10>>,[a,b],[a],{1425,368983,392344},b,[b],[],[{hackhack,103}],[]},[{unanimous_flus,[b]},{not_unanimous_flus,[{projection,1,<<125,66,48,143,209,3,150,235,225,93,216,21,70,165,243,106,106,243,105,176>>,[a,b],[b],{1425,368983,392326},a,[a],[],[{hackhack,100}],[]}]},{all_members_replied,true}]}
+[0.001 s] ok
+  machi_chain_manager1_test: convergence_demo_test_...16:49:43.396 a uses: [{epoch,1},{author,a},{upi,[a,b,c,d]},{repair,[]},{down,[]},{d,[{author_proc,call},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+XX1 1
+16:49:43.396 b uses: [{epoch,1},{author,a},{upi,[a,b,c,d]},{repair,[]},{down,[]},{d,[{author_proc,call},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.396 c uses: [{epoch,1},{author,a},{upi,[a,b,c,d]},{repair,[]},{down,[]},{d,[{author_proc,call},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+QQ unanimous
+? MGR : make_projection_summary ( QQP2 ) [{epoch,1},{author,a},{upi,[a,b,c,d]},{repair,[]},{down,[]},{d,[{author_proc,call},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[]}]
+QQE2 [{unanimous_flus,[a,b,c,d]},
+      {not_unanimous_flus,[]},
+      {all_members_replied,true}]
+16:49:43.397 d uses: [{epoch,1},{author,a},{upi,[a,b,c,d]},{repair,[]},{down,[]},{d,[{author_proc,call},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.398 c uses: [{epoch,4},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.399 d uses: [{epoch,10},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{d,a},{b,c},{b,d},{d,b},{c,d}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.401 d uses: [{epoch,19},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.403 d uses: [{epoch,26},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.404 d uses: [{epoch,28},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.405 d uses: [{epoch,34},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{b,a},{a,c},{c,a},{a,d},{b,c},{c,b},{d,b},{c,d},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.406 d uses: [{epoch,38},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.408 d uses: [{epoch,45},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.411 d uses: [{epoch,63},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.412 d uses: [{epoch,65},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.414 d uses: [{epoch,71},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.417 d uses: [{epoch,85},{author,d},{upi,[d]},{repair,[a,c]},{down,[b]},{d,[{author_proc,react},{ps,[{d,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.418 d uses: [{epoch,87},{author,d},{upi,[d]},{repair,[a,c,b]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.421 d uses: [{epoch,98},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.423 d uses: [{epoch,104},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.424 d uses: [{epoch,107},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.425 d uses: [{epoch,110},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.426 d uses: [{epoch,113},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{b,c},{c,b},{d,b},{c,d},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.426 d uses: [{epoch,116},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,b},{a,c},{a,d},{c,b},{b,d},{d,b},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.427 d uses: [{epoch,118},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{c,a},{a,d},{d,a},{b,c},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.428 d uses: [{epoch,121},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.428 d uses: [{epoch,123},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{b,a},{a,d},{b,d},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.430 d uses: [{epoch,129},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.433 d uses: [{epoch,139},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.436 d uses: [{epoch,149},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.503 c uses: [{epoch,149},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.506 b uses: [{epoch,153},{author,b},{upi,[a,b,c,d]},{repair,[]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.507 b uses: [{epoch,157},{author,b},{upi,[a,b,c,d]},{repair,[]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.508 b uses: [{epoch,161},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.509 b uses: [{epoch,166},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.510 b uses: [{epoch,170},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{c,a},{a,d},{d,a},{b,c},{c,b},{d,b},{c,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.513 b uses: [{epoch,181},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.515 b uses: [{epoch,184},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.516 b uses: [{epoch,187},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.516 b uses: [{epoch,189},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{b,a},{c,a},{a,d},{d,a},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.517 b uses: [{epoch,192},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.518 b uses: [{epoch,196},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.520 b uses: [{epoch,200},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.521 b uses: [{epoch,202},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.523 b uses: [{epoch,209},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.524 b uses: [{epoch,211},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.526 b uses: [{epoch,220},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.528 b uses: [{epoch,225},{author,b},{upi,[b]},{repair,[c]},{down,[a,d]},{d,[{author_proc,react},{ps,[{a,b},{a,c},{a,d},{d,a},{b,d},{d,b},{c,d}]},{nodes_up,[b,c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.529 b uses: [{epoch,228},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{a,d},{b,c},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.530 b uses: [{epoch,232},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{c,a},{a,d},{d,a},{c,b},{b,d},{d,b}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.531 b uses: [{epoch,235},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.534 b uses: [{epoch,243},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{d,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.563 c uses: [{epoch,243},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{d,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.563 c uses: [{epoch,245},{author,c},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.567 c uses: [{epoch,261},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{a,b},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.567 c uses: [{epoch,263},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.568 c uses: [{epoch,266},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{a,b},{a,c},{c,a},{a,d},{b,c},{d,b},{c,d},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.570 c uses: [{epoch,273},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.571 c uses: [{epoch,279},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.573 c uses: [{epoch,284},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.573 c uses: [{epoch,286},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.575 c uses: [{epoch,294},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.576 c uses: [{epoch,296},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{b,a},{a,c},{c,a},{d,a},{b,c},{c,b},{d,b},{c,d},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.576 c uses: [{epoch,299},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.579 c uses: [{epoch,311},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.580 c uses: [{epoch,314},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.582 c uses: [{epoch,322},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{b,a},{c,a},{a,d},{d,a},{b,c},{c,b},{d,b},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.583 c uses: [{epoch,325},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.583 c uses: [{epoch,327},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{a,b},{c,a},{c,b},{c,d}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.631 a uses: [{epoch,329},{author,a},{upi,[a,b,c,d]},{repair,[]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.633 a uses: [{epoch,338},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{a,c},{c,a},{a,d},{d,a},{c,b},{b,d},{c,d},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.635 a uses: [{epoch,344},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.635 a uses: [{epoch,347},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{c,d},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.636 a uses: [{epoch,349},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.636 a uses: [{epoch,351},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{c,a},{a,d},{d,b},{c,d},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.637 a uses: [{epoch,354},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{c,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.639 a uses: [{epoch,362},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{c,d},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.640 a uses: [{epoch,365},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.641 a uses: [{epoch,368},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{a,c},{c,a},{a,d},{d,a},{b,d},{d,b},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.642 a uses: [{epoch,372},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.642 a uses: [{epoch,374},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.643 a uses: [{epoch,376},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{b,c},{c,b},{b,d},{d,b},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.644 a uses: [{epoch,380},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{b,a},{c,a},{d,a},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.644 a uses: [{epoch,382},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{c,a},{a,d},{c,b}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.647 a uses: [{epoch,394},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.648 a uses: [{epoch,398},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.649 a uses: [{epoch,402},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.650 a uses: [{epoch,404},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.651 a uses: [{epoch,409},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.652 a uses: [{epoch,414},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{b,d},{d,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.654 a uses: [{epoch,420},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{c,b},{d,b},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.656 a uses: [{epoch,427},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.657 a uses: [{epoch,431},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.658 a uses: [{epoch,434},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{b,c},{c,b},{d,b},{c,d},{d,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+
+SET always_last_partitions ON ... we should see convergence to correct chains.
+16:49:43.701 d uses: [{epoch,434},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{b,c},{c,b},{d,b},{c,d},{d,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.799 c uses: [{epoch,434},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{b,c},{c,b},{d,b},{c,d},{d,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:43.999 a uses: [{epoch,436},{author,a},{upi,[a,c]},{repair,[d]},{down,[b]},{d,[{repair_airquote_done,{we_agree,434}},{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,43}}}]}]
+16:49:44.143 b uses: [{epoch,437},{author,b},{upi,[b]},{repair,[c,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,44}}}]}]
+16:49:44.171 d uses: [{epoch,438},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,44}}}]}]
+16:49:44.206 c uses: [{epoch,438},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,44}}}]}]
+16:49:44.403 a uses: [{epoch,439},{author,a},{upi,[a,c]},{repair,[d]},{down,[b]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,44}}}]}]
+16:49:44.695 d uses: [{epoch,440},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,44}}}]}]
+16:49:44.740 c uses: [{epoch,440},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,44}}}]}]
+16:49:44.808 a uses: [{epoch,441},{author,a},{upi,[a,c]},{repair,[d]},{down,[b]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,44}}}]}]
+16:49:44.810 b uses: [{epoch,442},{author,b},{upi,[b]},{repair,[c,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,44}}}]}]
+16:49:45.100 d uses: [{epoch,443},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,45}}}]}]
+16:49:45.144 c uses: [{epoch,443},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,45}}}]}]
+16:49:45.212 a uses: [{epoch,444},{author,a},{upi,[a,c]},{repair,[d]},{down,[b]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,45}}}]}]
+16:49:45.460 b uses: [{epoch,445},{author,b},{upi,[b]},{repair,[c,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,45}}}]}]
+16:49:45.502 d uses: [{epoch,446},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,45}}}]}]
+16:49:45.549 c uses: [{epoch,446},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,45}}}]}]
+16:49:45.618 a uses: [{epoch,447},{author,a},{upi,[a,c]},{repair,[d]},{down,[b]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,45}}}]}]
+16:49:46.025 d uses: [{epoch,448},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,46}}}]}]
+16:49:46.031 c uses: [{epoch,448},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,46}}}]}]
+16:49:46.117 b uses: [{epoch,449},{author,b},{upi,[b]},{repair,[c,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,46}}}]}]
+16:49:46.427 a uses: [{epoch,450},{author,a},{upi,[a,c]},{repair,[d]},{down,[b]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,46}}}]}]
+16:49:46.531 d uses: [{epoch,451},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,46}}}]}]
+16:49:46.580 c uses: [{epoch,451},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,46}}}]}]
+16:49:46.790 b uses: [{epoch,452},{author,b},{upi,[b]},{repair,[c,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,46}}}]}]
+16:49:46.833 a uses: [{epoch,453},{author,a},{upi,[a,c]},{repair,[d]},{down,[b]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,46}}}]}]
+16:49:47.039 d uses: [{epoch,454},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,47}}}]}]
+16:49:47.043 c uses: [{epoch,454},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,47}}}]}]
+16:49:47.237 a uses: [{epoch,455},{author,a},{upi,[a,c]},{repair,[d]},{down,[b]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,47}}}]}]
+16:49:47.451 b uses: [{epoch,456},{author,b},{upi,[b]},{repair,[c,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,47}}}]}]
+16:49:47.492 d uses: [{epoch,457},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,47}}}]}]
+16:49:47.538 c uses: [{epoch,457},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,47}}}]}]
+16:49:47.644 a uses: [{epoch,458},{author,a},{upi,[a,c]},{repair,[d]},{down,[b]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,47}}}]}]
+16:49:48.016 d uses: [{epoch,459},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,48}}}]}]
+16:49:48.022 c uses: [{epoch,459},{author,d},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,48}}}]}]
+16:49:48.049 a uses: [{epoch,460},{author,a},{upi,[a,c]},{repair,[d]},{down,[b]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,48}}}]}]
+16:49:48.111 b uses: [{epoch,461},{author,b},{upi,[b]},{repair,[c,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,b}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,48}}}]}]
+{FLAP: c flaps 5}!
+{FLAP: a flaps 5}!
+{FLAP: c flaps 6}!
+{FLAP: a flaps 6}!
+
+SET always_last_partitions OFF ... let loose the dogs of war!
+16:49:56.562 a uses: [{epoch,466},{author,a},{upi,[a,c]},{repair,[d,b]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.565 a uses: [{epoch,471},{author,a},{upi,[a,c]},{repair,[d,b]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.568 a uses: [{epoch,479},{author,a},{upi,[a,c]},{repair,[d,b]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.569 a uses: [{epoch,481},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.571 a uses: [{epoch,487},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.572 a uses: [{epoch,490},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.573 a uses: [{epoch,493},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.575 a uses: [{epoch,496},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.580 a uses: [{epoch,509},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.582 a uses: [{epoch,513},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.583 a uses: [{epoch,516},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{a,c},{d,a},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.585 a uses: [{epoch,520},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{c,d}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.587 a uses: [{epoch,524},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.589 a uses: [{epoch,529},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.590 a uses: [{epoch,532},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.591 a uses: [{epoch,535},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{d,a},{b,c},{c,b},{d,c}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.592 a uses: [{epoch,537},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.594 a uses: [{epoch,540},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.595 a uses: [{epoch,542},{author,a},{upi,[a]},{repair,[]},{down,[b,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d}]},{nodes_up,[a]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.598 a uses: [{epoch,547},{author,a},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.622 c uses: [{epoch,555},{author,c},{upi,[a]},{repair,[b,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.625 c uses: [{epoch,561},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.626 c uses: [{epoch,565},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.627 c uses: [{epoch,569},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{c,a},{b,c},{c,b},{b,d},{d,b},{c,d}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.628 c uses: [{epoch,573},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.629 c uses: [{epoch,575},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.630 c uses: [{epoch,578},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.632 c uses: [{epoch,582},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.636 d uses: [{epoch,596},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.636 c uses: [{epoch,596},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{a,d},{d,a},{b,c},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.640 d uses: [{epoch,604},{author,d},{upi,[d]},{repair,[a]},{down,[b,c]},{d,[{author_proc,react},{ps,[{a,c},{d,b},{c,d},{d,c}]},{nodes_up,[a,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.643 d uses: [{epoch,610},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.646 d uses: [{epoch,615},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,d},{d,a},{c,b},{d,b},{c,d}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.647 d uses: [{epoch,617},{author,d},{upi,[d]},{repair,[c]},{down,[a,b]},{d,[{author_proc,react},{ps,[{a,b},{c,a},{a,d},{c,b},{b,d}]},{nodes_up,[c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.649 d uses: [{epoch,623},{author,d},{upi,[d]},{repair,[c,a,b]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.652 d uses: [{epoch,627},{author,d},{upi,[d]},{repair,[c,a,b]},{down,[]},{d,[{author_proc,react},{ps,[{c,a},{b,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.653 d uses: [{epoch,629},{author,d},{upi,[d]},{repair,[c,a,b]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.659 d uses: [{epoch,643},{author,d},{upi,[d]},{repair,[c,a,b]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.662 d uses: [{epoch,651},{author,d},{upi,[d]},{repair,[c,a,b]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.668 d uses: [{epoch,668},{author,d},{upi,[d]},{repair,[c,a,b]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.677 d uses: [{epoch,685},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.679 d uses: [{epoch,688},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.680 d uses: [{epoch,691},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.685 d uses: [{epoch,699},{author,d},{upi,[d]},{repair,[a,b,c]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.688 d uses: [{epoch,701},{author,d},{upi,[d]},{repair,[a,c]},{down,[b]},{d,[{author_proc,react},{ps,[{d,b}]},{nodes_up,[a,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.690 d uses: [{epoch,706},{author,d},{upi,[d]},{repair,[a,c,b]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.690 d uses: [{epoch,708},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,c},{d,a},{b,c},{c,b},{b,d},{d,b},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.697 d uses: [{epoch,724},{author,d},{upi,[d]},{repair,[]},{down,[a,b,c]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.753 b uses: [{epoch,729},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.754 b uses: [{epoch,731},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.759 b uses: [{epoch,738},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{a,c},{c,a},{a,d},{d,a},{b,c},{d,b},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.767 b uses: [{epoch,753},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.768 b uses: [{epoch,755},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{b,a},{a,c},{d,a},{b,c},{b,d},{c,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.773 b uses: [{epoch,765},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.778 b uses: [{epoch,775},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.784 b uses: [{epoch,792},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.787 b uses: [{epoch,799},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.790 b uses: [{epoch,807},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.795 b uses: [{epoch,821},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.797 b uses: [{epoch,825},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.798 b uses: [{epoch,828},{author,b},{upi,[b]},{repair,[c,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,b},{d,a}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.799 b uses: [{epoch,832},{author,b},{upi,[b]},{repair,[c,d,a]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.800 b uses: [{epoch,834},{author,b},{upi,[b]},{repair,[c,d,a]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.801 b uses: [{epoch,837},{author,b},{upi,[b]},{repair,[c,d,a]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.802 b uses: [{epoch,840},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{d,a},{b,c},{c,b},{b,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.803 b uses: [{epoch,842},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.804 b uses: [{epoch,845},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{c,a},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.808 b uses: [{epoch,854},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{a,d},{d,a},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.808 b uses: [{epoch,856},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.815 b uses: [{epoch,870},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{a,c},{c,a},{a,d},{d,a},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.818 b uses: [{epoch,877},{author,b},{upi,[b]},{repair,[]},{down,[a,c,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{d,c}]},{nodes_up,[b]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.876 c uses: [{epoch,879},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.877 c uses: [{epoch,881},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.878 c uses: [{epoch,883},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.879 c uses: [{epoch,885},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.881 c uses: [{epoch,890},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.882 c uses: [{epoch,892},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{b,a},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.884 c uses: [{epoch,899},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.886 c uses: [{epoch,904},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{a,b},{a,c},{c,a},{a,d},{b,c},{c,b},{b,d},{c,d},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.887 c uses: [{epoch,906},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.888 c uses: [{epoch,910},{author,c},{upi,[c]},{repair,[]},{down,[a,b,d]},{d,[{author_proc,react},{ps,[{a,b},{b,a},{a,c},{c,a},{a,d},{d,a},{b,c},{c,b},{b,d},{d,b},{c,d},{d,c}]},{nodes_up,[c]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.894 c uses: [{epoch,925},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.896 c uses: [{epoch,929},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.902 c uses: [{epoch,943},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[islands_not_supported]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:56.903 c uses: [{epoch,945},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+
+SET always_last_partitions ON ... we should see convergence to correct chains2.
+16:49:56.946 d uses: [{epoch,945},{author,c},{upi,[c]},{repair,[a,b,d]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,56}}}]}]
+16:49:57.049 c uses: [{epoch,947},{author,c},{upi,[c]},{repair,[b,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,57}}}]}]
+16:49:57.147 b uses: [{epoch,948},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,57}}}]}]
+16:49:57.350 d uses: [{epoch,948},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,57}}}]}]
+16:49:57.579 a uses: [{epoch,949},{author,a},{upi,[a]},{repair,[b,d]},{down,[c]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,57}}}]}]
+16:49:57.625 c uses: [{epoch,950},{author,c},{upi,[c]},{repair,[b,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,57}}}]}]
+16:49:57.760 d uses: [{epoch,951},{author,d},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,57}}}]}]
+16:49:57.959 b uses: [{epoch,951},{author,d},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,57}}}]}]
+16:49:58.200 c uses: [{epoch,952},{author,c},{upi,[c]},{repair,[b,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,58}}}]}]
+16:49:58.321 a uses: [{epoch,953},{author,a},{upi,[a]},{repair,[b,d]},{down,[c]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,58}}}]}]
+16:49:58.366 b uses: [{epoch,954},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,58}}}]}]
+16:49:58.572 d uses: [{epoch,954},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,58}}}]}]
+16:49:58.802 c uses: [{epoch,955},{author,c},{upi,[c]},{repair,[b,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,58}}}]}]
+16:49:58.976 d uses: [{epoch,956},{author,d},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,58}}}]}]
+16:49:59.047 a uses: [{epoch,957},{author,a},{upi,[a]},{repair,[b,d]},{down,[c]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,59}}}]}]
+16:49:59.173 b uses: [{epoch,958},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,59}}}]}]
+16:49:59.382 d uses: [{epoch,958},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,59}}}]}]
+16:49:59.383 c uses: [{epoch,959},{author,c},{upi,[c]},{repair,[b,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,59}}}]}]
+16:49:59.576 b uses: [{epoch,960},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,59}}}]}]
+16:49:59.778 a uses: [{epoch,961},{author,a},{upi,[a]},{repair,[b,d]},{down,[c]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,59}}}]}]
+16:49:59.784 d uses: [{epoch,962},{author,d},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,59}}}]}]
+16:49:59.957 c uses: [{epoch,963},{author,c},{upi,[c]},{repair,[b,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,59}}}]}]
+16:49:59.980 b uses: [{epoch,964},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,49,59}}}]}]
+16:50:00.187 d uses: [{epoch,964},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,0}}}]}]
+16:50:00.532 a uses: [{epoch,965},{author,a},{upi,[a]},{repair,[b,d]},{down,[c]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,0}}}]}]
+16:50:00.552 c uses: [{epoch,966},{author,c},{upi,[c]},{repair,[b,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,0}}}]}]
+16:50:00.590 d uses: [{epoch,967},{author,d},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,0}}}]}]
+16:50:00.788 b uses: [{epoch,967},{author,d},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,0}}}]}]
+16:50:01.136 c uses: [{epoch,968},{author,c},{upi,[c]},{repair,[b,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,1}}}]}]
+16:50:01.194 b uses: [{epoch,969},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,1}}}]}]
+16:50:01.263 a uses: [{epoch,970},{author,a},{upi,[a]},{repair,[b,d]},{down,[c]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,1}}}]}]
+16:50:01.398 d uses: [{epoch,971},{author,d},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,1}}}]}]
+16:50:01.600 b uses: [{epoch,971},{author,d},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,1}}}]}]
+16:50:01.716 c uses: [{epoch,972},{author,c},{upi,[c]},{repair,[b,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,1}}}]}]
+16:50:01.803 d uses: [{epoch,973},{author,d},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,1}}}]}]
+16:50:02.002 a uses: [{epoch,974},{author,a},{upi,[a]},{repair,[b,d]},{down,[c]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,2}}}]}]
+16:50:02.003 b uses: [{epoch,975},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,2}}}]}]
+16:50:02.209 d uses: [{epoch,975},{author,b},{upi,[b]},{repair,[a,c,d]},{down,[]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,2}}}]}]
+16:50:02.291 c uses: [{epoch,976},{author,c},{upi,[c]},{repair,[b,d]},{down,[a]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[b,c,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,2}}}]}]
+{FLAP: b flaps 4}!
+{FLAP: b flaps 5}!
+{FLAP: d flaps 6}!
+{FLAP: b flaps 6}!
+16:50:02.761 a uses: [{epoch,977},{author,a},{upi,[a]},{repair,[b,d]},{down,[c]},{d,[{author_proc,react},{ps,[{a,c}]},{nodes_up,[a,b,d]}]},{d2,[{network_islands,[na_reset_by_always]},{hooray,{v2,{2015,3,3},{16,50,2}}}]}]
+
+SET always_last_partitions ON ... we should see convergence to correct chains3.
+16:50:12.096 c uses: [{epoch,978},{author,c},{upi,[c]},{repair,[b,d,a]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,12}}}]}]
+16:50:12.196 b uses: [{epoch,978},{author,c},{upi,[c]},{repair,[b,d,a]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,12}}}]}]
+16:50:12.294 a uses: [{epoch,978},{author,c},{upi,[c]},{repair,[b,d,a]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,12}}}]}]
+16:50:12.464 d uses: [{epoch,978},{author,c},{upi,[c]},{repair,[b,d,a]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,12}}}]}]
+16:50:12.503 c uses: [{epoch,980},{author,c},{upi,[c,b]},{repair,[d,a]},{down,[]},{d,[{repair_airquote_done,{we_agree,978}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,12}}}]}]
+16:50:12.601 b uses: [{epoch,980},{author,c},{upi,[c,b]},{repair,[d,a]},{down,[]},{d,[{repair_airquote_done,{we_agree,978}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,12}}}]}]
+16:50:12.700 a uses: [{epoch,980},{author,c},{upi,[c,b]},{repair,[d,a]},{down,[]},{d,[{repair_airquote_done,{we_agree,978}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,12}}}]}]
+16:50:12.871 d uses: [{epoch,980},{author,c},{upi,[c,b]},{repair,[d,a]},{down,[]},{d,[{repair_airquote_done,{we_agree,978}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,12}}}]}]
+16:50:12.910 c uses: [{epoch,982},{author,c},{upi,[c,b,d]},{repair,[a]},{down,[]},{d,[{repair_airquote_done,{we_agree,980}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,12}}}]}]
+16:50:13.008 b uses: [{epoch,982},{author,c},{upi,[c,b,d]},{repair,[a]},{down,[]},{d,[{repair_airquote_done,{we_agree,980}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,13}}}]}]
+16:50:13.106 a uses: [{epoch,982},{author,c},{upi,[c,b,d]},{repair,[a]},{down,[]},{d,[{repair_airquote_done,{we_agree,980}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,13}}}]}]
+16:50:13.278 d uses: [{epoch,982},{author,c},{upi,[c,b,d]},{repair,[a]},{down,[]},{d,[{repair_airquote_done,{we_agree,980}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,13}}}]}]
+16:50:13.316 c uses: [{epoch,984},{author,c},{upi,[c,b,d,a]},{repair,[]},{down,[]},{d,[{repair_airquote_done,{we_agree,982}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,13}}}]}]
+16:50:13.414 b uses: [{epoch,984},{author,c},{upi,[c,b,d,a]},{repair,[]},{down,[]},{d,[{repair_airquote_done,{we_agree,982}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,13}}}]}]
+16:50:13.513 a uses: [{epoch,984},{author,c},{upi,[c,b,d,a]},{repair,[]},{down,[]},{d,[{repair_airquote_done,{we_agree,982}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,13}}}]}]
+16:50:13.684 d uses: [{epoch,984},{author,c},{upi,[c,b,d,a]},{repair,[]},{down,[]},{d,[{repair_airquote_done,{we_agree,982}},{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,13}}}]}]
+16:50:14.088 d uses: [{epoch,986},{author,d},{upi,[c,b,d,a]},{repair,[]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,14}}}]}]
+16:50:14.128 c uses: [{epoch,986},{author,d},{upi,[c,b,d,a]},{repair,[]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,14}}}]}]
+16:50:14.224 b uses: [{epoch,986},{author,d},{upi,[c,b,d,a]},{repair,[]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,14}}}]}]
+16:50:14.326 a uses: [{epoch,986},{author,d},{upi,[c,b,d,a]},{repair,[]},{down,[]},{d,[{author_proc,react},{ps,[]},{nodes_up,[a,b,c,d]}]},{d2,[{network_islands,[[a,b,c,d]]},{hooray,{v2,{2015,3,3},{16,50,14}}}]}]
+Tue Mar  3 16:50:20 JST 2015
+
+[36.758 s] ok
+  [done in 36.773 s]
+module 'machi_chain_manager1_pulse'
+module 'machi_chain_manager1'
+module 'machi_chain_manager0_test'
+  machi_chain_manager0_test: smoke0_test...[0.001 s] ok
+  machi_chain_manager0_test: calc_projection_test_...
+{ time ( ) , Hack } {{16,50,20},0}
+.......................
+	NotCounted list was written to /tmp/manager-test.16.50.22
+	OKs length = 135
+	Transitions hit = 109
+	NotCounted length = 26
+	Least-counted transition = {{[a,c,b],[b]},1}
+	Most-counted transition  = {{[b],[b]},21051}
+[2.626 s] ok
+  machi_chain_manager0_test: pass1_smoke_test...ok
+  machi_chain_manager0_test: fail1_smoke_test...ok
+  machi_chain_manager0_test: fail2_smoke_test...ok
+  machi_chain_manager0_test: fail3_smoke_test...ok
+  machi_chain_manager0_test: fail4_smoke_test...ok
+  [done in 2.647 s]
+module 'machi_chain_manager0'
+module 'lamport_clock'
+module 'handle_errors'
+module 'event_logger'
+=======================================================
+  All 20 tests passed.
+
+=INFO REPORT==== 3-Mar-2015::16:50:22 ===
+    application: pulse
+    exited: stopped
+    type: temporary
+
+=INFO REPORT==== 3-Mar-2015::16:50:22 ===
+    application: inets
+    exited: killed
+    type: temporary

prototype/chain-manager/include/machi.hrl

@@ -45,16 +45,19 @@
             dbg2            :: list()  %proplist(), is not checksummed
         }).
 
+-define(NOT_FLAPPING, {0,0,0}).
+
 -record(ch_mgr, {
           init_finished   :: boolean(),
           name            :: m_server(),
           proj            :: #projection{},
           proj_history    :: queue(),
           myflu           :: pid() | atom(),
+          flap_limit      :: non_neg_integer(),
           %%
           runenv          :: list(), %proplist()
           opts            :: list(),  %proplist()
-flaps=0 :: integer(),
+          flaps=0         :: integer(),
 
           %% Deprecated ... TODO: remove when old test unit test code is removed
           proj_proposed   :: 'none' | #projection{}

prototype/chain-manager/src/machi_chain_manager1.erl

@@ -2,7 +2,7 @@
 %%
 %% Machi: a small village of replicated files
 %%
-%% Copyright (c) 2014 Basho Technologies, Inc.  All Rights Reserved.
+%% Copyright (c) 2014-2015 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
@@ -50,7 +50,8 @@
          test_calc_proposed_projection/1,
          test_write_proposed_projection/1,
          test_read_latest_public_projection/2,
-         test_react_to_env/1]).
+         test_react_to_env/1,
+         get_all_hosed/1]).
 
 -ifdef(EQC).
 -include_lib("eqc/include/eqc.hrl").
@@ -113,16 +114,22 @@ init({MyName, All_list, MyFLUPid, MgrOpts}) ->
               {seed, now()},
               {network_partitions, []},
               {network_islands, []},
+              {flapping_i, []},
               {up_nodes, not_init_yet}],
     BestProj = make_initial_projection(MyName, All_list, All_list,
-                                       [], [{author_proc, init_best}]),
+                                       [], []),
     NoneProj = make_initial_projection(MyName, All_list, [],
-                                       [], [{author_proc, init_none}]),
+                                       [], []),
     S = #ch_mgr{init_finished=false,
                 name=MyName,
                 proj=NoneProj,
                 proj_history=queue:new(),
                 myflu=MyFLUPid, % pid or atom local name
+                %% TODO 2015-03-04: revisit, should this constant be bigger?
+                %% Yes, this should be bigger, but it's a hack.  There is
+                %% no guarantee that all parties will advance to a minimum
+                %% flap awareness in the amount of time that this mgr will.
+                flap_limit=length(All_list) + 50,
                 runenv=RunEnv,
                 opts=MgrOpts},
 
@@ -145,7 +152,7 @@ handle_call(_Call, _From, #ch_mgr{init_finished=false} = S) ->
 handle_call({calculate_projection_internal_old}, _From,
             #ch_mgr{name=MyName}=S) ->
     RelativeToServer = MyName,
-    {Reply, S2} = calc_projection(S, RelativeToServer, [{author_proc, call}]),
+    {Reply, S2} = calc_projection(S, RelativeToServer),
     {reply, Reply, S2};
 handle_call({test_write_proposed_projection}, _From, S) ->
     if S#ch_mgr.proj_proposed == none ->
@@ -161,7 +168,7 @@ handle_call({stop}, _From, S) ->
 handle_call({test_calc_projection, KeepRunenvP}, _From,
             #ch_mgr{name=MyName}=S) ->
     RelativeToServer = MyName,
-    {P, S2} = calc_projection(S, RelativeToServer, [{author_proc, call}]),
+    {P, S2} = calc_projection(S, RelativeToServer),
     {reply, {ok, P}, if KeepRunenvP -> S2;
                         true        -> S
                      end};
@@ -180,7 +187,7 @@ handle_cast(_Cast, #ch_mgr{init_finished=false} = S) ->
     {noreply, S};
 handle_cast({test_calc_proposed_projection}, #ch_mgr{name=MyName}=S) ->
     RelativeToServer = MyName,
-    {Proj, S2} = calc_projection(S, RelativeToServer, [{author_proc, cast}]),
+    {Proj, S2} = calc_projection(S, RelativeToServer),
     {noreply, S2#ch_mgr{proj_proposed=Proj}};
 handle_cast(_Cast, S) ->
     ?D({cast_whaaaaaaaaaaa, _Cast}),
@@ -256,9 +263,13 @@ cl_write_public_proj_local(Epoch, Proj, SkipLocalWriteErrorP,
                end,
     case Res0 of
         ok ->
-            Continue();
+            {XX, SS} = Continue(),
+            {{qqq_local_write, ok, XX}, SS};
+            %% Continue();
         _Else when SkipLocalWriteErrorP ->
-            Continue();
+            {XX, SS} = Continue(),
+            {{qqq_local_write, _Else, XX}, SS};
+            %% Continue();
         Else when Else == error_written; Else == timeout; Else == t_timeout ->
             {Else, S2}
     end.
@@ -274,7 +285,7 @@ cl_write_public_proj_remote(FLUs, Partitions, Epoch, Proj, S) ->
 do_cl_read_latest_public_projection(ReadRepairP,
                                     #ch_mgr{proj=Proj1, myflu=_MyFLU} = S) ->
     _Epoch1 = Proj1#projection.epoch_number,
-    case cl_read_latest_public_projection(S) of
+    case cl_read_latest_projection(public, S) of
         {needs_repair, FLUsRs, Extra, S3} ->
             if not ReadRepairP ->
                     {not_unanimous, todoxyz, [{results, FLUsRs}|Extra], S3};
@@ -286,11 +297,11 @@ do_cl_read_latest_public_projection(ReadRepairP,
             {UnanimousTag, Proj2, Extra, S3}
     end.
 
-cl_read_latest_public_projection(#ch_mgr{proj=CurrentProj}=S) ->
+cl_read_latest_projection(ProjectionType, #ch_mgr{proj=CurrentProj}=S) ->
     #projection{all_members=All_list} = CurrentProj,
     {_UpNodes, Partitions, S2} = calc_up_nodes(S),
     DoIt = fun(X) ->
-                   case machi_flu0:proj_read_latest(X, public) of
+                   case machi_flu0:proj_read_latest(X, ProjectionType) of
                        {ok, P} -> P;
                        Else    -> Else
                    end
@@ -300,6 +311,11 @@ cl_read_latest_public_projection(#ch_mgr{proj=CurrentProj}=S) ->
     FLUsRs = lists:zip(All_list, Rs),
     UnwrittenRs = [x || error_unwritten <- Rs],
     Ps = [Proj || {_FLU, Proj} <- FLUsRs, is_record(Proj, projection)],
+    %% debug only:
+    %% BadAnswerFLUs = [{FLU,bad_answer,Answer} || {FLU, Answer} <- FLUsRs,
+    %%                         not is_record(Answer, projection)],
+    BadAnswerFLUs = [FLU || {FLU, Answer} <- FLUsRs,
+                            not is_record(Answer, projection)],
     if length(UnwrittenRs) == length(Rs) ->
             {error_unwritten, FLUsRs, [todo_fix_caller_perhaps], S2};
        UnwrittenRs /= [] ->
@@ -312,8 +328,17 @@ cl_read_latest_public_projection(#ch_mgr{proj=CurrentProj}=S) ->
                            end,
             Extra = [{all_members_replied, length(Rs) == length(All_list)}],
             Best_FLUs = [FLU || {FLU, Projx} <- FLUsRs, Projx == BestProj],
+            AllHosed = lists:usort(
+                         lists:flatten([get_all_hosed(P) || P <- Ps])),
+            AllFlapCounts = merge_flap_counts([get_all_flap_counts(P) ||
+                                                  P <- Ps]),
             Extra2 = [{unanimous_flus,Best_FLUs},
-                      {not_unanimous_flus, NotBestPs}|Extra],
+                      {not_unanimous_flus, All_list --
+                                                 (Best_FLUs ++ BadAnswerFLUs)},
+                      {bad_answer_flus, BadAnswerFLUs},
+                      {not_unanimous_answers, NotBestPs},
+                      {trans_all_hosed, AllHosed},
+                      {trans_all_flap_counts, AllFlapCounts}|Extra],
             {UnanimousTag, BestProj, Extra2, S2}
     end.
 
@@ -364,26 +389,31 @@ make_projection(EpochNum,
     P2 = update_projection_checksum(P),
     P2#projection{dbg2=Dbg2}.
 
-update_projection_checksum(P) ->
-    CSum = crypto:hash(sha, term_to_binary(P)),
-    P#projection{epoch_csum=CSum}.
+update_projection_checksum(#projection{dbg2=Dbg2} = P) ->
+    CSum = crypto:hash(sha, term_to_binary(P#projection{dbg2=[]})),
+    P#projection{epoch_csum=CSum, dbg2=Dbg2}.
 
 update_projection_dbg2(P, Dbg2) when is_list(Dbg2) ->
     P#projection{dbg2=Dbg2}.
 
+calc_projection(S, RelativeToServer) ->
+    AllHosed = [],
+    calc_projection(S, RelativeToServer, AllHosed).
+
 calc_projection(#ch_mgr{proj=LastProj, runenv=RunEnv} = S, RelativeToServer,
-                Dbg) ->
+                AllHosed) ->
+    Dbg = [],
     OldThreshold = proplists:get_value(old_threshold, RunEnv),
     NoPartitionThreshold = proplists:get_value(no_partition_threshold, RunEnv),
     calc_projection(OldThreshold, NoPartitionThreshold, LastProj,
-                    RelativeToServer, Dbg, S).
+                    RelativeToServer, AllHosed, Dbg, S).
 
 %% OldThreshold: Percent chance of using the old/previous network partition list
 %% NoPartitionThreshold: If the network partition changes, what percent chance
 %%                       that there are no partitions at all?
 
 calc_projection(_OldThreshold, _NoPartitionThreshold, LastProj,
-                RelativeToServer, Dbg,
+                RelativeToServer, _____TODO_delme_I_think__AllHosed, Dbg,
                 #ch_mgr{name=MyName, runenv=RunEnv1}=S) ->
     #projection{epoch_number=OldEpochNum,
                 all_members=All_list,
@@ -403,7 +433,7 @@ calc_projection(_OldThreshold, _NoPartitionThreshold, LastProj,
     {NewUPI_list3, Repairing_list3, RunEnv3} =
         case {NewUp, Repairing_list2} of
             {[], []} ->
-D_foo=[],
+                D_foo=[],
                 {NewUPI_list, [], RunEnv2};
             {[], [H|T]} when RelativeToServer == hd(NewUPI_list) ->
                 %% The author is head of the UPI list.  Let's see if
@@ -417,16 +447,19 @@ D_foo=[],
                                 tl(NewUPI_list) ++ Repairing_list2,
                                 S#ch_mgr.proj, Partitions, S),
                 if not SameEpoch_p ->
-D_foo=[],
+                        D_foo=[],
                         {NewUPI_list, OldRepairing_list, RunEnv2};
                    true ->
-D_foo=[{repair_airquote_done, {we_agree, (S#ch_mgr.proj)#projection.epoch_number}}],
+                        D_foo=[{repair_airquote_done,
+                                {we_agree,
+                                 (S#ch_mgr.proj)#projection.epoch_number}}],
                         {NewUPI_list ++ [H], T, RunEnv2}
                 end;
             {_, _} ->
-D_foo=[],
+                D_foo=[],
                 {NewUPI_list, OldRepairing_list, RunEnv2}
         end,
+
     Repairing_list4 = case NewUp of
                           []    -> Repairing_list3;
                           NewUp -> Repairing_list3 ++ NewUp
@@ -446,7 +479,7 @@ D_foo=[],
 
     P = make_projection(OldEpochNum + 1,
                         MyName, All_list, Down, NewUPI, NewRepairing,
-                        D_foo ++
+                        [da_hd] ++ D_foo ++
                         Dbg ++ [{ps, Partitions},{nodes_up, Up}]),
     {P, S#ch_mgr{runenv=RunEnv3}}.
 
@@ -529,6 +562,8 @@ rank_projections(Projs, CurrentProj) ->
     N = length(All_list),
     [{rank_projection(Proj, MemberRank, N), Proj} || Proj <- Projs].
 
+rank_projection(#projection{upi=[]}, _MemberRank, _N) ->
+    -100;
 rank_projection(#projection{author_server=Author,
                             upi=UPI_list,
                             repairing=Repairing_list}, MemberRank, N) ->
@@ -546,15 +581,8 @@ react_to_env_A10(S) ->
     ?REACT(a10),
     react_to_env_A20(0, S).
 
-react_to_env_A20(Retries, #ch_mgr{name=MyName} = S) ->
+react_to_env_A20(Retries, S) ->
     ?REACT(a20),
-    RelativeToServer = MyName,
-    {P_newprop, S2} = calc_projection(S, RelativeToServer,
-                                      [{author_proc, react}]),
-    react_to_env_A30(Retries, P_newprop, S2).
-
-react_to_env_A30(Retries, P_newprop, S) ->
-    ?REACT(a30),
     {UnanimousTag, P_latest, ReadExtra, S2} =
         do_cl_read_latest_public_projection(true, S),
 
@@ -562,29 +590,58 @@ react_to_env_A30(Retries, P_newprop, S) ->
     %% to determine if *all* of the UPI+Repairing FLUs are members of
     %% the unanimous server replies.
     UnanimousFLUs = lists:sort(proplists:get_value(unanimous_flus, ReadExtra)),
-    UPI_Repairing_FLUs = lists:sort(P_latest#projection.upi ++
-                                    P_latest#projection.repairing),
+    ?REACT({a20,?LINE,latest_epoch,P_latest#projection.epoch_number}),
+    ?REACT({a20,?LINE,latest_upi,P_latest#projection.upi}),
+    ?REACT({a20,?LINE,latest_repairing,P_latest#projection.repairing}),
+    ?REACT({a20,?LINE,flapping_i,get_raw_flapping_i(P_latest)}),
+
+    %% Reach into hosed compensation, if necessary, to find effective
+    %% UPI and Repairing lists.
+    {E_UPI, E_Repairing} = case get_flapping_hosed_compensation(P_latest) of
+                               undefined ->
+                                   {P_latest#projection.upi,
+                                    P_latest#projection.repairing};
+                               Answer ->
+                                   Answer
+                           end,
+    UPI_Repairing_FLUs = lists:sort(E_UPI ++ E_Repairing),
     All_UPI_Repairing_were_unanimous = UPI_Repairing_FLUs == UnanimousFLUs,
     %% TODO: investigate if the condition below is more correct?
     %% All_UPI_Repairing_were_unanimous = (UPI_Repairing_FLUs -- UnanimousFLUs) == [],
+    %% TODO: or:
+    %% All_UPI_Repairing_were_unanimous =
+    %%     ordsets:is_subset(ordsets:from_list(UPI_Repairing_FLUs),
+    %%                       ordsets:from_list(UnanimousFLUs)),
     LatestUnanimousP =
         if UnanimousTag == unanimous
            andalso
            All_UPI_Repairing_were_unanimous ->
-                ?REACT({a30,?LINE}),
+                ?REACT({a20,?LINE}),
                 true;
            UnanimousTag == unanimous ->
-                ?REACT({a30,?LINE,UPI_Repairing_FLUs,UnanimousFLUs}),
+                ?REACT({a20,?LINE,UPI_Repairing_FLUs,UnanimousFLUs}),
                 false;
            UnanimousTag == not_unanimous ->
-                ?REACT({a30,?LINE}),
+                ?REACT({a20,?LINE}),
                 false;
            true ->
                 exit({badbad, UnanimousTag})
         end,
-    react_to_env_A40(Retries, P_newprop, P_latest,
+    react_to_env_A30(Retries, P_latest,
                      LatestUnanimousP, S2).
 
+react_to_env_A30(Retries, P_latest, LatestUnanimousP,
+                 #ch_mgr{name=MyName, flap_limit=FlapLimit} = S) ->
+    ?REACT(a30),
+    RelativeToServer = MyName,
+    AllHosed = get_all_hosed(S),
+    {P_newprop1, S2} = calc_projection(S, RelativeToServer, AllHosed),
+
+    {P_newprop2, S3} = calculate_flaps(P_newprop1, FlapLimit, S2),
+
+    react_to_env_A40(Retries, P_newprop2, P_latest,
+                     LatestUnanimousP, S3).
+
 react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP,
                  #ch_mgr{name=MyName, proj=P_current}=S) ->
     ?REACT(a40),
@@ -690,33 +747,81 @@ react_to_env_A50(P_latest, S) ->
     {{no_change, P_latest#projection.epoch_number}, S}.
 
 react_to_env_B10(Retries, P_newprop, P_latest, LatestUnanimousP,
-                 Rank_newprop, Rank_latest, #ch_mgr{name=MyName}=S0) ->
+                 Rank_newprop, Rank_latest,
+                 #ch_mgr{name=MyName, flap_limit=FlapLimit}=S) ->
     ?REACT(b10),
 
-    S = calculate_flaps(P_newprop, S0),
-    FlapLimit = 3,                              % todo tweak
+    {_P_newprop_flap_time, P_newprop_flap_count} = get_flap_count(P_newprop),
+    LatestAllFlapCounts = get_all_flap_counts_counts(P_latest),
+    P_latest_trans_flap_count = my_find_minmost(LatestAllFlapCounts),
+
     if
         LatestUnanimousP ->
             ?REACT({b10, ?LINE}),
+            put(b10_hack, false),
 
             react_to_env_C100(P_newprop, P_latest, S);
 
-        S#ch_mgr.flaps > FlapLimit
-        andalso
-        Rank_latest =< Rank_newprop ->
-            if S#ch_mgr.flaps - FlapLimit - 3 =< 0 -> io:format(user, "{FLAP: ~w flaps ~w}!\n", [S#ch_mgr.name, S#ch_mgr.flaps]); true -> ok end,
-            {_, _, USec} = os:timestamp(),
-            %% If we always go to C200, then we can deadlock sometimes.
-            %% So we roll the dice.
-            %% TODO: make this PULSE-friendly!
-            if USec rem 3 == 0 ->
+        P_newprop_flap_count >= FlapLimit ->
+            %% I am flapping ... what else do I do?
+            B10Hack = get(b10_hack),
+            if B10Hack == false andalso P_newprop_flap_count - FlapLimit - 3 =< 0 -> io:format(user, "{FLAP: ~w flaps ~w}!\n", [S#ch_mgr.name, P_newprop_flap_count]), put(b10_hack, true); true -> ok end,
+
+            if
+                %% So, if we noticed a flap count by some FLU X with a
+                %% count below FlapLimit, then X crashes so that X's
+                %% flap count remains below FlapLimit, then we could get
+                %% stuck forever?  Hrm, except that 'crashes' ought to be
+                %% detected by our own failure detector and get us out of
+                %% this current flapping situation, right? TODO
+                %%
+                %% 2015-04-05: If we add 'orelse AllSettled' to this 'if'
+                %% clause, then we can end up short-circuiting too
+                %% early.  (Where AllSettled comes from the runenv's
+                %% flapping_i prop.)  So, I believe that we need to
+                %% rely on the failure detector to rescue us.
+                %%
+                %% TODO About the above ^^ I think that was based on buggy
+                %%      calculation of AllSettled.  Recheck!
+                %%
+                %% TODO Yay, another magic constant below, added to
+                %%      FlapLimit, that needs thorough examination and
+                %%      hopefully elimination.  I'm adding it to try to
+                %%      make it more likely that someone's private proj
+                %%      will include all_flap_counts_settled,true 100%
+                %%      of the time.  But I'm not sure how important that
+                %%      really is.
+                %%      That settled flag can lag behind after a change in
+                %%      network conditions, so I'm not sure how big its
+                %%      value is, if any.
+                P_latest_trans_flap_count >= FlapLimit + 20 ->
+                    %% Everyone that's flapping together now has flap_count
+                    %% that's larger than the limit.  So it's safe and good
+                    %% to stop here, so we can break the cycle of flapping.
                     react_to_env_A50(P_latest, S);
-               true ->
-                    react_to_env_C200(Retries, P_latest, S)
+
+                true ->
+                    %% It is our moral imperative to write so that the flap
+                    %% cycle continues enough times so that everyone notices
+                    %% and thus the earlier clause above fires.
+                    P_newprop2 = trim_proj_with_all_hosed(P_newprop, S),
+                    io:format(user, "GEE ~w\n", [self()]),
+                    io:format(user, "GEE1 ~w ~w\n", [self(), make_projection_summary(P_newprop)]),
+                    if P_newprop2#projection.upi == [] ->
+                            io:format(user, "GEE1-50  ~w ~w\n", [self(), make_projection_summary(P_newprop)]),
+                            ?REACT({b10, ?LINE}),
+                            react_to_env_A50(P_latest, S);
+                       true ->
+                            io:format(user, "GEE1-300 newprop ~w ~w\n", [self(), make_projection_summary(P_newprop)]),
+                            io:format(user, "GEE1-300 latest  ~w ~w\n", [self(), make_projection_summary(P_latest)]),
+                            ?REACT({b10, ?LINE}),
+                            react_to_env_C300(P_newprop2, P_latest, S)
+                    end
             end;
 
         Retries > 2 ->
             ?REACT({b10, ?LINE}),
+            put(b10_hack, false),
 
             %% The author of P_latest is too slow or crashed.
             %% Let's try to write P_newprop and see what happens!
@@ -726,6 +831,7 @@ react_to_env_B10(Retries, P_newprop, P_latest, LatestUnanimousP,
         andalso
         P_latest#projection.author_server /= MyName ->
             ?REACT({b10, ?LINE}),
+            put(b10_hack, false),
 
             %% Give the author of P_latest an opportunite to write a
             %% new projection in a new epoch to resolve this mixed
@@ -734,6 +840,7 @@ react_to_env_B10(Retries, P_newprop, P_latest, LatestUnanimousP,
 
         true ->
             ?REACT({b10, ?LINE}),
+            put(b10_hack, false),
 
             %% P_newprop is best, so let's write it.
             react_to_env_C300(P_newprop, P_latest, S)
@@ -781,8 +888,10 @@ react_to_env_C110(P_latest, #ch_mgr{myflu=MyFLU} = S) ->
     Islands = proplists:get_value(network_islands, RunEnv),
     P_latest2 = update_projection_dbg2(
                   P_latest,
-                  [{network_islands, Islands},
-                   {hooray, {v2, date(), time()}}|Extra_todo]),
+                  [%% {network_islands, Islands},
+                   %% {hooray, {v2, date(), time()}}
+                   Islands--Islands
+                   |Extra_todo]),
     Epoch = P_latest2#projection.epoch_number,
     ok = machi_flu0:proj_write(MyFLU, Epoch, private, P_latest2),
     case proplists:get_value(private_write_verbose, S#ch_mgr.opts) of
@@ -790,7 +899,7 @@ react_to_env_C110(P_latest, #ch_mgr{myflu=MyFLU} = S) ->
             {_,_,C} = os:timestamp(),
             MSec = trunc(C / 1000),
             {HH,MM,SS} = time(),
-            io:format(user, "~2..0w:~2..0w:~2..0w.~3..0w ~p uses: ~w\n",
+            io:format(user, "\n~2..0w:~2..0w:~2..0w.~3..0w ~p uses: ~w\n",
                       [HH,MM,SS,MSec, S#ch_mgr.name,
                        make_projection_summary(P_latest2)]);
         _ ->
@@ -802,6 +911,10 @@ react_to_env_C120(P_latest, #ch_mgr{proj_history=H} = S) ->
     ?REACT(c120),
     H2 = queue:in(P_latest, H),
     H3 = case queue:len(H2) of
+             %% TODO: revisit this constant?  Is this too long as a base?
+             %% My hunch is that it's fine and that the flap_limit needs to
+             %% be raised much higher (because it can increase several ticks
+             %% without a newer public epoch proposed anywhere).
              X when X > length(P_latest#projection.all_members) * 2 ->
                  {_V, Hxx} = queue:out(H2),
                  Hxx;
@@ -842,29 +955,183 @@ react_to_env_C310(P_newprop, S) ->
     ?REACT(c310),
     Epoch = P_newprop#projection.epoch_number,
     {_Res, S2} = cl_write_public_proj_skip_local_error(Epoch, P_newprop, S),
+    io:format(user, "GEE3 ~w ~w epoch ~w ~w\n", [self(), S#ch_mgr.name, P_newprop#projection.epoch_number, _Res]),
+    io:format(user, "GEE3 ~w ~w\n", [self(), make_projection_summary(P_newprop)]),
+    HH = lists:sublist(get(react), 90),
+    io:format(user, "GEE3 ~w ~p\n", [self(), lists:reverse(HH)]),
     ?REACT({c310,make_projection_summary(P_newprop)}),
     ?REACT({c310,_Res}),
     
     react_to_env_A10(S2).
 
-calculate_flaps(P_newprop, #ch_mgr{name=_MyName,
-                                   proj_history=H, flaps=Flaps} = S) ->
-    Ps = queue:to_list(H) ++ [P_newprop],
-    UPI_Repairing_combos =
-        lists:usort([{P#projection.upi, P#projection.repairing} || P <- Ps]),
-    Down_combos = lists:usort([P#projection.down || P <- Ps]),
-    case {queue:len(H), length(UPI_Repairing_combos), length(Down_combos)} of
-        {N, _, _} when N < length(P_newprop#projection.all_members) ->
-            S#ch_mgr{flaps=0};
-        %% {_, URs=_URs, 1=_Ds} when URs < 3 ->
-        {_, 1=_URs, 1=_Ds} ->
-            %%%%%% io:format(user, "F{~w,~w,~w..~w}!", [_MyName, _URs, _Ds, Flaps]),
-            S#ch_mgr{flaps=Flaps + 1};
-            %% todo_flapping;
-        _ ->
-            S#ch_mgr{flaps=0}
+proposals_are_flapping(Ps) ->
+    %% This works:
+    %% UniqueProposalSummaries = lists:usort([{P#projection.upi,
+    %%                                         P#projection.repairing,
+    %%                                         P#projection.down} || P <- Ps]),
+    %% length(UniqueProposalSummaries).
+    %% ... but refactor to use a fold, for later refactoring ease.
+    [First|Rest] = Ps,
+    case lists:foldl(
+           fun(#projection{upi=UPI,repairing=Repairing,down=Down}=NewP,
+               #projection{upi=UPI,repairing=Repairing,down=Down}) ->
+                   NewP;
+              (#projection{}=NewP,
+               #projection{upi=UPIo,repairing=Repairingo}=_OldP) ->
+                   case get_flapping_hosed_compensation(NewP) of
+                       {NewUnadjUPI, NewUnadjRepairing} ->
+                           OK = case get_flapping_hosed_compensation(NewP) of
+                                    {OldUnadjUPI, OldUnadjRepairing} ->
+                                        OldUnadjUPI == NewUnadjUPI
+                                        andalso
+                                        OldUnadjRepairing == NewUnadjRepairing;
+                                    _Else9 ->
+                                        UPIo == NewUnadjUPI
+                                        andalso
+                                        Repairingo == NewUnadjRepairing
+                                end,
+                           if not OK ->
+                                   bummer4;
+                              true ->
+                                   NewP
+                           end;
+                       undefined ->
+                           bummer2;
+                       _Else ->
+                           bummer3
+                   end;
+              (_, _Else) ->
+                   bummer
+           end, First, Rest) of
+        LastProj when is_record(LastProj, projection) ->
+            1;
+        _Else ->
+            -1 % arbitrary, anything but 1
     end.
 
+calculate_flaps(P_newprop, FlapLimit,
+                #ch_mgr{name=MyName, proj_history=H,
+                        flaps=Flaps, runenv=RunEnv0} = S) ->
+    Now = os:timestamp(),
+    RunEnv1 = replace(RunEnv0, [{flapping_i, []}]),
+    HistoryPs = queue:to_list(H),
+    Ps = HistoryPs ++ [P_newprop],
+    UniqueProposalSummaries = proposals_are_flapping(Ps),
+
+    {_WhateverUnanimous, BestP, Props, _S} =
+        cl_read_latest_projection(private, S),
+    NotBestPs = proplists:get_value(not_unanimous_answers, Props),
+    DownUnion = lists:usort(
+                  lists:flatten(
+                    [P#projection.down ||
+                        P <- [BestP|NotBestPs]])),
+    HosedTransUnion = proplists:get_value(trans_all_hosed, Props),
+    TransFlapCounts0 = proplists:get_value(trans_all_flap_counts, Props),
+
+    _Unanimous = proplists:get_value(unanimous_flus, Props),
+    _NotUnanimous = proplists:get_value(not_unanimous_flus, Props),
+    %% NOTE: bad_answer_flus are probably due to timeout or some other network
+    %%       glitch, i.e., anything other than {ok, P::projection()}
+    %%       response from machi_flu0:proj_read_latest().
+    BadFLUs = proplists:get_value(bad_answer_flus, Props),
+
+    RemoteTransFlapCounts1 = lists:keydelete(MyName, 1, TransFlapCounts0),
+    RemoteTransFlapCounts =
+        [X || {_FLU, {FlTime, _FlapCount}}=X <- RemoteTransFlapCounts1,
+              FlTime /= ?NOT_FLAPPING],
+    TempNewFlaps = Flaps + 1,
+    TempAllFlapCounts = lists:sort([{MyName, {Now, TempNewFlaps}}|
+                                    RemoteTransFlapCounts]),
+    %% Sanity check.
+    true = lists:all(fun({_,{_,_}}) -> true;
+                        (_)         -> false end, TempAllFlapCounts),
+
+    %% H is the bounded history of all of this manager's private
+    %% projection store writes.  If we've proposed the *same*
+    %% {UPI+Repairing, Down} combination for the entire length of our
+    %% bounded size of H, then we're flapping.
+    %%
+    %% If we're flapping, then we use our own flap counter and that of
+    %% all of our peer managers to see if we've all got flap counters
+    %% that exceed the flap_limit.  If that global condition appears
+    %% true, then we "blow the circuit breaker" by stopping our
+    %% participation in the flapping store (via the shortcut to A50).
+    %%
+    %% We reset our flap counter on any of several conditions:
+    %%
+    %% 1. If our bounded history H contains more than one proposal,
+    %%    then by definition we are not flapping.
+    %% 2. If a remote manager is flapping and has re-started a new
+    %%    flapping episode.
+    %% 3. If one of the remote managers that we saw earlier has
+    %%    stopped flapping.
+
+    case {queue:len(H), UniqueProposalSummaries} of
+        {N, 1} when N >= length(P_newprop#projection.all_members) ->
+            NewFlaps = TempNewFlaps,
+
+            %% Wow, this behavior is almost spooky.
+            %%
+            %% For an example partition map [{c,a}], on the very first
+            %% time this 'if' clause is hit by FLU b, AllHosed=[a,c].
+            %% How the heck does B know that??
+            %%
+            %% If I use:
+            %% DownUnionQQQ = [{P#projection.epoch_number, P#projection.author_server, P#projection.down} || P <- [BestP|NotBestPs]],
+            %% AllHosed = [x_1] ++ DownUnion ++ [x_2] ++ HosedTransUnion ++ [x_3] ++ BadFLUs ++ [{downunionqqq, DownUnionQQQ}];
+            %%
+            %% ... then b sees this when proposing epoch 451:
+            %%
+            %% {all_hosed,
+            %%  [x_1,a,c,x_2,x_3,
+            %%   {downunionqqq,
+            %%    [{450,a,[c]},{449,b,[]},{448,c,[a]},{441,d,[]}]}]},
+            %%
+            %% So b's working on epoch 451 at the same time that d's latest
+            %% public projection is only epoch 441.  But there's enough
+            %% lag so that b can "see" that a's bad=[c] (due to t_timeout!)
+            %% and c's bad=[a].  So voila, b magically knows about both
+            %% problem FLUs.  Weird/cool.
+
+            AllFlapCounts = TempAllFlapCounts,
+            AllHosed = lists:usort(DownUnion ++ HosedTransUnion ++ BadFLUs);
+        {_N, _} ->
+            NewFlaps = 0,
+            AllFlapCounts = [],
+            AllHosed = []
+    end,
+
+    %% If there's at least one count in AllFlapCounts that isn't my
+    %% flap count, and if it's over the flap limit, then consider them
+    %% settled.
+    AllFlapCountsSettled = lists:keydelete(MyName, 1, AllFlapCounts) /= []
+                           andalso
+                           my_find_minmost(AllFlapCounts) >= FlapLimit,
+    FlappingI = {flapping_i, [{flap_count, {Now, NewFlaps}},
+                              {all_hosed, AllHosed},
+                              {all_flap_counts, lists:sort(AllFlapCounts)},
+                              {all_flap_counts_settled, AllFlapCountsSettled},
+                              {bad,BadFLUs},
+                              {da_downu, DownUnion}, % debugging aid
+                              {da_hosedtu, HosedTransUnion}, % debugging aid
+                              {da_downreports, [{P#projection.epoch_number, P#projection.author_server, P#projection.down} || P <- [BestP|NotBestPs]]} % debugging aid
+                             ]},
+    Dbg2 = [FlappingI|P_newprop#projection.dbg],
+    %% SLF TODO: 2015-03-04: I'm growing increasingly suspicious of
+    %% the 'runenv' variable that's threaded through all this code.
+    %% It isn't doing what I'd originally intended.  And I think that
+    %% the flapping information that we've just constructed here is
+    %% going to get lost, and that's a shame.  Fix it.
+    RunEnv2 = replace(RunEnv1, [FlappingI]),
+    %% NOTE: If we'd increment of flaps here, that doesn't mean that
+    %%       someone's public proj store has been updated.  For example,
+    %%       if we loop through states C2xx a few times, we would incr
+    %%       flaps each time ... but the C2xx path doesn't write a new
+    %%       proposal to everyone's public proj stores, and there's no
+    %%       guarantee that anyone else as written a new public proj either.
+    {update_projection_checksum(P_newprop#projection{dbg=Dbg2}),
+     S#ch_mgr{flaps=NewFlaps, runenv=RunEnv2}}.
+
 projection_transitions_are_sane(Ps, RelativeToServer) ->
     projection_transitions_are_sane(Ps, RelativeToServer, false).
 
@@ -1134,6 +1401,82 @@ sleep_ranked_order(MinSleep, MaxSleep, FLU, FLU_list) ->
     timer:sleep(SleepTime),
     SleepTime.
 
+my_find_minmost([]) ->
+    0;
+my_find_minmost([{_,_}|_] = TransFlapCounts0) ->
+    lists:min([FlapCount || {_T, {_FlTime, FlapCount}} <- TransFlapCounts0]);
+my_find_minmost(TransFlapCounts0) ->
+    lists:min(TransFlapCounts0).
+
+get_raw_flapping_i(#projection{dbg=Dbg}) ->
+    proplists:get_value(flapping_i, Dbg, []).
+
+get_flap_count(P) ->
+    proplists:get_value(flap_count, get_raw_flapping_i(P), 0).
+
+get_all_flap_counts(P) ->
+    proplists:get_value(all_flap_counts, get_raw_flapping_i(P), []).
+
+get_all_flap_counts_counts(P) ->
+    case get_all_flap_counts(P) of
+        [] ->
+            [];
+        [{_,{_,_}}|_] = Cs ->
+            [Count || {_FLU, {_Time, Count}} <- Cs]
+    end.
+
+get_all_hosed(P) when is_record(P, projection)->
+    proplists:get_value(all_hosed, get_raw_flapping_i(P), []);
+get_all_hosed(S) when is_record(S, ch_mgr) ->
+    proplists:get_value(all_hosed,
+                        proplists:get_value(flapping_i, S#ch_mgr.runenv, []),
+                        []).
+
+get_flapping_hosed_compensation(P) ->
+    proplists:get_value(hosed_compensation, get_raw_flapping_i(P),
+                        undefined).
+
+merge_flap_counts(FlapCounts) ->
+    merge_flap_counts(FlapCounts, orddict:new()).
+
+merge_flap_counts([], D) ->
+    orddict:to_list(D);
+merge_flap_counts([FlapCount|Rest], D1) ->
+    %% We know that FlapCount is list({Actor, {FlapStartTime,NumFlaps}}).
+    D2 = orddict:from_list(FlapCount),
+    %% If the FlapStartTimes differ, then pick the larger start time tuple.
+    D3 = orddict:merge(fun(_Key, {T1,_NF1}= V1, {T2,_NF2}=_V2)
+                             when T1 > T2 ->
+                               V1;
+                          (_Key, {_T1,_NF1}=_V1, {_T2,_NF2}= V2) ->
+                               V2;
+                          (_Key, V1, V2) ->
+                               exit({bad_merge_2tuples,mod,?MODULE,line,?LINE,
+                                     _Key, V1, V2})
+                       end, D1, D2),
+    merge_flap_counts(Rest, D3).
+
+trim_proj_with_all_hosed(#projection{upi=UPI, repairing=Repairing,
+                                     dbg=Dbg}=P, S) ->
+    AllHosed = get_all_hosed(S),
+    HosedComp = get_flapping_hosed_compensation(P),
+    if AllHosed == [] orelse HosedComp /= undefined ->
+            P;
+       true ->
+            UPI2 = UPI -- AllHosed,
+            Repairing2 = Repairing -- AllHosed,
+            X = if AllHosed /= [] ->
+                        Compensation = {hosed_compensation, {UPI, Repairing}},
+                        [Compensation, {now_all_hosed, AllHosed}];
+                   true ->
+                        [no_comp]
+                end,
+            FI = get_raw_flapping_i(P),
+            Replace = [{flapping_i, X ++ FI}],
+            DbgB = replace(Dbg, Replace),
+            P#projection{upi=UPI2, repairing=Repairing2, dbg=DbgB}
+    end.
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 perhaps_call_t(S, Partitions, FLU, DoIt) ->

prototype/chain-manager/test/machi_chain_manager1_pulse.erl

@@ -134,6 +134,19 @@ change_partitions(OldThreshold, NoPartitionThreshold) ->
     machi_partition_simulator:reset_thresholds(OldThreshold,
                                                NoPartitionThreshold).
 
+always_last_partitions() ->
+    machi_partition_simulator:always_last_partitions().
+
+private_stable_check(FLUs) ->
+    {_FLU_pids, Mgr_pids} = get(manager_pids_hack),
+    Res = private_projections_are_stable_check(FLUs, Mgr_pids),
+    if not Res ->
+            io:format(user, "BUMMER: private stable check failed!\n", []);
+       true ->
+            ok
+    end,
+    Res.
+
 do_ticks(Num, PidsMaybe, OldThreshold, NoPartitionThreshold) ->
     io:format(user, "~p,~p,~p|", [Num, OldThreshold, NoPartitionThreshold]),
     {_FLU_pids, Mgr_pids} = case PidsMaybe of
@@ -229,34 +242,38 @@ prop_pulse() ->
         %% doesn't always allow unanimous private projection store values:
         %% FLU a might need one more tick to write its private projection, but
         %% it isn't given a chance at the end of the PULSE run.  So we cheat
+        Stabilize1 = [{set,{var,99999995},
+                      {call, ?MODULE, always_last_partitions, []}}],
+        Stabilize2 = [{set,{var,99999996},
+                       {call, ?MODULE, private_stable_check, [all_list()]}}],
         LastTriggerTicks = {set,{var,99999997},
                             {call, ?MODULE, do_ticks, [25, undefined, no, no]}},
         Cmds1 = lists:duplicate(2, LastTriggerTicks),
         %% Cmds1 = lists:duplicate(length(all_list())*2, LastTriggerTicks),
         Cmds = Cmds0 ++
-               Cmds1 ++ [{set,{var,99999999},
-                          {call, ?MODULE, dump_state, []}}],
+               Stabilize1 ++
+               Cmds1 ++
+               Stabilize2 ++
+            [{set,{var,99999999}, {call, ?MODULE, dump_state, []}}],
         {_H2, S2, Res} = pulse:run(
                            fun() ->
                                    {_H, _S, _R} = run_commands(?MODULE, Cmds)
                            end, [{seed, Seed},
                                  {strategy, unfair}]),
-        %% {FLU_pids, Mgr_pids} = S2#state.pids,
-        %% [ok = machi_flu0:stop(FLU) || FLU <- FLU_pids],
-        %% [ok = ?MGR:stop(Mgr) || Mgr <- Mgr_pids],
         ok = shutdown_hard(),
 
-%% ?QC_FMT("Cmds ~p\n", [Cmds]),
-%% ?QC_FMT("H2 ~p\n", [_H2]),
-%% ?QC_FMT("S2 ~p\n", [S2]),
         {Report, Diag} = S2#state.dump_state,
-%% ?QC_FMT("\nLast report = ~p\n", [lists:last(Report)]),
+
+        %% Report is ordered by Epoch.  For each private projection
+        %% written during any given epoch, confirm that all chain
+        %% members appear in only one unique chain, i.e., the sets of
+        %% unique chains are disjoint.
+        AllDisjointP = ?MGRTEST:all_reports_are_disjoint(Report),
 
         %% Given the report, we flip it around so that we observe the
         %% sets of chain transitions relative to each FLU.
         R_Chains = [?MGRTEST:extract_chains_relative_to_flu(FLU, Report) ||
                        FLU <- all_list()],
-        %% ?D(R_Chains),
         R_Projs = [{FLU, [?MGRTEST:chain_to_projection(
                              FLU, Epoch, UPI, Repairing, all_list()) ||
                              {Epoch, UPI, Repairing} <- E_Chains]} ||
@@ -268,16 +285,11 @@ prop_pulse() ->
           [{FLU,_SaneRes} = {FLU,?MGR:projection_transitions_are_sane_retrospective(
                                     Ps, FLU)} ||
               {FLU, Ps} <- R_Projs],
-%% ?QC_FMT("Sane ~p\n", [Sane]),
         SaneP = lists:all(fun({_FLU, SaneRes}) -> SaneRes == true end, Sane),
 
         %% The final report item should say that all are agreed_membership.
         {_LastEpoch, {ok_disjoint, LastRepXs}} = lists:last(Report),
-%% ?QC_FMT("LastEpoch=~p,", [_LastEpoch]),
-%% ?QC_FMT("Report ~P\n", [Report, 5000]),
-%% ?QC_FMT("Diag ~s\n", [Diag]),
         AgreedOrNot = lists:usort([element(1, X) || X <- LastRepXs]),
-%% ?QC_FMT("LastRepXs ~p", [LastRepXs]),
         
         %% TODO: Check that we've converged to a single chain with no repairs.
         SingleChainNoRepair = case LastRepXs of
@@ -296,7 +308,7 @@ prop_pulse() ->
             ?QC_FMT("SingleChainNoRepair failure =\n    ~p\n", [SingleChainNoRepair])
         end,
         conjunction([{res, Res == true orelse Res == ok},
-                     {all_disjoint, ?MGRTEST:all_reports_are_disjoint(Report)},
+                     {all_disjoint, AllDisjointP},
                      {sane, SaneP},
                      {all_agreed_at_end, AgreedOrNot == [agreed_membership]},
                      {single_chain_no_repair, SingleChainNoRepair}
@@ -329,7 +341,7 @@ shutdown_hard() ->
 
 exec_ticks(Num, Mgr_pids) ->
     Parent = self(),
-    Pids = [spawn(fun() ->
+    Pids = [spawn_link(fun() ->
                           [begin
                                erlang:yield(),
                                Max = 10,
@@ -349,4 +361,19 @@ exec_ticks(Num, Mgr_pids) ->
      end || _ <- Pids],    
     ok.
 
+private_projections_are_stable_check(All_list, Mgr_pids) ->
+    %% TODO: extend the check to look not only for latest num, but
+    %% also check for flapping, and if yes, to see if all_hosed are
+    %% all exactly equal.
+
+    _ = exec_ticks(40, Mgr_pids),
+    Private1 = [machi_flu0:proj_get_latest_num(FLU, private) ||
+                   FLU <- All_list],
+    _ = exec_ticks(5, Mgr_pids),
+    Private2 = [machi_flu0:proj_get_latest_num(FLU, private) ||
+                   FLU <- All_list],
+
+    (Private1 == Private2).
+
+
 -endif. % PULSE

prototype/chain-manager/test/machi_chain_manager1_test.erl

@@ -42,6 +42,95 @@
 -include_lib("eunit/include/eunit.hrl").
 -compile(export_all).
 
+unanimous_report(Namez) ->
+    UniquePrivateEs =
+        lists:usort(lists:flatten(
+                      [machi_flu0:proj_list_all(FLU, private) ||
+                          {_FLUName, FLU} <- Namez])),
+    [unanimous_report(Epoch, Namez) || Epoch <- UniquePrivateEs].
+
+unanimous_report(Epoch, Namez) ->
+    Projs = [{FLUName, case machi_flu0:proj_read(FLU, Epoch, private) of
+                           {ok, T} -> T;
+                           _Else   -> not_in_this_epoch
+                       end} || {FLUName, FLU} <- Namez],
+    UPI_R_Sums = [{Proj#projection.upi, Proj#projection.repairing,
+                   Proj#projection.epoch_csum} ||
+                     {_FLUname, Proj} <- Projs,
+                     is_record(Proj, projection)],
+    UniqueUPIs = lists:usort([UPI || {UPI, _Repairing, _CSum} <- UPI_R_Sums]),
+    Res =
+        [begin
+             case lists:usort([CSum || {U, _Repairing, CSum} <- UPI_R_Sums,
+                                       U == UPI]) of
+                 [_1CSum] ->
+                     %% Yay, there's only 1 checksum.  Let's check
+                     %% that all FLUs are in agreement.
+                     {UPI, Repairing, _CSum} =
+                         lists:keyfind(UPI, 1, UPI_R_Sums),
+                     %% TODO: make certain that this subtlety doesn't get
+                     %%       last in later implementations.
+
+                     %% So, this is a bit of a tricky thing.  If we're at
+                     %% upi=[c] and repairing=[a,b], then the transition
+                     %% (eventually!) to upi=[c,a] does not currently depend
+                     %% on b being an active participant in the repair.
+                     %%
+                     %% Yes, b's state is very important for making certain
+                     %% that all repair operations succeed both to a & b.
+                     %% However, in this simulation, we only consider that
+                     %% the head(Repairing) is sane.  Therefore, we use only
+                     %% the "HeadOfRepairing" in our considerations here.
+                     HeadOfRepairing = case Repairing of
+                                          [H_Rep|_] ->
+                                              [H_Rep];
+                                          _ ->
+                                              []
+                                      end,
+                     Tmp = [{FLU, case proplists:get_value(FLU, Projs) of
+                                      P when is_record(P, projection) ->
+                                          P#projection.epoch_csum;
+                                      Else ->
+                                          Else
+                                  end} || FLU <- UPI ++ HeadOfRepairing],
+                     case lists:usort([CSum || {_FLU, CSum} <- Tmp]) of
+                         [_] ->
+                             {agreed_membership, {UPI, Repairing}};
+                         Else2 ->
+                             {not_agreed, {UPI, Repairing}, Else2}
+                     end;
+                 _Else ->
+                     {UPI, not_unique, Epoch, _Else}
+             end
+         end || UPI <- UniqueUPIs],
+    AgreedResUPI_Rs = [UPI++Repairing ||
+                          {agreed_membership, {UPI, Repairing}} <- Res],
+    Tag = case lists:usort(lists:flatten(AgreedResUPI_Rs)) ==
+               lists:sort(lists:flatten(AgreedResUPI_Rs)) of
+              true ->
+                  ok_disjoint;
+              false ->
+                  bummer_NOT_DISJOINT
+          end,
+    {Epoch, {Tag, Res}}.
+
+all_reports_are_disjoint(Report) ->
+    [] == [X || {_Epoch, Tuple}=X <- Report,
+                element(1, Tuple) /= ok_disjoint].
+
+extract_chains_relative_to_flu(FLU, Report) ->
+    {FLU, [{Epoch, UPI, Repairing} ||
+              {Epoch, {ok_disjoint, Es}} <- Report,
+              {agreed_membership, {UPI, Repairing}} <- Es,
+              lists:member(FLU, UPI) orelse lists:member(FLU, Repairing)]}.
+
+chain_to_projection(MyName, Epoch, UPI_list, Repairing_list, All_list) ->
+    ?MGR:make_projection(Epoch, MyName, All_list,
+                         All_list -- (UPI_list ++ Repairing_list),
+                         UPI_list, Repairing_list, []).
+
+-ifndef(PULSE).
+
 smoke0_test() ->
     machi_partition_simulator:start_link({1,2,3}, 50, 50),
     {ok, FLUa} = machi_flu0:start_link(a),
@@ -138,99 +227,98 @@ nonunanimous_setup_and_fix_test() ->
         ok = machi_partition_simulator:stop()
     end.
 
-unanimous_report(Namez) ->
-    UniquePrivateEs =
-        lists:usort(lists:flatten(
-                      [machi_flu0:proj_list_all(FLU, private) ||
-                          {_FLUName, FLU} <- Namez])),
-    [unanimous_report(Epoch, Namez) || Epoch <- UniquePrivateEs].
+short_doc() ->
+"
+A visualization of the convergence behavior of the chain self-management
+algorithm for Machi.
+  1. Set up 4 FLUs and chain manager pairs.
+  2. Create a number of different network partition scenarios, where
+     (simulated) partitions may be symmetric or asymmetric.  Then halt changing
+     the partitions and keep the simulated network stable and broken.
+  3. Run a number of iterations of the algorithm in parallel by poking each
+     of the manager processes on a random'ish basis.
+  4. Afterward, fetch the chain transition changes made by each FLU and
+     verify that no transition was unsafe.
 
-unanimous_report(Epoch, Namez) ->
-    Projs = [{FLUName, case machi_flu0:proj_read(FLU, Epoch, private) of
-                           {ok, T} -> T;
-                           _Else   -> not_in_this_epoch
-                       end} || {FLUName, FLU} <- Namez],
-    UPI_R_Sums = [{Proj#projection.upi, Proj#projection.repairing,
-                   Proj#projection.epoch_csum} ||
-                     {_FLUname, Proj} <- Projs,
-                     is_record(Proj, projection)],
-    UniqueUPIs = lists:usort([UPI || {UPI, _Repairing, _CSum} <- UPI_R_Sums]),
-    Res =
-        [begin
-             case lists:usort([CSum || {U, _Repairing, CSum} <- UPI_R_Sums,
-                                       U == UPI]) of
-                 [_1CSum] ->
-                     %% Yay, there's only 1 checksum.  Let's check
-                     %% that all FLUs are in agreement.
-                     {UPI, Repairing, _CSum} =
-                         lists:keyfind(UPI, 1, UPI_R_Sums),
-                     %% TODO: make certain that this subtlety doesn't get
-                     %%       last in later implementations.
+During the iteration periods, the following is a cheatsheet for the output.
+See the internal source for interpreting the rest of the output.
 
-                     %% So, this is a bit of a tricky thing.  If we're at
-                     %% upi=[c] and repairing=[a,b], then the transition
-                     %% (eventually!) to upi=[c,a] does not currently depend
-                     %% on b being an active participant in the repair.
-                     %%
-                     %% Yes, b's state is very important for making certain
-                     %% that all repair operations succeed both to a & b.
-                     %% However, in this simulation, we only consider that
-                     %% the head(Repairing) is sane.  Therefore, we use only
-                     %% the "HeadOfRepairing" in our considerations here.
-                     HeadOfRepairing = case Repairing of
-                                          [H_Rep|_] ->
-                                              [H_Rep];
-                                          _ ->
-                                              []
-                                      end,
-                     Tmp = [{FLU, case proplists:get_value(FLU, Projs) of
-                                      P when is_record(P, projection) ->
-                                          P#projection.epoch_csum;
-                                      Else ->
-                                          Else
-                                  end} || FLU <- UPI ++ HeadOfRepairing],
-                     case lists:usort([CSum || {_FLU, CSum} <- Tmp]) of
-                         [_] ->
-                             {agreed_membership, {UPI, Repairing}};
-                         Else2 ->
-                             {not_agreed, {UPI, Repairing}, Else2}
-                     end;
-                 _Else ->
-                     {UPI, not_unique, Epoch, _Else}
-             end
-         end || UPI <- UniqueUPIs],
-    AgreedResUPI_Rs = [UPI++Repairing ||
-                          {agreed_membership, {UPI, Repairing}} <- Res],
-    Tag = case lists:usort(lists:flatten(AgreedResUPI_Rs)) ==
-               lists:sort(lists:flatten(AgreedResUPI_Rs)) of
-              true ->
-                  ok_disjoint;
-              false ->
-                  bummer_NOT_DISJOINT
-          end,
-    {Epoch, {Tag, Res}}.
+    'Let loose the dogs of war!'  Network instability
+    'SET partitions = '           Network stability (but broken)
+    'x uses:' The FLU x has made an internal state transition.  The rest of
+              the line is a dump of internal state.
+    '{t}'     This is a tick event which triggers one of the manager processes
+              to evaluate its environment and perhaps make a state transition.
 
-all_reports_are_disjoint(Report) ->
-    [] == [X || {_Epoch, Tuple}=X <- Report,
-                element(1, Tuple) /= ok_disjoint].
+A long chain of '{t}{t}{t}{t}' means that the chain state has settled
+to a stable configuration, which is the goal of the algorithm.
+Press control-c to interrupt....".
 
-extract_chains_relative_to_flu(FLU, Report) ->
-    {FLU, [{Epoch, UPI, Repairing} ||
-              {Epoch, {ok_disjoint, Es}} <- Report,
-              {agreed_membership, {UPI, Repairing}} <- Es,
-              lists:member(FLU, UPI) orelse lists:member(FLU, Repairing)]}.
+long_doc() ->
+    "
+'Let loose the dogs of war!'
 
-chain_to_projection(MyName, Epoch, UPI_list, Repairing_list, All_list) ->
-    ?MGR:make_projection(Epoch, MyName, All_list,
-                         All_list -- (UPI_list ++ Repairing_list),
-                         UPI_list, Repairing_list, []).
+    The simulated network is very unstable for a few seconds.
 
--ifndef(PULSE).
+'x uses'
+
+    After a single iteration, server x has determined that the chain
+    should be defined by the upi, repair, and down list in this record.
+    If all participants reach the same conclusion at the same epoch
+    number (and checksum, see next item below), then the chain is
+    stable, fully configured, and can provide full service.
+
+'epoch,E'
+
+    The epoch number for this decision is E.  The checksum of the full
+    record is not shown.  For purposes of the protocol, a server will
+    'wedge' itself and refuse service (until a new config is chosen)
+    whenever: a). it sees a bigger epoch number mentioned somewhere, or
+    b). it sees the same epoch number but a different checksum.  In case
+    of b), there was a network partition that has healed, and both sides
+    had chosen to operate with an identical epoch number but different
+    chain configs.
+
+'upi', 'repair', and 'down'
+
+    Members in the chain that are fully in sync and thus preserving the
+    Update Propagation Invariant, up but under repair (simulated), and
+    down, respectively.
+
+'ps,[some list]'
+
+    The list of asymmetric network partitions.  {a,b} means that a
+    cannot send to b, but b can send to a.
+
+    This partition list is recorded for debugging purposes but is *not*
+    used by the algorithm.  The algorithm only 'feels' its effects via
+    simulated timeout whenever there's a partition in one of the
+    messaging directions.
+
+'nodes_up,[list]'
+
+    The best guess right now of which ndoes are up, relative to the
+    author node, specified by '{author,X}'
+
+'SET partitions = [some list]'
+
+    All subsequent iterations should have a stable list of partitions,
+    i.e. the 'ps' list described should be stable.
+
+'{FLAP: x flaps n}!'
+
+    Server x has detected that it's flapping/oscillating after iteration
+    n of a naive/1st draft detection algorithm.
+".
 
 convergence_demo_test_() ->
-    {timeout, 300, fun() -> convergence_demo_test(x) end}.
+    {timeout, 98*300, fun() -> convergence_demo_test(x) end}.
 
 convergence_demo_test(_) ->
+    timer:sleep(100),
+    io:format(user, short_doc(), []),
+    timer:sleep(3000),
+
     All_list = [a,b,c,d],
     machi_partition_simulator:start_link({111,222,33}, 0, 100),
     _ = machi_partition_simulator:get(All_list),
@@ -248,106 +336,137 @@ convergence_demo_test(_) ->
     {ok, Md} = ?MGR:start_link(d, All_list, d, MgrOpts),
     try
         {ok, P1} = ?MGR:test_calc_projection(Ma, false),
-        P1Epoch = P1#projection.epoch_number,
-        ok = machi_flu0:proj_write(FLUa, P1Epoch, public, P1),
-        ok = machi_flu0:proj_write(FLUb, P1Epoch, public, P1),
-        ok = machi_flu0:proj_write(FLUc, P1Epoch, public, P1),
-        ok = machi_flu0:proj_write(FLUd, P1Epoch, public, P1),
+      P1Epoch = P1#projection.epoch_number,
+      ok = machi_flu0:proj_write(FLUa, P1Epoch, public, P1),
+      ok = machi_flu0:proj_write(FLUb, P1Epoch, public, P1),
+      ok = machi_flu0:proj_write(FLUc, P1Epoch, public, P1),
+      ok = machi_flu0:proj_write(FLUd, P1Epoch, public, P1),
 
-        {now_using, XX1} = ?MGR:test_react_to_env(Ma),
-        ?D(XX1),
-        {now_using, _} = ?MGR:test_react_to_env(Mb),
-        {now_using, _} = ?MGR:test_react_to_env(Mc),
-        {QQ,QQP2,QQE2} = ?MGR:test_read_latest_public_projection(Ma, false),
-        ?D(QQ),
-        ?Dw(?MGR:make_projection_summary(QQP2)),
-        ?D(QQE2),
-        %% {unanimous,P2,E2} = test_read_latest_public_projection(Ma, false),
+      machi_partition_simulator:reset_thresholds(10, 50),
+      _ = machi_partition_simulator:get(All_list),
 
-        machi_partition_simulator:reset_thresholds(10, 50),
-        _ = machi_partition_simulator:get(All_list),
+      Parent = self(),
+      DoIt = fun(Iters, S_min, S_max) ->
+                     io:format(user, "\nDoIt: top\n\n", []),
+                     Pids = [spawn(fun() ->
+                                           random:seed(now()),
+                                           [begin
+                                                erlang:yield(),
+                                                S_max_rand = random:uniform(
+                                                               S_max + 1),
+                                                io:format(user, "{t}", []),
+                                                Elapsed =
+                                                    ?MGR:sleep_ranked_order(
+                                                       S_min, S_max_rand,
+                                                       M_name, All_list),
+                                                _ = ?MGR:test_react_to_env(MMM),
+                                                %% Be more unfair by not
+                                                %% sleeping here.
+                                                %% timer:sleep(S_max - Elapsed),
+                                                Elapsed
+                                            end || _ <- lists:seq(1, Iters)],
+                                           Parent ! done
+                                   end) || {M_name, MMM} <- [{a, Ma},
+                                                             {b, Mb},
+                                                             {c, Mc},
+                                                             {d, Md}] ],
+                     [receive
+                          done ->
+                              ok
+                      after 995000 ->
+                              exit(icky_timeout)
+                      end || _ <- Pids]
+             end,
 
-        Parent = self(),
-        DoIt = fun(Iters, S_min, S_max) ->
-                       Pids = [spawn(fun() ->
-                                             [begin
-                                                  erlang:yield(),
-                                                  Elapsed =
-                                                      ?MGR:sleep_ranked_order(S_min, S_max, M_name, All_list),
-                                                  Res = ?MGR:test_react_to_env(MMM),
-                                                  timer:sleep(S_max - Elapsed),
-                                                  Res=Res %% ?D({self(), Res})
-                                              end || _ <- lists:seq(1, Iters)],
-                                             Parent ! done
-                                     end) || {M_name, MMM} <- [{a, Ma},
-                                                               {b, Mb},
-                                                               {c, Mc},
-                                                               {d, Md}] ],
-                       [receive
-                            done ->
-                                ok
-                        after 995000 ->
-                                exit(icky_timeout)
-                        end || _ <- Pids]
-               end,
+      XandYs1 = [[{X,Y}] || X <- All_list, Y <- All_list, X /= Y],
+      XandYs2 = [[{X,Y}, {A,B}] || X <- All_list, Y <- All_list, X /= Y,
+                                   A <- All_list, B <- All_list, A /= B,
+                                   X /= A],
+      %% XandYs3 = [[{X,Y}, {A,B}, {C,D}] || X <- All_list, Y <- All_list, X /= Y,
+      %%                                     A <- All_list, B <- All_list, A /= B,
+      %%                                     C <- All_list, D <- All_list, C /= D,
+      %%                                     X /= A, X /= C, A /= C],
+      AllPartitionCombinations = XandYs1 ++ XandYs2,
+      %% AllPartitionCombinations = XandYs3,
+      ?D({?LINE, length(AllPartitionCombinations)}),
 
-        DoIt(30, 0, 0),
-        io:format(user, "\nSET always_last_partitions ON ... we should see convergence to correct chains.\n", []),
-        %% machi_partition_simulator:always_these_partitions([{b,a}]),
-        machi_partition_simulator:always_these_partitions([{a,b}]),
-        %% machi_partition_simulator:always_these_partitions([{b,c}]),
-        %% machi_partition_simulator:always_these_partitions([{a,c},{c,b}]),
-        %% machi_partition_simulator:always_last_partitions(),
-        [DoIt(25, 40, 400) || _ <- [1]],
-        %% TODO: We should be stable now ... analyze it.
+      machi_partition_simulator:reset_thresholds(10, 50),
+      io:format(user, "\nLet loose the dogs of war!\n", []),
+      DoIt(30, 0, 0),
+      [begin
+           %% machi_partition_simulator:reset_thresholds(10, 50),
+           %% io:format(user, "\nLet loose the dogs of war!\n", []),
+           %% DoIt(30, 0, 0),
+           machi_partition_simulator:always_these_partitions(Partition),
+           io:format(user, "\nSET partitions = ~w.\n", [Partition]),
+           [DoIt(50, 10, 100) || _ <- [1,2,3,4] ],
+           true = private_projections_are_stable(Namez, DoIt),
+           true = all_hosed_lists_are_identical(Namez, Partition),
+           io:format(user, "\nSweet, we converged & all_hosed are unanimous-or-islands-inconclusive.\n", []),
+           %% PPP =
+           %%     [begin
+           %%          PPPallPubs = machi_flu0:proj_list_all(FLU, public),
+           %%          [begin
+           %%               {ok, Pr} = machi_flu0:proj_read(FLU, PPPepoch, public),
+           %%               {Pr#projection.epoch_number, FLUName, Pr}
+           %%           end || PPPepoch <- PPPallPubs]
+           %%      end || {FLUName, FLU} <- Namez],
+           %% io:format(user, "PPP ~p\n", [lists:sort(lists:append(PPP))]),
+           timer:sleep(1000),
+           ok
+       end || Partition <- AllPartitionCombinations
+       %% end || Partition <- [ [{c,a}] ]
+       %% end || Partition <- [ [{c,a}], [{c,b}, {a, b}] ]
+       %% end || Partition <- [ [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}],
+       %%                       [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}, {b,c}],
+       %%                       [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}, {c,d}] ]
+      ],
+       %% exit(end_experiment),
 
-        io:format(user, "\nSET always_last_partitions OFF ... let loose the dogs of war!\n", []),
-        machi_partition_simulator:reset_thresholds(10, 50),
-        DoIt(30, 0, 0),
-        io:format(user, "\nSET always_last_partitions ON ... we should see convergence to correct chains2.\n", []),
-        %% machi_partition_simulator:always_last_partitions(),
-        machi_partition_simulator:always_these_partitions([{a,c}]),
-        [DoIt(25, 40, 400) || _ <- [1]],
+      io:format(user, "\nSET partitions = []\n", []),
+      io:format(user, "Sweet, finishing early\n", []), exit(yoyoyo_testing_hack),
+      io:format(user, "We should see convergence to 1 correct chain.\n", []),
+      machi_partition_simulator:no_partitions(),
+      [DoIt(50, 10, 100) || _ <- [1]],
+      true = private_projections_are_stable(Namez, DoIt),
+      io:format(user, "~s\n", [os:cmd("date")]),
 
-        io:format(user, "\nSET always_last_partitions ON ... we should see convergence to correct chains3.\n", []),
-        machi_partition_simulator:no_partitions(),
-        [DoIt(20, 40, 400) || _ <- [1]],
-        %% TODO: We should be stable now ... analyze it.
-        io:format(user, "~s\n", [os:cmd("date")]),
+      %% We are stable now ... analyze it.
 
-        %% Create a report where at least one FLU has written a
-        %% private projection.
-        Report = unanimous_report(Namez),
-        %% ?D(Report),
+      %% Create a report where at least one FLU has written a
+      %% private projection.
+      Report = unanimous_report(Namez),
+      %% ?D(Report),
 
-        %% Report is ordered by Epoch.  For each private projection
-        %% written during any given epoch, confirm that all chain
-        %% members appear in only one unique chain, i.e., the sets of
-        %% unique chains are disjoint.
-        true = all_reports_are_disjoint(Report),
+      %% Report is ordered by Epoch.  For each private projection
+      %% written during any given epoch, confirm that all chain
+      %% members appear in only one unique chain, i.e., the sets of
+      %% unique chains are disjoint.
+      true = all_reports_are_disjoint(Report),
 
-        %% Given the report, we flip it around so that we observe the
-        %% sets of chain transitions relative to each FLU.
-        R_Chains = [extract_chains_relative_to_flu(FLU, Report) ||
-                       FLU <- All_list],
-        %% ?D(R_Chains),
-        R_Projs = [{FLU, [chain_to_projection(FLU, Epoch, UPI, Repairing,
-                                              All_list) ||
-                             {Epoch, UPI, Repairing} <- E_Chains]} ||
-                      {FLU, E_Chains} <- R_Chains],
+      %% Given the report, we flip it around so that we observe the
+      %% sets of chain transitions relative to each FLU.
+      R_Chains = [extract_chains_relative_to_flu(FLU, Report) ||
+                     FLU <- All_list],
+      %% ?D(R_Chains),
+      R_Projs = [{FLU, [chain_to_projection(FLU, Epoch, UPI, Repairing,
+                                            All_list) ||
+                           {Epoch, UPI, Repairing} <- E_Chains]} ||
+                    {FLU, E_Chains} <- R_Chains],
 
-        %% For each chain transition experienced by a particular FLU,
-        %% confirm that each state transition is OK.
-        try
-            [{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane(Ps, FLU)} ||
-                {FLU, Ps} <- R_Projs]
-        catch _Err:_What ->
-                io:format(user, "Report ~p\n", [Report]),
-                exit({line, ?LINE, _Err, _What})
-        end,
-        %% ?D(R_Projs),
+      %% For each chain transition experienced by a particular FLU,
+      %% confirm that each state transition is OK.
+      try
+          [{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane(Ps, FLU)} ||
+              {FLU, Ps} <- R_Projs],
+          io:format(user, "\nAll sanity checks pass, hooray!\n", [])
+      catch _Err:_What ->
+              io:format(user, "Report ~p\n", [Report]),
+              exit({line, ?LINE, _Err, _What})
+      end,
+      %% ?D(R_Projs),
 
-        ok
+      ok
     after
         ok = ?MGR:stop(Ma),
         ok = ?MGR:stop(Mb),
@@ -356,5 +475,36 @@ convergence_demo_test(_) ->
         ok = machi_partition_simulator:stop()
     end.
 
+private_projections_are_stable(Namez, PollFunc) ->
+    Private1 = [machi_flu0:proj_get_latest_num(FLU, private) ||
+                   {_Name, FLU} <- Namez],
+    PollFunc(5, 1, 10),
+    Private2 = [machi_flu0:proj_get_latest_num(FLU, private) ||
+                   {_Name, FLU} <- Namez],
+    true = (Private1 == Private2).
+
+all_hosed_lists_are_identical(Namez, Partition) ->
+    Ps = [machi_flu0:proj_read_latest(FLU, private) || {_Name, FLU} <- Namez],
+    Uniques = lists:usort([machi_chain_manager1:get_all_hosed(P) ||
+                              {ok, P} <- Ps]),
+    Members = [M || {M, _Pid} <- Namez],
+    Islands = machi_partition_simulator:partitions2num_islands(
+                Members, Partition),
+    %% io:format(user, "all_hosed_lists_are_identical:\n", []),
+    %% io:format(user, "  Uniques = ~p Islands ~p\n  Partition ~p\n",
+    %%           [Uniques, Islands, Partition]),
+    case length(Uniques) of
+        1 ->
+            true;
+        _ when Islands == 'many' ->
+            %% There are at least two partitions, so yes, it's quite
+            %% possible that the all_hosed lists may differ.
+            %% TODO Fix this up to be smarter about fully-isolated
+            %% islands of partition.
+            true;
+        _ ->
+            false
+    end.
+
 -endif. % not PULSE
 -endif. % TEST

prototype/chain-manager/test/machi_partition_simulator.erl

@@ -38,6 +38,8 @@
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
+-export([islands2partitions/1, partitions2num_islands/2]).
+
 -define(TAB, ?MODULE).
 
 -record(state, {
@@ -178,6 +180,16 @@ islands2partitions([Island|Rest]) ->
     ++
     islands2partitions(Rest).
 
-
+partitions2num_islands(Members, Partition) ->
+    Connections0 = [{X,Y} || X <- Members, Y <- Members, X /= Y],
+    Connections1 = Connections0 -- Partition,
+    Cs = [lists:member({X,Y}, Connections1)
+          orelse
+          lists:member({Y,X}, Connections1) || X <- Members, Y <- Members,
+                                               X /= Y],
+    case lists:usort(Cs) of
+        [true]        -> 1;
+        [false, true] -> many                   % TODO too lazy to finish
+    end.
 
 -endif. % TEST

prototype/corfurl/ebin/corfurl.app

@@ -0,0 +1,9 @@
+{application,corfurl,
+             [{description,"Quick prototype of CORFU in Erlang."},
+              {vsn,"0.0.0"},
+              {applications,[kernel,stdlib,lager]},
+              {mod,{corfurl_unfinished_app,[]}},
+              {registered,[]},
+              {env,[{ring_size,32}]},
+              {modules,[corfurl,corfurl_client,corfurl_flu,corfurl_sequencer,
+                        corfurl_util]}]}.

prototype/tango/ebin/tango.app

@@ -0,0 +1,10 @@
+{application,tango,
+             [{description,"Really quick hack prototype of Tango on top of corfurl."},
+              {vsn,"0.0.0"},
+              {applications,[kernel,stdlib,lager]},
+              {mod,{tango_does_not_exist_app,[]}},
+              {registered,[]},
+              {env,[]},
+              {modules,[corfurl,corfurl_client,corfurl_flu,corfurl_sequencer,
+                        corfurl_util,tango,tango_dt,tango_dt_map,
+                        tango_dt_queue,tango_dt_register,tango_oid]}]}.