Add public/private split in projection store of machi_flu0.erl

prototype/poc-machi/src/machi_chain_manager0.erl

@@ -0,0 +1,191 @@
+%% -------------------------------------------------------------------
+%%
+%% Machi: a small village of replicated files
+%%
+%% Copyright (c) 2014 Basho Technologies, Inc.  All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+%% -------------------------------------------------------------------
+-module(machi_chain_manager0).
+
+-include("machi.hrl").
+
+-export([make_initial_state/3, calc_projection/3, make_projection_summary/1]).
+
+-ifdef(TEST).
+-compile(export_all).
+-ifdef(EQC).
+-include_lib("eqc/include/eqc.hrl").
+-endif.
+-ifdef(PULSE).
+-compile({parse_transform, pulse_instrument}).
+-endif.
+
+-endif. %TEST
+
+make_initial_state(MyName, All_list, Seed) ->
+    RunEnv = [{seed, Seed},
+              {network_partitions, []}],
+    #ch_mgr{name=MyName,
+           proj=make_initial_projection(MyName, All_list, All_list, [], []),
+           runenv=RunEnv}.
+
+make_initial_projection(MyName, All_list, UPI_list, Repairing_list, Ps) ->
+    make_projection(1, 0, <<>>,
+                    MyName, All_list, [], UPI_list, Repairing_list, Ps).
+
+make_projection(EpochNum, PrevEpochNum, PrevEpochCSum,
+                MyName, All_list, Down_list, UPI_list, Repairing_list, Ps) ->
+    P = #projection{epoch_number=EpochNum,
+                    epoch_csum= <<>>,
+                    prev_epoch_num=PrevEpochNum,
+                    prev_epoch_csum=PrevEpochCSum,
+                    creation_time=now(),
+                    author_server=MyName,
+                    all_members=All_list,
+                    down=Down_list,
+                    upi=UPI_list,
+                    repairing=Repairing_list,
+                    dbg=Ps},
+    CSum = crypto:hash(sha, term_to_binary(P)),
+    P#projection{epoch_csum=CSum}.
+
+%% OldThreshold: Percent chance of using the old/previous network partition list
+%% NoPartitionThreshold: If the network partition changes, what are the odds
+%%                       that there are no partitions at all?
+
+calc_projection(OldThreshold, NoPartitionThreshold,
+                #ch_mgr{name=MyName, proj=LastProj, runenv=RunEnv1} = S) ->
+    #projection{epoch_number=OldEpochNum,
+                epoch_csum=OldEpochCsum,
+                all_members=All_list,
+                upi=OldUPI_list,
+                repairing=OldRepairing_list
+               } = LastProj,
+    LastUp = lists:usort(OldUPI_list ++ OldRepairing_list),
+    AllMembers = (S#ch_mgr.proj)#projection.all_members,
+    {Up, RunEnv2} = calc_up_nodes(MyName, OldThreshold, NoPartitionThreshold,
+                                  AllMembers, RunEnv1),
+    NewUp = Up -- LastUp,
+    Down = AllMembers -- Up,
+
+    NewUPI_list = [X || X <- OldUPI_list, lists:member(X, Up)],
+    Repairing_list2 = [X || X <- OldRepairing_list, lists:member(X, Up)],
+    {NewUPI_list3, Repairing_list3, RunEnv3} =
+        case {NewUp, Repairing_list2} of
+            {[], []} ->
+                {NewUPI_list, [], RunEnv2};
+            {[], [H|T]} ->
+                {Prob, RunEnvX} = roll_dice(100, RunEnv2),
+                if Prob =< 50 ->
+                        {NewUPI_list ++ [H], T, RunEnvX};
+                   true ->
+                        {NewUPI_list, OldRepairing_list, RunEnvX}
+                end;
+            {_, _} ->
+                {NewUPI_list, OldRepairing_list, RunEnv2}
+        end,
+    Repairing_list4 = case NewUp of
+                          []    -> Repairing_list3;
+                          NewUp -> Repairing_list3 ++ NewUp
+                      end,
+    Repairing_list5 = Repairing_list4 -- Down,
+    P = make_projection(OldEpochNum + 1, OldEpochNum, OldEpochCsum,
+                        MyName, All_list, Down, NewUPI_list3, Repairing_list5,
+                        [goo]),
+    {P, S#ch_mgr{runenv=RunEnv3}}.
+
+calc_up_nodes(MyName, OldThreshold, NoPartitionThreshold,
+              AllMembers, RunEnv1) ->
+    Seed1 = proplists:get_value(seed, RunEnv1),
+    Partitions1 = proplists:get_value(network_partitions, RunEnv1),
+    {Seed2, Partitions2} =
+        calc_network_partitions(AllMembers, Seed1, Partitions1,
+                                OldThreshold, NoPartitionThreshold),
+    UpNodes = lists:sort(
+                [Node || Node <- AllMembers,
+                         not lists:member({MyName, Node}, Partitions2),
+                         not lists:member({Node, MyName}, Partitions2)]),
+    RunEnv2 = replace(RunEnv1,
+                      [{seed, Seed2}, {network_partitions, Partitions2}]),
+    {UpNodes, RunEnv2}.
+
+
+calc_network_partitions(Nodes, Seed1, OldPartition,
+                        OldThreshold, NoPartitionThreshold) ->
+    {Cutoff2, Seed2} = random:uniform_s(100, Seed1),
+    if Cutoff2 < OldThreshold ->
+            {Seed2, OldPartition};
+       true ->
+            {Cutoff3, Seed3} = random:uniform_s(100, Seed1),
+            if Cutoff3 < NoPartitionThreshold ->
+                    {Seed3, []};
+               %% Cutoff3 rem 10 < 5 ->
+               %%      %% case get(goofus) of undefined -> put(goofus, true), io:format(user, "~w", [Cutoff3 rem 10]); _ -> ok end,
+               %%      OldSeed = case random:seed(Seed3) of undefined -> now();
+               %%                                           Else      -> Else
+               %%                end,
+               %%      {Down, _} = lists:partition(fun(X) -> X /= 'a' andalso random:uniform() < 0.5 end, Nodes),
+               %%      %% case get(goofus) of undefined -> put(goofus, true), io:format(user, "~w", [Down]); _ -> ok end,
+               %%      Partitions = [{X, Y} || X <- Nodes, Y <- Down],
+               %%      random:seed(OldSeed),
+               %%      {Seed3, Partitions};
+               true ->
+                    make_network_partition_locations(Nodes, Seed3)
+            end
+    end.
+
+replace(PropList, Items) ->
+    lists:foldl(fun({Key, Val}, Ps) ->
+                        lists:keyreplace(Key, 1, Ps, {Key,Val})
+                end, PropList, Items).
+
+make_projection_summary(#projection{epoch_number=EpochNum,
+                                    all_members=_All_list,
+                                    down=Down_list,
+                                    upi=UPI_list,
+                                    repairing=Repairing_list}) ->
+    [{epoch,EpochNum},
+     {upi,UPI_list},{repair,Repairing_list},{down,Down_list}].
+
+roll_dice(N, RunEnv) ->
+    Seed1 = proplists:get_value(seed, RunEnv),
+    {Val, Seed2} = random:uniform_s(N, Seed1),
+    {Val, replace(RunEnv, [{seed, Seed2}])}.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+make_network_partition_locations(Nodes, Seed1) ->
+    Pairs = make_all_pairs(Nodes),
+    Num = length(Pairs),
+    {Seed2, Weights} = lists:foldl(
+                         fun(_, {Seeda, Acc}) ->
+                                 {Cutoff, Seedb} = random:uniform_s(100, Seeda),
+                                 {Seedb, [Cutoff|Acc]}
+                         end, {Seed1, []}, lists:seq(1, Num)),
+    {Cutoff3, Seed3} = random:uniform_s(100, Seed2),
+    {Seed3, [X || {Weight, X} <- lists:zip(Weights, Pairs),
+                  Weight < Cutoff3]}.
+
+make_all_pairs(L) ->
+    lists:flatten(make_all_pairs2(lists:usort(L))).
+
+make_all_pairs2([]) ->
+    [];
+make_all_pairs2([_]) ->
+    [];
+make_all_pairs2([H1|T]) ->
+    [[{H1, X}, {X, H1}] || X <- T] ++ make_all_pairs(T).

prototype/poc-machi/src/machi_flu0.erl

@@ -27,7 +27,7 @@
 
 -export([start_link/1, stop/1,
          write/3, read/2, trim/2,
-         proj_write/3, proj_read/2, proj_get_latest_num/1, proj_read_latest/1]).
+         proj_write/4, proj_read/3, proj_get_latest_num/2, proj_read_latest/2]).
 -export([set_fake_repairing_status/2, get_fake_repairing_status/1]).
 -export([make_proj/1, make_proj/2]).
 
@@ -54,15 +54,16 @@
 -type register() :: 'unwritten' | binary() | 'trimmed'.
 
 -record(state, {
-          name :: list(),
+          name :: atom(),
           wedged = false :: boolean(),
           register = 'unwritten' :: register(),
           fake_repairing = false :: boolean(),
           proj_epoch :: non_neg_integer(),
-          proj_store :: dict()
+          proj_store_pub :: dict(),
+          proj_store_priv :: dict()
          }).
 
-start_link(Name) when is_list(Name) ->
+start_link(Name) when is_atom(Name) ->
     gen_server:start_link(?MODULE, [Name], []).
 
 stop(Pid) ->
@@ -77,17 +78,21 @@ write(Pid, Epoch, Bin) ->
 trim(Pid, Epoch) ->
     g_call(Pid, {reg_op, Epoch, trim}, ?LONG_TIME).
 
-proj_write(Pid, Epoch, Proj) ->
+proj_write(Pid, Epoch, StoreType, Proj)
-    g_call(Pid, {proj_write, Epoch, Proj}, ?LONG_TIME).
+  when StoreType == public; StoreType == private ->
+    g_call(Pid, {proj_write, Epoch, StoreType, Proj}, ?LONG_TIME).
 
-proj_read(Pid, Epoch) ->
+proj_read(Pid, Epoch, StoreType)
-    g_call(Pid, {proj_read, Epoch}, ?LONG_TIME).
+  when StoreType == public; StoreType == private ->
+    g_call(Pid, {proj_read, Epoch, StoreType}, ?LONG_TIME).
 
-proj_get_latest_num(Pid) ->
+proj_get_latest_num(Pid, StoreType)
-    g_call(Pid, {proj_get_latest_num}, ?LONG_TIME).
+  when StoreType == public; StoreType == private ->
+    g_call(Pid, {proj_get_latest_num, StoreType}, ?LONG_TIME).
 
-proj_read_latest(Pid) ->
+proj_read_latest(Pid, StoreType)
-    g_call(Pid, {proj_read_latest}, ?LONG_TIME).
+  when StoreType == public; StoreType == private ->
+    g_call(Pid, {proj_read_latest, StoreType}, ?LONG_TIME).
 
 set_fake_repairing_status(Pid, Status) ->
     gen_server:call(Pid, {set_fake_repairing_status, Status}, ?LONG_TIME).
@@ -115,7 +120,8 @@ init([Name]) ->
     lclock_init(),
     {ok, #state{name=Name,
                 proj_epoch=-42,
-                proj_store=orddict:new()}}.
+                proj_store_pub=orddict:new(),
+                proj_store_priv=orddict:new()}}.
 
 handle_call({{reg_op, _Epoch, _}, LC1}, _From, #state{wedged=true} = S) ->
     LC2 = lclock_update(LC1),
@@ -160,25 +166,25 @@ handle_call({{reg_op, _Epoch, trim}, LC1}, _From, #state{register=trimmed} = S)
     LC2 = lclock_update(LC1),
     {reply, {error_trimmed, LC2}, S};
 
-handle_call({{proj_write, Epoch, Proj}, LC1}, _From, S) ->
+handle_call({{proj_write, Epoch, StoreType, Proj}, LC1}, _From, S) ->
     LC2 = lclock_update(LC1),
-    {Reply, NewS} = do_proj_write(Epoch, Proj, S),
+    {Reply, NewS} = do_proj_write(Epoch, StoreType, Proj, S),
     {reply, {Reply, LC2}, NewS};
-handle_call({{proj_read, Epoch}, LC1}, _From, S) ->
+handle_call({{proj_read, Epoch, StoreType}, LC1}, _From, S) ->
     LC2 = lclock_update(LC1),
-    {Reply, NewS} = do_proj_read(Epoch, S),
+    {Reply, NewS} = do_proj_read(Epoch, StoreType, S),
     {reply, {Reply, LC2}, NewS};
-handle_call({{proj_get_latest_num}, LC1}, _From, S) ->
+handle_call({{proj_get_latest_num, StoreType}, LC1}, _From, S) ->
     LC2 = lclock_update(LC1),
-    {Reply, NewS} = do_proj_get_latest_num(S),
+    {Reply, NewS} = do_proj_get_latest_num(StoreType, S),
     {reply, {Reply, LC2}, NewS};
-handle_call({{proj_read_latest}, LC1}, _From, S) ->
+handle_call({{proj_read_latest, StoreType}, LC1}, _From, S) ->
     LC2 = lclock_update(LC1),
-    case do_proj_get_latest_num(S) of
+    case do_proj_get_latest_num(StoreType, S) of
         {error_unwritten, _S} ->
             {reply, {error_unwritten, LC2}, S};
         {{ok, Epoch}, _S} ->
-            Proj = orddict:fetch(Epoch, S#state.proj_store),
+            Proj = orddict:fetch(Epoch, get_store_dict(StoreType, S)),
             {reply, {{ok, Proj}, LC2}, S}
     end;
 handle_call({stop, LC1}, _From, MLP) ->
@@ -206,24 +212,25 @@ code_change(_OldVsn, MLP, _Extra) ->
 
 %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%% %%%%
 
-do_proj_write(Epoch, Proj, #state{proj_epoch=MyEpoch, proj_store=D,
+do_proj_write(Epoch, StoreType, Proj, #state{proj_epoch=MyEpoch,
-                                    wedged=MyWedged} = S) ->
+                                             wedged=MyWedged} = S) ->
+    D = get_store_dict(StoreType, S),
     case orddict:find(Epoch, D) of
         error ->
             D2 = orddict:store(Epoch, Proj, D),
             {NewEpoch, NewWedged} = if Epoch > MyEpoch ->
-                                              {Epoch, false};
+                                            {Epoch, false};
-                                         true ->
+                                       true ->
-                                              {MyEpoch, MyWedged}
+                                            {MyEpoch, MyWedged}
-                                      end,
+                                    end,
-            {ok, S#state{wedged=NewWedged,
+            {ok, set_store_dict(StoreType, D2, S#state{wedged=NewWedged,
-                         proj_epoch=NewEpoch,
+                                                       proj_epoch=NewEpoch})};
-                         proj_store=D2}};
         {ok, _} ->
             {error_written, S}
     end.
 
-do_proj_read(Epoch, #state{proj_store=D} = S) ->
+do_proj_read(Epoch, StoreType, S) ->
+    D = get_store_dict(StoreType, S),
     case orddict:find(Epoch, D) of
         error ->
             {error_unwritten, S};
@@ -231,7 +238,8 @@ do_proj_read(Epoch, #state{proj_store=D} = S) ->
             {{ok, Proj}, S}
     end.
 
-do_proj_get_latest_num(#state{proj_store=D} = S) ->
+do_proj_get_latest_num(StoreType, S) ->
+    D = get_store_dict(StoreType, S),
     case lists:sort(orddict:to_list(D)) of
         [] ->
             {error_unwritten, S};
@@ -240,6 +248,16 @@ do_proj_get_latest_num(#state{proj_store=D} = S) ->
             {{ok, Epoch}, S}
     end.
 
+get_store_dict(public, #state{proj_store_pub=D}) ->
+    D;
+get_store_dict(private, #state{proj_store_priv=D}) ->
+    D.
+
+set_store_dict(public, D, S) ->
+    S#state{proj_store_pub=D};
+set_store_dict(private, D, S) ->
+    S#state{proj_store_priv=D}.
+
 -ifdef(TEST).
 
 lclock_init() ->

prototype/poc-machi/test/machi_chain_manager_test0.erl

@@ -0,0 +1,468 @@
+%% -------------------------------------------------------------------
+%%
+%% Machi: a small village of replicated files
+%%
+%% Copyright (c) 2014 Basho Technologies, Inc.  All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+%% -------------------------------------------------------------------
+-module(machi_chain_manager_test0).
+
+-include("machi.hrl").
+
+-define(MGR, machi_chain_manager0).
+-define(D(X), io:format(user, "~s ~p\n", [??X, X])).
+
+-export([]).
+
+-ifdef(TEST).
+
+-ifdef(EQC).
+-include_lib("eqc/include/eqc.hrl").
+%% -include_lib("eqc/include/eqc_statem.hrl").
+-define(QC_OUT(P),
+        eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)).
+-endif.
+
+-include_lib("eunit/include/eunit.hrl").
+-compile(export_all).
+
+smoke0_test() ->
+    S0 = ?MGR:make_initial_state(a, [a,b,c,d], {4,5,6}),
+    lists:foldl(fun(_, S) ->
+                        {P1, S1} = ?MGR:calc_projection(20, 1, S),
+                        %% io:format(user, "~p\n", [?MGR:make_projection_summary(P1)]),
+                        S1#ch_mgr{proj=P1}
+                end, S0, lists:seq(1,10)).
+
+-ifdef(CRUFT_DELETE_ME_MAYBE).
+
+-record(s, {
+          step = 0 :: non_neg_integer(),
+          seed :: {integer(), integer(), integer()}
+         }).
+
+gen_all_nodes() ->
+    [a, b, c].
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+initial_state() ->
+    #s{}.
+
+command(#s{step=0}) ->
+    L = gen_all_nodes(),
+    {call, ?MODULE, init_run, [gen_rand_seed(), L]};
+command(_S) ->
+    foo.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+prop_m() ->
+    ?FORALL({_Cmds, _PulseSeed}, {commands(?MODULE), pulse:seed()},
+            begin
+                true
+            end).
+
+-endif. % CRUFT_DELETE_ME_MAYBE
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+gen_rand_seed() ->
+    noshrink({gen_num(), gen_num(), gen_num()}).
+
+gen_num() ->
+    choose(1, 50000).
+
+prop_calc_projection() ->
+    prop_calc_projection([a,b,c], false).
+
+prop_calc_projection(Nodes, TrackUseTab) ->
+    ?FORALL(
+       {Seed, OldThreshold, NoPartitionThreshold, Steps, HeadNode},
+       {gen_rand_seed(),
+        oneof([0,15,35,55,75,85,100]),
+        oneof([0,15,35,55,75,85,100]),
+        5000,
+        oneof(Nodes)},
+       begin
+           erase(goofus),
+           {RandN, _} = random:uniform_s(length(Nodes), Seed),
+           NodesShuffle = lists:nth(RandN, perms(Nodes)),
+           S0 = ?MGR:make_initial_state(HeadNode, NodesShuffle, Seed),
+           F = fun(_, {S, Acc}) ->
+                       {P1, S1} = ?MGR:calc_projection(
+                                    OldThreshold, NoPartitionThreshold, S),
+                       %%io:format(user, "~p\n", [make_projection_summary(P1)]),
+                       {S1#ch_mgr{proj=P1}, [P1|Acc]}
+               end,
+           {_, Projs0} = lists:foldl(F, {S0, []}, lists:seq(1,Steps)),
+           Projs = lists:reverse(Projs0),
+           if TrackUseTab == false ->
+                   ok;
+              true ->
+                   Transitions = extract_upi_transitions(Projs),
+                   %% io:format(user, "\n~P\n", [lists:usort(Transitions), 10]),
+                   [ets:update_counter(TrackUseTab, Trans, 1) ||
+                       Trans <- Transitions]
+           end,
+           true = projection_transitions_are_sane(Projs)
+       end).
+
+extract_upi_transitions([]) ->
+    [];
+extract_upi_transitions([_]) ->
+    [];
+extract_upi_transitions([P1, P2|T]) ->
+    Trans = {P1#projection.upi, P2#projection.upi},
+    [Trans|extract_upi_transitions([P2|T])].
+
+find_common_prefix([], _) ->
+    [];
+find_common_prefix(_, []) ->
+    [];
+find_common_prefix([H|L1], [H|L2]) ->
+    [H|find_common_prefix(L1, L2)];
+find_common_prefix(_, _) ->
+    [].
+
+calc_projection_test_() ->
+    Runtime = 15, %% Runtime = 60*60,
+    {timeout, Runtime * 500,
+     fun() ->
+             Nodes = [a,b,c],
+             Cs = combinations(Nodes),
+             Combos = lists:sort([{UPI1, UPI2} || UPI1 <- Cs, UPI2 <- Cs]),
+             put(hack, 0),
+             timer:sleep(500),
+             io:format(user, "\n", []),
+             Rs = [begin
+                       Hack = get(hack),if Hack rem 10000 == 0 -> ?D({time(), Hack}); true -> ok end,put(hack, Hack + 1),
+                       P1 = ?MGR:make_projection(2, 1, <<>>, HdNd, Nodes,
+                                                 [], PsUPI1, Nodes -- PsUPI1, []),
+                       P2 = ?MGR:make_projection(3, 2, <<>>, HdNd, Nodes,
+                                                 [], PsUPI2, Nodes -- PsUPI2, []),
+                       Res = case projection_transition_is_sane(P1, P2) of
+                                 true -> true;
+                                 _    -> false
+                             end,
+                       {Res, PsUPI1, PsUPI2}
+                   end || HdNd <- Nodes,
+                          {PsUPI1,PsUPI2} <- Combos,
+                          %% We assume that the author appears in any UPI list
+                          PsUPI1 /= [],
+                          PsUPI2 /= [],
+                          %% HdNd is the author for all of these
+                          %% tests, so it must be present in UPI1 & UPI2
+                          lists:member(HdNd, PsUPI1),
+                          lists:member(HdNd, PsUPI2)],
+             OKs = [begin
+                        {UPI1,UPI2}
+                    end || {true, UPI1, UPI2} <- Rs],
+             %% io:format(user, "OKs = ~p\n", [lists:usort(OKs)]),
+             Tab = ets:new(count, [public, set, {keypos, 1}]),
+             [ets:insert(Tab, {Transition, 0}) || Transition <- OKs],
+
+             true = eqc:quickcheck(
+                        eqc:testing_time(Runtime,
+                        ?QC_OUT(prop_calc_projection(Nodes, Tab)))),
+
+             NotCounted = lists:sort([Transition || {Transition, 0} <- ets:tab2list(Tab)]),
+             Counted = [X || {_, N}=X <- ets:tab2list(Tab),
+                             N > 0],
+             CountedX = lists:sort(fun(X, Y) ->
+                                           element(2, X) < element(2, Y)
+                                   end, Counted),
+             timer:sleep(100),
+             File = io_lib:format("/tmp/manager-test.~w.~w.~w",
+                                  tuple_to_list(time())),
+             {ok, FH} = file:open(File, [write]),
+             io:format(FH, "% NotCounted =\n~p.\n", [NotCounted]),
+             file:close(FH),
+             %% io:format(user, "\tNotCounted = ~p\n", [NotCounted]),
+             io:format(user, "\n\tNotCounted list was written to ~s\n", [File]),
+             io:format(user, "\tOKs length = ~p\n", [length(OKs)]),
+             io:format(user, "\tTransitions hit = ~p\n",
+                       [length(OKs) - length(NotCounted)]),
+             %% io:format(user, "\tTransitions = ~p\n", [lists:sort(_Counted)]),
+             io:format(user, "\tNotCounted length = ~p\n",
+                       [length(NotCounted)]),
+             io:format(user, "\tLeast-counted transition = ~p\n",
+                       [hd(CountedX)]),
+             io:format(user, "\tMost-counted transition  = ~p\n",
+                       [lists:last(CountedX)]),
+             ok
+     end}.
+
+projection_transitions_are_sane([]) ->
+    true;
+projection_transitions_are_sane([_]) ->
+    true;
+projection_transitions_are_sane([P1, P2|T]) ->
+    case projection_transition_is_sane(P1, P2) of
+        true ->
+            projection_transitions_are_sane([P2|T]);
+        Else ->
+            Else
+    end.
+
+projection_transition_is_sane(
+  #projection{epoch_number=Epoch1,
+              epoch_csum=CSum1,
+              prev_epoch_num=PrevEpoch1,
+              prev_epoch_csum=PrevCSum1,
+              creation_time=CreationTime1,
+              author_server=AuthorServer1,
+              all_members=All_list1,
+              down=Down_list1,
+              upi=UPI_list1,
+              repairing=Repairing_list1,
+              dbg=Dbg1} = P1,
+  #projection{epoch_number=Epoch2,
+              epoch_csum=CSum2,
+              prev_epoch_num=PrevEpoch2,
+              prev_epoch_csum=PrevCSum2,
+              creation_time=CreationTime2,
+              author_server=AuthorServer2,
+              all_members=All_list2,
+              down=Down_list2,
+              upi=UPI_list2,
+              repairing=Repairing_list2,
+              dbg=Dbg2} = P2) ->
+ try
+    true = is_integer(Epoch1) andalso is_integer(Epoch2),
+    true = is_binary(CSum1) andalso is_binary(CSum2),
+    true = is_integer(PrevEpoch1) andalso is_integer(PrevEpoch2),
+    true = is_binary(PrevCSum1) andalso is_binary(PrevCSum2),
+    {_,_,_} = CreationTime1,
+    {_,_,_} = CreationTime2,
+    true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2), % todo will probably change
+    true = is_list(All_list1) andalso is_list(All_list2),
+    true = is_list(Down_list1) andalso is_list(Down_list2),
+    true = is_list(UPI_list1) andalso is_list(UPI_list2),
+    true = is_list(Repairing_list1) andalso is_list(Repairing_list2),
+    true = is_list(Dbg1) andalso is_list(Dbg2),
+
+    true = Epoch2 > Epoch1,
+    true = PrevEpoch2 > PrevEpoch1,
+    All_list1 = All_list2,                 % todo will probably change
+
+    %% No duplicates
+    true = lists:sort(Down_list2) == lists:usort(Down_list2),
+    true = lists:sort(UPI_list2) == lists:usort(UPI_list2),
+    true = lists:sort(Repairing_list2) == lists:usort(Repairing_list2),
+
+    %% Disjoint-ness
+    true = lists:sort(All_list2) == lists:sort(Down_list2 ++ UPI_list2 ++
+                                                   Repairing_list2),
+    [] = [X || X <- Down_list2, not lists:member(X, All_list2)],
+    [] = [X || X <- UPI_list2, not lists:member(X, All_list2)],
+    [] = [X || X <- Repairing_list2, not lists:member(X, All_list2)],
+    DownS2 = sets:from_list(Down_list2),
+    UPIS2 = sets:from_list(UPI_list2),
+    RepairingS2 = sets:from_list(Repairing_list2),
+    true = sets:is_disjoint(DownS2, UPIS2),
+    true = sets:is_disjoint(DownS2, RepairingS2),
+    true = sets:is_disjoint(UPIS2, RepairingS2),
+
+    %% The author must not be down.
+    false = lists:member(AuthorServer1, Down_list1),
+    false = lists:member(AuthorServer2, Down_list2),
+    %% The author must be in either the UPI or repairing list.
+    true = lists:member(AuthorServer1, UPI_list1 ++ Repairing_list1),
+    true = lists:member(AuthorServer2, UPI_list2 ++ Repairing_list2),
+
+    %% Additions to the UPI chain may only be at the tail
+    UPI_common_prefix = find_common_prefix(UPI_list1, UPI_list2),
+    if UPI_common_prefix == [] ->
+            if UPI_list1 == [] orelse UPI_list2 == [] ->
+                    %% If the common prefix is empty, then one of the
+                    %% inputs must be empty.
+                    true;
+               true ->
+                    %% Otherwise, we have a case of UPI changing from
+                    %% one of these two situations:
+                    %%
+                    %% UPI_list1 -> UPI_list2
+                    %% [d,c,b,a] -> [c,a]
+                    %% [d,c,b,a] -> [c,a,repair_finished_added_to_tail].
+                    NotUPI2 = (Down_list2 ++ Repairing_list2),
+                    true = lists:prefix(UPI_list1 -- NotUPI2,
+                                        UPI_list2)
+            end;
+       true ->
+            true
+    end,
+    true = lists:prefix(UPI_common_prefix, UPI_list1),
+    true = lists:prefix(UPI_common_prefix, UPI_list2),
+    UPI_1_suffix = UPI_list1 -- UPI_common_prefix,
+    UPI_2_suffix = UPI_list2 -- UPI_common_prefix,
+
+    %% Where did elements in UPI_2_suffix come from?
+    %% Only two sources are permitted.
+    [true = lists:member(X, Repairing_list1) % X added after repair done
+            orelse
+            lists:member(X, UPI_list1)       % X in UPI_list1 after common pref
+     || X <- UPI_2_suffix],
+
+    %% The UPI_2_suffix must exactly be equal to: ordered items from
+    %% UPI_list1 concat'ed with ordered items from Repairing_list1.
+    %% Both temp vars below preserve relative order!
+    UPI_2_suffix_from_UPI1 = [X || X <- UPI_1_suffix,
+                                   lists:member(X, UPI_list2)],
+    UPI_2_suffix_from_Repairing1 = [X || X <- UPI_2_suffix,
+                                         lists:member(X, Repairing_list1)],
+    %% true?
+    UPI_2_suffix = UPI_2_suffix_from_UPI1 ++ UPI_2_suffix_from_Repairing1,
+
+    true
+ catch
+     _Type:_Err ->
+         S1 = ?MGR:make_projection_summary(P1),
+         S2 = ?MGR:make_projection_summary(P2),
+         Trace = erlang:get_stacktrace(),
+         {err, from, S1, to, S2, stack, Trace}
+ end.
+
+pass1_smoke_test() ->
+    P2 = ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d],
+                              [],      [d,c,b,a], [], []),
+    P3 = ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d],
+                              [b,c,d], [a],       [], []),
+    true = projection_transition_is_sane(P2, P3),
+
+    P2b= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d],
+                              [],      [d,c,b,a], [], []),
+    P3b= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d],
+                              [b,d],   [c,a],       [], []),
+    true = projection_transition_is_sane(P2b, P3b),
+
+    P2c= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d,e],
+                              [],      [e,d,c,b], [a], []),
+    P3c= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d,e],
+                              [e,c],   [d,b,a],   [], []),
+    true = projection_transition_is_sane(P2c, P3c),
+
+    true.
+
+fail1_smoke_test() ->
+    P15 = ?MGR:make_projection(15, 14, <<>>, a, [a,b,c,d],
+                              [b,d], [a],   [c], []),
+    P16 = ?MGR:make_projection(16, 15, <<>>, a, [a,b,c,d],
+                              [b,d], [c,a], [],  []),
+    true = projection_transition_is_sane(P15, P16) /= true,
+
+    P2d= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d,e],
+                              [],      [e,d,c,b], [a], []),
+    P3d= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d,e],
+                              [e,c],   [d,a,b],   [], []),
+    true = projection_transition_is_sane(P2d, P3d) /= true,
+
+    P2e= ?MGR:make_projection(2, 1, <<>>, a, [a,b,c,d,e],
+                              [],      [e,d,c,b], [a], []),
+    P3e= ?MGR:make_projection(3, 2, <<>>, a, [a,b,c,d,e],
+                              [e,c],   [a,d,b],   [], []),
+    true = projection_transition_is_sane(P2e, P3e) /= true,
+
+    true.
+
+fail2_smoke_test() ->
+    AB = [a,b],
+    BadSets = [{1, [b,a],[a,b]},
+               {2, [b],[a,b]},
+               %% {3, [a],[b]},  % weird but valid: [a] means b is repairing;
+                                 % so if a fails and b finishes repairing,
+                                 % swapping both at the same time is "ok".
+               {4, [a,b],[b,a]},
+               %% {5, [b],[a]},  % weird but valid: see above
+               {6, [a],[b,a]}
+              ],
+    [begin
+         P2f= ?MGR:make_projection(2, 1, <<>>, a, AB,
+                                   [], Two,   AB -- Two, []),
+         P3f= ?MGR:make_projection(3, 2, <<>>, a, AB,
+                                   [], Three, AB -- Three, []),
+         {Label, true} =
+             {Label, (projection_transition_is_sane(P2f, P3f) /= true)}
+     end || {Label, Two, Three} <- BadSets],
+
+    true.
+
+fail3_smoke_test() ->
+    Nodes = [a,b],
+    UPI1 = [a,b],
+    UPI2 = [b,a],
+    [begin
+         PsUPI1 = UPI1,
+         PsUPI2 = UPI2,
+         P1 = ?MGR:make_projection(2, 1, <<>>, HdNd, Nodes,
+                                   [], PsUPI1, Nodes -- PsUPI1, []),
+         P2 = ?MGR:make_projection(3, 2, <<>>, HdNd, Nodes,
+                                   [], PsUPI2, Nodes -- PsUPI2, []),
+         Res = case projection_transition_is_sane(P1, P2) of
+                   true -> true;
+                   _    -> false
+               end,
+         {Res, HdNd, PsUPI1, PsUPI2} = {false, HdNd, PsUPI1, PsUPI2}
+     end || HdNd <- Nodes],
+
+    true.
+
+fail4_smoke_test() ->
+    Nodes = [a,b,c],
+    Two = [c,b,a],
+    Three = [c,a,b],
+    P2a= ?MGR:make_projection(2, 1, <<>>, a, Nodes,
+                              [], Two,   Nodes -- Two, []),
+    P3a= ?MGR:make_projection(3, 2, <<>>, a, Nodes,
+                              [], Three, Nodes -- Three, []),
+    true = (projection_transition_is_sane(P2a, P3a) /= true),
+
+    true.
+
+
+%% aaa_smoke_test() ->
+%%     L = [a,b,c,d],
+%%     Cs = combinations(L),
+%%     Combos = lists:sort([{X, Y} || X <- Cs, Y <- Cs]),
+%%     timer:sleep(500),
+%%     io:format(user, "\n", []),
+%%     Rs = [begin
+%%               P1 = ?MGR:make_projection(2, 1, <<>>, a, L,
+%%                                         [], X, L -- X, []),
+%%               P2 = ?MGR:make_projection(3, 2, <<>>, a, L,
+%%                                         [], Y, L -- Y, []),
+%%               Res = case projection_transition_is_sane(P1, P2) of
+%%                         true -> true;
+%%                         _    -> false
+%%                     end,
+%%               {Res, P1, P2}
+%%           end || {X,Y} <- Combos],
+%%     OKs = [{X,Y} || {true, X, Y} <- Rs],
+%%     io:format(user, "Cs are ~p\n", [length(Cs)]),
+%%     io:format(user, "OKs are ~p\n", [length(OKs)]),
+%%     Bads = [{X,Y} || {false, X, Y} <- Rs],
+%%     io:format(user, "Bads are ~p\n", [length(Bads)]),
+%%     %% [io:format(user, "~p -> ~p: ~p\n", [X, Y, Res]);
+%%     ok.
+
+combinations(L) ->
+    lists:usort(perms(L) ++ lists:append([ combinations(L -- [X]) || X <- L])).
+
+perms([]) -> [[]];
+perms(L)  -> [[H|T] || H <- L, T <- perms(L--[H])].
+
+-endif.

prototype/poc-machi/test/machi_flu0_test.erl

@@ -31,7 +31,7 @@
 -ifdef(TEST).
 
 repair_status_test() ->
-    {ok, F} = machi_flu0:start_link("one"),
+    {ok, F} = machi_flu0:start_link(one),
     try
         ok = machi_flu0:set_fake_repairing_status(F, true),
         true = machi_flu0:get_fake_repairing_status(F),
@@ -47,13 +47,13 @@ concuerror1_test() ->
     ok.
 
 concuerror2_test() ->
-    {ok, F} = machi_flu0:start_link("one"),
+    {ok, F} = machi_flu0:start_link(one),
     ok = machi_flu0:stop(F),
     ok.
 
 concuerror3_test() ->
     Me = self(),
-    Fun = fun() -> {ok, F1} = machi_flu0:start_link("one"),
+    Fun = fun() -> {ok, F1} = machi_flu0:start_link(one),
                         ok = machi_flu0:stop(F1),
                         Me ! done
                end,
@@ -65,7 +65,7 @@ concuerror3_test() ->
 
 concuerror4_test() ->
     event_setup(),
-    {ok, F1} = machi_flu0:start_link("one"),
+    {ok, F1} = machi_flu0:start_link(one),
     Epoch = 1,
     ok = m_proj_write(F1, Epoch, dontcare),
 
@@ -103,7 +103,7 @@ concuerror4_test() ->
     
 proj_store_test() ->
     event_setup(),
-    {ok, F1} = machi_flu0:start_link("one"),
+    {ok, F1} = machi_flu0:start_link(one),
 
     error_unwritten = m_proj_get_latest_num(F1),
     error_unwritten = m_proj_read_latest(F1),
@@ -120,7 +120,7 @@ proj_store_test() ->
 
 wedge_test() ->
     event_setup(),
-    {ok, F1} = machi_flu0:start_link("one"),
+    {ok, F1} = machi_flu0:start_link(one),
     Epoch1 = 1,
     ok = m_proj_write(F1, Epoch1, dontcare),
 
@@ -148,8 +148,8 @@ wedge_test() ->
 proj0_test() ->
     Me = self(),
     event_setup(),
-    {ok, F1} = machi_flu0:start_link("one"),
+    {ok, F1} = machi_flu0:start_link(one),
-    {ok, F2} = machi_flu0:start_link("two"),
+    {ok, F2} = machi_flu0:start_link(two),
     FLUs = [F1, F2],
     FirstProj = machi_flu0:make_proj(1, FLUs),
     Epoch1 = FirstProj#proj.epoch,
@@ -207,7 +207,7 @@ m_stop(Pid) ->
     Res.
 
 m_proj_write(Pid, Epoch, Proj) ->
-    Res = machi_flu0:proj_write(Pid, Epoch, Proj),
+    Res = machi_flu0:proj_write(Pid, Epoch, public, Proj),
     event_add(proj_write, Pid, Res),
     Res.
 
@@ -227,17 +227,17 @@ m_proj_write_with_check(Pid, Epoch, Proj) ->
     end.
 
 m_proj_read(Pid, Epoch) ->
-    Res = machi_flu0:proj_read(Pid, Epoch),
+    Res = machi_flu0:proj_read(Pid, Epoch, public),
     event_add(proj_read, Pid, Res),
     Res.
 
 m_proj_get_latest_num(Pid) ->
-    Res = machi_flu0:proj_get_latest_num(Pid),
+    Res = machi_flu0:proj_get_latest_num(Pid, public),
     event_add(proj_get_latest_num, Pid, Res),
     Res.
 
 m_proj_read_latest(Pid) ->
-    Res = machi_flu0:proj_read_latest(Pid),
+    Res = machi_flu0:proj_read_latest(Pid, public),
     event_add(proj_read_latest, Pid, Res),
     Res.