984 lines
43 KiB
Erlang
984 lines
43 KiB
Erlang
%% -------------------------------------------------------------------
|
|
%%
|
|
%% Machi: a small village of replicated files
|
|
%%
|
|
%% Copyright (c) 2014-2015 Basho Technologies, Inc. All Rights Reserved.
|
|
%%
|
|
%% This file is provided to you under the Apache License,
|
|
%% Version 2.0 (the "License"); you may not use this file
|
|
%% except in compliance with the License. You may obtain
|
|
%% a copy of the License at
|
|
%%
|
|
%% http://www.apache.org/licenses/LICENSE-2.0
|
|
%%
|
|
%% Unless required by applicable law or agreed to in writing,
|
|
%% software distributed under the License is distributed on an
|
|
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
%% KIND, either express or implied. See the License for the
|
|
%% specific language governing permissions and limitations
|
|
%% under the License.
|
|
%%
|
|
%% -------------------------------------------------------------------
|
|
-module(machi_chain_manager1_converge_demo).
|
|
|
|
-include("machi.hrl").
|
|
-include("machi_projection.hrl").
|
|
|
|
-define(MGR, machi_chain_manager1).
|
|
|
|
-define(D(X), io:format(user, "~s ~p\n", [??X, X])).
|
|
-define(Dw(X), io:format(user, "~s ~w\n", [??X, X])).
|
|
-define(FLU_C, machi_flu1_client).
|
|
-define(FLU_PC, machi_proxy_flu1_client).
|
|
|
|
-compile(export_all).
|
|
|
|
-ifdef(TEST).
|
|
-ifndef(PULSE).
|
|
|
|
-ifdef(EQC).
|
|
-include_lib("eqc/include/eqc.hrl").
|
|
%% -include_lib("eqc/include/eqc_statem.hrl").
|
|
-define(QC_OUT(P),
|
|
eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)).
|
|
-endif.
|
|
|
|
-include_lib("eunit/include/eunit.hrl").
|
|
|
|
help() ->
|
|
io:format("~s\n", [short_doc()]).
|
|
|
|
short_doc() ->
|
|
"
|
|
A visualization of the convergence behavior of the chain self-management
|
|
algorithm for Machi.
|
|
|
|
1. Set up some server and chain manager pairs.
|
|
2. Create a number of different network partition scenarios, where
|
|
(simulated) partitions may be symmetric or asymmetric. Then stop changing
|
|
the partitions and keep the simulated network stable (and perhaps broken).
|
|
3. Run a number of iterations of the algorithm in parallel by poking each
|
|
of the manager processes on a random'ish basis.
|
|
4. Afterward, fetch the chain transition changes made by each FLU and
|
|
verify that no transition was unsafe.
|
|
|
|
During the iteration periods, the following is a cheatsheet for the output.
|
|
See the internal source for interpreting the rest of the output.
|
|
|
|
'SET partitions = '
|
|
|
|
A pair-wise list of actors which cannot send messages. The
|
|
list is uni-directional. If there are three servers (a,b,c),
|
|
and if the partitions list is '[{a,b},{b,c}]' then all
|
|
messages from a->b and b->c will be dropped, but any other
|
|
sender->recipient messages will be delivered successfully.
|
|
|
|
'x uses:'
|
|
|
|
The FLU x has made an internal state transition and is using
|
|
this epoch's projection as operating chain configuration. The
|
|
rest of the line is a summary of the projection.
|
|
|
|
'CONFIRM epoch {N}'
|
|
|
|
This message confirms that all of the servers listed in the
|
|
UPI and repairing lists of the projection at epoch {N} have
|
|
agreed to use this projection because they all have written
|
|
this projection to their respective private projection stores.
|
|
The chain is now usable by/available to all clients.
|
|
|
|
'Sweet, private projections are stable'
|
|
|
|
This report announces that this iteration of the test cycle
|
|
has passed successfully. The report that follows briefly
|
|
summarizes the latest private projection used by each
|
|
participating server. For example, when in strong consistency
|
|
mode with 'a' as a witness and 'b' and 'c' as real servers:
|
|
|
|
%% Legend:
|
|
%% server name, epoch ID, UPI list, repairing list, down list, ...
|
|
%% ... witness list, 'false' (a constant value)
|
|
|
|
[{a,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}},
|
|
{b,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}}]
|
|
|
|
Both servers 'a' and 'b' agree on epoch 1116 with epoch ID
|
|
{1116,<<23,143,246,55>>} where UPI=[a,b], repairing=[],
|
|
down=[c], and witnesses=[a].
|
|
|
|
Server 'c' is not shown because 'c' has wedged itself OOS (out
|
|
of service) by configuring a chain length of zero.
|
|
|
|
If no servers are listed in the report (i.e. only '[]' is
|
|
displayed), then all servers have wedged themselves OOS, and
|
|
the chain is unavailable.
|
|
|
|
'DoIt,'
|
|
|
|
This marks a group of tick events which trigger the manager
|
|
processes to evaluate their environment and perhaps make a
|
|
state transition.
|
|
|
|
A long chain of 'DoIt,DoIt,DoIt,' means that the chain state has
|
|
(probably) settled to a stable configuration, which is the goal of the
|
|
algorithm.
|
|
|
|
Press control-c to interrupt the test....".
|
|
|
|
%% ' silly Emacs syntax highlighting....
|
|
|
|
%% convergence_demo_test_() ->
|
|
%% {timeout, 98*300, fun() -> convergence_demo_testfun() end}.
|
|
|
|
%% convergence_demo_testfun() ->
|
|
%% convergence_demo_testfun(3).
|
|
|
|
-define(DEFAULT_MGR_OPTS, [{private_write_verbose, false},
|
|
{active_mode,false},
|
|
{use_partition_simulator, true}]).
|
|
|
|
t() ->
|
|
t(3).
|
|
|
|
t(N) ->
|
|
t(N, ?DEFAULT_MGR_OPTS).
|
|
|
|
t(N, MgrOpts) ->
|
|
convergence_demo_testfun(N, MgrOpts).
|
|
|
|
convergence_demo_testfun(NumFLUs, MgrOpts0) ->
|
|
%% Faster test startup, commented: io:format(user, short_doc(), []),
|
|
%% Faster test startup, commented: timer:sleep(3000),
|
|
|
|
application:start(sasl),
|
|
|
|
MgrOpts = MgrOpts0 ++ ?DEFAULT_MGR_OPTS,
|
|
TcpPort = proplists:get_value(port_base, MgrOpts, 62877),
|
|
TDir = proplists:get_value(tmp_dir, MgrOpts, "/tmp/c"),
|
|
ok = filelib:ensure_dir(TDir ++ "/not-used"),
|
|
_ = os:cmd("rm -rf " ++ TDir ++ "/*"),
|
|
FluInfo = [
|
|
{a,TcpPort+0,TDir++"/data.a"}, {b,TcpPort+1,TDir++"/data.b"},
|
|
{c,TcpPort+2,TDir++"/data.c"}, {d,TcpPort+3,TDir++"/data.d"},
|
|
{e,TcpPort+4,TDir++"/data.e"}, {f,TcpPort+5,TDir++"/data.f"},
|
|
{g,TcpPort+6,TDir++"/data.g"}, {h,TcpPort+7,TDir++"/data.h"},
|
|
{i,TcpPort+8,TDir++"/data.i"}, {j,TcpPort+9,TDir++"/data.j"},
|
|
{k,TcpPort+10,TDir++"/data.k"}, {l,TcpPort+11,TDir++"/data.l"},
|
|
{m,TcpPort+12,TDir++"/data.m"}, {n,TcpPort+13,TDir++"/data.n"},
|
|
{o,TcpPort+14,TDir++"/data.o"}, {p,TcpPort+15,TDir++"/data.p"},
|
|
{q,TcpPort+16,TDir++"/data.q"}, {r,TcpPort+17,TDir++"/data.r"}
|
|
],
|
|
FLU_biglist = [X || {X,_,_} <- FluInfo],
|
|
All_list = lists:sublist(FLU_biglist, NumFLUs),
|
|
io:format(user, "\nSET # of FLUs = ~w members ~w).\n",
|
|
[NumFLUs, All_list]),
|
|
|
|
machi_partition_simulator:start_link({111,222,33}, 0, 100),
|
|
_ = machi_partition_simulator:get(All_list),
|
|
|
|
Ps = [#p_srvr{name=Name,address="localhost",port=Port,
|
|
props=[{data_dir,Dir}|MgrOpts]} ||
|
|
{Name,Port,Dir} <- lists:sublist(FluInfo, NumFLUs)],
|
|
{ok, SupPid} = machi_flu_sup:start_link(),
|
|
[{ok, _} = machi_flu_psup:start_flu_package(P) || P <- Ps],
|
|
Namez = [begin
|
|
{ok, PPid} = ?FLU_PC:start_link(P),
|
|
{Name, PPid}
|
|
end || #p_srvr{name=Name}=P <- Ps],
|
|
MembersDict = machi_projection:make_members_dict(Ps),
|
|
Witnesses = proplists:get_value(witnesses, MgrOpts, []),
|
|
MgrNamez = [begin
|
|
MgrName = machi_flu_psup:make_mgr_supname(Name),
|
|
ok = ?MGR:set_chain_members(MgrName,MembersDict,Witnesses),
|
|
{Name, MgrName}
|
|
end || #p_srvr{name=Name} <- Ps],
|
|
CpApMode = case Witnesses /= [] of
|
|
true -> cp_mode;
|
|
false -> ap_mode
|
|
end,
|
|
|
|
try
|
|
[{_, Ma}|_] = MgrNamez,
|
|
{ok, P1} = ?MGR:test_calc_projection(Ma, false),
|
|
[_ = ?FLU_PC:write_projection(FLUPid, public, P1) ||
|
|
{_, FLUPid} <- Namez, FLUPid /= Ma],
|
|
|
|
machi_partition_simulator:reset_thresholds(10, 50),
|
|
_ = machi_partition_simulator:get(All_list),
|
|
|
|
Parent = self(),
|
|
DoIt = fun(Iters, S_min, S_max) ->
|
|
%% io:format(user, "\nDoIt: top\n\n", []),
|
|
io:format(user, "DoIt, ", []),
|
|
Pids = [{spawn(fun() ->
|
|
random:seed(now()),
|
|
[begin
|
|
erlang:yield(),
|
|
perhaps_adjust_pstore_sleep(),
|
|
S_max_rand = random:uniform(
|
|
S_max + 1),
|
|
%% io:format(user, "{t}", []),
|
|
Elapsed =
|
|
?MGR:sleep_ranked_order(
|
|
S_min, S_max_rand,
|
|
M_name, All_list),
|
|
_ = (catch ?MGR:trigger_react_to_env(MMM)),
|
|
%% Be more unfair by not
|
|
%% sleeping here.
|
|
% timer:sleep(S_max - Elapsed),
|
|
Elapsed
|
|
end || _ <- lists:seq(1, Iters)],
|
|
Parent ! {done, self()}
|
|
end), M_name} || {M_name, MMM} <- MgrNamez ],
|
|
[receive
|
|
{done, ThePid} ->
|
|
ok
|
|
after 120*1000 ->
|
|
[begin
|
|
case whereis(XX) of
|
|
undefined -> ok;
|
|
XXPid -> {_, XXbin} = process_info(XXPid, backtrace),
|
|
{_, XXdict} = process_info(XXPid, dictionary),
|
|
TTT = proplists:get_value(ttt, XXdict),
|
|
io:format(user, "BACK ~w: ttt=~w\n~s\n", [XX, TTT, XXbin])
|
|
end
|
|
end || XX <- [file_server_2] ++
|
|
[a_chmgr,b_chmgr,c_chmgr,d_chmgr,e_chmgr,f_chmgr,g_chmgr,h_chmgr,i_chmgr,j_chmgr] ++
|
|
[a_pstore,b_pstore,c_pstore,d_pstore,e_pstore,f_pstore,g_pstore,h_pstore,i_pstore,j_pstore] ++
|
|
[a_fitness,b_fitness,c_fitness,d_fitness,e_fitness,f_fitness,g_fitness,h_fitness,i_fitness,j_fitness] ],
|
|
[begin
|
|
[begin
|
|
case whereis(XX) of
|
|
undefined -> ok;
|
|
XXPid -> {_, XXbin} = process_info(XXPid, backtrace),
|
|
io:format(user, "BACK ~w: ~w\n~s\n", [XX, time(), XXbin])
|
|
end
|
|
end || XX <- [a_pstore,b_pstore,c_pstore,d_pstore,e_pstore,f_pstore,g_pstore,h_pstore,i_pstore,j_pstore] ],
|
|
timer:sleep(20)
|
|
end || _ <- lists:seq(1,30)],
|
|
exit({icky_timeout, M_name})
|
|
end || {ThePid,M_name} <- Pids]
|
|
end,
|
|
|
|
%% machi_partition_simulator:reset_thresholds(10, 50),
|
|
%% io:format(user, "\nLet loose the dogs of war!\n", []),
|
|
%% [DoIt(20, 0, 0) || _ <- lists:seq(1,9)],
|
|
%% %% io:format(user, "\nVariations of puppies and dogs of war!\n", []),
|
|
%% %% [begin
|
|
%% %% machi_partition_simulator:reset_thresholds(90, 90),
|
|
%% %% DoIt(7, 0, 0),
|
|
%% %% machi_partition_simulator:always_these_partitions([]),
|
|
%% %% DoIt(7, 0, 0)
|
|
%% %% end || _ <- lists:seq(1, 3)],
|
|
machi_partition_simulator:always_these_partitions([]),
|
|
io:format(user, "\nPuppies for everyone!\n", []),
|
|
[DoIt(20, 0, 0) || _ <- lists:seq(1,9)],
|
|
|
|
AllPs = make_partition_list(All_list),
|
|
PartitionCounts = lists:zip(AllPs, lists:seq(1, length(AllPs))),
|
|
MaxIters = NumFLUs * (NumFLUs + 1) * 6,
|
|
[begin
|
|
machi_partition_simulator:always_these_partitions(Partition),
|
|
io:format(user, "\n~s SET partitions = ~w (~w of ~w)\n",
|
|
[machi_util:pretty_time(), Partition, Count, length(AllPs)]),
|
|
true = lists:foldl(
|
|
fun(_, true) ->
|
|
true;
|
|
(_, _) ->
|
|
%% Run a few iterations
|
|
[DoIt(10, 10, 50) || _ <- lists:seq(1, 6)],
|
|
%% If stable, return true to short circuit remaining
|
|
private_projections_are_stable(Namez, DoIt)
|
|
end, false, lists:seq(0, MaxIters)),
|
|
io:format(user, "\n~s Sweet, private projections are stable\n", [machi_util:pretty_time()]),
|
|
io:format(user, "\t~P\n", [get(stable), 24]),
|
|
io:format(user, "Rolling sanity check ... ", []),
|
|
PrivProjs = [{Name, begin
|
|
{ok, Ps8} = ?FLU_PC:get_all_projections(
|
|
FLU, private, infinity),
|
|
[P || P <- Ps8,
|
|
P#projection_v1.epoch_number /= 0]
|
|
end} || {Name, FLU} <- Namez],
|
|
try
|
|
[{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane_retrospective(Psx, FLU)} ||
|
|
{FLU, Psx} <- PrivProjs]
|
|
catch
|
|
_Err:_What when CpApMode == cp_mode ->
|
|
io:format(user, "none proj skip detected, TODO? ", []);
|
|
_Err:_What when CpApMode == ap_mode ->
|
|
io:format(user, "PrivProjs ~p\n", [PrivProjs]),
|
|
exit({line, ?LINE, _Err, _What})
|
|
end,
|
|
io:format(user, "Yay!\n", []),
|
|
%% io:format(user, "\n\nEXITING!\n\n", []), timer:sleep(500), erlang:halt(0),
|
|
ReportXX = machi_chain_manager1_test:unanimous_report(Namez),
|
|
true = machi_chain_manager1_test:all_reports_are_disjoint(ReportXX),
|
|
io:format(user, "Yay for ReportXX!\n", []),
|
|
timer:sleep(1234),
|
|
|
|
%% TODO: static count is not sufficient. Must not delete the last
|
|
%% private_proj_is_upi_unanimous projection!
|
|
MaxFiles = 123,
|
|
[begin
|
|
Privs = filelib:wildcard(Dir ++ "/projection/private/*"),
|
|
FilesToDel1 = lists:sublist(Privs,
|
|
max(0, length(Privs)-MaxFiles)),
|
|
[_ = file:delete(File) || File <- FilesToDel1],
|
|
Pubs = filelib:wildcard(Dir ++ "/projection/public/*"),
|
|
FilesToDel2 = lists:sublist(Pubs,
|
|
max(0, length(Pubs)-MaxFiles)),
|
|
[_ = file:delete(File) || File <- FilesToDel2],
|
|
io:format(user, "Yay, now prune: ~w ~w, ", [length(FilesToDel1), length(FilesToDel2)])
|
|
end || Dir <- filelib:wildcard(TDir ++ "/data*")],
|
|
io:format(user, "\n", []),
|
|
|
|
timer:sleep(1250),
|
|
ok
|
|
end || {Partition, Count} <- PartitionCounts
|
|
],
|
|
|
|
io:format(user, "\nSET partitions = []\n", []),
|
|
io:format(user, "We should see convergence to 1 correct chain.\n", []),
|
|
machi_partition_simulator:no_partitions(),
|
|
[DoIt(50, 10, 50) || _ <- [1,2,3]],
|
|
true = private_projections_are_stable(Namez, DoIt),
|
|
io:format(user, "~s\n", [os:cmd("date")]),
|
|
|
|
%% We are stable now ... analyze it.
|
|
|
|
%% Create a report where at least one FLU has written a
|
|
%% private projection.
|
|
Report = machi_chain_manager1_test:unanimous_report(Namez),
|
|
%% ?D(Report),
|
|
|
|
%% Report is ordered by Epoch. For each private projection
|
|
%% written during any given epoch, confirm that all chain
|
|
%% members appear in only one unique chain, i.e., the sets of
|
|
%% unique chains are disjoint.
|
|
true = machi_chain_manager1_test:all_reports_are_disjoint(Report),
|
|
%% io:format(user, "\nLast Reports: ~p\n", [lists:nthtail(length(Report)-8,Report)]),
|
|
|
|
%% For each chain transition experienced by a particular FLU,
|
|
%% confirm that each state transition is OK.
|
|
PrivProjs = [{Name, begin
|
|
{ok, Ps9} = ?FLU_PC:get_all_projections(FLU,
|
|
private),
|
|
[P || P <- Ps9,
|
|
P#projection_v1.epoch_number /= 0]
|
|
end} || {Name, FLU} <- Namez],
|
|
try
|
|
[{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane_retrospective(Psx, FLU)} ||
|
|
{FLU, Psx} <- PrivProjs],
|
|
io:format(user, "\nAll sanity checks pass, hooray!\n", [])
|
|
catch
|
|
_Err:_What when CpApMode == cp_mode ->
|
|
io:format(user, "none proj skip detected, TODO? ", []);
|
|
_Err:_What when CpApMode == ap_mode ->
|
|
io:format(user, "Report ~p\n", [Report]),
|
|
io:format(user, "PrivProjs ~p\n", [PrivProjs]),
|
|
exit({line, ?LINE, _Err, _What})
|
|
end,
|
|
%% ?D(R_Projs),
|
|
|
|
ok
|
|
catch
|
|
XX:YY ->
|
|
io:format(user, "BUMMER ~p ~p @ ~p\n",
|
|
[XX, YY, erlang:get_stacktrace()]),
|
|
exit({bummer,XX,YY})
|
|
after
|
|
exit(SupPid, normal),
|
|
ok = machi_partition_simulator:stop(),
|
|
[ok = ?FLU_PC:quit(PPid) || {_, PPid} <- Namez],
|
|
machi_util:wait_for_death(SupPid, 100)
|
|
end.
|
|
|
|
%% Many of the static partition lists below have been problematic at one
|
|
%% time or another.....
|
|
%%
|
|
%% Uncomment *one* of the following make_partition_list() bodies.
|
|
|
|
make_partition_list(All_list) ->
|
|
%% [ [{a,c},{b,c}],
|
|
%% [{a,b},{c,b}] ].
|
|
|
|
%% Island1 = [hd(All_list), lists:last(All_list)],
|
|
%% Island2 = All_list -- Island1,
|
|
%% [
|
|
%% [{X,Y} || X <- Island1, Y <- Island2]
|
|
%% ++
|
|
%% [{X,Y} || X <- Island2, Y <- Island1]
|
|
%% ].
|
|
|
|
_X_Ys1 = [[{X,Y}] || X <- All_list, Y <- All_list, X /= Y],
|
|
_X_Ys2 = [[{X,Y}, {A,B}] || X <- All_list, Y <- All_list, X /= Y,
|
|
A <- All_list, B <- All_list, A /= B,
|
|
X /= A],
|
|
_X_Ys3 = [[{X,Y}, {A,B}, {C,D}] || X <- All_list, Y <- All_list, X /= Y,
|
|
A <- All_list, B <- All_list, A /= B,
|
|
C <- All_list, D <- All_list, C /= D,
|
|
X /= A, X /= C, A /= C],
|
|
%% _X_Ys4 = [[{X,Y}, {A,B}, {C,D}, {E,F}] ||
|
|
%% X <- All_list, Y <- All_list, X /= Y,
|
|
%% A <- All_list, B <- All_list, A /= B,
|
|
%% C <- All_list, D <- All_list, C /= D,
|
|
%% E <- All_list, F <- All_list, E /= F,
|
|
%% X /= A, X /= C, X /= E, A /= C, A /= E,
|
|
%% C /= E],
|
|
%% Concat = _X_Ys1,
|
|
%% Concat = _X_Ys2,
|
|
%% Concat = _X_Ys1 ++ _X_Ys2,
|
|
%% %% Concat = _X_Ys3,
|
|
Concat = _X_Ys1 ++ _X_Ys2 ++ _X_Ys3,
|
|
%% Concat = _X_Ys1 ++ _X_Ys2 ++ _X_Ys3 ++ _X_Ys4,
|
|
NoPartitions = lists:duplicate(trunc(length(Concat) * 0.1), []),
|
|
uniq_reverse(random_sort(lists:usort([lists:sort(L) || L <- Concat])
|
|
++ NoPartitions)).
|
|
|
|
%% %% for len=5 and 2 witnesses
|
|
%% [
|
|
%% [{b,c}],
|
|
%% [],
|
|
%% [{b,c}],
|
|
%% [{a,c},{b,c}],
|
|
%% [{b,c}],
|
|
%% [],
|
|
%% [{c,d}],
|
|
%% [],
|
|
%% [{d,e}],
|
|
%% [],
|
|
%% [{d,c}],
|
|
%% [{b,c}],
|
|
%% [],
|
|
%% [{c,e}]
|
|
%% ].
|
|
%% [
|
|
%% [{b,c}],
|
|
%% [{a,c},{b,c}]
|
|
%% %% [{b,c}],
|
|
%% %% [],
|
|
%% %% [{c,d}],
|
|
%% %% [],
|
|
%% %% [{d,e}],
|
|
%% %% [],
|
|
%% %% [{c,e}]
|
|
%% ].
|
|
|
|
%% [
|
|
%% [{b,c}],
|
|
%% [{b,c},{c,d},{e,a}],
|
|
%% [{a,c},{a,d},{a,e},{c,a},{d,a},{e,a},{b,c},{b,d},{b,e},{b,c},{b,d},{b,e}, % iof2
|
|
%% {c,a},{c,b},{c,d},{c,e},{a,c},{b,c},{d,c},{e,c}, % island of 1
|
|
%% {d,a},{d,b},{d,c},{d,e},{a,d},{b,d},{c,d},{e,d}, % island of 1
|
|
%% {e,a},{e,b},{e,c},{e,d},{a,e},{b,e},{c,e},{d,e}],% island of 1
|
|
%% [{a,e},{b,c},{d,e}] % the stinker?
|
|
%% ,
|
|
%% [],
|
|
%% [{b,a},{d,e},{e,a}],
|
|
%% [{b,c},{c,d}],
|
|
%% [{a,c},{c,a},{d,b}],
|
|
%% [{a,e},{c,e},{e,d}],
|
|
%% [{a,e},{c,d},{d,b}],
|
|
%% [{b,e},{c,a},{e,d}],
|
|
%% [{b,c},{c,d},{e,a}],
|
|
%% [{d,e},{e,c}],
|
|
%% [{a,e},{b,c},{d,e}] % the stinker?
|
|
%% ,
|
|
%% [],
|
|
%% [{e,a},{g,d}],
|
|
%% [{b,f},{f,b}],
|
|
%% [{a,g},{c,d}]
|
|
%% ]. % for 5 in AP, yay, working now.
|
|
|
|
|
|
%% [ [{a,b},{b,d},{c,b}],
|
|
%% [{a,b},{b,d},{c,b}, {a,b},{b,a},{a,c},{c,a},{a,d},{d,a}],
|
|
%% %% [{a,b},{b,d},{c,b}, {b,a},{a,b},{b,c},{c,b},{b,d},{d,b}],
|
|
%% [{a,b},{b,d},{c,b}, {c,a},{a,c},{c,b},{b,c},{c,d},{d,c}],
|
|
%% [{a,b},{b,d},{c,b}, {d,a},{a,d},{d,b},{b,d},{d,c},{c,d}] ].
|
|
|
|
%% [ [{a,b}, {b,c}],
|
|
%% [{a,b}, {a,c}] ].
|
|
%% Q = [ {X,Y} || X <- [a], Y <- [b,c,d,e,f,g,h,i,j,k,l,m,n,o,p] ],
|
|
%% %% [ [{d,e}], Q]. %% len=7 problem: bad inner flip when ps=[] at end!
|
|
%% [ Q, [{a,b},{c,d},{e,f}] ]. %% len=7 problem: WTF, double-check please!
|
|
|
|
%% len=7 problem: insane evil-near-infinite-loop sometimes
|
|
|
|
%% [ [{a,b}], Q, [{c,d}], Q, [{d,e}], Q].
|
|
|
|
%% [ [{a,b}, {b,c}] ]. %% hosed-not-equal @ 3 FLUs
|
|
|
|
%% [ [{b,d}] ].
|
|
|
|
%% [ [{a,b}], [], [{a,b}], [], [{a,b}] ].
|
|
|
|
%% [
|
|
%% %% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
|
|
%% %% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
|
|
%% %% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
|
|
%% %% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
|
|
%% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
|
|
%% [{b,a},{d,e}],
|
|
%% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], []
|
|
%% ].
|
|
|
|
%% [
|
|
%% [{a,b}],
|
|
%% [],
|
|
%% [{a,b}, {b,a}],
|
|
%% [],
|
|
%% [{b,c}]
|
|
%% ].
|
|
|
|
%% [
|
|
%% [{a,c},{b,a},{c,b}],
|
|
%% [{b,a}]
|
|
%% ].
|
|
|
|
%% [
|
|
%% [{c,b}, {c,a}],
|
|
%% [{b,c}, {b,a}],
|
|
%% [],
|
|
%% [{c,b}, {c,a}],
|
|
%% [{b,c}, {b,a}]
|
|
%% ].
|
|
|
|
%% [
|
|
%% [{a,b}], [],
|
|
%% [{b,a}, {b,c}], [],
|
|
%% [{c,b}, {c,a}, {d,c}], [],
|
|
%% [{c,b}, {c,a}], [],
|
|
|
|
%% [{b,a}, {c,a}], [],
|
|
%% [{a,b}, {c,b}], [],
|
|
%% [{b,c}, {a,c}]
|
|
%% ].
|
|
|
|
%% [ [{a,b},{b,c},{c,a}],
|
|
%% [{a,b}, {b,a}, {a,c},{c,a}] ].
|
|
|
|
%% [ [{a,b}, {c,b}],
|
|
%% [{a,b}, {b,c}] ].
|
|
|
|
%% [ [{a,b}, {b,c}, {c,d}],
|
|
%% [{a,b}, {b,c},{b,d}, {c,d}],
|
|
%% [{b,a}, {b,c}, {c,d}],
|
|
%% [{a,b}, {c,b}, {c,d}],
|
|
%% [{a,b}, {b,c}, {d,c}] ].
|
|
|
|
%% [
|
|
%% %% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
|
|
%% %% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
|
|
%% %% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
|
|
%% %% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
|
|
%% [{a,b}], [], [{a,b}], [], [{a,b}]
|
|
%% %% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
|
|
%% %% [{b,a},{d,e}],
|
|
%% %% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], []
|
|
%% ].
|
|
%% [ [{a,b}, {b,c}, {c,d}, {d,e}],
|
|
%% [{b,a}, {b,c}, {c,d}, {d,e}],
|
|
%% [{a,b}, {c,b}, {c,d}, {d,e}],
|
|
%% [{a,b}, {b,c}, {d,c}, {d,e}],
|
|
%% [{a,b}, {b,c}, {c,d}, {e,d}] ].
|
|
|
|
%% [ [{c,a}] ]. %% TODO double-check for total repair stability at SET=[]!!
|
|
|
|
%% [ [{c,a}],
|
|
%% [{c,b}, {a, b}] ].
|
|
|
|
%% [ [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}],
|
|
%% [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}, {b,c}],
|
|
%% [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}, {c,d}] ].
|
|
|
|
%% [ [{a,b}, {a,b},{b,a},{a,c},{c,a},{a,d},{d,a}],
|
|
%% [{a,b}, {b,a},{a,b},{b,c},{c,b},{b,d},{d,b}],
|
|
%% [{a,b}],
|
|
%% [{a,b}, {c,a},{a,c},{c,b},{b,c},{c,d},{d,c}],
|
|
%% [{a,b}, {d,a},{a,d},{d,b},{b,d},{d,c},{c,d}] ].
|
|
|
|
todo_why_does_this_crash_sometimes(FLUName, FLU, PPPepoch) ->
|
|
try
|
|
{ok, _}=Res = ?FLU_PC:read_projection(FLU, public, PPPepoch),
|
|
Res
|
|
catch _:_ ->
|
|
io:format(user, "QQQ Whoa, it crashed this time for ~p at epoch ~p\n",
|
|
[FLUName, PPPepoch]),
|
|
timer:sleep(1000),
|
|
exit(still_a_problem),
|
|
?FLU_PC:read_projection(FLU, public, PPPepoch)
|
|
end.
|
|
|
|
private_projections_are_stable(Namez, PollFunc) ->
|
|
FilterNoneProj = fun({_EpochID,[],[],_Dn,_W,InnerP}) -> false;
|
|
(_) -> true
|
|
end,
|
|
Private1x = [{Name, get_latest_inner_proj_summ(FLU)} || {Name,FLU} <- Namez],
|
|
Private1 = [X || X={_,Proj} <- Private1x, FilterNoneProj(Proj)],
|
|
[PollFunc(15, 1, 10) || _ <- lists:seq(1,6)],
|
|
Private2x = [{Name, get_latest_inner_proj_summ(FLU)} || {Name,FLU} <- Namez],
|
|
Private2 = [X || X={_,Proj} <- Private2x, FilterNoneProj(Proj)],
|
|
%% Is = [Inner_p || {_,_,_,_,Inner_p} <- Private1],
|
|
put(stable, lists:sort(Private1)),
|
|
%% We want either all true or all false (inner or not) ... except
|
|
%% that it isn't quite that simple. I've now witnessed a case
|
|
%% where the projections are stable but not everyone is
|
|
%% unanimously outer or unanimously inner!
|
|
%% Old partitions: [{a,b},{b,c},{c,a}]
|
|
%% result: all 3 had inner proj of [self]
|
|
%% New partitions: [{b,a},{c,b}]
|
|
%% Priv1 [{342,[c,a],[],[b],[],false},
|
|
%% {326,[b],[],[a,c],[],true},
|
|
%% {342,[c,a],[],[b],[],false}]
|
|
%% ... and it stays completely stable with these epoch #s.
|
|
%%
|
|
%% So, instead, if inner/outer status isn't unanimous, then we
|
|
%% should check to see if the sets of unique UPIs are disjoint.
|
|
%%
|
|
FLUs = [FLU || {FLU,_Pid} <- Namez],
|
|
U_UPI_Rs = lists:usort([UPI++Rep ||
|
|
{_Nm,{_EpochID,UPI,Rep,_Dn,_W,InnerP}} <- Private2]),
|
|
FLU_uses = [{Name, EpochID} ||
|
|
{Name,{EpochID,_UPI,Rep,_Dn,_W,InnerP}} <- Private2],
|
|
Witnesses = hd([Ws ||
|
|
{_Name,{_EpochID,_UPI,Rep,_Dn,Ws,InnerP}} <- Private2x]),
|
|
HaveWitnesses_p = Witnesses /= [],
|
|
CMode = if HaveWitnesses_p -> cp_mode;
|
|
true -> ap_mode
|
|
end,
|
|
Unanimous_with_all_peers_p =
|
|
lists:all(fun({FLU, UsesEpochID}) ->
|
|
WhoInEpochID = [Name ||
|
|
{Name,{EpochID,_UPI,_Rep,_Dn,_W,I_}}<-Private2,
|
|
EpochID == UsesEpochID],
|
|
WhoInEpochID_s = ordsets:from_list(WhoInEpochID),
|
|
UPI_R_versions = [UPI++Rep ||
|
|
{_Name,{EpochID,UPI,Rep,_Dn,_W,I_}}<-Private2,
|
|
EpochID == UsesEpochID],
|
|
UPI_R_vers_s = ordsets:from_list(hd(UPI_R_versions)),
|
|
UPI_R_versions == [ [] ] % This FLU in minority partition
|
|
orelse
|
|
(length(lists:usort(UPI_R_versions)) == 1
|
|
andalso
|
|
(ordsets:is_subset(UPI_R_vers_s, WhoInEpochID_s) orelse
|
|
(CMode == cp_mode andalso
|
|
ordsets:is_disjoint(UPI_R_vers_s, WhoInEpochID_s))))
|
|
end, FLU_uses),
|
|
Flat_U_UPI_Rs = lists:flatten(U_UPI_Rs),
|
|
%% Pubs = [begin
|
|
%% {ok, P} = ?FLU_PC:read_latest_projection(FLU, public),
|
|
%% {Name, P#projection_v1.epoch_number}
|
|
%% end || {Name, FLU} <- Namez],
|
|
|
|
%% In AP mode, if not disjoint, then a FLU will appear twice in
|
|
%% flattented U_UPIs.
|
|
AP_mode_disjoint_test_p =
|
|
if CMode == cp_mode ->
|
|
true;
|
|
CMode == ap_mode ->
|
|
lists:sort(Flat_U_UPI_Rs) == lists:usort(Flat_U_UPI_Rs)
|
|
end,
|
|
|
|
CP_mode_agree_test_p =
|
|
if CMode == cp_mode ->
|
|
FullMajority = (length(Namez) div 2) + 1,
|
|
EpochIDs = lists:sort(
|
|
[EpochID || {_Name,{EpochID,_UPI,_Rep,_Dn,_W,I_}}<-Private2]),
|
|
case lists:reverse(lists:sort(uniq_c(EpochIDs))) of
|
|
[{Count,EpochID}|_] when Count >= FullMajority ->
|
|
[{UPI, Rep}] = lists:usort(
|
|
[{_UPI,_Rep} || {_Name,{EpochIDx,_UPI,_Rep,_Dn,_W,I_}}<-Private2,
|
|
EpochIDx == EpochID]),
|
|
ExpectedFLUs = lists:sort(UPI ++ Rep),
|
|
UsingFLUs = lists:sort(
|
|
[Name || {Name,{EpochIDx,_UPI,_Rep,_Dn,_W,I_}}<-Private2,
|
|
EpochIDx == EpochID]),
|
|
io:format(user, "Priv2: EID ~W e ~w u ~w\n", [EpochID, 7, ExpectedFLUs, UsingFLUs]),
|
|
ordsets:is_subset(ordsets:from_list(ExpectedFLUs),
|
|
ordsets:from_list(UsingFLUs));
|
|
[{1=_Count,_EpochID}|_] ->
|
|
%% Our list is sorted & reversed, so 1=_Count
|
|
%% is biggest. If a majority is using the none proj,
|
|
%% then we're OK.
|
|
Private2None = [X || {_,{_,[],[],_,_,_}}=X <- Private2],
|
|
length(Private2None) >= FullMajority;
|
|
[] when Private2 == [], Private2x /= [] ->
|
|
%% Everyone is using none proj, chain is unavailable.
|
|
true;
|
|
Else ->
|
|
%% This is bad: we have a count that's less than
|
|
%% FullMajority but greater than 1.
|
|
false
|
|
end;
|
|
CMode == ap_mode ->
|
|
true
|
|
end,
|
|
|
|
%% io:format(user, "\nPriv1 ~p\nPriv2 ~p\n1==2 ~w ap_disjoint ~w u_all_peers ~w cp_mode_agree ~w\n", [lists:sort(Private1), lists:sort(Private2), Private1 == Private2, AP_mode_disjoint_test_p, Unanimous_with_all_peers_p, CP_mode_agree_test_p]),
|
|
Private1 == Private2 andalso
|
|
AP_mode_disjoint_test_p andalso
|
|
(
|
|
%% Another property that we want is that for each participant
|
|
%% X mentioned in a UPI or Repairing list of some epoch E that
|
|
%% X is using the same epoch E.
|
|
%%
|
|
%% It's possible (in theory) for humming consensus to agree on
|
|
%% the membership of UPI+Repairing but arrive those lists at
|
|
%% different epoch numbers. Machi chain replication won't
|
|
%% work in that case: all participants need to be using the
|
|
%% same epoch (and csum)!
|
|
(CMode == ap_mode andalso Unanimous_with_all_peers_p)
|
|
orelse
|
|
(CMode == cp_mode andalso CP_mode_agree_test_p)
|
|
).
|
|
|
|
get_latest_inner_proj_summ(FLU) ->
|
|
{ok, Proj} = ?FLU_PC:read_latest_projection(FLU, private),
|
|
#projection_v1{epoch_number=E, epoch_csum= <<CSum4:4/binary, _/binary>>,
|
|
upi=UPI, repairing=Repairing,
|
|
witnesses=Witnesses, down=Down} = Proj,
|
|
Inner_p = false,
|
|
EpochID = {E, CSum4},
|
|
{EpochID, UPI, Repairing, Down, Witnesses, Inner_p}.
|
|
|
|
uniq_reverse(L) ->
|
|
uniq_reverse(L, []).
|
|
|
|
uniq_reverse([], Acc) ->
|
|
Acc;
|
|
uniq_reverse([H|T], []) ->
|
|
uniq_reverse(T, [H]);
|
|
uniq_reverse([Same|T], [Same|_]=Acc) ->
|
|
uniq_reverse(T, Acc);
|
|
uniq_reverse([H|T], Acc) ->
|
|
uniq_reverse(T, [H|Acc]).
|
|
|
|
random_sort(L) ->
|
|
random:seed(now()),
|
|
L1 = [{random:uniform(), X} || X <- L],
|
|
[X || {_, X} <- lists:sort(L1)].
|
|
|
|
foo(NumFLUs, MgrOpts0) ->
|
|
timer:sleep(100),
|
|
%% Faster test startup, commented: io:format(user, short_doc(), []),
|
|
%% Faster test startup, commented: timer:sleep(3000),
|
|
|
|
TcpPort = 62877,
|
|
ok = filelib:ensure_dir("/tmp/c/not-used"),
|
|
FluInfo = [
|
|
{a,TcpPort+0,"/tmp/c/data.a"}, {b,TcpPort+1,"/tmp/c/data.b"},
|
|
{c,TcpPort+2,"/tmp/c/data.c"}, {d,TcpPort+3,"/tmp/c/data.d"},
|
|
{e,TcpPort+4,"/tmp/c/data.e"}, {f,TcpPort+5,"/tmp/c/data.f"},
|
|
{g,TcpPort+6,"/tmp/c/data.g"}, {h,TcpPort+7,"/tmp/c/data.h"},
|
|
{i,TcpPort+8,"/tmp/c/data.i"}, {j,TcpPort+9,"/tmp/c/data.j"},
|
|
{k,TcpPort+10,"/tmp/c/data.k"}, {l,TcpPort+11,"/tmp/c/data.l"},
|
|
{m,TcpPort+12,"/tmp/c/data.m"}, {n,TcpPort+13,"/tmp/c/data.n"},
|
|
{o,TcpPort+14,"/tmp/c/data.o"}, {p,TcpPort+15,"/tmp/c/data.p"},
|
|
{q,TcpPort+16,"/tmp/c/data.q"}, {r,TcpPort+17,"/tmp/c/data.r"}
|
|
],
|
|
FLU_biglist = [X || {X,_,_} <- FluInfo],
|
|
All_list = lists:sublist(FLU_biglist, NumFLUs),
|
|
io:format(user, "\nSET # of FLUs = ~w members ~w).\n",
|
|
[NumFLUs, All_list]),
|
|
machi_partition_simulator:start_link({111,222,33}, 0, 100),
|
|
_ = machi_partition_simulator:get(All_list),
|
|
|
|
Ps = [#p_srvr{name=Name,address="localhost",port=Port} ||
|
|
{Name,Port,_Dir} <- lists:sublist(FluInfo, NumFLUs)],
|
|
PsDirs = lists:zip(Ps,
|
|
[Dir || {_,_,Dir} <- lists:sublist(FluInfo, NumFLUs)]),
|
|
FLU_pids = [machi_flu1_test:setup_test_flu(Name, Port, Dir) ||
|
|
{#p_srvr{name=Name,port=Port}, Dir} <- PsDirs],
|
|
Namez = [begin
|
|
{ok, PPid} = ?FLU_PC:start_link(P),
|
|
{Name, PPid}
|
|
end || {#p_srvr{name=Name}=P, _Dir} <- PsDirs],
|
|
MembersDict = machi_projection:make_members_dict(Ps),
|
|
MgrOpts = MgrOpts0 ++ ?DEFAULT_MGR_OPTS,
|
|
MgrNamez =
|
|
[begin
|
|
{ok, MPid} = ?MGR:start_link(P#p_srvr.name, MembersDict, MgrOpts),
|
|
{P#p_srvr.name, MPid}
|
|
end || P <- Ps],
|
|
|
|
try
|
|
[{_, Ma}|_] = MgrNamez,
|
|
{ok, P1} = ?MGR:test_calc_projection(Ma, false),
|
|
[ok = ?FLU_PC:write_projection(FLUPid, public, P1) ||
|
|
{_, FLUPid} <- Namez, FLUPid /= Ma],
|
|
|
|
machi_partition_simulator:reset_thresholds(10, 50),
|
|
_ = machi_partition_simulator:get(All_list),
|
|
|
|
Parent = self(),
|
|
DoIt = fun(Iters, S_min, S_max) ->
|
|
%% io:format(user, "\nDoIt: top\n\n", []),
|
|
io:format(user, "DoIt, ", []),
|
|
Pids = [spawn(fun() ->
|
|
random:seed(now()),
|
|
[begin
|
|
erlang:yield(),
|
|
S_max_rand = random:uniform(
|
|
S_max + 1),
|
|
%% io:format(user, "{t}", []),
|
|
Elapsed =
|
|
?MGR:sleep_ranked_order(
|
|
S_min, S_max_rand,
|
|
M_name, All_list),
|
|
_ = ?MGR:trigger_react_to_env(MMM),
|
|
%% Be more unfair by not
|
|
%% sleeping here.
|
|
% timer:sleep(S_max - Elapsed),
|
|
Elapsed
|
|
end || _ <- lists:seq(1, Iters)],
|
|
Parent ! done
|
|
end) || {M_name, MMM} <- MgrNamez ],
|
|
[receive
|
|
done ->
|
|
ok
|
|
after 120*1000 ->
|
|
exit(icky_timeout)
|
|
end || _ <- Pids]
|
|
end,
|
|
|
|
%% machi_partition_simulator:reset_thresholds(10, 50),
|
|
%% io:format(user, "\nLet loose the dogs of war!\n", []),
|
|
machi_partition_simulator:always_these_partitions([]),
|
|
io:format(user, "\nPuppies for everyone!\n", []),
|
|
[DoIt(30, 0, 0) || _ <- lists:seq(1,5)],
|
|
|
|
DoIt
|
|
catch XXX:YYY ->
|
|
{XXX,YYY}
|
|
end.
|
|
|
|
uniq_c(L) ->
|
|
uniq_c(L, 0, unused).
|
|
|
|
uniq_c([], 0, _Last) ->
|
|
[];
|
|
uniq_c([], Count, Last) ->
|
|
[{Count, Last}];
|
|
uniq_c([H|T], 0, _Last) ->
|
|
uniq_c(T, 1, H);
|
|
uniq_c([H|T], Count, H) ->
|
|
uniq_c(T, Count+1, H);
|
|
uniq_c([H|T], Count, Last) ->
|
|
[{Count, Last}|uniq_c(T, 1, H)].
|
|
|
|
perhaps_adjust_pstore_sleep() ->
|
|
try
|
|
{ok, Bin} = file:read_file("/tmp/pstore_sleep_msec"),
|
|
{MSec,_} = string:to_integer(binary_to_list(Bin)),
|
|
ets:insert(?TEST_ETS_TABLE, {projection_store_sleep_time, MSec})
|
|
catch _:_ ->
|
|
ok
|
|
end.
|
|
|
|
|
|
%% MaxIters = NumFLUs * (NumFLUs + 1) * 6,
|
|
%% Stable = fun(S_Namez) ->
|
|
%% true = lists:foldl(
|
|
%% fun(_, true) ->
|
|
%% true;
|
|
%% (_, _) ->
|
|
%% %% Run a few iterations
|
|
%% [DoIt(10, 10, 50) || _ <- lists:seq(1, 6)],
|
|
%% %% If stable, return true to short circuit remaining
|
|
%% private_projections_are_stable(S_Namez, DoIt)
|
|
%% end, false, lists:seq(0, MaxIters))
|
|
%% end,
|
|
|
|
%% %% Part_b = [{a,b},{c,d}],
|
|
%% Part_b = [{c,d}],
|
|
%% %% Part_b = [{X,Y} || {X,_} <- Namez, {Y,_} <- Namez, X == b orelse Y == b],
|
|
%% %% Part_d = [{X,Y} || {X,_} <- Namez, {Y,_} <- Namez, X == d orelse Y == d],
|
|
|
|
%% %% machi_partition_simulator:always_these_partitions(Part_b),
|
|
%% %% io:format(user, "\nSET partitions = ~w at ~w\n", [Part_b, time()]),
|
|
%% %% true = Stable(Namez), io:format(user, "\nSweet, private projections are stable\n", []), io:format(user, "\t~P\n", [get(stable), 14]), (fun() -> ReportXX = machi_chain_manager1_test:unanimous_report(Namez), true = machi_chain_manager1_test:all_reports_are_disjoint(ReportXX), io:format(user, "Yay for ReportXX!\n", []) end)(),
|
|
|
|
%% %% Part_bd = Part_b ++ Part_d,
|
|
%% %% machi_partition_simulator:always_these_partitions(Part_bd),
|
|
%% %% io:format(user, "\nSET partitions = ~w at ~w\n", [Part_bd, time()]),
|
|
%% %% true = Stable(Namez), io:format(user, "\nSweet, private projections are stable\n", []), io:format(user, "\t~P\n", [get(stable), 14]), (fun() -> ReportXX = machi_chain_manager1_test:unanimous_report(Namez), true = machi_chain_manager1_test:all_reports_are_disjoint(ReportXX), io:format(user, "Yay for ReportXX!\n", []) end)(),
|
|
|
|
%% os:cmd("rm /tmp/signal"),
|
|
%% Part_b_partial_d = Part_b ++ [{e,f}],
|
|
%% %% Part_b_partial_d = [{a,b}, {b,c}, {c,d}, {d,e}],
|
|
%% %% Part_b_partial_d = [{a,b}, {b,d}, {d,e}],
|
|
%% machi_partition_simulator:always_these_partitions(Part_b_partial_d),
|
|
%% io:format(user, "\nSET partitions = ~w at ~w\n", [Part_b_partial_d, time()]),
|
|
%% %% Only_ab_namez = [T || T={Name, _} <- Namez, lists:member(Name, [a,b])],
|
|
%% true = Stable(Namez), io:format(user, "\nSweet, private projections are stable\n", []), io:format(user, "\t~P\n", [get(stable), 14]), (fun() -> ReportXX = machi_chain_manager1_test:unanimous_report(Namez), true = machi_chain_manager1_test:all_reports_are_disjoint(ReportXX), io:format(user, "Yay for ReportXX!\n", []) end)(),
|
|
|
|
%% machi_partition_simulator:always_these_partitions([{b,c}]),
|
|
%% io:format(user, "\nSET partitions = ~w at ~w\n", [[{b,c}], time()]),
|
|
%% %% Only_ab_namez = [T || T={Name, _} <- Namez, lists:member(Name, [a,b])],
|
|
%% [true = Stable(Namez) || _ <- [1,2,3] ], io:format(user, "\nSweet, private projections are stable\n", []), io:format(user, "\t~P\n", [get(stable), 14]), (fun() -> ReportXX = machi_chain_manager1_test:unanimous_report(Namez), true = machi_chain_manager1_test:all_reports_are_disjoint(ReportXX), io:format(user, "Yay for ReportXX!\n", []) end)(),
|
|
|
|
%% %% [begin
|
|
%% %% QQQ = lists:sublist(Part_b_partial_d, NNN),
|
|
%% %% machi_partition_simulator:always_these_partitions(QQQ),
|
|
%% %% io:format(user, "\nSET partitions = ~w at ~w\n", [QQQ, time()]),
|
|
%% %% %% Only_ab_namez = [T || T={Name, _} <- Namez, lists:member(Name, [a,b])],
|
|
%% %% true = Stable(Namez), io:format(user, "\nSweet, private projections are stable\n", []), io:format(user, "\t~P\n", [get(stable), 14]), (fun() -> ReportXX = machi_chain_manager1_test:unanimous_report(Namez), true = machi_chain_manager1_test:all_reports_are_disjoint(ReportXX), io:format(user, "Yay for ReportXX!\n", []) end)()
|
|
%% %% end || NNN <- lists:seq(1, length(Part_b_partial_d))],
|
|
|
|
%% io:format(user, "\nSET partitions = []\n", []),
|
|
%% io:format(user, "We should see convergence to 1 correct chain.\n", []),
|
|
%% machi_partition_simulator:no_partitions(),
|
|
%% [DoIt(50, 10, 50) || _ <- [1,2,3]],
|
|
%% true = private_projections_are_stable(Namez, DoIt),
|
|
%% io:format(user, "~s\n", [os:cmd("date")]),
|
|
|
|
%% %% We are stable now ... analyze it.
|
|
|
|
%% %% Create a report where at least one FLU has written a
|
|
%% %% private projection.
|
|
%% Report = machi_chain_manager1_test:unanimous_report(Namez),
|
|
%% %% ?D(Report),
|
|
|
|
%% %% Report is ordered by Epoch. For each private projection
|
|
%% %% written during any given epoch, confirm that all chain
|
|
%% %% members appear in only one unique chain, i.e., the sets of
|
|
%% %% unique chains are disjoint.
|
|
%% true = machi_chain_manager1_test:all_reports_are_disjoint(Report),
|
|
%% %% io:format(user, "\nLast Reports: ~p\n", [lists:nthtail(length(Report)-8,Report)]),
|
|
|
|
%% %% For each chain transition experienced by a particular FLU,
|
|
%% %% confirm that each state transition is OK.
|
|
%% PrivProjs = [{Name, begin
|
|
%% {ok, Ps9} = ?FLU_PC:get_all_projections(FLU,
|
|
%% private),
|
|
%% [P || P <- Ps9,
|
|
%% P#projection_v1.epoch_number /= 0]
|
|
%% end} || {Name, FLU} <- Namez],
|
|
%% try
|
|
%% [{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane_retrospective(Psx, FLU)} ||
|
|
%% {FLU, Psx} <- PrivProjs],
|
|
%% io:format(user, "\nAll sanity checks pass, hooray!\n", [])
|
|
%% catch _Err:_What ->
|
|
%% io:format(user, "Report ~p\n", [Report]),
|
|
%% io:format(user, "PrivProjs ~p\n", [PrivProjs]),
|
|
%% exit({line, ?LINE, _Err, _What})
|
|
%% end,
|
|
%% %% ?D(R_Projs),
|
|
|
|
%% ok
|
|
%% catch
|
|
%% XX:YY ->
|
|
%% io:format(user, "BUMMER ~p ~p @ ~p\n",
|
|
%% [XX, YY, erlang:get_stacktrace()]),
|
|
%% exit({bummer,XX,YY})
|
|
%% after
|
|
%% [ok = ?MGR:stop(MgrPid) || {_, MgrPid} <- MgrNamez],
|
|
%% [ok = ?FLU_PC:quit(PPid) || {_, PPid} <- Namez],
|
|
%% [ok = machi_flu1:stop(FLUPid) || FLUPid <- FLU_pids],
|
|
%% ok = machi_partition_simulator:stop()
|
|
%% end.
|
|
|
|
-endif. % !PULSE
|
|
-endif. % TEST
|
|
|