Keep good refactorings from commit a8390ee2

Also, add more misc details to the 'react' breadcrumb trail.  Also,
save get(react) results into dbg2 whenever we write a private projection,
very valuable for debugging.

Also: cleanup PULSE code, add regression commands as option and
controls with some new environment variables.  These regression
sequences were responsbile for several fruitful debugging sessions,
so we keep them for posterity and for their ability (with new seeds
and PULSE) to find new interleavings.
This commit is contained in:
Scott Lystig Fritchie 2015-07-10 15:04:50 +09:00
parent 297d29c79b
commit 2060b80830
6 changed files with 181 additions and 109 deletions

View file

@ -8,15 +8,17 @@ case PulseBuild of
true ->
PulseOpts =
[{pulse_no_side_effect,
[{erlang,display,1}
]},
[{erlang,display,1},
{os,getenv,1}
]},
{pulse_side_effect,
[ {does_not_exist_yet, some_func, '_'}
, {prim_file, '_', '_'}
, {file, '_', '_'}
, {filelib, '_', '_'}
, {os, '_', '_'} ]},
%% , {os, '_', '_'}
]},
{pulse_replace_module,
[ {gen_server, pulse_gen_server}

View file

@ -852,8 +852,22 @@ react_to_env_A20(Retries, #ch_mgr{name=MyName}=S) ->
UPI_Repairing_FLUs = lists:sort(P_latest#projection_v1.upi ++
P_latest#projection_v1.repairing),
All_UPI_Repairing_were_unanimous = UPI_Repairing_FLUs == UnanimousFLUs,
%% TODO: investigate if the condition below is more correct?
%% All_UPI_Repairing_were_unanimous = (UPI_Repairing_FLUs -- UnanimousFLUs) == [],
NotUnanimousFLUs = lists:sort(proplists:get_value(not_unanimous_flus,
ReadExtra, [xxx])),
NotUnanimousPs = lists:sort(proplists:get_value(not_unanimous_answers,
ReadExtra, [xxx])),
NotUnanimousSumms = [machi_projection:make_summary(
P#projection_v1{dbg2=[omitted]}) ||
P <- NotUnanimousPs,
is_record(P, projection_v1)],
BadAnswerFLUs = lists:sort(proplists:get_value(bad_answer_flus, ReadExtra)),
?REACT({a20,?LINE,[{unanimous_flus,UnanimousFLUs},
{upi_repairing,UPI_Repairing_FLUs},
{all_upi_repairing_were_unanimous,All_UPI_Repairing_were_unanimous},
{not_unanimous_flus, NotUnanimousFLUs},
{not_unanimous_answers, NotUnanimousSumms},
{bad_answer_flus, BadAnswerFLUs}
]}),
LatestUnanimousP =
if UnanimousTag == unanimous
andalso
@ -861,8 +875,7 @@ react_to_env_A20(Retries, #ch_mgr{name=MyName}=S) ->
?REACT({a20,?LINE}),
true;
UnanimousTag == unanimous ->
?REACT({a20,?LINE,[{upi_repairing,UPI_Repairing_FLUs},
{unanimous,UnanimousFLUs}]}),
?REACT({a20,?LINE}),
false;
UnanimousTag == not_unanimous ->
?REACT({a20,?LINE}),
@ -1458,7 +1471,7 @@ io:format(user, "YO: looping transition forced to none!\nNewProp: ~w\nLatest: ~w
react_to_env_C110(P_latest, #ch_mgr{name=MyName} = S) ->
?REACT(c110),
Extra_todo = [],
Extra_todo = [{react,get(react)}],
P_latest2 = machi_projection:update_dbg2(P_latest, Extra_todo),
MyNamePid = proxy_pid(MyName, S),
@ -1475,13 +1488,14 @@ react_to_env_C110(P_latest, #ch_mgr{name=MyName} = S) ->
{_,_,C} = os:timestamp(),
MSec = trunc(C / 1000),
{HH,MM,SS} = time(),
P_latest2x = P_latest2#projection_v1{dbg2=[]}, % limit verbose len.
case inner_projection_exists(P_latest2) of
false ->
case proplists:get_value(private_write_verbose, S#ch_mgr.opts) of
true ->
io:format(user, "\n~2..0w:~2..0w:~2..0w.~3..0w ~p uses plain: ~w\n",
[HH,MM,SS,MSec, S#ch_mgr.name,
machi_projection:make_summary(P_latest2)]);
machi_projection:make_summary(P_latest2x)]);
_ ->
ok
end;
@ -1489,9 +1503,10 @@ react_to_env_C110(P_latest, #ch_mgr{name=MyName} = S) ->
case proplists:get_value(private_write_verbose, S#ch_mgr.opts) of
true ->
P_inner = inner_projection_or_self(P_latest2),
P_innerx = P_inner#projection_v1{dbg2=[]}, % limit verbose len.
io:format(user, "\n~2..0w:~2..0w:~2..0w.~3..0w ~p uses inner: ~w\n",
[HH,MM,SS,MSec, S#ch_mgr.name,
machi_projection:make_summary(P_inner)]);
machi_projection:make_summary(P_innerx)]);
_ ->
ok
end

View file

@ -315,7 +315,8 @@ net_server_loop(Sock, S) ->
%% TODO: Weird that sometimes neither catch nor try/catch
%% can prevent OTP's SASL from logging an error here.
%% Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,.......
_ = (catch gen_tcp:send(Sock, Resp)), timer:sleep(1000),
%% TODO: is this what causes the intermittent PULSE deadlock errors?
%% _ = (catch gen_tcp:send(Sock, Resp)), timer:sleep(1000),
(catch gen_tcp:close(Sock)),
exit(normal)
end.

View file

@ -244,20 +244,14 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
{stable,true} = {stable,private_projections_are_stable(Namez, DoIt)},
io:format(user, "\nSweet, private projections are stable\n", []),
io:format(user, "Rolling sanity check ... ", []),
MaxFiles = 3*1000,
PrivProjs = [{Name, begin
{ok, Ps8} = ?FLU_PC:get_all_projections(FLU, private,
infinity),
Max = 3*1000,
Ps9 = if length(Ps8) < Max ->
{ok, Ps8} = ?FLU_PC:get_all_projections(
FLU, private, infinity),
Ps9 = if length(Ps8) < MaxFiles ->
Ps8;
true ->
NumToDel = length(Ps8) - Max,
io:format(user, "trunc a bit... ", []),
[begin
FilesToDel = lists:sublist(filelib:wildcard(Dir ++ "/projection/private/*"), NumToDel),
[_ = file:delete(File) || File <- FilesToDel]
end || Dir <- filelib:wildcard("/tmp/c/data*")],
lists:nthtail(Max, Ps8)
lists:nthtail(MaxFiles, Ps8)
end,
[P || P <- Ps9,
P#projection_v1.epoch_number /= 0]
@ -271,9 +265,20 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
end,
io:format(user, "Yay!\n", []),
ReportXX = machi_chain_manager1_test:unanimous_report(Namez),
io:format(user, "ReportXX ~P\n", [ReportXX, 30]),
true = machi_chain_manager1_test:all_reports_are_disjoint(ReportXX),
io:format(user, "Yay for ReportXX!\n", []),
[begin
Privs = filelib:wildcard(Dir ++ "/projection/private/*"),
FilesToDel1 = lists:sublist(Privs,
max(0, length(Privs)-MaxFiles)),
[_ = file:delete(File) || File <- FilesToDel1],
Pubs = filelib:wildcard(Dir ++ "/projection/public/*"),
FilesToDel2 = lists:sublist(Pubs,
max(0, length(Pubs)-MaxFiles)),
[_ = file:delete(File) || File <- FilesToDel2]
end || Dir <- filelib:wildcard("/tmp/c/data*")],
timer:sleep(1250),
ok
end || {Partition, Count} <- PartitionCounts
@ -348,10 +353,10 @@ make_partition_list(All_list) ->
X /= A, X /= C, A /= C],
%% Concat = _X_Ys1,
%% Concat = _X_Ys2,
Concat = _X_Ys1 ++ _X_Ys2,
%% Concat = _X_Ys3,
%% Concat = _X_Ys1 ++ _X_Ys2 ++ _X_Ys3,
random_sort(lists:usort([lists:sort(L) || L <- Concat])).
%% Concat = _X_Ys1 ++ _X_Ys2,
%% %% Concat = _X_Ys3,
%% %% Concat = _X_Ys1 ++ _X_Ys2 ++ _X_Ys3,
%% random_sort(lists:usort([lists:sort(L) || L <- Concat])).
%% [ [{a,b},{b,d},{c,b}],
%% [{a,b},{b,d},{c,b}, {a,b},{b,a},{a,c},{c,a},{a,d},{d,a}],
@ -392,6 +397,16 @@ make_partition_list(All_list) ->
%% [{a,b}, {c,b}, {c,d}],
%% [{a,b}, {b,c}, {d,c}] ].
[
%% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
%% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
%% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
%% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
[{a,b}], [], [{a,b}], [], [{a,b}]
%% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [],
%% [{b,a},{d,e}],
%% [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], [], [{a,b}], []
].
%% [ [{a,b}, {b,c}, {c,d}, {d,e}],
%% [{b,a}, {b,c}, {c,d}, {d,e}],
%% [{a,b}, {c,b}, {c,d}, {d,e}],
@ -407,11 +422,11 @@ make_partition_list(All_list) ->
%% [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}, {b,c}],
%% [{a,b},{b,a}, {a,c},{c,a}, {a,d},{d,a}, {c,d}] ].
% [ [{a,b}, {a,b},{b,a},{a,c},{c,a},{a,d},{d,a}],
% [{a,b}, {b,a},{a,b},{b,c},{c,b},{b,d},{d,b}],
% [{a,b}],
% [{a,b}, {c,a},{a,c},{c,b},{b,c},{c,d},{d,c}],
% [{a,b}, {d,a},{a,d},{d,b},{b,d},{d,c},{c,d}] ].
%% [ [{a,b}, {a,b},{b,a},{a,c},{c,a},{a,d},{d,a}],
%% [{a,b}, {b,a},{a,b},{b,c},{c,b},{b,d},{d,b}],
%% [{a,b}],
%% [{a,b}, {c,a},{a,c},{c,b},{b,c},{c,d},{d,c}],
%% [{a,b}, {d,a},{a,d},{d,b},{b,d},{d,c},{c,d}] ].
todo_why_does_this_crash_sometimes(FLUName, FLU, PPPepoch) ->
try

View file

@ -38,7 +38,8 @@
-compile({pulse_replace_module, [{application, pulse_application}]}).
%% The following functions contains side_effects but are run outside
%% PULSE, i.e. PULSE needs to leave them alone
-compile({pulse_skip,[{prop_pulse_test_,0}, {shutdown_hard,0}]}).
-compile({pulse_skip,[{prop_pulse_test_,0}, {prop_pulse_regression_test_,0},
{shutdown_hard,0}]}).
-compile({pulse_no_side_effect,[{file,'_','_'}, {erlang, now, 0}]}).
%% Used for output within EUnit...
@ -75,6 +76,44 @@ gen_old_threshold() ->
gen_no_partition_threshold() ->
noshrink(choose(1, 100)).
gen_commands(new) ->
non_empty(commands(?MODULE));
gen_commands(regression) ->
%% These regression tests include only few, very limited command
%% sequences that have been helpful in turning up bugs in the past.
%% For this style test, QuickCheck is basically just choosing random
%% seeds + PULSE execution to see if one of the oldies-but-goodies can
%% find another execution/interleaving that still shows a problem.
Cmd_a = [{set,{var,1},
{call,machi_chain_manager1_pulse,setup,[3,{846,1222,4424}]}},
{set,{var,2},
{call,machi_chain_manager1_pulse,do_ticks,[6,{var,1},13,48]}}],
Cmd_b = [{set,{var,1},
{call,machi_chain_manager1_pulse,setup,[4,{354,7401,1237}]}},
{set,{var,2},
{call,machi_chain_manager1_pulse,do_ticks,[10,{var,1},15,77]}},
{set,{var,3},
{call,machi_chain_manager1_pulse,do_ticks,[7,{var,1},92,39]}}],
Cmd_c = [{set,{var,1},
{call,machi_chain_manager1_pulse,setup,[2,{5202,467,3157}]}},
{set,{var,2},
{call,machi_chain_manager1_pulse,do_ticks,[8,{var,1},98,3]}},
{set,{var,3},
{call,machi_chain_manager1_pulse,do_ticks,[5,{var,1},56,49]}},
{set,{var,4},
{call,machi_chain_manager1_pulse,do_ticks,[10,{var,1},33,72]}},
{set,{var,5},
{call,machi_chain_manager1_pulse,do_ticks,[10,{var,1},88,20]}},
{set,{var,6},
{call,machi_chain_manager1_pulse,do_ticks,[8,{var,1},67,10]}},
{set,{var,7},
{call,machi_chain_manager1_pulse,do_ticks,[5,{var,1},86,25]}},
{set,{var,8},
{call,machi_chain_manager1_pulse,do_ticks,[6,{var,1},74,88]}},
{set,{var,9},
{call,machi_chain_manager1_pulse,do_ticks,[8,{var,1},78,39]}}],
noshrink(oneof([Cmd_a, Cmd_b, Cmd_c])).
command(#state{step=0}) ->
{call, ?MODULE, setup, [gen_num_pids(), gen_seed()]};
command(S) ->
@ -104,17 +143,18 @@ postcondition(_S, {call, _, _Func, _Args}, _Res) ->
true.
all_list_extra() ->
{PortBase, DirBase} = get_port_dir_base(),
[ %% Genenerators assume that this list is at least 2 items
{#p_srvr{name=a, address="localhost", port=7400,
props=[{chmgr, a_chmgr}]}, "/tmp/c/data.pulse.a"}
, {#p_srvr{name=b, address="localhost", port=7401,
props=[{chmgr, b_chmgr}]}, "/tmp/c/data.pulse.b"}
, {#p_srvr{name=c, address="localhost", port=7402,
props=[{chmgr, c_chmgr}]}, "/tmp/c//data.pulse.c"}
, {#p_srvr{name=d, address="localhost", port=7403,
props=[{chmgr, d_chmgr}]}, "/tmp/c/data.pulse.d"}
, {#p_srvr{name=e, address="localhost", port=7404,
props=[{chmgr, e_chmgr}]}, "/tmp/c/data.pulse.e"}
{#p_srvr{name=a, address="localhost", port=PortBase+0,
props=[{chmgr, a_chmgr}]}, DirBase ++ "/data.pulse.a"}
, {#p_srvr{name=b, address="localhost", port=PortBase+1,
props=[{chmgr, b_chmgr}]}, DirBase ++ "/data.pulse.b"}
, {#p_srvr{name=c, address="localhost", port=PortBase+2,
props=[{chmgr, c_chmgr}]}, DirBase ++ "//data.pulse.c"}
, {#p_srvr{name=d, address="localhost", port=PortBase+3,
props=[{chmgr, d_chmgr}]}, DirBase ++ "/data.pulse.d"}
, {#p_srvr{name=e, address="localhost", port=PortBase+4,
props=[{chmgr, e_chmgr}]}, DirBase ++ "/data.pulse.e"}
].
all_list() ->
@ -222,48 +262,6 @@ dump_state() ->
{_PSimPid, _SupPid, ProxiesDict, _AlE} = get(manager_pids_hack),
Report = ?MGRTEST:unanimous_report(ProxiesDict),
Namez = ProxiesDict,
%% ?QC_FMT("Report ~p\n", [Report]),
%% Diag1 = [begin
%% {ok, Ps} = ?FLU_PC:get_all_projections(Proxy, Type),
%% [io_lib:format("~p ~p ~p: ~w\n", [FLUName, Type, P#projection_v1.epoch_number, machi_projection:make_summary(P)]) || P <- Ps]
%% end || {FLUName, Proxy} <- orddict:to_list(ProxiesDict),
%% Type <- [public] ],
%% P_lists0 = [{FLUName, Type,
%% element(2,?FLU_PC:get_all_projections(Proxy, Type))} ||
%% {FLUName, Proxy} <- orddict:to_list(ProxiesDict),
%% Type <- [public,private]],
%% P_lists = [{FLUName, Type, P} || {FLUName, Type, Ps} <- P_lists0,
%% P <- Ps],
%% AllDict = lists:foldl(fun({FLU, Type, P}, D) ->
%% K = {FLU, Type, P#projection_v1.epoch_number},
%% dict:store(K, P, D)
%% end, dict:new(), lists:flatten(P_lists)),
%% DumbFinderBackward =
%% fun(FLUName) ->
%% fun(E, error_unwritten) ->
%% case dict:find({FLUName, private, E}, AllDict) of
%% {ok, T} -> T;
%% error -> error_unwritten
%% end;
%% (_E, Acc) ->
%% Acc
%% end
%% end,
%% UniquePrivateEs =
%% lists:usort(lists:flatten(
%% [element(2,?FLU_PC:list_all_projections(Proxy,private)) ||
%% {_FLUName, Proxy} <- orddict:to_list(ProxiesDict)])),
%% Diag2 = [[
%% io_lib:format("~p private: ~w\n",
%% [FLUName,
%% machi_projection:make_summary(
%% lists:foldl(DumbFinderBackward(FLUName),
%% error_unwritten,
%% lists:seq(Epoch, 0, -1)))])
%% || {FLUName, _FLU} <- Namez]
%% || Epoch <- UniquePrivateEs],
PrivProjs = [{Name, begin
{ok, Ps} = ?FLU_PC:get_all_projections(Proxy,
private),
@ -282,7 +280,10 @@ dump_state() ->
end.
prop_pulse() ->
?FORALL({Cmds0, Seed}, {non_empty(commands(?MODULE)), pulse:seed()},
prop_pulse(new).
prop_pulse(Style) when Style == new; Style == regression ->
?FORALL({Cmds0, Seed}, {gen_commands(Style), pulse:seed()},
?IMPLIES(1 < length(Cmds0) andalso length(Cmds0) < 11,
begin
ok = shutdown_hard(),
@ -308,13 +309,6 @@ prop_pulse() ->
{_H, _S, _R} = run_commands(?MODULE, Cmds)
end, [{seed, Seed},
{strategy, unfair}]),
%% ?QC_FMT("S2 ~p\n", [S2]),
case S2#state.dump_state of
undefined ->
?QC_FMT("BUMMER Cmds = ~p\n", [Cmds]);
_ ->
ok
end,
{Report, PrivProjs, Diag} = S2#state.dump_state,
%% Report is ordered by Epoch. For each private projection
@ -349,7 +343,7 @@ prop_pulse() ->
ok = shutdown_hard(),
?WHENFAIL(
begin
?QC_FMT("PrivProjs = ~p\n", [PrivProjs]),
?QC_FMT("PrivProjs = ~P\n", [PrivProjs, 100]),
?QC_FMT("Report = ~p\n", [Report]),
?QC_FMT("Cmds = ~p\n", [Cmds]),
?QC_FMT("Res = ~p\n", [Res]),
@ -366,7 +360,41 @@ prop_pulse() ->
]))
end)).
prop_pulse_test_() ->
-define(FIXTURE(TIMEOUT, EXTRATO, FUN), {timeout, (Timeout+ExtraTO+600), FUN}).
prop_pulse_new_test_() ->
{Timeout, ExtraTO} = get_timeouts(),
F = fun() ->
?assert(eqc:quickcheck(eqc:testing_time(Timeout,
?QC_OUT(prop_pulse(new)))))
end,
case os:getenv("PULSE_SKIP_NEW") of
false ->
?FIXTURE(Timeout, ExtraTO, F);
_ ->
{timeout, 5,
fun() -> timer:sleep(200),
io:format(user, " (skip new style) ", []) end}
end.
%% See gen_commands() for more detail on the regression tests.
prop_pulse_regression_test_() ->
{Timeout, ExtraTO} = get_timeouts(),
F = fun() ->
?assert(eqc:quickcheck(eqc:testing_time(Timeout,
?QC_OUT(prop_pulse(regression)))))
end,
case os:getenv("PULSE_SKIP_REGRESSION") of
false ->
?FIXTURE(Timeout, ExtraTO, F);
_ ->
{timeout, 5,
fun() -> timer:sleep(200),
io:format(user, " (skip regression style) ", []) end}
end.
get_timeouts() ->
Timeout = case os:getenv("PULSE_TIME") of
false -> 60;
Val -> list_to_integer(Val)
@ -375,20 +403,14 @@ prop_pulse_test_() ->
false -> 0;
Val2 -> list_to_integer(Val2)
end,
{timeout, (Timeout+ExtraTO+600), % 600 = a bit more fudge time
fun() ->
?assert(eqc:quickcheck(eqc:testing_time(Timeout,
?QC_OUT(prop_pulse()))))
end}.
{Timeout, ExtraTO}.
shutdown_hard() ->
?QC_FMT("shutdown(", []),
(catch unlink(whereis(machi_partition_simulator))),
[begin
Pid = whereis(X),
spawn(fun() -> (catch X:stop()) end),
timer:sleep(50),
(catch unlink(Pid)),
timer:sleep(10),
(catch exit(Pid, shutdown)),
timer:sleep(1),
@ -438,4 +460,20 @@ private_projections_are_stable_check(ProxiesDict, All_listE) ->
get_chmgr(#p_srvr{props=Ps}) ->
proplists:get_value(chmgr, Ps).
%% {PortBase, DirBase} = get_port_dir_base(),
get_port_dir_base() ->
I = case os:getenv("PULSE_BASE_PORT") of
false ->
0;
II ->
list_to_integer(II)
end,
D = case os:getenv("PULSE_BASE_DIR") of
false ->
"/tmp/c/";
DD ->
DD
end,
{7400 + (I * 100), D ++ "/" ++ integer_to_list(I)}.
-endif. % PULSE

View file

@ -401,13 +401,13 @@ unanimous_report_test() ->
E5 = 5,
UPI5 = [a,b],
Rep5 = [],
UPIRep5 = [{UPI5, Rep5}],
Report5 = [UPI5],
P5 = machi_projection:new(E5, a, MembersDict, [], UPI5, Rep5, []),
{ok_disjoint, UPIRep5} =
{ok_disjoint, Report5} =
unanimous_report2([{a, P5}, {b, P5}]),
{ok_disjoint, UPIRep5} =
{ok_disjoint, Report5} =
unanimous_report2([{a, not_in_this_epoch}, {b, P5}]),
{ok_disjoint, UPIRep5} =
{ok_disjoint, Report5} =
unanimous_report2([{a, P5}, {b, not_in_this_epoch}]),
UPI5_b = [a],
@ -427,15 +427,16 @@ unanimous_report_test() ->
UPI5_d = [c],
Rep5_d = [a],
Report5d = [UPI5, UPI5_d],
P5_d = machi_projection:new(E5, b, MembersDict3, [b], UPI5_d, Rep5_d, []),
{bummer_NOT_DISJOINT, _} = unanimous_report2([{a, P5}, {b, P5_d}]),
{ok_disjoint, Report5d} = unanimous_report2([{a, P5}, {b, P5_d}]),
UPI5_e = [b],
Rep5_e = [c],
UPIRep5be = [{UPI5_b, Rep5_b}, {UPI5_e, Rep5_e}],
Report5be = [UPI5_b, UPI5_e],
P5_e = machi_projection:new(E5, b, MembersDict3, [a], UPI5_e, Rep5_e, []),
{bummer_NOT_DISJOINT, _} = unanimous_report2([{a, P5}, {b, P5_e}]),
{ok_disjoint, UPIRep5be} = unanimous_report2([{a, P5_b}, {b, P5_e}]),
{ok_disjoint, Report5be} = unanimous_report2([{a, P5_b}, {b, P5_e}]),
ok.