greg/machi
1
0
Fork 0
Code Issues 11 Pull requests 1 Projects Releases Packages Wiki Activity Actions

WIP: fix annotation history on disk

Browse source
This commit is contained in:
Scott Lystig Fritchie 2015-08-28 18:37:11 +09:00
parent 8ca1ffdb13
commit 3dfe5c2677
3 changed files with 92 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
src/machi_chain_manager1.erl
View file

@ -1650,7 +1650,7 @@ react_to_env_A49(_P_latest, FinalProps, #ch_mgr{name=MyName,
members_dict=MembersDict} = P_current, members_dict=MembersDict} = P_current,
P_none = make_none_projection(MyName, All_list, Witness_list, P_none = make_none_projection(MyName, All_list, Witness_list,
MembersDict), MembersDict),
io:format(user, "Debug A49: ~w forced to none\n", [MyName]), io:format(user, "Debug A49: ~w forced to none\n\n ~P", [MyName, get(react), 120]),
react_to_env_A50(P_none, FinalProps, set_proj(S, P_none)). react_to_env_A50(P_none, FinalProps, set_proj(S, P_none)).
react_to_env_A50(P_latest, FinalProps, #ch_mgr{proj=P_current}=S) -> react_to_env_A50(P_latest, FinalProps, #ch_mgr{proj=P_current}=S) ->
@ -1659,6 +1659,7 @@ react_to_env_A50(P_latest, FinalProps, #ch_mgr{proj=P_current}=S) ->
{latest_epoch, P_latest#projection_v1.epoch_number}, {latest_epoch, P_latest#projection_v1.epoch_number},
{final_props, FinalProps}]}), {final_props, FinalProps}]}),
%% if S#ch_mgr.name == b; S#ch_mgr.name == c -> io:format(user, "A50: ~p: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end, %% if S#ch_mgr.name == b; S#ch_mgr.name == c -> io:format(user, "A50: ~p: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
%% io:format(user, "Debug A50: ~w P_current outer ~w ~w ~w\n", [S#ch_mgr.name, P_current#projection_v1.epoch_number,P_current#projection_v1.upi,P_current#projection_v1.repairing]),
{{no_change, FinalProps, P_current#projection_v1.epoch_number}, S}. {{no_change, FinalProps, P_current#projection_v1.epoch_number}, S}.
react_to_env_B10(Retries, P_newprop, P_latest, LatestUnanimousP, react_to_env_B10(Retries, P_newprop, P_latest, LatestUnanimousP,
@ -1860,7 +1861,7 @@ react_to_env_C100(P_newprop, #projection_v1{author_server=Author_latest,
Sane = projection_transition_is_sane(P_current, P_latest, MyName), Sane = projection_transition_is_sane(P_current, P_latest, MyName),
QQ_current = lists:flatten(io_lib:format("~w:~w,~w/~w:~w,~w", [P_current#projection_v1.epoch_number, P_current#projection_v1.upi, P_current#projection_v1.repairing, (inner_projection_or_self(P_current))#projection_v1.epoch_number, (inner_projection_or_self(P_current))#projection_v1.upi, (inner_projection_or_self(P_current))#projection_v1.repairing])), QQ_current = lists:flatten(io_lib:format("~w:~w,~w/~w:~w,~w", [P_current#projection_v1.epoch_number, P_current#projection_v1.upi, P_current#projection_v1.repairing, (inner_projection_or_self(P_current))#projection_v1.epoch_number, (inner_projection_or_self(P_current))#projection_v1.upi, (inner_projection_or_self(P_current))#projection_v1.repairing])),
QQ_latest = lists:flatten(io_lib:format("~w:~w,~w/~w:~w,~w", [P_latest#projection_v1.epoch_number, P_latest#projection_v1.upi, P_latest#projection_v1.repairing, (inner_projection_or_self(P_latest))#projection_v1.epoch_number, (inner_projection_or_self(P_latest))#projection_v1.upi, (inner_projection_or_self(P_latest))#projection_v1.repairing])), QQ_latest = lists:flatten(io_lib:format("~w:~w,~w/~w:~w,~w", [P_latest#projection_v1.epoch_number, P_latest#projection_v1.upi, P_latest#projection_v1.repairing, (inner_projection_or_self(P_latest))#projection_v1.epoch_number, (inner_projection_or_self(P_latest))#projection_v1.upi, (inner_projection_or_self(P_latest))#projection_v1.repairing])),
if Sane == true -> ok; true -> ?V("\n~w-insane-~w-auth=~w ~s -> ~s ~w\n", [?LINE, MyName, P_newprop#projection_v1.author_server, QQ_current, QQ_latest, Sane]) end, if Sane == true -> ok; true -> ?V("\n~w-insane-~w-auth=~w ~s -> ~s ~w\n ~p\n", [?LINE, MyName, P_newprop#projection_v1.author_server, QQ_current, QQ_latest, Sane, get(react)]) end,
Flap_latest = if is_record(Flap_latest0, flap_i) -> Flap_latest = if is_record(Flap_latest0, flap_i) ->
Flap_latest0; Flap_latest0;
true -> true ->
@ -1875,7 +1876,9 @@ react_to_env_C100(P_newprop, #projection_v1{author_server=Author_latest,
%% construction errors, checksum error, etc. %% construction errors, checksum error, etc.
case Sane of case Sane of
_ when P_current#projection_v1.epoch_number == 0 -> _ when P_current#projection_v1.epoch_number == 0 ->
%% Epoch == 0 is reserved for first-time, just booting conditions. %% Epoch == 0 is reserved for first-time, just booting conditions
%% or for when we got stuck in an insane projection transition
%% and were forced to the none projection to recover.
?REACT({c100, ?LINE, [first_write]}), ?REACT({c100, ?LINE, [first_write]}),
if Sane == true -> ok; true -> ?V("~w-insane-~w-~w:~w:~w,", [?LINE, MyName, P_newprop#projection_v1.epoch_number, P_newprop#projection_v1.upi, P_newprop#projection_v1.repairing]) end, %%% DELME!!! if Sane == true -> ok; true -> ?V("~w-insane-~w-~w:~w:~w,", [?LINE, MyName, P_newprop#projection_v1.epoch_number, P_newprop#projection_v1.upi, P_newprop#projection_v1.repairing]) end, %%% DELME!!!
react_to_env_C110(P_latest, S); react_to_env_C110(P_latest, S);
@ -2001,7 +2004,8 @@ react_to_env_C110(P_latest, #ch_mgr{name=MyName, proj=P_current,
inner_projection_or_self(P_latest)), inner_projection_or_self(P_latest)),
UnanimousTime = ProjUnanimous, UnanimousTime = ProjUnanimous,
A = make_annotation(EpochID, UnanimousTime), A = make_annotation(EpochID, UnanimousTime),
[A]; io:format(user, "\nCONFIRM debug C110 ~w annotates ~W outer ~w\n", [MyName, EpochID, 5, P_latest#projection_v1.epoch_number]),
[A, {annotated_by,c110}];
false -> false ->
[] []
end, end,
@ -2021,10 +2025,10 @@ react_to_env_C110(P_latest, #ch_mgr{name=MyName, proj=P_current,
?REACT({c120, [{write, ok}]}), ?REACT({c120, [{write, ok}]}),
%% We very intentionally do *not* pass P_latest2 forward: %% We very intentionally do *not* pass P_latest2 forward:
%% we must avoid bloating the dbg2 list! %% we must avoid bloating the dbg2 list!
P_latest_perhaps_annotated = P_latest2_perhaps_annotated =
machi_projection:update_dbg2(P_latest, Extra1), machi_projection:update_dbg2(P_latest, Extra1),
perhaps_verbose_c110(P_latest_perhaps_annotated, S), perhaps_verbose_c110(P_latest2_perhaps_annotated, S),
react_to_env_C120(P_latest_perhaps_annotated, [], S); react_to_env_C120(P_latest2_perhaps_annotated, [], S);
{{error, bad_arg}, _Goo} -> {{error, bad_arg}, _Goo} ->
?REACT({c120, [{write, bad_arg}]}), ?REACT({c120, [{write, bad_arg}]}),
@ -2177,6 +2181,14 @@ calculate_flaps(P_newprop, P_latest, _P_current, CurrentUp, _FlapLimit,
LastUpChange0 LastUpChange0
end, end,
LastUpChange_diff = timer:now_diff(now(), LastUpChange) / 1000000, LastUpChange_diff = timer:now_diff(now(), LastUpChange) / 1000000,
?REACT({calculate_flaps,?LINE,[{flap_start,FlapStart},
{flap_count,FlapCount},
{flap_last_up,FlapLastUp},
{flap_counts_last,FlapCountsLast},
{my_unique_prop_count,MyUniquePropCount},
{current_up,CurrentUp},
{last_up_change,LastUpChange},
{last_up_change_diff,LastUpChange_diff}]}),
%% TODO: Do we want to try to use BestP below to short-circuit %% TODO: Do we want to try to use BestP below to short-circuit
%% calculation if we notice that the best private epoch # from %% calculation if we notice that the best private epoch # from
@ -2246,12 +2258,12 @@ calculate_flaps(P_newprop, P_latest, _P_current, CurrentUp, _FlapLimit,
{N, _} when N >= MinQueueLen, {N, _} when N >= MinQueueLen,
P_latest_flap_start /= ?NOT_FLAPPING_START -> P_latest_flap_start /= ?NOT_FLAPPING_START ->
?REACT({calculate_flaps,?LINE, ?REACT({calculate_flaps,?LINE,
[{manifesto_clause,2}, [{manifesto_clause,{start,2}},
{latest_epoch, P_latest#projection_v1.epoch_number}, {latest_epoch, P_latest#projection_v1.epoch_number},
{latest_flap_count,P_latest_Flap#flap_i.flap_count}]}), {latest_flap_count,P_latest_Flap#flap_i.flap_count}]}),
true; true;
{N, [_]} when N >= MinQueueLen -> {N, [_]} when N >= MinQueueLen ->
?REACT({calculate_flaps,?LINE,[{manifesto_clause,1}]}), ?REACT({calculate_flaps,?LINE,[{manifesto_clause,{start,1}}]}),
true; true;
{_N, _} -> {_N, _} ->
?REACT({calculate_flaps,?LINE,[]}), ?REACT({calculate_flaps,?LINE,[]}),
@ -2272,7 +2284,7 @@ calculate_flaps(P_newprop, P_latest, _P_current, CurrentUp, _FlapLimit,
false; false;
AmFlappingNow_p andalso AmFlappingNow_p andalso
CurrentUp /= FlapLastUp -> CurrentUp /= FlapLastUp ->
?REACT({calculate_flaps,?LINE,[{manifesto_clause,1}]}), ?REACT({calculate_flaps,?LINE,[{manifesto_clause,{leave,1}}]}),
true; true;
AmFlappingNow_p -> AmFlappingNow_p ->
P_latest_LastStartTime = P_latest_LastStartTime =
@ -2292,7 +2304,7 @@ calculate_flaps(P_newprop, P_latest, _P_current, CurrentUp, _FlapLimit,
P_latest_LastStartTime /= ?NOT_FLAPPING_START -> P_latest_LastStartTime /= ?NOT_FLAPPING_START ->
?REACT({calculate_flaps,?LINE, ?REACT({calculate_flaps,?LINE,
[{manifesto_clause,2}, [{manifesto_clause,{leave,2}},
{p_latest, machi_projection:make_summary(P_latest)}, {p_latest, machi_projection:make_summary(P_latest)},
{curtime, Curtime}, {curtime, Curtime},
{flap_counts_last, FlapCountsLast}, {flap_counts_last, FlapCountsLast},
@ -2653,6 +2665,9 @@ poll_private_proj_is_upi_unanimous(#ch_mgr{consistency_mode=ap_mode} = S) ->
S; S;
poll_private_proj_is_upi_unanimous(#ch_mgr{consistency_mode=cp_mode, poll_private_proj_is_upi_unanimous(#ch_mgr{consistency_mode=cp_mode,
proj_unanimous={_,_,_}} = S) -> proj_unanimous={_,_,_}} = S) ->
%% #ch_mgr{name=MyName, proj=Proj} = S,
%% io:format(user, "\nCONFIRM debug ~w skip poll for inner ~w outer ~w\n",
%% [MyName, (inner_projection_or_self(Proj))#projection_v1.epoch_number, Proj#projection_v1.epoch_number]),
S; S;
poll_private_proj_is_upi_unanimous(#ch_mgr{consistency_mode=cp_mode, poll_private_proj_is_upi_unanimous(#ch_mgr{consistency_mode=cp_mode,
proj_unanimous=false, proj_unanimous=false,
@ -2682,17 +2697,35 @@ poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current,
UPI = Proj_ios#projection_v1.upi, UPI = Proj_ios#projection_v1.upi,
EpochID = machi_projection:make_epoch_id(Proj_ios), EpochID = machi_projection:make_epoch_id(Proj_ios),
{Rs, S2} = read_latest_projection_call_only2(private, UPI, S), {Rs, S2} = read_latest_projection_call_only2(private, UPI, S),
Rs2 = [if is_record(P, projection_v1) -> Rs2 = [if is_record(R, projection_v1) ->
machi_projection:make_epoch_id(inner_projection_or_self(P)); machi_projection:make_epoch_id(inner_projection_or_self(R));
true -> true ->
P R % probably {error, unwritten}
end || #projection_v1{}=P <- Rs], end || R <- Rs],
case lists:usort(Rs2) of case lists:usort(Rs2) of
[EID] when EID == EpochID -> [EID] when EID == EpochID ->
%% We have a debugging problem, alas. It would be really great
%% if we could preserve the dbg2 info that's in the current
%% projection that's on disk. However, the full dbg2 list
%% with 'react' trace data isn't in the #ch_mgr.proj copy of
%% the projection. So, go read it from the store.
%%
%% But of course there's another small problem. P_current could
%% be the result of make_zerf(), which helps us "fast forward" to
%% a newer CP mode projection. And so what we just read in the
%% 'Rs' at the top of this function may be for a new epoch that
%% we've never seen before and therefore doesn't exist in our
%% local private projection store. But if it came from
%% make_zerf(), by definition it must be annotated, so don't try
%% to proceed any further.
ProxyPid = proxy_pid(MyName, S),
OuterEpoch = P_current#projection_v1.epoch_number,
case ?FLU_PC:read_projection(ProxyPid, private, OuterEpoch) of
{ok, P_currentFull} ->
Now = os:timestamp(), Now = os:timestamp(),
Annotation = make_annotation(EpochID, Now), Annotation = make_annotation(EpochID, Now),
NewDbg2 = [Annotation|P_current#projection_v1.dbg2], NewDbg2 = [Annotation|P_currentFull#projection_v1.dbg2],
NewProj = P_current#projection_v1{dbg2=NewDbg2}, NewProj = P_currentFull#projection_v1{dbg2=NewDbg2},
ProjStore = case get_projection_store_regname(MgrOpts) of ProjStore = case get_projection_store_regname(MgrOpts) of
undefined -> undefined ->
machi_flu_psup:make_proj_supname(MyName); machi_flu_psup:make_proj_supname(MyName);
@ -2700,7 +2733,7 @@ poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current,
PStr PStr
end, end,
#projection_v1{epoch_number=_EpochRep, #projection_v1{epoch_number=_EpochRep,
epoch_csum= <<_CSumRep:4/binary, _/binary>>, epoch_csum= <<_CSumRep:4/binary,_/binary>>,
upi=_UPIRep, upi=_UPIRep,
repairing=_RepairingRep} = repairing=_RepairingRep} =
inner_projection_or_self(NewProj), inner_projection_or_self(NewProj),
@ -2710,6 +2743,9 @@ poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current,
{ok, NotifyPid} = machi_projection_store:get_wedge_notify_pid(ProjStore), {ok, NotifyPid} = machi_projection_store:get_wedge_notify_pid(ProjStore),
_ = machi_flu1:update_wedge_state(NotifyPid, false, EpochID), _ = machi_flu1:update_wedge_state(NotifyPid, false, EpochID),
S2#ch_mgr{proj_unanimous=Now}; S2#ch_mgr{proj_unanimous=Now};
_ ->
S2
end;
_Else -> _Else ->
%% io:format(user, "poll by ~w: want ~W got ~W\n", %% io:format(user, "poll by ~w: want ~W got ~W\n",
%% [MyName, EpochID, 6, _Else, 8]), %% [MyName, EpochID, 6, _Else, 8]),
@ -3308,14 +3344,18 @@ perhaps_verbose_c110(P_latest2, S) ->
{_,_,C} = os:timestamp(), {_,_,C} = os:timestamp(),
MSec = trunc(C / 1000), MSec = trunc(C / 1000),
{HH,MM,SS} = time(), {HH,MM,SS} = time(),
P_latest2x = P_latest2#projection_v1{dbg2=[]}, % limit verbose len. Dbg2X = lists:keydelete(react, 1,
P_latest2#projection_v1.dbg2) ++
[{is_annotated,is_annotated(P_latest2)}],
P_latest2x = P_latest2#projection_v1{dbg2=Dbg2X}, % limit verbose len.
case inner_projection_exists(P_latest2) of case inner_projection_exists(P_latest2) of
false -> false ->
Last2 = get(last_verbose), Last2 = get(last_verbose),
Summ2 = machi_projection:make_summary(P_latest2x), Summ2 = machi_projection:make_summary(P_latest2x),
case proplists:get_value(private_write_verbose, case proplists:get_value(private_write_verbose,
S#ch_mgr.opts) of S#ch_mgr.opts) of
true when Summ2 /= Last2 -> true ->
%% true when Summ2 /= Last2 ->
put(last_verbose, Summ2), put(last_verbose, Summ2),
?V("\n~2..0w:~2..0w:~2..0w.~3..0w ~p uses plain: ~w \n", ?V("\n~2..0w:~2..0w:~2..0w.~3..0w ~p uses plain: ~w \n",
[HH,MM,SS,MSec, S#ch_mgr.name, Summ2]); [HH,MM,SS,MSec, S#ch_mgr.name, Summ2]);
@ -3325,11 +3365,12 @@ perhaps_verbose_c110(P_latest2, S) ->
true -> true ->
Last2 = get(last_verbose), Last2 = get(last_verbose),
P_inner = inner_projection_or_self(P_latest2), P_inner = inner_projection_or_self(P_latest2),
P_innerx = P_inner#projection_v1{dbg2=[]}, % limit verbose len. P_innerx = P_inner#projection_v1{dbg2=Dbg2X}, % limit verbose len.
Summ2 = machi_projection:make_summary(P_innerx), Summ2 = machi_projection:make_summary(P_innerx),
case proplists:get_value(private_write_verbose, case proplists:get_value(private_write_verbose,
S#ch_mgr.opts) of S#ch_mgr.opts) of
true when Summ2 /= Last2 -> true ->
%% true when Summ2 /= Last2 ->
put(last_verbose, Summ2), put(last_verbose, Summ2),
?V("\n~2..0w:~2..0w:~2..0w.~3..0w ~p uses inner: ~w (outer ~w auth ~w flap ~w)\n", ?V("\n~2..0w:~2..0w:~2..0w.~3..0w ~p uses inner: ~w (outer ~w auth ~w flap ~w)\n",
[HH,MM,SS,MSec, S#ch_mgr.name, Summ2, P_latest2#projection_v1.epoch_number, P_latest2#projection_v1.author_server, P_latest2#projection_v1.flap]); [HH,MM,SS,MSec, S#ch_mgr.name, Summ2, P_latest2#projection_v1.epoch_number, P_latest2#projection_v1.author_server, P_latest2#projection_v1.flap]);

1
src/machi_projection_store.erl
View file

@ -309,6 +309,7 @@ do_proj_write3(ProjType, #projection_v1{epoch_number=Epoch,
io:format(user, "OUCH: on disk: ~w\n", [machi_projection:make_summary(binary_to_term(Bin))]), io:format(user, "OUCH: on disk: ~w\n", [machi_projection:make_summary(binary_to_term(Bin))]),
io:format(user, "OUCH: clobber: ~w\n", [machi_projection:make_summary(Proj)]), io:format(user, "OUCH: clobber: ~w\n", [machi_projection:make_summary(Proj)]),
io:format(user, "OUCH: clobber: ~p\n", [Proj#projection_v1.dbg2]), io:format(user, "OUCH: clobber: ~p\n", [Proj#projection_v1.dbg2]),
%% {{error, written}, S}
{{error, written, CurEpoch, Epoch, CurCSum, CSum}, S} {{error, written, CurEpoch, Epoch, CurCSum, CSum}, S}
end; end;
{error, enoent} -> {error, enoent} ->

11
test/machi_chain_manager1_converge_demo.erl
View file

@ -239,6 +239,14 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
%% machi_partition_simulator:reset_thresholds(10, 50), %% machi_partition_simulator:reset_thresholds(10, 50),
%% io:format(user, "\nLet loose the dogs of war!\n", []), %% io:format(user, "\nLet loose the dogs of war!\n", []),
%% [DoIt(20, 0, 0) || _ <- lists:seq(1,9)],
io:format(user, "\nVariations of puppies and dogs of war!\n", []),
[begin
machi_partition_simulator:reset_thresholds(90, 90),
DoIt(7, 0, 0),
machi_partition_simulator:always_these_partitions([]),
DoIt(7, 0, 0)
end || _ <- lists:seq(1, 3)],
machi_partition_simulator:always_these_partitions([]), machi_partition_simulator:always_these_partitions([]),
io:format(user, "\nPuppies for everyone!\n", []), io:format(user, "\nPuppies for everyone!\n", []),
[DoIt(20, 0, 0) || _ <- lists:seq(1,9)], [DoIt(20, 0, 0) || _ <- lists:seq(1,9)],
@ -371,7 +379,8 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
make_partition_list(All_list) -> make_partition_list(All_list) ->
[ [
[{b,c}] [{b,c}],
[{a,c},{b,c}]
%% [{b,c}], %% [{b,c}],
%% [], %% [],
%% [{c,d}], %% [{c,d}],