%% ------------------------------------------------------------------- %% %% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file %% except in compliance with the License. You may obtain %% a copy of the License at %% %% http://www.apache.org/licenses/LICENSE-2.0 %% %% Unless required by applicable law or agreed to in writing, %% software distributed under the License is distributed on an %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY %% KIND, either express or implied. See the License for the %% specific language governing permissions and limitations %% under the License. %% %% ------------------------------------------------------------------- -module(machi_fitness). -behaviour(gen_server). -include("machi.hrl"). -include("machi_projection.hrl"). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -endif. % TEST -define(LWWREG, riak_dt_lwwreg). -define(MAP, riak_dt_map). -define(DELAY_TIME_MS, 300). % TODO make configurable! %% API -export([start_link/1, get_unfit_list/1, update_local_down_list/3, add_admin_down/3, delete_admin_down/2, send_fitness_update_spam/3, send_spam_to_everyone/1]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -record(state, { my_flu_name :: atom() | binary(), reg_name :: atom(), local_down=[] :: list(), admin_down=[] :: list({term(),term()}), members_dict=orddict:new() :: orddict:orddict(), proxies_dict=orddict:new() :: orddict:orddict(), active_unfit=[] :: list(), pending_map=?MAP:new() :: ?MAP:riak_dt_map(), partition_simulator_p :: boolean() }). start_link(Args) -> gen_server:start_link(?MODULE, Args, []). get_unfit_list(PidSpec) -> gen_server:call(PidSpec, {get_unfit_list}, infinity). update_local_down_list(PidSpec, Down, MembersDict) -> gen_server:call(PidSpec, {update_local_down_list, Down, MembersDict}, infinity). add_admin_down(PidSpec, DownFLU, DownProps) -> gen_server:call(PidSpec, {add_admin_down, DownFLU, DownProps}, infinity). delete_admin_down(PidSpec, DownFLU) -> gen_server:call(PidSpec, {delete_admin_down, DownFLU}, infinity). send_fitness_update_spam(Pid, FromName, Dict) -> gen_server:call(Pid, {incoming_spam, FromName, Dict}, infinity). send_spam_to_everyone(Pid) -> gen_server:call(Pid, {send_spam_to_everyone}, infinity). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% init([{MyFluName}|Args]) -> RegName = machi_flu_psup:make_fitness_regname(MyFluName), register(RegName, self()), timer:send_interval(5000, debug_dump), UseSimulatorP = proplists:get_value(use_partition_simulator, Args, false), {ok, #state{my_flu_name=MyFluName, reg_name=RegName, partition_simulator_p=UseSimulatorP, local_down=[complete_bogus_here_to_trigger_initial_spam] }}. handle_call({get_unfit_list}, _From, #state{active_unfit=ActiveUnfit}=S) -> Reply = ActiveUnfit, {reply, Reply, S}; handle_call({update_local_down_list, Down, MembersDict}, _From, #state{my_flu_name=MyFluName, pending_map=OldMap, local_down=OldDown, members_dict=OldMembersDict, admin_down=AdminDown}=S) -> NewMap = store_in_map(OldMap, MyFluName, erlang:now(), Down, AdminDown, [props_yo]), S2 = if Down == OldDown, MembersDict == OldMembersDict -> %% Do nothing only if both are equal. If members_dict is %% changing, that's sufficient reason to spam. ok; true -> do_map_change(NewMap, [MyFluName], MembersDict, S) end, {reply, ok, S2#state{local_down=Down}}; handle_call({add_admin_down, DownFLU, DownProps}, _From, #state{local_down=OldDown, admin_down=AdminDown}=S) -> NewAdminDown = [{DownFLU,DownProps}|lists:keydelete(DownFLU, 1, AdminDown)], S3 = finish_admin_down(erlang:now(), OldDown, NewAdminDown, [props_yo], S), {reply, ok, S3}; handle_call({delete_admin_down, DownFLU}, _From, #state{local_down=OldDown, admin_down=AdminDown}=S) -> NewAdminDown = lists:keydelete(DownFLU, 1, AdminDown), S3 = finish_admin_down(erlang:now(), OldDown, NewAdminDown, [props_yo], S), {reply, ok, S3}; handle_call({incoming_spam, Author, Dict}, _From, S) -> {Res, S2} = do_incoming_spam(Author, Dict, S), {reply, Res, S2}; handle_call({send_spam_to_everyone}, _From, S) -> {Res, S2} = do_send_spam_to_everyone(S), {reply, Res, S2}; handle_call(_Request, _From, S) -> Reply = whhhhhhhhhhhhhhaaaaaaaaaaaaaaa, {reply, Reply, S}. handle_cast(_Msg, S) -> {noreply, S}. handle_info({adjust_down_list, FLU}, #state{active_unfit=ActiveUnfit}=S) -> NewUnfit = make_unfit_list(S), Added_to_new = NewUnfit -- ActiveUnfit, Dropped_from_new = ActiveUnfit -- NewUnfit, %% io:format(user, "adjust_down_list: ~w: adjust ~w: add ~p drop ~p\n", [S#state.my_flu_name, FLU, Added_to_new, Dropped_from_new]), %% We need to schedule a new round of adjustment messages. They might %% be redundant, or they might not. Here's a case where the current %% code needs the extra: %% %% SET partitions = [{a,c},{b,c},{c,b}] (11 of 26) at {23,37,44} %% We are stable spam/gossip at: %% [{a,problem_with,b},{b,problem_with,c}, %% {c,problem_with,a},{c,problem_with,b}] %% So everyone agrees unfit=[c]. %% %% SET partitions = [{c,a}] (12 of 26) at {23,37,48} %% We are stable spam/gossip at: %% [{a,problem_with,c},{c,problem_with,a}] %% So everyone *ought* to agree that unfit=[a]. %% %% In this case, when the partition list changes to [{c,a}], %% then we will discover via spam gossip that reports by B & C will %% change. However, our calc_unfit() via %% make_unfit_list() algorithm will decide that *a* is the bad guy %% and needs to go into our active_unfit list! And the only way %% to get added is via an {adjust_down_list,...} message. The %% usual place for generating them isn't wise enough because it %% doesn't call make_unfit_list(). %% %% The cost is that there will (at least) a 2x delay to the %% ?DELAY_TIME_MS waiting period to detect all partitions. %% %% Aside: for all I know right now, there may be a corner case %% hiding where we need this extra round of messages to *remove* a %% FLU from the active_unfit list? schedule_adjust_messages(lists:usort(Added_to_new ++ Dropped_from_new)), case {lists:member(FLU,Added_to_new), lists:member(FLU,Dropped_from_new)} of {true, true} -> error({bad, ?MODULE, ?LINE, FLU, ActiveUnfit, NewUnfit}); {true, false} -> {noreply, S#state{active_unfit=lists:usort(ActiveUnfit ++ [FLU])}}; {false, true} -> {noreply, S#state{active_unfit=ActiveUnfit -- [FLU]}}; {false, false} -> {noreply, S} end; handle_info(debug_dump, #state{my_flu_name=_MyFluName,active_unfit=_ActiveUnfit, pending_map=_Map}=S) -> %% io:format(user, "DUMP: ~w/~w: ~p ~W\n", [_MyFluName, self(), _ActiveUnfit, map_value(_Map), 13]), %% io:format(user, "DUMP ~w: ~w, ", [MyFluName, ActiveUnfit]), {noreply, S}; handle_info(_Info, S) -> {noreply, S}. terminate(_Reason, _S) -> ok. code_change(_OldVsn, S, _Extra) -> {ok, S}. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% make_unfit_list(#state{members_dict=MembersDict}=S) -> Now = erlang:now(), F = fun({Server, {UpdateTime, DownList, AdminDown, _Props}}, {ProblemAcc, AdminAcc}) -> case timer:now_diff(Now, UpdateTime) div (1000*1000) of N when N > 900 -> % TODO make configurable {ProblemAcc, AdminAcc}; _ -> Probs = [{Server,problem_with,D} || D <- DownList], {[Probs|ProblemAcc], AdminDown++AdminAcc} end end, {Problems0, AdminDown} = map_fold(F, {[], []}, S#state.pending_map), Problems = lists:flatten(Problems0), All_list = [K || {K,_V} <- orddict:to_list(MembersDict)], Unfit = calc_unfit(All_list, Problems), lists:usort(Unfit ++ AdminDown). store_in_map(Map, Name, Now, Down, AdminDown, Props) -> {AdminDownServers, AdminDownProps0} = lists:unzip(AdminDown), AdminDownProps = lists:append(AdminDownProps0), % flatten one level Val = {Now, Down, AdminDownServers, Props ++ AdminDownProps}, map_set(Name, Map, Name, Val). send_spam(NewMap, DontSendList, MembersDict, #state{my_flu_name=MyFluName}=S) -> Send = fun(FLU, #p_srvr{address=Host, port=TcpPort}) -> SpamProj = machi_projection:update_checksum( #projection_v1{epoch_number=?SPAM_PROJ_EPOCH, author_server=MyFluName, dbg=[NewMap], %% stuff only to make PB happy all_members=[], witnesses=[], creation_time={1,2,3}, mode=ap_mode, upi=[], repairing=[], down=[], dbg2=[], members_dict=[] }), %% Best effort, don't care about failure. spawn(fun() -> send_projection(FLU, Host, TcpPort, SpamProj, S) end) end, F = fun(FLU, P_srvr, Acc) -> case lists:member(FLU, DontSendList) of true -> Acc; false -> Send(FLU, P_srvr), [FLU|Acc] end end, _Sent = orddict:fold(F, [], MembersDict), ok. send_projection(FLU, _Host, _TcpPort, SpamProj, #state{my_flu_name=MyFluName, members_dict=MembersDict, partition_simulator_p=SimulatorP}=S) -> %% At the moment, we're using utterly-temporary-hack method of tunneling %% our messages through the write_projection API. Eventually the PB %% API should be expanded to accomodate this new fitness service. %% This is "best effort" only, use catch to ignore failures. ProxyPid = (catch proxy_pid(FLU, S)), DoIt = fun(_ArgIgnored) -> machi_proxy_flu1_client:write_projection(ProxyPid, public, SpamProj) end, ProxyPidPlaceholder = proxy_pid_unused, if SimulatorP -> AllMembers = [K || {K,_V} <- orddict:to_list(MembersDict)], {Partitions, _Islands} = machi_partition_simulator:get(AllMembers), machi_chain_manager1:init_remember_down_list(), Res = (catch machi_chain_manager1:perhaps_call(ProxyPidPlaceholder, MyFluName, Partitions, FLU, DoIt)), %% case machi_chain_manager1:get_remember_down_list() of %% [] -> %% ok; %% _ -> %% io:format(user, "fitness error ~w -> ~w\n", %% [MyFluName, FLU]) %% end, Res; true -> (catch DoIt(ProxyPidPlaceholder)) end. proxy_pid(Name, #state{proxies_dict=ProxiesDict}) -> orddict:fetch(Name, ProxiesDict). calc_unfit(All_list, HosedAnnotations) -> G = digraph:new(), [digraph:add_vertex(G, V) || V <- All_list], [digraph:add_edge(G, V1, V2) || {V1, problem_with, V2} <- HosedAnnotations], calc_unfit2(lists:sort(digraph:vertices(G)), G). calc_unfit2([], G) -> digraph:delete(G), []; calc_unfit2([H|T], G) -> case digraph:in_degree(G, H) of 0 -> calc_unfit2(T, G); 1 -> Neighbors = digraph:in_neighbours(G, H), case [V || V <- Neighbors, digraph:in_degree(G, V) == 1] of [AlsoOne|_] -> %% TODO: be smarter here about the choice of which is down. [H|calc_unfit2(T -- [AlsoOne], G)]; [] -> %% H is "on the end", e.g. 1-2-1, so it's OK. calc_unfit2(T, G) end; N when N > 1 -> [H|calc_unfit2(T, G)] end. do_incoming_spam(_Author, Map, #state{my_flu_name=MyFluName,pending_map=OldMap, members_dict=MembersDict}=S) -> OldMapV = map_value(OldMap), MapV = map_value(Map), if MapV == OldMapV -> {ok, S}; true -> %% io:format(user, "YY1 ~p\n", [OldMapV]), %% io:format(user, "YY2 ~p\n", [MapV]), NewMap = map_merge(OldMap, Map), %% NewMapV = map_value(NewMap), %% io:format(user, "YY3 ~p\n", [NewMapV]), %% Hrm, we may have changes that are interesting to the %% Author of this update, so perhaps we shouldn't exclude %% Author from our update, right? S2 = do_map_change(NewMap, [MyFluName], MembersDict, S), {ok, S2} end. do_send_spam_to_everyone(#state{my_flu_name=MyFluName, pending_map=Map,members_dict=MembersDict}=S) -> _ = send_spam(Map, [MyFluName], MembersDict, S), {ok, S}. do_map_change(NewMap, DontSendList, MembersDict, #state{my_flu_name=_MyFluName, pending_map=OldMap}=S) -> send_spam(NewMap, DontSendList, MembersDict, S), ChangedServers = find_changed_servers(OldMap, NewMap, _MyFluName), schedule_adjust_messages(ChangedServers), %% _OldMapV = map_value(OldMap), %% _MapV = map_value(NewMap), %% io:format(user, "TODO: ~w async tick trigger/scheduling... ~w for:\n" %% " ~p\n ~p\n",[_MyFluName,ChangedServers,_OldMapV,_MapV]), S2 = perhaps_adjust_members_proxies_dicts(MembersDict, S), S2#state{pending_map=NewMap}. perhaps_adjust_members_proxies_dicts(SameMembersDict, #state{members_dict=SameMembersDict}=S) -> S; perhaps_adjust_members_proxies_dicts(MembersDict, #state{proxies_dict=OldProxiesDict}=S) -> _ = machi_proxy_flu1_client:stop_proxies(OldProxiesDict), ProxiesDict = machi_proxy_flu1_client:start_proxies(MembersDict), S#state{members_dict=MembersDict, proxies_dict=ProxiesDict}. find_changed_servers(OldMap, NewMap, _MyFluName) -> AddBad = fun({_Who, {_Time, BadList, AdminDown, _Props}}, Acc) -> BadList ++ AdminDown ++ Acc end, OldBad = map_fold(AddBad, [], OldMap), NewBad = map_fold(AddBad, [], NewMap), lists:usort((OldBad -- NewBad) ++ (NewBad -- OldBad)). schedule_adjust_messages(FLU_list) -> [erlang:send_after(?DELAY_TIME_MS, self(), {adjust_down_list, FLU}) || FLU <- FLU_list]. finish_admin_down(Time, Down, NewAdminDown, Props, #state{my_flu_name=MyFluName, local_down=Down, pending_map=OldMap, members_dict=MembersDict}=S) -> NewMap = store_in_map(OldMap, MyFluName, Time, Down, NewAdminDown, Props), S2 = S#state{admin_down=NewAdminDown}, do_map_change(NewMap, [MyFluName], MembersDict, S2). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% map_set(Actor, Map, Key, ValTerm) -> Field = {Key, ?LWWREG}, Val = term_to_binary(ValTerm), {ok, Map2} = ?MAP:update({update, [{update, Field, {assign, Val}}]}, Actor, Map), Map2. -ifdef(TEST). map_get(Map, Key) -> Field = {Key, ?LWWREG}, case lists:keyfind(Field, 1, ?MAP:value(Map)) of false -> error; {Field, ValBin} -> {ok, binary_to_term(ValBin)} end. -endif. % TEST map_fold(Fun, Acc, Map) -> Vs = map_value(Map), lists:foldl(Fun, Acc, lists:sort(Vs)). map_value(Map) -> lists:sort([{K, binary_to_term(V)} || {{K, _Type}, V} <- ?MAP:value(Map)]). map_merge(Map1, Map2) -> ?MAP:merge(Map1, Map2). -ifdef(TEST). dt_understanding_test() -> F1 = {'X', riak_dt_lwwreg}, F2 = {'Y', riak_dt_lwwreg}, {ok, Map1} = ?MAP:update({update, [{update, F1, {assign, <<"A">>}}]}, a, ?MAP:new()), {ok, Map2} = ?MAP:update({update, [{update, F2, {assign, <<"B2">>}}]}, b, ?MAP:new()), %% io:format(user, "\n", []), %% io:format(user, "Merge comparison: ~p\n", [?MAP:merge(Map1, Map2) == ?MAP:merge(Map2, Map1)]), %% io:format(user, "M12 Val: ~p\n", [?MAP:value(?MAP:merge(Map1, Map2))]), %% io:format(user, "M21 Val: ~p\n", [?MAP:value(?MAP:merge(Map2, Map1))]), ?MAP:merge(Map1, Map2) == ?MAP:merge(Map2, Map1). smoke_test() -> Map1 = map_set(a, ?MAP:new(), k1, val1), Map2 = map_set(a, Map1, k2, val2), {ok, val1} = map_get(Map2, k1), {ok, val2} = map_get(Map2, k2), error = map_get(Map2, does_not_exist), Map3 = map_set(a, Map2, k3, val3), [{k3,1},{k2,1},{k1,1}] = map_fold(fun({K,_}, Acc) -> [{K,1}|Acc] end, [], Map3), ok. -endif. % TEST