greg/machi
1
0
Fork 0
Code Issues 11 Pull requests 1 Projects Releases Packages Wiki Activity Actions

Bootstrap chain @ app init: done, with an example.

Browse source 
For example:

% make clean
% make stage

And then configure 3 FLUs:

    % echo '{p_srvr, a, machi_flu1_client, "localhost", 39000, []}.' > rel/machi/etc/flu-config/a
    % echo '{p_srvr, b, machi_flu1_client, "localhost", 39001, []}.' > rel/machi/etc/flu-config/b
    % echo '{p_srvr, c, machi_flu1_client, "localhost", 39002, []}.' > rel/machi/etc/flu-config/c

And then configure a chain to use 2 of those 3 FLUs:

    % echo '{chain_def_v1,c1,ap_mode,[{p_srvr,a,machi_flu1_client,"localhost",39000,[]},{p_srvr,b,machi_flu1_client,"localhost",39001,[]}],[],[]}.' > rel/machi/etc/chain-config/c1

... then start Machi e.g.

    % ./rel/machi/bin/machi console

... you should see the following console messages scroll by (including a :

    =PROGRESS REPORT==== 8-Dec-2015::22:01:44 ===
              supervisor: {local,machi_flu_sup}
                 started: [{pid,<0.145.0>},
                           {name,a},
                           {mfargs,
                               {machi_flu_psup,start_link,
                                   [a,39000,"./data/flu/a",[]]}},
                           {restart_type,permanent},
                           {shutdown,5000},
                           {child_type,supervisor}]

    [... and also for the other two FLUs, including a bunch of progress
         reports for processes that started underneath that sub-supervisor.]

    22:01:44.446 [info] Running FLUs: [a,b,c]
    22:01:44.446 [info] Running FLUs at epoch 0: [a,b,c]
    22:01:44.532 [warning] The following FLUs are defined but are not also members of a defined chain: [c]
This commit is contained in:
Scott Lystig Fritchie 2015-12-08 21:16:54 +09:00
parent 37ac09a680
commit 8285899dba
7 changed files with 264 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
include/machi_projection.hrl
View file

@ -80,8 +80,9 @@
-record(chain_def_v1, {
name :: atom(),
mode :: pv1_consistency_mode(),
upi :: [p_srvr()],
witnesses :: [p_srvr()]
full :: [p_srvr()],
witnesses :: [p_srvr()],
props = [] :: list() % proplist for other related info
}).
-endif. % !MACHI_PROJECTION_HRL

16
rel/files/app.config
View file

@ -1,7 +1,7 @@
[
{machi, [
%% Data directory for all FLUs.
{flu_data_dir, "{{platform_data_dir}}"},
{flu_data_dir, "{{platform_data_dir}}/flu"},
%% FLU config directory
{flu_config_dir, "{{platform_etc_dir}}/flu-config"},
@ -10,15 +10,7 @@
{chain_config_dir, "{{platform_etc_dir}}/chain-config"},
%% FLUs to start at app start.
{initial_flus, [
%% Remember, this is a list, so separate all tuples
%% with a comma.
%%
%% {Name::atom(), Port::pos_integer(), proplist()}
%%
%% For example: {my_name_is_a, 12500, []}
]},
%% This task has moved to machi_flu_sup and machi_lifecycle_mgr.
%% Number of metadata manager processes to run per FLU.
%% Default = 10
@ -27,5 +19,9 @@
%% Do not delete, do not put Machi config items after this line.
{final_comma_stopper, do_not_delete}
]
},
{lager, [
{error_logger_hwm, 5000} % lager's default of 50/sec is too low
]
}
].

11
src/machi_chain_manager1.erl
View file

@ -2598,8 +2598,8 @@ do_repair(#ch_mgr{name=MyName,
T1 = os:timestamp(),
RepairId = proplists:get_value(repair_id, Opts, id1),
error_logger:info_msg(
"Repair start: tail ~p of ~p -> ~p, ~p ID ~w\n",
[MyName, UPI0, Repairing, RepairMode, RepairId]),
"Repair ~w start: tail ~p of ~p -> ~p, ~p\n",
[RepairId, MyName, UPI0, Repairing, RepairMode]),
UPI = UPI0 -- Witness_list,
Res = machi_chain_repair:repair(RepairMode, MyName, Repairing, UPI,
@ -2612,10 +2612,9 @@ do_repair(#ch_mgr{name=MyName,
end,
Stats = [{K, ets:lookup_element(ETS, K, 2)} || K <- ETS_T_Keys],
error_logger:info_msg(
"Repair ~s: tail ~p of ~p finished ~p repair ID ~w: "
"~p\nStats ~p\n",
[Summary, MyName, UPI0, RepairMode, RepairId,
Res, Stats]),
"Repair ~w ~s: tail ~p of ~p finished ~p: "
"~p Stats: ~p\n",
[RepairId, Summary, MyName, UPI0, RepairMode, Res, Stats]),
ets:delete(ETS),
exit({repair_final_status, Res});
_ ->

24
src/machi_chain_repair.erl
View file

@ -103,7 +103,8 @@ repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
Add = fun(Name, Pid) -> put(proxies_dict, orddict:store(Name, Pid, get(proxies_dict))) end,
OurFLUs = lists:usort([Src] ++ Repairing ++ UPI), % AP assumption!
RepairMode = proplists:get_value(repair_mode, Opts, repair),
Verb = proplists:get_value(verbose, Opts, true),
Verb = proplists:get_value(verbose, Opts, false),
RepairId = proplists:get_value(repair_id, Opts, id1),
Res = try
_ = [begin
{ok, Proxy} = machi_proxy_flu1_client:start_link(P),
@ -116,31 +117,38 @@ repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
get_file_lists(Proxy, FLU, Dict)
end, D, ProxiesDict),
MissingFileSummary = make_missing_file_summary(D2, OurFLUs),
?VERB("MissingFileSummary ~p\n", [MissingFileSummary]),
%% ?VERB("~w MissingFileSummary ~p\n",[RepairId,MissingFileSummary]),
lager:info("Repair ~w MissingFileSummary ~p\n",
[RepairId, MissingFileSummary]),
[ets:insert(ETS, {{directive_bytes, FLU}, 0}) || FLU <- OurFLUs],
%% Repair files from perspective of Src, i.e. tail(UPI).
SrcProxy = orddict:fetch(Src, ProxiesDict),
{ok, EpochID} = machi_proxy_flu1_client:get_epoch_id(
SrcProxy, ?SHORT_TIMEOUT),
?VERB("Make repair directives: "),
%% ?VERB("Make repair directives: "),
Ds =
[{File, make_repair_directives(
ConsistencyMode, RepairMode, File, Size, EpochID,
Verb,
Src, OurFLUs, ProxiesDict, ETS)} ||
{File, {Size, _MissingList}} <- MissingFileSummary],
?VERB(" done\n"),
%% ?VERB(" done\n"),
lager:info("Repair ~w repair directives finished\n", [RepairId]),
[begin
[{_, Bytes}] = ets:lookup(ETS, {directive_bytes, FLU}),
?VERB("Out-of-sync data for FLU ~p: ~s MBytes\n",
[FLU, mbytes(Bytes)])
%% ?VERB("Out-of-sync data for FLU ~p: ~s MBytes\n",
%% [FLU, mbytes(Bytes)]),
lager:info("Repair ~w "
"Out-of-sync data for FLU ~p: ~s MBytes\n",
[RepairId, FLU, mbytes(Bytes)])
end || FLU <- OurFLUs],
?VERB("Execute repair directives: "),
%% ?VERB("Execute repair directives: "),
ok = execute_repair_directives(ConsistencyMode, Ds, Src, EpochID,
Verb, OurFLUs, ProxiesDict, ETS),
?VERB(" done\n"),
%% ?VERB(" done\n"),
lager:info("Repair ~w repair directives finished\n", [RepairId]),
ok
catch
What:Why ->

11
src/machi_flu_sup.erl
View file

@ -21,6 +21,9 @@
%% @doc Supervisor for Machi FLU servers and their related support
%% servers.
%%
%% Responsibility for managing FLU & chain lifecycle after the initial
%% application startup is delegated to {@link machi_lifecycle_mgr}.
%%
%% See {@link machi_flu_psup} for an illustration of the entire Machi
%% application process structure.
@ -87,8 +90,12 @@ get_initial_flus() ->
load_rc_d_files_from_dir(Dir) ->
Files = filelib:wildcard(Dir ++ "/*"),
lists:append([case file:consult(File) of
{ok, X} -> X;
_ -> []
{ok, X} ->
X;
_ ->
lager:warning("Error parsing file '~s', ignoring",
[File]),
[]
end || File <- Files]).
sanitize_p_srvr_records(Ps) ->

250
src/machi_lifecycle_mgr.erl
View file

@ -18,7 +18,169 @@
%%
%% -------------------------------------------------------------------
-module(machi_chain_bootstrap).
%% @doc Lifecycle manager for Machi FLUs and chains.
%%
%% Over the lifetime of a Machi cluster, both the number and types of
%% FLUs and chains may change. The lifecycle manager is responsible
%% for implementing the lifecycle changes as expressed by "policy".
%% In our case, "policy" is created by an external administrative
%% entity that creates and deletes configuration files that define
%% FLUs and chains relative to this local machine.
%%
%% The "master configuration" for deciding which FLUs should be
%% running on this machine was inspired by BSD UNIX's `init(8)' and the
%% "rc.d" scheme. FLU definitions are found in a single directory,
%% with one file per FLU. Chains are defined similarly, with one
%% definition file per chain.
%%
%% If a definition file for a FLU (or chain) exists, then that
%% FLU/chain ought to be configured into being and running. If a
%% definition file for a FLU/chain is removed, then that FLU/chain
%% should be stopped gracefully. However, deleting of a file destroys
%% information that is stored inside of that file. Therefore, we will
%% <b>not allow arbitrary unlinking of lifecycle config files</b>. If
%% the administrator deletes these config files using `unlink(8)'
%% directly, then "the warranty has been broken".
%%
%% We will rely on using an administrative command to inform the
%% running system to stop and/or delete lifecycle resources. If the
%% Machi application is not running, sorry, please start Machi first.
%%
%% == Wheel reinvention ==
%%
%% There's a whole mess of configuration management research &amp;
%% libraries out there. I hope to ignore them all by doing something
%% quick &amp; dirty &amp; good enough here. If I fail, then I'll go
%% pay attention to That Other Stuff.
%%
%% == A note about policy ==
%%
%% It is outside of the scope of this local lifecycle manager to make
%% decisions about policy or to distribute policy info/files/whatever
%% to other machines. This is our machine. There are many like it,
%% but this one is ours.
%%
%% == Machi Application Variables ==
%%
%% All OTP application environment variables below are defined in the
%% `machi' application.
%%
%% <ul>
%% <li> <tt>flu_config_dir</tt>: Stores the `rc.d-'like config files for
%% FLU runtime policy.
%% </li>
%% <li> <tt>flu_data_dir</tt>: Stores the file data and metadata for
%% all FLUs.
%% </li>
%% <li> <tt>chain_config_dir</tt>: Stores the `rc.d'-like config files for
%% chain runtime policy.
%% </li>
%% </ul>
%%
%% == The FLU Lifecycle ==
%%
%% FLUs on the local machine may be started and stopped, as defined by
%% administrative policy. In order to do any useful work, however, a
%% running FLU must also be configured to be a member of a replication
%% chain. Thus, as a practical matter, both a FLU and the chain that
%% the FLU participates in must both be managed by this manager.
%%
%% When a new `rc.d'-style config file is written to the FLU
%% definition directory, a Machi server process will discover the file
%% within a certain period of time, e.g. 15 seconds. The FLU will be
%% started with the file's specified parameters. A FLU should be
%% defined and started before configuring its chain membership.
%%
%% Usually a FLU is removed implicitly by removing that FLU from the a
%% newer definition file for the chain, or by deleting the entire
%% chain definition. If a FLU has been started but never been a chain
%% member, then the FLU can be stopped &amp; removed explicitly.
%%
%% When a FLU has been removed by policy, the FLU's data files are set
%% aside into a temporary area. An additional policy command may be
%% used to permanently delete such FLUs' data files, i.e. to reclaim
%% disk space.
%%
%% Resources for the FLU are defined in {@link machi_projection.hrl}
%% in the `p_srvr{}' record. The major elements of this record are:
%%
%% <ul>
%%
%% <li> <tt>name :: atom()</tt>: The name of the FLU. This name
%% should be unique over the lifetime of the administrative
%% domain and thus managed by outside policy. This name must be
%% the same as the name of the `rc.d'-style config file that
%% defines the FLU.
%% </li>
%% <li> <tt>address :: string()</tt>: The DNS hostname or IP address
%% used by other servers to communicate with this FLU.
%% </li>
%% <li> <tt>port :: non_neg_integer() </tt>: The TCP port number that
%% the FLU listens to for incoming Protocol Buffers-serialized
%% communication.
%% </li>
%% <li> <tt>props :: property_list()</tt>: A general-purpose property
%% list. Its use is currently fluid &amp; not well-defined yet.
%% </li>
%% </ul>
%%
%% == The Chain Lifecycle ==
%%
%% If a FLU on the local machine is expected to participate in a
%% replication chain, then an `rc.d'-style chain definition file must
%% also be present on each machine that runs a FLU in the chain.
%%
%% Machi's chains are self-managing, via Humming Consensus; see the
%% [https://github.com/basho/machi/tree/master/doc/] directory for
%% much more detail about Humming Consensus. After FLUs have received
%% their initial chain configuration for Humming Consensus, the FLUs
%% will manage each other (and the chain) themselves.
%%
%% However, Humming Consensus does not handle three chain management
%% problems: 1. specifying the very first chain configuration,
%% 2. altering the membership of the chain (adding/removing FLUs from
%% the chain), or 3. stopping the chain permanently.
%%
%% FLUs in a new chain should have definition files created on each
%% FLU's respective machine prior to defining their chain. Similarly,
%% on each machine that hosts a chain member, a chain definition file
%% created. External policy is responsible for creating each of these
%% files.
%%
%% Resources for the chain are defined in {@link machi_projection.hrl}
%% in the `chain_def_v1{}' record. The major elements of this record are:
%%
%% <ul>
%%
%% <li> <tt>name :: atom()</tt>: The name of the chain. This name
%% should be unique over the lifetime of the administrative
%% domain and thus managed by outside policy. This name must be
%% the same as the name of the `rc.d'-style config file that
%% defines the chain.
%% </li>
%% <li> <tt>mode :: 'ap_mode' | 'cp_mode'</tt>: This is the consistency
%% to be used for managing the chain's replicated data: eventual
%% consistency and strong consistency, respectively.
%% </li>
%% <li> <tt>full :: [#p_srvr{}] </tt>: A list of `#p_srvr{}' records
%% to define the full-service members of the chain.
%% </li>
%% <li> <tt>witnesses :: [#p_srvr{}] </tt>: A list of `#p_srvr{}' records
%% to define the witness-only members of the chain. Witness servers
%% may only be used with strong consistency mode.
%% </li>
%% <li> <tt>props :: property_list()</tt>: A general-purpose property
%% list. Its use is currently fluid &amp; not well-defined yet.
%% </li>
%% </ul>
%%
%% == Conflicts with TCP ports, FLU & chain names, etc ==
%%
%% This manager is not responsible for managing conflicts in resource
%% namespaces, e.g., TCP port numbers, FLU names, chain names, etc.
%% Managing these namespaces is external policy's responsibility.
-module(machi_lifecycle_mgr).
-behaviour(gen_server).
@ -33,7 +195,10 @@
-define(SERVER, ?MODULE).
-record(state, {}).
-record(state, {
flus = [] :: [atom()],
chains = [] :: list()
}).
start_link() ->
gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
@ -50,13 +215,7 @@ handle_cast(_Msg, State) ->
{noreply, State}.
handle_info(finish_init, State) ->
FLUs = get_local_flu_names(),
FLU_Epochs = get_latest_public_epochs(FLUs),
FLUs_at_zero = [FLU || {FLU, 0} <- FLU_Epochs],
lager:info("FLUs at epoch 0: ~p\n", [FLUs_at_zero]),
ChainDefs = get_initial_chains(),
perhaps_bootstrap_chains(ChainDefs, FLUs),
{noreply, State};
{noreply, finish_init(State)};
handle_info(_Info, State) ->
{noreply, State}.
@ -68,7 +227,22 @@ code_change(_OldVsn, State, _Extra) ->
%%%%%
get_local_flu_names() ->
finish_init(S) ->
%% machi_flu_sup will start all FLUs that have a valid definition
%% file. That supervisor's structure + OTP supervisor behavior
%% guarantees that those FLUs should now be running.
%% (TODO: Unless they absolutely cannot keep running and the
%% supervisor has given up and terminated them.)
RunningFLUs = get_local_running_flus(),
RunningFLU_Epochs = get_latest_public_epochs(RunningFLUs),
RunningFLUs_at_zero = [FLU || {FLU, 0} <- RunningFLU_Epochs],
lager:info("Running FLUs: ~p\n", [RunningFLUs]),
lager:info("Running FLUs at epoch 0: ~p\n", [RunningFLUs_at_zero]),
ChainDefs = get_initial_chains(),
perhaps_bootstrap_chains(ChainDefs, RunningFLUs_at_zero, RunningFLUs),
S#state{flus=RunningFLUs, chains=ChainDefs}.
get_local_running_flus() ->
[Name || {Name,_,_,_} <- supervisor:which_children(machi_flu_sup)].
get_latest_public_epochs(FLUs) ->
@ -98,7 +272,7 @@ sanitize_chain_def_rec(Whole, {Acc, D}) ->
try
#chain_def_v1{name=Name,
mode=Mode,
upi=UPI,
full=Full,
witnesses=Witnesses} = Whole,
true = is_atom(Name),
NameK = {name, Name},
@ -107,34 +281,54 @@ sanitize_chain_def_rec(Whole, {Acc, D}) ->
IsPSrvr = fun(X) when is_record(X, p_srvr) -> true;
(_) -> false
end,
true = lists:all(IsPSrvr, UPI),
true = lists:all(IsPSrvr, Full),
true = lists:all(IsPSrvr, Witnesses),
%% All is sane enough.
D2 = dict:store(NameK, Name, D),
{[Whole|Acc], D2}
catch _:_ ->
catch _X:_Y ->
_ = lager:log(error, self(),
"~s: Bad chain_def record, skipping: ~P\n",
[?MODULE, Whole, 15]),
"~s: Bad chain_def record (~w ~w), skipping: ~P\n",
[?MODULE, _X, _Y, Whole, 15]),
{Acc, D}
end.
perhaps_bootstrap_chains([], _LocalFLUs_at_zero) ->
perhaps_bootstrap_chains([], LocalFLUs_at_zero, LocalFLUs) ->
if LocalFLUs == [] ->
ok;
true ->
lager:warning("The following FLUs are defined but are not also "
"members of a defined chain: ~w\n",
[LocalFLUs_at_zero])
end,
ok;
perhaps_bootstrap_chains([CD|ChainDefs], LocalFLUs_at_zero) ->
#chain_def_v1{upi=UPI, witnesses=Witnesses} = CD,
AllNames = [Name || #p_srvr{name=Name} <- UPI ++ Witnesses],
perhaps_bootstrap_chains([CD|ChainDefs], LocalFLUs_at_zero, LocalFLUs) ->
#chain_def_v1{full=Full, witnesses=Witnesses} = CD,
AllNames = [Name || #p_srvr{name=Name} <- Full ++ Witnesses],
case ordsets:intersection(ordsets:from_list(AllNames),
ordsets:from_list(LocalFLUs_at_zero)) of
[] ->
io:format(user, "TODO: no local flus in ~P\n", [CD, 10]),
ok;
[FLU1|_] ->
bootstrap_chain(CD, FLU1)
end,
perhaps_bootstrap_chains(ChainDefs, LocalFLUs_at_zero).
perhaps_bootstrap_chains(ChainDefs, LocalFLUs_at_zero, LocalFLUs);
[FLU1|_]=FLUs ->
%% One FLU is enough: Humming Consensus will config the remaining
bootstrap_chain(CD, FLU1),
perhaps_bootstrap_chains(ChainDefs, LocalFLUs_at_zero -- FLUs,
LocalFLUs -- FLUs)
end.
bootstrap_chain(CD, FLU) ->
io:format(user, "TODO: config ~p as bootstrap member of ~p\n", [FLU, CD]),
todo.
bootstrap_chain(#chain_def_v1{name=ChainName, mode=CMode, full=Full,
witnesses=Witnesses, props=Props}=CD, FLU) ->
All_p_srvrs = Full ++ Witnesses,
L = [{Name, P_srvr} || #p_srvr{name=Name}=P_srvr <- All_p_srvrs],
MembersDict = orddict:from_list(L),
Mgr = machi_chain_manager1:make_chmgr_regname(FLU),
case machi_chain_manager1:set_chain_members(Mgr, ChainName, 0, CMode,
MembersDict, Props) of
ok ->
ok;
Else ->
error_logger:warning("Attempt to bootstrap chain ~w via FLU ~w "
"failed: ~w (defn ~w)\n", [Else, CD]),
ok
end.

6
src/machi_sup.erl
View file

@ -62,8 +62,8 @@ init([]) ->
ServerSup =
{machi_flu_sup, {machi_flu_sup, start_link, []},
Restart, Shutdown, Type, []},
ChainBootstrap =
{machi_chain_bootstrap, {machi_chain_bootstrap, start_link, []},
LifecycleMgr =
{machi_lifecycle_mgr, {machi_lifecycle_mgr, start_link, []},
Restart, Shutdown, worker, []},
{ok, {SupFlags, [ServerSup, ChainBootstrap]}}.
{ok, {SupFlags, [ServerSup, LifecycleMgr]}}.