From 014ba89e3a3fa0853826d7e18e160e3ee9061f4d Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Mon, 19 Oct 2015 17:58:50 +0900 Subject: [PATCH 01/77] Add client-side plumbing for high proto append chunk CoC --- src/machi_pb_high_client.erl | 23 +++++++++++------------ test/machi_pb_high_client_test.erl | 10 ++++++---- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/machi_pb_high_client.erl b/src/machi_pb_high_client.erl index ef1e740..18dcc34 100644 --- a/src/machi_pb_high_client.erl +++ b/src/machi_pb_high_client.erl @@ -38,7 +38,7 @@ connected_p/1, echo/2, echo/3, auth/3, auth/4, - append_chunk/6, append_chunk/7, + append_chunk/7, append_chunk/8, write_chunk/5, write_chunk/6, read_chunk/5, read_chunk/6, trim_chunk/4, trim_chunk/5, @@ -96,21 +96,21 @@ auth(PidSpec, User, Pass) -> auth(PidSpec, User, Pass, Timeout) -> send_sync(PidSpec, {auth, User, Pass}, Timeout). --spec append_chunk(pid(), PlacementKey::binary(), Prefix::binary(), Chunk::binary(), +-spec append_chunk(pid(), CoC_namespace::binary(), CoC_locator::integer(), Prefix::binary(), Chunk::binary(), CSum::binary(), ChunkExtra::non_neg_integer()) -> {ok, Filename::string(), Offset::machi_dt:file_offset()} | {error, machi_client_error_reason()}. -append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra) -> - append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra, ?DEFAULT_TIMEOUT). +append_chunk(PidSpec, CoC_namespace, CoC_locator, Prefix, Chunk, CSum, ChunkExtra) -> + append_chunk(PidSpec, CoC_namespace, CoC_locator, Prefix, Chunk, CSum, ChunkExtra, ?DEFAULT_TIMEOUT). --spec append_chunk(pid(), PlacementKey::binary(), Prefix::binary(), +-spec append_chunk(pid(), CoC_namespace::binary(), CoC_locator::integer(), Prefix::binary(), Chunk::binary(), CSum::binary(), ChunkExtra::non_neg_integer(), Timeout::non_neg_integer()) -> {ok, Filename::string(), Offset::machi_dt:file_offset()} | {error, machi_client_error_reason()}. -append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra, Timeout) -> - send_sync(PidSpec, {append_chunk, PlacementKey, Prefix, Chunk, CSum, ChunkExtra}, Timeout). +append_chunk(PidSpec, CoC_namespace, CoC_locator, Prefix, Chunk, CSum, ChunkExtra, Timeout) -> + send_sync(PidSpec, {append_chunk, CoC_namespace, CoC_locator, Prefix, Chunk, CSum, ChunkExtra}, Timeout). -spec write_chunk(pid(), File::string(), machi_dt:file_offset(), Chunk::binary(), CSum::binary()) -> @@ -281,15 +281,14 @@ do_send_sync2({auth, User, Pass}, #state{sock=Sock}=S) -> Res = {bummer, {X, Y, erlang:get_stacktrace()}}, {Res, S} end; -do_send_sync2({append_chunk, PlacementKey, Prefix, Chunk, CSum, ChunkExtra}, +do_send_sync2({append_chunk, CoC_namespace, CoC_locator, + Prefix, Chunk, CSum, ChunkExtra}, #state{sock=Sock, sock_id=Index, count=Count}=S) -> try ReqID = <>, - PK = if PlacementKey == <<>> -> undefined; - true -> PlacementKey - end, CSumT = convert_csum_req(CSum, Chunk), - Req = #mpb_appendchunkreq{placement_key=PK, + Req = #mpb_appendchunkreq{coc_namespace=CoC_namespace, + coc_locator=CoC_locator, prefix=Prefix, chunk=Chunk, csum=CSumT, diff --git a/test/machi_pb_high_client_test.erl b/test/machi_pb_high_client_test.erl index 361eb55..9cba8bc 100644 --- a/test/machi_pb_high_client_test.erl +++ b/test/machi_pb_high_client_test.erl @@ -61,16 +61,17 @@ smoke_test2() -> %% a separate test module? Or separate test func? {error, _} = ?C:auth(Clnt, "foo", "bar"), - PK = <<>>, + CoC_n = <<>>, % CoC_namespace (not implemented) + CoC_l = 0, % CoC_locator (not implemented) Prefix = <<"prefix">>, Chunk1 = <<"Hello, chunk!">>, {ok, {Off1, Size1, File1}} = - ?C:append_chunk(Clnt, PK, Prefix, Chunk1, none, 0), + ?C:append_chunk(Clnt, CoC_n, CoC_l, Prefix, Chunk1, none, 0), true = is_binary(File1), Chunk2 = "It's another chunk", CSum2 = {client_sha, machi_util:checksum_chunk(Chunk2)}, {ok, {Off2, Size2, File2}} = - ?C:append_chunk(Clnt, PK, Prefix, Chunk2, CSum2, 1024), + ?C:append_chunk(Clnt, CoC_n, CoC_l, Prefix, Chunk2, CSum2, 1024), Chunk3 = ["This is a ", <<"test,">>, 32, [["Hello, world!"]]], File3 = File2, Off3 = Off2 + iolist_size(Chunk2), @@ -114,7 +115,8 @@ smoke_test2() -> LargeBytes = binary:copy(<<"x">>, 1024*1024), LBCsum = {client_sha, machi_util:checksum_chunk(LargeBytes)}, {ok, {Offx, Sizex, Filex}} = - ?C:append_chunk(Clnt, PK, Prefix, LargeBytes, LBCsum, 0), + ?C:append_chunk(Clnt, CoC_n, CoC_l, + Prefix, LargeBytes, LBCsum, 0), ok = ?C:trim_chunk(Clnt, Filex, Offx, Sizex), %% Make sure everything was trimmed -- 2.45.2 From 10a27ce7ddb5f2b60634f75787b4da17050637b7 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Mon, 19 Oct 2015 21:08:38 +0900 Subject: [PATCH 02/77] All eunit tests now passing again --- include/machi.hrl | 3 + src/machi.proto | 23 +++--- src/machi_cr_client.erl | 125 +++++++++++++++++++++-------- src/machi_dt.erl | 4 + src/machi_flu1.erl | 47 +++++++---- src/machi_flu1_client.erl | 104 +++++++++++++++++++++--- src/machi_pb_translate.erl | 24 +++--- src/machi_proxy_flu1_client.erl | 51 +++++++++--- test/machi_pb_high_client_test.erl | 2 +- 9 files changed, 295 insertions(+), 88 deletions(-) diff --git a/include/machi.hrl b/include/machi.hrl index 671fe01..f825556 100644 --- a/include/machi.hrl +++ b/include/machi.hrl @@ -40,3 +40,6 @@ %% TODO: it's used in flu_sup and elsewhere, change this to suitable name -define(TEST_ETS_TABLE, test_ets_table). + +-define(DEFAULT_COC_NAMESPACE, ""). +-define(DEFAULT_COC_LOCATOR, 0). diff --git a/src/machi.proto b/src/machi.proto index c9251cb..6228865 100644 --- a/src/machi.proto +++ b/src/machi.proto @@ -170,11 +170,12 @@ message Mpb_AuthResp { // High level API: append_chunk() request & response message Mpb_AppendChunkReq { - optional bytes placement_key = 1; - required string prefix = 2; - required bytes chunk = 3; - required Mpb_ChunkCSum csum = 4; - optional uint32 chunk_extra = 5; + required string coc_namespace = 1; + required uint32 coc_locator = 2; + required string prefix = 3; + required bytes chunk = 4; + required Mpb_ChunkCSum csum = 5; + optional uint32 chunk_extra = 6; } message Mpb_AppendChunkResp { @@ -378,11 +379,13 @@ message Mpb_ProjectionV1 { message Mpb_LL_AppendChunkReq { required Mpb_EpochID epoch_id = 1; - optional bytes placement_key = 2; - required string prefix = 3; - required bytes chunk = 4; - required Mpb_ChunkCSum csum = 5; - optional uint32 chunk_extra = 6; + /* To avoid CoC use, use coc_namespace="" and coc_locator=0 */ + required string coc_namespace = 2; + required uint32 coc_locator = 3; + required string prefix = 4; + required bytes chunk = 5; + required Mpb_ChunkCSum csum = 6; + optional uint32 chunk_extra = 7; } message Mpb_LL_AppendChunkResp { diff --git a/src/machi_cr_client.erl b/src/machi_cr_client.erl index e03262b..cec7c6a 100644 --- a/src/machi_cr_client.erl +++ b/src/machi_cr_client.erl @@ -119,7 +119,9 @@ -export([ %% File API append_chunk/3, append_chunk/4, + append_chunk/5, append_chunk/6, append_chunk_extra/4, append_chunk_extra/5, + append_chunk_extra/6, append_chunk_extra/7, write_chunk/4, write_chunk/5, read_chunk/5, read_chunk/6, trim_chunk/4, trim_chunk/5, @@ -164,13 +166,29 @@ start_link(P_srvr_list, Opts) -> %% with `Prefix'. append_chunk(PidSpec, Prefix, Chunk) -> - append_chunk(PidSpec, Prefix, Chunk, ?DEFAULT_TIMEOUT). + append_chunk_extra(PidSpec, ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR, + Prefix, Chunk, 0, ?DEFAULT_TIMEOUT). %% @doc Append a chunk (binary- or iolist-style) of data to a file %% with `Prefix'. append_chunk(PidSpec, Prefix, Chunk, Timeout) -> - append_chunk_extra(PidSpec, Prefix, Chunk, 0, Timeout). + append_chunk_extra(PidSpec, ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR, + Prefix, Chunk, 0, Timeout). + +%% @doc Append a chunk (binary- or iolist-style) of data to a file +%% with `Prefix'. + +append_chunk(PidSpec, CoC_Namespace, CoC_Locator, Prefix, Chunk) -> + append_chunk_extra(PidSpec, CoC_Namespace, CoC_Locator, + Prefix, Chunk, 0, ?DEFAULT_TIMEOUT). + +%% @doc Append a chunk (binary- or iolist-style) of data to a file +%% with `Prefix'. + +append_chunk(PidSpec, CoC_Namespace, CoC_Locator, Prefix, Chunk, Timeout) -> + append_chunk_extra(PidSpec, CoC_Namespace, CoC_Locator, + Prefix, Chunk, 0, Timeout). %% @doc Append a chunk (binary- or iolist-style) of data to a file %% with `Prefix'. @@ -184,7 +202,25 @@ append_chunk_extra(PidSpec, Prefix, Chunk, ChunkExtra) append_chunk_extra(PidSpec, Prefix, Chunk, ChunkExtra, Timeout0) -> {TO, Timeout} = timeout(Timeout0), - gen_server:call(PidSpec, {req, {append_chunk_extra, Prefix, + gen_server:call(PidSpec, {req, {append_chunk_extra, + ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR, + Prefix, + Chunk, ChunkExtra, TO}}, + Timeout). + +append_chunk_extra(PidSpec, CoC_Namespace, CoC_Locator, Prefix, Chunk, ChunkExtra) + when is_integer(ChunkExtra), ChunkExtra >= 0 -> + append_chunk_extra(PidSpec, CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra, ?DEFAULT_TIMEOUT). + +%% @doc Append a chunk (binary- or iolist-style) of data to a file +%% with `Prefix'. + +append_chunk_extra(PidSpec, CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra, Timeout0) -> + {TO, Timeout} = timeout(Timeout0), + gen_server:call(PidSpec, {req, {append_chunk_extra, + CoC_Namespace, CoC_Locator, Prefix, Chunk, ChunkExtra, TO}}, Timeout). @@ -288,8 +324,10 @@ code_change(_OldVsn, S, _Extra) -> %%%%%%%%%%%%%%%%%%%%%%%%%%% -handle_call2({append_chunk_extra, Prefix, Chunk, ChunkExtra, TO}, _From, S) -> - do_append_head(Prefix, Chunk, ChunkExtra, 0, os:timestamp(), TO, S); +handle_call2({append_chunk_extra, CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra, TO}, _From, S) -> + do_append_head(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, 0, os:timestamp(), TO, S); handle_call2({write_chunk, File, Offset, Chunk, TO}, _From, S) -> do_write_head(File, Offset, Chunk, 0, os:timestamp(), TO, S); handle_call2({read_chunk, File, Offset, Size, Opts, TO}, _From, S) -> @@ -301,9 +339,12 @@ handle_call2({checksum_list, File, TO}, _From, S) -> handle_call2({list_files, TO}, _From, S) -> do_list_files(0, os:timestamp(), TO, S). -do_append_head(Prefix, Chunk, ChunkExtra, 0=Depth, STime, TO, S) -> - do_append_head2(Prefix, Chunk, ChunkExtra, Depth + 1, STime, TO, S); -do_append_head(Prefix, Chunk, ChunkExtra, Depth, STime, TO, #state{proj=P}=S) -> +do_append_head(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, 0=Depth, STime, TO, S) -> + do_append_head2(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, Depth + 1, STime, TO, S); +do_append_head(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, Depth, STime, TO, #state{proj=P}=S) -> %% io:format(user, "head sleep1,", []), sleep_a_while(Depth), DiffMs = timer:now_diff(os:timestamp(), STime) div 1000, @@ -318,53 +359,62 @@ do_append_head(Prefix, Chunk, ChunkExtra, Depth, STime, TO, #state{proj=P}=S) -> case S2#state.proj of P2 when P2 == undefined orelse P2#projection_v1.upi == [] -> - do_append_head(Prefix, Chunk, ChunkExtra, Depth + 1, + do_append_head(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, Depth + 1, STime, TO, S2); _ -> - do_append_head2(Prefix, Chunk, ChunkExtra, Depth + 1, + do_append_head2(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, Depth + 1, STime, TO, S2) end end. -do_append_head2(Prefix, Chunk, ChunkExtra, Depth, STime, TO, +do_append_head2(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, Depth, STime, TO, #state{proj=P}=S) -> [HeadFLU|_RestFLUs] = mutation_flus(P), case is_witness_flu(HeadFLU, P) of true -> case witnesses_use_our_epoch(S) of true -> - do_append_head3(Prefix, Chunk, ChunkExtra, Depth, + do_append_head3(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, Depth, STime, TO, S); false -> %% Bummer, go back to the beginning and retry. - do_append_head(Prefix, Chunk, ChunkExtra, Depth, + do_append_head(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, Depth, STime, TO, S) end; false -> - do_append_head3(Prefix, Chunk, ChunkExtra, Depth, STime, TO, S) + do_append_head3(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, Depth, STime, TO, S) end. -do_append_head3(Prefix, Chunk, ChunkExtra, Depth, STime, TO, +do_append_head3(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, Depth, STime, TO, #state{epoch_id=EpochID, proj=P, proxies_dict=PD}=S) -> [HeadFLU|RestFLUs] = non_witness_flus(mutation_flus(P), P), Proxy = orddict:fetch(HeadFLU, PD), - case ?FLU_PC:append_chunk_extra(Proxy, - EpochID, Prefix, Chunk, ChunkExtra, - ?TIMEOUT) of + case ?FLU_PC:append_chunk_extra(Proxy, EpochID, + CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, ?TIMEOUT) of {ok, {Offset, _Size, File}=_X} -> - %% io:format(user, "append ~w,", [HeadFLU]), - do_append_midtail(RestFLUs, Prefix, File, Offset, Chunk, ChunkExtra, + do_append_midtail(RestFLUs, CoC_Namespace, CoC_Locator, Prefix, + File, Offset, Chunk, ChunkExtra, [HeadFLU], 0, STime, TO, S); {error, bad_checksum}=BadCS -> {reply, BadCS, S}; {error, Retry} when Retry == partition; Retry == bad_epoch; Retry == wedged -> - do_append_head(Prefix, Chunk, ChunkExtra, Depth, STime, TO, S); + do_append_head(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, Depth, STime, TO, S); {error, written} -> %% Implicit sequencing + this error = we don't know where this %% written block is. But we lost a race. Repeat, with a new %% sequencer assignment. - do_append_head(Prefix, Chunk, ChunkExtra, Depth, STime, TO, S); + do_append_head(CoC_Namespace, CoC_Locator, Prefix, + Chunk, ChunkExtra, Depth, STime, TO, S); {error, trimmed} = Err -> %% TODO: behaviour {reply, Err, S}; @@ -373,12 +423,15 @@ do_append_head3(Prefix, Chunk, ChunkExtra, Depth, STime, TO, Prefix,iolist_size(Chunk)}) end. -do_append_midtail(RestFLUs, Prefix, File, Offset, Chunk, ChunkExtra, +do_append_midtail(RestFLUs, CoC_Namespace, CoC_Locator, Prefix, + File, Offset, Chunk, ChunkExtra, Ws, Depth, STime, TO, S) when RestFLUs == [] orelse Depth == 0 -> - do_append_midtail2(RestFLUs, Prefix, File, Offset, Chunk, ChunkExtra, + do_append_midtail2(RestFLUs, CoC_Namespace, CoC_Locator, Prefix, + File, Offset, Chunk, ChunkExtra, Ws, Depth + 1, STime, TO, S); -do_append_midtail(_RestFLUs, Prefix, File, Offset, Chunk, ChunkExtra, +do_append_midtail(_RestFLUs, CoC_Namespace, CoC_Locator, Prefix, File, + Offset, Chunk, ChunkExtra, Ws, Depth, STime, TO, #state{proj=P}=S) -> %% io:format(user, "midtail sleep2,", []), sleep_a_while(Depth), @@ -405,36 +458,43 @@ do_append_midtail(_RestFLUs, Prefix, File, Offset, Chunk, ChunkExtra, if Prefix == undefined -> % atom! not binary()!! {error, partition}; true -> - do_append_head2(Prefix, Chunk, ChunkExtra, + do_append_head2(CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra, Depth, STime, TO, S2) end; RestFLUs3 -> - do_append_midtail2(RestFLUs3, Prefix, File, Offset, + do_append_midtail2(RestFLUs3, + CoC_Namespace, CoC_Locator, + Prefix, File, Offset, Chunk, ChunkExtra, Ws, Depth + 1, STime, TO, S2) end end end. -do_append_midtail2([], _Prefix, File, Offset, Chunk, +do_append_midtail2([], _CoC_Namespace, _CoC_Locator, + _Prefix, File, Offset, Chunk, _ChunkExtra, _Ws, _Depth, _STime, _TO, S) -> %% io:format(user, "ok!\n", []), {reply, {ok, {Offset, chunk_wrapper_size(Chunk), File}}, S}; -do_append_midtail2([FLU|RestFLUs]=FLUs, Prefix, File, Offset, Chunk, +do_append_midtail2([FLU|RestFLUs]=FLUs, CoC_Namespace, CoC_Locator, + Prefix, File, Offset, Chunk, ChunkExtra, Ws, Depth, STime, TO, #state{epoch_id=EpochID, proxies_dict=PD}=S) -> Proxy = orddict:fetch(FLU, PD), case ?FLU_PC:write_chunk(Proxy, EpochID, File, Offset, Chunk, ?TIMEOUT) of ok -> %% io:format(user, "write ~w,", [FLU]), - do_append_midtail2(RestFLUs, Prefix, File, Offset, Chunk, + do_append_midtail2(RestFLUs, CoC_Namespace, CoC_Locator, Prefix, + File, Offset, Chunk, ChunkExtra, [FLU|Ws], Depth, STime, TO, S); {error, bad_checksum}=BadCS -> %% TODO: alternate strategy? {reply, BadCS, S}; {error, Retry} when Retry == partition; Retry == bad_epoch; Retry == wedged -> - do_append_midtail(FLUs, Prefix, File, Offset, Chunk, + do_append_midtail(FLUs, CoC_Namespace, CoC_Locator, Prefix, + File, Offset, Chunk, ChunkExtra, Ws, Depth, STime, TO, S); {error, written} -> %% We know what the chunk ought to be, so jump to the @@ -499,7 +559,8 @@ do_write_head2(File, Offset, Chunk, Depth, STime, TO, ok -> %% From this point onward, we use the same code & logic path as %% append does. - do_append_midtail(RestFLUs, undefined, File, Offset, Chunk, + do_append_midtail(RestFLUs, undefined, undefined, undefined, + File, Offset, Chunk, undefined, [HeadFLU], 0, STime, TO, S); {error, bad_checksum}=BadCS -> {reply, BadCS, S}; diff --git a/src/machi_dt.erl b/src/machi_dt.erl index ab74f13..6df3eab 100644 --- a/src/machi_dt.erl +++ b/src/machi_dt.erl @@ -29,6 +29,8 @@ -type chunk_s() :: 'trimmed' | binary(). -type chunk_pos() :: {file_offset(), chunk_size(), file_name_s()}. -type chunk_size() :: non_neg_integer(). +-type coc_namespace() :: string(). +-type coc_locator() :: non_neg_integer(). -type error_general() :: 'bad_arg' | 'wedged' | 'bad_checksum'. -type epoch_csum() :: binary(). -type epoch_num() :: -1 | non_neg_integer(). @@ -58,6 +60,8 @@ chunk_s/0, chunk_pos/0, chunk_size/0, + coc_namespace/0, + coc_locator/0, error_general/0, epoch_csum/0, epoch_num/0, diff --git a/src/machi_flu1.erl b/src/machi_flu1.erl index 042eaed..057e665 100644 --- a/src/machi_flu1.erl +++ b/src/machi_flu1.erl @@ -229,24 +229,27 @@ append_server_loop(FluPid, #state{wedged=Wedged_p, witness=Witness_p, epoch_id=OldEpochId, flu_name=FluName}=S) -> receive - {seq_append, From, _Prefix, _Chunk, _CSum, _Extra, _EpochID} + {seq_append, From, _N, _L, _Prefix, _Chunk, _CSum, _Extra, _EpochID} when Witness_p -> %% The FLU's net_server_loop() process ought to filter all %% witness states, but we'll keep this clause for extra %% paranoia. From ! witness, append_server_loop(FluPid, S); - {seq_append, From, _Prefix, _Chunk, _CSum, _Extra, _EpochID} + {seq_append, From, _N, _L, _Prefix, _Chunk, _CSum, _Extra, _EpochID} when Wedged_p -> From ! wedged, append_server_loop(FluPid, S); - {seq_append, From, Prefix, Chunk, CSum, Extra, EpochID} -> + {seq_append, From, CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum, Extra, EpochID} -> %% Old is the one from our state, plain old 'EpochID' comes %% from the client. _ = case OldEpochId == EpochID of true -> spawn(fun() -> - append_server_dispatch(From, Prefix, Chunk, CSum, Extra, FluName, EpochID) + append_server_dispatch(From, CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum, Extra, + FluName, EpochID) end); false -> From ! {error, bad_epoch} @@ -401,10 +404,12 @@ do_pb_ll_request3({low_wedge_status, _EpochID}, S) -> do_pb_ll_request3({low_proj, PCMD}, S) -> {do_server_proj_request(PCMD, S), S}; %% Witness status *matters* below -do_pb_ll_request3({low_append_chunk, _EpochID, PKey, Prefix, Chunk, CSum_tag, +do_pb_ll_request3({low_append_chunk, _EpochID, CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, CSum, ChunkExtra}, #state{witness=false}=S) -> - {do_server_append_chunk(PKey, Prefix, Chunk, CSum_tag, CSum, + {do_server_append_chunk(CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, CSum, ChunkExtra, S), S}; do_pb_ll_request3({low_write_chunk, _EpochID, File, Offset, Chunk, CSum_tag, CSum}, @@ -444,10 +449,12 @@ do_pb_hl_request2({high_echo, Msg}, S) -> {Msg, S}; do_pb_hl_request2({high_auth, _User, _Pass}, S) -> {-77, S}; -do_pb_hl_request2({high_append_chunk, _todoPK, Prefix, ChunkBin, TaggedCSum, +do_pb_hl_request2({high_append_chunk, CoC_namespace, CoC_locator, + Prefix, ChunkBin, TaggedCSum, ChunkExtra}, #state{high_clnt=Clnt}=S) -> Chunk = {TaggedCSum, ChunkBin}, - Res = machi_cr_client:append_chunk_extra(Clnt, Prefix, Chunk, + Res = machi_cr_client:append_chunk_extra(Clnt, CoC_namespace, CoC_locator, + Prefix, Chunk, ChunkExtra), {Res, S}; do_pb_hl_request2({high_write_chunk, File, Offset, ChunkBin, TaggedCSum}, @@ -506,23 +513,27 @@ do_server_proj_request({kick_projection_reaction}, end), async_no_response. -do_server_append_chunk(PKey, Prefix, Chunk, CSum_tag, CSum, +do_server_append_chunk(CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, CSum, ChunkExtra, S) -> case sanitize_prefix(Prefix) of ok -> - do_server_append_chunk2(PKey, Prefix, Chunk, CSum_tag, CSum, + do_server_append_chunk2(CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, CSum, ChunkExtra, S); _ -> {error, bad_arg} end. -do_server_append_chunk2(_PKey, Prefix, Chunk, CSum_tag, Client_CSum, +do_server_append_chunk2(CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, Client_CSum, ChunkExtra, #state{flu_name=FluName, epoch_id=EpochID}=_S) -> %% TODO: Do anything with PKey? try TaggedCSum = check_or_make_tagged_checksum(CSum_tag, Client_CSum,Chunk), - R = {seq_append, self(), Prefix, Chunk, TaggedCSum, ChunkExtra, EpochID}, + R = {seq_append, self(), CoC_Namespace, CoC_Locator, + Prefix, Chunk, TaggedCSum, ChunkExtra, EpochID}, FluName ! R, receive {assignment, Offset, File} -> @@ -677,8 +688,10 @@ do_server_trunc_hack(File, #state{data_dir=DataDir}=_S) -> {error, bad_arg} end. -append_server_dispatch(From, Prefix, Chunk, CSum, Extra, FluName, EpochId) -> - Result = case handle_append(Prefix, Chunk, CSum, Extra, FluName, EpochId) of +append_server_dispatch(From, CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum, Extra, FluName, EpochId) -> + Result = case handle_append(CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum, Extra, FluName, EpochId) of {ok, File, Offset} -> {assignment, Offset, File}; Other -> @@ -687,9 +700,11 @@ append_server_dispatch(From, Prefix, Chunk, CSum, Extra, FluName, EpochId) -> From ! Result, exit(normal). -handle_append(_Prefix, <<>>, _Csum, _Extra, _FluName, _EpochId) -> +handle_append(_N, _L, _Prefix, <<>>, _Csum, _Extra, _FluName, _EpochId) -> {error, bad_arg}; -handle_append(Prefix, Chunk, Csum, Extra, FluName, EpochId) -> +handle_append(CoC_Namespace, CoC_Locator, + Prefix, Chunk, Csum, Extra, FluName, EpochId) -> + io:format(user, "TODO: CoC_Namespace, CoC_Locator ~p ~p\n", [CoC_Namespace, CoC_Locator]), Res = machi_flu_filename_mgr:find_or_make_filename_from_prefix(FluName, EpochId, {prefix, Prefix}), case Res of {file, F} -> diff --git a/src/machi_flu1_client.erl b/src/machi_flu1_client.erl index 119e154..e5b65fc 100644 --- a/src/machi_flu1_client.erl +++ b/src/machi_flu1_client.erl @@ -55,7 +55,9 @@ -export([ %% File API append_chunk/4, append_chunk/5, + append_chunk/6, append_chunk/7, append_chunk_extra/5, append_chunk_extra/6, + append_chunk_extra/7, append_chunk_extra/8, read_chunk/6, read_chunk/7, checksum_list/3, checksum_list/4, list_files/2, list_files/3, @@ -93,7 +95,9 @@ -spec append_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk()) -> {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}. append_chunk(Sock, EpochID, Prefix, Chunk) -> - append_chunk2(Sock, EpochID, Prefix, Chunk, 0). + append_chunk2(Sock, EpochID, + ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR, + Prefix, Chunk, 0). %% @doc Append a chunk (binary- or iolist-style) of data to a file %% with `Prefix'. @@ -104,7 +108,39 @@ append_chunk(Sock, EpochID, Prefix, Chunk) -> append_chunk(Host, TcpPort, EpochID, Prefix, Chunk) -> Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}), try - append_chunk2(Sock, EpochID, Prefix, Chunk, 0) + append_chunk2(Sock, EpochID, + ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR, + Prefix, Chunk, 0) + after + disconnect(Sock) + end. + +%% @doc Append a chunk (binary- or iolist-style) of data to a file +%% with `Prefix'. + +-spec append_chunk(port_wrap(), machi_dt:epoch_id(), + machi_dt:coc_namespace(), machi_dt:coc_locator(), + machi_dt:file_prefix(), machi_dt:chunk()) -> + {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}. +append_chunk(Sock, EpochID, CoC_Namespace, CoC_Locator, Prefix, Chunk) -> + append_chunk2(Sock, EpochID, + CoC_Namespace, CoC_Locator, + Prefix, Chunk, 0). + +%% @doc Append a chunk (binary- or iolist-style) of data to a file +%% with `Prefix'. + +-spec append_chunk(machi_dt:inet_host(), machi_dt:inet_port(), + machi_dt:epoch_id(), + machi_dt:coc_namespace(), machi_dt:coc_locator(), + machi_dt:file_prefix(), machi_dt:chunk()) -> + {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}. +append_chunk(Host, TcpPort, EpochID, CoC_Namespace, CoC_Locator, Prefix, Chunk) -> + Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}), + try + append_chunk2(Sock, EpochID, + CoC_Namespace, CoC_Locator, + Prefix, Chunk, 0) after disconnect(Sock) end. @@ -117,11 +153,14 @@ append_chunk(Host, TcpPort, EpochID, Prefix, Chunk) -> %% be reserved by the file sequencer for later write(s) by the %% `write_chunk()' API. --spec append_chunk_extra(port_wrap(), machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) -> +-spec append_chunk_extra(port_wrap(), machi_dt:epoch_id(), + machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) -> {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}. append_chunk_extra(Sock, EpochID, Prefix, Chunk, ChunkExtra) when is_integer(ChunkExtra), ChunkExtra >= 0 -> - append_chunk2(Sock, EpochID, Prefix, Chunk, ChunkExtra). + append_chunk2(Sock, EpochID, + ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR, + Prefix, Chunk, ChunkExtra). %% @doc Append a chunk (binary- or iolist-style) of data to a file %% with `Prefix' and also request an additional `Extra' bytes. @@ -138,7 +177,54 @@ append_chunk_extra(Host, TcpPort, EpochID, Prefix, Chunk, ChunkExtra) when is_integer(ChunkExtra), ChunkExtra >= 0 -> Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}), try - append_chunk2(Sock, EpochID, Prefix, Chunk, ChunkExtra) + append_chunk2(Sock, EpochID, + ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR, + Prefix, Chunk, ChunkExtra) + after + disconnect(Sock) + end. + +%% @doc Append a chunk (binary- or iolist-style) of data to a file +%% with `Prefix' and also request an additional `Extra' bytes. +%% +%% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then +%% the file offsets that follow `Chunk''s position for the following 4K will +%% be reserved by the file sequencer for later write(s) by the +%% `write_chunk()' API. + +-spec append_chunk_extra(port_wrap(), machi_dt:epoch_id(), + machi_dt:coc_namespace(), machi_dt:coc_locator(), + machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) -> + {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}. +append_chunk_extra(Sock, EpochID, CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra) + when is_integer(ChunkExtra), ChunkExtra >= 0 -> + append_chunk2(Sock, EpochID, + CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra). + +%% @doc Append a chunk (binary- or iolist-style) of data to a file +%% with `Prefix' and also request an additional `Extra' bytes. +%% +%% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then +%% the file offsets that follow `Chunk''s position for the following 4K will +%% be reserved by the file sequencer for later write(s) by the +%% `write_chunk()' API. + +-spec append_chunk_extra(machi_dt:inet_host(), machi_dt:inet_port(), + machi_dt:epoch_id(), + machi_dt:coc_namespace(), machi_dt:coc_locator(), + machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) -> + {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}. +append_chunk_extra(Host, TcpPort, EpochID, + CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra) + when is_integer(ChunkExtra), ChunkExtra >= 0 -> + Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}), + try + append_chunk2(Sock, EpochID, + CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra) after disconnect(Sock) end. @@ -542,7 +628,8 @@ read_chunk2(Sock, EpochID, File0, Offset, Size, Opts) -> {low_read_chunk, EpochID, File, Offset, Size, Opts}), do_pb_request_common(Sock, ReqID, Req). -append_chunk2(Sock, EpochID, Prefix0, Chunk0, ChunkExtra) -> +append_chunk2(Sock, EpochID, CoC_Namespace, CoC_Locator, + Prefix0, Chunk0, ChunkExtra) -> ReqID = <<"id">>, {Chunk, CSum_tag, CSum} = case Chunk0 of @@ -552,12 +639,11 @@ append_chunk2(Sock, EpochID, Prefix0, Chunk0, ChunkExtra) -> {Tag, CS} = machi_util:unmake_tagged_csum(ChunkCSum), {Chk, Tag, CS} end, - PKey = <<>>, % TODO Prefix = machi_util:make_binary(Prefix0), Req = machi_pb_translate:to_pb_request( ReqID, - {low_append_chunk, EpochID, PKey, Prefix, Chunk, CSum_tag, CSum, - ChunkExtra}), + {low_append_chunk, EpochID, CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, CSum, ChunkExtra}), do_pb_request_common(Sock, ReqID, Req). write_chunk2(Sock, EpochID, File0, Offset, Chunk0) -> diff --git a/src/machi_pb_translate.erl b/src/machi_pb_translate.erl index 0b49908..4c5472c 100644 --- a/src/machi_pb_translate.erl +++ b/src/machi_pb_translate.erl @@ -52,14 +52,16 @@ from_pb_request(#mpb_ll_request{ req_id=ReqID, append_chunk=#mpb_ll_appendchunkreq{ epoch_id=PB_EpochID, - placement_key=PKey, + coc_namespace=CoC_Namespace, + coc_locator=CoC_Locator, prefix=Prefix, chunk=Chunk, csum=#mpb_chunkcsum{type=CSum_type, csum=CSum}, chunk_extra=ChunkExtra}}) -> EpochID = conv_to_epoch_id(PB_EpochID), CSum_tag = conv_to_csum_tag(CSum_type), - {ReqID, {low_append_chunk, EpochID, PKey, Prefix, Chunk, CSum_tag, CSum, + {ReqID, {low_append_chunk, EpochID, CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, CSum, ChunkExtra}}; from_pb_request(#mpb_ll_request{ req_id=ReqID, @@ -170,14 +172,15 @@ from_pb_request(#mpb_request{req_id=ReqID, {ReqID, {high_auth, User, Pass}}; from_pb_request(#mpb_request{req_id=ReqID, append_chunk=IR=#mpb_appendchunkreq{}}) -> - #mpb_appendchunkreq{placement_key=__todoPK, + #mpb_appendchunkreq{coc_namespace=CoC_namespace, + coc_locator=CoC_locator, prefix=Prefix, chunk=Chunk, csum=CSum, chunk_extra=ChunkExtra} = IR, TaggedCSum = make_tagged_csum(CSum, Chunk), - {ReqID, {high_append_chunk, __todoPK, Prefix, Chunk, TaggedCSum, - ChunkExtra}}; + {ReqID, {high_append_chunk, CoC_namespace, CoC_locator, Prefix, Chunk, + TaggedCSum, ChunkExtra}}; from_pb_request(#mpb_request{req_id=ReqID, write_chunk=IR=#mpb_writechunkreq{}}) -> #mpb_writechunkreq{chunk=#mpb_chunk{file_name=File, @@ -388,15 +391,16 @@ to_pb_request(ReqID, {low_echo, _BogusEpochID, Msg}) -> to_pb_request(ReqID, {low_auth, _BogusEpochID, User, Pass}) -> #mpb_ll_request{req_id=ReqID, do_not_alter=2, auth=#mpb_authreq{user=User, password=Pass}}; -to_pb_request(ReqID, {low_append_chunk, EpochID, PKey, Prefix, Chunk, - CSum_tag, CSum, ChunkExtra}) -> +to_pb_request(ReqID, {low_append_chunk, EpochID, CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, CSum, ChunkExtra}) -> PB_EpochID = conv_from_epoch_id(EpochID), CSum_type = conv_from_csum_tag(CSum_tag), PB_CSum = #mpb_chunkcsum{type=CSum_type, csum=CSum}, #mpb_ll_request{req_id=ReqID, do_not_alter=2, append_chunk=#mpb_ll_appendchunkreq{ epoch_id=PB_EpochID, - placement_key=PKey, + coc_namespace=CoC_Namespace, + coc_locator=CoC_Locator, prefix=Prefix, chunk=Chunk, csum=PB_CSum, @@ -500,7 +504,7 @@ to_pb_response(ReqID, {low_auth, _, _, _}, __TODO_Resp) -> #mpb_ll_response{req_id=ReqID, generic=#mpb_errorresp{code=1, msg="AUTH not implemented"}}; -to_pb_response(ReqID, {low_append_chunk, _EID, _PKey, _Pfx, _Ch, _CST, _CS, _CE}, Resp)-> +to_pb_response(ReqID, {low_append_chunk, _EID, _N, _L, _Pfx, _Ch, _CST, _CS, _CE}, Resp)-> case Resp of {ok, {Offset, Size, File}} -> Where = #mpb_chunkpos{offset=Offset, @@ -687,7 +691,7 @@ to_pb_response(ReqID, {high_auth, _User, _Pass}, _Resp) -> #mpb_response{req_id=ReqID, generic=#mpb_errorresp{code=1, msg="AUTH not implemented"}}; -to_pb_response(ReqID, {high_append_chunk, _TODO, _Prefix, _Chunk, _TSum, _CE}, Resp)-> +to_pb_response(ReqID, {high_append_chunk, _CoC_n, _CoC_l, _Prefix, _Chunk, _TSum, _CE}, Resp)-> case Resp of {ok, {Offset, Size, File}} -> Where = #mpb_chunkpos{offset=Offset, diff --git a/src/machi_proxy_flu1_client.erl b/src/machi_proxy_flu1_client.erl index 2cbaabd..e4bc0d2 100644 --- a/src/machi_proxy_flu1_client.erl +++ b/src/machi_proxy_flu1_client.erl @@ -58,7 +58,9 @@ -export([ %% File API append_chunk/4, append_chunk/5, + append_chunk/6, append_chunk/7, append_chunk_extra/5, append_chunk_extra/6, + append_chunk_extra/7, append_chunk_extra/8, read_chunk/6, read_chunk/7, checksum_list/3, checksum_list/4, list_files/2, list_files/3, @@ -111,22 +113,51 @@ append_chunk(PidSpec, EpochID, Prefix, Chunk) -> %% with `Prefix'. append_chunk(PidSpec, EpochID, Prefix, Chunk, Timeout) -> - gen_server:call(PidSpec, {req, {append_chunk, EpochID, Prefix, Chunk}}, - Timeout). + append_chunk_extra(PidSpec, EpochID, + ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR, + Prefix, Chunk, 0, Timeout). + +%% @doc Append a chunk (binary- or iolist-style) of data to a file +%% with `Prefix'. + +append_chunk(PidSpec, EpochID, CoC_Namespace, CoC_Locator, Prefix, Chunk) -> + append_chunk(PidSpec, EpochID, CoC_Namespace, CoC_Locator, Prefix, Chunk, infinity). + +%% @doc Append a chunk (binary- or iolist-style) of data to a file +%% with `Prefix'. + +append_chunk(PidSpec, EpochID, CoC_Namespace, CoC_Locator, Prefix, Chunk, Timeout) -> + append_chunk_extra(PidSpec, EpochID, + CoC_Namespace, CoC_Locator, + Prefix, Chunk, 0, Timeout). %% @doc Append a chunk (binary- or iolist-style) of data to a file %% with `Prefix'. append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra) when is_integer(ChunkExtra), ChunkExtra >= 0 -> - append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra, infinity). + append_chunk_extra(PidSpec, EpochID, + ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR, + Prefix, Chunk, ChunkExtra, infinity). %% @doc Append a chunk (binary- or iolist-style) of data to a file %% with `Prefix'. append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra, Timeout) -> - gen_server:call(PidSpec, {req, {append_chunk_extra, EpochID, Prefix, - Chunk, ChunkExtra}}, + append_chunk_extra(PidSpec, EpochID, + ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR, + Prefix, Chunk, ChunkExtra, Timeout). + +append_chunk_extra(PidSpec, EpochID, CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra) -> + append_chunk_extra(PidSpec, EpochID, CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra, infinity). + +append_chunk_extra(PidSpec, EpochID, CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra, Timeout) -> + gen_server:call(PidSpec, {req, {append_chunk_extra, EpochID, + CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra}}, Timeout). %% @doc Read a chunk of data of size `Size' from `File' at `Offset'. @@ -384,12 +415,12 @@ do_req_retry(_Req, 2, Err, S) -> do_req_retry(Req, Depth, _Err, S) -> do_req(Req, Depth + 1, try_connect(disconnect(S))). -make_req_fun({append_chunk, EpochID, Prefix, Chunk}, +make_req_fun({append_chunk_extra, EpochID, CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra}, #state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) -> - fun() -> Mod:append_chunk(Sock, EpochID, Prefix, Chunk) end; -make_req_fun({append_chunk_extra, EpochID, Prefix, Chunk, ChunkExtra}, - #state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) -> - fun() -> Mod:append_chunk_extra(Sock, EpochID, Prefix, Chunk, ChunkExtra) end; + fun() -> Mod:append_chunk_extra(Sock, EpochID, CoC_Namespace, CoC_Locator, + Prefix, Chunk, ChunkExtra) + end; make_req_fun({read_chunk, EpochID, File, Offset, Size, Opts}, #state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) -> fun() -> Mod:read_chunk(Sock, EpochID, File, Offset, Size, Opts) end; diff --git a/test/machi_pb_high_client_test.erl b/test/machi_pb_high_client_test.erl index 9cba8bc..9f6984b 100644 --- a/test/machi_pb_high_client_test.erl +++ b/test/machi_pb_high_client_test.erl @@ -61,7 +61,7 @@ smoke_test2() -> %% a separate test module? Or separate test func? {error, _} = ?C:auth(Clnt, "foo", "bar"), - CoC_n = <<>>, % CoC_namespace (not implemented) + CoC_n = "", % CoC_namespace (not implemented) CoC_l = 0, % CoC_locator (not implemented) Prefix = <<"prefix">>, Chunk1 = <<"Hello, chunk!">>, -- 2.45.2 From c3002d985262be06d72a8ccd5bdf3d5b4408bf7a Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 20 Oct 2015 00:20:04 +0900 Subject: [PATCH 03/77] Push TODO closer to actual TODO work site, standardize var spelling --- src/machi_flu1.erl | 9 +++++---- src/machi_flu_filename_mgr.erl | 21 +++++++++++++-------- src/machi_pb_high_client.erl | 6 +++--- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/machi_flu1.erl b/src/machi_flu1.erl index 057e665..cd7fd98 100644 --- a/src/machi_flu1.erl +++ b/src/machi_flu1.erl @@ -449,11 +449,11 @@ do_pb_hl_request2({high_echo, Msg}, S) -> {Msg, S}; do_pb_hl_request2({high_auth, _User, _Pass}, S) -> {-77, S}; -do_pb_hl_request2({high_append_chunk, CoC_namespace, CoC_locator, +do_pb_hl_request2({high_append_chunk, CoC_Namespace, CoC_Locator, Prefix, ChunkBin, TaggedCSum, ChunkExtra}, #state{high_clnt=Clnt}=S) -> Chunk = {TaggedCSum, ChunkBin}, - Res = machi_cr_client:append_chunk_extra(Clnt, CoC_namespace, CoC_locator, + Res = machi_cr_client:append_chunk_extra(Clnt, CoC_Namespace, CoC_Locator, Prefix, Chunk, ChunkExtra), {Res, S}; @@ -704,8 +704,9 @@ handle_append(_N, _L, _Prefix, <<>>, _Csum, _Extra, _FluName, _EpochId) -> {error, bad_arg}; handle_append(CoC_Namespace, CoC_Locator, Prefix, Chunk, Csum, Extra, FluName, EpochId) -> - io:format(user, "TODO: CoC_Namespace, CoC_Locator ~p ~p\n", [CoC_Namespace, CoC_Locator]), - Res = machi_flu_filename_mgr:find_or_make_filename_from_prefix(FluName, EpochId, {prefix, Prefix}), + CoC = {coc, CoC_Namespace, CoC_Locator}, + Res = machi_flu_filename_mgr:find_or_make_filename_from_prefix( + FluName, EpochId, {prefix, Prefix}, CoC), case Res of {file, F} -> case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, F}) of diff --git a/src/machi_flu_filename_mgr.erl b/src/machi_flu_filename_mgr.erl index 7e8bb9d..9ceea0a 100644 --- a/src/machi_flu_filename_mgr.erl +++ b/src/machi_flu_filename_mgr.erl @@ -51,7 +51,7 @@ -export([ child_spec/2, start_link/2, - find_or_make_filename_from_prefix/3, + find_or_make_filename_from_prefix/4, increment_prefix_sequence/2, list_files_by_prefix/2 ]). @@ -87,18 +87,23 @@ start_link(FluName, DataDir) when is_atom(FluName) andalso is_list(DataDir) -> N = make_filename_mgr_name(FluName), gen_server:start_link({local, N}, ?MODULE, [FluName, DataDir], []). --spec find_or_make_filename_from_prefix( FluName :: atom(), - EpochId :: pv1_epoch_n(), - Prefix :: {prefix, string()} ) -> +-spec find_or_make_filename_from_prefix( FluName :: atom(), + EpochId :: pv1_epoch_n(), + Prefix :: {prefix, string()}, + {coc, riak_dt:coc_namespace(), riak_dt:coc_locator()}) -> {file, Filename :: string()} | {error, Reason :: term() } | timeout. % @doc Find the latest available or make a filename from a prefix. A prefix % should be in the form of a tagged tuple `{prefix, P}'. Returns a tagged % tuple in the form of `{file, F}' or an `{error, Reason}' -find_or_make_filename_from_prefix(FluName, EpochId, {prefix, Prefix}) when is_atom(FluName) -> +find_or_make_filename_from_prefix(FluName, EpochId, + {prefix, Prefix}, + {coc, CoC_Namespace, CoC_Locator}) + when is_atom(FluName) -> + io:format(user, "TODO: CoC_Namespace, CoC_Locator ~p ~p\n", [CoC_Namespace, CoC_Locator]), N = make_filename_mgr_name(FluName), - gen_server:call(N, {find_filename, EpochId, Prefix}, ?TIMEOUT); -find_or_make_filename_from_prefix(_FluName, _EpochId, Other) -> - lager:error("~p is not a valid prefix.", [Other]), + gen_server:call(N, {find_filename, EpochId, Prefix}, ?TIMEOUT); +find_or_make_filename_from_prefix(_FluName, _EpochId, Other, Other2) -> + lager:error("~p is not a valid prefix/CoC ~p", [Other, Other2]), error(badarg). -spec increment_prefix_sequence( FluName :: atom(), Prefix :: {prefix, string()} ) -> diff --git a/src/machi_pb_high_client.erl b/src/machi_pb_high_client.erl index 18dcc34..c8149ad 100644 --- a/src/machi_pb_high_client.erl +++ b/src/machi_pb_high_client.erl @@ -281,14 +281,14 @@ do_send_sync2({auth, User, Pass}, #state{sock=Sock}=S) -> Res = {bummer, {X, Y, erlang:get_stacktrace()}}, {Res, S} end; -do_send_sync2({append_chunk, CoC_namespace, CoC_locator, +do_send_sync2({append_chunk, CoC_Namespace, CoC_Locator, Prefix, Chunk, CSum, ChunkExtra}, #state{sock=Sock, sock_id=Index, count=Count}=S) -> try ReqID = <>, CSumT = convert_csum_req(CSum, Chunk), - Req = #mpb_appendchunkreq{coc_namespace=CoC_namespace, - coc_locator=CoC_locator, + Req = #mpb_appendchunkreq{coc_namespace=CoC_Namespace, + coc_locator=CoC_Locator, prefix=Prefix, chunk=Chunk, csum=CSumT, -- 2.45.2 From a1c834518d3da8f5ced1748bdc1c6306e1dfcbbb Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 20 Oct 2015 01:38:46 +0900 Subject: [PATCH 04/77] Attempt to address Mark's review comments --- doc/cluster-of-clusters/name-game-sketch.org | 126 +++++++++++++------ 1 file changed, 89 insertions(+), 37 deletions(-) diff --git a/doc/cluster-of-clusters/name-game-sketch.org b/doc/cluster-of-clusters/name-game-sketch.org index a7adb59..44b5df0 100644 --- a/doc/cluster-of-clusters/name-game-sketch.org +++ b/doc/cluster-of-clusters/name-game-sketch.org @@ -175,8 +175,8 @@ of chains: | Chain length | CoC namespace | Mode | Comment | |--------------+---------------+------+----------------------------------| | 3 | normal | AP | Normal storage redundancy & cost | -| 2 | cheap | AP | Reduced cost storage | -| 1 | risky | AP | Really cheap storage | +| 2 | reduced | AP | Reduced cost storage | +| 1 | risky | AP | Really, really cheap storage | | 9 | paranoid | AP | Safety-critical storage | | 3 | sequential | CP | Strong consistency | |--------------+---------------+------+----------------------------------| @@ -189,7 +189,7 @@ intention. Further, the CoC administrators may wish to use the namespace to provide separate storage for different applications. Jane's application may use the namespace "jane-normal" and Bob's app uses -"bob-cheap". The CoC administrators may definite separate groups of +"bob-reduced". The CoC administrators may definite separate groups of chains on separate servers to serve these two applications. * 6. Floating point is not required ... it is merely convenient for explanation @@ -218,8 +218,8 @@ Machi assigns file names based on: ~ClientSuppliedPrefix ++ "^" ++ SomeOpaqueFileNameSuffix~ What if the CoC client could peek inside of the opaque file name -suffix in order to remove (or add) the CoC location information that -we need? +suffix in order to look at the CoC location information that we might +code in the filename suffix? ** The notation we use @@ -263,6 +263,58 @@ we need? if ~r=0.5~, then ~L = 0.33 + 0.5*(0.58-0.33) = 0.455~, which is exactly in the middle of the ~(0.33,0.58]~ interval. +** A bit more about the CoC locator's meaning and use + +- If two files were written using exactly the same CoC locator and the + same CoC namespace, then the client is indicating that it wishes + that the two files be stored in the same chain. +- If two files have a different CoC locator, then the client has + absolutely no expectation of where the two files will be stored + relative to each other. + +Given the items above, then some consequences are: + +- If the client doesn't care about CoC placement, then picking a + random number is fine. Always choosing a different locator ~L~ for + each append will scatter data across the CoC as widely as possible. +- If the client believes that some physical locality is good, then the + client should reuse the same locator ~L~ for a batch of appends to + the same prefix ~p~ and namespace ~N~. We have no recommendations + for the batch size, yet; perhaps 10-1,000 might be a good start for + experiments? + +When the client choose CoC namespace ~N~ and CoC locator ~L~ (using +random number or target cluster technique), the client uses ~N~'s CoC +map to find the CoC target cluster, ~T~. The client has also chosen +the file prefix ~p~. The append op sent to cluster ~T~ would look +like: + +~append_chunk(N="reduced",L=0.25,p="myprefix",<<900-data-bytes>>,<>,...)~ + +A successful result would yield a chunk position: + +~{offset=883293,size=900,file="myprefix^reduced^0.25^OpaqueSuffix"}~ + +** A bit more about the CoC namespaces's meaning and use + +- The CoC framework will provide means of creating and managing + chains of different types, e.g., chain length, consistency mode. +- The CoC framework will manage the mapping of CoC namespace names to + the chains in the system. +- The CoC framework will provide a query service to map a CoC + namespace name to a Coc map, + e.g. ~coc_latest_map("reduced") -> Map{generation=7,...}~. + +For use by Riak CS, for example, we'd likely start with the following +namespaces ... working our way down the list as we add new features +and/or re-implement existing CS features. + +- "standard" = Chain length = 3, eventually consistency mode +- "reduced" = Chain length = 2, eventually consistency mode. +- "stanchion7" = Chain length = 7, strong consistency mode. Perhaps + use this namespace for the metadata required to re-implement the + operations that are performed by today's Stanchion application. + * 8. File migration (a.k.a. rebalancing/reparitioning/resharding/redistribution) ** What is "migration"? @@ -311,26 +363,26 @@ As an example: And the new Random Slicing map for some CoC namespace ~N~ might look like this: -| Generation number / Namespace | 7 / cheap | -|-------------------------------+------------| -| SubMap | 1 | -|-------------------------------+------------| -| Hash range | Cluster ID | -|-------------------------------+------------| -| 0.00 - 0.33 | Cluster1 | -| 0.33 - 0.66 | Cluster2 | -| 0.66 - 1.00 | Cluster3 | -|-------------------------------+------------| -| SubMap | 2 | -|-------------------------------+------------| -| Hash range | Cluster ID | -|-------------------------------+------------| -| 0.00 - 0.25 | Cluster1 | -| 0.25 - 0.33 | Cluster4 | -| 0.33 - 0.58 | Cluster2 | -| 0.58 - 0.66 | Cluster4 | -| 0.66 - 0.91 | Cluster3 | -| 0.91 - 1.00 | Cluster4 | +| Generation number / Namespace | 7 / reduced | +|-------------------------------+-------------| +| SubMap | 1 | +|-------------------------------+-------------| +| Hash range | Cluster ID | +|-------------------------------+-------------| +| 0.00 - 0.33 | Cluster1 | +| 0.33 - 0.66 | Cluster2 | +| 0.66 - 1.00 | Cluster3 | +|-------------------------------+-------------| +| SubMap | 2 | +|-------------------------------+-------------| +| Hash range | Cluster ID | +|-------------------------------+-------------| +| 0.00 - 0.25 | Cluster1 | +| 0.25 - 0.33 | Cluster4 | +| 0.33 - 0.58 | Cluster2 | +| 0.58 - 0.66 | Cluster4 | +| 0.66 - 0.91 | Cluster3 | +| 0.91 - 1.00 | Cluster4 | When a new Random Slicing map contains a single submap, then its use is identical to the original Random Slicing algorithm. If the map @@ -367,18 +419,18 @@ Cluster4. When the CoC manager is satisfied that all such files have been copied to Cluster4, then the CoC manager can create and distribute a new map, such as: -| Generation number / Namespace | 8 / cheap | -|-------------------------------+------------| -| SubMap | 1 | -|-------------------------------+------------| -| Hash range | Cluster ID | -|-------------------------------+------------| -| 0.00 - 0.25 | Cluster1 | -| 0.25 - 0.33 | Cluster4 | -| 0.33 - 0.58 | Cluster2 | -| 0.58 - 0.66 | Cluster4 | -| 0.66 - 0.91 | Cluster3 | -| 0.91 - 1.00 | Cluster4 | +| Generation number / Namespace | 8 / reduced | +|-------------------------------+-------------| +| SubMap | 1 | +|-------------------------------+-------------| +| Hash range | Cluster ID | +|-------------------------------+-------------| +| 0.00 - 0.25 | Cluster1 | +| 0.25 - 0.33 | Cluster4 | +| 0.33 - 0.58 | Cluster2 | +| 0.58 - 0.66 | Cluster4 | +| 0.66 - 0.91 | Cluster3 | +| 0.91 - 1.00 | Cluster4 | The HibariDB system performs data migrations in almost exactly this manner. However, one important -- 2.45.2 From 5477f3b6f85c0363cc710c1b7708fdc37afb5fd8 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 22 Oct 2015 17:28:15 +0900 Subject: [PATCH 05/77] WIP broken --- src/machi_flu_filename_mgr.erl | 45 +++++++++++++++++----------------- src/machi_util.erl | 42 +++++++++++++++++-------------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/src/machi_flu_filename_mgr.erl b/src/machi_flu_filename_mgr.erl index 9ceea0a..65a0a93 100644 --- a/src/machi_flu_filename_mgr.erl +++ b/src/machi_flu_filename_mgr.erl @@ -97,11 +97,10 @@ start_link(FluName, DataDir) when is_atom(FluName) andalso is_list(DataDir) -> % tuple in the form of `{file, F}' or an `{error, Reason}' find_or_make_filename_from_prefix(FluName, EpochId, {prefix, Prefix}, - {coc, CoC_Namespace, CoC_Locator}) + {coc, _CoC_Ns, _CoC_Loc}=CoC_NL) when is_atom(FluName) -> - io:format(user, "TODO: CoC_Namespace, CoC_Locator ~p ~p\n", [CoC_Namespace, CoC_Locator]), N = make_filename_mgr_name(FluName), - gen_server:call(N, {find_filename, EpochId, Prefix}, ?TIMEOUT); + gen_server:call(N, {find_filename, EpochId, CoC_NL, Prefix}, ?TIMEOUT); find_or_make_filename_from_prefix(_FluName, _EpochId, Other, Other2) -> lager:error("~p is not a valid prefix/CoC ~p", [Other, Other2]), error(badarg). @@ -143,19 +142,19 @@ handle_cast(Req, State) -> %% the FLU has already validated that the caller's epoch id and the FLU's epoch id %% are the same. So we *assume* that remains the case here - that is to say, we %% are not wedged. -handle_call({find_filename, EpochId, Prefix}, _From, S = #state{ datadir = DataDir, - epoch = EpochId, - tid = Tid}) -> +handle_call({find_filename, EpochId, CoC_NL, Prefix}, _From, S = #state{ datadir = DataDir, + epoch = EpochId, + tid = Tid }) -> %% Our state and the caller's epoch ids are the same. Business as usual. - File = handle_find_file(Tid, Prefix, DataDir), + File = handle_find_file(Tid, CoC_NL, Prefix, DataDir), {reply, {file, File}, S}; -handle_call({find_filename, EpochId, Prefix}, _From, S = #state{ datadir = DataDir, tid = Tid }) -> +handle_call({find_filename, EpochId, CoC_NL, Prefix}, _From, S = #state{ datadir = DataDir, tid = Tid }) -> %% If the epoch id in our state and the caller's epoch id were the same, it would've %% matched the above clause. Since we're here, we know that they are different. %% If epoch ids between our state and the caller's are different, we must increment the %% sequence number, generate a filename and then cache it. - File = increment_and_cache_filename(Tid, DataDir, Prefix), + File = increment_and_cache_filename(Tid, DataDir, CoC_NL, Prefix), {reply, {file, File}, S#state{epoch = EpochId}}; handle_call({increment_sequence, Prefix}, _From, S = #state{ datadir = DataDir }) -> @@ -203,11 +202,11 @@ list_files(DataDir, Prefix) -> make_filename_mgr_name(FluName) when is_atom(FluName) -> list_to_atom(atom_to_list(FluName) ++ "_filename_mgr"). -handle_find_file(Tid, Prefix, DataDir) -> - N = machi_util:read_max_filenum(DataDir, Prefix), +handle_find_file(Tid, {coc,CoC_Namespace,CoC_Locator}, Prefix, DataDir) -> + N = machi_util:read_max_filenum(DataDir, CoC_Namespace, CoC_Locator, Prefix), {File, Cleanup} = case find_file(DataDir, Prefix, N) of [] -> - {find_or_make_filename(Tid, DataDir, Prefix, N), false}; + {find_or_make_filename(Tid, DataDir, CoC_Namespace, CoC_Locator, Prefix, N), false}; [H] -> {H, true}; [Fn | _ ] = L -> lager:debug( @@ -218,20 +217,20 @@ handle_find_file(Tid, Prefix, DataDir) -> maybe_cleanup(Tid, {Prefix, N}, Cleanup), filename:basename(File). -find_or_make_filename(Tid, DataDir, Prefix, N) -> - case ets:lookup(Tid, {Prefix, N}) of +find_or_make_filename(Tid, DataDir, CoC_Namespace, CoC_Locator, Prefix, N) -> + case ets:lookup(Tid, {CoC_Namespace, CoC_Locator, Prefix, N}) of [] -> - F = generate_filename(DataDir, Prefix, N), - true = ets:insert_new(Tid, {{Prefix, N}, F}), + F = generate_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix, N), + true = ets:insert_new(Tid, {{CoC_Namespace, CoC_Locator, Prefix, N}, F}), F; [{_Key, File}] -> File end. -generate_filename(DataDir, Prefix, N) -> +generate_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix, N) -> {F, _} = machi_util:make_data_filename( DataDir, - Prefix, + CoC_Namespace, CoC_Locator, Prefix, generate_uuid_v4_str(), N), binary_to_list(F). @@ -241,11 +240,11 @@ maybe_cleanup(_Tid, _Key, false) -> maybe_cleanup(Tid, Key, true) -> true = ets:delete(Tid, Key). -increment_and_cache_filename(Tid, DataDir, Prefix) -> - ok = machi_util:increment_max_filenum(DataDir, Prefix), - N = machi_util:read_max_filenum(DataDir, Prefix), - F = generate_filename(DataDir, Prefix, N), - true = ets:insert_new(Tid, {{Prefix, N}, F}), +increment_and_cache_filename(Tid, DataDir, {coc,CoC_Namespace,CoC_Locator}, Prefix) -> + ok = machi_util:increment_max_filenum(DataDir, CoC_Namespace, CoC_Locator, Prefix), + N = machi_util:read_max_filenum(DataDir, CoC_Namespace, CoC_Locator, Prefix), + F = generate_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix, N), + true = ets:insert_new(Tid, {{CoC_Namespace, CoC_Locator, Prefix, N}, F}), filename:basename(F). diff --git a/src/machi_util.erl b/src/machi_util.erl index 8aa1972..83b4e6e 100644 --- a/src/machi_util.erl +++ b/src/machi_util.erl @@ -30,13 +30,13 @@ hexstr_to_int/1, int_to_hexstr/2, int_to_hexbin/2, make_binary/1, make_string/1, make_regname/1, - make_config_filename/2, + make_config_filename/4, make_checksum_filename/4, make_checksum_filename/2, - make_data_filename/4, make_data_filename/2, + make_data_filename/6, make_data_filename/2, make_projection_filename/2, is_valid_filename/1, parse_filename/1, - read_max_filenum/2, increment_max_filenum/2, + read_max_filenum/4, increment_max_filenum/4, info_msg/2, verb/1, verb/2, mbytes/1, pretty_time/0, pretty_time/2, @@ -68,10 +68,12 @@ make_regname(Prefix) when is_list(Prefix) -> %% @doc Calculate a config file path, by common convention. --spec make_config_filename(string(), string()) -> +-spec make_config_filename(string(), riak_dt:coc_namespace(), riak_dt:coc_locator(), string()) -> string(). -make_config_filename(DataDir, Prefix) -> - lists:flatten(io_lib:format("~s/config/~s", [DataDir, Prefix])). +make_config_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix) -> + Locator_str = int_to_hexstr(CoC_Locator, 32), + lists:flatten(io_lib:format("~s/config/~s^~s^~s", + [DataDir, Prefix, CoC_Namespace, Locator_str])). %% @doc Calculate a checksum file path, by common convention. @@ -92,17 +94,19 @@ make_checksum_filename(DataDir, FileName) -> %% @doc Calculate a file data file path, by common convention. --spec make_data_filename(string(), string(), atom()|string()|binary(), integer()|string()) -> +-spec make_data_filename(string(), riak_dt:coc_namespace(), riak_dt:coc_locator(), string(), atom()|string()|binary(), integer()|string()) -> {binary(), string()}. -make_data_filename(DataDir, Prefix, SequencerName, FileNum) +make_data_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix, SequencerName, FileNum) when is_integer(FileNum) -> - File = erlang:iolist_to_binary(io_lib:format("~s^~s^~w", - [Prefix, SequencerName, FileNum])), + Locator_str = int_to_hexstr(CoC_Locator, 32), + File = erlang:iolist_to_binary(io_lib:format("~s^~s^~s^~s^~w", + [Prefix, CoC_Namespace, Locator_str, SequencerName, FileNum])), make_data_filename2(DataDir, File); -make_data_filename(DataDir, Prefix, SequencerName, String) +make_data_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix, SequencerName, String) when is_list(String) -> - File = erlang:iolist_to_binary(io_lib:format("~s^~s^~s", - [Prefix, SequencerName, string])), + Locator_str = int_to_hexstr(CoC_Locator, 32), + File = erlang:iolist_to_binary(io_lib:format("~s^~s^~s^~s^~s", + [Prefix, CoC_Namespace, Locator_str, SequencerName, string])), make_data_filename2(DataDir, File). make_data_filename2(DataDir, File) -> @@ -158,10 +162,10 @@ parse_filename(Filename) -> %% @doc Read the file size of a config file, which is used as the %% basis for a minimum sequence number. --spec read_max_filenum(string(), string()) -> +-spec read_max_filenum(string(), riak_dt:coc_namespace(), riak_dt:coc_locator(), string()) -> non_neg_integer(). -read_max_filenum(DataDir, Prefix) -> - case file:read_file_info(make_config_filename(DataDir, Prefix)) of +read_max_filenum(DataDir, CoC_Namespace, CoC_Locator, Prefix) -> + case file:read_file_info(make_config_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix)) of {error, enoent} -> 0; {ok, FI} -> @@ -171,11 +175,11 @@ read_max_filenum(DataDir, Prefix) -> %% @doc Increase the file size of a config file, which is used as the %% basis for a minimum sequence number. --spec increment_max_filenum(string(), string()) -> +-spec increment_max_filenum(string(), riak_dt:coc_namespace(), riak_dt:coc_locator(), string()) -> ok | {error, term()}. -increment_max_filenum(DataDir, Prefix) -> +increment_max_filenum(DataDir, CoC_Namespace, CoC_Locator, Prefix) -> try - {ok, FH} = file:open(make_config_filename(DataDir, Prefix), [append]), + {ok, FH} = file:open(make_config_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix), [append]), ok = file:write(FH, "x"), ok = file:sync(FH), ok = file:close(FH) -- 2.45.2 From 916ac754d70a32d93c574c5c1a9917f7713aeee4 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 23 Oct 2015 00:24:55 +0900 Subject: [PATCH 06/77] WIP: still broken, almost passes suites=machi_cr_client_test tests=smoke_test_ --- src/machi_flu_filename_mgr.erl | 5 +++-- src/machi_util.erl | 21 +++++++++++++++++++-- test/machi_cr_client_test.erl | 9 ++++++--- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/machi_flu_filename_mgr.erl b/src/machi_flu_filename_mgr.erl index 65a0a93..4710a68 100644 --- a/src/machi_flu_filename_mgr.erl +++ b/src/machi_flu_filename_mgr.erl @@ -202,7 +202,8 @@ list_files(DataDir, Prefix) -> make_filename_mgr_name(FluName) when is_atom(FluName) -> list_to_atom(atom_to_list(FluName) ++ "_filename_mgr"). -handle_find_file(Tid, {coc,CoC_Namespace,CoC_Locator}, Prefix, DataDir) -> +handle_find_file(Tid, {coc,CoC_Namespace,CoC_Locator}=_CoC, Prefix, DataDir) -> +io:format(user, "\nCoC ~p\n", [_CoC]), N = machi_util:read_max_filenum(DataDir, CoC_Namespace, CoC_Locator, Prefix), {File, Cleanup} = case find_file(DataDir, Prefix, N) of [] -> @@ -214,7 +215,7 @@ handle_find_file(Tid, {coc,CoC_Namespace,CoC_Locator}, Prefix, DataDir) -> [Prefix, N, L]), {Fn, true} end, - maybe_cleanup(Tid, {Prefix, N}, Cleanup), + maybe_cleanup(Tid, {CoC_Namespace, CoC_Locator, Prefix, N}, Cleanup), filename:basename(File). find_or_make_filename(Tid, DataDir, CoC_Namespace, CoC_Locator, Prefix, N) -> diff --git a/src/machi_util.erl b/src/machi_util.erl index 83b4e6e..baba0f0 100644 --- a/src/machi_util.erl +++ b/src/machi_util.erl @@ -30,7 +30,7 @@ hexstr_to_int/1, int_to_hexstr/2, int_to_hexbin/2, make_binary/1, make_string/1, make_regname/1, - make_config_filename/4, + make_config_filename/4, make_config_filename/2, make_checksum_filename/4, make_checksum_filename/2, make_data_filename/6, make_data_filename/2, make_projection_filename/2, @@ -75,6 +75,14 @@ make_config_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix) -> lists:flatten(io_lib:format("~s/config/~s^~s^~s", [DataDir, Prefix, CoC_Namespace, Locator_str])). +%% @doc Calculate a config file path, by common convention. + +-spec make_config_filename(string(), string()) -> + string(). +make_config_filename(DataDir, Filename) -> + lists:flatten(io_lib:format("~s/config/~s", + [DataDir, Filename])). + %% @doc Calculate a checksum file path, by common convention. -spec make_checksum_filename(string(), string(), atom()|string()|binary(), integer()) -> @@ -154,7 +162,16 @@ is_valid_filename(Filename) -> -spec parse_filename( Filename :: string() ) -> [ string() ]. parse_filename(Filename) -> case string:tokens(Filename, "^") of - [_Prefix, _UUID, _SeqNo] = L -> L; + [_Prefix, _CoC_NS, _CoC_Loc, _UUID, _SeqNo] = L -> + L; + [Prefix, CoC_Loc, UUID, SeqNo] -> + %% string:tokens() doesn't consider "foo^^bar" as 3 tokens {sigh} + case re:replace(Filename, "[^^]+", "x", [global,{return,binary}]) of + <<"x^^x^x^x">> -> + [Prefix, "", CoC_Loc, UUID, SeqNo]; + _ -> + [] + end; _ -> [] end. diff --git a/test/machi_cr_client_test.erl b/test/machi_cr_client_test.erl index f5d3513..e127032 100644 --- a/test/machi_cr_client_test.erl +++ b/test/machi_cr_client_test.erl @@ -167,7 +167,9 @@ smoke_test2() -> true = is_binary(KludgeBin), {error, bad_arg} = machi_cr_client:checksum_list(C1, <<"!!!!">>), - %% Exactly one file right now +io:format(user, "\nFiles = ~p\n", [machi_cr_client:list_files(C1)]), + %% Exactly one file right now, e.g., + %% {ok,[{2098202,<<"pre^b144ef13-db4d-4c9f-96e7-caff02dc754f^1">>}]} {ok, [_]} = machi_cr_client:list_files(C1), %% Go back and test append_chunk_extra() and write_chunk() @@ -191,8 +193,9 @@ smoke_test2() -> end || Seq <- lists:seq(1, Extra10)], {ok, {Off11,Size11,File11}} = machi_cr_client:append_chunk(C1, Prefix, Chunk10), - %% Double-check that our reserved extra bytes were really honored! - true = (Off11 > (Off10 + (Extra10 * Size10))), + %% %% Double-check that our reserved extra bytes were really honored! + %% true = (Off11 > (Off10 + (Extra10 * Size10))), +io:format(user, "\nFiles = ~p\n", [machi_cr_client:list_files(C1)]), ok after -- 2.45.2 From 29c2ede275430f9a822a4e0896665a37ff211e19 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 28 Oct 2015 15:10:27 +0900 Subject: [PATCH 07/77] Add missing \ldots in chain repair figure --- doc/src.high-level/high-level-chain-mgr.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src.high-level/high-level-chain-mgr.tex b/doc/src.high-level/high-level-chain-mgr.tex index f139862..b88fa84 100644 --- a/doc/src.high-level/high-level-chain-mgr.tex +++ b/doc/src.high-level/high-level-chain-mgr.tex @@ -1279,7 +1279,7 @@ as the foundation for Machi's data loss prevention techniques. \begin{figure} \centering $ -[\overbrace{\underbrace{H_1}_\textbf{Head}, M_{11}, T_1, +[\overbrace{\underbrace{H_1}_\textbf{Head}, M_{11}, \ldots, T_1, H_2, M_{21}, \ldots \underbrace{T_2}_\textbf{Tail}}^\textbf{Chain (U.P.~Invariant preserving)} -- 2.45.2 From 2f9530529226246587f132cd2c7cc1f5db42f9cf Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 2 Dec 2015 15:24:24 +0900 Subject: [PATCH 08/77] Add machi_ap_repair_eqc:sublist() --- test/machi_ap_repair_eqc.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/machi_ap_repair_eqc.erl b/test/machi_ap_repair_eqc.erl index a42d7cc..ba9038c 100644 --- a/test/machi_ap_repair_eqc.erl +++ b/test/machi_ap_repair_eqc.erl @@ -218,6 +218,11 @@ sublist(L) -> ?LET(L2, eqc_gen:vector(K, eqc_gen:oneof(L)), lists:usort(L2))). +sublist(L) -> + ?LET(K, nat(), + ?LET(L2, eqc_gen:vector(K, eqc_gen:oneof(L)), + lists:usort(L2))). + %% Generator for possibly assymmetric partition information partition(FLUNames) -> frequency([{10, return([])}, -- 2.45.2 From 0d517d2377f8d3d9a080399ebd31719c3f64efd3 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 2 Dec 2015 15:36:41 +0900 Subject: [PATCH 09/77] fix machi_ap_repair_eqc:sublist() --- test/machi_ap_repair_eqc.erl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/machi_ap_repair_eqc.erl b/test/machi_ap_repair_eqc.erl index ba9038c..a42d7cc 100644 --- a/test/machi_ap_repair_eqc.erl +++ b/test/machi_ap_repair_eqc.erl @@ -218,11 +218,6 @@ sublist(L) -> ?LET(L2, eqc_gen:vector(K, eqc_gen:oneof(L)), lists:usort(L2))). -sublist(L) -> - ?LET(K, nat(), - ?LET(L2, eqc_gen:vector(K, eqc_gen:oneof(L)), - lists:usort(L2))). - %% Generator for possibly assymmetric partition information partition(FLUNames) -> frequency([{10, return([])}, -- 2.45.2 From d44e9dd542852240dc7882025b7978b84863e52d Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 2 Dec 2015 15:54:34 +0900 Subject: [PATCH 10/77] Fix plumbing for find_file() --- src/machi_flu_filename_mgr.erl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/machi_flu_filename_mgr.erl b/src/machi_flu_filename_mgr.erl index 4710a68..12fa364 100644 --- a/src/machi_flu_filename_mgr.erl +++ b/src/machi_flu_filename_mgr.erl @@ -191,8 +191,10 @@ generate_uuid_v4_str() -> io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b", [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]). -find_file(DataDir, Prefix, N) -> - {_Filename, Path} = machi_util:make_data_filename(DataDir, Prefix, "*", N), +find_file(DataDir, {coc,CoC_Namespace,CoC_Locator}=_CoC_NL, Prefix, N) -> + {_Filename, Path} = machi_util:make_data_filename(DataDir, + CoC_Namespace,CoC_Locator, + Prefix, "*", N), filelib:wildcard(Path). list_files(DataDir, Prefix) -> @@ -202,10 +204,9 @@ list_files(DataDir, Prefix) -> make_filename_mgr_name(FluName) when is_atom(FluName) -> list_to_atom(atom_to_list(FluName) ++ "_filename_mgr"). -handle_find_file(Tid, {coc,CoC_Namespace,CoC_Locator}=_CoC, Prefix, DataDir) -> -io:format(user, "\nCoC ~p\n", [_CoC]), +handle_find_file(Tid, {coc,CoC_Namespace,CoC_Locator}=CoC_NL, Prefix, DataDir) -> N = machi_util:read_max_filenum(DataDir, CoC_Namespace, CoC_Locator, Prefix), - {File, Cleanup} = case find_file(DataDir, Prefix, N) of + {File, Cleanup} = case find_file(DataDir, CoC_NL, Prefix, N) of [] -> {find_or_make_filename(Tid, DataDir, CoC_Namespace, CoC_Locator, Prefix, N), false}; [H] -> {H, true}; -- 2.45.2 From 37f33fae7b3a6d9c1024d84fdc815d329a4c169c Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 2 Dec 2015 15:57:25 +0900 Subject: [PATCH 11/77] Fix bad_arg errors in low level eunit tests ... all pass now, yay! --- test/machi_flu1_test.erl | 2 +- test/machi_proxy_flu1_client_test.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/machi_flu1_test.erl b/test/machi_flu1_test.erl index b8dded9..5491e2b 100644 --- a/test/machi_flu1_test.erl +++ b/test/machi_flu1_test.erl @@ -146,7 +146,7 @@ flu_smoke_test() -> Chunk2 = <<"yo yo">>, Len2 = byte_size(Chunk2), Off2 = ?MINIMUM_OFFSET + 77, - File2 = "smoke-whole-file^1^1", + File2 = "smoke-whole-file^^0^1^1", ok = ?FLU_C:write_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH, File2, Off2, Chunk2), {error, bad_arg} = ?FLU_C:write_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH, diff --git a/test/machi_proxy_flu1_client_test.erl b/test/machi_proxy_flu1_client_test.erl index 3adfad5..f505f55 100644 --- a/test/machi_proxy_flu1_client_test.erl +++ b/test/machi_proxy_flu1_client_test.erl @@ -72,7 +72,7 @@ api_smoke_test() -> {error, bad_checksum} = ?MUT:append_chunk(Prox1, FakeEpoch, Prefix, MyChunk_badcs), {error, bad_checksum} = ?MUT:write_chunk(Prox1, FakeEpoch, - <<"foo-file^1^1">>, 99832, + <<"foo-file^^0^1^1">>, 99832, MyChunk_badcs), %% Put kick_projection_reaction() in the middle of the test so -- 2.45.2 From 66de92490cfee93509b1e53cb7de69a4f7dc5e1b Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 3 Dec 2015 16:48:23 +0900 Subject: [PATCH 12/77] Introduce eleveldb, along with cuttlefish to avoid dependency confustion --- rebar.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rebar.config b/rebar.config index d6debc0..e7c8add 100644 --- a/rebar.config +++ b/rebar.config @@ -5,6 +5,8 @@ {edoc_opts, [{dir, "./edoc"}]}. {deps, [ + {cuttlefish, ".*", {git, "git://github.com/basho/cuttlefish.git", {branch, "develop"}}}, + {eleveldb, ".*", {git, "git://github.com/basho/eleveldb.git", {branch, "develop"}}}, {lager, ".*", {git, "git://github.com/basho/lager.git", {tag, "2.2.0"}}}, {protobuffs, "0.8.*", {git, "git://github.com/basho/erlang_protobuffs.git", {tag, "0.8.1p4"}}}, {riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {branch, "develop"}}}, -- 2.45.2 From 35c48300a5a48a5d10fd0fbc7d569b380497bd92 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 4 Dec 2015 15:21:44 +0900 Subject: [PATCH 13/77] Fix Dialyzer complaints, derp! --- src/machi_dt.erl | 2 ++ src/machi_flu_filename_mgr.erl | 18 +++++++++--------- src/machi_flu_metadata_mgr.erl | 7 +++++-- src/machi_util.erl | 26 ++++++++++++++------------ 4 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/machi_dt.erl b/src/machi_dt.erl index 6df3eab..410a982 100644 --- a/src/machi_dt.erl +++ b/src/machi_dt.erl @@ -30,6 +30,7 @@ -type chunk_pos() :: {file_offset(), chunk_size(), file_name_s()}. -type chunk_size() :: non_neg_integer(). -type coc_namespace() :: string(). +-type coc_nl() :: {coc, coc_namespace(), coc_locator()}. -type coc_locator() :: non_neg_integer(). -type error_general() :: 'bad_arg' | 'wedged' | 'bad_checksum'. -type epoch_csum() :: binary(). @@ -61,6 +62,7 @@ chunk_pos/0, chunk_size/0, coc_namespace/0, + coc_nl/0, coc_locator/0, error_general/0, epoch_csum/0, diff --git a/src/machi_flu_filename_mgr.erl b/src/machi_flu_filename_mgr.erl index 12fa364..45e580e 100644 --- a/src/machi_flu_filename_mgr.erl +++ b/src/machi_flu_filename_mgr.erl @@ -52,7 +52,7 @@ child_spec/2, start_link/2, find_or_make_filename_from_prefix/4, - increment_prefix_sequence/2, + increment_prefix_sequence/3, list_files_by_prefix/2 ]). @@ -90,7 +90,7 @@ start_link(FluName, DataDir) when is_atom(FluName) andalso is_list(DataDir) -> -spec find_or_make_filename_from_prefix( FluName :: atom(), EpochId :: pv1_epoch_n(), Prefix :: {prefix, string()}, - {coc, riak_dt:coc_namespace(), riak_dt:coc_locator()}) -> + machi_dt:coc_nl()) -> {file, Filename :: string()} | {error, Reason :: term() } | timeout. % @doc Find the latest available or make a filename from a prefix. A prefix % should be in the form of a tagged tuple `{prefix, P}'. Returns a tagged @@ -105,13 +105,13 @@ find_or_make_filename_from_prefix(_FluName, _EpochId, Other, Other2) -> lager:error("~p is not a valid prefix/CoC ~p", [Other, Other2]), error(badarg). --spec increment_prefix_sequence( FluName :: atom(), Prefix :: {prefix, string()} ) -> +-spec increment_prefix_sequence( FluName :: atom(), CoC_NL :: machi_dt:coc_nl(), Prefix :: {prefix, string()} ) -> ok | {error, Reason :: term() } | timeout. % @doc Increment the sequence counter for a given prefix. Prefix should % be in the form of `{prefix, P}'. -increment_prefix_sequence(FluName, {prefix, Prefix}) when is_atom(FluName) -> - gen_server:call(make_filename_mgr_name(FluName), {increment_sequence, Prefix}, ?TIMEOUT); -increment_prefix_sequence(_FluName, Other) -> +increment_prefix_sequence(FluName, {coc,_CoC_Namespace,_CoC_Locator}=CoC_NL, {prefix, Prefix}) when is_atom(FluName) -> + gen_server:call(make_filename_mgr_name(FluName), {increment_sequence, CoC_NL, Prefix}, ?TIMEOUT); +increment_prefix_sequence(_FluName, _CoC_NL, Other) -> lager:error("~p is not a valid prefix.", [Other]), error(badarg). @@ -157,8 +157,8 @@ handle_call({find_filename, EpochId, CoC_NL, Prefix}, _From, S = #state{ datadir File = increment_and_cache_filename(Tid, DataDir, CoC_NL, Prefix), {reply, {file, File}, S#state{epoch = EpochId}}; -handle_call({increment_sequence, Prefix}, _From, S = #state{ datadir = DataDir }) -> - ok = machi_util:increment_max_filenum(DataDir, Prefix), +handle_call({increment_sequence, {coc,CoC_Namespace,CoC_Locator}=_CoC_NL, Prefix}, _From, S = #state{ datadir = DataDir }) -> + ok = machi_util:increment_max_filenum(DataDir, CoC_Namespace,CoC_Locator, Prefix), {reply, ok, S}; handle_call({list_files, Prefix}, From, S = #state{ datadir = DataDir }) -> spawn(fun() -> @@ -198,7 +198,7 @@ find_file(DataDir, {coc,CoC_Namespace,CoC_Locator}=_CoC_NL, Prefix, N) -> filelib:wildcard(Path). list_files(DataDir, Prefix) -> - {F_bin, Path} = machi_util:make_data_filename(DataDir, Prefix, "*", "*"), + {F_bin, Path} = machi_util:make_data_filename(DataDir, "*^" ++ Prefix ++ "^*"), filelib:wildcard(binary_to_list(F_bin), filename:dirname(Path)). make_filename_mgr_name(FluName) when is_atom(FluName) -> diff --git a/src/machi_flu_metadata_mgr.erl b/src/machi_flu_metadata_mgr.erl index d4447ae..66274b3 100644 --- a/src/machi_flu_metadata_mgr.erl +++ b/src/machi_flu_metadata_mgr.erl @@ -185,14 +185,17 @@ handle_info({'DOWN', Mref, process, Pid, file_rollover}, State = #state{ fluname tid = Tid }) -> lager:info("file proxy ~p shutdown because of file rollover", [Pid]), R = get_md_record_by_mref(Tid, Mref), - [Prefix | _Rest] = machi_util:parse_filename(R#md.filename), + {Prefix, CoC_Namespace, CoC_Locator, _, _} = + machi_util:parse_filename(R#md.filename), + %% CoC_Namespace = list_to_binary(CoC_Namespace_str), + %% CoC_Locator = list_to_integer(CoC_Locator_str), %% We only increment the counter here. The filename will be generated on the %% next append request to that prefix and since the filename will have a new %% sequence number it probably will be associated with a different metadata %% manager. That's why we don't want to generate a new file name immediately %% and use it to start a new file proxy. - ok = machi_flu_filename_mgr:increment_prefix_sequence(FluName, {prefix, Prefix}), + ok = machi_flu_filename_mgr:increment_prefix_sequence(FluName, {coc, CoC_Namespace, CoC_Locator}, {prefix, Prefix}), %% purge our ets table of this entry completely since it is likely the %% new filename (whenever it comes) will be in a different manager than diff --git a/src/machi_util.erl b/src/machi_util.erl index baba0f0..aa5f070 100644 --- a/src/machi_util.erl +++ b/src/machi_util.erl @@ -68,7 +68,7 @@ make_regname(Prefix) when is_list(Prefix) -> %% @doc Calculate a config file path, by common convention. --spec make_config_filename(string(), riak_dt:coc_namespace(), riak_dt:coc_locator(), string()) -> +-spec make_config_filename(string(), machi_dt:coc_namespace(), machi_dt:coc_locator(), string()) -> string(). make_config_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix) -> Locator_str = int_to_hexstr(CoC_Locator, 32), @@ -102,7 +102,7 @@ make_checksum_filename(DataDir, FileName) -> %% @doc Calculate a file data file path, by common convention. --spec make_data_filename(string(), riak_dt:coc_namespace(), riak_dt:coc_locator(), string(), atom()|string()|binary(), integer()|string()) -> +-spec make_data_filename(string(), machi_dt:coc_namespace(), machi_dt:coc_locator(), string(), atom()|string()|binary(), integer()|string()) -> {binary(), string()}. make_data_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix, SequencerName, FileNum) when is_integer(FileNum) -> @@ -146,40 +146,42 @@ make_projection_filename(DataDir, File) -> -spec is_valid_filename( Filename :: string() ) -> true | false. is_valid_filename(Filename) -> case parse_filename(Filename) of - [] -> false; - _ -> true + {} -> false; + {_,_,_,_,_} -> true end. %% @doc Given a machi filename, return a set of components in a list. %% The components will be: %%
    %%
  • Prefix
    • +%%
  • CoC Namespace
    • +%%
  • CoC locator
    • %%
  • UUID
    • %%
  • Sequence number
    • %%
%% %% Invalid filenames will return an empty list. --spec parse_filename( Filename :: string() ) -> [ string() ]. +-spec parse_filename( Filename :: string() ) -> {} | {string(), machi_dt:coc_namespace(), machi_dt:coc_locator(), string(), string() }. parse_filename(Filename) -> case string:tokens(Filename, "^") of - [_Prefix, _CoC_NS, _CoC_Loc, _UUID, _SeqNo] = L -> - L; + [Prefix, CoC_NS, CoC_Loc, UUID, SeqNo] -> + {Prefix, CoC_NS, list_to_integer(CoC_Loc), UUID, SeqNo}; [Prefix, CoC_Loc, UUID, SeqNo] -> %% string:tokens() doesn't consider "foo^^bar" as 3 tokens {sigh} case re:replace(Filename, "[^^]+", "x", [global,{return,binary}]) of <<"x^^x^x^x">> -> - [Prefix, "", CoC_Loc, UUID, SeqNo]; + {Prefix, <<"">>, list_to_integer(CoC_Loc), UUID, SeqNo}; _ -> - [] + {} end; - _ -> [] + _ -> {} end. %% @doc Read the file size of a config file, which is used as the %% basis for a minimum sequence number. --spec read_max_filenum(string(), riak_dt:coc_namespace(), riak_dt:coc_locator(), string()) -> +-spec read_max_filenum(string(), machi_dt:coc_namespace(), machi_dt:coc_locator(), string()) -> non_neg_integer(). read_max_filenum(DataDir, CoC_Namespace, CoC_Locator, Prefix) -> case file:read_file_info(make_config_filename(DataDir, CoC_Namespace, CoC_Locator, Prefix)) of @@ -192,7 +194,7 @@ read_max_filenum(DataDir, CoC_Namespace, CoC_Locator, Prefix) -> %% @doc Increase the file size of a config file, which is used as the %% basis for a minimum sequence number. --spec increment_max_filenum(string(), riak_dt:coc_namespace(), riak_dt:coc_locator(), string()) -> +-spec increment_max_filenum(string(), machi_dt:coc_namespace(), machi_dt:coc_locator(), string()) -> ok | {error, term()}. increment_max_filenum(DataDir, CoC_Namespace, CoC_Locator, Prefix) -> try -- 2.45.2 From cf0829b934f58da2faeea8372cfa6b797fc34079 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 4 Dec 2015 16:37:05 +0900 Subject: [PATCH 14/77] Add rc.d style config dir for FLU server startup --- rel/files/app.config | 6 ++++ rel/reltool.config | 2 ++ src/machi_flu_psup.erl | 2 ++ src/machi_flu_sup.erl | 60 +++++++++++++++++++++++++++++++++++- test/machi_flu_psup_test.erl | 13 ++++++++ 5 files changed, 82 insertions(+), 1 deletion(-) diff --git a/rel/files/app.config b/rel/files/app.config index c65c526..ac3a6a8 100644 --- a/rel/files/app.config +++ b/rel/files/app.config @@ -3,6 +3,12 @@ %% Data directory for all FLUs. {flu_data_dir, "{{platform_data_dir}}"}, + %% FLU config directory + {flu_config_dir, "{{platform_etc_dir}}/flu-config"}, + + %% Chain config directory + {chain_config_dir, "{{platform_etc_dir}}/chain-config"}, + %% FLUs to start at app start. {initial_flus, [ %% Remember, this is a list, so separate all tuples diff --git a/rel/reltool.config b/rel/reltool.config index 1385ca1..ca68025 100644 --- a/rel/reltool.config +++ b/rel/reltool.config @@ -93,6 +93,8 @@ {template, "files/vm.args", "etc/vm.args"}, {template, "files/app.config", "etc/app.config"}, + {mkdir, "etc/chain-config"}, + {mkdir, "etc/flu-config"}, {mkdir, "lib/basho-patches"} %% {copy, "../apps/machi/ebin/etop_txt.beam", "lib/basho-patches"} ]}. diff --git a/src/machi_flu_psup.erl b/src/machi_flu_psup.erl index 9e568cd..d83d559 100644 --- a/src/machi_flu_psup.erl +++ b/src/machi_flu_psup.erl @@ -83,6 +83,8 @@ %% Supervisor callbacks -export([init/1]). +make_package_spec(#p_srvr{name=FluName, port=TcpPort, props=Props}) when is_list(Props) -> + make_package_spec({FluName, TcpPort, Props}); make_package_spec({FluName, TcpPort, Props}) when is_list(Props) -> FluDataDir = get_env(flu_data_dir, undefined_is_invalid), MyDataDir = filename:join(FluDataDir, atom_to_list(FluName)), diff --git a/src/machi_flu_sup.erl b/src/machi_flu_sup.erl index 74995bb..fe72499 100644 --- a/src/machi_flu_sup.erl +++ b/src/machi_flu_sup.erl @@ -29,8 +29,11 @@ -behaviour(supervisor). -include("machi.hrl"). +-include("machi_projection.hrl"). -include("machi_verbose.hrl"). +-ifdef(TEST). +-compile(export_all). -ifdef(PULSE). -compile({parse_transform, pulse_instrument}). -include_lib("pulse_otp/include/pulse_otp.hrl"). @@ -38,6 +41,7 @@ -else. -define(SHUTDOWN, 5000). -endif. +-endif. %TEST %% API -export([start_link/0]). @@ -69,5 +73,59 @@ get_initial_flus() -> []. -else. % PULSE get_initial_flus() -> - application:get_env(machi, initial_flus, []). + DoesNotExist = "/tmp/does/not/exist", + ConfigDir = case application:get_env(machi, flu_config_dir, DoesNotExist) of + DoesNotExist -> + DoesNotExist; + Dir -> + ok = filelib:ensure_dir(Dir ++ "/unused"), + Dir + end, + sanitize_p_srvr_records(load_rc_d_files_from_dir(ConfigDir)). -endif. % PULSE + +load_rc_d_files_from_dir(Dir) -> + Files = filelib:wildcard(Dir ++ "/*"), + lists:append([case file:consult(File) of + {ok, X} -> X; + _ -> [] + end || File <- Files]). + +sanitize_p_srvr_records(Ps) -> + {Sane, _} = lists:foldl(fun sanitize_p_srvr_rec/2, {[], dict:new()}, Ps), + Sane. + +sanitize_p_srvr_rec(Whole, {Acc, D}) -> + try + #p_srvr{name=Name, + proto_mod=PMod, + address=Address, + port=Port, + props=Props} = Whole, + true = is_atom(Name), + NameK = {name, Name}, + error = dict:find(NameK, D), + true = is_atom(PMod), + case code:is_loaded(PMod) of + {file, _} -> ok; + _ -> {module, _} = code:load_file(PMod), ok + end, + if is_list(Address) -> ok; + is_tuple(Address) -> ok % Erlang-style IPv4 or IPv6 + end, + true = is_integer(Port) andalso Port >= 1024 andalso Port =< 65534, + PortK = {port, Port}, + error = dict:find(PortK, D), + true = is_list(Props), + + %% All is sane enough. + D2 = dict:store(NameK, Name, + dict:store(PortK, Port, D)), + {[Whole|Acc], D2} + catch _:_ -> + _ = lager:log(error, self(), + "~s: Bad (or duplicate name/port) p_srvr record, " + "skipping: ~P\n", + [?MODULE, Whole, 15]), + {Acc, D} + end. diff --git a/test/machi_flu_psup_test.erl b/test/machi_flu_psup_test.erl index 1c7b015..fd93b42 100644 --- a/test/machi_flu_psup_test.erl +++ b/test/machi_flu_psup_test.erl @@ -173,6 +173,19 @@ partial_stop_restart2() -> ok end. +p_srvr_rec_test() -> + P = #p_srvr{name=a, address="localhost", port=1024, props=[yo]}, + [P] = machi_flu_sup:sanitize_p_srvr_records([P]), + [P] = machi_flu_sup:sanitize_p_srvr_records([P,P]), + [] = machi_flu_sup:sanitize_p_srvr_records([nope]), + [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{proto_mod=does_not_exist}]), + [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{proto_mod="lists"}]), + [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{address=7}]), + [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{port=5}]), + [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{port=foo}]), + [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{props=foo}]), + ok. + -endif. % !PULSE -endif. % TEST -- 2.45.2 From 85285679540ba735ce8897da024ded1aba3631e7 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Thu, 3 Dec 2015 18:55:39 +0900 Subject: [PATCH 15/77] Add eleveldb with sext to use it as metadata storage First step is to use as checksum table. It will also used for file names store and *ALL* other persistent metadata than files. --- rebar.config | 1 + src/machi_csum_table.erl | 356 +++++++++++++++------------------ test/machi_csum_table_test.erl | 34 ++-- 3 files changed, 177 insertions(+), 214 deletions(-) diff --git a/rebar.config b/rebar.config index e7c8add..7bd81ce 100644 --- a/rebar.config +++ b/rebar.config @@ -6,6 +6,7 @@ {deps, [ {cuttlefish, ".*", {git, "git://github.com/basho/cuttlefish.git", {branch, "develop"}}}, + {sext, ".*", {git, "git://github.com/basho/sext.git", {branch, "master"}}}, {eleveldb, ".*", {git, "git://github.com/basho/eleveldb.git", {branch, "develop"}}}, {lager, ".*", {git, "git://github.com/basho/lager.git", {tag, "2.2.0"}}}, {protobuffs, "0.8.*", {git, "git://github.com/basho/erlang_protobuffs.git", {tag, "0.8.1p4"}}}, diff --git a/src/machi_csum_table.erl b/src/machi_csum_table.erl index 6e5e2b8..e9e80f7 100644 --- a/src/machi_csum_table.erl +++ b/src/machi_csum_table.erl @@ -2,19 +2,15 @@ -export([open/2, find/3, - write/6, write/4, trim/5, trim/3, + write/6, write/4, trim/5, find_leftneighbor/2, find_rightneighbor/2, all_trimmed/3, any_trimmed/3, all_trimmed/2, - sync/1, calc_unwritten_bytes/1, split_checksum_list_blob_decode/1, close/1, delete/1, foldl_chunks/3]). --export([encode_csum_file_entry/3, encode_csum_file_entry_bin/3, - decode_csum_file_entry/1]). - -include("machi.hrl"). -ifdef(TEST). @@ -24,8 +20,7 @@ -record(machi_csum_table, {file :: string(), - fd :: file:io_device(), - table :: ets:tid()}). + table :: eleveldb:db_ref()}). -type table() :: #machi_csum_table{}. -type byte_sequence() :: { Offset :: non_neg_integer(), @@ -37,96 +32,93 @@ {ok, table()} | {error, file:posix()}. open(CSumFilename, _Opts) -> - T = ets:new(?MODULE, [private, ordered_set]), - CSum = machi_util:make_tagged_csum(none), - %% Dummy entry for headers - true = ets:insert_new(T, {0, ?MINIMUM_OFFSET, CSum}), + LevelDBOptions = [{create_if_missing, true}, + %% Keep this table small so as not to interfere + %% operating system's file cache, which is for + %% Machi's main read efficiency + {total_leveldb_mem_percent, 10}], + {ok, T} = eleveldb:open(CSumFilename, LevelDBOptions), + %% Dummy entry for reserved headers + ok = eleveldb:put(T, + sext:encode({0, ?MINIMUM_OFFSET}), + sext:encode(?CSUM_TAG_NONE_ATOM), + [{sync, true}]), C0 = #machi_csum_table{ file=CSumFilename, table=T}, - case file:read_file(CSumFilename) of - {ok, Bin} -> - List = case split_checksum_list_blob_decode(Bin) of - {List0, <<>>} -> - List0; - {List0, _Junk} -> - %% Partially written, needs repair TODO - List0 - end, - %% assuming all entries are strictly ordered by offset, - %% trim command should always come after checksum entry. - %% *if* by any chance that order cound not be kept, we - %% still can do ordering check and monotonic merge here. - %% TODO: make some random injection tests? - [begin %% Replay all file contents, Same logic as in write/6 - Chunks = find(C0, Offset, Size), - lists:foreach(fun({O, _, _}) -> - ets:delete(T, O) - end, Chunks), - true = ets:insert(T, {Offset, Size, CsumOrTrimmed}) - end - || {Offset, Size, CsumOrTrimmed} <- List], - ok; - {error, enoent} -> - ok; - Error -> - throw(Error) - end, - {ok, Fd} = file:open(CSumFilename, [raw, binary, append]), - {ok, C0#machi_csum_table{fd=Fd}}. + {ok, C0}. + +-spec split_checksum_list_blob_decode(binary())-> term(). +split_checksum_list_blob_decode(_Bin) -> + %% binary_to_term(Bin) + throw(not_yet). -spec find(table(), machi_dt:file_offset(), machi_dt:file_size()) -> list({machi_dt:file_offset(), machi_dt:file_size(), machi_dt:chunk_csum()|trimmed}). find(#machi_csum_table{table=T}, Offset, Size) -> - ets:select(T, [{{'$1', '$2', '$3'}, - [inclusion_match_spec(Offset, Size)], - ['$_']}]). + {ok, I} = eleveldb:iterator(T, [], keys_only), + EndKey = sext:encode({Offset+Size, 0}), + StartKey = sext:encode({Offset, Size}), --ifdef(TEST). -all(#machi_csum_table{table=T}) -> - ets:tab2list(T). --endif. + {ok, FirstKey} = case eleveldb:iterator_move(I, StartKey) of + {error, invalid_iterator} -> + eleveldb:iterator_move(I, first); + {ok, _} = R0 -> + case eleveldb:iterator_move(I, prev) of + {error, invalid_iterator} -> + R0; + {ok, _} = R1 -> + R1 + end + end, + _ = eleveldb:iterator_close(I), + FoldFun = fun({K, V}, Acc) -> + {TargetOffset, TargetSize} = sext:decode(K), + case has_overlap(TargetOffset, TargetSize, Offset, Size) of + true -> + [{TargetOffset, TargetSize, sext:decode(V)}|Acc]; + false -> + Acc + end; + (_KV, _) -> + throw(_KV) + end, + lists:reverse(eleveldb_fold(T, FirstKey, EndKey, FoldFun, [])). -write(#machi_csum_table{fd=Fd, table=T} = CsumT, - Offset, Size, CSum, +has_overlap(LeftOffset, LeftSize, RightOffset, RightSize) -> + (LeftOffset - (RightOffset+RightSize)) * (LeftOffset+LeftSize - RightOffset) < 0. + +%% @doc Updates all chunk info, by deleting existing entries if exists +%% and putting new chunk info +write(#machi_csum_table{table=T} = CsumT, Offset, Size, CSum, LeftUpdate, RightUpdate) -> - Binary = - [encode_csum_file_entry_bin(Offset, Size, CSum), - case LeftUpdate of - {LO, LS, LCsum} when LO + LS =:= Offset -> - encode_csum_file_entry_bin(LO, LS, LCsum); - undefined -> - <<>> - end, - case RightUpdate of - {RO, RS, RCsum} when RO =:= Offset + Size -> - encode_csum_file_entry_bin(RO, RS, RCsum); - undefined -> - <<>> - end], - case file:write(Fd, Binary) of - ok -> - Chunks = find(CsumT, Offset, Size), - lists:foreach(fun({O, _, _}) -> - ets:delete(T, O) + PutOps = + [{put, + sext:encode({Offset, Size}), + sext:encode(CSum)}] + ++ case LeftUpdate of + {LO, LS, LCsum} when LO + LS =:= Offset -> + [{put, + sext:encode({LO, LS}), + sext:encode(LCsum)}]; + undefined -> + [] + end + ++ case RightUpdate of + {RO, RS, RCsum} when RO =:= Offset + Size -> + [{put, + sext:encode({RO, RS}), + sext:encode({RCsum})}]; + undefined -> + [] + end, + Chunks = find(CsumT, Offset, Size), + DeleteOps = lists:map(fun({O, L, _}) -> + {delete, sext:encode({O, L})} end, Chunks), - case LeftUpdate of - {LO1, LS1, _} when LO1 + LS1 =:= Offset -> - ets:insert(T, LeftUpdate); - undefined -> noop - end, - case RightUpdate of - {RO1, _, _} when RO1 =:= Offset + Size -> - ets:insert(T, RightUpdate); - undefined -> noop - end, - true = ets:insert(T, {Offset, Size, CSum}), - ok; - Error -> - Error - end. + eleveldb:write(T, DeleteOps ++ PutOps, [{sync, true}]). -spec find_leftneighbor(table(), non_neg_integer()) -> undefined | @@ -156,33 +148,38 @@ write(CsumT, Offset, Size, CSum) -> write(CsumT, Offset, Size, CSum, undefined, undefined). trim(CsumT, Offset, Size, LeftUpdate, RightUpdate) -> - write(CsumT, Offset, Size, trimmed, LeftUpdate, RightUpdate). - --spec trim(table(), machi_dt:file_offset(), machi_dt:file_size()) -> - ok | {error, file:posix()}. -trim(#machi_csum_table{fd=Fd, table=T}, Offset, Size) -> - Binary = encode_csum_file_entry_bin(Offset, Size, trimmed), - case file:write(Fd, Binary) of - ok -> - true = ets:insert(T, {Offset, Size, trimmed}), - ok; - Error -> - Error - end. + write(CsumT, Offset, Size, + trimmed, %% Should this be much smaller like $t or just 't' + LeftUpdate, RightUpdate). +%% @doc returns whether all bytes in a specific window is continously +%% trimmed or not -spec all_trimmed(table(), non_neg_integer(), non_neg_integer()) -> boolean(). -all_trimmed(#machi_csum_table{table=T}, Left, Right) -> - runthru(ets:tab2list(T), Left, Right). +all_trimmed(CsumT, Left, Right) -> + Chunks = find(CsumT, Left, Right), + runthru(Chunks, Left, Right). +%% @doc returns whether all bytes 0-Pos0 is continously trimmed or +%% not, including header. -spec all_trimmed(table(), non_neg_integer()) -> boolean(). -all_trimmed(#machi_csum_table{table=T}, Pos) -> - case ets:tab2list(T) of - [{0, ?MINIMUM_OFFSET, _}|L] -> - %% tl/1 to remove header space {0, 1024, <<0>>} - runthru(L, ?MINIMUM_OFFSET, Pos); - List -> - %% In case a header is removed; - runthru(List, 0, Pos) +all_trimmed(#machi_csum_table{table=T}, Pos0) -> + FoldFun = fun({_, _}, false) -> + false; + ({K, V}, Pos) when is_integer(Pos) andalso Pos =< Pos0 -> + case {sext:decode(K), sext:decode(V)} of + {{Pos, Size}, trimmed} -> + Pos + Size; + _Eh -> + %% ?debugVal({_Eh, Pos}), + false + end + end, + case eleveldb:fold(T, FoldFun, 0, [{verify_checksums, true}]) of + false -> false; + Pos0 -> true; + LastTrimmed when LastTrimmed < Pos0 -> false; + _ -> %% LastTrimmed > Pos0, which is a irregular case but ok + true end. -spec any_trimmed(table(), @@ -192,13 +189,9 @@ any_trimmed(CsumT, Offset, Size) -> Chunks = find(CsumT, Offset, Size), lists:any(fun({_, _, State}) -> State =:= trimmed end, Chunks). --spec sync(table()) -> ok | {error, file:posix()}. -sync(#machi_csum_table{fd=Fd}) -> - file:sync(Fd). - -spec calc_unwritten_bytes(table()) -> [byte_sequence()]. -calc_unwritten_bytes(#machi_csum_table{table=T}) -> - case lists:sort(ets:tab2list(T)) of +calc_unwritten_bytes(#machi_csum_table{table=_} = CsumT) -> + case lists:sort(all(CsumT)) of [] -> [{?MINIMUM_OFFSET, infinity}]; Sorted -> @@ -206,17 +199,22 @@ calc_unwritten_bytes(#machi_csum_table{table=T}) -> build_unwritten_bytes_list(Sorted, LastOffset, []) end. +all(CsumT) -> + FoldFun = fun(E, Acc) -> [E|Acc] end, + lists:reverse(foldl_chunks(FoldFun, [], CsumT)). + -spec close(table()) -> ok. -close(#machi_csum_table{table=T, fd=Fd}) -> - true = ets:delete(T), - ok = file:close(Fd). +close(#machi_csum_table{table=T}) -> + ok = eleveldb:close(T). -spec delete(table()) -> ok. -delete(#machi_csum_table{file=F} = C) -> - catch close(C), - case file:delete(F) of +delete(#machi_csum_table{table=T, file=F}) -> + catch eleveldb:close(T), + %% TODO change this to directory walk + case os:cmd("rm -rf " ++ F) of ok -> ok; {error, enoent} -> ok; + "" -> ok; E -> E end. @@ -225,82 +223,11 @@ delete(#machi_csum_table{file=F} = C) -> -> Acc :: term()), Acc0 :: term(), table()) -> Acc :: term(). foldl_chunks(Fun, Acc0, #machi_csum_table{table=T}) -> - ets:foldl(Fun, Acc0, T). - -%% @doc Encode `Offset + Size + TaggedCSum' into an `iolist()' type for -%% internal storage by the FLU. - --spec encode_csum_file_entry( - machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()) -> - iolist(). -encode_csum_file_entry(Offset, Size, TaggedCSum) -> - Len = 8 + 4 + byte_size(TaggedCSum), - [<<$w, Len:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big>>, - TaggedCSum]. - -%% @doc Encode `Offset + Size + TaggedCSum' into an `binary()' type for -%% internal storage by the FLU. - --spec encode_csum_file_entry_bin( - machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()) -> - binary(). -encode_csum_file_entry_bin(Offset, Size, trimmed) -> - <<$t, Offset:64/unsigned-big, Size:32/unsigned-big>>; -encode_csum_file_entry_bin(Offset, Size, TaggedCSum) -> - Len = 8 + 4 + byte_size(TaggedCSum), - <<$w, Len:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big, - TaggedCSum/binary>>. - -%% @doc Decode a single `binary()' blob into an -%% `{Offset,Size,TaggedCSum}' tuple. -%% -%% The internal encoding (which is currently exposed to the outside world -%% via this function and related ones) is: -%% -%%
    -%%
  • 1 byte: record length -%%
    • -%%
  • 8 bytes (unsigned big-endian): byte offset -%%
    • -%%
  • 4 bytes (unsigned big-endian): chunk size -%%
    • -%%
  • all remaining bytes: tagged checksum (1st byte = type tag) -%%
    • -%%
-%% -%% See `machi.hrl' for the tagged checksum types, e.g., -%% `?CSUM_TAG_NONE'. - --spec decode_csum_file_entry(binary()) -> - error | - {machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()}. -decode_csum_file_entry(<<_:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big, TaggedCSum/binary>>) -> - {Offset, Size, TaggedCSum}; -decode_csum_file_entry(_Else) -> - error. - -%% @doc Split a `binary()' blob of `checksum_list' data into a list of -%% `{Offset,Size,TaggedCSum}' tuples. - --spec split_checksum_list_blob_decode(binary()) -> - {list({machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()}), - TrailingJunk::binary()}. -split_checksum_list_blob_decode(Bin) -> - split_checksum_list_blob_decode(Bin, []). - -split_checksum_list_blob_decode(<<$w, Len:8/unsigned-big, Part:Len/binary, Rest/binary>>, Acc)-> - One = <>, - case decode_csum_file_entry(One) of - error -> - split_checksum_list_blob_decode(Rest, Acc); - DecOne -> - split_checksum_list_blob_decode(Rest, [DecOne|Acc]) - end; -split_checksum_list_blob_decode(<<$t, Offset:64/unsigned-big, Size:32/unsigned-big, Rest/binary>>, Acc) -> - %% trimmed offset - split_checksum_list_blob_decode(Rest, [{Offset, Size, trimmed}|Acc]); -split_checksum_list_blob_decode(Rest, Acc) -> - {lists:reverse(Acc), Rest}. + FoldFun = fun({K, V}, Acc) -> + {Offset, Len} = sext:decode(K), + Fun({Offset, Len, sext:decode(V)}, Acc) + end, + eleveldb:fold(T, FoldFun, Acc0, [{verify_checksums, true}]). -spec build_unwritten_bytes_list( CsumData :: [{ Offset :: non_neg_integer(), Size :: pos_integer(), @@ -335,8 +262,37 @@ runthru(_L, _O, _P) -> %% b] where x < y and a < b; if (a-y)*(b-x) < 0 then there's a %% overlap, else, > 0 then there're no overlap. border condition = 0 %% is not overlap in this offset-size case. -inclusion_match_spec(Offset, Size) -> - {'>', 0, - {'*', - {'-', Offset + Size, '$1'}, - {'-', Offset, {'+', '$1', '$2'}}}}. +%% inclusion_match_spec(Offset, Size) -> +%% {'>', 0, +%% {'*', +%% {'-', Offset + Size, '$1'}, +%% {'-', Offset, {'+', '$1', '$2'}}}}. + + +eleveldb_fold(Ref, Start, End, FoldFun, InitAcc) -> + {ok, Iterator} = eleveldb:iterator(Ref, []), + try + eleveldb_do_fold(eleveldb:iterator_move(Iterator, Start), + Iterator, End, FoldFun, InitAcc) + catch throw:IteratorClosed -> + {error, IteratorClosed} + after + eleveldb:iterator_close(Iterator) + end. + +eleveldb_do_fold({ok, Key, Value}, _, End, FoldFun, Acc) + when End < Key -> + FoldFun({Key, Value}, Acc); +eleveldb_do_fold({ok, Key, Value}, Iterator, End, FoldFun, Acc) -> + eleveldb_do_fold(eleveldb:iterator_move(Iterator, next), + Iterator, End, FoldFun, + FoldFun({Key, Value}, Acc)); +eleveldb_do_fold({error, iterator_closed}, _, _, _, Acc) -> + %% It's really an error which is not expected + throw({iterator_closed, Acc}); +eleveldb_do_fold({error, invalid_iterator}, _, _, _, Acc) -> + %% Probably reached to end + Acc. + + + diff --git a/test/machi_csum_table_test.erl b/test/machi_csum_table_test.erl index c168d45..4059de4 100644 --- a/test/machi_csum_table_test.erl +++ b/test/machi_csum_table_test.erl @@ -2,26 +2,31 @@ -compile(export_all). -include_lib("eunit/include/eunit.hrl"). --define(HDR, {0, 1024, <<0>>}). +-define(HDR, {0, 1024, none}). + +cleanup(Dir) -> + os:cmd("rm -rf " ++ Dir). smoke_test() -> Filename = "./temp-checksum-dumb-file", - _ = file:delete(Filename), + _ = cleanup(Filename), {ok, MC} = machi_csum_table:open(Filename, []), - [{1024, infinity}] = machi_csum_table:calc_unwritten_bytes(MC), + ?assertEqual([{1024, infinity}], + machi_csum_table:calc_unwritten_bytes(MC)), Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>}, [] = machi_csum_table:find(MC, Offset, Size), ok = machi_csum_table:write(MC, Offset, Size, Checksum), [{1024, 40}, {1098, infinity}] = machi_csum_table:calc_unwritten_bytes(MC), - [Entry] = machi_csum_table:find(MC, Offset, Size), - ok = machi_csum_table:trim(MC, Offset, Size), - [{Offset, Size, trimmed}] = machi_csum_table:find(MC, Offset, Size), + ?assertEqual([Entry], machi_csum_table:find(MC, Offset, Size)), + ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined), + ?assertEqual([{Offset, Size, trimmed}], + machi_csum_table:find(MC, Offset, Size)), ok = machi_csum_table:close(MC), ok = machi_csum_table:delete(MC). close_test() -> Filename = "./temp-checksum-dumb-file-2", - _ = file:delete(Filename), + _ = cleanup(Filename), {ok, MC} = machi_csum_table:open(Filename, []), Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>}, [] = machi_csum_table:find(MC, Offset, Size), @@ -31,32 +36,33 @@ close_test() -> {ok, MC2} = machi_csum_table:open(Filename, []), [Entry] = machi_csum_table:find(MC2, Offset, Size), - ok = machi_csum_table:trim(MC2, Offset, Size), + ok = machi_csum_table:trim(MC2, Offset, Size, undefined, undefined), [{Offset, Size, trimmed}] = machi_csum_table:find(MC2, Offset, Size), ok = machi_csum_table:delete(MC2). smoke2_test() -> Filename = "./temp-checksum-dumb-file-3", - _ = file:delete(Filename), + _ = cleanup(Filename), {ok, MC} = machi_csum_table:open(Filename, []), Entry = {Offset, Size, Checksum} = {1025, 10, <<"deadbeef">>}, ok = machi_csum_table:write(MC, Offset, Size, Checksum), - [] = machi_csum_table:find(MC, 0, 0), - [?HDR] = machi_csum_table:find(MC, 0, 1), + ?assertEqual([], machi_csum_table:find(MC, 0, 0)), + ?assertEqual([?HDR], machi_csum_table:find(MC, 0, 1)), [Entry] = machi_csum_table:find(MC, Offset, Size), [?HDR] = machi_csum_table:find(MC, 1, 1024), - [?HDR, Entry] = machi_csum_table:find(MC, 1023, 1024), + ?assertEqual([?HDR, Entry], + machi_csum_table:find(MC, 1023, 1024)), [Entry] = machi_csum_table:find(MC, 1024, 1024), [Entry] = machi_csum_table:find(MC, 1025, 1024), - ok = machi_csum_table:trim(MC, Offset, Size), + ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined), [{Offset, Size, trimmed}] = machi_csum_table:find(MC, Offset, Size), ok = machi_csum_table:close(MC), ok = machi_csum_table:delete(MC). smoke3_test() -> Filename = "./temp-checksum-dumb-file-4", - _ = file:delete(Filename), + _ = cleanup(Filename), {ok, MC} = machi_csum_table:open(Filename, []), Scenario = [%% Command, {Offset, Size, Csum}, LeftNeighbor, RightNeibor -- 2.45.2 From a7ffef6b8e41427f58c594171bf64942e6fcd766 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 4 Dec 2015 17:18:15 +0900 Subject: [PATCH 16/77] Add src/machi_chain_bootstrap.erl --- src/machi_chain_bootstrap.erl | 76 +++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/machi_chain_bootstrap.erl diff --git a/src/machi_chain_bootstrap.erl b/src/machi_chain_bootstrap.erl new file mode 100644 index 0000000..55e6334 --- /dev/null +++ b/src/machi_chain_bootstrap.erl @@ -0,0 +1,76 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(machi_chain_bootstrap). + +-behaviour(gen_server). + +%% API +-export([start_link/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, {}). + +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). + +init([]) -> + self() ! finish_init, + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(finish_init, State) -> + FLUs = get_local_flu_names(), + FLU_Epochs = get_latest_public_epochs(FLUs), + FLUs_at_zero = [FLU || {FLU, 0} <- FLU_Epochs], + lager:info("FLUs at epoch 0: ~p\n", [FLUs_at_zero]), + {noreply, State}; +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%%% + +get_local_flu_names() -> + [Name || {Name,_,_,_} <- supervisor:which_children(machi_flu_sup)]. + +get_latest_public_epochs(FLUs) -> + [begin + PS = machi_flu1:make_projection_server_regname(FLU), + {ok, {Epoch, _CSum}} = machi_projection_store:get_latest_epochid( + PS, public), + {FLU, Epoch} + end || FLU <- FLUs]. -- 2.45.2 From befa776685e30404ea6fb5f59d892dca8ede1470 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 4 Dec 2015 17:16:18 +0900 Subject: [PATCH 17/77] Fix several new bugs --- src/machi_csum_table.erl | 2 +- src/machi_file_proxy.erl | 36 ++++++------- test/machi_file_proxy_eqc.erl | 2 +- test/machi_file_proxy_test.erl | 98 +++++++++++++++++++--------------- 4 files changed, 74 insertions(+), 64 deletions(-) diff --git a/src/machi_csum_table.erl b/src/machi_csum_table.erl index e9e80f7..0b7be8e 100644 --- a/src/machi_csum_table.erl +++ b/src/machi_csum_table.erl @@ -110,7 +110,7 @@ write(#machi_csum_table{table=T} = CsumT, Offset, Size, CSum, {RO, RS, RCsum} when RO =:= Offset + Size -> [{put, sext:encode({RO, RS}), - sext:encode({RCsum})}]; + sext:encode(RCsum)}]; undefined -> [] end, diff --git a/src/machi_file_proxy.erl b/src/machi_file_proxy.erl index ff6748f..980d5a7 100644 --- a/src/machi_file_proxy.erl +++ b/src/machi_file_proxy.erl @@ -249,27 +249,22 @@ handle_call({sync, data}, _From, State = #state{ data_filehandle = FHd }) -> R = file:sync(FHd), {reply, R, State}; -handle_call({sync, csum}, _From, State = #state{ csum_table = T }) -> - R = machi_csum_table:sync(T), - {reply, R, State}; +handle_call({sync, csum}, _From, State) -> + %% machi_csum_table always writes in {sync, true} option, so here + %% explicit sync isn't actually needed. + {reply, ok, State}; handle_call({sync, all}, _From, State = #state{filename = F, data_filehandle = FHd, - csum_table = T + csum_table = _T }) -> - R = machi_csum_table:sync(T), - R1 = file:sync(FHd), - Resp = case {R, R1} of - {ok, ok} -> ok; - {ok, O1} -> - lager:error("Got ~p during a data file sync on file ~p", [O1, F]), - O1; - {O2, ok} -> - lager:error("Got ~p during a csum file sync on file ~p", [O2, F]), - O2; - {O3, O4} -> - lager:error("Got ~p ~p syncing all files for file ~p", [O3, O4, F]), - {O3, O4} + Resp = case file:sync(FHd) of + ok -> + ok; + Error -> + lager:error("Got ~p syncing all files for file ~p", + [Error, F]), + Error end, {reply, Resp, State}; @@ -545,7 +540,6 @@ terminate(Reason, #state{filename = F, undefined -> noop; %% file deleted _ -> - ok = machi_csum_table:sync(T), ok = machi_csum_table:close(T) end, ok. @@ -633,7 +627,11 @@ read_all_ranges(FHd, Filename, [{Offset, Size, TaggedCsum}|T], ReadChunks, Trimm eof -> read_all_ranges(FHd, Filename, T, ReadChunks, TrimmedChunks); {ok, Bytes} when byte_size(Bytes) == Size -> - {Tag, Ck} = machi_util:unmake_tagged_csum(TaggedCsum), + {Tag, Ck} = case TaggedCsum of + none -> {?CSUM_TAG_NONE, ooops}; + _ -> + machi_util:unmake_tagged_csum(TaggedCsum) + end, case check_or_make_tagged_csum(Tag, Ck, Bytes) of {error, Bad} -> lager:error("Bad checksum; got ~p, expected ~p", diff --git a/test/machi_file_proxy_eqc.erl b/test/machi_file_proxy_eqc.erl index 00d470f..dd36787 100644 --- a/test/machi_file_proxy_eqc.erl +++ b/test/machi_file_proxy_eqc.erl @@ -38,7 +38,7 @@ eqc_test_() -> {timeout, 60, {spawn, [ - {timeout, 30, ?_assertEqual(true, eqc:quickcheck(eqc:testing_time(15, ?QC_OUT(prop_ok()))))} + ?_assertEqual(true, eqc:quickcheck(eqc:testing_time(30, ?QC_OUT(prop_ok())))) ] }}. diff --git a/test/machi_file_proxy_test.erl b/test/machi_file_proxy_test.erl index 8c4b60b..1bf6cde 100644 --- a/test/machi_file_proxy_test.erl +++ b/test/machi_file_proxy_test.erl @@ -76,54 +76,66 @@ random_binary(Start, End) -> binary:part(random_binary_single(), Start, End) end. +setup() -> + {ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR), + Pid. + +teardown(Pid) -> + catch machi_file_proxy:stop(Pid). + machi_file_proxy_test_() -> clean_up_data_dir(?TESTDIR), - {ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR), - [ - ?_assertEqual({error, bad_arg}, machi_file_proxy:read(Pid, -1, -1)), - ?_assertEqual({error, bad_arg}, machi_file_proxy:write(Pid, -1, <<"yo">>)), - ?_assertEqual({error, bad_arg}, machi_file_proxy:append(Pid, [], -1, <<"krep">>)), - ?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1)), - ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, 1)), - ?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1024)), - ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, ?HYOOGE)), - ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, ?HYOOGE, 1)), - ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, random_binary(0, ?HYOOGE))), - ?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))), - ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1024, <<"fail">>)), - ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, <<"fail">>)), - ?_assertMatch({ok, {[{_, _, _, _}], []}}, machi_file_proxy:read(Pid, 1025, 1000)), - ?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, [], 1024, <<"mind the gap">>)), - ?_assertEqual(ok, machi_file_proxy:write(Pid, 2060, [], random_binary(0, 1024))), - ?_assertException(exit, {normal, _}, machi_file_proxy:stop(Pid)) - ]. + {setup, + fun setup/0, + fun teardown/1, + fun(Pid) -> + [ + ?_assertEqual({error, bad_arg}, machi_file_proxy:read(Pid, -1, -1)), + ?_assertEqual({error, bad_arg}, machi_file_proxy:write(Pid, -1, <<"yo">>)), + ?_assertEqual({error, bad_arg}, machi_file_proxy:append(Pid, [], -1, <<"krep">>)), + ?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1)), + ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, 1)), + ?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1024)), + ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, ?HYOOGE)), + ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, ?HYOOGE, 1)), + ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, random_binary(0, ?HYOOGE))), + ?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))), + ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1024, <<"fail">>)), + ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, <<"fail">>)), + ?_assertMatch({ok, {[{_, _, _, _}], []}}, machi_file_proxy:read(Pid, 1025, 1000)), + ?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, [], 1024, <<"mind the gap">>)), + ?_assertEqual(ok, machi_file_proxy:write(Pid, 2060, [], random_binary(0, 1024))) + ] + end}. multiple_chunks_read_test_() -> clean_up_data_dir(?TESTDIR), - {ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR), - [ - ?_assertEqual(ok, machi_file_proxy:trim(Pid, 0, 1, false)), - ?_assertMatch({ok, {[], [{"test", 0, 1}]}}, - machi_file_proxy:read(Pid, 0, 1, - [{needs_trimmed, true}])), - ?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))), - ?_assertEqual(ok, machi_file_proxy:write(Pid, 10000, <<"fail">>)), - ?_assertEqual(ok, machi_file_proxy:write(Pid, 20000, <<"fail">>)), - ?_assertEqual(ok, machi_file_proxy:write(Pid, 30000, <<"fail">>)), - %% Freeza - ?_assertEqual(ok, machi_file_proxy:write(Pid, 530000, <<"fail">>)), - ?_assertMatch({ok, {[{"test", 1024, _, _}, - {"test", 10000, <<"fail">>, _}, - {"test", 20000, <<"fail">>, _}, - {"test", 30000, <<"fail">>, _}, - {"test", 530000, <<"fail">>, _}], []}}, - machi_file_proxy:read(Pid, 1024, 530000)), - ?_assertMatch({ok, {[{"test", 1, _, _}], [{"test", 0, 1}]}}, - machi_file_proxy:read(Pid, 0, 1024, - [{needs_trimmed, true}])), - ?_assertException(exit, {normal, _}, machi_file_proxy:stop(Pid)) - ]. - + {setup, + fun setup/0, + fun teardown/1, + fun(Pid) -> + [ + ?_assertEqual(ok, machi_file_proxy:trim(Pid, 0, 1, false)), + ?_assertMatch({ok, {[], [{"test", 0, 1}]}}, + machi_file_proxy:read(Pid, 0, 1, + [{needs_trimmed, true}])), + ?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))), + ?_assertEqual(ok, machi_file_proxy:write(Pid, 10000, <<"fail">>)), + ?_assertEqual(ok, machi_file_proxy:write(Pid, 20000, <<"fail">>)), + ?_assertEqual(ok, machi_file_proxy:write(Pid, 30000, <<"fail">>)), + %% Freeza + ?_assertEqual(ok, machi_file_proxy:write(Pid, 530000, <<"fail">>)), + ?_assertMatch({ok, {[{"test", 1024, _, _}, + {"test", 10000, <<"fail">>, _}, + {"test", 20000, <<"fail">>, _}, + {"test", 30000, <<"fail">>, _}, + {"test", 530000, <<"fail">>, _}], []}}, + machi_file_proxy:read(Pid, 1024, 530000)), + ?_assertMatch({ok, {[{"test", 1, _, _}], [{"test", 0, 1}]}}, + machi_file_proxy:read(Pid, 0, 1024, + [{needs_trimmed, true}])) + ] + end}. -endif. % !PULSE -endif. % TEST. -- 2.45.2 From 3c880dc437ecbaaa3c9e7e228c3c4e385f5b5dd6 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 4 Dec 2015 17:47:18 +0900 Subject: [PATCH 18/77] WIP: find 1st overlapping FLU in any #chain_def_v1{} --- include/machi_projection.hrl | 7 ++++ src/machi_chain_bootstrap.erl | 61 +++++++++++++++++++++++++++++++++++ src/machi_flu_sup.erl | 2 +- src/machi_sup.erl | 9 +++--- 4 files changed, 73 insertions(+), 6 deletions(-) diff --git a/include/machi_projection.hrl b/include/machi_projection.hrl index 0702586..ce6fe4a 100644 --- a/include/machi_projection.hrl +++ b/include/machi_projection.hrl @@ -75,4 +75,11 @@ %% create a consistent projection ranking score. -define(MAX_CHAIN_LENGTH, 64). +-record(chain_def_v1, { + name :: atom(), + mode :: pv1_consistency_mode(), + upi :: [pv1_server()], + witnesses :: [pv1_server()] + }). + -endif. % !MACHI_PROJECTION_HRL diff --git a/src/machi_chain_bootstrap.erl b/src/machi_chain_bootstrap.erl index 55e6334..306defc 100644 --- a/src/machi_chain_bootstrap.erl +++ b/src/machi_chain_bootstrap.erl @@ -22,6 +22,8 @@ -behaviour(gen_server). +-include("machi_projection.hrl"). + %% API -export([start_link/0]). @@ -52,6 +54,8 @@ handle_info(finish_init, State) -> FLU_Epochs = get_latest_public_epochs(FLUs), FLUs_at_zero = [FLU || {FLU, 0} <- FLU_Epochs], lager:info("FLUs at epoch 0: ~p\n", [FLUs_at_zero]), + ChainDefs = get_initial_chains(), + perhaps_bootstrap_chains(ChainDefs, FLUs), {noreply, State}; handle_info(_Info, State) -> {noreply, State}. @@ -74,3 +78,60 @@ get_latest_public_epochs(FLUs) -> PS, public), {FLU, Epoch} end || FLU <- FLUs]. + +get_initial_chains() -> + DoesNotExist = "/tmp/does/not/exist", + ConfigDir = case application:get_env(machi, chain_config_dir, DoesNotExist) of + DoesNotExist -> + DoesNotExist; + Dir -> + ok = filelib:ensure_dir(Dir ++ "/unused"), + Dir + end, + sanitize_chain_def_records(machi_flu_sup:load_rc_d_files_from_dir(ConfigDir)). + +sanitize_chain_def_records(Ps) -> + {Sane, _} = lists:foldl(fun sanitize_chain_def_rec/2, {[], dict:new()}, Ps), + Sane. + +sanitize_chain_def_rec(Whole, {Acc, D}) -> + try + #chain_def_v1{name=Name, + mode=Mode, + upi=UPI, + witnesses=Witnesses} = Whole, + true = is_atom(Name), + NameK = {name, Name}, + error = dict:find(NameK, D), + true = (Mode == ap_mode orelse Mode == cp_mode), + IsPSrvr = fun(X) when is_record(X, p_srvr) -> true; + (_) -> false + end, + true = lists:all(IsPSrvr, UPI), + true = lists:all(IsPSrvr, Witnesses), + + %% All is sane enough. + D2 = dict:store(NameK, Name, D), + {[Whole|Acc], D2} + catch _:_ -> + _ = lager:log(error, self(), + "~s: Bad chain_def record, skipping: ~P\n", + [?MODULE, Whole, 15]), + {Acc, D} + end. + +perhaps_bootstrap_chains([], _FLUs) -> + ok; +perhaps_bootstrap_chains([CD|ChainDefs], FLUs) -> + #chain_def_v1{upi=UPI, witnesses=Witnesses} = CD, + AllNames = [Name || #p_srvr{name=Name} <- UPI ++ Witnesses], + case ordsets:intersection(ordsets:from_list(AllNames), + ordsets:from_list(FLUs)) of + [] -> + io:format(user, "TODO: no local flus in ~P\n", [CD, 10]), + ok; + [FLU1|_] -> + io:format(user, "TODO: config ~p as bootstrap member of ~p\n", [FLU1, CD]), + yoyo + end, + perhaps_bootstrap_chains(ChainDefs, FLUs). diff --git a/src/machi_flu_sup.erl b/src/machi_flu_sup.erl index fe72499..f13a08d 100644 --- a/src/machi_flu_sup.erl +++ b/src/machi_flu_sup.erl @@ -44,7 +44,7 @@ -endif. %TEST %% API --export([start_link/0]). +-export([start_link/0, load_rc_d_files_from_dir/1]). %% Supervisor callbacks -export([init/1]). diff --git a/src/machi_sup.erl b/src/machi_sup.erl index d4ab71a..8d5be2a 100644 --- a/src/machi_sup.erl +++ b/src/machi_sup.erl @@ -62,9 +62,8 @@ init([]) -> ServerSup = {machi_flu_sup, {machi_flu_sup, start_link, []}, Restart, Shutdown, Type, []}, + ChainBootstrap = + {machi_chain_bootstrap, {machi_chain_bootstrap, start_link, []}, + Restart, Shutdown, worker, []}, - {ok, {SupFlags, [ServerSup]}}. - - %% AChild = {'AName', {'AModule', start_link, []}, - %% Restart, Shutdown, Type, ['AModule']}, - %% {ok, {SupFlags, [AChild]}}. + {ok, {SupFlags, [ServerSup, ChainBootstrap]}}. -- 2.45.2 From 1d3d121d83245441fdc885507eef28c76ca81040 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Mon, 7 Dec 2015 10:24:19 +0900 Subject: [PATCH 19/77] Simple fix for #52: file size matters --- test/machi_merkle_tree_test.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/machi_merkle_tree_test.erl b/test/machi_merkle_tree_test.erl index 922f0e2..96d4933 100644 --- a/test/machi_merkle_tree_test.erl +++ b/test/machi_merkle_tree_test.erl @@ -42,7 +42,9 @@ basic_test() -> T2 = machi_merkle_tree:build_tree(D1), ?assertNotEqual(T1#naive.root, T2#naive.root), - ?assertEqual(1, length(machi_merkle_tree:naive_diff(T1, T2))). + ?assertEqual(true, length(machi_merkle_tree:naive_diff(T1, T2)) == 1 + orelse + Filesize > ChunkSize). make_leaf_nodes(Filesize) -> -- 2.45.2 From 07c2b979186f0d5e0f35a27c29e5c3fc905b6a9a Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Fri, 4 Dec 2015 17:51:27 +0900 Subject: [PATCH 20/77] Change checksum_list API to return a t2b list --- src/machi_admin_util.erl | 6 ++++-- src/machi_csum_table.erl | 7 +++---- src/machi_file_proxy.erl | 11 ++++++++++- src/machi_flu1.erl | 33 +++++++-------------------------- test/machi_admin_util_test.erl | 11 ++++++++--- 5 files changed, 32 insertions(+), 36 deletions(-) diff --git a/src/machi_admin_util.erl b/src/machi_admin_util.erl index fb6dedb..46f6c3d 100644 --- a/src/machi_admin_util.erl +++ b/src/machi_admin_util.erl @@ -100,7 +100,7 @@ verify_file_checksums_common(Sock1, EpochID, File, ReadChunk) -> try case ?FLU_C:checksum_list(Sock1, EpochID, File) of {ok, InfoBin} -> - {Info, _} = machi_csum_table:split_checksum_list_blob_decode(InfoBin), + Info = machi_csum_table:split_checksum_list_blob_decode(InfoBin), Res = lists:foldl(verify_chunk_checksum(File, ReadChunk), [], Info), {ok, Res}; @@ -115,7 +115,9 @@ verify_file_checksums_common(Sock1, EpochID, File, ReadChunk) -> end. verify_chunk_checksum(File, ReadChunk) -> - fun({Offset, Size, <<_Tag:1/binary, CSum/binary>>}, Acc) -> + fun({0, ?MINIMUM_OFFSET, none}, []) -> + []; + ({Offset, Size, <<_Tag:1/binary, CSum/binary>>}, Acc) -> case ReadChunk(File, Offset, Size) of {ok, {[{_, Offset, Chunk, _}], _}} -> CSum2 = machi_util:checksum_chunk(Chunk), diff --git a/src/machi_csum_table.erl b/src/machi_csum_table.erl index 0b7be8e..88cc906 100644 --- a/src/machi_csum_table.erl +++ b/src/machi_csum_table.erl @@ -8,6 +8,7 @@ all_trimmed/2, calc_unwritten_bytes/1, split_checksum_list_blob_decode/1, + all/1, close/1, delete/1, foldl_chunks/3]). @@ -15,7 +16,6 @@ -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). --export([all/1]). -endif. -record(machi_csum_table, @@ -49,9 +49,8 @@ open(CSumFilename, _Opts) -> {ok, C0}. -spec split_checksum_list_blob_decode(binary())-> term(). -split_checksum_list_blob_decode(_Bin) -> - %% binary_to_term(Bin) - throw(not_yet). +split_checksum_list_blob_decode(Bin) -> + erlang:binary_to_term(Bin). -spec find(table(), machi_dt:file_offset(), machi_dt:file_size()) -> list({machi_dt:file_offset(), diff --git a/src/machi_file_proxy.erl b/src/machi_file_proxy.erl index 980d5a7..480aabd 100644 --- a/src/machi_file_proxy.erl +++ b/src/machi_file_proxy.erl @@ -57,7 +57,8 @@ write/4, trim/4, append/2, - append/4 + append/4, + checksum_list/1 ]). %% gen_server callbacks @@ -210,6 +211,10 @@ append(_Pid, ClientMeta, Extra, _Data) -> lager:warning("Bad arg to append: ClientMeta ~p, Extra ~p", [ClientMeta, Extra]), {error, bad_arg}. +-spec checksum_list(pid()) -> {ok, list()}. +checksum_list(Pid) -> + gen_server:call(Pid, {checksum_list}, ?TIMEOUT). + %% gen_server callbacks % @private @@ -430,6 +435,10 @@ handle_call({append, ClientMeta, Extra, Data}, _From, {reply, Resp, State#state{appends = {T+1, NewErr}, eof_position = NewEof}}; +handle_call({checksum_list}, _FRom, State = #state{csum_table=T}) -> + All = machi_csum_table:all(T), + {reply, {ok, All}, State}; + handle_call(Req, _From, State) -> lager:warning("Unknown call: ~p", [Req]), {reply, whoaaaaaaaaaaaa, State}. diff --git a/src/machi_flu1.erl b/src/machi_flu1.erl index 042eaed..c75a75d 100644 --- a/src/machi_flu1.erl +++ b/src/machi_flu1.erl @@ -595,26 +595,22 @@ do_server_trim_chunk(File, Offset, Size, TriggerGC, #state{flu_name=FluName}) -> do_server_checksum_listing(File, #state{flu_name=FluName, data_dir=DataDir}=_S) -> case sanitize_file_string(File) of ok -> - ok = sync_checksum_file(FluName, File), - CSumPath = machi_util:make_checksum_filename(DataDir, File), - %% TODO: If this file is legitimately bigger than our - %% {packet_size,N} limit, then we'll have a difficult time, eh? - case file:read_file(CSumPath) of - {ok, Bin} -> + case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of + {ok, Pid} -> + {ok, List} = machi_file_proxy:checksum_list(Pid), + Bin = erlang:term_to_binary(List), if byte_size(Bin) > (?PB_MAX_MSG_SIZE - 1024) -> %% TODO: Fix this limitation by streaming the %% binary in multiple smaller PB messages. %% Also, don't read the file all at once. ^_^ error_logger:error_msg("~s:~w oversize ~s\n", - [?MODULE, ?LINE, CSumPath]), + [?MODULE, ?LINE, DataDir]), {error, bad_arg}; true -> {ok, Bin} end; - {error, enoent} -> - {error, no_such_file}; - {error, _} -> - {error, bad_arg} + {error, trimmed} -> + {error, trimmed} end; _ -> {error, bad_arg} @@ -728,21 +724,6 @@ sanitize_prefix(Prefix) -> error end. -sync_checksum_file(FluName, File) -> - %% We just lookup the pid here - we don't start a proxy server. If - %% there isn't a pid for this file, then we just return ok. The - %% csum file was synced when the proxy was shutdown. - %% - %% If there *is* a pid, we call the sync function to ensure the - %% csum file is sync'd before we return. (Or an error if we get - %% an error). - case machi_flu_metadata_mgr:lookup_proxy_pid(FluName, {file, File}) of - undefined -> - ok; - Pid -> - machi_file_proxy:sync(Pid, csum) - end. - make_listener_regname(BaseName) -> list_to_atom(atom_to_list(BaseName) ++ "_listener"). diff --git a/test/machi_admin_util_test.erl b/test/machi_admin_util_test.erl index 54a75f4..e00fa31 100644 --- a/test/machi_admin_util_test.erl +++ b/test/machi_admin_util_test.erl @@ -33,7 +33,11 @@ -define(FLU_C, machi_flu1_client). verify_file_checksums_test_() -> - {timeout, 60, fun() -> verify_file_checksums_test2() end}. + {setup, + fun() -> os:cmd("rm -rf ./data") end, + fun(_) -> os:cmd("rm -rf ./data") end, + {timeout, 60, fun() -> verify_file_checksums_test2() end} + }. verify_file_checksums_test2() -> Host = "localhost", @@ -50,8 +54,9 @@ verify_file_checksums_test2() -> Prefix, <>) || X <- lists:seq(1, NumChunks)], {ok, [{_FileSize,File}]} = ?FLU_C:list_files(Sock1, ?DUMMY_PV1_EPOCH), - {ok, []} = machi_admin_util:verify_file_checksums_remote( - Host, TcpPort, ?DUMMY_PV1_EPOCH, File), + ?assertEqual({ok, []}, + machi_admin_util:verify_file_checksums_remote( + Host, TcpPort, ?DUMMY_PV1_EPOCH, File)), %% Clobber the first 3 chunks, which are sizes 1/2/3. {_, Path} = machi_util:make_data_filename(DataDir,binary_to_list(File)), -- 2.45.2 From 89e80a886294138686faa674b9d372094bcac4f6 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Mon, 7 Dec 2015 11:47:45 +0900 Subject: [PATCH 21/77] Fix GC not running --- src/machi_csum_table.erl | 54 +++++++++++++++------------------- src/machi_file_proxy.erl | 2 +- test/machi_csum_table_test.erl | 16 ++++++++++ 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/machi_csum_table.erl b/src/machi_csum_table.erl index 88cc906..b804992 100644 --- a/src/machi_csum_table.erl +++ b/src/machi_csum_table.erl @@ -154,32 +154,33 @@ trim(CsumT, Offset, Size, LeftUpdate, RightUpdate) -> %% @doc returns whether all bytes in a specific window is continously %% trimmed or not -spec all_trimmed(table(), non_neg_integer(), non_neg_integer()) -> boolean(). -all_trimmed(CsumT, Left, Right) -> - Chunks = find(CsumT, Left, Right), - runthru(Chunks, Left, Right). +all_trimmed(#machi_csum_table{table=T}, Left, Right) -> + FoldFun = fun({_, _}, false) -> + false; + ({K, V}, Pos) when is_integer(Pos) andalso Pos =< Right -> + case {sext:decode(K), sext:decode(V)} of + {{Pos, Size}, trimmed} -> + Pos + Size; + {{Offset, Size}, _} + when Offset + Size =< Left -> + Left; + _Eh -> + false + end + end, + case eleveldb:fold(T, FoldFun, Left, [{verify_checksums, true}]) of + false -> false; + Right -> true; + LastTrimmed when LastTrimmed < Right -> false; + _ -> %% LastTrimmed > Pos0, which is a irregular case but ok + true + end. %% @doc returns whether all bytes 0-Pos0 is continously trimmed or %% not, including header. -spec all_trimmed(table(), non_neg_integer()) -> boolean(). -all_trimmed(#machi_csum_table{table=T}, Pos0) -> - FoldFun = fun({_, _}, false) -> - false; - ({K, V}, Pos) when is_integer(Pos) andalso Pos =< Pos0 -> - case {sext:decode(K), sext:decode(V)} of - {{Pos, Size}, trimmed} -> - Pos + Size; - _Eh -> - %% ?debugVal({_Eh, Pos}), - false - end - end, - case eleveldb:fold(T, FoldFun, 0, [{verify_checksums, true}]) of - false -> false; - Pos0 -> true; - LastTrimmed when LastTrimmed < Pos0 -> false; - _ -> %% LastTrimmed > Pos0, which is a irregular case but ok - true - end. +all_trimmed(CsumT, Pos0) -> + all_trimmed(CsumT, 0, Pos0). -spec any_trimmed(table(), pos_integer(), @@ -248,15 +249,6 @@ build_unwritten_bytes_list([{CurrentOffset, CurrentSize, _Csum}|Rest], LastOffse build_unwritten_bytes_list([{CO, CS, _Ck}|Rest], _LastOffset, Acc) -> build_unwritten_bytes_list(Rest, CO + CS, Acc). -%% @doc make sure all trimmed chunks are continously chained -%% TODO: test with EQC -runthru([], Pos, Pos) -> true; -runthru([], Pos0, Pos) when Pos0 < Pos -> false; -runthru([{Offset0, Size0, trimmed}|T], Offset, Pos) when Offset0 =< Offset -> - runthru(T, Offset0+Size0, Pos); -runthru(_L, _O, _P) -> - false. - %% @doc If you want to find an overlap among two areas [x, y] and [a, %% b] where x < y and a < b; if (a-y)*(b-x) < 0 then there's a %% overlap, else, > 0 then there're no overlap. border condition = 0 diff --git a/src/machi_file_proxy.erl b/src/machi_file_proxy.erl index 480aabd..f6f9bea 100644 --- a/src/machi_file_proxy.erl +++ b/src/machi_file_proxy.erl @@ -834,7 +834,7 @@ maybe_gc(Reply, S = #state{fluname=FluName, filename = Filename, eof_position = Eof, csum_table=CsumTable}) -> - case machi_csum_table:all_trimmed(CsumTable, Eof) of + case machi_csum_table:all_trimmed(CsumTable, ?MINIMUM_OFFSET, Eof) of true -> lager:debug("GC? Let's do it: ~p.~n", [Filename]), %% Before unlinking a file, it should inform diff --git a/test/machi_csum_table_test.erl b/test/machi_csum_table_test.erl index 4059de4..f2b7a4f 100644 --- a/test/machi_csum_table_test.erl +++ b/test/machi_csum_table_test.erl @@ -113,3 +113,19 @@ smoke3_test() -> ok = machi_csum_table:delete(MC). %% TODO: add quickcheck test here + +%% Previous implementation +-spec all_trimmed2(machi_csum_table:table(), + non_neg_integer(), non_neg_integer()) -> boolean(). +all_trimmed2(CsumT, Left, Right) -> + Chunks = machi_csum_table:find(CsumT, Left, Right), + runthru(Chunks, Left, Right). + +%% @doc make sure all trimmed chunks are continously chained +%% TODO: test with EQC +runthru([], Pos, Pos) -> true; +runthru([], Pos0, Pos) when Pos0 < Pos -> false; +runthru([{Offset0, Size0, trimmed}|T], Offset, Pos) when Offset0 =< Offset -> + runthru(T, Offset0+Size0, Pos); +runthru(_L, _O, _P) -> + false. -- 2.45.2 From 5aeaf872d97f857ba2cf786c5238c66853005c11 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Mon, 7 Dec 2015 14:41:56 +0900 Subject: [PATCH 22/77] WIP: machi_chain_manager1:set_chain_members() API change, all tests pass, yay --- include/machi_projection.hrl | 4 +- src/machi.proto | 23 +++--- src/machi_chain_bootstrap.erl | 7 +- src/machi_chain_manager1.erl | 81 +++++++++++++-------- src/machi_pb_translate.erl | 4 + test/machi_ap_repair_eqc.erl | 4 +- test/machi_chain_manager1_converge_demo.erl | 21 +++--- test/machi_chain_manager1_test.erl | 4 +- test/machi_cr_client_test.erl | 12 ++- 9 files changed, 100 insertions(+), 60 deletions(-) diff --git a/include/machi_projection.hrl b/include/machi_projection.hrl index ce6fe4a..5ad4e54 100644 --- a/include/machi_projection.hrl +++ b/include/machi_projection.hrl @@ -22,10 +22,11 @@ -define(MACHI_PROJECTION_HRL, true). -type pv1_consistency_mode() :: 'ap_mode' | 'cp_mode'. +-type pv1_chain_name():: atom(). -type pv1_csum() :: binary(). -type pv1_epoch() :: {pv1_epoch_n(), pv1_csum()}. -type pv1_epoch_n() :: non_neg_integer(). --type pv1_server() :: atom() | binary(). +-type pv1_server() :: atom(). -type pv1_timestamp() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}. -record(p_srvr, { @@ -55,6 +56,7 @@ epoch_number :: pv1_epoch_n() | ?SPAM_PROJ_EPOCH, epoch_csum :: pv1_csum(), author_server :: pv1_server(), + chain_name = ch_not_def_yet :: pv1_chain_name(), all_members :: [pv1_server()], witnesses = [] :: [pv1_server()], creation_time :: pv1_timestamp(), diff --git a/src/machi.proto b/src/machi.proto index c9251cb..2fa8657 100644 --- a/src/machi.proto +++ b/src/machi.proto @@ -330,18 +330,17 @@ message Mpb_ProjectionV1 { required uint32 epoch_number = 1; required bytes epoch_csum = 2; required string author_server = 3; - repeated string all_members = 4; - repeated string witnesses = 5; - required Mpb_Now creation_time = 6; - required Mpb_Mode mode = 7; - repeated string upi = 8; - repeated string repairing = 9; - repeated string down = 10; - optional bytes opaque_flap = 11; - optional bytes opaque_inner = 12; - required bytes opaque_dbg = 13; - required bytes opaque_dbg2 = 14; - repeated Mpb_MembersDictEntry members_dict = 15; + required string chain_name = 4; + repeated string all_members = 5; + repeated string witnesses = 6; + required Mpb_Now creation_time = 7; + required Mpb_Mode mode = 8; + repeated string upi = 9; + repeated string repairing = 10; + repeated string down = 11; + required bytes opaque_dbg = 12; + required bytes opaque_dbg2 = 13; + repeated Mpb_MembersDictEntry members_dict = 14; } ////////////////////////////////////////// diff --git a/src/machi_chain_bootstrap.erl b/src/machi_chain_bootstrap.erl index 306defc..2406401 100644 --- a/src/machi_chain_bootstrap.erl +++ b/src/machi_chain_bootstrap.erl @@ -131,7 +131,10 @@ perhaps_bootstrap_chains([CD|ChainDefs], FLUs) -> io:format(user, "TODO: no local flus in ~P\n", [CD, 10]), ok; [FLU1|_] -> - io:format(user, "TODO: config ~p as bootstrap member of ~p\n", [FLU1, CD]), - yoyo + bootstrap_chain(CD, FLU1) end, perhaps_bootstrap_chains(ChainDefs, FLUs). + +bootstrap_chain(CD, FLU) -> + io:format(user, "TODO: config ~p as bootstrap member of ~p\n", [FLU, CD]), + todo. diff --git a/src/machi_chain_manager1.erl b/src/machi_chain_manager1.erl index 157ac5b..895a3d4 100644 --- a/src/machi_chain_manager1.erl +++ b/src/machi_chain_manager1.erl @@ -108,7 +108,7 @@ %% API -export([start_link/2, start_link/3, stop/1, ping/1, - set_chain_members/2, set_chain_members/3, set_active/2, + set_chain_members/2, set_chain_members/6, set_active/2, trigger_react_to_env/1]). -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, format_status/2, code_change/3]). @@ -168,13 +168,22 @@ ping(Pid) -> %% with lowest rank, i.e. name z* first, name a* last. set_chain_members(Pid, MembersDict) -> - set_chain_members(Pid, MembersDict, []). + set_chain_members(Pid, ch0_name, 0, ap_mode, MembersDict, []). -set_chain_members(Pid, MembersDict, Witness_list) -> - case lists:all(fun(Witness) -> orddict:is_key(Witness, MembersDict) end, - Witness_list) of +set_chain_members(Pid, ChainName, OldEpoch, CMode, MembersDict, Witness_list) + when is_atom(ChainName) andalso + is_integer(OldEpoch) andalso OldEpoch >= 0 andalso + (CMode == ap_mode orelse CMode == cp_mode) andalso + is_list(MembersDict) andalso + is_list(Witness_list) -> + case lists:all(fun({X, #p_srvr{name=X}}) -> true; + (_) -> false + end, MembersDict) + andalso + lists:all(fun(Witness) -> orddict:is_key(Witness, MembersDict) end, + Witness_list) of true -> - Cmd = {set_chain_members, MembersDict, Witness_list}, + Cmd = {set_chain_members, ChainName, OldEpoch, CMode, MembersDict, Witness_list}, gen_server:call(Pid, Cmd, infinity); false -> {error, bad_arg} @@ -291,7 +300,8 @@ init({MyName, InitMembersDict, MgrOpts}) -> handle_call({ping}, _From, S) -> {reply, pong, S}; -handle_call({set_chain_members, MembersDict, Witness_list}, _From, +handle_call({set_chain_members, ChainName, OldEpoch, CMode, + MembersDict, Witness_list}, _From, #ch_mgr{name=MyName, proj=#projection_v1{all_members=OldAll_list, epoch_number=OldEpoch, @@ -310,10 +320,10 @@ handle_call({set_chain_members, MembersDict, Witness_list}, _From, {NUPI, All_list -- NUPI} end, NewEpoch = OldEpoch + ?SET_CHAIN_MEMBERS_EPOCH_SKIP, - CMode = calc_consistency_mode(Witness_list), ok = set_consistency_mode(machi_flu_psup:make_proj_supname(MyName), CMode), NewProj = machi_projection:update_checksum( OldProj#projection_v1{author_server=MyName, + chain_name=ChainName, creation_time=now(), mode=CMode, epoch_number=NewEpoch, @@ -601,6 +611,8 @@ rank_and_sort_projections_with_extra(All_queried_list, FLUsRs, ProjectionType, Witness_list = CurrentProj#projection_v1.witnesses, NoneProj = make_none_projection(0, MyName, [], Witness_list, orddict:new()), + ChainName = CurrentProj#projection_v1.chain_name, + NoneProj2 = NoneProj#projection_v1{chain_name=ChainName}, Extra2 = [{all_members_replied, true}, {all_queried_list, All_queried_list}, {flus_rs, FLUsRs}, @@ -723,13 +735,14 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, runenv=RunEnv1, repair_final_status=RepairFS}=S) -> #projection_v1{epoch_number=OldEpochNum, + chain_name=ChainName, members_dict=MembersDict, witnesses=OldWitness_list, upi=OldUPI_list, repairing=OldRepairing_list } = LastProj, LastUp = lists:usort(OldUPI_list ++ OldRepairing_list), - AllMembers = (S#ch_mgr.proj)#projection_v1.all_members, + AllMembers = CurrentProj#projection_v1.all_members, {Up0, Partitions, RunEnv2} = calc_up_nodes(MyName, AllMembers, RunEnv1), Up = Up0 -- AllHosed, @@ -821,10 +834,11 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, end, ?REACT({calc,?LINE,[{new_upi, NewUPI},{new_rep, NewRepairing}]}), - P = machi_projection:new(OldEpochNum + 1, - MyName, MembersDict, Down, NewUPI, NewRepairing, - D_foo ++ - Dbg ++ [{ps, Partitions},{nodes_up, Up}]), + P0 = machi_projection:new(OldEpochNum + 1, + MyName, MembersDict, Down, NewUPI, NewRepairing, + D_foo ++ + Dbg ++ [{ps, Partitions},{nodes_up, Up}]), + P1 = P0#projection_v1{chain_name=ChainName}, P2 = if CMode == cp_mode -> UpWitnesses = [W || W <- Up, lists:member(W, OldWitness_list)], Majority = full_majority_size(AllMembers), @@ -833,7 +847,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, SoFar = length(NewUPI ++ NewRepairing), if SoFar >= Majority -> ?REACT({calc,?LINE,[]}), - P; + P1; true -> Need = Majority - SoFar, UpWitnesses = [W || W <- Up, @@ -842,7 +856,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, Ws = lists:sublist(UpWitnesses, Need), ?REACT({calc,?LINE,[{ws, Ws}]}), machi_projection:update_checksum( - P#projection_v1{upi=Ws++NewUPI}); + P1#projection_v1{upi=Ws++NewUPI}); true -> ?REACT({calc,?LINE,[]}), P_none0 = make_none_projection( @@ -855,6 +869,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, "Not enough witnesses are available now" end, P_none1 = P_none0#projection_v1{ + chain_name=ChainName, %% Stable creation time! creation_time={1,2,3}, dbg=[{none_projection,true}, @@ -875,7 +890,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg, end; CMode == ap_mode -> ?REACT({calc,?LINE,[]}), - P + P1 end, P3 = machi_projection:update_checksum( P2#projection_v1{mode=CMode, witnesses=OldWitness_list}), @@ -1045,13 +1060,13 @@ do_react_to_env(#ch_mgr{name=MyName, {{empty_members_dict, [], Epoch}, S}; true -> {_, S2} = do_set_chain_members_dict(NewMembersDict, S), - CMode = calc_consistency_mode(NewProj#projection_v1.witnesses), + CMode = NewProj#projection_v1.mode, {{empty_members_dict, [], Epoch}, set_proj(S2#ch_mgr{members_dict=NewMembersDict, consistency_mode=CMode}, NewProj)} end; do_react_to_env(S) -> -put(ttt, [?LINE]), + put(ttt, [?LINE]), %% The not_sanes manager counting dictionary is not strictly %% limited to flapping scenarios. (Though the mechanism first %% started as a way to deal with rare flapping scenarios.) @@ -1287,7 +1302,8 @@ react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra, end. react_to_env_A30(Retries, P_latest, LatestUnanimousP, P_current_calc, - #ch_mgr{name=MyName, consistency_mode=CMode} = S) -> + #ch_mgr{name=MyName, proj=P_current, + consistency_mode=CMode} = S) -> V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end, if V -> io:format(user, "A30: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end, ?REACT(a30), @@ -1307,15 +1323,17 @@ react_to_env_A30(Retries, P_latest, LatestUnanimousP, P_current_calc, P = #projection_v1{down=Down} = make_none_projection(Epoch + 1, MyName, All_list, Witness_list, MembersDict), + ChainName = P_current#projection_v1.chain_name, + P1 = P#projection_v1{chain_name=ChainName}, P_newprop = if CMode == ap_mode -> %% Not really none proj: just myself, AP style machi_projection:update_checksum( - P#projection_v1{upi=[MyName], + P1#projection_v1{upi=[MyName], down=Down -- [MyName], dbg=[{hosed_list,AllHosed}]}); CMode == cp_mode -> machi_projection:update_checksum( - P#projection_v1{dbg=[{hosed_list,AllHosed}]}) + P1#projection_v1{dbg=[{hosed_list,AllHosed}]}) end, react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP, P_current_calc, true, S); @@ -1850,7 +1868,9 @@ react_to_env_C103(#projection_v1{epoch_number=_Epoch_newprop} = _P_newprop, members_dict=MembersDict} = P_current, P_none0 = make_none_projection(Epoch_latest, MyName, All_list, Witness_list, MembersDict), - P_none1 = P_none0#projection_v1{dbg=[{none_projection,true}]}, + ChainName = P_current#projection_v1.chain_name, + P_none1 = P_none0#projection_v1{chain_name=ChainName, + dbg=[{none_projection,true}]}, P_none = machi_projection:update_checksum(P_none1), ?REACT({c103, ?LINE, [{current_epoch, P_current#projection_v1.epoch_number}, @@ -2206,6 +2226,7 @@ projection_transition_is_sane_except_si_epoch( creation_time=CreationTime1, mode=CMode1, author_server=AuthorServer1, + chain_name=ChainName1, all_members=All_list1, witnesses=Witness_list1, down=Down_list1, @@ -2217,6 +2238,7 @@ projection_transition_is_sane_except_si_epoch( creation_time=CreationTime2, mode=CMode2, author_server=AuthorServer2, + chain_name=ChainName2, all_members=All_list2, witnesses=Witness_list2, down=Down_list2, @@ -2237,7 +2259,8 @@ projection_transition_is_sane_except_si_epoch( true = is_binary(CSum1) andalso is_binary(CSum2), {_,_,_} = CreationTime1, {_,_,_} = CreationTime2, - true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2), % todo type may change? + true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2), + true = is_atom(ChainName1) andalso is_atom(ChainName2), true = is_list(All_list1) andalso is_list(All_list2), true = is_list(Witness_list1) andalso is_list(Witness_list2), true = is_list(Down_list1) andalso is_list(Down_list2), @@ -2249,6 +2272,9 @@ projection_transition_is_sane_except_si_epoch( %% projection_transition_is_sane_with_si_epoch(). true = Epoch2 >= Epoch1, + %% Don't change chain names in the middle of the stream. + true = (ChainName1 == ChainName2), + %% No duplicates true = lists:sort(Witness_list2) == lists:usort(Witness_list2), true = lists:sort(Down_list2) == lists:usort(Down_list2), @@ -2772,6 +2798,7 @@ full_majority_size(L) when is_list(L) -> full_majority_size(length(L)). make_zerf(#projection_v1{epoch_number=OldEpochNum, + chain_name=ChainName, all_members=AllMembers, members_dict=MembersDict, witnesses=OldWitness_list @@ -2794,7 +2821,8 @@ make_zerf(#projection_v1{epoch_number=OldEpochNum, MyName, AllMembers, OldWitness_list, MembersDict), machi_projection:update_checksum( - P#projection_v1{mode=cp_mode, + P#projection_v1{chain_name=ChainName, + mode=cp_mode, dbg2=[zerf_none,{up,Up},{maj,MajoritySize}]}); true -> make_zerf2(OldEpochNum, Up, MajoritySize, MyName, @@ -2916,11 +2944,6 @@ perhaps_verbose_c111(P_latest2, S) -> ok end. -calc_consistency_mode(_Witness_list = []) -> - ap_mode; -calc_consistency_mode(_Witness_list) -> - cp_mode. - set_proj(S, Proj) -> S#ch_mgr{proj=Proj, proj_unanimous=false}. diff --git a/src/machi_pb_translate.erl b/src/machi_pb_translate.erl index 0b49908..676c5a1 100644 --- a/src/machi_pb_translate.erl +++ b/src/machi_pb_translate.erl @@ -811,6 +811,7 @@ conv_to_epoch_id(#mpb_epochid{epoch_number=Epoch, conv_to_projection_v1(#mpb_projectionv1{epoch_number=Epoch, epoch_csum=CSum, author_server=Author, + chain_name=ChainName, all_members=AllMembers, witnesses=Witnesses, creation_time=CTime, @@ -824,6 +825,7 @@ conv_to_projection_v1(#mpb_projectionv1{epoch_number=Epoch, #projection_v1{epoch_number=Epoch, epoch_csum=CSum, author_server=to_atom(Author), + chain_name=to_atom(ChainName), all_members=[to_atom(X) || X <- AllMembers], witnesses=[to_atom(X) || X <- Witnesses], creation_time=conv_to_now(CTime), @@ -971,6 +973,7 @@ conv_from_boolean(true) -> conv_from_projection_v1(#projection_v1{epoch_number=Epoch, epoch_csum=CSum, author_server=Author, + chain_name=ChainName, all_members=AllMembers, witnesses=Witnesses, creation_time=CTime, @@ -984,6 +987,7 @@ conv_from_projection_v1(#projection_v1{epoch_number=Epoch, #mpb_projectionv1{epoch_number=Epoch, epoch_csum=CSum, author_server=to_list(Author), + chain_name=to_list(ChainName), all_members=[to_list(X) || X <- AllMembers], witnesses=[to_list(X) || X <- Witnesses], creation_time=conv_from_now(CTime), diff --git a/test/machi_ap_repair_eqc.erl b/test/machi_ap_repair_eqc.erl index a42d7cc..0c9f349 100644 --- a/test/machi_ap_repair_eqc.erl +++ b/test/machi_ap_repair_eqc.erl @@ -407,8 +407,8 @@ stabilize(0, _T) -> stabilize(_CmdsLen, #target{flu_names=FLUNames, mgr_names=MgrNames, verbose=Verbose}) -> machi_partition_simulator:no_partitions(), - wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames, - 100, Verbose), + true = wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames, + 100, Verbose), ok. chain_state_all_ok(FLUNames) -> diff --git a/test/machi_chain_manager1_converge_demo.erl b/test/machi_chain_manager1_converge_demo.erl index 9303701..cee7a78 100644 --- a/test/machi_chain_manager1_converge_demo.erl +++ b/test/machi_chain_manager1_converge_demo.erl @@ -187,15 +187,18 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) -> end || #p_srvr{name=Name}=P <- Ps], MembersDict = machi_projection:make_members_dict(Ps), Witnesses = proplists:get_value(witnesses, MgrOpts, []), + CMode = case {Witnesses, proplists:get_value(consistency_mode, MgrOpts, + ap_mode)} of + {[_|_], _} -> cp_mode; + {_, cp_mode} -> cp_mode; + {_, ap_mode} -> ap_mode + end, MgrNamez = [begin MgrName = machi_flu_psup:make_mgr_supname(Name), - ok = ?MGR:set_chain_members(MgrName,MembersDict,Witnesses), + ok = ?MGR:set_chain_members(MgrName, ch_demo, 0, CMode, + MembersDict,Witnesses), {Name, MgrName} end || #p_srvr{name=Name} <- Ps], - CpApMode = case Witnesses /= [] of - true -> cp_mode; - false -> ap_mode - end, try [{_, Ma}|_] = MgrNamez, @@ -303,9 +306,9 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) -> [{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane_retrospective(Psx, FLU)} || {FLU, Psx} <- PrivProjs] catch - _Err:_What when CpApMode == cp_mode -> + _Err:_What when CMode == cp_mode -> io:format(user, "none proj skip detected, TODO? ", []); - _Err:_What when CpApMode == ap_mode -> + _Err:_What when CMode == ap_mode -> io:format(user, "PrivProjs ~p\n", [PrivProjs]), exit({line, ?LINE, _Err, _What}) end, @@ -371,9 +374,9 @@ timer:sleep(1234), {FLU, Psx} <- PrivProjs], io:format(user, "\nAll sanity checks pass, hooray!\n", []) catch - _Err:_What when CpApMode == cp_mode -> + _Err:_What when CMode == cp_mode -> io:format(user, "none proj skip detected, TODO? ", []); - _Err:_What when CpApMode == ap_mode -> + _Err:_What when CMode == ap_mode -> io:format(user, "Report ~p\n", [Report]), io:format(user, "PrivProjs ~p\n", [PrivProjs]), exit({line, ?LINE, _Err, _What}) diff --git a/test/machi_chain_manager1_test.erl b/test/machi_chain_manager1_test.erl index e13193f..27e40ec 100644 --- a/test/machi_chain_manager1_test.erl +++ b/test/machi_chain_manager1_test.erl @@ -349,8 +349,8 @@ nonunanimous_setup_and_fix_test() -> [element(2,?FLU_PC:start_link(P)) || P <- P_s], MembersDict = machi_projection:make_members_dict(P_s), [Ma,Mb] = [a_chmgr, b_chmgr], - ok = machi_chain_manager1:set_chain_members(Ma, MembersDict, []), - ok = machi_chain_manager1:set_chain_members(Mb, MembersDict, []), + ok = machi_chain_manager1:set_chain_members(Ma, MembersDict), + ok = machi_chain_manager1:set_chain_members(Mb, MembersDict), try {ok, P1} = ?MGR:test_calc_projection(Ma, false), diff --git a/test/machi_cr_client_test.erl b/test/machi_cr_client_test.erl index f5d3513..299a78a 100644 --- a/test/machi_cr_client_test.erl +++ b/test/machi_cr_client_test.erl @@ -58,9 +58,15 @@ setup_smoke_test(Host, PortBase, Os, Witness_list) -> %% 4. Wait until all others are using epoch id from #3. %% %% Damn, this is a pain to make 100% deterministic, bleh. - ok = machi_chain_manager1:set_chain_members(a_chmgr, D, Witness_list), - ok = machi_chain_manager1:set_chain_members(b_chmgr, D, Witness_list), - ok = machi_chain_manager1:set_chain_members(c_chmgr, D, Witness_list), + CMode = if Witness_list == [] -> ap_mode; + Witness_list /= [] -> cp_mode + end, + ok = machi_chain_manager1:set_chain_members(a_chmgr, ch0, 0, CMode, + D, Witness_list), + ok = machi_chain_manager1:set_chain_members(b_chmgr, ch0, 0, CMode, + D, Witness_list), + ok = machi_chain_manager1:set_chain_members(c_chmgr, ch0, 0, CMode, + D, Witness_list), run_ticks([a_chmgr,b_chmgr,c_chmgr]), %% Everyone is settled on the same damn epoch id. {ok, EpochID} = machi_flu1_client:get_latest_epochid(Host, PortBase+0, -- 2.45.2 From 293eb4810f184c98abff27c384a0e9d5fd40807c Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Mon, 7 Dec 2015 14:07:00 +0900 Subject: [PATCH 23/77] Fix dialyzer error --- src/machi_basho_bench_driver.erl | 2 +- src/machi_chain_repair.erl | 4 +-- src/machi_csum_table.erl | 61 +++++++++++++++++++------------- src/machi_file_proxy.erl | 22 ++++++++---- src/machi_yessir_client.erl | 6 ++-- 5 files changed, 57 insertions(+), 38 deletions(-) diff --git a/src/machi_basho_bench_driver.erl b/src/machi_basho_bench_driver.erl index 2cf9e39..4d36328 100644 --- a/src/machi_basho_bench_driver.erl +++ b/src/machi_basho_bench_driver.erl @@ -136,7 +136,7 @@ load_ets_table(Conn, ETS) -> {ok, Fs} = machi_cr_client:list_files(Conn), [begin {ok, InfoBin} = machi_cr_client:checksum_list(Conn, File), - {PosList, _} = machi_csum_table:split_checksum_list_blob_decode(InfoBin), + PosList = machi_csum_table:split_checksum_list_blob_decode(InfoBin), StartKey = ets:update_counter(ETS, max_key, 0), %% _EndKey = lists:foldl(fun({Off,Sz,CSum}, K) -> %% V = {File, Off, Sz, CSum}, diff --git a/src/machi_chain_repair.erl b/src/machi_chain_repair.erl index 1138f9d..87ce73c 100644 --- a/src/machi_chain_repair.erl +++ b/src/machi_chain_repair.erl @@ -209,9 +209,7 @@ make_repair_directives(ConsistencyMode, RepairMode, File, Size, EpochID, case machi_proxy_flu1_client:checksum_list( Proxy, EpochID, File, ?LONG_TIMEOUT) of {ok, InfoBin} -> - {Info, _} = - machi_csum_table:split_checksum_list_blob_decode(InfoBin), - Info; + machi_csum_table:split_checksum_list_blob_decode(InfoBin); {error, no_such_file} -> [] end, diff --git a/src/machi_csum_table.erl b/src/machi_csum_table.erl index b804992..509dd36 100644 --- a/src/machi_csum_table.erl +++ b/src/machi_csum_table.erl @@ -25,6 +25,9 @@ -type table() :: #machi_csum_table{}. -type byte_sequence() :: { Offset :: non_neg_integer(), Size :: pos_integer()|infinity }. +-type chunk() :: {Offset :: machi_dt:file_offset(), + Size :: machi_dt:chunk_size(), + machi_dt:chunk_csum() | trimmed | none}. -export_type([table/0]). @@ -48,14 +51,16 @@ open(CSumFilename, _Opts) -> table=T}, {ok, C0}. --spec split_checksum_list_blob_decode(binary())-> term(). +-spec split_checksum_list_blob_decode(binary())-> [chunk()]. split_checksum_list_blob_decode(Bin) -> erlang:binary_to_term(Bin). --spec find(table(), machi_dt:file_offset(), machi_dt:file_size()) -> - list({machi_dt:file_offset(), - machi_dt:file_size(), - machi_dt:chunk_csum()|trimmed}). + +-define(has_overlap(LeftOffset, LeftSize, RightOffset, RightSize), + ((LeftOffset - (RightOffset+RightSize)) * (LeftOffset+LeftSize - RightOffset) < 0)). + +-spec find(table(), machi_dt:file_offset(), machi_dt:chunk_size()) + -> [chunk()]. find(#machi_csum_table{table=T}, Offset, Size) -> {ok, I} = eleveldb:iterator(T, [], keys_only), EndKey = sext:encode({Offset+Size, 0}), @@ -75,22 +80,26 @@ find(#machi_csum_table{table=T}, Offset, Size) -> _ = eleveldb:iterator_close(I), FoldFun = fun({K, V}, Acc) -> {TargetOffset, TargetSize} = sext:decode(K), - case has_overlap(TargetOffset, TargetSize, Offset, Size) of + case ?has_overlap(TargetOffset, TargetSize, Offset, Size) of true -> [{TargetOffset, TargetSize, sext:decode(V)}|Acc]; false -> Acc end; - (_KV, _) -> - throw(_KV) + (_K, Acc) -> + lager:error("~p wrong option", [_K]), + Acc end, lists:reverse(eleveldb_fold(T, FirstKey, EndKey, FoldFun, [])). -has_overlap(LeftOffset, LeftSize, RightOffset, RightSize) -> - (LeftOffset - (RightOffset+RightSize)) * (LeftOffset+LeftSize - RightOffset) < 0. %% @doc Updates all chunk info, by deleting existing entries if exists %% and putting new chunk info +-spec write(table(), + machi_dt:file_offset(), machi_dt:chunk_size(), + machi_dt:chunk_csum()|'none'|'trimmed', + undefined|chunk(), undefined|chunk()) -> + ok | {error, term()}. write(#machi_csum_table{table=T} = CsumT, Offset, Size, CSum, LeftUpdate, RightUpdate) -> PutOps = @@ -120,8 +129,7 @@ write(#machi_csum_table{table=T} = CsumT, Offset, Size, CSum, eleveldb:write(T, DeleteOps ++ PutOps, [{sync, true}]). -spec find_leftneighbor(table(), non_neg_integer()) -> - undefined | - {non_neg_integer(), machi_dt:chunk_size(), trimmed|machi_dt:chunk_csum()}. + undefined | chunk(). find_leftneighbor(CsumT, Offset) -> case find(CsumT, Offset, 1) of [] -> undefined; @@ -130,8 +138,7 @@ find_leftneighbor(CsumT, Offset) -> end. -spec find_rightneighbor(table(), non_neg_integer()) -> - undefined | - {non_neg_integer(), machi_dt:chunk_size(), trimmed|machi_dt:chunk_csum()}. + undefined | chunk(). find_rightneighbor(CsumT, Offset) -> case find(CsumT, Offset, 1) of [] -> undefined; @@ -141,7 +148,7 @@ find_rightneighbor(CsumT, Offset) -> end. -spec write(table(), machi_dt:file_offset(), machi_dt:file_size(), - machi_dt:chunk_csum()|trimmed) -> + machi_dt:chunk_csum()|none|trimmed) -> ok | {error, trimmed|file:posix()}. write(CsumT, Offset, Size, CSum) -> write(CsumT, Offset, Size, CSum, undefined, undefined). @@ -212,20 +219,19 @@ delete(#machi_csum_table{table=T, file=F}) -> catch eleveldb:close(T), %% TODO change this to directory walk case os:cmd("rm -rf " ++ F) of - ok -> ok; - {error, enoent} -> ok; "" -> ok; E -> E end. --spec foldl_chunks(fun(({non_neg_integer(), non_neg_integer(), term()}, - Acc0 :: term()) - -> Acc :: term()), +-spec foldl_chunks(fun((chunk(), Acc0 :: term()) -> Acc :: term()), Acc0 :: term(), table()) -> Acc :: term(). foldl_chunks(Fun, Acc0, #machi_csum_table{table=T}) -> FoldFun = fun({K, V}, Acc) -> {Offset, Len} = sext:decode(K), - Fun({Offset, Len, sext:decode(V)}, Acc) + Fun({Offset, Len, sext:decode(V)}, Acc); + (_K, Acc) -> + _ = lager:error("~p: wrong option?", [_K]), + Acc end, eleveldb:fold(T, FoldFun, Acc0, [{verify_checksums, true}]). @@ -259,7 +265,10 @@ build_unwritten_bytes_list([{CO, CS, _Ck}|Rest], _LastOffset, Acc) -> %% {'-', Offset + Size, '$1'}, %% {'-', Offset, {'+', '$1', '$2'}}}}. - +-spec eleveldb_fold(eleveldb:db_ref(), binary(), binary(), + fun(({binary(), binary()}, AccType::term()) -> AccType::term()), + AccType0::term()) -> + AccType::term(). eleveldb_fold(Ref, Start, End, FoldFun, InitAcc) -> {ok, Iterator} = eleveldb:iterator(Ref, []), try @@ -271,6 +280,11 @@ eleveldb_fold(Ref, Start, End, FoldFun, InitAcc) -> eleveldb:iterator_close(Iterator) end. +-spec eleveldb_do_fold({ok, binary(), binary()}|{error, iterator_closed|invalid_iterator}|{ok,binary()}, + eleveldb:itr_ref(), binary(), + fun(({binary(), binary()}, AccType::term()) -> AccType::term()), + AccType::term()) -> + AccType::term(). eleveldb_do_fold({ok, Key, Value}, _, End, FoldFun, Acc) when End < Key -> FoldFun({Key, Value}, Acc); @@ -284,6 +298,3 @@ eleveldb_do_fold({error, iterator_closed}, _, _, _, Acc) -> eleveldb_do_fold({error, invalid_iterator}, _, _, _, Acc) -> %% Probably reached to end Acc. - - - diff --git a/src/machi_file_proxy.erl b/src/machi_file_proxy.erl index f6f9bea..cae292c 100644 --- a/src/machi_file_proxy.erl +++ b/src/machi_file_proxy.erl @@ -600,7 +600,8 @@ check_or_make_tagged_csum(OtherTag, _ClientCsum, _Data) -> Size :: non_neg_integer(), NoChunk :: boolean(), NoChecksum :: boolean() - ) -> {ok, Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum :: binary()}]} | + ) -> {ok, {Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum :: binary()}], + Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]}} | {error, bad_checksum} | {error, partial_read} | {error, file:posix()} | @@ -624,6 +625,14 @@ do_read(FHd, Filename, CsumTable, Offset, Size, _, _) -> ChunkCsums = machi_csum_table:find(CsumTable, Offset, Size), read_all_ranges(FHd, Filename, ChunkCsums, [], []). +-spec read_all_ranges(file:io_device(), string(), + [{non_neg_integer(),non_neg_integer(),trimmed|binary()}], + Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum::binary()}], + Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]) -> + {ok, { + Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum::binary()}], + Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]}} | + {erorr, term()|partial_read}. read_all_ranges(_, _, [], ReadChunks, TrimmedChunks) -> %% TODO: currently returns empty list of trimmed chunks {ok, {lists:reverse(ReadChunks), lists:reverse(TrimmedChunks)}}; @@ -635,12 +644,13 @@ read_all_ranges(FHd, Filename, [{Offset, Size, TaggedCsum}|T], ReadChunks, Trimm case file:pread(FHd, Offset, Size) of eof -> read_all_ranges(FHd, Filename, T, ReadChunks, TrimmedChunks); + {ok, Bytes} when byte_size(Bytes) == Size, TaggedCsum =:= none -> + read_all_ranges(FHd, Filename, T, + [{Filename, Offset, Bytes, + machi_util:make_tagged_csum(none, <<>>)}|ReadChunks], + TrimmedChunks); {ok, Bytes} when byte_size(Bytes) == Size -> - {Tag, Ck} = case TaggedCsum of - none -> {?CSUM_TAG_NONE, ooops}; - _ -> - machi_util:unmake_tagged_csum(TaggedCsum) - end, + {Tag, Ck} = machi_util:unmake_tagged_csum(TaggedCsum), case check_or_make_tagged_csum(Tag, Ck, Bytes) of {error, Bad} -> lager:error("Bad checksum; got ~p, expected ~p", diff --git a/src/machi_yessir_client.erl b/src/machi_yessir_client.erl index f37b618..1bdef2a 100644 --- a/src/machi_yessir_client.erl +++ b/src/machi_yessir_client.erl @@ -180,9 +180,9 @@ checksum_list(#yessir{name=Name,chunk_size=ChunkSize}, _EpochID, File) -> MaxOffset -> C = machi_util:make_tagged_csum(client_sha, make_csum(Name, ChunkSize)), - Cs = [machi_csum_table:encode_csum_file_entry_bin(Offset, ChunkSize, C) || - Offset <- lists:seq(?MINIMUM_OFFSET, MaxOffset, ChunkSize)], - {ok, Cs} + Cs = [{Offset, ChunkSize, C} || + Offset <- lists:seq(?MINIMUM_OFFSET, MaxOffset, ChunkSize)], + {ok, term_to_binary(Cs)} end. %% @doc Fetch the list of chunk checksums for `File'. -- 2.45.2 From 38e63e8181f7f091ce9aa5bf25bfa972c393019c Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Mon, 7 Dec 2015 21:52:27 +0900 Subject: [PATCH 24/77] Add & remove, mostly working (2 eunit tests broken) --- src/machi_chain_manager1.erl | 148 ++++++++++++++++++++--------- src/machi_projection.erl | 1 + test/machi_chain_manager1_test.erl | 71 ++++++++++++-- test/machi_pb_high_client_test.erl | 3 + 4 files changed, 170 insertions(+), 53 deletions(-) diff --git a/src/machi_chain_manager1.erl b/src/machi_chain_manager1.erl index 895a3d4..da30e6e 100644 --- a/src/machi_chain_manager1.erl +++ b/src/machi_chain_manager1.erl @@ -290,7 +290,7 @@ init({MyName, InitMembersDict, MgrOpts}) -> last_down=[no_such_server_initial_value_only], fitness_svr=machi_flu_psup:make_fitness_regname(MyName) }, Proj), - {_, S2} = do_set_chain_members_dict(MembersDict, S), + S2 = do_set_chain_members_dict(MembersDict, S), S3 = if ActiveP == false -> S2; ActiveP == true -> @@ -300,13 +300,17 @@ init({MyName, InitMembersDict, MgrOpts}) -> handle_call({ping}, _From, S) -> {reply, pong, S}; -handle_call({set_chain_members, ChainName, OldEpoch, CMode, +handle_call({set_chain_members, SetChainName, SetOldEpoch, CMode, MembersDict, Witness_list}, _From, #ch_mgr{name=MyName, proj=#projection_v1{all_members=OldAll_list, epoch_number=OldEpoch, + chain_name=ChainName, upi=OldUPI}=OldProj}=S) -> - {Reply, S2} = do_set_chain_members_dict(MembersDict, S), + true = (OldEpoch == 0) % in this case we want unconditional set of ch name + orelse + (SetOldEpoch == OldEpoch andalso SetChainName == ChainName), + S2 = do_set_chain_members_dict(MembersDict, S), %% TODO: should there be any additional sanity checks? Right now, %% if someone does something bad, then do_react_to_env() will %% crash, which will crash us, and we'll restart in a sane & old @@ -335,7 +339,11 @@ handle_call({set_chain_members, ChainName, OldEpoch, CMode, members_dict=MembersDict}), S3 = set_proj(S2#ch_mgr{proj_history=queue:new(), consistency_mode=CMode}, NewProj), - {_QQ, S4} = do_react_to_env(S3), + {Res, S4} = do_react_to_env(S3), + Reply = case Res of + {_,_,_} -> ok; + _ -> Res + end, {reply, Reply, S4}; handle_call({set_active, Boolean}, _From, #ch_mgr{timer=TRef}=S) -> case {Boolean, TRef} of @@ -367,8 +375,8 @@ handle_call({test_read_latest_public_projection, ReadRepairP}, _From, S) -> {reply, Res, S2}; handle_call({trigger_react_to_env}=Call, _From, S) -> gobble_calls(Call), - {TODOtodo, S2} = do_react_to_env(S), - {reply, TODOtodo, S2}; + {Res, S2} = do_react_to_env(S), + {reply, Res, S2}; handle_call(_Call, _From, S) -> io:format(user, "\nBad call to ~p: ~p\n", [S#ch_mgr.name, _Call]), {reply, whaaaaaaaaaa, S}. @@ -545,6 +553,7 @@ cl_write_public_proj2(FLUs, Partitions, Epoch, Proj, IgnoreWrittenErrorP, S) -> end end, {true, []}, FLUs), %% io:format(user, "\nWrite public ~w by ~w: ~w\n", [Epoch, S#ch_mgr.name, Rs]), + %% io:format(user, "mgr ~w epoch ~w Rs ~p\n", [S#ch_mgr.name, Epoch, Rs]), {{remote_write_results, Rs}, S}. do_cl_read_latest_public_projection(ReadRepairP, @@ -566,12 +575,41 @@ do_cl_read_latest_public_projection(ReadRepairP, read_latest_projection_call_only(ProjectionType, AllHosed, #ch_mgr{proj=CurrentProj}=S) -> #projection_v1{all_members=All_list} = CurrentProj, - All_queried_list = All_list -- AllHosed, + All_queried_list = lists:sort(All_list -- AllHosed), + read_latest_projection_call_only1(ProjectionType, AllHosed, + All_queried_list, S). - {Rs, S2} = read_latest_projection_call_only2(ProjectionType, - All_queried_list, S), - FLUsRs = lists:zip(All_queried_list, Rs), - {All_queried_list, FLUsRs, S2}. +read_latest_projection_call_only1(ProjectionType, AllHosed, + All_queried_list, S) -> + {Rs_tmp, S2} = read_latest_projection_call_only2(ProjectionType, + All_queried_list, S), + New_all_maybe = + lists:usort( + lists:flatten( + [A_l || #projection_v1{all_members=A_l} <- Rs_tmp])) -- AllHosed, + case New_all_maybe -- All_queried_list of + [] -> + FLUsRs = lists:zip(All_queried_list, Rs_tmp), + {All_queried_list, FLUsRs, S2}; + [AnotherFLU|_] -> + %% Stop AnotherFLU proxy, in unexpected case where it's open + try + Proxy = proxy_pid(AnotherFLU, S2), + ?FLU_PC:stop_proxies([Proxy]) + catch _:_ -> ok + end, + MD = orddict:from_list( + lists:usort( + lists:flatten( + [orddict:to_list(D) || #projection_v1{members_dict=D} <- Rs_tmp]))), + Another_P_srvr = orddict:fetch(AnotherFLU, MD), + {ok, Proxy2} = ?FLU_PC:start_link(Another_P_srvr), + S3 = S2#ch_mgr{proxies_dict=orddict:store(AnotherFLU, Proxy2, + S2#ch_mgr.proxies_dict)}, + read_latest_projection_call_only1( + ProjectionType, AllHosed, + lists:usort([AnotherFLU|All_queried_list]), S3) + end. read_latest_projection_call_only2(ProjectionType, All_queried_list, S) -> {_UpNodes, Partitions, S2} = calc_up_nodes(S), @@ -621,7 +659,7 @@ rank_and_sort_projections_with_extra(All_queried_list, FLUsRs, ProjectionType, {bad_answer_flus, BadAnswerFLUs}, {bad_answers, BadAnswers}, {not_unanimous_answers, []}], - {not_unanimous, NoneProj, Extra2, S}; + {not_unanimous, NoneProj2, Extra2, S}; ProjectionType == public, UnwrittenRs /= [] -> {needs_repair, FLUsRs, [flarfus], S}; true -> @@ -1042,28 +1080,30 @@ rank_projection(#projection_v1{author_server=_Author, do_set_chain_members_dict(MembersDict, #ch_mgr{proxies_dict=OldProxiesDict}=S)-> _ = ?FLU_PC:stop_proxies(OldProxiesDict), ProxiesDict = ?FLU_PC:start_proxies(MembersDict), - {ok, S#ch_mgr{members_dict=MembersDict, - proxies_dict=ProxiesDict}}. + S#ch_mgr{members_dict=MembersDict, + proxies_dict=ProxiesDict}. do_react_to_env(#ch_mgr{name=MyName, proj=#projection_v1{epoch_number=Epoch, members_dict=[]=OldDict}=OldProj, opts=Opts}=S) -> + put(ttt, [?LINE]), %% Read from our local *public* projection store. If some other %% chain member has written something there, and if we are a %% member of that chain, then we'll adopt that projection and then %% start actively humming in that chain. - {NewMembersDict, NewProj} = + {NewMD, NewProj} = get_my_public_proj_boot_info(Opts, OldDict, OldProj), - case orddict:is_key(MyName, NewMembersDict) of + case orddict:is_key(MyName, NewMD) of false -> - {{empty_members_dict, [], Epoch}, S}; + {{empty_members_dict1, [], Epoch}, S}; true -> - {_, S2} = do_set_chain_members_dict(NewMembersDict, S), CMode = NewProj#projection_v1.mode, - {{empty_members_dict, [], Epoch}, - set_proj(S2#ch_mgr{members_dict=NewMembersDict, - consistency_mode=CMode}, NewProj)} + S2 = do_set_chain_members_dict(NewMD, S), + {Reply, S3} = react_to_env_C110(NewProj, + S2#ch_mgr{members_dict=NewMD, + consistency_mode=CMode}), + {Reply, S3} end; do_react_to_env(S) -> put(ttt, [?LINE]), @@ -1165,7 +1205,7 @@ react_to_env_A10(S) -> ?REACT(a10), react_to_env_A20(0, poll_private_proj_is_upi_unanimous(S)). -react_to_env_A20(Retries, #ch_mgr{name=MyName}=S) -> +react_to_env_A20(Retries, #ch_mgr{name=MyName, proj=P_current}=S) -> ?REACT(a20), init_remember_down_list(), {UnanimousTag, P_latest, ReadExtra, S2} = @@ -1193,17 +1233,34 @@ react_to_env_A20(Retries, #ch_mgr{name=MyName}=S) -> false when P_latest#projection_v1.epoch_number /= LastComplaint, P_latest#projection_v1.all_members /= [] -> put(rogue_server_epoch, P_latest#projection_v1.epoch_number), - error_logger:info_msg("Chain manager ~p found latest public " - "projection ~p has author ~p has a " - "members list ~p that does not include me.\n", + error_logger:info_msg("Chain manager ~w found latest public " + "projection ~w with author ~w has a " + "members list ~w that does not include me. " + "We assume this is a result of administrator " + "action and will thus wedge ourselves until " + "we are re-added to the chain or shutdown.\n", [S#ch_mgr.name, P_latest#projection_v1.epoch_number, P_latest#projection_v1.author_server, - P_latest#projection_v1.all_members]); + P_latest#projection_v1.all_members]), + EpochID = machi_projection:make_epoch_id(P_current), + ProjStore = get_projection_store_pid_or_regname(S), + {ok, NotifyPid} = machi_projection_store:get_wedge_notify_pid(ProjStore), + _QQ = machi_flu1:update_wedge_state(NotifyPid, true, EpochID), + #projection_v1{epoch_number=Epoch, + chain_name=ChainName, + all_members=All_list, + witnesses=Witness_list, + members_dict=MembersDict} = P_current, + P_none0 = make_none_projection(Epoch, + MyName, All_list, Witness_list, MembersDict), + P_none = P_none0#projection_v1{chain_name=ChainName}, + {{x,y,z,42,77}, S2#ch_mgr{proj=P_none}}; _ -> - ok - end, + react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S2) + end. +react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S) -> %% The UnanimousTag isn't quite sufficient for our needs. We need %% to determine if *all* of the UPI+Repairing FLUs are members of %% the unanimous server replies. All Repairing FLUs should be up @@ -1248,7 +1305,7 @@ react_to_env_A20(Retries, #ch_mgr{name=MyName}=S) -> true -> exit({badbad, UnanimousTag}) end, - react_to_env_A29(Retries, P_latest, LatestUnanimousP, ReadExtra, S2). + react_to_env_A29(Retries, P_latest, LatestUnanimousP, ReadExtra, S). react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra, #ch_mgr{consistency_mode=CMode, @@ -1282,7 +1339,6 @@ react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra, ?REACT({a29, ?LINE, [{zerf_backstop, true}, {zerf_in, machi_projection:make_summary(Zerf)}]}), - %% io:format(user, "zerf_in: A29: ~p: ~w\n\t~p\n", [MyName, machi_projection:make_summary(Zerf), get(yyy_hack)]), #projection_v1{dbg=ZerfDbg} = Zerf, Backstop = if Zerf#projection_v1.upi == [] -> []; @@ -1406,13 +1462,20 @@ react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP, %% we have a disagreement. not ordsets:is_disjoint(P_latest_s, Down_s) end, + AmExcludedFromLatestAll_p = + (not lists:member(MyName, P_latest#projection_v1.all_members)), ?REACT({a40, ?LINE, [{latest_author, P_latest#projection_v1.author_server}, + {am_excluded_from_latest_all_p, AmExcludedFromLatestAll_p}, {author_is_down_p, LatestAuthorDownP}, {rank_latest, Rank_latest}, {rank_newprop, Rank_newprop}]}), if + AmExcludedFromLatestAll_p -> + ?REACT({a40, ?LINE, []}), + react_to_env_A50(P_latest, [], S); + AmHosedP -> ExpectedUPI = if CMode == cp_mode -> []; CMode == ap_mode -> [MyName] @@ -1578,12 +1641,10 @@ react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP, end, if GoTo50_p -> ?REACT({a40, ?LINE, []}), -%% io:format(user, "CONFIRM debug question line ~w\n", [?LINE]), FinalProps = [{throttle_seconds, 0}], react_to_env_A50(P_latest, FinalProps, S); true -> ?REACT({a40, ?LINE, []}), -io:format(user, "CONFIRM debug question line ~w\n", [?LINE]), react_to_env_C300(P_newprop, P_latest, S) end end. @@ -1593,7 +1654,6 @@ react_to_env_A50(P_latest, FinalProps, #ch_mgr{proj=P_current}=S) -> ?REACT({a50, ?LINE, [{current_epoch, P_current#projection_v1.epoch_number}, {latest_epoch, P_latest#projection_v1.epoch_number}, {final_props, FinalProps}]}), - %% if S#ch_mgr.name == c -> io:format(user, "A50: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end, V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end, if V -> io:format(user, "A50: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end, {{no_change, FinalProps, P_current#projection_v1.epoch_number}, S}. @@ -2282,7 +2342,7 @@ projection_transition_is_sane_except_si_epoch( true = lists:sort(Repairing_list2) == lists:usort(Repairing_list2), %% Disjoint-ness - All_list1 = All_list2, % todo will probably change + %% %% %% %% %% %% %% %% All_list1 = All_list2, % todo will probably change %% true = lists:sort(All_list2) == lists:sort(Down_list2 ++ UPI_list2 ++ %% Repairing_list2), [] = [X || X <- Witness_list2, not lists:member(X, All_list2)], @@ -2387,8 +2447,7 @@ poll_private_proj_is_upi_unanimous_sleep(Count, #ch_mgr{runenv=RunEnv}=S) -> S2 end. -poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current, - opts=MgrOpts} = S) -> +poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current} = S) -> UPI = P_current#projection_v1.upi, EpochID = machi_projection:make_epoch_id(P_current), {Rs, S2} = read_latest_projection_call_only2(private, UPI, S), @@ -2421,12 +2480,7 @@ poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current, Annotation = make_annotation(EpochID, Now), NewDbg2 = [Annotation|P_currentFull#projection_v1.dbg2], NewProj = P_currentFull#projection_v1{dbg2=NewDbg2}, - ProjStore = case get_projection_store_regname(MgrOpts) of - undefined -> - machi_flu_psup:make_proj_supname(MyName); - PStr -> - PStr - end, + ProjStore = get_projection_store_pid_or_regname(S), #projection_v1{epoch_number=_EpochRep, epoch_csum= <<_CSumRep:4/binary,_/binary>>, upi=_UPIRep, @@ -2446,8 +2500,6 @@ poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current, S2 end; _Else -> - %% io:format(user, "poll by ~w: want ~W got ~W\n", - %% [MyName, EpochID, 6, _Else, 8]), S2 end. @@ -2837,7 +2889,6 @@ make_zerf2(OldEpochNum, Up, MajoritySize, MyName, AllMembers, OldWitness_list, Proj2 = Proj#projection_v1{dbg2=[{make_zerf,Epoch}, {yyy_hack, get(yyy_hack)}, {up,Up},{maj,MajoritySize}]}, - %% io:format(user, "ZERF ~w\n",[machi_projection:make_summary(Proj2)]), Proj2 catch throw:{zerf,no_common} -> @@ -2976,3 +3027,10 @@ get_unfit_list(FitnessServer) -> [] end. +get_projection_store_pid_or_regname(#ch_mgr{name=MyName, opts=MgrOpts}) -> + case get_projection_store_regname(MgrOpts) of + undefined -> + machi_flu_psup:make_proj_supname(MyName); + PStr -> + PStr + end. diff --git a/src/machi_projection.erl b/src/machi_projection.erl index f6e7cbc..c977fd9 100644 --- a/src/machi_projection.erl +++ b/src/machi_projection.erl @@ -174,6 +174,7 @@ make_summary(#projection_v1{epoch_number=EpochNum, repairing=Repairing_list, dbg=Dbg, dbg2=Dbg2}) -> [{epoch,EpochNum}, {csum,_CSum4}, +{all, _All_list}, {author,Author}, {mode,CMode},{witnesses, Witness_list}, {upi,UPI_list},{repair,Repairing_list},{down,Down_list}] ++ [{d,Dbg}, {d2,Dbg2}]. diff --git a/test/machi_chain_manager1_test.erl b/test/machi_chain_manager1_test.erl index 27e40ec..46aff1f 100644 --- a/test/machi_chain_manager1_test.erl +++ b/test/machi_chain_manager1_test.erl @@ -335,7 +335,8 @@ smoke1_test2() -> nonunanimous_setup_and_fix_test() -> machi_partition_simulator:start_link({1,2,3}, 100, 0), TcpPort = 62877, - FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}], + FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}, + {c,TcpPort+2,"./data.c"}], P_s = [#p_srvr{name=Name, address="localhost", port=Port} || {Name,Port,_Dir} <- FluInfo], @@ -345,10 +346,11 @@ nonunanimous_setup_and_fix_test() -> %% {ok, Mb} = ?MGR:start_link(b, MembersDict, [{active_mode, false}]++XX), [{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) || {Name,Port,Dir} <- FluInfo], - [Proxy_a, Proxy_b] = Proxies = + Proxies = [Proxy_a, Proxy_b, Proxy_c] = [element(2,?FLU_PC:start_link(P)) || P <- P_s], - MembersDict = machi_projection:make_members_dict(P_s), - [Ma,Mb] = [a_chmgr, b_chmgr], + %% MembersDict = machi_projection:make_members_dict(P_s), + MembersDict = machi_projection:make_members_dict(lists:sublist(P_s, 2)), + Mgrs = [Ma,Mb,Mc] = [a_chmgr, b_chmgr, c_chmgr], ok = machi_chain_manager1:set_chain_members(Ma, MembersDict), ok = machi_chain_manager1:set_chain_members(Mb, MembersDict), try @@ -387,10 +389,63 @@ nonunanimous_setup_and_fix_test() -> {ok, P2pb} = ?FLU_PC:read_latest_projection(Proxy_b, private), P2 = P2pb#projection_v1{dbg2=[]}, - %% Pspam = machi_projection:update_checksum( - %% P1b#projection_v1{epoch_number=?SPAM_PROJ_EPOCH, - %% dbg=[hello_spam]}), - %% ok = ?FLU_PC:write_projection(Proxy_b, public, Pspam), + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format(user, "\n+ Add a 3rd member to the chain.\n", []), + io:format(user, "\nNOTE: One INFO REPORT will follow.\n", []), + timer:sleep(50), + + MembersDict3 = machi_projection:make_members_dict(P_s), + ok = machi_chain_manager1:set_chain_members( + Ma, ch_not_def_yet, EpochNum_a, ap_mode, MembersDict3, []), + + [begin + {_rr1, _rr2, TheEpoch} = ?MGR:trigger_react_to_env(Mgr), + ok + end || _ <- lists:seq(1, 7), + {Mgr,Proxy} <- [{Ma, Proxy_a}, {Mb, Proxy_b}, {Mc, Proxy_c}]], + {_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Ma), + {_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Mb), + {_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Mc), + [{ok, #projection_v1{upi=[a,b], repairing=[c]}} = + ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies], + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format(user, "\n+ Remove 'a' from the chain.\n", []), + io:format(user, "\nNOTE: One INFO REPORT will follow.\n", []), + timer:sleep(50), + + MembersDict4 = machi_projection:make_members_dict(tl(P_s)), + ok = machi_chain_manager1:set_chain_members( + Mb, ch_not_def_yet, TheEpoch_3, ap_mode, MembersDict4, []), + + [begin + _ = ?MGR:trigger_react_to_env(Mgr), + ok + end || _ <- lists:seq(1, 7), + {Mgr,Proxy} <- [{Ma, Proxy_a}, {Mb, Proxy_b}, {Mc, Proxy_c}]], + %% {_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Ma), + {_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Mb), + {_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Mc), + [{ok, #projection_v1{upi=[b], repairing=[c]}} = + ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)], + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format(user, "\n+ Add a to the chain again.\n", []), + + MembersDict5 = machi_projection:make_members_dict(P_s), + ok = machi_chain_manager1:set_chain_members( + Mb, ch_not_def_yet, TheEpoch_4, ap_mode, MembersDict5, []), + + [begin + {_rr1, _rr2, TheEpoch} = ?MGR:trigger_react_to_env(Mgr), + ok + end || _ <- lists:seq(1, 7), + {Mgr,Proxy} <- [{Ma, Proxy_a}, {Mb, Proxy_b}, {Mc, Proxy_c}]], + {_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Ma), + {_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Mb), + {_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Mc), + [{ok, #projection_v1{upi=[b], repairing=[a,c]}} = + ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies], ok after diff --git a/test/machi_pb_high_client_test.erl b/test/machi_pb_high_client_test.erl index 361eb55..9769dd1 100644 --- a/test/machi_pb_high_client_test.erl +++ b/test/machi_pb_high_client_test.erl @@ -64,11 +64,13 @@ smoke_test2() -> PK = <<>>, Prefix = <<"prefix">>, Chunk1 = <<"Hello, chunk!">>, +io:format(user, "~s LINE ~p\n", [?MODULE, ?LINE]), {ok, {Off1, Size1, File1}} = ?C:append_chunk(Clnt, PK, Prefix, Chunk1, none, 0), true = is_binary(File1), Chunk2 = "It's another chunk", CSum2 = {client_sha, machi_util:checksum_chunk(Chunk2)}, +io:format(user, "~s LINE ~p\n", [?MODULE, ?LINE]), {ok, {Off2, Size2, File2}} = ?C:append_chunk(Clnt, PK, Prefix, Chunk2, CSum2, 1024), Chunk3 = ["This is a ", <<"test,">>, 32, [["Hello, world!"]]], @@ -113,6 +115,7 @@ smoke_test2() -> LargeBytes = binary:copy(<<"x">>, 1024*1024), LBCsum = {client_sha, machi_util:checksum_chunk(LargeBytes)}, +io:format(user, "~s LINE ~p\n", [?MODULE, ?LINE]), {ok, {Offx, Sizex, Filex}} = ?C:append_chunk(Clnt, PK, Prefix, LargeBytes, LBCsum, 0), ok = ?C:trim_chunk(Clnt, Filex, Offx, Sizex), -- 2.45.2 From 1bc9033076e6fe76821524e56a80a77bc2761ce0 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Mon, 7 Dec 2015 22:15:23 +0900 Subject: [PATCH 25/77] Yay, all tests pass! --- src/machi_chain_manager1.erl | 6 ++++-- test/machi_chain_manager1_test.erl | 10 ++++------ test/machi_pb_high_client_test.erl | 5 ++--- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/machi_chain_manager1.erl b/src/machi_chain_manager1.erl index da30e6e..768de36 100644 --- a/src/machi_chain_manager1.erl +++ b/src/machi_chain_manager1.erl @@ -1255,7 +1255,7 @@ react_to_env_A20(Retries, #ch_mgr{name=MyName, proj=P_current}=S) -> P_none0 = make_none_projection(Epoch, MyName, All_list, Witness_list, MembersDict), P_none = P_none0#projection_v1{chain_name=ChainName}, - {{x,y,z,42,77}, S2#ch_mgr{proj=P_none}}; + {{x,y,z,42,77}, set_proj(S2, P_none)}; _ -> react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S2) end. @@ -1463,6 +1463,8 @@ react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP, not ordsets:is_disjoint(P_latest_s, Down_s) end, AmExcludedFromLatestAll_p = + P_latest#projection_v1.epoch_number /= 0 + andalso (not lists:member(MyName, P_latest#projection_v1.all_members)), ?REACT({a40, ?LINE, [{latest_author, P_latest#projection_v1.author_server}, @@ -1473,7 +1475,7 @@ react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP, if AmExcludedFromLatestAll_p -> - ?REACT({a40, ?LINE, []}), + ?REACT({a40, ?LINE, [{latest,machi_projection:make_summary(P_latest)}]}), react_to_env_A50(P_latest, [], S); AmHosedP -> diff --git a/test/machi_chain_manager1_test.erl b/test/machi_chain_manager1_test.erl index 46aff1f..b791abc 100644 --- a/test/machi_chain_manager1_test.erl +++ b/test/machi_chain_manager1_test.erl @@ -390,9 +390,8 @@ nonunanimous_setup_and_fix_test() -> P2 = P2pb#projection_v1{dbg2=[]}, %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - io:format(user, "\n+ Add a 3rd member to the chain.\n", []), + %% io:format(user, "\n+ Add a 3rd member to the chain.\n", []), io:format(user, "\nNOTE: One INFO REPORT will follow.\n", []), - timer:sleep(50), MembersDict3 = machi_projection:make_members_dict(P_s), ok = machi_chain_manager1:set_chain_members( @@ -410,9 +409,8 @@ nonunanimous_setup_and_fix_test() -> ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies], %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - io:format(user, "\n+ Remove 'a' from the chain.\n", []), - io:format(user, "\nNOTE: One INFO REPORT will follow.\n", []), - timer:sleep(50), + %% io:format(user, "\n+ Remove 'a' from the chain.\n", []), + io:format(user, "NOTE: One INFO REPORT will follow.\n", []), MembersDict4 = machi_projection:make_members_dict(tl(P_s)), ok = machi_chain_manager1:set_chain_members( @@ -430,7 +428,7 @@ nonunanimous_setup_and_fix_test() -> ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)], %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - io:format(user, "\n+ Add a to the chain again.\n", []), + %% io:format(user, "\n+ Add a to the chain again.\n", []), MembersDict5 = machi_projection:make_members_dict(P_s), ok = machi_chain_manager1:set_chain_members( diff --git a/test/machi_pb_high_client_test.erl b/test/machi_pb_high_client_test.erl index 9769dd1..1283d00 100644 --- a/test/machi_pb_high_client_test.erl +++ b/test/machi_pb_high_client_test.erl @@ -49,6 +49,8 @@ smoke_test2() -> end || P <- Ps], ok = machi_chain_manager1:set_chain_members(a_chmgr, D), [machi_chain_manager1:trigger_react_to_env(a_chmgr) || _ <-lists:seq(1,5)], + {ok, PQQ} = machi_projection_store:read_latest_projection(a_pstore, public), + io:format(user, "a's proj: ~w\n", [machi_projection:make_summary(PQQ)]), {ok, Clnt} = ?C:start_link(Ps), try @@ -64,13 +66,11 @@ smoke_test2() -> PK = <<>>, Prefix = <<"prefix">>, Chunk1 = <<"Hello, chunk!">>, -io:format(user, "~s LINE ~p\n", [?MODULE, ?LINE]), {ok, {Off1, Size1, File1}} = ?C:append_chunk(Clnt, PK, Prefix, Chunk1, none, 0), true = is_binary(File1), Chunk2 = "It's another chunk", CSum2 = {client_sha, machi_util:checksum_chunk(Chunk2)}, -io:format(user, "~s LINE ~p\n", [?MODULE, ?LINE]), {ok, {Off2, Size2, File2}} = ?C:append_chunk(Clnt, PK, Prefix, Chunk2, CSum2, 1024), Chunk3 = ["This is a ", <<"test,">>, 32, [["Hello, world!"]]], @@ -115,7 +115,6 @@ io:format(user, "~s LINE ~p\n", [?MODULE, ?LINE]), LargeBytes = binary:copy(<<"x">>, 1024*1024), LBCsum = {client_sha, machi_util:checksum_chunk(LargeBytes)}, -io:format(user, "~s LINE ~p\n", [?MODULE, ?LINE]), {ok, {Offx, Sizex, Filex}} = ?C:append_chunk(Clnt, PK, Prefix, LargeBytes, LBCsum, 0), ok = ?C:trim_chunk(Clnt, Filex, Offx, Sizex), -- 2.45.2 From ef10ebed229d1ba5076424179ea42917f3314f30 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 8 Dec 2015 14:50:16 +0900 Subject: [PATCH 26/77] WIP: now trying to diagnose fitness server bug? --- include/machi_projection.hrl | 4 +- src/machi_chain_bootstrap.erl | 8 +- src/machi_fitness.erl | 2 + test/machi_chain_manager1_test.erl | 124 ++++++++++++++++++++++++----- 4 files changed, 110 insertions(+), 28 deletions(-) diff --git a/include/machi_projection.hrl b/include/machi_projection.hrl index 5ad4e54..c70eac0 100644 --- a/include/machi_projection.hrl +++ b/include/machi_projection.hrl @@ -80,8 +80,8 @@ -record(chain_def_v1, { name :: atom(), mode :: pv1_consistency_mode(), - upi :: [pv1_server()], - witnesses :: [pv1_server()] + upi :: [p_srvr()], + witnesses :: [p_srvr()] }). -endif. % !MACHI_PROJECTION_HRL diff --git a/src/machi_chain_bootstrap.erl b/src/machi_chain_bootstrap.erl index 2406401..c9dc49f 100644 --- a/src/machi_chain_bootstrap.erl +++ b/src/machi_chain_bootstrap.erl @@ -120,20 +120,20 @@ sanitize_chain_def_rec(Whole, {Acc, D}) -> {Acc, D} end. -perhaps_bootstrap_chains([], _FLUs) -> +perhaps_bootstrap_chains([], _LocalFLUs_at_zero) -> ok; -perhaps_bootstrap_chains([CD|ChainDefs], FLUs) -> +perhaps_bootstrap_chains([CD|ChainDefs], LocalFLUs_at_zero) -> #chain_def_v1{upi=UPI, witnesses=Witnesses} = CD, AllNames = [Name || #p_srvr{name=Name} <- UPI ++ Witnesses], case ordsets:intersection(ordsets:from_list(AllNames), - ordsets:from_list(FLUs)) of + ordsets:from_list(LocalFLUs_at_zero)) of [] -> io:format(user, "TODO: no local flus in ~P\n", [CD, 10]), ok; [FLU1|_] -> bootstrap_chain(CD, FLU1) end, - perhaps_bootstrap_chains(ChainDefs, FLUs). + perhaps_bootstrap_chains(ChainDefs, LocalFLUs_at_zero). bootstrap_chain(CD, FLU) -> io:format(user, "TODO: config ~p as bootstrap member of ~p\n", [FLU, CD]), diff --git a/src/machi_fitness.erl b/src/machi_fitness.erl index bf16198..f37281c 100644 --- a/src/machi_fitness.erl +++ b/src/machi_fitness.erl @@ -95,11 +95,13 @@ init([{MyFluName}|Args]) -> handle_call({get_unfit_list}, _From, #state{active_unfit=ActiveUnfit}=S) -> Reply = ActiveUnfit, +io:format(user, "get_unfit_list ~p: ~p\n", [S#state.my_flu_name, Reply]), {reply, Reply, S}; handle_call({update_local_down_list, Down, MembersDict}, _From, #state{my_flu_name=MyFluName, pending_map=OldMap, local_down=OldDown, members_dict=OldMembersDict, admin_down=AdminDown}=S) -> +io:format(user, "update_local_down_list: ~w: down ~w md ~W\n", [S#state.my_flu_name, Down, MembersDict, 10]), NewMap = store_in_map(OldMap, MyFluName, erlang:now(), Down, AdminDown, [props_yo]), S2 = if Down == OldDown, MembersDict == OldMembersDict -> diff --git a/test/machi_chain_manager1_test.erl b/test/machi_chain_manager1_test.erl index b791abc..efecf5e 100644 --- a/test/machi_chain_manager1_test.erl +++ b/test/machi_chain_manager1_test.erl @@ -332,7 +332,11 @@ smoke1_test2() -> ok = machi_partition_simulator:stop() end. -nonunanimous_setup_and_fix_test() -> +nonunanimous_setup_and_fix_test_() -> + os:cmd("rm -f /tmp/moomoo.*"), + {timeout, 1*60, fun() -> nonunanimous_setup_and_fix_test2() end}. + +nonunanimous_setup_and_fix_test2() -> machi_partition_simulator:start_link({1,2,3}, 100, 0), TcpPort = 62877, FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}, @@ -342,8 +346,7 @@ nonunanimous_setup_and_fix_test() -> [machi_flu1_test:clean_up_data_dir(Dir) || {_,_,Dir} <- FluInfo], {ok, SupPid} = machi_flu_sup:start_link(), - Opts = [{active_mode, false}], - %% {ok, Mb} = ?MGR:start_link(b, MembersDict, [{active_mode, false}]++XX), + Opts = [{active_mode, false}, {initial_wedged, true}], [{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) || {Name,Port,Dir} <- FluInfo], Proxies = [Proxy_a, Proxy_b, Proxy_c] = @@ -351,8 +354,17 @@ nonunanimous_setup_and_fix_test() -> %% MembersDict = machi_projection:make_members_dict(P_s), MembersDict = machi_projection:make_members_dict(lists:sublist(P_s, 2)), Mgrs = [Ma,Mb,Mc] = [a_chmgr, b_chmgr, c_chmgr], + MgrProxies = [{Ma, Proxy_a}, {Mb, Proxy_b}, {Mc, Proxy_c}], + Advance = fun() -> + [begin + catch ?MGR:trigger_react_to_env(Mgr), + ok + end || _ <- lists:seq(1, 7), + {Mgr,Proxy} <- MgrProxies] + end, ok = machi_chain_manager1:set_chain_members(Ma, MembersDict), ok = machi_chain_manager1:set_chain_members(Mb, MembersDict), + try {ok, P1} = ?MGR:test_calc_projection(Ma, false), @@ -390,18 +402,14 @@ nonunanimous_setup_and_fix_test() -> P2 = P2pb#projection_v1{dbg2=[]}, %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% io:format(user, "\n+ Add a 3rd member to the chain.\n", []), + %% io:format(user, "\nSTEP: Add a 3rd member to the chain.\n", []), io:format(user, "\nNOTE: One INFO REPORT will follow.\n", []), MembersDict3 = machi_projection:make_members_dict(P_s), ok = machi_chain_manager1:set_chain_members( Ma, ch_not_def_yet, EpochNum_a, ap_mode, MembersDict3, []), - [begin - {_rr1, _rr2, TheEpoch} = ?MGR:trigger_react_to_env(Mgr), - ok - end || _ <- lists:seq(1, 7), - {Mgr,Proxy} <- [{Ma, Proxy_a}, {Mb, Proxy_b}, {Mc, Proxy_c}]], + Advance(), {_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Ma), {_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Mb), {_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Mc), @@ -409,42 +417,114 @@ nonunanimous_setup_and_fix_test() -> ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies], %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% io:format(user, "\n+ Remove 'a' from the chain.\n", []), + %% io:format(user, "STEP: Remove 'a' from the chain.\n", []), io:format(user, "NOTE: One INFO REPORT will follow.\n", []), MembersDict4 = machi_projection:make_members_dict(tl(P_s)), ok = machi_chain_manager1:set_chain_members( Mb, ch_not_def_yet, TheEpoch_3, ap_mode, MembersDict4, []), - [begin - _ = ?MGR:trigger_react_to_env(Mgr), - ok - end || _ <- lists:seq(1, 7), - {Mgr,Proxy} <- [{Ma, Proxy_a}, {Mb, Proxy_b}, {Mc, Proxy_c}]], - %% {_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Ma), + Advance(), + {ok, {true, _}} = ?FLU_PC:wedge_status(Proxy_a), {_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Mb), {_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Mc), [{ok, #projection_v1{upi=[b], repairing=[c]}} = ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)], %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% io:format(user, "\n+ Add a to the chain again.\n", []), + %% io:format(user, "STEP: Add a to the chain again (a is running).\n", []), MembersDict5 = machi_projection:make_members_dict(P_s), ok = machi_chain_manager1:set_chain_members( Mb, ch_not_def_yet, TheEpoch_4, ap_mode, MembersDict5, []), - [begin - {_rr1, _rr2, TheEpoch} = ?MGR:trigger_react_to_env(Mgr), - ok - end || _ <- lists:seq(1, 7), - {Mgr,Proxy} <- [{Ma, Proxy_a}, {Mb, Proxy_b}, {Mc, Proxy_c}]], + Advance(), {_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Ma), {_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Mb), {_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Mc), [{ok, #projection_v1{upi=[b], repairing=[a,c]}} = ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies], + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format(user, "STEP: Stop a while a chain member, advance b&c.\n", []), + + ok = machi_flu_psup:stop_flu_package(a), + Advance(), + {_, _, TheEpoch_6} = ?MGR:trigger_react_to_env(Mb), + {_, _, TheEpoch_6} = ?MGR:trigger_react_to_env(Mc), + [{ok, #projection_v1{upi=[b], repairing=[c]}} = + ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)], + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format(user, "STEP: Remove 'a' from the chain.\n", []), + io:format(user, "NOTE: One INFO REPORT will follow.\n", []), + + MembersDict7 = machi_projection:make_members_dict(tl(P_s)), + ok = machi_chain_manager1:set_chain_members( + Mb, ch_not_def_yet, TheEpoch_6, ap_mode, MembersDict7, []), + + Advance(), + {_, _, TheEpoch_7} = ?MGR:trigger_react_to_env(Mb), + {_, _, TheEpoch_7} = ?MGR:trigger_react_to_env(Mc), + [{ok, #projection_v1{upi=[b], repairing=[c]}} = + ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)], + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format(user, "STEP: Start a, advance.\n", []), + + [{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) || + {Name,Port,Dir} <- [hd(FluInfo)]], + Advance(), + {ok, {true, _}} = ?FLU_PC:wedge_status(Proxy_a), + {ok, {false, EpochID_8}} = ?FLU_PC:wedge_status(Proxy_b), + {ok, {false, EpochID_8}} = ?FLU_PC:wedge_status(Proxy_c), + [{ok, #projection_v1{upi=[b], repairing=[c]}} = + ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)], + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format(user, "STEP: Stop a, delete a's data, leave it stopped\n", []), + ok = machi_flu_psup:stop_flu_package(a), + Advance(), + {_,_,Dir_a} = hd(FluInfo), + [machi_flu1_test:clean_up_data_dir(Dir) || {_,_,Dir} <- [hd(FluInfo)]], + {ok, {false, _}} = ?FLU_PC:wedge_status(Proxy_b), + {ok, {false, _}} = ?FLU_PC:wedge_status(Proxy_c), + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format(user, "STEP: Add a to the chain again (a is stopped).\n", []), + + MembersDict9 = machi_projection:make_members_dict(P_s), + {_, _, TheEpoch_9} = ?MGR:trigger_react_to_env(Mb), + ok = machi_chain_manager1:set_chain_members( + Mb, ch_not_def_yet, TheEpoch_9, ap_mode, MembersDict9, []), + Advance(), + [begin + {ok, Pqq} = ?FLU_PC:read_latest_projection(Pxy, private), + io:format(user, "At ~w: ~w\n", [Pxy, machi_projection:make_summary(Pqq#projection_v1{dbg2=[]})]) + end || Pxy <- tl(Proxies)], + {_, _, TheEpoch_9b} = ?MGR:trigger_react_to_env(Mb), + true = (TheEpoch_9b > TheEpoch_9), + [{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) || + {Name,Port,Dir} <- [hd(FluInfo)]], + Advance(), + os:cmd("touch /tmp/moomoo.c"), + [begin + Qzx = ?MGR:trigger_react_to_env(Mgr), + io:format(user, "dbg: ~w: ~w\n", [Mgr, Qzx]), + ok + end || _ <- lists:seq(1,1), + {Mgr,Proxy} <- MgrProxies], + [begin + {ok, Pqq} = ?FLU_PC:read_latest_projection(Pxy, private), + io:format(user, "At ~w: ~w\n", [Pxy, machi_projection:make_summary(Pqq#projection_v1{dbg2=[]})]) + end || Pxy <- Proxies], + [io:format(user, "Unfit @ ~w: ~p\n", [Xii, machi_fitness:get_unfit_list(machi_fitness)]) || Xii <- [a_fitness, b_fitness, c_fitness] ], + {ok, {true, {0,_}}} = ?FLU_PC:wedge_status(Proxy_a), + {_, _, TheEpoch_9c} = ?MGR:trigger_react_to_env(Ma), + {_, _, TheEpoch_9c} = ?MGR:trigger_react_to_env(Mb), + {_, _, TheEpoch_9c} = ?MGR:trigger_react_to_env(Mc), + [{ok, #projection_v1{upi=[b], repairing=[a,c]}} = + ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies], ok after exit(SupPid, normal), -- 2.45.2 From 27e8a31307fdce15768f90d513357962d0f11614 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 8 Dec 2015 15:27:47 +0900 Subject: [PATCH 27/77] Fix fitness timing problem with short-circuit +trigger_early_adjustment/2 --- src/machi_fitness.erl | 12 ++++++--- test/machi_chain_manager1_test.erl | 42 ++++++++++++------------------ 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/machi_fitness.erl b/src/machi_fitness.erl index f37281c..2b54244 100644 --- a/src/machi_fitness.erl +++ b/src/machi_fitness.erl @@ -39,7 +39,8 @@ get_unfit_list/1, update_local_down_list/3, add_admin_down/3, delete_admin_down/2, send_fitness_update_spam/3, - send_spam_to_everyone/1]). + send_spam_to_everyone/1, + trigger_early_adjustment/2]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -81,6 +82,13 @@ send_fitness_update_spam(Pid, FromName, Dict) -> send_spam_to_everyone(Pid) -> gen_server:call(Pid, {send_spam_to_everyone}, infinity). +%% @doc For testing purposes, we don't want a test to wait for +%% wall-clock time to elapse before the fitness server makes a +%% down->up status decision. + +trigger_early_adjustment(Pid, FLU) -> + Pid ! {adjust_down_list, FLU}. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% init([{MyFluName}|Args]) -> @@ -95,13 +103,11 @@ init([{MyFluName}|Args]) -> handle_call({get_unfit_list}, _From, #state{active_unfit=ActiveUnfit}=S) -> Reply = ActiveUnfit, -io:format(user, "get_unfit_list ~p: ~p\n", [S#state.my_flu_name, Reply]), {reply, Reply, S}; handle_call({update_local_down_list, Down, MembersDict}, _From, #state{my_flu_name=MyFluName, pending_map=OldMap, local_down=OldDown, members_dict=OldMembersDict, admin_down=AdminDown}=S) -> -io:format(user, "update_local_down_list: ~w: down ~w md ~W\n", [S#state.my_flu_name, Down, MembersDict, 10]), NewMap = store_in_map(OldMap, MyFluName, erlang:now(), Down, AdminDown, [props_yo]), S2 = if Down == OldDown, MembersDict == OldMembersDict -> diff --git a/test/machi_chain_manager1_test.erl b/test/machi_chain_manager1_test.erl index efecf5e..e143d76 100644 --- a/test/machi_chain_manager1_test.erl +++ b/test/machi_chain_manager1_test.erl @@ -356,11 +356,14 @@ nonunanimous_setup_and_fix_test2() -> Mgrs = [Ma,Mb,Mc] = [a_chmgr, b_chmgr, c_chmgr], MgrProxies = [{Ma, Proxy_a}, {Mb, Proxy_b}, {Mc, Proxy_c}], Advance = fun() -> - [begin - catch ?MGR:trigger_react_to_env(Mgr), - ok - end || _ <- lists:seq(1, 7), - {Mgr,Proxy} <- MgrProxies] + [begin + [catch machi_fitness:trigger_early_adjustment(Fit, Tgt) || + Fit <- [a_fitness,b_fitness,c_fitness], + Tgt <- [a,b,c] ], + [catch ?MGR:trigger_react_to_env(Mgr) || + {Mgr,_Proxy} <- MgrProxies], + ok + end || _ <- lists:seq(1, 3)] end, ok = machi_chain_manager1:set_chain_members(Ma, MembersDict), ok = machi_chain_manager1:set_chain_members(Mb, MembersDict), @@ -498,32 +501,19 @@ nonunanimous_setup_and_fix_test2() -> ok = machi_chain_manager1:set_chain_members( Mb, ch_not_def_yet, TheEpoch_9, ap_mode, MembersDict9, []), Advance(), - [begin - {ok, Pqq} = ?FLU_PC:read_latest_projection(Pxy, private), - io:format(user, "At ~w: ~w\n", [Pxy, machi_projection:make_summary(Pqq#projection_v1{dbg2=[]})]) - end || Pxy <- tl(Proxies)], {_, _, TheEpoch_9b} = ?MGR:trigger_react_to_env(Mb), true = (TheEpoch_9b > TheEpoch_9), + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format(user, "STEP: Start a, and it joins like it ought to\n", []), + [{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) || {Name,Port,Dir} <- [hd(FluInfo)]], Advance(), - os:cmd("touch /tmp/moomoo.c"), - [begin - Qzx = ?MGR:trigger_react_to_env(Mgr), - io:format(user, "dbg: ~w: ~w\n", [Mgr, Qzx]), - ok - end || _ <- lists:seq(1,1), - {Mgr,Proxy} <- MgrProxies], - [begin - {ok, Pqq} = ?FLU_PC:read_latest_projection(Pxy, private), - io:format(user, "At ~w: ~w\n", [Pxy, machi_projection:make_summary(Pqq#projection_v1{dbg2=[]})]) - end || Pxy <- Proxies], - [io:format(user, "Unfit @ ~w: ~p\n", [Xii, machi_fitness:get_unfit_list(machi_fitness)]) || Xii <- [a_fitness, b_fitness, c_fitness] ], - {ok, {true, {0,_}}} = ?FLU_PC:wedge_status(Proxy_a), - {_, _, TheEpoch_9c} = ?MGR:trigger_react_to_env(Ma), - {_, _, TheEpoch_9c} = ?MGR:trigger_react_to_env(Mb), - {_, _, TheEpoch_9c} = ?MGR:trigger_react_to_env(Mc), - [{ok, #projection_v1{upi=[b], repairing=[a,c]}} = + {ok, {false, {TheEpoch10,_}}} = ?FLU_PC:wedge_status(Proxy_a), + {ok, {false, {TheEpoch10,_}}} = ?FLU_PC:wedge_status(Proxy_b), + {ok, {false, {TheEpoch10,_}}} = ?FLU_PC:wedge_status(Proxy_c), + [{ok, #projection_v1{upi=[b], repairing=[c,a]}} = ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies], ok after -- 2.45.2 From 3391c898188ce72cd84dd6ea7104edee66b0fff9 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 8 Dec 2015 16:29:56 +0900 Subject: [PATCH 28/77] Clean up verbosity of nonunanimous_setup_and_fix_test2() --- test/machi_chain_manager1_test.erl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/machi_chain_manager1_test.erl b/test/machi_chain_manager1_test.erl index e143d76..7e77384 100644 --- a/test/machi_chain_manager1_test.erl +++ b/test/machi_chain_manager1_test.erl @@ -337,6 +337,7 @@ nonunanimous_setup_and_fix_test_() -> {timeout, 1*60, fun() -> nonunanimous_setup_and_fix_test2() end}. nonunanimous_setup_and_fix_test2() -> + error_logger:tty(false), machi_partition_simulator:start_link({1,2,3}, 100, 0), TcpPort = 62877, FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}, @@ -405,8 +406,7 @@ nonunanimous_setup_and_fix_test2() -> P2 = P2pb#projection_v1{dbg2=[]}, %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% io:format(user, "\nSTEP: Add a 3rd member to the chain.\n", []), - io:format(user, "\nNOTE: One INFO REPORT will follow.\n", []), + io:format("\nSTEP: Add a 3rd member to the chain.\n", []), MembersDict3 = machi_projection:make_members_dict(P_s), ok = machi_chain_manager1:set_chain_members( @@ -420,8 +420,7 @@ nonunanimous_setup_and_fix_test2() -> ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies], %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% io:format(user, "STEP: Remove 'a' from the chain.\n", []), - io:format(user, "NOTE: One INFO REPORT will follow.\n", []), + io:format("STEP: Remove 'a' from the chain.\n", []), MembersDict4 = machi_projection:make_members_dict(tl(P_s)), ok = machi_chain_manager1:set_chain_members( @@ -435,7 +434,7 @@ nonunanimous_setup_and_fix_test2() -> ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)], %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - %% io:format(user, "STEP: Add a to the chain again (a is running).\n", []), + io:format("STEP: Add a to the chain again (a is running).\n", []), MembersDict5 = machi_projection:make_members_dict(P_s), ok = machi_chain_manager1:set_chain_members( @@ -449,7 +448,7 @@ nonunanimous_setup_and_fix_test2() -> ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies], %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - io:format(user, "STEP: Stop a while a chain member, advance b&c.\n", []), + io:format("STEP: Stop a while a chain member, advance b&c.\n", []), ok = machi_flu_psup:stop_flu_package(a), Advance(), @@ -459,8 +458,7 @@ nonunanimous_setup_and_fix_test2() -> ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)], %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - io:format(user, "STEP: Remove 'a' from the chain.\n", []), - io:format(user, "NOTE: One INFO REPORT will follow.\n", []), + io:format("STEP: Remove 'a' from the chain.\n", []), MembersDict7 = machi_projection:make_members_dict(tl(P_s)), ok = machi_chain_manager1:set_chain_members( @@ -473,7 +471,7 @@ nonunanimous_setup_and_fix_test2() -> ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)], %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - io:format(user, "STEP: Start a, advance.\n", []), + io:format("STEP: Start a, advance.\n", []), [{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) || {Name,Port,Dir} <- [hd(FluInfo)]], @@ -485,7 +483,8 @@ nonunanimous_setup_and_fix_test2() -> ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)], %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - io:format(user, "STEP: Stop a, delete a's data, leave it stopped\n", []), + io:format("STEP: Stop a, delete a's data, leave it stopped\n", []), + ok = machi_flu_psup:stop_flu_package(a), Advance(), {_,_,Dir_a} = hd(FluInfo), @@ -494,7 +493,7 @@ nonunanimous_setup_and_fix_test2() -> {ok, {false, _}} = ?FLU_PC:wedge_status(Proxy_c), %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - io:format(user, "STEP: Add a to the chain again (a is stopped).\n", []), + io:format("STEP: Add a to the chain again (a is stopped).\n", []), MembersDict9 = machi_projection:make_members_dict(P_s), {_, _, TheEpoch_9} = ?MGR:trigger_react_to_env(Mb), @@ -505,7 +504,7 @@ nonunanimous_setup_and_fix_test2() -> true = (TheEpoch_9b > TheEpoch_9), %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - io:format(user, "STEP: Start a, and it joins like it ought to\n", []), + io:format("STEP: Start a, and it joins like it ought to\n", []), [{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) || {Name,Port,Dir} <- [hd(FluInfo)]], @@ -519,7 +518,8 @@ nonunanimous_setup_and_fix_test2() -> after exit(SupPid, normal), [ok = ?FLU_PC:quit(X) || X <- Proxies], - ok = machi_partition_simulator:stop() + ok = machi_partition_simulator:stop(), + error_logger:tty(true) end. unanimous_report_test() -> -- 2.45.2 From 37ac09a680e4f2929a1460fb170808110a095333 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 8 Dec 2015 17:46:11 +0900 Subject: [PATCH 29/77] Rename src/machi_chain_bootstrap.erl -> src/machi_lifecycle_mgr.erl --- src/{machi_chain_bootstrap.erl => machi_lifecycle_mgr.erl} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/{machi_chain_bootstrap.erl => machi_lifecycle_mgr.erl} (100%) diff --git a/src/machi_chain_bootstrap.erl b/src/machi_lifecycle_mgr.erl similarity index 100% rename from src/machi_chain_bootstrap.erl rename to src/machi_lifecycle_mgr.erl -- 2.45.2 From 8285899dbab5addd1d191288079cebd853ff1c36 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 8 Dec 2015 21:16:54 +0900 Subject: [PATCH 30/77] Bootstrap chain @ app init: done, with an example. For example: % make clean % make stage And then configure 3 FLUs: % echo '{p_srvr, a, machi_flu1_client, "localhost", 39000, []}.' > rel/machi/etc/flu-config/a % echo '{p_srvr, b, machi_flu1_client, "localhost", 39001, []}.' > rel/machi/etc/flu-config/b % echo '{p_srvr, c, machi_flu1_client, "localhost", 39002, []}.' > rel/machi/etc/flu-config/c And then configure a chain to use 2 of those 3 FLUs: % echo '{chain_def_v1,c1,ap_mode,[{p_srvr,a,machi_flu1_client,"localhost",39000,[]},{p_srvr,b,machi_flu1_client,"localhost",39001,[]}],[],[]}.' > rel/machi/etc/chain-config/c1 ... then start Machi e.g. % ./rel/machi/bin/machi console ... you should see the following console messages scroll by (including a : =PROGRESS REPORT==== 8-Dec-2015::22:01:44 === supervisor: {local,machi_flu_sup} started: [{pid,<0.145.0>}, {name,a}, {mfargs, {machi_flu_psup,start_link, [a,39000,"./data/flu/a",[]]}}, {restart_type,permanent}, {shutdown,5000}, {child_type,supervisor}] [... and also for the other two FLUs, including a bunch of progress reports for processes that started underneath that sub-supervisor.] 22:01:44.446 [info] Running FLUs: [a,b,c] 22:01:44.446 [info] Running FLUs at epoch 0: [a,b,c] 22:01:44.532 [warning] The following FLUs are defined but are not also members of a defined chain: [c] --- include/machi_projection.hrl | 5 +- rel/files/app.config | 16 +-- src/machi_chain_manager1.erl | 11 +- src/machi_chain_repair.erl | 24 ++-- src/machi_flu_sup.erl | 11 +- src/machi_lifecycle_mgr.erl | 250 +++++++++++++++++++++++++++++++---- src/machi_sup.erl | 6 +- 7 files changed, 264 insertions(+), 59 deletions(-) diff --git a/include/machi_projection.hrl b/include/machi_projection.hrl index c70eac0..e883fc6 100644 --- a/include/machi_projection.hrl +++ b/include/machi_projection.hrl @@ -80,8 +80,9 @@ -record(chain_def_v1, { name :: atom(), mode :: pv1_consistency_mode(), - upi :: [p_srvr()], - witnesses :: [p_srvr()] + full :: [p_srvr()], + witnesses :: [p_srvr()], + props = [] :: list() % proplist for other related info }). -endif. % !MACHI_PROJECTION_HRL diff --git a/rel/files/app.config b/rel/files/app.config index ac3a6a8..ff15eee 100644 --- a/rel/files/app.config +++ b/rel/files/app.config @@ -1,7 +1,7 @@ [ {machi, [ %% Data directory for all FLUs. - {flu_data_dir, "{{platform_data_dir}}"}, + {flu_data_dir, "{{platform_data_dir}}/flu"}, %% FLU config directory {flu_config_dir, "{{platform_etc_dir}}/flu-config"}, @@ -10,15 +10,7 @@ {chain_config_dir, "{{platform_etc_dir}}/chain-config"}, %% FLUs to start at app start. - {initial_flus, [ - %% Remember, this is a list, so separate all tuples - %% with a comma. - %% - %% {Name::atom(), Port::pos_integer(), proplist()} - %% - %% For example: {my_name_is_a, 12500, []} - - ]}, + %% This task has moved to machi_flu_sup and machi_lifecycle_mgr. %% Number of metadata manager processes to run per FLU. %% Default = 10 @@ -27,5 +19,9 @@ %% Do not delete, do not put Machi config items after this line. {final_comma_stopper, do_not_delete} ] + }, + {lager, [ + {error_logger_hwm, 5000} % lager's default of 50/sec is too low + ] } ]. diff --git a/src/machi_chain_manager1.erl b/src/machi_chain_manager1.erl index 768de36..75917d3 100644 --- a/src/machi_chain_manager1.erl +++ b/src/machi_chain_manager1.erl @@ -2598,8 +2598,8 @@ do_repair(#ch_mgr{name=MyName, T1 = os:timestamp(), RepairId = proplists:get_value(repair_id, Opts, id1), error_logger:info_msg( - "Repair start: tail ~p of ~p -> ~p, ~p ID ~w\n", - [MyName, UPI0, Repairing, RepairMode, RepairId]), + "Repair ~w start: tail ~p of ~p -> ~p, ~p\n", + [RepairId, MyName, UPI0, Repairing, RepairMode]), UPI = UPI0 -- Witness_list, Res = machi_chain_repair:repair(RepairMode, MyName, Repairing, UPI, @@ -2612,10 +2612,9 @@ do_repair(#ch_mgr{name=MyName, end, Stats = [{K, ets:lookup_element(ETS, K, 2)} || K <- ETS_T_Keys], error_logger:info_msg( - "Repair ~s: tail ~p of ~p finished ~p repair ID ~w: " - "~p\nStats ~p\n", - [Summary, MyName, UPI0, RepairMode, RepairId, - Res, Stats]), + "Repair ~w ~s: tail ~p of ~p finished ~p: " + "~p Stats: ~p\n", + [RepairId, Summary, MyName, UPI0, RepairMode, Res, Stats]), ets:delete(ETS), exit({repair_final_status, Res}); _ -> diff --git a/src/machi_chain_repair.erl b/src/machi_chain_repair.erl index 1138f9d..0e6606b 100644 --- a/src/machi_chain_repair.erl +++ b/src/machi_chain_repair.erl @@ -103,7 +103,8 @@ repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) -> Add = fun(Name, Pid) -> put(proxies_dict, orddict:store(Name, Pid, get(proxies_dict))) end, OurFLUs = lists:usort([Src] ++ Repairing ++ UPI), % AP assumption! RepairMode = proplists:get_value(repair_mode, Opts, repair), - Verb = proplists:get_value(verbose, Opts, true), + Verb = proplists:get_value(verbose, Opts, false), + RepairId = proplists:get_value(repair_id, Opts, id1), Res = try _ = [begin {ok, Proxy} = machi_proxy_flu1_client:start_link(P), @@ -116,31 +117,38 @@ repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) -> get_file_lists(Proxy, FLU, Dict) end, D, ProxiesDict), MissingFileSummary = make_missing_file_summary(D2, OurFLUs), - ?VERB("MissingFileSummary ~p\n", [MissingFileSummary]), + %% ?VERB("~w MissingFileSummary ~p\n",[RepairId,MissingFileSummary]), + lager:info("Repair ~w MissingFileSummary ~p\n", + [RepairId, MissingFileSummary]), [ets:insert(ETS, {{directive_bytes, FLU}, 0}) || FLU <- OurFLUs], %% Repair files from perspective of Src, i.e. tail(UPI). SrcProxy = orddict:fetch(Src, ProxiesDict), {ok, EpochID} = machi_proxy_flu1_client:get_epoch_id( SrcProxy, ?SHORT_TIMEOUT), - ?VERB("Make repair directives: "), + %% ?VERB("Make repair directives: "), Ds = [{File, make_repair_directives( ConsistencyMode, RepairMode, File, Size, EpochID, Verb, Src, OurFLUs, ProxiesDict, ETS)} || {File, {Size, _MissingList}} <- MissingFileSummary], - ?VERB(" done\n"), + %% ?VERB(" done\n"), + lager:info("Repair ~w repair directives finished\n", [RepairId]), [begin [{_, Bytes}] = ets:lookup(ETS, {directive_bytes, FLU}), - ?VERB("Out-of-sync data for FLU ~p: ~s MBytes\n", - [FLU, mbytes(Bytes)]) + %% ?VERB("Out-of-sync data for FLU ~p: ~s MBytes\n", + %% [FLU, mbytes(Bytes)]), + lager:info("Repair ~w " + "Out-of-sync data for FLU ~p: ~s MBytes\n", + [RepairId, FLU, mbytes(Bytes)]) end || FLU <- OurFLUs], - ?VERB("Execute repair directives: "), + %% ?VERB("Execute repair directives: "), ok = execute_repair_directives(ConsistencyMode, Ds, Src, EpochID, Verb, OurFLUs, ProxiesDict, ETS), - ?VERB(" done\n"), + %% ?VERB(" done\n"), + lager:info("Repair ~w repair directives finished\n", [RepairId]), ok catch What:Why -> diff --git a/src/machi_flu_sup.erl b/src/machi_flu_sup.erl index f13a08d..1b3a502 100644 --- a/src/machi_flu_sup.erl +++ b/src/machi_flu_sup.erl @@ -21,6 +21,9 @@ %% @doc Supervisor for Machi FLU servers and their related support %% servers. %% +%% Responsibility for managing FLU & chain lifecycle after the initial +%% application startup is delegated to {@link machi_lifecycle_mgr}. +%% %% See {@link machi_flu_psup} for an illustration of the entire Machi %% application process structure. @@ -87,8 +90,12 @@ get_initial_flus() -> load_rc_d_files_from_dir(Dir) -> Files = filelib:wildcard(Dir ++ "/*"), lists:append([case file:consult(File) of - {ok, X} -> X; - _ -> [] + {ok, X} -> + X; + _ -> + lager:warning("Error parsing file '~s', ignoring", + [File]), + [] end || File <- Files]). sanitize_p_srvr_records(Ps) -> diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index c9dc49f..b628d52 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -18,7 +18,169 @@ %% %% ------------------------------------------------------------------- --module(machi_chain_bootstrap). +%% @doc Lifecycle manager for Machi FLUs and chains. +%% +%% Over the lifetime of a Machi cluster, both the number and types of +%% FLUs and chains may change. The lifecycle manager is responsible +%% for implementing the lifecycle changes as expressed by "policy". +%% In our case, "policy" is created by an external administrative +%% entity that creates and deletes configuration files that define +%% FLUs and chains relative to this local machine. +%% +%% The "master configuration" for deciding which FLUs should be +%% running on this machine was inspired by BSD UNIX's `init(8)' and the +%% "rc.d" scheme. FLU definitions are found in a single directory, +%% with one file per FLU. Chains are defined similarly, with one +%% definition file per chain. +%% +%% If a definition file for a FLU (or chain) exists, then that +%% FLU/chain ought to be configured into being and running. If a +%% definition file for a FLU/chain is removed, then that FLU/chain +%% should be stopped gracefully. However, deleting of a file destroys +%% information that is stored inside of that file. Therefore, we will +%% not allow arbitrary unlinking of lifecycle config files. If +%% the administrator deletes these config files using `unlink(8)' +%% directly, then "the warranty has been broken". +%% +%% We will rely on using an administrative command to inform the +%% running system to stop and/or delete lifecycle resources. If the +%% Machi application is not running, sorry, please start Machi first. +%% +%% == Wheel reinvention == +%% +%% There's a whole mess of configuration management research & +%% libraries out there. I hope to ignore them all by doing something +%% quick & dirty & good enough here. If I fail, then I'll go +%% pay attention to That Other Stuff. +%% +%% == A note about policy == +%% +%% It is outside of the scope of this local lifecycle manager to make +%% decisions about policy or to distribute policy info/files/whatever +%% to other machines. This is our machine. There are many like it, +%% but this one is ours. +%% +%% == Machi Application Variables == +%% +%% All OTP application environment variables below are defined in the +%% `machi' application. +%% +%%
    +%%
  • flu_config_dir: Stores the `rc.d-'like config files for +%% FLU runtime policy. +%%
    • +%%
  • flu_data_dir: Stores the file data and metadata for +%% all FLUs. +%%
    • +%%
  • chain_config_dir: Stores the `rc.d'-like config files for +%% chain runtime policy. +%%
    • +%%
+%% +%% == The FLU Lifecycle == +%% +%% FLUs on the local machine may be started and stopped, as defined by +%% administrative policy. In order to do any useful work, however, a +%% running FLU must also be configured to be a member of a replication +%% chain. Thus, as a practical matter, both a FLU and the chain that +%% the FLU participates in must both be managed by this manager. +%% +%% When a new `rc.d'-style config file is written to the FLU +%% definition directory, a Machi server process will discover the file +%% within a certain period of time, e.g. 15 seconds. The FLU will be +%% started with the file's specified parameters. A FLU should be +%% defined and started before configuring its chain membership. +%% +%% Usually a FLU is removed implicitly by removing that FLU from the a +%% newer definition file for the chain, or by deleting the entire +%% chain definition. If a FLU has been started but never been a chain +%% member, then the FLU can be stopped & removed explicitly. +%% +%% When a FLU has been removed by policy, the FLU's data files are set +%% aside into a temporary area. An additional policy command may be +%% used to permanently delete such FLUs' data files, i.e. to reclaim +%% disk space. +%% +%% Resources for the FLU are defined in {@link machi_projection.hrl} +%% in the `p_srvr{}' record. The major elements of this record are: +%% +%%
    +%% +%%
  • name :: atom(): The name of the FLU. This name +%% should be unique over the lifetime of the administrative +%% domain and thus managed by outside policy. This name must be +%% the same as the name of the `rc.d'-style config file that +%% defines the FLU. +%%
    • +%%
  • address :: string(): The DNS hostname or IP address +%% used by other servers to communicate with this FLU. +%%
    • +%%
  • port :: non_neg_integer() : The TCP port number that +%% the FLU listens to for incoming Protocol Buffers-serialized +%% communication. +%%
    • +%%
  • props :: property_list(): A general-purpose property +%% list. Its use is currently fluid & not well-defined yet. +%%
    • +%%
+%% +%% == The Chain Lifecycle == +%% +%% If a FLU on the local machine is expected to participate in a +%% replication chain, then an `rc.d'-style chain definition file must +%% also be present on each machine that runs a FLU in the chain. +%% +%% Machi's chains are self-managing, via Humming Consensus; see the +%% [https://github.com/basho/machi/tree/master/doc/] directory for +%% much more detail about Humming Consensus. After FLUs have received +%% their initial chain configuration for Humming Consensus, the FLUs +%% will manage each other (and the chain) themselves. +%% +%% However, Humming Consensus does not handle three chain management +%% problems: 1. specifying the very first chain configuration, +%% 2. altering the membership of the chain (adding/removing FLUs from +%% the chain), or 3. stopping the chain permanently. +%% +%% FLUs in a new chain should have definition files created on each +%% FLU's respective machine prior to defining their chain. Similarly, +%% on each machine that hosts a chain member, a chain definition file +%% created. External policy is responsible for creating each of these +%% files. +%% +%% Resources for the chain are defined in {@link machi_projection.hrl} +%% in the `chain_def_v1{}' record. The major elements of this record are: +%% +%%
    +%% +%%
  • name :: atom(): The name of the chain. This name +%% should be unique over the lifetime of the administrative +%% domain and thus managed by outside policy. This name must be +%% the same as the name of the `rc.d'-style config file that +%% defines the chain. +%%
    • +%%
  • mode :: 'ap_mode' | 'cp_mode': This is the consistency +%% to be used for managing the chain's replicated data: eventual +%% consistency and strong consistency, respectively. +%%
    • +%%
  • full :: [#p_srvr{}] : A list of `#p_srvr{}' records +%% to define the full-service members of the chain. +%%
    • +%%
  • witnesses :: [#p_srvr{}] : A list of `#p_srvr{}' records +%% to define the witness-only members of the chain. Witness servers +%% may only be used with strong consistency mode. +%%
    • +%%
  • props :: property_list(): A general-purpose property +%% list. Its use is currently fluid & not well-defined yet. +%%
    • +%%
+%% +%% == Conflicts with TCP ports, FLU & chain names, etc == +%% +%% This manager is not responsible for managing conflicts in resource +%% namespaces, e.g., TCP port numbers, FLU names, chain names, etc. +%% Managing these namespaces is external policy's responsibility. + +-module(machi_lifecycle_mgr). -behaviour(gen_server). @@ -33,7 +195,10 @@ -define(SERVER, ?MODULE). --record(state, {}). +-record(state, { + flus = [] :: [atom()], + chains = [] :: list() + }). start_link() -> gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). @@ -50,13 +215,7 @@ handle_cast(_Msg, State) -> {noreply, State}. handle_info(finish_init, State) -> - FLUs = get_local_flu_names(), - FLU_Epochs = get_latest_public_epochs(FLUs), - FLUs_at_zero = [FLU || {FLU, 0} <- FLU_Epochs], - lager:info("FLUs at epoch 0: ~p\n", [FLUs_at_zero]), - ChainDefs = get_initial_chains(), - perhaps_bootstrap_chains(ChainDefs, FLUs), - {noreply, State}; + {noreply, finish_init(State)}; handle_info(_Info, State) -> {noreply, State}. @@ -68,7 +227,22 @@ code_change(_OldVsn, State, _Extra) -> %%%%% -get_local_flu_names() -> +finish_init(S) -> + %% machi_flu_sup will start all FLUs that have a valid definition + %% file. That supervisor's structure + OTP supervisor behavior + %% guarantees that those FLUs should now be running. + %% (TODO: Unless they absolutely cannot keep running and the + %% supervisor has given up and terminated them.) + RunningFLUs = get_local_running_flus(), + RunningFLU_Epochs = get_latest_public_epochs(RunningFLUs), + RunningFLUs_at_zero = [FLU || {FLU, 0} <- RunningFLU_Epochs], + lager:info("Running FLUs: ~p\n", [RunningFLUs]), + lager:info("Running FLUs at epoch 0: ~p\n", [RunningFLUs_at_zero]), + ChainDefs = get_initial_chains(), + perhaps_bootstrap_chains(ChainDefs, RunningFLUs_at_zero, RunningFLUs), + S#state{flus=RunningFLUs, chains=ChainDefs}. + +get_local_running_flus() -> [Name || {Name,_,_,_} <- supervisor:which_children(machi_flu_sup)]. get_latest_public_epochs(FLUs) -> @@ -98,7 +272,7 @@ sanitize_chain_def_rec(Whole, {Acc, D}) -> try #chain_def_v1{name=Name, mode=Mode, - upi=UPI, + full=Full, witnesses=Witnesses} = Whole, true = is_atom(Name), NameK = {name, Name}, @@ -107,34 +281,54 @@ sanitize_chain_def_rec(Whole, {Acc, D}) -> IsPSrvr = fun(X) when is_record(X, p_srvr) -> true; (_) -> false end, - true = lists:all(IsPSrvr, UPI), + true = lists:all(IsPSrvr, Full), true = lists:all(IsPSrvr, Witnesses), %% All is sane enough. D2 = dict:store(NameK, Name, D), {[Whole|Acc], D2} - catch _:_ -> + catch _X:_Y -> _ = lager:log(error, self(), - "~s: Bad chain_def record, skipping: ~P\n", - [?MODULE, Whole, 15]), + "~s: Bad chain_def record (~w ~w), skipping: ~P\n", + [?MODULE, _X, _Y, Whole, 15]), {Acc, D} end. -perhaps_bootstrap_chains([], _LocalFLUs_at_zero) -> +perhaps_bootstrap_chains([], LocalFLUs_at_zero, LocalFLUs) -> + if LocalFLUs == [] -> + ok; + true -> + lager:warning("The following FLUs are defined but are not also " + "members of a defined chain: ~w\n", + [LocalFLUs_at_zero]) + end, ok; -perhaps_bootstrap_chains([CD|ChainDefs], LocalFLUs_at_zero) -> - #chain_def_v1{upi=UPI, witnesses=Witnesses} = CD, - AllNames = [Name || #p_srvr{name=Name} <- UPI ++ Witnesses], +perhaps_bootstrap_chains([CD|ChainDefs], LocalFLUs_at_zero, LocalFLUs) -> + #chain_def_v1{full=Full, witnesses=Witnesses} = CD, + AllNames = [Name || #p_srvr{name=Name} <- Full ++ Witnesses], case ordsets:intersection(ordsets:from_list(AllNames), ordsets:from_list(LocalFLUs_at_zero)) of [] -> - io:format(user, "TODO: no local flus in ~P\n", [CD, 10]), - ok; - [FLU1|_] -> - bootstrap_chain(CD, FLU1) - end, - perhaps_bootstrap_chains(ChainDefs, LocalFLUs_at_zero). + perhaps_bootstrap_chains(ChainDefs, LocalFLUs_at_zero, LocalFLUs); + [FLU1|_]=FLUs -> + %% One FLU is enough: Humming Consensus will config the remaining + bootstrap_chain(CD, FLU1), + perhaps_bootstrap_chains(ChainDefs, LocalFLUs_at_zero -- FLUs, + LocalFLUs -- FLUs) + end. -bootstrap_chain(CD, FLU) -> - io:format(user, "TODO: config ~p as bootstrap member of ~p\n", [FLU, CD]), - todo. +bootstrap_chain(#chain_def_v1{name=ChainName, mode=CMode, full=Full, + witnesses=Witnesses, props=Props}=CD, FLU) -> + All_p_srvrs = Full ++ Witnesses, + L = [{Name, P_srvr} || #p_srvr{name=Name}=P_srvr <- All_p_srvrs], + MembersDict = orddict:from_list(L), + Mgr = machi_chain_manager1:make_chmgr_regname(FLU), + case machi_chain_manager1:set_chain_members(Mgr, ChainName, 0, CMode, + MembersDict, Props) of + ok -> + ok; + Else -> + error_logger:warning("Attempt to bootstrap chain ~w via FLU ~w " + "failed: ~w (defn ~w)\n", [Else, CD]), + ok + end. diff --git a/src/machi_sup.erl b/src/machi_sup.erl index 8d5be2a..3e02578 100644 --- a/src/machi_sup.erl +++ b/src/machi_sup.erl @@ -62,8 +62,8 @@ init([]) -> ServerSup = {machi_flu_sup, {machi_flu_sup, start_link, []}, Restart, Shutdown, Type, []}, - ChainBootstrap = - {machi_chain_bootstrap, {machi_chain_bootstrap, start_link, []}, + LifecycleMgr = + {machi_lifecycle_mgr, {machi_lifecycle_mgr, start_link, []}, Restart, Shutdown, worker, []}, - {ok, {SupFlags, [ServerSup, ChainBootstrap]}}. + {ok, {SupFlags, [ServerSup, LifecycleMgr]}}. -- 2.45.2 From 0fc7bc74b7b41363c0c302e8d7df0657550df5e1 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 8 Dec 2015 21:33:16 +0900 Subject: [PATCH 31/77] EDoc fixes --- Makefile | 3 +++ src/machi_csum_table.erl | 2 +- src/machi_dt.erl | 2 +- src/machi_flu_sup.erl | 2 +- src/machi_lifecycle_mgr.erl | 6 +++--- src/machi_pb_high_client.erl | 4 ++-- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 8cf5072..7ff19ed 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,9 @@ deps: clean: $(REBAR) -r clean +edoc: edoc-clean + $(REBAR) skip_deps=true doc + edoc-clean: rm -f edoc/*.png edoc/*.html edoc/*.css edoc/edoc-info diff --git a/src/machi_csum_table.erl b/src/machi_csum_table.erl index 6e5e2b8..f119465 100644 --- a/src/machi_csum_table.erl +++ b/src/machi_csum_table.erl @@ -332,7 +332,7 @@ runthru(_L, _O, _P) -> false. %% @doc If you want to find an overlap among two areas [x, y] and [a, -%% b] where x < y and a < b; if (a-y)*(b-x) < 0 then there's a +%% b] where x < y and a < b; if (a-y)*(b-x) < 0 then there's a %% overlap, else, > 0 then there're no overlap. border condition = 0 %% is not overlap in this offset-size case. inclusion_match_spec(Offset, Size) -> diff --git a/src/machi_dt.erl b/src/machi_dt.erl index ab74f13..9bcc540 100644 --- a/src/machi_dt.erl +++ b/src/machi_dt.erl @@ -44,7 +44,7 @@ -type projection() :: #projection_v1{}. -type projection_type() :: 'public' | 'private'. -%% @doc Tags that stand for how that checksum was generated. See +%% Tags that stand for how that checksum was generated. See %% machi_util:make_tagged_csum/{1,2} for further documentation and %% implementation. -type csum_tag() :: none | client_sha | server_sha | server_regen_sha. diff --git a/src/machi_flu_sup.erl b/src/machi_flu_sup.erl index 1b3a502..6974f36 100644 --- a/src/machi_flu_sup.erl +++ b/src/machi_flu_sup.erl @@ -21,7 +21,7 @@ %% @doc Supervisor for Machi FLU servers and their related support %% servers. %% -%% Responsibility for managing FLU & chain lifecycle after the initial +%% Responsibility for managing FLU and chain lifecycle after the initial %% application startup is delegated to {@link machi_lifecycle_mgr}. %% %% See {@link machi_flu_psup} for an illustration of the entire Machi diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index b628d52..6cc8a47 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -101,7 +101,7 @@ %% used to permanently delete such FLUs' data files, i.e. to reclaim %% disk space. %% -%% Resources for the FLU are defined in {@link machi_projection.hrl} +%% Resources for the FLU are defined in `machi_projection.hrl' %% in the `p_srvr{}' record. The major elements of this record are: %% %%
    @@ -147,7 +147,7 @@ %% created. External policy is responsible for creating each of these %% files. %% -%% Resources for the chain are defined in {@link machi_projection.hrl} +%% Resources for the chain are defined in `machi_projection.hrl' %% in the `chain_def_v1{}' record. The major elements of this record are: %% %%
      @@ -174,7 +174,7 @@ %% %%
    %% -%% == Conflicts with TCP ports, FLU & chain names, etc == +%% == Conflicts with TCP ports, FLU & chain names, etc == %% %% This manager is not responsible for managing conflicts in resource %% namespaces, e.g., TCP port numbers, FLU names, chain names, etc. diff --git a/src/machi_pb_high_client.erl b/src/machi_pb_high_client.erl index ef1e740..8cccbfd 100644 --- a/src/machi_pb_high_client.erl +++ b/src/machi_pb_high_client.erl @@ -58,7 +58,7 @@ count=0 :: non_neg_integer() }). -%% @doc official error types that is specific in Machi +%% Official error types that is specific in Machi -type machi_client_error_reason() :: bad_arg | wedged | bad_checksum | partition | not_written | written | trimmed | no_such_file | partial_read | @@ -145,7 +145,7 @@ read_chunk(PidSpec, File, Offset, Size, Options, Timeout) -> send_sync(PidSpec, {read_chunk, File, Offset, Size, Options}, Timeout). %% @doc Trims arbitrary binary range of any file. If a specified range -%% has any byte trimmed, it fails and returns `{error, trimmed}`. +%% has any byte trimmed, it fails and returns `{error, trimmed}'. %% Otherwise it trims all bytes in that range. If there are %% overlapping chunks with client-specified checksum, they will cut %% off and checksum are re-calculated in server side. TODO: Add -- 2.45.2 From 69280bfb4f8804cf4f6f4dd8f36d3eb029332247 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 8 Dec 2015 22:19:26 +0900 Subject: [PATCH 32/77] Fix typo/thinko: correct chain name @ bootstrap --- src/machi_chain_manager1.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/machi_chain_manager1.erl b/src/machi_chain_manager1.erl index 75917d3..93eb026 100644 --- a/src/machi_chain_manager1.erl +++ b/src/machi_chain_manager1.erl @@ -327,7 +327,7 @@ handle_call({set_chain_members, SetChainName, SetOldEpoch, CMode, ok = set_consistency_mode(machi_flu_psup:make_proj_supname(MyName), CMode), NewProj = machi_projection:update_checksum( OldProj#projection_v1{author_server=MyName, - chain_name=ChainName, + chain_name=SetChainName, creation_time=now(), mode=CMode, epoch_number=NewEpoch, -- 2.45.2 From a8785e44b1ff25b067ecfe6234a47cb05c16f108 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Mon, 7 Dec 2015 13:19:30 +0900 Subject: [PATCH 33/77] Set longer timeout for hyooge binary write test case --- test/machi_file_proxy_test.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/machi_file_proxy_test.erl b/test/machi_file_proxy_test.erl index 1bf6cde..a04d880 100644 --- a/test/machi_file_proxy_test.erl +++ b/test/machi_file_proxy_test.erl @@ -98,7 +98,8 @@ machi_file_proxy_test_() -> ?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1024)), ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, ?HYOOGE)), ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, ?HYOOGE, 1)), - ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, random_binary(0, ?HYOOGE))), + {timeout, 10, + ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, random_binary(0, ?HYOOGE)))}, ?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))), ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1024, <<"fail">>)), ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, <<"fail">>)), -- 2.45.2 From 9579b1b8b259806876eb57aa4fb4a4b7e6ce1a07 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Wed, 18 Nov 2015 15:16:04 +0900 Subject: [PATCH 34/77] Introduce ranch and add transport callback --- rebar.config | 1 + src/machi_flu1.erl | 465 +-------------------------- src/machi_flu_psup.erl | 13 +- src/machi_pb_protocol.erl | 645 ++++++++++++++++++++++++++++++++++++++ src/machi_sup.erl | 16 +- 5 files changed, 666 insertions(+), 474 deletions(-) create mode 100644 src/machi_pb_protocol.erl diff --git a/rebar.config b/rebar.config index 7bd81ce..be9b0d3 100644 --- a/rebar.config +++ b/rebar.config @@ -11,6 +11,7 @@ {lager, ".*", {git, "git://github.com/basho/lager.git", {tag, "2.2.0"}}}, {protobuffs, "0.8.*", {git, "git://github.com/basho/erlang_protobuffs.git", {tag, "0.8.1p4"}}}, {riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {branch, "develop"}}}, + {ranch, ".*", {git, "git://github.com/ninenines/ranch.git", {branch, "master"}}}, {node_package, ".*", {git, "git://github.com/basho/node_package.git", {branch, "develop"}}}, {eper, ".*", {git, "git://github.com/basho/eper.git", {tag, "0.92-basho1"}}}, {cluster_info, ".*", {git, "git://github.com/basho/cluster_info", {branch, "develop"}}} diff --git a/src/machi_flu1.erl b/src/machi_flu1.erl index bfe599b..c02772d 100644 --- a/src/machi_flu1.erl +++ b/src/machi_flu1.erl @@ -60,7 +60,8 @@ update_wedge_state/3, wedge_myself/2]). -export([make_listener_regname/1, make_projection_server_regname/1]). %% TODO: remove or replace in OTP way after gen_*'ified --export([main2/4, run_append_server/2, run_listen_server/1, +-export([main2/4, run_append_server/2, + %% run_listen_server/1, current_state/1, format_state/1]). -record(state, { @@ -188,27 +189,13 @@ main2(FluName, TcpPort, DataDir, Props) -> (catch exit(ListenPid, kill)), ok. -start_listen_server(S) -> - proc_lib:start_link(?MODULE, run_listen_server, [S], ?INIT_TIMEOUT). - start_append_server(S, AckPid) -> proc_lib:start_link(?MODULE, run_append_server, [AckPid, S], ?INIT_TIMEOUT). -run_listen_server(#state{flu_name=FluName, tcp_port=TcpPort}=S) -> - register(make_listener_regname(FluName), self()), - SockOpts = ?PB_PACKET_OPTS ++ - [{reuseaddr, true}, {mode, binary}, {active, false}, - {backlog,8192}], - case gen_tcp:listen(TcpPort, SockOpts) of - {ok, LSock} -> - proc_lib:init_ack({ok, self()}), - listen_server_loop(LSock, S); - Else -> - error_logger:warning_msg("~s:run_listen_server: " - "listen to TCP port ~w: ~w\n", - [?MODULE, TcpPort, Else]), - exit({?MODULE, run_listen_server, tcp_port, TcpPort, Else}) - end. +start_listen_server(_S) -> + %% FIXMEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE + %% proc_lib:start_link(?MODULE, run_listen_server, [S], ?INIT_TIMEOUT). + {ok, dummy}. run_append_server(FluPid, #state{flu_name=Name, wedged=Wedged_p,epoch_id=EpochId}=S) -> @@ -220,11 +207,6 @@ run_append_server(FluPid, #state{flu_name=Name, proc_lib:init_ack({ok, self()}), append_server_loop(FluPid, S#state{etstab=TID}). -listen_server_loop(LSock, S) -> - {ok, Sock} = gen_tcp:accept(LSock), - spawn_link(fun() -> net_server_loop(Sock, S) end), - listen_server_loop(LSock, S). - append_server_loop(FluPid, #state{wedged=Wedged_p, witness=Witness_p, epoch_id=OldEpochId, flu_name=FluName}=S) -> @@ -292,398 +274,6 @@ append_server_loop(FluPid, #state{wedged=Wedged_p, append_server_loop(FluPid, S) end. -net_server_loop(Sock, S) -> - case gen_tcp:recv(Sock, 0, ?SERVER_CMD_READ_TIMEOUT) of - {ok, Bin} -> - {RespBin, S2} = - case machi_pb:decode_mpb_ll_request(Bin) of - LL_req when LL_req#mpb_ll_request.do_not_alter == 2 -> - {R, NewS} = do_pb_ll_request(LL_req, S), - {maybe_encode_response(R), mode(low, NewS)}; - _ -> - HL_req = machi_pb:decode_mpb_request(Bin), - 1 = HL_req#mpb_request.do_not_alter, - {R, NewS} = do_pb_hl_request(HL_req, make_high_clnt(S)), - {machi_pb:encode_mpb_response(R), mode(high, NewS)} - end, - if RespBin == async_no_response -> - net_server_loop(Sock, S2); - true -> - case gen_tcp:send(Sock, RespBin) of - ok -> - net_server_loop(Sock, S2); - {error, _} -> - (catch gen_tcp:close(Sock)), - exit(normal) - end - end; - {error, SockError} -> - Msg = io_lib:format("Socket error ~w", [SockError]), - R = #mpb_ll_response{req_id= <<>>, - generic=#mpb_errorresp{code=1, msg=Msg}}, - _Resp = machi_pb:encode_mpb_ll_response(R), - %% TODO: Weird that sometimes neither catch nor try/catch - %% can prevent OTP's SASL from logging an error here. - %% Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,....... - %% TODO: is this what causes the intermittent PULSE deadlock errors? - %% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000), - (catch gen_tcp:close(Sock)), - exit(normal) - end. - -maybe_encode_response(async_no_response=X) -> - X; -maybe_encode_response(R) -> - machi_pb:encode_mpb_ll_response(R). - -mode(Mode, #state{pb_mode=undefined}=S) -> - S#state{pb_mode=Mode}; -mode(_, S) -> - S. - -make_high_clnt(#state{high_clnt=undefined}=S) -> - {ok, Proj} = machi_projection_store:read_latest_projection( - S#state.proj_store, private), - Ps = [P_srvr || {_, P_srvr} <- orddict:to_list( - Proj#projection_v1.members_dict)], - {ok, Clnt} = machi_cr_client:start_link(Ps), - S#state{high_clnt=Clnt}; -make_high_clnt(S) -> - S. - -do_pb_ll_request(#mpb_ll_request{req_id=ReqID}, #state{pb_mode=high}=S) -> - Result = {high_error, 41, "Low protocol request while in high mode"}, - {machi_pb_translate:to_pb_response(ReqID, unused, Result), S}; -do_pb_ll_request(PB_request, S) -> - Req = machi_pb_translate:from_pb_request(PB_request), - {ReqID, Cmd, Result, S2} = - case Req of - {RqID, {LowCmd, _}=CMD} - when LowCmd == low_proj; - LowCmd == low_wedge_status; LowCmd == low_list_files -> - %% Skip wedge check for projection commands! - %% Skip wedge check for these unprivileged commands - {Rs, NewS} = do_pb_ll_request3(CMD, S), - {RqID, CMD, Rs, NewS}; - {RqID, CMD} -> - EpochID = element(2, CMD), % by common convention - {Rs, NewS} = do_pb_ll_request2(EpochID, CMD, S), - {RqID, CMD, Rs, NewS} - end, - {machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}. - -do_pb_ll_request2(EpochID, CMD, S) -> - {Wedged_p, CurrentEpochID} = ets:lookup_element(S#state.etstab, epoch, 2), - if Wedged_p == true -> - {{error, wedged}, S#state{epoch_id=CurrentEpochID}}; - is_tuple(EpochID) - andalso - EpochID /= CurrentEpochID -> - {Epoch, _} = EpochID, - {CurrentEpoch, _} = CurrentEpochID, - if Epoch < CurrentEpoch -> - ok; - true -> - %% We're at same epoch # but different checksum, or - %% we're at a newer/bigger epoch #. - _ = wedge_myself(S#state.flu_name, CurrentEpochID), - ok - end, - {{error, bad_epoch}, S#state{epoch_id=CurrentEpochID}}; - true -> - do_pb_ll_request3(CMD, S#state{epoch_id=CurrentEpochID}) - end. - -%% Witness status does not matter below. -do_pb_ll_request3({low_echo, _BogusEpochID, Msg}, S) -> - {Msg, S}; -do_pb_ll_request3({low_auth, _BogusEpochID, _User, _Pass}, S) -> - {-6, S}; -do_pb_ll_request3({low_wedge_status, _EpochID}, S) -> - {do_server_wedge_status(S), S}; -do_pb_ll_request3({low_proj, PCMD}, S) -> - {do_server_proj_request(PCMD, S), S}; -%% Witness status *matters* below -do_pb_ll_request3({low_append_chunk, _EpochID, CoC_Namespace, CoC_Locator, - Prefix, Chunk, CSum_tag, - CSum, ChunkExtra}, - #state{witness=false}=S) -> - {do_server_append_chunk(CoC_Namespace, CoC_Locator, - Prefix, Chunk, CSum_tag, CSum, - ChunkExtra, S), S}; -do_pb_ll_request3({low_write_chunk, _EpochID, File, Offset, Chunk, CSum_tag, - CSum}, - #state{witness=false}=S) -> - {do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, S), S}; -do_pb_ll_request3({low_read_chunk, _EpochID, File, Offset, Size, Opts}, - #state{witness=false} = S) -> - {do_server_read_chunk(File, Offset, Size, Opts, S), S}; -do_pb_ll_request3({low_trim_chunk, _EpochID, File, Offset, Size, TriggerGC}, - #state{witness=false}=S) -> - {do_server_trim_chunk(File, Offset, Size, TriggerGC, S), S}; -do_pb_ll_request3({low_checksum_list, _EpochID, File}, - #state{witness=false}=S) -> - {do_server_checksum_listing(File, S), S}; -do_pb_ll_request3({low_list_files, _EpochID}, - #state{witness=false}=S) -> - {do_server_list_files(S), S}; -do_pb_ll_request3({low_delete_migration, _EpochID, File}, - #state{witness=false}=S) -> - {do_server_delete_migration(File, S), - #state{witness=false}=S}; -do_pb_ll_request3({low_trunc_hack, _EpochID, File}, - #state{witness=false}=S) -> - {do_server_trunc_hack(File, S), S}; -do_pb_ll_request3(_, #state{witness=true}=S) -> - {{error, bad_arg}, S}. % TODO: new status code?? - -do_pb_hl_request(#mpb_request{req_id=ReqID}, #state{pb_mode=low}=S) -> - Result = {low_error, 41, "High protocol request while in low mode"}, - {machi_pb_translate:to_pb_response(ReqID, unused, Result), S}; -do_pb_hl_request(PB_request, S) -> - {ReqID, Cmd} = machi_pb_translate:from_pb_request(PB_request), - {Result, S2} = do_pb_hl_request2(Cmd, S), - {machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}. - -do_pb_hl_request2({high_echo, Msg}, S) -> - {Msg, S}; -do_pb_hl_request2({high_auth, _User, _Pass}, S) -> - {-77, S}; -do_pb_hl_request2({high_append_chunk, CoC_Namespace, CoC_Locator, - Prefix, ChunkBin, TaggedCSum, - ChunkExtra}, #state{high_clnt=Clnt}=S) -> - Chunk = {TaggedCSum, ChunkBin}, - Res = machi_cr_client:append_chunk_extra(Clnt, CoC_Namespace, CoC_Locator, - Prefix, Chunk, - ChunkExtra), - {Res, S}; -do_pb_hl_request2({high_write_chunk, File, Offset, ChunkBin, TaggedCSum}, - #state{high_clnt=Clnt}=S) -> - Chunk = {TaggedCSum, ChunkBin}, - Res = machi_cr_client:write_chunk(Clnt, File, Offset, Chunk), - {Res, S}; -do_pb_hl_request2({high_read_chunk, File, Offset, Size, Opts}, - #state{high_clnt=Clnt}=S) -> - Res = machi_cr_client:read_chunk(Clnt, File, Offset, Size, Opts), - {Res, S}; -do_pb_hl_request2({high_trim_chunk, File, Offset, Size}, - #state{high_clnt=Clnt}=S) -> - Res = machi_cr_client:trim_chunk(Clnt, File, Offset, Size), - {Res, S}; -do_pb_hl_request2({high_checksum_list, File}, #state{high_clnt=Clnt}=S) -> - Res = machi_cr_client:checksum_list(Clnt, File), - {Res, S}; -do_pb_hl_request2({high_list_files}, #state{high_clnt=Clnt}=S) -> - Res = machi_cr_client:list_files(Clnt), - {Res, S}. - -do_server_proj_request({get_latest_epochid, ProjType}, - #state{proj_store=ProjStore}) -> - machi_projection_store:get_latest_epochid(ProjStore, ProjType); -do_server_proj_request({read_latest_projection, ProjType}, - #state{proj_store=ProjStore}) -> - machi_projection_store:read_latest_projection(ProjStore, ProjType); -do_server_proj_request({read_projection, ProjType, Epoch}, - #state{proj_store=ProjStore}) -> - machi_projection_store:read(ProjStore, ProjType, Epoch); -do_server_proj_request({write_projection, ProjType, Proj}, - #state{flu_name=FluName, proj_store=ProjStore}) -> - if Proj#projection_v1.epoch_number == ?SPAM_PROJ_EPOCH -> - %% io:format(user, "DBG ~s ~w ~P\n", [?MODULE, ?LINE, Proj, 5]), - Chmgr = machi_flu_psup:make_fitness_regname(FluName), - [Map] = Proj#projection_v1.dbg, - catch machi_fitness:send_fitness_update_spam( - Chmgr, Proj#projection_v1.author_server, Map); - true -> - catch machi_projection_store:write(ProjStore, ProjType, Proj) - end; -do_server_proj_request({get_all_projections, ProjType}, - #state{proj_store=ProjStore}) -> - machi_projection_store:get_all_projections(ProjStore, ProjType); -do_server_proj_request({list_all_projections, ProjType}, - #state{proj_store=ProjStore}) -> - machi_projection_store:list_all_projections(ProjStore, ProjType); -do_server_proj_request({kick_projection_reaction}, - #state{flu_name=FluName}) -> - %% Tell my chain manager that it might want to react to - %% this new world. - Chmgr = machi_chain_manager1:make_chmgr_regname(FluName), - spawn(fun() -> - catch machi_chain_manager1:trigger_react_to_env(Chmgr) - end), - async_no_response. - -do_server_append_chunk(CoC_Namespace, CoC_Locator, - Prefix, Chunk, CSum_tag, CSum, - ChunkExtra, S) -> - case sanitize_prefix(Prefix) of - ok -> - do_server_append_chunk2(CoC_Namespace, CoC_Locator, - Prefix, Chunk, CSum_tag, CSum, - ChunkExtra, S); - _ -> - {error, bad_arg} - end. - -do_server_append_chunk2(CoC_Namespace, CoC_Locator, - Prefix, Chunk, CSum_tag, Client_CSum, - ChunkExtra, #state{flu_name=FluName, - epoch_id=EpochID}=_S) -> - %% TODO: Do anything with PKey? - try - TaggedCSum = check_or_make_tagged_checksum(CSum_tag, Client_CSum,Chunk), - R = {seq_append, self(), CoC_Namespace, CoC_Locator, - Prefix, Chunk, TaggedCSum, ChunkExtra, EpochID}, - FluName ! R, - receive - {assignment, Offset, File} -> - Size = iolist_size(Chunk), - {ok, {Offset, Size, File}}; - witness -> - {error, bad_arg}; - wedged -> - {error, wedged} - after 10*1000 -> - {error, partition} - end - catch - throw:{bad_csum, _CS} -> - {error, bad_checksum}; - error:badarg -> - error_logger:error_msg("Message send to ~p gave badarg, make certain server is running with correct registered name\n", [?MODULE]), - {error, bad_arg} - end. - -do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, #state{flu_name=FluName}) -> - case sanitize_file_string(File) of - ok -> - case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of - {ok, Pid} -> - Meta = [{client_csum_tag, CSum_tag}, {client_csum, CSum}], - machi_file_proxy:write(Pid, Offset, Meta, Chunk); - {error, trimmed} = Error -> - Error - end; - _ -> - {error, bad_arg} - end. - -do_server_read_chunk(File, Offset, Size, Opts, #state{flu_name=FluName})-> - case sanitize_file_string(File) of - ok -> - case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of - {ok, Pid} -> - case machi_file_proxy:read(Pid, Offset, Size, Opts) of - %% XXX FIXME - %% For now we are omiting the checksum data because it blows up - %% protobufs. - {ok, ChunksAndTrimmed} -> {ok, ChunksAndTrimmed}; - Other -> Other - end; - {error, trimmed} = Error -> - Error - end; - _ -> - {error, bad_arg} - end. - -do_server_trim_chunk(File, Offset, Size, TriggerGC, #state{flu_name=FluName}) -> - lager:debug("Hi there! I'm trimming this: ~s, (~p, ~p), ~p~n", - [File, Offset, Size, TriggerGC]), - case sanitize_file_string(File) of - ok -> - case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of - {ok, Pid} -> - machi_file_proxy:trim(Pid, Offset, Size, TriggerGC); - {error, trimmed} = Trimmed -> - %% Should be returned back to (maybe) trigger repair - Trimmed - end; - _ -> - {error, bad_arg} - end. - -do_server_checksum_listing(File, #state{flu_name=FluName, data_dir=DataDir}=_S) -> - case sanitize_file_string(File) of - ok -> - case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of - {ok, Pid} -> - {ok, List} = machi_file_proxy:checksum_list(Pid), - Bin = erlang:term_to_binary(List), - if byte_size(Bin) > (?PB_MAX_MSG_SIZE - 1024) -> - %% TODO: Fix this limitation by streaming the - %% binary in multiple smaller PB messages. - %% Also, don't read the file all at once. ^_^ - error_logger:error_msg("~s:~w oversize ~s\n", - [?MODULE, ?LINE, DataDir]), - {error, bad_arg}; - true -> - {ok, Bin} - end; - {error, trimmed} -> - {error, trimmed} - end; - _ -> - {error, bad_arg} - end. - -do_server_list_files(#state{data_dir=DataDir}=_S) -> - {_, WildPath} = machi_util:make_data_filename(DataDir, ""), - Files = filelib:wildcard("*", WildPath), - {ok, [begin - {ok, FI} = file:read_file_info(WildPath ++ "/" ++ File), - Size = FI#file_info.size, - {Size, File} - end || File <- Files]}. - -do_server_wedge_status(S) -> - {Wedged_p, CurrentEpochID0} = ets:lookup_element(S#state.etstab, epoch, 2), - CurrentEpochID = if CurrentEpochID0 == undefined -> - ?DUMMY_PV1_EPOCH; - true -> - CurrentEpochID0 - end, - {Wedged_p, CurrentEpochID}. - -do_server_delete_migration(File, #state{data_dir=DataDir}=_S) -> - case sanitize_file_string(File) of - ok -> - {_, Path} = machi_util:make_data_filename(DataDir, File), - case file:delete(Path) of - ok -> - ok; - {error, enoent} -> - {error, no_such_file}; - _ -> - {error, bad_arg} - end; - _ -> - {error, bad_arg} - end. - -do_server_trunc_hack(File, #state{data_dir=DataDir}=_S) -> - case sanitize_file_string(File) of - ok -> - {_, Path} = machi_util:make_data_filename(DataDir, File), - case file:open(Path, [read, write, binary, raw]) of - {ok, FH} -> - try - {ok, ?MINIMUM_OFFSET} = file:position(FH, - ?MINIMUM_OFFSET), - ok = file:truncate(FH), - ok - after - file:close(FH) - end; - {error, enoent} -> - {error, no_such_file}; - _ -> - {error, bad_arg} - end; - _ -> - {error, bad_arg} - end. - append_server_dispatch(From, CoC_Namespace, CoC_Locator, Prefix, Chunk, CSum, Extra, FluName, EpochId) -> Result = case handle_append(CoC_Namespace, CoC_Locator, @@ -717,29 +307,6 @@ handle_append(CoC_Namespace, CoC_Locator, Error end. -sanitize_file_string(Str) -> - case has_no_prohibited_chars(Str) andalso machi_util:is_valid_filename(Str) of - true -> ok; - false -> error - end. - -has_no_prohibited_chars(Str) -> - case re:run(Str, "/") of - nomatch -> - true; - _ -> - true - end. - -sanitize_prefix(Prefix) -> - %% We are using '^' as our component delimiter - case re:run(Prefix, "/|\\^") of - nomatch -> - ok; - _ -> - error - end. - make_listener_regname(BaseName) -> list_to_atom(atom_to_list(BaseName) ++ "_listener"). @@ -753,26 +320,6 @@ make_listener_regname(BaseName) -> make_projection_server_regname(BaseName) -> list_to_atom(atom_to_list(BaseName) ++ "_pstore"). -check_or_make_tagged_checksum(?CSUM_TAG_NONE, _Client_CSum, Chunk) -> - %% TODO: If the client was foolish enough to use - %% this type of non-checksum, then the client gets - %% what it deserves wrt data integrity, alas. In - %% the client-side Chain Replication method, each - %% server will calculated this independently, which - %% isn't exactly what ought to happen for best data - %% integrity checking. In server-side CR, the csum - %% should be calculated by the head and passed down - %% the chain together with the value. - CS = machi_util:checksum_chunk(Chunk), - machi_util:make_tagged_csum(server_sha, CS); -check_or_make_tagged_checksum(?CSUM_TAG_CLIENT_SHA, Client_CSum, Chunk) -> - CS = machi_util:checksum_chunk(Chunk), - if CS == Client_CSum -> - machi_util:make_tagged_csum(server_sha, - Client_CSum); - true -> - throw({bad_csum, CS}) - end. -ifdef(TEST). diff --git a/src/machi_flu_psup.erl b/src/machi_flu_psup.erl index 9e568cd..b2ccdcc 100644 --- a/src/machi_flu_psup.erl +++ b/src/machi_flu_psup.erl @@ -143,16 +143,21 @@ init([FluName, TcpPort, DataDir, Props0]) -> FProxySupSpec = machi_file_proxy_sup:child_spec(FluName), + ListenerRegName = machi_flu1:make_listener_regname(FluName), + NbAcceptors = 100, + ListenerSpec = ranch:child_spec(ListenerRegName, NbAcceptors, + ranch_tcp, [{port, TcpPort}], + machi_pb_protocol, []), FluSpec = {FluName, {machi_flu1, start_link, - [ [{FluName, TcpPort, DataDir}|Props] ]}, + [ [{FluName, TcpPort+1, DataDir}|Props] ]}, permanent, ?SHUTDOWN, worker, []}, {ok, {SupFlags, [ - ProjSpec, FitnessSpec, MgrSpec, - FProxySupSpec, FNameMgrSpec, MetaSupSpec, - FluSpec]}}. + ProjSpec, FitnessSpec, MgrSpec, + FProxySupSpec, FNameMgrSpec, MetaSupSpec, + FluSpec, ListenerSpec]}}. make_flu_regname(FluName) when is_atom(FluName) -> FluName. diff --git a/src/machi_pb_protocol.erl b/src/machi_pb_protocol.erl new file mode 100644 index 0000000..28ce626 --- /dev/null +++ b/src/machi_pb_protocol.erl @@ -0,0 +1,645 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% @doc Ranch protocol callback module to handle PB protocol over +%% transport + +%% TODO +%% - Two modes, high and low should be separated at listener level? + +-module(machi_pb_protocol). + +-behaviour(gen_server). +-behaviour(ranch_protocol). +-export([start_link/4]). +-export([init/1]). +-export([handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-include("machi.hrl"). +-include("machi_pb.hrl"). +-include("machi_projection.hrl"). +-define(V(X,Y), ok). +%% -include("machi_verbose.hrl"). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. % TEST + +-record(state, {ref, + socket, + transport, + opts, + pb_mode, + data_dir, + witness, + %% - Used in projection related requests in low mode + %% - Used in spawning CR client in high mode + proj_store, + %%%% Low mode only + %% Current best knowledge, used for wedge_self / bad_epoch check + epoch_id, + %% Used in dispatching append_chunk* reqs to the + %% append serializing process + flu_name, + %% Stored in ETS before factorization, can be stored in the recored? + wedged, + %% Used in server_wedge_status to lookup the table + etstab, + %%%% High mode only + high_clnt, + %%%% to be removed + eof + }). + +%% -record(state, { + %% used in append serializer to trigger chain mgr react_to_env +%% flu_name :: atom(), +%% proj_store :: pid(), +%% witness = false :: boolean(), +%% append_pid :: pid(), +%% tcp_port :: non_neg_integer(), +%% data_dir :: string(), +%% wedged = true :: boolean(), +%% etstab :: ets:tid(), +%% epoch_id :: 'undefined' | machi_dt:epoch_id(), +%% pb_mode = undefined :: 'undefined' | 'high' | 'low', +%% high_clnt :: 'undefined' | pid(), +%% trim_table :: ets:tid(), +%% props = [] :: list() % proplist +%% }). + +-spec start_link(ranch:ref(), any(), module(), any()) -> {ok, pid()}. +start_link(Ref, Socket, Transport, Opts) -> + proc_lib:start_link(?MODULE, init, [#state{ref=Ref, socket=Socket, + transport=Transport, + opts=Opts}]). + +init(#state{ref=Ref, socket=Socket, transport=Transport, opts=_Opts}=State) -> + ok = proc_lib:init_ack({ok, self()}), + %% TODO: Perform any required state initialization here. + ok = ranch:accept_ack(Ref), + ok = Transport:setopts(Socket, [{active, once}]), + gen_server:enter_loop(?MODULE, [], State). + +handle_call(Request, _From, S) -> + Reply = {error, {unknown_message, Request}}, + {reply, Reply, S}. + +handle_cast(_Msg, S) -> + io:format(user, "~s:handle_cast: ~p\n", [?MODULE, _Msg]), + {noreply, S}. + +handle_info({tcp, Sock, Data}=_Info, S) -> + io:format(user, "~s:handle_info: ~p\n", [?MODULE, _Info]), + transport_received(Sock, Data, S); +handle_info({tcp_closed, Sock}=_Info, S) -> + io:format(user, "~s:handle_info: ~p\n", [?MODULE, _Info]), + transport_closed(Sock, S); +handle_info({tcp_error, Sock, Reason}=_Info, S) -> + io:format(user, "~s:handle_info: ~p\n", [?MODULE, _Info]), + transport_error(Sock, Reason, S); +handle_info(_Info, S) -> + io:format(user, "~s:handle_info: ~p\n", [?MODULE, _Info]), + {noreply, S}. + +terminate(_Reason, _S) -> + io:format(user, "~s:terminate: ~p\n", [?MODULE, _Reason]), + ok. + +code_change(_OldVsn, S, _Extra) -> + {ok, S}. + +%% Internal functions, or copy-n-paste'd thingie + +%%%% Just copied and will be removed %%%% + +%% TODO: sock opts should be migrated to ranch equivalent +%% run_listen_server(#state{flu_name=FluName, tcp_port=TcpPort}=S) -> +%% register(make_listener_regname(FluName), self()), +%% SockOpts = ?PB_PACKET_OPTS ++ +%% [{reuseaddr, true}, {mode, binary}, {active, false}, +%% {backlog,8192}], +%% case gen_tcp:listen(TcpPort, SockOpts) of +%% {ok, LSock} -> +%% proc_lib:init_ack({ok, self()}), +%% listen_server_loop(LSock, S); +%% Else -> +%% error_logger:warning_msg("~s:run_listen_server: " +%% "listen to TCP port ~w: ~w\n", +%% [?MODULE, TcpPort, Else]), +%% exit({?MODULE, run_listen_server, tcp_port, TcpPort, Else}) +%% end. + +%% listen_server_loop(LSock, S) -> +%% {ok, Sock} = gen_tcp:accept(LSock), +%% spawn_link(fun() -> net_server_loop(Sock, S) end), +%% listen_server_loop(LSock, S). + +%% net_server_loop(Sock, S) -> +%% case gen_tcp:recv(Sock, 0, ?SERVER_CMD_READ_TIMEOUT) of +%% {ok, Bin} -> +%% {RespBin, S2} = +%% case machi_pb:decode_mpb_ll_request(Bin) of +%% LL_req when LL_req#mpb_ll_request.do_not_alter == 2 -> +%% {R, NewS} = do_pb_ll_request(LL_req, S), +%% {maybe_encode_response(R), mode(low, NewS)}; +%% _ -> +%% HL_req = machi_pb:decode_mpb_request(Bin), +%% 1 = HL_req#mpb_request.do_not_alter, +%% {R, NewS} = do_pb_hl_request(HL_req, make_high_clnt(S)), +%% {machi_pb:encode_mpb_response(R), mode(high, NewS)} +%% end, +%% if RespBin == async_no_response -> +%% net_server_loop(Sock, S2); +%% true -> +%% case gen_tcp:send(Sock, RespBin) of +%% ok -> +%% net_server_loop(Sock, S2); +%% {error, _} -> +%% (catch gen_tcp:close(Sock)), +%% exit(normal) +%% end +%% end; +%% {error, SockError} -> +%% Msg = io_lib:format("Socket error ~w", [SockError]), +%% R = #mpb_ll_response{req_id= <<>>, +%% generic=#mpb_errorresp{code=1, msg=Msg}}, +%% _Resp = machi_pb:encode_mpb_ll_response(R), +%% %% TODO: Weird that sometimes neither catch nor try/catch +%% %% can prevent OTP's SASL from logging an error here. +%% %% Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,....... +%% %% TODO: is this what causes the intermittent PULSE deadlock errors? +%% %% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000), +%% (catch gen_tcp:close(Sock)), +%% exit(normal) +%% end. + +%%%% Common transport handling + +transport_received(Sock, Bin, #state{transport=Transport}=S) -> + {RespBin, S2} = + case machi_pb:decode_mpb_ll_request(Bin) of + LL_req when LL_req#mpb_ll_request.do_not_alter == 2 -> + {R, NewS} = do_pb_ll_request(LL_req, S), + {maybe_encode_response(R), mode(low, NewS)}; + _ -> + HL_req = machi_pb:decode_mpb_request(Bin), + 1 = HL_req#mpb_request.do_not_alter, + {R, NewS} = do_pb_hl_request(HL_req, make_high_clnt(S)), + {machi_pb:encode_mpb_response(R), mode(high, NewS)} + end, + if RespBin == async_no_response -> + {noreply, S2}; + true -> + case Transport:send(Sock, RespBin) of + ok -> + {noreply, S2}; + {error, Reason} -> + transport_error(Sock, Reason, S2) + end + end. + +transport_closed(Sock, S) -> + (catch gen_tcp:close(Sock)), + {stop, normal, S#state{sock=undefined}}. + +transport_error(Sock, Reason, S) -> + Msg = io_lib:format("Socket error ~w", [SockError]), + R = #mpb_ll_response{req_id= <<>>, + generic=#mpb_errorresp{code=1, msg=Msg}}, + _Resp = machi_pb:encode_mpb_ll_response(R), + %% TODO of TODO comments: comments below with four %s are copy-n-paste'd, + %% then it should be considered they are still open and should be addressed. + %%%% TODO: Weird that sometimes neither catch nor try/catch + %%%% can prevent OTP's SASL from logging an error here. + %%%% Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,....... + %%%% TODO: is this what causes the intermittent PULSE deadlock errors? + %%%% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000), + (catch gen_tcp:close(Sock)), + %% TODO: better to exit with `Reason'? + exit(normal). + +maybe_encode_response(async_no_response=X) -> + X; +maybe_encode_response(R) -> + machi_pb:encode_mpb_ll_response(R). + +%%%% Not categorized / not-yet-well-understood items +%% TODO: may be external API +mode(Mode, #state{pb_mode=undefined}=S) -> + S#state{pb_mode=Mode}; +mode(_, S) -> + S. + + +%%%% Low PB mode %%%% + +do_pb_ll_request(#mpb_ll_request{req_id=ReqID}, #state{pb_mode=high}=S) -> + Result = {high_error, 41, "Low protocol request while in high mode"}, + {machi_pb_translate:to_pb_response(ReqID, unused, Result), S}; +do_pb_ll_request(PB_request, S) -> + Req = machi_pb_translate:from_pb_request(PB_request), + {ReqID, Cmd, Result, S2} = + case Req of + {RqID, {LowCmd, _}=CMD} + when LowCmd == low_proj; + LowCmd == low_wedge_status; LowCmd == low_list_files -> + %% Skip wedge check for projection commands! + %% Skip wedge check for these unprivileged commands + {Rs, NewS} = do_pb_ll_request3(CMD, S), + {RqID, CMD, Rs, NewS}; + {RqID, CMD} -> + EpochID = element(2, CMD), % by common convention + {Rs, NewS} = do_pb_ll_request2(EpochID, CMD, S), + {RqID, CMD, Rs, NewS} + end, + {machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}. + +do_pb_ll_request2(EpochID, CMD, S) -> + {Wedged_p, CurrentEpochID} = ets:lookup_element(S#state.etstab, epoch, 2), + if Wedged_p == true -> + {{error, wedged}, S#state{epoch_id=CurrentEpochID}}; + is_tuple(EpochID) + andalso + EpochID /= CurrentEpochID -> + {Epoch, _} = EpochID, + {CurrentEpoch, _} = CurrentEpochID, + if Epoch < CurrentEpoch -> + ok; + true -> + %% We're at same epoch # but different checksum, or + %% we're at a newer/bigger epoch #. + _ = wedge_myself(S#state.flu_name, CurrentEpochID), + ok + end, + {{error, bad_epoch}, S#state{epoch_id=CurrentEpochID}}; + true -> + do_pb_ll_request3(CMD, S#state{epoch_id=CurrentEpochID}) + end. + +%% Witness status does not matter below. +do_pb_ll_request3({low_echo, _BogusEpochID, Msg}, S) -> + {Msg, S}; +do_pb_ll_request3({low_auth, _BogusEpochID, _User, _Pass}, S) -> + {-6, S}; +do_pb_ll_request3({low_wedge_status, _EpochID}, S) -> + {do_server_wedge_status(S), S}; +do_pb_ll_request3({low_proj, PCMD}, S) -> + {do_server_proj_request(PCMD, S), S}; +%% Witness status *matters* below +do_pb_ll_request3({low_append_chunk, _EpochID, CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, + CSum, ChunkExtra}, + #state{witness=false}=S) -> + {do_server_append_chunk(CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, CSum, + ChunkExtra, S), S}; +do_pb_ll_request3({low_write_chunk, _EpochID, File, Offset, Chunk, CSum_tag, + CSum}, + #state{witness=false}=S) -> + {do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, S), S}; +do_pb_ll_request3({low_read_chunk, _EpochID, File, Offset, Size, Opts}, + #state{witness=false} = S) -> + {do_server_read_chunk(File, Offset, Size, Opts, S), S}; +do_pb_ll_request3({low_trim_chunk, _EpochID, File, Offset, Size, TriggerGC}, + #state{witness=false}=S) -> + {do_server_trim_chunk(File, Offset, Size, TriggerGC, S), S}; +do_pb_ll_request3({low_checksum_list, _EpochID, File}, + #state{witness=false}=S) -> + {do_server_checksum_listing(File, S), S}; +do_pb_ll_request3({low_list_files, _EpochID}, + #state{witness=false}=S) -> + {do_server_list_files(S), S}; +do_pb_ll_request3({low_delete_migration, _EpochID, File}, + #state{witness=false}=S) -> + {do_server_delete_migration(File, S), + #state{witness=false}=S}; +do_pb_ll_request3({low_trunc_hack, _EpochID, File}, + #state{witness=false}=S) -> + {do_server_trunc_hack(File, S), S}; +do_pb_ll_request3(_, #state{witness=true}=S) -> + {{error, bad_arg}, S}. % TODO: new status code?? + +do_server_proj_request({get_latest_epochid, ProjType}, + #state{proj_store=ProjStore}) -> + machi_projection_store:get_latest_epochid(ProjStore, ProjType); +do_server_proj_request({read_latest_projection, ProjType}, + #state{proj_store=ProjStore}) -> + machi_projection_store:read_latest_projection(ProjStore, ProjType); +do_server_proj_request({read_projection, ProjType, Epoch}, + #state{proj_store=ProjStore}) -> + machi_projection_store:read(ProjStore, ProjType, Epoch); +do_server_proj_request({write_projection, ProjType, Proj}, + #state{flu_name=FluName, proj_store=ProjStore}) -> + if Proj#projection_v1.epoch_number == ?SPAM_PROJ_EPOCH -> + %% io:format(user, "DBG ~s ~w ~P\n", [?MODULE, ?LINE, Proj, 5]), + Chmgr = machi_flu_psup:make_fitness_regname(FluName), + [Map] = Proj#projection_v1.dbg, + catch machi_fitness:send_fitness_update_spam( + Chmgr, Proj#projection_v1.author_server, Map); + true -> + catch machi_projection_store:write(ProjStore, ProjType, Proj) + end; +do_server_proj_request({get_all_projections, ProjType}, + #state{proj_store=ProjStore}) -> + machi_projection_store:get_all_projections(ProjStore, ProjType); +do_server_proj_request({list_all_projections, ProjType}, + #state{proj_store=ProjStore}) -> + machi_projection_store:list_all_projections(ProjStore, ProjType); +do_server_proj_request({kick_projection_reaction}, + #state{flu_name=FluName}) -> + %% Tell my chain manager that it might want to react to + %% this new world. + Chmgr = machi_chain_manager1:make_chmgr_regname(FluName), + spawn(fun() -> + catch machi_chain_manager1:trigger_react_to_env(Chmgr) + end), + async_no_response. + +do_server_append_chunk(CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, CSum, + ChunkExtra, S) -> + case sanitize_prefix(Prefix) of + ok -> + do_server_append_chunk2(CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, CSum, + ChunkExtra, S); + _ -> + {error, bad_arg} + end. + +do_server_append_chunk2(CoC_Namespace, CoC_Locator, + Prefix, Chunk, CSum_tag, Client_CSum, + ChunkExtra, #state{flu_name=FluName, + epoch_id=EpochID}=_S) -> + %% TODO: Do anything with PKey? + try + TaggedCSum = check_or_make_tagged_checksum(CSum_tag, Client_CSum,Chunk), + R = {seq_append, self(), CoC_Namespace, CoC_Locator, + Prefix, Chunk, TaggedCSum, ChunkExtra, EpochID}, + FluName ! R, + receive + {assignment, Offset, File} -> + Size = iolist_size(Chunk), + {ok, {Offset, Size, File}}; + witness -> + {error, bad_arg}; + wedged -> + {error, wedged} + after 10*1000 -> + {error, partition} + end + catch + throw:{bad_csum, _CS} -> + {error, bad_checksum}; + error:badarg -> + error_logger:error_msg("Message send to ~p gave badarg, make certain server is running with correct registered name\n", [?MODULE]), + {error, bad_arg} + end. + +do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, #state{flu_name=FluName}) -> + case sanitize_file_string(File) of + ok -> + case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of + {ok, Pid} -> + Meta = [{client_csum_tag, CSum_tag}, {client_csum, CSum}], + machi_file_proxy:write(Pid, Offset, Meta, Chunk); + {error, trimmed} = Error -> + Error + end; + _ -> + {error, bad_arg} + end. + +do_server_read_chunk(File, Offset, Size, Opts, #state{flu_name=FluName})-> + case sanitize_file_string(File) of + ok -> + case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of + {ok, Pid} -> + case machi_file_proxy:read(Pid, Offset, Size, Opts) of + %% XXX FIXME + %% For now we are omiting the checksum data because it blows up + %% protobufs. + {ok, ChunksAndTrimmed} -> {ok, ChunksAndTrimmed}; + Other -> Other + end; + {error, trimmed} = Error -> + Error + end; + _ -> + {error, bad_arg} + end. + +do_server_trim_chunk(File, Offset, Size, TriggerGC, #state{flu_name=FluName}) -> + lager:debug("Hi there! I'm trimming this: ~s, (~p, ~p), ~p~n", + [File, Offset, Size, TriggerGC]), + case sanitize_file_string(File) of + ok -> + case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of + {ok, Pid} -> + machi_file_proxy:trim(Pid, Offset, Size, TriggerGC); + {error, trimmed} = Trimmed -> + %% Should be returned back to (maybe) trigger repair + Trimmed + end; + _ -> + {error, bad_arg} + end. + +do_server_checksum_listing(File, #state{flu_name=FluName, data_dir=DataDir}=_S) -> + case sanitize_file_string(File) of + ok -> + case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of + {ok, Pid} -> + {ok, List} = machi_file_proxy:checksum_list(Pid), + Bin = erlang:term_to_binary(List), + if byte_size(Bin) > (?PB_MAX_MSG_SIZE - 1024) -> + %% TODO: Fix this limitation by streaming the + %% binary in multiple smaller PB messages. + %% Also, don't read the file all at once. ^_^ + error_logger:error_msg("~s:~w oversize ~s\n", + [?MODULE, ?LINE, DataDir]), + {error, bad_arg}; + true -> + {ok, Bin} + end; + {error, trimmed} -> + {error, trimmed} + end; + _ -> + {error, bad_arg} + end. + +do_server_list_files(#state{data_dir=DataDir}=_S) -> + {_, WildPath} = machi_util:make_data_filename(DataDir, ""), + Files = filelib:wildcard("*", WildPath), + {ok, [begin + {ok, FI} = file:read_file_info(WildPath ++ "/" ++ File), + Size = FI#file_info.size, + {Size, File} + end || File <- Files]}. + +do_server_wedge_status(S) -> + {Wedged_p, CurrentEpochID0} = ets:lookup_element(S#state.etstab, epoch, 2), + CurrentEpochID = if CurrentEpochID0 == undefined -> + ?DUMMY_PV1_EPOCH; + true -> + CurrentEpochID0 + end, + {Wedged_p, CurrentEpochID}. + +do_server_delete_migration(File, #state{data_dir=DataDir}=_S) -> + case sanitize_file_string(File) of + ok -> + {_, Path} = machi_util:make_data_filename(DataDir, File), + case file:delete(Path) of + ok -> + ok; + {error, enoent} -> + {error, no_such_file}; + _ -> + {error, bad_arg} + end; + _ -> + {error, bad_arg} + end. + +do_server_trunc_hack(File, #state{data_dir=DataDir}=_S) -> + case sanitize_file_string(File) of + ok -> + {_, Path} = machi_util:make_data_filename(DataDir, File), + case file:open(Path, [read, write, binary, raw]) of + {ok, FH} -> + try + {ok, ?MINIMUM_OFFSET} = file:position(FH, + ?MINIMUM_OFFSET), + ok = file:truncate(FH), + ok + after + file:close(FH) + end; + {error, enoent} -> + {error, no_such_file}; + _ -> + {error, bad_arg} + end; + _ -> + {error, bad_arg} + end. + +sanitize_file_string(Str) -> + case has_no_prohibited_chars(Str) andalso machi_util:is_valid_filename(Str) of + true -> ok; + false -> error + end. + +has_no_prohibited_chars(Str) -> + case re:run(Str, "/") of + nomatch -> + true; + _ -> + true + end. + +sanitize_prefix(Prefix) -> + %% We are using '^' as our component delimiter + case re:run(Prefix, "/|\\^") of + nomatch -> + ok; + _ -> + error + end. + +check_or_make_tagged_checksum(?CSUM_TAG_NONE, _Client_CSum, Chunk) -> + %% TODO: If the client was foolish enough to use + %% this type of non-checksum, then the client gets + %% what it deserves wrt data integrity, alas. In + %% the client-side Chain Replication method, each + %% server will calculated this independently, which + %% isn't exactly what ought to happen for best data + %% integrity checking. In server-side CR, the csum + %% should be calculated by the head and passed down + %% the chain together with the value. + CS = machi_util:checksum_chunk(Chunk), + machi_util:make_tagged_csum(server_sha, CS); +check_or_make_tagged_checksum(?CSUM_TAG_CLIENT_SHA, Client_CSum, Chunk) -> + CS = machi_util:checksum_chunk(Chunk), + if CS == Client_CSum -> + machi_util:make_tagged_csum(server_sha, + Client_CSum); + true -> + throw({bad_csum, CS}) + end. + + + + +%%%% High PB mode %%%% + +do_pb_hl_request(#mpb_request{req_id=ReqID}, #state{pb_mode=low}=S) -> + Result = {low_error, 41, "High protocol request while in low mode"}, + {machi_pb_translate:to_pb_response(ReqID, unused, Result), S}; +do_pb_hl_request(PB_request, S) -> + {ReqID, Cmd} = machi_pb_translate:from_pb_request(PB_request), + {Result, S2} = do_pb_hl_request2(Cmd, S), + {machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}. + +do_pb_hl_request2({high_echo, Msg}, S) -> + {Msg, S}; +do_pb_hl_request2({high_auth, _User, _Pass}, S) -> + {-77, S}; +do_pb_hl_request2({high_append_chunk, CoC_Namespace, CoC_Locator, + Prefix, ChunkBin, TaggedCSum, + ChunkExtra}, #state{high_clnt=Clnt}=S) -> + Chunk = {TaggedCSum, ChunkBin}, + Res = machi_cr_client:append_chunk_extra(Clnt, CoC_Namespace, CoC_Locator, + Prefix, Chunk, + ChunkExtra), + {Res, S}; +do_pb_hl_request2({high_write_chunk, File, Offset, ChunkBin, TaggedCSum}, + #state{high_clnt=Clnt}=S) -> + Chunk = {TaggedCSum, ChunkBin}, + Res = machi_cr_client:write_chunk(Clnt, File, Offset, Chunk), + {Res, S}; +do_pb_hl_request2({high_read_chunk, File, Offset, Size, Opts}, + #state{high_clnt=Clnt}=S) -> + Res = machi_cr_client:read_chunk(Clnt, File, Offset, Size, Opts), + {Res, S}; +do_pb_hl_request2({high_trim_chunk, File, Offset, Size}, + #state{high_clnt=Clnt}=S) -> + Res = machi_cr_client:trim_chunk(Clnt, File, Offset, Size), + {Res, S}; +do_pb_hl_request2({high_checksum_list, File}, #state{high_clnt=Clnt}=S) -> + Res = machi_cr_client:checksum_list(Clnt, File), + {Res, S}; +do_pb_hl_request2({high_list_files}, #state{high_clnt=Clnt}=S) -> + Res = machi_cr_client:list_files(Clnt), + {Res, S}. + +make_high_clnt(#state{high_clnt=undefined}=S) -> + {ok, Proj} = machi_projection_store:read_latest_projection( + S#state.proj_store, private), + Ps = [P_srvr || {_, P_srvr} <- orddict:to_list( + Proj#projection_v1.members_dict)], + {ok, Clnt} = machi_cr_client:start_link(Ps), + S#state{high_clnt=Clnt}; +make_high_clnt(S) -> + S. diff --git a/src/machi_sup.erl b/src/machi_sup.erl index d4ab71a..729b6a7 100644 --- a/src/machi_sup.erl +++ b/src/machi_sup.erl @@ -47,8 +47,6 @@ start_link() -> supervisor:start_link({local, ?SERVER}, ?MODULE, []). init([]) -> - %% {_, Ps} = process_info(self(), links), - %% [unlink(P) || P <- Ps], RestartStrategy = one_for_one, MaxRestarts = 1000, MaxSecondsBetweenRestarts = 3600, @@ -59,12 +57,8 @@ init([]) -> Shutdown = ?SHUTDOWN, Type = supervisor, - ServerSup = - {machi_flu_sup, {machi_flu_sup, start_link, []}, - Restart, Shutdown, Type, []}, - - {ok, {SupFlags, [ServerSup]}}. - - %% AChild = {'AName', {'AModule', start_link, []}, - %% Restart, Shutdown, Type, ['AModule']}, - %% {ok, {SupFlags, [AChild]}}. + FluSup = {machi_flu_sup, {machi_flu_sup, start_link, []}, + Restart, Shutdown, Type, []}, + RanchSup = {ranch_sup, {ranch_sup, start_link, []}, + Restart, Shutdown, supervisor, [ranch_sup]}, + {ok, {SupFlags, [FluSup, RanchSup]}}. -- 2.45.2 From 7614910f36ca2074ae523ff6d2abe04de44ec3f4 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Wed, 18 Nov 2015 17:00:14 +0900 Subject: [PATCH 35/77] Initialize FLU package with ranch listener --- src/machi_flu1.erl | 29 ++--- src/machi_flu_psup.erl | 14 +- src/machi_listener_sup.erl | 78 ++++++++++++ src/machi_pb_protocol.erl | 253 +++++++++++++++---------------------- 4 files changed, 193 insertions(+), 181 deletions(-) create mode 100644 src/machi_listener_sup.erl diff --git a/src/machi_flu1.erl b/src/machi_flu1.erl index c02772d..56c412c 100644 --- a/src/machi_flu1.erl +++ b/src/machi_flu1.erl @@ -58,10 +58,9 @@ -export([start_link/1, stop/1, update_wedge_state/3, wedge_myself/2]). --export([make_listener_regname/1, make_projection_server_regname/1]). +-export([make_projection_server_regname/1]). %% TODO: remove or replace in OTP way after gen_*'ified -export([main2/4, run_append_server/2, - %% run_listen_server/1, current_state/1, format_state/1]). -record(state, { @@ -69,14 +68,9 @@ proj_store :: pid(), witness = false :: boolean(), append_pid :: pid(), - tcp_port :: non_neg_integer(), - data_dir :: string(), wedged = true :: boolean(), etstab :: ets:tid(), epoch_id :: 'undefined' | machi_dt:epoch_id(), - pb_mode = undefined :: 'undefined' | 'high' | 'low', - high_clnt :: 'undefined' | pid(), - trim_table :: ets:tid(), props = [] :: list() % proplist }). @@ -153,8 +147,6 @@ main2(FluName, TcpPort, DataDir, Props) -> S0 = #state{flu_name=FluName, proj_store=ProjectionPid, - tcp_port=TcpPort, - data_dir=DataDir, wedged=Wedged_p, witness=Witness_p, etstab=ets_table_name(FluName), @@ -168,7 +160,8 @@ main2(FluName, TcpPort, DataDir, Props) -> ok end, S1 = S0#state{append_pid=AppendPid}, - {ok, ListenPid} = start_listen_server(S1), + {ok, ListenerPid} = start_listen_server(TcpPort, DataDir, S1), + io:format(user, "Listener started: ~w~n", [{FluName, ListenerPid}]), Config_e = machi_util:make_config_filename(DataDir, "unused"), ok = filelib:ensure_dir(Config_e), @@ -180,22 +173,23 @@ main2(FluName, TcpPort, DataDir, Props) -> put(flu_flu_name, FluName), put(flu_append_pid, S1#state.append_pid), put(flu_projection_pid, ProjectionPid), - put(flu_listen_pid, ListenPid), + put(flu_listen_pid, ListenerPid), proc_lib:init_ack({ok, self()}), receive killme -> ok end, (catch exit(S1#state.append_pid, kill)), (catch exit(ProjectionPid, kill)), - (catch exit(ListenPid, kill)), + (catch exit(ListenerPid, kill)), ok. start_append_server(S, AckPid) -> proc_lib:start_link(?MODULE, run_append_server, [AckPid, S], ?INIT_TIMEOUT). -start_listen_server(_S) -> - %% FIXMEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE - %% proc_lib:start_link(?MODULE, run_listen_server, [S], ?INIT_TIMEOUT). - {ok, dummy}. +start_listen_server(TcpPort, DataDir, + #state{flu_name=FluName, witness=Witness, etstab=EtsTab, + proj_store=ProjStore}=_S) -> + machi_listener_sup:start_listener(FluName, TcpPort, Witness, DataDir, + EtsTab, ProjStore). run_append_server(FluPid, #state{flu_name=Name, wedged=Wedged_p,epoch_id=EpochId}=S) -> @@ -307,9 +301,6 @@ handle_append(CoC_Namespace, CoC_Locator, Error end. -make_listener_regname(BaseName) -> - list_to_atom(atom_to_list(BaseName) ++ "_listener"). - %% This is the name of the projection store that is spawned by the %% *flu*, for use primarily in testing scenarios. In normal use, we %% ought to be using the OTP style of managing processes, via diff --git a/src/machi_flu_psup.erl b/src/machi_flu_psup.erl index b2ccdcc..55584d5 100644 --- a/src/machi_flu_psup.erl +++ b/src/machi_flu_psup.erl @@ -143,21 +143,19 @@ init([FluName, TcpPort, DataDir, Props0]) -> FProxySupSpec = machi_file_proxy_sup:child_spec(FluName), - ListenerRegName = machi_flu1:make_listener_regname(FluName), - NbAcceptors = 100, - ListenerSpec = ranch:child_spec(ListenerRegName, NbAcceptors, - ranch_tcp, [{port, TcpPort}], - machi_pb_protocol, []), + ListenerSupSpec = {machi_listener_sup:make_listener_sup_name(FluName), + {machi_listener_sup, start_link, [FluName]}, + permanent, ?SHUTDOWN, supervisor, []}, + FluSpec = {FluName, {machi_flu1, start_link, - [ [{FluName, TcpPort+1, DataDir}|Props] ]}, + [ [{FluName, TcpPort, DataDir}|Props] ]}, permanent, ?SHUTDOWN, worker, []}, - {ok, {SupFlags, [ ProjSpec, FitnessSpec, MgrSpec, FProxySupSpec, FNameMgrSpec, MetaSupSpec, - FluSpec, ListenerSpec]}}. + ListenerSupSpec, FluSpec]}}. make_flu_regname(FluName) when is_atom(FluName) -> FluName. diff --git a/src/machi_listener_sup.erl b/src/machi_listener_sup.erl new file mode 100644 index 0000000..dc004ce --- /dev/null +++ b/src/machi_listener_sup.erl @@ -0,0 +1,78 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% @doc This is the supervisor to hold ranch listener for sigle FLU, +%% holds at most one child worker. + +%% TODO: This supervisor is maybe useless. First introduced by workaround +%% to start listener dynamically in flu1 initialization time. +%% Because psup is blocked in flu1 initialization time, adding a child +%% to psup leads to deadlock. +%% By the result of refactoring process, if initialization can be done +%% only by static arguments, then this supervisor should be removed +%% and add listener as a direct child of psup. + +-module(machi_listener_sup). +-behaviour(supervisor). + +%% public API +-export([start_link/1, + start_listener/6, + stop_listener/1, + make_listener_sup_name/1, + make_listener_name/1]). + +%% supervisor callback +-export([init/1]). + +-define(BACKLOG, 8192). + +start_link(FluName) -> + supervisor:start_link({local, make_listener_sup_name(FluName)}, ?MODULE, []). + +start_listener(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore) -> + supervisor:start_child(make_listener_sup_name(FluName), + child_spec(FluName, TcpPort, Witness, DataDir, + EpochTab, ProjStore)). + +stop_listener(FluName) -> + SupName = make_listener_sup_name(FluName), + ListenerName = make_listener_name(FluName), + ok = supervisor:terminate_child(SupName, ListenerName), + ok = supervisor:delete_child(SupName, ListenerName). + +make_listener_sup_name(FluName) when is_atom(FluName) -> + list_to_atom(atom_to_list(FluName) ++ "_listener_sup"). + +make_listener_name(FluName) -> + list_to_atom(atom_to_list(FluName) ++ "_listener"). + +init([]) -> + SupFlags = {one_for_one, 1000, 10}, + {ok, {SupFlags, []}}. + +child_spec(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore) -> + ListenerName = make_listener_name(FluName), + NbAcceptors = 100, + TcpOpts = [{port, TcpPort}, {backlog, ?BACKLOG}], + ProtoOpts = [FluName, Witness, DataDir, EpochTab, ProjStore], + ranch:child_spec(ListenerName, NbAcceptors, + ranch_tcp, TcpOpts, + machi_pb_protocol, ProtoOpts). diff --git a/src/machi_pb_protocol.erl b/src/machi_pb_protocol.erl index 28ce626..9368a2a 100644 --- a/src/machi_pb_protocol.erl +++ b/src/machi_pb_protocol.erl @@ -19,7 +19,7 @@ %% ------------------------------------------------------------------- %% @doc Ranch protocol callback module to handle PB protocol over -%% transport +%% transport, including both high and low modes. %% TODO %% - Two modes, high and low should be separated at listener level? @@ -28,172 +28,114 @@ -behaviour(gen_server). -behaviour(ranch_protocol). + -export([start_link/4]). -export([init/1]). -export([handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-include_lib("kernel/include/file.hrl"). + -include("machi.hrl"). -include("machi_pb.hrl"). -include("machi_projection.hrl"). --define(V(X,Y), ok). -%% -include("machi_verbose.hrl"). -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). -endif. % TEST --record(state, {ref, - socket, - transport, - opts, - pb_mode, - data_dir, - witness, - %% - Used in projection related requests in low mode - %% - Used in spawning CR client in high mode - proj_store, - %%%% Low mode only - %% Current best knowledge, used for wedge_self / bad_epoch check - epoch_id, - %% Used in dispatching append_chunk* reqs to the - %% append serializing process - flu_name, - %% Stored in ETS before factorization, can be stored in the recored? - wedged, - %% Used in server_wedge_status to lookup the table - etstab, - %%%% High mode only - high_clnt, - %%%% to be removed - eof - }). +-record(state, { + %% Transport related items passed from Ranch + ref :: ranch:ref(), + socket :: socket(), + transport :: module(), -%% -record(state, { - %% used in append serializer to trigger chain mgr react_to_env -%% flu_name :: atom(), -%% proj_store :: pid(), -%% witness = false :: boolean(), -%% append_pid :: pid(), -%% tcp_port :: non_neg_integer(), -%% data_dir :: string(), -%% wedged = true :: boolean(), -%% etstab :: ets:tid(), -%% epoch_id :: 'undefined' | machi_dt:epoch_id(), -%% pb_mode = undefined :: 'undefined' | 'high' | 'low', -%% high_clnt :: 'undefined' | pid(), -%% trim_table :: ets:tid(), -%% props = [] :: list() % proplist -%% }). + %% Machi application related items below + data_dir :: string(), + witness :: boolean(), + pb_mode :: undefined | high | low, + %% - Used in projection related requests in low mode + %% - Used in spawning CR client in high mode + proj_store :: pid(), --spec start_link(ranch:ref(), any(), module(), any()) -> {ok, pid()}. -start_link(Ref, Socket, Transport, Opts) -> - proc_lib:start_link(?MODULE, init, [#state{ref=Ref, socket=Socket, - transport=Transport, - opts=Opts}]). + %% Low mode only + %% Current best knowledge, used for wedge_self / bad_epoch check + epoch_id :: undefined | machi_dt:epoch_id(), + %% Used in dispatching append_chunk* reqs to the + %% append serializing process + flu_name :: atom(), + %% Used in server_wedge_status to lookup the table + epoch_tab :: ets:tid(), -init(#state{ref=Ref, socket=Socket, transport=Transport, opts=_Opts}=State) -> - ok = proc_lib:init_ack({ok, self()}), - %% TODO: Perform any required state initialization here. - ok = ranch:accept_ack(Ref), - ok = Transport:setopts(Socket, [{active, once}]), - gen_server:enter_loop(?MODULE, [], State). + %% High mode only + high_clnt :: pid(), + + %% anything you want + props = [] :: list() % proplist + }). + +-type socket() :: any(). +-type state() :: #state{}. + +-spec start_link(ranch:ref(), socket(), module(), [term()]) -> {ok, pid()}. +start_link(Ref, Socket, Transport, [FluName, Witness, DataDir, EpochTab, ProjStore]) -> + proc_lib:start_link(?MODULE, init, [#state{ref=Ref, + socket=Socket, + transport=Transport, + flu_name=FluName, + witness=Witness, + data_dir=DataDir, + epoch_tab=EpochTab, + proj_store=ProjStore}]). + +-spec init(state()) -> no_return(). +init(#state{ref=Ref, socket=Socket, transport=Transport}=State) -> + ok = proc_lib:init_ack({ok, self()}), + ok = ranch:accept_ack(Ref), + {_Wedged_p, CurrentEpochID} = lookup_epoch(State), + ok = Transport:setopts(Socket, [{active, once}|?PB_PACKET_OPTS]), + gen_server:enter_loop(?MODULE, [], State#state{epoch_id=CurrentEpochID}). handle_call(Request, _From, S) -> + lager:warning("~s:handle_call UNKNOWN message: ~w", [?MODULE, Request]), Reply = {error, {unknown_message, Request}}, {reply, Reply, S}. handle_cast(_Msg, S) -> - io:format(user, "~s:handle_cast: ~p\n", [?MODULE, _Msg]), + lager:warning("~s:handle_cast UNKNOWN message: ~w", [?MODULE, _Msg]), {noreply, S}. -handle_info({tcp, Sock, Data}=_Info, S) -> - io:format(user, "~s:handle_info: ~p\n", [?MODULE, _Info]), +%% TODO: Other transport support needed?? TLS/SSL, SCTP +handle_info({tcp, Sock, Data}=_Info, #state{socket=Sock}=S) -> + lager:debug("~s:handle_info: ~w", [?MODULE, _Info]), transport_received(Sock, Data, S); -handle_info({tcp_closed, Sock}=_Info, S) -> - io:format(user, "~s:handle_info: ~p\n", [?MODULE, _Info]), +handle_info({tcp_closed, Sock}=_Info, #state{socket=Sock}=S) -> + lager:debug("~s:handle_info: ~w", [?MODULE, _Info]), transport_closed(Sock, S); -handle_info({tcp_error, Sock, Reason}=_Info, S) -> - io:format(user, "~s:handle_info: ~p\n", [?MODULE, _Info]), +handle_info({tcp_error, Sock, Reason}=_Info, #state{socket=Sock}=S) -> + lager:debug("~s:handle_info: ~w", [?MODULE, _Info]), transport_error(Sock, Reason, S); handle_info(_Info, S) -> - io:format(user, "~s:handle_info: ~p\n", [?MODULE, _Info]), + lager:warning("~s:handle_info UNKNOWN message: ~w", [?MODULE, _Info]), {noreply, S}. -terminate(_Reason, _S) -> - io:format(user, "~s:terminate: ~p\n", [?MODULE, _Reason]), +terminate(_Reason, #state{socket=undefined}=_S) -> + lager:debug("~s:terminate: ~w", [?MODULE, _Reason]), + ok; +terminate(_Reason, #state{socket=Socket}=_S) -> + lager:debug("~s:terminate: ~w", [?MODULE, _Reason]), + (catch gen_tcp:close(Socket)), ok. code_change(_OldVsn, S, _Extra) -> {ok, S}. -%% Internal functions, or copy-n-paste'd thingie - -%%%% Just copied and will be removed %%%% - -%% TODO: sock opts should be migrated to ranch equivalent -%% run_listen_server(#state{flu_name=FluName, tcp_port=TcpPort}=S) -> -%% register(make_listener_regname(FluName), self()), -%% SockOpts = ?PB_PACKET_OPTS ++ -%% [{reuseaddr, true}, {mode, binary}, {active, false}, -%% {backlog,8192}], -%% case gen_tcp:listen(TcpPort, SockOpts) of -%% {ok, LSock} -> -%% proc_lib:init_ack({ok, self()}), -%% listen_server_loop(LSock, S); -%% Else -> -%% error_logger:warning_msg("~s:run_listen_server: " -%% "listen to TCP port ~w: ~w\n", -%% [?MODULE, TcpPort, Else]), -%% exit({?MODULE, run_listen_server, tcp_port, TcpPort, Else}) -%% end. - -%% listen_server_loop(LSock, S) -> -%% {ok, Sock} = gen_tcp:accept(LSock), -%% spawn_link(fun() -> net_server_loop(Sock, S) end), -%% listen_server_loop(LSock, S). - -%% net_server_loop(Sock, S) -> -%% case gen_tcp:recv(Sock, 0, ?SERVER_CMD_READ_TIMEOUT) of -%% {ok, Bin} -> -%% {RespBin, S2} = -%% case machi_pb:decode_mpb_ll_request(Bin) of -%% LL_req when LL_req#mpb_ll_request.do_not_alter == 2 -> -%% {R, NewS} = do_pb_ll_request(LL_req, S), -%% {maybe_encode_response(R), mode(low, NewS)}; -%% _ -> -%% HL_req = machi_pb:decode_mpb_request(Bin), -%% 1 = HL_req#mpb_request.do_not_alter, -%% {R, NewS} = do_pb_hl_request(HL_req, make_high_clnt(S)), -%% {machi_pb:encode_mpb_response(R), mode(high, NewS)} -%% end, -%% if RespBin == async_no_response -> -%% net_server_loop(Sock, S2); -%% true -> -%% case gen_tcp:send(Sock, RespBin) of -%% ok -> -%% net_server_loop(Sock, S2); -%% {error, _} -> -%% (catch gen_tcp:close(Sock)), -%% exit(normal) -%% end -%% end; -%% {error, SockError} -> -%% Msg = io_lib:format("Socket error ~w", [SockError]), -%% R = #mpb_ll_response{req_id= <<>>, -%% generic=#mpb_errorresp{code=1, msg=Msg}}, -%% _Resp = machi_pb:encode_mpb_ll_response(R), -%% %% TODO: Weird that sometimes neither catch nor try/catch -%% %% can prevent OTP's SASL from logging an error here. -%% %% Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,....... -%% %% TODO: is this what causes the intermittent PULSE deadlock errors? -%% %% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000), -%% (catch gen_tcp:close(Sock)), -%% exit(normal) -%% end. +%% -- private %%%% Common transport handling +-spec transport_received(socket(), machi_dt:chunk(), state()) -> + {noreply, state()}. transport_received(Sock, Bin, #state{transport=Transport}=S) -> {RespBin, S2} = case machi_pb:decode_mpb_ll_request(Bin) of @@ -207,33 +149,36 @@ transport_received(Sock, Bin, #state{transport=Transport}=S) -> {machi_pb:encode_mpb_response(R), mode(high, NewS)} end, if RespBin == async_no_response -> + Transport:setopts(Sock, [{active, once}]), {noreply, S2}; true -> case Transport:send(Sock, RespBin) of ok -> + Transport:setopts(Sock, [{active, once}]), {noreply, S2}; {error, Reason} -> transport_error(Sock, Reason, S2) end end. -transport_closed(Sock, S) -> - (catch gen_tcp:close(Sock)), - {stop, normal, S#state{sock=undefined}}. +-spec transport_closed(socket(), state()) -> {stop, term(), state()}. +transport_closed(_Socket, S) -> + {stop, normal, S}. -transport_error(Sock, Reason, S) -> - Msg = io_lib:format("Socket error ~w", [SockError]), +-spec transport_error(socket(), term(), state()) -> no_return(). +transport_error(Sock, Reason, #state{transport=Transport}=_S) -> + Msg = io_lib:format("Socket error ~w", [Reason]), R = #mpb_ll_response{req_id= <<>>, generic=#mpb_errorresp{code=1, msg=Msg}}, _Resp = machi_pb:encode_mpb_ll_response(R), - %% TODO of TODO comments: comments below with four %s are copy-n-paste'd, + %% TODO for TODO comments: comments below with four %s are copy-n-paste'd, %% then it should be considered they are still open and should be addressed. %%%% TODO: Weird that sometimes neither catch nor try/catch %%%% can prevent OTP's SASL from logging an error here. %%%% Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,....... %%%% TODO: is this what causes the intermittent PULSE deadlock errors? %%%% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000), - (catch gen_tcp:close(Sock)), + (catch Transport:close(Sock)), %% TODO: better to exit with `Reason'? exit(normal). @@ -242,14 +187,11 @@ maybe_encode_response(async_no_response=X) -> maybe_encode_response(R) -> machi_pb:encode_mpb_ll_response(R). -%%%% Not categorized / not-yet-well-understood items -%% TODO: may be external API mode(Mode, #state{pb_mode=undefined}=S) -> S#state{pb_mode=Mode}; mode(_, S) -> S. - %%%% Low PB mode %%%% do_pb_ll_request(#mpb_ll_request{req_id=ReqID}, #state{pb_mode=high}=S) -> @@ -257,24 +199,27 @@ do_pb_ll_request(#mpb_ll_request{req_id=ReqID}, #state{pb_mode=high}=S) -> {machi_pb_translate:to_pb_response(ReqID, unused, Result), S}; do_pb_ll_request(PB_request, S) -> Req = machi_pb_translate:from_pb_request(PB_request), + %% io:format(user, "[~w] do_pb_ll_request Req: ~w~n", [S#state.flu_name, Req]), {ReqID, Cmd, Result, S2} = case Req of - {RqID, {LowCmd, _}=CMD} - when LowCmd == low_proj; - LowCmd == low_wedge_status; LowCmd == low_list_files -> + {RqID, {LowCmd, _}=Cmd0} + when LowCmd =:= low_proj; + LowCmd =:= low_wedge_status; + LowCmd =:= low_list_files -> %% Skip wedge check for projection commands! %% Skip wedge check for these unprivileged commands - {Rs, NewS} = do_pb_ll_request3(CMD, S), - {RqID, CMD, Rs, NewS}; - {RqID, CMD} -> - EpochID = element(2, CMD), % by common convention - {Rs, NewS} = do_pb_ll_request2(EpochID, CMD, S), - {RqID, CMD, Rs, NewS} + {Rs, NewS} = do_pb_ll_request3(Cmd0, S), + {RqID, Cmd0, Rs, NewS}; + {RqID, Cmd0} -> + EpochID = element(2, Cmd0), % by common convention + {Rs, NewS} = do_pb_ll_request2(EpochID, Cmd0, S), + {RqID, Cmd0, Rs, NewS} end, {machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}. do_pb_ll_request2(EpochID, CMD, S) -> - {Wedged_p, CurrentEpochID} = ets:lookup_element(S#state.etstab, epoch, 2), + {Wedged_p, CurrentEpochID} = lookup_epoch(S), + %% io:format(user, "{Wedged_p, CurrentEpochID}: ~w~n", [{Wedged_p, CurrentEpochID}]), if Wedged_p == true -> {{error, wedged}, S#state{epoch_id=CurrentEpochID}}; is_tuple(EpochID) @@ -287,7 +232,7 @@ do_pb_ll_request2(EpochID, CMD, S) -> true -> %% We're at same epoch # but different checksum, or %% we're at a newer/bigger epoch #. - _ = wedge_myself(S#state.flu_name, CurrentEpochID), + _ = machi_flu1:wedge_myself(S#state.flu_name, CurrentEpochID), ok end, {{error, bad_epoch}, S#state{epoch_id=CurrentEpochID}}; @@ -295,6 +240,9 @@ do_pb_ll_request2(EpochID, CMD, S) -> do_pb_ll_request3(CMD, S#state{epoch_id=CurrentEpochID}) end. +lookup_epoch(#state{epoch_tab=T}) -> + ets:lookup_element(T, epoch, 2). + %% Witness status does not matter below. do_pb_ll_request3({low_echo, _BogusEpochID, Msg}, S) -> {Msg, S}; @@ -498,7 +446,7 @@ do_server_list_files(#state{data_dir=DataDir}=_S) -> end || File <- Files]}. do_server_wedge_status(S) -> - {Wedged_p, CurrentEpochID0} = ets:lookup_element(S#state.etstab, epoch, 2), + {Wedged_p, CurrentEpochID0} = lookup_epoch(S), CurrentEpochID = if CurrentEpochID0 == undefined -> ?DUMMY_PV1_EPOCH; true -> @@ -589,9 +537,6 @@ check_or_make_tagged_checksum(?CSUM_TAG_CLIENT_SHA, Client_CSum, Chunk) -> throw({bad_csum, CS}) end. - - - %%%% High PB mode %%%% do_pb_hl_request(#mpb_request{req_id=ReqID}, #state{pb_mode=low}=S) -> -- 2.45.2 From a1f5a6ce6201f2760176addacb82fb83c44c3ce8 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Thu, 19 Nov 2015 17:42:01 +0900 Subject: [PATCH 36/77] Fix unit test cases around flu1 startup --- src/machi_flu1.erl | 2 +- test/machi_admin_util_test.erl | 4 +- test/machi_chain_manager1_test.erl | 66 +++++++++---------------- test/machi_cinfo_test.erl | 2 +- test/machi_cr_client_test.erl | 4 +- test/machi_flu_psup_test.erl | 4 +- test/machi_pb_high_client_test.erl | 2 +- test/machi_proxy_flu1_client_test.erl | 8 ++- test/machi_psup_test_util.erl | 71 +++++++++++++++++++++++++++ 9 files changed, 109 insertions(+), 54 deletions(-) create mode 100644 test/machi_psup_test_util.erl diff --git a/src/machi_flu1.erl b/src/machi_flu1.erl index 56c412c..41e5125 100644 --- a/src/machi_flu1.erl +++ b/src/machi_flu1.erl @@ -161,7 +161,7 @@ main2(FluName, TcpPort, DataDir, Props) -> end, S1 = S0#state{append_pid=AppendPid}, {ok, ListenerPid} = start_listen_server(TcpPort, DataDir, S1), - io:format(user, "Listener started: ~w~n", [{FluName, ListenerPid}]), + %% io:format(user, "Listener started: ~w~n", [{FluName, ListenerPid}]), Config_e = machi_util:make_config_filename(DataDir, "unused"), ok = filelib:ensure_dir(Config_e), diff --git a/test/machi_admin_util_test.erl b/test/machi_admin_util_test.erl index e00fa31..8a43c39 100644 --- a/test/machi_admin_util_test.erl +++ b/test/machi_admin_util_test.erl @@ -44,6 +44,7 @@ verify_file_checksums_test2() -> TcpPort = 32958, DataDir = "./data", W_props = [{initial_wedged, false}], + {ok, SupPid} = machi_sup:start_link(), machi_flu1_test:start_flu_package(verify1_flu, TcpPort, DataDir, W_props), Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}), @@ -80,7 +81,8 @@ verify_file_checksums_test2() -> ok after catch ?FLU_C:quit(Sock1), - catch machi_flu1_test:stop_flu_package(verify1_flu) + catch machi_flu1_test:stop_flu_package(verify1_flu), + exit(SupPid, normal) end. -endif. % !PULSE diff --git a/test/machi_chain_manager1_test.erl b/test/machi_chain_manager1_test.erl index e13193f..3f95955 100644 --- a/test/machi_chain_manager1_test.erl +++ b/test/machi_chain_manager1_test.erl @@ -274,42 +274,30 @@ make_prop_ets() -> -endif. % EQC smoke0_test() -> - {ok, _} = machi_partition_simulator:start_link({1,2,3}, 50, 50), - Host = "localhost", TcpPort = 6623, - {ok, FLUa} = machi_flu1:start_link([{a,TcpPort,"./data.a"}]), - Pa = #p_srvr{name=a, address=Host, port=TcpPort}, - Members_Dict = machi_projection:make_members_dict([Pa]), - %% Egadz, more racing on startup, yay. TODO fix. - timer:sleep(1), + {_, [Pa], [M0]} = machi_psup_test_util:start_flu_packages( + 1, "./data.", TcpPort, []), {ok, FLUaP} = ?FLU_PC:start_link(Pa), - {ok, M0} = ?MGR:start_link(a, Members_Dict, [{active_mode, false}]), try pong = ?MGR:ping(M0) after - ok = ?MGR:stop(M0), - ok = machi_flu1:stop(FLUa), ok = ?FLU_PC:quit(FLUaP), - ok = machi_partition_simulator:stop() + machi_psup_test_util:stop_flu_packages() end. smoke1_test_() -> {timeout, 1*60, fun() -> smoke1_test2() end}. smoke1_test2() -> - machi_partition_simulator:start_link({1,2,3}, 100, 0), TcpPort = 62777, - FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}, {c,TcpPort+2,"./data.c"}], - P_s = [#p_srvr{name=Name, address="localhost", port=Port} || - {Name,Port,_Dir} <- FluInfo], - - [machi_flu1_test:clean_up_data_dir(Dir) || {_,_,Dir} <- FluInfo], - FLUs = [element(2, machi_flu1:start_link([{Name,Port,Dir}])) || - {Name,Port,Dir} <- FluInfo], - MembersDict = machi_projection:make_members_dict(P_s), - {ok, M0} = ?MGR:start_link(a, MembersDict, [{active_mode,false}]), + MgrOpts = [{active_mode,false}], + {_, Ps, MgrNames} = machi_psup_test_util:start_flu_packages( + 3, "./data.", TcpPort, MgrOpts), + MembersDict = machi_projection:make_members_dict(Ps), + [machi_chain_manager1:set_chain_members(M, MembersDict) || M <- MgrNames], + Ma = hd(MgrNames), try - {ok, P1} = ?MGR:test_calc_projection(M0, false), + {ok, P1} = ?MGR:test_calc_projection(Ma, false), % DERP! Check for race with manager's proxy vs. proj listener ok = lists:foldl( fun(_, {_,{true,[{c,ok},{b,ok},{a,ok}]}}) -> @@ -318,37 +306,28 @@ smoke1_test2() -> ok; % Skip remaining! (_, _Else) -> timer:sleep(10), - ?MGR:test_write_public_projection(M0, P1) + ?MGR:test_write_public_projection(Ma, P1) end, not_ok, lists:seq(1, 1000)), %% Writing the exact same projection multiple times returns ok: %% no change! - {_,{true,[{c,ok},{b,ok},{a,ok}]}} = ?MGR:test_write_public_projection(M0, P1), - {unanimous, P1, Extra1} = ?MGR:test_read_latest_public_projection(M0, false), + {_,{true,[{c,ok},{b,ok},{a,ok}]}} = ?MGR:test_write_public_projection(Ma, P1), + {unanimous, P1, Extra1} = ?MGR:test_read_latest_public_projection(Ma, false), ok after - ok = ?MGR:stop(M0), - [ok = machi_flu1:stop(X) || X <- FLUs], - ok = machi_partition_simulator:stop() + machi_psup_test_util:stop_flu_packages() end. nonunanimous_setup_and_fix_test() -> - machi_partition_simulator:start_link({1,2,3}, 100, 0), TcpPort = 62877, - FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}], - P_s = [#p_srvr{name=Name, address="localhost", port=Port} || - {Name,Port,_Dir} <- FluInfo], - - [machi_flu1_test:clean_up_data_dir(Dir) || {_,_,Dir} <- FluInfo], - {ok, SupPid} = machi_flu_sup:start_link(), - Opts = [{active_mode, false}], - %% {ok, Mb} = ?MGR:start_link(b, MembersDict, [{active_mode, false}]++XX), - [{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) || - {Name,Port,Dir} <- FluInfo], + MgrOpts = [{active_mode,false}], + {_, Ps, [Ma,Mb]} = machi_psup_test_util:start_flu_packages( + 2, "./data.", TcpPort, MgrOpts), + MembersDict = machi_projection:make_members_dict(Ps), + [machi_chain_manager1:set_chain_members(M, MembersDict) || M <- [Ma, Mb]], + [Proxy_a, Proxy_b] = Proxies = - [element(2,?FLU_PC:start_link(P)) || P <- P_s], - MembersDict = machi_projection:make_members_dict(P_s), - [Ma,Mb] = [a_chmgr, b_chmgr], + [element(2, ?FLU_PC:start_link(P)) || P <- Ps], ok = machi_chain_manager1:set_chain_members(Ma, MembersDict, []), ok = machi_chain_manager1:set_chain_members(Mb, MembersDict, []), try @@ -394,9 +373,8 @@ nonunanimous_setup_and_fix_test() -> ok after - exit(SupPid, normal), [ok = ?FLU_PC:quit(X) || X <- Proxies], - ok = machi_partition_simulator:stop() + machi_psup_test_util:stop_flu_packages() end. unanimous_report_test() -> diff --git a/test/machi_cinfo_test.erl b/test/machi_cinfo_test.erl index dcb611e..9699df3 100644 --- a/test/machi_cinfo_test.erl +++ b/test/machi_cinfo_test.erl @@ -48,7 +48,7 @@ setup() -> {c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}} ], [os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps], - {ok, SupPid} = machi_flu_sup:start_link(), + {ok, SupPid} = machi_sup:start_link(), %% Only run a, don't run b & c so we have 100% failures talking to them [begin #p_srvr{name=Name, port=Port, props=Dir} = P, diff --git a/test/machi_cr_client_test.erl b/test/machi_cr_client_test.erl index e127032..a370436 100644 --- a/test/machi_cr_client_test.erl +++ b/test/machi_cr_client_test.erl @@ -96,7 +96,7 @@ run_ticks(MgrList) -> ok. smoke_test2() -> - {ok, SupPid} = machi_flu_sup:start_link(), + {ok, SupPid} = machi_sup:start_link(), error_logger:tty(false), try Prefix = <<"pre">>, @@ -208,7 +208,7 @@ io:format(user, "\nFiles = ~p\n", [machi_cr_client:list_files(C1)]), witness_smoke_test_() -> {timeout, 1*60, fun() -> witness_smoke_test2() end}. witness_smoke_test2() -> - SupPid = case machi_flu_sup:start_link() of + SupPid = case machi_sup:start_link() of {ok, P} -> P; {error, {already_started, P1}} -> P1; Other -> error(Other) diff --git a/test/machi_flu_psup_test.erl b/test/machi_flu_psup_test.erl index 1c7b015..afd2d0f 100644 --- a/test/machi_flu_psup_test.erl +++ b/test/machi_flu_psup_test.erl @@ -43,7 +43,7 @@ smoke_test2() -> {c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}} ], [os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps], - {ok, SupPid} = machi_flu_sup:start_link(), + {ok, SupPid} = machi_sup:start_link(), try %% Only run a, don't run b & c so we have 100% failures talking to them [begin @@ -74,7 +74,7 @@ partial_stop_restart2() -> PStores = [machi_flu_psup:make_proj_supname(P#p_srvr.name) || {_,P} <-Ps], Dict = orddict:from_list(Ps), [os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps], - {ok, SupPid} = machi_flu_sup:start_link(), + {ok, SupPid} = machi_sup:start_link(), DbgProps = [{initial_wedged, true}], Start = fun({_,P}) -> #p_srvr{name=Name, port=Port, props=Dir} = P, diff --git a/test/machi_pb_high_client_test.erl b/test/machi_pb_high_client_test.erl index 9f6984b..3273bc2 100644 --- a/test/machi_pb_high_client_test.erl +++ b/test/machi_pb_high_client_test.erl @@ -41,7 +41,7 @@ smoke_test2() -> ok = application:set_env(machi, max_file_size, 1024*1024), [os:cmd("rm -rf " ++ P#p_srvr.props) || P <- Ps], - {ok, SupPid} = machi_flu_sup:start_link(), + {ok, SupPid} = machi_sup:start_link(), try [begin #p_srvr{name=Name, port=Port, props=Dir} = P, diff --git a/test/machi_proxy_flu1_client_test.erl b/test/machi_proxy_flu1_client_test.erl index f505f55..422dce1 100644 --- a/test/machi_proxy_flu1_client_test.erl +++ b/test/machi_proxy_flu1_client_test.erl @@ -38,6 +38,7 @@ api_smoke_test() -> W_props = [{active_mode, false},{initial_wedged, false}], Prefix = <<"prefix">>, + {ok, SupPid} = machi_sup:start_link(), machi_flu1_test:start_flu_package(RegName, TcpPort, DataDir, W_props), try @@ -102,7 +103,8 @@ api_smoke_test() -> _ = (catch ?MUT:quit(Prox1)) end after - (catch machi_flu1_test:stop_flu_package(RegName)) + (catch machi_flu1_test:stop_flu_package(RegName)), + exit(SupPid, normal) end. flu_restart_test_() -> @@ -114,6 +116,7 @@ flu_restart_test2() -> TcpPort = 57125, DataDir = "./data.api_smoke_flu2", W_props = [{initial_wedged, false}, {active_mode, false}], + {ok, SupPid} = machi_sup:start_link(), machi_flu1_test:start_flu_package(RegName, TcpPort, DataDir, W_props), try @@ -306,7 +309,8 @@ flu_restart_test2() -> end || Fun <- ExpectedOps ], ok after - _ = (catch ?MUT:quit(Prox1)) + _ = (catch ?MUT:quit(Prox1)), + exit(SupPid, normal) end after (catch machi_flu1_test:stop_flu_package(RegName)) diff --git a/test/machi_psup_test_util.erl b/test/machi_psup_test_util.erl new file mode 100644 index 0000000..6446bf7 --- /dev/null +++ b/test/machi_psup_test_util.erl @@ -0,0 +1,71 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(machi_psup_test_util). +-compile(export_all). + +-ifdef(TEST). +-ifndef(PULSE). + +-include_lib("eunit/include/eunit.hrl"). + +-include("machi.hrl"). +-include("machi_projection.hrl"). + +-define(FLU, machi_flu1). +-define(FLU_C, machi_flu1_client). + +-spec start_flu_packages(pos_integer(), string(), inet:port(), list()) -> + {MachiSup::pid(), Ps::[#p_srvr{}], MgrNames::[atom()]}. +start_flu_packages(FluCount, DirPrefix, BaseTcpPort, MgrOpts) -> + FluInfo = flu_info(FluCount, DirPrefix, BaseTcpPort), + _ = stop_machi_sup(), + _ = [machi_flu1_test:clean_up_data_dir(Dir) || {_, Dir, _} <- FluInfo], + {ok, SupPid} = machi_sup:start_link(), + [{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, MgrOpts) || + {#p_srvr{name=Name, port=Port}, Dir, _} <- FluInfo], + {Ps, _Dirs, MgrNames} = lists:unzip3(FluInfo), + {SupPid, Ps, MgrNames}. + +stop_flu_packages() -> + stop_machi_sup(). + +flu_info(FluCount, DirPrefix, BaseTcpPort) -> + [begin + FLUNameStr = [$a + I - 1], + FLUName = list_to_atom(FLUNameStr), + MgrName = machi_flu_psup:make_mgr_supname(FLUName), + {#p_srvr{name=FLUName, address="localhost", port=BaseTcpPort + I, + props=[{chmgr, MgrName}]}, + DirPrefix ++ "/data.eqc." ++ FLUNameStr, + MgrName} + end || I <- lists:seq(1, FluCount)]. + +stop_machi_sup() -> + case whereis(machi_sup) of + undefined -> ok; + Pid -> + catch exit(whereis(machi_sup), normal), + machi_util:wait_for_death(Pid, 30) + end. + +-endif. % !PULSE +-endif. % TEST + -- 2.45.2 From 14765a7279ec028609b7779deece6fa39ab7deed Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Mon, 7 Dec 2015 14:46:08 +0900 Subject: [PATCH 37/77] Change ranch callback module name --- src/machi_flu1.erl | 2 +- ...i_pb_protocol.erl => machi_flu1_net_server.erl} | 14 ++++++++------ src/machi_listener_sup.erl | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) rename src/{machi_pb_protocol.erl => machi_flu1_net_server.erl} (98%) diff --git a/src/machi_flu1.erl b/src/machi_flu1.erl index 41e5125..fade024 100644 --- a/src/machi_flu1.erl +++ b/src/machi_flu1.erl @@ -207,7 +207,7 @@ append_server_loop(FluPid, #state{wedged=Wedged_p, receive {seq_append, From, _N, _L, _Prefix, _Chunk, _CSum, _Extra, _EpochID} when Witness_p -> - %% The FLU's net_server_loop() process ought to filter all + %% The FLU's machi_flu1_net_server process ought to filter all %% witness states, but we'll keep this clause for extra %% paranoia. From ! witness, diff --git a/src/machi_pb_protocol.erl b/src/machi_flu1_net_server.erl similarity index 98% rename from src/machi_pb_protocol.erl rename to src/machi_flu1_net_server.erl index 9368a2a..3f11500 100644 --- a/src/machi_pb_protocol.erl +++ b/src/machi_flu1_net_server.erl @@ -24,7 +24,7 @@ %% TODO %% - Two modes, high and low should be separated at listener level? --module(machi_pb_protocol). +-module(machi_flu1_net_server). -behaviour(gen_server). -behaviour(ranch_protocol). @@ -136,7 +136,9 @@ code_change(_OldVsn, S, _Extra) -> -spec transport_received(socket(), machi_dt:chunk(), state()) -> {noreply, state()}. -transport_received(Sock, Bin, #state{transport=Transport}=S) -> +transport_received(Socket, <<"QUIT\n">>, #state{socket=Socket}=S) -> + {stop, normal, S}; +transport_received(Socket, Bin, #state{transport=Transport}=S) -> {RespBin, S2} = case machi_pb:decode_mpb_ll_request(Bin) of LL_req when LL_req#mpb_ll_request.do_not_alter == 2 -> @@ -149,15 +151,15 @@ transport_received(Sock, Bin, #state{transport=Transport}=S) -> {machi_pb:encode_mpb_response(R), mode(high, NewS)} end, if RespBin == async_no_response -> - Transport:setopts(Sock, [{active, once}]), + Transport:setopts(Socket, [{active, once}]), {noreply, S2}; true -> - case Transport:send(Sock, RespBin) of + case Transport:send(Socket, RespBin) of ok -> - Transport:setopts(Sock, [{active, once}]), + Transport:setopts(Socket, [{active, once}]), {noreply, S2}; {error, Reason} -> - transport_error(Sock, Reason, S2) + transport_error(Socket, Reason, S2) end end. diff --git a/src/machi_listener_sup.erl b/src/machi_listener_sup.erl index dc004ce..f227611 100644 --- a/src/machi_listener_sup.erl +++ b/src/machi_listener_sup.erl @@ -72,7 +72,7 @@ child_spec(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore) -> ListenerName = make_listener_name(FluName), NbAcceptors = 100, TcpOpts = [{port, TcpPort}, {backlog, ?BACKLOG}], - ProtoOpts = [FluName, Witness, DataDir, EpochTab, ProjStore], + NetServerOpts = [FluName, Witness, DataDir, EpochTab, ProjStore], ranch:child_spec(ListenerName, NbAcceptors, ranch_tcp, TcpOpts, - machi_pb_protocol, ProtoOpts). + machi_flu1_net_server, NetServerOpts). -- 2.45.2 From aa0a0413d1b57873e6d59f0a15ebc489519af39e Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Mon, 7 Dec 2015 15:26:39 +0900 Subject: [PATCH 38/77] Cosmetics of comments, specs, whitespaces and unit tests refactoring --- src/machi_flu1_net_server.erl | 53 +++++++------ src/machi_listener_sup.erl | 29 ++++--- test/machi_admin_util_test.erl | 67 ++++++++-------- test/machi_ap_repair_eqc.erl | 2 +- test/machi_chain_manager1_test.erl | 30 ++++---- test/machi_flu1_test.erl | 60 ++------------- test/machi_pb_high_client_test.erl | 31 +++----- test/machi_projection_store_test.erl | 4 +- test/machi_proxy_flu1_client_test.erl | 35 ++++----- test/machi_psup_test_util.erl | 71 ----------------- test/machi_test_util.erl | 107 ++++++++++++++++++++++++++ 11 files changed, 243 insertions(+), 246 deletions(-) delete mode 100644 test/machi_psup_test_util.erl create mode 100644 test/machi_test_util.erl diff --git a/src/machi_flu1_net_server.erl b/src/machi_flu1_net_server.erl index 3f11500..6bc9d3b 100644 --- a/src/machi_flu1_net_server.erl +++ b/src/machi_flu1_net_server.erl @@ -45,12 +45,12 @@ -endif. % TEST -record(state, { - %% Transport related items passed from Ranch + %% Ranch's transport management stuff ref :: ranch:ref(), socket :: socket(), transport :: module(), - %% Machi application related items below + %% Machi FLU configurations, common for low and high data_dir :: string(), witness :: boolean(), pb_mode :: undefined | high | low, @@ -58,14 +58,14 @@ %% - Used in spawning CR client in high mode proj_store :: pid(), - %% Low mode only + %% Low mode only items %% Current best knowledge, used for wedge_self / bad_epoch check epoch_id :: undefined | machi_dt:epoch_id(), %% Used in dispatching append_chunk* reqs to the %% append serializing process - flu_name :: atom(), + flu_name :: pv1_server(), %% Used in server_wedge_status to lookup the table - epoch_tab :: ets:tid(), + epoch_tab :: ets:tab(), %% High mode only high_clnt :: pid(), @@ -106,15 +106,15 @@ handle_cast(_Msg, S) -> {noreply, S}. %% TODO: Other transport support needed?? TLS/SSL, SCTP -handle_info({tcp, Sock, Data}=_Info, #state{socket=Sock}=S) -> +handle_info({tcp, Socket, Data}=_Info, #state{socket=Socket}=S) -> lager:debug("~s:handle_info: ~w", [?MODULE, _Info]), - transport_received(Sock, Data, S); -handle_info({tcp_closed, Sock}=_Info, #state{socket=Sock}=S) -> + transport_received(Socket, Data, S); +handle_info({tcp_closed, Socket}=_Info, #state{socket=Socket}=S) -> lager:debug("~s:handle_info: ~w", [?MODULE, _Info]), - transport_closed(Sock, S); -handle_info({tcp_error, Sock, Reason}=_Info, #state{socket=Sock}=S) -> + transport_closed(Socket, S); +handle_info({tcp_error, Socket, Reason}=_Info, #state{socket=Socket}=S) -> lager:debug("~s:handle_info: ~w", [?MODULE, _Info]), - transport_error(Sock, Reason, S); + transport_error(Socket, Reason, S); handle_info(_Info, S) -> lager:warning("~s:handle_info UNKNOWN message: ~w", [?MODULE, _Info]), {noreply, S}. @@ -143,17 +143,18 @@ transport_received(Socket, Bin, #state{transport=Transport}=S) -> case machi_pb:decode_mpb_ll_request(Bin) of LL_req when LL_req#mpb_ll_request.do_not_alter == 2 -> {R, NewS} = do_pb_ll_request(LL_req, S), - {maybe_encode_response(R), mode(low, NewS)}; + {maybe_encode_response(R), set_mode(low, NewS)}; _ -> HL_req = machi_pb:decode_mpb_request(Bin), 1 = HL_req#mpb_request.do_not_alter, {R, NewS} = do_pb_hl_request(HL_req, make_high_clnt(S)), - {machi_pb:encode_mpb_response(R), mode(high, NewS)} + {machi_pb:encode_mpb_response(R), set_mode(high, NewS)} end, - if RespBin == async_no_response -> + case RespBin of + async_no_response -> Transport:setopts(Socket, [{active, once}]), {noreply, S2}; - true -> + _ -> case Transport:send(Socket, RespBin) of ok -> Transport:setopts(Socket, [{active, once}]), @@ -168,7 +169,7 @@ transport_closed(_Socket, S) -> {stop, normal, S}. -spec transport_error(socket(), term(), state()) -> no_return(). -transport_error(Sock, Reason, #state{transport=Transport}=_S) -> +transport_error(Socket, Reason, #state{transport=Transport}=_S) -> Msg = io_lib:format("Socket error ~w", [Reason]), R = #mpb_ll_response{req_id= <<>>, generic=#mpb_errorresp{code=1, msg=Msg}}, @@ -180,18 +181,20 @@ transport_error(Sock, Reason, #state{transport=Transport}=_S) -> %%%% Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,....... %%%% TODO: is this what causes the intermittent PULSE deadlock errors? %%%% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000), - (catch Transport:close(Sock)), - %% TODO: better to exit with `Reason'? + (catch Transport:close(Socket)), + _ = lager:warning("Socket error (~w -> ~w): ~w", + [Transport:sockname(Socket), Transport:peername(Socket), Reason]), + %% TODO: better to exit with `Reason' without logging? exit(normal). -maybe_encode_response(async_no_response=X) -> - X; +maybe_encode_response(async_no_response=R) -> + R; maybe_encode_response(R) -> machi_pb:encode_mpb_ll_response(R). -mode(Mode, #state{pb_mode=undefined}=S) -> +set_mode(Mode, #state{pb_mode=undefined}=S) -> S#state{pb_mode=Mode}; -mode(_, S) -> +set_mode(_, S) -> S. %%%% Low PB mode %%%% @@ -208,7 +211,6 @@ do_pb_ll_request(PB_request, S) -> when LowCmd =:= low_proj; LowCmd =:= low_wedge_status; LowCmd =:= low_list_files -> - %% Skip wedge check for projection commands! %% Skip wedge check for these unprivileged commands {Rs, NewS} = do_pb_ll_request3(Cmd0, S), {RqID, Cmd0, Rs, NewS}; @@ -254,6 +256,7 @@ do_pb_ll_request3({low_wedge_status, _EpochID}, S) -> {do_server_wedge_status(S), S}; do_pb_ll_request3({low_proj, PCMD}, S) -> {do_server_proj_request(PCMD, S), S}; + %% Witness status *matters* below do_pb_ll_request3({low_append_chunk, _EpochID, CoC_Namespace, CoC_Locator, Prefix, Chunk, CSum_tag, @@ -285,6 +288,7 @@ do_pb_ll_request3({low_delete_migration, _EpochID, File}, do_pb_ll_request3({low_trunc_hack, _EpochID, File}, #state{witness=false}=S) -> {do_server_trunc_hack(File, S), S}; + do_pb_ll_request3(_, #state{witness=true}=S) -> {{error, bad_arg}, S}. % TODO: new status code?? @@ -361,7 +365,8 @@ do_server_append_chunk2(CoC_Namespace, CoC_Locator, throw:{bad_csum, _CS} -> {error, bad_checksum}; error:badarg -> - error_logger:error_msg("Message send to ~p gave badarg, make certain server is running with correct registered name\n", [?MODULE]), + lager:error("badarg at ~w:do_server_append_chunk2:~w ~w", + [?MODULE, ?LINE, erlang:get_stacktrace()]), {error, bad_arg} end. diff --git a/src/machi_listener_sup.erl b/src/machi_listener_sup.erl index f227611..f2362ad 100644 --- a/src/machi_listener_sup.erl +++ b/src/machi_listener_sup.erl @@ -18,16 +18,15 @@ %% %% ------------------------------------------------------------------- -%% @doc This is the supervisor to hold ranch listener for sigle FLU, -%% holds at most one child worker. +%% @doc A supervisor to hold ranch listener for sigle FLU. +%% It holds at most one child worker. -%% TODO: This supervisor is maybe useless. First introduced by workaround -%% to start listener dynamically in flu1 initialization time. -%% Because psup is blocked in flu1 initialization time, adding a child -%% to psup leads to deadlock. -%% By the result of refactoring process, if initialization can be done -%% only by static arguments, then this supervisor should be removed -%% and add listener as a direct child of psup. +%% TODO: This supervisor is maybe useless. First introduced for +%% workaround to start listener dynamically in flu1 initialization +%% time. Because psup is being blocked in flu1 initialization time, +%% adding a child to psup leads to deadlock. If initialization can be +%% done only by static arguments, then this supervisor should be +%% removed and added as a direct child of `machi_flu_psup'. -module(machi_listener_sup). -behaviour(supervisor). @@ -42,32 +41,44 @@ %% supervisor callback -export([init/1]). +-include("machi_projection.hrl"). + -define(BACKLOG, 8192). +-spec start_link(pv1_server()) -> {ok, pid()}. start_link(FluName) -> supervisor:start_link({local, make_listener_sup_name(FluName)}, ?MODULE, []). +-spec start_listener(pv1_server(), inet:port_number(), boolean(), + string(), ets:tab(), atom() | pid()) -> {ok, pid()}. start_listener(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore) -> supervisor:start_child(make_listener_sup_name(FluName), child_spec(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore)). +-spec stop_listener(pv1_server()) -> ok. stop_listener(FluName) -> SupName = make_listener_sup_name(FluName), ListenerName = make_listener_name(FluName), ok = supervisor:terminate_child(SupName, ListenerName), ok = supervisor:delete_child(SupName, ListenerName). +-spec make_listener_name(pv1_server()) -> atom(). make_listener_sup_name(FluName) when is_atom(FluName) -> list_to_atom(atom_to_list(FluName) ++ "_listener_sup"). +-spec make_listener_sup_name(pv1_server()) -> atom(). make_listener_name(FluName) -> list_to_atom(atom_to_list(FluName) ++ "_listener"). +%% Supervisor callback + init([]) -> SupFlags = {one_for_one, 1000, 10}, {ok, {SupFlags, []}}. +-spec child_spec(pv1_server(), inet:port_number(), boolean(), + string(), ets:tab(), atom() | pid()) -> supervisor:child_spec(). child_spec(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore) -> ListenerName = make_listener_name(FluName), NbAcceptors = 100, diff --git a/test/machi_admin_util_test.erl b/test/machi_admin_util_test.erl index 8a43c39..1ebbbf3 100644 --- a/test/machi_admin_util_test.erl +++ b/test/machi_admin_util_test.erl @@ -44,45 +44,46 @@ verify_file_checksums_test2() -> TcpPort = 32958, DataDir = "./data", W_props = [{initial_wedged, false}], - {ok, SupPid} = machi_sup:start_link(), - machi_flu1_test:start_flu_package(verify1_flu, TcpPort, DataDir, - W_props), - Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}), try - Prefix = <<"verify_prefix">>, - NumChunks = 10, - [{ok, _} = ?FLU_C:append_chunk(Sock1, ?DUMMY_PV1_EPOCH, - Prefix, <>) || - X <- lists:seq(1, NumChunks)], - {ok, [{_FileSize,File}]} = ?FLU_C:list_files(Sock1, ?DUMMY_PV1_EPOCH), - ?assertEqual({ok, []}, - machi_admin_util:verify_file_checksums_remote( - Host, TcpPort, ?DUMMY_PV1_EPOCH, File)), + machi_test_util:start_flu_package(verify1_flu, TcpPort, DataDir, + W_props), + Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}), + try + Prefix = <<"verify_prefix">>, + NumChunks = 10, + [{ok, _} = ?FLU_C:append_chunk(Sock1, ?DUMMY_PV1_EPOCH, + Prefix, <>) || + X <- lists:seq(1, NumChunks)], + {ok, [{_FileSize,File}]} = ?FLU_C:list_files(Sock1, ?DUMMY_PV1_EPOCH), + ?assertEqual({ok, []}, + machi_admin_util:verify_file_checksums_remote( + Host, TcpPort, ?DUMMY_PV1_EPOCH, File)), - %% Clobber the first 3 chunks, which are sizes 1/2/3. - {_, Path} = machi_util:make_data_filename(DataDir,binary_to_list(File)), - {ok, FH} = file:open(Path, [read,write]), - {ok, _} = file:position(FH, ?MINIMUM_OFFSET), - ok = file:write(FH, "y"), - ok = file:write(FH, "yo"), - ok = file:write(FH, "yo!"), - ok = file:close(FH), + %% Clobber the first 3 chunks, which are sizes 1/2/3. + {_, Path} = machi_util:make_data_filename(DataDir,binary_to_list(File)), + {ok, FH} = file:open(Path, [read,write]), + {ok, _} = file:position(FH, ?MINIMUM_OFFSET), + ok = file:write(FH, "y"), + ok = file:write(FH, "yo"), + ok = file:write(FH, "yo!"), + ok = file:close(FH), - %% Check the local flavor of the API: should be 3 bad checksums - {ok, Res1} = machi_admin_util:verify_file_checksums_local( - Host, TcpPort, ?DUMMY_PV1_EPOCH, Path), - 3 = length(Res1), + %% Check the local flavor of the API: should be 3 bad checksums + {ok, Res1} = machi_admin_util:verify_file_checksums_local( + Host, TcpPort, ?DUMMY_PV1_EPOCH, Path), + 3 = length(Res1), - %% Check the remote flavor of the API: should be 3 bad checksums - {ok, Res2} = machi_admin_util:verify_file_checksums_remote( - Host, TcpPort, ?DUMMY_PV1_EPOCH, File), - 3 = length(Res2), + %% Check the remote flavor of the API: should be 3 bad checksums + {ok, Res2} = machi_admin_util:verify_file_checksums_remote( + Host, TcpPort, ?DUMMY_PV1_EPOCH, File), + 3 = length(Res2), - ok + ok + after + catch ?FLU_C:quit(Sock1) + end after - catch ?FLU_C:quit(Sock1), - catch machi_flu1_test:stop_flu_package(verify1_flu), - exit(SupPid, normal) + catch machi_test_util:stop_flu_package() end. -endif. % !PULSE diff --git a/test/machi_ap_repair_eqc.erl b/test/machi_ap_repair_eqc.erl index a42d7cc..23e0e93 100644 --- a/test/machi_ap_repair_eqc.erl +++ b/test/machi_ap_repair_eqc.erl @@ -342,7 +342,7 @@ setup_target(Num, Seed, Verbose) -> setup_chain(Seed, AllListE, FLUNames, MgrNames, Dict) -> ok = shutdown_hard(), [begin - machi_flu1_test:clean_up_data_dir(Dir), + machi_test_util:clean_up_dir(Dir), filelib:ensure_dir(Dir ++ "/not-used") end || {_P, Dir} <- AllListE], [catch ets:delete(T) || T <- tabs()], diff --git a/test/machi_chain_manager1_test.erl b/test/machi_chain_manager1_test.erl index 3f95955..99ecb6a 100644 --- a/test/machi_chain_manager1_test.erl +++ b/test/machi_chain_manager1_test.erl @@ -275,14 +275,14 @@ make_prop_ets() -> smoke0_test() -> TcpPort = 6623, - {_, [Pa], [M0]} = machi_psup_test_util:start_flu_packages( - 1, "./data.", TcpPort, []), + {[Pa], [M0], _Dirs} = machi_test_util:start_flu_packages( + 1, TcpPort, "./data.", []), {ok, FLUaP} = ?FLU_PC:start_link(Pa), try pong = ?MGR:ping(M0) after ok = ?FLU_PC:quit(FLUaP), - machi_psup_test_util:stop_flu_packages() + machi_test_util:stop_flu_packages() end. smoke1_test_() -> @@ -291,12 +291,13 @@ smoke1_test_() -> smoke1_test2() -> TcpPort = 62777, MgrOpts = [{active_mode,false}], - {_, Ps, MgrNames} = machi_psup_test_util:start_flu_packages( - 3, "./data.", TcpPort, MgrOpts), - MembersDict = machi_projection:make_members_dict(Ps), - [machi_chain_manager1:set_chain_members(M, MembersDict) || M <- MgrNames], - Ma = hd(MgrNames), try + {Ps, MgrNames, _Dirs} = machi_test_util:start_flu_packages( + 3, TcpPort, "./data.", MgrOpts), + MembersDict = machi_projection:make_members_dict(Ps), + [machi_chain_manager1:set_chain_members(M, MembersDict) || M <- MgrNames], + Ma = hd(MgrNames), + {ok, P1} = ?MGR:test_calc_projection(Ma, false), % DERP! Check for race with manager's proxy vs. proj listener ok = lists:foldl( @@ -315,22 +316,23 @@ smoke1_test2() -> ok after - machi_psup_test_util:stop_flu_packages() + machi_test_util:stop_flu_packages() end. nonunanimous_setup_and_fix_test() -> TcpPort = 62877, MgrOpts = [{active_mode,false}], - {_, Ps, [Ma,Mb]} = machi_psup_test_util:start_flu_packages( - 2, "./data.", TcpPort, MgrOpts), + {Ps, [Ma,Mb], _Dirs} = machi_test_util:start_flu_packages( + 2, TcpPort, "./data.", MgrOpts), MembersDict = machi_projection:make_members_dict(Ps), [machi_chain_manager1:set_chain_members(M, MembersDict) || M <- [Ma, Mb]], [Proxy_a, Proxy_b] = Proxies = [element(2, ?FLU_PC:start_link(P)) || P <- Ps], - ok = machi_chain_manager1:set_chain_members(Ma, MembersDict, []), - ok = machi_chain_manager1:set_chain_members(Mb, MembersDict, []), + try + ok = machi_chain_manager1:set_chain_members(Ma, MembersDict, []), + ok = machi_chain_manager1:set_chain_members(Mb, MembersDict, []), {ok, P1} = ?MGR:test_calc_projection(Ma, false), P1a = machi_projection:update_checksum( @@ -374,7 +376,7 @@ nonunanimous_setup_and_fix_test() -> ok after [ok = ?FLU_PC:quit(X) || X <- Proxies], - machi_psup_test_util:stop_flu_packages() + machi_test_util:stop_flu_packages() end. unanimous_report_test() -> diff --git a/test/machi_flu1_test.erl b/test/machi_flu1_test.erl index 5491e2b..942d2da 100644 --- a/test/machi_flu1_test.erl +++ b/test/machi_flu1_test.erl @@ -30,46 +30,6 @@ -define(FLU, machi_flu1). -define(FLU_C, machi_flu1_client). -clean_up_data_dir(DataDir) -> - [begin - Fs = filelib:wildcard(DataDir ++ Glob), - [file:delete(F) || F <- Fs], - [file:del_dir(F) || F <- Fs] - end || Glob <- ["*/*/*/*", "*/*/*", "*/*", "*"] ], - _ = file:del_dir(DataDir), - ok. - -start_flu_package(RegName, TcpPort, DataDir) -> - start_flu_package(RegName, TcpPort, DataDir, []). - -start_flu_package(RegName, TcpPort, DataDir, Props) -> - case proplists:get_value(save_data_dir, Props) of - true -> - ok; - _ -> - clean_up_data_dir(DataDir) - end, - - maybe_start_sup(), - machi_flu_psup:start_flu_package(RegName, TcpPort, DataDir, Props). - -stop_flu_package(FluName) -> - machi_flu_psup:stop_flu_package(FluName), - Pid = whereis(machi_sup), - exit(Pid, normal), - machi_util:wait_for_death(Pid, 100). - -maybe_start_sup() -> - case whereis(machi_sup) of - undefined -> - machi_sup:start_link(), - %% evil but we have to let stuff start up - timer:sleep(10), - maybe_start_sup(); - Pid -> Pid - end. - - -ifndef(PULSE). flu_smoke_test() -> @@ -78,10 +38,9 @@ flu_smoke_test() -> DataDir = "./data", Prefix = <<"prefix!">>, BadPrefix = BadFile = "no/good", - W_props = [{initial_wedged, false}], - start_flu_package(smoke_flu, TcpPort, DataDir, W_props), try + _ = machi_test_util:start_flu_package(smoke_flu, TcpPort, DataDir, W_props), Msg = "Hello, world!", Msg = ?FLU_C:echo(Host, TcpPort, Msg), {error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort, @@ -178,22 +137,21 @@ flu_smoke_test() -> ok = ?FLU_C:quit(?FLU_C:connect(#p_srvr{address=Host, port=TcpPort})) after - stop_flu_package(smoke_flu) + machi_test_util:stop_flu_package() end. flu_projection_smoke_test() -> Host = "localhost", TcpPort = 32959, DataDir = "./data.projst", - - start_flu_package(projection_test_flu, TcpPort, DataDir), try + machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir), [ok = flu_projection_common(Host, TcpPort, T) || T <- [public, private] ] %% , {ok, {false, EpochID1}} = ?FLU_C:wedge_status(Host, TcpPort), %% io:format(user, "EpochID1 ~p\n", [EpochID1]) after - stop_flu_package(projection_test_flu) + machi_test_util:stop_flu_package() end. flu_projection_common(Host, TcpPort, T) -> @@ -223,10 +181,9 @@ bad_checksum_test() -> Host = "localhost", TcpPort = 32960, DataDir = "./data.bct", - Opts = [{initial_wedged, false}], - start_flu_package(projection_test_flu, TcpPort, DataDir, Opts), try + machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts), Prefix = <<"some prefix">>, Chunk1 = <<"yo yo yo">>, Chunk1_badcs = {<>, Chunk1}, @@ -235,17 +192,16 @@ bad_checksum_test() -> Prefix, Chunk1_badcs), ok after - stop_flu_package(projection_test_flu) + machi_test_util:stop_flu_package() end. witness_test() -> Host = "localhost", TcpPort = 32961, DataDir = "./data.witness", - Opts = [{initial_wedged, false}, {witness_mode, true}], - start_flu_package(projection_test_flu, TcpPort, DataDir, Opts), try + machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts), Prefix = <<"some prefix">>, Chunk1 = <<"yo yo yo">>, @@ -276,7 +232,7 @@ witness_test() -> ok after - stop_flu_package(projection_test_flu) + machi_test_util:stop_flu_package() end. %% The purpose of timing_pb_encoding_test_ and timing_bif_encoding_test_ is diff --git a/test/machi_pb_high_client_test.erl b/test/machi_pb_high_client_test.erl index 3273bc2..16b125c 100644 --- a/test/machi_pb_high_client_test.erl +++ b/test/machi_pb_high_client_test.erl @@ -34,21 +34,15 @@ smoke_test_() -> {timeout, 5*60, fun() -> smoke_test2() end}. smoke_test2() -> - Port = 5720, - Ps = [#p_srvr{name=a, address="localhost", port=Port, props="./data.a"} - ], - D = orddict:from_list([{P#p_srvr.name, P} || P <- Ps]), + PortBase = 5720, ok = application:set_env(machi, max_file_size, 1024*1024), - - [os:cmd("rm -rf " ++ P#p_srvr.props) || P <- Ps], - {ok, SupPid} = machi_sup:start_link(), try - [begin - #p_srvr{name=Name, port=Port, props=Dir} = P, - {ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, []) - end || P <- Ps], - ok = machi_chain_manager1:set_chain_members(a_chmgr, D), - [machi_chain_manager1:trigger_react_to_env(a_chmgr) || _ <-lists:seq(1,5)], + {Ps, MgrNames, Dirs} = machi_test_util:start_flu_packages( + 1, PortBase, "./data.", []), + D = orddict:from_list([{P#p_srvr.name, P} || P <- Ps]), + M0 = hd(MgrNames), + ok = machi_chain_manager1:set_chain_members(M0, D), + [machi_chain_manager1:trigger_react_to_env(M0) || _ <-lists:seq(1,5)], {ok, Clnt} = ?C:start_link(Ps), try @@ -94,7 +88,8 @@ smoke_test2() -> File1Bin = binary_to_list(File1), [begin - #p_srvr{name=Name, port=Port, props=Dir} = P, + #p_srvr{name=Name, props=Props} = P, + Dir = proplists:get_value(data_dir, Props), ?assertEqual({ok, [File1Bin]}, file:list_dir(filename:join([Dir, "data"]))), FileListFileName = filename:join([Dir, "known_files_" ++ atom_to_list(Name)]), @@ -122,7 +117,8 @@ smoke_test2() -> %% Make sure everything was trimmed File = binary_to_list(Filex), [begin - #p_srvr{name=Name, port=_Port, props=Dir} = P, + #p_srvr{name=Name, props=Props} = P, + Dir = proplists:get_value(data_dir, Props), ?assertEqual({ok, []}, file:list_dir(filename:join([Dir, "data"]))), FileListFileName = filename:join([Dir, "known_files_" ++ atom_to_list(Name)]), @@ -139,10 +135,7 @@ smoke_test2() -> (catch ?C:quit(Clnt)) end after - exit(SupPid, normal), - [os:cmd("rm -rf " ++ P#p_srvr.props) || P <- Ps], - machi_util:wait_for_death(SupPid, 100), - ok + machi_test_util:stop_flu_packages() end. -endif. % !PULSE diff --git a/test/machi_projection_store_test.erl b/test/machi_projection_store_test.erl index 665553b..eab42a9 100644 --- a/test/machi_projection_store_test.erl +++ b/test/machi_projection_store_test.erl @@ -33,7 +33,7 @@ smoke_test() -> Dir = "./data.a", Os = [{ignore_stability_time, true}, {active_mode, false}], os:cmd("rm -rf " ++ Dir), - machi_flu1_test:start_flu_package(a, PortBase, "./data.a", Os), + machi_test_util:start_flu_package(a, PortBase, "./data.a", Os), try P1 = machi_projection:new(1, a, [], [], [], [], []), @@ -58,7 +58,7 @@ smoke_test() -> ok after - machi_flu1_test:stop_flu_package(a) + machi_test_util:stop_flu_package() end. -endif. % !PULSE diff --git a/test/machi_proxy_flu1_client_test.erl b/test/machi_proxy_flu1_client_test.erl index 422dce1..dc0bdba 100644 --- a/test/machi_proxy_flu1_client_test.erl +++ b/test/machi_proxy_flu1_client_test.erl @@ -32,17 +32,14 @@ api_smoke_test() -> RegName = api_smoke_flu, - Host = "localhost", TcpPort = 57124, DataDir = "./data.api_smoke_flu", W_props = [{active_mode, false},{initial_wedged, false}], Prefix = <<"prefix">>, - {ok, SupPid} = machi_sup:start_link(), - machi_flu1_test:start_flu_package(RegName, TcpPort, DataDir, W_props), - try - I = #p_srvr{name=RegName, address=Host, port=TcpPort}, + {[I], _, _} = machi_test_util:start_flu_package( + RegName, TcpPort, DataDir, W_props), {ok, Prox1} = ?MUT:start_link(I), try FakeEpoch = ?DUMMY_PV1_EPOCH, @@ -50,13 +47,13 @@ api_smoke_test() -> FakeEpoch, Prefix, <<"data">>, infinity) || _ <- lists:seq(1,5)], %% Stop the FLU, what happens? - machi_flu1_test:stop_flu_package(RegName), + machi_test_util:stop_flu_package(), [{error,partition} = ?MUT:append_chunk(Prox1, FakeEpoch, Prefix, <<"data-stopped1">>, infinity) || _ <- lists:seq(1,3)], %% Start the FLU again, we should be able to do stuff immediately - machi_flu1_test:start_flu_package(RegName, TcpPort, DataDir, - [save_data_dir|W_props]), + machi_test_util:start_flu_package(RegName, TcpPort, DataDir, + [save_data_dir|W_props]), MyChunk = <<"my chunk data">>, {ok, {MyOff,MySize,MyFile}} = ?MUT:append_chunk(Prox1, FakeEpoch, Prefix, MyChunk, @@ -103,8 +100,7 @@ api_smoke_test() -> _ = (catch ?MUT:quit(Prox1)) end after - (catch machi_flu1_test:stop_flu_package(RegName)), - exit(SupPid, normal) + (catch machi_test_util:stop_flu_package()) end. flu_restart_test_() -> @@ -112,15 +108,13 @@ flu_restart_test_() -> flu_restart_test2() -> RegName = a, - Host = "localhost", TcpPort = 57125, DataDir = "./data.api_smoke_flu2", W_props = [{initial_wedged, false}, {active_mode, false}], - {ok, SupPid} = machi_sup:start_link(), - machi_flu1_test:start_flu_package(RegName, TcpPort, DataDir, W_props), try - I = #p_srvr{name=RegName, address=Host, port=TcpPort}, + {[I], _, _} = machi_test_util:start_flu_package( + RegName, TcpPort, DataDir, W_props), {ok, Prox1} = ?MUT:start_link(I), try FakeEpoch = ?DUMMY_PV1_EPOCH, @@ -140,7 +134,7 @@ flu_restart_test2() -> {ok, EpochID} = ?MUT:get_epoch_id(Prox1), {ok, EpochID} = ?MUT:get_latest_epochid(Prox1, public), {ok, EpochID} = ?MUT:get_latest_epochid(Prox1, private), - ok = machi_flu1_test:stop_flu_package(RegName), timer:sleep(50), + ok = machi_test_util:stop_flu_package(), timer:sleep(50), %% Now that the last proxy op was successful and only %% after did we stop the FLU, let's check that both the @@ -296,25 +290,24 @@ flu_restart_test2() -> ], [begin - machi_flu1_test:start_flu_package( + machi_test_util:start_flu_package( RegName, TcpPort, DataDir, [save_data_dir|W_props]), _ = Fun(line), ok = Fun(run), ok = Fun(run), - ok = machi_flu1_test:stop_flu_package(RegName), + ok = machi_test_util:stop_flu_package(), {error, partition} = Fun(stop), {error, partition} = Fun(stop), ok end || Fun <- ExpectedOps ], ok after - _ = (catch ?MUT:quit(Prox1)), - exit(SupPid, normal) + _ = (catch ?MUT:quit(Prox1)) end after - (catch machi_flu1_test:stop_flu_package(RegName)) + (catch machi_test_util:stop_flu_package()) end. - + -endif. % !PULSE -endif. % TEST diff --git a/test/machi_psup_test_util.erl b/test/machi_psup_test_util.erl deleted file mode 100644 index 6446bf7..0000000 --- a/test/machi_psup_test_util.erl +++ /dev/null @@ -1,71 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(machi_psup_test_util). --compile(export_all). - --ifdef(TEST). --ifndef(PULSE). - --include_lib("eunit/include/eunit.hrl"). - --include("machi.hrl"). --include("machi_projection.hrl"). - --define(FLU, machi_flu1). --define(FLU_C, machi_flu1_client). - --spec start_flu_packages(pos_integer(), string(), inet:port(), list()) -> - {MachiSup::pid(), Ps::[#p_srvr{}], MgrNames::[atom()]}. -start_flu_packages(FluCount, DirPrefix, BaseTcpPort, MgrOpts) -> - FluInfo = flu_info(FluCount, DirPrefix, BaseTcpPort), - _ = stop_machi_sup(), - _ = [machi_flu1_test:clean_up_data_dir(Dir) || {_, Dir, _} <- FluInfo], - {ok, SupPid} = machi_sup:start_link(), - [{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, MgrOpts) || - {#p_srvr{name=Name, port=Port}, Dir, _} <- FluInfo], - {Ps, _Dirs, MgrNames} = lists:unzip3(FluInfo), - {SupPid, Ps, MgrNames}. - -stop_flu_packages() -> - stop_machi_sup(). - -flu_info(FluCount, DirPrefix, BaseTcpPort) -> - [begin - FLUNameStr = [$a + I - 1], - FLUName = list_to_atom(FLUNameStr), - MgrName = machi_flu_psup:make_mgr_supname(FLUName), - {#p_srvr{name=FLUName, address="localhost", port=BaseTcpPort + I, - props=[{chmgr, MgrName}]}, - DirPrefix ++ "/data.eqc." ++ FLUNameStr, - MgrName} - end || I <- lists:seq(1, FluCount)]. - -stop_machi_sup() -> - case whereis(machi_sup) of - undefined -> ok; - Pid -> - catch exit(whereis(machi_sup), normal), - machi_util:wait_for_death(Pid, 30) - end. - --endif. % !PULSE --endif. % TEST - diff --git a/test/machi_test_util.erl b/test/machi_test_util.erl new file mode 100644 index 0000000..21d5ca9 --- /dev/null +++ b/test/machi_test_util.erl @@ -0,0 +1,107 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(machi_test_util). +-compile(export_all). + +-ifdef(TEST). +-ifndef(PULSE). + +-include_lib("eunit/include/eunit.hrl"). + +-include("machi.hrl"). +-include("machi_projection.hrl"). + +-define(FLU, machi_flu1). +-define(FLU_C, machi_flu1_client). + +-spec start_flu_package(atom(), inet:port_number(), string()) -> + {Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}. +start_flu_package(FluName, TcpPort, DataDir) -> + start_flu_package(FluName, TcpPort, DataDir, []). + +-spec start_flu_package(atom(), inet:port_number(), string(), list()) -> + {Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}. +start_flu_package(FluName, TcpPort, DataDir, Props) -> + MgrName = machi_flu_psup:make_mgr_supname(FluName), + FluInfo = [{#p_srvr{name=FluName, address="localhost", port=TcpPort, + props=[{chmgr, MgrName}, {data_dir, DataDir} | Props]}, + DataDir, MgrName}], + start_flu_packages(FluInfo). + +-spec start_flu_packages(pos_integer(), inet:port_number(), string(), list()) -> + {Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}. +start_flu_packages(FluCount, BaseTcpPort, DirPrefix, Props) -> + FluInfo = flu_info(FluCount, BaseTcpPort, DirPrefix, Props), + start_flu_packages(FluInfo). + +start_flu_packages(FluInfo) -> + _ = stop_machi_sup(), + clean_up_data_dirs(FluInfo), + {ok, _SupPid} = machi_sup:start_link(), + [{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, Props) || + {#p_srvr{name=Name, port=Port, props=Props}, Dir, _} <- FluInfo], + {Ps, Dirs, MgrNames} = lists:unzip3(FluInfo), + {Ps, MgrNames, Dirs}. + +stop_flu_package() -> + stop_flu_packages(). + +stop_flu_packages() -> + stop_machi_sup(). + +flu_info(FluCount, BaseTcpPort, DirPrefix, Props) -> + [begin + FLUNameStr = [$a + I - 1], + FLUName = list_to_atom(FLUNameStr), + MgrName = machi_flu_psup:make_mgr_supname(FLUName), + DataDir = DirPrefix ++ "/data.eqc." ++ FLUNameStr, + {#p_srvr{name=FLUName, address="localhost", port=BaseTcpPort + I, + props=[{chmgr, MgrName}, {data_dir, DataDir} | Props]}, + DataDir, MgrName} + end || I <- lists:seq(1, FluCount)]. + +stop_machi_sup() -> + case whereis(machi_sup) of + undefined -> ok; + Pid -> + catch exit(whereis(machi_sup), normal), + machi_util:wait_for_death(Pid, 30) + end. + +clean_up_data_dirs(FluInfo) -> + _ = [case proplists:get_value(save_data_dir, Propx) of + true -> ok; + _ -> clean_up_dir(Dir) + end || {#p_srvr{props=Propx}, Dir, _} <- FluInfo], + ok. + +clean_up_dir(Dir) -> + [begin + Fs = filelib:wildcard(Dir ++ Glob), + [file:delete(F) || F <- Fs], + [file:del_dir(F) || F <- Fs] + end || Glob <- ["*/*/*/*", "*/*/*", "*/*", "*"] ], + _ = file:del_dir(Dir), + ok. + +-endif. % !PULSE +-endif. % TEST + -- 2.45.2 From ade4430d303d0ad2df3fe1dafc05802eeaa40d7d Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Tue, 8 Dec 2015 15:38:27 +0900 Subject: [PATCH 39/77] More cleaner clean up --- src/machi_flu1.erl | 7 ++++++- test/machi_proxy_flu1_client_test.erl | 6 +++--- test/machi_test_util.erl | 16 ++++++++++------ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/machi_flu1.erl b/src/machi_flu1.erl index fade024..e620308 100644 --- a/src/machi_flu1.erl +++ b/src/machi_flu1.erl @@ -82,7 +82,12 @@ start_link([{FluName, TcpPort, DataDir}|Rest]) proc_lib:start_link(?MODULE, main2, [FluName, TcpPort, DataDir, Rest], ?INIT_TIMEOUT). -stop(Pid) -> +stop(RegName) when is_atom(RegName) -> + case whereis(RegName) of + undefined -> ok; + Pid -> stop(Pid) + end; +stop(Pid) when is_pid(Pid) -> case erlang:is_process_alive(Pid) of true -> Pid ! killme, diff --git a/test/machi_proxy_flu1_client_test.erl b/test/machi_proxy_flu1_client_test.erl index dc0bdba..439b1a7 100644 --- a/test/machi_proxy_flu1_client_test.erl +++ b/test/machi_proxy_flu1_client_test.erl @@ -53,7 +53,7 @@ api_smoke_test() -> infinity) || _ <- lists:seq(1,3)], %% Start the FLU again, we should be able to do stuff immediately machi_test_util:start_flu_package(RegName, TcpPort, DataDir, - [save_data_dir|W_props]), + [no_cleanup|W_props]), MyChunk = <<"my chunk data">>, {ok, {MyOff,MySize,MyFile}} = ?MUT:append_chunk(Prox1, FakeEpoch, Prefix, MyChunk, @@ -148,7 +148,7 @@ flu_restart_test2() -> ExpectedOps = [ - fun(run) -> {ok, EpochID} = ?MUT:get_epoch_id(Prox1), + fun(run) -> ?assertEqual({ok, EpochID}, ?MUT:get_epoch_id(Prox1)), ok; (line) -> io:format("line ~p, ", [?LINE]); (stop) -> ?MUT:get_epoch_id(Prox1) end, @@ -292,7 +292,7 @@ flu_restart_test2() -> [begin machi_test_util:start_flu_package( RegName, TcpPort, DataDir, - [save_data_dir|W_props]), + [no_cleanup|W_props]), _ = Fun(line), ok = Fun(run), ok = Fun(run), diff --git a/test/machi_test_util.erl b/test/machi_test_util.erl index 21d5ca9..ff908b7 100644 --- a/test/machi_test_util.erl +++ b/test/machi_test_util.erl @@ -54,7 +54,7 @@ start_flu_packages(FluCount, BaseTcpPort, DirPrefix, Props) -> start_flu_packages(FluInfo) -> _ = stop_machi_sup(), - clean_up_data_dirs(FluInfo), + clean_up(FluInfo), {ok, _SupPid} = machi_sup:start_link(), [{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, Props) || {#p_srvr{name=Name, port=Port, props=Props}, Dir, _} <- FluInfo], @@ -86,11 +86,15 @@ stop_machi_sup() -> machi_util:wait_for_death(Pid, 30) end. -clean_up_data_dirs(FluInfo) -> - _ = [case proplists:get_value(save_data_dir, Propx) of - true -> ok; - _ -> clean_up_dir(Dir) - end || {#p_srvr{props=Propx}, Dir, _} <- FluInfo], +clean_up(FluInfo) -> + _ = [begin + case proplists:get_value(no_cleanup, Props) of + true -> ok; + _ -> + _ = machi_flu1:stop(FLUName), + clean_up_dir(Dir) + end + end || {#p_srvr{name=FLUName, props=Props}, Dir, _} <- FluInfo], ok. clean_up_dir(Dir) -> -- 2.45.2 From f23e500993e8f96158092057b9bc491af31bdeb5 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 9 Dec 2015 11:32:05 +0900 Subject: [PATCH 40/77] WIP comments --- src/machi_lifecycle_mgr.erl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 6cc8a47..746f65a 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -332,3 +332,25 @@ bootstrap_chain(#chain_def_v1{name=ChainName, mode=CMode, full=Full, "failed: ~w (defn ~w)\n", [Else, CD]), ok end. + +%% 1. Don't worry about partial writes to config dir: that's a Policy +%% implementation worry: create a temp file, fsync(2), then rename(2) +%% and possibly fsync(2) on the dir. +%% +%% 2. Add periodic re-scan check FLUs & chains (in that order). +%% +%% 3. Add force re-rescan of FLUs *or* chains. (separate is better?) +%% +%% 4. For chain modification: add "change" directory? Scan looks in +%% "change"? +%% +%% 5. Easy comparison for current vs. new chain config. +%% - identify new FLUs: no action required, HC will deal with it? +%% - identify removed FLUs: set_chain_members() to change HC +%% : stop FLU package (after short pause) +%% : move config file -> just-in-case +%% : move data files -> just-in-case +%% - identify j-i-c dir in case resuming from interruption??? to avoid problem of putting config & data files in different places? +%% - move data files itself may be interrupted? +%% - move chain def'n from change -> chain_config_dir + -- 2.45.2 From b243a9b863018ae21ac2c801daa73ea0f0a4e4ce Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 9 Dec 2015 12:29:59 +0900 Subject: [PATCH 41/77] Avoid TCP port # reuse in machi_flu_psup_test tests --- test/machi_flu_psup_test.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/machi_flu_psup_test.erl b/test/machi_flu_psup_test.erl index 1c7b015..024418d 100644 --- a/test/machi_flu_psup_test.erl +++ b/test/machi_flu_psup_test.erl @@ -38,9 +38,9 @@ smoke_test_() -> {timeout, 5*60, fun() -> smoke_test2() end}. smoke_test2() -> - Ps = [{a,#p_srvr{name=a, address="localhost", port=5555, props="./data.a"}}, - {b,#p_srvr{name=b, address="localhost", port=5556, props="./data.b"}}, - {c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}} + Ps = [{a,#p_srvr{name=a, address="localhost", port=5550, props="./data.a"}}, + {b,#p_srvr{name=b, address="localhost", port=5551, props="./data.b"}}, + {c,#p_srvr{name=c, address="localhost", port=5552, props="./data.c"}} ], [os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps], {ok, SupPid} = machi_flu_sup:start_link(), @@ -66,9 +66,9 @@ partial_stop_restart_test_() -> {timeout, 5*60, fun() -> partial_stop_restart2() end}. partial_stop_restart2() -> - Ps = [{a,#p_srvr{name=a, address="localhost", port=5555, props="./data.a"}}, - {b,#p_srvr{name=b, address="localhost", port=5556, props="./data.b"}}, - {c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}} + Ps = [{a,#p_srvr{name=a, address="localhost", port=5560, props="./data.a"}}, + {b,#p_srvr{name=b, address="localhost", port=5561, props="./data.b"}}, + {c,#p_srvr{name=c, address="localhost", port=5562, props="./data.c"}} ], ChMgrs = [machi_flu_psup:make_mgr_supname(P#p_srvr.name) || {_,P} <-Ps], PStores = [machi_flu_psup:make_proj_supname(P#p_srvr.name) || {_,P} <-Ps], -- 2.45.2 From 7301c8308e7c878463f559418b410a25913050bf Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 9 Dec 2015 14:07:27 +0900 Subject: [PATCH 42/77] Clarify the initial docs, thanks @mrallen1! --- src/machi_lifecycle_mgr.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 746f65a..d11ff49 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -25,7 +25,9 @@ %% for implementing the lifecycle changes as expressed by "policy". %% In our case, "policy" is created by an external administrative %% entity that creates and deletes configuration files that define -%% FLUs and chains relative to this local machine. +%% FLUs and chains relative to this local machine. (This "policy" +%% may be a human administrator or (as the Machi project matures) +%% partial or full automatic implementation of policy.) %% %% The "master configuration" for deciding which FLUs should be %% running on this machine was inspired by BSD UNIX's `init(8)' and the @@ -108,7 +110,7 @@ %% %%
  • name :: atom(): The name of the FLU. This name %% should be unique over the lifetime of the administrative -%% domain and thus managed by outside policy. This name must be +%% domain and thus managed by external policy. This name must be %% the same as the name of the `rc.d'-style config file that %% defines the FLU. %%
    • @@ -154,7 +156,7 @@ %% %%
  • name :: atom(): The name of the chain. This name %% should be unique over the lifetime of the administrative -%% domain and thus managed by outside policy. This name must be +%% domain and thus managed by external policy. This name must be %% the same as the name of the `rc.d'-style config file that %% defines the chain. %%
    • -- 2.45.2 From 65eec61f82ebe8853e2cbac638db93139b99f956 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 9 Dec 2015 14:48:46 +0900 Subject: [PATCH 43/77] Basic stuff to add new flus via 'pending' dir --- rel/files/app.config | 4 ++ rel/reltool.config | 2 + src/machi_flu_psup.erl | 5 ++- src/machi_flu_sup.erl | 24 +++++----- src/machi_lifecycle_mgr.erl | 90 +++++++++++++++++++++++++++++++++++-- 5 files changed, 110 insertions(+), 15 deletions(-) diff --git a/rel/files/app.config b/rel/files/app.config index ff15eee..eb330f3 100644 --- a/rel/files/app.config +++ b/rel/files/app.config @@ -16,6 +16,10 @@ %% Default = 10 %% {metadata_manager_count, 2}, + %% Platform vars (mirror of reltool packaging) + {platform_data_dir, "{{platform_data_dir}}"}, + {platform_etc_dir, "{{platform_etc_dir}}"}, + %% Do not delete, do not put Machi config items after this line. {final_comma_stopper, do_not_delete} ] diff --git a/rel/reltool.config b/rel/reltool.config index ca68025..64cb243 100644 --- a/rel/reltool.config +++ b/rel/reltool.config @@ -95,6 +95,8 @@ {template, "files/app.config", "etc/app.config"}, {mkdir, "etc/chain-config"}, {mkdir, "etc/flu-config"}, + {mkdir, "etc/pending"}, + {mkdir, "etc/rejected"}, {mkdir, "lib/basho-patches"} %% {copy, "../apps/machi/ebin/etop_txt.beam", "lib/basho-patches"} ]}. diff --git a/src/machi_flu_psup.erl b/src/machi_flu_psup.erl index d83d559..3b8ac3b 100644 --- a/src/machi_flu_psup.erl +++ b/src/machi_flu_psup.erl @@ -180,5 +180,8 @@ get_env(Setting, Default) -> get_data_dir(Props) -> case proplists:get_value(data_dir, Props) of Path when is_list(Path) -> - Path + Path; + undefined -> + {ok, Dir} = application:get_env(machi, flu_data_dir), + Dir end. diff --git a/src/machi_flu_sup.erl b/src/machi_flu_sup.erl index 6974f36..9f0e4fd 100644 --- a/src/machi_flu_sup.erl +++ b/src/machi_flu_sup.erl @@ -47,7 +47,9 @@ -endif. %TEST %% API --export([start_link/0, load_rc_d_files_from_dir/1]). +-export([start_link/0, + get_initial_flus/0, load_rc_d_files_from_dir/1, + sanitize_p_srvr_records/1]). %% Supervisor callbacks -export([init/1]). @@ -81,22 +83,22 @@ get_initial_flus() -> DoesNotExist -> DoesNotExist; Dir -> - ok = filelib:ensure_dir(Dir ++ "/unused"), Dir end, - sanitize_p_srvr_records(load_rc_d_files_from_dir(ConfigDir)). + Ps = [P || {_File, P} <- load_rc_d_files_from_dir(ConfigDir)], + sanitize_p_srvr_records(Ps). -endif. % PULSE load_rc_d_files_from_dir(Dir) -> Files = filelib:wildcard(Dir ++ "/*"), - lists:append([case file:consult(File) of - {ok, X} -> - X; - _ -> - lager:warning("Error parsing file '~s', ignoring", - [File]), - [] - end || File <- Files]). + [case file:consult(File) of + {ok, [X]} -> + {File, X}; + _ -> + lager:warning("Error parsing file '~s', ignoring", + [File]), + {File, []} + end || File <- Files]. sanitize_p_srvr_records(Ps) -> {Sane, _} = lists:foldl(fun sanitize_p_srvr_rec/2, {[], dict:new()}, Ps), diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index d11ff49..1946f14 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -189,7 +189,8 @@ -include("machi_projection.hrl"). %% API --export([start_link/0]). +-export([start_link/0, + process_pending/0]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -205,12 +206,20 @@ start_link() -> gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). +process_pending() -> + gen_server:call(?SERVER, {process_pending}, infinity). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + init([]) -> self() ! finish_init, {ok, #state{}}. +handle_call({process_pending}, _From, State) -> + {Reply, NewState} = do_process_pending(State), + {reply, Reply, NewState}; handle_call(_Request, _From, State) -> - Reply = ok, + Reply = 'whatwatwha????????????????????', {reply, Reply, State}. handle_cast(_Msg, State) -> @@ -227,7 +236,7 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% finish_init(S) -> %% machi_flu_sup will start all FLUs that have a valid definition @@ -335,6 +344,81 @@ bootstrap_chain(#chain_def_v1{name=ChainName, mode=CMode, full=Full, ok end. +do_process_pending(S) -> + PendingDir = get_pending_dir(S), + %% PendingFiles = get_pending_files(PendingDir, S), + PendingParsed = machi_flu_sup:load_rc_d_files_from_dir(PendingDir), + P_FLUs = [X || {_File, #p_srvr{}}=X <- PendingParsed], + P_Chains = [X || {_File, #chain_def_v1{}}=X <- PendingParsed], + BadFiles = [File || {File, []} <- PendingParsed], + S2 = process_pending_flus(P_FLUs, S), + S3 = process_pending_chains(P_Chains, S2), + S4 = process_bad_files(BadFiles, S3), + {{P_FLUs, P_Chains}, S4}. + +get_pending_dir(_S) -> + {ok, EtcDir} = application:get_env(machi, platform_etc_dir), + EtcDir ++ "/pending". + +get_rejected_dir(_S) -> + {ok, EtcDir} = application:get_env(machi, platform_etc_dir), + EtcDir ++ "/rejected". + +get_flu_config_dir(_S) -> + {ok, Dir} = application:get_env(machi, flu_config_dir), + Dir. + +get_chain_config_dir(_S) -> + {ok, Dir} = application:get_env(machi, chain_config_dir), + Dir. + +process_pending_flus(P_FLUs, S) -> + lists:foldl(fun process_pending_flu/2, S, P_FLUs). + +process_pending_flu({File, P}, S) -> + #p_srvr{name=FLU} = P, + CurrentPs = machi_flu_sup:get_initial_flus(), + Valid_Ps = machi_flu_sup:sanitize_p_srvr_records(CurrentPs ++ [P]), + case lists:member(P, Valid_Ps) + andalso + (not lists:keymember(FLU, #p_srvr.name, CurrentPs)) of + false -> + lager:error("Pending FLU config file ~s has been rejected\n", + [File]), + _ = move_to_rejected(File, S), + S; + true -> + {ok, SupPid} = machi_flu_psup:start_flu_package(P), + lager:info("Started FLU ~w with supervisor pid ~p\n", + [FLU, SupPid]), + _ = move_to_flu_config(FLU, File, S), + S + end. + +process_pending_chains(P_Chains, S) -> + lists:foldl(fun process_pending_chain/2, S, P_Chains). + +process_pending_chain({_File, #chain_def_v1{}}, S) -> + io:format(user, "TODO: ~s LINE ~p\n", [?MODULE, ?LINE]), + S. + +process_bad_files(Files, S) -> + lists:foldl(fun move_to_rejected/2, S, Files). + +move_to_rejected(File, S) -> + lager:error("Pending unknown config file ~s has been rejected\n", [File]), + Dst = get_rejected_dir(S), + Suffix = lists:flatten(io_lib:format("~w,~w,~w", + tuple_to_list(os:timestamp()))), + ok = file:rename(File, Dst ++ "/" ++ filename:basename(File) ++ Suffix), + S. + +move_to_flu_config(FLU, File, S) -> + lager:info("Creating FLU config file ~w\n", [FLU]), + Dst = get_flu_config_dir(S), + ok = file:rename(File, Dst ++ "/" ++ atom_to_list(FLU)), + S. + %% 1. Don't worry about partial writes to config dir: that's a Policy %% implementation worry: create a temp file, fsync(2), then rename(2) %% and possibly fsync(2) on the dir. -- 2.45.2 From 2871f8397cab25e84cc777530fd845f410355bad Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 9 Dec 2015 17:19:02 +0900 Subject: [PATCH 44/77] WIP: modify chain still a bit broken --- include/machi_projection.hrl | 4 +- src/machi_chain_manager1.erl | 2 +- src/machi_lifecycle_mgr.erl | 156 +++++++++++++++++++++++++---------- 3 files changed, 117 insertions(+), 45 deletions(-) diff --git a/include/machi_projection.hrl b/include/machi_projection.hrl index e883fc6..4794bd6 100644 --- a/include/machi_projection.hrl +++ b/include/machi_projection.hrl @@ -82,7 +82,9 @@ mode :: pv1_consistency_mode(), full :: [p_srvr()], witnesses :: [p_srvr()], - props = [] :: list() % proplist for other related info + old_all :: [pv1_server()], + old_witnesses :: [pv1_server()], + props = [] :: list() % proplist for other related info }). -endif. % !MACHI_PROJECTION_HRL diff --git a/src/machi_chain_manager1.erl b/src/machi_chain_manager1.erl index 93eb026..9068958 100644 --- a/src/machi_chain_manager1.erl +++ b/src/machi_chain_manager1.erl @@ -1255,7 +1255,7 @@ react_to_env_A20(Retries, #ch_mgr{name=MyName, proj=P_current}=S) -> P_none0 = make_none_projection(Epoch, MyName, All_list, Witness_list, MembersDict), P_none = P_none0#projection_v1{chain_name=ChainName}, - {{x,y,z,42,77}, set_proj(S2, P_none)}; + {{now_using,[],Epoch}, set_proj(S2, P_none)}; _ -> react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S2) end. diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 1946f14..5a49dac 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -126,6 +126,9 @@ %% %%
%% +%% A FLU may be created or removed (via implicit or explicit policy). +%% An existing FLU may not be reconfigured. +%% %% == The Chain Lifecycle == %% %% If a FLU on the local machine is expected to participate in a @@ -176,6 +179,17 @@ %% %% %% +%% A chain may be created, removed, or modified. +%% +%% A modification request writes a `#chain_def_v1{}' record with the +%% same name but different `full' and/or `witnesses' list to each +%% machine that hosts a FLU in the chain (in both the old and new +%% versions of the chain). +%% +%% A deletion request writes a `#chain_def_v1{}' record with the same +%% name but empty lists for `full' and `witnesses' to each machine +%% that hosts a FLU in the chain (in the old version of the chain). +%% %% == Conflicts with TCP ports, FLU & chain names, etc == %% %% This manager is not responsible for managing conflicts in resource @@ -273,7 +287,9 @@ get_initial_chains() -> ok = filelib:ensure_dir(Dir ++ "/unused"), Dir end, - sanitize_chain_def_records(machi_flu_sup:load_rc_d_files_from_dir(ConfigDir)). + CDefs = [CDef || {_File, CDef} <- machi_flu_sup:load_rc_d_files_from_dir( + ConfigDir)], + sanitize_chain_def_records(CDefs). sanitize_chain_def_records(Ps) -> {Sane, _} = lists:foldl(fun sanitize_chain_def_rec/2, {[], dict:new()}, Ps), @@ -285,23 +301,22 @@ sanitize_chain_def_rec(Whole, {Acc, D}) -> mode=Mode, full=Full, witnesses=Witnesses} = Whole, - true = is_atom(Name), + {true, 10} = {is_atom(Name), 10}, NameK = {name, Name}, - error = dict:find(NameK, D), - true = (Mode == ap_mode orelse Mode == cp_mode), + {error, 20} = {dict:find(NameK, D), 20}, + {true, 30} = {(Mode == ap_mode orelse Mode == cp_mode), 30}, IsPSrvr = fun(X) when is_record(X, p_srvr) -> true; (_) -> false end, - true = lists:all(IsPSrvr, Full), - true = lists:all(IsPSrvr, Witnesses), + {true, 40} = {lists:all(IsPSrvr, Full), 40}, + {true, 50} = {lists:all(IsPSrvr, Witnesses), 50}, %% All is sane enough. D2 = dict:store(NameK, Name, D), {[Whole|Acc], D2} - catch _X:_Y -> - _ = lager:log(error, self(), - "~s: Bad chain_def record (~w ~w), skipping: ~P\n", - [?MODULE, _X, _Y, Whole, 15]), + catch X:Y -> + lager:error("~s: Bad chain_def record (~w ~w), skipping: ~P\n", + [?MODULE, X, Y, Whole, 15]), {Acc, D} end. @@ -323,25 +338,62 @@ perhaps_bootstrap_chains([CD|ChainDefs], LocalFLUs_at_zero, LocalFLUs) -> perhaps_bootstrap_chains(ChainDefs, LocalFLUs_at_zero, LocalFLUs); [FLU1|_]=FLUs -> %% One FLU is enough: Humming Consensus will config the remaining - bootstrap_chain(CD, FLU1), + _ = bootstrap_chain(CD, FLU1), perhaps_bootstrap_chains(ChainDefs, LocalFLUs_at_zero -- FLUs, LocalFLUs -- FLUs) end. -bootstrap_chain(#chain_def_v1{name=ChainName, mode=CMode, full=Full, - witnesses=Witnesses, props=Props}=CD, FLU) -> - All_p_srvrs = Full ++ Witnesses, +bootstrap_chain(CD, FLU) -> + bootstrap_chain2(CD, FLU, 20). + +bootstrap_chain2(CD, FLU, 0) -> + lager:warning("Failed all attempts to bootstrap chain ~w via FLU ~w ", + [CD,FLU]), + failed; +bootstrap_chain2(#chain_def_v1{name=NewChainName, mode=CMode, full=Full, + witnesses=Witnesses, props=Props}=CD, + FLU, N) -> + All_p_srvrs = Witnesses ++ Full, L = [{Name, P_srvr} || #p_srvr{name=Name}=P_srvr <- All_p_srvrs], MembersDict = orddict:from_list(L), Mgr = machi_chain_manager1:make_chmgr_regname(FLU), - case machi_chain_manager1:set_chain_members(Mgr, ChainName, 0, CMode, - MembersDict, Props) of + PStore = machi_flu1:make_projection_server_regname(FLU), + {ok, #projection_v1{epoch_number=OldEpoch, chain_name=OldChainName, + all_members=OldAll_list, witnesses=OldWitnesses}} = + machi_projection_store:read_latest_projection(PStore, private), + NewAll_list = [Name || #p_srvr{name=Name} <- All_p_srvrs], + case set_chain_members(OldChainName, NewChainName, + OldAll_list, OldWitnesses, + NewAll_list, NewWitnesses, + Mgr, NewChainName, OldEpoch, + CMode, MembersDict, NewWitnesses) of ok -> - ok; + lager:info("Bootstrapped chain ~w via FLU ~w to " + "mode=~w all=~w witnesses=~w\n", + [NewChainName, FLU, CMode, All_list, NewWitnesses]), + AddedFLUs = NewAll_list -- OldAll_list, + RemovedFLUs = OldAll_list -- NewAll_list, + {ok, AddedFLUs, RemovedFLUs}; Else -> - error_logger:warning("Attempt to bootstrap chain ~w via FLU ~w " - "failed: ~w (defn ~w)\n", [Else, CD]), - ok + lager:error("Attempt to bootstrap chain ~w via FLU ~w " + "failed: ~w (defn ~w)\n", + [NewChainName, FLU, Else, CD]), + timer:sleep(555), + bootstrap_chain2(CD, FLU, N-1) + end. + +set_chain_members(OldChainName, NewChainName, + OldAll_list, OldWitnesses, NewAll_list, NewWitnesses, + Mgr, ChainName, OldEpoch, CMode, MembersDict, Props) -> + if OldChainName == NewChainName, + OldAll_list == NewAll_list, OldWitnesses == NewWitnesses -> + ok; + OldEpoch == 0 orelse (OldChainName == NewChainName) -> + machi_chain_manager1:set_chain_members(Mgr, ChainName, OldEpoch, + CMode, + MembersDict, NewWitnesses); + true -> + chain_bad_arg end. do_process_pending(S) -> @@ -398,7 +450,47 @@ process_pending_flu({File, P}, S) -> process_pending_chains(P_Chains, S) -> lists:foldl(fun process_pending_chain/2, S, P_Chains). -process_pending_chain({_File, #chain_def_v1{}}, S) -> +process_pending_chain({File, CD}, S) -> + #chain_def_v1{name=Name, mode=CMode, full=Full, witnesses=Witnesses} = CD, + case sanitize_chain_def_records([CD]) of + [CD] -> + RunningFLUs = get_local_running_flus(), + AllNames = [Name || #p_srvr{name=Name} <- Full ++ Witnesses], + case ordsets:intersection(ordsets:from_list(AllNames), + ordsets:from_list(RunningFLUs)) of + [] -> + lager:error("Pending chain config file ~s " + "has no local FLUs on this machine, rejected\n", + [File]), + _ = move_to_rejected(File, S), + S; + [FLU|_]=LocalFLUs -> + %% TODO: Between the successful chain change inside of + %% bootstrap_chain() (and before it returns to us!) and + %% the return of process_pending_chain2(), we have a race + %% window if this process crashes. + case bootstrap_chain(CD, FLU) of + {ok, AddedFLUs, RemovedFLUs} -> + process_pending_chain2(File, CD, LocalFLUs, + AddedFLUs, RemovedFLUs, S); + Else -> + lager:error("Pending chain config file ~s " + "has failed (~w), rejected\n", + [Else, File]), + _ = move_to_rejected(File, S), + S + end + end; + [] -> + lager:error("Pending chain config file ~s has been rejected\n", + [File]), + _ = move_to_rejected(File, S), + S + end. + +process_pending_chain2(File, CD, LocalFLUs, AddedFLUs, RemovedFLUs, S) -> + %% AddedFLUs: no any extra work. + %% RemovedFLUs: Stop the FLU, config -> j-i-c, data -> j-i-c io:format(user, "TODO: ~s LINE ~p\n", [?MODULE, ?LINE]), S. @@ -418,25 +510,3 @@ move_to_flu_config(FLU, File, S) -> Dst = get_flu_config_dir(S), ok = file:rename(File, Dst ++ "/" ++ atom_to_list(FLU)), S. - -%% 1. Don't worry about partial writes to config dir: that's a Policy -%% implementation worry: create a temp file, fsync(2), then rename(2) -%% and possibly fsync(2) on the dir. -%% -%% 2. Add periodic re-scan check FLUs & chains (in that order). -%% -%% 3. Add force re-rescan of FLUs *or* chains. (separate is better?) -%% -%% 4. For chain modification: add "change" directory? Scan looks in -%% "change"? -%% -%% 5. Easy comparison for current vs. new chain config. -%% - identify new FLUs: no action required, HC will deal with it? -%% - identify removed FLUs: set_chain_members() to change HC -%% : stop FLU package (after short pause) -%% : move config file -> just-in-case -%% : move data files -> just-in-case -%% - identify j-i-c dir in case resuming from interruption??? to avoid problem of putting config & data files in different places? -%% - move data files itself may be interrupted? -%% - move chain def'n from change -> chain_config_dir - -- 2.45.2 From 2e2d282afc8385c224c01cea7d75024407c86830 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Wed, 9 Dec 2015 18:04:50 +0900 Subject: [PATCH 45/77] Use outside of ephemeral port range to listen on When there is TCP_WAIT connection whose local part has port to be listened, listen (bind) will fail by eaddrinuse _on Linux_ (won't on Mac OS X). This commit also adds some logs and pattern matches. Reference - Ephemeral port - Wikipedia, the free encyclopedia https://en.wikipedia.org/wiki/Ephemeral_port "Many Linux kernels use the port range 32768 to 61000.[note 2] FreeBSD has used the IANA port range since release 4.6. Previous versions, including the Berkeley Software Distribution (BSD), use ports 1024 to 5000 as ephemeral ports.[2]" - Demostration of collision between already-closed ephemeral port and listen port on Linux (Mac OS X allows) https://gist.github.com/shino/36ae1e01608366d52236 --- src/machi_flu1_net_server.erl | 15 ++++++++++----- test/machi_flu1_test.erl | 16 ++++++++-------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/machi_flu1_net_server.erl b/src/machi_flu1_net_server.erl index 6bc9d3b..a244a5f 100644 --- a/src/machi_flu1_net_server.erl +++ b/src/machi_flu1_net_server.erl @@ -113,17 +113,22 @@ handle_info({tcp_closed, Socket}=_Info, #state{socket=Socket}=S) -> lager:debug("~s:handle_info: ~w", [?MODULE, _Info]), transport_closed(Socket, S); handle_info({tcp_error, Socket, Reason}=_Info, #state{socket=Socket}=S) -> - lager:debug("~s:handle_info: ~w", [?MODULE, _Info]), + lager:warning("~s:handle_info (socket=~w) tcp_error: ~w", [?MODULE, Socket, Reason]), transport_error(Socket, Reason, S); handle_info(_Info, S) -> lager:warning("~s:handle_info UNKNOWN message: ~w", [?MODULE, _Info]), {noreply, S}. -terminate(_Reason, #state{socket=undefined}=_S) -> - lager:debug("~s:terminate: ~w", [?MODULE, _Reason]), +terminate(normal, #state{socket=undefined}=_S) -> ok; -terminate(_Reason, #state{socket=Socket}=_S) -> - lager:debug("~s:terminate: ~w", [?MODULE, _Reason]), +terminate(Reason, #state{socket=undefined}=_S) -> + lager:warning("~s:terminate (socket=undefined): ~w", [?MODULE, Reason]), + ok; +terminate(normal, #state{socket=Socket}=_S) -> + (catch gen_tcp:close(Socket)), + ok; +terminate(Reason, #state{socket=Socket}=_S) -> + lager:warning("~s:terminate (socket=Socket): ~w", [?MODULE, Reason]), (catch gen_tcp:close(Socket)), ok. diff --git a/test/machi_flu1_test.erl b/test/machi_flu1_test.erl index 942d2da..0577033 100644 --- a/test/machi_flu1_test.erl +++ b/test/machi_flu1_test.erl @@ -34,13 +34,13 @@ flu_smoke_test() -> Host = "localhost", - TcpPort = 32957, + TcpPort = 12957, DataDir = "./data", Prefix = <<"prefix!">>, BadPrefix = BadFile = "no/good", W_props = [{initial_wedged, false}], + {_, _, _} = machi_test_util:start_flu_package(smoke_flu, TcpPort, DataDir, W_props), try - _ = machi_test_util:start_flu_package(smoke_flu, TcpPort, DataDir, W_props), Msg = "Hello, world!", Msg = ?FLU_C:echo(Host, TcpPort, Msg), {error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort, @@ -142,10 +142,10 @@ flu_smoke_test() -> flu_projection_smoke_test() -> Host = "localhost", - TcpPort = 32959, + TcpPort = 12959, DataDir = "./data.projst", + {_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir), try - machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir), [ok = flu_projection_common(Host, TcpPort, T) || T <- [public, private] ] %% , {ok, {false, EpochID1}} = ?FLU_C:wedge_status(Host, TcpPort), @@ -179,11 +179,11 @@ flu_projection_common(Host, TcpPort, T) -> bad_checksum_test() -> Host = "localhost", - TcpPort = 32960, + TcpPort = 12960, DataDir = "./data.bct", Opts = [{initial_wedged, false}], + {_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts), try - machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts), Prefix = <<"some prefix">>, Chunk1 = <<"yo yo yo">>, Chunk1_badcs = {<>, Chunk1}, @@ -197,11 +197,11 @@ bad_checksum_test() -> witness_test() -> Host = "localhost", - TcpPort = 32961, + TcpPort = 12961, DataDir = "./data.witness", Opts = [{initial_wedged, false}, {witness_mode, true}], + {_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts), try - machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts), Prefix = <<"some prefix">>, Chunk1 = <<"yo yo yo">>, -- 2.45.2 From cd9bf9eeabed6c26bc92ec0643b7107e80c5bc4a Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 9 Dec 2015 18:17:26 +0900 Subject: [PATCH 46/77] Modify chain mostly works, 2 TODOs remain --- src/machi_lifecycle_mgr.erl | 88 ++++++++++++++++++++++++++++--------- 1 file changed, 67 insertions(+), 21 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 5a49dac..cab8f5b 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -350,27 +350,34 @@ bootstrap_chain2(CD, FLU, 0) -> lager:warning("Failed all attempts to bootstrap chain ~w via FLU ~w ", [CD,FLU]), failed; -bootstrap_chain2(#chain_def_v1{name=NewChainName, mode=CMode, full=Full, - witnesses=Witnesses, props=Props}=CD, +bootstrap_chain2(#chain_def_v1{name=NewChainName, mode=NewCMode, + full=Full, witnesses=Witnesses, + old_all=ReqOldAll, old_witnesses=ReqOldWitnesses, + props=_Props}=CD, FLU, N) -> All_p_srvrs = Witnesses ++ Full, L = [{Name, P_srvr} || #p_srvr{name=Name}=P_srvr <- All_p_srvrs], MembersDict = orddict:from_list(L), + NewAll_list = [Name || #p_srvr{name=Name} <- All_p_srvrs], + NewWitnesses_list = [Name || #p_srvr{name=Name} <- Witnesses], + Mgr = machi_chain_manager1:make_chmgr_regname(FLU), PStore = machi_flu1:make_projection_server_regname(FLU), {ok, #projection_v1{epoch_number=OldEpoch, chain_name=OldChainName, + mode=OldCMode, all_members=OldAll_list, witnesses=OldWitnesses}} = machi_projection_store:read_latest_projection(PStore, private), - NewAll_list = [Name || #p_srvr{name=Name} <- All_p_srvrs], - case set_chain_members(OldChainName, NewChainName, + case set_chain_members(OldChainName, NewChainName, OldCMode, + ReqOldAll, ReqOldWitnesses, OldAll_list, OldWitnesses, - NewAll_list, NewWitnesses, + NewAll_list, Witnesses, Mgr, NewChainName, OldEpoch, - CMode, MembersDict, NewWitnesses) of + NewCMode, MembersDict, NewWitnesses_list) of ok -> lager:info("Bootstrapped chain ~w via FLU ~w to " "mode=~w all=~w witnesses=~w\n", - [NewChainName, FLU, CMode, All_list, NewWitnesses]), + [NewChainName, FLU, NewCMode, + NewAll_list, NewWitnesses_list]), AddedFLUs = NewAll_list -- OldAll_list, RemovedFLUs = OldAll_list -- NewAll_list, {ok, AddedFLUs, RemovedFLUs}; @@ -382,15 +389,25 @@ bootstrap_chain2(#chain_def_v1{name=NewChainName, mode=CMode, full=Full, bootstrap_chain2(CD, FLU, N-1) end. -set_chain_members(OldChainName, NewChainName, +set_chain_members(OldChainName, NewChainName, OldCMode, + ReqOldAll, ReqOldWitnesses, OldAll_list, OldWitnesses, NewAll_list, NewWitnesses, - Mgr, ChainName, OldEpoch, CMode, MembersDict, Props) -> - if OldChainName == NewChainName, + Mgr, ChainName, OldEpoch, NewCMode, MembersDict, _Props) -> + if OldChainName == NewChainName, OldCMode == NewCMode, OldAll_list == NewAll_list, OldWitnesses == NewWitnesses -> + %% The chain's current definition at this FLU is already what we + %% want. Let's pretend that we sent the command and that it was + %% successful. ok; - OldEpoch == 0 orelse (OldChainName == NewChainName) -> + OldEpoch == 0 orelse (OldChainName == NewChainName andalso + OldCMode == NewCMode andalso + ReqOldAll == OldAll_list andalso + ReqOldWitnesses == OldWitnesses) -> + %% The old epoch is 0 (no chain defined) or else the prerequisites + %% for our chain change request are indeed matched by the FLU's + %% current private projection. machi_chain_manager1:set_chain_members(Mgr, ChainName, OldEpoch, - CMode, + NewCMode, MembersDict, NewWitnesses); true -> chain_bad_arg @@ -440,22 +457,28 @@ process_pending_flu({File, P}, S) -> _ = move_to_rejected(File, S), S; true -> - {ok, SupPid} = machi_flu_psup:start_flu_package(P), - lager:info("Started FLU ~w with supervisor pid ~p\n", - [FLU, SupPid]), - _ = move_to_flu_config(FLU, File, S), - S + case machi_flu_psup:start_flu_package(P) of + {ok, SupPid} -> + lager:info("Started FLU ~w with supervisor pid ~p\n", + [FLU, SupPid]), + _ = move_to_flu_config(FLU, File, S), + S; + Else -> + lager:error("Start FLU ~w failed: ~p\n", [FLU, Else]), + _ = move_to_rejected(File, S), + S + end end. process_pending_chains(P_Chains, S) -> lists:foldl(fun process_pending_chain/2, S, P_Chains). process_pending_chain({File, CD}, S) -> - #chain_def_v1{name=Name, mode=CMode, full=Full, witnesses=Witnesses} = CD, + #chain_def_v1{full=Full, witnesses=Witnesses} = CD, case sanitize_chain_def_records([CD]) of [CD] -> RunningFLUs = get_local_running_flus(), - AllNames = [Name || #p_srvr{name=Name} <- Full ++ Witnesses], + AllNames = [FLUName || #p_srvr{name=FLUName} <- Full ++ Witnesses], case ordsets:intersection(ordsets:from_list(AllNames), ordsets:from_list(RunningFLUs)) of [] -> @@ -488,10 +511,27 @@ process_pending_chain({File, CD}, S) -> S end. -process_pending_chain2(File, CD, LocalFLUs, AddedFLUs, RemovedFLUs, S) -> +process_pending_chain2(File, CD, LocalFLUs, _AddedFLUs, RemovedFLUs, S) -> %% AddedFLUs: no any extra work. %% RemovedFLUs: Stop the FLU, config -> j-i-c, data -> j-i-c - io:format(user, "TODO: ~s LINE ~p\n", [?MODULE, ?LINE]), + case ordsets:intersection(ordsets:from_list(LocalFLUs), + ordsets:from_list(RemovedFLUs)) of + [] -> + ok; + LocalRemovedFLUs -> + %% Sleep for a little bit to allow HC to settle. + timer:sleep(3000), + [begin + %% We may be retrying this, so be liberal with any pattern + %% matching on return values. + _ = machi_flu_psup:stop_flu_package(FLU), + io:format(user, "TODO: move config ~s LINE ~p\n", [?MODULE, ?LINE]), + io:format(user, "TODO: move data ~s LINE ~p\n", [?MODULE, ?LINE]), + ok + end || FLU <- LocalRemovedFLUs] + end, + #chain_def_v1{name=Name} = CD, + _ = move_to_chain_config(Name, File, S), S. process_bad_files(Files, S) -> @@ -510,3 +550,9 @@ move_to_flu_config(FLU, File, S) -> Dst = get_flu_config_dir(S), ok = file:rename(File, Dst ++ "/" ++ atom_to_list(FLU)), S. + +move_to_chain_config(Name, File, S) -> + lager:info("Creating chain config file ~w\n", [Name]), + Dst = get_chain_config_dir(S), + ok = file:rename(File, Dst ++ "/" ++ atom_to_list(Name)), + S. -- 2.45.2 From 7f25fcc8f8b22014bf53b2c44c395d4a1d40c31e Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 9 Dec 2015 19:02:16 +0900 Subject: [PATCH 47/77] Modify chain mostly works --- rel/reltool.config | 1 + src/machi_lifecycle_mgr.erl | 58 +++++++++++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/rel/reltool.config b/rel/reltool.config index 64cb243..6f9663f 100644 --- a/rel/reltool.config +++ b/rel/reltool.config @@ -47,6 +47,7 @@ {overlay, [ {mkdir, "data"}, + {mkdir, "data/^PRESERVE"}, {mkdir, "log"}, %% Copy base files for starting and interacting w/ node diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index cab8f5b..2b9177f 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -425,6 +425,15 @@ do_process_pending(S) -> S4 = process_bad_files(BadFiles, S3), {{P_FLUs, P_Chains}, S4}. +flu_config_exists(FLU, S) -> + ConfigDir = get_flu_config_dir(S), + case file:read_file_info(ConfigDir ++ "/" ++ atom_to_list(FLU)) of + {ok, _} -> + true; + _ -> + false + end. + get_pending_dir(_S) -> {ok, EtcDir} = application:get_env(machi, platform_etc_dir), EtcDir ++ "/pending". @@ -437,10 +446,21 @@ get_flu_config_dir(_S) -> {ok, Dir} = application:get_env(machi, flu_config_dir), Dir. +get_flu_data_dir(_S) -> + {ok, Dir} = application:get_env(machi, flu_data_dir), + Dir. + get_chain_config_dir(_S) -> {ok, Dir} = application:get_env(machi, chain_config_dir), Dir. +get_data_dir(_S) -> + {ok, Dir} = application:get_env(machi, platform_data_dir), + Dir. + +get_preserve_dir(S) -> + get_data_dir(S) ++ "/^PRESERVE". + process_pending_flus(P_FLUs, S) -> lists:foldl(fun process_pending_flu/2, S, P_FLUs). @@ -487,14 +507,14 @@ process_pending_chain({File, CD}, S) -> [File]), _ = move_to_rejected(File, S), S; - [FLU|_]=LocalFLUs -> + [FLU|_] -> %% TODO: Between the successful chain change inside of %% bootstrap_chain() (and before it returns to us!) and %% the return of process_pending_chain2(), we have a race %% window if this process crashes. case bootstrap_chain(CD, FLU) of {ok, AddedFLUs, RemovedFLUs} -> - process_pending_chain2(File, CD, LocalFLUs, + process_pending_chain2(File, CD, AddedFLUs, RemovedFLUs, S); Else -> lager:error("Pending chain config file ~s " @@ -511,22 +531,34 @@ process_pending_chain({File, CD}, S) -> S end. -process_pending_chain2(File, CD, LocalFLUs, _AddedFLUs, RemovedFLUs, S) -> - %% AddedFLUs: no any extra work. - %% RemovedFLUs: Stop the FLU, config -> j-i-c, data -> j-i-c - case ordsets:intersection(ordsets:from_list(LocalFLUs), - ordsets:from_list(RemovedFLUs)) of +process_pending_chain2(File, CD, _AddedFLUs, RemovedFLUs, S) -> + LocalRemovedFLUs = [FLU || FLU <- RemovedFLUs, + flu_config_exists(FLU, S)], + case LocalRemovedFLUs of [] -> ok; - LocalRemovedFLUs -> + [_|_] -> %% Sleep for a little bit to allow HC to settle. timer:sleep(3000), [begin %% We may be retrying this, so be liberal with any pattern %% matching on return values. _ = machi_flu_psup:stop_flu_package(FLU), - io:format(user, "TODO: move config ~s LINE ~p\n", [?MODULE, ?LINE]), - io:format(user, "TODO: move data ~s LINE ~p\n", [?MODULE, ?LINE]), + ConfigDir = get_flu_config_dir(S), + FluDataDir = get_flu_data_dir(S), + PreserveDir = get_preserve_dir(S), + Suffix = make_ts_suffix(), + FLU_str = atom_to_list(FLU), + MyPreserveDir = PreserveDir ++ "/" ++ FLU_str ++ "." ++ Suffix, + ok = filelib:ensure_dir(MyPreserveDir ++ "/unused"), +io:format(user, "PRE ~s\n", [MyPreserveDir]), + _ = file:make_dir(MyPreserveDir), +io:format(user, "PRE rename ~s ~s\n", [ConfigDir ++ "/" ++ FLU_str, MyPreserveDir ++ "/" ++ FLU_str ++ ".config"]), + _ = file:rename(ConfigDir ++ "/" ++ FLU_str, + MyPreserveDir ++ "/" ++ FLU_str ++ ".config"), +io:format(user, "PRE rename ~s ~s\n", [FluDataDir ++ "/" ++ FLU_str, MyPreserveDir ++ "/" ++ FLU_str ++ ".data"]), + _ = file:rename(FluDataDir ++ "/" ++ FLU_str, + MyPreserveDir ++ "/" ++ FLU_str ++ ".data"), ok end || FLU <- LocalRemovedFLUs] end, @@ -540,11 +572,13 @@ process_bad_files(Files, S) -> move_to_rejected(File, S) -> lager:error("Pending unknown config file ~s has been rejected\n", [File]), Dst = get_rejected_dir(S), - Suffix = lists:flatten(io_lib:format("~w,~w,~w", - tuple_to_list(os:timestamp()))), + Suffix = make_ts_suffix(), ok = file:rename(File, Dst ++ "/" ++ filename:basename(File) ++ Suffix), S. +make_ts_suffix() -> + lists:flatten(io_lib:format("~w,~w,~w", tuple_to_list(os:timestamp()))). + move_to_flu_config(FLU, File, S) -> lager:info("Creating FLU config file ~w\n", [FLU]), Dst = get_flu_config_dir(S), -- 2.45.2 From 95e2df304eb650bfb10029dd71797ded1dbad2fd Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 9 Dec 2015 22:25:43 +0900 Subject: [PATCH 48/77] WIP: minor cleanup --- src/machi_lifecycle_mgr.erl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 2b9177f..9c18b54 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -381,6 +381,11 @@ bootstrap_chain2(#chain_def_v1{name=NewChainName, mode=NewCMode, AddedFLUs = NewAll_list -- OldAll_list, RemovedFLUs = OldAll_list -- NewAll_list, {ok, AddedFLUs, RemovedFLUs}; + chain_bad_state=Else -> + lager:error("Attempt to bootstrap chain ~w via FLU ~w " + "failed (no retries): ~w (defn ~w)\n", + [NewChainName, FLU, Else, CD]), + Else; Else -> lager:error("Attempt to bootstrap chain ~w via FLU ~w " "failed: ~w (defn ~w)\n", @@ -410,7 +415,7 @@ set_chain_members(OldChainName, NewChainName, OldCMode, NewCMode, MembersDict, NewWitnesses); true -> - chain_bad_arg + chain_bad_state end. do_process_pending(S) -> @@ -513,9 +518,8 @@ process_pending_chain({File, CD}, S) -> %% the return of process_pending_chain2(), we have a race %% window if this process crashes. case bootstrap_chain(CD, FLU) of - {ok, AddedFLUs, RemovedFLUs} -> - process_pending_chain2(File, CD, - AddedFLUs, RemovedFLUs, S); + {ok, _AddedFLUs, RemovedFLUs} -> + process_pending_chain2(File, CD, RemovedFLUs, S); Else -> lager:error("Pending chain config file ~s " "has failed (~w), rejected\n", @@ -531,7 +535,7 @@ process_pending_chain({File, CD}, S) -> S end. -process_pending_chain2(File, CD, _AddedFLUs, RemovedFLUs, S) -> +process_pending_chain2(File, CD, RemovedFLUs, S) -> LocalRemovedFLUs = [FLU || FLU <- RemovedFLUs, flu_config_exists(FLU, S)], case LocalRemovedFLUs of @@ -551,12 +555,9 @@ process_pending_chain2(File, CD, _AddedFLUs, RemovedFLUs, S) -> FLU_str = atom_to_list(FLU), MyPreserveDir = PreserveDir ++ "/" ++ FLU_str ++ "." ++ Suffix, ok = filelib:ensure_dir(MyPreserveDir ++ "/unused"), -io:format(user, "PRE ~s\n", [MyPreserveDir]), _ = file:make_dir(MyPreserveDir), -io:format(user, "PRE rename ~s ~s\n", [ConfigDir ++ "/" ++ FLU_str, MyPreserveDir ++ "/" ++ FLU_str ++ ".config"]), _ = file:rename(ConfigDir ++ "/" ++ FLU_str, MyPreserveDir ++ "/" ++ FLU_str ++ ".config"), -io:format(user, "PRE rename ~s ~s\n", [FluDataDir ++ "/" ++ FLU_str, MyPreserveDir ++ "/" ++ FLU_str ++ ".data"]), _ = file:rename(FluDataDir ++ "/" ++ FLU_str, MyPreserveDir ++ "/" ++ FLU_str ++ ".data"), ok -- 2.45.2 From b0a9e65ca2b035b97128a6518aecb2503ff5c567 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 9 Dec 2015 23:00:27 +0900 Subject: [PATCH 49/77] WIP: trying to shut down entire chain, but buggy, derp --- src/machi_lifecycle_mgr.erl | 77 ++++++++++++++++++++++++++++++------- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 9c18b54..d67bac2 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -499,7 +499,7 @@ process_pending_chains(P_Chains, S) -> lists:foldl(fun process_pending_chain/2, S, P_Chains). process_pending_chain({File, CD}, S) -> - #chain_def_v1{full=Full, witnesses=Witnesses} = CD, + #chain_def_v1{name=Name, full=Full, witnesses=Witnesses} = CD, case sanitize_chain_def_records([CD]) of [CD] -> RunningFLUs = get_local_running_flus(), @@ -507,11 +507,34 @@ process_pending_chain({File, CD}, S) -> case ordsets:intersection(ordsets:from_list(AllNames), ordsets:from_list(RunningFLUs)) of [] -> - lager:error("Pending chain config file ~s " - "has no local FLUs on this machine, rejected\n", - [File]), - _ = move_to_rejected(File, S), - S; + %% TODO: hahahah, /me cries ... so, there's a problem if + %% we crash at the end of process_pending_chain2(), then + %% we don't delete the pending #chain_def_v1{} file ... so + %% we'll come to this same path the next time, and + %% get_current_chain_for_running_flus() may tell us that + %% there are zero FLUs running for this chain ... because + %% they were stopped before the crash. Silly. + %% + %% TODO: fix this gap by adding another field to + %% #chain_def_v1{} that specifies what *policy* + %% believes/states/defines/THE_TRUTH about which + %% FLUs in the chain are configured on this machine. + %% Then always use exactly that truth to guide us. + case get_current_chain_for_running_flus(CD, RunningFLUs) of + [] -> + lager:error("Pending chain config file ~s has no " + "FLUs on this machine, rejected\n", + [File]), + _ = move_to_rejected(File, S), + S; + RemovedFLUs -> + lager:info("Pending chain config file ~s creates " + "chain ~w of length 0, " + "stopping FLUS: ~w\n", + [File, Name, RemovedFLUs]), + process_pending_chain2(File, CD, RemovedFLUs, + delete, S) + end; [FLU|_] -> %% TODO: Between the successful chain change inside of %% bootstrap_chain() (and before it returns to us!) and @@ -519,7 +542,8 @@ process_pending_chain({File, CD}, S) -> %% window if this process crashes. case bootstrap_chain(CD, FLU) of {ok, _AddedFLUs, RemovedFLUs} -> - process_pending_chain2(File, CD, RemovedFLUs, S); + process_pending_chain2(File, CD, RemovedFLUs, + move, S); Else -> lager:error("Pending chain config file ~s " "has failed (~w), rejected\n", @@ -535,7 +559,7 @@ process_pending_chain({File, CD}, S) -> S end. -process_pending_chain2(File, CD, RemovedFLUs, S) -> +process_pending_chain2(File, CD, RemovedFLUs, ChainConfigAction, S) -> LocalRemovedFLUs = [FLU || FLU <- RemovedFLUs, flu_config_exists(FLU, S)], case LocalRemovedFLUs of @@ -556,15 +580,25 @@ process_pending_chain2(File, CD, RemovedFLUs, S) -> MyPreserveDir = PreserveDir ++ "/" ++ FLU_str ++ "." ++ Suffix, ok = filelib:ensure_dir(MyPreserveDir ++ "/unused"), _ = file:make_dir(MyPreserveDir), - _ = file:rename(ConfigDir ++ "/" ++ FLU_str, - MyPreserveDir ++ "/" ++ FLU_str ++ ".config"), - _ = file:rename(FluDataDir ++ "/" ++ FLU_str, - MyPreserveDir ++ "/" ++ FLU_str ++ ".data"), + Src1 = ConfigDir ++ "/" ++ FLU_str, + Dst1 = MyPreserveDir ++ "/" ++ FLU_str ++ ".config", + lager:info("Stopped FLU ~w: rename ~s ~s\n", + [FLU, Src1, Dst1]), + _ = file:rename(Src1, Dst1), + Src2 = FluDataDir ++ "/" ++ FLU_str, + Dst2 = MyPreserveDir ++ "/" ++ FLU_str ++ ".data", + lager:info("Stopped FLU ~w: rename ~s ~s\n", + [FLU, Src2, Dst2]), + _ = file:rename(Src2, Dst2), ok end || FLU <- LocalRemovedFLUs] end, #chain_def_v1{name=Name} = CD, - _ = move_to_chain_config(Name, File, S), + if ChainConfigAction == move -> + _ = move_to_chain_config(Name, File, S); + ChainConfigAction == delete -> + _ = delete_chain_config(Name, File, S) + end, S. process_bad_files(Files, S) -> @@ -591,3 +625,20 @@ move_to_chain_config(Name, File, S) -> Dst = get_chain_config_dir(S), ok = file:rename(File, Dst ++ "/" ++ atom_to_list(Name)), S. + +delete_chain_config(Name, File, S) -> + lager:info("Deleting chain config file ~w for chain ~w\n", [File, Name]), + Dst = get_chain_config_dir(S), + ok = file:delete(Dst ++ "/" ++ atom_to_list(Name)), + S. + +get_current_chain_for_running_flus(#chain_def_v1{name=Name}, RunningFLUs) -> + FLU_CurChs = [begin + PStore = machi_flu1:make_projection_server_regname(FLU), + {ok, #projection_v1{chain_name=Ch}} = + machi_projection_store:read_latest_projection( + PStore, private), + {FLU, Ch} + end || FLU <- RunningFLUs], + [FLU || {FLU, CurChain} <- FLU_CurChs, + CurChain == Name]. -- 2.45.2 From 61ef7739cda288acbbe13e1cc2601bea94762b47 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 10 Dec 2015 00:12:34 +0900 Subject: [PATCH 50/77] Modify chain mostly works, better --- include/machi_projection.hrl | 8 +++-- src/machi_lifecycle_mgr.erl | 57 +++++++++--------------------------- 2 files changed, 19 insertions(+), 46 deletions(-) diff --git a/include/machi_projection.hrl b/include/machi_projection.hrl index 4794bd6..23a540d 100644 --- a/include/machi_projection.hrl +++ b/include/machi_projection.hrl @@ -82,9 +82,11 @@ mode :: pv1_consistency_mode(), full :: [p_srvr()], witnesses :: [p_srvr()], - old_all :: [pv1_server()], - old_witnesses :: [pv1_server()], - props = [] :: list() % proplist for other related info + old_all :: [pv1_server()], % guard against some races + old_witnesses :: [pv1_server()], % guard against some races + local_run :: [pv1_server()], % must be tailored to each machine! + local_stop :: [pv1_server()], % must be tailored to each machine! + props = [] :: list() % proplist for other related info }). -endif. % !MACHI_PROJECTION_HRL diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index d67bac2..df0459a 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -378,9 +378,7 @@ bootstrap_chain2(#chain_def_v1{name=NewChainName, mode=NewCMode, "mode=~w all=~w witnesses=~w\n", [NewChainName, FLU, NewCMode, NewAll_list, NewWitnesses_list]), - AddedFLUs = NewAll_list -- OldAll_list, - RemovedFLUs = OldAll_list -- NewAll_list, - {ok, AddedFLUs, RemovedFLUs}; + ok; chain_bad_state=Else -> lager:error("Attempt to bootstrap chain ~w via FLU ~w " "failed (no retries): ~w (defn ~w)\n", @@ -499,50 +497,34 @@ process_pending_chains(P_Chains, S) -> lists:foldl(fun process_pending_chain/2, S, P_Chains). process_pending_chain({File, CD}, S) -> - #chain_def_v1{name=Name, full=Full, witnesses=Witnesses} = CD, + #chain_def_v1{name=Name, + local_stop=LocalStopFLUs, local_run=LocalRunFLUs} = CD, case sanitize_chain_def_records([CD]) of [CD] -> - RunningFLUs = get_local_running_flus(), - AllNames = [FLUName || #p_srvr{name=FLUName} <- Full ++ Witnesses], - case ordsets:intersection(ordsets:from_list(AllNames), - ordsets:from_list(RunningFLUs)) of + case LocalRunFLUs of [] -> - %% TODO: hahahah, /me cries ... so, there's a problem if - %% we crash at the end of process_pending_chain2(), then - %% we don't delete the pending #chain_def_v1{} file ... so - %% we'll come to this same path the next time, and - %% get_current_chain_for_running_flus() may tell us that - %% there are zero FLUs running for this chain ... because - %% they were stopped before the crash. Silly. - %% - %% TODO: fix this gap by adding another field to - %% #chain_def_v1{} that specifies what *policy* - %% believes/states/defines/THE_TRUTH about which - %% FLUs in the chain are configured on this machine. - %% Then always use exactly that truth to guide us. - case get_current_chain_for_running_flus(CD, RunningFLUs) of + case LocalStopFLUs of [] -> lager:error("Pending chain config file ~s has no " "FLUs on this machine, rejected\n", [File]), _ = move_to_rejected(File, S), S; - RemovedFLUs -> - lager:info("Pending chain config file ~s creates " - "chain ~w of length 0, " - "stopping FLUS: ~w\n", - [File, Name, RemovedFLUs]), - process_pending_chain2(File, CD, RemovedFLUs, + [_|_] -> + lager:info("Pending chain config file ~s stops " + "all local members of chain ~w: ~w\n", + [File, Name, LocalStopFLUs]), + process_pending_chain2(File, CD, LocalStopFLUs, delete, S) end; [FLU|_] -> %% TODO: Between the successful chain change inside of %% bootstrap_chain() (and before it returns to us!) and %% the return of process_pending_chain2(), we have a race - %% window if this process crashes. + %% window if this process crashes. (Review again!) case bootstrap_chain(CD, FLU) of - {ok, _AddedFLUs, RemovedFLUs} -> - process_pending_chain2(File, CD, RemovedFLUs, + ok -> + process_pending_chain2(File, CD, LocalStopFLUs, move, S); Else -> lager:error("Pending chain config file ~s " @@ -627,18 +609,7 @@ move_to_chain_config(Name, File, S) -> S. delete_chain_config(Name, File, S) -> - lager:info("Deleting chain config file ~w for chain ~w\n", [File, Name]), + lager:info("Deleting chain config file ~s for chain ~w\n", [File, Name]), Dst = get_chain_config_dir(S), ok = file:delete(Dst ++ "/" ++ atom_to_list(Name)), S. - -get_current_chain_for_running_flus(#chain_def_v1{name=Name}, RunningFLUs) -> - FLU_CurChs = [begin - PStore = machi_flu1:make_projection_server_regname(FLU), - {ok, #projection_v1{chain_name=Ch}} = - machi_projection_store:read_latest_projection( - PStore, private), - {FLU, Ch} - end || FLU <- RunningFLUs], - [FLU || {FLU, CurChain} <- FLU_CurChs, - CurChain == Name]. -- 2.45.2 From cb706f0d2317077a83efee54c3b1091176f57daf Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 10 Dec 2015 15:20:56 +0900 Subject: [PATCH 51/77] Add test/machi_lifecycle_mgr_test.erl --- test/machi_lifecycle_mgr_test.erl | 155 ++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 test/machi_lifecycle_mgr_test.erl diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl new file mode 100644 index 0000000..f4a4f56 --- /dev/null +++ b/test/machi_lifecycle_mgr_test.erl @@ -0,0 +1,155 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(machi_lifecycle_mgr_test). +-compile(export_all). + +-ifdef(TEST). +-ifndef(PULSE). + +-include_lib("eunit/include/eunit.hrl"). + +-include("machi.hrl"). +-include("machi_projection.hrl"). + +-define(MGR, machi_chain_manager1). + +smoke_test_() -> + {timeout, 120, fun() -> smoke_test2() end}. + +smoke_test2() -> + catch application:stop(machi), + {ok, SupPid} = machi_sup:start_link(), + error_logger:tty(false), + Dir = "./" ++ atom_to_list(?MODULE) ++ ".datadir", + machi_flu1_test:clean_up_data_dir(Dir ++ "/*/*"), + machi_flu1_test:clean_up_data_dir(Dir), + Envs = [{flu_data_dir, Dir ++ "/data/flu"}, + {flu_config_dir, Dir ++ "/etc/flu-config"}, + {chain_config_dir, Dir ++ "/etc/chain-config"}, + {platform_data_dir, Dir ++ "/data"}, + {platform_etc_dir, Dir ++ "/etc"}, + {not_used_pending, Dir ++ "/etc/pending"} + ], + EnvKeys = [K || {K, V} <- Envs], + undefined = application:get_env(machi, yo), + Cleanup = machi_flu1_test:get_env_vars(machi, EnvKeys ++ [yo]), + [begin + filelib:ensure_dir(V ++ "/unused"), + application:set_env(machi, K, V) + end || {K, V} <- Envs], + try + Prefix = <<"pre">>, + Chunk1 = <<"yochunk">>, + Host = "localhost", + PortBase = 60120, + + Pa = #p_srvr{name=a,address="localhost",port=PortBase+0}, + Pb = #p_srvr{name=b,address="localhost",port=PortBase+1}, + Pc = #p_srvr{name=c,address="localhost",port=PortBase+2}, + %% Pstore_a = machi_flu1:make_projection_server_regname(a), + %% Pstore_b = machi_flu1:make_projection_server_regname(b), + %% Pstore_c = machi_flu1:make_projection_server_regname(c), + Pstores = [Pstore_a, Pstore_b, Pstore_c] = + [machi_flu1:make_projection_server_regname(a), + machi_flu1:make_projection_server_regname(b), + machi_flu1:make_projection_server_regname(c)], + ChMgrs = [ChMgr_a, ChMgr_b, ChMgr_c] = + [machi_chain_manager1:make_chmgr_regname(a), + machi_chain_manager1:make_chmgr_regname(b), + machi_chain_manager1:make_chmgr_regname(c)], + Fits = [Fit_a, Fit_b, Fit_c] = + [machi_flu_psup:make_fitness_regname(a), + machi_flu_psup:make_fitness_regname(b), + machi_flu_psup:make_fitness_regname(c)], + Advance = machi_chain_manager1_test:make_advance_fun( + Fits, [a,b,c], ChMgrs, 3), + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format("\nSTEP: Start 3 FLUs, no chain.\n", []), + + [make_pending_config(P) || P <- [Pa,Pb,Pc] ], + {[_,_,_],[]} = machi_lifecycle_mgr:process_pending(), + [{ok, #projection_v1{epoch_number=0}} = + machi_projection_store:read_latest_projection(PSTORE, private) + || PSTORE <- Pstores], + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format("\nSTEP: Start chain = [a,b,c]\n", []), + + C1 = #chain_def_v1{name=cx, mode=ap_mode, full=[Pa,Pb,Pc], + local_run=[a,b,c]}, + make_pending_config(C1), + {[],[_]} = machi_lifecycle_mgr:process_pending(), + Advance(), + [{ok, #projection_v1{all_members=[a,b,c]}} = + machi_projection_store:read_latest_projection(PSTORE, private) + || PSTORE <- Pstores], + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format("\nSTEP: Reset chain = [b,c]\n", []), + + C2 = #chain_def_v1{name=cx, mode=ap_mode, full=[Pb,Pc], + old_all=[a,b,c], old_witnesses=[], + local_stop=[a], local_run=[b,c]}, + make_pending_config(C2), + {[],[_]} = machi_lifecycle_mgr:process_pending(), + Advance(), + %% a should be down + {'EXIT', _} = (catch machi_projection_store:read_latest_projection( + hd(Pstores), private)), + [{ok, #projection_v1{all_members=[b,c]}} = + machi_projection_store:read_latest_projection(PSTORE, private) + || PSTORE <- tl(Pstores)], + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + io:format("\nSTEP: Reset chain = []\n", []), + + C3 = #chain_def_v1{name=cx, mode=ap_mode, full=[], + old_all=[b,c], old_witnesses=[], + local_stop=[b,c], local_run=[]}, + make_pending_config(C3), + {[],[_]} = machi_lifecycle_mgr:process_pending(), + Advance(), + %% a,b,c should be down + [{'EXIT', _} = (catch machi_projection_store:read_latest_projection( + PSTORE, private)) + || PSTORE <- Pstores], + + ok + after + exit(SupPid, normal), + machi_util:wait_for_death(SupPid, 100), + error_logger:tty(true), + catch application:stop(machi), + %% machi_flu1_test:clean_up_data_dir(Dir ++ "/*/*"), + %% machi_flu1_test:clean_up_data_dir(Dir), + machi_flu1_test:clean_up_env_vars(Cleanup), + undefined = application:get_env(machi, yo) + end. + +make_pending_config(Term) -> + Dir = machi_lifecycle_mgr:get_pending_dir(x), + Blob = io_lib:format("~w.\n", [Term]), + ok = file:write_file(Dir ++ "/" ++ lists:flatten(io_lib:format("~w,~w,~w", tuple_to_list(os:timestamp()))), + Blob). + +-endif. % !PULSE +-endif. % TEST -- 2.45.2 From 9472bad37b48a137e987412eb0c21b0595da6d56 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 10 Dec 2015 15:57:35 +0900 Subject: [PATCH 52/77] Clean up test errors --- include/machi_projection.hrl | 20 +++++++------- src/machi_flu_psup.erl | 6 ++--- src/machi_lifecycle_mgr.erl | 7 +++-- src/machi_pb_translate.erl | 2 +- test/machi_chain_manager1_test.erl | 43 +++++++++++++++++------------- test/machi_flu1_test.erl | 16 +++++++++++ 6 files changed, 60 insertions(+), 34 deletions(-) diff --git a/include/machi_projection.hrl b/include/machi_projection.hrl index 23a540d..4181625 100644 --- a/include/machi_projection.hrl +++ b/include/machi_projection.hrl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved. +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -78,15 +78,15 @@ -define(MAX_CHAIN_LENGTH, 64). -record(chain_def_v1, { - name :: atom(), - mode :: pv1_consistency_mode(), - full :: [p_srvr()], - witnesses :: [p_srvr()], - old_all :: [pv1_server()], % guard against some races - old_witnesses :: [pv1_server()], % guard against some races - local_run :: [pv1_server()], % must be tailored to each machine! - local_stop :: [pv1_server()], % must be tailored to each machine! - props = [] :: list() % proplist for other related info + name :: atom(), % chain name + mode :: pv1_consistency_mode(), + full = [] :: [p_srvr()], + witnesses = [] :: [p_srvr()], + old_all = [] :: [pv1_server()], % guard against some races + old_witnesses=[] :: [pv1_server()], % guard against some races + local_run = [] :: [pv1_server()], % must be tailored to each machine! + local_stop = [] :: [pv1_server()], % must be tailored to each machine! + props = [] :: list() % proplist for other related info }). -endif. % !MACHI_PROJECTION_HRL diff --git a/src/machi_flu_psup.erl b/src/machi_flu_psup.erl index 3b8ac3b..a8fe946 100644 --- a/src/machi_flu_psup.erl +++ b/src/machi_flu_psup.erl @@ -96,7 +96,7 @@ make_package_spec(FluName, TcpPort, DataDir, Props) -> permanent, ?SHUTDOWN, supervisor, []}. start_flu_package(#p_srvr{name=FluName, port=TcpPort, props=Props}) -> - DataDir = get_data_dir(Props), + DataDir = get_data_dir(FluName, Props), start_flu_package(FluName, TcpPort, DataDir, Props). start_flu_package(FluName, TcpPort, DataDir, Props) -> @@ -177,11 +177,11 @@ get_env(Setting, Default) -> {ok, V} -> V end. -get_data_dir(Props) -> +get_data_dir(FluName, Props) -> case proplists:get_value(data_dir, Props) of Path when is_list(Path) -> Path; undefined -> {ok, Dir} = application:get_env(machi, flu_data_dir), - Dir + Dir ++ "/" ++ atom_to_list(FluName) end. diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index df0459a..6b58a8d 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -205,6 +205,9 @@ %% API -export([start_link/0, process_pending/0]). +-ifdef(TEST). +-compile(export_all). +-endif. % TEST %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -548,8 +551,8 @@ process_pending_chain2(File, CD, RemovedFLUs, ChainConfigAction, S) -> [] -> ok; [_|_] -> - %% Sleep for a little bit to allow HC to settle. - timer:sleep(3000), + %% %% Sleep for a little bit to allow HC to settle. + %% timer:sleep(1000), [begin %% We may be retrying this, so be liberal with any pattern %% matching on return values. diff --git a/src/machi_pb_translate.erl b/src/machi_pb_translate.erl index 676c5a1..cb8cef2 100644 --- a/src/machi_pb_translate.erl +++ b/src/machi_pb_translate.erl @@ -955,7 +955,7 @@ conv_from_status({error, partial_read}) -> conv_from_status({error, bad_epoch}) -> 'BAD_EPOCH'; conv_from_status(_OOPS) -> - io:format(user, "HEY, ~s:~w got ~w\n", [?MODULE, ?LINE, _OOPS]), + io:format(user, "HEY, ~s:~w got ~p\n", [?MODULE, ?LINE, _OOPS]), 'BAD_JOSS'. conv_to_boolean(undefined) -> diff --git a/test/machi_chain_manager1_test.erl b/test/machi_chain_manager1_test.erl index 7e77384..094443b 100644 --- a/test/machi_chain_manager1_test.erl +++ b/test/machi_chain_manager1_test.erl @@ -273,6 +273,17 @@ make_prop_ets() -> -endif. % EQC +make_advance_fun(FitList, FLUList, MgrList, Num) -> + fun() -> + [begin + [catch machi_fitness:trigger_early_adjustment(Fit, Tgt) || + Fit <- FitList, + Tgt <- FLUList ], + [catch ?MGR:trigger_react_to_env(Mgr) || Mgr <- MgrList], + ok + end || _ <- lists:seq(1, Num)] + end. + smoke0_test() -> {ok, _} = machi_partition_simulator:start_link({1,2,3}, 50, 50), Host = "localhost", @@ -348,6 +359,7 @@ nonunanimous_setup_and_fix_test2() -> [machi_flu1_test:clean_up_data_dir(Dir) || {_,_,Dir} <- FluInfo], {ok, SupPid} = machi_flu_sup:start_link(), Opts = [{active_mode, false}, {initial_wedged, true}], + ChainName = my_little_chain, [{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) || {Name,Port,Dir} <- FluInfo], Proxies = [Proxy_a, Proxy_b, Proxy_c] = @@ -356,19 +368,14 @@ nonunanimous_setup_and_fix_test2() -> MembersDict = machi_projection:make_members_dict(lists:sublist(P_s, 2)), Mgrs = [Ma,Mb,Mc] = [a_chmgr, b_chmgr, c_chmgr], MgrProxies = [{Ma, Proxy_a}, {Mb, Proxy_b}, {Mc, Proxy_c}], - Advance = fun() -> - [begin - [catch machi_fitness:trigger_early_adjustment(Fit, Tgt) || - Fit <- [a_fitness,b_fitness,c_fitness], - Tgt <- [a,b,c] ], - [catch ?MGR:trigger_react_to_env(Mgr) || - {Mgr,_Proxy} <- MgrProxies], - ok - end || _ <- lists:seq(1, 3)] - end, - ok = machi_chain_manager1:set_chain_members(Ma, MembersDict), - ok = machi_chain_manager1:set_chain_members(Mb, MembersDict), - + Advance = make_advance_fun([a_fitness,b_fitness,c_fitness], + [a,b,c], + [Mgr || {Mgr,_Proxy} <- MgrProxies], + 3), + ok = machi_chain_manager1:set_chain_members(Ma, ChainName, 0, ap_mode, + MembersDict, []), + ok = machi_chain_manager1:set_chain_members(Mb, ChainName, 0, ap_mode, + MembersDict, []), try {ok, P1} = ?MGR:test_calc_projection(Ma, false), @@ -410,7 +417,7 @@ nonunanimous_setup_and_fix_test2() -> MembersDict3 = machi_projection:make_members_dict(P_s), ok = machi_chain_manager1:set_chain_members( - Ma, ch_not_def_yet, EpochNum_a, ap_mode, MembersDict3, []), + Ma, ChainName, EpochNum_a, ap_mode, MembersDict3, []), Advance(), {_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Ma), @@ -424,7 +431,7 @@ nonunanimous_setup_and_fix_test2() -> MembersDict4 = machi_projection:make_members_dict(tl(P_s)), ok = machi_chain_manager1:set_chain_members( - Mb, ch_not_def_yet, TheEpoch_3, ap_mode, MembersDict4, []), + Mb, ChainName, TheEpoch_3, ap_mode, MembersDict4, []), Advance(), {ok, {true, _}} = ?FLU_PC:wedge_status(Proxy_a), @@ -438,7 +445,7 @@ nonunanimous_setup_and_fix_test2() -> MembersDict5 = machi_projection:make_members_dict(P_s), ok = machi_chain_manager1:set_chain_members( - Mb, ch_not_def_yet, TheEpoch_4, ap_mode, MembersDict5, []), + Mb, ChainName, TheEpoch_4, ap_mode, MembersDict5, []), Advance(), {_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Ma), @@ -462,7 +469,7 @@ nonunanimous_setup_and_fix_test2() -> MembersDict7 = machi_projection:make_members_dict(tl(P_s)), ok = machi_chain_manager1:set_chain_members( - Mb, ch_not_def_yet, TheEpoch_6, ap_mode, MembersDict7, []), + Mb, ChainName, TheEpoch_6, ap_mode, MembersDict7, []), Advance(), {_, _, TheEpoch_7} = ?MGR:trigger_react_to_env(Mb), @@ -498,7 +505,7 @@ nonunanimous_setup_and_fix_test2() -> MembersDict9 = machi_projection:make_members_dict(P_s), {_, _, TheEpoch_9} = ?MGR:trigger_react_to_env(Mb), ok = machi_chain_manager1:set_chain_members( - Mb, ch_not_def_yet, TheEpoch_9, ap_mode, MembersDict9, []), + Mb, ChainName, TheEpoch_9, ap_mode, MembersDict9, []), Advance(), {_, _, TheEpoch_9b} = ?MGR:trigger_react_to_env(Mb), true = (TheEpoch_9b > TheEpoch_9), diff --git a/test/machi_flu1_test.erl b/test/machi_flu1_test.erl index b8dded9..ea8702a 100644 --- a/test/machi_flu1_test.erl +++ b/test/machi_flu1_test.erl @@ -30,6 +30,22 @@ -define(FLU, machi_flu1). -define(FLU_C, machi_flu1_client). +get_env_vars(App, Ks) -> + Raw = [application:get_env(App, K) || K <- Ks], + Old = lists:zip(Ks, Raw), + {App, Old}. + +clean_up_env_vars({App, Old}) -> + [case Res of + undefined -> + application:unset_env(App, K); + {ok, V} -> + application:set_env(App, K, V) + end || {K, Res} <- Old]. + +filter_env_var({ok, V}) -> V; +filter_env_var(Else) -> Else. + clean_up_data_dir(DataDir) -> [begin Fs = filelib:wildcard(DataDir ++ Glob), -- 2.45.2 From eef00e4f8f27189bb7f09e030d1ec629cc57e721 Mon Sep 17 00:00:00 2001 From: Shunichi Shinohara Date: Thu, 10 Dec 2015 15:58:17 +0900 Subject: [PATCH 53/77] Add TODO comment for possible race condition --- src/machi_flu1_net_server.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/machi_flu1_net_server.erl b/src/machi_flu1_net_server.erl index a244a5f..93e3675 100644 --- a/src/machi_flu1_net_server.erl +++ b/src/machi_flu1_net_server.erl @@ -250,6 +250,7 @@ do_pb_ll_request2(EpochID, CMD, S) -> end. lookup_epoch(#state{epoch_tab=T}) -> + %% TODO: race in shutdown to access ets table after owner dies ets:lookup_element(T, epoch, 2). %% Witness status does not matter below. -- 2.45.2 From 9cec53eea634bb8d78779ccc0f117c01ebb4a6df Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 10 Dec 2015 19:18:25 +0900 Subject: [PATCH 54/77] Yet another strawman AST --- src/machi_lifecycle_mgr.erl | 38 +++++++++++++++++++++++++++++++ test/machi_lifecycle_mgr_test.erl | 26 +++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 6b58a8d..4cc9610 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -616,3 +616,41 @@ delete_chain_config(Name, File, S) -> Dst = get_chain_config_dir(S), ok = file:delete(Dst ++ "/" ++ atom_to_list(Name)), S. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +check_ast_tuple_syntax(Ts) -> + lists:partition(fun check_a_ast_tuple/1, Ts). + +check_a_ast_tuple({host, Name, Props}) -> + is_stringy(Name) andalso is_proplisty(Props) andalso + lists:all(fun({admin_interface, X}) -> is_stringy(X); + ({client_interface, X}) -> is_stringy(X); + (_) -> false + end, Props); +check_a_ast_tuple({flu, Name, HostName, Port, Props}) -> + is_stringy(Name) andalso is_stringy(HostName) andalso + is_porty(Port) andalso is_proplisty(Props); +check_a_ast_tuple({chain, Name, AddList, RemoveList, Props}) -> + is_stringy(Name) andalso + lists:all(fun is_stringy/1, AddList) andalso + lists:all(fun is_stringy/1, RemoveList) andalso + is_proplisty(Props); +check_a_ast_tuple(_) -> + false. + +is_proplisty(Props) -> + is_list(Props) andalso + lists:all(fun({_,_}) -> true; + (X) when is_atom(X) -> true; + (_) -> false + end, Props). + +is_stringy(L) -> + is_list(L) andalso + lists:all(fun(C) when 33 =< C, C =< 126 -> true; + (_) -> false + end, L). + +is_porty(Port) -> + is_integer(Port) andalso 1024 =< Port andalso Port =< 65535. diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index f4a4f56..31c650b 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -151,5 +151,31 @@ make_pending_config(Term) -> ok = file:write_file(Dir ++ "/" ++ lists:flatten(io_lib:format("~w,~w,~w", tuple_to_list(os:timestamp()))), Blob). +ast_tuple_syntax_test() -> + T = fun(L) -> machi_lifecycle_mgr:check_ast_tuple_syntax(L) end, + {_Good,[]=_Bad} = + T([ {host, "localhost", []}, + {host, "localhost", [{client_interface, "1.2.3.4"}, + {admin_interface, "5.6.7.8"}]}, + {flu, "fx", "foohost", 4000, []}, + {chain, "cy", ["fx", "fy"], ["fz"], [foo,{bar,baz}]} ]), + {[],[_,_,_,_]} = + T([ {host, 'localhost', []}, + {host, 'localhost', yo}, + {host, "localhost", [{client_interface, 77.88293829832}]}, + {host, "localhost", [{client_interface, "1.2.3.4"}, + {bummer, "5.6.7.8"}]} ]), + {[],[_,_,_,_,_,_]} = + T([ {flu, 'fx', "foohost", 4000, []}, + {flu, "fx", <<"foohost">>, 4000, []}, + {flu, "fx", "foohost", -4000, []}, + {flu, "fx", "foohost", 40009999, []}, + {flu, "fx", "foohost", 4000, gack}, + {flu, "fx", "foohost", 4000, [22]} ]), + {[],[_,_]} = + T([ {chain, 'cy', ["fx", "fy"], ["fz"], [foo,{bar,baz}]}, + {chain, "cy", ["fx", 27], ["fz"], oops,arity,way,way,way,too,big,x} + ]). + -endif. % !PULSE -endif. % TEST -- 2.45.2 From c37f23d97adf9cb112413949d9a2d3daefbe99d2 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 10 Dec 2015 22:53:17 +0900 Subject: [PATCH 55/77] WIP: 'Run' AST thingie ha, take that, wheel! --- src/machi_lifecycle_mgr.erl | 158 ++++++++++++++++++++++++++++-- test/machi_lifecycle_mgr_test.erl | 38 +++++-- 2 files changed, 180 insertions(+), 16 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 4cc9610..3f39f82 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -620,31 +620,173 @@ delete_chain_config(Name, File, S) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%% check_ast_tuple_syntax(Ts) -> - lists:partition(fun check_a_ast_tuple/1, Ts). + lists:partition(fun check_an_ast_tuple/1, Ts). -check_a_ast_tuple({host, Name, Props}) -> +check_an_ast_tuple({host, Name, Props}) -> is_stringy(Name) andalso is_proplisty(Props) andalso lists:all(fun({admin_interface, X}) -> is_stringy(X); ({client_interface, X}) -> is_stringy(X); (_) -> false end, Props); -check_a_ast_tuple({flu, Name, HostName, Port, Props}) -> +check_an_ast_tuple({host, Name, AdminI, ClientI, Props}) -> + is_stringy(Name) andalso + is_stringy(AdminI) andalso is_stringy(ClientI) andalso + is_proplisty(Props) andalso + lists:all(fun({admin_interface, X}) -> is_stringy(X); + ({client_interface, X}) -> is_stringy(X); + (_) -> false + end, Props); +check_an_ast_tuple({flu, Name, HostName, Port, Props}) -> is_stringy(Name) andalso is_stringy(HostName) andalso is_porty(Port) andalso is_proplisty(Props); -check_a_ast_tuple({chain, Name, AddList, RemoveList, Props}) -> +check_an_ast_tuple({chain, Name, AddList, RemoveList, Props}) -> is_stringy(Name) andalso lists:all(fun is_stringy/1, AddList) andalso lists:all(fun is_stringy/1, RemoveList) andalso - is_proplisty(Props); -check_a_ast_tuple(_) -> + is_proplisty(Props); +check_an_ast_tuple({chain, Name, CMode, AddList, Add_Witnesses, + RemoveList, Remove_Witnesses, Props}) -> + is_stringy(Name) andalso + (CMode == ap_mode orelse CMode == cp_mode) andalso + lists:all(fun is_stringy/1, AddList) andalso + lists:all(fun is_stringy/1, Add_Witnesses) andalso + lists:all(fun is_stringy/1, RemoveList) andalso + lists:all(fun is_stringy/1, Remove_Witnesses) andalso + is_proplisty(Props); +check_an_ast_tuple(switch_old_and_new) -> + true; +check_an_ast_tuple(_) -> false. +%% Prerequisite: all tuples are approved by check_ast_tuple_syntax(). + +normalize_ast_tuple_syntax(Ts) -> + lists:map(fun normalize_an_ast_tuple/1, Ts). + +normalize_an_ast_tuple({host, Name, Props}) -> + AdminI = proplists:get_value(admin_interface, Props, Name), + ClientI = proplists:get_value(client_interface, Props, Name), + Props2 = lists:keydelete(admin_interface, 1, + lists:keydelete(client_interface, 1, Props)), + {host, Name, AdminI, ClientI, n(Props2)}; +normalize_an_ast_tuple({host, Name, AdminI, ClientI, Props}) -> + Props2 = lists:keydelete(admin_interface, 1, + lists:keydelete(client_interface, 1, Props)), + {host, Name, AdminI, ClientI, n(Props2)}; +normalize_an_ast_tuple({flu, Name, HostName, Port, Props}) -> + {flu, Name, HostName, Port, n(Props)}; +normalize_an_ast_tuple({chain, Name, AddList, RemoveList, Props}) -> + {chain, Name, ap_mode, n(AddList), [], n(RemoveList), [], n(Props)}; +normalize_an_ast_tuple({chain, Name, CMode, AddList, Add_Witnesses, + RemoveList, Remove_Witnesses, Props}) -> + {chain, Name, CMode, n(AddList), n(Add_Witnesses), + n(RemoveList), n(Remove_Witnesses), n(Props)}; +normalize_an_ast_tuple(A=switch_old_and_new) -> + A. + +run_ast(Ts) -> + {_, []} = check_ast_tuple_syntax(Ts), + Ts2 = normalize_ast_tuple_syntax(Ts), + Env1 = make_ast_run_env(), + try + Env2 = lists:foldl(fun run_ast_cmd/2, Env1, Ts2), + {ok, Env2} + catch throw:DbgStuff -> + {error, DbgStuff} + end. + +run_ast_cmd({host, Name, _AdminI, _ClientI, _Props}=T, E) -> + Key = {kv, {host, Name}}, + case d_find(Key, E) of + error -> + d_store(Key, T, E); + {ok, _} -> + err("Duplicate host definition ~p: ~p", [Name], T) + end; +run_ast_cmd({flu, Name, HostName, Port, _Props}=T, E) -> + Key = {kv, {flu, Name}}, + HostExists_p = env_host_exists(HostName, E), + case d_find(Key, E) of + error when HostExists_p -> + case host_port_is_assigned(HostName, Port, E) of + false -> + d_store(Key, T, E); + {true, UsedBy} -> + err("Host ~p port ~p already in use by FLU ~p", + [HostName, Port, UsedBy], T) + end; + error -> + err("Unknown host ~p", [HostName], T); + {ok, _} -> + err("Duplicate flu ~p", [Name], T) + end; +run_ast_cmd(switch_old_and_new, E) -> + switch_env_dict(E); +run_ast_cmd(Unknown, _E) -> + err("Unknown AST thingie", [], Unknown). + +make_ast_run_env() -> + {_KV_old=dict:new(), _KV_new=dict:new(), _IsNew=false}. + +env_host_exists(HostName, E) -> + Key = {kv, {host, HostName}}, + case d_find(Key, E) of + error -> + false; + {ok, _} -> + true + end. + +host_port_is_assigned(HostName, Port, {KV_old, KV_new, _}) -> + L = dict:to_list(KV_old) ++ dict:to_list(KV_new), + FLU_Ts = [V || {{kv, {flu, _}}, V} <- L], + case [V || {flu, _Nm, Host_, Port_, _Ps}=V <- FLU_Ts, + Host_ == HostName, Port_ == Port] of + [{flu, Name, _Host, _Port, _Ps}] -> + {true, Name}; + [] -> + false + end. + +d_find(Key, {KV_old, KV_new, IsNew}) -> + case dict:find(Key, KV_new) of + {ok, Val} when IsNew -> + Val; + _ -> + dict:find(Key, KV_old) + end. + +d_store(Key, Val, {KV_old, KV_new, false}) -> + {dict:store(Key, Val, KV_old), KV_new, false}; +d_store(Key, Val, {KV_old, KV_new, true}) -> + {KV_old, dict:store(Key, Val, KV_new), true}. + +switch_env_dict({KV_old, KV_new, false}) -> + {KV_old, KV_new, true}; +switch_env_dict({_, _, true}) -> + A = switch_old_and_new, + err("Duplicate ~p", [A], A). + +n(L) -> + lists:sort(L). + +err(Fmt, Args, AST) -> + throw({lists:flatten(io_lib:format(Fmt, Args)), AST}). + +%% We won't allow 'atom' style proplist members: too difficult to normalize. +%% Also, no duplicates, again because normalizing useful for checksums but +%% bad for order-based traditional proplists (first key wins). + is_proplisty(Props) -> is_list(Props) andalso lists:all(fun({_,_}) -> true; - (X) when is_atom(X) -> true; + %% nope: (X) when is_atom(X) -> true; (_) -> false - end, Props). + end, Props) andalso + begin + Ks = [K || {K,_V} <- Props], + lists:sort(Ks) == lists:usort(Ks) + end. is_stringy(L) -> is_list(L) andalso diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index 31c650b..6e44532 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -48,7 +48,7 @@ smoke_test2() -> {platform_etc_dir, Dir ++ "/etc"}, {not_used_pending, Dir ++ "/etc/pending"} ], - EnvKeys = [K || {K, V} <- Envs], + EnvKeys = [K || {K,_V} <- Envs], undefined = application:get_env(machi, yo), Cleanup = machi_flu1_test:get_env_vars(machi, EnvKeys ++ [yo]), [begin @@ -153,12 +153,19 @@ make_pending_config(Term) -> ast_tuple_syntax_test() -> T = fun(L) -> machi_lifecycle_mgr:check_ast_tuple_syntax(L) end, - {_Good,[]=_Bad} = - T([ {host, "localhost", []}, - {host, "localhost", [{client_interface, "1.2.3.4"}, - {admin_interface, "5.6.7.8"}]}, - {flu, "fx", "foohost", 4000, []}, - {chain, "cy", ["fx", "fy"], ["fz"], [foo,{bar,baz}]} ]), + Canon1 = [ {host, "localhost", []}, + {host, "localhost", [{client_interface, "1.2.3.4"}, + {admin_interface, "5.6.7.8"}]}, + {flu, "fx", "foohost", 4000, []}, + switch_old_and_new, + {chain, "cy", ["fx", "fy"], ["fz"], [{foo,"yay"},{bar,baz}]} ], + + {_Good,[]=_Bad} = T(Canon1), + Canon1_norm = machi_lifecycle_mgr:normalize_ast_tuple_syntax(Canon1), + true = (length(Canon1) == length(Canon1_norm)), + {Canon1_norm_b, []} = T(Canon1_norm), + true = (length(Canon1_norm) == length(Canon1_norm_b)), + {[],[_,_,_,_]} = T([ {host, 'localhost', []}, {host, 'localhost', yo}, @@ -172,10 +179,25 @@ ast_tuple_syntax_test() -> {flu, "fx", "foohost", 40009999, []}, {flu, "fx", "foohost", 4000, gack}, {flu, "fx", "foohost", 4000, [22]} ]), - {[],[_,_]} = + {[],[_,_,_]} = T([ {chain, 'cy', ["fx", "fy"], ["fz"], [foo,{bar,baz}]}, + yoloyolo, {chain, "cy", ["fx", 27], ["fz"], oops,arity,way,way,way,too,big,x} ]). +ast_run_test() -> + PortBase = 20300, + R1 = [ + {host, "localhost", "localhost", "localhost", []}, + switch_old_and_new, + {flu, "f1", "localhost", PortBase+0, []}, + {flu, "f2", "localhost", PortBase+1, []} + ], + {ok, X1} = machi_lifecycle_mgr:run_ast(R1), + Y1 = {lists:sort(dict:to_list(element(1, X1))), + lists:sort(dict:to_list(element(2, X1))), + element(3, X1)}, + io:format(user, "\nY1 ~p\n", [Y1]). + -endif. % !PULSE -endif. % TEST -- 2.45.2 From 6a5c590ad1f777b08bbfa0273f1fd095a880aae4 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 10 Dec 2015 23:05:08 +0900 Subject: [PATCH 56/77] WIP: AST change {chain,...} thingie --- src/machi_lifecycle_mgr.erl | 26 +++++++++++--------------- test/machi_lifecycle_mgr_test.erl | 6 +++--- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 3f39f82..c4a36e0 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -574,6 +574,8 @@ process_pending_chain2(File, CD, RemovedFLUs, ChainConfigAction, S) -> Dst2 = MyPreserveDir ++ "/" ++ FLU_str ++ ".data", lager:info("Stopped FLU ~w: rename ~s ~s\n", [FLU, Src2, Dst2]), + %% TODO: If EXDEV, then we should rename to + %% another dir on same device, but ... where? _ = file:rename(Src2, Dst2), ok end || FLU <- LocalRemovedFLUs] @@ -639,19 +641,15 @@ check_an_ast_tuple({host, Name, AdminI, ClientI, Props}) -> check_an_ast_tuple({flu, Name, HostName, Port, Props}) -> is_stringy(Name) andalso is_stringy(HostName) andalso is_porty(Port) andalso is_proplisty(Props); -check_an_ast_tuple({chain, Name, AddList, RemoveList, Props}) -> +check_an_ast_tuple({chain, Name, FullList, Props}) -> is_stringy(Name) andalso - lists:all(fun is_stringy/1, AddList) andalso - lists:all(fun is_stringy/1, RemoveList) andalso + lists:all(fun is_stringy/1, FullList) andalso is_proplisty(Props); -check_an_ast_tuple({chain, Name, CMode, AddList, Add_Witnesses, - RemoveList, Remove_Witnesses, Props}) -> +check_an_ast_tuple({chain, Name, CMode, FullList, Witnesses, Props}) -> is_stringy(Name) andalso (CMode == ap_mode orelse CMode == cp_mode) andalso - lists:all(fun is_stringy/1, AddList) andalso - lists:all(fun is_stringy/1, Add_Witnesses) andalso - lists:all(fun is_stringy/1, RemoveList) andalso - lists:all(fun is_stringy/1, Remove_Witnesses) andalso + lists:all(fun is_stringy/1, FullList) andalso + lists:all(fun is_stringy/1, Witnesses) andalso is_proplisty(Props); check_an_ast_tuple(switch_old_and_new) -> true; @@ -675,12 +673,10 @@ normalize_an_ast_tuple({host, Name, AdminI, ClientI, Props}) -> {host, Name, AdminI, ClientI, n(Props2)}; normalize_an_ast_tuple({flu, Name, HostName, Port, Props}) -> {flu, Name, HostName, Port, n(Props)}; -normalize_an_ast_tuple({chain, Name, AddList, RemoveList, Props}) -> - {chain, Name, ap_mode, n(AddList), [], n(RemoveList), [], n(Props)}; -normalize_an_ast_tuple({chain, Name, CMode, AddList, Add_Witnesses, - RemoveList, Remove_Witnesses, Props}) -> - {chain, Name, CMode, n(AddList), n(Add_Witnesses), - n(RemoveList), n(Remove_Witnesses), n(Props)}; +normalize_an_ast_tuple({chain, Name, FullList, Props}) -> + {chain, Name, ap_mode, n(FullList), [], n(Props)}; +normalize_an_ast_tuple({chain, Name, CMode, FullList, Witnesses, Props}) -> + {chain, Name, CMode, n(FullList), n(Witnesses), n(Props)}; normalize_an_ast_tuple(A=switch_old_and_new) -> A. diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index 6e44532..6b5074f 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -158,7 +158,7 @@ ast_tuple_syntax_test() -> {admin_interface, "5.6.7.8"}]}, {flu, "fx", "foohost", 4000, []}, switch_old_and_new, - {chain, "cy", ["fx", "fy"], ["fz"], [{foo,"yay"},{bar,baz}]} ], + {chain, "cy", ["fx", "fy"], [{foo,"yay"},{bar,baz}]} ], {_Good,[]=_Bad} = T(Canon1), Canon1_norm = machi_lifecycle_mgr:normalize_ast_tuple_syntax(Canon1), @@ -180,9 +180,9 @@ ast_tuple_syntax_test() -> {flu, "fx", "foohost", 4000, gack}, {flu, "fx", "foohost", 4000, [22]} ]), {[],[_,_,_]} = - T([ {chain, 'cy', ["fx", "fy"], ["fz"], [foo,{bar,baz}]}, + T([ {chain, 'cy', ["fx", "fy"], [foo,{bar,baz}]}, yoloyolo, - {chain, "cy", ["fx", 27], ["fz"], oops,arity,way,way,way,too,big,x} + {chain, "cy", ["fx", 27], oops,arity,way,way,way,too,big,x} ]). ast_run_test() -> -- 2.45.2 From 3ee3de1aaf49a4c6c9ad931ca3dba3e659b0c511 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 10 Dec 2015 23:44:27 +0900 Subject: [PATCH 57/77] WIP: end of day --- src/machi_lifecycle_mgr.erl | 52 ++++++++++++++++++++++++------- test/machi_lifecycle_mgr_test.erl | 6 ++-- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index c4a36e0..9ba698a 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -641,14 +641,14 @@ check_an_ast_tuple({host, Name, AdminI, ClientI, Props}) -> check_an_ast_tuple({flu, Name, HostName, Port, Props}) -> is_stringy(Name) andalso is_stringy(HostName) andalso is_porty(Port) andalso is_proplisty(Props); -check_an_ast_tuple({chain, Name, FullList, Props}) -> +check_an_ast_tuple({chain, Name, AllList, Props}) -> is_stringy(Name) andalso - lists:all(fun is_stringy/1, FullList) andalso + lists:all(fun is_stringy/1, AllList) andalso is_proplisty(Props); -check_an_ast_tuple({chain, Name, CMode, FullList, Witnesses, Props}) -> +check_an_ast_tuple({chain, Name, CMode, AllList, Witnesses, Props}) -> is_stringy(Name) andalso (CMode == ap_mode orelse CMode == cp_mode) andalso - lists:all(fun is_stringy/1, FullList) andalso + lists:all(fun is_stringy/1, AllList) andalso lists:all(fun is_stringy/1, Witnesses) andalso is_proplisty(Props); check_an_ast_tuple(switch_old_and_new) -> @@ -673,10 +673,10 @@ normalize_an_ast_tuple({host, Name, AdminI, ClientI, Props}) -> {host, Name, AdminI, ClientI, n(Props2)}; normalize_an_ast_tuple({flu, Name, HostName, Port, Props}) -> {flu, Name, HostName, Port, n(Props)}; -normalize_an_ast_tuple({chain, Name, FullList, Props}) -> - {chain, Name, ap_mode, n(FullList), [], n(Props)}; -normalize_an_ast_tuple({chain, Name, CMode, FullList, Witnesses, Props}) -> - {chain, Name, CMode, n(FullList), n(Witnesses), n(Props)}; +normalize_an_ast_tuple({chain, Name, AllList, Props}) -> + {chain, Name, ap_mode, n(AllList), [], n(Props)}; +normalize_an_ast_tuple({chain, Name, CMode, AllList, Witnesses, Props}) -> + {chain, Name, CMode, n(AllList), n(Witnesses), n(Props)}; normalize_an_ast_tuple(A=switch_old_and_new) -> A. @@ -699,14 +699,21 @@ run_ast_cmd({host, Name, _AdminI, _ClientI, _Props}=T, E) -> {ok, _} -> err("Duplicate host definition ~p: ~p", [Name], T) end; -run_ast_cmd({flu, Name, HostName, Port, _Props}=T, E) -> - Key = {kv, {flu, Name}}, +run_ast_cmd({flu, Name, HostName, Port, Props}=T, E) -> + Key = {kv, {flu, Name}}, + Key_tmp = {tmp, {flu, Name}}, HostExists_p = env_host_exists(HostName, E), case d_find(Key, E) of error when HostExists_p -> case host_port_is_assigned(HostName, Port, E) of false -> - d_store(Key, T, E); + {ok, ClientI} = get_host_client_interface(HostName, E), + Mod = proplists:get_value( + proto_mod, Props, 'machi_flu1_client'), + Val_tmp = #p_srvr{name=Name, proto_mod=Mod, + address=ClientI, port=Port, props=Props}, + d_store(Key, T, + d_store(Key_tmp, Val_tmp, E)); {true, UsedBy} -> err("Host ~p port ~p already in use by FLU ~p", [HostName, Port, UsedBy], T) @@ -716,11 +723,25 @@ run_ast_cmd({flu, Name, HostName, Port, _Props}=T, E) -> {ok, _} -> err("Duplicate flu ~p", [Name], T) end; +run_ast_cmd({chain, Name, _CMode, _AllList, _Witnesses, _Props}=T, E) -> + Key = {kv, {chain, Name}}, + case d_find(Key, E) of + error -> + run_ast_new_chain(T, E); + {ok, _} -> + run_ast_modify_chain(T, E) + end; run_ast_cmd(switch_old_and_new, E) -> switch_env_dict(E); run_ast_cmd(Unknown, _E) -> err("Unknown AST thingie", [], Unknown). +run_ast_new_chain({chain, Name, CMode, AllList, Witnesses, Props}=T, E) -> + err("YO", [], T). + +run_ast_modify_chain(T, _E) -> + err("TODO modify chain", [], T). + make_ast_run_env() -> {_KV_old=dict:new(), _KV_new=dict:new(), _IsNew=false}. @@ -733,6 +754,15 @@ env_host_exists(HostName, E) -> true end. +get_host_client_interface(HostName, E) -> + Key = {kv, {host, HostName}}, + case d_find(Key, E) of + error -> + false; + {ok, {host, _Name, _AdminI, ClientI, _Props}} -> + {ok, ClientI} + end. + host_port_is_assigned(HostName, Port, {KV_old, KV_new, _}) -> L = dict:to_list(KV_old) ++ dict:to_list(KV_new), FLU_Ts = [V || {{kv, {flu, _}}, V} <- L], diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index 6b5074f..9213d6f 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -189,9 +189,11 @@ ast_run_test() -> PortBase = 20300, R1 = [ {host, "localhost", "localhost", "localhost", []}, + {flu, "f0", "localhost", PortBase+0, []}, switch_old_and_new, - {flu, "f1", "localhost", PortBase+0, []}, - {flu, "f2", "localhost", PortBase+1, []} + {flu, "f1", "localhost", PortBase+1, []}, + {flu, "f2", "localhost", PortBase+2, []}, + {chain, "ca", ["f0", "f1", "f2"], []} ], {ok, X1} = machi_lifecycle_mgr:run_ast(R1), Y1 = {lists:sort(dict:to_list(element(1, X1))), -- 2.45.2 From 61eae1300fbf81f133c1c859381173bb72ea4294 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 11 Dec 2015 12:43:38 +0900 Subject: [PATCH 58/77] WIP: finish basic 'run', add negative tests --- src/machi_lifecycle_mgr.erl | 136 ++++++++++++++++++++++-------- test/machi_lifecycle_mgr_test.erl | 19 ++++- 2 files changed, 121 insertions(+), 34 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 9ba698a..b147cf5 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -681,27 +681,49 @@ normalize_an_ast_tuple(A=switch_old_and_new) -> A. run_ast(Ts) -> - {_, []} = check_ast_tuple_syntax(Ts), - Ts2 = normalize_ast_tuple_syntax(Ts), - Env1 = make_ast_run_env(), - try - Env2 = lists:foldl(fun run_ast_cmd/2, Env1, Ts2), - {ok, Env2} - catch throw:DbgStuff -> - {error, DbgStuff} + case check_ast_tuple_syntax(Ts) of + {_, []} -> + Ts2 = normalize_ast_tuple_syntax(Ts), + Env1 = make_ast_run_env(), + try + Env2 = lists:foldl(fun run_ast_cmd/2, Env1, Ts2), + {ok, Env2} + catch throw:DbgStuff -> + {error, DbgStuff} + end; + {_, Else} -> + {error, {bad_syntax, Else}} end. +%% Legend for env key naming scheme +%% +%% {kv, X} +%% Mutable: no. +%% Reference KV store for X. Variations of X are: +%% {host, Name} | {flu, Name} | {chain, Name} +%% +%% {tmp, X} +%% Mutable: yes. +%% Tmp scratch for X. Variations of X are: +%% {p_srvr, Name} +%% #p_srvr{} record for FLU Name, for cache/convenience purposes. +%% If a FLU has been defined via {kv,_}, this key must also exist. +%% +%% {flu_assigned_to, ChainName} +%% If a FLU is currently assigned to a chain, map to ChainName. +%% If a FLU is not currently assigned to a chain, key does not exist. + run_ast_cmd({host, Name, _AdminI, _ClientI, _Props}=T, E) -> - Key = {kv, {host, Name}}, + Key = {kv,{host,Name}}, case d_find(Key, E) of error -> d_store(Key, T, E); {ok, _} -> - err("Duplicate host definition ~p: ~p", [Name], T) + err("Duplicate host definition ~p", [Name], T) end; run_ast_cmd({flu, Name, HostName, Port, Props}=T, E) -> - Key = {kv, {flu, Name}}, - Key_tmp = {tmp, {flu, Name}}, + Key = {kv,{flu,Name}}, + Key_p = {kv,{p_srvr,Name}}, HostExists_p = env_host_exists(HostName, E), case d_find(Key, E) of error when HostExists_p -> @@ -710,10 +732,10 @@ run_ast_cmd({flu, Name, HostName, Port, Props}=T, E) -> {ok, ClientI} = get_host_client_interface(HostName, E), Mod = proplists:get_value( proto_mod, Props, 'machi_flu1_client'), - Val_tmp = #p_srvr{name=Name, proto_mod=Mod, - address=ClientI, port=Port, props=Props}, + Val_p = #p_srvr{name=Name, proto_mod=Mod, + address=ClientI, port=Port, props=Props}, d_store(Key, T, - d_store(Key_tmp, Val_tmp, E)); + d_store(Key_p, Val_p, E)); {true, UsedBy} -> err("Host ~p port ~p already in use by FLU ~p", [HostName, Port, UsedBy], T) @@ -723,30 +745,75 @@ run_ast_cmd({flu, Name, HostName, Port, Props}=T, E) -> {ok, _} -> err("Duplicate flu ~p", [Name], T) end; -run_ast_cmd({chain, Name, _CMode, _AllList, _Witnesses, _Props}=T, E) -> - Key = {kv, {chain, Name}}, +run_ast_cmd({chain, Name, CMode, AllList, Witnesses, _Props}=T, E) -> + Key = {kv,{chain,Name}}, + AllFLUs = AllList ++ Witnesses, + + %% All FLUs must exist. + case lists:sort(AllFLUs) == lists:usort(AllFLUs) of + true -> ok; + false -> err("Duplicate FLU(s) specified", [], T) + end, + MissingFLUs = [FLU || FLU <- AllFLUs, not env_flu_exists(FLU, E)], + case MissingFLUs of + [] -> ok; + [_|_] -> err("Undefined FLU(s) ~p", [MissingFLUs], T) + end, + + %% All FLUs must not be assigned to another chain. + AssignedFLUs = + [case d_find({tmp,{flu_assigned_to,FLU}}, E) of + error -> + []; + {ok, Ch} when Ch == Name -> + []; % It's assigned to me already + {ok, Ch} -> + [{flu, FLU, assigned_to_chain, Ch}] + end || FLU <- AllFLUs], + case lists:append(AssignedFLUs) of + [] -> ok; + As -> err("Some FLUs are assigned to other chains: ~p\n", [As], T) + end, + + %% If chain already exists, then the consistency mode cannot change. case d_find(Key, E) of error -> - run_ast_new_chain(T, E); - {ok, _} -> - run_ast_modify_chain(T, E) - end; + ok; + {ok, C_old} -> + {chain, _, OldCMode, _, _, _} = C_old, + if CMode == OldCMode -> ok; + true -> err("Invalid consistency mode: ~w -> ~w", + [OldCMode, CMode], T) + end + end, + + E2 = lists:foldl(fun(FLU, Env) -> + d_erase({tmp,{flu_assigned_to,FLU}}, Env) + end, E, AllFLUs), + E3 = lists:foldl(fun(FLU, Env) -> + d_store({tmp,{flu_assigned_to,FLU}}, Name, Env) + end, E2, AllFLUs), + + %% It's good, let's roll. + d_store(Key, T, E3); run_ast_cmd(switch_old_and_new, E) -> switch_env_dict(E); run_ast_cmd(Unknown, _E) -> err("Unknown AST thingie", [], Unknown). -run_ast_new_chain({chain, Name, CMode, AllList, Witnesses, Props}=T, E) -> - err("YO", [], T). - -run_ast_modify_chain(T, _E) -> - err("TODO modify chain", [], T). - make_ast_run_env() -> {_KV_old=dict:new(), _KV_new=dict:new(), _IsNew=false}. -env_host_exists(HostName, E) -> - Key = {kv, {host, HostName}}, +env_host_exists(Name, E) -> + Key = {kv,{host, Name}}, + case d_find(Key, E) of error -> + false; + {ok, _} -> + true + end. + +env_flu_exists(Name, E) -> + Key = {kv,{flu,Name}}, case d_find(Key, E) of error -> false; @@ -755,7 +822,7 @@ env_host_exists(HostName, E) -> end. get_host_client_interface(HostName, E) -> - Key = {kv, {host, HostName}}, + Key = {kv,{host,HostName}}, case d_find(Key, E) of error -> false; @@ -765,7 +832,7 @@ get_host_client_interface(HostName, E) -> host_port_is_assigned(HostName, Port, {KV_old, KV_new, _}) -> L = dict:to_list(KV_old) ++ dict:to_list(KV_new), - FLU_Ts = [V || {{kv, {flu, _}}, V} <- L], + FLU_Ts = [V || {{kv,{flu,_}}, V} <- L], case [V || {flu, _Nm, Host_, Port_, _Ps}=V <- FLU_Ts, Host_ == HostName, Port_ == Port] of [{flu, Name, _Host, _Port, _Ps}] -> @@ -776,8 +843,8 @@ host_port_is_assigned(HostName, Port, {KV_old, KV_new, _}) -> d_find(Key, {KV_old, KV_new, IsNew}) -> case dict:find(Key, KV_new) of - {ok, Val} when IsNew -> - Val; + {ok, _Val}=Res when IsNew -> + Res; _ -> dict:find(Key, KV_old) end. @@ -787,6 +854,9 @@ d_store(Key, Val, {KV_old, KV_new, false}) -> d_store(Key, Val, {KV_old, KV_new, true}) -> {KV_old, dict:store(Key, Val, KV_new), true}. +d_erase({tmp,_}=Key, {KV_old, KV_new, IsNew}) -> + {dict:erase(Key, KV_old), dict:erase(Key, KV_new), IsNew}. + switch_env_dict({KV_old, KV_new, false}) -> {KV_old, KV_new, true}; switch_env_dict({_, _, true}) -> diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index 9213d6f..0d172bf 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -195,11 +195,28 @@ ast_run_test() -> {flu, "f2", "localhost", PortBase+2, []}, {chain, "ca", ["f0", "f1", "f2"], []} ], + + {ok, X1} = machi_lifecycle_mgr:run_ast(R1), Y1 = {lists:sort(dict:to_list(element(1, X1))), lists:sort(dict:to_list(element(2, X1))), element(3, X1)}, - io:format(user, "\nY1 ~p\n", [Y1]). + io:format(user, "\nY1 ~p\n", [Y1]), + + Negative_after_R1 = + [ + {host, "localhost", "foo", "foo", []}, % dupe host + {flu, "f1", "other", PortBase+9999999, []}, % bogus port # (syntax) + {flu, "f1", "other", PortBase+888, []}, % dupe flu name + {flu, "f7", "localhost", PortBase+1, []}, % dupe host+port + {chain, "ca", ["f7"], []}, % unknown flu + {chain, "cb", ["f0"], []}, % flu previously assigned + {chain, "ca", cp_mode, ["f0", "f1", "f2"], [], []} % mode change + ], + [begin + io:format(user, "Neg ~p\n", [Neg]), + {error, _} = machi_lifecycle_mgr:run_ast(R1 ++ [Neg]) + end || Neg <- Negative_after_R1]. -endif. % !PULSE -endif. % TEST -- 2.45.2 From df8eea8c102440e883893111439f052a66a26b8c Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 11 Dec 2015 12:54:54 +0900 Subject: [PATCH 59/77] WIP: dict -> gb_trees, 1 of 2 --- src/machi_lifecycle_mgr.erl | 9 ++++++--- test/machi_lifecycle_mgr_test.erl | 17 +++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index b147cf5..234bce6 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -295,7 +295,7 @@ get_initial_chains() -> sanitize_chain_def_records(CDefs). sanitize_chain_def_records(Ps) -> - {Sane, _} = lists:foldl(fun sanitize_chain_def_rec/2, {[], dict:new()}, Ps), + {Sane, _} = lists:foldl(fun sanitize_chain_def_rec/2, {[], gb_trees:empty()}, Ps), Sane. sanitize_chain_def_rec(Whole, {Acc, D}) -> @@ -306,7 +306,7 @@ sanitize_chain_def_rec(Whole, {Acc, D}) -> witnesses=Witnesses} = Whole, {true, 10} = {is_atom(Name), 10}, NameK = {name, Name}, - {error, 20} = {dict:find(NameK, D), 20}, + {none, 20} = {gb_trees:lookup(NameK, D), 20}, {true, 30} = {(Mode == ap_mode orelse Mode == cp_mode), 30}, IsPSrvr = fun(X) when is_record(X, p_srvr) -> true; (_) -> false @@ -315,7 +315,7 @@ sanitize_chain_def_rec(Whole, {Acc, D}) -> {true, 50} = {lists:all(IsPSrvr, Witnesses), 50}, %% All is sane enough. - D2 = dict:store(NameK, Name, D), + D2 = gb_trees:enter(NameK, Name, D), {[Whole|Acc], D2} catch X:Y -> lager:error("~s: Bad chain_def record (~w ~w), skipping: ~P\n", @@ -892,3 +892,6 @@ is_stringy(L) -> is_porty(Port) -> is_integer(Port) andalso 1024 =< Port andalso Port =< 65535. + +diff_env({KV_old, KV_new, _IsNew}) -> + yo. diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index 0d172bf..6c5363e 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -196,11 +196,10 @@ ast_run_test() -> {chain, "ca", ["f0", "f1", "f2"], []} ], - - {ok, X1} = machi_lifecycle_mgr:run_ast(R1), - Y1 = {lists:sort(dict:to_list(element(1, X1))), - lists:sort(dict:to_list(element(2, X1))), - element(3, X1)}, + {ok, Env1} = machi_lifecycle_mgr:run_ast(R1), + Y1 = {lists:sort(dict:to_list(element(1, Env1))), + lists:sort(dict:to_list(element(2, Env1))), + element(3, Env1)}, io:format(user, "\nY1 ~p\n", [Y1]), Negative_after_R1 = @@ -216,7 +215,13 @@ ast_run_test() -> [begin io:format(user, "Neg ~p\n", [Neg]), {error, _} = machi_lifecycle_mgr:run_ast(R1 ++ [Neg]) - end || Neg <- Negative_after_R1]. + end || Neg <- Negative_after_R1], + + %% %% The 'run' phase doesn't blow smoke. What about 'diff'. + %% {ok, X2} = machi_lifecycle_mgr:diff_env(Env1), + %% io:format(user, "X2: ~p\n", [X2]), + + ok. -endif. % !PULSE -endif. % TEST -- 2.45.2 From 3826af8ee2a9823ce9fc5a440196a3c0580cf2dc Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 11 Dec 2015 13:17:33 +0900 Subject: [PATCH 60/77] WIP: dict -> gb_trees, 2 of 2 --- src/machi_lifecycle_mgr.erl | 29 ++++++++++++++++++----------- test/machi_lifecycle_mgr_test.erl | 4 ++-- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 234bce6..60d9418 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -802,11 +802,12 @@ run_ast_cmd(Unknown, _E) -> err("Unknown AST thingie", [], Unknown). make_ast_run_env() -> - {_KV_old=dict:new(), _KV_new=dict:new(), _IsNew=false}. + {_KV_old=gb_trees:empty(), _KV_new=gb_trees:empty(), _IsNew=false}. env_host_exists(Name, E) -> - Key = {kv,{host, Name}}, - case d_find(Key, E) of error -> + Key = {kv,{host,Name}}, + case d_find(Key, E) of + error -> false; {ok, _} -> true @@ -831,7 +832,7 @@ get_host_client_interface(HostName, E) -> end. host_port_is_assigned(HostName, Port, {KV_old, KV_new, _}) -> - L = dict:to_list(KV_old) ++ dict:to_list(KV_new), + L = gb_trees:to_list(KV_old) ++ gb_trees:to_list(KV_new), FLU_Ts = [V || {{kv,{flu,_}}, V} <- L], case [V || {flu, _Nm, Host_, Port_, _Ps}=V <- FLU_Ts, Host_ == HostName, Port_ == Port] of @@ -842,20 +843,26 @@ host_port_is_assigned(HostName, Port, {KV_old, KV_new, _}) -> end. d_find(Key, {KV_old, KV_new, IsNew}) -> - case dict:find(Key, KV_new) of - {ok, _Val}=Res when IsNew -> - Res; + %% Bah, use 'dict' return value convention. + case gb_trees:lookup(Key, KV_new) of + {value, Val} when IsNew -> + {ok, Val}; _ -> - dict:find(Key, KV_old) + case gb_trees:lookup(Key, KV_old) of + {value, Val} -> + {ok, Val}; + _ -> + error + end end. d_store(Key, Val, {KV_old, KV_new, false}) -> - {dict:store(Key, Val, KV_old), KV_new, false}; + {gb_trees:enter(Key, Val, KV_old), KV_new, false}; d_store(Key, Val, {KV_old, KV_new, true}) -> - {KV_old, dict:store(Key, Val, KV_new), true}. + {KV_old, gb_trees:enter(Key, Val, KV_new), true}. d_erase({tmp,_}=Key, {KV_old, KV_new, IsNew}) -> - {dict:erase(Key, KV_old), dict:erase(Key, KV_new), IsNew}. + {gb_trees:delete_any(Key, KV_old), gb_trees:delete_any(Key, KV_new), IsNew}. switch_env_dict({KV_old, KV_new, false}) -> {KV_old, KV_new, true}; diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index 6c5363e..9cd0509 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -197,8 +197,8 @@ ast_run_test() -> ], {ok, Env1} = machi_lifecycle_mgr:run_ast(R1), - Y1 = {lists:sort(dict:to_list(element(1, Env1))), - lists:sort(dict:to_list(element(2, Env1))), + Y1 = {lists:sort(gb_trees:to_list(element(1, Env1))), + lists:sort(gb_trees:to_list(element(2, Env1))), element(3, Env1)}, io:format(user, "\nY1 ~p\n", [Y1]), -- 2.45.2 From 1db232db1be852d4e8712b40a30908b6d76b0515 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 11 Dec 2015 15:33:31 +0900 Subject: [PATCH 61/77] WIP: diff in progress --- src/machi_lifecycle_mgr.erl | 68 ++++++++++++++++++++++++++++++- test/machi_lifecycle_mgr_test.erl | 18 +++++--- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 60d9418..516dc4f 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -856,6 +856,18 @@ d_find(Key, {KV_old, KV_new, IsNew}) -> end end. +d_get(Key, {KV_old, KV_new, IsNew}) -> + %% Bah, use 'dict' return value convention. + case gb_trees:lookup(Key, KV_new) of + {value, Val} when IsNew -> + Val; + _ -> + case gb_trees:lookup(Key, KV_old) of + {value, Val} -> + Val + end + end. + d_store(Key, Val, {KV_old, KV_new, false}) -> {gb_trees:enter(Key, Val, KV_old), KV_new, false}; d_store(Key, Val, {KV_old, KV_new, true}) -> @@ -900,5 +912,57 @@ is_stringy(L) -> is_porty(Port) -> is_integer(Port) andalso 1024 =< Port andalso Port =< 65535. -diff_env({KV_old, KV_new, _IsNew}) -> - yo. +diff_env({KV_old, KV_new, _IsNew}=E, RelativeHost) -> + Old_list = gb_trees:to_list(KV_old), + New_list = gb_trees:to_list(KV_new), + Keys_old = [K || {K,_V} <- Old_list], + Keys_new = [K || {K,_V} <- New_list], + Append = fun(Item) -> + fun(K, D) -> + L = gb_trees:get(K, D), + gb_trees:enter(K, L ++ [Item], D) + end + end, + + DiffD = lists:foldl(fun(K, D) -> + gb_trees:enter(K, [], D) + end, gb_trees:empty(), lists:usort(Keys_old++Keys_new)), + DiffD2 = lists:foldl(Append(old), DiffD, Keys_old), + DiffD3 = lists:foldl(Append(new), DiffD2, Keys_new), + %% DiffD3 values will be exactly one of: [old], [old,new], [new] + + put(final, []), + Add = fun(X) -> put(final, [X|get(final)]) end, + + %% Find all new FLUs and define them. + [begin + {flu, Name, Host, _Port, _Ps} = V, + if Host == RelativeHost; Host == any -> + {ok, P_srvr} = d_find({kv,{p_srvr,Name}}, E), + Add(P_srvr) + end + end || {{kv,{flu,Name}}, V} <- New_list], + + %% Find new chains on this host and define them. + %% Find modified chains on this host and re-define them. + [begin + {chain, Name, CMode, AllList, Witnesses, Props} = V, + FLUsA = [d_get({kv,{flu,FLU}}, E) || FLU <- AllList], + FLUsW = [d_get({kv,{flu,FLU}}, E) || FLU <- Witnesses], + TheFLU_Hosts = + [Host || {flu, _Name, Host, _Port, _Ps} <- FLUsA ++ FLUsW], + case (lists:member(RelativeHost, TheFLU_Hosts) + orelse RelativeHost == all) of + true -> + case gb_trees:lookup({kv,{chain,Name}}, KV_old) of + {value, OldT} -> + Add({yo,change,Name}); + none -> + Add({yo,new,Name}) + end; + false -> + ok + end + end || {{kv,{chain,Name}}, V} <- New_list], + + {DiffD3, lists:reverse(get(final))}. diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index 9cd0509..c6dc34e 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -190,10 +190,15 @@ ast_run_test() -> R1 = [ {host, "localhost", "localhost", "localhost", []}, {flu, "f0", "localhost", PortBase+0, []}, - switch_old_and_new, {flu, "f1", "localhost", PortBase+1, []}, + {chain, "ca", ["f0"], []}, + {chain, "cb", ["f1"], []}, + switch_old_and_new, {flu, "f2", "localhost", PortBase+2, []}, - {chain, "ca", ["f0", "f1", "f2"], []} + {flu, "f3", "localhost", PortBase+3, []}, + {flu, "f4", "localhost", PortBase+4, []}, + {chain, "ca", ["f0", "f2"], []}, + {chain, "cc", ["f3", "f4"], []} ], {ok, Env1} = machi_lifecycle_mgr:run_ast(R1), @@ -209,7 +214,7 @@ ast_run_test() -> {flu, "f1", "other", PortBase+888, []}, % dupe flu name {flu, "f7", "localhost", PortBase+1, []}, % dupe host+port {chain, "ca", ["f7"], []}, % unknown flu - {chain, "cb", ["f0"], []}, % flu previously assigned + {chain, "cc", ["f0"], []}, % flu previously assigned {chain, "ca", cp_mode, ["f0", "f1", "f2"], [], []} % mode change ], [begin @@ -217,9 +222,10 @@ ast_run_test() -> {error, _} = machi_lifecycle_mgr:run_ast(R1 ++ [Neg]) end || Neg <- Negative_after_R1], - %% %% The 'run' phase doesn't blow smoke. What about 'diff'. - %% {ok, X2} = machi_lifecycle_mgr:diff_env(Env1), - %% io:format(user, "X2: ~p\n", [X2]), + %% The 'run' phase doesn't blow smoke. What about 'diff'? + {X2a, X2b} = machi_lifecycle_mgr:diff_env(Env1, "localhost"), + io:format(user, "X2a: ~p\n", [gb_trees:to_list(X2a)]), + io:format(user, "X2b: ~p\n", [X2b]), ok. -- 2.45.2 From 6b7d871adab64e3823774bdad1d6b37963a2b5be Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 11 Dec 2015 16:26:13 +0900 Subject: [PATCH 62/77] WIP: diff in progress 2 --- include/machi_projection.hrl | 2 +- src/machi_lifecycle_mgr.erl | 93 +++++++++++++++++++------------ test/machi_lifecycle_mgr_test.erl | 4 +- 3 files changed, 61 insertions(+), 38 deletions(-) diff --git a/include/machi_projection.hrl b/include/machi_projection.hrl index 4181625..ce161e4 100644 --- a/include/machi_projection.hrl +++ b/include/machi_projection.hrl @@ -82,7 +82,7 @@ mode :: pv1_consistency_mode(), full = [] :: [p_srvr()], witnesses = [] :: [p_srvr()], - old_all = [] :: [pv1_server()], % guard against some races + old_full = [] :: [pv1_server()], % guard against some races old_witnesses=[] :: [pv1_server()], % guard against some races local_run = [] :: [pv1_server()], % must be tailored to each machine! local_stop = [] :: [pv1_server()], % must be tailored to each machine! diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 516dc4f..45d3967 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -355,7 +355,8 @@ bootstrap_chain2(CD, FLU, 0) -> failed; bootstrap_chain2(#chain_def_v1{name=NewChainName, mode=NewCMode, full=Full, witnesses=Witnesses, - old_all=ReqOldAll, old_witnesses=ReqOldWitnesses, + old_full=ReqOldFull, + old_witnesses=ReqOldWitnesses, props=_Props}=CD, FLU, N) -> All_p_srvrs = Witnesses ++ Full, @@ -371,7 +372,7 @@ bootstrap_chain2(#chain_def_v1{name=NewChainName, mode=NewCMode, all_members=OldAll_list, witnesses=OldWitnesses}} = machi_projection_store:read_latest_projection(PStore, private), case set_chain_members(OldChainName, NewChainName, OldCMode, - ReqOldAll, ReqOldWitnesses, + ReqOldFull, ReqOldWitnesses, OldAll_list, OldWitnesses, NewAll_list, Witnesses, Mgr, NewChainName, OldEpoch, @@ -396,18 +397,18 @@ bootstrap_chain2(#chain_def_v1{name=NewChainName, mode=NewCMode, end. set_chain_members(OldChainName, NewChainName, OldCMode, - ReqOldAll, ReqOldWitnesses, - OldAll_list, OldWitnesses, NewAll_list, NewWitnesses, + ReqOldFull, ReqOldWitnesses, + OldFull_list, OldWitnesses, NewAll_list, NewWitnesses, Mgr, ChainName, OldEpoch, NewCMode, MembersDict, _Props) -> if OldChainName == NewChainName, OldCMode == NewCMode, - OldAll_list == NewAll_list, OldWitnesses == NewWitnesses -> + OldFull_list == NewAll_list, OldWitnesses == NewWitnesses -> %% The chain's current definition at this FLU is already what we %% want. Let's pretend that we sent the command and that it was %% successful. ok; OldEpoch == 0 orelse (OldChainName == NewChainName andalso OldCMode == NewCMode andalso - ReqOldAll == OldAll_list andalso + ReqOldFull == OldFull_list andalso ReqOldWitnesses == OldWitnesses) -> %% The old epoch is 0 (no chain defined) or else the prerequisites %% for our chain change request are indeed matched by the FLU's @@ -641,14 +642,14 @@ check_an_ast_tuple({host, Name, AdminI, ClientI, Props}) -> check_an_ast_tuple({flu, Name, HostName, Port, Props}) -> is_stringy(Name) andalso is_stringy(HostName) andalso is_porty(Port) andalso is_proplisty(Props); -check_an_ast_tuple({chain, Name, AllList, Props}) -> +check_an_ast_tuple({chain, Name, FullList, Props}) -> is_stringy(Name) andalso - lists:all(fun is_stringy/1, AllList) andalso + lists:all(fun is_stringy/1, FullList) andalso is_proplisty(Props); -check_an_ast_tuple({chain, Name, CMode, AllList, Witnesses, Props}) -> +check_an_ast_tuple({chain, Name, CMode, FullList, Witnesses, Props}) -> is_stringy(Name) andalso (CMode == ap_mode orelse CMode == cp_mode) andalso - lists:all(fun is_stringy/1, AllList) andalso + lists:all(fun is_stringy/1, FullList) andalso lists:all(fun is_stringy/1, Witnesses) andalso is_proplisty(Props); check_an_ast_tuple(switch_old_and_new) -> @@ -673,10 +674,10 @@ normalize_an_ast_tuple({host, Name, AdminI, ClientI, Props}) -> {host, Name, AdminI, ClientI, n(Props2)}; normalize_an_ast_tuple({flu, Name, HostName, Port, Props}) -> {flu, Name, HostName, Port, n(Props)}; -normalize_an_ast_tuple({chain, Name, AllList, Props}) -> - {chain, Name, ap_mode, n(AllList), [], n(Props)}; -normalize_an_ast_tuple({chain, Name, CMode, AllList, Witnesses, Props}) -> - {chain, Name, CMode, n(AllList), n(Witnesses), n(Props)}; +normalize_an_ast_tuple({chain, Name, FullList, Props}) -> + {chain, Name, ap_mode, n(FullList), [], n(Props)}; +normalize_an_ast_tuple({chain, Name, CMode, FullList, Witnesses, Props}) -> + {chain, Name, CMode, n(FullList), n(Witnesses), n(Props)}; normalize_an_ast_tuple(A=switch_old_and_new) -> A. @@ -701,14 +702,16 @@ run_ast(Ts) -> %% Mutable: no. %% Reference KV store for X. Variations of X are: %% {host, Name} | {flu, Name} | {chain, Name} +%% Value is a {host,...} or {flu,...}, or {chain,...} AST tuple. %% -%% {tmp, X} -%% Mutable: yes. -%% Tmp scratch for X. Variations of X are: %% {p_srvr, Name} %% #p_srvr{} record for FLU Name, for cache/convenience purposes. %% If a FLU has been defined via {kv,_}, this key must also exist. %% +%% +%% {tmp, X} +%% Mutable: yes. +%% Tmp scratch for X. Variations of X are: %% {flu_assigned_to, ChainName} %% If a FLU is currently assigned to a chain, map to ChainName. %% If a FLU is not currently assigned to a chain, key does not exist. @@ -732,7 +735,7 @@ run_ast_cmd({flu, Name, HostName, Port, Props}=T, E) -> {ok, ClientI} = get_host_client_interface(HostName, E), Mod = proplists:get_value( proto_mod, Props, 'machi_flu1_client'), - Val_p = #p_srvr{name=Name, proto_mod=Mod, + Val_p = #p_srvr{name=list_to_atom(Name), proto_mod=Mod, address=ClientI, port=Port, props=Props}, d_store(Key, T, d_store(Key_p, Val_p, E)); @@ -745,9 +748,9 @@ run_ast_cmd({flu, Name, HostName, Port, Props}=T, E) -> {ok, _} -> err("Duplicate flu ~p", [Name], T) end; -run_ast_cmd({chain, Name, CMode, AllList, Witnesses, _Props}=T, E) -> +run_ast_cmd({chain, Name, CMode, FullList, Witnesses, _Props}=T, E) -> Key = {kv,{chain,Name}}, - AllFLUs = AllList ++ Witnesses, + AllFLUs = FullList ++ Witnesses, %% All FLUs must exist. case lists:sort(AllFLUs) == lists:usort(AllFLUs) of @@ -858,6 +861,7 @@ d_find(Key, {KV_old, KV_new, IsNew}) -> d_get(Key, {KV_old, KV_new, IsNew}) -> %% Bah, use 'dict' return value convention. + %% Assume key exists, fail if not found. case gb_trees:lookup(Key, KV_new) of {value, Val} when IsNew -> Val; @@ -924,12 +928,12 @@ diff_env({KV_old, KV_new, _IsNew}=E, RelativeHost) -> end end, - DiffD = lists:foldl(fun(K, D) -> - gb_trees:enter(K, [], D) - end, gb_trees:empty(), lists:usort(Keys_old++Keys_new)), - DiffD2 = lists:foldl(Append(old), DiffD, Keys_old), - DiffD3 = lists:foldl(Append(new), DiffD2, Keys_new), - %% DiffD3 values will be exactly one of: [old], [old,new], [new] + %% DiffD = lists:foldl(fun(K, D) -> + %% gb_trees:enter(K, [], D) + %% end, gb_trees:empty(), lists:usort(Keys_old++Keys_new)), + %% DiffD2 = lists:foldl(Append(old), DiffD, Keys_old), + %% DiffD3 = lists:foldl(Append(new), DiffD2, Keys_new), + %% %% DiffD3 values will be exactly one of: [old], [old,new], [new] put(final, []), Add = fun(X) -> put(final, [X|get(final)]) end, @@ -946,23 +950,42 @@ diff_env({KV_old, KV_new, _IsNew}=E, RelativeHost) -> %% Find new chains on this host and define them. %% Find modified chains on this host and re-define them. [begin - {chain, Name, CMode, AllList, Witnesses, Props} = V, - FLUsA = [d_get({kv,{flu,FLU}}, E) || FLU <- AllList], + {chain, Name, CMode, FullList, Witnesses, Props} = V, + FLUsF = [d_get({kv,{flu,FLU}}, E) || FLU <- FullList], FLUsW = [d_get({kv,{flu,FLU}}, E) || FLU <- Witnesses], - TheFLU_Hosts = - [Host || {flu, _Name, Host, _Port, _Ps} <- FLUsA ++ FLUsW], - case (lists:member(RelativeHost, TheFLU_Hosts) + TheFLU_Hosts = [{list_to_atom(FLU_str), Host} || + {flu, FLU_str, Host, _Port, _Ps} <- FLUsF ++ FLUsW], + case (lists:keymember(RelativeHost, 2, TheFLU_Hosts) orelse RelativeHost == all) of true -> + Ps_F = [d_get({kv,{p_srvr,FLU}}, E) || FLU <- FullList], + Ps_W = [d_get({kv,{p_srvr,FLU}}, E) || FLU <- Witnesses], + case gb_trees:lookup({kv,{chain,Name}}, KV_old) of {value, OldT} -> - Add({yo,change,Name}); + {chain, _, _, OldFull_ss, OldWitnesses_ss, _} = OldT, + OldFull = [Str || Str <- OldFull_ss], + OldWitnesses = [Str || Str <- OldWitnesses_ss], +io:format(user, "\nOldT: ~p\n", [OldT]), + Run = [FLU || {FLU, Hst} <- TheFLU_Hosts, + Hst == RelativeHost + orelse RelativeHost == all, + not lists:member(FLU, + OldFull++OldWitnesses)], + Stop = [yolo]; none -> - Add({yo,new,Name}) - end; + OldFull = [], + OldWitnesses = [], + Run = [derp,derp], + Stop = [] + end, + Add(#chain_def_v1{name=list_to_atom(Name), + mode=CMode, full=Ps_F, witnesses=Ps_W, + old_full=OldFull, old_witnesses=OldWitnesses, + local_run=Run, local_stop=Stop}); false -> ok end end || {{kv,{chain,Name}}, V} <- New_list], - {DiffD3, lists:reverse(get(final))}. + {gb_trees:empty(), lists:reverse(get(final))}. diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index c6dc34e..fd5d8bb 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -107,7 +107,7 @@ smoke_test2() -> io:format("\nSTEP: Reset chain = [b,c]\n", []), C2 = #chain_def_v1{name=cx, mode=ap_mode, full=[Pb,Pc], - old_all=[a,b,c], old_witnesses=[], + old_full=[a,b,c], old_witnesses=[], local_stop=[a], local_run=[b,c]}, make_pending_config(C2), {[],[_]} = machi_lifecycle_mgr:process_pending(), @@ -123,7 +123,7 @@ smoke_test2() -> io:format("\nSTEP: Reset chain = []\n", []), C3 = #chain_def_v1{name=cx, mode=ap_mode, full=[], - old_all=[b,c], old_witnesses=[], + old_full=[b,c], old_witnesses=[], local_stop=[b,c], local_run=[]}, make_pending_config(C3), {[],[_]} = machi_lifecycle_mgr:process_pending(), -- 2.45.2 From 009bad230fd67d06f493ed61491007cdd6f8ffad Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 11 Dec 2015 16:36:18 +0900 Subject: [PATCH 63/77] WIP: change internal types for most strings -> atom to match chmgr internal use --- src/machi_lifecycle_mgr.erl | 43 +++++++++++++++-------------- test/machi_lifecycle_mgr_test.erl | 46 +++++++++++++++---------------- 2 files changed, 45 insertions(+), 44 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 45d3967..54ccf42 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -640,17 +640,17 @@ check_an_ast_tuple({host, Name, AdminI, ClientI, Props}) -> (_) -> false end, Props); check_an_ast_tuple({flu, Name, HostName, Port, Props}) -> - is_stringy(Name) andalso is_stringy(HostName) andalso + is_atom(Name) andalso is_stringy(HostName) andalso is_porty(Port) andalso is_proplisty(Props); check_an_ast_tuple({chain, Name, FullList, Props}) -> - is_stringy(Name) andalso - lists:all(fun is_stringy/1, FullList) andalso + is_atom(Name) andalso + lists:all(fun erlang:is_atom/1, FullList) andalso is_proplisty(Props); check_an_ast_tuple({chain, Name, CMode, FullList, Witnesses, Props}) -> - is_stringy(Name) andalso + is_atom(Name) andalso (CMode == ap_mode orelse CMode == cp_mode) andalso - lists:all(fun is_stringy/1, FullList) andalso - lists:all(fun is_stringy/1, Witnesses) andalso + lists:all(fun erlang:is_atom/1, FullList) andalso + lists:all(fun erlang:is_atom/1, Witnesses) andalso is_proplisty(Props); check_an_ast_tuple(switch_old_and_new) -> true; @@ -735,7 +735,7 @@ run_ast_cmd({flu, Name, HostName, Port, Props}=T, E) -> {ok, ClientI} = get_host_client_interface(HostName, E), Mod = proplists:get_value( proto_mod, Props, 'machi_flu1_client'), - Val_p = #p_srvr{name=list_to_atom(Name), proto_mod=Mod, + Val_p = #p_srvr{name=Name, proto_mod=Mod, address=ClientI, port=Port, props=Props}, d_store(Key, T, d_store(Key_p, Val_p, E)); @@ -953,8 +953,8 @@ diff_env({KV_old, KV_new, _IsNew}=E, RelativeHost) -> {chain, Name, CMode, FullList, Witnesses, Props} = V, FLUsF = [d_get({kv,{flu,FLU}}, E) || FLU <- FullList], FLUsW = [d_get({kv,{flu,FLU}}, E) || FLU <- Witnesses], - TheFLU_Hosts = [{list_to_atom(FLU_str), Host} || - {flu, FLU_str, Host, _Port, _Ps} <- FLUsF ++ FLUsW], + TheFLU_Hosts = + [{FLU, Host} || {flu, FLU, Host, _Port, _Ps} <- FLUsF ++ FLUsW], case (lists:keymember(RelativeHost, 2, TheFLU_Hosts) orelse RelativeHost == all) of true -> @@ -965,21 +965,22 @@ diff_env({KV_old, KV_new, _IsNew}=E, RelativeHost) -> {value, OldT} -> {chain, _, _, OldFull_ss, OldWitnesses_ss, _} = OldT, OldFull = [Str || Str <- OldFull_ss], - OldWitnesses = [Str || Str <- OldWitnesses_ss], -io:format(user, "\nOldT: ~p\n", [OldT]), - Run = [FLU || {FLU, Hst} <- TheFLU_Hosts, - Hst == RelativeHost - orelse RelativeHost == all, - not lists:member(FLU, - OldFull++OldWitnesses)], - Stop = [yolo]; + OldWitnesses = [Str || Str <- OldWitnesses_ss]; none -> OldFull = [], - OldWitnesses = [], - Run = [derp,derp], - Stop = [] + OldWitnesses = [] end, - Add(#chain_def_v1{name=list_to_atom(Name), + Run = [FLU || {FLU, Hst} <- TheFLU_Hosts, + Hst == RelativeHost + orelse RelativeHost == all, + not lists:member(FLU, + OldFull++OldWitnesses)], + Stop = [FLU || {FLU, Hst} <- TheFLU_Hosts, + Hst == RelativeHost + orelse RelativeHost == all, + lists:member(FLU, + OldFull++OldWitnesses)], + Add(#chain_def_v1{name=Name, mode=CMode, full=Ps_F, witnesses=Ps_W, old_full=OldFull, old_witnesses=OldWitnesses, local_run=Run, local_stop=Stop}); diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index fd5d8bb..9fda3a0 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -156,9 +156,9 @@ ast_tuple_syntax_test() -> Canon1 = [ {host, "localhost", []}, {host, "localhost", [{client_interface, "1.2.3.4"}, {admin_interface, "5.6.7.8"}]}, - {flu, "fx", "foohost", 4000, []}, + {flu, 'fx', "foohost", 4000, []}, switch_old_and_new, - {chain, "cy", ["fx", "fy"], [{foo,"yay"},{bar,baz}]} ], + {chain, 'cy', ['fx', 'fy'], [{foo,"yay"},{bar,baz}]} ], {_Good,[]=_Bad} = T(Canon1), Canon1_norm = machi_lifecycle_mgr:normalize_ast_tuple_syntax(Canon1), @@ -173,12 +173,12 @@ ast_tuple_syntax_test() -> {host, "localhost", [{client_interface, "1.2.3.4"}, {bummer, "5.6.7.8"}]} ]), {[],[_,_,_,_,_,_]} = - T([ {flu, 'fx', "foohost", 4000, []}, - {flu, "fx", <<"foohost">>, 4000, []}, - {flu, "fx", "foohost", -4000, []}, - {flu, "fx", "foohost", 40009999, []}, - {flu, "fx", "foohost", 4000, gack}, - {flu, "fx", "foohost", 4000, [22]} ]), + T([ {flu, 'fx', 'foohost', 4000, []}, + {flu, 'fx', <<"foohost">>, 4000, []}, + {flu, 'fx', "foohost", -4000, []}, + {flu, 'fx', "foohost", 40009999, []}, + {flu, 'fx', "foohost", 4000, gack}, + {flu, 'fx', "foohost", 4000, [22]} ]), {[],[_,_,_]} = T([ {chain, 'cy', ["fx", "fy"], [foo,{bar,baz}]}, yoloyolo, @@ -189,16 +189,16 @@ ast_run_test() -> PortBase = 20300, R1 = [ {host, "localhost", "localhost", "localhost", []}, - {flu, "f0", "localhost", PortBase+0, []}, - {flu, "f1", "localhost", PortBase+1, []}, - {chain, "ca", ["f0"], []}, - {chain, "cb", ["f1"], []}, + {flu, 'f0', "localhost", PortBase+0, []}, + {flu, 'f1', "localhost", PortBase+1, []}, + {chain, 'ca', ['f0'], []}, + {chain, 'cb', ['f1'], []}, switch_old_and_new, - {flu, "f2", "localhost", PortBase+2, []}, - {flu, "f3", "localhost", PortBase+3, []}, - {flu, "f4", "localhost", PortBase+4, []}, - {chain, "ca", ["f0", "f2"], []}, - {chain, "cc", ["f3", "f4"], []} + {flu, 'f2', "localhost", PortBase+2, []}, + {flu, 'f3', "localhost", PortBase+3, []}, + {flu, 'f4', "localhost", PortBase+4, []}, + {chain, 'ca', ['f0', 'f2'], []}, + {chain, 'cc', ['f3', 'f4'], []} ], {ok, Env1} = machi_lifecycle_mgr:run_ast(R1), @@ -210,12 +210,12 @@ ast_run_test() -> Negative_after_R1 = [ {host, "localhost", "foo", "foo", []}, % dupe host - {flu, "f1", "other", PortBase+9999999, []}, % bogus port # (syntax) - {flu, "f1", "other", PortBase+888, []}, % dupe flu name - {flu, "f7", "localhost", PortBase+1, []}, % dupe host+port - {chain, "ca", ["f7"], []}, % unknown flu - {chain, "cc", ["f0"], []}, % flu previously assigned - {chain, "ca", cp_mode, ["f0", "f1", "f2"], [], []} % mode change + {flu, 'f1', "other", PortBase+9999999, []}, % bogus port # (syntax) + {flu, 'f1', "other", PortBase+888, []}, % dupe flu name + {flu, 'f7', "localhost", PortBase+1, []}, % dupe host+port + {chain, 'ca', ['f7'], []}, % unknown flu + {chain, 'cc', ['f0'], []}, % flu previously assigned + {chain, 'ca', cp_mode, ['f0', 'f1', 'f2'], [], []} % mode change ], [begin io:format(user, "Neg ~p\n", [Neg]), -- 2.45.2 From e55115fdba4fa9c70788285eac2b59cbbcf01284 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 11 Dec 2015 17:28:27 +0900 Subject: [PATCH 64/77] All EUnit tests in machi_lifecycle_mgr_test pass! --- src/machi_lifecycle_mgr.erl | 33 +++++++++++++++++++++--------- test/machi_lifecycle_mgr_test.erl | 34 +++++++++++++++++++++++-------- 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 54ccf42..f18f199 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -941,9 +941,11 @@ diff_env({KV_old, KV_new, _IsNew}=E, RelativeHost) -> %% Find all new FLUs and define them. [begin {flu, Name, Host, _Port, _Ps} = V, - if Host == RelativeHost; Host == any -> + if Host == RelativeHost orelse RelativeHost == all -> {ok, P_srvr} = d_find({kv,{p_srvr,Name}}, E), - Add(P_srvr) + Add(P_srvr); + true -> + ok end end || {{kv,{flu,Name}}, V} <- New_list], @@ -975,18 +977,29 @@ diff_env({KV_old, KV_new, _IsNew}=E, RelativeHost) -> orelse RelativeHost == all, not lists:member(FLU, OldFull++OldWitnesses)], - Stop = [FLU || {FLU, Hst} <- TheFLU_Hosts, - Hst == RelativeHost - orelse RelativeHost == all, - lists:member(FLU, - OldFull++OldWitnesses)], - Add(#chain_def_v1{name=Name, + %% Gaaah, need to find the host for FLUs not present + %% in FLUsF ++ FLUsW. + OldFLUsF = [d_get({kv,{flu,FLU}}, E) || FLU <- OldFull], + OldFLUsW = [d_get({kv,{flu,FLU}}, E) || FLU <- OldWitnesses], + OldTheFLU_Hosts = + [{FLU, Host} || {flu, FLU, Host, _Port, _Ps} <- OldFLUsF ++ OldFLUsW], + %% Yay, now we have the info we need for local FLU Stop list. + Stop = [FLU || FLU <- OldFull++OldWitnesses, + not (lists:member(FLU, FullList) + orelse + lists:member(FLU, Witnesses)), + lists:member({FLU, RelativeHost}, OldTheFLU_Hosts) + orelse RelativeHost == all], + PropsExtra = [], + %% PropsExtra = [{auto_gen,true}], + Add(#chain_def_v1{name=Name, mode=CMode, full=Ps_F, witnesses=Ps_W, old_full=OldFull, old_witnesses=OldWitnesses, - local_run=Run, local_stop=Stop}); + local_run=Run, local_stop=Stop, + props=Props ++ PropsExtra}); false -> ok end end || {{kv,{chain,Name}}, V} <- New_list], - {gb_trees:empty(), lists:reverse(get(final))}. + {x, lists:reverse(get(final))}. diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index 9fda3a0..7efb937 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -202,10 +202,11 @@ ast_run_test() -> ], {ok, Env1} = machi_lifecycle_mgr:run_ast(R1), - Y1 = {lists:sort(gb_trees:to_list(element(1, Env1))), - lists:sort(gb_trees:to_list(element(2, Env1))), - element(3, Env1)}, - io:format(user, "\nY1 ~p\n", [Y1]), + %% Uncomment to examine the Env trees. + %% Y1 = {lists:sort(gb_trees:to_list(element(1, Env1))), + %% lists:sort(gb_trees:to_list(element(2, Env1))), + %% element(3, Env1)}, + %% io:format(user, "\nY1 ~p\n", [Y1]), Negative_after_R1 = [ @@ -218,14 +219,31 @@ ast_run_test() -> {chain, 'ca', cp_mode, ['f0', 'f1', 'f2'], [], []} % mode change ], [begin - io:format(user, "Neg ~p\n", [Neg]), + %% io:format(user, "dbg: Neg ~p\n", [Neg]), {error, _} = machi_lifecycle_mgr:run_ast(R1 ++ [Neg]) end || Neg <- Negative_after_R1], %% The 'run' phase doesn't blow smoke. What about 'diff'? - {X2a, X2b} = machi_lifecycle_mgr:diff_env(Env1, "localhost"), - io:format(user, "X2a: ~p\n", [gb_trees:to_list(X2a)]), - io:format(user, "X2b: ~p\n", [X2b]), + {X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, "localhost"), + %% There's only one host, "localhost", so 'all' should be exactly equal. + {X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, all), + %% io:format(user, "X1b: ~p\n", [X1b]), + + %% Append to the R1 scenario: for chain cc: add f5, remove f4 + %% Expect: see pattern matching below on X2b. + R2 = (R1 -- [switch_old_and_new]) ++ + [switch_old_and_new, + {flu, 'f5', "localhost", PortBase+5, []}, + {chain, 'cc', ['f3','f5'], []}], + {ok, Env2} = machi_lifecycle_mgr:run_ast(R2), + {X2a, X2b} = machi_lifecycle_mgr:diff_env(Env2, "localhost"), + %% io:format(user, "X2b: ~p\n", [X2b]), + F5_port = PortBase+5, + [#p_srvr{name='f5',address="localhost",port=F5_port}, + #chain_def_v1{name='cc', + full=[#p_srvr{name='f3'},#p_srvr{name='f5'}], witnesses=[], + old_full=[f3,f4], old_witnesses=[], + local_run=[f5], local_stop=[f4]}] = X2b, ok. -- 2.45.2 From 6f077fbb622e9210041e6cc6b30619093787447d Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 11 Dec 2015 18:57:57 +0900 Subject: [PATCH 65/77] New machi_lifecycle_mgr_test, AST spec -> running FLUs & chains works! --- src/machi_projection_store.erl | 3 +- test/machi_lifecycle_mgr_test.erl | 92 ++++++++++++++++++++++++++----- 2 files changed, 79 insertions(+), 16 deletions(-) diff --git a/src/machi_projection_store.erl b/src/machi_projection_store.erl index ba50544..d1d36d9 100644 --- a/src/machi_projection_store.erl +++ b/src/machi_projection_store.erl @@ -321,7 +321,7 @@ do_proj_write3(ProjType, #projection_v1{epoch_number=Epoch, end. do_proj_write4(ProjType, Proj, Path, Epoch, #state{consistency_mode=CMode}=S) -> - {ok, FH} = file:open(Path, [write, raw, binary]), + {{ok, FH}, Epoch, Path} = {file:open(Path, [write, raw, binary]), Epoch, Path}, ok = file:write(FH, term_to_binary(Proj)), ok = file:sync(FH), ok = file:close(FH), @@ -387,7 +387,6 @@ wait_for_liveness(PidSpec, StartTime, WaitTime) -> undefined -> case timer:now_diff(os:timestamp(), StartTime) div 1000 of X when X < WaitTime -> - io:format(user, "\nYOO ~p ~p\n", [PidSpec, lists:sort(registered())]), timer:sleep(1), wait_for_liveness(PidSpec, StartTime, WaitTime) end; diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index 7efb937..38f5132 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -31,10 +31,7 @@ -define(MGR, machi_chain_manager1). -smoke_test_() -> - {timeout, 120, fun() -> smoke_test2() end}. - -smoke_test2() -> +setup() -> catch application:stop(machi), {ok, SupPid} = machi_sup:start_link(), error_logger:tty(false), @@ -55,6 +52,24 @@ smoke_test2() -> filelib:ensure_dir(V ++ "/unused"), application:set_env(machi, K, V) end || {K, V} <- Envs], + {SupPid, Dir, Cleanup}. + +cleanup({SupPid, Dir, Cleanup}) -> + exit(SupPid, normal), + machi_util:wait_for_death(SupPid, 100), + error_logger:tty(true), + catch application:stop(machi), + machi_flu1_test:clean_up_data_dir(Dir ++ "/*/*"), + machi_flu1_test:clean_up_data_dir(Dir), + machi_flu1_test:clean_up_env_vars(Cleanup), + undefined = application:get_env(machi, yo), + ok. + +smoke_test_() -> + {timeout, 60, fun() -> smoke_test2() end}. + +smoke_test2() -> + YoCleanup = setup(), try Prefix = <<"pre">>, Chunk1 = <<"yochunk">>, @@ -135,20 +150,15 @@ smoke_test2() -> ok after - exit(SupPid, normal), - machi_util:wait_for_death(SupPid, 100), - error_logger:tty(true), - catch application:stop(machi), - %% machi_flu1_test:clean_up_data_dir(Dir ++ "/*/*"), - %% machi_flu1_test:clean_up_data_dir(Dir), - machi_flu1_test:clean_up_env_vars(Cleanup), - undefined = application:get_env(machi, yo) + cleanup(YoCleanup) end. make_pending_config(Term) -> Dir = machi_lifecycle_mgr:get_pending_dir(x), Blob = io_lib:format("~w.\n", [Term]), - ok = file:write_file(Dir ++ "/" ++ lists:flatten(io_lib:format("~w,~w,~w", tuple_to_list(os:timestamp()))), + {A,B,C} = os:timestamp(), + ok = file:write_file(Dir ++ "/" ++ + lists:flatten(io_lib:format("~w.~6..0w",[A*1000000+B,C])), Blob). ast_tuple_syntax_test() -> @@ -236,7 +246,7 @@ ast_run_test() -> {flu, 'f5', "localhost", PortBase+5, []}, {chain, 'cc', ['f3','f5'], []}], {ok, Env2} = machi_lifecycle_mgr:run_ast(R2), - {X2a, X2b} = machi_lifecycle_mgr:diff_env(Env2, "localhost"), + {_X2a, X2b} = machi_lifecycle_mgr:diff_env(Env2, "localhost"), %% io:format(user, "X2b: ~p\n", [X2b]), F5_port = PortBase+5, [#p_srvr{name='f5',address="localhost",port=F5_port}, @@ -247,5 +257,59 @@ ast_run_test() -> ok. +ast_then_apply_test_() -> + {timeout, 60, fun() -> ast_then_apply_test2() end}. + +ast_then_apply_test2() -> + YoCleanup = setup(), + try + PortBase = 20400, + NumChains = 4, + ChainLen = 3, + FLU_num = NumChains * ChainLen, + FLU_defs = [{flu, list_to_atom("f"++integer_to_list(X)), + "localhost", PortBase+X, []} || X <- lists:seq(1,FLU_num)], + FLU_names = [FLU || {flu,FLU,_,_,_} <- FLU_defs], + Ch_defs = [{chain, list_to_atom("c"++integer_to_list(X)), + lists:sublist(FLU_names, X, 3), + []} || X <- lists:seq(1, FLU_num, 3)], + + R1 = [switch_old_and_new, + {host, "localhost", "localhost", "localhost", []}] + ++ FLU_defs ++ Ch_defs, + {ok, Env1} = machi_lifecycle_mgr:run_ast(R1), + {_X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, "localhost"), + %% io:format(user, "X1b ~p\n", [X1b]), + [make_pending_config(X) || X <- X1b], + {PassFLUs, PassChains} = machi_lifecycle_mgr:process_pending(), + true = (length(PassFLUs) == length(FLU_defs)), + true = (length(PassChains) == length(Ch_defs)), + + %% Kick the chain managers into doing something useful right now. + Pstores = [list_to_atom(atom_to_list(X) ++ "_pstore") || X <- FLU_names], + Fits = [list_to_atom(atom_to_list(X) ++ "_fitness") || X <- FLU_names], + ChMgrs = [list_to_atom(atom_to_list(X) ++ "_chmgr") || X <- FLU_names], + Advance = machi_chain_manager1_test:make_advance_fun( + Fits, FLU_names, ChMgrs, 3), + Advance(), + + %% Sanity check: everyone is configured properly. + [begin + {ok, #projection_v1{epoch_number=Epoch, all_members=All, + chain_name=ChainName, upi=UPI}} = + machi_projection_store:read_latest_projection(PStore, private), + %% io:format(user, "~p: epoch ~p all ~p\n", [PStore, Epoch, All]), + true = Epoch > 0, + ChainLen = length(All), + true = (length(UPI) > 0), + {chain, _, Full, []} = lists:keyfind(ChainName, 2, Ch_defs), + true = lists:sort(Full) == lists:sort(All) + end || PStore <- Pstores], + + ok + after + cleanup(YoCleanup) + end. + -endif. % !PULSE -endif. % TEST -- 2.45.2 From e4a784d3dd5e1c9120ec6ea009d8bce64ea1a52d Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Mon, 14 Dec 2015 19:14:56 +0900 Subject: [PATCH 66/77] Part 1 of X --- README.md | 69 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index d802080..80850ba 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,46 @@ -# Machi +# Machi: a robust & reliable, distributed, highly available, large file store [Travis-CI](http://travis-ci.org/basho/machi) :: ![Travis-CI](https://secure.travis-ci.org/basho/machi.png) -Our goal is a robust & reliable, distributed, highly available(*), -large file store based upon write-once registers, append-only files, -Chain Replication, and client-server style architecture. All members -of the cluster store all of the files. Distributed load -balancing/sharding of files is __outside__ of the scope of this -system. However, it is a high priority that this system be able to -integrate easily into systems that do provide distributed load -balancing, e.g., Riak Core. Although strong consistency is a major -feature of Chain Replication, first use cases will focus mainly on -eventual consistency features --- strong consistency design will be -discussed in a separate design document (read more below). +Our goal is a robust & reliable, distributed, highly available, large +file store. Such stores already exist, both in the open source world +and in the commercial world. Why reinvent the wheel? We believe +there are three reasons, ordered by decreasing rarity. -The ability for Machi to maintain strong consistency will make it -attractive as a toolkit for building things like CORFU and Tango as -well as better-known open source software such as Kafka's file -replication. (See the bibliography of the [Machi high level design -doc](./doc/high-level-machi.pdf) for further references.) +1. We want end-to-end checksums for all file data, from the initial + file writer to every file reader, anywhere, all the time. +2. We need flexibility to trade consistency for availability: + e.g. weak consistency in exchange for being available in cases + of partial system failure. +3. We want to manage file replicas in a way that's provably correct + and also easy to test. - (*) When operating in strong consistency mode (supporting - sequential or linearizable semantics), the availability of the - system is restricted to quorum majority availability. When in - eventual consistency mode, service can be provided by any - available server. +Of all the file stores in the open source & commercial worlds, only +criteria #3 is a viable option. Or so we hope. Or we just don't +care, and if data gets lost or corrupted, then ... so be it. -## Status: mid-October 2015: work is underway +If we have app use cases where availability is more important than +consistency, then systems that meet criteria #2 are also rare. +Most file stores provide only strong consistency and therefore +have unavoidable, unavailable behavior when parts of the system +fail. +What if we want a file store that is always available to write new +file data and attempts best-effort file reads? + +If we really do care about data loss and/or data corruption, then we +really want both #3 and #1. Unfortunately, systems that meet criteria +#1 are *very* +rare. Why? This is 2015. We have decades of research that shows +that computer hardware can (and +indeed does) corrupt data at nearly every level of the modern +client/server application stack. Systems with end-to-end data +corruption detection should be ubiquitous today. Alas, they are not. +Machi is an effort to change the deplorable state of the world, one +Erlang function at a time. + +## Status: mid-December 2015: work is underway + +TODO: status update here. * The chain manager is ready for both eventual consistency use ("available mode") and strong consistency use ("consistent mode"). Both modes use a new @@ -53,9 +67,18 @@ If you'd like to work on a protocol such as Thrift, UBF, msgpack over UDP, or some other protocol, let us know by [opening an issue to discuss it](./issues/new). +## Where to learn more about Machi + The two major design documents for Machi are now mostly stable. Please see the [doc](./doc) directory's [README](./doc) for details. +Scott recently (November 2015) gave a presentation at the +[RICON 2015 conference](http://ricon.io) about one of the techniques +used by Machi; "Managing Chain Replication Metadata with +Humming Consensus" is available online now. +* [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf) +* [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q) + ## Contributing to Machi: source code, documentation, etc. Basho Technologies, Inc. as committed to licensing all work for Machi -- 2.45.2 From d196dbcee5c1e24c1929add648814dd4c76990a3 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Mon, 14 Dec 2015 19:21:20 +0900 Subject: [PATCH 67/77] MD fixup --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 80850ba..6a8a64d 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,8 @@ What if we want a file store that is always available to write new file data and attempts best-effort file reads? If we really do care about data loss and/or data corruption, then we -really want both #3 and #1. Unfortunately, systems that meet criteria -#1 are *very* +really want both #3 and #1. Unfortunately, systems that meet +criteria #1 are *very* rare. Why? This is 2015. We have decades of research that shows that computer hardware can (and indeed does) corrupt data at nearly every level of the modern -- 2.45.2 From e5f7f3ba9ad62d36f3fa566122b1b0930b7c9b88 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 15 Dec 2015 15:07:18 +0900 Subject: [PATCH 68/77] Remove doc/overview.edoc --- doc/overview.edoc | 170 ---------------------------------------------- 1 file changed, 170 deletions(-) delete mode 100644 doc/overview.edoc diff --git a/doc/overview.edoc b/doc/overview.edoc deleted file mode 100644 index 6182f6b..0000000 --- a/doc/overview.edoc +++ /dev/null @@ -1,170 +0,0 @@ - -@title Machi: a small village of replicated files - -@doc - -== About This EDoc Documentation == - -This EDoc-style documentation will concern itself only with Erlang -function APIs and function & data types. Higher-level design and -commentary will remain outside of the Erlang EDoc system; please see -the "Pointers to Other Machi Documentation" section below for more -details. - -Readers should beware that this documentation may be out-of-sync with -the source code. When in doubt, use the `make edoc' command to -regenerate all HTML pages. - -It is the developer's responsibility to re-generate the documentation -periodically and commit it to the Git repo. - -== Machi Code Overview == - -=== Chain Manager === - -The Chain Manager is responsible for managing the state of Machi's -"Chain Replication" state. This role is roughly analogous to the -"Riak Core" application inside of Riak, which takes care of -coordinating replica placement and replica repair. - -For each primitive data server in the cluster, a Machi FLU, there is a -Chain Manager process that manages its FLU's role within the Machi -cluster's Chain Replication scheme. Each Chain Manager process -executes locally and independently to manage the distributed state of -a single Machi Chain Replication chain. - -
    - -
  • To contrast with Riak Core ... Riak Core's claimant process is - solely responsible for managing certain critical aspects of - Riak Core distributed state. Machi's Chain Manager process - performs similar tasks as Riak Core's claimant. However, Machi - has several active Chain Manager processes, one per FLU server, - instead of a single active process like Core's claimant. Each - Chain Manager process acts independently; each is constrained - so that it will reach consensus via independent computation - & action. - - Full discussion of this distributed consensus is outside the - scope of this document; see the "Pointers to Other Machi - Documentation" section below for more information. -
    • -
  • Machi differs from a Riak Core application because Machi's - replica placement policy is simply, "All Machi servers store - replicas of all Machi files". - Machi is intended to be a primitive building block for creating larger - cluster-of-clusters where files are - distributed/fragmented/sharded across a large pool of - independent Machi clusters. -
    • -
  • See - [https://www.usenix.org/legacy/events/osdi04/tech/renesse.html] - for a copy of the paper, "Chain Replication for Supporting High - Throughput and Availability" by Robbert van Renesse and Fred - B. Schneider. -
    • -
- -=== FLU === - -The FLU is the basic storage server for Machi. - -
    -
  • The name FLU is taken from "flash storage unit" from the paper - "CORFU: A Shared Log Design for Flash Clusters" by - Balakrishnan, Malkhi, Prabhakaran, and Wobber. See - [https://www.usenix.org/conference/nsdi12/technical-sessions/presentation/balakrishnan] -
    • -
  • In CORFU, the sequencer step is a prerequisite step that is - performed by a separate component, the Sequencer. - In Machi, the `append_chunk()' protocol message has - an implicit "sequencer" operation applied by the "head" of the - Machi Chain Replication chain. If a client wishes to write - data that has already been assigned a sequencer position, then - the `write_chunk()' API function is used. -
    • -
- -For each FLU, there are three independent tasks that are implemented -using three different Erlang processes: - -
    -
  • A FLU server, implemented primarily by `machi_flu.erl'. -
    • -
  • A projection store server, implemented primarily by - `machi_projection_store.erl'. -
    • -
  • A chain state manager server, implemented primarily by - `machi_chain_manager1.erl'. -
    • -
- -From the perspective of failure detection, it is very convenient that -all three FLU-related services (file server, sequencer server, and -projection server) are accessed using the same single TCP port. - -=== Projection (data structure) === - -The projection is a data structure that specifies the current state -of the Machi cluster: all FLUs, which FLUS are considered -up/running or down/crashed/stopped, which FLUs are actively -participants in the Chain Replication protocol, and which FLUs are -under "repair" (i.e., having their data resyncronized when -newly-added to a cluster or when restarting after a crash). - -=== Projection Store (server) === - -The projection store is a storage service that is implemented by an -Erlang/OTP `gen_server' process that is associated with each -FLU. Conceptually, the projection store is an array of -write-once registers. For each projection store register, the -key is a 2-tuple of an epoch number (`non_neg_integer()' type) -and a projection type (`public' or `private' type); the value is -a projection data structure (`projection_v1()' type). - -=== Client and Proxy Client === - -Machi is intentionally avoiding using distributed Erlang for Machi's -communication. This design decision makes Erlang-side code more -difficult & complex but allows us the freedom of implementing -parts of Machi in other languages without major -protocol&API&glue code changes later in the product's -lifetime. - -There are two layers of interface for Machi clients. - -
    -
  • The `machi_flu1_client' module implements an API that uses a - TCP socket directly. -
    • -
  • The `machi_proxy_flu1_client' module implements an API that - uses a local, long-lived `gen_server' process as a proxy for - the remote, perhaps disconnected-or-crashed Machi FLU server. -
    • -
- -The types for both modules ought to be the same. However, due to -rapid code churn, some differences might exist. Any major difference -is (almost by definition) a bug: please open a GitHub issue to request -a correction. - -== TODO notes == - -Any use of the string "TODO" in upper/lower/mixed case, anywhere in -the code, is a reminder signal of unfinished work. - -== Pointers to Other Machi Documentation == - -
    -
  • If you are viewing this document locally, please look in the - `../doc/' directory, -
    • -
  • If you are viewing this document via the Web, please find the - documentation via this link: - [http://github.com/basho/machi/tree/master/doc/] - Please be aware that this link points to the `master' branch - of the Machi source repository and therefore may be - out-of-sync with non-`master' branch code. -
    • - -
-- 2.45.2 From ec56164bd167084ad2eaa784c27ae4c78cbe1c07 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 15 Dec 2015 15:52:29 +0900 Subject: [PATCH 69/77] Part 2 of X --- README.md | 119 ++++++++++++++++++++++---------------------------- doc/README.md | 37 ++++++++-------- 2 files changed, 72 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 6a8a64d..f589f32 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,16 @@ [Travis-CI](http://travis-ci.org/basho/machi) :: ![Travis-CI](https://secure.travis-ci.org/basho/machi.png) +Outline + +1. [Why another file store?](#sec1) +2. [Development status summary](#sec2) +3. [Where to learn more about Machi](#sec3) +4. [Contributing to Machi's development](#sec4) + + +## 1. Why another file store? + Our goal is a robust & reliable, distributed, highly available, large file store. Such stores already exist, both in the open source world and in the commercial world. Why reinvent the wheel? We believe @@ -38,40 +48,51 @@ corruption detection should be ubiquitous today. Alas, they are not. Machi is an effort to change the deplorable state of the world, one Erlang function at a time. -## Status: mid-December 2015: work is underway + +## 2. Development status summary -TODO: status update here. +Mid-December 2015: work is underway. -* The chain manager is ready for both eventual consistency use ("available - mode") and strong consistency use ("consistent mode"). Both modes use a new - consensus technique, Humming Consensus. - * Scott will be - [speaking about Humming Consensus](http://ricon.io/agenda/#managing-chain-replication-metadata-with-humming-consensus) - at the [Ricon 2015 conference] (http://ricon.io) in San Francisco, - CA, USA on Thursday, November 5th, 2015. - * If you would like to run the network partition simulator - mentioned in that Ricon presentation, please see the - [partition simulator convergence test doc.](./doc/machi_chain_manager1_converge_demo.md) - * Implementation of the file repair process for strong consistency - is still in progress. +* In progress: + * Code refactoring: metadata management using + [ELevelDB](https://github.com/basho/eleveldb) + * File repair using file-centric, Merkle-style hash tree. + * QuickCheck tests for file repair correctness + * 2015-12-15: The EUnit test `machi_ap_repair_eqc` is + currently failing occasionally because it (correctly) detects + double-write errors. Double-write errors will be eliminated + when the ELevelDB integration work is complete. + * Chain Replication management using the Humming Consensus + algorithm to manage chain state is stable. + * ... with the caveat that it runs very well in a very harsh + and unforgiving network partition simulator but has not run + much yet in the real world. + * All Machi client/server protocols are based on + [Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview). + * The current specification for Machi's protocols can be found at + [https://github.com/basho/machi/blob/master/src/machi.proto](https://github.com/basho/machi/blob/master/src/machi.proto). + * The Machi PB protocol is not yet stable. Expect change! + * The Erlang language client implementation of the high-level + protocol flavor is brittle (e.g., little error handling yet). -* All Machi client/server protocols are based on - [Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview). - * The current specification for Machi's protocols can be found at - [https://github.com/basho/machi/blob/master/src/machi.proto](https://github.com/basho/machi/blob/master/src/machi.proto). - * The Machi PB protocol is not yet stable. Expect change! - * The Erlang language client implementation of the high-level - protocol flavor is brittle (e.g., little error handling yet). +If you would like to run the network partition simulator +mentioned in the Ricon 2015 presentation about Humming Consensus, +please see the +[partition simulator convergence test doc.](./doc/machi_chain_manager1_converge_demo.md) If you'd like to work on a protocol such as Thrift, UBF, msgpack over UDP, or some other protocol, let us know by [opening an issue to discuss it](./issues/new). -## Where to learn more about Machi + +## 3. Where to learn more about Machi The two major design documents for Machi are now mostly stable. Please see the [doc](./doc) directory's [README](./doc) for details. +We also have a +[Frequently Asked Questions (FAQ) list](./FAQ.md). + Scott recently (November 2015) gave a presentation at the [RICON 2015 conference](http://ricon.io) about one of the techniques used by Machi; "Managing Chain Replication Metadata with @@ -79,7 +100,10 @@ Humming Consensus" is available online now. * [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf) * [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q) -## Contributing to Machi: source code, documentation, etc. + +## 4. Contributing to Machi's development + +### 4.1 License Basho Technologies, Inc. as committed to licensing all work for Machi under the @@ -95,26 +119,7 @@ We invite all contributors to review the [CONTRIBUTING.md](./CONTRIBUTING.md) document for guidelines for working with the Basho development team. -## A brief survey of this directories in this repository - -* A list of Frequently Asked Questions, a.k.a. - [the Machi FAQ](./FAQ.md). - -* The [doc](./doc/) directory: home for major documents about Machi: - high level design documents as well as exploration of features still - under design & review within Basho. - -* The `ebin` directory: used for compiled application code - -* The `include`, `src`, and `test` directories: contain the header - files, source files, and test code for Machi, respectively. - -* The [prototype](./prototype/) directory: contains proof of concept - code, scaffolding libraries, and other exploratory code. Curious - readers should see the [prototype/README.md](./prototype/README.md) - file for more explanation of the small sub-projects found here. - -## Development environment requirements +### 4.2 Development environment requirements All development to date has been done with Erlang/OTP version 17 on OS X. The only known limitations for using R16 are minor type @@ -126,26 +131,8 @@ tool chain for C and C++ applications. Specifically, we assume `make` is available. The utility used to compile the Machi source code, `rebar`, is pre-compiled and included in the repo. -There are no known OS limits at this time: any platform that supports -Erlang/OTP should be sufficient for Machi. This may change over time -(e.g., adding NIFs which can make full portability to Windows OTP -environments difficult), but it hasn't happened yet. - -## Contributions - -Basho encourages contributions to Riak from the community. Here’s how -to get started. - -* Fork the appropriate sub-projects that are affected by your change. -* Create a topic branch for your change and checkout that branch. - git checkout -b some-topic-branch -* Make your changes and run the test suite if one is provided. (see below) -* Commit your changes and push them to your fork. -* Open pull-requests for the appropriate projects. -* Contributors will review your pull request, suggest changes, and merge it when it’s ready and/or offer feedback. -* To report a bug or issue, please open a new issue against this repository. - --The Machi team at Basho, -[Scott Lystig Fritchie](mailto:scott@basho.com), technical lead, and -[Matt Brender](mailto:mbrender@basho.com), your developer advocate. - +Machi has a dependency on the +[ELevelDB](https://github.com/basho/eleveldb) library. ELevelDB only +supports UNIX/Linux OSes and 64-bit versions of Erlang/OTP only; we +apologize to Windows-based and 32-bit-based Erlang developers for this +restriction. diff --git a/doc/README.md b/doc/README.md index 181278b..3ad424c 100644 --- a/doc/README.md +++ b/doc/README.md @@ -6,20 +6,6 @@ Erlang documentation, please use this link: ## Documents in this directory -### chain-self-management-sketch.org - -[chain-self-management-sketch.org](chain-self-management-sketch.org) -is a mostly-deprecated draft of -an introduction to the -self-management algorithm proposed for Machi. Most material has been -moved to the [high-level-chain-mgr.pdf](high-level-chain-mgr.pdf) document. - -### cluster-of-clusters (directory) - -This directory contains the sketch of the "cluster of clusters" design -strawman for partitioning/distributing/sharding files across a large -number of independent Machi clusters. - ### high-level-machi.pdf [high-level-machi.pdf](high-level-machi.pdf) @@ -50,9 +36,9 @@ introduction to the Humming Consensus algorithm. Its abstract: > of file updates to all replica servers in a Machi cluster. Chain > Replication is a variation of primary/backup replication where the > order of updates between the primary server and each of the backup -> servers is strictly ordered into a single ``chain''. Management of -> Chain Replication's metadata, e.g., ``What is the current order of -> servers in the chain?'', remains an open research problem. The +> servers is strictly ordered into a single "chain". Management of +> Chain Replication's metadata, e.g., "What is the current order of +> servers in the chain?", remains an open research problem. The > current state of the art for Chain Replication metadata management > relies on an external oracle (e.g., ZooKeeper) or the Elastic > Replication algorithm. @@ -60,7 +46,7 @@ introduction to the Humming Consensus algorithm. Its abstract: > This document describes the Machi chain manager, the component > responsible for managing Chain Replication metadata state. The chain > manager uses a new technique, based on a variation of CORFU, called -> ``humming consensus''. +> "humming consensus". > Humming consensus does not require active participation by all or even > a majority of participants to make decisions. Machi's chain manager > bases its logic on humming consensus to make decisions about how to @@ -71,3 +57,18 @@ introduction to the Humming Consensus algorithm. Its abstract: > decision during that epoch. When a differing decision is discovered, > new time epochs are proposed in which a new consensus is reached and > disseminated to all available participants. + +### chain-self-management-sketch.org + +[chain-self-management-sketch.org](chain-self-management-sketch.org) +is a mostly-deprecated draft of +an introduction to the +self-management algorithm proposed for Machi. Most material has been +moved to the [high-level-chain-mgr.pdf](high-level-chain-mgr.pdf) document. + +### cluster-of-clusters (directory) + +This directory contains the sketch of the "cluster of clusters" design +strawman for partitioning/distributing/sharding files across a large +number of independent Machi clusters. + -- 2.45.2 From 9bd885ccb42a6af58186a05e982a2ab45b3755f8 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 15 Dec 2015 16:20:57 +0900 Subject: [PATCH 70/77] Part 3 of X --- FAQ.md | 63 ++++++++++++++++++++++++++++--------------------------- README.md | 50 ++++++++++++++++++++++++------------------- 2 files changed, 61 insertions(+), 52 deletions(-) diff --git a/FAQ.md b/FAQ.md index 5885bc5..f2e37c1 100644 --- a/FAQ.md +++ b/FAQ.md @@ -13,8 +13,8 @@ + [1.1 What is Machi?](#n1.1) + [1.2 What is a Machi "cluster of clusters"?](#n1.2) + [1.2.1 This "cluster of clusters" idea needs a better name, don't you agree?](#n1.2.1) - + [1.3 What is Machi like when operating in "eventually consistent"/"AP mode"?](#n1.3) - + [1.4 What is Machi like when operating in "strongly consistent"/"CP mode"?](#n1.4) + + [1.3 What is Machi like when operating in "eventually consistent" mode?](#n1.3) + + [1.4 What is Machi like when operating in "strongly consistent" mode?](#n1.4) + [1.5 What does Machi's API look like?](#n1.5) + [1.6 What licensing terms are used by Machi?](#n1.6) + [1.7 Where can I find the Machi source code and documentation? Can I contribute?](#n1.7) @@ -120,7 +120,7 @@ For proof that naming things is hard, see [http://martinfowler.com/bliki/TwoHardThings.html](http://martinfowler.com/bliki/TwoHardThings.html) -### 1.3. What is Machi like when operating in "eventually consistent"/"AP mode"? +### 1.3. What is Machi like when operating in "eventually consistent" mode? Machi's operating mode dictates how a Machi cluster will react to network partitions. A network partition may be caused by: @@ -130,17 +130,15 @@ network partitions. A network partition may be caused by: * An extreme server software "hang" or "pause", e.g. caused by OS scheduling problems such as a failing/stuttering disk device. -"AP mode" refers to the "A" and "P" properties of the "CAP -conjecture", meaning that the cluster will be "Available" and -"Partition tolerant". - -The consistency semantics of file operations while in "AP mode" are -eventually consistent during and after network partitions: +The consistency semantics of file operations while in eventual +consistency mode during and after network partitions are: * File write operations are permitted by any client on the "same side" of the network partition. * File read operations are successful for any file contents where the client & server are on the "same side" of the network partition. + * File read operations will probably fail for any file contents where the + client & server are on "different sides" of the network partition. * After the network partition(s) is resolved, files are merged together from "all sides" of the partition(s). * Unique files are copied in their entirety. @@ -151,16 +149,10 @@ eventually consistent during and after network partitions: to rules which guarantee safe mergeability.). -### 1.4. What is Machi like when operating in "strongly consistent"/"CP mode"? +### 1.4. What is Machi like when operating in "strongly consistent" mode? -Machi's operating mode dictates how a Machi cluster will react to -network partitions. -"CP mode" refers to the "C" and "P" properties of the "CAP -conjecture", meaning that the cluster will be "Consistent" and -"Partition tolerant". - -The consistency semantics of file operations while in "CP mode" are -strongly consistent during and after network partitions: +The consistency semantics of file operations while in strongly +consistency mode during and after network partitions are: * File write operations are permitted by any client on the "same side" of the network partition if and only if a quorum majority of Machi servers @@ -205,7 +197,12 @@ Internally, there is a more complex protocol used by individual cluster members to manage file contents and to repair damaged/missing files. See Figure 3 in [Machi high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-machi.pdf) -for more details. +for more description. + +The definitions of both the "high level" external protocol and "low +level" internal protocol are in a +[Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview) +definition at [./src/machi.proto](./src/machi.proto). ### 1.6. What licensing terms are used by Machi? @@ -232,8 +229,8 @@ guidelines at ### 1.8. What is Machi's expected release schedule, packaging, and operating system/OS distribution support? -Basho expects that Machi's first release will take place near the end -of calendar year 2015. +Basho expects that Machi's first major product release will take place +during the 2nd quarter of 2016. Basho's official support for operating systems (e.g. Linux, FreeBSD), operating system packaging (e.g. CentOS rpm/yum package management, @@ -537,6 +534,10 @@ change several times during any single test case) and a random series of cluster operations, an event trace of all cluster activity is used to verify that no safety-critical rules have been violated. +All test code is available in the [./test](./test) subdirectory. +Modules that use QuickCheck will use a file suffix of `_eqc`, for +example, [./test/machi_ap_repair_eqc.erl](./test/machi_ap_repair_eqc.erl). + ### 3.5. Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks @@ -567,7 +568,10 @@ deploy multiple Machi servers per machine: one Machi server per disk. ### 3.7. What language(s) is Machi written in? -So far, Machi is written in 100% Erlang. +So far, Machi is written in 100% Erlang. Machi uses at least one +library, [ELevelDB](https://github.com/basho/eleveldb), that is +implemented both in C++ and in Erlang, using Erlang NIFs (Native +Interface Functions) to allow Erlang code to call C++ functions. In the event that we encounter a performance problem that cannot be solved within the Erlang/OTP runtime environment, all of Machi's @@ -588,19 +592,16 @@ bit-twiddling magicSPEED ... without also having to find a replacement for disterl. (Or without having to re-invent disterl's features in another language.) - -In the first drafts of the Machi code, the inter-node communication -uses a hand-crafted, artisanal, mostly ASCII protocol as part of a -"demo day" quick & dirty prototype. Work is underway (summer of 2015) -to replace that protocol gradually with a well-structured, -well-documented protocol based on Protocol Buffers data serialization. +All wire protocols used by Machi are defined & implemented using +[Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview). +The definition file can be found at [./src/machi.proto](./src/machi.proto). ### 3.9. Can I use HTTP to write/read stuff into/from Machi? -Yes, sort of. For as long as the legacy of -Machi's first internal protocol & code still -survives, it's possible to use a +Short answer: No, not yet. + +Longer answer: No, but it was possible as a hack, many months ago, see [primitive/hack'y HTTP interface that is described in this source code commit log](https://github.com/basho/machi/commit/6cebf397232cba8e63c5c9a0a8c02ba391b20fef). Please note that commit `6cebf397232cba8e63c5c9a0a8c02ba391b20fef` is required to try using this feature: the code has since bit-rotted and diff --git a/README.md b/README.md index f589f32..28f77d2 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ Outline 1. [Why another file store?](#sec1) -2. [Development status summary](#sec2) -3. [Where to learn more about Machi](#sec3) +2. [Where to learn more about Machi](#sec2) +3. [Development status summary](#sec3) 4. [Contributing to Machi's development](#sec4) @@ -39,8 +39,8 @@ file data and attempts best-effort file reads? If we really do care about data loss and/or data corruption, then we really want both #3 and #1. Unfortunately, systems that meet -criteria #1 are *very* -rare. Why? This is 2015. We have decades of research that shows +criteria #1 are _very rare_. +Why? This is 2015. We have decades of research that shows that computer hardware can (and indeed does) corrupt data at nearly every level of the modern client/server application stack. Systems with end-to-end data @@ -49,7 +49,23 @@ Machi is an effort to change the deplorable state of the world, one Erlang function at a time. -## 2. Development status summary +## 2. Where to learn more about Machi + +The two major design documents for Machi are now mostly stable. +Please see the [doc](./doc) directory's [README](./doc) for details. + +We also have a +[Frequently Asked Questions (FAQ) list](./FAQ.md). + +Scott recently (November 2015) gave a presentation at the +[RICON 2015 conference](http://ricon.io) about one of the techniques +used by Machi; "Managing Chain Replication Metadata with +Humming Consensus" is available online now. +* [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf) +* [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q) + + +## 3. Development status summary Mid-December 2015: work is underway. @@ -57,11 +73,19 @@ Mid-December 2015: work is underway. * Code refactoring: metadata management using [ELevelDB](https://github.com/basho/eleveldb) * File repair using file-centric, Merkle-style hash tree. + * Server-side socket handling is now performed by + [ranch](https://github.com/ninenines/ranch) * QuickCheck tests for file repair correctness * 2015-12-15: The EUnit test `machi_ap_repair_eqc` is currently failing occasionally because it (correctly) detects double-write errors. Double-write errors will be eliminated when the ELevelDB integration work is complete. + * The `make stage` and `make release` commands can be used to + create a primitive "package". Use `./rel/machi/bin/machi console` + to start the Machi app in interactive mode. Substitute the word + `start` instead of console to start Machi in background/daemon + mode. The `./rel/machi/bin/machi` command without any arguments + will give a short usage summary. * Chain Replication management using the Humming Consensus algorithm to manage chain state is stable. * ... with the caveat that it runs very well in a very harsh @@ -84,22 +108,6 @@ If you'd like to work on a protocol such as Thrift, UBF, msgpack over UDP, or some other protocol, let us know by [opening an issue to discuss it](./issues/new). - -## 3. Where to learn more about Machi - -The two major design documents for Machi are now mostly stable. -Please see the [doc](./doc) directory's [README](./doc) for details. - -We also have a -[Frequently Asked Questions (FAQ) list](./FAQ.md). - -Scott recently (November 2015) gave a presentation at the -[RICON 2015 conference](http://ricon.io) about one of the techniques -used by Machi; "Managing Chain Replication Metadata with -Humming Consensus" is available online now. -* [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf) -* [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q) - ## 4. Contributing to Machi's development -- 2.45.2 From 463d20a9fd9ae12cd688903bf3e2663c1f51b861 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 16 Dec 2015 16:41:11 +0900 Subject: [PATCH 71/77] Add 'quick admin' config management tool/hack --- priv/quick-admin-examples/000 | 1 + priv/quick-admin-examples/001 | 4 + priv/quick-admin-examples/002 | 4 + rel/reltool.config | 9 ++ src/machi_lifecycle_mgr.erl | 142 ++++++++++++++++++++---------- test/machi_lifecycle_mgr_test.erl | 18 ++-- 6 files changed, 119 insertions(+), 59 deletions(-) create mode 100644 priv/quick-admin-examples/000 create mode 100644 priv/quick-admin-examples/001 create mode 100644 priv/quick-admin-examples/002 diff --git a/priv/quick-admin-examples/000 b/priv/quick-admin-examples/000 new file mode 100644 index 0000000..45eee80 --- /dev/null +++ b/priv/quick-admin-examples/000 @@ -0,0 +1 @@ +{host, "localhost", []}. diff --git a/priv/quick-admin-examples/001 b/priv/quick-admin-examples/001 new file mode 100644 index 0000000..e5df9dd --- /dev/null +++ b/priv/quick-admin-examples/001 @@ -0,0 +1,4 @@ +{flu,f1,"localhost",20401,[]}. +{flu,f2,"localhost",20402,[]}. +{flu,f3,"localhost",20403,[]}. +{chain,c1,[f1,f2,f3],[]}. diff --git a/priv/quick-admin-examples/002 b/priv/quick-admin-examples/002 new file mode 100644 index 0000000..5656558 --- /dev/null +++ b/priv/quick-admin-examples/002 @@ -0,0 +1,4 @@ +{flu,f4,"localhost",20404,[]}. +{flu,f5,"localhost",20405,[]}. +{flu,f6,"localhost",20406,[]}. +{chain,c2,[f4,f5,f6],[]}. diff --git a/rel/reltool.config b/rel/reltool.config index 6f9663f..00df339 100644 --- a/rel/reltool.config +++ b/rel/reltool.config @@ -98,6 +98,15 @@ {mkdir, "etc/flu-config"}, {mkdir, "etc/pending"}, {mkdir, "etc/rejected"}, + + %% Experiment: quick-admin + {mkdir, "etc/quick-admin"}, + {mkdir, "priv"}, + {mkdir, "priv/quick-admin-examples"}, + {copy, "../priv/quick-admin-examples/000", "priv/quick-admin-examples"}, + {copy, "../priv/quick-admin-examples/001", "priv/quick-admin-examples"}, + {copy, "../priv/quick-admin-examples/002", "priv/quick-admin-examples"}, + {mkdir, "lib/basho-patches"} %% {copy, "../apps/machi/ebin/etop_txt.beam", "lib/basho-patches"} ]}. diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index f18f199..2bedf8c 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -205,6 +205,10 @@ %% API -export([start_link/0, process_pending/0]). +-export([get_pending_dir/0, get_rejected_dir/0, get_flu_config_dir/0, + get_flu_data_dir/0, get_chain_config_dir/0, get_data_dir/0]). +-export([quick_admin_sanity_check/0, quick_admin_apply/1]). +-export([make_pending_config/1, run_ast/1, diff_env/2]). -ifdef(TEST). -compile(export_all). -endif. % TEST @@ -282,14 +286,7 @@ get_latest_public_epochs(FLUs) -> end || FLU <- FLUs]. get_initial_chains() -> - DoesNotExist = "/tmp/does/not/exist", - ConfigDir = case application:get_env(machi, chain_config_dir, DoesNotExist) of - DoesNotExist -> - DoesNotExist; - Dir -> - ok = filelib:ensure_dir(Dir ++ "/unused"), - Dir - end, + ConfigDir = get_chain_config_dir(), CDefs = [CDef || {_File, CDef} <- machi_flu_sup:load_rc_d_files_from_dir( ConfigDir)], sanitize_chain_def_records(CDefs). @@ -378,7 +375,7 @@ bootstrap_chain2(#chain_def_v1{name=NewChainName, mode=NewCMode, Mgr, NewChainName, OldEpoch, NewCMode, MembersDict, NewWitnesses_list) of ok -> - lager:info("Bootstrapped chain ~w via FLU ~w to " + lager:info("Configured chain ~w via FLU ~w to " "mode=~w all=~w witnesses=~w\n", [NewChainName, FLU, NewCMode, NewAll_list, NewWitnesses_list]), @@ -421,8 +418,7 @@ set_chain_members(OldChainName, NewChainName, OldCMode, end. do_process_pending(S) -> - PendingDir = get_pending_dir(S), - %% PendingFiles = get_pending_files(PendingDir, S), + PendingDir = get_pending_dir(), PendingParsed = machi_flu_sup:load_rc_d_files_from_dir(PendingDir), P_FLUs = [X || {_File, #p_srvr{}}=X <- PendingParsed], P_Chains = [X || {_File, #chain_def_v1{}}=X <- PendingParsed], @@ -432,8 +428,8 @@ do_process_pending(S) -> S4 = process_bad_files(BadFiles, S3), {{P_FLUs, P_Chains}, S4}. -flu_config_exists(FLU, S) -> - ConfigDir = get_flu_config_dir(S), +flu_config_exists(FLU) -> + ConfigDir = get_flu_config_dir(), case file:read_file_info(ConfigDir ++ "/" ++ atom_to_list(FLU)) of {ok, _} -> true; @@ -441,32 +437,36 @@ flu_config_exists(FLU, S) -> false end. -get_pending_dir(_S) -> +get_pending_dir() -> {ok, EtcDir} = application:get_env(machi, platform_etc_dir), EtcDir ++ "/pending". -get_rejected_dir(_S) -> +get_rejected_dir() -> {ok, EtcDir} = application:get_env(machi, platform_etc_dir), EtcDir ++ "/rejected". -get_flu_config_dir(_S) -> +get_flu_config_dir() -> {ok, Dir} = application:get_env(machi, flu_config_dir), Dir. -get_flu_data_dir(_S) -> +get_flu_data_dir() -> {ok, Dir} = application:get_env(machi, flu_data_dir), Dir. -get_chain_config_dir(_S) -> +get_chain_config_dir() -> {ok, Dir} = application:get_env(machi, chain_config_dir), Dir. -get_data_dir(_S) -> +get_data_dir() -> {ok, Dir} = application:get_env(machi, platform_data_dir), Dir. -get_preserve_dir(S) -> - get_data_dir(S) ++ "/^PRESERVE". +get_preserve_dir() -> + get_data_dir() ++ "/^PRESERVE". + +get_quick_admin_dir() -> + {ok, EtcDir} = application:get_env(machi, platform_etc_dir), + EtcDir ++ "/quick-admin". process_pending_flus(P_FLUs, S) -> lists:foldl(fun process_pending_flu/2, S, P_FLUs). @@ -547,7 +547,7 @@ process_pending_chain({File, CD}, S) -> process_pending_chain2(File, CD, RemovedFLUs, ChainConfigAction, S) -> LocalRemovedFLUs = [FLU || FLU <- RemovedFLUs, - flu_config_exists(FLU, S)], + flu_config_exists(FLU)], case LocalRemovedFLUs of [] -> ok; @@ -558,9 +558,9 @@ process_pending_chain2(File, CD, RemovedFLUs, ChainConfigAction, S) -> %% We may be retrying this, so be liberal with any pattern %% matching on return values. _ = machi_flu_psup:stop_flu_package(FLU), - ConfigDir = get_flu_config_dir(S), - FluDataDir = get_flu_data_dir(S), - PreserveDir = get_preserve_dir(S), + ConfigDir = get_flu_config_dir(), + FluDataDir = get_flu_data_dir(), + PreserveDir = get_preserve_dir(), Suffix = make_ts_suffix(), FLU_str = atom_to_list(FLU), MyPreserveDir = PreserveDir ++ "/" ++ FLU_str ++ "." ++ Suffix, @@ -594,7 +594,7 @@ process_bad_files(Files, S) -> move_to_rejected(File, S) -> lager:error("Pending unknown config file ~s has been rejected\n", [File]), - Dst = get_rejected_dir(S), + Dst = get_rejected_dir(), Suffix = make_ts_suffix(), ok = file:rename(File, Dst ++ "/" ++ filename:basename(File) ++ Suffix), S. @@ -604,19 +604,19 @@ make_ts_suffix() -> move_to_flu_config(FLU, File, S) -> lager:info("Creating FLU config file ~w\n", [FLU]), - Dst = get_flu_config_dir(S), + Dst = get_flu_config_dir(), ok = file:rename(File, Dst ++ "/" ++ atom_to_list(FLU)), S. move_to_chain_config(Name, File, S) -> lager:info("Creating chain config file ~w\n", [Name]), - Dst = get_chain_config_dir(S), + Dst = get_chain_config_dir(), ok = file:rename(File, Dst ++ "/" ++ atom_to_list(Name)), S. delete_chain_config(Name, File, S) -> lager:info("Deleting chain config file ~s for chain ~w\n", [File, Name]), - Dst = get_chain_config_dir(S), + Dst = get_chain_config_dir(), ok = file:delete(Dst ++ "/" ++ atom_to_list(Name)), S. @@ -917,24 +917,7 @@ is_porty(Port) -> is_integer(Port) andalso 1024 =< Port andalso Port =< 65535. diff_env({KV_old, KV_new, _IsNew}=E, RelativeHost) -> - Old_list = gb_trees:to_list(KV_old), New_list = gb_trees:to_list(KV_new), - Keys_old = [K || {K,_V} <- Old_list], - Keys_new = [K || {K,_V} <- New_list], - Append = fun(Item) -> - fun(K, D) -> - L = gb_trees:get(K, D), - gb_trees:enter(K, L ++ [Item], D) - end - end, - - %% DiffD = lists:foldl(fun(K, D) -> - %% gb_trees:enter(K, [], D) - %% end, gb_trees:empty(), lists:usort(Keys_old++Keys_new)), - %% DiffD2 = lists:foldl(Append(old), DiffD, Keys_old), - %% DiffD3 = lists:foldl(Append(new), DiffD2, Keys_new), - %% %% DiffD3 values will be exactly one of: [old], [old,new], [new] - put(final, []), Add = fun(X) -> put(final, [X|get(final)]) end, @@ -1003,3 +986,70 @@ diff_env({KV_old, KV_new, _IsNew}=E, RelativeHost) -> end || {{kv,{chain,Name}}, V} <- New_list], {x, lists:reverse(get(final))}. + +make_pending_config(Term) -> + Dir = get_pending_dir(), + Blob = io_lib:format("~p.\n", [Term]), + {A,B,C} = os:timestamp(), + Path = lists:flatten(io_lib:format("~s/~w.~6..0w",[Dir, A*1000000+B, C])), + ok = file:write_file(Path, Blob). + +%% @doc Check a "quick admin" directory's for sanity +%% +%% This is a quick admin tool, though perhaps "tool" is too big of a word. +%% The meaning of "quick" is closer to "quick & dirty hack". Any +%% violation of the assumptions of these quick admin functions will result in +%% unspecified behavior, bugs, plagues, and perhaps lost data. +%% +%% Add files in this directory are assumed to have names of the same length +%% (e.g. 4 bytes), ASCII formatted numbers, greater than 0, left-padded with +%% "0". Each file is assumed to have zero or more AST tuples within, parsable +%% using {ok, ListOfAST_tuples} = file:consult(QuickAdminFile). +%% +%% The largest numbered file is assumed to be all of the AST changes that we +%% want to apply in a single batch. The AST tuples of all files with smaller +%% numbers will be concatenated together to create the prior history of +%% cluster-of-clusters. We assume that all transitions inside these earlier +%% files were actually safe & sane, therefore any sanity problem can only +%% be caused by the contents of the largest numbered file. + +quick_admin_sanity_check() -> + try + {ok, Env} = quick_admin_run_ast(), + {true, diff_env(Env, all)} + catch X:Y -> + {false, {X,Y}} + end. + +quick_admin_apply(HostName) -> + try + {ok, Env} = quick_admin_run_ast(), + {_, Cs} = diff_env(Env, HostName), + [ok = make_pending_config(C) || C <- Cs], + {PassFLUs, PassChains} = machi_lifecycle_mgr:process_pending(), + case length(PassFLUs) + length(PassChains) of + N when N == length(Cs) -> + yay; + _ -> + {sad, expected, length(Cs), Cs, got, PassFLUs, PassChains} + end + catch X:Y -> + {false, {X,Y}, erlang:get_stacktrace()} + end. + + +quick_admin_parse_quick(F) -> + {ok, Terms} = file:consult(F), + Terms. + +quick_admin_run_ast() -> + case filelib:wildcard(get_quick_admin_dir() ++ "/*") of + [] -> + PrevASTs = LatestASTs = []; + Quicks -> + LatestQuick = lists:last(Quicks), + Prev = Quicks -- [LatestQuick], + PrevASTs = lists:append([quick_admin_parse_quick(F) || F <- Prev]), + LatestASTs = quick_admin_parse_quick(LatestQuick) + end, + {ok, _Env} = run_ast(PrevASTs ++ [switch_old_and_new] ++ LatestASTs). diff --git a/test/machi_lifecycle_mgr_test.erl b/test/machi_lifecycle_mgr_test.erl index 38f5132..8b17861 100644 --- a/test/machi_lifecycle_mgr_test.erl +++ b/test/machi_lifecycle_mgr_test.erl @@ -100,7 +100,7 @@ smoke_test2() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% io:format("\nSTEP: Start 3 FLUs, no chain.\n", []), - [make_pending_config(P) || P <- [Pa,Pb,Pc] ], + [machi_lifecycle_mgr:make_pending_config(P) || P <- [Pa,Pb,Pc] ], {[_,_,_],[]} = machi_lifecycle_mgr:process_pending(), [{ok, #projection_v1{epoch_number=0}} = machi_projection_store:read_latest_projection(PSTORE, private) @@ -111,7 +111,7 @@ smoke_test2() -> C1 = #chain_def_v1{name=cx, mode=ap_mode, full=[Pa,Pb,Pc], local_run=[a,b,c]}, - make_pending_config(C1), + machi_lifecycle_mgr:make_pending_config(C1), {[],[_]} = machi_lifecycle_mgr:process_pending(), Advance(), [{ok, #projection_v1{all_members=[a,b,c]}} = @@ -124,7 +124,7 @@ smoke_test2() -> C2 = #chain_def_v1{name=cx, mode=ap_mode, full=[Pb,Pc], old_full=[a,b,c], old_witnesses=[], local_stop=[a], local_run=[b,c]}, - make_pending_config(C2), + machi_lifecycle_mgr:make_pending_config(C2), {[],[_]} = machi_lifecycle_mgr:process_pending(), Advance(), %% a should be down @@ -140,7 +140,7 @@ smoke_test2() -> C3 = #chain_def_v1{name=cx, mode=ap_mode, full=[], old_full=[b,c], old_witnesses=[], local_stop=[b,c], local_run=[]}, - make_pending_config(C3), + machi_lifecycle_mgr:make_pending_config(C3), {[],[_]} = machi_lifecycle_mgr:process_pending(), Advance(), %% a,b,c should be down @@ -153,14 +153,6 @@ smoke_test2() -> cleanup(YoCleanup) end. -make_pending_config(Term) -> - Dir = machi_lifecycle_mgr:get_pending_dir(x), - Blob = io_lib:format("~w.\n", [Term]), - {A,B,C} = os:timestamp(), - ok = file:write_file(Dir ++ "/" ++ - lists:flatten(io_lib:format("~w.~6..0w",[A*1000000+B,C])), - Blob). - ast_tuple_syntax_test() -> T = fun(L) -> machi_lifecycle_mgr:check_ast_tuple_syntax(L) end, Canon1 = [ {host, "localhost", []}, @@ -280,7 +272,7 @@ ast_then_apply_test2() -> {ok, Env1} = machi_lifecycle_mgr:run_ast(R1), {_X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, "localhost"), %% io:format(user, "X1b ~p\n", [X1b]), - [make_pending_config(X) || X <- X1b], + [machi_lifecycle_mgr:make_pending_config(X) || X <- X1b], {PassFLUs, PassChains} = machi_lifecycle_mgr:process_pending(), true = (length(PassFLUs) == length(FLU_defs)), true = (length(PassChains) == length(Ch_defs)), -- 2.45.2 From f98b4da45b1569b87834dcce2e78762fd7ec9a0a Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Wed, 16 Dec 2015 19:05:25 +0900 Subject: [PATCH 72/77] Add 'quick admin' config management: better file handling --- rel/files/machi-admin | 32 +++++++++++++---- rel/reltool.config | 2 +- src/machi_lifecycle_mgr.erl | 72 ++++++++++++++++++++++++------------- 3 files changed, 74 insertions(+), 32 deletions(-) diff --git a/rel/files/machi-admin b/rel/files/machi-admin index 627705f..fd07634 100755 --- a/rel/files/machi-admin +++ b/rel/files/machi-admin @@ -22,23 +22,41 @@ cd $RUNNER_BASE_DIR SCRIPT=`basename $0` usage() { - echo "Usage: $SCRIPT { test | " + echo "Usage: $SCRIPT { quick-admin-check | quick-admin-apply | " echo " top }" } case "$1" in - test) + quick-admin-check) # Make sure the local node IS running node_up_check shift - # Parse out the node name to pass to the client - NODE_NAME=${NAME_ARG#* } + NODE_NAME=${NAME_ARG#* } # target machi server node name + IN_FILE="$1" - $ERTS_PATH/erl -noshell $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \ - -pa $RUNNER_LIB_DIR/basho-patches \ - -eval "case catch(machi:client_test(\"$NODE_NAME\")) of \ + $ERTS_PATH/erl -noshell -noinput $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \ + -remsh $NODE_NAME \ + -eval "Me = self(), spawn('"$NODE_NAME"', fun() -> X = (catch(machi_lifecycle_mgr:quick_admin_sanity_check(\"$IN_FILE\"))), Me ! {res, X} end), XX = receive {res, Res} -> Res after 10*1000 -> timeout end, io:format(user, \"Result: ~p\n\", [XX]), case XX of \ + ok -> init:stop(); \ + _ -> init:stop(1) \ + end." + + ;; + quick-admin-apply) + # Make sure the local node IS running + node_up_check + + shift + + NODE_NAME=${NAME_ARG#* } # target machi server node name + IN_FILE="$1" + RELATIVE_HOST="$2" + + $ERTS_PATH/erl -noshell -noinput $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \ + -remsh $NODE_NAME \ + -eval "Me = self(), spawn('"$NODE_NAME"', fun() -> X = (catch(machi_lifecycle_mgr:quick_admin_apply(\"$IN_FILE\", \"$RELATIVE_HOST\"))), Me ! {res, X} end), XX = receive {res, Res} -> Res after 10*1000 -> timeout end, io:format(user, \"Result: ~p\n\", [XX]), case XX of \ ok -> init:stop(); \ _ -> init:stop(1) \ end." diff --git a/rel/reltool.config b/rel/reltool.config index 00df339..33df951 100644 --- a/rel/reltool.config +++ b/rel/reltool.config @@ -100,7 +100,7 @@ {mkdir, "etc/rejected"}, %% Experiment: quick-admin - {mkdir, "etc/quick-admin"}, + {mkdir, "etc/quick-admin-archive"}, {mkdir, "priv"}, {mkdir, "priv/quick-admin-examples"}, {copy, "../priv/quick-admin-examples/000", "priv/quick-admin-examples"}, diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 2bedf8c..c8374d2 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -207,7 +207,7 @@ process_pending/0]). -export([get_pending_dir/0, get_rejected_dir/0, get_flu_config_dir/0, get_flu_data_dir/0, get_chain_config_dir/0, get_data_dir/0]). --export([quick_admin_sanity_check/0, quick_admin_apply/1]). +-export([quick_admin_sanity_check/1, quick_admin_apply/2]). -export([make_pending_config/1, run_ast/1, diff_env/2]). -ifdef(TEST). -compile(export_all). @@ -323,6 +323,8 @@ sanitize_chain_def_rec(Whole, {Acc, D}) -> perhaps_bootstrap_chains([], LocalFLUs_at_zero, LocalFLUs) -> if LocalFLUs == [] -> ok; + LocalFLUs_at_zero == [] -> + ok; true -> lager:warning("The following FLUs are defined but are not also " "members of a defined chain: ~w\n", @@ -420,6 +422,7 @@ set_chain_members(OldChainName, NewChainName, OldCMode, do_process_pending(S) -> PendingDir = get_pending_dir(), PendingParsed = machi_flu_sup:load_rc_d_files_from_dir(PendingDir), + %% A pending file has exactly one record (#p_srvr{} or #chain_def_v1{}). P_FLUs = [X || {_File, #p_srvr{}}=X <- PendingParsed], P_Chains = [X || {_File, #chain_def_v1{}}=X <- PendingParsed], BadFiles = [File || {File, []} <- PendingParsed], @@ -466,7 +469,7 @@ get_preserve_dir() -> get_quick_admin_dir() -> {ok, EtcDir} = application:get_env(machi, platform_etc_dir), - EtcDir ++ "/quick-admin". + EtcDir ++ "/quick-admin-archive". process_pending_flus(P_FLUs, S) -> lists:foldl(fun process_pending_flu/2, S, P_FLUs). @@ -600,7 +603,7 @@ move_to_rejected(File, S) -> S. make_ts_suffix() -> - lists:flatten(io_lib:format("~w,~w,~w", tuple_to_list(os:timestamp()))). + str("~w,~w,~w", tuple_to_list(os:timestamp())). move_to_flu_config(FLU, File, S) -> lager:info("Creating FLU config file ~w\n", [FLU]), @@ -890,7 +893,10 @@ n(L) -> lists:sort(L). err(Fmt, Args, AST) -> - throw({lists:flatten(io_lib:format(Fmt, Args)), AST}). + throw({str(Fmt, Args), AST}). + +str(Fmt, Args) -> + lists:flatten(io_lib:format(Fmt, Args)). %% We won't allow 'atom' style proplist members: too difficult to normalize. %% Also, no duplicates, again because normalizing useful for checksums but @@ -991,7 +997,7 @@ make_pending_config(Term) -> Dir = get_pending_dir(), Blob = io_lib:format("~p.\n", [Term]), {A,B,C} = os:timestamp(), - Path = lists:flatten(io_lib:format("~s/~w.~6..0w",[Dir, A*1000000+B, C])), + Path = str("~s/~w.~6..0w",[Dir, A*1000000+B, C]), ok = file:write_file(Path, Blob). %% @doc Check a "quick admin" directory's for sanity @@ -1013,28 +1019,29 @@ make_pending_config(Term) -> %% files were actually safe & sane, therefore any sanity problem can only %% be caused by the contents of the largest numbered file. -quick_admin_sanity_check() -> +quick_admin_sanity_check(File) -> try - {ok, Env} = quick_admin_run_ast(), - {true, diff_env(Env, all)} + {ok, Env} = quick_admin_run_ast(File), + ok catch X:Y -> - {false, {X,Y}} + {error, {X,Y, erlang:get_stacktrace()}} end. -quick_admin_apply(HostName) -> +quick_admin_apply(File, HostName) -> try - {ok, Env} = quick_admin_run_ast(), + {ok, Env} = quick_admin_run_ast(File), {_, Cs} = diff_env(Env, HostName), [ok = make_pending_config(C) || C <- Cs], {PassFLUs, PassChains} = machi_lifecycle_mgr:process_pending(), case length(PassFLUs) + length(PassChains) of N when N == length(Cs) -> - yay; + ok = quick_admin_add_archive_file(File), + ok; _ -> - {sad, expected, length(Cs), Cs, got, PassFLUs, PassChains} + {error, {expected, length(Cs), Cs, got, PassFLUs, PassChains}} end catch X:Y -> - {false, {X,Y}, erlang:get_stacktrace()} + {error, {X,Y, erlang:get_stacktrace()}} end. @@ -1042,14 +1049,31 @@ quick_admin_parse_quick(F) -> {ok, Terms} = file:consult(F), Terms. -quick_admin_run_ast() -> - case filelib:wildcard(get_quick_admin_dir() ++ "/*") of - [] -> - PrevASTs = LatestASTs = []; - Quicks -> - LatestQuick = lists:last(Quicks), - Prev = Quicks -- [LatestQuick], - PrevASTs = lists:append([quick_admin_parse_quick(F) || F <- Prev]), - LatestASTs = quick_admin_parse_quick(LatestQuick) - end, +quick_admin_run_ast(File) -> + Prevs = quick_admin_list_archive_files(), + PrevASTs = lists:append([quick_admin_parse_quick(F) || F <- Prevs]), + LatestASTs = quick_admin_parse_quick(File), {ok, _Env} = run_ast(PrevASTs ++ [switch_old_and_new] ++ LatestASTs). + +quick_admin_list_archive_files() -> + Prevs0 = filelib:wildcard(get_quick_admin_dir() ++ "/*"), + [Fn || Fn <- Prevs0, base_fn_all_digits(Fn)]. + +base_fn_all_digits(Path) -> + Base = filename:basename(Path), + lists:all(fun is_ascii_digit/1, Base). + +is_ascii_digit(X) when $0 =< X, X =< $9 -> + true; +is_ascii_digit(_) -> + false. + +quick_admin_add_archive_file(File) -> + Prevs = quick_admin_list_archive_files(), + N = case [list_to_integer(filename:basename(Fn)) || Fn <- Prevs] of + [] -> 0; + Ns -> lists:max(Ns) + end, + Dir = get_quick_admin_dir(), + NewName = str("~s/~6..0w", [Dir, N + 1]), + ok = file:rename(File, NewName). -- 2.45.2 From 1d1bfadb96a8fcd64e1abd5e9869db7962dd0440 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 17 Dec 2015 12:31:32 +0900 Subject: [PATCH 73/77] Corrections from review --- src/machi_flu_sup.erl | 7 +++++-- src/machi_lifecycle_mgr.erl | 27 ++++++++++++++------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/machi_flu_sup.erl b/src/machi_flu_sup.erl index 9f0e4fd..450f505 100644 --- a/src/machi_flu_sup.erl +++ b/src/machi_flu_sup.erl @@ -116,8 +116,11 @@ sanitize_p_srvr_rec(Whole, {Acc, D}) -> error = dict:find(NameK, D), true = is_atom(PMod), case code:is_loaded(PMod) of - {file, _} -> ok; - _ -> {module, _} = code:load_file(PMod), ok + {file, _} -> + ok; + _ -> + {module, _} = code:load_file(PMod), + ok end, if is_list(Address) -> ok; is_tuple(Address) -> ok % Erlang-style IPv4 or IPv6 diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index c8374d2..94d0fd5 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -301,20 +301,20 @@ sanitize_chain_def_rec(Whole, {Acc, D}) -> mode=Mode, full=Full, witnesses=Witnesses} = Whole, - {true, 10} = {is_atom(Name), 10}, + {true, ?LINE} = {is_atom(Name), ?LINE}, NameK = {name, Name}, - {none, 20} = {gb_trees:lookup(NameK, D), 20}, - {true, 30} = {(Mode == ap_mode orelse Mode == cp_mode), 30}, + {none, ?LINE} = {gb_trees:lookup(NameK, D), ?LINE}, + {true, ?LINE} = {(Mode == ap_mode orelse Mode == cp_mode), ?LINE}, IsPSrvr = fun(X) when is_record(X, p_srvr) -> true; (_) -> false end, - {true, 40} = {lists:all(IsPSrvr, Full), 40}, - {true, 50} = {lists:all(IsPSrvr, Witnesses), 50}, + {true, ?LINE} = {lists:all(IsPSrvr, Full), ?LINE}, + {true, ?LINE} = {lists:all(IsPSrvr, Witnesses), ?LINE}, %% All is sane enough. D2 = gb_trees:enter(NameK, Name, D), {[Whole|Acc], D2} - catch X:Y -> + catch X:Y -> % badmatch will include ?LINE lager:error("~s: Bad chain_def record (~w ~w), skipping: ~P\n", [?MODULE, X, Y, Whole, 15]), {Acc, D} @@ -388,10 +388,9 @@ bootstrap_chain2(#chain_def_v1{name=NewChainName, mode=NewCMode, [NewChainName, FLU, Else, CD]), Else; Else -> - lager:error("Attempt to bootstrap chain ~w via FLU ~w " - "failed: ~w (defn ~w)\n", - [NewChainName, FLU, Else, CD]), - timer:sleep(555), + lager:error("Attempt ~w to bootstrap chain ~w via FLU ~w " + "failed: ~w (may retry with defn ~w)\n", + [N, NewChainName, FLU, Else, CD]), bootstrap_chain2(CD, FLU, N-1) end. @@ -787,9 +786,11 @@ run_ast_cmd({chain, Name, CMode, FullList, Witnesses, _Props}=T, E) -> ok; {ok, C_old} -> {chain, _, OldCMode, _, _, _} = C_old, - if CMode == OldCMode -> ok; - true -> err("Invalid consistency mode: ~w -> ~w", - [OldCMode, CMode], T) + if CMode == OldCMode -> + ok; + true -> + err("Consistency mode change ~w -> ~w is not permitted\n", + [OldCMode, CMode], T) end end, -- 2.45.2 From 51a05ba7707356cfaf4b93bb24282e63d90fe935 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 17 Dec 2015 12:44:10 +0900 Subject: [PATCH 74/77] Fix dialyzer complaints in machi_lifecycle_mgr.erl --- src/machi_lifecycle_mgr.erl | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 94d0fd5..13e7ff3 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -486,13 +486,13 @@ process_pending_flu({File, P}, S) -> _ = move_to_rejected(File, S), S; true -> - case machi_flu_psup:start_flu_package(P) of - {ok, SupPid} -> - lager:info("Started FLU ~w with supervisor pid ~p\n", - [FLU, SupPid]), - _ = move_to_flu_config(FLU, File, S), - S; - Else -> + try + {ok, SupPid} = machi_flu_psup:start_flu_package(P), + lager:info("Started FLU ~w with supervisor pid ~p\n", + [FLU, SupPid]), + _ = move_to_flu_config(FLU, File, S), + S + catch error:Else -> lager:error("Start FLU ~w failed: ~p\n", [FLU, Else]), _ = move_to_rejected(File, S), S @@ -581,7 +581,8 @@ process_pending_chain2(File, CD, RemovedFLUs, ChainConfigAction, S) -> %% another dir on same device, but ... where? _ = file:rename(Src2, Dst2), ok - end || FLU <- LocalRemovedFLUs] + end || FLU <- LocalRemovedFLUs], + ok end, #chain_def_v1{name=Name} = CD, if ChainConfigAction == move -> @@ -933,7 +934,8 @@ diff_env({KV_old, KV_new, _IsNew}=E, RelativeHost) -> {flu, Name, Host, _Port, _Ps} = V, if Host == RelativeHost orelse RelativeHost == all -> {ok, P_srvr} = d_find({kv,{p_srvr,Name}}, E), - Add(P_srvr); + Add(P_srvr), + ok; true -> ok end @@ -986,7 +988,8 @@ diff_env({KV_old, KV_new, _IsNew}=E, RelativeHost) -> mode=CMode, full=Ps_F, witnesses=Ps_W, old_full=OldFull, old_witnesses=OldWitnesses, local_run=Run, local_stop=Stop, - props=Props ++ PropsExtra}); + props=Props ++ PropsExtra}), + ok; false -> ok end -- 2.45.2 From bb0e67f6e0379a13f69370a3e63782d6f8e530ae Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Thu, 17 Dec 2015 18:30:33 +0900 Subject: [PATCH 75/77] Add doc/flu-and-chain-lifecycle.org --- doc/flu-and-chain-lifecycle.org | 620 ++++++++++++++++++++++++++++++++ src/machi_lifecycle_mgr.erl | 93 +---- 2 files changed, 633 insertions(+), 80 deletions(-) create mode 100644 doc/flu-and-chain-lifecycle.org diff --git a/doc/flu-and-chain-lifecycle.org b/doc/flu-and-chain-lifecycle.org new file mode 100644 index 0000000..7b445ce --- /dev/null +++ b/doc/flu-and-chain-lifecycle.org @@ -0,0 +1,620 @@ +FLU and Chain Life Cycle Management -*- mode: org; -*- +#+STARTUP: lognotedone hidestars indent showall inlineimages +#+COMMENT: To generate the outline section: egrep '^\*[*]* ' doc/flu-and-chain-lifecycle.org | egrep -v '^\* Outline' | sed -e 's/^\*\*\* / + /' -e 's/^\*\* / + /' -e 's/^\* /+ /' + +* FLU and Chain Life Cycle Management + +In an ideal world, we (the Machi development team) would have a full +vision of how Machi would be managed, down to the last detail of +beautiful CLI character and network protocol bit. Our vision isn't +complete yet, so we are working one small step at a time. + +* Outline + ++ FLU and Chain Life Cycle Management ++ Terminology review + + Terminology: Machi run-time components/services/thingies + + Terminology: Machi data structures + + Terminology: Cluster-of-cluster (CoC) data structures ++ Overview of administrative life cycles + + Cluster-of-clusters (CoC) administrative life cycle + + Chain administrative life cycle + + FLU server administrative life cycle ++ Quick admin: declarative management of Machi FLU and chain life cycles + + Quick admin uses the "rc.d" config scheme for life cycle management + + Quick admin's declarative "language": an Erlang-flavored AST + + Verb 'host': define a new host for FLU services + + Verb 'flu': define a new FLU + + Verb 'chain': define or reconfigure a chain + + Executing quick admin AST files via the 'machi-admin' utility + + Checking the syntax of an AST file + + Executing an AST file + + Using quick admin to manage multiple machines ++ The "rc.d" style configuration file scheme + + Riak had a similar configuration file editing problem (and its solution) + + Machi's "rc.d" file scheme. + + FLU life cycle management using "rc.d" style files + + The key configuration components of a FLU + + Chain life cycle management using "rc.d" style files + + The key configuration components of a chain + +* Terminology review + +** Terminology: Machi run-time components/services/thingies + ++ FLU: a basic Machi server, responsible for managing a collection of + files. + ++ Chain: a small collection of FLUs that maintain replicas of the same + collection of files. A chain is usually small, 1-3 servers, where + more than 3 would be used only in cases when availability of + certain data is critical despite failures of several machines. + + The length of a chain is directly proportional to its + replication factor, e.g., a chain length=3 will maintain + (nominally) 3 replicas of each file. + + To maintain file availability when ~F~ failures have occurred, a + chain must be at least ~F+1~ members long. (In comparison, the + quorum replication technique requires ~2F+1~ members in the + general case.) + ++ Cluster: this word can be used interchangeably with "chain". + ++ Cluster-of-clusters: A collection of Machi clusters where files are + horizontally partitioned/sharded/distributed across + +** Terminology: Machi data structures + ++ Projection: used to define a single chain: the chain's consistency + mode (strong or eventual consistency), all members (from an + administrative point of view), all active members (from a runtime, + automatically-managed point of view), repairing/file-syncing members + (also runtime, auto-managed), and so on + ++ Epoch: A version number of a projection. The epoch number is used + by both clients & servers to manage transitions from one projection + to another, e.g., when the chain is temporarily shortened by the + failure of a member FLU server. + +** Terminology: Cluster-of-cluster (CoC) data structures + ++ Namespace: A collection of human-friendly names that are mapped to + groups of Machi chains that provide the same type of storage + service: consistency mode, replication policy, etc. + + A single namespace name, e.g. ~normal-ec~, is paired with a single + CoC chart (see below). + + Example: ~normal-ec~ might be a collection of Machi chains in + eventually-consistent mode that are of length=3. + + Example: ~risky-ec~ might be a collection of Machi chains in + eventually-consistent mode that are of length=1. + + Example: ~mgmt-critical~ might be a collection of Machi chains in + strongly-consistent mode that are of length=7. + ++ CoC chart: Encodes the rules which partition/shard/distribute a + particular namespace across a group of chains that collectively + store the namespace's files. + + "chart: noun, a geographical map or plan, especially on used for + navigation by sea or air." + ++ Chain weight: A value assigned to each chain within a CoC chart + structure that defines the relative storage capacity of a chain + within the namespace. For example, a chain weight=150 has 50% more + capacity than a chain weight=100. + ++ CoC chart epoch: The version number assigned to a CoC chart. + +* Overview of administrative life cycles + +** Cluster-of-clusters (CoC) administrative life cycle + ++ CoC is first created ++ CoC adds namespaces (e.g. consistency policy + chain length policy) ++ CoC adds/removes chains to a namespace to increase/decrease the + namespace's storage capacity. ++ CoC adjusts chain weights within a namespace, e.g., to shift files + within the namespace to chains with greater storage capacity + resources and/or runtime I/O resources. + +A CoC "file migration" is the process of moving files from one +namespace member chain to another for purposes of shifting & +re-balancing storage capacity and/or runtime I/O capacity. + +** Chain administrative life cycle + ++ A chain is created with an initial FLU membership list. ++ Chain may be administratively modified zero or more times to + add/remove member FLU servers. ++ A chain may be decommissioned. + +See also: http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html + +** FLU server administrative life cycle + ++ A FLU is created after an administrator chooses the FLU's runtime + location is selected by the administrator: which machine/virtual + machine, IP address and TCP port allocation, etc. ++ An unassigned FLU may be added to a chain by chain administrative + policy. ++ A FLU that is assigned to a chain may be removed from that chain by + chain administrative policy. + + In the current implementation, the FLU's Erlang processes will be + halted. Then the FLU's data and metadata files will be moved to + another area of the disk for safekeeping. Later, a "garbage + collection" process can be used for reclaiming disk space used by + halted FLU servers. + +See also: http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html + +* Quick admin: declarative management of Machi FLU and chain life cycles + +The "quick admin" scheme is a temporary (?) tool for managing Machi +FLU server and chain life cycles in a declarative manner. The API is +described in this section. + +** Quick admin uses the "rc.d" config scheme for life cycle management + +As described at the top of +http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html, the "rc.d" +config files do not manage "policy". "Policy" is doing the right +thing with a Machi cluster-of-clusters from a systems administrator's +point of view. The "rc.d" config files can only implement decisions +made according to policy. + +The "quick admin" tool is a first attempt at automating policy +decisions in a safe way (we hope) that is also easy to implement (we +hope) with a variety of systems management tools, e.g. Chef, Puppet, +Ansible, Saltstack, or plain-old-human-at-a-keyboard. + +** Quick admin's declarative "language": an Erlang-flavored AST + +The "language" that an administrator uses to express desired policy +changes is not (yet) a true language. As a quick implementation hack, +the current language is an Erlang-flavored abstract syntax tree +(AST). The tree isn't very deep, either, frequently just one +element tall. (Not much of a tree, is it?) + +There are three verbs in the language currently: + ++ ~host~, define a new host that can execute FLU servers ++ ~flu~, define a new FLU ++ ~chain~, define a new chain or re-configure an existing chain with + the same name + +*** Verb 'host': define a new host for FLU services + +In this context, a host is a machine, virtual machine, or container +that can execute the Machi application and can therefore provide FLU +services, i.e. file service, Humming Consensus management. + +Two formats may be used to define a new host: + +#+BEGIN_SRC +{host, Name, Props}. +{host, Name, AdminI, ClientI, Props}. +#+END_SRC + +The shorter tuple is shorthand notation for the latter. If the +shorthand form is used, then it will be converted automatically to the +long form as: + +#+BEGIN_SRC +{host, Name, AdminI=Name, ClientI=Name, Props}. +#+END_SRC + +Type information, description, and restrictions: + ++ ~Name::string()~ The ~Name~ attribute must be unique. Note that it + is possible to define two different hosts, one using a DNS hostname + and one using an IP address. The user must avoid this + double-definition because it is not enforced by quick admin. + + The ~Name~ field is used for cross-reference purposes with other + verbs, e.g., ~flu~ and ~chain~. + + There is no syntax yet for removing a host definition. + ++ ~AdminI::string()~ A DNS hostname or IP address for cluster + administration purposes, e.g. SSH access. + + This field is unused at the present time. + ++ ~ClientI::string()~ A DNS hostname or IP address for Machi's client + protocol access, e.g., Protocol Buffers network API service. + + This field is unused at the present time. + ++ ~props::proplist()~ is an Erlang-style property list for specifying + additional configuration options, debugging information, sysadmin + comments, etc. + ++ A full-featured admin tool should also include managing several + other aspects of configuration related to a "host". For example, + for any single IP address, quick admin assumes that there will be + exactly one Erlang VM that is running the Machi application. Of + course, it is possible to have dozens of Erlang VMs on the same + (let's assume for clarity) hardware machine and all running Machi + ... but there are additional aspects of such a machine that quick + admin does not account for + + multiple IP addresses per machine + + multiple Machi package installation paths + + multiple Machi config files (e.g. cuttlefish config, ~etc.conf~, + ~vm.args~) + + multiple data directories/file system mount points + + This is also a management problem for quick admin for a single + Machi package on a machine to take advantage of bulk data + storage using multiple multiple file system mount points. + + multiple Erlang VM host names, required for distributed Erlang, + which is used for communication with ~machi~ and ~machi-admin~ + command line utilities. + + and others.... + +*** Verb 'flu': define a new FLU + +A new FLU is defined relative to a previously-defined ~host~ entities; +an exception will be thrown if the ~host~ cannot be cross-referenced. + +#+BEGIN_SRC +{flu, Name, HostName, Port, Props} +#+END_SRC + +Type information, description, and restrictions: + ++ ~Name::atom()~ The name of the FLU, as a human-friendly name and + also for internal management use; please note the ~atom()~ type. + This name must be unique. + + The ~Name~ field is used for cross-reference purposes with the + ~chain~ verb. + + There is no syntax yet for removing a FLU definition. + ++ ~Hostname::string()~ The cross-reference name of the ~host~ that + this FLU should run on. + ++ ~Port::non_neg_integer()~ The TCP port used by this FLU server's + Protocol Buffers network API listener service + ++ ~props::proplist()~ is an Erlang-style property list for specifying + additional configuration options, debugging information, sysadmin + comments, etc. + +*** Verb 'chain': define or reconfigure a chain + +A chain is defined relative to zero or more previously-defined ~flu~ +entities; an exception will be thrown if any ~flu~ cannot be +cross-referenced. + +Two formats may be used to define/reconfigure a chain: + +#+BEGIN_SRC +{chain, Name, FullList, Props}. +{chain, Name, CMode, FullList, Witnesses, Props}. +#+END_SRC + +The shorter tuple is shorthand notation for the latter. If the +shorthand form is used, then it will be converted automatically to the +long form as: + +#+BEGIN_SRC +{chain, Name, ap_mode, FullList, [], Props}. +#+END_SRC + +Type information, description, and restrictions: + ++ ~Name::atom()~ The name of the chain, as a human-friendly name and + also for internal management use; please note the ~atom()~ type. + This name must be unique. + + There is no syntax yet for removing a chain definition. + ++ ~CMode::'ap_mode'|'cp_mode'~ Defines the consistency mode of the + chain, either eventual consistency or strong consistency, + respectively. + + A chain cannot change consistency mode, e.g., from + strong~->~eventual consistency. + ++ ~FullList::list(atom())~ Specifies the list of full-service FLU + servers, i.e. servers that provide file data & metadata services as + well as Humming Consensus. Each atom in the list must + cross-reference with a previously defined ~chain~; an exception will + be thrown if any ~flu~ cannot be cross-referenced. + ++ ~Witnesses::list(atom())~ Specifies the list of witness-only + servers, i.e. servers that only participate in Humming Consensus. + Each atom in the list must cross-reference with a previously defined + ~chain~; an exception will be thrown if any ~flu~ cannot be + cross-referenced. + + This list must be empty for eventual consistency chains. + ++ ~props::proplist()~ is an Erlang-style property list for specifying + additional configuration options, debugging information, sysadmin + comments, etc. + ++ If this verb specifies a new ~chain~ name, then all of the member + FLU servers (full & witness types) will be bootstrapped to a + starting configuration. + ++ If this verb specifies a previously-defined ~chain~ name, then all + of the member FLU servers (full & witness types, respectively) will + be adjusted to add or remove members, as appropriate. + + Any FLU servers added to either list must not be assigned to any + other chain, or they must be a member of this specific chain. + + Any FLU servers removed from either list will be halted. + (See the "FLU server administrative life cycle" section above.) + +** Executing quick admin AST files via the 'machi-admin' utility + +Examples of quick admin AST files can be found in the +~priv/quick-admin/examples~ directory. Below is an example that will +define a new host ( ~"localhost"~ ), three new FLU servers ( ~f1~ & ~f2~ +and ~f3~ ), and an eventually consistent chain ( ~c1~ ) that uses the new +FLU servers: + +#+BEGIN_SRC +{host, "localhost", []}. +{flu,f1,"localhost",20401,[]}. +{flu,f2,"localhost",20402,[]}. +{flu,f3,"localhost",20403,[]}. +{chain,c1,[f1,f2,f3],[]}. +#+END_SRC + +*** Checking the syntax of an AST file + +Given an AST config file, ~/path/to/ast/file~, its basic syntax and +correctness can be checked without executing it. + +#+BEGIN_SRC +./rel/machi/bin/machi-admin quick-admin-check /path/to/ast/file +#+END_SRC + ++ The utility will exit with status zero and output ~ok~ if the syntax + and proposed configuration appears to be correct. ++ If there is an error, the utility will exit with status one, and an + error message will be printed. + +*** Executing an AST file + +Given an AST config file, ~/path/to/ast/file~, it can be executed +using the command: + +#+BEGIN_SRC +./rel/machi/bin/machi-admin quick-admin-apply /path/to/ast/file RelativeHost +#+END_SRC + +... where the last argument, ~RelativeHost~, should be the exact +spelling of one of the previously defined AST ~host~ entities, +*and also* is the same host that the ~machi-admin~ utility is being +executed on. + +Restrictions and warnings: + ++ This is alpha quality software. + ++ There is no "undo". + + Of course there is, but you need to resort to doing things like + using ~machi attach~ to attach to the server's CLI to then execute + magic Erlang incantations to stop FLUs, unconfigure chains, etc. + + Oh, and delete some files with magic paths, also. + +** Using quick admin to manage multiple machines + +A quick sketch follows: + +1. Create the AST file to specify all of the changes that you wish to + make to all hosts, FLUs, and/or chains, e.g., ~/tmp/ast.txt~. +2. Check the basic syntax with the ~quick-admin-check~ argument to + ~machi-admin~. +3. If the syntax is good, then copy ~/tmp/ast.txt~ to all hosts in the + cluster, using the same path, ~/tmp/ast.txt~. +4. For each machine in the cluster, run: +#+BEGIN_SRC +./rel/machi/bin/machi-admin quick-admin-apply /tmp/ast.txt RelativeHost +#+END_SRC + +... where RelativeHost is the AST ~host~ name of the machine that you +are executing the ~machi-admin~ command on. The command should be +successful, with exit status 0 and outputting the string ~ok~. + +Finally, for each machine in the cluster, a listing of all files in +the directory ~rel/machi/etc/quick-admin-archive~ should show exactly +the same files, one for each time that ~quick-admin-apply~ has been +run successfully on that machine. + +* The "rc.d" style configuration file scheme + +This configuration scheme is inspired by BSD UNIX's ~init(8)~ process +manager's configuration style, called "rc.d" after the name of the +directory where these files are stored, ~/etc/rc.d~. The ~init~ +process is responsible for (among other things) starting UNIX +processes at machine boot time and stopping them when the machine is +shut down. + +The original scheme used by ~init~ to start processes at boot time was +a single Bourne shell script called ~/etc/rc~. When a new software +package was installed that required a daemon to be started at boot +time, text was added to the ~/etc/rc~ file. Uninstalling packages was +much trickier, because it meant removing lines from a file that +*is a computer program (run by the Bourne shell, a Turing-complete +programming language)*. Error-free editing of the ~/etc/rc~ script +was impossible in all cases. + +Later, ~init~'s configuration was split into a few master Bourne shell +scripts and a subdirectory, ~/etc/rc.d~. The subdirectory contained +shell scripts that were responsible for boot time starting of a single +daemon or service, e.g. NFS or an HTTP server. When a new software +package was added, a new file was added to the ~rc.d~ subdirectory. +When a package was removed, the corresponding file in ~rc.d~ was +removed. With this simple scheme, addition & removal of boot time +scripts was vastly simplified. + +** Riak had a similar configuration file editing problem (and its solution) + +Another software product from Basho Technologies, Riak, had a similar +configuration file editing problem. One file in particular, +~app.config~, had a syntax that made it difficult both for human +systems administrators and also computer programs to edit the file in +a syntactically correct manner. + +Later releases of Riak switched to an alternative configuration file +format, one inspired by the BSD UNIX ~sysctl(8)~ utility and +~sysctl.conf(5)~ file syntax. The ~sysctl.conf~ format is much easier +to manage by computer programs to add items. Removing items is not +100% simple, however: the correct lines must be identified and then +removed (e.g. with Perl or a text editor or combination of ~grep -v~ +and ~mv~), but removing any comment lines that "belong" to the removed +config item(s) is not any easy for a 1-line shell script to do 100% +correctly. + +Machi will use the ~sysctl.conf~ style configuration for some +application configuration variables. However, adding & removing FLUs +and chains will be managed using the "rc.d" style because of the +"rc.d" scheme's simplicity and tolerance of mistakes by administrators +(human or computer). + +** Machi's "rc.d" file scheme. + +Machi will use a single subdirectory that will contain configuration +files for some life cycle management task, e.g. a single FLU or a +single chain. + +The contents of the file should be a single Erlang term, serialized in +ASCII form as Erlang source code statement, i.e. a single Erlang term +~T~ that is formatted by ~io:format("~w.",[T]).~. This file must be +parseable by the Erlang function ~file:consult()~. + +Later versions of Machi may change the file format to be more familiar +to administrators who are unaccustomed to Erlang language syntax. + +** FLU life cycle management using "rc.d" style files + +*** The key configuration components of a FLU + +1. The machine (or virtual machine) to run it on. +2. The Machi software package's artifacts to execute. +3. The disk device(s) used to store Machi file data & metadata, "rc.d" + style config files, etc. +4. The name, IP address and TCP port assigned to the FLU service. +5. Its chain assignment. + +Notes: + ++ Items 1-3 are currently outside of the scope of this life cycle + document. We assume that human administrators know how to do these + things. ++ Item 4's properties are explicitly managed by a FLU-defining "rc.d" + style config file. ++ Item 5 is managed by the chain life cycle management system. + +Here is an example of a properly formatted FLU config file: + +#+BEGIN_SRC +{p_srvr,f1,machi_flu1_client,"192.168.72.23",20401,[]}. +#+END_SRC + +... which corresponds to the following Erlang record definition: + +#+BEGIN_SRC +-record(p_srvr, { + name :: atom(), + proto_mod = 'machi_flu1_client' :: atom(), % Module name + address :: term(), % Protocol-specific + port :: term(), % Protocol-specific + props = [] :: list() % proplist for other related info + }). +#+END_SRC + ++ ~name~ is ~f1~. This is name of the FLU. This name should be + unique over the lifetime of the administrative domain and thus + managed by external policy. This name must be the same as the name + of the config file that defines the FLU. ++ ~proto_mod~ is used for internal management purposes and should be + considered a mandatory constant. ++ ~address~ is "192.168.72.23". The DNS hostname or IP address used + by other servers to communicate with this FLU. This must be a valid + IP address, previously assigned to this machine/VM using the + appropriate operating system-specific procedure. ++ ~port~ is TCP port 20401. The TCP port number that the FLU listens + to for incoming Protocol Buffers-serialized communication. This TCP + port must not be in use (now or in the future) by another Machi FLU + or any other process running on this machine/VM. ++ ~props~ is an Erlang-style property list for specifying additional + configuration options, debugging information, sysadmin comments, + etc. + +** Chain life cycle management using "rc.d" style files + +Unlike FLUs, chains have a self-management aspect that makes a chain +life cycle different from a single FLU server. Machi's chains are +self-managing, via Humming Consensus; see the +https://github.com/basho/machi/tree/master/doc/ directory for much +more detail about Humming Consensus. After FLUs have received their +initial chain configuration for Humming Consensus, the FLUs will +manage the chain (and each other) by themselves. + +However, Humming Consensus does not handle three chain management +problems: + +1. Specifying the very first chain configuration, +2. Altering the membership of the chain (i.e. adding/removing FLUs + from the chain), +3. Stopping the chain permanently. + +A chain "rc.d" file will only be used to bootstrap a newly-defined FLU +server. It's like a piece of glue information to introduce the new +FLU to the Humming Consensus group that is managing the chain's +dynamic state (e.g. which members are up or down). In all other +respects, chain config files are ignored by life cycle management code. +However, to mimic the life cycle of the FLU server's "rc.d" config +files, a chain "rc.d" files is not deleted until the chain has been +decommissioned (i.e. defined with length=0). + +*** The key configuration components of a chain + +1. The name of the chain. +2. Consistency mode: eventually consistent or strongly consistent. +3. The membership list of all FLU servers in the chain. + + Remember, all servers in a single chain will manage full replicas + of the same collection of Machi files. +4. If the chain is defined to use strongly consistent mode, then a + list of "witness servers" may also be defined. See the + [https://github.com/basho/machi/tree/master/doc/] documentation for + more information on witness servers. + + The witness list must be empty for all chains in eventual + consistency mode. + +Here is an example of a properly formatted chain config file: + +#+BEGIN_SRC +{chain_def_v1,c1,ap_mode, + [{p_srvr,f1,machi_flu1_client,"localhost",20401,[]}, + {p_srvr,f2,machi_flu1_client,"localhost",20402,[]}, + {p_srvr,f3,machi_flu1_client,"localhost",20403,[]}], + [],[],[], + [f1,f2,f3], + [],[]}. +#+END_SRC + +... which corresponds to the following Erlang record definition: + +#+BEGIN_SRC +-record(chain_def_v1, { + name :: atom(), % chain name + mode :: 'ap_mode' | 'cp_mode', + full = [] :: [p_srvr()], + witnesses = [] :: [p_srvr()], + old_full = [] :: [atom()], % guard against some races + old_witnesses=[] :: [atom()], % guard against some races + local_run = [] :: [atom()], % must be tailored to each machine! + local_stop = [] :: [atom()], % must be tailored to each machine! + props = [] :: list() % proplist for other related info + }). +#+END_SRC + ++ ~name~ is ~c1~, the name of the chain. This name should be unique + over the lifetime of the administrative domain and thus managed by + external policy. This name must be the same as the name of the + config file that defines the chain. ++ ~mode~ is ~ap_mode~, an internal code symbol for eventual + consistency mode. ++ ~full~ is a list of Erlang ~#p_srvr{}~ records for full-service + members of the chain, i.e., providing Machi file data & metadata + storage services. ++ ~witnesses~ is a list of Erlang ~#p_srvr{}~ records for witness-only + FLU servers, i.e., providing only Humming Consensus service. ++ The next four fields are used for internal management only. ++ ~props~ is an Erlang-style property list for specifying additional + configuration options, debugging information, sysadmin comments, + etc. + diff --git a/src/machi_lifecycle_mgr.erl b/src/machi_lifecycle_mgr.erl index 13e7ff3..9d4a688 100644 --- a/src/machi_lifecycle_mgr.erl +++ b/src/machi_lifecycle_mgr.erl @@ -81,115 +81,47 @@ %% %% == The FLU Lifecycle == %% +%% See also: [https://github.com/basho/machi/tree/master/doc/flu-and-chain-lifecycle.org] +%% %% FLUs on the local machine may be started and stopped, as defined by %% administrative policy. In order to do any useful work, however, a %% running FLU must also be configured to be a member of a replication %% chain. Thus, as a practical matter, both a FLU and the chain that %% the FLU participates in must both be managed by this manager. %% -%% When a new `rc.d'-style config file is written to the FLU -%% definition directory, a Machi server process will discover the file -%% within a certain period of time, e.g. 15 seconds. The FLU will be -%% started with the file's specified parameters. A FLU should be -%% defined and started before configuring its chain membership. +%% The FLU will be started with the file's specified parameters. A +%% FLU should be defined and started before configuring its chain +%% membership. %% %% Usually a FLU is removed implicitly by removing that FLU from the a -%% newer definition file for the chain, or by deleting the entire -%% chain definition. If a FLU has been started but never been a chain -%% member, then the FLU can be stopped & removed explicitly. -%% -%% When a FLU has been removed by policy, the FLU's data files are set -%% aside into a temporary area. An additional policy command may be -%% used to permanently delete such FLUs' data files, i.e. to reclaim -%% disk space. -%% -%% Resources for the FLU are defined in `machi_projection.hrl' -%% in the `p_srvr{}' record. The major elements of this record are: -%% -%%
    -%% -%%
  • name :: atom(): The name of the FLU. This name -%% should be unique over the lifetime of the administrative -%% domain and thus managed by external policy. This name must be -%% the same as the name of the `rc.d'-style config file that -%% defines the FLU. -%%
    • -%%
  • address :: string(): The DNS hostname or IP address -%% used by other servers to communicate with this FLU. -%%
    • -%%
  • port :: non_neg_integer() : The TCP port number that -%% the FLU listens to for incoming Protocol Buffers-serialized -%% communication. -%%
    • -%%
  • props :: property_list(): A general-purpose property -%% list. Its use is currently fluid & not well-defined yet. -%%
    • -%%
+%% newer definition of its chain (by omitting the FLU's name). +%% If a FLU has been started but never been a chain +%% member, then the FLU can be stopped and removed explicitly. %% %% A FLU may be created or removed (via implicit or explicit policy). %% An existing FLU may not be reconfigured. %% %% == The Chain Lifecycle == %% +%% See also: [https://github.com/basho/machi/tree/master/doc/flu-and-chain-lifecycle.org] +%% %% If a FLU on the local machine is expected to participate in a %% replication chain, then an `rc.d'-style chain definition file must %% also be present on each machine that runs a FLU in the chain. %% -%% Machi's chains are self-managing, via Humming Consensus; see the -%% [https://github.com/basho/machi/tree/master/doc/] directory for -%% much more detail about Humming Consensus. After FLUs have received -%% their initial chain configuration for Humming Consensus, the FLUs -%% will manage each other (and the chain) themselves. -%% -%% However, Humming Consensus does not handle three chain management -%% problems: 1. specifying the very first chain configuration, -%% 2. altering the membership of the chain (adding/removing FLUs from -%% the chain), or 3. stopping the chain permanently. -%% %% FLUs in a new chain should have definition files created on each %% FLU's respective machine prior to defining their chain. Similarly, %% on each machine that hosts a chain member, a chain definition file %% created. External policy is responsible for creating each of these %% files. %% -%% Resources for the chain are defined in `machi_projection.hrl' -%% in the `chain_def_v1{}' record. The major elements of this record are: -%% -%%
    -%% -%%
  • name :: atom(): The name of the chain. This name -%% should be unique over the lifetime of the administrative -%% domain and thus managed by external policy. This name must be -%% the same as the name of the `rc.d'-style config file that -%% defines the chain. -%%
    • -%%
  • mode :: 'ap_mode' | 'cp_mode': This is the consistency -%% to be used for managing the chain's replicated data: eventual -%% consistency and strong consistency, respectively. -%%
    • -%%
  • full :: [#p_srvr{}] : A list of `#p_srvr{}' records -%% to define the full-service members of the chain. -%%
    • -%%
  • witnesses :: [#p_srvr{}] : A list of `#p_srvr{}' records -%% to define the witness-only members of the chain. Witness servers -%% may only be used with strong consistency mode. -%%
    • -%%
  • props :: property_list(): A general-purpose property -%% list. Its use is currently fluid & not well-defined yet. -%%
    • -%%
-%% -%% A chain may be created, removed, or modified. +%% A chain may be created or modified. %% %% A modification request writes a `#chain_def_v1{}' record with the %% same name but different `full' and/or `witnesses' list to each %% machine that hosts a FLU in the chain (in both the old and new %% versions of the chain). %% -%% A deletion request writes a `#chain_def_v1{}' record with the same -%% name but empty lists for `full' and `witnesses' to each machine -%% that hosts a FLU in the chain (in the old version of the chain). -%% %% == Conflicts with TCP ports, FLU & chain names, etc == %% %% This manager is not responsible for managing conflicts in resource @@ -1080,4 +1012,5 @@ quick_admin_add_archive_file(File) -> end, Dir = get_quick_admin_dir(), NewName = str("~s/~6..0w", [Dir, N + 1]), - ok = file:rename(File, NewName). + {ok, Contents} = file:read_file(File), + ok = file:write_file(NewName, Contents). -- 2.45.2 From 0922def0d690d4a7ec7cd5aab25a8d7d66caea2e Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 18 Dec 2015 11:50:15 +0900 Subject: [PATCH 76/77] s/verb/term/gi --- doc/flu-and-chain-lifecycle.org | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/flu-and-chain-lifecycle.org b/doc/flu-and-chain-lifecycle.org index 7b445ce..4672080 100644 --- a/doc/flu-and-chain-lifecycle.org +++ b/doc/flu-and-chain-lifecycle.org @@ -23,9 +23,9 @@ complete yet, so we are working one small step at a time. + Quick admin: declarative management of Machi FLU and chain life cycles + Quick admin uses the "rc.d" config scheme for life cycle management + Quick admin's declarative "language": an Erlang-flavored AST - + Verb 'host': define a new host for FLU services - + Verb 'flu': define a new FLU - + Verb 'chain': define or reconfigure a chain + + Term 'host': define a new host for FLU services + + Term 'flu': define a new FLU + + Term 'chain': define or reconfigure a chain + Executing quick admin AST files via the 'machi-admin' utility + Checking the syntax of an AST file + Executing an AST file @@ -172,14 +172,14 @@ the current language is an Erlang-flavored abstract syntax tree (AST). The tree isn't very deep, either, frequently just one element tall. (Not much of a tree, is it?) -There are three verbs in the language currently: +There are three terms in the language currently: + ~host~, define a new host that can execute FLU servers + ~flu~, define a new FLU + ~chain~, define a new chain or re-configure an existing chain with the same name -*** Verb 'host': define a new host for FLU services +*** Term 'host': define a new host for FLU services In this context, a host is a machine, virtual machine, or container that can execute the Machi application and can therefore provide FLU @@ -207,7 +207,7 @@ Type information, description, and restrictions: and one using an IP address. The user must avoid this double-definition because it is not enforced by quick admin. + The ~Name~ field is used for cross-reference purposes with other - verbs, e.g., ~flu~ and ~chain~. + terms, e.g., ~flu~ and ~chain~. + There is no syntax yet for removing a host definition. + ~AdminI::string()~ A DNS hostname or IP address for cluster @@ -243,7 +243,7 @@ Type information, description, and restrictions: command line utilities. + and others.... -*** Verb 'flu': define a new FLU +*** Term 'flu': define a new FLU A new FLU is defined relative to a previously-defined ~host~ entities; an exception will be thrown if the ~host~ cannot be cross-referenced. @@ -258,7 +258,7 @@ Type information, description, and restrictions: also for internal management use; please note the ~atom()~ type. This name must be unique. + The ~Name~ field is used for cross-reference purposes with the - ~chain~ verb. + ~chain~ term. + There is no syntax yet for removing a FLU definition. + ~Hostname::string()~ The cross-reference name of the ~host~ that @@ -271,7 +271,7 @@ Type information, description, and restrictions: additional configuration options, debugging information, sysadmin comments, etc. -*** Verb 'chain': define or reconfigure a chain +*** Term 'chain': define or reconfigure a chain A chain is defined relative to zero or more previously-defined ~flu~ entities; an exception will be thrown if any ~flu~ cannot be @@ -322,11 +322,11 @@ Type information, description, and restrictions: additional configuration options, debugging information, sysadmin comments, etc. -+ If this verb specifies a new ~chain~ name, then all of the member ++ If this term specifies a new ~chain~ name, then all of the member FLU servers (full & witness types) will be bootstrapped to a starting configuration. -+ If this verb specifies a previously-defined ~chain~ name, then all ++ If this term specifies a previously-defined ~chain~ name, then all of the member FLU servers (full & witness types, respectively) will be adjusted to add or remove members, as appropriate. + Any FLU servers added to either list must not be assigned to any -- 2.45.2 From d602663060fdb9610a8024b8a4a856899cbfa1d7 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Fri, 18 Dec 2015 13:43:18 +0900 Subject: [PATCH 77/77] Ignore RUNLOG* --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0f6b627..3af54ff 100644 --- a/.gitignore +++ b/.gitignore @@ -25,5 +25,6 @@ rel/machi *.patch current_counterexample.eqc foo* +RUNLOG* typescript* *.swp -- 2.45.2