greg/machi
1
0
Fork 0
Code Issues 11 Pull requests 1 Projects Releases Packages Wiki Activity Actions

All eunit tests now passing again

Browse source
This commit is contained in:
Scott Lystig Fritchie 2015-10-19 21:08:38 +09:00
parent 014ba89e3a
commit 10a27ce7dd
9 changed files with 295 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
include/machi.hrl
View file

@ -40,3 +40,6 @@
%% TODO: it's used in flu_sup and elsewhere, change this to suitable name
-define(TEST_ETS_TABLE, test_ets_table).
-define(DEFAULT_COC_NAMESPACE, "").
-define(DEFAULT_COC_LOCATOR, 0).

23
src/machi.proto
View file

@ -170,11 +170,12 @@ message Mpb_AuthResp {
// High level API: append_chunk() request & response
message Mpb_AppendChunkReq {
optional bytes placement_key = 1;
required string prefix = 2;
required bytes chunk = 3;
required Mpb_ChunkCSum csum = 4;
optional uint32 chunk_extra = 5;
required string coc_namespace = 1;
required uint32 coc_locator = 2;
required string prefix = 3;
required bytes chunk = 4;
required Mpb_ChunkCSum csum = 5;
optional uint32 chunk_extra = 6;
}
message Mpb_AppendChunkResp {
@ -378,11 +379,13 @@ message Mpb_ProjectionV1 {
message Mpb_LL_AppendChunkReq {
required Mpb_EpochID epoch_id = 1;
optional bytes placement_key = 2;
required string prefix = 3;
required bytes chunk = 4;
required Mpb_ChunkCSum csum = 5;
optional uint32 chunk_extra = 6;
/* To avoid CoC use, use coc_namespace="" and coc_locator=0 */
required string coc_namespace = 2;
required uint32 coc_locator = 3;
required string prefix = 4;
required bytes chunk = 5;
required Mpb_ChunkCSum csum = 6;
optional uint32 chunk_extra = 7;
}
message Mpb_LL_AppendChunkResp {

125
src/machi_cr_client.erl
View file

@ -119,7 +119,9 @@
-export([
%% File API
append_chunk/3, append_chunk/4,
append_chunk/5, append_chunk/6,
append_chunk_extra/4, append_chunk_extra/5,
append_chunk_extra/6, append_chunk_extra/7,
write_chunk/4, write_chunk/5,
read_chunk/5, read_chunk/6,
trim_chunk/4, trim_chunk/5,
@ -164,13 +166,29 @@ start_link(P_srvr_list, Opts) ->
%% with `Prefix'.
append_chunk(PidSpec, Prefix, Chunk) ->
append_chunk(PidSpec, Prefix, Chunk, ?DEFAULT_TIMEOUT).
append_chunk_extra(PidSpec, ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR,
Prefix, Chunk, 0, ?DEFAULT_TIMEOUT).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
append_chunk(PidSpec, Prefix, Chunk, Timeout) ->
append_chunk_extra(PidSpec, Prefix, Chunk, 0, Timeout).
append_chunk_extra(PidSpec, ?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR,
Prefix, Chunk, 0, Timeout).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
append_chunk(PidSpec, CoC_Namespace, CoC_Locator, Prefix, Chunk) ->
append_chunk_extra(PidSpec, CoC_Namespace, CoC_Locator,
Prefix, Chunk, 0, ?DEFAULT_TIMEOUT).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
append_chunk(PidSpec, CoC_Namespace, CoC_Locator, Prefix, Chunk, Timeout) ->
append_chunk_extra(PidSpec, CoC_Namespace, CoC_Locator,
Prefix, Chunk, 0, Timeout).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
@ -184,7 +202,25 @@ append_chunk_extra(PidSpec, Prefix, Chunk, ChunkExtra)
append_chunk_extra(PidSpec, Prefix, Chunk, ChunkExtra, Timeout0) ->
{TO, Timeout} = timeout(Timeout0),
gen_server:call(PidSpec, {req, {append_chunk_extra, Prefix,
gen_server:call(PidSpec, {req, {append_chunk_extra,
?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR,
Prefix,
Chunk, ChunkExtra, TO}},
Timeout).
append_chunk_extra(PidSpec, CoC_Namespace, CoC_Locator, Prefix, Chunk, ChunkExtra)
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
append_chunk_extra(PidSpec, CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra, ?DEFAULT_TIMEOUT).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
append_chunk_extra(PidSpec, CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra, Timeout0) ->
{TO, Timeout} = timeout(Timeout0),
gen_server:call(PidSpec, {req, {append_chunk_extra,
CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, TO}},
Timeout).
@ -288,8 +324,10 @@ code_change(_OldVsn, S, _Extra) ->
%%%%%%%%%%%%%%%%%%%%%%%%%%%
handle_call2({append_chunk_extra, Prefix, Chunk, ChunkExtra, TO}, _From, S) ->
do_append_head(Prefix, Chunk, ChunkExtra, 0, os:timestamp(), TO, S);
handle_call2({append_chunk_extra, CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra, TO}, _From, S) ->
do_append_head(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, 0, os:timestamp(), TO, S);
handle_call2({write_chunk, File, Offset, Chunk, TO}, _From, S) ->
do_write_head(File, Offset, Chunk, 0, os:timestamp(), TO, S);
handle_call2({read_chunk, File, Offset, Size, Opts, TO}, _From, S) ->
@ -301,9 +339,12 @@ handle_call2({checksum_list, File, TO}, _From, S) ->
handle_call2({list_files, TO}, _From, S) ->
do_list_files(0, os:timestamp(), TO, S).
do_append_head(Prefix, Chunk, ChunkExtra, 0=Depth, STime, TO, S) ->
do_append_head2(Prefix, Chunk, ChunkExtra, Depth + 1, STime, TO, S);
do_append_head(Prefix, Chunk, ChunkExtra, Depth, STime, TO, #state{proj=P}=S) ->
do_append_head(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, 0=Depth, STime, TO, S) ->
do_append_head2(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, Depth + 1, STime, TO, S);
do_append_head(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, Depth, STime, TO, #state{proj=P}=S) ->
%% io:format(user, "head sleep1,", []),
sleep_a_while(Depth),
DiffMs = timer:now_diff(os:timestamp(), STime) div 1000,
@ -318,53 +359,62 @@ do_append_head(Prefix, Chunk, ChunkExtra, Depth, STime, TO, #state{proj=P}=S) ->
case S2#state.proj of
P2 when P2 == undefined orelse
P2#projection_v1.upi == [] ->
do_append_head(Prefix, Chunk, ChunkExtra, Depth + 1,
do_append_head(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, Depth + 1,
STime, TO, S2);
_ ->
do_append_head2(Prefix, Chunk, ChunkExtra, Depth + 1,
do_append_head2(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, Depth + 1,
STime, TO, S2)
end
end.
do_append_head2(Prefix, Chunk, ChunkExtra, Depth, STime, TO,
do_append_head2(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, Depth, STime, TO,
#state{proj=P}=S) ->
[HeadFLU|_RestFLUs] = mutation_flus(P),
case is_witness_flu(HeadFLU, P) of
true ->
case witnesses_use_our_epoch(S) of
true ->
do_append_head3(Prefix, Chunk, ChunkExtra, Depth,
do_append_head3(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, Depth,
STime, TO, S);
false ->
%% Bummer, go back to the beginning and retry.
do_append_head(Prefix, Chunk, ChunkExtra, Depth,
do_append_head(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, Depth,
STime, TO, S)
end;
false ->
do_append_head3(Prefix, Chunk, ChunkExtra, Depth, STime, TO, S)
do_append_head3(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, Depth, STime, TO, S)
end.
do_append_head3(Prefix, Chunk, ChunkExtra, Depth, STime, TO,
do_append_head3(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, Depth, STime, TO,
#state{epoch_id=EpochID, proj=P, proxies_dict=PD}=S) ->
[HeadFLU|RestFLUs] = non_witness_flus(mutation_flus(P), P),
Proxy = orddict:fetch(HeadFLU, PD),
case ?FLU_PC:append_chunk_extra(Proxy,
EpochID, Prefix, Chunk, ChunkExtra,
?TIMEOUT) of
case ?FLU_PC:append_chunk_extra(Proxy, EpochID,
CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, ?TIMEOUT) of
{ok, {Offset, _Size, File}=_X} ->
%% io:format(user, "append ~w,", [HeadFLU]),
do_append_midtail(RestFLUs, Prefix, File, Offset, Chunk, ChunkExtra,
do_append_midtail(RestFLUs, CoC_Namespace, CoC_Locator, Prefix,
File, Offset, Chunk, ChunkExtra,
[HeadFLU], 0, STime, TO, S);
{error, bad_checksum}=BadCS ->
{reply, BadCS, S};
{error, Retry}
when Retry == partition; Retry == bad_epoch; Retry == wedged ->
do_append_head(Prefix, Chunk, ChunkExtra, Depth, STime, TO, S);
do_append_head(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, Depth, STime, TO, S);
{error, written} ->
%% Implicit sequencing + this error = we don't know where this
%% written block is. But we lost a race. Repeat, with a new
%% sequencer assignment.
do_append_head(Prefix, Chunk, ChunkExtra, Depth, STime, TO, S);
do_append_head(CoC_Namespace, CoC_Locator, Prefix,
Chunk, ChunkExtra, Depth, STime, TO, S);
{error, trimmed} = Err ->
%% TODO: behaviour
{reply, Err, S};
@ -373,12 +423,15 @@ do_append_head3(Prefix, Chunk, ChunkExtra, Depth, STime, TO,
Prefix,iolist_size(Chunk)})
end.
do_append_midtail(RestFLUs, Prefix, File, Offset, Chunk, ChunkExtra,
do_append_midtail(RestFLUs, CoC_Namespace, CoC_Locator, Prefix,
File, Offset, Chunk, ChunkExtra,
Ws, Depth, STime, TO, S)
when RestFLUs == [] orelse Depth == 0 ->
do_append_midtail2(RestFLUs, Prefix, File, Offset, Chunk, ChunkExtra,
do_append_midtail2(RestFLUs, CoC_Namespace, CoC_Locator, Prefix,
File, Offset, Chunk, ChunkExtra,
Ws, Depth + 1, STime, TO, S);
do_append_midtail(_RestFLUs, Prefix, File, Offset, Chunk, ChunkExtra,
do_append_midtail(_RestFLUs, CoC_Namespace, CoC_Locator, Prefix, File,
Offset, Chunk, ChunkExtra,
Ws, Depth, STime, TO, #state{proj=P}=S) ->
%% io:format(user, "midtail sleep2,", []),
sleep_a_while(Depth),
@ -405,36 +458,43 @@ do_append_midtail(_RestFLUs, Prefix, File, Offset, Chunk, ChunkExtra,
if Prefix == undefined -> % atom! not binary()!!
{error, partition};
true ->
do_append_head2(Prefix, Chunk, ChunkExtra,
do_append_head2(CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra,
Depth, STime, TO, S2)
end;
RestFLUs3 ->
do_append_midtail2(RestFLUs3, Prefix, File, Offset,
do_append_midtail2(RestFLUs3,
CoC_Namespace, CoC_Locator,
Prefix, File, Offset,
Chunk, ChunkExtra,
Ws, Depth + 1, STime, TO, S2)
end
end
end.
do_append_midtail2([], _Prefix, File, Offset, Chunk,
do_append_midtail2([], _CoC_Namespace, _CoC_Locator,
_Prefix, File, Offset, Chunk,
_ChunkExtra, _Ws, _Depth, _STime, _TO, S) ->
%% io:format(user, "ok!\n", []),
{reply, {ok, {Offset, chunk_wrapper_size(Chunk), File}}, S};
do_append_midtail2([FLU|RestFLUs]=FLUs, Prefix, File, Offset, Chunk,
do_append_midtail2([FLU|RestFLUs]=FLUs, CoC_Namespace, CoC_Locator,
Prefix, File, Offset, Chunk,
ChunkExtra, Ws, Depth, STime, TO,
#state{epoch_id=EpochID, proxies_dict=PD}=S) ->
Proxy = orddict:fetch(FLU, PD),
case ?FLU_PC:write_chunk(Proxy, EpochID, File, Offset, Chunk, ?TIMEOUT) of
ok ->
%% io:format(user, "write ~w,", [FLU]),
do_append_midtail2(RestFLUs, Prefix, File, Offset, Chunk,
do_append_midtail2(RestFLUs, CoC_Namespace, CoC_Locator, Prefix,
File, Offset, Chunk,
ChunkExtra, [FLU|Ws], Depth, STime, TO, S);
{error, bad_checksum}=BadCS ->
%% TODO: alternate strategy?
{reply, BadCS, S};
{error, Retry}
when Retry == partition; Retry == bad_epoch; Retry == wedged ->
do_append_midtail(FLUs, Prefix, File, Offset, Chunk,
do_append_midtail(FLUs, CoC_Namespace, CoC_Locator, Prefix,
File, Offset, Chunk,
ChunkExtra, Ws, Depth, STime, TO, S);
{error, written} ->
%% We know what the chunk ought to be, so jump to the
@ -499,7 +559,8 @@ do_write_head2(File, Offset, Chunk, Depth, STime, TO,
ok ->
%% From this point onward, we use the same code & logic path as
%% append does.
do_append_midtail(RestFLUs, undefined, File, Offset, Chunk,
do_append_midtail(RestFLUs, undefined, undefined, undefined,
File, Offset, Chunk,
undefined, [HeadFLU], 0, STime, TO, S);
{error, bad_checksum}=BadCS ->
{reply, BadCS, S};

4
src/machi_dt.erl
View file

@ -29,6 +29,8 @@
-type chunk_s() :: 'trimmed' | binary().
-type chunk_pos() :: {file_offset(), chunk_size(), file_name_s()}.
-type chunk_size() :: non_neg_integer().
-type coc_namespace() :: string().
-type coc_locator() :: non_neg_integer().
-type error_general() :: 'bad_arg' | 'wedged' | 'bad_checksum'.
-type epoch_csum() :: binary().
-type epoch_num() :: -1 | non_neg_integer().
@ -58,6 +60,8 @@
chunk_s/0,
chunk_pos/0,
chunk_size/0,
coc_namespace/0,
coc_locator/0,
error_general/0,
epoch_csum/0,
epoch_num/0,

47
src/machi_flu1.erl
View file

@ -229,24 +229,27 @@ append_server_loop(FluPid, #state{wedged=Wedged_p,
witness=Witness_p,
epoch_id=OldEpochId, flu_name=FluName}=S) ->
receive
{seq_append, From, _Prefix, _Chunk, _CSum, _Extra, _EpochID}
{seq_append, From, _N, _L, _Prefix, _Chunk, _CSum, _Extra, _EpochID}
when Witness_p ->
%% The FLU's net_server_loop() process ought to filter all
%% witness states, but we'll keep this clause for extra
%% paranoia.
From ! witness,
append_server_loop(FluPid, S);
{seq_append, From, _Prefix, _Chunk, _CSum, _Extra, _EpochID}
{seq_append, From, _N, _L, _Prefix, _Chunk, _CSum, _Extra, _EpochID}
when Wedged_p ->
From ! wedged,
append_server_loop(FluPid, S);
{seq_append, From, Prefix, Chunk, CSum, Extra, EpochID} ->
{seq_append, From, CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum, Extra, EpochID} ->
%% Old is the one from our state, plain old 'EpochID' comes
%% from the client.
_ = case OldEpochId == EpochID of
true ->
spawn(fun() ->
append_server_dispatch(From, Prefix, Chunk, CSum, Extra, FluName, EpochID)
append_server_dispatch(From, CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum, Extra,
FluName, EpochID)
end);
false ->
From ! {error, bad_epoch}
@ -401,10 +404,12 @@ do_pb_ll_request3({low_wedge_status, _EpochID}, S) ->
do_pb_ll_request3({low_proj, PCMD}, S) ->
{do_server_proj_request(PCMD, S), S};
%% Witness status *matters* below
do_pb_ll_request3({low_append_chunk, _EpochID, PKey, Prefix, Chunk, CSum_tag,
do_pb_ll_request3({low_append_chunk, _EpochID, CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum_tag,
CSum, ChunkExtra},
#state{witness=false}=S) ->
{do_server_append_chunk(PKey, Prefix, Chunk, CSum_tag, CSum,
{do_server_append_chunk(CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum_tag, CSum,
ChunkExtra, S), S};
do_pb_ll_request3({low_write_chunk, _EpochID, File, Offset, Chunk, CSum_tag,
CSum},
@ -444,10 +449,12 @@ do_pb_hl_request2({high_echo, Msg}, S) ->
{Msg, S};
do_pb_hl_request2({high_auth, _User, _Pass}, S) ->
{-77, S};
do_pb_hl_request2({high_append_chunk, _todoPK, Prefix, ChunkBin, TaggedCSum,
do_pb_hl_request2({high_append_chunk, CoC_namespace, CoC_locator,
Prefix, ChunkBin, TaggedCSum,
ChunkExtra}, #state{high_clnt=Clnt}=S) ->
Chunk = {TaggedCSum, ChunkBin},
Res = machi_cr_client:append_chunk_extra(Clnt, Prefix, Chunk,
Res = machi_cr_client:append_chunk_extra(Clnt, CoC_namespace, CoC_locator,
Prefix, Chunk,
ChunkExtra),
{Res, S};
do_pb_hl_request2({high_write_chunk, File, Offset, ChunkBin, TaggedCSum},
@ -506,23 +513,27 @@ do_server_proj_request({kick_projection_reaction},
end),
async_no_response.
do_server_append_chunk(PKey, Prefix, Chunk, CSum_tag, CSum,
do_server_append_chunk(CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum_tag, CSum,
ChunkExtra, S) ->
case sanitize_prefix(Prefix) of
ok ->
do_server_append_chunk2(PKey, Prefix, Chunk, CSum_tag, CSum,
do_server_append_chunk2(CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum_tag, CSum,
ChunkExtra, S);
_ ->
{error, bad_arg}
end.
do_server_append_chunk2(_PKey, Prefix, Chunk, CSum_tag, Client_CSum,
do_server_append_chunk2(CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum_tag, Client_CSum,
ChunkExtra, #state{flu_name=FluName,
epoch_id=EpochID}=_S) ->
%% TODO: Do anything with PKey?
try
TaggedCSum = check_or_make_tagged_checksum(CSum_tag, Client_CSum,Chunk),
R = {seq_append, self(), Prefix, Chunk, TaggedCSum, ChunkExtra, EpochID},
R = {seq_append, self(), CoC_Namespace, CoC_Locator,
Prefix, Chunk, TaggedCSum, ChunkExtra, EpochID},
FluName ! R,
receive
{assignment, Offset, File} ->
@ -677,8 +688,10 @@ do_server_trunc_hack(File, #state{data_dir=DataDir}=_S) ->
{error, bad_arg}
end.
append_server_dispatch(From, Prefix, Chunk, CSum, Extra, FluName, EpochId) ->
Result = case handle_append(Prefix, Chunk, CSum, Extra, FluName, EpochId) of
append_server_dispatch(From, CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum, Extra, FluName, EpochId) ->
Result = case handle_append(CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum, Extra, FluName, EpochId) of
{ok, File, Offset} ->
{assignment, Offset, File};
Other ->
@ -687,9 +700,11 @@ append_server_dispatch(From, Prefix, Chunk, CSum, Extra, FluName, EpochId) ->
From ! Result,
exit(normal).
handle_append(_Prefix, <<>>, _Csum, _Extra, _FluName, _EpochId) ->
handle_append(_N, _L, _Prefix, <<>>, _Csum, _Extra, _FluName, _EpochId) ->
{error, bad_arg};
handle_append(Prefix, Chunk, Csum, Extra, FluName, EpochId) ->
handle_append(CoC_Namespace, CoC_Locator,
Prefix, Chunk, Csum, Extra, FluName, EpochId) ->
io:format(user, "TODO: CoC_Namespace, CoC_Locator ~p ~p\n", [CoC_Namespace, CoC_Locator]),
Res = machi_flu_filename_mgr:find_or_make_filename_from_prefix(FluName, EpochId, {prefix, Prefix}),
case Res of
{file, F} ->

104
src/machi_flu1_client.erl
View file

@ -55,7 +55,9 @@
-export([
%% File API
append_chunk/4, append_chunk/5,
append_chunk/6, append_chunk/7,
append_chunk_extra/5, append_chunk_extra/6,
append_chunk_extra/7, append_chunk_extra/8,
read_chunk/6, read_chunk/7,
checksum_list/3, checksum_list/4,
list_files/2, list_files/3,
@ -93,7 +95,9 @@
-spec append_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk()) ->
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
append_chunk(Sock, EpochID, Prefix, Chunk) ->
append_chunk2(Sock, EpochID, Prefix, Chunk, 0).
append_chunk2(Sock, EpochID,
?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR,
Prefix, Chunk, 0).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
@ -104,7 +108,39 @@ append_chunk(Sock, EpochID, Prefix, Chunk) ->
append_chunk(Host, TcpPort, EpochID, Prefix, Chunk) ->
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
try
append_chunk2(Sock, EpochID, Prefix, Chunk, 0)
append_chunk2(Sock, EpochID,
?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR,
Prefix, Chunk, 0)
after
disconnect(Sock)
end.
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
-spec append_chunk(port_wrap(), machi_dt:epoch_id(),
machi_dt:coc_namespace(), machi_dt:coc_locator(),
machi_dt:file_prefix(), machi_dt:chunk()) ->
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
append_chunk(Sock, EpochID, CoC_Namespace, CoC_Locator, Prefix, Chunk) ->
append_chunk2(Sock, EpochID,
CoC_Namespace, CoC_Locator,
Prefix, Chunk, 0).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
-spec append_chunk(machi_dt:inet_host(), machi_dt:inet_port(),
machi_dt:epoch_id(),
machi_dt:coc_namespace(), machi_dt:coc_locator(),
machi_dt:file_prefix(), machi_dt:chunk()) ->
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
append_chunk(Host, TcpPort, EpochID, CoC_Namespace, CoC_Locator, Prefix, Chunk) ->
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
try
append_chunk2(Sock, EpochID,
CoC_Namespace, CoC_Locator,
Prefix, Chunk, 0)
after
disconnect(Sock)
end.
@ -117,11 +153,14 @@ append_chunk(Host, TcpPort, EpochID, Prefix, Chunk) ->
%% be reserved by the file sequencer for later write(s) by the
%% `write_chunk()' API.
-spec append_chunk_extra(port_wrap(), machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) ->
-spec append_chunk_extra(port_wrap(), machi_dt:epoch_id(),
machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) ->
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
append_chunk_extra(Sock, EpochID, Prefix, Chunk, ChunkExtra)
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
append_chunk2(Sock, EpochID, Prefix, Chunk, ChunkExtra).
append_chunk2(Sock, EpochID,
?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR,
Prefix, Chunk, ChunkExtra).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix' and also request an additional `Extra' bytes.
@ -138,7 +177,54 @@ append_chunk_extra(Host, TcpPort, EpochID, Prefix, Chunk, ChunkExtra)
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
try
append_chunk2(Sock, EpochID, Prefix, Chunk, ChunkExtra)
append_chunk2(Sock, EpochID,
?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR,
Prefix, Chunk, ChunkExtra)
after
disconnect(Sock)
end.
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix' and also request an additional `Extra' bytes.
%%
%% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
%% the file offsets that follow `Chunk''s position for the following 4K will
%% be reserved by the file sequencer for later write(s) by the
%% `write_chunk()' API.
-spec append_chunk_extra(port_wrap(), machi_dt:epoch_id(),
machi_dt:coc_namespace(), machi_dt:coc_locator(),
machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) ->
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
append_chunk_extra(Sock, EpochID, CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra)
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
append_chunk2(Sock, EpochID,
CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix' and also request an additional `Extra' bytes.
%%
%% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
%% the file offsets that follow `Chunk''s position for the following 4K will
%% be reserved by the file sequencer for later write(s) by the
%% `write_chunk()' API.
-spec append_chunk_extra(machi_dt:inet_host(), machi_dt:inet_port(),
machi_dt:epoch_id(),
machi_dt:coc_namespace(), machi_dt:coc_locator(),
machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) ->
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
append_chunk_extra(Host, TcpPort, EpochID,
CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra)
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
try
append_chunk2(Sock, EpochID,
CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra)
after
disconnect(Sock)
end.
@ -542,7 +628,8 @@ read_chunk2(Sock, EpochID, File0, Offset, Size, Opts) ->
{low_read_chunk, EpochID, File, Offset, Size, Opts}),
do_pb_request_common(Sock, ReqID, Req).
append_chunk2(Sock, EpochID, Prefix0, Chunk0, ChunkExtra) ->
append_chunk2(Sock, EpochID, CoC_Namespace, CoC_Locator,
Prefix0, Chunk0, ChunkExtra) ->
ReqID = <<"id">>,
{Chunk, CSum_tag, CSum} =
case Chunk0 of
@ -552,12 +639,11 @@ append_chunk2(Sock, EpochID, Prefix0, Chunk0, ChunkExtra) ->
{Tag, CS} = machi_util:unmake_tagged_csum(ChunkCSum),
{Chk, Tag, CS}
end,
PKey = <<>>, % TODO
Prefix = machi_util:make_binary(Prefix0),
Req = machi_pb_translate:to_pb_request(
ReqID,
{low_append_chunk, EpochID, PKey, Prefix, Chunk, CSum_tag, CSum,
ChunkExtra}),
{low_append_chunk, EpochID, CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum_tag, CSum, ChunkExtra}),
do_pb_request_common(Sock, ReqID, Req).
write_chunk2(Sock, EpochID, File0, Offset, Chunk0) ->

24
src/machi_pb_translate.erl
View file

@ -52,14 +52,16 @@ from_pb_request(#mpb_ll_request{
req_id=ReqID,
append_chunk=#mpb_ll_appendchunkreq{
epoch_id=PB_EpochID,
placement_key=PKey,
coc_namespace=CoC_Namespace,
coc_locator=CoC_Locator,
prefix=Prefix,
chunk=Chunk,
csum=#mpb_chunkcsum{type=CSum_type, csum=CSum},
chunk_extra=ChunkExtra}}) ->
EpochID = conv_to_epoch_id(PB_EpochID),
CSum_tag = conv_to_csum_tag(CSum_type),
{ReqID, {low_append_chunk, EpochID, PKey, Prefix, Chunk, CSum_tag, CSum,
{ReqID, {low_append_chunk, EpochID, CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum_tag, CSum,
ChunkExtra}};
from_pb_request(#mpb_ll_request{
req_id=ReqID,
@ -170,14 +172,15 @@ from_pb_request(#mpb_request{req_id=ReqID,
{ReqID, {high_auth, User, Pass}};
from_pb_request(#mpb_request{req_id=ReqID,
append_chunk=IR=#mpb_appendchunkreq{}}) ->
#mpb_appendchunkreq{placement_key=__todoPK,
#mpb_appendchunkreq{coc_namespace=CoC_namespace,
coc_locator=CoC_locator,
prefix=Prefix,
chunk=Chunk,
csum=CSum,
chunk_extra=ChunkExtra} = IR,
TaggedCSum = make_tagged_csum(CSum, Chunk),
{ReqID, {high_append_chunk, __todoPK, Prefix, Chunk, TaggedCSum,
ChunkExtra}};
{ReqID, {high_append_chunk, CoC_namespace, CoC_locator, Prefix, Chunk,
TaggedCSum, ChunkExtra}};
from_pb_request(#mpb_request{req_id=ReqID,
write_chunk=IR=#mpb_writechunkreq{}}) ->
#mpb_writechunkreq{chunk=#mpb_chunk{file_name=File,
@ -388,15 +391,16 @@ to_pb_request(ReqID, {low_echo, _BogusEpochID, Msg}) ->
to_pb_request(ReqID, {low_auth, _BogusEpochID, User, Pass}) ->
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
auth=#mpb_authreq{user=User, password=Pass}};
to_pb_request(ReqID, {low_append_chunk, EpochID, PKey, Prefix, Chunk,
CSum_tag, CSum, ChunkExtra}) ->
to_pb_request(ReqID, {low_append_chunk, EpochID, CoC_Namespace, CoC_Locator,
Prefix, Chunk, CSum_tag, CSum, ChunkExtra}) ->
PB_EpochID = conv_from_epoch_id(EpochID),
CSum_type = conv_from_csum_tag(CSum_tag),
PB_CSum = #mpb_chunkcsum{type=CSum_type, csum=CSum},
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
append_chunk=#mpb_ll_appendchunkreq{
epoch_id=PB_EpochID,
placement_key=PKey,
coc_namespace=CoC_Namespace,
coc_locator=CoC_Locator,
prefix=Prefix,
chunk=Chunk,
csum=PB_CSum,
@ -500,7 +504,7 @@ to_pb_response(ReqID, {low_auth, _, _, _}, __TODO_Resp) ->
#mpb_ll_response{req_id=ReqID,
generic=#mpb_errorresp{code=1,
msg="AUTH not implemented"}};
to_pb_response(ReqID, {low_append_chunk, _EID, _PKey, _Pfx, _Ch, _CST, _CS, _CE}, Resp)->
to_pb_response(ReqID, {low_append_chunk, _EID, _N, _L, _Pfx, _Ch, _CST, _CS, _CE}, Resp)->
case Resp of
{ok, {Offset, Size, File}} ->
Where = #mpb_chunkpos{offset=Offset,
@ -687,7 +691,7 @@ to_pb_response(ReqID, {high_auth, _User, _Pass}, _Resp) ->
#mpb_response{req_id=ReqID,
generic=#mpb_errorresp{code=1,
msg="AUTH not implemented"}};
to_pb_response(ReqID, {high_append_chunk, _TODO, _Prefix, _Chunk, _TSum, _CE}, Resp)->
to_pb_response(ReqID, {high_append_chunk, _CoC_n, _CoC_l, _Prefix, _Chunk, _TSum, _CE}, Resp)->
case Resp of
{ok, {Offset, Size, File}} ->
Where = #mpb_chunkpos{offset=Offset,

51
src/machi_proxy_flu1_client.erl
View file

@ -58,7 +58,9 @@
-export([
%% File API
append_chunk/4, append_chunk/5,
append_chunk/6, append_chunk/7,
append_chunk_extra/5, append_chunk_extra/6,
append_chunk_extra/7, append_chunk_extra/8,
read_chunk/6, read_chunk/7,
checksum_list/3, checksum_list/4,
list_files/2, list_files/3,
@ -111,22 +113,51 @@ append_chunk(PidSpec, EpochID, Prefix, Chunk) ->
%% with `Prefix'.
append_chunk(PidSpec, EpochID, Prefix, Chunk, Timeout) ->
gen_server:call(PidSpec, {req, {append_chunk, EpochID, Prefix, Chunk}},
Timeout).
append_chunk_extra(PidSpec, EpochID,
?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR,
Prefix, Chunk, 0, Timeout).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
append_chunk(PidSpec, EpochID, CoC_Namespace, CoC_Locator, Prefix, Chunk) ->
append_chunk(PidSpec, EpochID, CoC_Namespace, CoC_Locator, Prefix, Chunk, infinity).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
append_chunk(PidSpec, EpochID, CoC_Namespace, CoC_Locator, Prefix, Chunk, Timeout) ->
append_chunk_extra(PidSpec, EpochID,
CoC_Namespace, CoC_Locator,
Prefix, Chunk, 0, Timeout).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra)
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra, infinity).
append_chunk_extra(PidSpec, EpochID,
?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR,
Prefix, Chunk, ChunkExtra, infinity).
%% @doc Append a chunk (binary- or iolist-style) of data to a file
%% with `Prefix'.
append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra, Timeout) ->
gen_server:call(PidSpec, {req, {append_chunk_extra, EpochID, Prefix,
Chunk, ChunkExtra}},
append_chunk_extra(PidSpec, EpochID,
?DEFAULT_COC_NAMESPACE, ?DEFAULT_COC_LOCATOR,
Prefix, Chunk, ChunkExtra, Timeout).
append_chunk_extra(PidSpec, EpochID, CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra) ->
append_chunk_extra(PidSpec, EpochID, CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra, infinity).
append_chunk_extra(PidSpec, EpochID, CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra, Timeout) ->
gen_server:call(PidSpec, {req, {append_chunk_extra, EpochID,
CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra}},
Timeout).
%% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
@ -384,12 +415,12 @@ do_req_retry(_Req, 2, Err, S) ->
do_req_retry(Req, Depth, _Err, S) ->
do_req(Req, Depth + 1, try_connect(disconnect(S))).
make_req_fun({append_chunk, EpochID, Prefix, Chunk},
make_req_fun({append_chunk_extra, EpochID, CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra},
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
fun() -> Mod:append_chunk(Sock, EpochID, Prefix, Chunk) end;
make_req_fun({append_chunk_extra, EpochID, Prefix, Chunk, ChunkExtra},
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
fun() -> Mod:append_chunk_extra(Sock, EpochID, Prefix, Chunk, ChunkExtra) end;
fun() -> Mod:append_chunk_extra(Sock, EpochID, CoC_Namespace, CoC_Locator,
Prefix, Chunk, ChunkExtra)
end;
make_req_fun({read_chunk, EpochID, File, Offset, Size, Opts},
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
fun() -> Mod:read_chunk(Sock, EpochID, File, Offset, Size, Opts) end;

2
test/machi_pb_high_client_test.erl
View file

@ -61,7 +61,7 @@ smoke_test2() ->
%% a separate test module? Or separate test func?
{error, _} = ?C:auth(Clnt, "foo", "bar"),
CoC_n = <<>>, % CoC_namespace (not implemented)
CoC_n = "", % CoC_namespace (not implemented)
CoC_l = 0, % CoC_locator (not implemented)
Prefix = <<"prefix">>,
Chunk1 = <<"Hello, chunk!">>,