Move FLU client 'verify checksums + local path' code from prototype/demo-day-hack

This commit is contained in:
Scott Lystig Fritchie 2015-04-02 12:38:12 +09:00
parent 76fcd4d931
commit f8263c15cc
5 changed files with 98 additions and 26 deletions

View file

@ -34,5 +34,8 @@ build_plt: deps compile
dialyzer: deps compile dialyzer: deps compile
dialyzer -Wno_return --plt $(PLT) ebin dialyzer -Wno_return --plt $(PLT) ebin
dialyzer-test: deps compile
dialyzer -Wno_return --plt $(PLT) .eunit
clean_plt: clean_plt:
rm $(PLT) rm $(PLT)

View file

@ -20,7 +20,13 @@
-module(machi_admin_util). -module(machi_admin_util).
%% TODO Move these types to a common header file? (also machi_flu1_client.erl?)
-type inet_host() :: inet:ip_address() | inet:hostname().
-type inet_port() :: inet:port_number().
-export([ -export([
%% verify_file_checksums_local/2,
verify_file_checksums_local/3,
verify_file_checksums_remote/2, verify_file_checksums_remote/3 verify_file_checksums_remote/2, verify_file_checksums_remote/3
]). ]).
-compile(export_all). -compile(export_all).
@ -29,28 +35,69 @@
-define(FLU_C, machi_flu1_client). -define(FLU_C, machi_flu1_client).
-spec verify_file_checksums_local(inet_host(), inet_port(), binary()|list()) ->
{ok, [tuple()]} | {error, term()}.
verify_file_checksums_local(Host, TcpPort, Path) ->
Sock1 = machi_util:connect(Host, TcpPort),
verify_file_checksums_local2(Sock1, Path).
-spec verify_file_checksums_remote(port(), binary()|list()) ->
{ok, [tuple()]} | {error, term()}.
verify_file_checksums_remote(Sock1, File) when is_port(Sock1) -> verify_file_checksums_remote(Sock1, File) when is_port(Sock1) ->
verify_file_checksums_remote2(Sock1, File). verify_file_checksums_remote2(Sock1, File).
-spec verify_file_checksums_remote(inet_host(), inet_port(), binary()|list()) ->
{ok, [tuple()]} | {error, term()}.
verify_file_checksums_remote(Host, TcpPort, File) -> verify_file_checksums_remote(Host, TcpPort, File) ->
Sock1 = machi_util:connect(Host, TcpPort), Sock1 = machi_util:connect(Host, TcpPort),
verify_file_checksums_remote2(Sock1, File). verify_file_checksums_remote2(Sock1, File).
%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%
verify_file_checksums_local2(Sock1, Path0) ->
Path = machi_util:make_string(Path0),
case file:open(Path, [read, binary, raw]) of
{ok, FH} ->
File = re:replace(Path, ".*/", "", [{return, binary}]),
try
ReadChunk = fun(_File, Offset, Size) ->
file:pread(FH, Offset, Size)
end,
verify_file_checksums_common(Sock1, File, ReadChunk)
after
file:close(FH)
end;
Else ->
Else
end.
verify_file_checksums_remote2(Sock1, File) -> verify_file_checksums_remote2(Sock1, File) ->
ReadChunk = fun(File_name, Offset, Size) ->
?FLU_C:read_chunk(Sock1, File_name, Offset, Size)
end,
verify_file_checksums_common(Sock1, File, ReadChunk).
verify_file_checksums_common(Sock1, File, ReadChunk) ->
try try
{ok, Info} = ?FLU_C:checksum_list(Sock1, File), case ?FLU_C:checksum_list(Sock1, File) of
Res = lists:foldl(verify_chunk_checksum(Sock1, File), [], Info), {ok, Info} ->
{ok, Res} ?FLU_C:checksum_list(Sock1, File),
Res = lists:foldl(verify_chunk_checksum(File, ReadChunk),
[], Info),
{ok, Res};
{error, no_such_file}=Nope ->
Nope;
{error, _}=Else ->
Else
end
catch catch
What:Why -> What:Why ->
{error, {What, Why, erlang:get_stacktrace()}} {error, {What, Why, erlang:get_stacktrace()}}
end. end.
verify_chunk_checksum(Sock1, File) -> verify_chunk_checksum(File, ReadChunk) ->
fun({Offset, Size, CSum}, Acc) -> fun({Offset, Size, CSum}, Acc) ->
case ?FLU_C:read_chunk(Sock1, File, Offset, Size) of case ReadChunk(File, Offset, Size) of
{ok, Chunk} -> {ok, Chunk} ->
CSum2 = machi_util:checksum(Chunk), CSum2 = machi_util:checksum(Chunk),
if CSum == CSum2 -> if CSum == CSum2 ->

View file

@ -36,15 +36,18 @@
trunc_hack/2, trunc_hack/3 trunc_hack/2, trunc_hack/3
]). ]).
-type chunk() :: iolist(). -type chunk() :: binary() | iolist(). % client can use either
-type chunk_s() :: binary(). -type chunk_csum() :: {file_offset(), chunk_size(), binary()}.
-type chunk_s() :: binary(). % server always uses binary()
-type chunk_pos() :: {file_offset(), chunk_size(), file_name_s()}. -type chunk_pos() :: {file_offset(), chunk_size(), file_name_s()}.
-type chunk_size() :: non_neg_integer(). -type chunk_size() :: non_neg_integer().
-type inet_host() :: inet:ip_address() | inet:hostname(). -type inet_host() :: inet:ip_address() | inet:hostname().
-type inet_port() :: inet:port_number(). -type inet_port() :: inet:port_number().
-type file_info() :: {file_size(), file_name_s()}.
-type file_name() :: binary() | list(). -type file_name() :: binary() | list().
-type file_name_s() :: binary(). % server reply -type file_name_s() :: binary(). % server reply
-type file_offset() :: non_neg_integer(). -type file_offset() :: non_neg_integer().
-type file_size() :: non_neg_integer().
-type file_prefix() :: binary() | list(). -type file_prefix() :: binary() | list().
%% @doc Append a chunk (binary- or iolist-style) of data to a file %% @doc Append a chunk (binary- or iolist-style) of data to a file
@ -72,14 +75,16 @@ append_chunk(Host, TcpPort, Prefix, Chunk) ->
-spec read_chunk(port(), file_name(), file_offset(), chunk_size()) -> -spec read_chunk(port(), file_name(), file_offset(), chunk_size()) ->
{ok, chunk_s()} | {error, term()}. {ok, chunk_s()} | {error, term()}.
read_chunk(Sock, File, Offset, Size) -> read_chunk(Sock, File, Offset, Size)
when Offset >= ?MINIMUM_OFFSET, Size >= 0 ->
read_chunk2(Sock, File, Offset, Size). read_chunk2(Sock, File, Offset, Size).
%% @doc Read a chunk of data of size `Size' from `File' at `Offset'. %% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
-spec read_chunk(inet_host(), inet_port(), file_name(), file_offset(), chunk_size()) -> -spec read_chunk(inet_host(), inet_port(), file_name(), file_offset(), chunk_size()) ->
{ok, chunk_s()} | {error, term()}. {ok, chunk_s()} | {error, term()}.
read_chunk(Host, TcpPort, File, Offset, Size) -> read_chunk(Host, TcpPort, File, Offset, Size)
when Offset >= ?MINIMUM_OFFSET, Size >= 0 ->
Sock = machi_util:connect(Host, TcpPort), Sock = machi_util:connect(Host, TcpPort),
try try
read_chunk2(Sock, File, Offset, Size) read_chunk2(Sock, File, Offset, Size)
@ -90,14 +95,14 @@ read_chunk(Host, TcpPort, File, Offset, Size) ->
%% @doc Fetch the list of chunk checksums for `File'. %% @doc Fetch the list of chunk checksums for `File'.
-spec checksum_list(port(), file_name()) -> -spec checksum_list(port(), file_name()) ->
{ok, [file_name()]} | {error, term()}. {ok, [chunk_csum()]} | {error, term()}.
checksum_list(Sock, File) when is_port(Sock) -> checksum_list(Sock, File) when is_port(Sock) ->
checksum_list2(Sock, File). checksum_list2(Sock, File).
%% @doc Fetch the list of chunk checksums for `File'. %% @doc Fetch the list of chunk checksums for `File'.
-spec checksum_list(inet_host(), inet_port(), file_name()) -> -spec checksum_list(inet_host(), inet_port(), file_name()) ->
{ok, [file_name()]} | {error, term()}. {ok, [chunk_csum()]} | {error, term()}.
checksum_list(Host, TcpPort, File) when is_integer(TcpPort) -> checksum_list(Host, TcpPort, File) when is_integer(TcpPort) ->
Sock = machi_util:connect(Host, TcpPort), Sock = machi_util:connect(Host, TcpPort),
try try
@ -109,14 +114,14 @@ checksum_list(Host, TcpPort, File) when is_integer(TcpPort) ->
%% @doc Fetch the list of all files on the remote FLU. %% @doc Fetch the list of all files on the remote FLU.
-spec list_files(port()) -> -spec list_files(port()) ->
{ok, [file_name()]} | {error, term()}. {ok, [file_info()]} | {error, term()}.
list_files(Sock) when is_port(Sock) -> list_files(Sock) when is_port(Sock) ->
list2(Sock). list2(Sock).
%% @doc Fetch the list of all files on the remote FLU. %% @doc Fetch the list of all files on the remote FLU.
-spec list_files(inet_host(), inet_port()) -> -spec list_files(inet_host(), inet_port()) ->
{ok, [file_name()]} | {error, term()}. {ok, [file_info()]} | {error, term()}.
list_files(Host, TcpPort) when is_integer(TcpPort) -> list_files(Host, TcpPort) when is_integer(TcpPort) ->
Sock = machi_util:connect(Host, TcpPort), Sock = machi_util:connect(Host, TcpPort),
try try
@ -140,16 +145,18 @@ quit(Sock) when is_port(Sock) ->
%% `File' at `Offset'. %% `File' at `Offset'.
-spec write_chunk(port(), file_name(), file_offset(), chunk()) -> -spec write_chunk(port(), file_name(), file_offset(), chunk()) ->
{ok, chunk_s()} | {error, term()}. ok | {error, term()}.
write_chunk(Sock, File, Offset, Chunk) -> write_chunk(Sock, File, Offset, Chunk)
when Offset >= ?MINIMUM_OFFSET ->
write_chunk2(Sock, File, Offset, Chunk). write_chunk2(Sock, File, Offset, Chunk).
%% @doc Restricted API: Write a chunk of already-sequenced data to %% @doc Restricted API: Write a chunk of already-sequenced data to
%% `File' at `Offset'. %% `File' at `Offset'.
-spec write_chunk(inet_host(), inet_port(), file_name(), file_offset(), chunk()) -> -spec write_chunk(inet_host(), inet_port(), file_name(), file_offset(), chunk()) ->
{ok, chunk_s()} | {error, term()}. ok | {error, term()}.
write_chunk(Host, TcpPort, File, Offset, Chunk) -> write_chunk(Host, TcpPort, File, Offset, Chunk)
when Offset >= ?MINIMUM_OFFSET ->
Sock = machi_util:connect(Host, TcpPort), Sock = machi_util:connect(Host, TcpPort),
try try
write_chunk2(Sock, File, Offset, Chunk) write_chunk2(Sock, File, Offset, Chunk)
@ -161,7 +168,7 @@ write_chunk(Host, TcpPort, File, Offset, Chunk) ->
%% migrated. %% migrated.
-spec delete_migration(port(), file_name()) -> -spec delete_migration(port(), file_name()) ->
{ok, [file_name()]} | {error, term()}. ok | {error, term()}.
delete_migration(Sock, File) when is_port(Sock) -> delete_migration(Sock, File) when is_port(Sock) ->
delete_migration2(Sock, File). delete_migration2(Sock, File).
@ -169,7 +176,7 @@ delete_migration(Sock, File) when is_port(Sock) ->
%% migrated. %% migrated.
-spec delete_migration(inet_host(), inet_port(), file_name()) -> -spec delete_migration(inet_host(), inet_port(), file_name()) ->
{ok, [file_name()]} | {error, term()}. ok | {error, term()}.
delete_migration(Host, TcpPort, File) when is_integer(TcpPort) -> delete_migration(Host, TcpPort, File) when is_integer(TcpPort) ->
Sock = machi_util:connect(Host, TcpPort), Sock = machi_util:connect(Host, TcpPort),
try try
@ -182,7 +189,7 @@ delete_migration(Host, TcpPort, File) when is_integer(TcpPort) ->
%% erasure coded. %% erasure coded.
-spec trunc_hack(port(), file_name()) -> -spec trunc_hack(port(), file_name()) ->
{ok, [file_name()]} | {error, term()}. ok | {error, term()}.
trunc_hack(Sock, File) when is_port(Sock) -> trunc_hack(Sock, File) when is_port(Sock) ->
trunc_hack2(Sock, File). trunc_hack2(Sock, File).
@ -190,7 +197,7 @@ trunc_hack(Sock, File) when is_port(Sock) ->
%% erasure coded. %% erasure coded.
-spec trunc_hack(inet_host(), inet_port(), file_name()) -> -spec trunc_hack(inet_host(), inet_port(), file_name()) ->
{ok, [file_name()]} | {error, term()}. ok | {error, term()}.
trunc_hack(Host, TcpPort, File) when is_integer(TcpPort) -> trunc_hack(Host, TcpPort, File) when is_integer(TcpPort) ->
Sock = machi_util:connect(Host, TcpPort), Sock = machi_util:connect(Host, TcpPort),
try try

View file

@ -24,7 +24,7 @@
checksum/1, checksum/1,
hexstr_to_bin/1, bin_to_hexstr/1, hexstr_to_bin/1, bin_to_hexstr/1,
hexstr_to_int/1, int_to_hexstr/2, int_to_hexbin/2, hexstr_to_int/1, int_to_hexstr/2, int_to_hexbin/2,
make_binary/1, make_binary/1, make_string/1,
make_regname/1, make_regname/1,
make_checksum_filename/2, make_data_filename/2, make_checksum_filename/2, make_data_filename/2,
read_max_filenum/2, increment_max_filenum/2, read_max_filenum/2, increment_max_filenum/2,
@ -118,6 +118,11 @@ make_binary(X) when is_binary(X) ->
make_binary(X) when is_list(X) -> make_binary(X) when is_list(X) ->
iolist_to_binary(X). iolist_to_binary(X).
make_string(X) when is_list(X) ->
lists:flatten(X);
make_string(X) when is_binary(X) ->
binary_to_list(X).
hexstr_to_int(X) -> hexstr_to_int(X) ->
B = hexstr_to_bin(X), B = hexstr_to_bin(X),
B_size = byte_size(B) * 8, B_size = byte_size(B) * 8,

View file

@ -28,7 +28,7 @@
-define(FLU, machi_flu1). -define(FLU, machi_flu1).
-define(FLU_C, machi_flu1_client). -define(FLU_C, machi_flu1_client).
verify_file_checksums_remote_test() -> verify_file_checksums_test() ->
Host = "localhost", Host = "localhost",
TcpPort = 32958, TcpPort = 32958,
DataDir = "./data", DataDir = "./data",
@ -49,10 +49,20 @@ verify_file_checksums_remote_test() ->
ok = file:write(FH, "yo"), ok = file:write(FH, "yo"),
ok = file:write(FH, "yo!"), ok = file:write(FH, "yo!"),
ok = file:close(FH), ok = file:close(FH),
{ok, [_,_,_]} = machi_admin_util:verify_file_checksums_remote(
Host, TcpPort, File) %% Check the local flavor of the API
{ok, Res1} = machi_admin_util:verify_file_checksums_local(
Host, TcpPort, Path),
3 = length(Res1),
%% Check the remote flavor of the API
{ok, Res2} = machi_admin_util:verify_file_checksums_remote(
Host, TcpPort, File),
3 = length(Res2),
ok
after after
catch ?FLU_C:quick(Sock1), catch ?FLU_C:quit(Sock1),
ok = ?FLU:stop(FLU1) ok = ?FLU:stop(FLU1)
end. end.