WIP
This commit is contained in:
parent
d482902509
commit
8103b424c0
6 changed files with 198 additions and 116 deletions
|
@ -21,7 +21,7 @@
|
||||||
%% @doc Now 4GiBytes, could be up to 64bit due to PB message limit of
|
%% @doc Now 4GiBytes, could be up to 64bit due to PB message limit of
|
||||||
%% chunk size
|
%% chunk size
|
||||||
-define(DEFAULT_MAX_FILE_SIZE, ((1 bsl 32) - 1)).
|
-define(DEFAULT_MAX_FILE_SIZE, ((1 bsl 32) - 1)).
|
||||||
-define(MINIMUM_OFFSET, 1024).
|
-define(MINIMUM_OFFSET, 0).
|
||||||
|
|
||||||
%% 0th draft of checksum typing with 1st byte.
|
%% 0th draft of checksum typing with 1st byte.
|
||||||
-define(CSUM_TAG_NONE, 0). % No csum provided by client
|
-define(CSUM_TAG_NONE, 0). % No csum provided by client
|
||||||
|
|
|
@ -43,12 +43,8 @@ open(CSumFilename, _Opts) ->
|
||||||
%% operating system's file cache, which is for
|
%% operating system's file cache, which is for
|
||||||
%% Machi's main read efficiency
|
%% Machi's main read efficiency
|
||||||
{total_leveldb_mem_percent, 10}],
|
{total_leveldb_mem_percent, 10}],
|
||||||
|
ok = filelib:ensure_dir(CSumFilename),
|
||||||
{ok, T} = eleveldb:open(CSumFilename, LevelDBOptions),
|
{ok, T} = eleveldb:open(CSumFilename, LevelDBOptions),
|
||||||
%% Dummy entry for reserved headers
|
|
||||||
%% ok = eleveldb:put(T,
|
|
||||||
%% sext:encode({0, ?MINIMUM_OFFSET}),
|
|
||||||
%% sext:encode(?CSUM_TAG_NONE_ATOM),
|
|
||||||
%% [{sync, true}]),
|
|
||||||
C0 = #machi_csum_table{
|
C0 = #machi_csum_table{
|
||||||
file=CSumFilename,
|
file=CSumFilename,
|
||||||
table=T},
|
table=T},
|
||||||
|
@ -64,22 +60,12 @@ split_checksum_list_blob_decode(Bin) ->
|
||||||
-spec find(table(), binary(), machi_dt:file_offset(), machi_dt:chunk_size())
|
-spec find(table(), binary(), machi_dt:file_offset(), machi_dt:chunk_size())
|
||||||
-> [chunk()].
|
-> [chunk()].
|
||||||
find(#machi_csum_table{table=T}, Filename, Offset, Size) when is_binary(Filename) ->
|
find(#machi_csum_table{table=T}, Filename, Offset, Size) when is_binary(Filename) ->
|
||||||
{ok, I} = eleveldb:iterator(T, [], keys_only),
|
|
||||||
EndKey = sext:encode({Filename, Offset+Size, 0}),
|
EndKey = sext:encode({Filename, Offset+Size, 0}),
|
||||||
StartKey = sext:encode({Filename, Offset, Size}),
|
|
||||||
|
|
||||||
{ok, FirstKey} = case eleveldb:iterator_move(I, StartKey) of
|
case search_for_start_key(T, Filename, Offset, Size) of
|
||||||
{error, invalid_iterator} ->
|
undefined -> [];
|
||||||
eleveldb:iterator_move(I, first);
|
FirstKey ->
|
||||||
{ok, _} = R0 ->
|
|
||||||
case eleveldb:iterator_move(I, prev) of
|
|
||||||
{error, invalid_iterator} ->
|
|
||||||
R0;
|
|
||||||
{ok, _} = R1 ->
|
|
||||||
R1
|
|
||||||
end
|
|
||||||
end,
|
|
||||||
_ = eleveldb:iterator_close(I),
|
|
||||||
FoldFun = fun({K, V}, Acc) ->
|
FoldFun = fun({K, V}, Acc) ->
|
||||||
{Filename, TargetOffset, TargetSize} = sext:decode(K),
|
{Filename, TargetOffset, TargetSize} = sext:decode(K),
|
||||||
case ?has_overlap(TargetOffset, TargetSize, Offset, Size) of
|
case ?has_overlap(TargetOffset, TargetSize, Offset, Size) of
|
||||||
|
@ -92,7 +78,8 @@ find(#machi_csum_table{table=T}, Filename, Offset, Size) when is_binary(Filename
|
||||||
lager:error("~p wrong option", [_K]),
|
lager:error("~p wrong option", [_K]),
|
||||||
Acc
|
Acc
|
||||||
end,
|
end,
|
||||||
lists:reverse(eleveldb_fold(T, FirstKey, EndKey, FoldFun, [])).
|
lists:reverse(eleveldb_fold(T, FirstKey, EndKey, FoldFun, []))
|
||||||
|
end.
|
||||||
|
|
||||||
|
|
||||||
%% @doc Updates all chunk info, by deleting existing entries if exists
|
%% @doc Updates all chunk info, by deleting existing entries if exists
|
||||||
|
@ -196,10 +183,11 @@ any_trimmed(CsumT, Filename, Offset, Size) ->
|
||||||
calc_unwritten_bytes(#machi_csum_table{table=_} = CsumT, Filename) ->
|
calc_unwritten_bytes(#machi_csum_table{table=_} = CsumT, Filename) ->
|
||||||
case lists:sort(all(CsumT, Filename)) of
|
case lists:sort(all(CsumT, Filename)) of
|
||||||
[] ->
|
[] ->
|
||||||
[{?MINIMUM_OFFSET, infinity}];
|
[{0, infinity}];
|
||||||
Sorted ->
|
[{0, _, _}|_] = Sorted ->
|
||||||
{LastOffset, _, _} = hd(Sorted),
|
build_unwritten_bytes_list(Sorted, 0, []);
|
||||||
build_unwritten_bytes_list(Sorted, LastOffset, [])
|
[{LastOffset, _, _}|_] = Sorted ->
|
||||||
|
build_unwritten_bytes_list(Sorted, LastOffset, [{0, LastOffset}])
|
||||||
end.
|
end.
|
||||||
|
|
||||||
all(CsumT, Filename) ->
|
all(CsumT, Filename) ->
|
||||||
|
@ -233,7 +221,7 @@ maybe_trim_file(#machi_csum_table{table=T} = CsumT, Filename, EofP) when is_bina
|
||||||
%% @doc Folds over all chunks of a file
|
%% @doc Folds over all chunks of a file
|
||||||
-spec foldl_file_chunks(fun((chunk(), Acc0 :: term()) -> Acc :: term()),
|
-spec foldl_file_chunks(fun((chunk(), Acc0 :: term()) -> Acc :: term()),
|
||||||
Acc0 :: term(), table(), binary()) -> Acc :: term().
|
Acc0 :: term(), table(), binary()) -> Acc :: term().
|
||||||
foldl_file_chunks(Fun, Acc0, #machi_csum_table{table=T}, Filename) ->
|
foldl_file_chunks(Fun, Acc0, #machi_csum_table{table=T}, Filename) when is_binary(Filename) ->
|
||||||
FoldFun = fun({K, V}, Acc) ->
|
FoldFun = fun({K, V}, Acc) ->
|
||||||
{Filename, Offset, Len} = sext:decode(K),
|
{Filename, Offset, Len} = sext:decode(K),
|
||||||
Fun({Offset, Len, sext:decode(V)}, Acc);
|
Fun({Offset, Len, sext:decode(V)}, Acc);
|
||||||
|
@ -260,6 +248,8 @@ foldl_chunks(Fun, Acc0, #machi_csum_table{table=T}) ->
|
||||||
end,
|
end,
|
||||||
eleveldb:fold(T, FoldFun, Acc0, [{verify_checksums, true}]).
|
eleveldb:fold(T, FoldFun, Acc0, [{verify_checksums, true}]).
|
||||||
|
|
||||||
|
%% == internal functions ==
|
||||||
|
|
||||||
-spec build_unwritten_bytes_list( CsumData :: [{ Offset :: non_neg_integer(),
|
-spec build_unwritten_bytes_list( CsumData :: [{ Offset :: non_neg_integer(),
|
||||||
Size :: pos_integer(),
|
Size :: pos_integer(),
|
||||||
Checksum :: binary() }],
|
Checksum :: binary() }],
|
||||||
|
@ -323,3 +313,50 @@ eleveldb_do_fold({error, iterator_closed}, _, _, _, Acc) ->
|
||||||
eleveldb_do_fold({error, invalid_iterator}, _, _, _, Acc) ->
|
eleveldb_do_fold({error, invalid_iterator}, _, _, _, Acc) ->
|
||||||
%% Probably reached to end
|
%% Probably reached to end
|
||||||
Acc.
|
Acc.
|
||||||
|
|
||||||
|
|
||||||
|
%% Key1 < MaybeStartKey =< Key
|
||||||
|
%% FirstKey =< MaybeStartKey
|
||||||
|
search_for_start_key(T, Filename, Offset, Size) ->
|
||||||
|
MaybeStartKey = sext:encode({Filename, Offset, Size}),
|
||||||
|
FirstKey = sext:encode({Filename, 0, 0}),
|
||||||
|
{ok, I} = eleveldb:iterator(T, [], keys_only),
|
||||||
|
|
||||||
|
try
|
||||||
|
case eleveldb:iterator_move(I, MaybeStartKey) of
|
||||||
|
{error, invalid_iterator} ->
|
||||||
|
%% No key in right - go for probably first key in the file
|
||||||
|
case eleveldb:iterator_move(I, FirstKey) of
|
||||||
|
{error, _} -> undefined;
|
||||||
|
{ok, Key0} -> goto_end(I, Key0, Offset)
|
||||||
|
end;
|
||||||
|
{ok, Key} when Key < FirstKey ->
|
||||||
|
FirstKey;
|
||||||
|
{ok, Key} ->
|
||||||
|
case eleveldb:iterator_move(I, prev) of
|
||||||
|
{error, invalid_iterator} ->
|
||||||
|
Key;
|
||||||
|
{ok, Key1} when Key1 < FirstKey ->
|
||||||
|
Key;
|
||||||
|
{ok, Key1} ->
|
||||||
|
Key1
|
||||||
|
end
|
||||||
|
end
|
||||||
|
after
|
||||||
|
_ = eleveldb:iterator_close(I)
|
||||||
|
end.
|
||||||
|
|
||||||
|
goto_end(I, Key, Offset) ->
|
||||||
|
case sext:decode(Key) of
|
||||||
|
{_Filename, O, L} when Offset =< O + L ->
|
||||||
|
Key;
|
||||||
|
{_Filename, O, L} when O + L < Offset ->
|
||||||
|
case eleveldb:iterator_move(I, next) of
|
||||||
|
{ok, NextKey} ->
|
||||||
|
goto_end(I, NextKey, Offset);
|
||||||
|
{error, _} ->
|
||||||
|
Key
|
||||||
|
end
|
||||||
|
end.
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -221,7 +221,7 @@ checksum_list(Pid) ->
|
||||||
init({Filename, DataDir, CsumTable}) ->
|
init({Filename, DataDir, CsumTable}) ->
|
||||||
{_, DPath} = machi_util:make_data_filename(DataDir, Filename),
|
{_, DPath} = machi_util:make_data_filename(DataDir, Filename),
|
||||||
ok = filelib:ensure_dir(DPath),
|
ok = filelib:ensure_dir(DPath),
|
||||||
UnwrittenBytes = machi_csum_table:calc_unwritten_bytes(CsumTable),
|
UnwrittenBytes = machi_csum_table:calc_unwritten_bytes(CsumTable, iolist_to_binary(Filename)),
|
||||||
{Eof, infinity} = lists:last(UnwrittenBytes),
|
{Eof, infinity} = lists:last(UnwrittenBytes),
|
||||||
{ok, FHd} = file:open(DPath, [read, write, binary, raw]),
|
{ok, FHd} = file:open(DPath, [read, write, binary, raw]),
|
||||||
%% Reserve for EC and stuff, to prevent eof when read
|
%% Reserve for EC and stuff, to prevent eof when read
|
||||||
|
@ -343,7 +343,7 @@ handle_call({write, Offset, ClientMeta, Data}, _From,
|
||||||
{Error, Err + 1}
|
{Error, Err + 1}
|
||||||
end
|
end
|
||||||
end,
|
end,
|
||||||
{NewEof, infinity} = lists:last(machi_csum_table:calc_unwritten_bytes(CsumTable, F)),
|
{NewEof, infinity} = lists:last(machi_csum_table:calc_unwritten_bytes(CsumTable, iolist_to_binary(F))),
|
||||||
lager:debug("Wrote ~p bytes at ~p of file ~p, NewEOF = ~p~n",
|
lager:debug("Wrote ~p bytes at ~p of file ~p, NewEOF = ~p~n",
|
||||||
[iolist_size(Data), Offset, F, NewEof]),
|
[iolist_size(Data), Offset, F, NewEof]),
|
||||||
{reply, Resp, State#state{writes = {T+1, NewErr},
|
{reply, Resp, State#state{writes = {T+1, NewErr},
|
||||||
|
@ -365,7 +365,8 @@ handle_call({trim, Offset, Size, _TriggerGC}, _From,
|
||||||
trims = {T, Err},
|
trims = {T, Err},
|
||||||
csum_table = CsumTable}) ->
|
csum_table = CsumTable}) ->
|
||||||
|
|
||||||
case machi_csum_table:all_trimmed(CsumTable, Filename,
|
F = iolist_to_binary(Filename),
|
||||||
|
case machi_csum_table:all_trimmed(CsumTable, F,
|
||||||
Offset, Offset+Size) of
|
Offset, Offset+Size) of
|
||||||
true ->
|
true ->
|
||||||
NewState = State#state{ops=Ops+1, trims={T, Err+1}},
|
NewState = State#state{ops=Ops+1, trims={T, Err+1}},
|
||||||
|
@ -377,18 +378,18 @@ handle_call({trim, Offset, Size, _TriggerGC}, _From,
|
||||||
LUpdate = maybe_regenerate_checksum(
|
LUpdate = maybe_regenerate_checksum(
|
||||||
FHd,
|
FHd,
|
||||||
machi_csum_table:find_leftneighbor(CsumTable,
|
machi_csum_table:find_leftneighbor(CsumTable,
|
||||||
Filename,
|
F,
|
||||||
Offset)),
|
Offset)),
|
||||||
RUpdate = maybe_regenerate_checksum(
|
RUpdate = maybe_regenerate_checksum(
|
||||||
FHd,
|
FHd,
|
||||||
machi_csum_table:find_rightneighbor(CsumTable,
|
machi_csum_table:find_rightneighbor(CsumTable,
|
||||||
Filename,
|
F,
|
||||||
Offset+Size)),
|
Offset+Size)),
|
||||||
|
|
||||||
case machi_csum_table:trim(CsumTable, Filename, Offset,
|
case machi_csum_table:trim(CsumTable, F, Offset,
|
||||||
Size, LUpdate, RUpdate) of
|
Size, LUpdate, RUpdate) of
|
||||||
ok ->
|
ok ->
|
||||||
{NewEof, infinity} = lists:last(machi_csum_table:calc_unwritten_bytes(CsumTable, Filename)),
|
{NewEof, infinity} = lists:last(machi_csum_table:calc_unwritten_bytes(CsumTable, F)),
|
||||||
NewState = State#state{ops=Ops+1,
|
NewState = State#state{ops=Ops+1,
|
||||||
trims={T+1, Err},
|
trims={T+1, Err},
|
||||||
eof_position=NewEof},
|
eof_position=NewEof},
|
||||||
|
@ -439,7 +440,7 @@ handle_call({append, ClientMeta, Extra, Data}, _From,
|
||||||
|
|
||||||
handle_call({checksum_list}, _FRom, State = #state{filename=Filename,
|
handle_call({checksum_list}, _FRom, State = #state{filename=Filename,
|
||||||
csum_table=T}) ->
|
csum_table=T}) ->
|
||||||
All = machi_csum_table:all(T, Filename),
|
All = machi_csum_table:all(T,iolist_to_binary(Filename)),
|
||||||
{reply, {ok, All}, State};
|
{reply, {ok, All}, State};
|
||||||
|
|
||||||
handle_call(Req, _From, State) ->
|
handle_call(Req, _From, State) ->
|
||||||
|
@ -617,7 +618,7 @@ check_or_make_tagged_csum(OtherTag, _ClientCsum, _Data) ->
|
||||||
do_read(FHd, Filename, CsumTable, Offset, Size, _, _) ->
|
do_read(FHd, Filename, CsumTable, Offset, Size, _, _) ->
|
||||||
%% Note that find/3 only returns overlapping chunks, both borders
|
%% Note that find/3 only returns overlapping chunks, both borders
|
||||||
%% are not aligned to original Offset and Size.
|
%% are not aligned to original Offset and Size.
|
||||||
ChunkCsums = machi_csum_table:find(CsumTable, Filename,
|
ChunkCsums = machi_csum_table:find(CsumTable, iolist_to_binary(Filename),
|
||||||
Offset, Size),
|
Offset, Size),
|
||||||
read_all_ranges(FHd, Filename, ChunkCsums, [], []).
|
read_all_ranges(FHd, Filename, ChunkCsums, [], []).
|
||||||
|
|
||||||
|
@ -696,7 +697,7 @@ read_all_ranges(FHd, Filename, [{Offset, Size, TaggedCsum}|T], ReadChunks, Trimm
|
||||||
handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
||||||
Size = iolist_size(Data),
|
Size = iolist_size(Data),
|
||||||
|
|
||||||
case machi_csum_table:find(CsumTable, Filename, Offset, Size) of
|
case machi_csum_table:find(CsumTable, iolist_to_binary(Filename), Offset, Size) of
|
||||||
[] -> %% Nothing should be there
|
[] -> %% Nothing should be there
|
||||||
try
|
try
|
||||||
do_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Size, Data)
|
do_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Size, Data)
|
||||||
|
@ -719,6 +720,7 @@ handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
||||||
ok;
|
ok;
|
||||||
{ok, _Other} ->
|
{ok, _Other} ->
|
||||||
%% TODO: leave some debug/warning message here?
|
%% TODO: leave some debug/warning message here?
|
||||||
|
io:format(user, "baposdifa;lsdfkj<<<<<<<~n", []),
|
||||||
{error, written}
|
{error, written}
|
||||||
end;
|
end;
|
||||||
[{Offset, Size, OtherCsum}] ->
|
[{Offset, Size, OtherCsum}] ->
|
||||||
|
@ -727,11 +729,12 @@ handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
||||||
" a check for unwritten bytes gave us checksum ~p"
|
" a check for unwritten bytes gave us checksum ~p"
|
||||||
" but the data we were trying to write has checksum ~p",
|
" but the data we were trying to write has checksum ~p",
|
||||||
[Offset, Filename, OtherCsum, TaggedCsum]),
|
[Offset, Filename, OtherCsum, TaggedCsum]),
|
||||||
|
io:format(user, "baposdifa;lsdfkj*************8~n", []),
|
||||||
{error, written};
|
{error, written};
|
||||||
_Chunks ->
|
_Chunks ->
|
||||||
%% TODO: Do we try to read all continuous chunks to see
|
%% TODO: Do we try to read all continuous chunks to see
|
||||||
%% wether its total checksum matches client-provided checksum?
|
%% wether its total checksum matches client-provided checksum?
|
||||||
case machi_csum_table:any_trimmed(CsumTable, Filename,
|
case machi_csum_table:any_trimmed(CsumTable, iolist_to_binary(Filename),
|
||||||
Offset, Size) of
|
Offset, Size) of
|
||||||
true ->
|
true ->
|
||||||
%% More than a byte is trimmed, besides, do we
|
%% More than a byte is trimmed, besides, do we
|
||||||
|
@ -741,6 +744,7 @@ handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
||||||
{error, trimmed};
|
{error, trimmed};
|
||||||
false ->
|
false ->
|
||||||
%% No byte is trimmed, but at least one byte is written
|
%% No byte is trimmed, but at least one byte is written
|
||||||
|
io:format(user, "baposdifa;lsdfkj*************8 ~p~n", [_Chunks]),
|
||||||
{error, written}
|
{error, written}
|
||||||
end
|
end
|
||||||
end.
|
end.
|
||||||
|
@ -758,6 +762,7 @@ handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
||||||
do_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Size, Data) ->
|
do_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Size, Data) ->
|
||||||
case file:pwrite(FHd, Offset, Data) of
|
case file:pwrite(FHd, Offset, Data) of
|
||||||
ok ->
|
ok ->
|
||||||
|
F = iolist_to_binary(Filename),
|
||||||
lager:debug("Successful write in file ~p at offset ~p, length ~p",
|
lager:debug("Successful write in file ~p at offset ~p, length ~p",
|
||||||
[Filename, Offset, Size]),
|
[Filename, Offset, Size]),
|
||||||
|
|
||||||
|
@ -767,14 +772,14 @@ do_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Size, Data) ->
|
||||||
LUpdate = maybe_regenerate_checksum(
|
LUpdate = maybe_regenerate_checksum(
|
||||||
FHd,
|
FHd,
|
||||||
machi_csum_table:find_leftneighbor(CsumTable,
|
machi_csum_table:find_leftneighbor(CsumTable,
|
||||||
Filename,
|
F,
|
||||||
Offset)),
|
Offset)),
|
||||||
RUpdate = maybe_regenerate_checksum(
|
RUpdate = maybe_regenerate_checksum(
|
||||||
FHd,
|
FHd,
|
||||||
machi_csum_table:find_rightneighbor(CsumTable,
|
machi_csum_table:find_rightneighbor(CsumTable,
|
||||||
Filename,
|
F,
|
||||||
Offset+Size)),
|
Offset+Size)),
|
||||||
ok = machi_csum_table:write(CsumTable, Filename, Offset, Size,
|
ok = machi_csum_table:write(CsumTable, F, Offset, Size,
|
||||||
TaggedCsum, LUpdate, RUpdate),
|
TaggedCsum, LUpdate, RUpdate),
|
||||||
lager:debug("Successful write to checksum file for ~p",
|
lager:debug("Successful write to checksum file for ~p",
|
||||||
[Filename]),
|
[Filename]),
|
||||||
|
@ -845,7 +850,7 @@ maybe_gc(Reply, S = #state{data_filehandle = FHd,
|
||||||
eof_position = Eof,
|
eof_position = Eof,
|
||||||
csum_table=CsumTable}) ->
|
csum_table=CsumTable}) ->
|
||||||
lager:debug("GC? Let's try it: ~p.~n", [Filename]),
|
lager:debug("GC? Let's try it: ~p.~n", [Filename]),
|
||||||
case machi_csum_table:maybe_trim_file(CsumTable, Filename, Eof) of
|
case machi_csum_table:maybe_trim_file(CsumTable, iolist_to_binary(Filename), Eof) of
|
||||||
{ok, trimmed} ->
|
{ok, trimmed} ->
|
||||||
%% Checksum table entries are all trimmed now, unlinking
|
%% Checksum table entries are all trimmed now, unlinking
|
||||||
%% file from operating system
|
%% file from operating system
|
||||||
|
|
|
@ -2,68 +2,69 @@
|
||||||
-compile(export_all).
|
-compile(export_all).
|
||||||
|
|
||||||
-include_lib("eunit/include/eunit.hrl").
|
-include_lib("eunit/include/eunit.hrl").
|
||||||
-define(HDR, {0, 1024, none}).
|
|
||||||
|
|
||||||
cleanup(Dir) ->
|
cleanup(Dir) ->
|
||||||
os:cmd("rm -rf " ++ Dir).
|
os:cmd("rm -rf " ++ Dir).
|
||||||
|
|
||||||
smoke_test() ->
|
smoke_test() ->
|
||||||
Filename = "./temp-checksum-dumb-file",
|
DBFile = "./temp-checksum-dumb-file",
|
||||||
_ = cleanup(Filename),
|
Filename = <<"/some/puppy/and/cats^^^42">>,
|
||||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
_ = cleanup(DBFile),
|
||||||
?assertEqual([{1024, infinity}],
|
{ok, MC} = machi_csum_table:open(DBFile, []),
|
||||||
machi_csum_table:calc_unwritten_bytes(MC)),
|
?assertEqual([{0, infinity}],
|
||||||
|
machi_csum_table:calc_unwritten_bytes(MC, Filename)),
|
||||||
Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
|
Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
|
||||||
[] = machi_csum_table:find(MC, Offset, Size),
|
[] = machi_csum_table:find(MC, Filename, Offset, Size),
|
||||||
ok = machi_csum_table:write(MC, Offset, Size, Checksum),
|
ok = machi_csum_table:write(MC, Filename, Offset, Size, Checksum),
|
||||||
[{1024, 40}, {1098, infinity}] = machi_csum_table:calc_unwritten_bytes(MC),
|
[{0, 1064}, {1098, infinity}] = machi_csum_table:calc_unwritten_bytes(MC, Filename),
|
||||||
?assertEqual([Entry], machi_csum_table:find(MC, Offset, Size)),
|
?assertEqual([Entry], machi_csum_table:find(MC, Filename, Offset, Size)),
|
||||||
ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined),
|
ok = machi_csum_table:trim(MC, Filename, Offset, Size, undefined, undefined),
|
||||||
?assertEqual([{Offset, Size, trimmed}],
|
?assertEqual([{Offset, Size, trimmed}],
|
||||||
machi_csum_table:find(MC, Offset, Size)),
|
machi_csum_table:find(MC, Filename, Offset, Size)),
|
||||||
ok = machi_csum_table:close(MC),
|
ok = machi_csum_table:close(MC).
|
||||||
ok = machi_csum_table:delete(MC).
|
|
||||||
|
|
||||||
close_test() ->
|
close_test() ->
|
||||||
Filename = "./temp-checksum-dumb-file-2",
|
DBFile = "./temp-checksum-dumb-file-2",
|
||||||
_ = cleanup(Filename),
|
Filename = <<"/some/puppy/and/cats^^^43">>,
|
||||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
_ = cleanup(DBFile),
|
||||||
|
{ok, MC} = machi_csum_table:open(DBFile, []),
|
||||||
Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
|
Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
|
||||||
[] = machi_csum_table:find(MC, Offset, Size),
|
[] = machi_csum_table:find(MC, Filename, Offset, Size),
|
||||||
ok = machi_csum_table:write(MC, Offset, Size, Checksum),
|
ok = machi_csum_table:write(MC, Filename, Offset, Size, Checksum),
|
||||||
[Entry] = machi_csum_table:find(MC, Offset, Size),
|
[Entry] = machi_csum_table:find(MC, Filename, Offset, Size),
|
||||||
ok = machi_csum_table:close(MC),
|
ok = machi_csum_table:close(MC),
|
||||||
|
|
||||||
{ok, MC2} = machi_csum_table:open(Filename, []),
|
{ok, MC2} = machi_csum_table:open(DBFile, []),
|
||||||
[Entry] = machi_csum_table:find(MC2, Offset, Size),
|
[Entry] = machi_csum_table:find(MC2, Filename, Offset, Size),
|
||||||
ok = machi_csum_table:trim(MC2, Offset, Size, undefined, undefined),
|
ok = machi_csum_table:trim(MC2, Filename, Offset, Size, undefined, undefined),
|
||||||
[{Offset, Size, trimmed}] = machi_csum_table:find(MC2, Offset, Size),
|
[{Offset, Size, trimmed}] = machi_csum_table:find(MC2, Filename, Offset, Size),
|
||||||
ok = machi_csum_table:delete(MC2).
|
ok = machi_csum_table:close(MC2).
|
||||||
|
|
||||||
smoke2_test() ->
|
smoke2_test() ->
|
||||||
Filename = "./temp-checksum-dumb-file-3",
|
DBFile = "./temp-checksum-dumb-file-3",
|
||||||
_ = cleanup(Filename),
|
Filename = <<"/some/puppy/and/cats^^^43">>,
|
||||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
_ = cleanup(DBFile),
|
||||||
|
{ok, MC} = machi_csum_table:open(DBFile, []),
|
||||||
Entry = {Offset, Size, Checksum} = {1025, 10, <<"deadbeef">>},
|
Entry = {Offset, Size, Checksum} = {1025, 10, <<"deadbeef">>},
|
||||||
ok = machi_csum_table:write(MC, Offset, Size, Checksum),
|
ok = machi_csum_table:write(MC, Filename, Offset, Size, Checksum),
|
||||||
?assertEqual([], machi_csum_table:find(MC, 0, 0)),
|
?assertEqual([], machi_csum_table:find(MC, Filename, 0, 0)),
|
||||||
?assertEqual([?HDR], machi_csum_table:find(MC, 0, 1)),
|
?assertEqual([], machi_csum_table:find(MC, Filename, 0, 1)),
|
||||||
[Entry] = machi_csum_table:find(MC, Offset, Size),
|
[Entry] = machi_csum_table:find(MC, Filename, Offset, Size),
|
||||||
[?HDR] = machi_csum_table:find(MC, 1, 1024),
|
[] = machi_csum_table:find(MC, Filename, 1, 1024),
|
||||||
?assertEqual([?HDR, Entry],
|
?assertEqual([Entry],
|
||||||
machi_csum_table:find(MC, 1023, 1024)),
|
machi_csum_table:find(MC, Filename, 1023, 1024)),
|
||||||
[Entry] = machi_csum_table:find(MC, 1024, 1024),
|
[Entry] = machi_csum_table:find(MC, Filename, 1024, 1024),
|
||||||
[Entry] = machi_csum_table:find(MC, 1025, 1024),
|
[Entry] = machi_csum_table:find(MC, Filename, 1025, 1024),
|
||||||
|
|
||||||
ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined),
|
ok = machi_csum_table:trim(MC, Filename, Offset, Size, undefined, undefined),
|
||||||
[{Offset, Size, trimmed}] = machi_csum_table:find(MC, Offset, Size),
|
[{Offset, Size, trimmed}] = machi_csum_table:find(MC, Filename, Offset, Size),
|
||||||
ok = machi_csum_table:close(MC),
|
ok = machi_csum_table:close(MC).
|
||||||
ok = machi_csum_table:delete(MC).
|
|
||||||
|
|
||||||
smoke3_test() ->
|
smoke3_test() ->
|
||||||
Filename = "./temp-checksum-dumb-file-4",
|
DBFile = "./temp-checksum-dumb-file-4",
|
||||||
_ = cleanup(Filename),
|
Filename = <<"/some/puppy/and/cats^^^44">>,
|
||||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
_ = cleanup(DBFile),
|
||||||
|
{ok, MC} = machi_csum_table:open(DBFile, []),
|
||||||
Scenario =
|
Scenario =
|
||||||
[%% Command, {Offset, Size, Csum}, LeftNeighbor, RightNeibor
|
[%% Command, {Offset, Size, Csum}, LeftNeighbor, RightNeibor
|
||||||
{?LINE, write, {2000, 10, <<"heh">>}, undefined, undefined},
|
{?LINE, write, {2000, 10, <<"heh">>}, undefined, undefined},
|
||||||
|
@ -84,9 +85,9 @@ smoke3_test() ->
|
||||||
%% ?debugVal({_Line, Chunk}),
|
%% ?debugVal({_Line, Chunk}),
|
||||||
{Offset, Size, Csum} = Chunk,
|
{Offset, Size, Csum} = Chunk,
|
||||||
?assertEqual(LeftN0,
|
?assertEqual(LeftN0,
|
||||||
machi_csum_table:find_leftneighbor(MC, Offset)),
|
machi_csum_table:find_leftneighbor(MC, Filename, Offset)),
|
||||||
?assertEqual(RightN0,
|
?assertEqual(RightN0,
|
||||||
machi_csum_table:find_rightneighbor(MC, Offset+Size)),
|
machi_csum_table:find_rightneighbor(MC, Filename, Offset+Size)),
|
||||||
LeftN = case LeftN0 of
|
LeftN = case LeftN0 of
|
||||||
{OffsL, SizeL, trimmed} -> {OffsL, SizeL, trimmed};
|
{OffsL, SizeL, trimmed} -> {OffsL, SizeL, trimmed};
|
||||||
{OffsL, SizeL, _} -> {OffsL, SizeL, <<"boom">>};
|
{OffsL, SizeL, _} -> {OffsL, SizeL, <<"boom">>};
|
||||||
|
@ -98,19 +99,18 @@ smoke3_test() ->
|
||||||
end,
|
end,
|
||||||
case Cmd of
|
case Cmd of
|
||||||
write ->
|
write ->
|
||||||
ok = machi_csum_table:write(MC, Offset, Size, Csum,
|
ok = machi_csum_table:write(MC, Filename, Offset, Size, Csum,
|
||||||
LeftN, RightN);
|
LeftN, RightN);
|
||||||
trim ->
|
trim ->
|
||||||
ok = machi_csum_table:trim(MC, Offset, Size,
|
ok = machi_csum_table:trim(MC, Filename, Offset, Size,
|
||||||
LeftN, RightN)
|
LeftN, RightN)
|
||||||
end
|
end
|
||||||
end || {_Line, Cmd, Chunk, LeftN0, RightN0} <- Scenario ],
|
end || {_Line, Cmd, Chunk, LeftN0, RightN0} <- Scenario ],
|
||||||
?assert(not machi_csum_table:all_trimmed(MC, 10000)),
|
?assert(not machi_csum_table:all_trimmed(MC, Filename, 0, 10000)),
|
||||||
machi_csum_table:trim(MC, 0, 10000, undefined, undefined),
|
machi_csum_table:trim(MC, Filename, 0, 10000, undefined, undefined),
|
||||||
?assert(machi_csum_table:all_trimmed(MC, 10000)),
|
?assert(machi_csum_table:all_trimmed(MC, Filename, 0, 10000)),
|
||||||
|
|
||||||
ok = machi_csum_table:close(MC),
|
ok = machi_csum_table:close(MC).
|
||||||
ok = machi_csum_table:delete(MC).
|
|
||||||
|
|
||||||
%% TODO: add quickcheck test here
|
%% TODO: add quickcheck test here
|
||||||
|
|
||||||
|
|
|
@ -116,11 +116,11 @@ get_written_interval(L) ->
|
||||||
initial_state() ->
|
initial_state() ->
|
||||||
{_, _, MS} = os:timestamp(),
|
{_, _, MS} = os:timestamp(),
|
||||||
Filename = test_server:temp_name("eqc_data") ++ "." ++ integer_to_list(MS),
|
Filename = test_server:temp_name("eqc_data") ++ "." ++ integer_to_list(MS),
|
||||||
#state{filename=Filename, written=[{0,1024}]}.
|
#state{filename=Filename, written=[]}.
|
||||||
|
|
||||||
initial_state(I, T) ->
|
initial_state(I, T) ->
|
||||||
S=initial_state(),
|
S=initial_state(),
|
||||||
S#state{written=[{0,1024}],
|
S#state{written=[],
|
||||||
planned_writes=I,
|
planned_writes=I,
|
||||||
planned_trims=T}.
|
planned_trims=T}.
|
||||||
|
|
||||||
|
@ -230,7 +230,8 @@ start_command(S) ->
|
||||||
{call, ?MODULE, start, [S]}.
|
{call, ?MODULE, start, [S]}.
|
||||||
|
|
||||||
start(#state{filename=File}) ->
|
start(#state{filename=File}) ->
|
||||||
{ok, Pid} = machi_file_proxy:start_link(some_flu, File, ?TESTDIR),
|
CsumT = get_csum_table(),
|
||||||
|
{ok, Pid} = machi_file_proxy:start_link(File, ?TESTDIR, CsumT),
|
||||||
unlink(Pid),
|
unlink(Pid),
|
||||||
Pid.
|
Pid.
|
||||||
|
|
||||||
|
@ -432,6 +433,40 @@ stop_post(_, _, _) -> true.
|
||||||
stop_next(S, _, _) ->
|
stop_next(S, _, _) ->
|
||||||
S#state{pid=undefined, prev_extra=0}.
|
S#state{pid=undefined, prev_extra=0}.
|
||||||
|
|
||||||
|
csum_table_holder() ->
|
||||||
|
Parent = self(),
|
||||||
|
spawn_link(fun() ->
|
||||||
|
CsumFile = test_server:temp_name("eqc_data-csum"),
|
||||||
|
filelib:ensure_dir(CsumFile),
|
||||||
|
{ok, CsumT} = machi_csum_table:open(CsumFile, []),
|
||||||
|
erlang:register(csum_table_holder, self()),
|
||||||
|
Parent ! ok,
|
||||||
|
csum_table_holder_loop(CsumT),
|
||||||
|
machi_csum_table:close(CsumT),
|
||||||
|
erlang:unregister(csum_table_holder)
|
||||||
|
end),
|
||||||
|
receive
|
||||||
|
Other -> Other
|
||||||
|
after 1000 ->
|
||||||
|
timeout
|
||||||
|
end.
|
||||||
|
|
||||||
|
csum_table_holder_loop(CsumT) ->
|
||||||
|
receive
|
||||||
|
{get, From} ->
|
||||||
|
From ! CsumT;
|
||||||
|
stop ->
|
||||||
|
ok
|
||||||
|
end.
|
||||||
|
|
||||||
|
get_csum_table() ->
|
||||||
|
csum_table_holder ! {get, self()},
|
||||||
|
receive CsumT -> CsumT
|
||||||
|
end.
|
||||||
|
|
||||||
|
stop_csum_table_holder() ->
|
||||||
|
catch csum_table_holder ! stop.
|
||||||
|
|
||||||
%% Property
|
%% Property
|
||||||
|
|
||||||
prop_ok() ->
|
prop_ok() ->
|
||||||
|
@ -440,7 +475,9 @@ prop_ok() ->
|
||||||
{shuffle_interval(), shuffle_interval()},
|
{shuffle_interval(), shuffle_interval()},
|
||||||
?FORALL(Cmds, parallel_commands(?MODULE, initial_state(I, T)),
|
?FORALL(Cmds, parallel_commands(?MODULE, initial_state(I, T)),
|
||||||
begin
|
begin
|
||||||
|
ok = csum_table_holder(),
|
||||||
{H, S, Res} = run_parallel_commands(?MODULE, Cmds),
|
{H, S, Res} = run_parallel_commands(?MODULE, Cmds),
|
||||||
|
stop_csum_table_holder(),
|
||||||
cleanup(),
|
cleanup(),
|
||||||
pretty_commands(?MODULE, Cmds, {H, S, Res},
|
pretty_commands(?MODULE, Cmds, {H, S, Res},
|
||||||
aggregate(command_names(Cmds), Res == ok))
|
aggregate(command_names(Cmds), Res == ok))
|
||||||
|
|
|
@ -77,25 +77,28 @@ random_binary(Start, End) ->
|
||||||
end.
|
end.
|
||||||
|
|
||||||
setup() ->
|
setup() ->
|
||||||
{ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR),
|
{ok, CsumT} = machi_csum_table:open(filename:join([?TESTDIR, "csumfile"]), []),
|
||||||
Pid.
|
{ok, Pid} = machi_file_proxy:start_link("test", ?TESTDIR, CsumT),
|
||||||
|
{Pid, CsumT}.
|
||||||
|
|
||||||
teardown(Pid) ->
|
teardown({Pid, CsumT}) ->
|
||||||
catch machi_file_proxy:stop(Pid).
|
catch machi_file_proxy:stop(Pid),
|
||||||
|
catch machi_csum_table:close(CsumT).
|
||||||
|
|
||||||
machi_file_proxy_test_() ->
|
machi_file_proxy_test_() ->
|
||||||
clean_up_data_dir(?TESTDIR),
|
clean_up_data_dir(?TESTDIR),
|
||||||
{setup,
|
{setup,
|
||||||
fun setup/0,
|
fun setup/0,
|
||||||
fun teardown/1,
|
fun teardown/1,
|
||||||
fun(Pid) ->
|
fun({Pid, _}) ->
|
||||||
[
|
[
|
||||||
?_assertEqual({error, bad_arg}, machi_file_proxy:read(Pid, -1, -1)),
|
?_assertEqual({error, bad_arg}, machi_file_proxy:read(Pid, -1, -1)),
|
||||||
?_assertEqual({error, bad_arg}, machi_file_proxy:write(Pid, -1, <<"yo">>)),
|
?_assertEqual({error, bad_arg}, machi_file_proxy:write(Pid, -1, <<"yo">>)),
|
||||||
?_assertEqual({error, bad_arg}, machi_file_proxy:append(Pid, [], -1, <<"krep">>)),
|
?_assertEqual({error, bad_arg}, machi_file_proxy:append(Pid, [], -1, <<"krep">>)),
|
||||||
?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1)),
|
?_assertMatch({error, not_written}, machi_file_proxy:read(Pid, 1, 1)),
|
||||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, 1)),
|
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, 1)),
|
||||||
?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1024)),
|
?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))),
|
||||||
|
?_assertMatch({ok, _}, machi_file_proxy:read(Pid, 1, 1024)),
|
||||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, ?HYOOGE)),
|
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, ?HYOOGE)),
|
||||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, ?HYOOGE, 1)),
|
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, ?HYOOGE, 1)),
|
||||||
{timeout, 10,
|
{timeout, 10,
|
||||||
|
@ -114,7 +117,7 @@ multiple_chunks_read_test_() ->
|
||||||
{setup,
|
{setup,
|
||||||
fun setup/0,
|
fun setup/0,
|
||||||
fun teardown/1,
|
fun teardown/1,
|
||||||
fun(Pid) ->
|
fun({Pid, _}) ->
|
||||||
[
|
[
|
||||||
?_assertEqual(ok, machi_file_proxy:trim(Pid, 0, 1, false)),
|
?_assertEqual(ok, machi_file_proxy:trim(Pid, 0, 1, false)),
|
||||||
?_assertMatch({ok, {[], [{"test", 0, 1}]}},
|
?_assertMatch({ok, {[], [{"test", 0, 1}]}},
|
||||||
|
|
Loading…
Reference in a new issue