WIP
This commit is contained in:
UENISHI Kota
parent
d482902509
commit
8103b424c0
6 changed files with 198 additions and 116 deletions
|
|
@ -21,7 +21,7 @@
|
|||
%% @doc Now 4GiBytes, could be up to 64bit due to PB message limit of
|
||||
%% chunk size
|
||||
-define(DEFAULT_MAX_FILE_SIZE, ((1 bsl 32) - 1)).
|
||||
-define(MINIMUM_OFFSET, 1024).
|
||||
-define(MINIMUM_OFFSET, 0).
|
||||
|
||||
%% 0th draft of checksum typing with 1st byte.
|
||||
-define(CSUM_TAG_NONE, 0). % No csum provided by client
|
||||
|
|
|
|||
|
|
@ -43,12 +43,8 @@ open(CSumFilename, _Opts) ->
|
|||
%% operating system's file cache, which is for
|
||||
%% Machi's main read efficiency
|
||||
{total_leveldb_mem_percent, 10}],
|
||||
ok = filelib:ensure_dir(CSumFilename),
|
||||
{ok, T} = eleveldb:open(CSumFilename, LevelDBOptions),
|
||||
%% Dummy entry for reserved headers
|
||||
%% ok = eleveldb:put(T,
|
||||
%% sext:encode({0, ?MINIMUM_OFFSET}),
|
||||
%% sext:encode(?CSUM_TAG_NONE_ATOM),
|
||||
%% [{sync, true}]),
|
||||
C0 = #machi_csum_table{
|
||||
file=CSumFilename,
|
||||
table=T},
|
||||
|
|
@ -64,35 +60,26 @@ split_checksum_list_blob_decode(Bin) ->
|
|||
-spec find(table(), binary(), machi_dt:file_offset(), machi_dt:chunk_size())
|
||||
-> [chunk()].
|
||||
find(#machi_csum_table{table=T}, Filename, Offset, Size) when is_binary(Filename) ->
|
||||
{ok, I} = eleveldb:iterator(T, [], keys_only),
|
||||
EndKey = sext:encode({Filename, Offset+Size, 0}),
|
||||
StartKey = sext:encode({Filename, Offset, Size}),
|
||||
|
||||
{ok, FirstKey} = case eleveldb:iterator_move(I, StartKey) of
|
||||
{error, invalid_iterator} ->
|
||||
eleveldb:iterator_move(I, first);
|
||||
{ok, _} = R0 ->
|
||||
case eleveldb:iterator_move(I, prev) of
|
||||
{error, invalid_iterator} ->
|
||||
R0;
|
||||
{ok, _} = R1 ->
|
||||
R1
|
||||
end
|
||||
end,
|
||||
_ = eleveldb:iterator_close(I),
|
||||
FoldFun = fun({K, V}, Acc) ->
|
||||
{Filename, TargetOffset, TargetSize} = sext:decode(K),
|
||||
case ?has_overlap(TargetOffset, TargetSize, Offset, Size) of
|
||||
true ->
|
||||
[{TargetOffset, TargetSize, sext:decode(V)}|Acc];
|
||||
false ->
|
||||
case search_for_start_key(T, Filename, Offset, Size) of
|
||||
undefined -> [];
|
||||
FirstKey ->
|
||||
|
||||
FoldFun = fun({K, V}, Acc) ->
|
||||
{Filename, TargetOffset, TargetSize} = sext:decode(K),
|
||||
case ?has_overlap(TargetOffset, TargetSize, Offset, Size) of
|
||||
true ->
|
||||
[{TargetOffset, TargetSize, sext:decode(V)}|Acc];
|
||||
false ->
|
||||
Acc
|
||||
end;
|
||||
(_K, Acc) ->
|
||||
lager:error("~p wrong option", [_K]),
|
||||
Acc
|
||||
end;
|
||||
(_K, Acc) ->
|
||||
lager:error("~p wrong option", [_K]),
|
||||
Acc
|
||||
end,
|
||||
lists:reverse(eleveldb_fold(T, FirstKey, EndKey, FoldFun, [])).
|
||||
end,
|
||||
lists:reverse(eleveldb_fold(T, FirstKey, EndKey, FoldFun, []))
|
||||
end.
|
||||
|
||||
|
||||
%% @doc Updates all chunk info, by deleting existing entries if exists
|
||||
|
|
@ -196,10 +183,11 @@ any_trimmed(CsumT, Filename, Offset, Size) ->
|
|||
calc_unwritten_bytes(#machi_csum_table{table=_} = CsumT, Filename) ->
|
||||
case lists:sort(all(CsumT, Filename)) of
|
||||
[] ->
|
||||
[{?MINIMUM_OFFSET, infinity}];
|
||||
Sorted ->
|
||||
{LastOffset, _, _} = hd(Sorted),
|
||||
build_unwritten_bytes_list(Sorted, LastOffset, [])
|
||||
[{0, infinity}];
|
||||
[{0, _, _}|_] = Sorted ->
|
||||
build_unwritten_bytes_list(Sorted, 0, []);
|
||||
[{LastOffset, _, _}|_] = Sorted ->
|
||||
build_unwritten_bytes_list(Sorted, LastOffset, [{0, LastOffset}])
|
||||
end.
|
||||
|
||||
all(CsumT, Filename) ->
|
||||
|
|
@ -233,7 +221,7 @@ maybe_trim_file(#machi_csum_table{table=T} = CsumT, Filename, EofP) when is_bina
|
|||
%% @doc Folds over all chunks of a file
|
||||
-spec foldl_file_chunks(fun((chunk(), Acc0 :: term()) -> Acc :: term()),
|
||||
Acc0 :: term(), table(), binary()) -> Acc :: term().
|
||||
foldl_file_chunks(Fun, Acc0, #machi_csum_table{table=T}, Filename) ->
|
||||
foldl_file_chunks(Fun, Acc0, #machi_csum_table{table=T}, Filename) when is_binary(Filename) ->
|
||||
FoldFun = fun({K, V}, Acc) ->
|
||||
{Filename, Offset, Len} = sext:decode(K),
|
||||
Fun({Offset, Len, sext:decode(V)}, Acc);
|
||||
|
|
@ -260,6 +248,8 @@ foldl_chunks(Fun, Acc0, #machi_csum_table{table=T}) ->
|
|||
end,
|
||||
eleveldb:fold(T, FoldFun, Acc0, [{verify_checksums, true}]).
|
||||
|
||||
%% == internal functions ==
|
||||
|
||||
-spec build_unwritten_bytes_list( CsumData :: [{ Offset :: non_neg_integer(),
|
||||
Size :: pos_integer(),
|
||||
Checksum :: binary() }],
|
||||
|
|
@ -323,3 +313,50 @@ eleveldb_do_fold({error, iterator_closed}, _, _, _, Acc) ->
|
|||
eleveldb_do_fold({error, invalid_iterator}, _, _, _, Acc) ->
|
||||
%% Probably reached to end
|
||||
Acc.
|
||||
|
||||
|
||||
%% Key1 < MaybeStartKey =< Key
|
||||
%% FirstKey =< MaybeStartKey
|
||||
search_for_start_key(T, Filename, Offset, Size) ->
|
||||
MaybeStartKey = sext:encode({Filename, Offset, Size}),
|
||||
FirstKey = sext:encode({Filename, 0, 0}),
|
||||
{ok, I} = eleveldb:iterator(T, [], keys_only),
|
||||
|
||||
try
|
||||
case eleveldb:iterator_move(I, MaybeStartKey) of
|
||||
{error, invalid_iterator} ->
|
||||
%% No key in right - go for probably first key in the file
|
||||
case eleveldb:iterator_move(I, FirstKey) of
|
||||
{error, _} -> undefined;
|
||||
{ok, Key0} -> goto_end(I, Key0, Offset)
|
||||
end;
|
||||
{ok, Key} when Key < FirstKey ->
|
||||
FirstKey;
|
||||
{ok, Key} ->
|
||||
case eleveldb:iterator_move(I, prev) of
|
||||
{error, invalid_iterator} ->
|
||||
Key;
|
||||
{ok, Key1} when Key1 < FirstKey ->
|
||||
Key;
|
||||
{ok, Key1} ->
|
||||
Key1
|
||||
end
|
||||
end
|
||||
after
|
||||
_ = eleveldb:iterator_close(I)
|
||||
end.
|
||||
|
||||
goto_end(I, Key, Offset) ->
|
||||
case sext:decode(Key) of
|
||||
{_Filename, O, L} when Offset =< O + L ->
|
||||
Key;
|
||||
{_Filename, O, L} when O + L < Offset ->
|
||||
case eleveldb:iterator_move(I, next) of
|
||||
{ok, NextKey} ->
|
||||
goto_end(I, NextKey, Offset);
|
||||
{error, _} ->
|
||||
Key
|
||||
end
|
||||
end.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -221,7 +221,7 @@ checksum_list(Pid) ->
|
|||
init({Filename, DataDir, CsumTable}) ->
|
||||
{_, DPath} = machi_util:make_data_filename(DataDir, Filename),
|
||||
ok = filelib:ensure_dir(DPath),
|
||||
UnwrittenBytes = machi_csum_table:calc_unwritten_bytes(CsumTable),
|
||||
UnwrittenBytes = machi_csum_table:calc_unwritten_bytes(CsumTable, iolist_to_binary(Filename)),
|
||||
{Eof, infinity} = lists:last(UnwrittenBytes),
|
||||
{ok, FHd} = file:open(DPath, [read, write, binary, raw]),
|
||||
%% Reserve for EC and stuff, to prevent eof when read
|
||||
|
|
@ -343,7 +343,7 @@ handle_call({write, Offset, ClientMeta, Data}, _From,
|
|||
{Error, Err + 1}
|
||||
end
|
||||
end,
|
||||
{NewEof, infinity} = lists:last(machi_csum_table:calc_unwritten_bytes(CsumTable, F)),
|
||||
{NewEof, infinity} = lists:last(machi_csum_table:calc_unwritten_bytes(CsumTable, iolist_to_binary(F))),
|
||||
lager:debug("Wrote ~p bytes at ~p of file ~p, NewEOF = ~p~n",
|
||||
[iolist_size(Data), Offset, F, NewEof]),
|
||||
{reply, Resp, State#state{writes = {T+1, NewErr},
|
||||
|
|
@ -365,7 +365,8 @@ handle_call({trim, Offset, Size, _TriggerGC}, _From,
|
|||
trims = {T, Err},
|
||||
csum_table = CsumTable}) ->
|
||||
|
||||
case machi_csum_table:all_trimmed(CsumTable, Filename,
|
||||
F = iolist_to_binary(Filename),
|
||||
case machi_csum_table:all_trimmed(CsumTable, F,
|
||||
Offset, Offset+Size) of
|
||||
true ->
|
||||
NewState = State#state{ops=Ops+1, trims={T, Err+1}},
|
||||
|
|
@ -377,18 +378,18 @@ handle_call({trim, Offset, Size, _TriggerGC}, _From,
|
|||
LUpdate = maybe_regenerate_checksum(
|
||||
FHd,
|
||||
machi_csum_table:find_leftneighbor(CsumTable,
|
||||
Filename,
|
||||
F,
|
||||
Offset)),
|
||||
RUpdate = maybe_regenerate_checksum(
|
||||
FHd,
|
||||
machi_csum_table:find_rightneighbor(CsumTable,
|
||||
Filename,
|
||||
F,
|
||||
Offset+Size)),
|
||||
|
||||
case machi_csum_table:trim(CsumTable, Filename, Offset,
|
||||
case machi_csum_table:trim(CsumTable, F, Offset,
|
||||
Size, LUpdate, RUpdate) of
|
||||
ok ->
|
||||
{NewEof, infinity} = lists:last(machi_csum_table:calc_unwritten_bytes(CsumTable, Filename)),
|
||||
{NewEof, infinity} = lists:last(machi_csum_table:calc_unwritten_bytes(CsumTable, F)),
|
||||
NewState = State#state{ops=Ops+1,
|
||||
trims={T+1, Err},
|
||||
eof_position=NewEof},
|
||||
|
|
@ -439,7 +440,7 @@ handle_call({append, ClientMeta, Extra, Data}, _From,
|
|||
|
||||
handle_call({checksum_list}, _FRom, State = #state{filename=Filename,
|
||||
csum_table=T}) ->
|
||||
All = machi_csum_table:all(T, Filename),
|
||||
All = machi_csum_table:all(T,iolist_to_binary(Filename)),
|
||||
{reply, {ok, All}, State};
|
||||
|
||||
handle_call(Req, _From, State) ->
|
||||
|
|
@ -617,7 +618,7 @@ check_or_make_tagged_csum(OtherTag, _ClientCsum, _Data) ->
|
|||
do_read(FHd, Filename, CsumTable, Offset, Size, _, _) ->
|
||||
%% Note that find/3 only returns overlapping chunks, both borders
|
||||
%% are not aligned to original Offset and Size.
|
||||
ChunkCsums = machi_csum_table:find(CsumTable, Filename,
|
||||
ChunkCsums = machi_csum_table:find(CsumTable, iolist_to_binary(Filename),
|
||||
Offset, Size),
|
||||
read_all_ranges(FHd, Filename, ChunkCsums, [], []).
|
||||
|
||||
|
|
@ -696,7 +697,7 @@ read_all_ranges(FHd, Filename, [{Offset, Size, TaggedCsum}|T], ReadChunks, Trimm
|
|||
handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
||||
Size = iolist_size(Data),
|
||||
|
||||
case machi_csum_table:find(CsumTable, Filename, Offset, Size) of
|
||||
case machi_csum_table:find(CsumTable, iolist_to_binary(Filename), Offset, Size) of
|
||||
[] -> %% Nothing should be there
|
||||
try
|
||||
do_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Size, Data)
|
||||
|
|
@ -719,6 +720,7 @@ handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
|||
ok;
|
||||
{ok, _Other} ->
|
||||
%% TODO: leave some debug/warning message here?
|
||||
io:format(user, "baposdifa;lsdfkj<<<<<<<~n", []),
|
||||
{error, written}
|
||||
end;
|
||||
[{Offset, Size, OtherCsum}] ->
|
||||
|
|
@ -727,11 +729,12 @@ handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
|||
" a check for unwritten bytes gave us checksum ~p"
|
||||
" but the data we were trying to write has checksum ~p",
|
||||
[Offset, Filename, OtherCsum, TaggedCsum]),
|
||||
io:format(user, "baposdifa;lsdfkj*************8~n", []),
|
||||
{error, written};
|
||||
_Chunks ->
|
||||
%% TODO: Do we try to read all continuous chunks to see
|
||||
%% wether its total checksum matches client-provided checksum?
|
||||
case machi_csum_table:any_trimmed(CsumTable, Filename,
|
||||
case machi_csum_table:any_trimmed(CsumTable, iolist_to_binary(Filename),
|
||||
Offset, Size) of
|
||||
true ->
|
||||
%% More than a byte is trimmed, besides, do we
|
||||
|
|
@ -741,6 +744,7 @@ handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
|||
{error, trimmed};
|
||||
false ->
|
||||
%% No byte is trimmed, but at least one byte is written
|
||||
io:format(user, "baposdifa;lsdfkj*************8 ~p~n", [_Chunks]),
|
||||
{error, written}
|
||||
end
|
||||
end.
|
||||
|
|
@ -758,6 +762,7 @@ handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
|||
do_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Size, Data) ->
|
||||
case file:pwrite(FHd, Offset, Data) of
|
||||
ok ->
|
||||
F = iolist_to_binary(Filename),
|
||||
lager:debug("Successful write in file ~p at offset ~p, length ~p",
|
||||
[Filename, Offset, Size]),
|
||||
|
||||
|
|
@ -767,14 +772,14 @@ do_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Size, Data) ->
|
|||
LUpdate = maybe_regenerate_checksum(
|
||||
FHd,
|
||||
machi_csum_table:find_leftneighbor(CsumTable,
|
||||
Filename,
|
||||
F,
|
||||
Offset)),
|
||||
RUpdate = maybe_regenerate_checksum(
|
||||
FHd,
|
||||
machi_csum_table:find_rightneighbor(CsumTable,
|
||||
Filename,
|
||||
F,
|
||||
Offset+Size)),
|
||||
ok = machi_csum_table:write(CsumTable, Filename, Offset, Size,
|
||||
ok = machi_csum_table:write(CsumTable, F, Offset, Size,
|
||||
TaggedCsum, LUpdate, RUpdate),
|
||||
lager:debug("Successful write to checksum file for ~p",
|
||||
[Filename]),
|
||||
|
|
@ -845,7 +850,7 @@ maybe_gc(Reply, S = #state{data_filehandle = FHd,
|
|||
eof_position = Eof,
|
||||
csum_table=CsumTable}) ->
|
||||
lager:debug("GC? Let's try it: ~p.~n", [Filename]),
|
||||
case machi_csum_table:maybe_trim_file(CsumTable, Filename, Eof) of
|
||||
case machi_csum_table:maybe_trim_file(CsumTable, iolist_to_binary(Filename), Eof) of
|
||||
{ok, trimmed} ->
|
||||
%% Checksum table entries are all trimmed now, unlinking
|
||||
%% file from operating system
|
||||
|
|
|
|||
|
|
@ -2,68 +2,69 @@
|
|||
-compile(export_all).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-define(HDR, {0, 1024, none}).
|
||||
|
||||
cleanup(Dir) ->
|
||||
os:cmd("rm -rf " ++ Dir).
|
||||
|
||||
smoke_test() ->
|
||||
Filename = "./temp-checksum-dumb-file",
|
||||
_ = cleanup(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
?assertEqual([{1024, infinity}],
|
||||
machi_csum_table:calc_unwritten_bytes(MC)),
|
||||
DBFile = "./temp-checksum-dumb-file",
|
||||
Filename = <<"/some/puppy/and/cats^^^42">>,
|
||||
_ = cleanup(DBFile),
|
||||
{ok, MC} = machi_csum_table:open(DBFile, []),
|
||||
?assertEqual([{0, infinity}],
|
||||
machi_csum_table:calc_unwritten_bytes(MC, Filename)),
|
||||
Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
|
||||
[] = machi_csum_table:find(MC, Offset, Size),
|
||||
ok = machi_csum_table:write(MC, Offset, Size, Checksum),
|
||||
[{1024, 40}, {1098, infinity}] = machi_csum_table:calc_unwritten_bytes(MC),
|
||||
?assertEqual([Entry], machi_csum_table:find(MC, Offset, Size)),
|
||||
ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined),
|
||||
[] = machi_csum_table:find(MC, Filename, Offset, Size),
|
||||
ok = machi_csum_table:write(MC, Filename, Offset, Size, Checksum),
|
||||
[{0, 1064}, {1098, infinity}] = machi_csum_table:calc_unwritten_bytes(MC, Filename),
|
||||
?assertEqual([Entry], machi_csum_table:find(MC, Filename, Offset, Size)),
|
||||
ok = machi_csum_table:trim(MC, Filename, Offset, Size, undefined, undefined),
|
||||
?assertEqual([{Offset, Size, trimmed}],
|
||||
machi_csum_table:find(MC, Offset, Size)),
|
||||
ok = machi_csum_table:close(MC),
|
||||
ok = machi_csum_table:delete(MC).
|
||||
machi_csum_table:find(MC, Filename, Offset, Size)),
|
||||
ok = machi_csum_table:close(MC).
|
||||
|
||||
close_test() ->
|
||||
Filename = "./temp-checksum-dumb-file-2",
|
||||
_ = cleanup(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
DBFile = "./temp-checksum-dumb-file-2",
|
||||
Filename = <<"/some/puppy/and/cats^^^43">>,
|
||||
_ = cleanup(DBFile),
|
||||
{ok, MC} = machi_csum_table:open(DBFile, []),
|
||||
Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
|
||||
[] = machi_csum_table:find(MC, Offset, Size),
|
||||
ok = machi_csum_table:write(MC, Offset, Size, Checksum),
|
||||
[Entry] = machi_csum_table:find(MC, Offset, Size),
|
||||
[] = machi_csum_table:find(MC, Filename, Offset, Size),
|
||||
ok = machi_csum_table:write(MC, Filename, Offset, Size, Checksum),
|
||||
[Entry] = machi_csum_table:find(MC, Filename, Offset, Size),
|
||||
ok = machi_csum_table:close(MC),
|
||||
|
||||
{ok, MC2} = machi_csum_table:open(Filename, []),
|
||||
[Entry] = machi_csum_table:find(MC2, Offset, Size),
|
||||
ok = machi_csum_table:trim(MC2, Offset, Size, undefined, undefined),
|
||||
[{Offset, Size, trimmed}] = machi_csum_table:find(MC2, Offset, Size),
|
||||
ok = machi_csum_table:delete(MC2).
|
||||
{ok, MC2} = machi_csum_table:open(DBFile, []),
|
||||
[Entry] = machi_csum_table:find(MC2, Filename, Offset, Size),
|
||||
ok = machi_csum_table:trim(MC2, Filename, Offset, Size, undefined, undefined),
|
||||
[{Offset, Size, trimmed}] = machi_csum_table:find(MC2, Filename, Offset, Size),
|
||||
ok = machi_csum_table:close(MC2).
|
||||
|
||||
smoke2_test() ->
|
||||
Filename = "./temp-checksum-dumb-file-3",
|
||||
_ = cleanup(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
DBFile = "./temp-checksum-dumb-file-3",
|
||||
Filename = <<"/some/puppy/and/cats^^^43">>,
|
||||
_ = cleanup(DBFile),
|
||||
{ok, MC} = machi_csum_table:open(DBFile, []),
|
||||
Entry = {Offset, Size, Checksum} = {1025, 10, <<"deadbeef">>},
|
||||
ok = machi_csum_table:write(MC, Offset, Size, Checksum),
|
||||
?assertEqual([], machi_csum_table:find(MC, 0, 0)),
|
||||
?assertEqual([?HDR], machi_csum_table:find(MC, 0, 1)),
|
||||
[Entry] = machi_csum_table:find(MC, Offset, Size),
|
||||
[?HDR] = machi_csum_table:find(MC, 1, 1024),
|
||||
?assertEqual([?HDR, Entry],
|
||||
machi_csum_table:find(MC, 1023, 1024)),
|
||||
[Entry] = machi_csum_table:find(MC, 1024, 1024),
|
||||
[Entry] = machi_csum_table:find(MC, 1025, 1024),
|
||||
ok = machi_csum_table:write(MC, Filename, Offset, Size, Checksum),
|
||||
?assertEqual([], machi_csum_table:find(MC, Filename, 0, 0)),
|
||||
?assertEqual([], machi_csum_table:find(MC, Filename, 0, 1)),
|
||||
[Entry] = machi_csum_table:find(MC, Filename, Offset, Size),
|
||||
[] = machi_csum_table:find(MC, Filename, 1, 1024),
|
||||
?assertEqual([Entry],
|
||||
machi_csum_table:find(MC, Filename, 1023, 1024)),
|
||||
[Entry] = machi_csum_table:find(MC, Filename, 1024, 1024),
|
||||
[Entry] = machi_csum_table:find(MC, Filename, 1025, 1024),
|
||||
|
||||
ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined),
|
||||
[{Offset, Size, trimmed}] = machi_csum_table:find(MC, Offset, Size),
|
||||
ok = machi_csum_table:close(MC),
|
||||
ok = machi_csum_table:delete(MC).
|
||||
ok = machi_csum_table:trim(MC, Filename, Offset, Size, undefined, undefined),
|
||||
[{Offset, Size, trimmed}] = machi_csum_table:find(MC, Filename, Offset, Size),
|
||||
ok = machi_csum_table:close(MC).
|
||||
|
||||
smoke3_test() ->
|
||||
Filename = "./temp-checksum-dumb-file-4",
|
||||
_ = cleanup(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
DBFile = "./temp-checksum-dumb-file-4",
|
||||
Filename = <<"/some/puppy/and/cats^^^44">>,
|
||||
_ = cleanup(DBFile),
|
||||
{ok, MC} = machi_csum_table:open(DBFile, []),
|
||||
Scenario =
|
||||
[%% Command, {Offset, Size, Csum}, LeftNeighbor, RightNeibor
|
||||
{?LINE, write, {2000, 10, <<"heh">>}, undefined, undefined},
|
||||
|
|
@ -84,9 +85,9 @@ smoke3_test() ->
|
|||
%% ?debugVal({_Line, Chunk}),
|
||||
{Offset, Size, Csum} = Chunk,
|
||||
?assertEqual(LeftN0,
|
||||
machi_csum_table:find_leftneighbor(MC, Offset)),
|
||||
machi_csum_table:find_leftneighbor(MC, Filename, Offset)),
|
||||
?assertEqual(RightN0,
|
||||
machi_csum_table:find_rightneighbor(MC, Offset+Size)),
|
||||
machi_csum_table:find_rightneighbor(MC, Filename, Offset+Size)),
|
||||
LeftN = case LeftN0 of
|
||||
{OffsL, SizeL, trimmed} -> {OffsL, SizeL, trimmed};
|
||||
{OffsL, SizeL, _} -> {OffsL, SizeL, <<"boom">>};
|
||||
|
|
@ -98,19 +99,18 @@ smoke3_test() ->
|
|||
end,
|
||||
case Cmd of
|
||||
write ->
|
||||
ok = machi_csum_table:write(MC, Offset, Size, Csum,
|
||||
ok = machi_csum_table:write(MC, Filename, Offset, Size, Csum,
|
||||
LeftN, RightN);
|
||||
trim ->
|
||||
ok = machi_csum_table:trim(MC, Offset, Size,
|
||||
ok = machi_csum_table:trim(MC, Filename, Offset, Size,
|
||||
LeftN, RightN)
|
||||
end
|
||||
end || {_Line, Cmd, Chunk, LeftN0, RightN0} <- Scenario ],
|
||||
?assert(not machi_csum_table:all_trimmed(MC, 10000)),
|
||||
machi_csum_table:trim(MC, 0, 10000, undefined, undefined),
|
||||
?assert(machi_csum_table:all_trimmed(MC, 10000)),
|
||||
?assert(not machi_csum_table:all_trimmed(MC, Filename, 0, 10000)),
|
||||
machi_csum_table:trim(MC, Filename, 0, 10000, undefined, undefined),
|
||||
?assert(machi_csum_table:all_trimmed(MC, Filename, 0, 10000)),
|
||||
|
||||
ok = machi_csum_table:close(MC),
|
||||
ok = machi_csum_table:delete(MC).
|
||||
ok = machi_csum_table:close(MC).
|
||||
|
||||
%% TODO: add quickcheck test here
|
||||
|
||||
|
|
|
|||
|
|
@ -116,11 +116,11 @@ get_written_interval(L) ->
|
|||
initial_state() ->
|
||||
{_, _, MS} = os:timestamp(),
|
||||
Filename = test_server:temp_name("eqc_data") ++ "." ++ integer_to_list(MS),
|
||||
#state{filename=Filename, written=[{0,1024}]}.
|
||||
#state{filename=Filename, written=[]}.
|
||||
|
||||
initial_state(I, T) ->
|
||||
S=initial_state(),
|
||||
S#state{written=[{0,1024}],
|
||||
S#state{written=[],
|
||||
planned_writes=I,
|
||||
planned_trims=T}.
|
||||
|
||||
|
|
@ -230,7 +230,8 @@ start_command(S) ->
|
|||
{call, ?MODULE, start, [S]}.
|
||||
|
||||
start(#state{filename=File}) ->
|
||||
{ok, Pid} = machi_file_proxy:start_link(some_flu, File, ?TESTDIR),
|
||||
CsumT = get_csum_table(),
|
||||
{ok, Pid} = machi_file_proxy:start_link(File, ?TESTDIR, CsumT),
|
||||
unlink(Pid),
|
||||
Pid.
|
||||
|
||||
|
|
@ -432,6 +433,40 @@ stop_post(_, _, _) -> true.
|
|||
stop_next(S, _, _) ->
|
||||
S#state{pid=undefined, prev_extra=0}.
|
||||
|
||||
csum_table_holder() ->
|
||||
Parent = self(),
|
||||
spawn_link(fun() ->
|
||||
CsumFile = test_server:temp_name("eqc_data-csum"),
|
||||
filelib:ensure_dir(CsumFile),
|
||||
{ok, CsumT} = machi_csum_table:open(CsumFile, []),
|
||||
erlang:register(csum_table_holder, self()),
|
||||
Parent ! ok,
|
||||
csum_table_holder_loop(CsumT),
|
||||
machi_csum_table:close(CsumT),
|
||||
erlang:unregister(csum_table_holder)
|
||||
end),
|
||||
receive
|
||||
Other -> Other
|
||||
after 1000 ->
|
||||
timeout
|
||||
end.
|
||||
|
||||
csum_table_holder_loop(CsumT) ->
|
||||
receive
|
||||
{get, From} ->
|
||||
From ! CsumT;
|
||||
stop ->
|
||||
ok
|
||||
end.
|
||||
|
||||
get_csum_table() ->
|
||||
csum_table_holder ! {get, self()},
|
||||
receive CsumT -> CsumT
|
||||
end.
|
||||
|
||||
stop_csum_table_holder() ->
|
||||
catch csum_table_holder ! stop.
|
||||
|
||||
%% Property
|
||||
|
||||
prop_ok() ->
|
||||
|
|
@ -440,7 +475,9 @@ prop_ok() ->
|
|||
{shuffle_interval(), shuffle_interval()},
|
||||
?FORALL(Cmds, parallel_commands(?MODULE, initial_state(I, T)),
|
||||
begin
|
||||
ok = csum_table_holder(),
|
||||
{H, S, Res} = run_parallel_commands(?MODULE, Cmds),
|
||||
stop_csum_table_holder(),
|
||||
cleanup(),
|
||||
pretty_commands(?MODULE, Cmds, {H, S, Res},
|
||||
aggregate(command_names(Cmds), Res == ok))
|
||||
|
|
|
|||
|
|
@ -77,25 +77,28 @@ random_binary(Start, End) ->
|
|||
end.
|
||||
|
||||
setup() ->
|
||||
{ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR),
|
||||
Pid.
|
||||
{ok, CsumT} = machi_csum_table:open(filename:join([?TESTDIR, "csumfile"]), []),
|
||||
{ok, Pid} = machi_file_proxy:start_link("test", ?TESTDIR, CsumT),
|
||||
{Pid, CsumT}.
|
||||
|
||||
teardown(Pid) ->
|
||||
catch machi_file_proxy:stop(Pid).
|
||||
teardown({Pid, CsumT}) ->
|
||||
catch machi_file_proxy:stop(Pid),
|
||||
catch machi_csum_table:close(CsumT).
|
||||
|
||||
machi_file_proxy_test_() ->
|
||||
clean_up_data_dir(?TESTDIR),
|
||||
{setup,
|
||||
fun setup/0,
|
||||
fun teardown/1,
|
||||
fun(Pid) ->
|
||||
fun({Pid, _}) ->
|
||||
[
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:read(Pid, -1, -1)),
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:write(Pid, -1, <<"yo">>)),
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:append(Pid, [], -1, <<"krep">>)),
|
||||
?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1)),
|
||||
?_assertMatch({error, not_written}, machi_file_proxy:read(Pid, 1, 1)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, 1)),
|
||||
?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1024)),
|
||||
?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))),
|
||||
?_assertMatch({ok, _}, machi_file_proxy:read(Pid, 1, 1024)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, ?HYOOGE)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, ?HYOOGE, 1)),
|
||||
{timeout, 10,
|
||||
|
|
@ -114,7 +117,7 @@ multiple_chunks_read_test_() ->
|
|||
{setup,
|
||||
fun setup/0,
|
||||
fun teardown/1,
|
||||
fun(Pid) ->
|
||||
fun({Pid, _}) ->
|
||||
[
|
||||
?_assertEqual(ok, machi_file_proxy:trim(Pid, 0, 1, false)),
|
||||
?_assertMatch({ok, {[], [{"test", 0, 1}]}},
|
||||
|
|
|
|||
Loading…
Reference in a new issue