From 5913531e32d743b79c85dbdb49f215ee23046166 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Tue, 27 Oct 2015 11:00:05 +0900 Subject: [PATCH 1/6] Introduce machi_config.erl --- src/machi_config.erl | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 src/machi_config.erl diff --git a/src/machi_config.erl b/src/machi_config.erl new file mode 100644 index 0000000..0da4112 --- /dev/null +++ b/src/machi_config.erl @@ -0,0 +1,43 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +%% @doc Configuration consulting utilities. Some conventions: +%% - The function name should match with exact configuration +%% name in `app.config' or `advanced.config' of `machi' section. +%% - The default value of that configuration is expected to be in +%% cuttlefish schema file. Otherwise some macro in headers may +%% be chosen. +%% - Documentation of the configuration is supposed to be written +%% in cuttlefish schema file, rather than @doc section of the function. +%% - spec of the function should be written. +%% - Returning `undefined' is strongly discouraged. Return some default +%% value instead. +%% - `application:get_env/3' is recommended. See `max_file_size/0' for +%% example. + +-module(machi_config). + +-include("machi.hrl"). + +-export([max_file_size/0]). + +-spec max_file_size() -> pos_integer(). +max_file_size() -> + application:get_env(machi, max_file_size, ?MAX_FILE_SIZE). From b2eb3e089c9c13fe77c73b80b82d9444c06daad5 Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Tue, 27 Oct 2015 11:05:07 +0900 Subject: [PATCH 2/6] Cleanup MACROs and changed default value * machi_file_proxy now uses application environment value `max_file_size` via machi_config * changed name from MAX_FILE_SIZE to DEFAULT_MAX_FILE_SIZE --- include/machi.hrl | 8 ++++---- src/machi.app.src | 5 +---- src/machi_config.erl | 2 +- src/machi_file_proxy.erl | 11 +++++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/machi.hrl b/include/machi.hrl index c135214..b445c85 100644 --- a/include/machi.hrl +++ b/include/machi.hrl @@ -18,10 +18,9 @@ %% %% ------------------------------------------------------------------- --define(MAX_FILE_SIZE, 256*1024*1024). % 256 MBytes --define(MAX_CHUNK_SIZE, ((1 bsl 32) - 1)). -%% -define(DATA_DIR, "/Volumes/SAM1/seq-tests/data"). --define(DATA_DIR, "./data"). +%% @doc Now 4GiBytes, could be up to 64bit due to PB message limit of +%% chunk size +-define(DEFAULT_MAX_FILE_SIZE, ((1 bsl 32) - 1)). -define(MINIMUM_OFFSET, 1024). %% 0th draft of checksum typing with 1st byte. @@ -34,4 +33,5 @@ -define(PB_MAX_MSG_SIZE, (33*1024*1024)). -define(PB_PACKET_OPTS, [{packet, 4}, {packet_size, ?PB_MAX_MSG_SIZE}]). +%% TODO: it's used in flu_sup and elsewhere, change this to suitable name -define(TEST_ETS_TABLE, test_ets_table). diff --git a/src/machi.app.src b/src/machi.app.src index a15b20c..6498cf5 100644 --- a/src/machi.app.src +++ b/src/machi.app.src @@ -5,9 +5,6 @@ {mod,{machi_app,[]}}, {registered, []}, {env, [ - {flu_list, - [ - %%%%%% {flu_a, 32900, "./data.flu_a"} - ]} + %% Don't use this env for defaults, or I'll give some poopies ]} ]}. diff --git a/src/machi_config.erl b/src/machi_config.erl index 0da4112..0647d3d 100644 --- a/src/machi_config.erl +++ b/src/machi_config.erl @@ -40,4 +40,4 @@ -spec max_file_size() -> pos_integer(). max_file_size() -> - application:get_env(machi, max_file_size, ?MAX_FILE_SIZE). + application:get_env(machi, max_file_size, ?DEFAULT_MAX_FILE_SIZE). diff --git a/src/machi_file_proxy.erl b/src/machi_file_proxy.erl index 7a5c996..c293777 100644 --- a/src/machi_file_proxy.erl +++ b/src/machi_file_proxy.erl @@ -86,7 +86,8 @@ csum_path :: string()|undefined, data_filehandle :: file:io_device(), csum_table :: machi_csum_table:table(), - eof_position = 0 :: non_neg_integer(), + eof_position = 0 :: machi_dt:chunk_pos(), + max_file_size = ?DEFAULT_MAX_FILE_SIZE :: machi_dt:chunk_pos(), tref :: reference(), %% timer ref ticks = 0 :: non_neg_integer(), %% ticks elapsed with no new operations ops = 0 :: non_neg_integer(), %% sum of all ops @@ -222,7 +223,8 @@ init({Filename, DataDir}) -> data_filehandle = FHd, csum_table = CsumTable, tref = Tref, - eof_position = Eof}, + eof_position = Eof, + max_file_size = machi_config:max_file_size()}, lager:debug("Starting file proxy ~p for filename ~p, state = ~p, Eof = ~p", [self(), Filename, St, Eof]), {ok, St}. @@ -390,9 +392,10 @@ handle_cast(Cast, State) -> {noreply, State}. % @private -handle_info(tick, State = #state{eof_position = Eof}) when Eof >= ?MAX_FILE_SIZE -> +handle_info(tick, State = #state{eof_position = Eof, + max_file_size = MaxFileSize}) when Eof >= MaxFileSize -> lager:notice("Eof position ~p >= max file size ~p. Shutting down.", - [Eof, ?MAX_FILE_SIZE]), + [Eof, MaxFileSize]), {stop, file_rollover, State}; %% XXX Is this a good idea? Need to think this through a bit. From d59c1fae31767eab96519c0a70cf112f0179dd1f Mon Sep 17 00:00:00 2001 From: UENISHI Kota Date: Tue, 27 Oct 2015 13:27:20 +0900 Subject: [PATCH 3/6] Fix dialyzer issue --- src/machi_file_proxy.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/machi_file_proxy.erl b/src/machi_file_proxy.erl index c293777..a69dda1 100644 --- a/src/machi_file_proxy.erl +++ b/src/machi_file_proxy.erl @@ -86,8 +86,8 @@ csum_path :: string()|undefined, data_filehandle :: file:io_device(), csum_table :: machi_csum_table:table(), - eof_position = 0 :: machi_dt:chunk_pos(), - max_file_size = ?DEFAULT_MAX_FILE_SIZE :: machi_dt:chunk_pos(), + eof_position = 0 :: non_neg_integer(), + max_file_size = ?DEFAULT_MAX_FILE_SIZE :: pos_integer(), tref :: reference(), %% timer ref ticks = 0 :: non_neg_integer(), %% ticks elapsed with no new operations ops = 0 :: non_neg_integer(), %% sum of all ops From dd17b1de0a79cf1a5a2b5d2b4150f42598fd550a Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 27 Oct 2015 14:07:07 +0900 Subject: [PATCH 4/6] Remove compiler warnings --- src/machi_file_proxy.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/machi_file_proxy.erl b/src/machi_file_proxy.erl index a69dda1..6a12e4c 100644 --- a/src/machi_file_proxy.erl +++ b/src/machi_file_proxy.erl @@ -283,7 +283,6 @@ handle_call({read, Offset, Length, Opts}, _From, State = #state{filename = F, data_filehandle = FH, csum_table = CsumTable, - eof_position = EofP, reads = {T, Err} }) -> NoChecksum = proplists:get_value(no_checksum, Opts, false), @@ -317,7 +316,6 @@ handle_call({write, Offset, ClientMeta, Data}, _From, State = #state{filename = F, writes = {T, Err}, data_filehandle = FHd, - eof_position=EofP, csum_table = CsumTable}) -> ClientCsumTag = proplists:get_value(client_csum_tag, ClientMeta, ?CSUM_TAG_NONE), From b500d5f4490efd0fb64ac95db8b76c03c33af421 Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 27 Oct 2015 14:07:45 +0900 Subject: [PATCH 5/6] config hell --- src/machi.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/machi.app.src b/src/machi.app.src index 6498cf5..c26154f 100644 --- a/src/machi.app.src +++ b/src/machi.app.src @@ -5,6 +5,6 @@ {mod,{machi_app,[]}}, {registered, []}, {env, [ - %% Don't use this env for defaults, or I'll give some poopies + %% Don't use this static env for defaults, or we will fall into config hell. ]} ]}. From cfaed63fa78e968b9fa610d5a6e9308b2b6988ee Mon Sep 17 00:00:00 2001 From: Scott Lystig Fritchie Date: Tue, 27 Oct 2015 14:08:45 +0900 Subject: [PATCH 6/6] Experimental: add 'make dialyzer' to priv/test-for-gh-pr.sh --- priv/test-for-gh-pr.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/priv/test-for-gh-pr.sh b/priv/test-for-gh-pr.sh index 55f8a8d..b8e2acf 100755 --- a/priv/test-for-gh-pr.sh +++ b/priv/test-for-gh-pr.sh @@ -4,6 +4,7 @@ if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then echo '$TRAVIS_PULL_REQUEST is false, skipping tests' exit 0 else - echo '$TRAVIS_PULL_REQUEST is not false, running tests' + echo '$TRAVIS_PULL_REQUEST is not false ($TRAVIS_PULL_REQUEST), running tests' make test + make dialyzer fi