diff --git a/.gitignore b/.gitignore index 6ad1c94..72f5f57 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ edoc # PB artifacts for Erlang include/machi_pb.hrl +# Release packaging +rel/machi + # Misc Scott cruft *.patch current_counterexample.eqc diff --git a/Makefile b/Makefile index dbc3d25..3f36477 100644 --- a/Makefile +++ b/Makefile @@ -1,36 +1,45 @@ -REBAR_BIN := $(shell which rebar) -ifeq ($(REBAR_BIN),) -REBAR_BIN = ./rebar +REPO ?= machi +PKG_REVISION ?= $(shell git describe --tags) +PKG_BUILD = 1 +BASE_DIR = $(shell pwd) +ERLANG_BIN = $(shell dirname $(shell which erl)) +REBAR := $(shell which rebar) +ifeq ($(REBAR),) +REBAR = $(BASE_DIR)/rebar endif +OVERLAY_VARS ?= .PHONY: rel deps package pkgclean edoc all: deps compile compile: - $(REBAR_BIN) compile + $(REBAR) compile + +generate: + $(REBAR) generate $(OVERLAY_VARS) 2>&1 | grep -v 'command does not apply to directory' deps: - $(REBAR_BIN) get-deps + $(REBAR) get-deps clean: - $(REBAR_BIN) -r clean + $(REBAR) -r clean test: deps compile eunit eunit: - $(REBAR_BIN) -v skip_deps=true eunit + $(REBAR) -v skip_deps=true eunit edoc: edoc-clean - $(REBAR_BIN) skip_deps=true doc + $(REBAR) skip_deps=true doc edoc-clean: rm -f edoc/*.png edoc/*.html edoc/*.css edoc/edoc-info pulse: compile @echo Sorry, PULSE test needs maintenance. -SLF - #env USE_PULSE=1 $(REBAR_BIN) skip_deps=true clean compile - #env USE_PULSE=1 $(REBAR_BIN) skip_deps=true -D PULSE eunit -v + #env USE_PULSE=1 $(REBAR) skip_deps=true clean compile + #env USE_PULSE=1 $(REBAR) skip_deps=true -D PULSE eunit -v APPS = kernel stdlib sasl erts ssl compiler eunit crypto PLT = $(HOME)/.machi_dialyzer_plt @@ -53,3 +62,14 @@ dialyzer-test: deps compile clean_plt: rm $(PLT) + +## +## Release targets +## +rel: deps compile generate + +relclean: + rm -rf rel/$(REPO) + +stage : rel + $(foreach dep,$(wildcard deps/*), rm -rf rel/$(REPO)/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) rel/$(REPO)/lib;) diff --git a/apps/machi/ebin/machi.app b/apps/machi/ebin/machi.app new file mode 100644 index 0000000..9a79074 --- /dev/null +++ b/apps/machi/ebin/machi.app @@ -0,0 +1,5 @@ +{application,machi, + [{description,"Machi extremely-early boot items"}, + %% {modules,[etop_txt]} + {modules,[]} + ]}. diff --git a/rebar.config b/rebar.config index 9296f39..782fa10 100644 --- a/rebar.config +++ b/rebar.config @@ -7,6 +7,10 @@ {deps, [ {lager, ".*", {git, "git://github.com/basho/lager.git", {tag, "2.2.0"}}}, {protobuffs, "0.8.*", {git, "git://github.com/basho/erlang_protobuffs.git", {tag, "0.8.1p4"}}}, - {riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {branch, "develop"}}} + {riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {branch, "develop"}}}, + {node_package, ".*", {git, "git://github.com/basho/node_package.git", {branch, "develop"}}}, + {eper, ".*", {git, "git://github.com/basho/eper.git", {tag, "0.78"}}} ]}. +{sub_dirs, ["rel", "apps/machi"]}. +{lib_dirs, ["apps/machi"]}. diff --git a/rel/files/app.config b/rel/files/app.config new file mode 100644 index 0000000..57afcca --- /dev/null +++ b/rel/files/app.config @@ -0,0 +1 @@ +[]. diff --git a/rel/files/machi-admin b/rel/files/machi-admin new file mode 100755 index 0000000..2b16f04 --- /dev/null +++ b/rel/files/machi-admin @@ -0,0 +1,68 @@ +#!/bin/sh +# -*- tab-width:4;indent-tabs-mode:nil -*- +# ex: ts=4 sw=4 et + +# Pull environment for this install +. "{{runner_base_dir}}/lib/env.sh" + +# Make sure the user running this script is the owner and/or su to that user +check_user "$@" +ES=$? +if [ "$ES" -ne 0 ]; then + exit $ES +fi + +# Keep track of where script was invoked +ORIGINAL_DIR=$(pwd) + +# Make sure CWD is set to runner run dir +cd $RUNNER_BASE_DIR + +# Identify the script name +SCRIPT=`basename $0` + +usage() { + echo "Usage: $SCRIPT { test | " + echo " top }" +} + +stat_admin() +{ + case "$1" in + test) + # Make sure the local node IS running + node_up_check + + shift + + # Parse out the node name to pass to the client + NODE_NAME=${NAME_ARG#* } + + $ERTS_PATH/erl -noshell $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \ + -pa $RUNNER_LIB_DIR/basho-patches \ + -eval "case catch(machi:client_test(\"$NODE_NAME\")) of \ + ok -> init:stop(); \ + _ -> init:stop(1) \ + end." + + ;; + top) + # Make sure the local node IS running + node_up_check + + shift + + MYPID=$$ + NODE_NAME=${NAME_ARG#* } + $ERTS_PATH/erl -noshell -noinput \ + -pa $RUNNER_LIB_DIR/basho-patches \ + -hidden $NAME_PARAM machi_etop$MYPID$NAME_HOST $COOKIE_ARG \ + -s etop -s erlang halt -output text \ + -node $NODE_NAME \ + $* -tracing off + ;; + *) + usage + exit 1 + ;; +esac diff --git a/rel/files/machi.schema b/rel/files/machi.schema new file mode 100644 index 0000000..cc58596 --- /dev/null +++ b/rel/files/machi.schema @@ -0,0 +1,220 @@ +%%-*- mode: erlang -*- + +%% @doc Where to emit the default log messages (typically at 'info' +%% severity): +%% off: disabled +%% file: the file specified by log.console.file +%% console: to standard output (seen when using `machi attach-direct`) +%% both: log.console.file and standard out. +{mapping, "log.console", "lager.handlers", [ + {default, {{console_log_default}} }, + {datatype, {enum, [off, file, console, both]}} +]}. + +%% @doc The severity level of the console log, default is 'info'. +{mapping, "log.console.level", "lager.handlers", [ + {default, info}, + {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}} +]}. + +%% @doc When 'log.console' is set to 'file' or 'both', the file where +%% console messages will be logged. +{mapping, "log.console.file", "lager.handlers", [ + {default, "$(platform_log_dir)/console.log"}, + {datatype, file} +]}. + +%% @doc The file where error messages will be logged. +{mapping, "log.error.file", "lager.handlers", [ + {default, "$(platform_log_dir)/error.log"}, + {datatype, file} +]}. + +%% @doc When set to 'on', enables log output to syslog. +{mapping, "log.syslog", "lager.handlers", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc When set to 'on', enables log output to syslog. +{mapping, "log.syslog.ident", "lager.handlers", [ + {default, "machi"}, + hidden +]}. + +%% @doc Syslog facility to log entries from Riak. +{mapping, "log.syslog.facility", "lager.handlers", [ + {default, daemon}, + {datatype, {enum,[kern, user, mail, daemon, auth, syslog, + lpr, news, uucp, clock, authpriv, ftp, + cron, local0, local1, local2, local3, + local4, local5, local6, local7]}}, + hidden +]}. + +%% @doc The severity level at which to log entries to syslog, default is 'info'. +{mapping, "log.syslog.level", "lager.handlers", [ + {default, info}, + {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}}, + hidden +]}. + +{translation, + "lager.handlers", + fun(Conf) -> + SyslogHandler = case cuttlefish:conf_get("log.syslog", Conf) of + true -> + Ident = cuttlefish:conf_get("log.syslog.ident", Conf), + Facility = cuttlefish:conf_get("log.syslog.facility", Conf), + LogLevel = cuttlefish:conf_get("log.syslog.level", Conf), + [{lager_syslog_backend, [Ident, Facility, LogLevel]}]; + _ -> [] + end, + ErrorHandler = case cuttlefish:conf_get("log.error.file", Conf) of + undefined -> []; + ErrorFilename -> [{lager_file_backend, [{file, ErrorFilename}, + {level, error}, + {size, 10485760}, + {date, "$D0"}, + {count, 5}]}] + end, + + ConsoleLogLevel = cuttlefish:conf_get("log.console.level", Conf), + ConsoleLogFile = cuttlefish:conf_get("log.console.file", Conf), + + ConsoleHandler = {lager_console_backend, ConsoleLogLevel}, + ConsoleFileHandler = {lager_file_backend, [{file, ConsoleLogFile}, + {level, ConsoleLogLevel}, + {size, 10485760}, + {date, "$D0"}, + {count, 5}]}, + + ConsoleHandlers = case cuttlefish:conf_get("log.console", Conf) of + off -> []; + file -> [ConsoleFileHandler]; + console -> [ConsoleHandler]; + both -> [ConsoleHandler, ConsoleFileHandler]; + _ -> [] + end, + SyslogHandler ++ ConsoleHandlers ++ ErrorHandler + end +}. + + +%% @doc Whether to enable Erlang's built-in error logger. +{mapping, "sasl", "sasl.sasl_error_logger", [ + {default, off}, + {datatype, flag}, + hidden +]}. + +%% @doc Whether to enable the crash log. +{mapping, "log.crash", "lager.crash_log", [ + {default, on}, + {datatype, flag} +]}. + +%% @doc If the crash log is enabled, the file where its messages will +%% be written. +{mapping, "log.crash.file", "lager.crash_log", [ + {default, "$(platform_log_dir)/crash.log"}, + {datatype, file} +]}. + +{translation, + "lager.crash_log", + fun(Conf) -> + case cuttlefish:conf_get("log.crash", Conf) of + false -> undefined; + _ -> + cuttlefish:conf_get("log.crash.file", Conf, "{{platform_log_dir}}/crash.log") + end + end}. + +%% @doc Maximum size in bytes of individual messages in the crash log +{mapping, "log.crash.maximum_message_size", "lager.crash_log_msg_size", [ + {default, "64KB"}, + {datatype, bytesize} +]}. + +%% @doc Maximum size of the crash log in bytes, before it is rotated +{mapping, "log.crash.size", "lager.crash_log_size", [ + {default, "10MB"}, + {datatype, bytesize} +]}. + +%% @doc The schedule on which to rotate the crash log. For more +%% information see: +%% https://github.com/basho/lager/blob/master/README.md#internal-log-rotation +{mapping, "log.crash.rotation", "lager.crash_log_date", [ + {default, "$D0"} +]}. + +%% @doc The number of rotated crash logs to keep. When set to +%% 'current', only the current open log file is kept. +{mapping, "log.crash.rotation.keep", "lager.crash_log_count", [ + {default, 5}, + {datatype, [integer, {atom, current}]}, + {validators, ["rotation_count"]} +]}. + +{validator, + "rotation_count", + "must be 'current' or a positive integer", + fun(current) -> true; + (Int) when is_integer(Int) andalso Int >= 0 -> true; + (_) -> false + end}. + +{translation, + "lager.crash_log_count", + fun(Conf) -> + case cuttlefish:conf_get("log.crash.rotation.keep", Conf) of + current -> 0; + Int -> Int + end + end}. + +%% @doc Whether to redirect error_logger messages into lager - +%% defaults to true +{mapping, "log.error.redirect", "lager.error_logger_redirect", [ + {default, on}, + {datatype, flag}, + hidden +]}. + +%% @doc Maximum number of error_logger messages to handle in a second +{mapping, "log.error.messages_per_second", "lager.error_logger_hwm", [ + {default, 100}, + {datatype, integer}, + hidden +]}. + + +%% @doc Cookie for distributed node communication. All nodes in the +%% same cluster should use the same cookie or they will not be able to +%% communicate. +{mapping, "distributed_cookie", "vm_args.-setcookie", [ + {default, "machi"} +]}. + + +%% override zdbbl from 1mb to 32mb +{mapping, "erlang.distribution_buffer_size", "vm_args.+zdbbl", [ + {default, "32MB"}, + merge +]}. + +%% VM scheduler collapse, part 1 of 2 +{mapping, "erlang.schedulers.force_wakeup_interval", "vm_args.+sfwi", [ + {default, 500}, + {datatype, integer}, + merge +]}. + +%% VM scheduler collapse, part 2 of 2 +{mapping, "erlang.schedulers.compaction_of_load", "vm_args.+scl", [ + {default, false}, + merge +]}. + diff --git a/rel/files/vm.args b/rel/files/vm.args new file mode 100644 index 0000000..4afcaf5 --- /dev/null +++ b/rel/files/vm.args @@ -0,0 +1,27 @@ +## Name of the riak node +-name {{node}} + +## Cookie for distributed erlang. All nodes in the same cluster +## should use the same cookie or they will not be able to communicate. +-setcookie machi + +## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive +## (Disabled by default..use with caution!) +##-heart + +## Enable kernel poll and a few async threads ++K true ++A 64 + +## Treat error_logger warnings as warnings ++W w + +## Increase number of concurrent ports/sockets +-env ERL_MAX_PORTS 4096 + +## ## Tweak GC to run more often +## -env ERL_FULLSWEEP_AFTER 0 + +## Set the location of crash dumps +-env ERL_CRASH_DUMP {{crash_dump}} + diff --git a/rel/rebar.config b/rel/rebar.config new file mode 100644 index 0000000..9959478 --- /dev/null +++ b/rel/rebar.config @@ -0,0 +1,4 @@ +{lib_dirs, ["../deps"]}. +%% {plugin_dir, "../deps/cuttlefish/src"}. +%% {plugins, [cuttlefish_rebar_plugin]}. +%% {cuttlefish_filename, "machi.conf"}. diff --git a/rel/reltool.config b/rel/reltool.config new file mode 100644 index 0000000..b264857 --- /dev/null +++ b/rel/reltool.config @@ -0,0 +1,98 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ft=erlang ts=4 sw=4 et +{sys, [ + {lib_dirs, ["../deps", "../apps"]}, + {rel, "machi", "0.0.0", + [ + kernel, + stdlib, + lager, + sasl, + public_key, + ssl, +%% riak_sysmon, +%% os_mon, + crypto, + runtime_tools, + machi +%% cluster_info, +%% exometer_core, + ]}, + {rel, "start_clean", "", + [ + kernel, + stdlib + ]}, + {boot_rel, "machi"}, + {profile, embedded}, + {excl_sys_filters, ["^bin/.*", + "^erts.*/bin/(dialyzer|typer)", + "^erts.*/doc", + "^erts.*/man"]}, + {excl_archive_filters, [".*"]}, +%% {app, cuttlefish, [{incl_cond, include}]}, +%% {app, cluster_info, [{incl_cond, include}]}, + {app, eper, [{incl_cond, include}]}, + {app, sasl, [{incl_cond, include}]}, +%% {app, syslog, [{incl_cond, include}]}, +%% {app, lager_syslog, [{incl_cond, include}]}, + {app, lager, [{incl_cond, include}]} +%% {app, exometer_core, [{incl_cond, include}]}, + ]}. + + +{target_dir, "machi"}. + +{overlay_vars, "vars.config"}. + +{overlay, [ + {mkdir, "data"}, + {mkdir, "log"}, + + %% Copy base files for starting and interacting w/ node + {copy, "../deps/node_package/priv/base/erl", + "{{erts_vsn}}/bin/erl"}, + {copy, "../deps/node_package/priv/base/nodetool", + "{{erts_vsn}}/bin/nodetool"}, +%% {copy, "../deps/cuttlefish/cuttlefish", +%% "{{erts_vsn}}/bin/cuttlefish"}, + {template, "../deps/node_package/priv/base/runner", + "bin/machi"}, + {template, "../deps/node_package/priv/base/env.sh", + "lib/env.sh"}, + {template, "../deps/node_package/priv/base/app_epath.sh", + "lib/app_epath.sh"}, + + %% Copy config files + + %% Cuttlefish Schema Files have a priority order. + %% Anything in a file prefixed with 00- will override + %% anything in a file with a higher numbered prefix. + + %% Please only use 0[0-9]-*.schema for development purposes + %% NOTHING PERMANENT + + %% {template, "files/riak.schema", "lib/10-riak.schema"}, + %% {template, "../deps/cuttlefish/priv/erlang_vm.schema", "lib/11-erlang_vm.schema"}, + + %% {template, "../deps/riak_core/priv/riak_core.schema", "lib/12-riak_core.schema"}, + %% {template, "../deps/riak_api/priv/riak_api.schema", "lib/13-riak_api.schema"}, + %% {template, "../deps/riak_kv/priv/riak_kv.schema", "lib/14-riak_kv.schema"}, + %% {template, "../deps/riak_sysmon/priv/riak_sysmon.schema", "lib/15-riak_sysmon.schema"}, + %% {template, "../deps/bitcask/priv/bitcask.schema", "lib/16-bitcask.schema"}, + %% {template, "../deps/bitcask/priv/bitcask_multi.schema", "lib/17-bitcask_multi.schema"}, + %% {template, "../deps/riak_control/priv/riak_control.schema", "lib/18-riak_control.schema"}, + + %% {template, "../deps/riak_kv/priv/multi_backend.schema", "lib/20-multi_backend.schema"}, + %% {template, "../deps/eleveldb/priv/eleveldb.schema", "lib/21-leveldb.schema"}, + %% {template, "../deps/eleveldb/priv/eleveldb_multi.schema", "lib/22-leveldb_multi.schema"}, + %% {template, "../deps/yokozuna/priv/yokozuna.schema", "lib/30-yokozuna.schema"}, + + %% Copy additional bin scripts + {template, "files/machi-admin", "bin/machi-admin"}, + + {template, "files/vm.args", "etc/vm.args"}, + {template, "files/app.config", "etc/app.config"}, + {mkdir, "lib/basho-patches"} + %% {copy, "../apps/machi/ebin/etop_txt.beam", "lib/basho-patches"} + ]}. diff --git a/rel/vars.config b/rel/vars.config new file mode 100644 index 0000000..a328191 --- /dev/null +++ b/rel/vars.config @@ -0,0 +1,45 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ft=erlang ts=4 sw=4 et + +%% Platform-specific installation paths +{platform_bin_dir, "./bin"}. +{platform_data_dir, "./data"}. +{platform_etc_dir, "./etc"}. +{platform_lib_dir, "./lib"}. +{platform_log_dir, "./log"}. + +%% +%% etc/app.config +%% +{sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}. +{sasl_log_dir, "{{platform_log_dir}}/sasl"}. + +%% lager +{console_log_default, file}. + +%% +%% etc/vm.args +%% +{node, "machi@127.0.0.1"}. +{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}. + +%% +%% bin/machi +%% +{runner_script_dir, "\`cd \\`dirname $0\\` 1>/dev/null && /bin/pwd\`"}. +{runner_base_dir, "{{runner_script_dir}}/.."}. +{runner_etc_dir, "$RUNNER_BASE_DIR/etc"}. +{runner_log_dir, "$RUNNER_BASE_DIR/log"}. +{runner_lib_dir, "$RUNNER_BASE_DIR/lib"}. +{runner_patch_dir, "$RUNNER_BASE_DIR/lib/basho-patches"}. +{pipe_dir, "/tmp/$RUNNER_BASE_DIR/"}. +{runner_user, ""}. +{runner_wait_process, "riak_core_node_watcher"}. +{runner_ulimit_warn, 65536}. + +%% +%% cuttlefish +%% +{cuttlefish, ""}. % blank = off +{cuttlefish_conf, "machi.conf"}. +