Very first packaging draft, no cuttlefish

* "make rel" does something not horrible
* `rel/machi/bin/machi console` launches a console to an idle VM

.gitignore

@@ -11,6 +11,9 @@ edoc
 # PB artifacts for Erlang
 include/machi_pb.hrl
 
+# Release packaging
+rel/machi
+
 # Misc Scott cruft
 *.patch
 current_counterexample.eqc

Makefile

@@ -1,36 +1,45 @@
-REBAR_BIN := $(shell which rebar)
+REPO            ?= machi
-ifeq ($(REBAR_BIN),)
+PKG_REVISION    ?= $(shell git describe --tags)
-REBAR_BIN = ./rebar
+PKG_BUILD        = 1
+BASE_DIR         = $(shell pwd)
+ERLANG_BIN       = $(shell dirname $(shell which erl))
+REBAR := $(shell which rebar)
+ifeq ($(REBAR),)
+REBAR = $(BASE_DIR)/rebar
 endif
+OVERLAY_VARS    ?=
 
 .PHONY: rel deps package pkgclean edoc
 
 all: deps compile
 
 compile:
-	$(REBAR_BIN) compile
+	$(REBAR) compile
+
+generate:
+	$(REBAR) generate $(OVERLAY_VARS) 2>&1 | grep -v 'command does not apply to directory'
 
 deps:
-	$(REBAR_BIN) get-deps
+	$(REBAR) get-deps
 
 clean:
-	$(REBAR_BIN) -r clean
+	$(REBAR) -r clean
 
 test: deps compile eunit
 
 eunit:
-	$(REBAR_BIN) -v skip_deps=true eunit
+	$(REBAR) -v skip_deps=true eunit
 
 edoc: edoc-clean
-	$(REBAR_BIN) skip_deps=true doc
+	$(REBAR) skip_deps=true doc
 
 edoc-clean:
 	rm -f edoc/*.png edoc/*.html edoc/*.css edoc/edoc-info
 
 pulse: compile
 	@echo Sorry, PULSE test needs maintenance. -SLF
-	#env USE_PULSE=1 $(REBAR_BIN) skip_deps=true clean compile
+	#env USE_PULSE=1 $(REBAR) skip_deps=true clean compile
-	#env USE_PULSE=1 $(REBAR_BIN) skip_deps=true -D PULSE eunit -v
+	#env USE_PULSE=1 $(REBAR) skip_deps=true -D PULSE eunit -v
 
 APPS = kernel stdlib sasl erts ssl compiler eunit crypto
 PLT = $(HOME)/.machi_dialyzer_plt
@@ -53,3 +62,14 @@ dialyzer-test: deps compile
 
 clean_plt:
 	rm $(PLT)
+
+##
+## Release targets
+##
+rel: deps compile generate
+
+relclean:
+	rm -rf rel/$(REPO)
+
+stage : rel
+	$(foreach dep,$(wildcard deps/*), rm -rf rel/$(REPO)/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) rel/$(REPO)/lib;)

apps/machi/ebin/machi.app

@@ -0,0 +1,5 @@
+{application,machi,
+             [{description,"Machi extremely-early boot items"},
+              %% {modules,[etop_txt]}
+              {modules,[]}
+             ]}.

rebar.config

@@ -7,6 +7,10 @@
 {deps, [
         {lager, ".*", {git, "git://github.com/basho/lager.git", {tag, "2.2.0"}}},
         {protobuffs, "0.8.*", {git, "git://github.com/basho/erlang_protobuffs.git", {tag, "0.8.1p4"}}},
-        {riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {branch, "develop"}}}
+        {riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {branch, "develop"}}},
+        {node_package, ".*", {git, "git://github.com/basho/node_package.git", {branch, "develop"}}},
+        {eper, ".*", {git, "git://github.com/basho/eper.git", {tag, "0.78"}}}
         ]}.
 
+{sub_dirs, ["rel", "apps/machi"]}.
+{lib_dirs, ["apps/machi"]}.

rel/files/app.config

@@ -0,0 +1 @@
+[].

rel/files/machi-admin

@@ -0,0 +1,68 @@
+#!/bin/sh
+# -*- tab-width:4;indent-tabs-mode:nil -*-
+# ex: ts=4 sw=4 et
+
+# Pull environment for this install
+. "{{runner_base_dir}}/lib/env.sh"
+
+# Make sure the user running this script is the owner and/or su to that user
+check_user "$@"
+ES=$?
+if [ "$ES" -ne 0 ]; then
+    exit $ES
+fi
+
+# Keep track of where script was invoked
+ORIGINAL_DIR=$(pwd)
+
+# Make sure CWD is set to runner run dir
+cd $RUNNER_BASE_DIR
+
+# Identify the script name
+SCRIPT=`basename $0`
+
+usage() {
+    echo "Usage: $SCRIPT { test | "
+    echo "                    top }"
+}
+
+stat_admin()
+{
+    case "$1" in
+    test)
+        # Make sure the local node IS running
+        node_up_check
+
+        shift
+
+        # Parse out the node name to pass to the client
+        NODE_NAME=${NAME_ARG#* }
+
+        $ERTS_PATH/erl -noshell $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \
+                       -pa $RUNNER_LIB_DIR/basho-patches \
+                       -eval "case catch(machi:client_test(\"$NODE_NAME\")) of \
+                               ok -> init:stop();                             \
+                               _  -> init:stop(1)                             \
+                              end."
+
+        ;;
+    top)
+        # Make sure the local node IS running
+        node_up_check
+
+        shift
+
+        MYPID=$$
+        NODE_NAME=${NAME_ARG#* }
+        $ERTS_PATH/erl -noshell -noinput \
+            -pa $RUNNER_LIB_DIR/basho-patches \
+            -hidden $NAME_PARAM machi_etop$MYPID$NAME_HOST $COOKIE_ARG \
+            -s etop -s erlang halt -output text \
+            -node $NODE_NAME \
+            $* -tracing off
+        ;;
+    *)
+        usage
+        exit 1
+        ;;
+esac

rel/files/machi.schema

@@ -0,0 +1,220 @@
+%%-*- mode: erlang -*-
+
+%% @doc Where to emit the default log messages (typically at 'info'
+%% severity):
+%%     off: disabled
+%%    file: the file specified by log.console.file
+%% console: to standard output (seen when using `machi attach-direct`)
+%%    both: log.console.file and standard out.
+{mapping, "log.console", "lager.handlers", [
+  {default, {{console_log_default}} },
+  {datatype, {enum, [off, file, console, both]}}
+]}.
+
+%% @doc The severity level of the console log, default is 'info'.
+{mapping, "log.console.level", "lager.handlers", [
+  {default, info},
+  {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}}
+]}.
+
+%% @doc When 'log.console' is set to 'file' or 'both', the file where
+%% console messages will be logged.
+{mapping, "log.console.file", "lager.handlers", [
+  {default, "$(platform_log_dir)/console.log"},
+  {datatype, file}
+]}.
+
+%% @doc The file where error messages will be logged.
+{mapping, "log.error.file", "lager.handlers", [
+  {default, "$(platform_log_dir)/error.log"},
+  {datatype, file}
+]}.
+
+%% @doc When set to 'on', enables log output to syslog.
+{mapping, "log.syslog", "lager.handlers", [
+  {default, off},
+  {datatype, flag}
+]}.
+
+%% @doc When set to 'on', enables log output to syslog.
+{mapping, "log.syslog.ident", "lager.handlers", [
+  {default, "machi"},
+  hidden
+]}.
+
+%% @doc Syslog facility to log entries from Riak.
+{mapping, "log.syslog.facility", "lager.handlers", [
+  {default, daemon},
+  {datatype, {enum,[kern, user, mail, daemon, auth, syslog,
+                    lpr, news, uucp, clock, authpriv, ftp,
+                    cron, local0, local1, local2, local3,
+                    local4, local5, local6, local7]}},
+  hidden
+]}.
+
+%% @doc The severity level at which to log entries to syslog, default is 'info'.
+{mapping, "log.syslog.level", "lager.handlers", [
+  {default, info},
+  {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}},
+  hidden
+]}.
+
+{translation,
+ "lager.handlers",
+ fun(Conf) ->
+    SyslogHandler = case cuttlefish:conf_get("log.syslog", Conf) of
+      true ->
+        Ident = cuttlefish:conf_get("log.syslog.ident", Conf),
+        Facility = cuttlefish:conf_get("log.syslog.facility", Conf),
+        LogLevel = cuttlefish:conf_get("log.syslog.level", Conf),
+        [{lager_syslog_backend, [Ident, Facility, LogLevel]}];
+      _ -> []
+    end,
+    ErrorHandler = case cuttlefish:conf_get("log.error.file", Conf) of
+      undefined -> [];
+      ErrorFilename -> [{lager_file_backend, [{file, ErrorFilename},
+                                              {level, error},
+                                              {size, 10485760},
+                                              {date, "$D0"},
+                                              {count, 5}]}]
+    end,
+
+    ConsoleLogLevel = cuttlefish:conf_get("log.console.level", Conf),
+    ConsoleLogFile = cuttlefish:conf_get("log.console.file", Conf),
+
+    ConsoleHandler = {lager_console_backend, ConsoleLogLevel},
+    ConsoleFileHandler = {lager_file_backend, [{file, ConsoleLogFile},
+                                               {level, ConsoleLogLevel},
+                                               {size, 10485760},
+                                               {date, "$D0"},
+                                               {count, 5}]},
+
+    ConsoleHandlers = case cuttlefish:conf_get("log.console", Conf) of
+      off -> [];
+      file -> [ConsoleFileHandler];
+      console -> [ConsoleHandler];
+      both -> [ConsoleHandler, ConsoleFileHandler];
+      _ -> []
+    end,
+    SyslogHandler ++ ConsoleHandlers ++ ErrorHandler
+  end
+}.
+
+
+%% @doc Whether to enable Erlang's built-in error logger.
+{mapping, "sasl", "sasl.sasl_error_logger", [
+  {default, off},
+  {datatype, flag},
+  hidden
+]}.
+
+%% @doc Whether to enable the crash log.
+{mapping, "log.crash", "lager.crash_log", [
+  {default, on},
+  {datatype, flag}
+]}.
+
+%% @doc If the crash log is enabled, the file where its messages will
+%% be written.
+{mapping, "log.crash.file", "lager.crash_log", [
+  {default, "$(platform_log_dir)/crash.log"},
+  {datatype, file}
+]}.
+
+{translation,
+ "lager.crash_log",
+ fun(Conf) ->
+     case cuttlefish:conf_get("log.crash", Conf) of
+         false -> undefined;
+         _ ->
+             cuttlefish:conf_get("log.crash.file", Conf, "{{platform_log_dir}}/crash.log")
+     end
+ end}.
+
+%% @doc Maximum size in bytes of individual messages in the crash log
+{mapping, "log.crash.maximum_message_size", "lager.crash_log_msg_size", [
+  {default, "64KB"},
+  {datatype, bytesize}
+]}.
+
+%% @doc Maximum size of the crash log in bytes, before it is rotated
+{mapping, "log.crash.size", "lager.crash_log_size", [
+  {default, "10MB"},
+  {datatype, bytesize}
+]}.
+
+%% @doc The schedule on which to rotate the crash log.  For more
+%% information see:
+%% https://github.com/basho/lager/blob/master/README.md#internal-log-rotation
+{mapping, "log.crash.rotation", "lager.crash_log_date", [
+  {default, "$D0"}
+]}.
+
+%% @doc The number of rotated crash logs to keep. When set to
+%% 'current', only the current open log file is kept.
+{mapping, "log.crash.rotation.keep", "lager.crash_log_count", [
+  {default, 5},
+  {datatype, [integer, {atom, current}]},
+  {validators, ["rotation_count"]}
+]}.
+
+{validator,
+ "rotation_count",
+ "must be 'current' or a positive integer",
+ fun(current) -> true;
+    (Int) when is_integer(Int) andalso Int >= 0 -> true;
+    (_) -> false
+ end}.
+
+{translation,
+ "lager.crash_log_count",
+ fun(Conf) ->
+    case cuttlefish:conf_get("log.crash.rotation.keep", Conf) of
+       current -> 0;
+       Int -> Int
+    end
+ end}.
+
+%% @doc Whether to redirect error_logger messages into lager -
+%% defaults to true
+{mapping, "log.error.redirect", "lager.error_logger_redirect", [
+  {default, on},
+  {datatype, flag},
+  hidden
+]}.
+
+%% @doc Maximum number of error_logger messages to handle in a second
+{mapping, "log.error.messages_per_second", "lager.error_logger_hwm", [
+  {default, 100},
+  {datatype, integer},
+  hidden
+]}.
+
+
+%% @doc Cookie for distributed node communication.  All nodes in the
+%% same cluster should use the same cookie or they will not be able to
+%% communicate.
+{mapping, "distributed_cookie", "vm_args.-setcookie", [
+  {default, "machi"}
+]}.
+
+
+%% override zdbbl from 1mb to 32mb
+{mapping, "erlang.distribution_buffer_size", "vm_args.+zdbbl", [
+  {default, "32MB"},
+  merge
+]}.
+
+%% VM scheduler collapse, part 1 of 2
+{mapping, "erlang.schedulers.force_wakeup_interval", "vm_args.+sfwi", [
+  {default, 500},
+  {datatype, integer},
+  merge
+]}.
+
+%% VM scheduler collapse, part 2 of 2
+{mapping, "erlang.schedulers.compaction_of_load", "vm_args.+scl", [
+  {default, false},
+  merge
+]}.
+

rel/files/vm.args

@@ -0,0 +1,27 @@
+## Name of the riak node
+-name {{node}}
+
+## Cookie for distributed erlang.  All nodes in the same cluster
+## should use the same cookie or they will not be able to communicate.
+-setcookie machi
+
+## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive
+## (Disabled by default..use with caution!)
+##-heart
+
+## Enable kernel poll and a few async threads
++K true
++A 64
+
+## Treat error_logger warnings as warnings
++W w
+
+## Increase number of concurrent ports/sockets
+-env ERL_MAX_PORTS 4096
+
+## ## Tweak GC to run more often 
+## -env ERL_FULLSWEEP_AFTER 0
+
+## Set the location of crash dumps
+-env ERL_CRASH_DUMP {{crash_dump}}
+

rel/rebar.config

@@ -0,0 +1,4 @@
+{lib_dirs, ["../deps"]}.
+%% {plugin_dir, "../deps/cuttlefish/src"}.
+%% {plugins, [cuttlefish_rebar_plugin]}.
+%% {cuttlefish_filename, "machi.conf"}.

rel/reltool.config

@@ -0,0 +1,98 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ft=erlang ts=4 sw=4 et
+{sys, [
+       {lib_dirs, ["../deps", "../apps"]},
+       {rel, "machi", "0.0.0",
+        [
+         kernel,
+         stdlib,
+         lager,
+         sasl,
+         public_key,
+         ssl,
+%%         riak_sysmon,
+%%         os_mon,
+         crypto,
+         runtime_tools,
+         machi
+%%         cluster_info,
+%%         exometer_core,
+        ]},
+       {rel, "start_clean", "",
+        [
+         kernel,
+         stdlib
+        ]},
+       {boot_rel, "machi"},
+       {profile, embedded},
+       {excl_sys_filters, ["^bin/.*",
+                           "^erts.*/bin/(dialyzer|typer)",
+                           "^erts.*/doc",
+                           "^erts.*/man"]},
+       {excl_archive_filters, [".*"]},
+%%       {app, cuttlefish, [{incl_cond, include}]},
+%%       {app, cluster_info, [{incl_cond, include}]},
+       {app, eper, [{incl_cond, include}]},
+       {app, sasl, [{incl_cond, include}]},
+%%       {app, syslog, [{incl_cond, include}]},
+%%       {app, lager_syslog, [{incl_cond, include}]},
+       {app, lager, [{incl_cond, include}]}
+%%       {app, exometer_core, [{incl_cond, include}]},
+      ]}.
+
+
+{target_dir, "machi"}.
+
+{overlay_vars, "vars.config"}.
+
+{overlay, [
+           {mkdir, "data"},
+           {mkdir, "log"},
+
+           %% Copy base files for starting and interacting w/ node
+           {copy, "../deps/node_package/priv/base/erl",
+                  "{{erts_vsn}}/bin/erl"},
+           {copy, "../deps/node_package/priv/base/nodetool",
+                  "{{erts_vsn}}/bin/nodetool"},
+%%           {copy, "../deps/cuttlefish/cuttlefish",
+%%                  "{{erts_vsn}}/bin/cuttlefish"},
+           {template, "../deps/node_package/priv/base/runner",
+                  "bin/machi"},
+           {template, "../deps/node_package/priv/base/env.sh",
+                  "lib/env.sh"},
+           {template, "../deps/node_package/priv/base/app_epath.sh",
+                  "lib/app_epath.sh"},
+
+           %% Copy config files
+
+           %% Cuttlefish Schema Files have a priority order.
+           %% Anything in a file prefixed with 00- will override
+           %% anything in a file with a higher numbered prefix.
+
+           %% Please only use 0[0-9]-*.schema for development purposes
+           %% NOTHING PERMANENT
+
+           %% {template, "files/riak.schema", "lib/10-riak.schema"},
+           %% {template, "../deps/cuttlefish/priv/erlang_vm.schema", "lib/11-erlang_vm.schema"},
+
+           %% {template, "../deps/riak_core/priv/riak_core.schema", "lib/12-riak_core.schema"},
+           %% {template, "../deps/riak_api/priv/riak_api.schema", "lib/13-riak_api.schema"},
+           %% {template, "../deps/riak_kv/priv/riak_kv.schema", "lib/14-riak_kv.schema"},
+           %% {template, "../deps/riak_sysmon/priv/riak_sysmon.schema", "lib/15-riak_sysmon.schema"},
+           %% {template, "../deps/bitcask/priv/bitcask.schema", "lib/16-bitcask.schema"},
+           %% {template, "../deps/bitcask/priv/bitcask_multi.schema", "lib/17-bitcask_multi.schema"},
+           %% {template, "../deps/riak_control/priv/riak_control.schema", "lib/18-riak_control.schema"},
+
+           %% {template, "../deps/riak_kv/priv/multi_backend.schema", "lib/20-multi_backend.schema"},
+           %% {template, "../deps/eleveldb/priv/eleveldb.schema", "lib/21-leveldb.schema"},
+           %% {template, "../deps/eleveldb/priv/eleveldb_multi.schema", "lib/22-leveldb_multi.schema"},
+           %% {template, "../deps/yokozuna/priv/yokozuna.schema", "lib/30-yokozuna.schema"},
+
+           %% Copy additional bin scripts
+           {template, "files/machi-admin", "bin/machi-admin"},
+
+           {template, "files/vm.args", "etc/vm.args"},
+           {template, "files/app.config", "etc/app.config"},
+           {mkdir, "lib/basho-patches"}
+           %% {copy, "../apps/machi/ebin/etop_txt.beam", "lib/basho-patches"}
+          ]}.

rel/vars.config

@@ -0,0 +1,45 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ft=erlang ts=4 sw=4 et
+
+%% Platform-specific installation paths
+{platform_bin_dir,  "./bin"}.
+{platform_data_dir, "./data"}.
+{platform_etc_dir,  "./etc"}.
+{platform_lib_dir,  "./lib"}.
+{platform_log_dir,  "./log"}.
+
+%%
+%% etc/app.config
+%%
+{sasl_error_log,    "{{platform_log_dir}}/sasl-error.log"}.
+{sasl_log_dir,      "{{platform_log_dir}}/sasl"}.
+
+%% lager
+{console_log_default, file}.
+
+%%
+%% etc/vm.args
+%%
+{node,         "machi@127.0.0.1"}.
+{crash_dump,   "{{platform_log_dir}}/erl_crash.dump"}.
+
+%%
+%% bin/machi
+%%
+{runner_script_dir,  "\`cd \\`dirname $0\\` 1>/dev/null && /bin/pwd\`"}.
+{runner_base_dir,    "{{runner_script_dir}}/.."}.
+{runner_etc_dir,     "$RUNNER_BASE_DIR/etc"}.
+{runner_log_dir,     "$RUNNER_BASE_DIR/log"}.
+{runner_lib_dir,     "$RUNNER_BASE_DIR/lib"}.
+{runner_patch_dir,   "$RUNNER_BASE_DIR/lib/basho-patches"}.
+{pipe_dir,           "/tmp/$RUNNER_BASE_DIR/"}.
+{runner_user,        ""}.
+{runner_wait_process, "riak_core_node_watcher"}.
+{runner_ulimit_warn, 65536}.
+
+%%
+%% cuttlefish
+%%
+{cuttlefish,         ""}. % blank = off
+{cuttlefish_conf,    "machi.conf"}.
+