diff --git a/.gitignore b/.gitignore
index 3af54ff..ef440c4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@ prototype/chain-manager/patch.*
.eqc-info
.eunit
deps
+dev
erl_crash.dump
.concrete/DEV_MODE
.rebar
diff --git a/Makefile b/Makefile
index 7ff19ed..01b1e99 100644
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@ endif
OVERLAY_VARS ?=
EUNIT_OPTS = -v
-.PHONY: rel deps package pkgclean edoc
+.PHONY: rel stagedevrel deps package pkgclean edoc
all: deps compile
@@ -57,6 +57,37 @@ relclean:
stage : rel
$(foreach dep,$(wildcard deps/*), rm -rf rel/$(REPO)/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) rel/$(REPO)/lib;)
+##
+## Developer targets
+##
+## devN - Make a dev build for node N
+## stagedevN - Make a stage dev build for node N (symlink libraries)
+## devrel - Make a dev build for 1..$DEVNODES
+## stagedevrel Make a stagedev build for 1..$DEVNODES
+##
+## Example, make a 68 node devrel cluster
+## make stagedevrel DEVNODES=68
+
+.PHONY : stagedevrel devrel
+DEVNODES ?= 3
+
+# 'seq' is not available on all *BSD, so using an alternate in awk
+SEQ = $(shell awk 'BEGIN { for (i = 1; i < '$(DEVNODES)'; i++) printf("%i ", i); print i ;exit(0);}')
+
+$(eval stagedevrel : $(foreach n,$(SEQ),stagedev$(n)))
+$(eval devrel : $(foreach n,$(SEQ),dev$(n)))
+
+dev% : all
+ mkdir -p dev
+ rel/gen_dev $@ rel/vars/dev_vars.config.src rel/vars/$@_vars.config
+ (cd rel && ../rebar generate target_dir=../dev/$@ overlay_vars=vars/$@_vars.config)
+
+stagedev% : dev%
+ $(foreach dep,$(wildcard deps/*), rm -rf dev/$^/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) dev/$^/lib;)
+
+devclean: clean
+ rm -rf dev
+
DIALYZER_APPS = kernel stdlib sasl erts ssl compiler eunit crypto public_key syntax_tools
PLT = $(HOME)/.machi_dialyzer_plt
diff --git a/README.md b/README.md
index 28f77d2..37db1e0 100644
--- a/README.md
+++ b/README.md
@@ -64,6 +64,9 @@ Humming Consensus" is available online now.
* [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf)
* [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q)
+See later in this document for how to run the Humming Consensus demos,
+including the network partition simulator.
+
## 3. Development status summary
@@ -99,10 +102,10 @@ Mid-December 2015: work is underway.
* The Erlang language client implementation of the high-level
protocol flavor is brittle (e.g., little error handling yet).
-If you would like to run the network partition simulator
-mentioned in the Ricon 2015 presentation about Humming Consensus,
-please see the
-[partition simulator convergence test doc.](./doc/machi_chain_manager1_converge_demo.md)
+If you would like to run the Humming Consensus code (with or without
+the network partition simulator) as described in the RICON 2015
+presentation, please see the
+[Humming Consensus demo doc.](./doc/humming_consensus_demo.md).
If you'd like to work on a protocol such as Thrift, UBF,
msgpack over UDP, or some other protocol, let us know by
diff --git a/doc/machi_chain_manager1_converge_demo.md b/doc/humming_consensus_demo.md
similarity index 76%
rename from doc/machi_chain_manager1_converge_demo.md
rename to doc/humming_consensus_demo.md
index 2844bfa..eb66ebe 100644
--- a/doc/machi_chain_manager1_converge_demo.md
+++ b/doc/humming_consensus_demo.md
@@ -1,6 +1,75 @@
+# Table of contents
+
+* [Hand-on experiments with Machi and Humming Consensus](#hands-on)
+* [Using the network partition simulator and convergence demo test code](#partition-simulator)
+
+
+# Hand-on experiments with Machi and Humming Consensus
+
+
+## Prerequisites
+
+1. Machi requires a OS X, FreeBSD, Linux, or Solaris machine.
+2. You'll need the `git` source management utility.
+3. You'll need the Erlang/OTP 17 runtime environment. Please don't
+ use earlier or later versions until we have a chance to fix the
+ compilation warnings that versions R16B and 18 will trigger.
+
+For `git` and the Erlang runtime, please use your OS-specific
+package manager to install these. If your package manager doesn't
+have Erlang/OTP version 17 available, then we recommend using the
+[precompiled packages available at Erlang Solutions](https://www.erlang-solutions.com/resources/download.html).
+
+All of the commands that should be run at your login shell (e.g. Bash,
+c-shell) can be cut-and-pasted from this document directly to your
+login shell prompt.
+
+
+## Clone and compile the code
+
+Clone the Machi source repo and compile the source and test code. Run
+the following commands at your login shell:
+
+ cd /tmp
+ git clone https://github.com/basho/machi.git
+ cd machi
+ git checkout master
+ make
+
+Then run the unit test suite. This may take up to two minutes or so
+to finish.
+
+ make test
+
+At the end, the test suite should report that all tests passed.
+
+If you had a test failure, a likely cause may be a limit on the number
+of file descriptors available to your user process. (Recent releases
+of OS X have a limit of 1024 file descriptors, which may be too slow.)
+The output of the `limit -n` will tell you your file descriptor limit.
+
+## Running three Machi instances on a single machine
+
+Run the following command:
+
+ make stagedevrel
+
+This will create a directory structure like this:
+
+ |-dev1-|... stand-alone Machi app directories
+ |-dev-|-dev2-|... stand-alone Machi app directories
+ |-dev3-|... stand-alone Machi app directories
+
+
# Using the network partition simulator and convergence demo test code
+This is the demo code mentioned in the presentation that Scott Lystig
+Fritchie gave at the
+[RICON 2015 conference](http://ricon.io).
+* [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf)
+* [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q)
+
## A complete example of all input and output
If you don't have an Erlang/OTP 17 runtime environment available,
@@ -15,31 +84,15 @@ To help interpret the output of the test, please skip ahead to the
## Prerequisites
-1. You'll need the `git` source management
-2. You'll need the Erlang/OTP 17 runtime environment. Please don't
- use earlier or later versions until we have a chance to fix the
- compilation warnings that versions R16B and 18 will trigger.
-
-All of the commands that should be run at your login shell (e.g. Bash,
-c-shell) can be cut-and-pasted from this document directly to your
-login shell prompt.
+If you don't have `git` and/or the Erlang 17 runtime system available
+on your OS X, FreeBSD, Linux, or Solaris machine, please take a look
+at the [Prerequistes section](#prerequisites) first. When you have
+installed the prerequisite software, please return back here.
## Clone and compile the code
-Clone the Machi source repo and compile the source and test code. Run
-the following commands at your login shell:
-
- cd /tmp
- git clone https://github.com/basho/machi.git
- cd machi
- git checkout master
- make
-
-Then run the unit test suite. This may take up to two minutes or so
-to finish. Most of the tests will be silent; please be patient until
-the tests finish.
-
- make test
+Please briefly visit the [Clone and compile the code](#clone-compile)
+section. When finished, please return back here.
## Run an interactive Erlang CLI shell
diff --git a/rel/gen_dev b/rel/gen_dev
new file mode 100755
index 0000000..1b8ce1b
--- /dev/null
+++ b/rel/gen_dev
@@ -0,0 +1,16 @@
+#! /bin/sh
+#
+# Example usage: gen_dev dev4 vars.src vars
+#
+# Generate an overlay config for devNNN from vars.src and write to vars
+#
+
+NAME=$1
+TEMPLATE=$2
+VARFILE=$3
+
+NODE="$NAME@127.0.0.1"
+
+echo "Generating $NAME - node='$NODE'"
+sed -e "s/@NODE@/$NODE/" \
+ < $TEMPLATE > $VARFILE
diff --git a/rel/vars.config b/rel/vars.config
index 06b3aa0..b1bb405 100644
--- a/rel/vars.config
+++ b/rel/vars.config
@@ -1,6 +1,9 @@
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ft=erlang ts=4 sw=4 et
+%% NOTE: When modifying this file, also keep its near cousin
+%% config file rel/vars/dev_vars.config.src in sync!
+
%% Platform-specific installation paths
{platform_bin_dir, "./bin"}.
{platform_data_dir, "./data"}.
diff --git a/rel/vars/dev_vars.config.src b/rel/vars/dev_vars.config.src
new file mode 100644
index 0000000..a5a3828
--- /dev/null
+++ b/rel/vars/dev_vars.config.src
@@ -0,0 +1,48 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ft=erlang ts=4 sw=4 et
+
+%% NOTE: When modifying this file, also keep its near cousin
+%% config file rel/vars/dev_vars.config.src in sync!
+
+%% Platform-specific installation paths
+{platform_bin_dir, "./bin"}.
+{platform_data_dir, "./data"}.
+{platform_etc_dir, "./etc"}.
+{platform_lib_dir, "./lib"}.
+{platform_log_dir, "./log"}.
+
+%%
+%% etc/app.config
+%%
+{sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}.
+{sasl_log_dir, "{{platform_log_dir}}/sasl"}.
+
+%% lager
+{console_log_default, file}.
+
+%%
+%% etc/vm.args
+%%
+{node, "@NODE@"}.
+{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}.
+
+%%
+%% bin/machi
+%%
+{runner_script_dir, "\`cd \\`dirname $0\\` 1>/dev/null && /bin/pwd\`"}.
+{runner_base_dir, "{{runner_script_dir}}/.."}.
+{runner_etc_dir, "$RUNNER_BASE_DIR/etc"}.
+{runner_log_dir, "$RUNNER_BASE_DIR/log"}.
+{runner_lib_dir, "$RUNNER_BASE_DIR/lib"}.
+{runner_patch_dir, "$RUNNER_BASE_DIR/lib/basho-patches"}.
+{pipe_dir, "/tmp/$RUNNER_BASE_DIR/"}.
+{runner_user, ""}.
+{runner_wait_process, "machi_flu_sup"}.
+{runner_ulimit_warn, 65536}.
+
+%%
+%% cuttlefish
+%%
+{cuttlefish, ""}. % blank = off
+{cuttlefish_conf, "machi.conf"}.
+