89 changed files with 3447 additions and 8565 deletions
--- a/.gitignore
+++ b/.gitignore
@ -2,9 +2,7 @@ prototype/chain-manager/patch.*
 .eqc-info
 .eunit
 deps
 dev
 erl_crash.dump
 eqc
 .concrete/DEV_MODE
 .rebar
 edoc
@ -22,12 +20,9 @@ include/machi_pb.hrl
 # Release packaging
 rel/machi
 rel/vars/dev*vars.config
 # Misc Scott cruft
 *.patch
 current_counterexample.eqc
 foo*
 RUNLOG*
 typescript*
 *.swp
--- a/.travis.yml
+++ b/.travis.yml
@ -4,4 +4,3 @@ notifications:
 script: "priv/test-for-gh-pr.sh"
 otp_release:
  - 17.5
 ## No, Dialyzer is too different between 17 & 18:  - 18.1
--- a/FAQ.md
+++ b/FAQ.md
@ -11,14 +11,14 @@
 + [1 Questions about Machi in general](#n1)
    + [1.1 What is Machi?](#n1.1)
-    + [1.2 What is a Machi chain?](#n1.2)
+    + [1.2 What is a Machi "cluster of clusters"?](#n1.2)
-    + [1.3 What is a Machi cluster?](#n1.3)
+        + [1.2.1 This "cluster of clusters" idea needs a better name, don't you agree?](#n1.2.1)
-    + [1.4 What is Machi like when operating in "eventually consistent" mode?](#n1.4)
+    + [1.3 What is Machi like when operating in "eventually consistent"/"AP mode"?](#n1.3)
-    + [1.5 What is Machi like when operating in "strongly consistent" mode?](#n1.5)
+    + [1.4 What is Machi like when operating in "strongly consistent"/"CP mode"?](#n1.4)
-    + [1.6 What does Machi's API look like?](#n1.6)
+    + [1.5 What does Machi's API look like?](#n1.5)
-    + [1.7 What licensing terms are used by Machi?](#n1.7)
+    + [1.6 What licensing terms are used by Machi?](#n1.6)
-    + [1.8 Where can I find the Machi source code and documentation?  Can I contribute?](#n1.8)
+    + [1.7 Where can I find the Machi source code and documentation?  Can I contribute?](#n1.7)
-    + [1.9 What is Machi's expected release schedule, packaging, and operating system/OS distribution support?](#n1.9)
+    + [1.8 What is Machi's expected release schedule, packaging, and operating system/OS distribution support?](#n1.8)
 + [2 Questions about Machi relative to {{something else}}](#n2)
    + [2.1 How is Machi better than Hadoop?](#n2.1)
    + [2.2 How does Machi differ from HadoopFS/HDFS?](#n2.2)
@ -28,15 +28,13 @@
 + [3 Machi's specifics](#n3)
    + [3.1 What technique is used to replicate Machi's files?  Can other techniques be used?](#n3.1)
    + [3.2 Does Machi have a reliance on a coordination service such as ZooKeeper or etcd?](#n3.2)
-    + [3.3 Are there any presentations available about Humming Consensus](#n3.3)
+    + [3.3 Is it true that there's an allegory written to describe humming consensus?](#n3.3)
-    + [3.4 Is it true that there's an allegory written to describe Humming Consensus?](#n3.4)
+    + [3.4 How is Machi tested?](#n3.4)
-    + [3.5 How is Machi tested?](#n3.5)
+    + [3.5 Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks](#n3.5)
-    + [3.6 Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks](#n3.6)
+    + [3.6 Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?](#n3.6)
-    + [3.7 Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?](#n3.7)
+    + [3.7 What language(s) is Machi written in?](#n3.7)
-    + [3.8 What language(s) is Machi written in?](#n3.8)
+    + [3.8 Does Machi use the Erlang/OTP network distribution system (aka "disterl")?](#n3.8)
-    + [3.9 Can Machi run on Windows?  Can Machi run on 32-bit platforms?](#n3.9)
+    + [3.9 Can I use HTTP to write/read stuff into/from Machi?](#n3.9)
    + [3.10 Does Machi use the Erlang/OTP network distribution system (aka "disterl")?](#n3.10)
    + [3.11 Can I use HTTP to write/read stuff into/from Machi?](#n3.11)
 <!-- ENDOUTLINE -->
@ -46,13 +44,13 @@
 <a name="n1.1">
 ### 1.1.  What is Machi?
-Very briefly, Machi is a very simple append-only blob/file store.
+Very briefly, Machi is a very simple append-only file store.
 Machi is
 "dumber" than many other file stores (i.e., lacking many features
-found in other file stores) such as HadoopFS or a simple NFS or CIFS file
+found in other file stores) such as HadoopFS or simple NFS or CIFS file
 server.
-However, Machi is a distributed blob/file store, which makes it different
+However, Machi is a distributed file store, which makes it different
 (and, in some ways, more complicated) than a simple NFS or CIFS file
 server.
@ -84,39 +82,45 @@ For a much longer answer, please see the
 [Machi high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-machi.pdf).
 <a name="n1.2">
-### 1.2.  What is a Machi chain?
+### 1.2.  What is a Machi "cluster of clusters"?
-A Machi chain is a small number of machines that maintain a common set
+Machi's design is based on using small, well-understood and provable
-of replicated files.  A typical chain is of length 2 or 3.  For
+(mathematically) techniques to maintain multiple file copies without
-critical data that must be available despite several simultaneous
+data loss or data corruption.  At its lowest level, Machi contains no
-server failures, a chain length of 6 or 7 might be used.
+support for distribution/partitioning/sharding of files across many
 servers.  A typical, fully-functional Machi cluster will likely be two
 or three machines.
-<a name="n1.3">
+However, Machi is designed to be an excellent building block for
-### 1.3.  What is a Machi cluster?
+building larger systems.  A deployment of Machi "cluster of clusters"
 will use the "random slicing" technique for partitioning files across
 multiple Machi clusters that, as individuals, are unaware of the
 larger cluster-of-clusters scheme.
-A Machi cluster is a collection of Machi chains that
+The cluster-of-clusters management service will be fully decentralized
 partitions/shards/distributes files (based on file name) across the
 collection of chains.  Machi uses the "random slicing" algorithm (a
 variation of consistent hashing) to define the mapping of file name to
 chain name.
 The cluster management service will be fully decentralized
 and run as a separate software service installed on each Machi
 cluster.  This manager will appear to the local Machi server as simply
-another Machi file client.  The cluster managers will take
+another Machi file client.  The cluster-of-clusters managers will take
 care of file migration as the cluster grows and shrinks in capacity
 and in response to day-to-day changes in workload.
-Though the cluster manager has not yet been implemented,
+Though the cluster-of-clusters manager has not yet been implemented,
 its design is fully decentralized and capable of operating despite
-multiple partial failure of its member chains.  We expect this
+multiple partial failure of its member clusters.  We expect this
 design to scale easily to at least one thousand servers.
 Please see the
 [Machi source repository's 'doc' directory for more details](https://github.com/basho/machi/tree/master/doc/).
-<a name="n1.4">
+<a name="n1.2.1">
-### 1.4.  What is Machi like when operating in "eventually consistent" mode?
+#### 1.2.1.  This "cluster of clusters" idea needs a better name, don't you agree?
 Yes.  Please help us: we are bad at naming things.
 For proof that naming things is hard, see
 [http://martinfowler.com/bliki/TwoHardThings.html](http://martinfowler.com/bliki/TwoHardThings.html)
 <a name="n1.3">
 ### 1.3.  What is Machi like when operating in "eventually consistent"/"AP mode"?
 Machi's operating mode dictates how a Machi cluster will react to
 network partitions.  A network partition may be caused by:
@ -126,30 +130,37 @@ network partitions.  A network partition may be caused by:
 * An extreme server software "hang" or "pause", e.g. caused by OS
  scheduling problems such as a failing/stuttering disk device.
-The consistency semantics of file operations while in eventual
+"AP mode" refers to the "A" and "P" properties of the "CAP
-consistency mode during and after network partitions are:
+conjecture", meaning that the cluster will be "Available" and
 "Partition tolerant".
 The consistency semantics of file operations while in "AP mode" are
 eventually consistent during and after network partitions:
 * File write operations are permitted by any client on the "same side"
  of the network partition.
 * File read operations are successful for any file contents where the
  client & server are on the "same side" of the network partition.
    * File read operations will probably fail for any file contents where the
      client & server are on "different sides" of the network partition.
 * After the network partition(s) is resolved, files are merged
  together from "all sides" of the partition(s).
    * Unique files are copied in their entirety.
    * Byte ranges within the same file are merged.  This is possible
-      due to Machi's restrictions on file naming and file offset
+      due to Machi's restrictions on file naming (files names are
-      assignment.  Both file names and file offsets are always chosen
+      alwoys assigned by Machi servers) and file offset assignments
-      by Machi servers according to rules which guarantee safe
+      (byte offsets are also always chosen by Machi servers according
-      mergeability.  Server-assigned names are a characteristic of a
+      to rules which guarantee safe mergeability.).
      "blob store".
-<a name="n1.5">
+<a name="n1.4">
-### 1.5.  What is Machi like when operating in "strongly consistent" mode?
+### 1.4.  What is Machi like when operating in "strongly consistent"/"CP mode"?
-The consistency semantics of file operations while in strongly
+Machi's operating mode dictates how a Machi cluster will react to
-consistency mode during and after network partitions are:
+network partitions.
 "CP mode" refers to the "C" and "P" properties of the "CAP
 conjecture", meaning that the cluster will be "Consistent" and
 "Partition tolerant".
 The consistency semantics of file operations while in "CP mode" are
 strongly consistent during and after network partitions:
 * File write operations are permitted by any client on the "same side"
  of the network partition if and only if a quorum majority of Machi servers
@ -164,19 +175,19 @@ consistency mode during and after network partitions are:
 Machi's design can provide the illusion of quorum minority write
 availability if the cluster is configured to operate with "witness
-servers".  (This feaure partially implemented, as of December 2015.)
+servers".  (This feaure is not implemented yet, as of June 2015.)
 See Section 11 of
 [Machi chain manager high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-chain-mgr.pdf)
 for more details.
-<a name="n1.6">
+<a name="n1.5">
-### 1.6.  What does Machi's API look like?
+### 1.5.  What does Machi's API look like?
 The Machi API only contains a handful of API operations.  The function
-arguments shown below (in simplifed form) use Erlang-style type annotations.
+arguments shown below use Erlang-style type annotations.
-    append_chunk(Prefix:binary(), Chunk:binary(), CheckSum:binary()).
+    append_chunk(Prefix:binary(), Chunk:binary()).
-    append_chunk_extra(Prefix:binary(), Chunk:binary(), CheckSum:binary(), ExtraSpace:non_neg_integer()).
+    append_chunk_extra(Prefix:binary(), Chunk:binary(), ExtraSpace:non_neg_integer()).
    read_chunk(File:binary(), Offset:non_neg_integer(), Size:non_neg_integer()).
    checksum_list(File:binary()).
@ -194,22 +205,17 @@ Internally, there is a more complex protocol used by individual
 cluster members to manage file contents and to repair damaged/missing
 files.  See Figure 3 in
 [Machi high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-machi.pdf)
-for more description.
+for more details.
-The definitions of both the "high level" external protocol and "low
+<a name="n1.6">
-level" internal protocol are in a
+### 1.6.  What licensing terms are used by Machi?
 [Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview)
 definition at [./src/machi.proto](./src/machi.proto).
 <a name="n1.7">
 ### 1.7.  What licensing terms are used by Machi?
 All Machi source code and documentation is licensed by
 [Basho Technologies, Inc.](http://www.basho.com/)
 under the [Apache Public License version 2](https://github.com/basho/machi/tree/master/LICENSE).
-<a name="n1.8">
+<a name="n1.7">
-### 1.8.  Where can I find the Machi source code and documentation?  Can I contribute?
+### 1.7.  Where can I find the Machi source code and documentation?  Can I contribute?
 All Machi source code and documentation can be found at GitHub:
 [https://github.com/basho/machi](https://github.com/basho/machi).
@ -223,11 +229,11 @@ ideas for improvement, please see our contributing & collaboration
 guidelines at
 [https://github.com/basho/machi/blob/master/CONTRIBUTING.md](https://github.com/basho/machi/blob/master/CONTRIBUTING.md).
-<a name="n1.9">
+<a name="n1.8">
-### 1.9.  What is Machi's expected release schedule, packaging, and operating system/OS distribution support?
+### 1.8.  What is Machi's expected release schedule, packaging, and operating system/OS distribution support?
-Basho expects that Machi's first major product release will take place
+Basho expects that Machi's first release will take place near the end
-during the 2nd quarter of 2016.
+of calendar year 2015.
 Basho's official support for operating systems (e.g. Linux, FreeBSD),
 operating system packaging (e.g. CentOS rpm/yum package management,
@ -302,15 +308,15 @@ file's writable phase).
 <tr>
 <td> Does not have any file distribution/partitioning/sharding across
-Machi chains: in a single Machi chain, all files are replicated by
+Machi clusters: in a single Machi cluster, all files are replicated by
-all servers in the chain.  The "random slicing" technique is used
+all servers in the cluster.  The "cluster of clusters" concept is used
 to distribute/partition/shard files across multiple Machi clusters.
 <td> File distribution/partitioning/sharding is performed
 automatically by the HDFS "name node".
 <tr>
-<td> Machi requires no central "name node" for single chain use or
+<td> Machi requires no central "name node" for single cluster use.
-for multi-chain cluster use.
+Machi requires no central "name node" for "cluster of clusters" use
 <td> Requires a single "namenode" server to maintain file system contents
 and file content mapping.  (May be deployed with a "secondary
 namenode" to reduce unavailability when the primary namenode fails.)
@ -476,8 +482,8 @@ difficult to adapt to Machi's design goals:
 * Both protocols use quorum majority consensus, which requires a
  minimum of *2F + 1* working servers to tolerate *F* failures.  For
  example, to tolerate 2 server failures, quorum majority protocols
-  require a minimum of 5 servers.  To tolerate the same number of
+  require a minium of 5 servers.  To tolerate the same number of
-  failures, Chain Replication requires a minimum of only 3 servers.
+  failures, Chain replication requires only 3 servers.
 * Machi's use of "humming consensus" to manage internal server
  metadata state would also (probably) require conversion to Paxos or
  Raft.  (Or "outsourced" to a service such as ZooKeeper.)
@ -494,17 +500,7 @@ Humming consensus is described in the
 [Machi chain manager high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-chain-mgr.pdf).
 <a name="n3.3">
-### 3.3.  Are there any presentations available about Humming Consensus
+### 3.3.  Is it true that there's an allegory written to describe humming consensus?
 Scott recently (November 2015) gave a presentation at the
 [RICON 2015 conference](http://ricon.io) about one of the techniques
 used by Machi; "Managing Chain Replication Metadata with
 Humming Consensus" is available online now.
 * [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf)
 * [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q)
 <a name="n3.4">
 ### 3.4.  Is it true that there's an allegory written to describe Humming Consensus?
 Yes.  In homage to Leslie Lamport's original paper about the Paxos
 protocol, "The Part-time Parliamant", there is an allegorical story
@ -515,8 +511,8 @@ The full story, full of wonder and mystery, is called
 There is also a
 [short followup blog posting](http://www.snookles.com/slf-blog/2015/03/20/on-humming-consensus-an-allegory-part-2/).
-<a name="n3.5">
+<a name="n3.4">
-### 3.5.  How is Machi tested?
+### 3.4.  How is Machi tested?
 While not formally proven yet, Machi's implementation of Chain
 Replication and of humming consensus have been extensively tested with
@ -541,20 +537,16 @@ change several times during any single test case) and a random series
 of cluster operations, an event trace of all cluster activity is used
 to verify that no safety-critical rules have been violated.
-All test code is available in the [./test](./test) subdirectory.
+<a name="n3.5">
-Modules that use QuickCheck will use a file suffix of `_eqc`, for
+### 3.5.  Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks
 example, [./test/machi_ap_repair_eqc.erl](./test/machi_ap_repair_eqc.erl).
 <a name="n3.6">
 ### 3.6.  Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks
 No, Machi's design assumes that each Machi server is a fully
 independent hardware and assumes only standard local disks (Winchester
 and/or SSD style) with local-only interfaces (e.g. SATA, SCSI, PCI) in
 each machine.
-<a name="n3.7">
+<a name="n3.6">
-### 3.7.  Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?
+### 3.6.  Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?
 No.  When used with servers with multiple disks, the intent is to
 deploy multiple Machi servers per machine: one Machi server per disk.
@ -572,13 +564,10 @@ deploy multiple Machi servers per machine: one Machi server per disk.
  placement relative to 12 servers is smaller than a placement problem
  of managing 264 seprate disks (if each of 12 servers has 22 disks).
-<a name="n3.8">
+<a name="n3.7">
-### 3.8.  What language(s) is Machi written in?
+### 3.7.  What language(s) is Machi written in?
-So far, Machi is written in Erlang, mostly.  Machi uses at least one
+So far, Machi is written in 100% Erlang.
 library, [ELevelDB](https://github.com/basho/eleveldb), that is
 implemented both in C++ and in Erlang, using Erlang NIFs (Native
 Interface Functions) to allow Erlang code to call C++ functions.
 In the event that we encounter a performance problem that cannot be
 solved within the Erlang/OTP runtime environment, all of Machi's
@ -587,16 +576,8 @@ in C, Java, or other "gotta go fast fast FAST!!"  programming
 language.  We expect that the Chain Replication manager and other
 critical "control plane" software will remain in Erlang.
-<a name="n3.9">
+<a name="n3.8">
-### 3.9.  Can Machi run on Windows?  Can Machi run on 32-bit platforms?
+### 3.8.  Does Machi use the Erlang/OTP network distribution system (aka "disterl")?
 The ELevelDB NIF does not compile or run correctly on Erlang/OTP
 Windows platforms, nor does it compile correctly on 32-bit platforms.
 Machi should support all 64-bit UNIX-like platforms that are supported
 by Erlang/OTP and ELevelDB.
 <a name="n3.10">
 ### 3.10.  Does Machi use the Erlang/OTP network distribution system (aka "disterl")?
 No, Machi doesn't use Erlang/OTP's built-in distributed message
 passing system.  The code would be *much* simpler if we did use
@ -607,16 +588,19 @@ bit-twiddling magicSPEED ... without also having to find a replacement
 for disterl.  (Or without having to re-invent disterl's features in
 another language.)
-All wire protocols used by Machi are defined & implemented using
+<a name="artisanal-protocol">
-[Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview).
+In the first drafts of the Machi code, the inter-node communication
-The definition file can be found at [./src/machi.proto](./src/machi.proto).
+uses a hand-crafted, artisanal, mostly ASCII protocol as part of a
 "demo day" quick & dirty prototype.  Work is underway (summer of 2015)
 to replace that protocol gradually with a well-structured,
 well-documented protocol based on Protocol Buffers data serialization.
-<a name="n3.11">
+<a name="n3.9">
-### 3.11.  Can I use HTTP to write/read stuff into/from Machi?
+### 3.9.  Can I use HTTP to write/read stuff into/from Machi?
-Short answer: No, not yet.
+Yes, sort of.  For as long as the legacy of
-
+Machi's first internal protocol & code still
-Longer answer: No, but it was possible as a hack, many months ago, see 
+survives, it's possible to use a
 [primitive/hack'y HTTP interface that is described in this source code commit log](https://github.com/basho/machi/commit/6cebf397232cba8e63c5c9a0a8c02ba391b20fef).
 Please note that commit `6cebf397232cba8e63c5c9a0a8c02ba391b20fef` is
 required to try using this feature: the code has since bit-rotted and
--- a/39
+++ b/39
@ -8,9 +8,8 @@ ifeq ($(REBAR),)
 REBAR = $(BASE_DIR)/rebar
 endif
 OVERLAY_VARS    ?=
 EUNIT_OPTS       = -v
-.PHONY: rel stagedevrel deps package pkgclean edoc
+.PHONY: rel deps package pkgclean edoc
 all: deps compile
@ -35,9 +34,6 @@ deps:
 clean:
 	$(REBAR) -r clean
 edoc: edoc-clean
 	$(REBAR) skip_deps=true doc
 edoc-clean:
 	rm -f edoc/*.png edoc/*.html edoc/*.css edoc/edoc-info
@ -57,38 +53,7 @@ relclean:
 stage : rel
 	$(foreach dep,$(wildcard deps/*), rm -rf rel/$(REPO)/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) rel/$(REPO)/lib;)
-##
+DIALYZER_APPS = kernel stdlib sasl erts ssl compiler eunit crypto
 ## Developer targets
 ##
 ##  devN - Make a dev build for node N
 ##  stagedevN - Make a stage dev build for node N (symlink libraries)
 ##  devrel - Make a dev build for 1..$DEVNODES
 ##  stagedevrel Make a stagedev build for 1..$DEVNODES
 ##
 ##  Example, make a 68 node devrel cluster
 ##    make stagedevrel DEVNODES=68
 .PHONY : stagedevrel devrel
 DEVNODES ?= 3
 # 'seq' is not available on all *BSD, so using an alternate in awk
 SEQ = $(shell awk 'BEGIN { for (i = 1; i < '$(DEVNODES)'; i++) printf("%i ", i); print i ;exit(0);}')
 $(eval stagedevrel : $(foreach n,$(SEQ),stagedev$(n)))
 $(eval devrel : $(foreach n,$(SEQ),dev$(n)))
 dev% : all
 	mkdir -p dev
 	rel/gen_dev $@ rel/vars/dev_vars.config.src rel/vars/$@_vars.config
 	(cd rel && ../rebar generate target_dir=../dev/$@ overlay_vars=vars/$@_vars.config)
 stagedev% : dev%
 	  $(foreach dep,$(wildcard deps/*), rm -rf dev/$^/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) dev/$^/lib;)
 devclean: clean
 	rm -rf dev
 DIALYZER_APPS = kernel stdlib sasl erts ssl compiler eunit crypto public_key syntax_tools
 PLT = $(HOME)/.machi_dialyzer_plt
 include tools.mk
--- a/README.md
+++ b/README.md
@ -1,136 +1,59 @@
-# Machi: a distributed, decentralized blob/large file store
+# Machi
  [Travis-CI](http://travis-ci.org/basho/machi) :: ![Travis-CI](https://secure.travis-ci.org/basho/machi.png)
-Outline
+Our goal is a robust & reliable, distributed, highly available(*),
 large file store based upon write-once registers, append-only files,
 Chain Replication, and client-server style architecture.  All members
 of the cluster store all of the files.  Distributed load
 balancing/sharding of files is __outside__ of the scope of this
 system.  However, it is a high priority that this system be able to
 integrate easily into systems that do provide distributed load
 balancing, e.g., Riak Core.  Although strong consistency is a major
 feature of Chain Replication, first use cases will focus mainly on
 eventual consistency features --- strong consistency design will be
 discussed in a separate design document (read more below).
-1. [Why another blob/file store?](#sec1)
+The ability for Machi to maintain strong consistency will make it
-2. [Where to learn more about Machi](#sec2)
+attractive as a toolkit for building things like CORFU and Tango as
-3. [Development status summary](#sec3)
+well as better-known open source software such as Kafka's file
-4. [Contributing to Machi's development](#sec4)
+replication.  (See the bibliography of the [Machi high level design
 doc](./doc/high-level-machi.pdf) for further references.)
-<a name="sec1">
+    (*) When operating in strong consistency mode (supporting
-## 1. Why another blob/file store?
+    sequential or linearizable semantics), the availability of the
    system is restricted to quorum majority availability.  When in
    eventual consistency mode, service can be provided by any
    available server.
-Our goal is a robust & reliable, distributed, highly available, large
+## Status: mid-October 2015: work is underway
 file and blob store.  Such stores already exist, both in the open source world
 and in the commercial world.  Why reinvent the wheel?  We believe
 there are three reasons, ordered by decreasing rarity.
-1. We want end-to-end checksums for all file data, from the initial
+* The chain manager is ready for both eventual consistency use ("AP
-   file writer to every file reader, anywhere, all the time.
+  mode") and strong constency use ("CP mode").  Both modes use a new
-2. We need flexibility to trade consistency for availability:
+  consensus technique, Humming Consensus.
-   e.g. weak consistency in exchange for being available in cases
+    * Scott will be
-   of partial system failure.
+      [speaking about Humming Consensus](http://ricon.io/agenda/#managing-chain-replication-metadata-with-humming-consensus)
-3. We want to manage file replicas in a way that's provably correct
+      at the [Ricon 2015 conference] (http://ricon.io) in San Francisco,
-   and also easy to test.
+      CA, USA on Thursday, November 5th, 2015.
    * Implementation of the file repair process for strong consistency
      is still in progress.
-Criteria #3 is difficult to find in the open source world but perhaps
+* All Machi client/server protocols are based on
-not impossible.
+  [Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview).
    * The current specification for Machi's protocols can be found at
      [https://github.com/basho/machi/blob/master/src/machi.proto](https://github.com/basho/machi/blob/master/src/machi.proto).
    * The Machi PB protocol is not yet stable.  Expect change!
    * The Erlang language client implementation of the high-level
      protocol flavor is brittle (e.g., little error handling yet).
-If we have app use cases where availability is more important than
+If you'd like to work on a protocol such as Thrift, UBF,
-consistency, then systems that meet criteria #2 are also rare.
+msgpack over UDP, or some other protocol, let us know by
-Most file stores provide only strong consistency and therefore
+[opening an issue to discuss it](./issues/new).
 have unavoidable, unavailable behavior when parts of the system
 fail.
 What if we want a file store that is always available to write new
 file data and attempts best-effort file reads?
 If we really do care about data loss and/or data corruption, then we
 really want both #3 and #1.  Unfortunately, systems that meet
 criteria #1 are _very rare_.  (Nonexistant?)
 Why?  This is 2015.  We have decades of research that shows
 that computer hardware can (and
 indeed does) corrupt data at nearly every level of the modern
 client/server application stack.  Systems with end-to-end data
 corruption detection should be ubiquitous today.  Alas, they are not.
 Machi is an effort to change the deplorable state of the world, one
 Erlang function at a time.
 <a name="sec2">
 ## 2. Where to learn more about Machi
 The two major design documents for Machi are now mostly stable.
 Please see the [doc](./doc) directory's [README](./doc) for details.
-We also have a
+## Contributing to Machi: source code, documentation, etc.
 [Frequently Asked Questions (FAQ) list](./FAQ.md).
 Scott recently (November 2015) gave a presentation at the
 [RICON 2015 conference](http://ricon.io) about one of the techniques
 used by Machi; "Managing Chain Replication Metadata with
 Humming Consensus" is available online now.
 * [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf)
 * [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q)
 See later in this document for how to run the Humming Consensus demos,
 including the network partition simulator.
 <a name="sec3">
 ## 3. Development status summary
 Mid-March 2016: The Machi development team has been downsized in
 recent months, and the pace of development has slowed.  Here is a
 summary of the status of Machi's major components.
 * Humming Consensus and the chain manager
  * No new safety bugs have been found by model-checking tests.
  * A new document,
    [Hands-on experiments with Machi and Humming Consensus](doc/humming-consensus-demo.md)
    is now available.  It is a tutorial for setting up a 3 virtual
    machine Machi cluster and how to demonstrate the chain manager's
    reactions to server stops & starts, crashes & restarts, and pauses
    (simulated by `SIGSTOP` and `SIGCONT`).
  * The chain manager can still make suboptimal-but-safe choices for
    chain transitions when a server hangs/pauses temporarily.
    * Recent chain manager changes have made the instability window
      much shorter when the slow/paused server resumes execution.
    * Scott believes that a modest change to the chain manager's
      calculation of a new projection can reduce flapping in this (and
      many other cases) less likely.  Currently, the new local
      projection is calculated using only local state (i.e., the chain
      manager's internal state + the fitness server's state).
      However, if the "latest" projection read from the public
      projection stores were also input to the new projection
      calculation function, then many obviously bad projections can be
      avoided without needing rounds of Humming Consensus to
      demonstrate that a bad projection is bad.
 * FLU/data server process
  * All known correctness bugs have been fixed.
  * Performance has not yet been measured.  Performance measurement
    and enhancements are scheduled to start in the middle of March 2016.
    (This will include a much-needed update to the `basho_bench` driver.)
 * Access protocols and client libraries
  * The protocol used by both external clients and internally (instead
    of using Erlang's native message passing mechanisms) is based on
    Protocol Buffers.
    * (Machi PB protocol specification: ./src/machi.proto)[./src/machi.proto]
    * At the moment, the PB specification contains two protocols.
      Sometime in the near future, the spec will be split to separate
      the external client API (the "high" protocol) from the internal
      communication API (the "low" protocol).
 * Recent conference talks about Machi
  * Erlang Factory San Francisco 2016
    [the slides and video recording](http://www.erlang-factory.com/sfbay2016/scott-lystig-fritchie)
    will be available a few weeks after the conference ends on March
    11, 2016.
  * Ricon 2015
    * [The slides](http://ricon.io/archive/2015/slides/Scott_Fritchie_Ricon_2015.pdf)
    * and the [video recording](https://www.youtube.com/watch?v=yR5kHL1bu1Q&index=13&list=PL9Jh2HsAWHxIc7Tt2M6xez_TOP21GBH6M)
    are now available.
    * If you would like to run the Humming Consensus code (with or without
    the network partition simulator) as described in the RICON 2015
    presentation, please see the
    [Humming Consensus demo doc](./doc/humming_consensus_demo.md).
 <a name="sec4">
 ## 4. Contributing to Machi's development
 ### 4.1 License
 Basho Technologies, Inc. as committed to licensing all work for Machi
 under the
@ -146,29 +69,57 @@ We invite all contributors to review the
 [CONTRIBUTING.md](./CONTRIBUTING.md) document for guidelines for
 working with the Basho development team.
-### 4.2 Development environment requirements
+## A brief survey of this directories in this repository
 * A list of Frequently Asked Questions, a.k.a.
  [the Machi FAQ](./FAQ.md).
 * The [doc](./doc/) directory: home for major documents about Machi:
  high level design documents as well as exploration of features still
  under design & review within Basho.
 * The `ebin` directory: used for compiled application code
 * The `include`, `src`, and `test` directories: contain the header
  files, source files, and test code for Machi, respectively.
 * The [prototype](./prototype/) directory: contains proof of concept
  code, scaffolding libraries, and other exploratory code.  Curious
  readers should see the [prototype/README.md](./prototype/README.md)
  file for more explanation of the small sub-projects found here.
 ## Development environment requirements
 All development to date has been done with Erlang/OTP version 17 on OS
 X.  The only known limitations for using R16 are minor type
 specification difference between R16 and 17, but we strongly suggest
 continuing development using version 17.
-We also assume that you have the standard UNIX/Linux developer
+We also assume that you have the standard UNIX/Linux developers
-tool chain for C and C++ applications.  Also, we assume
+tool chain for C and C++ applications.  Specifically, we assume `make`
-that Git and GNU Make are available.
+is available.  The utility used to compile the Machi source code,
 The utility used to compile the Machi source code,
 `rebar`, is pre-compiled and included in the repo.
 For more details, please see the
 [Machi development environment prerequisites doc](./doc/dev-prerequisites.md).
-Machi has a dependency on the
+There are no known OS limits at this time: any platform that supports
-[ELevelDB](https://github.com/basho/eleveldb) library.  ELevelDB only
+Erlang/OTP should be sufficient for Machi.  This may change over time
-supports UNIX/Linux OSes and 64-bit versions of Erlang/OTP only; we
+(e.g., adding NIFs which can make full portability to Windows OTP
-apologize to Windows-based and 32-bit-based Erlang developers for this
+environments difficult), but it hasn't happened yet.
 restriction.
-### 4.3 New protocols and features
+## Contributions 
 Basho encourages contributions to Riak from the community. Here’s how
 to get started.
 * Fork the appropriate sub-projects that are affected by your change. 
 * Create a topic branch for your change and checkout that branch.
     git checkout -b some-topic-branch
 * Make your changes and run the test suite if one is provided. (see below)
 * Commit your changes and push them to your fork.
 * Open pull-requests for the appropriate projects.
 * Contributors will review your pull request, suggest changes, and merge it when it’s ready and/or offer feedback.
 * To report a bug or issue, please open a new issue against this repository.
 -The Machi team at Basho, 
 [Scott Lystig Fritchie](mailto:scott@basho.com), technical lead, and
 [Matt Brender](mailto:mbrender@basho.com), your developer advocate.
 If you'd like to work on a protocol such as Thrift, UBF,
 msgpack over UDP, or some other protocol, let us know by
 [opening an issue to discuss it](./issues/new).
--- a/dialyzer.ignore-warnings
+++ b/dialyzer.ignore-warnings
@ -3,13 +3,12 @@ machi_pb.erl:0:
 ##################################################
 ########  Specific types     #####################
 ##################################################
 Unknown types:
  basho_bench_config:get/2
  machi_partition_simulator:get/1
  hamcrest:matchspec/0
 ##################################################
 ########  Specific messages  #####################
 ##################################################
 machi_chain_manager1.erl:2473: The created fun has no local return
-machi_chain_manager1.erl:2184: The pattern <_P1, P2, Else = {'expected_author2', UPI1_tail, _}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'>
+machi_chain_manager1.erl:2102: The pattern <_P1, P2, Else = {'expected_author2', UPI1_tail, _}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom() | binary(),all_members::'undefined' | [atom() | binary()],witnesses::[atom() | binary()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom() | binary()],repairing::'undefined' | [atom() | binary()],down::'undefined' | [atom() | binary()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom() | binary(),all_members::'undefined' | [atom() | binary()],witnesses::[atom() | binary()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom() | binary()],repairing::'undefined' | [atom() | binary()],down::'undefined' | [atom() | binary()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'>
-machi_chain_manager1.erl:2233: The pattern <_P1 = {'projection_v1', _, _, _, _, _, _, _, 'cp_mode', UPI1, Repairing1, _, _, _, _}, _P2 = {'projection_v1', _, _, _, _, _, _, _, 'cp_mode', UPI2, Repairing2, _, _, _, _}, Else = {'epoch_not_si', EpochX, 'not_gt', EpochY}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'>
+machi_chain_manager1.erl:2151: The pattern <_P1 = {'projection_v1', _, _, _, _, _, _, 'cp_mode', UPI1, Repairing1, _, _, _, _}, _P2 = {'projection_v1', _, _, _, _, _, _, 'cp_mode', UPI2, Repairing2, _, _, _, _}, Else = {'epoch_not_si', EpochX, 'not_gt', EpochY}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom() | binary(),all_members::'undefined' | [atom() | binary()],witnesses::[atom() | binary()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom() | binary()],repairing::'undefined' | [atom() | binary()],down::'undefined' | [atom() | binary()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom() | binary(),all_members::'undefined' | [atom() | binary()],witnesses::[atom() | binary()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom() | binary()],repairing::'undefined' | [atom() | binary()],down::'undefined' | [atom() | binary()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'>
 machi_flu1.erl:246: The created fun has no local return
--- a/doc/README.md
+++ b/doc/README.md
@ -6,6 +6,20 @@ Erlang documentation, please use this link:
 ## Documents in this directory
 ### chain-self-management-sketch.org
 [chain-self-management-sketch.org](chain-self-management-sketch.org)
 is a mostly-deprecated draft of
 an introduction to the
 self-management algorithm proposed for Machi.  Most material has been
 moved to the [high-level-chain-mgr.pdf](high-level-chain-mgr.pdf) document.
 ### cluster-of-clusters (directory)
 This directory contains the sketch of the "cluster of clusters" design
 strawman for partitioning/distributing/sharding files across a large
 number of independent Machi clusters.
 ### high-level-machi.pdf
 [high-level-machi.pdf](high-level-machi.pdf)
@ -36,9 +50,9 @@ introduction to the Humming Consensus algorithm.  Its abstract:
 > of file updates to all replica servers in a Machi cluster.  Chain
 > Replication is a variation of primary/backup replication where the
 > order of updates between the primary server and each of the backup
-> servers is strictly ordered into a single "chain".  Management of
+> servers is strictly ordered into a single ``chain''.  Management of
-> Chain Replication's metadata, e.g., "What is the current order of
+> Chain Replication's metadata, e.g., ``What is the current order of
-> servers in the chain?", remains an open research problem.  The
+> servers in the chain?'', remains an open research problem.  The
 > current state of the art for Chain Replication metadata management
 > relies on an external oracle (e.g., ZooKeeper) or the Elastic
 > Replication algorithm.
@ -46,7 +60,7 @@ introduction to the Humming Consensus algorithm.  Its abstract:
 > This document describes the Machi chain manager, the component
 > responsible for managing Chain Replication metadata state.  The chain
 > manager uses a new technique, based on a variation of CORFU, called
-> "humming consensus".
+> ``humming consensus''.
 > Humming consensus does not require active participation by all or even
 > a majority of participants to make decisions.  Machi's chain manager
 > bases its logic on humming consensus to make decisions about how to
@ -57,18 +71,3 @@ introduction to the Humming Consensus algorithm.  Its abstract:
 > decision during that epoch.  When a differing decision is discovered,
 > new time epochs are proposed in which a new consensus is reached and
 > disseminated to all available participants.
 ### chain-self-management-sketch.org
 [chain-self-management-sketch.org](chain-self-management-sketch.org)
 is a mostly-deprecated draft of
 an introduction to the
 self-management algorithm proposed for Machi.  Most material has been
 moved to the [high-level-chain-mgr.pdf](high-level-chain-mgr.pdf) document.
 ### cluster (directory)
 This directory contains the sketch of the cluster design
 strawman for partitioning/distributing/sharding files across a large
 number of independent Machi chains.
--- a/doc/cluster-of-clusters/migration-3to4.fig
+++ b/doc/cluster-of-clusters/migration-3to4.fig
@ -88,16 +88,16 @@ Single
 4 0 0 50 -1 2 14 0.0000 4 180 495 4425 3525 ~8%\001
 4 0 0 50 -1 2 14 0.0000 4 240 1710 5025 3525 ~25% total keys\001
 4 0 0 50 -1 2 14 0.0000 4 180 495 6825 3525 ~8%\001
 4 0 0 50 -1 2 24 0.0000 4 270 1485 600 600 Cluster1\001
 4 0 0 50 -1 2 24 0.0000 4 270 1485 3000 600 Cluster2\001
 4 0 0 50 -1 2 24 0.0000 4 270 1485 5400 600 Cluster3\001
 4 0 0 50 -1 2 24 0.0000 4 270 1485 300 2850 Cluster1\001
 4 0 0 50 -1 2 24 0.0000 4 270 1485 2700 2850 Cluster2\001
 4 0 0 50 -1 2 24 0.0000 4 270 1485 5175 2850 Cluster3\001
 4 0 0 50 -1 2 24 0.0000 4 270 405 2100 2625 Cl\001
 4 0 0 50 -1 2 24 0.0000 4 270 405 6900 2625 Cl\001
 4 0 0 50 -1 2 24 0.0000 4 270 195 2175 3075 4\001
 4 0 0 50 -1 2 24 0.0000 4 270 195 4575 3075 4\001
 4 0 0 50 -1 2 24 0.0000 4 270 195 6975 3075 4\001
-4 0 0 50 -1 2 24 0.0000 4 270 1245 600 600 Chain1\001
+4 0 0 50 -1 2 24 0.0000 4 270 405 4500 2625 Cl\001
-4 0 0 50 -1 2 24 0.0000 4 270 1245 3000 600 Chain2\001
+4 0 0 50 -1 2 18 0.0000 4 240 3990 1200 4875 CoC locator, on the unit interval\001
 4 0 0 50 -1 2 24 0.0000 4 270 1245 5400 600 Chain3\001
 4 0 0 50 -1 2 24 0.0000 4 270 285 2100 2625 C\001
 4 0 0 50 -1 2 24 0.0000 4 270 285 4500 2625 C\001
 4 0 0 50 -1 2 24 0.0000 4 270 285 6900 2625 C\001
 4 0 0 50 -1 2 24 0.0000 4 270 1245 525 2850 Chain1\001
 4 0 0 50 -1 2 24 0.0000 4 270 1245 2925 2850 Chain2\001
 4 0 0 50 -1 2 24 0.0000 4 270 1245 5325 2850 Chain3\001
 4 0 0 50 -1 2 18 0.0000 4 240 4350 1350 4875 Cluster locator, on the unit interval\001
--- a/doc/cluster-of-clusters/migration-3to4.png
+++ b/doc/cluster-of-clusters/migration-3to4.png
--- a/doc/cluster-of-clusters/migration-4.png
+++ b/doc/cluster-of-clusters/migration-4.png
--- a/doc/cluster-of-clusters/name-game-sketch.org
+++ b/doc/cluster-of-clusters/name-game-sketch.org
@ -0,0 +1,427 @@
 -*- mode: org; -*-
 #+TITLE: Machi cluster-of-clusters "name game" sketch
 #+AUTHOR: Scott
 #+STARTUP: lognotedone hidestars indent showall inlineimages
 #+SEQ_TODO: TODO WORKING WAITING DONE
 #+COMMENT: M-x visual-line-mode
 #+COMMENT: Also, disable auto-fill-mode
 * 1. "Name Games" with random-slicing style consistent hashing
 Our goal: to distribute lots of files very evenly across a cluster of
 Machi clusters (hereafter called a "cluster of clusters" or "CoC").
 * 2. Assumptions
 ** Basic familiarity with Machi high level design and Machi's "projection"
 The [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] contains all of the basic
 background assumed by the rest of this document.
 ** Analogy: "neighborhood : city :: Machi : cluster-of-clusters"
 Analogy: The word "machi" in Japanese means small town or
 neighborhood.  As the Tokyo Metropolitan Area is built from many
 machis and smaller cities, therefore a big, partitioned file store can
 be built out of many small Machi clusters.
 ** Familiarity with the Machi cluster-of-clusters/CoC concept
 It's clear (I hope!) from
 the [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] that Machi alone does not support
 any kind of file partitioning/distribution/sharding across multiple
 small Machi clusters.  There must be another layer above a Machi cluster to
 provide such partitioning services.
 The name "cluster of clusters" originated within Basho to avoid
 conflicting use of the word "cluster".  A Machi cluster is usually
 synonymous with a single Chain Replication chain and a single set of
 machines (e.g. 2-5 machines).  However, in the not-so-far future, we
 expect much more complicated patterns of Chain Replication to be used
 in real-world deployments.
 "Cluster of clusters" is clunky and long, but we haven't found a good
 substitute yet.  If you have a good suggestion, please contact us!
 ~^_^~
 Using the [[https://github.com/basho/machi/tree/master/prototype/demo-day-hack][cluster-of-clusters quick-and-dirty prototype]] as an
 architecture sketch, let's now assume that we have ~n~ independent Machi
 clusters.  We assume that each of these clusters has roughly the same
 chain length in the nominal case, e.g. chain length of 3.
 We wish to provide partitioned/distributed file storage
 across all ~n~ clusters.  We call the entire collection of ~n~ Machi
 clusters a "cluster of clusters", or abbreviated "CoC".
 We may wish to have several types of Machi clusters, e.g. chain length
 of 3 for normal data, longer for cannot-afford-data-loss files, and
 shorter for don't-care-if-it-gets-lost files.  Each of these types of
 chains will have a name ~N~ in the CoC namespace.  The role of the CoC
 namespace will be demonstrated in Section 3 below.
 ** Continue CoC prototype's assumption: a Machi cluster is unaware of CoC
 Let's continue with an assumption that an individual Machi cluster
 inside of the cluster-of-clusters is completely unaware of the
 cluster-of-clusters layer.
 TODO: We may need to break this assumption sometime in the future?
 ** The reader is familiar with the random slicing technique
 I'd done something very-very-nearly-identical for the Hibari database
 6 years ago.  But the Hibari technique was based on stuff I did at
 Sendmail, Inc, so it felt old news to me.  {shrug}
 The Hibari documentation has a brief photo illustration of how random
 slicing works, see [[http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration][Hibari Sysadmin Guide, chain migration]]
 For a comprehensive description, please see these two papers:
 #+BEGIN_QUOTE
 Reliable and Randomized Data Distribution Strategies for Large Scale Storage Systems
 Alberto Miranda et al.
 http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.226.5609
                                                  (short version, HIPC'11)
 Random Slicing: Efficient and Scalable Data Placement for Large-Scale
    Storage Systems 
 Alberto Miranda et al.
 DOI: http://dx.doi.org/10.1145/2632230 (long version, ACM Transactions
                              on Storage, Vol. 10, No. 3, Article 9, 2014)
 #+END_QUOTE
 ** CoC locator: We borrow from random slicing but do not hash any strings!
 We will use the general technique of random slicing, but we adapt the
 technique to fit our use case.
 In general, random slicing says:
 - Hash a string onto the unit interval [0.0, 1.0)
 - Calculate h(unit interval point, Map) -> bin, where ~Map~ partitions
  the unit interval into bins.
 Our adaptation is in step 1: we do not hash any strings.  Instead, we
 store & use the unit interval point as-is, without using a hash
 function in this step.  This number is called the "CoC locator".
 As described later in this doc, Machi file names are structured into
 several components.  One component of the file name contains the "CoC
 locator"; we use the number as-is for step 2 above.
 * 3. A simple illustration
 We use a variation of the Random Slicing hash that we will call
 ~rs_hash_with_float()~.  The Erlang-style function type is shown
 below.
 #+BEGIN_SRC erlang
 %% type specs, Erlang-style
 -spec rs_hash_with_float(float(), rs_hash:map()) -> rs_hash:cluster_id().
 #+END_SRC
 I'm borrowing an illustration from the HibariDB documentation here,
 but it fits my purposes quite well.  (I am the original creator of that
 image, and also the use license is compatible.)
 #+CAPTION: Illustration of 'Map', using four Machi clusters
 [[./migration-4.png]]
 Assume that we have a random slicing map called ~Map~.  This particular
 ~Map~ maps the unit interval onto 4 Machi clusters:
 | Hash range  | Cluster ID |
 |-------------+------------|
 | 0.00 - 0.25 | Cluster1   |
 | 0.25 - 0.33 | Cluster4   |
 | 0.33 - 0.58 | Cluster2   |
 | 0.58 - 0.66 | Cluster4   |
 | 0.66 - 0.91 | Cluster3   |
 | 0.91 - 1.00 | Cluster4   |
 Assume that the system chooses a CoC locator of 0.05.
 According to ~Map~, the value of
 ~rs_hash_with_float(0.05,Map) = Cluster1~.
 Similarly, ~rs_hash_with_float(0.26,Map) = Cluster4~.
 * 4. An additional assumption: clients will want some control over file location
 We will continue to use the 4-cluster diagram from the previous
 section.
 ** Our new assumption: client control over initial file location
 The CoC management scheme may decide that files need to migrate to
 other clusters.  The reason could be for storage load or I/O load
 balancing reasons.  It could be because a cluster is being
 decommissioned by its owners.  There are many legitimate reasons why a
 file that is initially created on cluster ID X has been moved to
 cluster ID Y.
 However, there are also legitimate reasons for why the client would want
 control over the choice of Machi cluster when the data is first
 written.  The single biggest reason is load balancing.  Assuming that
 the client (or the CoC management layer acting on behalf of the CoC
 client) knows the current utilization across the participating Machi
 clusters, then it may be very helpful to send new append() requests to
 under-utilized clusters.
 * 5. Use of the CoC namespace: name separation plus chain type
 Let us assume that the CoC framework provides several different types
 of chains:
 | Chain length | CoC namespace | Mode | Comment                          |
 |--------------+---------------+------+----------------------------------|
 |            3 | normal        | AP   | Normal storage redundancy & cost |
 |            2 | cheap         | AP   | Reduced cost storage             |
 |            1 | risky         | AP   | Really cheap storage             |
 |            9 | paranoid      | AP   | Safety-critical storage          |
 |            3 | sequential    | CP   | Strong consistency               |
 |--------------+---------------+------+----------------------------------|
 The client may want to choose the amount of redundancy that its
 application requires: normal, reduced cost, or perhaps even a single
 copy.  The CoC namespace is used by the client to signal this
 intention.
 Further, the CoC administrators may wish to use the namespace to
 provide separate storage for different applications.  Jane's
 application may use the namespace "jane-normal" and Bob's app uses
 "bob-cheap".  The CoC administrators may definite separate groups of
 chains on separate servers to serve these two applications.
 * 6. Floating point is not required ... it is merely convenient for explanation
 NOTE: Use of floating point terms is not required.  For example,
 integer arithmetic could be used, if using a sufficiently large
 interval to create an even & smooth distribution of hashes across the
 expected maximum number of clusters.
 For example, if the maximum CoC cluster size would be 4,000 individual
 Machi clusters, then a minimum of 12 bits of integer space is required
 to assign one integer per Machi cluster.  However, for load balancing
 purposes, a finer grain of (for example) 100 integers per Machi
 cluster would permit file migration to move increments of
 approximately 1% of single Machi cluster's storage capacity.  A
 minimum of 12+7=19 bits of hash space would be necessary to accommodate
 these constraints.
 It is likely that Machi's final implementation will choose a 24 bit
 integer to represent the CoC locator.
 * 7. Proposal: Break the opacity of Machi file names
 Machi assigns file names based on:
 ~ClientSuppliedPrefix ++ "^" ++ SomeOpaqueFileNameSuffix~
 What if the CoC client could peek inside of the opaque file name
 suffix in order to remove (or add) the CoC location information that
 we need?
 ** The notation we use
 - ~T~   = the target CoC member/Cluster ID chosen by the CoC client at the time of ~append()~
 - ~p~   = file prefix, chosen by the CoC client.
 - ~L~   = the CoC locator
 - ~N~   = the CoC namespace
 - ~u~ = the Machi file server unique opaque file name suffix, e.g. a GUID string
 - ~F~   = a Machi file name, i.e., ~p^L^N^u~
 ** The details: CoC file write
 1. CoC client chooses ~p~, ~T~, and ~N~ (i.e., the file prefix, target
   cluster, and target cluster namespace)
 2. CoC client knows the CoC ~Map~ for namespace ~N~.
 3. CoC client choose some CoC locator value ~L~ such that
   ~rs_hash_with_float(L,Map) = T~ (see below).
 4. CoC client sends its request to cluster
   ~T~: ~append_chunk(p,L,N,...) -> {ok,p^L^N^u,ByteOffset}~
 5. CoC stores/uses the file name ~F = p^L^N^u~.
 ** The details: CoC file read
 1. CoC client knows the file name ~F~ and parses it to find
   the values of ~L~ and ~N~ (recall, ~F = p^L^N^u~).
 2. CoC client knows the CoC ~Map~ for type ~N~.
 3. CoC calculates ~rs_hash_with_float(L,Map) = T~
 4. CoC client sends request to cluster ~T~: ~read_chunk(F,...) ->~ ... success!
 ** The details: calculating 'L' (the CoC locator) to match a desired target cluster
 1. We know ~Map~, the current CoC mapping for a CoC namespace ~N~.
 2. We look inside of ~Map~, and we find all of the unit interval ranges
   that map to our desired target cluster ~T~.  Let's call this list
   ~MapList = [Range1=(start,end],Range2=(start,end],...]~.
 3. In our example, ~T=Cluster2~.  The example ~Map~ contains a single
   unit interval range for ~Cluster2~, ~[(0.33,0.58]]~.
 4. Choose a uniformly random number ~r~ on the unit interval.
 5. Calculate locator ~L~ by mapping ~r~ onto the concatenation
   of the CoC hash space range intervals in ~MapList~.  For example,
   if ~r=0.5~, then ~L = 0.33 + 0.5*(0.58-0.33) = 0.455~, which is
   exactly in the middle of the ~(0.33,0.58]~ interval.
 * 8. File migration (a.k.a. rebalancing/reparitioning/resharding/redistribution)
 ** What is "migration"?
 This section describes Machi's file migration.  Other storage systems
 call this process as "rebalancing", "repartitioning", "resharding" or
 "redistribution".
 For Riak Core applications, it is called "handoff" and "ring resizing"
 (depending on the context).
 See also the [[http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsUserGuide.html#Balancer][Hadoop file balancer]] for another example of a data
 migration process.
 As discussed in section 5, the client can have good reason for wanting
 to have some control of the initial location of the file within the
 cluster.  However, the cluster manager has an ongoing interest in
 balancing resources throughout the lifetime of the file.  Disks will
 get full, hardware will change, read workload will fluctuate,
 etc etc.
 This document uses the word "migration" to describe moving data from
 one Machi chain to another within a CoC system.
 A simple variation of the Random Slicing hash algorithm can easily
 accommodate Machi's need to migrate files without interfering with
 availability.  Machi's migration task is much simpler due to the
 immutable nature of Machi file data.
 ** Change to Random Slicing
 The map used by the Random Slicing hash algorithm needs a few simple
 changes to make file migration straightforward.
 - Add a "generation number", a strictly increasing number (similar to
  a Machi cluster's "epoch number") that reflects the history of
  changes made to the Random Slicing map
 - Use a list of Random Slicing maps instead of a single map, one map
  per chance that files may not have been migrated yet out of
  that map.
 As an example:
 #+CAPTION: Illustration of 'Map', using four Machi clusters
 [[./migration-3to4.png]]
 And the new Random Slicing map for some CoC namespace ~N~ might look
 like this:
 | Generation number / Namespace | 7 / cheap  |
 |-------------------------------+------------|
 | SubMap                        | 1          |
 |-------------------------------+------------|
 | Hash range                    | Cluster ID |
 |-------------------------------+------------|
 | 0.00 - 0.33                   | Cluster1   |
 | 0.33 - 0.66                   | Cluster2   |
 | 0.66 - 1.00                   | Cluster3   |
 |-------------------------------+------------|
 | SubMap                        | 2          |
 |-------------------------------+------------|
 | Hash range                    | Cluster ID |
 |-------------------------------+------------|
 | 0.00 - 0.25                   | Cluster1   |
 | 0.25 - 0.33                   | Cluster4   |
 | 0.33 - 0.58                   | Cluster2   |
 | 0.58 - 0.66                   | Cluster4   |
 | 0.66 - 0.91                   | Cluster3   |
 | 0.91 - 1.00                   | Cluster4   |
 When a new Random Slicing map contains a single submap, then its use
 is identical to the original Random Slicing algorithm.  If the map
 contains multiple submaps, then the access rules change a bit:
 - Write operations always go to the newest/largest submap.
 - Read operations attempt to read from all unique submaps.
  - Skip searching submaps that refer to the same cluster ID.
    - In this example, unit interval value 0.10 is mapped to Cluster1
      by both submaps.
  - Read from newest/largest submap to oldest/smallest submap.
  - If not found in any submap, search a second time (to handle races
    with file copying between submaps).
  - If the requested data is found, optionally copy it directly to the
    newest submap.   (This is a variation of read repair (RR). RR here
    accelerates the migration process and can reduce the number of
    operations required to query servers in multiple submaps).
 The cluster-of-clusters manager is responsible for:
 - Managing the various generations of the CoC Random Slicing maps for
  all namespaces.
 - Distributing namespace maps to CoC clients.
 - Managing the processes that are responsible for copying "cold" data,
  i.e., files data that is not regularly accessed, to its new submap
  location.
 - When migration of a file to its new cluster is confirmed successful,
  delete it from the old cluster.
 In example map #7, the CoC manager will copy files with unit interval
 assignments in ~(0.25,0.33]~, ~(0.58,0.66]~, and ~(0.91,1.00]~ from their
 old locations in cluster IDs Cluster1/2/3 to their new cluster,
 Cluster4.  When the CoC manager is satisfied that all such files have
 been copied to Cluster4, then the CoC manager can create and
 distribute a new map, such as:
 | Generation number / Namespace | 8 / cheap  |
 |-------------------------------+------------|
 | SubMap                        | 1          |
 |-------------------------------+------------|
 | Hash range                    | Cluster ID |
 |-------------------------------+------------|
 | 0.00 - 0.25                   | Cluster1   |
 | 0.25 - 0.33                   | Cluster4   |
 | 0.33 - 0.58                   | Cluster2   |
 | 0.58 - 0.66                   | Cluster4   |
 | 0.66 - 0.91                   | Cluster3   |
 | 0.91 - 1.00                   | Cluster4   |
 The HibariDB system performs data migrations in almost exactly this
 manner.  However, one important
 limitation of HibariDB is not being able to
 perform more than one migration at a time.  HibariDB's data is
 mutable, and mutation causes many problems already when migrating data
 across two submaps; three or more submaps was too complex to implement
 quickly.
 Fortunately for Machi, its file data is immutable and therefore can
 easily manage many migrations in parallel, i.e., its submap list may
 be several maps long, each one for an in-progress file migration.
 * 9. Other considerations for FLU/sequencer implementations
 ** Append to existing file when possible
 In the earliest Machi FLU implementation, it was impossible to append
 to the same file after ~30 seconds.  For example:
 - Client: ~append(prefix="foo",...) -> {ok,"foo^suffix1",Offset1}~
 - Client: ~append(prefix="foo",...) -> {ok,"foo^suffix1",Offset2}~
 - Client: ~append(prefix="foo",...) -> {ok,"foo^suffix1",Offset3}~
 - Client: sleep 40 seconds
 - Server: after 30 seconds idle time, stop Erlang server process for
  the ~"foo^suffix1"~ file
 - Client: ...wakes up...
 - Client: ~append(prefix="foo",...) -> {ok,"foo^suffix2",Offset4}~
 Our ideal append behavior is to always append to the same file.  Why?
 It would be nice if Machi didn't create zillions of tiny files if the
 client appends to some prefix very infrequently.  In general, it is
 better to create fewer & bigger files by re-using a Machi file name
 when possible.
 The sequencer should always assign new offsets to the latest/newest
 file for any prefix, as long as all prerequisites are also true,
 - The epoch has not changed.  (In AP mode, epoch change -> mandatory file name suffix change.)
 - The latest file for prefix ~p~ is smaller than maximum file size for a FLU's configuration.
 * 10. Acknowledgments
 The source for the "migration-4.png" and "migration-3to4.png" images
 come from the [[http://hibari.github.io/hibari-doc/images/migration-3to4.png][HibariDB documentation]].
--- a/doc/cluster/migration-3to4.png
+++ b/doc/cluster/migration-3to4.png
--- a/doc/cluster/migration-4.png
+++ b/doc/cluster/migration-4.png
--- a/doc/cluster/name-game-sketch.org
+++ b/doc/cluster/name-game-sketch.org
@ -1,481 +0,0 @@
 -*- mode: org; -*-
 #+TITLE: Machi cluster "name game" sketch
 #+AUTHOR: Scott
 #+STARTUP: lognotedone hidestars indent showall inlineimages
 #+SEQ_TODO: TODO WORKING WAITING DONE
 #+COMMENT: M-x visual-line-mode
 #+COMMENT: Also, disable auto-fill-mode
 * 1. "Name Games" with random-slicing style consistent hashing
 Our goal: to distribute lots of files very evenly across a large
 collection of individual, small Machi chains.
 * 2. Assumptions
 ** Basic familiarity with Machi high level design and Machi's "projection"
 The [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] contains all of the basic
 background assumed by the rest of this document.
 ** Analogy: "neighborhood : city :: Machi chain : Machi cluster"
 Analogy: The word "machi" in Japanese means small town or
 neighborhood.  As the Tokyo Metropolitan Area is built from many
 machis and smaller cities, therefore a big, partitioned file store can
 be built out of many small Machi chains.
 ** Familiarity with the Machi chain concept
 It's clear (I hope!) from
 the [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] that Machi alone does not support
 any kind of file partitioning/distribution/sharding across multiple
 small Machi chains.  There must be another layer above a Machi chain to
 provide such partitioning services.
 Using the [[https://github.com/basho/machi/tree/master/prototype/demo-day-hack][cluster quick-and-dirty prototype]] as an
 architecture sketch, let's now assume that we have ~n~ independent Machi
 chains.  We assume that each of these chains has the same
 chain length in the nominal case, e.g. chain length of 3.
 We wish to provide partitioned/distributed file storage
 across all ~n~ chains.  We call the entire collection of ~n~ Machi
 chains a "cluster".
 We may wish to have several types of Machi clusters.  For example:
 + Chain length of 1 for "don't care if it gets lost,
  store stuff very very cheaply" data.
 + Chain length of 2 for normal data.
  + Equivalent to quorum replication's reliability with 3 copies.
 + Chain length of 7 for critical, unreplaceable data.
  + Equivalent to quorum replication's reliability with 15 copies.
 Each of these types of chains will have a name ~N~ in the
 namespace.  The role of the cluster namespace will be demonstrated in
 Section 3 below.
 ** Continue an early assumption: a Machi chain is unaware of clustering
 Let's continue with an assumption that an individual Machi chain
 inside of a cluster is completely unaware of the cluster layer.
 ** The reader is familiar with the random slicing technique
 I'd done something very-very-nearly-like-this for the Hibari database
 6 years ago.  But the Hibari technique was based on stuff I did at
 Sendmail, Inc, in 2000, so this technique feels like old news to me.
 {shrug}
 The following section provides an illustrated example.
 Very quickly, the random slicing algorithm is:
 - Hash a string onto the unit interval [0.0, 1.0)
 - Calculate h(unit interval point, Map) -> bin, where ~Map~ divides
  the unit interval into bins (or partitions or shards).
 Machi's adaptation is in step 1: we do not hash any strings.  Instead, we
 simply choose a number on the unit interval.  This number is called
 the "cluster locator number".
 As described later in this doc, Machi file names are structured into
 several components.  One component of the file name contains the cluster
 locator number; we use the number as-is for step 2 above.
 *** For more information about Random Slicing
 For a comprehensive description of random slicing, please see the
 first two papers.  For a quicker summary, please see the third
 reference.
 #+BEGIN_QUOTE
 Reliable and Randomized Data Distribution Strategies for Large Scale Storage Systems
 Alberto Miranda et al.
 http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.226.5609
                                                  (short version, HIPC'11)
 Random Slicing: Efficient and Scalable Data Placement for Large-Scale
    Storage Systems 
 Alberto Miranda et al.
 DOI: http://dx.doi.org/10.1145/2632230 (long version, ACM Transactions
                              on Storage, Vol. 10, No. 3, Article 9, 2014)
 [[http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration][Hibari Sysadmin Guide, chain migration section]].
 http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration
 #+END_QUOTE
 * 3. A simple illustration
 We use a variation of the Random Slicing hash that we will call
 ~rs_hash_with_float()~.  The Erlang-style function type is shown
 below.
 #+BEGIN_SRC erlang
 %% type specs, Erlang-style
 -spec rs_hash_with_float(float(), rs_hash:map()) -> rs_hash:chain_id().
 #+END_SRC
 I'm borrowing an illustration from the HibariDB documentation here,
 but it fits my purposes quite well.  (I am the original creator of that
 image, and also the use license is compatible.)
 #+CAPTION: Illustration of 'Map', using four Machi chains
 [[./migration-4.png]]
 Assume that we have a random slicing map called ~Map~.  This particular
 ~Map~ maps the unit interval onto 4 Machi chains:
 | Hash range  | Chain ID |
 |-------------+----------|
 | 0.00 - 0.25 | Chain1   |
 | 0.25 - 0.33 | Chain4   |
 | 0.33 - 0.58 | Chain2   |
 | 0.58 - 0.66 | Chain4   |
 | 0.66 - 0.91 | Chain3   |
 | 0.91 - 1.00 | Chain4   |
 Assume that the system chooses a cluster locator of 0.05.
 According to ~Map~, the value of
 ~rs_hash_with_float(0.05,Map) = Chain1~.
 Similarly, ~rs_hash_with_float(0.26,Map) = Chain4~.
 This example should look very similar to Hibari's technique.
 The Hibari documentation has a brief photo illustration of how random
 slicing works, see [[http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration][Hibari Sysadmin Guide, chain migration]].
 * 4. Use of the cluster namespace: name separation plus chain type
 Let us assume that the cluster framework provides several different types
 of chains:
 | Chain length | Namespace    | Consistency Mode | Comment                          |
 |--------------+--------------+------------------+----------------------------------|
 |            3 | ~normal~     | eventual         | Normal storage redundancy & cost |
 |            2 | ~reduced~    | eventual         | Reduced cost storage             |
 |            1 | ~risky~      | eventual         | Really, really cheap storage     |
 |            7 | ~paranoid~   | eventual         | Safety-critical storage          |
 |            3 | ~sequential~ | strong           | Strong consistency               |
 |--------------+--------------+------------------+----------------------------------|
 The client may want to choose the amount of redundancy that its
 application requires: normal, reduced cost, or perhaps even a single
 copy.  The cluster namespace is used by the client to signal this
 intention.
 Further, the cluster administrators may wish to use the namespace to
 provide separate storage for different applications.  Jane's
 application may use the namespace "jane-normal" and Bob's app uses
 "bob-reduced".  Administrators may definine separate groups of
 chains on separate servers to serve these two applications.
 * 5. In its lifetime, a file may be moved to different chains
 The cluster management scheme may decide that files need to migrate to
 other chains -- i.e., file that is initially created on chain ID ~X~
 has been moved to chain ID ~Y~.
 + For storage load or I/O load balancing reasons.
 + Because a chain is being decommissioned by the sysadmin.
 * 6. Floating point is not required ... it is merely convenient for explanation
 NOTE: Use of floating point terms is not required.  For example,
 integer arithmetic could be used, if using a sufficiently large
 interval to create an even & smooth distribution of hashes across the
 expected maximum number of chains.
 For example, if the maximum cluster size would be 4,000 individual
 Machi chains, then a minimum of 12 bits of integer space is required
 to assign one integer per Machi chain.  However, for load balancing
 purposes, a finer grain of (for example) 100 integers per Machi
 chain would permit file migration to move increments of
 approximately 1% of single Machi chain's storage capacity.  A
 minimum of 12+7=19 bits of hash space would be necessary to accommodate
 these constraints.
 It is likely that Machi's final implementation will choose a 24 bit
 integer (or perhaps 32 bits) to represent the cluster locator.
 * 7. Proposal: Break the opacity of Machi file names, slightly.
 Machi assigns file names based on:
 ~ClientSuppliedPrefix ++ "^" ++ SomeOpaqueFileNameSuffix~
 What if some parts of the system could peek inside of the opaque file name
 suffix in order to look at the cluster location information that we might
 code in the filename suffix?
 We break the system into parts that speak two levels of protocols,
 "high" and "low".
 + The high level protocol is used outside of the Machi cluster
 + The low level protocol is used inside of the Machi cluster
 Both protocols are based on a Protocol Buffers specification and
 implementation.  Other protocols, such as HTTP, will be added later.
 #+BEGIN_SRC
     +-----------------------+
     | Machi external client |
     | e.g. Riak CS          |
     +-----------------------+
          ^
          | Machi "high" API
          | ProtoBuffs protocol     Machi cluster boundary: outside
 .........................................................................
          |                         Machi cluster boundary: inside
          v
     +--------------------------+    +------------------------+
     | Machi "high" API service |    | Machi HTTP API service |
     +--------------------------+    +------------------------+
          ^                                       |
          |              +------------------------+
          v              v
     +------------------------+
     | Cluster bridge service |
     +------------------------+
          ^
          | Machi "low" API
          | ProtoBuffs protocol                                             
          +----------------------------------------+----+----+
          |                                        |    |    |  
          v                                        v    v    v  
       +-------------------------+              ... other chains...
       | Chain C1 (logical view) |
       |  +--------------+       |  
       |  | FLU server 1 |       |  
       |  |  +--------------+    |  
       |  +--| FLU server 2 |    |  
       |     +--------------+    |  In reality, API bridge talks directly
       +-------------------------+  to each FLU server in a chain.       
 #+END_SRC
 ** The notation we use
 - ~N~   = the cluster namespace, chosen by the client.
 - ~p~   = file prefix, chosen by the client.
 - ~L~   = the cluster locator (a number, type is implementation-dependent)
 - ~Map~ = a mapping of cluster locators to chains
 - ~T~   = the target chain ID/name
 - ~u~   = a unique opaque file name suffix, e.g. a GUID string
 - ~F~   = a Machi file name, i.e., a concatenation of ~p^L^N^u~
 ** The details: cluster file append
 0. Cluster client chooses ~N~ and ~p~ (i.e., cluster namespace and
   file prefix) and sends the append request to a Machi cluster member
   via the Protocol Buffers "high" API.
 1. Cluster bridge chooses ~T~ (i.e., target chain), based on criteria
   such as disk utilization percentage.
 2. Cluster bridge knows the cluster ~Map~ for namespace ~N~.
 3. Cluster bridge choose some cluster locator value ~L~ such that
   ~rs_hash_with_float(L,Map) = T~ (see algorithm below).
 4. Cluster bridge sends its request to chain
   ~T~: ~append_chunk(p,L,N,...) -> {ok,p^L^N^u,ByteOffset}~
 5. Cluster bridge forwards the reply tuple to the client.
 6. Client stores/uses the file name ~F = p^L^N^u~.
 ** The details: Cluster file read
 0. Cluster client sends the read request to a Machi cluster member via
   the Protocol Buffers "high" API.
 1. Cluster bridge parses the file name ~F~  to find
   the values of ~L~ and ~N~ (recall, ~F = p^L^N^u~).
 2. Cluster bridge knows the Cluster ~Map~ for type ~N~.
 3. Cluster bridge calculates ~rs_hash_with_float(L,Map) = T~
 4. Cluster bridge sends request to chain ~T~:
   ~read_chunk(F,...) ->~ ... reply
 5. Cluster bridge forwards the reply to the client.
 ** The details: calculating 'L' (the cluster locator number) to match a desired target chain
 1. We know ~Map~, the current cluster mapping for a cluster namespace ~N~.
 2. We look inside of ~Map~, and we find all of the unit interval ranges
   that map to our desired target chain ~T~.  Let's call this list
   ~MapList = [Range1=(start,end],Range2=(start,end],...]~.
 3. In our example, ~T=Chain2~.  The example ~Map~ contains a single
   unit interval range for ~Chain2~, ~[(0.33,0.58]]~.
 4. Choose a uniformly random number ~r~ on the unit interval.
 5. Calculate the cluster locator ~L~ by mapping ~r~ onto the concatenation
   of the cluster hash space range intervals in ~MapList~.  For example,
   if ~r=0.5~, then ~L = 0.33 + 0.5*(0.58-0.33) = 0.455~, which is
   exactly in the middle of the ~(0.33,0.58]~ interval.
 ** A bit more about the cluster namespaces's meaning and use
 For use by Riak CS, for example, we'd likely start with the following
 namespaces ... working our way down the list as we add new features
 and/or re-implement existing CS features.
 - "standard" = Chain length = 3, eventually consistency mode
 - "reduced" = Chain length = 2, eventually consistency mode.
 - "stanchion7" = Chain length = 7, strong consistency mode.  Perhaps
  use this namespace for the metadata required to re-implement the
  operations that are performed by today's Stanchion application.
 We want the cluster framework to:
 - provide means of creating and managing
  chains of different types, e.g., chain length, consistency mode.
 - manage the mapping of cluster namespace
  names to the chains in the system.
 - provide query functions to map a cluster
  namespace name to a cluster map,
  e.g. ~get_cluster_latest_map("reduced") -> Map{generation=7,...}~.
 * 8. File migration (a.k.a. rebalancing/reparitioning/resharding/redistribution)
 ** What is "migration"?
 This section describes Machi's file migration.  Other storage systems
 call this process as "rebalancing", "repartitioning", "resharding" or
 "redistribution".
 For Riak Core applications, it is called "handoff" and "ring resizing"
 (depending on the context).
 See also the [[http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsUserGuide.html#Balancer][Hadoop file balancer]] for another example of a data
 migration process.
 As discussed in section 5, the client can have good reason for wanting
 to have some control of the initial location of the file within the
 chain.  However, the chain manager has an ongoing interest in
 balancing resources throughout the lifetime of the file.  Disks will
 get full, hardware will change, read workload will fluctuate,
 etc etc.
 This document uses the word "migration" to describe moving data from
 one Machi chain to another chain within a cluster system.
 A simple variation of the Random Slicing hash algorithm can easily
 accommodate Machi's need to migrate files without interfering with
 availability.  Machi's migration task is much simpler due to the
 immutable nature of Machi file data.
 ** Change to Random Slicing
 The map used by the Random Slicing hash algorithm needs a few simple
 changes to make file migration straightforward.
 - Add a "generation number", a strictly increasing number (similar to
  a Machi chain's "epoch number") that reflects the history of
  changes made to the Random Slicing map
 - Use a list of Random Slicing maps instead of a single map, one map
  per chance that files may not have been migrated yet out of
  that map.
 As an example:
 #+CAPTION: Illustration of 'Map', using four Machi chains
 [[./migration-3to4.png]]
 And the new Random Slicing map for some cluster namespace ~N~ might look
 like this:
 | Generation number / Namespace | 7 / reduced |
 |-------------------------------+-------------|
 | SubMap                        | 1           |
 |-------------------------------+-------------|
 | Hash range                    | Chain ID    |
 |-------------------------------+-------------|
 | 0.00 - 0.33                   | Chain1      |
 | 0.33 - 0.66                   | Chain2      |
 | 0.66 - 1.00                   | Chain3      |
 |-------------------------------+-------------|
 | SubMap                        | 2           |
 |-------------------------------+-------------|
 | Hash range                    | Chain ID    |
 |-------------------------------+-------------|
 | 0.00 - 0.25                   | Chain1      |
 | 0.25 - 0.33                   | Chain4      |
 | 0.33 - 0.58                   | Chain2      |
 | 0.58 - 0.66                   | Chain4      |
 | 0.66 - 0.91                   | Chain3      |
 | 0.91 - 1.00                   | Chain4      |
 When a new Random Slicing map contains a single submap, then its use
 is identical to the original Random Slicing algorithm.  If the map
 contains multiple submaps, then the access rules change a bit:
 - Write operations always go to the newest/largest submap.
 - Read operations attempt to read from all unique submaps.
  - Skip searching submaps that refer to the same chain ID.
    - In this example, unit interval value 0.10 is mapped to Chain1
      by both submaps.
  - Read from newest/largest submap to oldest/smallest submap.
  - If not found in any submap, search a second time (to handle races
    with file copying between submaps).
  - If the requested data is found, optionally copy it directly to the
    newest submap.   (This is a variation of read repair (RR). RR here
    accelerates the migration process and can reduce the number of
    operations required to query servers in multiple submaps).
 The cluster manager is responsible for:
 - Managing the various generations of the cluster Random Slicing maps for
  all namespaces.
 - Distributing namespace maps to cluster bridges.
 - Managing the processes that are responsible for copying "cold" data,
  i.e., files data that is not regularly accessed, to its new submap
  location.
 - When migration of a file to its new chain is confirmed successful,
  delete it from the old chain.
 In example map #7, the cluster manager will copy files with unit interval
 assignments in ~(0.25,0.33]~, ~(0.58,0.66]~, and ~(0.91,1.00]~ from their
 old locations in chain IDs Chain1/2/3 to their new chain,
 Chain4.  When the cluster manager is satisfied that all such files have
 been copied to Chain4, then the cluster manager can create and
 distribute a new map, such as:
 | Generation number / Namespace | 8 / reduced |
 |-------------------------------+-------------|
 | SubMap                        | 1           |
 |-------------------------------+-------------|
 | Hash range                    | Chain ID    |
 |-------------------------------+-------------|
 | 0.00 - 0.25                   | Chain1      |
 | 0.25 - 0.33                   | Chain4      |
 | 0.33 - 0.58                   | Chain2      |
 | 0.58 - 0.66                   | Chain4      |
 | 0.66 - 0.91                   | Chain3      |
 | 0.91 - 1.00                   | Chain4      |
 The HibariDB system performs data migrations in almost exactly this
 manner.  However, one important
 limitation of HibariDB is not being able to
 perform more than one migration at a time.  HibariDB's data is
 mutable.  Mutation causes many problems when migrating data
 across two submaps; three or more submaps was too complex to implement
 quickly and correctly.
 Fortunately for Machi, its file data is immutable and therefore can
 easily manage many migrations in parallel, i.e., its submap list may
 be several maps long, each one for an in-progress file migration.
 * 9. Other considerations for FLU/sequencer implementations
 ** Append to existing file when possible
 The sequencer should always assign new offsets to the latest/newest
 file for any prefix, as long as all prerequisites are also true,
 - The epoch has not changed.  (In AP mode, epoch change -> mandatory
  file name suffix change.)
 - The cluster locator number is stable.
 - The latest file for prefix ~p~ is smaller than maximum file size for
  a FLU's configuration.
 The stability of the cluster locator number is an implementation detail that
 must be managed by the cluster bridge.
 Reuse of the same file is not possible if the bridge always chooses a
 different cluster locator number ~L~ or if the client always uses a unique
 file prefix ~p~.  The latter is a sign of a misbehaved client; the
 former is a poorly-implemented bridge.
 * 10. Acknowledgments
 The original source for the "migration-4.png" and "migration-3to4.png" images
 come from the [[http://hibari.github.io/hibari-doc/images/migration-3to4.png][HibariDB documentation]].
--- a/doc/dev-clone-compile.md
+++ b/doc/dev-clone-compile.md
@ -1,30 +0,0 @@
 # Clone and compile Machi
 Clone the Machi source repo and compile the source and test code.  Run
 the following commands at your login shell:
    cd /tmp
    git clone https://github.com/basho/machi.git
    cd machi
    git checkout master
    make          # or 'gmake' if GNU make uses an alternate name
 Then run the unit test suite.  This may take up to two minutes or so
 to finish.
    make test
 At the end, the test suite should report that all tests passed.  The
 actual number of tests shown in the "All `X` tests passed" line may be
 different than the example below.
    [... many lines omitted ...]
    module 'event_logger'
    module 'chain_mgr_legacy'
    =======================================================
      All 90 tests passed.
 If you had a test failure, a likely cause may be a limit on the number
 of file descriptors available to your user process.  (Recent releases
 of OS X have a limit of 1024 file descriptors, which may be too slow.)
 The output of the `limit -n` will tell you your file descriptor limit.
--- a/doc/dev-prerequisites.md
+++ b/doc/dev-prerequisites.md
@ -1,38 +0,0 @@
 ## Machi developer environment prerequisites
 1. Machi requires an 64-bit variant of UNIX: OS X, FreeBSD, Linux, or
   Solaris machine is a standard developer environment for C and C++
   applications (64-bit versions).
 2. You'll need the `git` source management utility.
 3. You'll need the 64-bit Erlang/OTP 17 runtime environment.  Please
   don't use earlier or later versions until we have a chance to fix
   the compilation warnings that versions R16B and 18 will trigger.
   Also, please verify that you are not using a 32-bit Erlang/OTP
   runtime package.
 For `git` and the Erlang runtime, please use your OS-specific
 package manager to install these.  If your package manager doesn't
 have 64-bit Erlang/OTP version 17 available, then we recommend using the
 [precompiled packages available at Erlang Solutions](https://www.erlang-solutions.com/resources/download.html).
 Also, please verify that you have enough file descriptors available to
 your user processes.  The output of `ulimit -n` should report at least
 4,000 file descriptors available.  If your limit is lower (a frequent
 problem for OS X users), please increase it to at least 4,000.
 # Using Vagrant to set up a developer environment for Machi
 The Machi source directory contains a `Vagrantfile` for creating an
 Ubuntu Linux-based virtual machine for compiling and running Machi.
 This file is in the
 [$SRC_TOP/priv/humming-consensus-demo.vagrant](../priv/humming-consensus-demo.vagrant)
 directory.
 If used as-is, the virtual machine specification is modest.
 * 1 virtual CPU
 * 512MB virtual memory
 * 768MB swap space
 * 79GB sparse virtual disk image.  After installing prerequisites and
  compiling Machi, the root file system uses approximately 2.7 GBytes.
--- a/doc/flu-and-chain-lifecycle.org
+++ b/doc/flu-and-chain-lifecycle.org
@ -1,617 +0,0 @@
 FLU and Chain Life Cycle Management -*- mode: org; -*-
 #+STARTUP: lognotedone hidestars indent showall inlineimages
 #+COMMENT: To generate the outline section: egrep '^\*[*]* ' doc/flu-and-chain-lifecycle.org | egrep -v '^\* Outline' | sed -e 's/^\*\*\* /    + /' -e 's/^\*\* /  + /' -e 's/^\* /+ /'
 * FLU and Chain Life Cycle Management
 In an ideal world, we (the Machi development team) would have a full
 vision of how Machi would be managed, down to the last detail of
 beautiful CLI character and network protocol bit.  Our vision isn't
 complete yet, so we are working one small step at a time.
 * Outline
 + FLU and Chain Life Cycle Management
 + Terminology review
  + Terminology: Machi run-time components/services/thingies
  + Terminology: Machi chain data structures
  + Terminology: Machi cluster data structures
 + Overview of administrative life cycles
  + Cluster administrative life cycle
  + Chain administrative life cycle
  + FLU server administrative life cycle
 + Quick admin: declarative management of Machi FLU and chain life cycles
  + Quick admin uses the "rc.d" config scheme for life cycle management
  + Quick admin's declarative "language": an Erlang-flavored AST
    + Term 'host': define a new host for FLU services
    + Term 'flu': define a new FLU
    + Term 'chain': define or reconfigure a chain
  + Executing quick admin AST files via the 'machi-admin' utility
    + Checking the syntax of an AST file
    + Executing an AST file
  + Using quick admin to manage multiple machines
 + The "rc.d" style configuration file scheme
  + Riak had a similar configuration file editing problem (and its solution)
  + Machi's "rc.d" file scheme.
  + FLU life cycle management using "rc.d" style files
    + The key configuration components of a FLU
  + Chain life cycle management using "rc.d" style files
    + The key configuration components of a chain
 * Terminology review
 ** Terminology: Machi run-time components/services/thingies
 + FLU: a basic Machi server, responsible for managing a collection of
  files.
 + Chain: a small collection of FLUs that maintain replicas of the same
  collection of files.  A chain is usually small, 1-3 servers, where
  more than 3 would be used only in cases when availability of
  certain data is critical despite failures of several machines.
  + The length of a chain is directly proportional to its
    replication factor, e.g., a chain length=3 will maintain
    (nominally) 3 replicas of each file.
  + To maintain file availability when ~F~ failures have occurred, a
    chain must be at least ~F+1~ members long.  (In comparison, the
    quorum replication technique requires ~2F+1~ members in the
    general case.)
 + Cluster: A collection of Machi chains that are used to store files
  in a horizontally partitioned/sharded/distributed manner.
 ** Terminology: Machi data structures
 + Projection: used to define a single chain: the chain's consistency
  mode (strong or eventual consistency), all members (from an
  administrative point of view), all active members (from a runtime,
  automatically-managed point of view), repairing/file-syncing members
  (also runtime, auto-managed), and so on
 + Epoch: A version number of a projection.  The epoch number is used
  by both clients & servers to manage transitions from one projection
  to another, e.g., when the chain is temporarily shortened by the
  failure of a member FLU server.
 ** Terminology: Machi cluster data structures
 + Namespace: A collection of human-friendly names that are mapped to
  groups of Machi chains that provide the same type of storage
  service: consistency mode, replication policy, etc.
  + A single namespace name, e.g. ~normal-ec~, is paired with a single
    cluster map (see below).
  + Example: ~normal-ec~ might be a collection of Machi chains in
    eventually-consistent mode that are of length=3.
  + Example: ~risky-ec~ might be a collection of Machi chains in
    eventually-consistent mode that are of length=1.
  + Example: ~mgmt-critical~ might be a collection of Machi chains in
    strongly-consistent mode that are of length=7.
 + Cluster map: Encodes the rules which partition/shard/distribute
  the files stored in a particular namespace across a group of chains
  that collectively store the namespace's files.
 + Chain weight: A value assigned to each chain within a cluster map
  structure that defines the relative storage capacity of a chain
  within the namespace.  For example, a chain weight=150 has 50% more
  capacity than a chain weight=100.
 + Cluster map epoch: The version number assigned to a cluster map.
 * Overview of administrative life cycles
 ** Cluster administrative life cycle
 + Cluster is first created
 + Adds namespaces (e.g. consistency policy + chain length policy) to
  the cluster
 + Chains are added to/removed from a namespace to increase/decrease the
  namespace's storage capacity.
 + Adjust chain weights within a namespace, e.g., to shift files
  within the namespace to chains with greater storage capacity
  resources and/or runtime I/O resources.
 A cluster "file migration" is the process of moving files from one
 namespace member chain to another for purposes of shifting &
 re-balancing storage capacity and/or runtime I/O capacity.
 ** Chain administrative life cycle
 + A chain is created with an initial FLU membership list.
 + Chain may be administratively modified zero or more times to
  add/remove member FLU servers.
 + A chain may be decommissioned.
 See also: http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html
 ** FLU server administrative life cycle
 + A FLU is created after an administrator chooses the FLU's runtime
  location is selected by the administrator: which machine/virtual
  machine, IP address and TCP port allocation, etc.
 + An unassigned FLU may be added to a chain by chain administrative
  policy.
 + A FLU that is assigned to a chain may be removed from that chain by
  chain administrative policy.
  + In the current implementation, the FLU's Erlang processes will be
    halted.  Then the FLU's data and metadata files will be moved to
    another area of the disk for safekeeping.  Later, a "garbage
    collection" process can be used for reclaiming disk space used by
    halted FLU servers.
 See also: http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html
 * Quick admin: declarative management of Machi FLU and chain life cycles
 The "quick admin" scheme is a temporary (?) tool for managing Machi
 FLU server and chain life cycles in a declarative manner.  The API is
 described in this section.
 ** Quick admin uses the "rc.d" config scheme for life cycle management
 As described at the top of
 http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html, the "rc.d"
 config files do not manage "policy".  "Policy" is doing the right
 thing with a Machi cluster from a systems administrator's
 point of view.  The "rc.d" config files can only implement decisions
 made according to policy.
 The "quick admin" tool is a first attempt at automating policy
 decisions in a safe way (we hope) that is also easy to implement (we
 hope) with a variety of systems management tools, e.g. Chef, Puppet,
 Ansible, Saltstack, or plain-old-human-at-a-keyboard.
 ** Quick admin's declarative "language": an Erlang-flavored AST
 The "language" that an administrator uses to express desired policy
 changes is not (yet) a true language.  As a quick implementation hack,
 the current language is an Erlang-flavored abstract syntax tree
 (AST).  The tree isn't very deep, either, frequently just one
 element tall.  (Not much of a tree, is it?)
 There are three terms in the language currently:
 + ~host~, define a new host that can execute FLU servers
 + ~flu~, define a new FLU
 + ~chain~, define a new chain or re-configure an existing chain with
  the same name
 *** Term 'host': define a new host for FLU services
 In this context, a host is a machine, virtual machine, or container
 that can execute the Machi application and can therefore provide FLU
 services, i.e. file service, Humming Consensus management.
 Two formats may be used to define a new host:
 #+BEGIN_SRC
 {host, Name, Props}.
 {host, Name, AdminI, ClientI, Props}.
 #+END_SRC
 The shorter tuple is shorthand notation for the latter.  If the
 shorthand form is used, then it will be converted automatically to the
 long form as:
 #+BEGIN_SRC
 {host, Name, AdminI=Name, ClientI=Name, Props}.
 #+END_SRC
 Type information, description, and restrictions:
 + ~Name::string()~ The ~Name~ attribute must be unique.  Note that it
  is possible to define two different hosts, one using a DNS hostname
  and one using an IP address.  The user must avoid this
  double-definition because it is not enforced by quick admin.
  + The ~Name~ field is used for cross-reference purposes with other
    terms, e.g., ~flu~ and ~chain~.
  + There is no syntax yet for removing a host definition.
 + ~AdminI::string()~ A DNS hostname or IP address for cluster
  administration purposes, e.g. SSH access.
  + This field is unused at the present time.
 + ~ClientI::string()~ A DNS hostname or IP address for Machi's client
  protocol access, e.g., Protocol Buffers network API service.
  + This field is unused at the present time.
 + ~props::proplist()~ is an Erlang-style property list for specifying
  additional configuration options, debugging information, sysadmin
  comments, etc.
 + A full-featured admin tool should also include managing several
  other aspects of configuration related to a "host".  For example,
  for any single IP address, quick admin assumes that there will be
  exactly one Erlang VM that is running the Machi application.  Of
  course, it is possible to have dozens of Erlang VMs on the same
  (let's assume for clarity) hardware machine and all running Machi
  ... but there are additional aspects of such a machine that quick
  admin does not account for
  + multiple IP addresses per machine
  + multiple Machi package installation paths
  + multiple Machi config files (e.g. cuttlefish config, ~etc.conf~,
    ~vm.args~)
  + multiple data directories/file system mount points
    + This is also a management problem for quick admin for a single
      Machi package on a machine to take advantage of bulk data
      storage using multiple multiple file system mount points.
  + multiple Erlang VM host names, required for distributed Erlang,
    which is used for communication with ~machi~ and ~machi-admin~
    command line utilities.
  + and others....
 *** Term 'flu': define a new FLU
 A new FLU is defined relative to a previously-defined ~host~ entities;
 an exception will be thrown if the ~host~ cannot be cross-referenced.
 #+BEGIN_SRC
 {flu, Name, HostName, Port, Props}
 #+END_SRC
 Type information, description, and restrictions:
 + ~Name::atom()~ The name of the FLU, as a human-friendly name and
  also for internal management use; please note the ~atom()~ type.
  This name must be unique.
  + The ~Name~ field is used for cross-reference purposes with the
    ~chain~ term.
  + There is no syntax yet for removing a FLU definition.
 + ~Hostname::string()~ The cross-reference name of the ~host~ that
  this FLU should run on.
 + ~Port::non_neg_integer()~ The TCP port used by this FLU server's
  Protocol Buffers network API listener service
 + ~props::proplist()~ is an Erlang-style property list for specifying
  additional configuration options, debugging information, sysadmin
  comments, etc.
 *** Term 'chain': define or reconfigure a chain
 A chain is defined relative to zero or more previously-defined ~flu~
 entities; an exception will be thrown if any ~flu~ cannot be
 cross-referenced.
 Two formats may be used to define/reconfigure a chain:
 #+BEGIN_SRC
 {chain, Name, FullList, Props}.
 {chain, Name, CMode, FullList, Witnesses, Props}.
 #+END_SRC
 The shorter tuple is shorthand notation for the latter.  If the
 shorthand form is used, then it will be converted automatically to the
 long form as:
 #+BEGIN_SRC
 {chain, Name, ap_mode, FullList, [], Props}.
 #+END_SRC
 Type information, description, and restrictions:
 + ~Name::atom()~ The name of the chain, as a human-friendly name and
  also for internal management use; please note the ~atom()~ type.
  This name must be unique.
  + There is no syntax yet for removing a chain definition.
 + ~CMode::'ap_mode'|'cp_mode'~ Defines the consistency mode of the
  chain, either eventual consistency or strong consistency,
  respectively.
  + A chain cannot change consistency mode, e.g., from
    strong~->~eventual consistency.
 + ~FullList::list(atom())~ Specifies the list of full-service FLU
  servers, i.e. servers that provide file data & metadata services as
  well as Humming Consensus.  Each atom in the list must
  cross-reference with a previously defined ~chain~; an exception will
  be thrown if any ~flu~ cannot be cross-referenced.
 + ~Witnesses::list(atom())~ Specifies the list of witness-only
  servers, i.e. servers that only participate in Humming Consensus.
  Each atom in the list must cross-reference with a previously defined
  ~chain~; an exception will be thrown if any ~flu~ cannot be
  cross-referenced.
  + This list must be empty for eventual consistency chains.
 + ~props::proplist()~ is an Erlang-style property list for specifying
  additional configuration options, debugging information, sysadmin
  comments, etc.
 + If this term specifies a new ~chain~ name, then all of the member
  FLU servers (full & witness types) will be bootstrapped to a
  starting configuration.
 + If this term specifies a previously-defined ~chain~ name, then all
  of the member FLU servers (full & witness types, respectively) will
  be adjusted to add or remove members, as appropriate.
  + Any FLU servers added to either list must not be assigned to any
    other chain, or they must be a member of this specific chain.
  + Any FLU servers removed from either list will be halted.
    (See the "FLU server administrative life cycle" section above.)
 ** Executing quick admin AST files via the 'machi-admin' utility
 Examples of quick admin AST files can be found in the
 ~priv/quick-admin/examples~ directory.  Below is an example that will
 define a new host ( ~"localhost"~ ), three new FLU servers ( ~f1~ & ~f2~
 and ~f3~ ), and an eventually consistent chain ( ~c1~ ) that uses the new
 FLU servers:
 #+BEGIN_SRC
 {host, "localhost", []}.
 {flu,f1,"localhost",20401,[]}.
 {flu,f2,"localhost",20402,[]}.
 {flu,f3,"localhost",20403,[]}.
 {chain,c1,[f1,f2,f3],[]}.
 #+END_SRC
 *** Checking the syntax of an AST file
 Given an AST config file, ~/path/to/ast/file~, its basic syntax and
 correctness can be checked without executing it.
 #+BEGIN_SRC
 ./rel/machi/bin/machi-admin quick-admin-check /path/to/ast/file
 #+END_SRC
 + The utility will exit with status zero and output ~ok~ if the syntax
  and proposed configuration appears to be correct.
 + If there is an error, the utility will exit with status one, and an
  error message will be printed.
 *** Executing an AST file
 Given an AST config file, ~/path/to/ast/file~, it can be executed
 using the command:
 #+BEGIN_SRC
 ./rel/machi/bin/machi-admin quick-admin-apply /path/to/ast/file RelativeHost
 #+END_SRC
 ... where the last argument, ~RelativeHost~, should be the exact
 spelling of one of the previously defined AST ~host~ entities,
 *and also* is the same host that the ~machi-admin~ utility is being
 executed on.
 Restrictions and warnings:
 + This is alpha quality software.
 + There is no "undo".
  + Of course there is, but you need to resort to doing things like
    using ~machi attach~ to attach to the server's CLI to then execute
    magic Erlang incantations to stop FLUs, unconfigure chains, etc.
    + Oh, and delete some files with magic paths, also.
 ** Using quick admin to manage multiple machines
 A quick sketch follows:
 1. Create the AST file to specify all of the changes that you wish to
   make to all hosts, FLUs, and/or chains, e.g., ~/tmp/ast.txt~.
 2. Check the basic syntax with the ~quick-admin-check~ argument to
   ~machi-admin~. 
 3. If the syntax is good, then copy ~/tmp/ast.txt~ to all hosts in the
   cluster, using the same path, ~/tmp/ast.txt~.
 4. For each machine in the cluster, run:
 #+BEGIN_SRC
 ./rel/machi/bin/machi-admin quick-admin-apply /tmp/ast.txt RelativeHost
 #+END_SRC
 ... where RelativeHost is the AST ~host~ name of the machine that you
 are executing the ~machi-admin~ command on.  The command should be
 successful, with exit status 0 and outputting the string ~ok~.
 Finally, for each machine in the cluster, a listing of all files in
 the directory ~rel/machi/etc/quick-admin-archive~ should show exactly
 the same files, one for each time that ~quick-admin-apply~ has been
 run successfully on that machine.
 * The "rc.d" style configuration file scheme
 This configuration scheme is inspired by BSD UNIX's ~init(8)~ process
 manager's configuration style, called "rc.d" after the name of the
 directory where these files are stored, ~/etc/rc.d~.  The ~init~
 process is responsible for (among other things) starting UNIX
 processes at machine boot time and stopping them when the machine is
 shut down.
 The original scheme used by ~init~ to start processes at boot time was
 a single Bourne shell script called ~/etc/rc~.  When a new software
 package was installed that required a daemon to be started at boot
 time, text was added to the ~/etc/rc~ file.  Uninstalling packages was
 much trickier, because it meant removing lines from a file that
 *is a computer program (run by the Bourne shell, a Turing-complete
 programming language)*.  Error-free editing of the ~/etc/rc~ script
 was impossible in all cases.
 Later, ~init~'s configuration was split into a few master Bourne shell
 scripts and a subdirectory, ~/etc/rc.d~.  The subdirectory contained
 shell scripts that were responsible for boot time starting of a single
 daemon or service, e.g. NFS or an HTTP server.  When a new software
 package was added, a new file was added to the ~rc.d~ subdirectory.
 When a package was removed, the corresponding file in ~rc.d~ was
 removed.  With this simple scheme, addition & removal of boot time
 scripts was vastly simplified.
 ** Riak had a similar configuration file editing problem (and its solution)
 Another software product from Basho Technologies, Riak, had a similar
 configuration file editing problem.  One file in particular,
 ~app.config~, had a syntax that made it difficult both for human
 systems administrators and also computer programs to edit the file in
 a syntactically correct manner.
 Later releases of Riak switched to an alternative configuration file
 format, one inspired by the BSD UNIX ~sysctl(8)~ utility and
 ~sysctl.conf(5)~ file syntax.  The ~sysctl.conf~ format is much easier
 to manage by computer programs to add items.  Removing items is not
 100% simple, however: the correct lines must be identified and then
 removed (e.g. with Perl or a text editor or combination of ~grep -v~
 and ~mv~), but removing any comment lines that "belong" to the removed
 config item(s) is not any easy for a 1-line shell script to do 100%
 correctly.
 Machi will use the ~sysctl.conf~ style configuration for some
 application configuration variables.  However, adding & removing FLUs
 and chains will be managed using the "rc.d" style because of the
 "rc.d" scheme's simplicity and tolerance of mistakes by administrators
 (human or computer).
 ** Machi's "rc.d" file scheme.
 Machi will use a single subdirectory that will contain configuration
 files for some life cycle management task, e.g. a single FLU or a
 single chain.
 The contents of the file should be a single Erlang term, serialized in
 ASCII form as Erlang source code statement, i.e. a single Erlang term
 ~T~ that is formatted by ~io:format("~w.",[T]).~.  This file must be
 parseable by the Erlang function ~file:consult()~.
 Later versions of Machi may change the file format to be more familiar
 to administrators who are unaccustomed to Erlang language syntax.
 ** FLU life cycle management using "rc.d" style files
 *** The key configuration components of a FLU
 1. The machine (or virtual machine) to run it on.
 2. The Machi software package's artifacts to execute.
 3. The disk device(s) used to store Machi file data & metadata, "rc.d"
   style config files, etc.
 4. The name, IP address and TCP port assigned to the FLU service.
 5. Its chain assignment.
 Notes:
 + Items 1-3 are currently outside of the scope of this life cycle
  document.  We assume that human administrators know how to do these
  things.
 + Item 4's properties are explicitly managed by a FLU-defining "rc.d"
  style config file.
 + Item 5 is managed by the chain life cycle management system.
 Here is an example of a properly formatted FLU config file:
 #+BEGIN_SRC
 {p_srvr,f1,machi_flu1_client,"192.168.72.23",20401,[]}.
 #+END_SRC
 ... which corresponds to the following Erlang record definition:
 #+BEGIN_SRC
 -record(p_srvr, {
          name            :: atom(),
          proto_mod = 'machi_flu1_client' :: atom(), % Module name
          address         :: term(), % Protocol-specific
          port            :: term(), % Protocol-specific
          props = []      :: list()  % proplist for other related info
         }).
 #+END_SRC
 + ~name~ is ~f1~.  This is name of the FLU.  This name should be
  unique over the lifetime of the administrative domain and thus
  managed by external policy.  This name must be the same as the name
  of the config file that defines the FLU.
 + ~proto_mod~ is used for internal management purposes and should be
  considered a mandatory constant.
 + ~address~ is "192.168.72.23".  The DNS hostname or IP address used
  by other servers to communicate with this FLU. This must be a valid
  IP address, previously assigned to this machine/VM using the
  appropriate operating system-specific procedure.
 + ~port~ is TCP port 20401.  The TCP port number that the FLU listens
  to for incoming Protocol Buffers-serialized communication.  This TCP
  port must not be in use (now or in the future) by another Machi FLU
  or any other process running on this machine/VM.
 + ~props~ is an Erlang-style property list for specifying additional
  configuration options, debugging information, sysadmin comments,
  etc.
 ** Chain life cycle management using "rc.d" style files
 Unlike FLUs, chains have a self-management aspect that makes a chain
 life cycle different from a single FLU server.  Machi's chains are
 self-managing, via Humming Consensus; see the
 https://github.com/basho/machi/tree/master/doc/ directory for much
 more detail about Humming Consensus.  After FLUs have received their
 initial chain configuration for Humming Consensus, the FLUs will
 manage the chain (and each other) by themselves.
 However, Humming Consensus does not handle three chain management
 problems:
 1. Specifying the very first chain configuration,
 2. Altering the membership of the chain (i.e. adding/removing FLUs
   from the chain),
 3. Stopping the chain permanently.
 A chain "rc.d" file will only be used to bootstrap a newly-defined FLU
 server.  It's like a piece of glue information to introduce the new
 FLU to the Humming Consensus group that is managing the chain's
 dynamic state (e.g. which members are up or down).  In all other
 respects, chain config files are ignored by life cycle management code.
 However, to mimic the life cycle of the FLU server's "rc.d" config
 files, a chain "rc.d" files is not deleted until the chain has been
 decommissioned (i.e. defined with length=0).
 *** The key configuration components of a chain
 1. The name of the chain.
 2. Consistency mode: eventually consistent or strongly consistent.
 3. The membership list of all FLU servers in the chain.
   + Remember, all servers in a single chain will manage full replicas
     of the same collection of Machi files.
 4. If the chain is defined to use strongly consistent mode, then a
   list of "witness servers" may also be defined.  See the
   [https://github.com/basho/machi/tree/master/doc/] documentation for
   more information on witness servers.
   + The witness list must be empty for all chains in eventual
     consistency mode. 
 Here is an example of a properly formatted chain config file:
 #+BEGIN_SRC
 {chain_def_v1,c1,ap_mode,
              [{p_srvr,f1,machi_flu1_client,"localhost",20401,[]},
               {p_srvr,f2,machi_flu1_client,"localhost",20402,[]},
               {p_srvr,f3,machi_flu1_client,"localhost",20403,[]}],
              [],[],[],
              [f1,f2,f3],
              [],[]}.
 #+END_SRC
 ... which corresponds to the following Erlang record definition:
 #+BEGIN_SRC
 -record(chain_def_v1, {
          name            :: atom(),         % chain name
          mode            :: 'ap_mode' | 'cp_mode',
          full = []       :: [p_srvr()],
          witnesses = []  :: [p_srvr()],
          old_full = []   :: [atom()], % guard against some races
          old_witnesses=[] :: [atom()], % guard against some races
          local_run = []  :: [atom()], % must be tailored to each machine!
          local_stop = [] :: [atom()], % must be tailored to each machine!
          props = []      :: list()   % proplist for other related info
         }).
 #+END_SRC
 + ~name~ is ~c1~, the name of the chain.  This name should be unique
  over the lifetime of the administrative domain and thus managed by
  external policy.  This name must be the same as the name of the
  config file that defines the chain.
 + ~mode~ is ~ap_mode~, an internal code symbol for eventual
  consistency mode.
 + ~full~ is a list of Erlang ~#p_srvr{}~ records for full-service
  members of the chain, i.e., providing Machi file data & metadata
  storage services.
 + ~witnesses~ is a list of Erlang ~#p_srvr{}~ records for witness-only
  FLU servers, i.e., providing only Humming Consensus service.
 + The next four fields are used for internal management only.
 + ~props~ is an Erlang-style property list for specifying additional
  configuration options, debugging information, sysadmin comments,
  etc.
--- a/doc/high-level-chain-mgr.pdf
+++ b/doc/high-level-chain-mgr.pdf
--- a/doc/humming-consensus-demo.md
+++ b/doc/humming-consensus-demo.md
@ -1,372 +0,0 @@
 # Table of contents
 * [Hands-on experiments with Machi and Humming Consensus](#hands-on)
 * [Using the network partition simulator and convergence demo test code](#partition-simulator)
 <a name="hands-on">
 # Hands-on experiments with Machi and Humming Consensus
 ## Prerequisites
 Please refer to the
 [Machi development environment prerequisites doc](./dev-prerequisites.md)
 for Machi developer environment prerequisites.
 If you do not have an Erlang/OTP runtime system available, but you do
 have [the Vagrant virtual machine](https://www.vagrantup.com/) manager
 available, then please refer to the instructions in the prerequisites
 doc for using Vagrant.
 <a name="clone-compile">
 ## Clone and compile the code
 Please see the
 [Machi 'clone and compile' doc](./dev-clone-compile.md)
 for the short list of steps required to fetch the Machi source code
 from GitHub and to compile &amp; test Machi.
 ## Running three Machi instances on a single machine
 All of the commands that should be run at your login shell (e.g. Bash,
 c-shell) can be cut-and-pasted from this document directly to your
 login shell prompt.
 Run the following command:
    make stagedevrel
 This will create a directory structure like this:
          |-dev1-|... stand-alone Machi app + subdirectories
    |-dev-|-dev2-|... stand-alone Machi app + directories
          |-dev3-|... stand-alone Machi app + directories
 Each of the `dev/dev1`, `dev/dev2`, and `dev/dev3` are stand-alone
 application instances of Machi and can be run independently of each
 other on the same machine.  This demo will use all three.
 The lifecycle management utilities for Machi are a bit immature,
 currently.  They assume that each Machi server runs on a host with a
 unique hostname -- there is no flexibility built-in yet to easily run
 multiple Machi instances on the same machine.  To continue with the
 demo, we need to use `sudo` or `su` to obtain superuser privileges to
 edit the `/etc/hosts` file.
 Please add the following line to `/etc/hosts`, using this command:
    sudo sh -c 'echo "127.0.0.1 machi1 machi2 machi3" >> /etc/hosts'
 Next, we will use a shell script to finish setting up our cluster.  It
 will do the following for us:
 * Verify that the new line that was added to `/etc/hosts` is correct.
 * Modify the `etc/app.config` files to configure the Humming Consensus
  chain manager's actions logged to the `log/console.log` file.
 * Start the three application instances.
 * Verify that the three instances are running correctly.
 * Configure a single chain, with one FLU server per application
  instance.
 Please run this script using this command:
    ./priv/humming-consensus-demo.setup.sh
 If the output looks like this (and exits with status zero), then the
 script was successful.
    Step: Verify that the required entries in /etc/hosts are present
    Step: add a verbose logging option to app.config
    Step: start three three Machi application instances
    pong
    pong
    pong
    Step: configure one chain to start a Humming Consensus group with three members
    Result: ok
    Result: ok
    Result: ok
 We have now created a single replica chain, called `c1`, that has
 three file servers participating in the chain.  Thanks to the
 hostnames that we added to `/etc/hosts`, all are using the localhost
 network interface.
    | App instance | Pseudo   | FLU name | TCP port |
    | directory    | Hostname |          |   number |
    |--------------+----------+----------+----------|
    | dev1         | machi1   | flu1     |    20401 |
    | dev2         | machi2   | flu2     |    20402 |
    | dev3         | machi3   | flu3     |    20403 |
 The log files for each application instance can be found in the
 `./dev/devN/log/console.log` file, where the `N` is the instance
 number: 1, 2, or 3.
 ## Understanding the chain manager's log file output
 After running the `./priv/humming-consensus-demo.setup.sh` script,
 let's look at the last few lines of the `./dev/dev1/log/console.log`
 log file for Erlang VM process #1.
    2016-03-09 10:16:35.676 [info] <0.105.0>@machi_lifecycle_mgr:process_pending_flu:422 Started FLU f1 with supervisor pid <0.128.0>
    2016-03-09 10:16:35.676 [info] <0.105.0>@machi_lifecycle_mgr:move_to_flu_config:540 Creating FLU config file f1
    2016-03-09 10:16:35.790 [info] <0.105.0>@machi_lifecycle_mgr:bootstrap_chain2:312 Configured chain c1 via FLU f1 to mode=ap_mode all=[f1,f2,f3] witnesses=[]
    2016-03-09 10:16:35.790 [info] <0.105.0>@machi_lifecycle_mgr:move_to_chain_config:546 Creating chain config file c1
    2016-03-09 10:16:44.139 [info] <0.132.0> CONFIRM epoch 1141 <<155,42,7,221>> upi [] rep [] auth f1 by f1
    2016-03-09 10:16:44.271 [info] <0.132.0> CONFIRM epoch 1148 <<57,213,154,16>> upi [f1] rep [] auth f1 by f1
    2016-03-09 10:16:44.864 [info] <0.132.0> CONFIRM epoch 1151 <<239,29,39,70>> upi [f1] rep [f3] auth f1 by f1
    2016-03-09 10:16:45.235 [info] <0.132.0> CONFIRM epoch 1152 <<173,17,66,225>> upi [f2] rep [f1,f3] auth f2 by f1
    2016-03-09 10:16:47.343 [info] <0.132.0> CONFIRM epoch 1154 <<154,231,224,149>> upi [f2,f1,f3] rep [] auth f2 by f1
 Let's pick apart some of these lines.  We have started all three
 servers at about the same time.  We see some race conditions happen,
 and some jostling and readjustment happens pretty quickly in the first
 few seconds.
 * `Started FLU f1 with supervisor pid <0.128.0>`
  * This VM, #1,
  started a FLU (Machi data server) with the name `f1`.  In the Erlang
  process supervisor hierarchy, the process ID of the top supervisor
  is `<0.128.0>`.
 * `Configured chain c1 via FLU f1 to mode=ap_mode all=[f1,f2,f3] witnesses=[]`
  * A bootstrap configuration for a chain named `c1` has been created.
  * The FLUs/data servers that are eligible for participation in the
    chain have names `f1`, `f2`, and `f3`.
  * The chain will operate in eventual consistency mode (`ap_mode`)
  * The witness server list is empty.  Witness servers are never used
    in eventual consistency mode.
 * `CONFIRM epoch 1141 <<155,42,7,221>> upi [] rep [] auth f1 by f1`
  * All participants in epoch 1141 are unanimous in adopting epoch
    1141's projection.  All active membership lists are empty, so
    there is no functional chain replication yet, at least as far as
    server `f1` knows
  * The epoch's abbreviated checksum is `<<155,42,7,221>>`.
  * The UPI list, i.e. the replicas whose data is 100% in sync is
    `[]`, the empty list.  (UPI = Update Propagation Invariant)
  * The list of servers that are under data repair (`rep`) is also
    empty, `[]`.
  * This projection was authored by server `f1`.
  * The log message was generated by server `f1`.
 * `CONFIRM epoch 1148 <<57,213,154,16>> upi [f1] rep [] auth f1 by f1`
  * Now the server `f1` has created a chain of length 1, `[f1]`.
  * Chain repair/file re-sync is not required when the UPI server list
    changes from length 0 -> 1.
 * `CONFIRM epoch 1151 <<239,29,39,70>> upi [f1] rep [f3] auth f1 by f1`
  * Server `f1` has noticed that server `f3` is alive.  Apparently it
    has not yet noticed that server `f2` is also running.
  * Server `f3` is in the repair list.
 * `CONFIRM epoch 1152 <<173,17,66,225>> upi [f2] rep [f1,f3] auth f2 by f1`
  * Server `f2` is apparently now aware that all three servers are running.
  * The previous configuration used by `f2` was `upi [f2]`, i.e., `f2`
    was running in a chain of one.  `f2` noticed that `f1` and `f3`
    were now available and has started adding them to the chain.
  * All new servers are always added to the tail of the chain in the
    repair list.
  * In eventual consistency mode, a UPI change like this is OK.
    * When performing a read, a client must read from both tail of the
      UPI list and also from all repairing servers.
    * When performing a write, the client writes to both the UPI
      server list and also the repairing list, in that order.
      * I.e., the client concatenates both lists,
      `UPI ++ Repairing`, for its chain configuration for the write.
  * Server `f2` will trigger file repair/re-sync shortly.
    * The waiting time for starting repair has been configured to be
      extremely short, 1 second.  The default waiting time is 10
      seconds, in case Humming Consensus remains unstable.
 * `CONFIRM epoch 1154 <<154,231,224,149>> upi [f2,f1,f3] rep [] auth f2 by f1`
  * File repair/re-sync has finished.  All file data on all servers
    are now in sync.
  * The UPI/in-sync part of the chain is now `[f2,f1,f3]`, and there
    are no servers under repair.
 ## Let's create some failures
 Here are some suggestions for creating failures.
 * Use the `./dev/devN/bin/machi stop` and `./dev/devN/bin/machi start`
  commands to stop & start VM #`N`.
 * Stop a VM abnormally by using `kill`.  The OS process name to look
  for is `beam.smp`.
 * Suspend and resume a VM, using the `SIGSTOP` and `SIGCONT` signals.
  * E.g. `kill -STOP 9823` and `kill -CONT 9823`
 The network partition simulator is not (yet) available when running
 Machi in this mode.  Please see the next section for instructions on
 how to use partition simulator.
 <a name="partition-simulator">
 # Using the network partition simulator and convergence demo test code
 This is the demo code mentioned in the presentation that Scott Lystig
 Fritchie gave at the
 [RICON 2015 conference](http://ricon.io).
 * [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf)
 * [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q)
 ## A complete example of all input and output
 If you don't have an Erlang/OTP 17 runtime environment available,
 please see this file for full input and output of a strong consistency
 length=3 chain test:
 https://gist.github.com/slfritchie/8352efc88cc18e62c72c
 This file contains all commands input and all simulator output from a
 sample run of the simulator.
 To help interpret the output of the test, please skip ahead to the
 "The test output is very verbose" section.
 ## Prerequisites
 If you don't have `git` and/or the Erlang 17 runtime system available
 on your OS X, FreeBSD, Linux, or Solaris machine, please take a look
 at the [Prerequisites section](#prerequisites) first.  When you have
 installed the prerequisite software, please return back here.
 ## Clone and compile the code
 Please briefly visit the [Clone and compile the code](#clone-compile)
 section.  When finished, please return back here.
 ## Run an interactive Erlang CLI shell
 Run the following command at your login shell:
    erl -pz .eunit ebin deps/*/ebin
 If you are using Erlang/OTP version 17, you should see some CLI output
 that looks like this:
    Erlang/OTP 17 [erts-6.4] [source] [64-bit] [smp:8:8] [async-threads:10] [hipe] [kernel-poll:false] [dtrace]
    Eshell V6.4  (abort with ^G)
    1>
 ## The test output is very verbose ... what are the important parts?
 The output of the Erlang command
 `machi_chain_manager1_converge_demo:help()` will display the following
 guide to the output of the tests.
    A visualization of the convergence behavior of the chain self-management
    algorithm for Machi.
      1. Set up some server and chain manager pairs.
      2. Create a number of different network partition scenarios, where
         (simulated) partitions may be symmetric or asymmetric.  Then stop changing
         the partitions and keep the simulated network stable (and perhaps broken).
      3. Run a number of iterations of the algorithm in parallel by poking each
         of the manager processes on a random'ish basis.
      4. Afterward, fetch the chain transition changes made by each FLU and
         verify that no transition was unsafe.
    During the iteration periods, the following is a cheatsheet for the output.
    See the internal source for interpreting the rest of the output.
        'SET partitions = '
            A pair-wise list of actors which cannot send messages.  The
            list is uni-directional.  If there are three servers (a,b,c),
            and if the partitions list is '[{a,b},{b,c}]' then all
            messages from a->b and b->c will be dropped, but any other
            sender->recipient messages will be delivered successfully.
        'x uses:'
            The FLU x has made an internal state transition and is using
            this epoch's projection as operating chain configuration.  The
            rest of the line is a summary of the projection.
        'CONFIRM epoch {N}'
            This message confirms that all of the servers listed in the
            UPI and repairing lists of the projection at epoch {N} have
            agreed to use this projection because they all have written
            this projection to their respective private projection stores.
            The chain is now usable by/available to all clients.
        'Sweet, private projections are stable'
            This report announces that this iteration of the test cycle
            has passed successfully.  The report that follows briefly
            summarizes the latest private projection used by each
            participating server.  For example, when in strong consistency
            mode with 'a' as a witness and 'b' and 'c' as real servers:
            %% Legend:
            %% server name, epoch ID, UPI list, repairing list, down list, ...
            %%                         ... witness list, 'false' (a constant value)
            [{a,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}},
             {b,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}}]
            Both servers 'a' and 'b' agree on epoch 1116 with epoch ID
            {1116,<<23,143,246,55>>} where UPI=[a,b], repairing=[],
            down=[c], and witnesses=[a].
            Server 'c' is not shown because 'c' has wedged itself OOS (out
            of service) by configuring a chain length of zero.
            If no servers are listed in the report (i.e. only '[]' is
            displayed), then all servers have wedged themselves OOS, and
            the chain is unavailable.
        'DoIt,' 
            This marks a group of tick events which trigger the manager
            processes to evaluate their environment and perhaps make a
            state transition.
    A long chain of 'DoIt,DoIt,DoIt,' means that the chain state has
    (probably) settled to a stable configuration, which is the goal of the
    algorithm.
    Press control-c to interrupt the test....".
 ## Run a test in eventual consistency mode
 Run the following command at the Erlang CLI prompt:
    machi_chain_manager1_converge_demo:t(3, [{private_write_verbose,true}]).
 The first argument, `3`, is the number of servers to participate in
 the chain.  Please note:
 * Chain lengths as short as 1 or 2 are valid, but the results are a
  bit boring.
 * Chain lengths as long as 7 or 9 can be used, but they may
  suffer from longer periods of churn/instability before all chain
  managers reach agreement via humming consensus.  (It is future work
  to shorten the worst of the unstable churn latencies.)
 * In eventual consistency mode, chain lengths may be even numbers,
  e.g. 2, 4, or 6.
 * The simulator will choose partition events from the permutations of
  all 1, 2, and 3 node partition pairs.  The total runtime will
  increase *dramatically* with chain length.
    * Chain length 2: about 3 partition cases
    * Chain length 3: about 35 partition cases
    * Chain length 4: about 230 partition cases
    * Chain length 5: about 1100 partition cases
 ## Run a test in strong consistency mode (with witnesses):
 *NOTE:* Due to a bug in the test code, please do not try to run the
 convergence test in strong consistency mode and also without the
 correct minority number of witness servers!  If in doubt, please run
 the commands shown below exactly.
 Run the following command at the Erlang CLI prompt:
    machi_chain_manager1_converge_demo:t(3, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a]}]).
 The first argument, `3`, is the number of servers to participate in
 the chain.  Chain lengths as long as 7 or 9 can be used, but they may
 suffer from longer periods of churn/instability before all chain
 managers reach agreement via humming consensus.
 Due to the bug mentioned above, please use the following
 commands when running with chain lengths of 5 or 7, respectively.
    machi_chain_manager1_converge_demo:t(5, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a,b]}]).
    machi_chain_manager1_converge_demo:t(7, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a,b,c]}]).
--- a/doc/overview.edoc
+++ b/doc/overview.edoc
@ -0,0 +1,170 @@
@title Machi: a small village of replicated files
@doc 
 == About This EDoc Documentation ==
 This EDoc-style documentation will concern itself only with Erlang
 function APIs and function &amp; data types.  Higher-level design and
 commentary will remain outside of the Erlang EDoc system; please see
 the "Pointers to Other Machi Documentation" section below for more
 details.
 Readers should beware that this documentation may be out-of-sync with
 the source code.  When in doubt, use the `make edoc' command to
 regenerate all HTML pages.
 It is the developer's responsibility to re-generate the documentation
 periodically and commit it to the Git repo.
 == Machi Code Overview ==
 === Chain Manager ===
 The Chain Manager is responsible for managing the state of Machi's
 "Chain Replication" state.  This role is roughly analogous to the
 "Riak Core" application inside of Riak, which takes care of
 coordinating replica placement and replica repair.
 For each primitive data server in the cluster, a Machi FLU, there is a
 Chain Manager process that manages its FLU's role within the Machi
 cluster's Chain Replication scheme.  Each Chain Manager process
 executes locally and independently to manage the distributed state of
 a single Machi Chain Replication chain.
  <ul>
  <li> To contrast with Riak Core ... Riak Core's claimant process is
       solely responsible for managing certain critical aspects of
       Riak Core distributed state.  Machi's Chain Manager process
       performs similar tasks as Riak Core's claimant.  However, Machi
       has several active Chain Manager processes, one per FLU server,
       instead of a single active process like Core's claimant.  Each
       Chain Manager process acts independently; each is constrained
       so that it will reach consensus via independent computation
       &amp; action.
       Full discussion of this distributed consensus is outside the
       scope of this document; see the "Pointers to Other Machi
       Documentation" section below for more information.
  </li>
  <li> Machi differs from a Riak Core application because Machi's
       replica placement policy is simply, "All Machi servers store
       replicas of all Machi files".
       Machi is intended to be a primitive building block for creating larger
       cluster-of-clusters where files are
       distributed/fragmented/sharded across a large pool of
       independent Machi clusters.
  </li>
  <li> See
       [https://www.usenix.org/legacy/events/osdi04/tech/renesse.html]
       for a copy of the paper, "Chain Replication for Supporting High
       Throughput and Availability" by Robbert van Renesse and Fred
       B. Schneider.
  </li>
  </ul>
 === FLU ===
 The FLU is the basic storage server for Machi.
  <ul>
  <li> The name FLU is taken from "flash storage unit" from the paper
       "CORFU: A Shared Log Design for Flash Clusters" by
       Balakrishnan, Malkhi, Prabhakaran, and Wobber.  See
       [https://www.usenix.org/conference/nsdi12/technical-sessions/presentation/balakrishnan]
  </li>
  <li> In CORFU, the sequencer step is a prerequisite step that is
       performed by a separate component, the Sequencer.
       In Machi, the `append_chunk()' protocol message has
       an implicit "sequencer" operation applied by the "head" of the
       Machi Chain Replication chain.  If a client wishes to write
       data that has already been assigned a sequencer position, then
       the `write_chunk()' API function is used.
  </li>
  </ul>
 For each FLU, there are three independent tasks that are implemented
 using three different Erlang processes:
  <ul>
  <li> A FLU server, implemented primarily by `machi_flu.erl'.
  </li>
  <li> A projection store server, implemented primarily by
      `machi_projection_store.erl'. 
  </li>
  <li> A chain state manager server, implemented primarily by
       `machi_chain_manager1.erl'.
  </li>
  </ul>
 From the perspective of failure detection, it is very convenient that
 all three FLU-related services (file server, sequencer server, and
 projection server) are accessed using the same single TCP port.
 === Projection (data structure) ===
 The projection is a data structure that specifies the current state
 of the Machi cluster: all FLUs, which FLUS are considered
 up/running or down/crashed/stopped, which FLUs are actively
 participants in the Chain Replication protocol, and which FLUs are
 under "repair" (i.e., having their data resyncronized when
 newly-added to a cluster or when restarting after a crash).
 === Projection Store (server) ===
 The projection store is a storage service that is implemented by an
 Erlang/OTP `gen_server' process that is associated with each
 FLU.  Conceptually, the projection store is an array of
 write-once registers.  For each projection store register, the
 key is a 2-tuple of an epoch number (`non_neg_integer()' type)
 and a projection type (`public' or `private' type); the value is
 a projection data structure (`projection_v1()' type).
 === Client and Proxy Client ===
 Machi is intentionally avoiding using distributed Erlang for Machi's
 communication.  This design decision makes Erlang-side code more
 difficult &amp; complex but allows us the freedom of implementing
 parts of Machi in other languages without major
 protocol&amp;API&amp;glue code changes later in the product's
 lifetime.
 There are two layers of interface for Machi clients.
  <ul>
  <li> The `machi_flu1_client' module implements an API that uses a
       TCP socket directly.
  </li>
  <li> The `machi_proxy_flu1_client' module implements an API that
       uses a local, long-lived `gen_server' process as a proxy for
       the remote, perhaps disconnected-or-crashed Machi FLU server.
  </li>
  </ul>
 The types for both modules ought to be the same.  However, due to
 rapid code churn, some differences might exist.  Any major difference
 is (almost by definition) a bug: please open a GitHub issue to request
 a correction.
 == TODO notes ==
 Any use of the string "TODO" in upper/lower/mixed case, anywhere in
 the code, is a reminder signal of unfinished work.
 == Pointers to Other Machi Documentation ==
 <ul>
 <li> If you are viewing this document locally, please look in the
     `../doc/' directory,
 </li>
 <li> If you are viewing this document via the Web, please find the
     documentation via this link: 
     [http://github.com/basho/machi/tree/master/doc/]
     Please be aware that this link points to the `master' branch
     of the Machi source repository and therefore may be
     out-of-sync with non-`master' branch code.
 </li>
 </ul>
--- a/doc/process-protocol-module-overview.jpg
+++ b/doc/process-protocol-module-overview.jpg
--- a/doc/src.high-level/high-level-chain-mgr.tex
+++ b/doc/src.high-level/high-level-chain-mgr.tex
@ -1279,7 +1279,7 @@ as the foundation for Machi's data loss prevention techniques.
 \begin{figure}
 \centering
 $
-[\overbrace{\underbrace{H_1}_\textbf{Head}, M_{11}, \ldots, T_1,
+[\overbrace{\underbrace{H_1}_\textbf{Head}, M_{11}, T_1,
                        H_2, M_{21},
                        \ldots
                        \underbrace{T_2}_\textbf{Tail}}^\textbf{Chain (U.P.~Invariant preserving)}
--- a/include/machi.hrl
+++ b/include/machi.hrl
@ -40,24 +40,3 @@
 %% TODO: it's used in flu_sup and elsewhere, change this to suitable name
 -define(TEST_ETS_TABLE, test_ets_table).
 -define(DEFAULT_COC_NAMESPACE, "").
 -define(DEFAULT_COC_LOCATOR,    0).
 -record(ns_info, {
          version = 0  :: machi_dt:namespace_version(),
          name = <<>>  :: machi_dt:namespace(),
          locator = 0  :: machi_dt:locator()
         }).
 -record(append_opts, {
          chunk_extra = 0             :: machi_dt:chunk_size(),
          preferred_file_name         :: 'undefined' | machi_dt:file_name_s(),
          flag_fail_preferred = false :: boolean()
         }).
 -record(read_opts, {
          no_checksum = false         :: boolean(),
          no_chunk = false            :: boolean(),
          needs_trimmed = false       :: boolean()
         }).
--- a/include/machi_merkle_tree.hrl
+++ b/include/machi_merkle_tree.hrl
@ -1,20 +0,0 @@
 %% machi merkle tree records
 -record(naive, {
          chunk_size = 1048576   :: pos_integer(), %% default 1 MB
          recalc = true          :: boolean(),
          root                   :: 'undefined' | binary(),
          lvl1 = []              :: [ binary() ],
          lvl2 = []              :: [ binary() ],
          lvl3 = []              :: [ binary() ],
          leaves = []            :: [ { Offset :: pos_integer(),
                                        Size :: pos_integer(),
                                        Csum :: binary()} ]
         }).
 -record(mt, {
          filename               :: string(),
          tree                   :: #naive{},
          backend = 'naive'      :: 'naive'
         }).
--- a/include/machi_projection.hrl
+++ b/include/machi_projection.hrl
@ -1,6 +1,6 @@
 %% -------------------------------------------------------------------
 %%
-%% Copyright (c) 2007-2015 Basho Technologies, Inc.  All Rights Reserved.
+%% Copyright (c) 2007-2014 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
@ -22,11 +22,10 @@
 -define(MACHI_PROJECTION_HRL, true).
 -type pv1_consistency_mode() :: 'ap_mode' | 'cp_mode'.
 -type pv1_chain_name():: atom().
 -type pv1_csum()      :: binary().
 -type pv1_epoch()     :: {pv1_epoch_n(), pv1_csum()}.
 -type pv1_epoch_n()   :: non_neg_integer().
-type pv1_server()    :: atom().
+-type pv1_server()    :: atom() | binary().
 -type pv1_timestamp() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}.
 -record(p_srvr, {
@ -56,7 +55,6 @@
          epoch_number    :: pv1_epoch_n() | ?SPAM_PROJ_EPOCH,
          epoch_csum      :: pv1_csum(),
          author_server   :: pv1_server(),
          chain_name = ch_not_def_yet :: pv1_chain_name(),
          all_members     :: [pv1_server()],
          witnesses = []  :: [pv1_server()],
          creation_time   :: pv1_timestamp(),
@ -77,16 +75,4 @@
 %% create a consistent projection ranking score.
 -define(MAX_CHAIN_LENGTH, 64).
 -record(chain_def_v1, {
          name            :: atom(),         % chain name
          mode            :: pv1_consistency_mode(),
          full = []       :: [p_srvr()],
          witnesses = []  :: [p_srvr()],
          old_full = []   :: [pv1_server()], % guard against some races
          old_witnesses=[] :: [pv1_server()], % guard against some races
          local_run = []  :: [pv1_server()], % must be tailored to each machine!
          local_stop = [] :: [pv1_server()], % must be tailored to each machine!
          props = []      :: list()   % proplist for other related info
         }).
 -endif. % !MACHI_PROJECTION_HRL
--- a/priv/humming-consensus-demo.setup.sh
+++ b/priv/humming-consensus-demo.setup.sh
@ -1,56 +0,0 @@
 #!/bin/sh
 echo "Step: Verify that the required entries in /etc/hosts are present"
 for i in 1 2 3; do
    grep machi$i /etc/hosts | egrep -s '^127.0.0.1' > /dev/null 2>&1
    if [ $? -ne 0 ]; then
        echo ""
        echo "'grep -s machi$i' failed. Aborting, sorry."
        exit 1
    fi
    ping -c 1 machi$i > /dev/null 2>&1
    if [ $? -ne 0 ]; then
        echo ""
        echo "Ping attempt on host machi$i failed. Aborting."
        echo ""
        ping -c 1 machi$i
        exit 1
    fi
 done
 echo "Step: add a verbose logging option to app.config"
 for i in 1 2 3; do
    ed ./dev/dev$i/etc/app.config <<EOF > /dev/null 2>&1
 /verbose_confirm
 a
 {chain_manager_opts, [{private_write_verbose_confirm,true}]},
 {stability_time, 1},
 .
 w
 q
 EOF
 done
 echo "Step: start three three Machi application instances"
 for i in 1 2 3; do
    ./dev/dev$i/bin/machi start
    ./dev/dev$i/bin/machi ping
    if [ $? -ne 0 ]; then
        echo "Sorry, a 'ping' check for instance dev$i failed. Aborting."
        exit 1
    fi
 done
 echo "Step: configure one chain to start a Humming Consensus group with three members"
 # Note: $CWD of each Machi proc is two levels below the source code root dir.
 LIFECYCLE000=../../priv/quick-admin-examples/demo-000
 for i in 3 2 1; do
    ./dev/dev$i/bin/machi-admin quick-admin-apply $LIFECYCLE000 machi$i
    if [ $? -ne 0 ]; then
        echo "Sorry, 'machi-admin quick-admin-apply failed' on machi$i. Aborting."
        exit 1
    fi
 done
 exit 0
--- a/priv/humming-consensus-demo.vagrant/Vagrantfile
+++ b/priv/humming-consensus-demo.vagrant/Vagrantfile
@ -1,93 +0,0 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 # All Vagrant configuration is done below. The "2" in Vagrant.configure
 # configures the configuration version (we support older styles for
 # backwards compatibility). Please don't change it unless you know what
 # you're doing.
 Vagrant.configure(2) do |config|
  # The most common configuration options are documented and commented below.
  # For a complete reference, please see the online documentation at
  # https://docs.vagrantup.com.
  # Every Vagrant development environment requires a box. You can search for
  # boxes at https://atlas.hashicorp.com/search.
  # If this Vagrant box has not been downloaded before (e.g. using "vagrant box add"),
  # then Vagrant will automatically download the VM image from HashiCorp.
  config.vm.box = "hashicorp/precise64"
  # If using a FreeBSD box, Bash may not be installed.
  # Use the config.ssh.shell setting to specify an alternate shell.
  # Note, however, that any code in the 'config.vm.provision' section
  # would then have to use this shell's syntax!
  #   config.ssh.shell = "/bin/csh -l"
  # Disable automatic box update checking. If you disable this, then
  # boxes will only be checked for updates when the user runs
  # `vagrant box outdated`. This is not recommended.
  # config.vm.box_check_update = false
  # Create a forwarded port mapping which allows access to a specific port
  # within the machine from a port on the host machine. In the example below,
  # accessing "localhost:8080" will access port 80 on the guest machine.
  # config.vm.network "forwarded_port", guest: 80, host: 8080
  # Create a private network, which allows host-only access to the machine
  # using a specific IP.
  # config.vm.network "private_network", ip: "192.168.33.10"
  # Create a public network, which generally matched to bridged network.
  # Bridged networks make the machine appear as another physical device on
  # your network.
  # config.vm.network "public_network"
  # Share an additional folder to the guest VM. The first argument is
  # the path on the host to the actual folder. The second argument is
  # the path on the guest to mount the folder. And the optional third
  # argument is a set of non-required options.
  # config.vm.synced_folder "../data", "/vagrant_data"
  # Provider-specific configuration so you can fine-tune various
  # backing providers for Vagrant. These expose provider-specific options.
  # Example for VirtualBox:
  #
  config.vm.provider "virtualbox" do |vb|
    # Display the VirtualBox GUI when booting the machine
    # vb.gui = true
    # Customize the amount of memory on the VM:
    vb.memory = "512"
  end
  #
  # View the documentation for the provider you are using for more
  # information on available options.
  # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies
  # such as FTP and Heroku are also available. See the documentation at
  # https://docs.vagrantup.com/v2/push/atlas.html for more information.
  # config.push.define "atlas" do |push|
  #   push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME"
  # end
  # Enable provisioning with a shell script. Additional provisioners such as
  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
  # documentation for more information about their specific syntax and use.
  config.vm.provision "shell", inline: <<-SHELL
    # Install prerequsites
    # Support here for FreeBSD is experimental
    apt-get update ; sudo apt-get install -y git sudo rsync ; # Ubuntu Linux
    env ASSUME_ALWAYS_YES=yes pkg install -f git sudo rsync ; # FreeBSD 10
    # Install dependent packages, using slf-configurator
    git clone https://github.com/slfritchie/slf-configurator.git
    chown -R vagrant ./slf-configurator
    (cd slf-configurator ; sudo sh -x ./ALL.sh)
    echo 'export PATH=${PATH}:/usr/local/erlang/17.5/bin' >> ~vagrant/.bashrc
    export PATH=${PATH}:/usr/local/erlang/17.5/bin
    ## echo 'set path = ( $path  /usr/local/erlang/17.5/bin )' >> ~vagrant/.cshrc
    ## setenv PATH /usr/local/erlang/17.5/bin:$PATH
    git clone https://github.com/basho/machi.git
    (cd machi ; git checkout master ; make && make test )
    chown -R vagrant ./machi
  SHELL
 end
--- a/priv/make-faq.pl
+++ b/priv/make-faq.pl
@ -36,7 +36,7 @@ while (<I>) {
            $indent = " " x ($count * 4);
            s/^#*\s*[0-9. ]*//;
            $anchor = "n$label";
-            printf T1 "%s+ [%s. %s](#%s)\n", $indent, $label, $_, $anchor;
+            printf T1 "%s+ [%s %s](#%s)\n", $indent, $label, $_, $anchor;
            printf T2 "<a name=\"%s\">\n", $anchor;
            $line =~ s/(#+)\s*[0-9. ]*/$1 $label.  /;
            print T2 $line;
--- a/priv/quick-admin-examples/000
+++ b/priv/quick-admin-examples/000
@ -1 +0,0 @@
 {host, "localhost", []}.
--- a/priv/quick-admin-examples/001
+++ b/priv/quick-admin-examples/001
@ -1,4 +0,0 @@
 {flu,f1,"localhost",20401,[]}.
 {flu,f2,"localhost",20402,[]}.
 {flu,f3,"localhost",20403,[]}.
 {chain,c1,[f1,f2,f3],[]}.
--- a/priv/quick-admin-examples/002
+++ b/priv/quick-admin-examples/002
@ -1,4 +0,0 @@
 {flu,f4,"localhost",20404,[]}.
 {flu,f5,"localhost",20405,[]}.
 {flu,f6,"localhost",20406,[]}.
 {chain,c2,[f4,f5,f6],[]}.
--- a/priv/quick-admin-examples/demo-000
+++ b/priv/quick-admin-examples/demo-000
@ -1,7 +0,0 @@
 {host, "machi1", []}.
 {host, "machi2", []}.
 {host, "machi3", []}.
 {flu,f1,"machi1",20401,[]}.
 {flu,f2,"machi2",20402,[]}.
 {flu,f3,"machi3",20403,[]}.
 {chain,c1,[f1,f2,f3],[]}.
--- a/rebar.config
+++ b/rebar.config
@ -1,17 +1,13 @@
-{require_otp_vsn, "17|18"}.
+{require_otp_vsn, "17"}.
 %%% {erl_opts, [warnings_as_errors, {parse_transform, lager_transform}, debug_info]}.
 {erl_opts, [{parse_transform, lager_transform}, debug_info]}.
 {edoc_opts, [{dir, "./edoc"}]}.
 {deps, [
        {cuttlefish, ".*", {git, "git://github.com/basho/cuttlefish.git", {branch, "develop"}}},
        {sext, ".*", {git, "git://github.com/basho/sext.git", {branch, "master"}}},
        {eleveldb, ".*", {git, "git://github.com/basho/eleveldb.git", {branch, "develop"}}},
        {lager, ".*", {git, "git://github.com/basho/lager.git", {tag, "2.2.0"}}},
        {protobuffs, "0.8.*", {git, "git://github.com/basho/erlang_protobuffs.git", {tag, "0.8.1p4"}}},
        {riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {branch, "develop"}}},
        {ranch, ".*", {git, "git://github.com/ninenines/ranch.git", {branch, "master"}}},
        {node_package, ".*", {git, "git://github.com/basho/node_package.git", {branch, "develop"}}},
        {eper, ".*", {git, "git://github.com/basho/eper.git", {tag, "0.92-basho1"}}},
        {cluster_info, ".*", {git, "git://github.com/basho/cluster_info", {branch, "develop"}}}
--- a/rel/files/app.config
+++ b/rel/files/app.config
@ -1,35 +1,25 @@
 [
 {machi, [
          %% Data directory for all FLUs.
-          {flu_data_dir, "{{platform_data_dir}}/flu"},
+          {flu_data_dir, "{{platform_data_dir}}"},
          %% FLU config directory
          {flu_config_dir, "{{platform_etc_dir}}/flu-config"},
          %% Chain config directory
          {chain_config_dir, "{{platform_etc_dir}}/chain-config"},
          %% FLUs to start at app start.
-          %% This task has moved to machi_flu_sup and machi_lifecycle_mgr.
+          {initial_flus, [
                          %% Remember, this is a list, so separate all tuples
                          %% with a comma.
                          %%
                          %% {Name::atom(), Port::pos_integer(), proplist()}
                          %%
                          %% For example: {my_name_is_a, 12500, []}
                         ]},
          %% Number of metadata manager processes to run per FLU.
          %% Default = 10
          %% {metadata_manager_count, 2},
          %% Default options for chain manager processes.
          %% {chain_manager_opts, [{private_write_verbose,true},
          %%                       {private_write_verbose_confirm,true}]},
          %% Platform vars (mirror of reltool packaging)
          {platform_data_dir, "{{platform_data_dir}}"},
          {platform_etc_dir,  "{{platform_etc_dir}}"},
          %% Do not delete, do not put Machi config items after this line.
          {final_comma_stopper, do_not_delete}
         ]
  },
  {lager, [
           {error_logger_hwm, 5000} % lager's default of 50/sec is too low
          ]
  }
 ].
--- a/rel/files/machi-admin
+++ b/rel/files/machi-admin
@ -22,41 +22,23 @@ cd $RUNNER_BASE_DIR
 SCRIPT=`basename $0`
 usage() {
-    echo "Usage: $SCRIPT { quick-admin-check | quick-admin-apply | "
+    echo "Usage: $SCRIPT { test | "
    echo "                    top }"
 }
 case "$1" in
-    quick-admin-check)
+    test)
        # Make sure the local node IS running
        node_up_check
        shift
-        NODE_NAME=${NAME_ARG#* } # target machi server node name
+        # Parse out the node name to pass to the client
-        IN_FILE="$1"
+        NODE_NAME=${NAME_ARG#* }
-        $ERTS_PATH/erl -noshell -noinput $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \
+        $ERTS_PATH/erl -noshell $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \
-                       -remsh $NODE_NAME \
+                       -pa $RUNNER_LIB_DIR/basho-patches \
-                       -eval "Me = self(), spawn('"$NODE_NAME"', fun() -> X = (catch(machi_lifecycle_mgr:quick_admin_sanity_check(\"$IN_FILE\"))), Me ! {res, X} end), XX = receive {res, Res} -> Res after 10*1000 -> timeout end, io:format(user, \"Result: ~p\n\", [XX]), case XX of \
+                       -eval "case catch(machi:client_test(\"$NODE_NAME\")) of \
                               ok -> init:stop();                             \
                               _  -> init:stop(1)                             \
                              end."
        ;;
    quick-admin-apply)
        # Make sure the local node IS running
        node_up_check
        shift
        NODE_NAME=${NAME_ARG#* } # target machi server node name
        IN_FILE="$1"
        RELATIVE_HOST="$2"
        $ERTS_PATH/erl -noshell -noinput $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \
                       -remsh $NODE_NAME \
                       -eval "Me = self(), spawn('"$NODE_NAME"', fun() -> X = (catch(machi_lifecycle_mgr:quick_admin_apply(\"$IN_FILE\", \"$RELATIVE_HOST\"))), Me ! {res, X} end), XX = receive {res, Res} -> Res after 10*1000 -> timeout end, io:format(user, \"Result: ~p\n\", [XX]), case XX of \
                               ok -> init:stop();                             \
                               _  -> init:stop(1)                             \
                              end."
--- a/rel/gen_dev
+++ b/rel/gen_dev
@ -1,16 +0,0 @@
 #! /bin/sh
 #
 # Example usage: gen_dev dev4 vars.src vars
 #
 # Generate an overlay config for devNNN from vars.src and write to vars
 #
 NAME=$1
 TEMPLATE=$2
 VARFILE=$3
 NODE="$NAME@127.0.0.1"
 echo "Generating $NAME - node='$NODE'"
 sed -e "s/@NODE@/$NODE/" \
    < $TEMPLATE > $VARFILE
--- a/rel/reltool.config
+++ b/rel/reltool.config
@ -47,7 +47,6 @@
 {overlay, [
           {mkdir, "data"},
           {mkdir, "data/^PRESERVE"},
           {mkdir, "log"},
           %% Copy base files for starting and interacting w/ node
@ -94,20 +93,6 @@
           {template, "files/vm.args", "etc/vm.args"},
           {template, "files/app.config", "etc/app.config"},
           {mkdir, "etc/chain-config"},
           {mkdir, "etc/flu-config"},
           {mkdir, "etc/pending"},
           {mkdir, "etc/rejected"},
           %% Experiment: quick-admin
           {mkdir, "etc/quick-admin-archive"},
           {mkdir, "priv"},
           {mkdir, "priv/quick-admin-examples"},
           {copy, "../priv/quick-admin-examples/000", "priv/quick-admin-examples"},
           {copy, "../priv/quick-admin-examples/001", "priv/quick-admin-examples"},
           {copy, "../priv/quick-admin-examples/002", "priv/quick-admin-examples"},
           {copy, "../priv/quick-admin-examples/demo-000", "priv/quick-admin-examples/demo-000"},
           {mkdir, "lib/basho-patches"}
           %% {copy, "../apps/machi/ebin/etop_txt.beam", "lib/basho-patches"}
          ]}.
--- a/rel/vars.config
+++ b/rel/vars.config
@ -1,9 +1,6 @@
 %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
 %% ex: ft=erlang ts=4 sw=4 et
 %% NOTE: When modifying this file, also keep its near cousin
 %%       config file rel/vars/dev_vars.config.src in sync!
 %% Platform-specific installation paths
 {platform_bin_dir,  "./bin"}.
 {platform_data_dir, "./data"}.
--- a/rel/vars/dev_vars.config.src
+++ b/rel/vars/dev_vars.config.src
@ -1,48 +0,0 @@
 %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
 %% ex: ft=erlang ts=4 sw=4 et
 %% NOTE: When modifying this file, also keep its near cousin
 %%       config file rel/vars/dev_vars.config.src in sync!
 %% Platform-specific installation paths
 {platform_bin_dir,  "./bin"}.
 {platform_data_dir, "./data"}.
 {platform_etc_dir,  "./etc"}.
 {platform_lib_dir,  "./lib"}.
 {platform_log_dir,  "./log"}.
 %%
 %% etc/app.config
 %%
 {sasl_error_log,    "{{platform_log_dir}}/sasl-error.log"}.
 {sasl_log_dir,      "{{platform_log_dir}}/sasl"}.
 %% lager
 {console_log_default, file}.
 %%
 %% etc/vm.args
 %%
 {node,         "@NODE@"}.
 {crash_dump,   "{{platform_log_dir}}/erl_crash.dump"}.
 %%
 %% bin/machi
 %%
 {runner_script_dir,  "\`cd \\`dirname $0\\` 1>/dev/null && /bin/pwd\`"}.
 {runner_base_dir,    "{{runner_script_dir}}/.."}.
 {runner_etc_dir,     "$RUNNER_BASE_DIR/etc"}.
 {runner_log_dir,     "$RUNNER_BASE_DIR/log"}.
 {runner_lib_dir,     "$RUNNER_BASE_DIR/lib"}.
 {runner_patch_dir,   "$RUNNER_BASE_DIR/lib/basho-patches"}.
 {pipe_dir,           "/tmp/$RUNNER_BASE_DIR/"}.
 {runner_user,        ""}.
 {runner_wait_process, "machi_flu_sup"}.
 {runner_ulimit_warn, 65536}.
 %%
 %% cuttlefish
 %%
 {cuttlefish,         ""}. % blank = off
 {cuttlefish_conf,    "machi.conf"}.
--- a/src/machi.app.src
+++ b/src/machi.app.src
@ -1,7 +1,7 @@
 {application, machi, [
    {description, "A village of write-once files."},
-    {vsn, "0.0.1"},
+    {vsn, "0.0.0"},
-    {applications, [kernel, stdlib, crypto, cluster_info, ranch]},
+    {applications, [kernel, stdlib, crypto, cluster_info]},
    {mod,{machi_app,[]}},
    {registered, []},
    {env, [
--- a/src/machi.proto
+++ b/src/machi.proto
@ -170,18 +170,11 @@ message Mpb_AuthResp {
 // High level API: append_chunk() request & response
 message Mpb_AppendChunkReq {
-    // General namespace arguments
+    optional bytes placement_key = 1;
-    /* In single chain/non-clustered environment, use namespace="" */
+    required string prefix = 2;
-    required string namespace = 1;
+    required bytes chunk = 3;
-
+    required Mpb_ChunkCSum csum = 4;
-    required string prefix = 10;
+    optional uint32 chunk_extra = 5;
    required bytes chunk = 11;
    required Mpb_ChunkCSum csum = 12;
    optional uint32 chunk_extra = 20;
    optional string preferred_file_name = 21;
    /* Fail the operation if our preferred file name is not available */
    optional bool flag_fail_preferred = 22 [default=false];
 }
 message Mpb_AppendChunkResp {
@ -193,7 +186,7 @@ message Mpb_AppendChunkResp {
 // High level API: write_chunk() request & response
 message Mpb_WriteChunkReq {
-    required Mpb_Chunk chunk = 10;
+    required Mpb_Chunk chunk = 1;
 }
 message Mpb_WriteChunkResp {
@ -203,22 +196,19 @@ message Mpb_WriteChunkResp {
 // High level API: read_chunk() request & response
 message Mpb_ReadChunkReq {
-    // No namespace arguments are required because NS is embedded
+    required Mpb_ChunkPos chunk_pos = 1;
    // inside of the file name.
    required Mpb_ChunkPos chunk_pos = 10;
    // Use flag_no_checksum=non-zero to skip returning the chunk's checksum.
    // TODO: not implemented yet.
-    optional bool flag_no_checksum = 20 [default=false];
+    optional uint32 flag_no_checksum = 2 [default=0];
    // Use flag_no_chunk=non-zero to skip returning the chunk (which
    // only makes sense if flag_no_checksum is not set).
    // TODO: not implemented yet.
-    optional bool flag_no_chunk = 21 [default=false];
+    optional uint32 flag_no_chunk = 3 [default=0];
    // TODO: not implemented yet.
-    optional bool flag_needs_trimmed = 22 [default=false];
+    optional uint32 flag_needs_trimmed = 4 [default=0];
 }
 message Mpb_ReadChunkResp {
@ -254,8 +244,6 @@ message Mpb_ChecksumListResp {
 // High level API: list_files() request & response
 message Mpb_ListFilesReq {
    // TODO: Add flag for file glob/regexp/other filter type
    // TODO: What else could go wrong?
 }
 message Mpb_ListFilesResp {
@ -342,17 +330,18 @@ message Mpb_ProjectionV1 {
    required uint32 epoch_number = 1;
    required bytes epoch_csum = 2;
    required string author_server = 3;
-    required string chain_name = 4;
+    repeated string all_members = 4;
-    repeated string all_members = 5;
+    repeated string witnesses = 5;
-    repeated string witnesses = 6;
+    required Mpb_Now creation_time = 6;
-    required Mpb_Now creation_time = 7;
+    required Mpb_Mode mode = 7;
-    required Mpb_Mode mode = 8;
+    repeated string upi = 8;
-    repeated string upi = 9;
+    repeated string repairing = 9;
-    repeated string repairing = 10;
+    repeated string down = 10;
-    repeated string down = 11;
+    optional bytes opaque_flap = 11;
-    required bytes opaque_dbg = 12;
+    optional bytes opaque_inner = 12;
-    required bytes opaque_dbg2 = 13;
+    required bytes opaque_dbg = 13;
-    repeated Mpb_MembersDictEntry members_dict = 14;
+    required bytes opaque_dbg2 = 14;
    repeated Mpb_MembersDictEntry members_dict = 15;
 }
 //////////////////////////////////////////
@ -388,20 +377,12 @@ message Mpb_ProjectionV1 {
 // Low level API: append_chunk()
 message Mpb_LL_AppendChunkReq {
-    // General namespace arguments
+    required Mpb_EpochID epoch_id = 1;
-    required uint32 namespace_version = 1;
+    optional bytes placement_key = 2;
-    required string namespace = 2;
+    required string prefix = 3;
-    required uint32 locator = 3;
+    required bytes chunk = 4;
-
+    required Mpb_ChunkCSum csum = 5;
-    required Mpb_EpochID epoch_id = 10;
+    optional uint32 chunk_extra = 6;
    required string prefix = 11;
    required bytes chunk = 12;
    required Mpb_ChunkCSum csum = 13;
    optional uint32 chunk_extra = 20;
    optional string preferred_file_name = 21;
    /* Fail the operation if our preferred file name is not available */
    optional bool flag_fail_preferred = 22 [default=false];
 }
 message Mpb_LL_AppendChunkResp {
@ -413,12 +394,8 @@ message Mpb_LL_AppendChunkResp {
 // Low level API: write_chunk()
 message Mpb_LL_WriteChunkReq {
-    // General namespace arguments
+    required Mpb_EpochID epoch_id = 1;
-    required uint32 namespace_version = 1;
+    required Mpb_Chunk chunk = 2;
    required string namespace = 2;
    required Mpb_EpochID epoch_id = 10;
    required Mpb_Chunk chunk = 11;
 }
 message Mpb_LL_WriteChunkResp {
@ -428,23 +405,19 @@ message Mpb_LL_WriteChunkResp {
 // Low level API: read_chunk()
 message Mpb_LL_ReadChunkReq {
-    // General namespace arguments
+    required Mpb_EpochID epoch_id = 1;
-    required uint32 namespace_version = 1;
+    required Mpb_ChunkPos chunk_pos = 2;
    required string namespace = 2;
    required Mpb_EpochID epoch_id = 10;
    required Mpb_ChunkPos chunk_pos = 11;
    // Use flag_no_checksum=non-zero to skip returning the chunk's checksum.
    // TODO: not implemented yet.
-    optional bool flag_no_checksum = 20 [default=false];
+    optional uint32 flag_no_checksum = 3 [default=0];
    // Use flag_no_chunk=non-zero to skip returning the chunk (which
    // only makes sense if flag_checksum is not set).
    // TODO: not implemented yet.
-    optional bool flag_no_chunk = 21 [default=false];
+    optional uint32 flag_no_chunk = 4 [default=0];
-    optional bool flag_needs_trimmed = 22 [default=false];
+    optional uint32 flag_needs_trimmed = 5 [default=0];
 }
 message Mpb_LL_ReadChunkResp {
@ -456,16 +429,11 @@ message Mpb_LL_ReadChunkResp {
 // Low level API: trim_chunk()
 message Mpb_LL_TrimChunkReq {
-    // General namespace arguments
+    required Mpb_EpochID epoch_id = 1;
-    required uint32 namespace_version = 1;
+    required string file = 2;
-    required string namespace = 2;
+    required uint64 offset = 3;
-
+    required uint32 size = 4;
-    required Mpb_EpochID epoch_id = 10;
+    optional uint32 trigger_gc = 5 [default=0];
    required string file = 11;
    required uint64 offset = 12;
    required uint32 size = 13;
    optional bool trigger_gc = 20 [default=false];
 }
 message Mpb_LL_TrimChunkResp {
@ -475,7 +443,8 @@ message Mpb_LL_TrimChunkResp {
 // Low level API: checksum_list()
 message Mpb_LL_ChecksumListReq {
-    required string file = 1;
+    required Mpb_EpochID epoch_id = 1;
    required string file = 2;
 }
 message Mpb_LL_ChecksumListResp {
@ -506,9 +475,7 @@ message Mpb_LL_WedgeStatusReq {
 message Mpb_LL_WedgeStatusResp {
    required Mpb_GeneralStatusCode status = 1;
    optional Mpb_EpochID epoch_id = 2;
-    optional bool wedged_flag = 3;
+    optional uint32 wedged_flag = 3;
    optional uint32 namespace_version = 4;
    optional string namespace = 5;
 }
 // Low level API: delete_migration()
--- a/src/machi_admin_util.erl
+++ b/src/machi_admin_util.erl
@ -90,18 +90,17 @@ verify_file_checksums_local2(Sock1, EpochID, Path0) ->
    end.
 verify_file_checksums_remote2(Sock1, EpochID, File) ->
    NSInfo = undefined,
    ReadChunk = fun(File_name, Offset, Size) ->
-                        ?FLU_C:read_chunk(Sock1, NSInfo, EpochID,
+                        ?FLU_C:read_chunk(Sock1, EpochID,
-                                          File_name, Offset, Size, undefined)
+                                          File_name, Offset, Size, [])
                end,
    verify_file_checksums_common(Sock1, EpochID, File, ReadChunk).
-verify_file_checksums_common(Sock1, _EpochID, File, ReadChunk) ->
+verify_file_checksums_common(Sock1, EpochID, File, ReadChunk) ->
    try
-        case ?FLU_C:checksum_list(Sock1, File) of
+        case ?FLU_C:checksum_list(Sock1, EpochID, File) of
            {ok, InfoBin} ->
-                Info = machi_csum_table:split_checksum_list_blob_decode(InfoBin),
+                {Info, _} = machi_csum_table:split_checksum_list_blob_decode(InfoBin),
                Res = lists:foldl(verify_chunk_checksum(File, ReadChunk),
                                  [], Info),
                {ok, Res};
@ -116,9 +115,7 @@ verify_file_checksums_common(Sock1, _EpochID, File, ReadChunk) ->
    end.
 verify_chunk_checksum(File, ReadChunk) ->
-    fun({0, ?MINIMUM_OFFSET, none}, []) ->
+    fun({Offset, Size, <<_Tag:1/binary, CSum/binary>>}, Acc) ->
            [];
       ({Offset, Size, <<_Tag:1/binary, CSum/binary>>}, Acc) ->
            case ReadChunk(File, Offset, Size) of
                {ok, {[{_, Offset, Chunk, _}], _}} ->
                    CSum2 = machi_util:checksum_chunk(Chunk),
--- a/src/machi_basho_bench_driver.erl
+++ b/src/machi_basho_bench_driver.erl
@ -1,6 +1,6 @@
 %% -------------------------------------------------------------------
 %%
-%% Copyright (c) 2007-2016 Basho Technologies, Inc.  All Rights Reserved.
+%% Copyright (c) 2007-2015 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
@ -43,25 +43,23 @@
 %% could add new entries to this ETS table.
 %%
 %% Now we can use various integer-centric key generators that are
-%% already bundled with basho_bench.  NOTE: this scheme does not allow
+%% already bundled with basho_bench.
 %% mixing of 'append' and 'read' operations in the same config.  Basho
 %% Bench does not support different key generators for different
 %% operations, unfortunately.  The work-around is to run two different
 %% Basho Bench instances: on for 'append' ops with a key generator for
 %% the desired prefix(es), and the other for 'read' ops with an
 %% integer key generator.
 %%
-%% TODO: The 'read' operator will always read chunks at exactly the
+%% TODO: Add CRC checking, when feasible and when supported on the
-%% byte offset & size as the original append/write ops.  If reads are
+%% server side.
-%% desired at any arbitrary offset & size, then a new strategy is
+%%
-%% required.
+%% TODO: As an alternate idea, if we know that the chunks written are
 %% always the same size, and if we don't care about CRC checking, then
 %% all we need to know are the file names &amp; file sizes on the server:
 %% we can then pick any valid offset within that file.  That would
 %% certainly be more scalable than the zillion-row-ETS-table, which is
 %% definitely RAM-hungry.
 -module(machi_basho_bench_driver).
 -export([new/1, run/4]).
 -record(m, {
          id,
          conn,
          max_key
         }).
@ -83,7 +81,7 @@ new(Id) ->
                                     {read_concurrency, true}]),
            ets:insert(ETS, {max_key, 0}),
            ets:insert(ETS, {total_bytes, 0}),
-            MaxKeys = load_ets_table_maybe(Conn, ETS),
+            MaxKeys = load_ets_table(Conn, ETS),
            ?INFO("Key preload: finished, ~w keys loaded", [MaxKeys]),
            Bytes = ets:lookup_element(ETS, total_bytes, 2),
            ?INFO("Key preload: finished, chunk list specifies ~s MBytes of chunks",
@ -92,14 +90,12 @@ new(Id) ->
       true ->
            ok
    end,
-    {ok, #m{id=Id, conn=Conn}}.
+    {ok, #m{conn=Conn}}.
 run(append, KeyGen, ValueGen, #m{conn=Conn}=S) ->
    Prefix = KeyGen(),
    Value = ValueGen(),
-    CSum = machi_util:make_client_csum(Value),
+    case machi_cr_client:append_chunk(Conn, Prefix, Value, ?THE_TIMEOUT) of
    AppendOpts = {append_opts,0,undefined,false}, % HACK FIXME
    case machi_cr_client:append_chunk(Conn, undefined, Prefix, Value, CSum, AppendOpts, ?THE_TIMEOUT) of
        {ok, Pos} ->
            EtsKey = ets:update_counter(?ETS_TAB, max_key, 1),
            true = ets:insert(?ETS_TAB, {EtsKey, Pos}),
@ -116,26 +112,9 @@ run(read, KeyGen, _ValueGen, #m{conn=Conn, max_key=MaxKey}=S) ->
    Idx = KeyGen() rem MaxKey,
    %% {File, Offset, Size, _CSum} = ets:lookup_element(?ETS_TAB, Idx, 2),
    {File, Offset, Size} = ets:lookup_element(?ETS_TAB, Idx, 2),
-    ReadOpts = {read_opts,false,false,false}, % HACK FIXME
+    case machi_cr_client:read_chunk(Conn, File, Offset, Size, [], ?THE_TIMEOUT) of
-    case machi_cr_client:read_chunk(Conn, undefined, File, Offset, Size, ReadOpts, ?THE_TIMEOUT) of
+        {ok, _Chunk} ->
        {ok, {Chunks, _Trimmed}} ->
            %% io:format(user, "Chunks ~P\n", [Chunks, 15]),
            %% {ok, S};
            case lists:all(fun({File2, Offset2, Chunk, CSum}) ->
                                   {_Tag, CS} = machi_util:unmake_tagged_csum(CSum),
                                   CS2 = machi_util:checksum_chunk(Chunk),
                                   if CS == CS2 ->
                                           true;
                                      CS /= CS2 ->
                                           ?ERROR("Client-side checksum error for file ~p offset ~p expected ~p got ~p\n", [File2, Offset2, CS, CS2]),
                                           false
                                   end
                           end, Chunks) of
                true ->
            {ok, S};
                false ->
                    {error, bad_checksum, S}
            end;
        {error, _}=Err ->
            ?ERROR("read file ~p offset ~w size ~w: ~w\n",
                   [File, Offset, Size, Err]),
@ -153,40 +132,21 @@ find_server_info(_Id) ->
            Ps
    end.
 load_ets_table_maybe(Conn, ETS) ->
    case basho_bench_config:get(operations, undefined) of
        undefined ->
            ?ERROR("The 'operations' key is missing from the config file, aborting", []),
            exit(bad_config);
        Ops when is_list(Ops) ->
            case lists:keyfind(read, 1, Ops) of
                {read,_} ->
                    load_ets_table(Conn, ETS);
                false ->
                    ?INFO("No 'read' op in the 'operations' list ~p, skipping ETS table load.", [Ops]),
                    0
            end
    end.
 load_ets_table(Conn, ETS) ->
    {ok, Fs} = machi_cr_client:list_files(Conn),
    [begin
-         {ok, InfoBin} = machi_cr_client:checksum_list(Conn, File, ?THE_TIMEOUT),
+         {ok, InfoBin} = machi_cr_client:checksum_list(Conn, File),
-         PosList = machi_csum_table:split_checksum_list_blob_decode(InfoBin),
+         {PosList, _} = machi_csum_table:split_checksum_list_blob_decode(InfoBin),
         ?INFO("File ~s len PosList ~p\n", [File, length(PosList)]),
         StartKey = ets:update_counter(ETS, max_key, 0),
-         {_, C, Bytes} = lists:foldl(fun({_Off,0,_CSum}, {_K, _C, _Bs}=Acc) ->
+         %% _EndKey = lists:foldl(fun({Off,Sz,CSum}, K) ->
-                                             Acc;
+         %%                               V = {File, Off, Sz, CSum},
-                                        ({0,_Sz,_CSum}, {_K, _C, _Bs}=Acc) ->
+         {_, Bytes} = lists:foldl(fun({Off,Sz,_CSum}, {K, Bs}) ->
                                             Acc;
                                        ({Off,Sz,_CSum}, {K, C, Bs}) ->
                                          V = {File, Off, Sz},
                                          ets:insert(ETS, {K, V}),
-                                             {K + 1, C + 1, Bs + Sz}
+                                          {K + 1, Bs + Sz}
-                                     end, {StartKey, 0, 0}, PosList),
+                                  end, {StartKey, 0}, PosList),
-         _ = ets:update_counter(ETS, max_key, C),
+         ets:update_counter(ETS, max_key, length(PosList)),
-         _ = ets:update_counter(ETS, total_bytes, Bytes),
+         ets:update_counter(ETS, total_bytes, Bytes)
         ok
     end || {_Size, File} <- Fs],
    ets:update_counter(?ETS_TAB, max_key, 0).
--- a/src/machi_chain_manager1.erl
+++ b/src/machi_chain_manager1.erl
@ -92,11 +92,8 @@
 -define(REPAIR_START_STABILITY_TIME, 10).
 -endif. % TEST
-%% Maximum length of the history of adopted projections (via C120).
+%% Magic constant for looping "too frequently" breaker.  TODO revisit & revise.
-define(MAX_HISTORY_LENGTH, 8).
+-define(TOO_FREQUENT_BREAKER, 10).
 %% Magic constant for looping "too frequently" breaker.
 -define(TOO_FREQUENT_BREAKER, (?MAX_HISTORY_LENGTH+5)).
 -define(RETURN2(X), begin (catch put(why2, [?LINE|get(why2)])), X end).
@ -106,9 +103,12 @@
 %% Amount of epoch number skip-ahead for set_chain_members call
 -define(SET_CHAIN_MEMBERS_EPOCH_SKIP, 1111).
 %% Maximum length of the history of adopted projections (via C120).
 -define(MAX_HISTORY_LENGTH, 30).
 %% API
 -export([start_link/2, start_link/3, stop/1, ping/1,
-         set_chain_members/2, set_chain_members/6, set_active/2,
+         set_chain_members/2, set_chain_members/3, set_active/2,
         trigger_react_to_env/1]).
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, format_status/2, code_change/3]).
@ -168,22 +168,13 @@ ping(Pid) ->
 %% with lowest rank, i.e. name z* first, name a* last.
 set_chain_members(Pid, MembersDict) ->
-    set_chain_members(Pid, ch0_name, 0, ap_mode, MembersDict, []).
+    set_chain_members(Pid, MembersDict, []).
-set_chain_members(Pid, ChainName, OldEpoch, CMode, MembersDict, Witness_list)
+set_chain_members(Pid, MembersDict, Witness_list) ->
-  when is_atom(ChainName) andalso
+    case lists:all(fun(Witness) -> orddict:is_key(Witness, MembersDict) end,
       is_integer(OldEpoch) andalso OldEpoch >= 0 andalso
       (CMode == ap_mode orelse CMode == cp_mode) andalso
       is_list(MembersDict) andalso
       is_list(Witness_list) ->
    case lists:all(fun({X, #p_srvr{name=X}}) -> true;
                      (_)                    -> false
                   end, MembersDict)
         andalso
         lists:all(fun(Witness) -> orddict:is_key(Witness, MembersDict) end,
                   Witness_list) of
        true ->
-            Cmd = {set_chain_members, ChainName, OldEpoch, CMode, MembersDict, Witness_list},
+            Cmd = {set_chain_members, MembersDict, Witness_list},
            gen_server:call(Pid, Cmd, infinity);
        false ->
            {error, bad_arg}
@ -234,13 +225,11 @@ test_read_latest_public_projection(Pid, ReadRepairP) ->
 %% manager's pid in MgrOpts and use direct gen_server calls to the
 %% local projection store.
-init({MyName, InitMembersDict, MgrOpts0}) ->
+init({MyName, InitMembersDict, MgrOpts}) ->
    put(ttt, [?LINE]),
    _ = random:seed(now()),
    init_remember_down_list(),
    MgrOpts = MgrOpts0 ++ application:get_env(machi, chain_manager_opts, []),
    Opt = fun(Key, Default) -> proplists:get_value(Key, MgrOpts, Default) end,
    InitWitness_list = Opt(witnesses, []),
    ZeroAll_list = [P#p_srvr.name || {_,P} <- orddict:to_list(InitMembersDict)],
    ZeroProj = make_none_projection(0, MyName, ZeroAll_list,
@ -292,7 +281,7 @@ init({MyName, InitMembersDict, MgrOpts0}) ->
                         last_down=[no_such_server_initial_value_only],
                         fitness_svr=machi_flu_psup:make_fitness_regname(MyName)
                        }, Proj),
-    S2 = do_set_chain_members_dict(MembersDict, S),
+    {_, S2} = do_set_chain_members_dict(MembersDict, S),
    S3 = if ActiveP == false ->
                 S2;
            ActiveP == true ->
@ -302,17 +291,12 @@ init({MyName, InitMembersDict, MgrOpts0}) ->
 handle_call({ping}, _From, S) ->
    {reply, pong, S};
-handle_call({set_chain_members, SetChainName, SetOldEpoch, CMode,
+handle_call({set_chain_members, MembersDict, Witness_list}, _From,
             MembersDict, Witness_list}, _From,
            #ch_mgr{name=MyName,
                    proj=#projection_v1{all_members=OldAll_list,
                                        epoch_number=OldEpoch,
                                        chain_name=ChainName,
                                        upi=OldUPI}=OldProj}=S) ->
-    true = (OldEpoch == 0) % in this case we want unconditional set of ch name
+    {Reply, S2} = do_set_chain_members_dict(MembersDict, S),
           orelse
           (SetOldEpoch == OldEpoch andalso SetChainName == ChainName),
    S2 = do_set_chain_members_dict(MembersDict, S),
    %% TODO: should there be any additional sanity checks?  Right now,
    %% if someone does something bad, then do_react_to_env() will
    %% crash, which will crash us, and we'll restart in a sane & old
@ -326,10 +310,10 @@ handle_call({set_chain_members, SetChainName, SetOldEpoch, CMode,
                                {NUPI, All_list -- NUPI}
                        end,
    NewEpoch = OldEpoch + ?SET_CHAIN_MEMBERS_EPOCH_SKIP,
    CMode = calc_consistency_mode(Witness_list),
    ok = set_consistency_mode(machi_flu_psup:make_proj_supname(MyName), CMode),
    NewProj = machi_projection:update_checksum(
                OldProj#projection_v1{author_server=MyName,
                                      chain_name=SetChainName,
                                      creation_time=now(),
                                      mode=CMode,
                                      epoch_number=NewEpoch,
@ -341,11 +325,7 @@ handle_call({set_chain_members, SetChainName, SetOldEpoch, CMode,
                                      members_dict=MembersDict}),
    S3 = set_proj(S2#ch_mgr{proj_history=queue:new(),
                            consistency_mode=CMode}, NewProj),
-    {Res, S4} = do_react_to_env(S3),
+    {_QQ, S4} = do_react_to_env(S3),
    Reply = case Res of
                {_,_,_} -> ok
                % Dialyzer says that all vals match with the 3-tuple pattern
            end,
    {reply, Reply, S4};
 handle_call({set_active, Boolean}, _From, #ch_mgr{timer=TRef}=S) ->
    case {Boolean, TRef} of
@ -377,8 +357,8 @@ handle_call({test_read_latest_public_projection, ReadRepairP}, _From, S) ->
    {reply, Res, S2};
 handle_call({trigger_react_to_env}=Call, _From, S) ->
    gobble_calls(Call),
-    {Res, S2} = do_react_to_env(S),
+    {TODOtodo, S2} = do_react_to_env(S),
-    {reply, Res, S2};
+    {reply, TODOtodo, S2};
 handle_call(_Call, _From, S) ->
    io:format(user, "\nBad call to ~p: ~p\n", [S#ch_mgr.name, _Call]),
    {reply, whaaaaaaaaaa, S}.
@ -390,7 +370,6 @@ handle_cast(_Cast, S) ->
 handle_info(tick_check_environment, #ch_mgr{ignore_timer=true}=S) ->
    {noreply, S};
 handle_info(tick_check_environment, S) ->
    gobble_ticks(),
    {{_Delta, Props, _Epoch}, S1} = do_react_to_env(S),
    S2 = sanitize_repair_state(S1),
    S3 = perhaps_start_repair(S2),
@ -463,7 +442,7 @@ get_my_proj_boot_info(MgrOpts, DefaultDict, DefaultProj, ProjType) ->
            {DefaultDict, DefaultProj};
        Store ->
            {ok, P} = machi_projection_store:read_latest_projection(Store,
-                                                                    ProjType, 7789),
+                                                                    ProjType),
            {P#projection_v1.members_dict, P}
    end.
@ -556,7 +535,6 @@ cl_write_public_proj2(FLUs, Partitions, Epoch, Proj, IgnoreWrittenErrorP, S) ->
                   end
           end, {true, []}, FLUs),
    %% io:format(user, "\nWrite public ~w by ~w: ~w\n", [Epoch, S#ch_mgr.name, Rs]),
    %% io:format(user, "mgr ~w epoch ~w Rs ~p\n", [S#ch_mgr.name, Epoch, Rs]),
    {{remote_write_results, Rs}, S}.
 do_cl_read_latest_public_projection(ReadRepairP,
@ -578,41 +556,12 @@ do_cl_read_latest_public_projection(ReadRepairP,
 read_latest_projection_call_only(ProjectionType, AllHosed,
                                 #ch_mgr{proj=CurrentProj}=S) ->
    #projection_v1{all_members=All_list} = CurrentProj,
-    All_queried_list = lists:sort(All_list -- AllHosed),
+    All_queried_list = All_list -- AllHosed,
    read_latest_projection_call_only1(ProjectionType, AllHosed,
                                      All_queried_list, S).
-read_latest_projection_call_only1(ProjectionType, AllHosed,
+    {Rs, S2} = read_latest_projection_call_only2(ProjectionType,
                                  All_queried_list, S) ->
    {Rs_tmp, S2} = read_latest_projection_call_only2(ProjectionType,
                                                 All_queried_list, S),
-    New_all_maybe =
+    FLUsRs = lists:zip(All_queried_list, Rs),
-        lists:usort(
+    {All_queried_list, FLUsRs, S2}.
          lists:flatten(
            [A_l || #projection_v1{all_members=A_l} <- Rs_tmp])) -- AllHosed,
    case New_all_maybe -- All_queried_list of
        [] ->
            FLUsRs = lists:zip(All_queried_list, Rs_tmp),
            {All_queried_list, FLUsRs, S2};
        [AnotherFLU|_] ->
            %% Stop AnotherFLU proxy, in unexpected case where it's open
            try
                Proxy = proxy_pid(AnotherFLU, S2),
                ?FLU_PC:stop_proxies([Proxy])
            catch _:_ -> ok
            end,
            MD = orddict:from_list(
                  lists:usort(
                   lists:flatten(
                    [orddict:to_list(D) || #projection_v1{members_dict=D} <- Rs_tmp]))),
            Another_P_srvr = orddict:fetch(AnotherFLU, MD),
            {ok, Proxy2} = ?FLU_PC:start_link(Another_P_srvr),
            S3 = S2#ch_mgr{proxies_dict=orddict:store(AnotherFLU, Proxy2,
                                                      S2#ch_mgr.proxies_dict)},
            read_latest_projection_call_only1(
              ProjectionType, AllHosed,
              lists:usort([AnotherFLU|All_queried_list]), S3)
    end.
 read_latest_projection_call_only2(ProjectionType, All_queried_list, S) ->
    {_UpNodes, Partitions, S2} = calc_up_nodes(S),
@ -652,8 +601,6 @@ rank_and_sort_projections_with_extra(All_queried_list, FLUsRs, ProjectionType,
            Witness_list = CurrentProj#projection_v1.witnesses,
            NoneProj = make_none_projection(0, MyName, [], Witness_list,
                                            orddict:new()),
            ChainName = CurrentProj#projection_v1.chain_name,
            NoneProj2 = NoneProj#projection_v1{chain_name=ChainName},
            Extra2 = [{all_members_replied, true},
                      {all_queried_list, All_queried_list},
                      {flus_rs, FLUsRs},
@ -662,7 +609,7 @@ rank_and_sort_projections_with_extra(All_queried_list, FLUsRs, ProjectionType,
                      {bad_answer_flus, BadAnswerFLUs},
                      {bad_answers, BadAnswers},
                      {not_unanimous_answers, []}],
-            {not_unanimous, NoneProj2, Extra2, S};
+            {not_unanimous, NoneProj, Extra2, S};
       ProjectionType == public, UnwrittenRs /= [] ->
            {needs_repair, FLUsRs, [flarfus], S};
       true ->
@ -776,14 +723,13 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
                         runenv=RunEnv1,
                         repair_final_status=RepairFS}=S) ->
    #projection_v1{epoch_number=OldEpochNum,
                   chain_name=ChainName,
                   members_dict=MembersDict,
                   witnesses=OldWitness_list,
                   upi=OldUPI_list,
                   repairing=OldRepairing_list
                  } = LastProj,
    LastUp = lists:usort(OldUPI_list ++ OldRepairing_list),
-    AllMembers = CurrentProj#projection_v1.all_members,
+    AllMembers = (S#ch_mgr.proj)#projection_v1.all_members,
    {Up0, Partitions, RunEnv2} = calc_up_nodes(MyName,
                                               AllMembers, RunEnv1),
    Up = Up0 -- AllHosed,
@ -840,10 +786,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
                        D_foo=[{repair_done, {repair_final_status, ok, (S#ch_mgr.proj)#projection_v1.epoch_number}}],
                        {NewUPI_list ++ Repairing_list2, [], RunEnv2};
                   true ->
-                        D_foo=[d_foo2, {sim_p,Simulator_p},
+                        D_foo=[d_foo2],
                               {simr_p,SimRepair_p}, {same_epoch,SameEpoch_p},
                               {rel_to,RelativeToServer},
                               {repch,RepChk_LastInUPI}, {repair_fs,RepairFS}],
                        {NewUPI_list, OldRepairing_list, RunEnv2}
                end;
            {_ABC, _XYZ} ->
@ -878,11 +821,10 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
        end,
    ?REACT({calc,?LINE,[{new_upi, NewUPI},{new_rep, NewRepairing}]}),
-    P0 = machi_projection:new(OldEpochNum + 1,
+    P = machi_projection:new(OldEpochNum + 1,
                             MyName, MembersDict, Down, NewUPI, NewRepairing,
                             D_foo ++
                                 Dbg ++ [{ps, Partitions},{nodes_up, Up}]),
    P1 = P0#projection_v1{chain_name=ChainName},
    P2 = if CMode == cp_mode ->
                 UpWitnesses = [W || W <- Up, lists:member(W, OldWitness_list)],
                 Majority = full_majority_size(AllMembers),
@ -891,7 +833,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
                 SoFar = length(NewUPI ++ NewRepairing),
                 if SoFar >= Majority ->
                         ?REACT({calc,?LINE,[]}),
-                         P1;
+                         P;
                    true ->
                         Need = Majority - SoFar,
                         UpWitnesses = [W || W <- Up,
@ -900,7 +842,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
                                 Ws = lists:sublist(UpWitnesses, Need),
                                 ?REACT({calc,?LINE,[{ws, Ws}]}),
                                 machi_projection:update_checksum(
-                                   P1#projection_v1{upi=Ws++NewUPI});
+                                   P#projection_v1{upi=Ws++NewUPI});
                            true ->
                                 ?REACT({calc,?LINE,[]}),
                                 P_none0 = make_none_projection(
@ -913,7 +855,6 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
                                               "Not enough witnesses are available now"
                                       end,
                                 P_none1 = P_none0#projection_v1{
                                             chain_name=ChainName,
                                             %% Stable creation time!
                                             creation_time={1,2,3},
                                             dbg=[{none_projection,true},
@ -934,7 +875,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
                 end;
            CMode == ap_mode ->
                 ?REACT({calc,?LINE,[]}),
-                 P1
+                 P
         end,
    P3 = machi_projection:update_checksum(
           P2#projection_v1{mode=CMode, witnesses=OldWitness_list}),
@ -1086,30 +1027,28 @@ rank_projection(#projection_v1{author_server=_Author,
 do_set_chain_members_dict(MembersDict, #ch_mgr{proxies_dict=OldProxiesDict}=S)->
    _ = ?FLU_PC:stop_proxies(OldProxiesDict),
    ProxiesDict = ?FLU_PC:start_proxies(MembersDict),
-    S#ch_mgr{members_dict=MembersDict,
+    {ok, S#ch_mgr{members_dict=MembersDict,
-             proxies_dict=ProxiesDict}.
+                  proxies_dict=ProxiesDict}}.
 do_react_to_env(#ch_mgr{name=MyName,
                        proj=#projection_v1{epoch_number=Epoch,
                                            members_dict=[]=OldDict}=OldProj,
                        opts=Opts}=S) ->
    put(ttt, [?LINE]),
    %% Read from our local *public* projection store.  If some other
    %% chain member has written something there, and if we are a
    %% member of that chain, then we'll adopt that projection and then
    %% start actively humming in that chain.
-    {NewMD, NewProj} =
+    {NewMembersDict, NewProj} =
        get_my_public_proj_boot_info(Opts, OldDict, OldProj),
-    case orddict:is_key(MyName, NewMD) of
+    case orddict:is_key(MyName, NewMembersDict) of
        false ->
-            {{empty_members_dict1, [], Epoch}, S};
+            {{empty_members_dict, [], Epoch}, S};
        true ->
-            CMode = NewProj#projection_v1.mode,
+            {_, S2} = do_set_chain_members_dict(NewMembersDict, S),
-            S2 = do_set_chain_members_dict(NewMD, S),
+            CMode = calc_consistency_mode(NewProj#projection_v1.witnesses),
-            {Reply, S3} = react_to_env_C110(NewProj,
+            {{empty_members_dict, [], Epoch},
-                                            S2#ch_mgr{members_dict=NewMD,
+             set_proj(S2#ch_mgr{members_dict=NewMembersDict,
-                                                      consistency_mode=CMode}),
+                                consistency_mode=CMode}, NewProj)}
            {Reply, S3}
    end;
 do_react_to_env(S) ->
 put(ttt, [?LINE]),
@ -1211,7 +1150,7 @@ react_to_env_A10(S) ->
    ?REACT(a10),
    react_to_env_A20(0, poll_private_proj_is_upi_unanimous(S)).
-react_to_env_A20(Retries, #ch_mgr{name=MyName, proj=P_current}=S) ->
+react_to_env_A20(Retries, #ch_mgr{name=MyName}=S) ->
    ?REACT(a20),
    init_remember_down_list(),
    {UnanimousTag, P_latest, ReadExtra, S2} =
@ -1239,34 +1178,17 @@ react_to_env_A20(Retries, #ch_mgr{name=MyName, proj=P_current}=S) ->
        false when P_latest#projection_v1.epoch_number /= LastComplaint,
                   P_latest#projection_v1.all_members /= [] ->
            put(rogue_server_epoch, P_latest#projection_v1.epoch_number),
-            error_logger:info_msg("Chain manager ~w found latest public "
+            error_logger:info_msg("Chain manager ~p found latest public "
-                                  "projection ~w with author ~w has a "
+                                  "projection ~p has author ~p has a "
-                                  "members list ~w that does not include me.  "
+                                  "members list ~p that does not include me.\n",
                                  "We assume this is a result of administrator "
                                  "action and will thus wedge ourselves until "
                                  "we are re-added to the chain or shutdown.\n",
                                  [S#ch_mgr.name,
                                   P_latest#projection_v1.epoch_number,
                                   P_latest#projection_v1.author_server,
-                                   P_latest#projection_v1.all_members]),
+                                   P_latest#projection_v1.all_members]);
            EpochID = machi_projection:make_epoch_id(P_current),
            ProjStore = get_projection_store_pid_or_regname(S),
            {ok, NotifyPid} = machi_projection_store:get_wedge_notify_pid(ProjStore),
            _QQ = machi_flu1:update_wedge_state(NotifyPid, true, EpochID),
            #projection_v1{epoch_number=Epoch,
                           chain_name=ChainName,
                           all_members=All_list,
                           witnesses=Witness_list,
                           members_dict=MembersDict} = P_current,
            P_none0 = make_none_projection(Epoch,
                                           MyName, All_list, Witness_list, MembersDict),
            P_none = P_none0#projection_v1{chain_name=ChainName},
            {{now_using,[],Epoch}, set_proj(S2, P_none)};
        _ ->
-            react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S2)
+            ok
-    end.
+    end,
 react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S) ->
    %% The UnanimousTag isn't quite sufficient for our needs.  We need
    %% to determine if *all* of the UPI+Repairing FLUs are members of
    %% the unanimous server replies.  All Repairing FLUs should be up
@ -1311,7 +1233,7 @@ react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S) ->
           true ->
                exit({badbad, UnanimousTag})
        end,
-    react_to_env_A29(Retries, P_latest, LatestUnanimousP, ReadExtra, S).
+    react_to_env_A29(Retries, P_latest, LatestUnanimousP, ReadExtra, S2).
 react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra,
                 #ch_mgr{consistency_mode=CMode,
@ -1345,6 +1267,7 @@ react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra,
                    ?REACT({a29, ?LINE,
                            [{zerf_backstop, true},
                             {zerf_in, machi_projection:make_summary(Zerf)}]}),
                    %% io:format(user, "zerf_in: A29: ~p: ~w\n\t~p\n", [MyName,  machi_projection:make_summary(Zerf), get(yyy_hack)]),
                    #projection_v1{dbg=ZerfDbg} = Zerf,
                    Backstop = if Zerf#projection_v1.upi == [] ->
                                       [];
@ -1364,8 +1287,7 @@ react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra,
    end.
 react_to_env_A30(Retries, P_latest, LatestUnanimousP, P_current_calc,
-                 #ch_mgr{name=MyName, proj=P_current,
+                 #ch_mgr{name=MyName, consistency_mode=CMode} = S) ->
                         consistency_mode=CMode} = S) ->
    V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end,
    if V -> io:format(user, "A30: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
    ?REACT(a30),
@ -1385,17 +1307,15 @@ react_to_env_A30(Retries, P_latest, LatestUnanimousP, P_current_calc,
            P = #projection_v1{down=Down} =
                make_none_projection(Epoch + 1, MyName, All_list,
                                     Witness_list, MembersDict),
            ChainName = P_current#projection_v1.chain_name,
            P1 = P#projection_v1{chain_name=ChainName},
            P_newprop = if CMode == ap_mode ->
                                %% Not really none proj: just myself, AP style
                                machi_projection:update_checksum(
-                                  P1#projection_v1{upi=[MyName],
+                                  P#projection_v1{upi=[MyName],
                                                  down=Down -- [MyName],
                                                  dbg=[{hosed_list,AllHosed}]});
                           CMode == cp_mode ->
                                machi_projection:update_checksum(
-                                  P1#projection_v1{dbg=[{hosed_list,AllHosed}]})
+                                  P#projection_v1{dbg=[{hosed_list,AllHosed}]})
                        end,
            react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP,
                             P_current_calc, true, S);
@ -1468,22 +1388,13 @@ react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP,
                %% we have a disagreement.
                not ordsets:is_disjoint(P_latest_s, Down_s)
        end,
    AmExcludedFromLatestAll_p =
        P_latest#projection_v1.epoch_number /= 0
        andalso
        (not lists:member(MyName, P_latest#projection_v1.all_members)),
    ?REACT({a40, ?LINE,
            [{latest_author, P_latest#projection_v1.author_server},
             {am_excluded_from_latest_all_p, AmExcludedFromLatestAll_p},
             {author_is_down_p, LatestAuthorDownP},
             {rank_latest, Rank_latest},
             {rank_newprop, Rank_newprop}]}),
    if
        AmExcludedFromLatestAll_p ->
            ?REACT({a40, ?LINE, [{latest,machi_projection:make_summary(P_latest)}]}),
            react_to_env_A50(P_latest, [], S);
        AmHosedP ->
            ExpectedUPI = if CMode == cp_mode -> [];
                             CMode == ap_mode -> [MyName]
@ -1649,10 +1560,12 @@ react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP,
                       end,
            if GoTo50_p ->
                    ?REACT({a40, ?LINE, []}),
 %% io:format(user, "CONFIRM debug question line ~w\n", [?LINE]),
                    FinalProps = [{throttle_seconds, 0}],
                    react_to_env_A50(P_latest, FinalProps, S);
               true ->
                    ?REACT({a40, ?LINE, []}),
 io:format(user, "CONFIRM debug question line ~w\n", [?LINE]),
                    react_to_env_C300(P_newprop, P_latest, S)
            end
    end.
@ -1662,6 +1575,7 @@ react_to_env_A50(P_latest, FinalProps, #ch_mgr{proj=P_current}=S) ->
    ?REACT({a50, ?LINE, [{current_epoch, P_current#projection_v1.epoch_number},
                         {latest_epoch, P_latest#projection_v1.epoch_number},
                         {final_props, FinalProps}]}),
    %% if S#ch_mgr.name == c -> io:format(user, "A50: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
    V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end,
    if V -> io:format(user, "A50: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
    {{no_change, FinalProps, P_current#projection_v1.epoch_number}, S}.
@ -1915,7 +1829,7 @@ react_to_env_C100_inner(Author_latest, NotSanesDict0, _MyName,
    S2 = S#ch_mgr{not_sanes=NotSanesDict, sane_transitions=0},
    case orddict:fetch(Author_latest, NotSanesDict) of
        N when N > ?TOO_FREQUENT_BREAKER ->
-            ?V("\n\nYOYO ~w breaking the cycle insane-freq=~w by-author=~w of:\n  current: ~w\n  new    : ~w\n", [_MyName, N, Author_latest, machi_projection:make_summary(S#ch_mgr.proj), machi_projection:make_summary(P_latest)]),
+            %% ?V("\n\nYOYO ~w breaking the cycle of:\n  current: ~w\n  new    : ~w\n", [_MyName, machi_projection:make_summary(S#ch_mgr.proj), machi_projection:make_summary(P_latest)]),
            ?REACT({c100, ?LINE, [{not_sanes_author_count, N}]}),
            react_to_env_C103(P_newprop, P_latest, P_current_calc, S2);
        N ->
@ -1936,14 +1850,12 @@ react_to_env_C103(#projection_v1{epoch_number=_Epoch_newprop} = _P_newprop,
                   members_dict=MembersDict} = P_current,
    P_none0 = make_none_projection(Epoch_latest,
                                   MyName, All_list, Witness_list, MembersDict),
-    ChainName = P_current#projection_v1.chain_name,
+    P_none1 = P_none0#projection_v1{dbg=[{none_projection,true}]},
    P_none1 = P_none0#projection_v1{chain_name=ChainName,
                                    dbg=[{none_projection,true}]},
    P_none = machi_projection:update_checksum(P_none1),
    ?REACT({c103, ?LINE,
            [{current_epoch, P_current#projection_v1.epoch_number},
             {none_projection_epoch, P_none#projection_v1.epoch_number}]}),
-    io:format(user, "SET add_admin_down(~w) at ~w current_epoch ~w none_proj_epoch ~w =====================================\n", [MyName, time(), P_current#projection_v1.epoch_number, P_none#projection_v1.epoch_number]),
+    io:format(user, "SET add_admin_down(~w) at ~w =====================================\n", [MyName, time()]),
    machi_fitness:add_admin_down(S#ch_mgr.fitness_svr, MyName, []),
    timer:sleep(5*1000),
    io:format(user, "SET delete_admin_down(~w) at ~w =====================================\n", [MyName, time()]),
@ -1980,7 +1892,7 @@ react_to_env_C110(P_latest, #ch_mgr{name=MyName} = S) ->
    %% In contrast to the public projection store writes, Humming Consensus
    %% doesn't care about the status of writes to the public store: it's
    %% always relying only on successful reads of the public store.
-    case {?FLU_PC:write_projection(MyStorePid, private, P_latest2,?TO*30+66),Goo} of
+    case {?FLU_PC:write_projection(MyStorePid, private, P_latest2,?TO*30),Goo} of
        {ok, Goo} ->
            ?REACT({c110, [{write, ok}]}),
            react_to_env_C111(P_latest, P_latest2, Extra1, MyStorePid, S);
@ -2066,6 +1978,7 @@ react_to_env_C120(P_latest, FinalProps, #ch_mgr{proj_history=H,
    ?REACT(c120),
    H2   = add_and_trunc_history(P_latest, H, ?MAX_HISTORY_LENGTH),
    %% diversion_c120_verbose_goop(P_latest, S),
    ?REACT({c120, [{latest, machi_projection:make_summary(P_latest)}]}),
    S2 = set_proj(S#ch_mgr{proj_history=H2,
                           sane_transitions=Xtns + 1}, P_latest),
@ -2073,21 +1986,20 @@ react_to_env_C120(P_latest, FinalProps, #ch_mgr{proj_history=H,
             false ->
                 S2;
             {{_ConfEpoch, _ConfCSum}, ConfTime} ->
-                 P_latestEpoch = P_latest#projection_v1.epoch_number,
+                 io:format(user, "\nCONFIRM debug C120 ~w was annotated ~w\n", [S#ch_mgr.name, P_latest#projection_v1.epoch_number]),
                 io:format(user, "\nCONFIRM debug C120 ~w was annotated ~w\n", [S#ch_mgr.name, P_latestEpoch]),
                 S2#ch_mgr{proj_unanimous=ConfTime}
         end,
    V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end,
    if V -> io:format("C120: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
    {{now_using, FinalProps, P_latest#projection_v1.epoch_number}, S3}.
-add_and_trunc_history(#projection_v1{epoch_number=0}, H, _MaxLength) ->
+add_and_trunc_history(P_latest, H, MaxLength) ->
    H;
 add_and_trunc_history(#projection_v1{} = P_latest, H, MaxLength) ->
    Latest_U_R = {P_latest#projection_v1.upi, P_latest#projection_v1.repairing},
-    add_and_trunc_history(Latest_U_R, H, MaxLength);
+    H2 = if P_latest#projection_v1.epoch_number > 0 ->
-add_and_trunc_history(Item, H, MaxLength) ->
+                 queue:in(Latest_U_R, H);
-    H2 = queue:in(Item, H),
+            true ->
                 H
         end,
    case queue:len(H2) of
        X when X > MaxLength ->
            {_V, Hxx} = queue:out(H2),
@ -2100,9 +2012,10 @@ react_to_env_C200(Retries, P_latest, S) ->
    ?REACT(c200),
    try
        AuthorProxyPid = proxy_pid(P_latest#projection_v1.author_server, S),
-        %% This is just advisory, we don't need a sync reply.
+        ?FLU_PC:kick_projection_reaction(AuthorProxyPid, [])
        ?FLU_PC:kick_projection_reaction(AuthorProxyPid, [], 100)
    catch _Type:_Err ->
            %% ?V("TODO: tell_author_yo is broken: ~p ~p\n",
            %%           [_Type, _Err]),
            ok
    end,
    react_to_env_C210(Retries, S).
@ -2293,7 +2206,6 @@ projection_transition_is_sane_except_si_epoch(
                 creation_time=CreationTime1,
                 mode=CMode1,
                 author_server=AuthorServer1,
                 chain_name=ChainName1,
                 all_members=All_list1,
                 witnesses=Witness_list1,
                 down=Down_list1,
@ -2305,7 +2217,6 @@ projection_transition_is_sane_except_si_epoch(
                 creation_time=CreationTime2,
                 mode=CMode2,
                 author_server=AuthorServer2,
                 chain_name=ChainName2,
                 all_members=All_list2,
                 witnesses=Witness_list2,
                 down=Down_list2,
@ -2326,8 +2237,7 @@ projection_transition_is_sane_except_si_epoch(
    true = is_binary(CSum1) andalso is_binary(CSum2),
    {_,_,_} = CreationTime1,
    {_,_,_} = CreationTime2,
-    true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2),
+    true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2), % todo type may change?
    true = is_atom(ChainName1) andalso is_atom(ChainName2),
    true = is_list(All_list1) andalso is_list(All_list2),
    true = is_list(Witness_list1) andalso is_list(Witness_list2),
    true = is_list(Down_list1) andalso is_list(Down_list2),
@ -2339,9 +2249,6 @@ projection_transition_is_sane_except_si_epoch(
    %% projection_transition_is_sane_with_si_epoch().
    true = Epoch2 >= Epoch1,
    %% Don't change chain names in the middle of the stream.
    true = (ChainName1 == ChainName2),
    %% No duplicates
    true = lists:sort(Witness_list2) == lists:usort(Witness_list2),
    true = lists:sort(Down_list2) == lists:usort(Down_list2),
@ -2349,7 +2256,7 @@ projection_transition_is_sane_except_si_epoch(
    true = lists:sort(Repairing_list2) == lists:usort(Repairing_list2),
    %% Disjoint-ness
-    %% %% %% %% %% %% %% %% All_list1 = All_list2,                 % todo will probably change
+    All_list1 = All_list2,                 % todo will probably change
    %% true = lists:sort(All_list2) == lists:sort(Down_list2 ++ UPI_list2 ++
    %%                                                Repairing_list2),
    [] = [X || X <- Witness_list2, not lists:member(X, All_list2)],
@ -2454,7 +2361,8 @@ poll_private_proj_is_upi_unanimous_sleep(Count, #ch_mgr{runenv=RunEnv}=S) ->
            S2
    end.
-poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current} = S) ->
+poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current,
                                            opts=MgrOpts} = S) ->
    UPI = P_current#projection_v1.upi,
    EpochID = machi_projection:make_epoch_id(P_current),
    {Rs, S2} = read_latest_projection_call_only2(private, UPI, S),
@ -2487,30 +2395,33 @@ poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current} = S) ->
                    Annotation = make_annotation(EpochID, Now),
                    NewDbg2 = [Annotation|P_currentFull#projection_v1.dbg2],
                    NewProj = P_currentFull#projection_v1{dbg2=NewDbg2},
-                    ProjStore = get_projection_store_pid_or_regname(S),
+                    ProjStore = case get_projection_store_regname(MgrOpts) of
                                   undefined ->
                                       machi_flu_psup:make_proj_supname(MyName);
                                   PStr ->
                                       PStr
                                end,
                    #projection_v1{epoch_number=_EpochRep,
                                   epoch_csum= <<_CSumRep:4/binary,_/binary>>,
                                   author_server=AuthRep,
                                   upi=_UPIRep,
                                   repairing=_RepairingRep} = NewProj,
                    ok = machi_projection_store:write(ProjStore, private, NewProj),
-                    case proplists:get_value(private_write_verbose_confirm, S#ch_mgr.opts) of
+                    case proplists:get_value(private_write_verbose, S#ch_mgr.opts) of
                        true ->
-                            error_logger:info_msg("CONFIRM epoch ~w ~w upi ~w rep ~w auth ~w by ~w\n", [_EpochRep, _CSumRep, _UPIRep, _RepairingRep, AuthRep, MyName]);
+                            io:format(user, "\n~s CONFIRM epoch ~w ~w upi ~w rep ~w by ~w\n", [machi_util:pretty_time(), _EpochRep, _CSumRep, _UPIRep, _RepairingRep, MyName]);
                        _ ->
                            ok
                    end,
                    %% Unwedge our FLU.
                    {ok, NotifyPid} = machi_projection_store:get_wedge_notify_pid(ProjStore),
                    _ = machi_flu1:update_wedge_state(NotifyPid, false, EpochID),
-                    #ch_mgr{proj_history=H} = S2,
+                    S2#ch_mgr{proj_unanimous=Now};
                    H2 = add_and_trunc_history({confirm, Epoch}, H,
                                               ?MAX_HISTORY_LENGTH),
                    S2#ch_mgr{proj_unanimous=Now, proj_history=H2};
                _ ->
                    S2
            end;
        _Else ->
            %% io:format(user, "poll by ~w: want ~W got ~W\n",
            %%           [MyName, EpochID, 6, _Else, 8]),
            S2
    end.
@ -2546,14 +2457,6 @@ gobble_calls(StaticCall) ->
            ok
    end.
 gobble_ticks() ->
    receive
        tick_check_environment ->
            gobble_ticks()
    after 0 ->
            ok
    end.
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 perhaps_start_repair(#ch_mgr{name=MyName,
@ -2569,13 +2472,12 @@ perhaps_start_repair(#ch_mgr{name=MyName,
            %% RepairOpts = [{repair_mode, check}, verbose],
            RepairFun = fun() -> do_repair(S, RepairOpts, CMode) end,
            LastUPI = lists:last(UPI),
            StabilityTime = application:get_env(machi, stability_time, ?REPAIR_START_STABILITY_TIME),
            IgnoreStabilityTime_p = proplists:get_value(ignore_stability_time,
                                                        S#ch_mgr.opts, false),
            case timer:now_diff(os:timestamp(), Start) div 1000000 of
                N when MyName == LastUPI andalso
                       (IgnoreStabilityTime_p orelse
-                        N >= StabilityTime) ->
+                        N >= ?REPAIR_START_STABILITY_TIME) ->
                    {WorkerPid, _Ref} = spawn_monitor(RepairFun),
                    S#ch_mgr{repair_worker=WorkerPid,
                             repair_start=os:timestamp(),
@ -2616,8 +2518,8 @@ do_repair(#ch_mgr{name=MyName,
            T1 = os:timestamp(),
            RepairId = proplists:get_value(repair_id, Opts, id1),
            error_logger:info_msg(
-              "Repair ~w start: tail ~p of ~p -> ~p, ~p\n",
+              "Repair start: tail ~p of ~p -> ~p, ~p ID ~w\n",
-              [RepairId, MyName, UPI0, Repairing, RepairMode]),
+              [MyName, UPI0, Repairing, RepairMode, RepairId]),
            UPI = UPI0 -- Witness_list,
            Res = machi_chain_repair:repair(RepairMode, MyName, Repairing, UPI,
@ -2630,9 +2532,10 @@ do_repair(#ch_mgr{name=MyName,
                      end,
            Stats = [{K, ets:lookup_element(ETS, K, 2)} || K <- ETS_T_Keys],
            error_logger:info_msg(
-              "Repair ~w ~s: tail ~p of ~p finished ~p: "
+              "Repair ~s: tail ~p of ~p finished ~p repair ID ~w: "
-              "~p Stats: ~p\n",
+              "~p\nStats ~p\n",
-              [RepairId, Summary, MyName, UPI0, RepairMode, Res, Stats]),
+              [Summary, MyName, UPI0, RepairMode, RepairId,
               Res, Stats]),
            ets:delete(ETS),
            exit({repair_final_status, Res});
        _ ->
@ -2869,7 +2772,6 @@ full_majority_size(L) when is_list(L) ->
    full_majority_size(length(L)).
 make_zerf(#projection_v1{epoch_number=OldEpochNum,
                         chain_name=ChainName,
                         all_members=AllMembers,
                         members_dict=MembersDict,
                         witnesses=OldWitness_list
@ -2892,8 +2794,7 @@ make_zerf(#projection_v1{epoch_number=OldEpochNum,
                                     MyName, AllMembers, OldWitness_list,
                                     MembersDict),
            machi_projection:update_checksum(
-              P#projection_v1{chain_name=ChainName,
+              P#projection_v1{mode=cp_mode,
                              mode=cp_mode,
                              dbg2=[zerf_none,{up,Up},{maj,MajoritySize}]});
        true ->
            make_zerf2(OldEpochNum, Up, MajoritySize, MyName,
@ -2908,6 +2809,7 @@ make_zerf2(OldEpochNum, Up, MajoritySize, MyName, AllMembers, OldWitness_list,
        Proj2 = Proj#projection_v1{dbg2=[{make_zerf,Epoch},
                                         {yyy_hack, get(yyy_hack)},
                                         {up,Up},{maj,MajoritySize}]},
        %% io:format(user, "ZERF ~w\n",[machi_projection:make_summary(Proj2)]),
        Proj2
    catch
        throw:{zerf,no_common} ->
@ -2984,36 +2886,41 @@ zerf_find_last_annotated(FLU, MajoritySize, S) ->
            []                                  % lists:flatten() will destroy
    end.
-perhaps_verbose_c111(P_latest2, #ch_mgr{name=MyName, opts=Opts}=S) ->
+perhaps_verbose_c111(P_latest2, S) ->
-    PrivWriteVerb = proplists:get_value(private_write_verbose, Opts, false),
+    case proplists:get_value(private_write_verbose, S#ch_mgr.opts) of
-    PrivWriteVerbCONFIRM = proplists:get_value(private_write_verbose_confirm, Opts, false),
+        true ->
    if PrivWriteVerb orelse PrivWriteVerbCONFIRM ->
            Dbg2X = lists:keydelete(react, 1,
                                    P_latest2#projection_v1.dbg2) ++
                [{is_annotated,is_annotated(P_latest2)}],
            P_latest2x = P_latest2#projection_v1{dbg2=Dbg2X}, % limit verbose len.
            Last2 = get(last_verbose),
            Summ2 = machi_projection:make_summary(P_latest2x),
-            if PrivWriteVerb, Summ2 /= Last2 ->
+            if P_latest2#projection_v1.upi == [],
                    put(last_verbose, Summ2),
                    error_logger:info_msg("~p uses plain: ~w \n",
                       [MyName, Summ2]);
               true ->
                    ok
            end,
            if PrivWriteVerbCONFIRM,
               P_latest2#projection_v1.upi == [],
               (S#ch_mgr.proj)#projection_v1.upi /= [] ->
                    <<CSumRep:4/binary,_/binary>> =
                                          P_latest2#projection_v1.epoch_csum,
-                    error_logger:info_msg("CONFIRM epoch ~w ~w upi ~w rep ~w auth ~w by ~w\n", [(S#ch_mgr.proj)#projection_v1.epoch_number, CSumRep, P_latest2#projection_v1.upi, P_latest2#projection_v1.repairing, P_latest2#projection_v1.author_server, S#ch_mgr.name]);
+                    io:format(user, "\n~s CONFIRM epoch ~w ~w upi ~w rep ~w by ~w\n", [machi_util:pretty_time(), (S#ch_mgr.proj)#projection_v1.epoch_number, CSumRep, P_latest2#projection_v1.upi, P_latest2#projection_v1.repairing, S#ch_mgr.name]);
               true ->
                    ok
            end,
            case proplists:get_value(private_write_verbose,
                                     S#ch_mgr.opts) of
                true when Summ2 /= Last2 ->
                    put(last_verbose, Summ2),
                    ?V("\n~s ~p uses plain: ~w \n",
                       [machi_util:pretty_time(), S#ch_mgr.name, Summ2]);
                _ ->
                    ok
            end;
-       true ->
+        _ ->
            ok
    end.
 calc_consistency_mode(_Witness_list = []) ->
    ap_mode;
 calc_consistency_mode(_Witness_list) ->
    cp_mode.
 set_proj(S, Proj) ->
    S#ch_mgr{proj=Proj, proj_unanimous=false}.
@ -3046,10 +2953,3 @@ get_unfit_list(FitnessServer) ->
            []
    end.
 get_projection_store_pid_or_regname(#ch_mgr{name=MyName, opts=MgrOpts}) ->
    case get_projection_store_regname(MgrOpts) of
        undefined ->
            machi_flu_psup:make_proj_supname(MyName);
        PStr ->
            PStr
    end.
--- a/src/machi_chain_repair.erl
+++ b/src/machi_chain_repair.erl
@ -103,8 +103,7 @@ repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
    Add = fun(Name, Pid) -> put(proxies_dict, orddict:store(Name, Pid, get(proxies_dict))) end,
    OurFLUs = lists:usort([Src] ++ Repairing ++ UPI), % AP assumption!
    RepairMode = proplists:get_value(repair_mode, Opts, repair),
-    Verb = proplists:get_value(verbose, Opts, false),
+    Verb = proplists:get_value(verbose, Opts, true),
    RepairId = proplists:get_value(repair_id, Opts, id1),
    Res = try
              _ = [begin
                       {ok, Proxy} = machi_proxy_flu1_client:start_link(P),
@ -117,39 +116,31 @@ repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
                                       get_file_lists(Proxy, FLU, Dict)
                               end, D, ProxiesDict),
              MissingFileSummary = make_missing_file_summary(D2, OurFLUs),
-              %% ?VERB("~w MissingFileSummary ~p\n",[RepairId,MissingFileSummary]),
+              ?VERB("MissingFileSummary ~p\n", [MissingFileSummary]),
              lager:info("Repair ~w MissingFileSummary ~p\n",
                         [RepairId, MissingFileSummary]),
              [ets:insert(ETS, {{directive_bytes, FLU}, 0}) || FLU <- OurFLUs],
              %% Repair files from perspective of Src, i.e. tail(UPI).
              SrcProxy = orddict:fetch(Src, ProxiesDict),
              {ok, EpochID} = machi_proxy_flu1_client:get_epoch_id(
                                SrcProxy, ?SHORT_TIMEOUT),
-              %% ?VERB("Make repair directives: "),
+              ?VERB("Make repair directives: "),
              Ds =
                  [{File, make_repair_directives(
                            ConsistencyMode, RepairMode, File, Size, EpochID,
                            Verb,
                            Src, OurFLUs, ProxiesDict, ETS)} ||
                      {File, {Size, _MissingList}} <- MissingFileSummary],
-              %% ?VERB(" done\n"),
+              ?VERB(" done\n"),
              lager:info("Repair ~w repair directives finished\n", [RepairId]),
              [begin
                   [{_, Bytes}] = ets:lookup(ETS, {directive_bytes, FLU}),
-                   %% ?VERB("Out-of-sync data for FLU ~p: ~s MBytes\n",
+                   ?VERB("Out-of-sync data for FLU ~p: ~s MBytes\n",
-                   %%       [FLU, mbytes(Bytes)]),
+                         [FLU, mbytes(Bytes)])
                   lager:info("Repair ~w "
                              "Out-of-sync data for FLU ~p: ~s MBytes\n",
                         [RepairId, FLU, mbytes(Bytes)]),
                   ok
               end || FLU <- OurFLUs],
-              %% ?VERB("Execute repair directives: "),
+              ?VERB("Execute repair directives: "),
              ok = execute_repair_directives(ConsistencyMode, Ds, Src, EpochID,
                                             Verb, OurFLUs, ProxiesDict, ETS),
-              %% ?VERB(" done\n"),
+              ?VERB(" done\n"),
              lager:info("Repair ~w repair directives finished\n", [RepairId]),
              ok
          catch
              What:Why ->
@ -207,7 +198,7 @@ make_repair_compare_fun(SrcFLU) ->
            T_a =< T_b
    end.
-make_repair_directives(ConsistencyMode, RepairMode, File, Size, _EpochID,
+make_repair_directives(ConsistencyMode, RepairMode, File, Size, EpochID,
                       Verb, Src, FLUs0, ProxiesDict, ETS) ->
    true = (Size < ?MAX_OFFSET),
    FLUs = lists:usort(FLUs0),
@ -216,9 +207,11 @@ make_repair_directives(ConsistencyMode, RepairMode, File, Size, _EpochID,
              Proxy = orddict:fetch(FLU, ProxiesDict),
              OffSzCs =
                  case machi_proxy_flu1_client:checksum_list(
-                         Proxy, File, ?LONG_TIMEOUT) of
+                         Proxy, EpochID, File, ?LONG_TIMEOUT) of
                      {ok, InfoBin} ->
-                          machi_csum_table:split_checksum_list_blob_decode(InfoBin);
+                          {Info, _} =
                            machi_csum_table:split_checksum_list_blob_decode(InfoBin),
                          Info;
                      {error, no_such_file} ->
                          []
                  end,
@ -236,6 +229,7 @@ make_repair_directives(ConsistencyMode, RepairMode, File, Size, _EpochID,
 make_repair_directives2(C2, ConsistencyMode, RepairMode,
                       File, Verb, Src, FLUs, ProxiesDict, ETS) ->
    ?VERB("."),
    make_repair_directives3(C2, ConsistencyMode, RepairMode,
                            File, Verb, Src, FLUs, ProxiesDict, ETS, []).
@ -265,18 +259,7 @@ make_repair_directives3([{Offset, Size, CSum, _FLU}=A|Rest0],
            %%     byte range from all FLUs
            %% 3b. Log big warning about data loss.
            %% 4. Log any other checksum discrepencies as they are found.
-            QQ = [begin
+            exit({todo_repair_sanity_check, ?LINE, File, Offset, As})
                      Pxy = orddict:fetch(FLU, ProxiesDict),
                      {ok, EpochID} = machi_proxy_flu1_client:get_epoch_id(
                                        Pxy, ?SHORT_TIMEOUT),
                      NSInfo = undefined,
                      XX = machi_proxy_flu1_client:read_chunk(
                             Pxy, NSInfo, EpochID, File, Offset, Size, undefined,
                             ?SHORT_TIMEOUT),
                      {FLU, XX}
                  end || {__Offset, __Size, __CSum, FLU} <- As],
            exit({todo_repair_sanity_check, ?LINE, File, Offset, {as,As}, {qq,QQ}})
    end,
    %% List construction guarantees us that there's at least one ?MAX_OFFSET
    %% item remains.  Sort order + our "taking" of all exact Offset+Size
@ -329,25 +312,23 @@ execute_repair_directives(ap_mode=_ConsistencyMode, Ds, _Src, EpochID, Verb,
                            {ProxiesDict, EpochID, Verb, ETS}, Ds),
    ok.
-execute_repair_directive({File, Cmds}, {ProxiesDict, EpochID, _Verb, ETS}=Acc) ->
+execute_repair_directive({File, Cmds}, {ProxiesDict, EpochID, Verb, ETS}=Acc) ->
    EtsKeys = [{in_files, t_in_files}, {in_chunks, t_in_chunks},
               {in_bytes, t_in_bytes}, {out_files, t_out_files},
               {out_chunks, t_out_chunks}, {out_bytes, t_out_bytes}],
    [ets:insert(ETS, {L_K, 0}) || {L_K, _T_K} <- EtsKeys],
    F = fun({copy, {Offset, Size, TaggedCSum, MySrc}, MyDsts}, Acc2) ->
                SrcP = orddict:fetch(MySrc, ProxiesDict),
-                %% case ets:lookup_element(ETS, in_chunks, 2) rem 100 of
+                case ets:lookup_element(ETS, in_chunks, 2) rem 100 of
-                %%     0 -> ?VERB(".2", []);
+                    0 -> ?VERB(".", []);
-                %%     _ -> ok
+                    _ -> ok
-                %% end,
+                end,
                _T1 = os:timestamp(),
                %% TODO: support case multiple written or trimmed chunks returned
-                NSInfo = undefined,
+                {ok, {[{_, Offset, Chunk, _}], _}} =
                {ok, {[{_, Offset, Chunk, _ReadCSum}|OtherChunks], []=_TrimmedList}} =
                    machi_proxy_flu1_client:read_chunk(
-                      SrcP, NSInfo, EpochID, File, Offset, Size, undefined,
+                      SrcP, EpochID, File, Offset, Size, [],
                      ?SHORT_TIMEOUT),
                [] = OtherChunks,
                _T2 = os:timestamp(),
                <<_Tag:1/binary, CSum/binary>> = TaggedCSum,
                case machi_util:checksum_chunk(Chunk) of
@ -356,7 +337,7 @@ execute_repair_directive({File, Cmds}, {ProxiesDict, EpochID, _Verb, ETS}=Acc) -
                                 DstP = orddict:fetch(DstFLU, ProxiesDict),
                                 _T3 = os:timestamp(),
                                 ok = machi_proxy_flu1_client:write_chunk(
-                                        DstP, NSInfo, EpochID, File, Offset, Chunk, TaggedCSum,
+                                        DstP, EpochID, File, Offset, Chunk,
                                        ?SHORT_TIMEOUT),
                                 _T4 = os:timestamp()
                             end || DstFLU <- MyDsts],
--- a/src/machi_cinfo.erl
+++ b/src/machi_cinfo.erl
@ -85,8 +85,8 @@ fitness(FluName) ->
 -spec flu1(atom()) -> [{atom(), term()}].
 flu1(FluName) ->
-    State = machi_flu1_append_server:current_state(FluName),
+    State = machi_flu1:current_state(FluName),
-    machi_flu1_append_server:format_state(State).
+    machi_flu1:format_state(State).
 %% Internal functions
--- a/src/machi_cr_client.erl
+++ b/src/machi_cr_client.erl
--- a/src/machi_csum_table.erl
+++ b/src/machi_csum_table.erl
@ -1,53 +1,35 @@
 %% -------------------------------------------------------------------
 %%
 %% Copyright (c) 2007-2016 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
 %% except in compliance with the License.  You may obtain
 %% a copy of the License at
 %%
 %%   http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing,
 %% software distributed under the License is distributed on an
 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 %% KIND, either express or implied.  See the License for the
 %% specific language governing permissions and limitations
 %% under the License.
 %%
 %% -------------------------------------------------------------------
 -module(machi_csum_table).
 -export([open/2,
         find/3,
-         write/6, write/4, trim/5,
+         write/6, write/4, trim/5, trim/3,
         find_leftneighbor/2, find_rightneighbor/2,
         all_trimmed/3, any_trimmed/3,
         all_trimmed/2,
         sync/1,
         calc_unwritten_bytes/1,
         split_checksum_list_blob_decode/1,
         all/1,
         close/1, delete/1,
         foldl_chunks/3]).
 -export([encode_csum_file_entry/3, encode_csum_file_entry_bin/3,
         decode_csum_file_entry/1]).
 -include("machi.hrl").
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 -export([all/1]).
 -endif.
 -record(machi_csum_table,
        {file :: string(),
-         table :: eleveldb:db_ref()}).
+         fd   :: file:io_device(),
         table :: ets:tid()}).
 -type table() :: #machi_csum_table{}.
 -type byte_sequence() :: { Offset :: non_neg_integer(),
                           Size   :: pos_integer()|infinity }.
 -type chunk() :: {Offset :: machi_dt:file_offset(),
                  Size :: machi_dt:chunk_size(),
                  machi_dt:chunk_csum() | trimmed | none}.
 -export_type([table/0]).
@ -55,110 +37,101 @@
                  {ok, table()} | {error, file:posix()}.
 open(CSumFilename, _Opts) ->
-    LevelDBOptions = [{create_if_missing, true},
+    T = ets:new(?MODULE, [private, ordered_set]),
-                      %% Keep this table small so as not to interfere
+    CSum = machi_util:make_tagged_csum(none),
-                      %% operating system's file cache, which is for
+    %% Dummy entry for headers
-                      %% Machi's main read efficiency
+    true = ets:insert_new(T, {0, ?MINIMUM_OFFSET, CSum}),
                      {total_leveldb_mem_percent, 10}],
    {ok, T} = eleveldb:open(CSumFilename, LevelDBOptions),
    %% Dummy entry for reserved headers
    ok = eleveldb:put(T,
                      sext:encode({0, ?MINIMUM_OFFSET}),
                      sext:encode(?CSUM_TAG_NONE_ATOM),
                      [{sync, true}]),
    C0 = #machi_csum_table{
            file=CSumFilename,
            table=T},
-    {ok, C0}.
+    case file:read_file(CSumFilename) of
-
+        {ok, Bin} ->
-spec split_checksum_list_blob_decode(binary())-> [chunk()].
+            List = case split_checksum_list_blob_decode(Bin) of
-split_checksum_list_blob_decode(Bin) ->
+                       {List0, <<>>} ->
-    erlang:binary_to_term(Bin).
+                           List0;
-
+                       {List0, _Junk} ->
-
+                           %% Partially written, needs repair TODO
-define(has_overlap(LeftOffset, LeftSize, RightOffset, RightSize),
+                           List0
        ((LeftOffset - (RightOffset+RightSize)) * (LeftOffset+LeftSize - RightOffset) < 0)).
 -spec find(table(), machi_dt:file_offset(), machi_dt:chunk_size())
          -> [chunk()].
 find(#machi_csum_table{table=T}, Offset, Size) ->
    {ok, I} = eleveldb:iterator(T, [], keys_only),
    EndKey = sext:encode({Offset+Size, 0}),
    StartKey = sext:encode({Offset, Size}),
    {ok, FirstKey} = case eleveldb:iterator_move(I, StartKey) of
                         {error, invalid_iterator} ->
                             try
                                 %% Assume that the invalid_iterator is because
                                 %% we tried to move to the end via StartKey.
                                 %% Instead, move there directly.
                                 {ok, _} = eleveldb:iterator_move(I, last),
                                 {ok, _} = eleveldb:iterator_move(I, prev)
                             catch
                                 _:_ ->
                                     {ok, _} = eleveldb:iterator_move(I, first)
                             end;
                         {ok, _} = R0 ->
                             case eleveldb:iterator_move(I, prev) of
                                 {error, invalid_iterator} ->
                                     R0;
                                 {ok, _} = R1 ->
                                     R1
                             end
                   end,
-    _ = eleveldb:iterator_close(I),
+            %% assuming all entries are strictly ordered by offset,
-    FoldFun = fun({K, V}, Acc) ->
+            %% trim command should always come after checksum entry.
-                      {TargetOffset, TargetSize} = sext:decode(K),
+            %% *if* by any chance that order cound not be kept, we
-                      case ?has_overlap(TargetOffset, TargetSize, Offset, Size) of
+            %% still can do ordering check and monotonic merge here.
-                          true ->
+            %% TODO: make some random injection tests?
-                              [{TargetOffset, TargetSize, sext:decode(V)}|Acc];
+            [begin %% Replay all file contents, Same logic as in write/6
-                          false ->
+                 Chunks = find(C0, Offset, Size),
-                              Acc
+                 lists:foreach(fun({O, _, _}) ->
-                      end;
+                                       ets:delete(T, O)
                 (_K, Acc) ->
                      lager:error("~p wrong option", [_K]),
                      Acc
              end,
    lists:reverse(eleveldb_fold(T, FirstKey, EndKey, FoldFun, [])).
 %% @doc Updates all chunk info, by deleting existing entries if exists
 %% and putting new chunk info
 -spec write(table(),
            machi_dt:file_offset(), machi_dt:chunk_size(),
            machi_dt:chunk_csum()|'none'|'trimmed',
            undefined|chunk(), undefined|chunk()) ->
                   ok | {error, term()}.
 write(#machi_csum_table{table=T} = CsumT, Offset, Size, CSum,
      LeftUpdate, RightUpdate) ->
    PutOps =
        [{put,
          sext:encode({Offset, Size}),
          sext:encode(CSum)}]
        ++ case LeftUpdate of
               {LO, LS, LCsum} when LO + LS =:= Offset ->
                   [{put,
                     sext:encode({LO, LS}),
                     sext:encode(LCsum)}];
               undefined ->
                   []
           end
        ++ case RightUpdate of
               {RO, RS, RCsum} when RO =:= Offset + Size ->
                   [{put,
                     sext:encode({RO, RS}),
                     sext:encode(RCsum)}];
               undefined ->
                   []
           end,
    Chunks = find(CsumT, Offset, Size),
    DeleteOps = lists:map(fun({O, L, _}) ->
                                  {delete, sext:encode({O, L})}
                               end, Chunks),
-%% io:format(user, "PutOps: ~P\n", [PutOps, 20]),
+                 true = ets:insert(T, {Offset, Size, CsumOrTrimmed})
-%% io:format(user, "DelOps: ~P\n", [DeleteOps, 20]),
+             end
-    eleveldb:write(T, DeleteOps ++ PutOps, [{sync, true}]).
+             || {Offset, Size, CsumOrTrimmed} <- List],
            ok;
        {error, enoent} ->
            ok;
        Error ->
            throw(Error)
    end,
    {ok, Fd} = file:open(CSumFilename, [raw, binary, append]),
    {ok, C0#machi_csum_table{fd=Fd}}.
 -spec find(table(), machi_dt:file_offset(), machi_dt:file_size()) ->
                  list({machi_dt:file_offset(),
                        machi_dt:file_size(),
                        machi_dt:chunk_csum()|trimmed}).
 find(#machi_csum_table{table=T}, Offset, Size) ->
    ets:select(T, [{{'$1', '$2', '$3'},
                    [inclusion_match_spec(Offset, Size)],
                    ['$_']}]).
 -ifdef(TEST).
 all(#machi_csum_table{table=T}) ->
    ets:tab2list(T).
 -endif.
 write(#machi_csum_table{fd=Fd, table=T} = CsumT,
      Offset, Size, CSum,
      LeftUpdate, RightUpdate) ->
    Binary =
        [encode_csum_file_entry_bin(Offset, Size, CSum),
         case LeftUpdate of
             {LO, LS, LCsum} when LO + LS =:= Offset ->
                 encode_csum_file_entry_bin(LO, LS, LCsum);
             undefined ->
                 <<>>
         end,
         case RightUpdate of
             {RO, RS, RCsum} when RO =:= Offset + Size ->
                 encode_csum_file_entry_bin(RO, RS, RCsum);
             undefined ->
                 <<>>
         end],
    case file:write(Fd, Binary) of
        ok ->
            Chunks = find(CsumT, Offset, Size),
            lists:foreach(fun({O, _, _}) ->
                                  ets:delete(T, O)
                          end, Chunks),
            case LeftUpdate of
                {LO1, LS1, _} when LO1 + LS1 =:= Offset ->
                    ets:insert(T, LeftUpdate);
                undefined -> noop
            end,
            case RightUpdate of
                {RO1, _, _} when RO1 =:= Offset + Size ->
                    ets:insert(T, RightUpdate);
                undefined -> noop
            end,
            true = ets:insert(T, {Offset, Size, CSum}),
            ok;
        Error ->
            io:format(user, "boob *********************", []),
            Error
    end.
 -spec find_leftneighbor(table(), non_neg_integer()) ->
-                               undefined | chunk().
+                               undefined |
                               {non_neg_integer(), machi_dt:chunk_size(), trimmed|machi_dt:chunk_csum()}.
 find_leftneighbor(CsumT, Offset) ->
    case find(CsumT, Offset, 1) of
        [] -> undefined;
@ -167,7 +140,8 @@ find_leftneighbor(CsumT, Offset) ->
    end.
 -spec find_rightneighbor(table(), non_neg_integer()) ->
-                                undefined | chunk().
+                                undefined |
                                {non_neg_integer(), machi_dt:chunk_size(), trimmed|machi_dt:chunk_csum()}.
 find_rightneighbor(CsumT, Offset) ->
    case find(CsumT, Offset, 1) of
        [] -> undefined;
@ -177,46 +151,40 @@ find_rightneighbor(CsumT, Offset) ->
    end.
 -spec write(table(), machi_dt:file_offset(), machi_dt:file_size(),
-            machi_dt:chunk_csum()|none|trimmed) ->
+            machi_dt:chunk_csum()|trimmed) ->
                   ok | {error, trimmed|file:posix()}.
 write(CsumT, Offset, Size, CSum) ->
    write(CsumT, Offset, Size, CSum, undefined, undefined).
 trim(CsumT, Offset, Size, LeftUpdate, RightUpdate) ->
-    write(CsumT, Offset, Size,
+    write(CsumT, Offset, Size, trimmed, LeftUpdate, RightUpdate).
          trimmed, %% Should this be much smaller like $t or just 't'
          LeftUpdate, RightUpdate).
-%% @doc returns whether all bytes in a specific window is continously
+-spec trim(table(), machi_dt:file_offset(), machi_dt:file_size()) ->
-%% trimmed or not
+                  ok | {error, file:posix()}.
-spec all_trimmed(table(), non_neg_integer(), non_neg_integer()) -> boolean().
+trim(#machi_csum_table{fd=Fd, table=T}, Offset, Size) ->
-all_trimmed(#machi_csum_table{table=T}, Left, Right) ->
+    Binary = encode_csum_file_entry_bin(Offset, Size, trimmed),
-    FoldFun = fun({_, _}, false) ->
+    case file:write(Fd, Binary) of
-                      false;
+        ok ->
-                 ({K, V}, Pos) when is_integer(Pos) andalso Pos =< Right ->
+            true = ets:insert(T, {Offset, Size, trimmed}),
-                      case {sext:decode(K), sext:decode(V)} of
+            ok;
-                          {{Pos, Size}, trimmed} ->
+        Error ->
-                              Pos + Size;
+            Error
                          {{Offset, Size}, _}
                            when Offset + Size =< Left ->
                              Left;
                          _Eh ->
                              false
                      end
              end,
    case eleveldb:fold(T, FoldFun, Left, [{verify_checksums, true}]) of
        false -> false;
        Right -> true;
        LastTrimmed when LastTrimmed < Right -> false;
        _ -> %% LastTrimmed > Pos0, which is a irregular case but ok
            true
    end.
-%% @doc returns whether all bytes 0-Pos0 is continously trimmed or
+-spec all_trimmed(table(), non_neg_integer(), non_neg_integer()) -> boolean().
-%% not, including header.
+all_trimmed(#machi_csum_table{table=T}, Left, Right) ->
    runthru(ets:tab2list(T), Left, Right).
 -spec all_trimmed(table(), non_neg_integer()) -> boolean().
-all_trimmed(CsumT, Pos0) ->
+all_trimmed(#machi_csum_table{table=T}, Pos) ->
-    all_trimmed(CsumT, 0, Pos0).
+    case ets:tab2list(T) of
        [{0, ?MINIMUM_OFFSET, _}|L] ->
            %% tl/1 to remove header space {0, 1024, <<0>>}
            runthru(L, ?MINIMUM_OFFSET, Pos);
        List ->
            %% In case a header is removed;
            runthru(List, 0, Pos)
    end.
 -spec any_trimmed(table(),
                  pos_integer(),
@ -225,9 +193,13 @@ any_trimmed(CsumT, Offset, Size) ->
    Chunks = find(CsumT, Offset, Size),
    lists:any(fun({_, _, State}) -> State =:= trimmed end, Chunks).
 -spec sync(table()) -> ok | {error, file:posix()}.
 sync(#machi_csum_table{fd=Fd}) ->
    file:sync(Fd).
 -spec calc_unwritten_bytes(table()) -> [byte_sequence()].
-calc_unwritten_bytes(#machi_csum_table{table=_} = CsumT) ->
+calc_unwritten_bytes(#machi_csum_table{table=T}) ->
-    case lists:sort(all(CsumT)) of
+    case lists:sort(ets:tab2list(T)) of
        [] ->
            [{?MINIMUM_OFFSET, infinity}];
        Sorted ->
@ -235,34 +207,101 @@ calc_unwritten_bytes(#machi_csum_table{table=_} = CsumT) ->
            build_unwritten_bytes_list(Sorted, LastOffset, [])
    end.
 all(CsumT) ->
    FoldFun = fun(E, Acc) -> [E|Acc] end,
    lists:reverse(foldl_chunks(FoldFun, [], CsumT)).
 -spec close(table()) -> ok.
-close(#machi_csum_table{table=T}) ->
+close(#machi_csum_table{table=T, fd=Fd}) ->
-    ok = eleveldb:close(T).
+    true = ets:delete(T),
    ok = file:close(Fd).
 -spec delete(table()) -> ok.
-delete(#machi_csum_table{table=T, file=F}) ->
+delete(#machi_csum_table{file=F} = C) ->
-    catch eleveldb:close(T),
+    catch close(C),
-    %% TODO change this to directory walk
+    case file:delete(F) of
-    case os:cmd("rm -rf " ++ F) of
+        ok -> ok;
-        "" -> ok;
+        {error, enoent} -> ok;
        E -> E
    end.
-spec foldl_chunks(fun((chunk(),  Acc0 :: term()) -> Acc :: term()),
+-spec foldl_chunks(fun(({non_neg_integer(), non_neg_integer(), term()},
                        Acc0 :: term())
                       -> Acc :: term()),
                   Acc0 :: term(), table()) -> Acc :: term().
 foldl_chunks(Fun, Acc0, #machi_csum_table{table=T}) ->
-    FoldFun = fun({K, V}, Acc) ->
+    ets:foldl(Fun, Acc0, T).
-                      {Offset, Len} = sext:decode(K),
+
-                      Fun({Offset, Len, sext:decode(V)}, Acc);
+%% @doc Encode `Offset + Size + TaggedCSum' into an `iolist()' type for
-                 (_K, Acc) ->
+%% internal storage by the FLU.
-                      _ = lager:error("~p: wrong option?", [_K]),
+
-                      Acc
+-spec encode_csum_file_entry(
-              end,
+        machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()) ->
-    eleveldb:fold(T, FoldFun, Acc0, [{verify_checksums, true}]).
+        iolist().
 encode_csum_file_entry(Offset, Size, TaggedCSum) ->
    Len = 8 + 4 + byte_size(TaggedCSum),
    [<<$w, Len:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big>>,
     TaggedCSum].
 %% @doc Encode `Offset + Size + TaggedCSum' into an `binary()' type for
 %% internal storage by the FLU.
 -spec encode_csum_file_entry_bin(
        machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()) ->
        binary().
 encode_csum_file_entry_bin(Offset, Size, trimmed) ->
    <<$t, Offset:64/unsigned-big, Size:32/unsigned-big>>;
 encode_csum_file_entry_bin(Offset, Size, TaggedCSum) ->
    Len = 8 + 4 + byte_size(TaggedCSum),
    <<$w, Len:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big,
      TaggedCSum/binary>>.
 %% @doc Decode a single `binary()' blob into an
 %%      `{Offset,Size,TaggedCSum}' tuple.
 %%
 %% The internal encoding (which is currently exposed to the outside world
 %% via this function and related ones) is:
 %%
 %% <ul>
 %% <li> 1 byte: record length
 %% </li>
 %% <li> 8 bytes (unsigned big-endian): byte offset
 %% </li>
 %% <li> 4 bytes (unsigned big-endian): chunk size
 %% </li>
 %% <li> all remaining bytes: tagged checksum (1st byte = type tag)
 %% </li>
 %% </ul>
 %%
 %% See `machi.hrl' for the tagged checksum types, e.g.,
 %% `?CSUM_TAG_NONE'.
 -spec decode_csum_file_entry(binary()) ->
        error |
        {machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()}.
 decode_csum_file_entry(<<_:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big, TaggedCSum/binary>>) ->
    {Offset, Size, TaggedCSum};
 decode_csum_file_entry(_Else) ->
    error.
 %% @doc Split a `binary()' blob of `checksum_list' data into a list of
 %% `{Offset,Size,TaggedCSum}' tuples.
 -spec split_checksum_list_blob_decode(binary()) ->
  {list({machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()}),
   TrailingJunk::binary()}.
 split_checksum_list_blob_decode(Bin) ->
    split_checksum_list_blob_decode(Bin, []).
 split_checksum_list_blob_decode(<<$w, Len:8/unsigned-big, Part:Len/binary, Rest/binary>>, Acc)->
    One = <<Len:8/unsigned-big, Part/binary>>,
    case decode_csum_file_entry(One) of
        error ->
            split_checksum_list_blob_decode(Rest, Acc);
        DecOne ->
            split_checksum_list_blob_decode(Rest, [DecOne|Acc])
    end;
 split_checksum_list_blob_decode(<<$t, Offset:64/unsigned-big, Size:32/unsigned-big, Rest/binary>>, Acc) ->
    %% trimmed offset
    split_checksum_list_blob_decode(Rest, [{Offset, Size, trimmed}|Acc]);
 split_checksum_list_blob_decode(Rest, Acc) ->
    {lists:reverse(Acc), Rest}.
 -spec build_unwritten_bytes_list( CsumData   :: [{ Offset   :: non_neg_integer(),
                                                   Size     :: pos_integer(),
@ -284,46 +323,21 @@ build_unwritten_bytes_list([{CurrentOffset, CurrentSize, _Csum}|Rest], LastOffse
 build_unwritten_bytes_list([{CO, CS, _Ck}|Rest], _LastOffset, Acc) ->
    build_unwritten_bytes_list(Rest, CO + CS, Acc).
 %% @doc make sure all trimmed chunks are continously chained
 %% TODO: test with EQC
 runthru([], Pos, Pos) -> true;
 runthru([], Pos0, Pos) when Pos0 < Pos -> false;
 runthru([{Offset0, Size0, trimmed}|T], Offset, Pos) when Offset0 =< Offset ->
    runthru(T, Offset0+Size0, Pos);
 runthru(_L, _O, _P) ->
    false.
 %% @doc If you want to find an overlap among two areas [x, y] and [a,
-%% b] where x &lt; y and a &lt; b; if (a-y)*(b-x) &lt; 0 then there's a
+%% b] where x < y and a < b; if (a-y)*(b-x) < 0 then there's a
 %% overlap, else, > 0 then there're no overlap. border condition = 0
 %% is not overlap in this offset-size case.
-%% inclusion_match_spec(Offset, Size) ->
+inclusion_match_spec(Offset, Size) ->
-%%     {'>', 0,
+    {'>', 0,
-%%      {'*',
+     {'*',
-%%       {'-', Offset + Size, '$1'},
+      {'-', Offset + Size, '$1'},
-%%       {'-', Offset, {'+', '$1', '$2'}}}}.
+      {'-', Offset, {'+', '$1', '$2'}}}}.
 -spec eleveldb_fold(eleveldb:db_ref(), binary(), binary(),
                    fun(({binary(), binary()}, AccType::term()) -> AccType::term()),
                    AccType0::term()) ->
                           AccType::term().
 eleveldb_fold(Ref, Start, End, FoldFun, InitAcc) ->
    {ok, Iterator} = eleveldb:iterator(Ref, []),
    try
        eleveldb_do_fold(eleveldb:iterator_move(Iterator, Start),
                         Iterator, End, FoldFun, InitAcc)
    catch throw:IteratorClosed ->
            {error, IteratorClosed}
    after
        eleveldb:iterator_close(Iterator)
    end.
 -spec eleveldb_do_fold({ok, binary(), binary()}|{error, iterator_closed|invalid_iterator}|{ok,binary()},
                       eleveldb:itr_ref(), binary(),
                       fun(({binary(), binary()}, AccType::term()) -> AccType::term()),
                       AccType::term()) ->
                              AccType::term().
 eleveldb_do_fold({ok, Key, Value}, _, End, FoldFun, Acc)
  when End < Key ->
    FoldFun({Key, Value}, Acc);
 eleveldb_do_fold({ok, Key, Value}, Iterator, End, FoldFun, Acc) ->
    eleveldb_do_fold(eleveldb:iterator_move(Iterator, next),
                     Iterator, End, FoldFun,
                     FoldFun({Key, Value}, Acc));
 eleveldb_do_fold({error, iterator_closed}, _, _, _, Acc) ->
            %% It's really an error which is not expected
    throw({iterator_closed, Acc});
 eleveldb_do_fold({error, invalid_iterator}, _, _, _, Acc) ->
    %% Probably reached to end
    Acc.
--- a/src/machi_dt.erl
+++ b/src/machi_dt.erl
@ -20,24 +20,15 @@
 -module(machi_dt).
 -include("machi.hrl").
 -include("machi_projection.hrl").
-type append_opts() :: #append_opts{}.
+-type chunk()       :: chunk_bin() | {chunk_csum(), chunk_bin()}.
-type chunk()       :: chunk_bin() | iolist(). % client can choose either rep.
+-type chunk_bin()   :: binary() | iolist().    % client can use either
-type chunk_bin()   :: binary().               % server returns binary() only.
+-type chunk_csum()  :: binary().               % 1 byte tag, N-1 bytes checksum
-type chunk_csum()  :: <<>> | chunk_csum_bin() | {csum_tag(), binary()}.
+-type chunk_summary() :: {file_offset(), chunk_size(), binary()}.
-type chunk_csum_bin() :: binary().            % 1 byte tag, N-1 bytes checksum
+-type chunk_s()     :: 'trimmed' | binary().
 -type chunk_cstrm() :: 'trimmed' | chunk_csum().
 -type chunk_summary() :: {file_offset(), chunk_size(), chunk_bin(), chunk_cstrm()}.
 -type chunk_pos()   :: {file_offset(), chunk_size(), file_name_s()}.
 -type chunk_size()  :: non_neg_integer().
 %% Tags that stand for how that checksum was generated. See
 %% machi_util:make_tagged_csum/{1,2} for further documentation and
 %% implementation.
 -type csum_tag()    :: none | client_sha | server_sha | server_regen_sha.
 -type error_general() :: 'bad_arg' | 'wedged' | 'bad_checksum'.
 -type epoch_csum()  :: binary().
 -type epoch_num()   :: -1 | non_neg_integer().
@ -50,26 +41,23 @@
 -type file_prefix() :: binary() | list().
 -type inet_host()   :: inet:ip_address() | inet:hostname().
 -type inet_port()   :: inet:port_number().
 -type locator()     :: number().
 -type namespace()   :: binary().
 -type namespace_version() :: non_neg_integer().
 -type ns_info()     :: #ns_info{}.
 -type projection()      :: #projection_v1{}.
 -type projection_type() :: 'public' | 'private'.
-type read_opts()   :: #read_opts{}.
+
-type read_opts_x() :: 'undefined' | 'noopt' | 'none' | #read_opts{}.
+%% @doc Tags that stand for how that checksum was generated. See
 %% machi_util:make_tagged_csum/{1,2} for further documentation and
 %% implementation.
 -type csum_tag()    :: none | client_sha | server_sha | server_regen_sha.
 -export_type([
              append_opts/0,
              chunk/0,
              chunk_bin/0,
              chunk_csum/0,
-              chunk_csum_bin/0,
+              csum_tag/0,
              chunk_cstrm/0,
              chunk_summary/0,
              chunk_s/0,
              chunk_pos/0,
              chunk_size/0,
              csum_tag/0,
              error_general/0,
              epoch_csum/0,
              epoch_num/0,
@ -82,13 +70,7 @@
              file_prefix/0,
              inet_host/0,
              inet_port/0,
              locator/0,
              namespace/0,
              namespace_version/0,
              ns_info/0,
              projection/0,
-              projection_type/0,
+              projection_type/0
              read_opts/0,
              read_opts_x/0
             ]).
--- a/src/machi_file_proxy.erl
+++ b/src/machi_file_proxy.erl
@ -57,8 +57,7 @@
    write/4,
    trim/4,
    append/2,
-    append/4,
+    append/4
    checksum_list/1
 ]).
 %% gen_server callbacks
@ -71,7 +70,7 @@
    code_change/3
 ]).
-define(TICK, 5*1000).
+-define(TICK, 30*1000). %% XXX FIXME Should be something like 5 seconds
 -define(TICK_THRESHOLD, 5). %% After this + 1 more quiescent ticks, shutdown
 -define(TIMEOUT, 10*1000).
 -define(TOO_MANY_ERRORS_RATIO, 50).
@ -91,7 +90,6 @@
    csum_table            :: machi_csum_table:table(),
    eof_position = 0      :: non_neg_integer(),
    max_file_size = ?DEFAULT_MAX_FILE_SIZE :: pos_integer(),
    rollover = false      :: boolean(),
    tref                  :: reference(), %% timer ref
    ticks = 0             :: non_neg_integer(), %% ticks elapsed with no new operations
    ops = 0               :: non_neg_integer(), %% sum of all ops
@ -133,8 +131,9 @@ sync(_Pid, Type) ->
    {error, bad_arg}.
 % @doc Read file at offset for length. This returns a sequence of all
-% written and trimmed (optional) bytes that overlaps with requested
+% chunks that overlaps with requested offset and length. Note that
-% offset and length. Borders are not aligned.
+% borders are not aligned, not to mess up repair at cr_client with
 % checksums. They should be cut at cr_client.
 -spec read(Pid :: pid(),
           Offset :: non_neg_integer(),
           Length :: non_neg_integer()) ->
@ -142,18 +141,18 @@ sync(_Pid, Type) ->
                         Data :: binary(), Checksum :: binary()}]} |
                  {error, Reason :: term()}.
 read(Pid, Offset, Length) ->
-    read(Pid, Offset, Length, #read_opts{}).
+    read(Pid, Offset, Length, []).
 -spec read(Pid :: pid(),
           Offset :: non_neg_integer(),
           Length :: non_neg_integer(),
-           machi_dt:read_opts_x()) ->
+           [{no_checksum|no_chunk|needs_trimmed, boolean()}]) ->
                  {ok, [{Filename::string(), Offset :: non_neg_integer(),
                         Data :: binary(), Checksum :: binary()}]} |
                  {error, Reason :: term()}.
-read(Pid, Offset, Length, #read_opts{}=Opts)
+read(Pid, Offset, Length, Opts) when is_pid(Pid) andalso is_integer(Offset) andalso Offset >= 0
-  when is_pid(Pid) andalso is_integer(Offset) andalso Offset >= 0
+                                     andalso is_integer(Length) andalso Length > 0
-       andalso is_integer(Length) andalso Length > 0 ->
+                                     andalso is_list(Opts) ->
    gen_server:call(Pid, {read, Offset, Length, Opts}, ?TIMEOUT);
 read(_Pid, Offset, Length, Opts) ->
    lager:warning("Bad args to read: Offset ~p, Length ~p, Options ~p", [Offset, Length, Opts]),
@ -212,10 +211,6 @@ append(_Pid, ClientMeta, Extra, _Data) ->
    lager:warning("Bad arg to append: ClientMeta ~p, Extra ~p", [ClientMeta, Extra]),
    {error, bad_arg}.
 -spec checksum_list(pid()) -> {ok, list()}.
 checksum_list(Pid) ->
    gen_server:call(Pid, {checksum_list}, ?TIMEOUT).
 %% gen_server callbacks
 % @private
@ -228,8 +223,6 @@ init({FluName, Filename, DataDir}) ->
    UnwrittenBytes = machi_csum_table:calc_unwritten_bytes(CsumTable),
    {Eof, infinity} = lists:last(UnwrittenBytes),
    {ok, FHd} = file:open(DPath, [read, write, binary, raw]),
    %% Reserve for EC and stuff, to prevent eof when read
    ok = file:pwrite(FHd, 0, binary:copy(<<"so what?">>, ?MINIMUM_OFFSET div 8)),
    Tref = schedule_tick(),
    St = #state{
        fluname         = FluName,
@ -240,7 +233,7 @@ init({FluName, Filename, DataDir}) ->
        data_filehandle = FHd,
        csum_table      = CsumTable,
        tref            = Tref,
-        eof_position    = erlang:max(Eof, ?MINIMUM_OFFSET),
+        eof_position    = Eof,
        max_file_size   = machi_config:max_file_size()},
    lager:debug("Starting file proxy ~p for filename ~p, state = ~p, Eof = ~p",
                [self(), Filename, St, Eof]),
@ -255,22 +248,27 @@ handle_call({sync, data}, _From, State = #state{ data_filehandle = FHd }) ->
    R = file:sync(FHd),
    {reply, R, State};
-handle_call({sync, csum}, _From, State) ->
+handle_call({sync, csum}, _From, State = #state{ csum_table = T }) ->
-    %% machi_csum_table always writes in {sync, true} option, so here
+    R = machi_csum_table:sync(T),
-    %% explicit sync isn't actually needed.
+    {reply, R, State};
    {reply, ok, State};
 handle_call({sync, all}, _From, State = #state{filename = F,
                                               data_filehandle = FHd,
-                                               csum_table = _T
+                                               csum_table = T
                                              }) ->
-    Resp = case file:sync(FHd) of
+    R = machi_csum_table:sync(T),
-               ok ->
+    R1 = file:sync(FHd),
-                   ok;
+    Resp = case {R, R1} of
-               Error ->
+        {ok, ok} -> ok;
-                   lager:error("Got ~p syncing all files for file ~p",
+        {ok, O1} ->
-                               [Error, F]),
+                   lager:error("Got ~p during a data file sync on file ~p", [O1, F]),
-                   Error
+                   O1;
        {O2, ok} ->
                   lager:error("Got ~p during a csum file sync on file ~p", [O2, F]),
                   O2;
        {O3, O4} ->
                   lager:error("Got ~p ~p syncing all files for file ~p", [O3, O4, F]),
                   {O3, O4}
    end,
    {reply, Resp, State};
@ -299,15 +297,15 @@ handle_call({read, Offset, Length, Opts}, _From,
                          }) ->
    %% TODO: use these options - NoChunk prevents reading from disks
    %% NoChecksum doesn't check checksums
-    #read_opts{no_checksum=NoChecksum, no_chunk=NoChunk,
+    NoChecksum = proplists:get_value(no_checksum, Opts, false),
-               needs_trimmed=NeedsTrimmed} = Opts,
+    NoChunk = proplists:get_value(no_chunk, Opts, false),
    {Resp, NewErr} =
        case do_read(FH, F, CsumTable, Offset, Length, NoChunk, NoChecksum) of
            {ok, {[], []}} ->
                {{error, not_written}, Err + 1};
            {ok, {Chunks0, Trimmed0}} ->
                Chunks = slice_both_side(Chunks0, Offset, Offset+Length),
-                Trimmed = case NeedsTrimmed of
+                Trimmed = case proplists:get_value(needs_trimmed, Opts, false) of
                              true -> Trimmed0;
                              false -> []
                          end,
@ -370,7 +368,7 @@ handle_call({trim, Offset, Size, _TriggerGC}, _From,
                           trims = {T, Err},
                           csum_table = CsumTable}) ->
-    case machi_csum_table:all_trimmed(CsumTable, Offset, Offset+Size) of
+    case machi_csum_table:all_trimmed(CsumTable, Offset, Size) of
        true ->
            NewState = State#state{ops=Ops+1, trims={T, Err+1}},
            %% All bytes of that range was already trimmed returns ok
@ -387,10 +385,7 @@ handle_call({trim, Offset, Size, _TriggerGC}, _From,
            case machi_csum_table:trim(CsumTable, Offset, Size, LUpdate, RUpdate) of
                ok ->
-                    {NewEof, infinity} = lists:last(machi_csum_table:calc_unwritten_bytes(CsumTable)),
+                    NewState = State#state{ops=Ops+1, trims={T+1, Err}},
                    NewState = State#state{ops=Ops+1,
                                           trims={T+1, Err},
                                           eof_position=NewEof},
                    maybe_gc(ok, NewState);
                Error ->
                    {reply, Error, State#state{ops=Ops+1, trims={T, Err+1}}}
@ -436,10 +431,6 @@ handle_call({append, ClientMeta, Extra, Data}, _From,
    {reply, Resp, State#state{appends = {T+1, NewErr},
                              eof_position = NewEof}};
 handle_call({checksum_list}, _FRom, State = #state{csum_table=T}) ->
    All = machi_csum_table:all(T),
    {reply, {ok, All}, State};
 handle_call(Req, _From, State) ->
    lager:warning("Unknown call: ~p", [Req]),
    {reply, whoaaaaaaaaaaaa, State}.
@ -450,23 +441,11 @@ handle_cast(Cast, State) ->
    {noreply, State}.
 % @private
-handle_info(tick, State = #state{fluname = FluName,
+handle_info(tick, State = #state{eof_position = Eof,
                                 filename = F,
                                 eof_position = Eof,
                                 max_file_size = MaxFileSize}) when Eof >= MaxFileSize ->
-    %% Older code halted here with {stop, file_rollover, State}.
+    lager:notice("Eof position ~p >= max file size ~p. Shutting down.",
-    %% However, there may be other requests in our mailbox already
+                 [Eof, MaxFileSize]),
-    %% and/or not yet delivered but in a race with the
+    {stop, file_rollover, State};
    %% machi_flu_metadata_mgr.  So we close our eleveldb instance (to
    %% avoid double-open attempt by a new file proxy proc), tell
    %% machi_flu_metadata_mgr that we request a rollover, then stop.
    %% terminate() will take care of forwarding messages that are
    %% caught in the race.
    lager:notice("Eof ~s position ~p >= max file size ~p. Shutting down.",
                 [F, Eof, MaxFileSize]),
    State2 = close_files(State),
    machi_flu_metadata_mgr:stop_proxy_pid_rollover(FluName, {file, F}),
    {stop, normal, State2#state{rollover = true}};
 %% XXX Is this a good idea? Need to think this through a bit.
 handle_info(tick, State = #state{wedged = true}) ->
@ -480,7 +459,7 @@ handle_info(tick, State = #state{
                             writes = {WT, WE},
                             appends = {AT, AE}
                            }) when Ops > 100 andalso
-                               trunc(((RE+WE+AE) / (RT+WT+AT)) * 100) > ?TOO_MANY_ERRORS_RATIO ->
+                               trunc(((RE+WE+AE) / RT+WT+AT) * 100) > ?TOO_MANY_ERRORS_RATIO ->
    Errors = RE + WE + AE,
    lager:notice("Got ~p errors. Shutting down.", [Errors]),
    {stop, too_many_errors, State};
@ -539,9 +518,9 @@ handle_info(Req, State) ->
    {noreply, State}.
 % @private
-terminate(Reason, State = #state{fluname = FluName,
+terminate(Reason, #state{filename = F,
-                                 filename = F,
+                         data_filehandle = FHd,
-                                 rollover = Rollover_p,
+                         csum_table = T,
                         reads = {RT, RE},
                         writes = {WT, WE},
                         appends = {AT, AE}
@ -551,11 +530,19 @@ terminate(Reason, State = #state{fluname = FluName,
    lager:info("  Reads:  ~p/~p", [RT, RE]),
    lager:info(" Writes:  ~p/~p", [WT, WE]),
    lager:info("Appends:  ~p/~p", [AT, AE]),
-    close_files(State),
+    case FHd of
-    if Rollover_p ->
+        undefined ->
-            forward_late_messages(FluName, F, 500);
+            noop; %% file deleted
-       true ->
+        _ ->
-            ok
+            ok = file:sync(FHd),
            ok = file:close(FHd)
    end,
    case T of
        undefined ->
            noop; %% file deleted
        _ ->
            ok = machi_csum_table:sync(T),
            ok = machi_csum_table:close(T)
    end,
    ok.
@ -586,15 +573,6 @@ check_or_make_tagged_csum(Tag, InCsum, Data) when Tag == ?CSUM_TAG_CLIENT_SHA;
        false ->
            {error, {bad_csum, Csum}}
    end;
 check_or_make_tagged_csum(?CSUM_TAG_SERVER_REGEN_SHA,
                          InCsum, Data) ->
    Csum = machi_util:checksum_chunk(Data),
    case Csum =:= InCsum of
        true ->
            machi_util:make_tagged_csum(server_regen_sha, Csum);
        false ->
            {error, {bad_csum, Csum}}
    end;
 check_or_make_tagged_csum(OtherTag, _ClientCsum, _Data) ->
    lager:warning("Unknown checksum tag ~p", [OtherTag]),
    {error, bad_checksum}.
@ -606,8 +584,7 @@ check_or_make_tagged_csum(OtherTag, _ClientCsum, _Data) ->
              Size       :: non_neg_integer(),
              NoChunk    :: boolean(),
              NoChecksum :: boolean()
-             ) -> {ok, {Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum :: binary()}],
+             ) -> {ok, Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum :: binary()}]} |
                        Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]}} |
                  {error, bad_checksum} |
                  {error, partial_read} |
                  {error, file:posix()} |
@ -631,14 +608,6 @@ do_read(FHd, Filename, CsumTable, Offset, Size, _, _) ->
    ChunkCsums = machi_csum_table:find(CsumTable, Offset, Size),
    read_all_ranges(FHd, Filename, ChunkCsums, [], []).
 -spec read_all_ranges(file:io_device(), string(),
                      [{non_neg_integer(),non_neg_integer(),trimmed|binary()}],
                      Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum::binary()}],
                      Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]) ->
                             {ok, {
                                Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum::binary()}],
                                Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]}} |
                             {erorr, term()|partial_read}.
 read_all_ranges(_, _, [], ReadChunks, TrimmedChunks) ->
    %% TODO: currently returns empty list of trimmed chunks
    {ok, {lists:reverse(ReadChunks), lists:reverse(TrimmedChunks)}};
@ -650,11 +619,6 @@ read_all_ranges(FHd, Filename, [{Offset, Size, TaggedCsum}|T], ReadChunks, Trimm
    case file:pread(FHd, Offset, Size) of
        eof ->
            read_all_ranges(FHd, Filename, T, ReadChunks, TrimmedChunks);
        {ok, Bytes} when byte_size(Bytes) == Size, TaggedCsum =:= none ->
            read_all_ranges(FHd, Filename, T,
                            [{Filename, Offset, Bytes,
                              machi_util:make_tagged_csum(none, <<>>)}|ReadChunks],
                            TrimmedChunks);
        {ok, Bytes} when byte_size(Bytes) == Size ->
            {Tag, Ck} = machi_util:unmake_tagged_csum(TaggedCsum),
            case check_or_make_tagged_csum(Tag, Ck, Bytes) of
@ -705,7 +669,6 @@ read_all_ranges(FHd, Filename, [{Offset, Size, TaggedCsum}|T], ReadChunks, Trimm
 % caller as `ok'
 handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
    Size = iolist_size(Data),
    case machi_csum_table:find(CsumTable, Offset, Size) of
        [] -> %% Nothing should be there
            try
@ -799,7 +762,7 @@ slice_both_side([], _, _) ->
    [];
 slice_both_side([{F, Offset, Chunk, _Csum}|L], LeftPos, RightPos)
  when Offset < LeftPos andalso LeftPos < RightPos ->
-    TrashLen = (LeftPos - Offset),
+    TrashLen = 8 * (LeftPos - Offset),
    <<_:TrashLen/binary, NewChunk/binary>> = Chunk,
    NewChecksum = machi_util:make_tagged_csum(?CSUM_TAG_SERVER_REGEN_SHA_ATOM, Chunk),
    NewH = {F, LeftPos, NewChunk, NewChecksum},
@ -824,7 +787,7 @@ maybe_regenerate_checksum(_, {_, _, trimmed} = Change) ->
 maybe_regenerate_checksum(FHd, {Offset, Size, _Csum}) ->
    case file:pread(FHd, Offset, Size) of
        eof ->
-            error({eof, Offset, Size});
+            error(eof);
        {ok, Bytes} when byte_size(Bytes) =:= Size ->
            TaggedCsum = machi_util:make_tagged_csum(server_regen_sha,
@ -834,9 +797,9 @@ maybe_regenerate_checksum(FHd, {Offset, Size, _Csum}) ->
            throw(Error)
    end.
-%% GC: make sure unwritten bytes = [{Eof, infinity}] and Eof is > max
+%% GC: make sure unwritte bytes = [{Eof, infinity}] and Eof is > max file size
-%% file size walk through the checksum table and make sure all chunks
+%% walk through the checksum table and make sure all chunks trimmed
-%% trimmed Then unlink the file
+%% Then unlink the file
 -spec maybe_gc(term(), #state{}) ->
                      {reply, term(), #state{}} | {stop, normal, term(), #state{}}.
 maybe_gc(Reply, S = #state{eof_position = Eof,
@ -850,7 +813,7 @@ maybe_gc(Reply, S = #state{fluname=FluName,
                           filename = Filename,
                           eof_position = Eof,
                           csum_table=CsumTable}) ->
-    case machi_csum_table:all_trimmed(CsumTable, ?MINIMUM_OFFSET, Eof) of
+    case machi_csum_table:all_trimmed(CsumTable, Eof) of
        true ->
            lager:debug("GC? Let's do it: ~p.~n", [Filename]),
            %% Before unlinking a file, it should inform
@ -873,36 +836,3 @@ maybe_gc(Reply, S = #state{fluname=FluName,
        false ->
            {reply, Reply, S}
    end.
 close_files(State = #state{data_filehandle = FHd,
                           csum_table = T}) ->
    case FHd of
        undefined ->
            noop; %% file deleted
        _ ->
            ok = file:sync(FHd),
            ok = file:close(FHd)
    end,
    case T of
        undefined ->
            noop; %% file deleted
        _ ->
            ok = machi_csum_table:close(T)
    end,
    State#state{data_filehandle = undefined, csum_table = undefined}.
 forward_late_messages(FluName, F, Timeout) ->
    receive
        M ->
            case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, F}) of
                {ok, Pid} ->
                    Pid ! M;
                {error, trimmed} ->
                    lager:error("TODO: FLU ~p file ~p reports trimmed status "
                                "when forwarding ~P\n",
                                [FluName, F, M, 20])
            end,
            forward_late_messages(FluName, F, Timeout)
    after Timeout ->
            ok
    end.
--- a/src/machi_fitness.erl
+++ b/src/machi_fitness.erl
@ -39,8 +39,7 @@
         get_unfit_list/1, update_local_down_list/3,
         add_admin_down/3, delete_admin_down/2,
         send_fitness_update_spam/3,
-         send_spam_to_everyone/1,
+         send_spam_to_everyone/1]).
         trigger_early_adjustment/2]).
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
@ -82,13 +81,6 @@ send_fitness_update_spam(Pid, FromName, Dict) ->
 send_spam_to_everyone(Pid) ->
    gen_server:call(Pid, {send_spam_to_everyone}, infinity).
 %% @doc For testing purposes, we don't want a test to wait for
 %%      wall-clock time to elapse before the fitness server makes a
 %%      down->up status decision.
 trigger_early_adjustment(Pid, FLU) ->
    Pid ! {adjust_down_list, FLU}.
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 init([{MyFluName}|Args]) ->
@ -108,7 +100,6 @@ handle_call({update_local_down_list, Down, MembersDict}, _From,
            #state{my_flu_name=MyFluName, pending_map=OldMap,
                   local_down=OldDown, members_dict=OldMembersDict,
                   admin_down=AdminDown}=S) ->
    verbose("FITNESS: ~w has down suspect ~w\n", [MyFluName, Down]),
    NewMap = store_in_map(OldMap, MyFluName, erlang:now(), Down,
                          AdminDown, [props_yo]),
    S2 = if Down == OldDown, MembersDict == OldMembersDict ->
@ -120,17 +111,13 @@ handle_call({update_local_down_list, Down, MembersDict}, _From,
         end,
    {reply, ok, S2#state{local_down=Down}};
 handle_call({add_admin_down, DownFLU, DownProps}, _From,
-            #state{my_flu_name=MyFluName,
+            #state{local_down=OldDown, admin_down=AdminDown}=S) ->
                   local_down=OldDown, admin_down=AdminDown}=S) ->
    verbose("FITNESS: ~w add admin down ~w\n", [MyFluName, DownFLU]),
    NewAdminDown = [{DownFLU,DownProps}|lists:keydelete(DownFLU, 1, AdminDown)],
    S3 = finish_admin_down(erlang:now(), OldDown, NewAdminDown,
                           [props_yo], S),
    {reply, ok, S3};
 handle_call({delete_admin_down, DownFLU}, _From,
-            #state{my_flu_name=MyFluName,
+            #state{local_down=OldDown, admin_down=AdminDown}=S) ->
                   local_down=OldDown, admin_down=AdminDown}=S) ->
    verbose("FITNESS: ~w delete admin down ~w\n", [MyFluName, DownFLU]),
    NewAdminDown = lists:keydelete(DownFLU, 1, AdminDown),
    S3 = finish_admin_down(erlang:now(), OldDown, NewAdminDown,
                           [props_yo], S),
@ -148,8 +135,7 @@ handle_call(_Request, _From, S) ->
 handle_cast(_Msg, S) ->
    {noreply, S}.
-handle_info({adjust_down_list, FLU}, #state{my_flu_name=MyFluName,
+handle_info({adjust_down_list, FLU}, #state{active_unfit=ActiveUnfit}=S) ->
                                            active_unfit=ActiveUnfit}=S) ->
    NewUnfit = make_unfit_list(S),
    Added_to_new     = NewUnfit -- ActiveUnfit,
    Dropped_from_new = ActiveUnfit -- NewUnfit,
@ -190,11 +176,9 @@ handle_info({adjust_down_list, FLU}, #state{my_flu_name=MyFluName,
        {true, true} ->
            error({bad, ?MODULE, ?LINE, FLU, ActiveUnfit, NewUnfit});
        {true, false} ->
-            NewActive = wrap_active(MyFluName,lists:usort(ActiveUnfit++[FLU])),
+            {noreply, S#state{active_unfit=lists:usort(ActiveUnfit ++ [FLU])}};
            {noreply, S#state{active_unfit=NewActive}};
        {false, true} ->
-            NewActive = wrap_active(MyFluName,ActiveUnfit--[FLU]),
+            {noreply, S#state{active_unfit=ActiveUnfit -- [FLU]}};
            {noreply, S#state{active_unfit=NewActive}};
        {false, false} ->
            {noreply, S}
    end;
@ -432,18 +416,6 @@ map_value(Map) ->
 map_merge(Map1, Map2) ->
    ?MAP:merge(Map1, Map2).
 wrap_active(MyFluName, L) ->
    verbose("FITNESS: ~w has new down list ~w\n", [MyFluName, L]),
    L.
 verbose(Fmt, Args) ->
    case application:get_env(machi, fitness_verbose) of
        {ok, true} ->
            error_logger:info_msg(Fmt, Args);
        _ ->
            ok
    end.
 -ifdef(TEST).
 dt_understanding_test() ->
--- a/src/machi_flu1.erl
+++ b/src/machi_flu1.erl
@ -21,9 +21,7 @@
 %% @doc The Machi FLU file server + file location sequencer.
 %%
 %% This module implements only the Machi FLU file server and its
-%% implicit sequencer together with listener, append server,
+%% implicit sequencer.  
 %% file management and file proxy processes.
 %% Please see the EDoc "Overview" for details about the FLU as a
 %% primitive file server process vs. the larger Machi design of a FLU
 %% as a sequencer + file server + chain manager group of processes.
@ -56,16 +54,32 @@
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 -export([timing_demo_test_COMMENTED_/0, sort_2lines/2]). % Just to suppress warning
 -endif. % TEST
 -export([start_link/1, stop/1,
         update_wedge_state/3, wedge_myself/2]).
-export([make_projection_server_regname/1,
+-export([make_listener_regname/1, make_projection_server_regname/1]).
         ets_table_name/1]).
 %% TODO: remove or replace in OTP way after gen_*'ified
-export([main2/4]).
+-export([main2/4, run_append_server/2, run_listen_server/1,
         current_state/1, format_state/1]).
 -record(state, {
          flu_name        :: atom(),
          proj_store      :: pid(),
          witness = false :: boolean(),
          append_pid      :: pid(),
          tcp_port        :: non_neg_integer(),
          data_dir        :: string(),
          wedged = true   :: boolean(),
          etstab          :: ets:tid(),
          epoch_id        :: 'undefined' | machi_dt:epoch_id(),
          pb_mode = undefined  :: 'undefined' | 'high' | 'low',
          high_clnt       :: 'undefined' | pid(),
          trim_table      :: ets:tid(),
          props = []      :: list()  % proplist
         }).
 -define(SERVER_CMD_READ_TIMEOUT, 600*1000).
 -define(INIT_TIMEOUT, 60*1000).
 start_link([{FluName, TcpPort, DataDir}|Rest])
@ -73,12 +87,7 @@ start_link([{FluName, TcpPort, DataDir}|Rest])
    proc_lib:start_link(?MODULE, main2, [FluName, TcpPort, DataDir, Rest],
                        ?INIT_TIMEOUT).
-stop(RegName) when is_atom(RegName) ->
+stop(Pid) ->
    case whereis(RegName) of
        undefined -> ok;
        Pid -> stop(Pid)
    end;
 stop(Pid) when is_pid(Pid) ->
    case erlang:is_process_alive(Pid) of
        true ->
            Pid ! killme,
@ -87,14 +96,33 @@ stop(Pid) when is_pid(Pid) ->
            error
    end.
-update_wedge_state(PidSpec, Boolean, EpochId) ->
+update_wedge_state(PidSpec, Boolean, EpochId)
-    machi_flu1_append_server:int_update_wedge_state(PidSpec, Boolean, EpochId).
+  when (Boolean == true orelse Boolean == false), is_tuple(EpochId) ->
    PidSpec ! {wedge_state_change, Boolean, EpochId}.
-wedge_myself(PidSpec, EpochId) ->
+wedge_myself(PidSpec, EpochId)
-    machi_flu1_append_server:int_wedge_myself(PidSpec, EpochId).
+  when is_tuple(EpochId) ->
    PidSpec ! {wedge_myself, EpochId}.
 current_state(PidSpec) ->
    PidSpec ! {current_state, self()},
    %% TODO: Not so rubust f(^^;)
    receive
        Res -> Res
    after
        60*1000 -> {error, timeout}
    end.
 format_state(State) ->
    Fields = record_info(fields, state),
    [_Name | Values] = tuple_to_list(State),
    lists:zip(Fields, Values).
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
 ets_table_name(FluName) when is_atom(FluName) ->
    list_to_atom(atom_to_list(FluName) ++ "_epoch").
 main2(FluName, TcpPort, DataDir, Props) ->
    {SendAppendPidToProj_p, ProjectionPid} =
        case proplists:get_value(projection_store_registered_name, Props) of
@ -122,16 +150,24 @@ main2(FluName, TcpPort, DataDir, Props) ->
        end,
    Witness_p = proplists:get_value(witness_mode, Props, false),
-    {ok, AppendPid} = start_append_server(FluName, Witness_p, Wedged_p, EpochId),
+    S0 = #state{flu_name=FluName,
                proj_store=ProjectionPid,
                tcp_port=TcpPort,
                data_dir=DataDir,
                wedged=Wedged_p,
                witness=Witness_p,
                etstab=ets_table_name(FluName),
                epoch_id=EpochId,
                props=Props},
    {ok, AppendPid} = start_append_server(S0, self()),
    if SendAppendPidToProj_p ->
-            machi_projection_store:set_wedge_notify_pid(ProjectionPid, AppendPid);
+            machi_projection_store:set_wedge_notify_pid(ProjectionPid,
                                                        AppendPid);
       true ->
            ok
    end,
-    {ok, ListenerPid} = start_listen_server(FluName, TcpPort, Witness_p, DataDir,
+    S1 = S0#state{append_pid=AppendPid},
-                                            ets_table_name(FluName), ProjectionPid,
+    {ok, ListenPid} = start_listen_server(S1),
                                            Props),
    %% io:format(user, "Listener started: ~w~n", [{FluName, ListenerPid}]),
    Config_e = machi_util:make_config_filename(DataDir, "unused"),
    ok = filelib:ensure_dir(Config_e),
@ -141,24 +177,574 @@ main2(FluName, TcpPort, DataDir, Props) ->
    ok = filelib:ensure_dir(Projection_e),
    put(flu_flu_name, FluName),
-    put(flu_append_pid, AppendPid),
+    put(flu_append_pid, S1#state.append_pid),
    put(flu_projection_pid, ProjectionPid),
-    put(flu_listen_pid, ListenerPid),
+    put(flu_listen_pid, ListenPid),
    proc_lib:init_ack({ok, self()}),
    receive killme -> ok end,
-    (catch exit(AppendPid, kill)),
+    (catch exit(S1#state.append_pid, kill)),
    (catch exit(ProjectionPid, kill)),
-    (catch exit(ListenerPid, kill)),
+    (catch exit(ListenPid, kill)),
    ok.
-start_append_server(FluName, Witness_p, Wedged_p, EpochId) ->
+start_listen_server(S) ->
-    machi_flu1_subsup:start_append_server(FluName, Witness_p, Wedged_p, EpochId).
+    proc_lib:start_link(?MODULE, run_listen_server, [S], ?INIT_TIMEOUT).
-start_listen_server(FluName, TcpPort, Witness_p, DataDir, EtsTab, ProjectionPid,
+start_append_server(S, AckPid) ->
-                    Props) ->
+    proc_lib:start_link(?MODULE, run_append_server, [AckPid, S], ?INIT_TIMEOUT).
-    machi_flu1_subsup:start_listener(FluName, TcpPort, Witness_p, DataDir,
+
-                                     EtsTab, ProjectionPid, Props).
+run_listen_server(#state{flu_name=FluName, tcp_port=TcpPort}=S) ->
    register(make_listener_regname(FluName), self()),
    SockOpts = ?PB_PACKET_OPTS ++
        [{reuseaddr, true}, {mode, binary}, {active, false},
         {backlog,8192}],
    case gen_tcp:listen(TcpPort, SockOpts) of
        {ok, LSock} ->
            proc_lib:init_ack({ok, self()}),
            listen_server_loop(LSock, S);
        Else ->
            error_logger:warning_msg("~s:run_listen_server: "
                                     "listen to TCP port ~w: ~w\n",
                                     [?MODULE, TcpPort, Else]),
            exit({?MODULE, run_listen_server, tcp_port, TcpPort, Else})
    end.
 run_append_server(FluPid, #state{flu_name=Name,
                                 wedged=Wedged_p,epoch_id=EpochId}=S) ->
    %% Reminder: Name is the "main" name of the FLU, i.e., no suffix
    register(Name, self()),
    TID = ets:new(ets_table_name(Name),
                  [set, protected, named_table, {read_concurrency, true}]),
    ets:insert(TID, {epoch, {Wedged_p, EpochId}}),
    proc_lib:init_ack({ok, self()}),
    append_server_loop(FluPid, S#state{etstab=TID}).
 listen_server_loop(LSock, S) ->
    {ok, Sock} = gen_tcp:accept(LSock),
    spawn_link(fun() -> net_server_loop(Sock, S) end),
    listen_server_loop(LSock, S).
 append_server_loop(FluPid, #state{wedged=Wedged_p,
                                  witness=Witness_p,
                                  epoch_id=OldEpochId, flu_name=FluName}=S) ->
    receive
        {seq_append, From, _Prefix, _Chunk, _CSum, _Extra, _EpochID}
          when Witness_p ->
            %% The FLU's net_server_loop() process ought to filter all
            %% witness states, but we'll keep this clause for extra
            %% paranoia.
            From ! witness,
            append_server_loop(FluPid, S);
        {seq_append, From, _Prefix, _Chunk, _CSum, _Extra, _EpochID}
          when Wedged_p ->
            From ! wedged,
            append_server_loop(FluPid, S);
        {seq_append, From, Prefix, Chunk, CSum, Extra, EpochID} ->
            %% Old is the one from our state, plain old 'EpochID' comes
            %% from the client.
            _ = case OldEpochId == EpochID of
                true ->
                    spawn(fun() -> 
                        append_server_dispatch(From, Prefix, Chunk, CSum, Extra, FluName, EpochID) 
                    end);
                false ->
                    From ! {error, bad_epoch}
            end,
            append_server_loop(FluPid, S);
        {wedge_myself, WedgeEpochId} ->
            if not Wedged_p andalso WedgeEpochId == OldEpochId ->
                    true = ets:insert(S#state.etstab,
                                      {epoch, {true, OldEpochId}}),
                    %% Tell my chain manager that it might want to react to
                    %% this new world.
                    Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
                    spawn(fun() ->
                            catch machi_chain_manager1:trigger_react_to_env(Chmgr)
                          end),
                    append_server_loop(FluPid, S#state{wedged=true});
               true ->
                    append_server_loop(FluPid, S)
            end;
        {wedge_state_change, Boolean, {NewEpoch, _}=NewEpochId} ->
            OldEpoch = case OldEpochId of {OldE, _} -> OldE;
                                          undefined -> -1
                       end,
            if NewEpoch >= OldEpoch ->
                    true = ets:insert(S#state.etstab,
                                      {epoch, {Boolean, NewEpochId}}),
                    append_server_loop(FluPid, S#state{wedged=Boolean,
                                                       epoch_id=NewEpochId});
               true ->
                    append_server_loop(FluPid, S)
            end;
        {wedge_status, FromPid} ->
            #state{wedged=Wedged_p, epoch_id=EpochId} = S,
            FromPid ! {wedge_status_reply, Wedged_p, EpochId},
            append_server_loop(FluPid, S);
        {current_state, FromPid} ->
            FromPid ! S;
        Else ->
            io:format(user, "append_server_loop: WHA? ~p\n", [Else]),
            append_server_loop(FluPid, S)
    end.
 net_server_loop(Sock, S) ->
    case gen_tcp:recv(Sock, 0, ?SERVER_CMD_READ_TIMEOUT) of
        {ok, Bin} ->
            {RespBin, S2} = 
                case machi_pb:decode_mpb_ll_request(Bin) of
                    LL_req when LL_req#mpb_ll_request.do_not_alter == 2 ->
                        {R, NewS} = do_pb_ll_request(LL_req, S),
                        {maybe_encode_response(R), mode(low, NewS)};
                    _ ->
                        HL_req = machi_pb:decode_mpb_request(Bin),
                        1 = HL_req#mpb_request.do_not_alter,
                        {R, NewS} = do_pb_hl_request(HL_req, make_high_clnt(S)),
                        {machi_pb:encode_mpb_response(R), mode(high, NewS)}
                end,
            if RespBin == async_no_response ->
                    net_server_loop(Sock, S2);
               true ->
                    case gen_tcp:send(Sock, RespBin) of
                        ok ->
                            net_server_loop(Sock, S2);
                        {error, _} ->
                            (catch gen_tcp:close(Sock)),
                            exit(normal)
                    end
            end;
        {error, SockError} ->
            Msg = io_lib:format("Socket error ~w", [SockError]),
            R = #mpb_ll_response{req_id= <<>>,
                                 generic=#mpb_errorresp{code=1, msg=Msg}},
            _Resp = machi_pb:encode_mpb_ll_response(R),
            %% TODO: Weird that sometimes neither catch nor try/catch
            %%       can prevent OTP's SASL from logging an error here.
            %%       Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,.......
            %% TODO: is this what causes the intermittent PULSE deadlock errors?
            %% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000),
            (catch gen_tcp:close(Sock)),
            exit(normal)
    end.
 maybe_encode_response(async_no_response=X) ->
    X;
 maybe_encode_response(R) ->
    machi_pb:encode_mpb_ll_response(R).
 mode(Mode, #state{pb_mode=undefined}=S) ->
    S#state{pb_mode=Mode};
 mode(_, S) ->
    S.
 make_high_clnt(#state{high_clnt=undefined}=S) ->
    {ok, Proj} = machi_projection_store:read_latest_projection(
                   S#state.proj_store, private),
    Ps = [P_srvr || {_, P_srvr} <- orddict:to_list(
                                     Proj#projection_v1.members_dict)],
    {ok, Clnt} = machi_cr_client:start_link(Ps),
    S#state{high_clnt=Clnt};
 make_high_clnt(S) ->
    S.
 do_pb_ll_request(#mpb_ll_request{req_id=ReqID}, #state{pb_mode=high}=S) ->
    Result = {high_error, 41, "Low protocol request while in high mode"},
    {machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
 do_pb_ll_request(PB_request, S) ->
    Req = machi_pb_translate:from_pb_request(PB_request),
    {ReqID, Cmd, Result, S2} = 
        case Req of
            {RqID, {LowCmd, _}=CMD}
              when LowCmd == low_proj;
                   LowCmd == low_wedge_status; LowCmd == low_list_files ->
                %% Skip wedge check for projection commands!
                %% Skip wedge check for these unprivileged commands
                {Rs, NewS} = do_pb_ll_request3(CMD, S),
                {RqID, CMD, Rs, NewS};
            {RqID, CMD} ->
                EpochID = element(2, CMD),      % by common convention
                {Rs, NewS} = do_pb_ll_request2(EpochID, CMD, S),
                {RqID, CMD, Rs, NewS}
        end,
    {machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
 do_pb_ll_request2(EpochID, CMD, S) ->
    {Wedged_p, CurrentEpochID} = ets:lookup_element(S#state.etstab, epoch, 2),
    if Wedged_p == true ->
            {{error, wedged}, S#state{epoch_id=CurrentEpochID}};
       is_tuple(EpochID)
       andalso
       EpochID /= CurrentEpochID ->
            {Epoch, _} = EpochID,
            {CurrentEpoch, _} = CurrentEpochID,
            if Epoch < CurrentEpoch ->
                    ok;
               true ->
                    %% We're at same epoch # but different checksum, or
                    %% we're at a newer/bigger epoch #.
                    _ = wedge_myself(S#state.flu_name, CurrentEpochID),
                    ok
            end,
            {{error, bad_epoch}, S#state{epoch_id=CurrentEpochID}};
       true ->
            do_pb_ll_request3(CMD, S#state{epoch_id=CurrentEpochID})
    end.
 %% Witness status does not matter below.
 do_pb_ll_request3({low_echo, _BogusEpochID, Msg}, S) ->
    {Msg, S};
 do_pb_ll_request3({low_auth, _BogusEpochID, _User, _Pass}, S) ->
    {-6, S};
 do_pb_ll_request3({low_wedge_status, _EpochID}, S) ->
    {do_server_wedge_status(S), S};
 do_pb_ll_request3({low_proj, PCMD}, S) ->
    {do_server_proj_request(PCMD, S), S};
 %% Witness status *matters* below
 do_pb_ll_request3({low_append_chunk, _EpochID, PKey, Prefix, Chunk, CSum_tag,
                CSum, ChunkExtra},
                  #state{witness=false}=S) ->
    {do_server_append_chunk(PKey, Prefix, Chunk, CSum_tag, CSum,
                            ChunkExtra, S), S};
 do_pb_ll_request3({low_write_chunk, _EpochID, File, Offset, Chunk, CSum_tag,
                   CSum},
                  #state{witness=false}=S) ->
    {do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, S), S};
 do_pb_ll_request3({low_read_chunk, _EpochID, File, Offset, Size, Opts},
                  #state{witness=false} = S) ->
    {do_server_read_chunk(File, Offset, Size, Opts, S), S};
 do_pb_ll_request3({low_trim_chunk, _EpochID, File, Offset, Size, TriggerGC},
                  #state{witness=false}=S) ->
    {do_server_trim_chunk(File, Offset, Size, TriggerGC, S), S};
 do_pb_ll_request3({low_checksum_list, _EpochID, File},
                  #state{witness=false}=S) ->
    {do_server_checksum_listing(File, S), S};
 do_pb_ll_request3({low_list_files, _EpochID},
                  #state{witness=false}=S) ->
    {do_server_list_files(S), S};
 do_pb_ll_request3({low_delete_migration, _EpochID, File},
                  #state{witness=false}=S) ->
    {do_server_delete_migration(File, S),
     #state{witness=false}=S};
 do_pb_ll_request3({low_trunc_hack, _EpochID, File},
                  #state{witness=false}=S) ->
    {do_server_trunc_hack(File, S), S};
 do_pb_ll_request3(_, #state{witness=true}=S) ->
    {{error, bad_arg}, S}.                       % TODO: new status code??
 do_pb_hl_request(#mpb_request{req_id=ReqID}, #state{pb_mode=low}=S) ->
    Result = {low_error, 41, "High protocol request while in low mode"},
    {machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
 do_pb_hl_request(PB_request, S) ->
    {ReqID, Cmd} = machi_pb_translate:from_pb_request(PB_request),
    {Result, S2} = do_pb_hl_request2(Cmd, S),
    {machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
 do_pb_hl_request2({high_echo, Msg}, S) ->
    {Msg, S};
 do_pb_hl_request2({high_auth, _User, _Pass}, S) ->
    {-77, S};
 do_pb_hl_request2({high_append_chunk, _todoPK, Prefix, ChunkBin, TaggedCSum,
                   ChunkExtra}, #state{high_clnt=Clnt}=S) ->
    Chunk = {TaggedCSum, ChunkBin},
    Res = machi_cr_client:append_chunk_extra(Clnt, Prefix, Chunk,
                                             ChunkExtra),
    {Res, S};
 do_pb_hl_request2({high_write_chunk, File, Offset, ChunkBin, TaggedCSum},
                  #state{high_clnt=Clnt}=S) ->
    Chunk = {TaggedCSum, ChunkBin},
    Res = machi_cr_client:write_chunk(Clnt, File, Offset, Chunk),
    {Res, S};
 do_pb_hl_request2({high_read_chunk, File, Offset, Size, Opts},
                  #state{high_clnt=Clnt}=S) ->
    Res = machi_cr_client:read_chunk(Clnt, File, Offset, Size, Opts),
    {Res, S};
 do_pb_hl_request2({high_trim_chunk, File, Offset, Size},
                  #state{high_clnt=Clnt}=S) ->
    Res = machi_cr_client:trim_chunk(Clnt, File, Offset, Size),
    {Res, S};
 do_pb_hl_request2({high_checksum_list, File}, #state{high_clnt=Clnt}=S) ->
    Res = machi_cr_client:checksum_list(Clnt, File),
    {Res, S};
 do_pb_hl_request2({high_list_files}, #state{high_clnt=Clnt}=S) ->
    Res = machi_cr_client:list_files(Clnt),
    {Res, S}.
 do_server_proj_request({get_latest_epochid, ProjType},
                       #state{proj_store=ProjStore}) ->
    machi_projection_store:get_latest_epochid(ProjStore, ProjType);
 do_server_proj_request({read_latest_projection, ProjType},
                       #state{proj_store=ProjStore}) ->
    machi_projection_store:read_latest_projection(ProjStore, ProjType);
 do_server_proj_request({read_projection, ProjType, Epoch},
                       #state{proj_store=ProjStore}) ->
    machi_projection_store:read(ProjStore, ProjType, Epoch);
 do_server_proj_request({write_projection, ProjType, Proj},
                       #state{flu_name=FluName, proj_store=ProjStore}) ->
    if Proj#projection_v1.epoch_number == ?SPAM_PROJ_EPOCH ->
            %% io:format(user, "DBG ~s ~w ~P\n", [?MODULE, ?LINE, Proj, 5]),
            Chmgr = machi_flu_psup:make_fitness_regname(FluName),
            [Map] = Proj#projection_v1.dbg,
            catch machi_fitness:send_fitness_update_spam(
                    Chmgr, Proj#projection_v1.author_server, Map);
       true ->
            catch machi_projection_store:write(ProjStore, ProjType, Proj)
    end;
 do_server_proj_request({get_all_projections, ProjType},
                       #state{proj_store=ProjStore}) ->
    machi_projection_store:get_all_projections(ProjStore, ProjType);
 do_server_proj_request({list_all_projections, ProjType},
                       #state{proj_store=ProjStore}) ->
    machi_projection_store:list_all_projections(ProjStore, ProjType);
 do_server_proj_request({kick_projection_reaction},
                       #state{flu_name=FluName}) ->
    %% Tell my chain manager that it might want to react to
    %% this new world.
    Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
    spawn(fun() ->
                  catch machi_chain_manager1:trigger_react_to_env(Chmgr)
          end),
    async_no_response.
 do_server_append_chunk(PKey, Prefix, Chunk, CSum_tag, CSum,
                       ChunkExtra, S) ->
    case sanitize_prefix(Prefix) of
        ok ->
            do_server_append_chunk2(PKey, Prefix, Chunk, CSum_tag, CSum,
                                    ChunkExtra, S);
        _ ->
            {error, bad_arg}
    end.
 do_server_append_chunk2(_PKey, Prefix, Chunk, CSum_tag, Client_CSum,
                        ChunkExtra, #state{flu_name=FluName,
                                           epoch_id=EpochID}=_S) ->
    %% TODO: Do anything with PKey?
    try
        TaggedCSum = check_or_make_tagged_checksum(CSum_tag, Client_CSum,Chunk),
        R = {seq_append, self(), Prefix, Chunk, TaggedCSum, ChunkExtra, EpochID},
        FluName ! R,
        receive
            {assignment, Offset, File} ->
                Size = iolist_size(Chunk),
                {ok, {Offset, Size, File}};
            witness ->
                {error, bad_arg};
            wedged ->
                {error, wedged}
        after 10*1000 ->
                {error, partition}
        end
    catch
        throw:{bad_csum, _CS} ->
            {error, bad_checksum};
        error:badarg ->
            error_logger:error_msg("Message send to ~p gave badarg, make certain server is running with correct registered name\n", [?MODULE]),
            {error, bad_arg}
    end.
 do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, #state{flu_name=FluName}) ->
    case sanitize_file_string(File) of
        ok ->
            case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
                {ok, Pid} ->
                    Meta = [{client_csum_tag, CSum_tag}, {client_csum, CSum}],
                    machi_file_proxy:write(Pid, Offset, Meta, Chunk);
                {error, trimmed} = Error ->
                    Error
            end;
        _ ->
            {error, bad_arg}
    end.
 do_server_read_chunk(File, Offset, Size, Opts, #state{flu_name=FluName})->
    case sanitize_file_string(File) of
        ok ->
            case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
                {ok, Pid} ->
                    case machi_file_proxy:read(Pid, Offset, Size, Opts) of
                        %% XXX FIXME 
                        %% For now we are omiting the checksum data because it blows up
                        %% protobufs.
                        {ok, ChunksAndTrimmed} -> {ok, ChunksAndTrimmed};
                        Other -> Other
                    end;
                {error, trimmed} = Error ->
                    Error
            end;
        _ ->
            {error, bad_arg}
    end.
 do_server_trim_chunk(File, Offset, Size, TriggerGC, #state{flu_name=FluName}) ->
    lager:debug("Hi there! I'm trimming this: ~s, (~p, ~p), ~p~n",
                [File, Offset, Size, TriggerGC]),
    case sanitize_file_string(File) of
        ok ->
            case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
                {ok, Pid} ->
                    machi_file_proxy:trim(Pid, Offset, Size, TriggerGC);
                {error, trimmed} = Trimmed ->
                    %% Should be returned back to (maybe) trigger repair
                    Trimmed
            end;
        _ ->
            {error, bad_arg}
    end.
 do_server_checksum_listing(File, #state{flu_name=FluName, data_dir=DataDir}=_S) ->
    case sanitize_file_string(File) of
        ok ->
            ok = sync_checksum_file(FluName, File),
            CSumPath = machi_util:make_checksum_filename(DataDir, File),
            %% TODO: If this file is legitimately bigger than our
            %% {packet_size,N} limit, then we'll have a difficult time, eh?
            case file:read_file(CSumPath) of
                {ok, Bin} ->
                    if byte_size(Bin) > (?PB_MAX_MSG_SIZE - 1024) ->
                            %% TODO: Fix this limitation by streaming the
                            %% binary in multiple smaller PB messages.
                            %% Also, don't read the file all at once. ^_^
                            error_logger:error_msg("~s:~w oversize ~s\n",
                                                   [?MODULE, ?LINE, CSumPath]),
                            {error, bad_arg};
                       true ->
                            {ok, Bin}
                    end;
                {error, enoent} ->
                    {error, no_such_file};
                {error, _} ->
                    {error, bad_arg}
            end;
        _ ->
            {error, bad_arg}
    end.
 do_server_list_files(#state{data_dir=DataDir}=_S) ->
    {_, WildPath} = machi_util:make_data_filename(DataDir, ""),
    Files = filelib:wildcard("*", WildPath),
    {ok, [begin
              {ok, FI} = file:read_file_info(WildPath ++ "/" ++ File),
              Size = FI#file_info.size,
              {Size, File}
          end || File <- Files]}.
 do_server_wedge_status(S) ->
    {Wedged_p, CurrentEpochID0} = ets:lookup_element(S#state.etstab, epoch, 2),
    CurrentEpochID = if CurrentEpochID0 == undefined ->
                             ?DUMMY_PV1_EPOCH;
                        true ->
                             CurrentEpochID0
                     end,
    {Wedged_p, CurrentEpochID}.
 do_server_delete_migration(File, #state{data_dir=DataDir}=_S) ->
    case sanitize_file_string(File) of
        ok ->
            {_, Path} = machi_util:make_data_filename(DataDir, File),
            case file:delete(Path) of
                ok ->
                    ok;
                {error, enoent} ->
                    {error, no_such_file};
                _ ->
                    {error, bad_arg}
            end;
        _ ->
            {error, bad_arg}
    end.
 do_server_trunc_hack(File, #state{data_dir=DataDir}=_S) ->
    case sanitize_file_string(File) of
        ok ->
            {_, Path} = machi_util:make_data_filename(DataDir, File),
            case file:open(Path, [read, write, binary, raw]) of
                {ok, FH} ->
                    try
                        {ok, ?MINIMUM_OFFSET} = file:position(FH,
                                                              ?MINIMUM_OFFSET),
                        ok = file:truncate(FH),
                        ok
                    after
                        file:close(FH)
                    end;
                {error, enoent} ->
                    {error, no_such_file};
                _ ->
                    {error, bad_arg}
            end;
        _ ->
            {error, bad_arg}
    end.
 append_server_dispatch(From, Prefix, Chunk, CSum, Extra, FluName, EpochId) ->
    Result = case handle_append(Prefix, Chunk, CSum, Extra, FluName, EpochId) of
        {ok, File, Offset} ->
            {assignment, Offset, File};
        Other ->
            Other
    end,
    From ! Result,
    exit(normal).
 handle_append(_Prefix, <<>>, _Csum, _Extra, _FluName, _EpochId) ->
    {error, bad_arg};
 handle_append(Prefix, Chunk, Csum, Extra, FluName, EpochId) ->
    Res = machi_flu_filename_mgr:find_or_make_filename_from_prefix(FluName, EpochId, {prefix, Prefix}),
    case Res of
        {file, F} ->
            case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, F}) of
                {ok, Pid} ->
                    {Tag, CS} = machi_util:unmake_tagged_csum(Csum),
                    Meta = [{client_csum_tag, Tag}, {client_csum, CS}],
                    machi_file_proxy:append(Pid, Meta, Extra, Chunk);
                {error, trimmed} = E ->
                    E
            end;
        Error ->
            Error
    end.
 sanitize_file_string(Str) ->
    case has_no_prohibited_chars(Str) andalso machi_util:is_valid_filename(Str) of
        true -> ok;
        false -> error
    end.
 has_no_prohibited_chars(Str) ->
    case re:run(Str, "/") of
        nomatch ->
            true;
        _ ->
            true
    end.
 sanitize_prefix(Prefix) ->
    %% We are using '^' as our component delimiter
    case re:run(Prefix, "/|\\^") of
        nomatch ->
            ok;
        _ ->
            error
    end.
 sync_checksum_file(FluName, File) ->
    %% We just lookup the pid here - we don't start a proxy server. If
    %% there isn't a pid for this file, then we just return ok. The
    %% csum file was synced when the proxy was shutdown.
    %%
    %% If there *is* a pid, we call the sync function to ensure the
    %% csum file is sync'd before we return. (Or an error if we get
    %% an error).
    case machi_flu_metadata_mgr:lookup_proxy_pid(FluName, {file, File}) of
        undefined ->
            ok;
        Pid ->
          machi_file_proxy:sync(Pid, csum)
    end.
 make_listener_regname(BaseName) ->
    list_to_atom(atom_to_list(BaseName) ++ "_listener").
 %% This is the name of the projection store that is spawned by the
 %% *flu*, for use primarily in testing scenarios.  In normal use, we
@ -170,8 +756,26 @@ start_listen_server(FluName, TcpPort, Witness_p, DataDir, EtsTab, ProjectionPid,
 make_projection_server_regname(BaseName) ->
    list_to_atom(atom_to_list(BaseName) ++ "_pstore").
-ets_table_name(FluName) when is_atom(FluName) ->
+check_or_make_tagged_checksum(?CSUM_TAG_NONE, _Client_CSum, Chunk) ->
-    list_to_atom(atom_to_list(FluName) ++ "_epoch").
+    %% TODO: If the client was foolish enough to use
    %% this type of non-checksum, then the client gets
    %% what it deserves wrt data integrity, alas.  In
    %% the client-side Chain Replication method, each
    %% server will calculated this independently, which
    %% isn't exactly what ought to happen for best data
    %% integrity checking.  In server-side CR, the csum
    %% should be calculated by the head and passed down
    %% the chain together with the value.
    CS = machi_util:checksum_chunk(Chunk),
    machi_util:make_tagged_csum(server_sha, CS);
 check_or_make_tagged_checksum(?CSUM_TAG_CLIENT_SHA, Client_CSum, Chunk) ->
    CS = machi_util:checksum_chunk(Chunk),
    if CS == Client_CSum ->
            machi_util:make_tagged_csum(server_sha,
                                        Client_CSum);
       true ->
            throw({bad_csum, CS})
    end.
 -ifdef(TEST).
@ -213,7 +817,7 @@ timing_demo_test2() ->
                     lists:foldl(fun(X, _) ->
                                         B = machi_checksums:encode_csum_file_entry_hex(X, 100, CSum),
                                         %% file:write(ZZZ, [B, 10]),
-                                         decode_csum_file_entry_hex(list_to_binary(B))
+                                         machi_checksums:decode_csum_file_entry_hex(list_to_binary(B))
                                 end, x, Xs)
             end),
    io:format(user, "~.3f sec\n", [HexUSec / 1000000]),
--- a/src/machi_flu1_append_server.erl
+++ b/src/machi_flu1_append_server.erl
@ -1,193 +0,0 @@
 %% -------------------------------------------------------------------
 %%
 %% Copyright (c) 2007-2015 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
 %% except in compliance with the License.  You may obtain
 %% a copy of the License at
 %%
 %%   http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing,
 %% software distributed under the License is distributed on an
 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 %% KIND, either express or implied.  See the License for the
 %% specific language governing permissions and limitations
 %% under the License.
 %%
 %% -------------------------------------------------------------------
 %% @doc Machi FLU1 append serialization server process
 -module(machi_flu1_append_server).
 -behavior(gen_server).
 -include("machi.hrl").
 -include("machi_projection.hrl").
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 -endif. % TEST
 -export([start_link/4]).
 -export([init/1]).
 -export([handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, code_change/3]).
 -export([int_update_wedge_state/3, int_wedge_myself/2]).
 -export([current_state/1, format_state/1]).
 -record(state, {
          flu_name        :: atom(),
          witness = false :: boolean(),
          wedged = true   :: boolean(),
          etstab          :: ets:tid(),
          epoch_id        :: 'undefined' | machi_dt:epoch_id()
         }).
 -define(INIT_TIMEOUT, 60*1000).
 -define(CALL_TIMEOUT, 60*1000).
 -spec start_link(pv1_server(), boolean(), boolean(),
                 undefined | machi_dt:epoch_id()) -> {ok, pid()}.
 start_link(Fluname, Witness_p, Wedged_p, EpochId) ->
    %% Reminder: Name is the "main" name of the FLU, i.e., no suffix
    gen_server:start_link({local, Fluname},
                          ?MODULE, [Fluname, Witness_p, Wedged_p, EpochId],
                          [{timeout, ?INIT_TIMEOUT}]).
 -spec current_state(atom() | pid()) -> term().
 current_state(PidSpec) ->
    gen_server:call(PidSpec, current_state, ?CALL_TIMEOUT).
 format_state(State) ->
    Fields = record_info(fields, state),
    [_Name | Values] = tuple_to_list(State),
    lists:zip(Fields, Values).
 int_update_wedge_state(PidSpec, Boolean, EpochId)
  when is_boolean(Boolean), is_tuple(EpochId) ->
    gen_server:cast(PidSpec, {wedge_state_change, Boolean, EpochId}).
 int_wedge_myself(PidSpec, EpochId)
  when is_tuple(EpochId) ->
    gen_server:cast(PidSpec, {wedge_myself, EpochId}).
 init([Fluname, Witness_p, Wedged_p, EpochId]) ->
    TID = ets:new(machi_flu1:ets_table_name(Fluname),
                  [set, protected, named_table, {read_concurrency, true}]),
    ets:insert(TID, {epoch, {Wedged_p, EpochId}}),
    {ok, #state{flu_name=Fluname, witness=Witness_p, wedged=Wedged_p,
                etstab=TID, epoch_id=EpochId}}.
 handle_call({seq_append, _From2, _NSInfo, _EpochID, _Prefix, _Chunk, _TCSum, _Opts},
            _From, #state{witness=true}=S) ->
    %% The FLU's machi_flu1_net_server process ought to filter all
    %% witness states, but we'll keep this clause for extra
    %% paranoia.
    {reply, witness, S};
 handle_call({seq_append, _From2, _NSInfo, _EpochID, _Prefix, _Chunk, _TCSum, _Opts},
            _From, #state{wedged=true}=S) ->
    {reply, wedged, S};
 handle_call({seq_append, _From2, NSInfo, EpochID,
             Prefix, Chunk, TCSum, Opts},
            From, #state{flu_name=FluName, epoch_id=OldEpochId}=S) ->
    %% Old is the one from our state, plain old 'EpochID' comes
    %% from the client.
    _ = case OldEpochId of
            EpochID ->
                spawn(fun() ->
                              append_server_dispatch(From, NSInfo,
                                                     Prefix, Chunk, TCSum, Opts,
                                                     FluName, EpochID)
                      end),
                {noreply, S};
            _ ->
                {reply, {error, bad_epoch}, S}
        end;
 %% TODO: Who sends this message?
 handle_call(wedge_status, _From,
            #state{wedged=Wedged_p, epoch_id=EpochId} = S) ->
    {reply, {wedge_status_reply, Wedged_p, EpochId}, S};
 handle_call(current_state, _From, S) ->
    {reply, S, S};
 handle_call(Else, From, S) ->
    io:format(user, "~s:handle_call: WHA? from=~w ~w\n", [?MODULE, From, Else]),
    {noreply, S}.
 handle_cast({wedge_myself, WedgeEpochId},
            #state{flu_name=FluName, wedged=Wedged_p, epoch_id=OldEpochId}=S) ->
    if not Wedged_p andalso WedgeEpochId == OldEpochId ->
            true = ets:insert(S#state.etstab,
                              {epoch, {true, OldEpochId}}),
            %% Tell my chain manager that it might want to react to
            %% this new world.
            Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
            spawn(fun() ->
                          catch machi_chain_manager1:trigger_react_to_env(Chmgr)
                  end),
            {noreply, S#state{wedged=true}};
       true ->
            {noreply, S}
    end;
 handle_cast({wedge_state_change, Boolean, {NewEpoch, _}=NewEpochId},
            #state{epoch_id=OldEpochId}=S) ->
    OldEpoch = case OldEpochId of {OldE, _} -> OldE;
                   undefined -> -1
               end,
    if NewEpoch >= OldEpoch ->
            true = ets:insert(S#state.etstab,
                              {epoch, {Boolean, NewEpochId}}),
            {noreply, S#state{wedged=Boolean, epoch_id=NewEpochId}};
       true ->
            {noreply, S}
    end;
 handle_cast(Else, S) ->
    io:format(user, "~s:handle_cast: WHA? ~p\n", [?MODULE, Else]),
    {noreply, S}.
 handle_info(Else, S) ->
    io:format(user, "~s:handle_info: WHA? ~p\n", [?MODULE, Else]),
    {noreply, S}.
 terminate(normal, _S) ->
    ok;
 terminate(Reason, _S) ->
    lager:warning("~s:terminate: ~w", [?MODULE, Reason]),
    ok.
 code_change(_OldVsn, S, _Extra) ->
    {ok, S}.
 append_server_dispatch(From, NSInfo,
                       Prefix, Chunk, TCSum, Opts, FluName, EpochId) ->
    Result = case handle_append(NSInfo,
                                Prefix, Chunk, TCSum, Opts, FluName, EpochId) of
        {ok, File, Offset} ->
            {assignment, Offset, File};
        Other ->
            Other
    end,
    _ = gen_server:reply(From, Result),
    ok.
 handle_append(NSInfo,
              Prefix, Chunk, TCSum, Opts, FluName, EpochId) ->
    Res = machi_flu_filename_mgr:find_or_make_filename_from_prefix(
            FluName, EpochId, {prefix, Prefix}, NSInfo),
    case Res of
        {file, F} ->
            case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, F}) of
                {ok, Pid} ->
                    {Tag, CS} = machi_util:unmake_tagged_csum(TCSum),
                    Meta = [{client_csum_tag, Tag}, {client_csum, CS}],
                    Extra = Opts#append_opts.chunk_extra,
                    machi_file_proxy:append(Pid, Meta, Extra, Chunk);
                {error, trimmed} = E ->
                    E
            end;
        Error ->
            Error
    end.
--- a/src/machi_flu1_client.erl
+++ b/src/machi_flu1_client.erl
@ -38,71 +38,6 @@
 %% TODO This EDoc was written first, and the EDoc and also `-type' and
 %% `-spec' definitions for {@link machi_proxy_flu1_client} and {@link
 %% machi_cr_client} must be improved.
 %%
 %% == Client API implementation notes ==
 %%
 %% At the moment, there are several modules that implement various
 %% subsets of the Machi API. The table below attempts to show how and
 %% why they differ.
 %%
 %% ```
 %% |--------------------------+-------+-----+------+------+-------+----------------|
 %% |                          | PB    |     | #    |      | Conn  | Epoch & NS     |
 %% | Module name              | Level | CR? | FLUS | Impl | Life? | version aware? |
 %% |--------------------------+-------+-----+------+------+-------+----------------|
 %% | machi_pb_high_api_client | high  | yes | many | proc | long  | no             |
 %% | machi_cr_client          | low   | yes | many | proc | long  | no             |
 %% | machi_proxy_flu1_client  | low   | no  | 1    | proc | long  | yes            |
 %% | machi_flu1_client        | low   | no  | 1    | lib  | short | yes            |
 %% |--------------------------+-------+-----+------+------+-------+----------------|
 %% '''
 %%
 %% In terms of use and API layering, the table rows are in highest`->'lowest
 %% order: each level calls the layer immediately below it.
 %%
 %% <dl>
 %% <dt> <b> PB Level</b> </dt>
 %% <dd> The Protocol Buffers API is divided logically into two levels,
 %% "low" and "high".  The low-level protocol is used for intra-chain
 %% communication.  The high-level protocol is used for clients outside
 %% of a Machi chain or Machi cluster of chains.
 %% </dd>
 %% <dt> <b> CR?</b> </dt>
 %% <dd> Does this API support (directly or indirectly) Chain
 %% Replication?  If `no', then the API has no awareness of multiple
 %% replicas of any file or file chunk; unaware clients can only
 %% perform operations at a single Machi FLU's file service or
 %% projection store service.
 %% </dd>
 %% <dt> <b> # FLUs</b> </dt>
 %% <dd> Now many FLUs does this API layer communicate with
 %% simultaneously?  Note that there is a one-to-one correspondence
 %% between this value and the "CR?" column's value.
 %% </dd>
 %% <dt> <b> Impl</b> </dt>
 %% <dd> Implementation: library-only or an Erlang process,
 %%      e.g., `gen_server'.
 %% </dd>
 %% <dt> <b> Conn Life?</b> </dt>
 %% <dd> Expected TCP session connection life: short or long.  At the
 %% lowest level, the {@link machi_flu1_client} API implementation takes
 %% no effort to reconnect to a remote FLU when its single TCP session
 %% is broken.  For long-lived connection life APIs, the server side will
 %% automatically attempt to reconnect to remote FLUs when a TCP session
 %% is broken.
 %% </dd>
 %% <dt> <b> Epoch &amp; NS version aware?</b> </dt>
 %% <dd> Are clients of this API responsible for knowing a chain's EpochID
 %% and namespace version numbers?  If `no', then the server side of the
 %% API will automatically attempt to discover/re-discover the EpochID and
 %% namespace version numbers whenever they change.
 %% </dd>
 %% </dl>
 %%
 %% The only protocol that we expect to be used by entities outside of
 %% a single Machi chain or a multi-chain cluster is the "high"
 %% Protocol Buffers API.  The {@link riak_pb_high_api_client} module
 %% is an Erlang reference implementation of this PB API.
 -module(machi_flu1_client).
@ -115,15 +50,14 @@
 -include_lib("pulse_otp/include/pulse_otp.hrl").
 -endif.
-define(SHORT_TIMEOUT, 2500).
+-define(HARD_TIMEOUT, 2500).
 -define(LONG_TIMEOUT, (60*1000)).
 -export([
         %% File API
-         append_chunk/6, append_chunk/7,
+         append_chunk/4, append_chunk/5,
-         append_chunk/8, append_chunk/9,
+         append_chunk_extra/5, append_chunk_extra/6,
-         read_chunk/7, read_chunk/8,
+         read_chunk/6, read_chunk/7,
-         checksum_list/2, checksum_list/3,
+         checksum_list/3, checksum_list/4,
         list_files/2, list_files/3,
         wedge_status/1, wedge_status/2,
@ -145,113 +79,106 @@
        ]).
 %% For "internal" replication only.
 -export([
-         write_chunk/7, write_chunk/8,
+         write_chunk/5, write_chunk/6,
-         trim_chunk/6,
+         trim_chunk/5,
         delete_migration/3, delete_migration/4,
         trunc_hack/3, trunc_hack/4
        ]).
 -type port_wrap()   :: {w,atom(),term()}.
-spec append_chunk(port_wrap(),
+%% @doc Append a chunk (binary- or iolist-style) of data to a file
-                   'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
+%% with `Prefix'.
-                   machi_dt:file_prefix(), machi_dt:chunk(),
+
-                   machi_dt:chunk_csum()) ->
+-spec append_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk()) ->
      {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
-append_chunk(Sock, NSInfo, EpochID, Prefix, Chunk, CSum) ->
+append_chunk(Sock, EpochID, Prefix, Chunk) ->
-    append_chunk(Sock, NSInfo, EpochID, Prefix, Chunk, CSum,
+    append_chunk2(Sock, EpochID, Prefix, Chunk, 0).
                 #append_opts{}, ?LONG_TIMEOUT).
 %% @doc Append a chunk (binary- or iolist-style) of data to a file
-%% with `Prefix' and also request an additional `Extra' bytes.
+%% with `Prefix'.
 %%
 %% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
 %% the file offsets that follow `Chunk''s position for the following 4K will
 %% be reserved by the file sequencer for later write(s) by the
 %% `write_chunk()' API.
 -spec append_chunk(machi_dt:inet_host(), machi_dt:inet_port(),
-                   'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
+                   machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk()) ->
                   machi_dt:file_prefix(), machi_dt:chunk(),
                   machi_dt:chunk_csum()) ->
      {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
-append_chunk(Host, TcpPort, NSInfo, EpochID, Prefix, Chunk, CSum) ->
+append_chunk(Host, TcpPort, EpochID, Prefix, Chunk) ->
    append_chunk(Host, TcpPort, NSInfo, EpochID, Prefix, Chunk, CSum,
                 #append_opts{}, ?LONG_TIMEOUT).
 -spec append_chunk(port_wrap(),
                   'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
                   machi_dt:file_prefix(), machi_dt:chunk(),
                   machi_dt:chunk_csum(), machi_dt:append_opts(), timeout()) ->
      {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
 append_chunk(Sock, NSInfo0, EpochID, Prefix, Chunk, CSum, Opts, Timeout) ->
    NSInfo = machi_util:ns_info_default(NSInfo0),
    append_chunk2(Sock, NSInfo, EpochID, Prefix, Chunk, CSum, Opts, Timeout).
 %% @doc Append a chunk (binary- or iolist-style) of data to a file
 %% with `Prefix' and also request an additional `Extra' bytes.
 %%
 %% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
 %% the file offsets that follow `Chunk''s position for the following 4K will
 %% be reserved by the file sequencer for later write(s) by the
 %% `write_chunk()' API.
 -spec append_chunk(machi_dt:inet_host(), machi_dt:inet_port(),
                   'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
                   machi_dt:file_prefix(), machi_dt:chunk(),
                   machi_dt:chunk_csum(), machi_dt:append_opts(), timeout()) ->
      {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
 append_chunk(Host, TcpPort, NSInfo0, EpochID,
             Prefix, Chunk, CSum, Opts, Timeout) ->
    Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
    try
-        NSInfo = machi_util:ns_info_default(NSInfo0),
+        append_chunk2(Sock, EpochID, Prefix, Chunk, 0)
-        append_chunk2(Sock, NSInfo, EpochID,
+    after
-                      Prefix, Chunk, CSum, Opts, Timeout)
+        disconnect(Sock)
    end.
 %% @doc Append a chunk (binary- or iolist-style) of data to a file
 %% with `Prefix' and also request an additional `Extra' bytes.
 %%
 %% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
 %% the file offsets that follow `Chunk''s position for the following 4K will
 %% be reserved by the file sequencer for later write(s) by the
 %% `write_chunk()' API.
 -spec append_chunk_extra(port_wrap(), machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) ->
      {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
 append_chunk_extra(Sock, EpochID, Prefix, Chunk, ChunkExtra)
  when is_integer(ChunkExtra), ChunkExtra >= 0 ->
    append_chunk2(Sock, EpochID, Prefix, Chunk, ChunkExtra).
 %% @doc Append a chunk (binary- or iolist-style) of data to a file
 %% with `Prefix' and also request an additional `Extra' bytes.
 %%
 %% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
 %% the file offsets that follow `Chunk''s position for the following 4K will
 %% be reserved by the file sequencer for later write(s) by the
 %% `write_chunk()' API.
 -spec append_chunk_extra(machi_dt:inet_host(), machi_dt:inet_port(),
               machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) ->
      {ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
 append_chunk_extra(Host, TcpPort, EpochID, Prefix, Chunk, ChunkExtra)
  when is_integer(ChunkExtra), ChunkExtra >= 0 ->
    Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
    try
        append_chunk2(Sock, EpochID, Prefix, Chunk, ChunkExtra)
    after
        disconnect(Sock)
    end.
 %% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
-spec read_chunk(port_wrap(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size(),
+-spec read_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size(),
-                 machi_dt:read_opts_x()) ->
+                 proplists:proplist()) ->
-      {ok, {[machi_dt:chunk_summary()], [machi_dt:chunk_pos()]}} |
+      {ok, machi_dt:chunk_s()} |
      {error, machi_dt:error_general() | 'not_written' | 'partial_read'} |
      {error, term()}.
-read_chunk(Sock, NSInfo0, EpochID, File, Offset, Size, Opts0)
+read_chunk(Sock, EpochID, File, Offset, Size, Opts)
  when Offset >= ?MINIMUM_OFFSET, Size >= 0 ->
-    NSInfo = machi_util:ns_info_default(NSInfo0),
+    read_chunk2(Sock, EpochID, File, Offset, Size, Opts).
    Opts = machi_util:read_opts_default(Opts0),
    read_chunk2(Sock, NSInfo, EpochID, File, Offset, Size, Opts).
 %% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
-spec read_chunk(machi_dt:inet_host(), machi_dt:inet_port(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
+-spec read_chunk(machi_dt:inet_host(), machi_dt:inet_port(), machi_dt:epoch_id(),
                 machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size(),
-                 machi_dt:read_opts_x()) ->
+                 proplists:proplist()) ->
-      {ok, [machi_dt:chunk_summary()]} |
+      {ok, machi_dt:chunk_s()} |
      {error, machi_dt:error_general() | 'not_written' | 'partial_read'} |
      {error, term()}.
-read_chunk(Host, TcpPort, NSInfo0, EpochID, File, Offset, Size, Opts0)
+read_chunk(Host, TcpPort, EpochID, File, Offset, Size, Opts)
  when Offset >= ?MINIMUM_OFFSET, Size >= 0 ->
    Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
    NSInfo = machi_util:ns_info_default(NSInfo0),
    Opts = machi_util:read_opts_default(Opts0),
    try
-        read_chunk2(Sock, NSInfo, EpochID, File, Offset, Size, Opts)
+        read_chunk2(Sock, EpochID, File, Offset, Size, Opts)
    after
        disconnect(Sock)
    end.
 %% @doc Fetch the list of chunk checksums for `File'.
-spec checksum_list(port_wrap(), machi_dt:file_name()) ->
+-spec checksum_list(port_wrap(), machi_dt:epoch_id(), machi_dt:file_name()) ->
      {ok, binary()} |
      {error, machi_dt:error_general() | 'no_such_file' | 'partial_read'} |
      {error, term()}.
-checksum_list(Sock, File) ->
+checksum_list(Sock, EpochID, File) ->
-    checksum_list2(Sock, File).
+    checksum_list2(Sock, EpochID, File).
 %% @doc Fetch the list of chunk checksums for `File'.
 %%
@ -275,13 +202,13 @@ checksum_list(Sock, File) ->
 %% Details of the encoding used inside the `binary()' blog can be found
 %% in the EDoc comments for {@link machi_flu1:decode_csum_file_entry/1}.
-spec checksum_list(machi_dt:inet_host(), machi_dt:inet_port(), machi_dt:file_name()) ->
+-spec checksum_list(machi_dt:inet_host(), machi_dt:inet_port(), machi_dt:epoch_id(), machi_dt:file_name()) ->
      {ok, binary()} |
      {error, machi_dt:error_general() | 'no_such_file'} | {error, term()}.
-checksum_list(Host, TcpPort, File) when is_integer(TcpPort) ->
+checksum_list(Host, TcpPort, EpochID, File) when is_integer(TcpPort) ->
    Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
    try
-        checksum_list2(Sock, File)
+        checksum_list2(Sock, EpochID, File)
    after
        disconnect(Sock)
    end.
@ -308,7 +235,7 @@ list_files(Host, TcpPort, EpochID) when is_integer(TcpPort) ->
 %% @doc Fetch the wedge status from the remote FLU.
 -spec wedge_status(port_wrap()) ->
-      {ok, {boolean(), machi_dt:epoch_id(), machi_dt:namespace_version(),machi_dt:namespace()}} | {error, term()}.
+      {ok, {boolean(), machi_dt:epoch_id()}} | {error, term()}.
 wedge_status(Sock) ->
    wedge_status2(Sock).
@ -316,7 +243,7 @@ wedge_status(Sock) ->
 %% @doc Fetch the wedge status from the remote FLU.
 -spec wedge_status(machi_dt:inet_host(), machi_dt:inet_port()) ->
-      {ok, {boolean(), machi_dt:epoch_id(), machi_dt:namespace_version(),machi_dt:namespace()}} | {error, term()}.
+      {ok, {boolean(), machi_dt:epoch_id()}} | {error, term()}.
 wedge_status(Host, TcpPort) when is_integer(TcpPort) ->
    Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
    try
@ -527,25 +454,23 @@ disconnect(_) ->
 %% @doc Restricted API: Write a chunk of already-sequenced data to
 %% `File' at `Offset'.
-spec write_chunk(port_wrap(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk(), machi_dt:chunk_csum()) ->
+-spec write_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk()) ->
      ok | {error, machi_dt:error_general()} | {error, term()}.
-write_chunk(Sock, NSInfo0, EpochID, File, Offset, Chunk, CSum)
+write_chunk(Sock, EpochID, File, Offset, Chunk)
  when Offset >= ?MINIMUM_OFFSET ->
-    NSInfo = machi_util:ns_info_default(NSInfo0),
+    write_chunk2(Sock, EpochID, File, Offset, Chunk).
    write_chunk2(Sock, NSInfo, EpochID, File, Offset, Chunk, CSum).
 %% @doc Restricted API: Write a chunk of already-sequenced data to
 %% `File' at `Offset'.
 -spec write_chunk(machi_dt:inet_host(), machi_dt:inet_port(),
-                  'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk(), machi_dt:chunk_csum()) ->
+                  machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk()) ->
      ok | {error, machi_dt:error_general()} | {error, term()}.
-write_chunk(Host, TcpPort, NSInfo0, EpochID, File, Offset, Chunk, CSum)
+write_chunk(Host, TcpPort, EpochID, File, Offset, Chunk)
  when Offset >= ?MINIMUM_OFFSET ->
    Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
    try
-        NSInfo = machi_util:ns_info_default(NSInfo0),
+        write_chunk2(Sock, EpochID, File, Offset, Chunk)
        write_chunk2(Sock, NSInfo, EpochID, File, Offset, Chunk, CSum)
    after
        disconnect(Sock)
    end.
@ -553,18 +478,16 @@ write_chunk(Host, TcpPort, NSInfo0, EpochID, File, Offset, Chunk, CSum)
 %% @doc Restricted API: Write a chunk of already-sequenced data to
 %% `File' at `Offset'.
-spec trim_chunk(port_wrap(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size()) ->
+-spec trim_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size()) ->
      ok | {error, machi_dt:error_general()} | {error, term()}.
-trim_chunk(Sock, NSInfo0, EpochID, File0, Offset, Size)
+trim_chunk(Sock, EpochID, File0, Offset, Size)
  when Offset >= ?MINIMUM_OFFSET ->
    ReqID = <<"id">>,
    NSInfo = machi_util:ns_info_default(NSInfo0),
    #ns_info{version=NSVersion, name=NS} = NSInfo,
    File = machi_util:make_binary(File0),
    true = (Offset >= ?MINIMUM_OFFSET),
    Req = machi_pb_translate:to_pb_request(
            ReqID,
-            {low_trim_chunk, NSVersion, NS, EpochID, File, Offset, Size, 0}),
+            {low_trim_chunk, EpochID, File, Offset, Size, 0}),
    do_pb_request_common(Sock, ReqID, Req).
 %% @doc Restricted API: Delete a file after it has been successfully
@ -611,88 +534,83 @@ trunc_hack(Host, TcpPort, EpochID, File) when is_integer(TcpPort) ->
 %%%%%%%%%%%%%%%%%%%%%%%%%%%
-read_chunk2(Sock, NSInfo, EpochID, File0, Offset, Size, Opts) ->
+read_chunk2(Sock, EpochID, File0, Offset, Size, Opts) ->
    ReqID = <<"id">>,
    #ns_info{version=NSVersion, name=NS} = NSInfo,
    File = machi_util:make_binary(File0),
    Req = machi_pb_translate:to_pb_request(
            ReqID,
-            {low_read_chunk, NSVersion, NS, EpochID, File, Offset, Size, Opts}),
+            {low_read_chunk, EpochID, File, Offset, Size, Opts}),
    do_pb_request_common(Sock, ReqID, Req).
-append_chunk2(Sock, NSInfo, EpochID,
+append_chunk2(Sock, EpochID, Prefix0, Chunk0, ChunkExtra) ->
              Prefix0, Chunk, CSum0, Opts, Timeout) ->
    ReqID = <<"id">>,
-    Prefix = machi_util:make_binary(Prefix0),
+    {Chunk, CSum_tag, CSum} =
-    {CSum_tag, CSum} = case CSum0 of
+        case Chunk0 of
-                           <<>> ->
+            X when is_binary(X) ->
-                               {?CSUM_TAG_NONE, <<>>};
+                {Chunk0, ?CSUM_TAG_NONE, <<>>};
-                           {_Tag, _CS} ->
+            {ChunkCSum, Chk} ->
-                               CSum0;
+                {Tag, CS} = machi_util:unmake_tagged_csum(ChunkCSum),
-                           B when is_binary(B) ->
+                {Chk, Tag, CS}
                               machi_util:unmake_tagged_csum(CSum0)
        end,
-    #ns_info{version=NSVersion, name=NS, locator=NSLocator} = NSInfo,
+    PKey = <<>>,                                % TODO
-    %% NOTE: The tuple position of NSLocator is a bit odd, because EpochID
+    Prefix = machi_util:make_binary(Prefix0),
    %%       _must_ be in the 4th position (as NSV & NS must be in 2nd & 3rd).
    Req = machi_pb_translate:to_pb_request(
            ReqID,
-            {low_append_chunk, NSVersion, NS, EpochID, NSLocator,
+            {low_append_chunk, EpochID, PKey, Prefix, Chunk, CSum_tag, CSum,
-             Prefix, Chunk, CSum_tag, CSum, Opts}),
+             ChunkExtra}),
-    do_pb_request_common(Sock, ReqID, Req, true, Timeout).
+    do_pb_request_common(Sock, ReqID, Req).
-write_chunk2(Sock, NSInfo, EpochID, File0, Offset, Chunk, CSum0) ->
+write_chunk2(Sock, EpochID, File0, Offset, Chunk0) ->
    ReqID = <<"id">>,
    #ns_info{version=NSVersion, name=NS} = NSInfo,
    File = machi_util:make_binary(File0),
    true = (Offset >= ?MINIMUM_OFFSET),
-    {CSum_tag, CSum} = case CSum0 of
+    {Chunk, CSum_tag, CSum} =
-                           <<>> ->
+        case Chunk0 of
-                               {?CSUM_TAG_NONE, <<>>};
+            X when is_binary(X) ->
-                           {_Tag, _CS} ->
+                {Chunk0, ?CSUM_TAG_NONE, <<>>};
-                               CSum0;
+            {ChunkCSum, Chk} ->
-                           B when is_binary(B) ->
+                {Tag, CS} = machi_util:unmake_tagged_csum(ChunkCSum),
-                               machi_util:unmake_tagged_csum(CSum0)
+                {Chk, Tag, CS}
        end,
    Req = machi_pb_translate:to_pb_request(
            ReqID,
-            {low_write_chunk, NSVersion, NS, EpochID, File, Offset, Chunk, CSum_tag, CSum}),
+            {low_write_chunk, EpochID, File, Offset, Chunk, CSum_tag, CSum}),
    do_pb_request_common(Sock, ReqID, Req).
 list2(Sock, EpochID) ->
    ReqID = <<"id">>,
    Req = machi_pb_translate:to_pb_request(
-            ReqID, {low_skip_wedge, {low_list_files, EpochID}}),
+            ReqID, {low_list_files, EpochID}),
    do_pb_request_common(Sock, ReqID, Req).
 wedge_status2(Sock) ->
    ReqID = <<"id">>,
    Req = machi_pb_translate:to_pb_request(
-            ReqID, {low_skip_wedge, {low_wedge_status}}),
+            ReqID, {low_wedge_status, undefined}),
    do_pb_request_common(Sock, ReqID, Req).
 echo2(Sock, Message) ->
    ReqID = <<"id">>,
    Req = machi_pb_translate:to_pb_request(
-            ReqID, {low_skip_wedge, {low_echo, Message}}),
+            ReqID, {low_echo, undefined, Message}),
    do_pb_request_common(Sock, ReqID, Req).
-checksum_list2(Sock, File) ->
+checksum_list2(Sock, EpochID, File) ->
    ReqID = <<"id">>,
    Req = machi_pb_translate:to_pb_request(
-            ReqID, {low_skip_wedge, {low_checksum_list, File}}),
+            ReqID, {low_checksum_list, EpochID, File}),
    do_pb_request_common(Sock, ReqID, Req).
 delete_migration2(Sock, EpochID, File) ->
    ReqID = <<"id">>,
    Req = machi_pb_translate:to_pb_request(
-            ReqID, {low_skip_wedge, {low_delete_migration, EpochID, File}}),
+            ReqID, {low_delete_migration, EpochID, File}),
    do_pb_request_common(Sock, ReqID, Req).
 trunc_hack2(Sock, EpochID, File) ->
    ReqID = <<"id-trunc">>,
    Req = machi_pb_translate:to_pb_request(
-            ReqID, {low_skip_wedge, {low_trunc_hack, EpochID, File}}),
+            ReqID, {low_trunc_hack, EpochID, File}),
    do_pb_request_common(Sock, ReqID, Req).
 get_latest_epochid2(Sock, ProjType) ->
@ -735,18 +653,18 @@ kick_projection_reaction2(Sock, _Options) ->
    ReqID = <<42>>,
    Req = machi_pb_translate:to_pb_request(
                ReqID, {low_proj, {kick_projection_reaction}}),
-    do_pb_request_common(Sock, ReqID, Req, false, ?LONG_TIMEOUT).
+    do_pb_request_common(Sock, ReqID, Req, false).
 do_pb_request_common(Sock, ReqID, Req) ->
-    do_pb_request_common(Sock, ReqID, Req, true, ?LONG_TIMEOUT).
+    do_pb_request_common(Sock, ReqID, Req, true).
-do_pb_request_common(Sock, ReqID, Req, GetReply_p, Timeout) ->
+do_pb_request_common(Sock, ReqID, Req, GetReply_p) ->
    erase(bad_sock),
    try
        ReqBin = list_to_binary(machi_pb:encode_mpb_ll_request(Req)),
        ok = w_send(Sock, ReqBin),
        if GetReply_p ->
-                case w_recv(Sock, 0, Timeout) of
+                case w_recv(Sock, 0) of
                    {ok, RespBin} ->
                        Resp = machi_pb:decode_mpb_ll_response(RespBin),
                        {ReqID2, Reply} = machi_pb_translate:from_pb_response(Resp),
@ -792,7 +710,7 @@ w_connect(#p_srvr{proto_mod=?MODULE, address=Host, port=Port, props=Props}=_P)->
        case proplists:get_value(session_proto, Props, tcp) of
            tcp ->
 put(xxx, goofus),
-                Sock = machi_util:connect(Host, Port, ?SHORT_TIMEOUT),
+                Sock = machi_util:connect(Host, Port, ?HARD_TIMEOUT),
 put(xxx, Sock),
                ok = inet:setopts(Sock, ?PB_PACKET_OPTS),
                {w,tcp,Sock};
@ -816,8 +734,8 @@ w_close({w,tcp,Sock}) ->
    catch gen_tcp:close(Sock),
    ok.
-w_recv({w,tcp,Sock}, Amt, Timeout) ->
+w_recv({w,tcp,Sock}, Amt) ->
-    gen_tcp:recv(Sock, Amt, Timeout).
+    gen_tcp:recv(Sock, Amt, ?HARD_TIMEOUT).
 w_send({w,tcp,Sock}, IoData) ->
    gen_tcp:send(Sock, IoData).
--- a/src/machi_flu1_net_server.erl
+++ b/src/machi_flu1_net_server.erl
@ -1,634 +0,0 @@
 %% -------------------------------------------------------------------
 %%
 %% Copyright (c) 2007-2015 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
 %% except in compliance with the License.  You may obtain
 %% a copy of the License at
 %%
 %%   http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing,
 %% software distributed under the License is distributed on an
 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 %% KIND, either express or implied.  See the License for the
 %% specific language governing permissions and limitations
 %% under the License.
 %%
 %% -------------------------------------------------------------------
 %% @doc Ranch protocol callback module to handle PB protocol over
 %% transport, including both high and low modes.
 %% TODO
 %% - Two modes, high and low should be separated at listener level?
 -module(machi_flu1_net_server).
 -behaviour(gen_server).
 -behaviour(ranch_protocol).
 -export([start_link/4]).
 -export([init/1]).
 -export([handle_call/3, handle_cast/2, handle_info/2,
         terminate/2, code_change/3]).
 -include_lib("kernel/include/file.hrl").
 -include("machi.hrl").
 -include("machi_pb.hrl").
 -include("machi_projection.hrl").
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 -endif. % TEST
 -record(state, {
          %% Ranch's transport management stuff
          ref         :: ranch:ref(),
          socket      :: socket(),
          transport   :: module(),
          %% Machi FLU configurations, common for low and high
          data_dir    :: string(),
          witness     :: boolean(),
          pb_mode     :: undefined | high | low,
          %% - Used in projection related requests in low mode
          %% - Used in spawning CR client in high mode
          proj_store  :: pid(),
          %% Low mode only items
          %% Current best knowledge, used for wedge_self / bad_epoch check
          epoch_id    :: undefined | machi_dt:epoch_id(),
          %% Used in dispatching append_chunk* reqs to the
          %% append serializing process
          flu_name    :: pv1_server(),
          %% Used in server_wedge_status to lookup the table
          epoch_tab   :: ets:tab(),
          %% Clustering: cluster map version number
          namespace_version = 0 :: machi_dt:namespace_version(),
          %% Clustering: my (and my chain's) assignment to a specific namespace
          namespace = <<>> :: machi_dt:namespace(),
          %% High mode only
          high_clnt   :: pid(),
          %% anything you want
          props = []  :: proplists:proplist()
         }).
 -type socket() :: any().
 -type state()  :: #state{}.
 -spec start_link(ranch:ref(), socket(), module(), [term()]) -> {ok, pid()}.
 start_link(Ref, Socket, Transport, [FluName, Witness, DataDir, EpochTab, ProjStore, Props]) ->
    NS = proplists:get_value(namespace, Props, <<>>),
    true = is_binary(NS),
    proc_lib:start_link(?MODULE, init, [#state{ref=Ref,
                                               socket=Socket,
                                               transport=Transport,
                                               flu_name=FluName,
                                               witness=Witness,
                                               data_dir=DataDir,
                                               epoch_tab=EpochTab,
                                               proj_store=ProjStore,
                                               namespace=NS,
                                               props=Props}]).
 -spec init(state()) -> no_return().
 init(#state{ref=Ref, socket=Socket, transport=Transport}=State) ->
    ok = proc_lib:init_ack({ok, self()}),
    ok = ranch:accept_ack(Ref),
    {_Wedged_p, CurrentEpochID} = lookup_epoch(State),
    ok = Transport:setopts(Socket, [{active, once}|?PB_PACKET_OPTS]),
    gen_server:enter_loop(?MODULE, [], State#state{epoch_id=CurrentEpochID}).
 handle_call(Request, _From, S) ->
    lager:warning("~s:handle_call UNKNOWN message: ~w", [?MODULE, Request]),
    Reply = {error, {unknown_message, Request}},
    {reply, Reply, S}.
 handle_cast(_Msg, S) ->
    lager:warning("~s:handle_cast UNKNOWN message: ~w", [?MODULE, _Msg]),
    {noreply, S}.
 %% TODO: Other transport support needed?? TLS/SSL, SCTP
 handle_info({tcp, Socket, Data}=_Info, #state{socket=Socket}=S) ->
    lager:debug("~s:handle_info: ~w", [?MODULE, _Info]),
    transport_received(Socket, Data, S);
 handle_info({tcp_closed, Socket}=_Info, #state{socket=Socket}=S) ->
    lager:debug("~s:handle_info: ~w", [?MODULE, _Info]),
    transport_closed(Socket, S);
 handle_info({tcp_error, Socket, Reason}=_Info, #state{socket=Socket}=S) ->
    lager:warning("~s:handle_info (socket=~w) tcp_error: ~w", [?MODULE, Socket, Reason]),
    transport_error(Socket, Reason, S);
 handle_info(_Info, S) ->
    lager:warning("~s:handle_info UNKNOWN message: ~w", [?MODULE, _Info]),
    {noreply, S}.
 terminate(normal, #state{socket=undefined}=_S) ->
    ok;
 terminate(Reason, #state{socket=undefined}=_S) ->
    lager:warning("~s:terminate (socket=undefined): ~w", [?MODULE, Reason]),
    ok;
 terminate(normal, #state{socket=Socket}=_S) ->
    (catch gen_tcp:close(Socket)),
    ok;
 terminate(Reason, #state{socket=Socket}=_S) ->
    lager:warning("~s:terminate (socket=Socket): ~w", [?MODULE, Reason]),
    (catch gen_tcp:close(Socket)),
    ok.
 code_change(_OldVsn, S, _Extra) ->
    {ok, S}.
 %% -- private
 %%%% Common transport handling
 -spec transport_received(socket(), machi_dt:chunk(), state()) ->
                                {noreply, state()}.
 transport_received(Socket, <<"QUIT\n">>, #state{socket=Socket}=S) ->
    {stop, normal, S};
 transport_received(Socket, Bin, #state{transport=Transport}=S) ->
    {RespBin, S2} =
        case machi_pb:decode_mpb_ll_request(Bin) of
            LL_req when LL_req#mpb_ll_request.do_not_alter == 2 ->
                {R, NewS} = do_pb_ll_request(LL_req, S),
                {maybe_encode_response(R), set_mode(low, NewS)};
            _ ->
                HL_req = machi_pb:decode_mpb_request(Bin),
                1 = HL_req#mpb_request.do_not_alter,
                {R, NewS} = do_pb_hl_request(HL_req, make_high_clnt(S)),
                {machi_pb:encode_mpb_response(R), set_mode(high, NewS)}
        end,
    case RespBin of
        async_no_response ->
            Transport:setopts(Socket, [{active, once}]),
            {noreply, S2};
        _ ->
            case Transport:send(Socket, RespBin) of
                ok ->
                    Transport:setopts(Socket, [{active, once}]),
                    {noreply, S2};
                {error, Reason} ->
                    transport_error(Socket, Reason, S2)
            end
    end.
 -spec transport_closed(socket(), state()) -> {stop, term(), state()}.
 transport_closed(_Socket, S) ->
    {stop, normal, S}.
 -spec transport_error(socket(), term(), state()) -> no_return().
 transport_error(Socket, Reason, #state{transport=Transport}=_S) ->
    Msg = io_lib:format("Socket error ~w", [Reason]),
    R = #mpb_ll_response{req_id= <<>>,
                         generic=#mpb_errorresp{code=1, msg=Msg}},
    _Resp = machi_pb:encode_mpb_ll_response(R),
    %% TODO for TODO comments: comments below with four %s are copy-n-paste'd,
    %% then it should be considered they are still open and should be addressed.
    %%%% TODO: Weird that sometimes neither catch nor try/catch
    %%%%       can prevent OTP's SASL from logging an error here.
    %%%%       Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,.......
    %%%% TODO: is this what causes the intermittent PULSE deadlock errors?
    %%%% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000),
    (catch Transport:close(Socket)),
    _ = lager:warning("Socket error (~w -> ~w): ~w",
                      [Transport:sockname(Socket), Transport:peername(Socket), Reason]),
    %% TODO: better to exit with `Reason' without logging?
    exit(normal).
 maybe_encode_response(async_no_response=R) ->
    R;
 maybe_encode_response(R) ->
    machi_pb:encode_mpb_ll_response(R).
 set_mode(Mode, #state{pb_mode=undefined}=S) ->
    S#state{pb_mode=Mode};
 set_mode(_, S) ->
    S.
 %%%% Low PB mode %%%%
 do_pb_ll_request(#mpb_ll_request{req_id=ReqID}, #state{pb_mode=high}=S) ->
    Result = {high_error, 41, "Low protocol request while in high mode"},
    {machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
 do_pb_ll_request(PB_request, S) ->
    Req = machi_pb_translate:from_pb_request(PB_request),
    {ReqID, Cmd, Result, S2} = 
        case Req of
            {RqID, {low_skip_wedge, LowSubCmd}=Cmd0} ->
                %% Skip wedge check for these unprivileged commands
                {Rs, NewS} = do_pb_ll_request3(LowSubCmd, S),
                {RqID, Cmd0, Rs, NewS};
            {RqID, {low_proj, _LowSubCmd}=Cmd0} ->
                {Rs, NewS} = do_pb_ll_request3(Cmd0, S),
                {RqID, Cmd0, Rs, NewS};
            {RqID, Cmd0} ->
                %% All remaining must have NSVersion, NS, & EpochID at next pos
                NSVersion = element(2, Cmd0),
                NS = element(3, Cmd0),
                EpochID = element(4, Cmd0),
                {Rs, NewS} = do_pb_ll_request2(NSVersion, NS, EpochID, Cmd0, S),
                {RqID, Cmd0, Rs, NewS}
        end,
    {machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
 %% do_pb_ll_request2(): Verification of epoch details & namespace details.
 do_pb_ll_request2(NSVersion, NS, EpochID, CMD, S) ->
    {Wedged_p, CurrentEpochID} = lookup_epoch(S),
    if not is_tuple(EpochID) orelse tuple_size(EpochID) /= 2 ->
            exit({bad_epoch_id, EpochID, for, CMD});
       Wedged_p == true ->
            {{error, wedged}, S#state{epoch_id=CurrentEpochID}};
       EpochID /= CurrentEpochID ->
            {Epoch, _} = EpochID,
            {CurrentEpoch, _} = CurrentEpochID,
            if Epoch < CurrentEpoch ->
                    {{error, bad_epoch}, S};
               true ->
                    _ = machi_flu1:wedge_myself(S#state.flu_name, CurrentEpochID),
                    {{error, wedged}, S#state{epoch_id=CurrentEpochID}}
            end;
       true ->
            #state{namespace_version=MyNSVersion, namespace=MyNS} = S,
            if NSVersion /= MyNSVersion ->
                    {{error, bad_epoch}, S};
               NS /= MyNS ->
                    {{error, bad_arg}, S};
               true ->
                    do_pb_ll_request3(CMD, S)
            end
    end.
 lookup_epoch(#state{epoch_tab=T}) ->
    %% TODO: race in shutdown to access ets table after owner dies
    ets:lookup_element(T, epoch, 2).
 %% Witness status does not matter below.
 do_pb_ll_request3({low_echo, Msg}, S) ->
    {Msg, S};
 do_pb_ll_request3({low_auth, _User, _Pass}, S) ->
    {-6, S};
 do_pb_ll_request3({low_wedge_status}, S) ->
    {do_server_wedge_status(S), S};
 do_pb_ll_request3({low_proj, PCMD}, S) ->
    {do_server_proj_request(PCMD, S), S};
 %% Witness status *matters* below
 do_pb_ll_request3({low_append_chunk, NSVersion, NS, EpochID, NSLocator,
                   Prefix, Chunk, CSum_tag,
                   CSum, Opts},
                  #state{witness=false}=S) ->
    NSInfo = #ns_info{version=NSVersion, name=NS, locator=NSLocator},
    {do_server_append_chunk(NSInfo, EpochID,
                            Prefix, Chunk, CSum_tag, CSum,
                            Opts, S), S};
 do_pb_ll_request3({low_write_chunk, _NSVersion, _NS, _EpochID, File, Offset, Chunk, CSum_tag,
                   CSum},
                  #state{witness=false}=S) ->
    {do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, S), S};
 do_pb_ll_request3({low_read_chunk, _NSVersion, _NS, _EpochID, File, Offset, Size, Opts},
                  #state{witness=false} = S) ->
    {do_server_read_chunk(File, Offset, Size, Opts, S), S};
 do_pb_ll_request3({low_trim_chunk, _NSVersion, _NS, _EpochID, File, Offset, Size, TriggerGC},
                  #state{witness=false}=S) ->
    {do_server_trim_chunk(File, Offset, Size, TriggerGC, S), S};
 do_pb_ll_request3({low_checksum_list, File},
                  #state{witness=false}=S) ->
    {do_server_checksum_listing(File, S), S};
 do_pb_ll_request3({low_list_files, _EpochID},
                  #state{witness=false}=S) ->
    {do_server_list_files(S), S};
 do_pb_ll_request3({low_delete_migration, _EpochID, File},
                  #state{witness=false}=S) ->
    {do_server_delete_migration(File, S),
     #state{witness=false}=S};
 do_pb_ll_request3({low_trunc_hack, _EpochID, File},
                  #state{witness=false}=S) ->
    {do_server_trunc_hack(File, S), S};
 do_pb_ll_request3(_, #state{witness=true}=S) ->
    {{error, bad_arg}, S}.                       % TODO: new status code??
 do_server_proj_request({get_latest_epochid, ProjType},
                       #state{proj_store=ProjStore}) ->
    machi_projection_store:get_latest_epochid(ProjStore, ProjType);
 do_server_proj_request({read_latest_projection, ProjType},
                       #state{proj_store=ProjStore}) ->
    machi_projection_store:read_latest_projection(ProjStore, ProjType);
 do_server_proj_request({read_projection, ProjType, Epoch},
                       #state{proj_store=ProjStore}) ->
    machi_projection_store:read(ProjStore, ProjType, Epoch);
 do_server_proj_request({write_projection, ProjType, Proj},
                       #state{flu_name=FluName, proj_store=ProjStore}) ->
    if Proj#projection_v1.epoch_number == ?SPAM_PROJ_EPOCH ->
            %% io:format(user, "DBG ~s ~w ~P\n", [?MODULE, ?LINE, Proj, 5]),
            Chmgr = machi_flu_psup:make_fitness_regname(FluName),
            [Map] = Proj#projection_v1.dbg,
            catch machi_fitness:send_fitness_update_spam(
                    Chmgr, Proj#projection_v1.author_server, Map);
       true ->
            catch machi_projection_store:write(ProjStore, ProjType, Proj)
    end;
 do_server_proj_request({get_all_projections, ProjType},
                       #state{proj_store=ProjStore}) ->
    machi_projection_store:get_all_projections(ProjStore, ProjType);
 do_server_proj_request({list_all_projections, ProjType},
                       #state{proj_store=ProjStore}) ->
    machi_projection_store:list_all_projections(ProjStore, ProjType);
 do_server_proj_request({kick_projection_reaction},
                       #state{flu_name=FluName}) ->
    %% Tell my chain manager that it might want to react to
    %% this new world.
    Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
    spawn(fun() ->
                  catch machi_chain_manager1:trigger_react_to_env(Chmgr)
          end),
    async_no_response.
 do_server_append_chunk(NSInfo, EpochID,
                       Prefix, Chunk, CSum_tag, CSum,
                       Opts, S) ->
    case sanitize_prefix(Prefix) of
        ok ->
            do_server_append_chunk2(NSInfo, EpochID,
                                    Prefix, Chunk, CSum_tag, CSum,
                                    Opts, S);
        _ ->
            {error, bad_arg}
    end.
 do_server_append_chunk2(NSInfo, EpochID,
                        Prefix, Chunk, CSum_tag, Client_CSum,
                        Opts, #state{flu_name=FluName,
                                     epoch_id=EpochID}=_S) ->
    %% TODO: Do anything with PKey?
    try
        TaggedCSum = check_or_make_tagged_checksum(CSum_tag, Client_CSum,Chunk),
        R = {seq_append, self(), NSInfo, EpochID,
             Prefix, Chunk, TaggedCSum, Opts},
        case gen_server:call(FluName, R, 10*1000) of
            {assignment, Offset, File} ->
                Size = iolist_size(Chunk),
                {ok, {Offset, Size, File}};
            witness ->
                {error, bad_arg};
            wedged ->
                {error, wedged};
            {error, timeout} ->
                {error, partition}
        end
    catch
        throw:{bad_csum, _CS} ->
            {error, bad_checksum};
        error:badarg ->
            lager:error("badarg at ~w:do_server_append_chunk2:~w ~w",
                        [?MODULE, ?LINE, erlang:get_stacktrace()]),
            {error, bad_arg}
    end.
 do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, #state{flu_name=FluName}) ->
    case sanitize_file_string(File) of
        ok ->
            case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
                {ok, Pid} ->
                    Meta = [{client_csum_tag, CSum_tag}, {client_csum, CSum}],
                    machi_file_proxy:write(Pid, Offset, Meta, Chunk);
                {error, trimmed} = Error ->
                    Error
            end;
        _ ->
            {error, bad_arg}
    end.
 do_server_read_chunk(File, Offset, Size, Opts, #state{flu_name=FluName})->
    case sanitize_file_string(File) of
        ok ->
            case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
                {ok, Pid} ->
                    case machi_file_proxy:read(Pid, Offset, Size, Opts) of
                        %% XXX FIXME 
                        %% For now we are omiting the checksum data because it blows up
                        %% protobufs.
                        {ok, ChunksAndTrimmed} -> {ok, ChunksAndTrimmed};
                        Other -> Other
                    end;
                {error, trimmed} = Error ->
                    Error
            end;
        _ ->
            {error, bad_arg}
    end.
 do_server_trim_chunk(File, Offset, Size, TriggerGC, #state{flu_name=FluName}) ->
    lager:debug("Hi there! I'm trimming this: ~s, (~p, ~p), ~p~n",
                [File, Offset, Size, TriggerGC]),
    case sanitize_file_string(File) of
        ok ->
            case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
                {ok, Pid} ->
                    machi_file_proxy:trim(Pid, Offset, Size, TriggerGC);
                {error, trimmed} = Trimmed ->
                    %% Should be returned back to (maybe) trigger repair
                    Trimmed
            end;
        _ ->
            {error, bad_arg}
    end.
 do_server_checksum_listing(File, #state{flu_name=FluName, data_dir=DataDir}=_S) ->
    case sanitize_file_string(File) of
        ok ->
            case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
                {ok, Pid} ->
                    {ok, List} = machi_file_proxy:checksum_list(Pid),
                    Bin = erlang:term_to_binary(List),
                    if byte_size(Bin) > (?PB_MAX_MSG_SIZE - 1024) ->
                            %% TODO: Fix this limitation by streaming the
                            %% binary in multiple smaller PB messages.
                            %% Also, don't read the file all at once. ^_^
                            error_logger:error_msg("~s:~w oversize ~s\n",
                                                   [?MODULE, ?LINE, DataDir]),
                            {error, bad_arg};
                       true ->
                            {ok, Bin}
                    end;
                {error, trimmed} ->
                    {error, trimmed}
            end;
        _ ->
            {error, bad_arg}
    end.
 do_server_list_files(#state{data_dir=DataDir}=_S) ->
    {_, WildPath} = machi_util:make_data_filename(DataDir, ""),
    Files = filelib:wildcard("*", WildPath),
    {ok, [begin
              {ok, FI} = file:read_file_info(WildPath ++ "/" ++ File),
              Size = FI#file_info.size,
              {Size, File}
          end || File <- Files]}.
 do_server_wedge_status(#state{namespace_version=NSVersion, namespace=NS}=S) ->
    {Wedged_p, CurrentEpochID0} = lookup_epoch(S),
    CurrentEpochID = if CurrentEpochID0 == undefined ->
                             ?DUMMY_PV1_EPOCH;
                        true ->
                             CurrentEpochID0
                     end,
    {Wedged_p, CurrentEpochID, NSVersion, NS}.
 do_server_delete_migration(File, #state{data_dir=DataDir}=_S) ->
    case sanitize_file_string(File) of
        ok ->
            {_, Path} = machi_util:make_data_filename(DataDir, File),
            case file:delete(Path) of
                ok ->
                    ok;
                {error, enoent} ->
                    {error, no_such_file};
                _ ->
                    {error, bad_arg}
            end;
        _ ->
            {error, bad_arg}
    end.
 do_server_trunc_hack(File, #state{data_dir=DataDir}=_S) ->
    case sanitize_file_string(File) of
        ok ->
            {_, Path} = machi_util:make_data_filename(DataDir, File),
            case file:open(Path, [read, write, binary, raw]) of
                {ok, FH} ->
                    try
                        {ok, ?MINIMUM_OFFSET} = file:position(FH,
                                                              ?MINIMUM_OFFSET),
                        ok = file:truncate(FH),
                        ok
                    after
                        file:close(FH)
                    end;
                {error, enoent} ->
                    {error, no_such_file};
                _ ->
                    {error, bad_arg}
            end;
        _ ->
            {error, bad_arg}
    end.
 sanitize_file_string(Str) ->
    case has_no_prohibited_chars(Str) andalso machi_util:is_valid_filename(Str) of
        true -> ok;
        false -> error
    end.
 has_no_prohibited_chars(Str) ->
    case re:run(Str, "/") of
        nomatch ->
            true;
        _ ->
            true
    end.
 sanitize_prefix(Prefix) ->
    %% We are using '^' as our component delimiter
    case re:run(Prefix, "/|\\^") of
        nomatch ->
            ok;
        _ ->
            error
    end.
 check_or_make_tagged_checksum(?CSUM_TAG_NONE, _Client_CSum, Chunk) ->
    %% TODO: If the client was foolish enough to use
    %% this type of non-checksum, then the client gets
    %% what it deserves wrt data integrity, alas.  In
    %% the client-side Chain Replication method, each
    %% server will calculated this independently, which
    %% isn't exactly what ought to happen for best data
    %% integrity checking.  In server-side CR, the csum
    %% should be calculated by the head and passed down
    %% the chain together with the value.
    CS = machi_util:checksum_chunk(Chunk),
    machi_util:make_tagged_csum(server_sha, CS);
 check_or_make_tagged_checksum(?CSUM_TAG_CLIENT_SHA, Client_CSum, Chunk) ->
    CS = machi_util:checksum_chunk(Chunk),
    if CS == Client_CSum ->
            machi_util:make_tagged_csum(server_sha,
                                        Client_CSum);
       true ->
            throw({bad_csum, CS})
    end.
 %%%% High PB mode %%%%
 do_pb_hl_request(#mpb_request{req_id=ReqID}, #state{pb_mode=low}=S) ->
    Result = {low_error, 41, "High protocol request while in low mode"},
    {machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
 do_pb_hl_request(PB_request, S) ->
    {ReqID, Cmd} = machi_pb_translate:from_pb_request(PB_request),
    {Result, S2} = do_pb_hl_request2(Cmd, S),
    {machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
 do_pb_hl_request2({high_echo, Msg}, S) ->
    {Msg, S};
 do_pb_hl_request2({high_auth, _User, _Pass}, S) ->
    {-77, S};
 do_pb_hl_request2({high_append_chunk=Op, NS, Prefix, Chunk, TaggedCSum, Opts},
                  #state{high_clnt=Clnt}=S) ->
    NSInfo = #ns_info{name=NS},                 % TODO populate other fields
    todo_perhaps_remind_ns_locator_not_chosen(Op),
    Res = machi_cr_client:append_chunk(Clnt, NSInfo,
                                       Prefix, Chunk, TaggedCSum, Opts),
    {Res, S};
 do_pb_hl_request2({high_write_chunk=Op, File, Offset, Chunk, CSum},
                  #state{high_clnt=Clnt}=S) ->
    NSInfo = undefined,
    todo_perhaps_remind_ns_locator_not_chosen(Op),
    Res = machi_cr_client:write_chunk(Clnt, NSInfo, File, Offset, Chunk, CSum),
    {Res, S};
 do_pb_hl_request2({high_read_chunk=Op, File, Offset, Size, Opts},
                  #state{high_clnt=Clnt}=S) ->
    NSInfo = undefined,
    todo_perhaps_remind_ns_locator_not_chosen(Op),
    Res = machi_cr_client:read_chunk(Clnt, NSInfo, File, Offset, Size, Opts),
    {Res, S};
 do_pb_hl_request2({high_trim_chunk=Op, File, Offset, Size},
                  #state{high_clnt=Clnt}=S) ->
    NSInfo = undefined,
    todo_perhaps_remind_ns_locator_not_chosen(Op),
    Res = machi_cr_client:trim_chunk(Clnt, NSInfo, File, Offset, Size),
    {Res, S};
 do_pb_hl_request2({high_checksum_list, File}, #state{high_clnt=Clnt}=S) ->
    Res = machi_cr_client:checksum_list(Clnt, File),
    {Res, S};
 do_pb_hl_request2({high_list_files}, #state{high_clnt=Clnt}=S) ->
    Res = machi_cr_client:list_files(Clnt),
    {Res, S}.
 make_high_clnt(#state{high_clnt=undefined}=S) ->
    {ok, Proj} = machi_projection_store:read_latest_projection(
                   S#state.proj_store, private),
    Ps = [P_srvr || {_, P_srvr} <- orddict:to_list(
                                     Proj#projection_v1.members_dict)],
    {ok, Clnt} = machi_cr_client:start_link(Ps),
    S#state{high_clnt=Clnt};
 make_high_clnt(S) ->
    S.
 todo_perhaps_remind_ns_locator_not_chosen(Op) ->
    Key = {?MODULE, Op},
    case get(Key) of
        undefined ->
            io:format(user, "TODO op ~w is using default locator value\n",
                      [Op]),
            put(Key, true);
        _ ->
            ok
    end.
--- a/src/machi_flu1_subsup.erl
+++ b/src/machi_flu1_subsup.erl
@ -1,118 +0,0 @@
 %% -------------------------------------------------------------------
 %%
 %% Copyright (c) 2007-2015 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
 %% except in compliance with the License.  You may obtain
 %% a copy of the License at
 %%
 %%   http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing,
 %% software distributed under the License is distributed on an
 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 %% KIND, either express or implied.  See the License for the
 %% specific language governing permissions and limitations
 %% under the License.
 %%
 %% -------------------------------------------------------------------
 %% @doc A supervisor to hold dynamic processes inside single
 %% FLU service, ranch listener and append server.
 %% TODO: This supervisor is maybe useless. First introduced for
 %% workaround to start listener dynamically in flu1 initialization
 %% phase.  Because `machi_flu_psup' is being blocked in flu1
 %% initialization time, adding a child to the supervisor leads to
 %% deadlock.  If initialization can be done only by static arguments,
 %% then this supervisor should be removed and added as a direct child
 %% of `machi_flu_psup'.
 -module(machi_flu1_subsup).
 -behaviour(supervisor).
 %% public API
 -export([start_link/1,
         start_append_server/4,
         stop_append_server/1,
         start_listener/7,
         stop_listener/1,
         subsup_name/1,
         listener_name/1]).
 %% supervisor callback
 -export([init/1]).
 -include("machi_projection.hrl").
 -define(SHUTDOWN, 5000).
 -define(BACKLOG, 8192).
 -spec start_link(pv1_server()) -> {ok, pid()}.
 start_link(FluName) ->
    supervisor:start_link({local, subsup_name(FluName)}, ?MODULE, []).
 -spec start_append_server(pv1_server(), boolean(), boolean(),
                          undefined | machi_dt:epoch_id()) ->
                                 {ok, pid()}.
 start_append_server(FluName, Witness_p, Wedged_p, EpochId) ->
    supervisor:start_child(subsup_name(FluName),
                           append_server_spec(FluName, Witness_p, Wedged_p, EpochId)).
 -spec stop_append_server(pv1_server()) -> ok.
 stop_append_server(FluName) ->
    SubSup = listener_name(FluName),
    ok = supervisor:terminate_child(SubSup, FluName),
    ok = supervisor:delete_child(SubSup, FluName).
 -spec start_listener(pv1_server(), inet:port_number(), boolean(),
                     string(), ets:tab(), atom() | pid(),
                     proplists:proplist()) -> {ok, pid()}.
 start_listener(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore,
               Props) ->
    supervisor:start_child(subsup_name(FluName),
                           listener_spec(FluName, TcpPort, Witness, DataDir,
                                         EpochTab, ProjStore, Props)).
 -spec stop_listener(pv1_server()) -> ok.
 stop_listener(FluName) ->
    SupName = subsup_name(FluName),
    ListenerName = listener_name(FluName),
    ok = supervisor:terminate_child(SupName, ListenerName),
    ok = supervisor:delete_child(SupName, ListenerName).
 -spec subsup_name(pv1_server()) -> atom().
 subsup_name(FluName) when is_atom(FluName) ->
    list_to_atom(atom_to_list(FluName) ++ "_flu1_subsup").
 -spec listener_name(pv1_server()) -> atom().
 listener_name(FluName) ->
    list_to_atom(atom_to_list(FluName) ++ "_listener").
 %% Supervisor callback
 init([]) ->
    SupFlags = {one_for_all, 1000, 10},
    {ok, {SupFlags, []}}.
 %% private
 -spec listener_spec(pv1_server(), inet:port_number(), boolean(),
                    string(), ets:tab(), atom() | pid(),
                    proplists:proplist()) -> supervisor:child_spec().
 listener_spec(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore, Props) ->
    ListenerName = listener_name(FluName),
    NbAcceptors = 10,
    TcpOpts = [{port, TcpPort}, {backlog, ?BACKLOG}],
    NetServerOpts = [FluName, Witness, DataDir, EpochTab, ProjStore, Props],
    ranch:child_spec(ListenerName, NbAcceptors,
                     ranch_tcp, TcpOpts,
                     machi_flu1_net_server, NetServerOpts).
 -spec append_server_spec(pv1_server(), boolean(), boolean(),
                         undefined | machi_dt:epoch_id()) -> supervisor:child_spec().
 append_server_spec(FluName, Witness_p, Wedged_p, EpochId) ->
    {FluName, {machi_flu1_append_server, start_link,
               [FluName, Witness_p, Wedged_p, EpochId]},
     permanent, ?SHUTDOWN, worker, [machi_flu1_append_server]}.
--- a/src/machi_flu_filename_mgr.erl
+++ b/src/machi_flu_filename_mgr.erl
@ -51,8 +51,8 @@
 -export([
    child_spec/2,
    start_link/2,
-    find_or_make_filename_from_prefix/4,
+    find_or_make_filename_from_prefix/3,
-    increment_prefix_sequence/3,
+    increment_prefix_sequence/2,
    list_files_by_prefix/2
    ]).
@ -67,13 +67,12 @@
    ]).
 -define(TIMEOUT, 10 * 1000).
-include("machi.hrl").            %% included for #ns_info record
+-include("machi_projection.hrl"). %% included for pv1_epoch_n type
 -include("machi_projection.hrl"). %% included for pv1_epoch type
 -record(state, {fluname :: atom(),
                tid     :: ets:tid(),
                datadir :: string(),
-                epoch   :: pv1_epoch()
+                epoch   :: pv1_epoch_n()
               }).
 %% public API
@ -89,30 +88,26 @@ start_link(FluName, DataDir) when is_atom(FluName) andalso is_list(DataDir) ->
    gen_server:start_link({local, N}, ?MODULE, [FluName, DataDir], []).
 -spec find_or_make_filename_from_prefix( FluName :: atom(),
-                                         EpochId :: pv1_epoch(),
+                                         EpochId :: pv1_epoch_n(),
-                                         Prefix :: {prefix, string()},
+                                         Prefix :: {prefix, string()} ) ->
                                         machi_dt:ns_info()) ->
        {file, Filename :: string()} | {error, Reason :: term() } | timeout.
 % @doc Find the latest available or make a filename from a prefix. A prefix
 % should be in the form of a tagged tuple `{prefix, P}'. Returns a tagged
 % tuple in the form of `{file, F}' or an `{error, Reason}'
-find_or_make_filename_from_prefix(FluName, EpochId,
+find_or_make_filename_from_prefix(FluName, EpochId, {prefix, Prefix}) when is_atom(FluName) ->
                                  {prefix, Prefix},
                                  #ns_info{}=NSInfo)
  when is_atom(FluName) ->
    N = make_filename_mgr_name(FluName),
-    gen_server:call(N, {find_filename, FluName, EpochId, NSInfo, Prefix}, ?TIMEOUT);
+    gen_server:call(N, {find_filename, EpochId, Prefix}, ?TIMEOUT);
-find_or_make_filename_from_prefix(_FluName, _EpochId, Other, Other2) ->
+find_or_make_filename_from_prefix(_FluName, _EpochId, Other) ->
-    lager:error("~p is not a valid prefix/locator ~p", [Other, Other2]),
+    lager:error("~p is not a valid prefix.", [Other]),
    error(badarg).
-spec increment_prefix_sequence( FluName :: atom(), NSInfo :: machi_dt:ns_info(), Prefix :: {prefix, string()} ) ->
+-spec increment_prefix_sequence( FluName :: atom(), Prefix :: {prefix, string()} ) ->
        ok | {error, Reason :: term() } | timeout.
 % @doc Increment the sequence counter for a given prefix. Prefix should
 % be in the form of `{prefix, P}'.
-increment_prefix_sequence(FluName, #ns_info{}=NSInfo, {prefix, Prefix}) when is_atom(FluName) ->
+increment_prefix_sequence(FluName, {prefix, Prefix}) when is_atom(FluName) ->
-    gen_server:call(make_filename_mgr_name(FluName), {increment_sequence, NSInfo, Prefix}, ?TIMEOUT);
+    gen_server:call(make_filename_mgr_name(FluName), {increment_sequence, Prefix}, ?TIMEOUT);
-increment_prefix_sequence(_FluName, _NSInfo, Other) ->
+increment_prefix_sequence(_FluName, Other) ->
    lager:error("~p is not a valid prefix.", [Other]),
    error(badarg).
@ -131,7 +126,7 @@ list_files_by_prefix(_FluName, Other) ->
 init([FluName, DataDir]) ->
    Tid = ets:new(make_filename_mgr_name(FluName), [named_table, {read_concurrency, true}]),
    {ok, #state{fluname = FluName,
-                epoch = ?DUMMY_PV1_EPOCH,
+                epoch = 0,
                datadir = DataDir,
                tid = Tid}}.
@ -143,23 +138,23 @@ handle_cast(Req, State) ->
 %% the FLU has already validated that the caller's epoch id and the FLU's epoch id
 %% are the same. So we *assume* that remains the case here - that is to say, we
 %% are not wedged.
-handle_call({find_filename, FluName, EpochId, NSInfo, Prefix}, _From,
+handle_call({find_filename, EpochId, Prefix}, _From, S = #state{ datadir = DataDir,
-            S = #state{ datadir = DataDir, epoch = EpochId, tid = Tid }) ->
+                                                                 epoch = EpochId,
                                                                 tid = Tid}) ->
    %% Our state and the caller's epoch ids are the same. Business as usual.
-    File = handle_find_file(FluName, Tid, NSInfo, Prefix, DataDir),
+    File = handle_find_file(Tid, Prefix, DataDir),
    {reply, {file, File}, S};
-handle_call({find_filename, _FluName, EpochId, NSInfo, Prefix}, _From, S = #state{ datadir = DataDir, tid = Tid }) ->
+handle_call({find_filename, EpochId, Prefix}, _From, S = #state{ datadir = DataDir, tid = Tid }) ->
    %% If the epoch id in our state and the caller's epoch id were the same, it would've
    %% matched the above clause. Since we're here, we know that they are different.
    %% If epoch ids between our state and the caller's are different, we must increment the
    %% sequence number, generate a filename and then cache it.
-    File = increment_and_cache_filename(Tid, DataDir, NSInfo, Prefix),
+    File = increment_and_cache_filename(Tid, DataDir, Prefix),
    {reply, {file, File}, S#state{epoch = EpochId}};
-handle_call({increment_sequence, #ns_info{name=NS, locator=NSLocator}, Prefix}, _From, S = #state{ datadir = DataDir, tid=Tid }) ->
+handle_call({increment_sequence, Prefix}, _From, S = #state{ datadir = DataDir }) ->
-    NSInfo = #ns_info{name=NS, locator=NSLocator},
+    ok = machi_util:increment_max_filenum(DataDir, Prefix),
    _File = increment_and_cache_filename(Tid, DataDir, NSInfo, Prefix),
    {reply, ok, S};
 handle_call({list_files, Prefix}, From, S = #state{ datadir = DataDir }) ->
    spawn(fun() ->
@ -192,38 +187,61 @@ generate_uuid_v4_str() ->
    io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b",
                        [A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]).
 find_file(DataDir, Prefix, N) ->
    {_Filename, Path} = machi_util:make_data_filename(DataDir, Prefix, "*", N),
    filelib:wildcard(Path).
 list_files(DataDir, Prefix) ->
-    {F_bin, Path} = machi_util:make_data_filename(DataDir, "*^" ++ Prefix ++ "^*"),
+    {F_bin, Path} = machi_util:make_data_filename(DataDir, Prefix, "*", "*"),
    filelib:wildcard(binary_to_list(F_bin), filename:dirname(Path)).
 make_filename_mgr_name(FluName) when is_atom(FluName) ->
    list_to_atom(atom_to_list(FluName) ++ "_filename_mgr").
-handle_find_file(_FluName, Tid, #ns_info{name=NS, locator=NSLocator}, Prefix, DataDir) ->
+handle_find_file(Tid, Prefix, DataDir) ->
-    case ets:lookup(Tid, {NS, NSLocator, Prefix}) of
+    N = machi_util:read_max_filenum(DataDir, Prefix),
    {File, Cleanup} = case find_file(DataDir, Prefix, N) of
        [] ->
-            N = machi_util:read_max_filenum(DataDir, NS, NSLocator, Prefix),
+            {find_or_make_filename(Tid, DataDir, Prefix, N), false};
-            F = generate_filename(DataDir, NS, NSLocator, Prefix, N),
+        [H] -> {H, true};
-            true = ets:insert(Tid, {{NS, NSLocator, Prefix}, F}),
+        [Fn | _ ] = L ->
            lager:warning(
              "Searching for a matching file to prefix ~p and sequence number ~p gave multiples: ~p",
              [Prefix, N, L]),
            {Fn, true}
    end,
    maybe_cleanup(Tid, {Prefix, N}, Cleanup),
    filename:basename(File).
 find_or_make_filename(Tid, DataDir, Prefix, N) ->
    case ets:lookup(Tid, {Prefix, N}) of
         [] ->
            F = generate_filename(DataDir, Prefix, N),
            true = ets:insert_new(Tid, {{Prefix, N}, F}),
            F;
        [{_Key, File}] ->
            File
    end.
-generate_filename(DataDir, NS, NSLocator, Prefix, N) ->
+generate_filename(DataDir, Prefix, N) ->
-    {F, _Q} = machi_util:make_data_filename(
+    {F, _} = machi_util:make_data_filename(
              DataDir,
-              NS, NSLocator, Prefix,
+              Prefix,
              generate_uuid_v4_str(),
              N),
    binary_to_list(F).
-increment_and_cache_filename(Tid, DataDir, #ns_info{name=NS,locator=NSLocator}, Prefix) ->
+maybe_cleanup(_Tid, _Key, false) ->
-    ok = machi_util:increment_max_filenum(DataDir, NS, NSLocator, Prefix),
+    ok;
-    N = machi_util:read_max_filenum(DataDir, NS, NSLocator, Prefix),
+maybe_cleanup(Tid, Key, true) ->
-    F = generate_filename(DataDir, NS, NSLocator, Prefix, N),
+    true = ets:delete(Tid, Key).
-    true = ets:insert(Tid, {{NS, NSLocator, Prefix}, F}),
+
-    F.
+increment_and_cache_filename(Tid, DataDir, Prefix) ->
    ok = machi_util:increment_max_filenum(DataDir, Prefix),
    N = machi_util:read_max_filenum(DataDir, Prefix),
    F = generate_filename(DataDir, Prefix, N),
    true = ets:insert_new(Tid, {{Prefix, N}, F}),
    filename:basename(F).
--- a/src/machi_flu_metadata_mgr.erl
+++ b/src/machi_flu_metadata_mgr.erl
@ -34,7 +34,6 @@
 -module(machi_flu_metadata_mgr).
 -behaviour(gen_server).
 -include("machi.hrl").
 -define(MAX_MGRS, 10). %% number of managers to start by default.
 -define(HASH(X), erlang:phash2(X)). %% hash algorithm to use
@ -63,7 +62,6 @@
         lookup_proxy_pid/2,
         start_proxy_pid/2,
         stop_proxy_pid/2,
         stop_proxy_pid_rollover/2,
         build_metadata_mgr_name/2,
         trim_file/2
        ]).
@ -101,10 +99,7 @@ start_proxy_pid(FluName, {file, Filename}) ->
    gen_server:call(get_manager_atom(FluName, Filename), {start_proxy_pid, Filename}, ?TIMEOUT).
 stop_proxy_pid(FluName, {file, Filename}) ->
-    gen_server:call(get_manager_atom(FluName, Filename), {stop_proxy_pid, false, Filename}, ?TIMEOUT).
+    gen_server:call(get_manager_atom(FluName, Filename), {stop_proxy_pid, Filename}, ?TIMEOUT).
 stop_proxy_pid_rollover(FluName, {file, Filename}) ->
    gen_server:call(get_manager_atom(FluName, Filename), {stop_proxy_pid, true, Filename}, ?TIMEOUT).
 trim_file(FluName, {file, Filename}) ->
    gen_server:call(get_manager_atom(FluName, Filename), {trim_file, Filename}, ?TIMEOUT).
@ -155,7 +150,7 @@ handle_call({start_proxy_pid, Filename}, _From,
            {reply, {error, trimmed}, State}
    end;
-handle_call({stop_proxy_pid, Rollover_p, Filename}, _From, State = #state{ tid = Tid }) ->
+handle_call({stop_proxy_pid, Filename}, _From, State = #state{ tid = Tid }) ->
    case lookup_md(Tid, Filename) of
        not_found ->
            ok;
@ -163,13 +158,8 @@ handle_call({stop_proxy_pid, Rollover_p, Filename}, _From, State = #state{ tid =
            ok;
        #md{ proxy_pid = Pid, mref = M } = R ->
            demonitor(M, [flush]),
            if Rollover_p ->
                    do_rollover(Filename, State);
               true ->
            machi_file_proxy:stop(Pid),
-                    update_ets(Tid, R#md{ proxy_pid = undefined,
+            update_ets(Tid, R#md{ proxy_pid = undefined, mref = undefined })
                                          mref = undefined })
            end
    end,
    {reply, ok, State};
@ -191,6 +181,25 @@ handle_info({'DOWN', Mref, process, Pid, normal}, State = #state{ tid = Tid }) -
    clear_ets(Tid, Mref),
    {noreply, State};
 handle_info({'DOWN', Mref, process, Pid, file_rollover}, State = #state{ fluname = FluName, 
                                                                         tid = Tid }) ->
    lager:info("file proxy ~p shutdown because of file rollover", [Pid]),
    R = get_md_record_by_mref(Tid, Mref),
    [Prefix | _Rest] = machi_util:parse_filename(R#md.filename),
    %% We only increment the counter here. The filename will be generated on the 
    %% next append request to that prefix and since the filename will have a new
    %% sequence number it probably will be associated with a different metadata
    %% manager. That's why we don't want to generate a new file name immediately
    %% and use it to start a new file proxy.
    ok = machi_flu_filename_mgr:increment_prefix_sequence(FluName, {prefix, Prefix}),
    %% purge our ets table of this entry completely since it is likely the
    %% new filename (whenever it comes) will be in a different manager than
    %% us.
    purge_ets(Tid, R),
    {noreply, State};
 handle_info({'DOWN', Mref, process, Pid, wedged}, State = #state{ tid = Tid }) ->
    lager:error("file proxy ~p shutdown because it's wedged", [Pid]),
    clear_ets(Tid, Mref),
@ -263,35 +272,8 @@ get_md_record_by_mref(Tid, Mref) ->
    [R] = ets:match_object(Tid, {md, '_', '_', Mref}),
    R.
 get_md_record_by_filename(Tid, Filename) ->
    [R] = ets:lookup(Tid, Filename),
    R.
 get_env(Setting, Default) ->
    case application:get_env(machi, Setting) of
        undefined -> Default;
        {ok, V} -> V
    end.
 do_rollover(Filename, _State = #state{ fluname = FluName,
                                       tid = Tid }) ->
    R = get_md_record_by_filename(Tid, Filename),
    lager:info("file ~p proxy ~p shutdown because of file rollover",
               [Filename, R#md.proxy_pid]),
    {Prefix, NS, NSLocator, _, _} =
        machi_util:parse_filename(R#md.filename),
    %% We only increment the counter here. The filename will be generated on the
    %% next append request to that prefix and since the filename will have a new
    %% sequence number it probably will be associated with a different metadata
    %% manager. That's why we don't want to generate a new file name immediately
    %% and use it to start a new file proxy.
    NSInfo = #ns_info{name=NS, locator=NSLocator},
 lager:warning("INCR: ~p ~p\n", [FluName, Prefix]),
    ok = machi_flu_filename_mgr:increment_prefix_sequence(FluName, NSInfo, {prefix, Prefix}),
    %% purge our ets table of this entry completely since it is likely the
    %% new filename (whenever it comes) will be in a different manager than
    %% us.
    purge_ets(Tid, R),
    ok.
--- a/src/machi_flu_psup.erl
+++ b/src/machi_flu_psup.erl
@ -83,8 +83,6 @@
 %% Supervisor callbacks
 -export([init/1]).
 make_package_spec(#p_srvr{name=FluName, port=TcpPort, props=Props}) when is_list(Props) ->
    make_package_spec({FluName, TcpPort, Props});
 make_package_spec({FluName, TcpPort, Props}) when is_list(Props) ->
    FluDataDir = get_env(flu_data_dir, undefined_is_invalid),
    MyDataDir = filename:join(FluDataDir, atom_to_list(FluName)),
@ -96,7 +94,7 @@ make_package_spec(FluName, TcpPort, DataDir, Props) ->
     permanent, ?SHUTDOWN, supervisor, []}.
 start_flu_package(#p_srvr{name=FluName, port=TcpPort, props=Props}) ->
-    DataDir = get_data_dir(FluName, Props),
+    DataDir = get_data_dir(Props),
    start_flu_package(FluName, TcpPort, DataDir, Props).
 start_flu_package(FluName, TcpPort, DataDir, Props) ->
@ -145,19 +143,16 @@ init([FluName, TcpPort, DataDir, Props0]) ->
    FProxySupSpec = machi_file_proxy_sup:child_spec(FluName),
    Flu1SubSupSpec = {machi_flu1_subsup:subsup_name(FluName),
                      {machi_flu1_subsup, start_link, [FluName]},
                      permanent, ?SHUTDOWN, supervisor, []},
    FluSpec = {FluName,
               {machi_flu1, start_link,
                [ [{FluName, TcpPort, DataDir}|Props] ]},
               permanent, ?SHUTDOWN, worker, []},
    {ok, {SupFlags, [
            ProjSpec, FitnessSpec, MgrSpec, 
            FProxySupSpec, FNameMgrSpec, MetaSupSpec, 
-            Flu1SubSupSpec, FluSpec]}}.
+            FluSpec]}}.
 make_flu_regname(FluName) when is_atom(FluName) ->
    FluName.
@ -180,11 +175,8 @@ get_env(Setting, Default) ->
        {ok, V} -> V
    end.
-get_data_dir(FluName, Props) ->
+get_data_dir(Props) ->
    case proplists:get_value(data_dir, Props) of
        Path when is_list(Path) ->
-            Path;
+            Path
        undefined ->
            {ok, Dir} = application:get_env(machi, flu_data_dir),
            Dir ++ "/" ++ atom_to_list(FluName)
    end.
--- a/src/machi_flu_sup.erl
+++ b/src/machi_flu_sup.erl
@ -21,9 +21,6 @@
 %% @doc Supervisor for Machi FLU servers and their related support
 %% servers.
 %%
 %% Responsibility for managing FLU and chain lifecycle after the initial
 %% application startup is delegated to {@link machi_lifecycle_mgr}.
 %%
 %% See {@link machi_flu_psup} for an illustration of the entire Machi
 %% application process structure.
@ -32,11 +29,8 @@
 -behaviour(supervisor).
 -include("machi.hrl").
 -include("machi_projection.hrl").
 -include("machi_verbose.hrl").
 -ifdef(TEST).
 -compile(export_all).
 -ifdef(PULSE).
 -compile({parse_transform, pulse_instrument}).
 -include_lib("pulse_otp/include/pulse_otp.hrl").
@ -44,12 +38,9 @@
 -else.
 -define(SHUTDOWN, 5000).
 -endif.
 -endif. %TEST
 %% API
-export([start_link/0,
+-export([start_link/0]).
         get_initial_flus/0, load_rc_d_files_from_dir/1,
         sanitize_p_srvr_records/1]).
 %% Supervisor callbacks
 -export([init/1]).
@ -78,66 +69,5 @@ get_initial_flus() ->
    [].
 -else.  % PULSE
 get_initial_flus() ->
-    DoesNotExist = "/tmp/does/not/exist",
+    application:get_env(machi, initial_flus, []).
    ConfigDir = case application:get_env(machi, flu_config_dir, DoesNotExist) of
                    DoesNotExist ->
                        DoesNotExist;
                    Dir ->
                        Dir
                end,
    Ps = [P || {_File, P} <- load_rc_d_files_from_dir(ConfigDir)],
    sanitize_p_srvr_records(Ps).
 -endif. % PULSE
 load_rc_d_files_from_dir(Dir) ->
    Files = filelib:wildcard(Dir ++ "/*"),
    [case file:consult(File) of
         {ok, [X]} ->
             {File, X};
         _ ->
             lager:warning("Error parsing file '~s', ignoring",
                           [File]),
             {File, []}
     end || File <- Files].
 sanitize_p_srvr_records(Ps) ->
    {Sane, _} = lists:foldl(fun sanitize_p_srvr_rec/2, {[], dict:new()}, Ps),
    Sane.
 sanitize_p_srvr_rec(Whole, {Acc, D}) ->
    try
        #p_srvr{name=Name,
                proto_mod=PMod,
                address=Address,
                port=Port,
                props=Props} = Whole,
        true = is_atom(Name),
        NameK = {name, Name},
        error = dict:find(NameK, D),
        true = is_atom(PMod),
        case code:is_loaded(PMod) of
            {file, _} ->
                ok;
            _ ->
                {module, _} = code:load_file(PMod),
                ok
        end,
        if is_list(Address)  -> ok;
           is_tuple(Address) -> ok              % Erlang-style IPv4 or IPv6
        end,
        true = is_integer(Port) andalso Port >= 1024 andalso Port =< 65534,
        PortK = {port, Port},
        error = dict:find(PortK, D),
        true = is_list(Props),
        %% All is sane enough.
        D2 = dict:store(NameK, Name,
                        dict:store(PortK, Port, D)),
        {[Whole|Acc], D2}
    catch _:_ ->
            _ = lager:log(error, self(),
                          "~s: Bad (or duplicate name/port) p_srvr record, "
                          "skipping: ~P\n",
                          [?MODULE, Whole, 15]),
            {Acc, D}
    end.
--- a/src/machi_lifecycle_mgr.erl
+++ b/src/machi_lifecycle_mgr.erl
--- a/src/machi_merkle_tree.erl
+++ b/src/machi_merkle_tree.erl
@ -1,156 +0,0 @@
 %% -------------------------------------------------------------------
 %%
 %% Copyright (c) 2007-2015 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
 %% except in compliance with the License.  You may obtain
 %% a copy of the License at
 %%
 %%   http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing,
 %% software distributed under the License is distributed on an
 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 %% KIND, either express or implied.  See the License for the
 %% specific language governing permissions and limitations
 %% under the License.
 %%
 %% -------------------------------------------------------------------
 %% @doc Creates a Merkle tree per file based on the checksum data for
 %% a given data file.
 %%
 %% The `naive' implementation representation is:
 %%
 %% `<<Length:64, Offset:32, 0>>' for unwritten bytes
 %% `<<Length:64, Offset:32, 1>>' for trimmed bytes
 %% `<<Length:64, Offset:32, Csum/binary>>' for written bytes
 %%
 %% The tree feeds these leaf nodes into hashes representing chunks of a minimum
 %% size of at least 1024 KB (1 MB), but if the file size is larger, we will try
 %% to get about 100 chunks for the first rollup "Level 1." We aim for around 10
 %% hashes at level 2, and then 2 hashes level 3 and finally the root.
 -module(machi_merkle_tree).
 -include("machi.hrl").
 -include("machi_merkle_tree.hrl").
 -ifdef(TEST).
 -compile(export_all).
 -else.
 -export([
    open/2,
    open/3,
    tree/1,
    filename/1,
    diff/2
 ]).
 -endif.
 -define(TRIMMED, <<1>>).
 -define(UNWRITTEN, <<0>>).
 -define(NAIVE_ENCODE(Offset, Size, Data), <<Offset:64/unsigned-big, Size:32/unsigned-big, Data/binary>>).
 -define(MINIMUM_CHUNK, 1048576). %% 1024 * 1024
 -define(LEVEL_SIZE, 10).
 -define(H, sha).
 %% public API
 open(Filename, DataDir) ->
    open(Filename, DataDir, naive).
 open(Filename, DataDir, Type) ->
    Tree = load_filename(Filename, DataDir, Type),
    {ok, #mt{ filename = Filename, tree = Tree, backend = Type}}.
 tree(#mt{ tree = T, backend = naive }) ->
    case T#naive.recalc of
         true -> build_tree(T);
        false -> T
    end.
 filename(#mt{ filename = F }) -> F.
 diff(#mt{backend = naive, tree = T1}, #mt{backend = naive, tree = T2}) ->
    case T1#naive.root == T2#naive.root of
        true -> same;
        false -> naive_diff(T1, T2) 
    end;
 diff(_, _) -> error(badarg).
 %% private
 % @private
 load_filename(Filename, DataDir, naive) ->
    {Last, M} = do_load(Filename, DataDir, fun insert_csum_naive/2, []),
    ChunkSize = max(?MINIMUM_CHUNK, Last div 100),
    T = #naive{ leaves = lists:reverse(M), chunk_size = ChunkSize, recalc = true },
    build_tree(T).
 do_load(Filename, DataDir, FoldFun, AccInit) ->
    CsumFile = machi_util:make_checksum_filename(DataDir, Filename),
    {ok, T} = machi_csum_table:open(CsumFile, []),
    Acc = machi_csum_table:foldl_chunks(FoldFun, {0, AccInit}, T),
    ok = machi_csum_table:close(T),
    Acc.
 % @private
 insert_csum_naive({Last, Size, _Csum}=In, {Last, MT}) ->
    %% no gap
    {Last+Size, update_acc(In, MT)};
 insert_csum_naive({Offset, Size, _Csum}=In, {Last, MT}) ->
    Hole = Offset - Last,
    MT0 = update_acc({Last, Hole, unwritten}, MT),
    {Offset+Size, update_acc(In, MT0)}.
 % @private
 update_acc({Offset, Size, unwritten}, MT) ->
    [ {Offset, Size, ?NAIVE_ENCODE(Offset, Size, ?UNWRITTEN)} | MT ];
 update_acc({Offset, Size, trimmed}, MT) ->
    [ {Offset, Size, ?NAIVE_ENCODE(Offset, Size, ?TRIMMED)} | MT ];
 update_acc({Offset, Size, <<_Tag:8, Csum/binary>>}, MT) ->
    [ {Offset, Size, ?NAIVE_ENCODE(Offset, Size, Csum)} | MT ].
 build_tree(MT = #naive{ leaves = L, chunk_size = ChunkSize }) ->
    Lvl1s = build_level_1(ChunkSize, L, 1, [ crypto:hash_init(?H) ]),
    Mod2 = length(Lvl1s) div ?LEVEL_SIZE,
    Lvl2s = build_int_level(Mod2, Lvl1s, 1, [ crypto:hash_init(?H) ]),
    Mod3 = length(Lvl2s) div 2,
    Lvl3s = build_int_level(Mod3, Lvl2s, 1, [ crypto:hash_init(?H) ]),
    Root = build_root(Lvl3s, crypto:hash_init(?H)),
    MT#naive{ root = Root, lvl1 = Lvl1s, lvl2 = Lvl2s, lvl3 = Lvl3s, recalc = false }.
 build_root([], Ctx) ->
    crypto:hash_final(Ctx);
 build_root([H|T], Ctx) ->
    build_root(T, crypto:hash_update(Ctx, H)).
 build_int_level(_Mod, [], _Cnt, [ Ctx | Rest ]) ->
    lists:reverse( [ crypto:hash_final(Ctx) | Rest ] );
 build_int_level(Mod, [H|T], Cnt, [ Ctx | Rest ]) when Cnt rem Mod == 0 ->
    NewCtx = crypto:hash_init(?H),
    build_int_level(Mod, T, Cnt + 1, [ crypto:hash_update(NewCtx, H), crypto:hash_final(Ctx) | Rest ]);
 build_int_level(Mod, [H|T], Cnt, [ Ctx | Rest ]) ->
    build_int_level(Mod, T, Cnt+1, [ crypto:hash_update(Ctx, H) | Rest ]).
 build_level_1(_Size, [], _Multiple, [ Ctx | Rest ]) ->
    lists:reverse([ crypto:hash_final(Ctx) | Rest ]);
 build_level_1(Size, [{Pos, Len, Hash}|T], Multiple, [ Ctx | Rest ])
                                    when ( Pos + Len ) > ( Size * Multiple ) ->
    NewCtx = crypto:hash_init(?H),
    build_level_1(Size, T, Multiple+1,
                  [ crypto:hash_update(NewCtx, Hash), crypto:hash_final(Ctx) | Rest ]);
 build_level_1(Size, [{Pos, Len, Hash}|T], Multiple, [ Ctx | Rest ])
                                    when ( Pos + Len ) =< ( Size * Multiple ) ->
    build_level_1(Size, T, Multiple, [ crypto:hash_update(Ctx, Hash) | Rest ]).
 naive_diff(#naive{lvl1 = L1}, #naive{lvl1=L2, chunk_size=CS2}) ->
    Set1 = gb_sets:from_list(lists:zip(lists:seq(1, length(L1)), L1)),
    Set2 = gb_sets:from_list(lists:zip(lists:seq(1, length(L2)), L2)),
    %% The byte ranges in list 2 that do not match in list 1
    %% Or should we do something else?
    [ {(X-1)*CS2, CS2, SHA} || {X, SHA} <- gb_sets:to_list(gb_sets:subtract(Set1, Set2)) ].
--- a/src/machi_pb_high_client.erl
+++ b/src/machi_pb_high_client.erl
@ -25,10 +25,6 @@
 %% to a single socket connection, and there is no code to deal with
 %% multiple connections/load balancing/error handling to several/all
 %% Machi cluster servers.
 %%
 %% Please see {@link machi_flu1_client} the "Client API implemntation notes"
 %% section for how this module relates to the rest of the client API
 %% implementation.
 -module(machi_pb_high_client).
@ -62,114 +58,68 @@
          count=0     :: non_neg_integer()
         }).
 %% Official error types that is specific in Machi
 -type machi_client_error_reason() :: bad_arg | wedged | bad_checksum |
                                     partition | not_written | written |
                                     trimmed | no_such_file | partial_read |
                                     bad_epoch | inet:posix().
 %% @doc Creates a client process
 -spec start_link(p_srvr_dict()) -> {ok, pid()} | {error, machi_client_error_reason()}.
 start_link(P_srvr_list) ->
    gen_server:start_link(?MODULE, [P_srvr_list], []).
 %% @doc Stops a client process.
 -spec quit(pid()) -> ok.
 quit(PidSpec) ->
    gen_server:call(PidSpec, quit, infinity).
 connected_p(PidSpec) ->
    gen_server:call(PidSpec, connected_p, infinity).
 -spec echo(pid(), string()) -> {ok, string()} | {error, machi_client_error_reason()}.
 echo(PidSpec, String) ->
    echo(PidSpec, String, ?DEFAULT_TIMEOUT).
 -spec echo(pid(), string(), non_neg_integer()) -> {ok, string()} | {error, machi_client_error_reason()}.
 echo(PidSpec, String, Timeout) ->
    send_sync(PidSpec, {echo, String}, Timeout).
 %% TODO: auth() is not implemented.  Auth requires SSL, and this client
 %% doesn't support SSL yet.  This is just a placeholder and reminder.
 -spec auth(pid(), string(), string()) -> ok | {error, machi_client_error_reason()}.
 auth(PidSpec, User, Pass) ->
    auth(PidSpec, User, Pass, ?DEFAULT_TIMEOUT).
 -spec auth(pid(), string(), string(), non_neg_integer()) -> ok | {error, machi_client_error_reason()}.
 auth(PidSpec, User, Pass, Timeout) ->
    send_sync(PidSpec, {auth, User, Pass}, Timeout).
-spec append_chunk(pid(),
+append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra) ->
-                   NS::machi_dt:namespace(), Prefix::machi_dt:file_prefix(),
+    append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra, ?DEFAULT_TIMEOUT).
                   Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum(),
                   Opts::machi_dt:append_opts()) ->
                          {ok, Filename::string(), Offset::machi_dt:file_offset()} |
                          {error, machi_client_error_reason()}.
 append_chunk(PidSpec, NS, Prefix, Chunk, CSum, Opts) ->
    append_chunk(PidSpec, NS, Prefix, Chunk, CSum, Opts, ?DEFAULT_TIMEOUT).
-spec append_chunk(pid(),
+append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra, Timeout) ->
-                   NS::machi_dt:namespace(), Prefix::machi_dt:file_prefix(),
+    send_sync(PidSpec, {append_chunk, PlacementKey, Prefix, Chunk, CSum, ChunkExtra}, Timeout).
                   Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum(),
                   Opts::machi_dt:append_opts(),
                   Timeout::non_neg_integer()) ->
                          {ok, Filename::string(), Offset::machi_dt:file_offset()} |
                          {error, machi_client_error_reason()}.
 append_chunk(PidSpec, NS, Prefix, Chunk, CSum, Opts, Timeout) ->
    send_sync(PidSpec, {append_chunk, NS, Prefix, Chunk, CSum, Opts}, Timeout).
 -spec write_chunk(pid(), File::string(), machi_dt:file_offset(),
                  Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum()) ->
                         ok | {error, machi_client_error_reason()}.
 write_chunk(PidSpec, File, Offset, Chunk, CSum) ->
    write_chunk(PidSpec, File, Offset, Chunk, CSum, ?DEFAULT_TIMEOUT).
 -spec write_chunk(pid(), File::string(), machi_dt:file_offset(),
                  Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum(), Timeout::non_neg_integer()) ->
                         ok | {error, machi_client_error_reason()}.
 write_chunk(PidSpec, File, Offset, Chunk, CSum, Timeout) ->
    send_sync(PidSpec, {write_chunk, File, Offset, Chunk, CSum}, Timeout).
 %% @doc Tries to read a chunk of a specified file. It returns `{ok,
 %% {Chunks, TrimmedChunks}}' for live file while it returns `{error,
 %% trimmed}' if all bytes of the file was trimmed.
-spec read_chunk(pid(), File::string(), machi_dt:file_offset(), machi_dt:chunk_size(),
+-spec read_chunk(pid(), string(), pos_integer(), pos_integer(),
-                 machi_dt:read_opts_x()) ->
+                 [{flag_no_checksum | flag_no_chunk | needs_trimmed, boolean()}]) ->
-                        {ok, {Chunks::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size(), binary()}],
+                        {ok, {list(), list()}} | {error, term()}.
-                              Trimmed::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size()}]}} |
+read_chunk(PidSpec, File, Offset, Size, Options) ->
-                        {error, machi_client_error_reason()}.
+    read_chunk(PidSpec, File, Offset, Size, Options, ?DEFAULT_TIMEOUT).
 read_chunk(PidSpec, File, Offset, Size, Opts) ->
    read_chunk(PidSpec, File, Offset, Size, Opts, ?DEFAULT_TIMEOUT).
-spec read_chunk(pid(), File::string(), machi_dt:file_offset(), machi_dt:chunk_size(),
+-spec read_chunk(pid(), string(), pos_integer(), pos_integer(),
-                 machi_dt:read_opts_x(),
+                 [{no_checksum | no_chunk | needs_trimmed, boolean()}],
-                 Timeout::non_neg_integer()) ->
+                 pos_integer()) ->
-                        {ok, {Chunks::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size(), binary()}],
+                        {ok, {list(), list()}} | {error, term()}.
-                              Trimmed::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size()}]}} |
+read_chunk(PidSpec, File, Offset, Size, Options, Timeout) ->
-                        {error, machi_client_error_reason()}.
+    send_sync(PidSpec, {read_chunk, File, Offset, Size, Options}, Timeout).
 read_chunk(PidSpec, File, Offset, Size, Opts0, Timeout) ->
    Opts = machi_util:read_opts_default(Opts0),
    send_sync(PidSpec, {read_chunk, File, Offset, Size, Opts}, Timeout).
-%% @doc Trims arbitrary binary range of any file. If a specified range
+%% @doc Trims arbitrary binary range of any file. TODO: Add option
-%% has any byte trimmed, it fails and returns `{error, trimmed}'.
+%% specifying whether to trigger GC.
 %% Otherwise it trims all bytes in that range. If there are
 %% overlapping chunks with client-specified checksum, they will cut
 %% off and checksum are re-calculated in server side.  TODO: Add
 %% option specifying whether to trigger GC.
 -spec trim_chunk(pid(), string(), non_neg_integer(), machi_dt:chunk_size()) ->
-                        ok | {error, machi_client_error_reason()}.
+                        ok | {error, term()}.
 trim_chunk(PidSpec, File, Offset, Size) ->
    trim_chunk(PidSpec, File, Offset, Size, ?DEFAULT_TIMEOUT).
 trim_chunk(PidSpec, File, Offset, Size, Timeout) ->
    send_sync(PidSpec, {trim_chunk, File, Offset, Size}, Timeout).
 %% @doc Returns a binary that has checksums and chunks encoded inside
 %% (This is because encoding-decoding them are inefficient). TODO:
 %% return a structured list of them.
 -spec checksum_list(pid(), string()) -> {ok, binary()} | {error, machi_client_error_reason()}.
 checksum_list(PidSpec, File) ->
    checksum_list(PidSpec, File, ?DEFAULT_TIMEOUT).
@ -289,19 +239,19 @@ do_send_sync2({auth, User, Pass}, #state{sock=Sock}=S) ->
            Res = {bummer, {X, Y, erlang:get_stacktrace()}},
            {Res, S}
    end;
-do_send_sync2({append_chunk, NS, Prefix, Chunk, CSum, Opts},
+do_send_sync2({append_chunk, PlacementKey, Prefix, Chunk, CSum, ChunkExtra},
             #state{sock=Sock, sock_id=Index, count=Count}=S) ->
    try
        ReqID = <<Index:64/big, Count:64/big>>,
        PK = if PlacementKey == <<>> -> undefined;
                true                 -> PlacementKey
             end,
        CSumT = convert_csum_req(CSum, Chunk),
-        {ChunkExtra, Pref, FailPref} = machi_pb_translate:conv_from_append_opts(Opts),
+        Req = #mpb_appendchunkreq{placement_key=PK,
        Req = #mpb_appendchunkreq{namespace=NS,
                                  prefix=Prefix,
                                  chunk=Chunk,
                                  csum=CSumT,
-                                  chunk_extra=ChunkExtra,
+                                  chunk_extra=ChunkExtra},
                                  preferred_file_name=Pref,
                                  flag_fail_preferred=FailPref},
        R1a = #mpb_request{req_id=ReqID, do_not_alter=1,
                           append_chunk=Req},
        Bin1a = machi_pb:encode_mpb_request(R1a),
@ -346,13 +296,13 @@ do_send_sync2({write_chunk, File, Offset, Chunk, CSum},
            Res = {bummer, {X, Y, erlang:get_stacktrace()}},
            {Res, S#state{count=Count+1}}
    end;
-do_send_sync2({read_chunk, File, Offset, Size, Opts},
+do_send_sync2({read_chunk, File, Offset, Size, Options},
             #state{sock=Sock, sock_id=Index, count=Count}=S) ->
    try
        ReqID = <<Index:64/big, Count:64/big>>,
-        #read_opts{no_checksum=FlagNoChecksum,
+        FlagNoChecksum = proplists:get_value(no_checksum, Options, false),
-                   no_chunk=FlagNoChunk,
+        FlagNoChunk = proplists:get_value(no_chunk, Options, false),
-                   needs_trimmed=NeedsTrimmed} = Opts,
+        NeedsTrimmed = proplists:get_value(needs_trimmed, Options, false),
        Req = #mpb_readchunkreq{chunk_pos=#mpb_chunkpos{file_name=File,
                                                        offset=Offset,
                                                        chunk_size=Size},
@ -445,15 +395,9 @@ do_send_sync2({list_files},
            {Res, S#state{count=Count+1}}
    end.
 %% We only convert the checksum types that make sense here:
 %% none or client_sha.  None of the other types should be sent
 %% to us via the PB high protocol.
 convert_csum_req(none, Chunk) ->
    #mpb_chunkcsum{type='CSUM_TAG_CLIENT_SHA',
                   csum=machi_util:checksum_chunk(Chunk)};
 convert_csum_req(<<>>, Chunk) ->
    convert_csum_req(none, Chunk);
 convert_csum_req({client_sha, CSumBin}, _Chunk) ->
    #mpb_chunkcsum{type='CSUM_TAG_CLIENT_SHA',
                   csum=CSumBin}.
@ -501,12 +445,12 @@ convert_read_chunk_resp(#mpb_readchunkresp{status='OK', chunks=PB_Chunks, trimme
                                      csum=#mpb_chunkcsum{type=T, csum=Ck}}) ->
                               %% TODO: cleanup export
                               Csum = <<(machi_pb_translate:conv_to_csum_tag(T)):8, Ck/binary>>,
-                               {list_to_binary(File), Offset, Chunk, Csum}
+                               {File, Offset, Chunk, Csum}
                       end, PB_Chunks),
    Trimmed = lists:map(fun(#mpb_chunkpos{file_name=File,
                                          offset=Offset,
                                          chunk_size=Size}) ->
-                                {list_to_binary(File), Offset, Size}
+                                {File, Offset, Size}
                        end, PB_Trimmed),
    {ok, {Chunks, Trimmed}};
 convert_read_chunk_resp(#mpb_readchunkresp{status=Status}) ->
--- a/src/machi_pb_translate.erl
+++ b/src/machi_pb_translate.erl
@ -34,9 +34,7 @@
 -export([from_pb_request/1,
         from_pb_response/1,
         to_pb_request/2,
-         to_pb_response/3,
+         to_pb_response/3
         conv_from_append_opts/1,
         conv_to_append_opts/1
        ]).
 %% TODO: fixme cleanup
@ -45,104 +43,93 @@
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
                   echo=#mpb_echoreq{message=Msg}}) ->
-    {ReqID, {low_skip_wedge, {low_echo, Msg}}};
+    {ReqID, {low_echo, undefined, Msg}};
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
                   auth=#mpb_authreq{user=User, password=Pass}}) ->
-    {ReqID, {low_skip_wedge, {low_auth, User, Pass}}};
+    {ReqID, {low_auth, undefined, User, Pass}};
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
-                   append_chunk=IR=#mpb_ll_appendchunkreq{
+                   append_chunk=#mpb_ll_appendchunkreq{
                     namespace_version=NSVersion,
                     namespace=NS_str,
                     locator=NSLocator,
                     epoch_id=PB_EpochID,
                     placement_key=PKey,
                     prefix=Prefix,
                     chunk=Chunk,
-                     csum=#mpb_chunkcsum{type=CSum_type, csum=CSum}}}) ->
+                     csum=#mpb_chunkcsum{type=CSum_type, csum=CSum},
-    NS = list_to_binary(NS_str),
+                     chunk_extra=ChunkExtra}}) ->
    EpochID = conv_to_epoch_id(PB_EpochID),
    CSum_tag = conv_to_csum_tag(CSum_type),
-    Opts = conv_to_append_opts(IR),
+    {ReqID, {low_append_chunk, EpochID, PKey, Prefix, Chunk, CSum_tag, CSum,
-    %% NOTE: The tuple position of NSLocator is a bit odd, because EpochID
+             ChunkExtra}};
    %%       _must_ be in the 4th position (as NSV & NS must be in 2nd & 3rd).
    {ReqID, {low_append_chunk, NSVersion, NS, EpochID, NSLocator,
             Prefix, Chunk, CSum_tag, CSum, Opts}};
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
                   write_chunk=#mpb_ll_writechunkreq{
                     namespace_version=NSVersion,
                     namespace=NS_str,
                     epoch_id=PB_EpochID,
                     chunk=#mpb_chunk{file_name=File,
                                      offset=Offset,
                                      chunk=Chunk,
                                      csum=#mpb_chunkcsum{type=CSum_type, csum=CSum}}}}) ->
    NS = list_to_binary(NS_str),
    EpochID = conv_to_epoch_id(PB_EpochID),
    CSum_tag = conv_to_csum_tag(CSum_type),
-    {ReqID, {low_write_chunk, NSVersion, NS, EpochID, File, Offset, Chunk, CSum_tag, CSum}};
+    {ReqID, {low_write_chunk, EpochID, File, Offset, Chunk, CSum_tag, CSum}};
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
                   read_chunk=#mpb_ll_readchunkreq{
                                 namespace_version=NSVersion,
                                 namespace=NS_str,
                                 epoch_id=PB_EpochID,
                                 chunk_pos=ChunkPos,
                                 flag_no_checksum=PB_GetNoChecksum,
                                 flag_no_chunk=PB_GetNoChunk,
                                 flag_needs_trimmed=PB_NeedsTrimmed}}) ->
    NS = list_to_binary(NS_str),
    EpochID = conv_to_epoch_id(PB_EpochID),
-    Opts = #read_opts{no_checksum=PB_GetNoChecksum,
+    Opts = [{no_checksum, conv_to_boolean(PB_GetNoChecksum)},
-                      no_chunk=PB_GetNoChunk,
+            {no_chunk, conv_to_boolean(PB_GetNoChunk)},
-                      needs_trimmed=PB_NeedsTrimmed},
+            {needs_trimmed, conv_to_boolean(PB_NeedsTrimmed)}],
    #mpb_chunkpos{file_name=File,
                  offset=Offset,
                  chunk_size=Size} = ChunkPos,
-    {ReqID, {low_read_chunk, NSVersion, NS, EpochID, File, Offset, Size, Opts}};
+    {ReqID, {low_read_chunk, EpochID, File, Offset, Size, Opts}};
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
                   trim_chunk=#mpb_ll_trimchunkreq{
                     namespace_version=NSVersion,
                     namespace=NS_str,
                     epoch_id=PB_EpochID,
                     file=File,
                     offset=Offset,
                     size=Size,
-                     trigger_gc=TriggerGC}}) ->
+                                 trigger_gc=PB_TriggerGC}}) ->
    NS = list_to_binary(NS_str),
    EpochID = conv_to_epoch_id(PB_EpochID),
-    {ReqID, {low_trim_chunk, NSVersion, NS, EpochID, File, Offset, Size, TriggerGC}};
+    TriggerGC = conv_to_boolean(PB_TriggerGC),
    {ReqID, {low_trim_chunk, EpochID, File, Offset, Size, TriggerGC}};
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
                   checksum_list=#mpb_ll_checksumlistreq{
                     epoch_id=PB_EpochID,
                     file=File}}) ->
-    {ReqID, {low_skip_wedge, {low_checksum_list, File}}};
+    EpochID = conv_to_epoch_id(PB_EpochID),
    {ReqID, {low_checksum_list, EpochID, File}};
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
                   list_files=#mpb_ll_listfilesreq{
                     epoch_id=PB_EpochID}}) ->
    EpochID = conv_to_epoch_id(PB_EpochID),
-    {ReqID, {low_skip_wedge, {low_list_files, EpochID}}};
+    {ReqID, {low_list_files, EpochID}};
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
                   wedge_status=#mpb_ll_wedgestatusreq{}}) ->
-    {ReqID, {low_skip_wedge, {low_wedge_status}}};
+    {ReqID, {low_wedge_status, undefined}};
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
                   delete_migration=#mpb_ll_deletemigrationreq{
                    epoch_id=PB_EpochID,
                    file=File}}) ->
    EpochID = conv_to_epoch_id(PB_EpochID),
-    {ReqID, {low_skip_wedge, {low_delete_migration, EpochID, File}}};
+    {ReqID, {low_delete_migration, EpochID, File}};
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
                   trunc_hack=#mpb_ll_trunchackreq{
                    epoch_id=PB_EpochID,
                    file=File}}) ->
    EpochID = conv_to_epoch_id(PB_EpochID),
-    {ReqID, {low_skip_wedge, {low_trunc_hack, EpochID, File}}};
+    {ReqID, {low_trunc_hack, EpochID, File}};
 from_pb_request(#mpb_ll_request{
                   req_id=ReqID,
                   proj_gl=#mpb_ll_getlatestepochidreq{type=ProjType}}) ->
@ -183,22 +170,22 @@ from_pb_request(#mpb_request{req_id=ReqID,
    {ReqID, {high_auth, User, Pass}};
 from_pb_request(#mpb_request{req_id=ReqID,
                             append_chunk=IR=#mpb_appendchunkreq{}}) ->
-    #mpb_appendchunkreq{namespace=NS_str,
+    #mpb_appendchunkreq{placement_key=__todoPK,
                        prefix=Prefix,
                        chunk=Chunk,
-                        csum=CSum} = IR,
+                        csum=CSum,
-    NS = list_to_binary(NS_str),
+                        chunk_extra=ChunkExtra} = IR,
    TaggedCSum = make_tagged_csum(CSum, Chunk),
-    Opts = conv_to_append_opts(IR),
+    {ReqID, {high_append_chunk, __todoPK, Prefix, Chunk, TaggedCSum,
-    {ReqID, {high_append_chunk, NS, Prefix, Chunk, TaggedCSum, Opts}};
+             ChunkExtra}};
 from_pb_request(#mpb_request{req_id=ReqID,
                             write_chunk=IR=#mpb_writechunkreq{}}) ->
    #mpb_writechunkreq{chunk=#mpb_chunk{file_name=File,
                                        offset=Offset,
                                        chunk=Chunk,
-                                        csum=CSumRec}} = IR,
+                                        csum=CSum}} = IR,
-    CSum = make_tagged_csum(CSumRec, Chunk),
+    TaggedCSum = make_tagged_csum(CSum, Chunk),
-    {ReqID, {high_write_chunk, File, Offset, Chunk, CSum}};
+    {ReqID, {high_write_chunk, File, Offset, Chunk, TaggedCSum}};
 from_pb_request(#mpb_request{req_id=ReqID,
                             read_chunk=IR=#mpb_readchunkreq{}}) ->
    #mpb_readchunkreq{chunk_pos=#mpb_chunkpos{file_name=File,
@ -207,10 +194,11 @@ from_pb_request(#mpb_request{req_id=ReqID,
                     flag_no_checksum=FlagNoChecksum,
                     flag_no_chunk=FlagNoChunk,
                     flag_needs_trimmed=NeedsTrimmed} = IR,
-    Opts = #read_opts{no_checksum=FlagNoChecksum,
+    %% I want MAPS
-                      no_chunk=FlagNoChunk,
+    Options = [{no_checksum, machi_util:int2bool(FlagNoChecksum)},
-                      needs_trimmed=NeedsTrimmed},
+               {no_chunk, machi_util:int2bool(FlagNoChunk)},
-    {ReqID, {high_read_chunk, File, Offset, Size, Opts}};
+               {needs_trimmed, machi_util:int2bool(NeedsTrimmed)}],
    {ReqID, {high_read_chunk, File, Offset, Size, Options}};
 from_pb_request(#mpb_request{req_id=ReqID,
                             trim_chunk=IR=#mpb_trimchunkreq{}}) ->
    #mpb_trimchunkreq{chunk_pos=#mpb_chunkpos{file_name=File,
@ -274,12 +262,12 @@ from_pb_response(#mpb_ll_response{
                                          chunk=Bytes,
                                          csum=#mpb_chunkcsum{type=T,csum=Ck}}) ->
                                       Csum = <<(conv_to_csum_tag(T)):8, Ck/binary>>,
-                                      {list_to_binary(File), Offset, Bytes, Csum}
+                                      {File, Offset, Bytes, Csum}
                              end, PB_Chunks),
            Trimmed = lists:map(fun(#mpb_chunkpos{file_name=File,
                                                  offset=Offset,
                                                  chunk_size=Size}) ->
-                                        {list_to_binary(File), Offset, Size}
+                                        {File, Offset, Size}
                                end, PB_Trimmed),
            {ReqID, {ok, {Chunks, Trimmed}}};
        _ ->
@ -315,16 +303,12 @@ from_pb_response(#mpb_ll_response{
 from_pb_response(#mpb_ll_response{
                    req_id=ReqID,
                    wedge_status=#mpb_ll_wedgestatusresp{
-                      status=Status,
+                      epoch_id=PB_EpochID, wedged_flag=PB_Wedged}}) ->
                      epoch_id=PB_EpochID, wedged_flag=Wedged_p,
                      namespace_version=NSVersion, namespace=NS_str}}) ->
    GeneralStatus = case machi_pb_high_client:convert_general_status_code(Status) of
                        ok    -> ok;
                        _Else -> {yukky, _Else}
                    end,
    EpochID = conv_to_epoch_id(PB_EpochID),
-    NS = list_to_binary(NS_str),
+    Wedged_p = if PB_Wedged == 1 -> true;
-    {ReqID, {GeneralStatus, {Wedged_p, EpochID, NSVersion, NS}}};
+                  PB_Wedged == 0 -> false
               end,
    {ReqID, {ok, {Wedged_p, EpochID}}};
 from_pb_response(#mpb_ll_response{
                    req_id=ReqID,
                    delete_migration=#mpb_ll_deletemigrationresp{
@ -390,100 +374,89 @@ from_pb_response(#mpb_ll_response{
        'OK' ->
            {ReqID, {ok, Epochs}};
        _ ->
-            {ReqID, machi_pb_high_client:convert_general_status_code(Status)}
+            {ReqID< machi_pb_high_client:convert_general_status_code(Status)}
    end.
 %% No response for proj_kp/kick_projection_reaction
 %% TODO: move the #mbp_* record making code from
 %%       machi_pb_high_client:do_send_sync() clauses into to_pb_request().
-to_pb_request(ReqID, {low_skip_wedge, {low_echo, Msg}}) ->
+to_pb_request(ReqID, {low_echo, _BogusEpochID, Msg}) ->
    #mpb_ll_request{
               req_id=ReqID, do_not_alter=2,
               echo=#mpb_echoreq{message=Msg}};
-to_pb_request(ReqID, {low_skip_wedge, {low_auth, User, Pass}}) ->
+to_pb_request(ReqID, {low_auth, _BogusEpochID, User, Pass}) ->
    #mpb_ll_request{req_id=ReqID, do_not_alter=2,
                    auth=#mpb_authreq{user=User, password=Pass}};
-%% NOTE: The tuple position of NSLocator is a bit odd, because EpochID
+to_pb_request(ReqID, {low_append_chunk, EpochID, PKey, Prefix, Chunk,
-%%       _must_ be in the 4th position (as NSV & NS must be in 2nd & 3rd).
+                      CSum_tag, CSum, ChunkExtra}) ->
 to_pb_request(ReqID, {low_append_chunk, NSVersion, NS, EpochID, NSLocator,
                      Prefix, Chunk, CSum_tag, CSum, Opts}) ->
    PB_EpochID = conv_from_epoch_id(EpochID),
    CSum_type = conv_from_csum_tag(CSum_tag),
    PB_CSum = #mpb_chunkcsum{type=CSum_type, csum=CSum},
    {ChunkExtra, Pref, FailPref} = conv_from_append_opts(Opts),
    #mpb_ll_request{req_id=ReqID, do_not_alter=2,
                    append_chunk=#mpb_ll_appendchunkreq{
                      namespace_version=NSVersion,
                      namespace=NS,
                      locator=NSLocator,
                      epoch_id=PB_EpochID,
                      placement_key=PKey,
                      prefix=Prefix,
                      chunk=Chunk,
                      csum=PB_CSum,
-                      chunk_extra=ChunkExtra,
+                      chunk_extra=ChunkExtra}};
-                      preferred_file_name=Pref,
+to_pb_request(ReqID, {low_write_chunk, EpochID, File, Offset, Chunk, CSum_tag, CSum}) ->
                      flag_fail_preferred=FailPref}};
 to_pb_request(ReqID, {low_write_chunk, NSVersion, NS, EpochID, File, Offset, Chunk, CSum_tag, CSum}) ->
    PB_EpochID = conv_from_epoch_id(EpochID),
    CSum_type = conv_from_csum_tag(CSum_tag),
    PB_CSum = #mpb_chunkcsum{type=CSum_type, csum=CSum},
    #mpb_ll_request{req_id=ReqID, do_not_alter=2,
                    write_chunk=#mpb_ll_writechunkreq{
                      namespace_version=NSVersion,
                      namespace=NS,
                      epoch_id=PB_EpochID,
                                   chunk=#mpb_chunk{file_name=File,
                                                    offset=Offset,
                                                    chunk=Chunk,
                                                    csum=PB_CSum}}};
-to_pb_request(ReqID, {low_read_chunk, NSVersion, NS, EpochID, File, Offset, Size, Opts}) ->
+to_pb_request(ReqID, {low_read_chunk, EpochID, File, Offset, Size, Opts}) ->
    PB_EpochID = conv_from_epoch_id(EpochID),
-    #read_opts{no_checksum=FNChecksum,
+    FNChecksum = proplists:get_value(no_checksum, Opts, false),
-               no_chunk=FNChunk,
+    FNChunk = proplists:get_value(no_chunk, Opts, false),
-               needs_trimmed=NeedsTrimmed} = Opts,
+    NeedsTrimmed = proplists:get_value(needs_trimmed, Opts, false),
    #mpb_ll_request{
               req_id=ReqID, do_not_alter=2,
               read_chunk=#mpb_ll_readchunkreq{
                            namespace_version=NSVersion,
                            namespace=NS,
                             epoch_id=PB_EpochID,
                             chunk_pos=#mpb_chunkpos{
                                          file_name=File,
                                          offset=Offset,
                                          chunk_size=Size},
-                            flag_no_checksum=FNChecksum,
+                            flag_no_checksum=machi_util:bool2int(FNChecksum),
-                            flag_no_chunk=FNChunk,
+                            flag_no_chunk=machi_util:bool2int(FNChunk),
-                            flag_needs_trimmed=NeedsTrimmed}};
+                            flag_needs_trimmed=machi_util:bool2int(NeedsTrimmed)}};
-to_pb_request(ReqID, {low_trim_chunk, NSVersion, NS, EpochID, File, Offset, Size, TriggerGC}) ->
+to_pb_request(ReqID, {low_trim_chunk, EpochID, File, Offset, Size, TriggerGC}) ->
    PB_EpochID = conv_from_epoch_id(EpochID),
    #mpb_ll_request{req_id=ReqID, do_not_alter=2,
                    trim_chunk=#mpb_ll_trimchunkreq{
                                  namespace_version=NSVersion,
                                  namespace=NS,
                                  epoch_id=PB_EpochID,
                                  file=File,
                                  offset=Offset,
                                  size=Size,
                                  trigger_gc=TriggerGC}};
-to_pb_request(ReqID, {low_skip_wedge, {low_checksum_list, File}}) ->
+to_pb_request(ReqID, {low_checksum_list, EpochID, File}) ->
    PB_EpochID = conv_from_epoch_id(EpochID),
    #mpb_ll_request{req_id=ReqID, do_not_alter=2,
                    checksum_list=#mpb_ll_checksumlistreq{
                      epoch_id=PB_EpochID,
                      file=File}};
-to_pb_request(ReqID, {low_skip_wedge, {low_list_files, EpochID}}) ->
+to_pb_request(ReqID, {low_list_files, EpochID}) ->
    PB_EpochID = conv_from_epoch_id(EpochID),
    #mpb_ll_request{req_id=ReqID, do_not_alter=2,
                    list_files=#mpb_ll_listfilesreq{epoch_id=PB_EpochID}};
-to_pb_request(ReqID, {low_skip_wedge, {low_wedge_status}}) ->
+to_pb_request(ReqID, {low_wedge_status, _BogusEpochID}) ->
    #mpb_ll_request{req_id=ReqID, do_not_alter=2,
                    wedge_status=#mpb_ll_wedgestatusreq{}};
-to_pb_request(ReqID, {low_skip_wedge, {low_delete_migration, EpochID, File}}) ->
+to_pb_request(ReqID, {low_delete_migration, EpochID, File}) ->
    PB_EpochID = conv_from_epoch_id(EpochID),
    #mpb_ll_request{req_id=ReqID, do_not_alter=2,
                    delete_migration=#mpb_ll_deletemigrationreq{
                     epoch_id=PB_EpochID,
                      file=File}};
-to_pb_request(ReqID, {low_skip_wedge, {low_trunc_hack, EpochID, File}}) ->
+to_pb_request(ReqID, {low_trunc_hack, EpochID, File}) ->
    PB_EpochID = conv_from_epoch_id(EpochID),
    #mpb_ll_request{req_id=ReqID, do_not_alter=2,
                    trunc_hack=#mpb_ll_trunchackreq{
@ -519,15 +492,15 @@ to_pb_response(_ReqID, _, async_no_response=X) ->
    X;
 to_pb_response(ReqID, _, {low_error, ErrCode, ErrMsg}) ->
    make_ll_error_resp(ReqID, ErrCode, ErrMsg);
-to_pb_response(ReqID, {low_skip_wedge, {low_echo, _Msg}}, Resp) ->
+to_pb_response(ReqID, {low_echo, _BogusEpochID, _Msg}, Resp) ->
    #mpb_ll_response{
                req_id=ReqID,
                echo=#mpb_echoresp{message=Resp}};
-to_pb_response(ReqID, {low_skip_wedge, {low_auth, _, _}}, __TODO_Resp) ->
+to_pb_response(ReqID, {low_auth, _, _, _}, __TODO_Resp) ->
    #mpb_ll_response{req_id=ReqID,
                     generic=#mpb_errorresp{code=1,
                                            msg="AUTH not implemented"}};
-to_pb_response(ReqID, {low_append_chunk, _NSV, _NS, _EID, _NSL, _Pfx, _Ch, _CST, _CS, _O}, Resp)->
+to_pb_response(ReqID, {low_append_chunk, _EID, _PKey, _Pfx, _Ch, _CST, _CS, _CE}, Resp)->
    case Resp of
        {ok, {Offset, Size, File}} ->
            Where = #mpb_chunkpos{offset=Offset,
@ -543,11 +516,11 @@ to_pb_response(ReqID, {low_append_chunk, _NSV, _NS, _EID, _NSL, _Pfx, _Ch, _CST,
        _Else ->
            make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
    end;
-to_pb_response(ReqID, {low_write_chunk, _NSV, _NS, _EID, _Fl, _Off, _Ch, _CST, _CS},Resp)->
+to_pb_response(ReqID, {low_write_chunk, _EID, _Fl, _Off, _Ch, _CST, _CS},Resp)->
    Status = conv_from_status(Resp),
    #mpb_ll_response{req_id=ReqID,
                     write_chunk=#mpb_ll_writechunkresp{status=Status}};
-to_pb_response(ReqID, {low_read_chunk, _NSV, _NS, _EID, _Fl, _Off, _Sz, _Opts}, Resp)->
+to_pb_response(ReqID, {low_read_chunk, _EID, _Fl, _Off, _Sz, _Opts}, Resp)->
    case Resp of
        {ok, {Chunks, Trimmed}} ->
            PB_Chunks = lists:map(fun({File, Offset, Bytes, Csum}) ->
@ -574,7 +547,7 @@ to_pb_response(ReqID, {low_read_chunk, _NSV, _NS, _EID, _Fl, _Off, _Sz, _Opts},
        _Else ->
            make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
    end;
-to_pb_response(ReqID, {low_trim_chunk, _, _, _, _, _, _, _}, Resp) ->
+to_pb_response(ReqID, {low_trim_chunk, _, _, _, _, _}, Resp) ->
    case Resp of
        ok ->
            #mpb_ll_response{req_id=ReqID,
@ -582,11 +555,11 @@ to_pb_response(ReqID, {low_trim_chunk, _, _, _, _, _, _, _}, Resp) ->
        {error, _}=Error ->
            Status = conv_from_status(Error),
            #mpb_ll_response{req_id=ReqID,
-                             trim_chunk=#mpb_ll_trimchunkresp{status=Status}};
+                             read_chunk=#mpb_ll_trimchunkresp{status=Status}};
        _Else ->
            make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
    end;
-to_pb_response(ReqID, {low_skip_wedge, {low_checksum_list, _File}}, Resp) ->
+to_pb_response(ReqID, {low_checksum_list, _EpochID, _File}, Resp) ->
    case Resp of
        {ok, Chunk} ->
            #mpb_ll_response{req_id=ReqID,
@ -599,7 +572,7 @@ to_pb_response(ReqID, {low_skip_wedge, {low_checksum_list, _File}}, Resp) ->
        _Else ->
            make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
    end;
-to_pb_response(ReqID, {low_skip_wedge, {low_list_files, _EpochID}}, Resp) ->
+to_pb_response(ReqID, {low_list_files, _EpochID}, Resp) ->
    case Resp of
        {ok, FileInfo} ->
            PB_Files = [#mpb_fileinfo{file_size=Size, file_name=Name} ||
@ -614,28 +587,26 @@ to_pb_response(ReqID, {low_skip_wedge, {low_list_files, _EpochID}}, Resp) ->
        _Else ->
            make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
    end;
-to_pb_response(ReqID, {low_skip_wedge, {low_wedge_status}}, Resp) ->
+to_pb_response(ReqID, {low_wedge_status, _BogusEpochID}, Resp) ->
    case Resp of
        {error, _}=Error ->
            Status = conv_from_status(Error),
            #mpb_ll_response{req_id=ReqID,
                           wedge_status=#mpb_ll_wedgestatusresp{status=Status}};
-        {Wedged_p, EpochID, NSVersion, NS} ->
+        {Wedged_p, EpochID} ->
            PB_Wedged = conv_from_boolean(Wedged_p),
            PB_EpochID = conv_from_epoch_id(EpochID),
            #mpb_ll_response{req_id=ReqID,
                             wedge_status=#mpb_ll_wedgestatusresp{
                               status='OK',
                               epoch_id=PB_EpochID,
-                               wedged_flag=Wedged_p,
+                               wedged_flag=PB_Wedged}}
                               namespace_version=NSVersion,
                               namespace=NS
                              }}
    end;
-to_pb_response(ReqID, {low_skip_wedge, {low_delete_migration, _EID, _Fl}}, Resp)->
+to_pb_response(ReqID, {low_delete_migration, _EID, _Fl}, Resp)->
    Status = conv_from_status(Resp),
    #mpb_ll_response{req_id=ReqID,
                   delete_migration=#mpb_ll_deletemigrationresp{status=Status}};
-to_pb_response(ReqID, {low_skip_wedge, {low_trunc_hack, _EID, _Fl}}, Resp)->
+to_pb_response(ReqID, {low_trunc_hack, _EID, _Fl}, Resp)->
    Status = conv_from_status(Resp),
    #mpb_ll_response{req_id=ReqID,
                     trunc_hack=#mpb_ll_trunchackresp{status=Status}};
@ -716,7 +687,7 @@ to_pb_response(ReqID, {high_auth, _User, _Pass}, _Resp) ->
    #mpb_response{req_id=ReqID,
                  generic=#mpb_errorresp{code=1,
                                         msg="AUTH not implemented"}};
-to_pb_response(ReqID, {high_append_chunk, _NS, _Prefix, _Chunk, _TSum, _O}, Resp)->
+to_pb_response(ReqID, {high_append_chunk, _TODO, _Prefix, _Chunk, _TSum, _CE}, Resp)->
    case Resp of
        {ok, {Offset, Size, File}} ->
            Where = #mpb_chunkpos{offset=Offset,
@ -732,7 +703,7 @@ to_pb_response(ReqID, {high_append_chunk, _NS, _Prefix, _Chunk, _TSum, _O}, Resp
        _Else ->
            make_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
    end;
-to_pb_response(ReqID, {high_write_chunk, _File, _Offset, _Chunk, _CSum}, Resp) ->
+to_pb_response(ReqID, {high_write_chunk, _File, _Offset, _Chunk, _TaggedCSum}, Resp) ->
    case Resp of
        {ok, {_,_,_}} ->
            %% machi_cr_client returns ok 2-tuple, convert to simple ok.
@ -840,7 +811,6 @@ conv_to_epoch_id(#mpb_epochid{epoch_number=Epoch,
 conv_to_projection_v1(#mpb_projectionv1{epoch_number=Epoch,
                                        epoch_csum=CSum,
                                        author_server=Author,
                                        chain_name=ChainName,
                                        all_members=AllMembers,
                                        witnesses=Witnesses,
                                        creation_time=CTime,
@ -854,7 +824,6 @@ conv_to_projection_v1(#mpb_projectionv1{epoch_number=Epoch,
    #projection_v1{epoch_number=Epoch,
                   epoch_csum=CSum,
                   author_server=to_atom(Author),
                   chain_name=to_atom(ChainName),
                   all_members=[to_atom(X) || X <- AllMembers],
                   witnesses=[to_atom(X) || X <- Witnesses],
                   creation_time=conv_to_now(CTime),
@ -984,34 +953,24 @@ conv_from_status({error, partial_read}) ->
 conv_from_status({error, bad_epoch}) ->
    'BAD_EPOCH';
 conv_from_status(_OOPS) ->
-    io:format(user, "HEY, ~s:~w got ~p\n", [?MODULE, ?LINE, _OOPS]),
+    io:format(user, "HEY, ~s:~w got ~w\n", [?MODULE, ?LINE, _OOPS]),
    'BAD_JOSS'.
-conv_from_append_opts(#append_opts{chunk_extra=ChunkExtra,
+conv_to_boolean(undefined) ->
-                                   preferred_file_name=Pref,
+    false;
-                                   flag_fail_preferred=FailPref}) ->
+conv_to_boolean(0) ->
-    {ChunkExtra, Pref, FailPref}.
+    false;
 conv_to_boolean(N) when is_integer(N) ->
    true.
-
+conv_from_boolean(false) ->
-conv_to_append_opts(#mpb_appendchunkreq{
+    0;
-                       chunk_extra=ChunkExtra,
+conv_from_boolean(true) ->
-                       preferred_file_name=Pref,
+    1.
                       flag_fail_preferred=FailPref}) ->
    #append_opts{chunk_extra=ChunkExtra,
                 preferred_file_name=Pref,
                 flag_fail_preferred=FailPref};
 conv_to_append_opts(#mpb_ll_appendchunkreq{
                       chunk_extra=ChunkExtra,
                       preferred_file_name=Pref,
                       flag_fail_preferred=FailPref}) ->
    #append_opts{chunk_extra=ChunkExtra,
                 preferred_file_name=Pref,
                 flag_fail_preferred=FailPref}.
 conv_from_projection_v1(#projection_v1{epoch_number=Epoch,
                                       epoch_csum=CSum,
                                       author_server=Author,
                                       chain_name=ChainName,
                                       all_members=AllMembers,
                                       witnesses=Witnesses,
                                       creation_time=CTime,
@ -1025,7 +984,6 @@ conv_from_projection_v1(#projection_v1{epoch_number=Epoch,
    #mpb_projectionv1{epoch_number=Epoch,
                      epoch_csum=CSum,
                      author_server=to_list(Author),
                      chain_name=to_list(ChainName),
                      all_members=[to_list(X) || X <- AllMembers],
                      witnesses=[to_list(X) || X <- Witnesses],
                      creation_time=conv_from_now(CTime),
--- a/src/machi_plist.erl
+++ b/src/machi_plist.erl
@ -1,23 +1,3 @@
 %% -------------------------------------------------------------------
 %%
 %% Copyright (c) 2007-2016 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
 %% except in compliance with the License.  You may obtain
 %% a copy of the License at
 %%
 %%   http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing,
 %% software distributed under the License is distributed on an
 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 %% KIND, either express or implied.  See the License for the
 %% specific language governing permissions and limitations
 %% under the License.
 %%
 %% -------------------------------------------------------------------
 -module(machi_plist).
 %%% @doc persistent list of binaries
--- a/src/machi_projection.erl
+++ b/src/machi_projection.erl
@ -174,7 +174,6 @@ make_summary(#projection_v1{epoch_number=EpochNum,
                            repairing=Repairing_list,
                            dbg=Dbg, dbg2=Dbg2}) ->
    [{epoch,EpochNum}, {csum,_CSum4},
 {all, _All_list},
     {author,Author}, {mode,CMode},{witnesses, Witness_list},
     {upi,UPI_list},{repair,Repairing_list},{down,Down_list}] ++
        [{d,Dbg}, {d2,Dbg2}].
--- a/src/machi_projection_store.erl
+++ b/src/machi_projection_store.erl
@ -321,7 +321,7 @@ do_proj_write3(ProjType, #projection_v1{epoch_number=Epoch,
    end.
 do_proj_write4(ProjType, Proj, Path, Epoch, #state{consistency_mode=CMode}=S) ->
-    {{ok, FH}, Epoch, Path} = {file:open(Path, [write, raw, binary]), Epoch, Path},
+    {ok, FH} = file:open(Path, [write, raw, binary]),
    ok = file:write(FH, term_to_binary(Proj)),
    ok = file:sync(FH),
    ok = file:close(FH),
@ -387,6 +387,7 @@ wait_for_liveness(PidSpec, StartTime, WaitTime) ->
        undefined ->
            case timer:now_diff(os:timestamp(), StartTime) div 1000 of
                X when X < WaitTime ->
                    io:format(user, "\nYOO ~p ~p\n", [PidSpec, lists:sort(registered())]),
                    timer:sleep(1),
                    wait_for_liveness(PidSpec, StartTime, WaitTime)
            end;
--- a/src/machi_proxy_flu1_client.erl
+++ b/src/machi_proxy_flu1_client.erl
@ -22,10 +22,6 @@
 %% proxy-process style API for hiding messy details such as TCP
 %% connection/disconnection with the remote Machi server.
 %%
 %% Please see {@link machi_flu1_client} the "Client API implemntation notes"
 %% section for how this module relates to the rest of the client API
 %% implementation.
 %%
 %% Machi is intentionally avoiding using distributed Erlang for
 %% Machi's communication.  This design decision makes Erlang-side code
 %% more difficult &amp; complex, but it's the price to pay for some
@ -61,9 +57,10 @@
 %% FLU1 API
 -export([
         %% File API
-         append_chunk/6, append_chunk/8,
+         append_chunk/4, append_chunk/5,
-         read_chunk/7, read_chunk/8,
+         append_chunk_extra/5, append_chunk_extra/6,
-         checksum_list/2, checksum_list/3,
+         read_chunk/6, read_chunk/7,
         checksum_list/3, checksum_list/4,
         list_files/2, list_files/3,
         wedge_status/1, wedge_status/2,
@ -81,8 +78,8 @@
         quit/1,
         %% Internal API
-         write_chunk/7, write_chunk/8,
+         write_chunk/5, write_chunk/6,
-         trim_chunk/6, trim_chunk/7,
+         trim_chunk/5, trim_chunk/6,
         %% Helpers
         stop_proxies/1, start_proxies/1
@ -107,39 +104,51 @@ start_link(#p_srvr{}=I) ->
 %% @doc Append a chunk (binary- or iolist-style) of data to a file
 %% with `Prefix'.
-append_chunk(PidSpec, NSInfo, EpochID, Prefix, Chunk, CSum) ->
+append_chunk(PidSpec, EpochID, Prefix, Chunk) ->
-    append_chunk(PidSpec, NSInfo, EpochID, Prefix, Chunk, CSum,
+    append_chunk(PidSpec, EpochID, Prefix, Chunk, infinity).
                 #append_opts{}, infinity).
 %% @doc Append a chunk (binary- or iolist-style) of data to a file
 %% with `Prefix'.
-append_chunk(PidSpec, NSInfo, EpochID, Prefix, Chunk, CSum, Opts,
+append_chunk(PidSpec, EpochID, Prefix, Chunk, Timeout) ->
-             Timeout) ->
+    gen_server:call(PidSpec, {req, {append_chunk, EpochID, Prefix, Chunk}},
-    gen_server:call(PidSpec, {req, {append_chunk, NSInfo, EpochID,
+                    Timeout).
-                                    Prefix, Chunk, CSum, Opts, Timeout}},
+
 %% @doc Append a chunk (binary- or iolist-style) of data to a file
 %% with `Prefix'.
 append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra)
  when is_integer(ChunkExtra), ChunkExtra >= 0 ->
    append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra, infinity).
 %% @doc Append a chunk (binary- or iolist-style) of data to a file
 %% with `Prefix'.
 append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra, Timeout) ->
    gen_server:call(PidSpec, {req, {append_chunk_extra, EpochID, Prefix,
                                    Chunk, ChunkExtra}},
                    Timeout).
 %% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
-read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, Opts) ->
+read_chunk(PidSpec, EpochID, File, Offset, Size, Opts) ->
-    read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, Opts, infinity).
+    read_chunk(PidSpec, EpochID, File, Offset, Size, Opts, infinity).
 %% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
-read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, Opts, Timeout) ->
+read_chunk(PidSpec, EpochID, File, Offset, Size, Opts, Timeout) ->
-    gen_server:call(PidSpec, {req, {read_chunk, NSInfo, EpochID, File, Offset, Size, Opts}},
+    gen_server:call(PidSpec, {req, {read_chunk, EpochID, File, Offset, Size, Opts}},
                    Timeout).
 %% @doc Fetch the list of chunk checksums for `File'.
-checksum_list(PidSpec, File) ->
+checksum_list(PidSpec, EpochID, File) ->
-    checksum_list(PidSpec, File, infinity).
+    checksum_list(PidSpec, EpochID, File, infinity).
 %% @doc Fetch the list of chunk checksums for `File'.
-checksum_list(PidSpec, File, Timeout) ->
+checksum_list(PidSpec, EpochID, File, Timeout) ->
-    gen_server:call(PidSpec, {req, {checksum_list, File}},
+    gen_server:call(PidSpec, {req, {checksum_list, EpochID, File}},
                    Timeout).
 %% @doc Fetch the list of all files on the remote FLU.
@ -280,18 +289,18 @@ quit(PidSpec) ->
 %% @doc Write a chunk (binary- or iolist-style) of data to a file
 %% with `Prefix' at `Offset'.
-write_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, CSum) ->
+write_chunk(PidSpec, EpochID, File, Offset, Chunk) ->
-    write_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, CSum, infinity).
+    write_chunk(PidSpec, EpochID, File, Offset, Chunk, infinity).
 %% @doc Write a chunk (binary- or iolist-style) of data to a file
 %% with `Prefix' at `Offset'.
-write_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, CSum, Timeout) ->
+write_chunk(PidSpec, EpochID, File, Offset, Chunk, Timeout) ->
-    case gen_server:call(PidSpec, {req, {write_chunk, NSInfo, EpochID, File, Offset, Chunk, CSum}},
+    case gen_server:call(PidSpec, {req, {write_chunk, EpochID, File, Offset, Chunk}},
                         Timeout) of
        {error, written}=Err ->
            Size = byte_size(Chunk),
-            case read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, undefined, Timeout) of
+            case read_chunk(PidSpec, EpochID, File, Offset, Size, [], Timeout) of
                {ok, {[{File, Offset, Chunk2, _}], []}} when Chunk2 == Chunk ->
                    %% See equivalent comment inside write_projection().
                    ok;
@ -303,15 +312,15 @@ write_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, CSum, Timeout) ->
    end.
-trim_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size) ->
+trim_chunk(PidSpec, EpochID, File, Offset, Size) ->
-    trim_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, infinity).
+    trim_chunk(PidSpec, EpochID, File, Offset, Size, infinity).
 %% @doc Write a chunk (binary- or iolist-style) of data to a file
 %% with `Prefix' at `Offset'.
-trim_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, Timeout) ->
+trim_chunk(PidSpec, EpochID, File, Offset, Chunk, Timeout) ->
    gen_server:call(PidSpec,
-                    {req, {trim_chunk, NSInfo, EpochID, File, Offset, Chunk}},
+                    {req, {trim_chunk, EpochID, File, Offset, Chunk}},
                    Timeout).
 %%%%%%%%%%%%%%%%%%%%%%%%%%%
@ -375,24 +384,24 @@ do_req_retry(_Req, 2, Err, S) ->
 do_req_retry(Req, Depth, _Err, S) ->
    do_req(Req, Depth + 1, try_connect(disconnect(S))).
-make_req_fun({append_chunk, NSInfo, EpochID,
+make_req_fun({append_chunk, EpochID, Prefix, Chunk},
              Prefix, Chunk, CSum, Opts, Timeout},
             #state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
-    fun() -> Mod:append_chunk(Sock, NSInfo, EpochID,
+    fun() -> Mod:append_chunk(Sock, EpochID, Prefix, Chunk) end;
-                              Prefix, Chunk, CSum, Opts, Timeout)
+make_req_fun({append_chunk_extra, EpochID, Prefix, Chunk, ChunkExtra},
    end;
 make_req_fun({read_chunk, NSInfo, EpochID, File, Offset, Size, Opts},
             #state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
-    fun() -> Mod:read_chunk(Sock, NSInfo, EpochID, File, Offset, Size, Opts) end;
+    fun() -> Mod:append_chunk_extra(Sock, EpochID, Prefix, Chunk, ChunkExtra) end;
-make_req_fun({write_chunk, NSInfo, EpochID, File, Offset, Chunk, CSum},
+make_req_fun({read_chunk, EpochID, File, Offset, Size, Opts},
             #state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
-    fun() -> Mod:write_chunk(Sock, NSInfo, EpochID, File, Offset, Chunk, CSum) end;
+    fun() -> Mod:read_chunk(Sock, EpochID, File, Offset, Size, Opts) end;
-make_req_fun({trim_chunk, NSInfo, EpochID, File, Offset, Size},
+make_req_fun({write_chunk, EpochID, File, Offset, Chunk},
             #state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
-    fun() -> Mod:trim_chunk(Sock, NSInfo, EpochID, File, Offset, Size) end;
+    fun() -> Mod:write_chunk(Sock, EpochID, File, Offset, Chunk) end;
-make_req_fun({checksum_list, File},
+make_req_fun({trim_chunk, EpochID, File, Offset, Size},
             #state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
-    fun() -> Mod:checksum_list(Sock, File) end;
+    fun() -> Mod:trim_chunk(Sock, EpochID, File, Offset, Size) end;
 make_req_fun({checksum_list, EpochID, File},
             #state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
    fun() -> Mod:checksum_list(Sock, EpochID, File) end;
 make_req_fun({list_files, EpochID},
             #state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
    fun() -> Mod:list_files(Sock, EpochID) end;
--- a/src/machi_sup.erl
+++ b/src/machi_sup.erl
@ -47,6 +47,8 @@ start_link() ->
    supervisor:start_link({local, ?SERVER}, ?MODULE, []).
 init([]) ->
    %% {_, Ps} = process_info(self(), links),
    %% [unlink(P) || P <- Ps],
    RestartStrategy = one_for_one,
    MaxRestarts = 1000,
    MaxSecondsBetweenRestarts = 3600,
@ -60,16 +62,9 @@ init([]) ->
    ServerSup =
        {machi_flu_sup, {machi_flu_sup, start_link, []},
         Restart, Shutdown, Type, []},
-    RanchSup = {ranch_sup, {ranch_sup, start_link, []},
+
-                Restart, Shutdown, supervisor, [ranch_sup]},
+    {ok, {SupFlags, [ServerSup]}}.
-    LifecycleMgr =
+
-        {machi_lifecycle_mgr, {machi_lifecycle_mgr, start_link, []},
+    %% AChild = {'AName', {'AModule', start_link, []},
-         Restart, Shutdown, worker, []},
+    %%           Restart, Shutdown, Type, ['AModule']},
-    RunningApps = [A || {A,_D,_V} <- application:which_applications()],
+    %% {ok, {SupFlags, [AChild]}}.
    Specs = case lists:member(ranch, RunningApps) of
                true ->
                    [ServerSup, LifecycleMgr];
                false ->
                    [ServerSup, RanchSup, LifecycleMgr]
            end,
    {ok, {SupFlags, Specs}}.
--- a/src/machi_util.erl
+++ b/src/machi_util.erl
@ -25,19 +25,18 @@
 -export([
         checksum_chunk/1,
         make_tagged_csum/1, make_tagged_csum/2,
         make_client_csum/1,
         unmake_tagged_csum/1,
         hexstr_to_bin/1, bin_to_hexstr/1,
         hexstr_to_int/1, int_to_hexstr/2, int_to_hexbin/2,
         make_binary/1, make_string/1,
         make_regname/1,
-         make_config_filename/4, make_config_filename/2,
+         make_config_filename/2,
         make_checksum_filename/4, make_checksum_filename/2,
-         make_data_filename/6, make_data_filename/2,
+         make_data_filename/4, make_data_filename/2,
         make_projection_filename/2, 
         is_valid_filename/1,
         parse_filename/1,
-         read_max_filenum/4, increment_max_filenum/4,
+         read_max_filenum/2, increment_max_filenum/2,
         info_msg/2, verb/1, verb/2,
         mbytes/1, 
         pretty_time/0, pretty_time/2,
@ -50,9 +49,7 @@
         %% Other
         wait_for_death/2, wait_for_life/2,
         bool2int/1,
-         int2bool/1,
+         int2bool/1
         read_opts_default/1,
         ns_info_default/1
        ]).
 -include("machi.hrl").
@ -71,20 +68,10 @@ make_regname(Prefix) when is_list(Prefix) ->
 %% @doc Calculate a config file path, by common convention.
 -spec make_config_filename(string(), machi_dt:namespace(), machi_dt:locator(), string()) ->
      string().
 make_config_filename(DataDir, NS, NSLocator, Prefix) ->
    NSLocator_str = int_to_hexstr(NSLocator, 32),
    lists:flatten(io_lib:format("~s/config/~s^~s^~s",
                                [DataDir, Prefix, NS, NSLocator_str])).
 %% @doc Calculate a config file path, by common convention.
 -spec make_config_filename(string(), string()) ->
      string().
-make_config_filename(DataDir, Filename) ->
+make_config_filename(DataDir, Prefix) ->
-    lists:flatten(io_lib:format("~s/config/~s",
+    lists:flatten(io_lib:format("~s/config/~s", [DataDir, Prefix])).
                                [DataDir, Filename])).
 %% @doc Calculate a checksum file path, by common convention.
@ -105,19 +92,17 @@ make_checksum_filename(DataDir, FileName) ->
 %% @doc Calculate a file data file path, by common convention.
-spec make_data_filename(string(), machi_dt:namespace(), machi_dt:locator(), string(), atom()|string()|binary(), integer()|string()) ->
+-spec make_data_filename(string(), string(), atom()|string()|binary(), integer()|string()) ->
      {binary(), string()}.
-make_data_filename(DataDir, NS, NSLocator, Prefix, SequencerName, FileNum)
+make_data_filename(DataDir, Prefix, SequencerName, FileNum)
  when is_integer(FileNum) ->
-    NSLocator_str = int_to_hexstr(NSLocator, 32),
+    File = erlang:iolist_to_binary(io_lib:format("~s^~s^~w",
-    File = erlang:iolist_to_binary(io_lib:format("~s^~s^~s^~s^~w",
+                                                 [Prefix, SequencerName, FileNum])),
                                                 [Prefix, NS, NSLocator_str, SequencerName, FileNum])),
    make_data_filename2(DataDir, File);
-make_data_filename(DataDir, NS, NSLocator, Prefix, SequencerName, String)
+make_data_filename(DataDir, Prefix, SequencerName, String)
  when is_list(String) ->
-    NSLocator_str = int_to_hexstr(NSLocator, 32),
+    File = erlang:iolist_to_binary(io_lib:format("~s^~s^~s",
-    File = erlang:iolist_to_binary(io_lib:format("~s^~s^~s^~s^~s",
+                                                 [Prefix, SequencerName, string])),
                                                 [Prefix, NS, NSLocator_str, SequencerName, string])),
    make_data_filename2(DataDir, File).
 make_data_filename2(DataDir, File) ->
@ -149,44 +134,34 @@ make_projection_filename(DataDir, File) ->
 -spec is_valid_filename( Filename :: string() ) -> true | false.
 is_valid_filename(Filename) ->
    case parse_filename(Filename) of
-        {}          -> false;
+        [] -> false;
-        {_,_,_,_,_} -> true
+        _ -> true
    end.
 %% @doc Given a machi filename, return a set of components in a list.
 %% The components will be:
 %% <ul>
 %%      <li>Prefix</li>
 %%      <li>Cluster namespace</li>
 %%      <li>Cluster locator</li>
 %%      <li>UUID</li>
 %%      <li>Sequence number</li>
 %% </ul>
 %%
 %% Invalid filenames will return an empty list.
-spec parse_filename( Filename :: string() ) -> {} | {string(), machi_dt:namespace(), machi_dt:locator(), string(), string() }.
+-spec parse_filename( Filename :: string() ) -> [ string() ].
 parse_filename(Filename) ->
    case string:tokens(Filename, "^") of
-        [Prefix, NS, NSLocator, UUID, SeqNo] ->
+        [_Prefix, _UUID, _SeqNo] = L -> L;
-            {Prefix, NS, list_to_integer(NSLocator), UUID, SeqNo};
+        _ -> []
        [Prefix,          NSLocator, UUID, SeqNo] ->
            %% string:tokens() doesn't consider "foo^^bar" as 3 tokens {sigh}
            case re:replace(Filename, "[^^]+", "x", [global,{return,binary}]) of
                <<"x^^x^x^x">> ->
                    {Prefix, <<"">>, list_to_integer(NSLocator), UUID, SeqNo};
                _ ->
                    {}
            end;
        _ -> {}
    end.
 %% @doc Read the file size of a config file, which is used as the
 %% basis for a minimum sequence number.
-spec read_max_filenum(string(), machi_dt:namespace(), machi_dt:locator(), string()) ->
+-spec read_max_filenum(string(), string()) ->
      non_neg_integer().
-read_max_filenum(DataDir, NS, NSLocator, Prefix) ->
+read_max_filenum(DataDir, Prefix) ->
-    case file:read_file_info(make_config_filename(DataDir, NS, NSLocator, Prefix)) of
+    case file:read_file_info(make_config_filename(DataDir, Prefix)) of
        {error, enoent} ->
            0;
        {ok, FI} ->
@ -196,11 +171,11 @@ read_max_filenum(DataDir, NS, NSLocator, Prefix) ->
 %% @doc Increase the file size of a config file, which is used as the
 %% basis for a minimum sequence number.
-spec increment_max_filenum(string(), machi_dt:namespace(), machi_dt:locator(), string()) ->
+-spec increment_max_filenum(string(), string()) ->
      ok | {error, term()}.
-increment_max_filenum(DataDir, NS, NSLocator, Prefix) ->
+increment_max_filenum(DataDir, Prefix) ->
    try
-        {ok, FH} = file:open(make_config_filename(DataDir, NS, NSLocator, Prefix), [append]),
+        {ok, FH} = file:open(make_config_filename(DataDir, Prefix), [append]),
        ok = file:write(FH, "x"),
        ok = file:sync(FH),
        ok = file:close(FH)
@ -289,25 +264,12 @@ int_to_hexbin(I, I_size) ->
 checksum_chunk(Chunk) when is_binary(Chunk); is_list(Chunk) ->
    crypto:hash(sha, Chunk).
 convert_csum_tag(A) when is_atom(A)->
    A;
 convert_csum_tag(?CSUM_TAG_NONE) ->
    ?CSUM_TAG_NONE_ATOM;
 convert_csum_tag(?CSUM_TAG_CLIENT_SHA) ->
    ?CSUM_TAG_CLIENT_SHA_ATOM;
 convert_csum_tag(?CSUM_TAG_SERVER_SHA) ->
    ?CSUM_TAG_SERVER_SHA_ATOM;
 convert_csum_tag(?CSUM_TAG_SERVER_REGEN_SHA) ->
    ?CSUM_TAG_SERVER_REGEN_SHA_ATOM.
 %% @doc Create a tagged checksum
 make_tagged_csum(none) ->
    <<?CSUM_TAG_NONE:8>>;
 make_tagged_csum(<<>>) ->
    <<?CSUM_TAG_NONE:8>>;
 make_tagged_csum({Tag, CSum}) ->
-    make_tagged_csum(convert_csum_tag(Tag), CSum).
+    make_tagged_csum(Tag, CSum).
 %% @doc Makes tagged csum. Each meanings are:
 %% none / ?CSUM_TAG_NONE
@ -328,9 +290,6 @@ make_tagged_csum(?CSUM_TAG_SERVER_SHA_ATOM, SHA) ->
 make_tagged_csum(?CSUM_TAG_SERVER_REGEN_SHA_ATOM, SHA) ->
    <<?CSUM_TAG_SERVER_REGEN_SHA:8, SHA/binary>>.
 make_client_csum(BinOrList) ->
    make_tagged_csum(?CSUM_TAG_CLIENT_SHA_ATOM, checksum_chunk(BinOrList)).
 unmake_tagged_csum(<<Tag:8, Rest/binary>>) ->
    {Tag, Rest}.
@ -378,7 +337,7 @@ wait_for_death(Pid, Iters) when is_pid(Pid) ->
        false ->
            ok;
        true ->
-            timer:sleep(10),
+            timer:sleep(1),
            wait_for_death(Pid, Iters-1)
    end.
@ -449,17 +408,3 @@ bool2int(true) -> 1;
 bool2int(false) -> 0.
 int2bool(0) -> false;
 int2bool(I) when is_integer(I) -> true.
 read_opts_default(#read_opts{}=NSInfo) ->
    NSInfo;
 read_opts_default(A) when A == 'undefined'; A == 'noopt'; A == 'none' ->
    #read_opts{};
 read_opts_default(A) when is_atom(A) ->
    #read_opts{}.
 ns_info_default(#ns_info{}=NSInfo) ->
    NSInfo;
 ns_info_default(A) when is_atom(A) ->
    #ns_info{}.
--- a/src/machi_yessir_client.erl
+++ b/src/machi_yessir_client.erl
@ -22,8 +22,6 @@
 -module(machi_yessir_client).
 -ifdef(TODO_refactoring_deferred).
 -include("machi.hrl").
 -include("machi_projection.hrl").
@ -32,7 +30,7 @@
         append_chunk/4, append_chunk/5,
         append_chunk_extra/5, append_chunk_extra/6,
         read_chunk/5, read_chunk/6,
-         checksum_list/2, checksum_list/3,
+         checksum_list/3, checksum_list/4,
         list_files/2, list_files/3,
         wedge_status/1, wedge_status/2,
@ -175,24 +173,24 @@ read_chunk(_Host, _TcpPort, EpochID, File, Offset, Size)
 %% @doc Fetch the list of chunk checksums for `File'.
-checksum_list(#yessir{name=Name,chunk_size=ChunkSize}, File) ->
+checksum_list(#yessir{name=Name,chunk_size=ChunkSize}, _EpochID, File) ->
    case get({Name,offset,File}) of
        undefined ->
            {error, no_such_file};
        MaxOffset ->
            C = machi_util:make_tagged_csum(client_sha,
                                            make_csum(Name, ChunkSize)),
-            Cs = [{Offset, ChunkSize, C} ||
+            Cs = [machi_csum_table:encode_csum_file_entry_bin(Offset, ChunkSize, C) ||
                    Offset <- lists:seq(?MINIMUM_OFFSET, MaxOffset, ChunkSize)],
-            {ok, term_to_binary(Cs)}
+            {ok, Cs}
    end.
 %% @doc Fetch the list of chunk checksums for `File'.
-checksum_list(_Host, _TcpPort, File) ->
+checksum_list(_Host, _TcpPort, EpochID, File) ->
    Sock = connect(#p_srvr{proto_mod=?MODULE}),
    try
-        checksum_list(Sock, File)
+        checksum_list(Sock, EpochID, File)
    after
        disconnect(Sock)
    end.
@ -511,5 +509,3 @@ disconnect(#yessir{name=Name}) ->
 %% =INFO REPORT==== 17-May-2015::18:57:52 ===
 %% Repair success: tail a of [a] finished ap_mode repair ID {a,{1431,856671,140404}}: ok
 %% Stats [{t_in_files,0},{t_in_chunks,10413},{t_in_bytes,682426368},{t_out_files,0},{t_out_chunks,10413},{t_out_bytes,682426368},{t_bad_chunks,0},{t_elapsed_seconds,1.591}]
 -endif. % TODO_refactoring_deferred
--- a/test/machi_admin_util_test.erl
+++ b/test/machi_admin_util_test.erl
@ -33,33 +33,25 @@
 -define(FLU_C, machi_flu1_client).
 verify_file_checksums_test_() ->
-    {setup,
+    {timeout, 60, fun() -> verify_file_checksums_test2() end}.
     fun() -> os:cmd("rm -rf ./data") end,
     fun(_) -> os:cmd("rm -rf ./data")  end,
     {timeout, 60, fun() -> verify_file_checksums_test2() end}
    }.
 verify_file_checksums_test2() ->
    Host = "localhost",
    TcpPort = 32958,
    DataDir = "./data",
    W_props = [{initial_wedged, false}],
-    NSInfo = undefined,
+    machi_flu1_test:start_flu_package(verify1_flu, TcpPort, DataDir,
    NoCSum = <<>>,
    try
        machi_test_util:start_flu_package(verify1_flu, TcpPort, DataDir,
                                          W_props),
    Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}),
    try
        Prefix = <<"verify_prefix">>,
        NumChunks = 10,
-            [{ok, _} = ?FLU_C:append_chunk(Sock1, NSInfo, ?DUMMY_PV1_EPOCH,
+        [{ok, _} = ?FLU_C:append_chunk(Sock1, ?DUMMY_PV1_EPOCH,
-                                           Prefix, <<X:(X*8)/big>>, NoCSum) ||
+                                       Prefix, <<X:(X*8)/big>>) ||
            X <- lists:seq(1, NumChunks)],
        {ok, [{_FileSize,File}]} = ?FLU_C:list_files(Sock1, ?DUMMY_PV1_EPOCH),
-            ?assertEqual({ok, []},
+        {ok, []} = machi_admin_util:verify_file_checksums_remote(
-                         machi_admin_util:verify_file_checksums_remote(
+                     Host, TcpPort, ?DUMMY_PV1_EPOCH, File),
                           Host, TcpPort, ?DUMMY_PV1_EPOCH, File)),
        %% Clobber the first 3 chunks, which are sizes 1/2/3.
        {_, Path} = machi_util:make_data_filename(DataDir,binary_to_list(File)),
@ -82,10 +74,8 @@ verify_file_checksums_test2() ->
        ok
    after
-            catch ?FLU_C:quit(Sock1)
+        catch ?FLU_C:quit(Sock1),
-        end
+        catch machi_flu1_test:stop_flu_package(verify1_flu)
    after
        catch machi_test_util:stop_flu_package()
    end.
 -endif. % !PULSE
--- a/test/machi_ap_repair_eqc.erl
+++ b/test/machi_ap_repair_eqc.erl
@ -1,616 +0,0 @@
 %% -------------------------------------------------------------------
 %%
 %% Copyright (c) 2007-2015 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
 %% except in compliance with the License.  You may obtain
 %% a copy of the License at
 %%
 %%   http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing,
 %% software distributed under the License is distributed on an
 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 %% KIND, either express or implied.  See the License for the
 %% specific language governing permissions and limitations
 %% under the License.
 %%
 %% -------------------------------------------------------------------
 %% EQC single-threaded and concurrent test for file operations and repair
 %% under simulated network partition.
 %% The main purpose is to confirm no dataloss, i.e. every chunk that
 %% has been successfully written (ACK received) by append/write
 %% opration will be read after partition heals.
 %%
 %% All updating -- append, write and trim -- operations are executed
 %% through CR client, not directly by flu1 client, in order to be
 %% end-to-end test (in single chain point of veiw.) There may be churn
 %% for projections by simulated network partition.
 %%
 %% Test steps
 %% 1. Setup single chain.
 %% 2. Execute updating operations and simulated partition (by eqc_statem).
 %%    Every updating results are recorded in ETS tables.
 %% 3. When {error, timeout|partition} happens, trigger management tick for
 %%    every chain manager process.
 %% 4. After commands are executed, remove patition and wait for the chain
 %%    without down nodes nor repairing nodes.
 %% 5. Asserting written results so that each record be read from the
 %%    chain and data be the same with written one.
 %% Improvements to-do's
 %% - Use higher concurrency, e.g. 10+
 %% - Random length for binary to write
 %% - Operations other than append, write, trim
 %% - Use checksum instead of binary to save memory
 %% - More variety for partitioning pattern: non-constant failure
 %% - Stop and restart
 %% - Suspend and resume of some erlang processes
 -module(machi_ap_repair_eqc).
 -ifdef(TEST).
 -ifdef(EQC).
 -compile(export_all).
 -include("machi.hrl").
 -include("machi_projection.hrl").
 -include("machi_verbose.hrl").
 -include_lib("eqc/include/eqc.hrl").
 -include_lib("eqc/include/eqc_statem.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -record(target, {verbose=false,
                 flu_names,
                 mgr_names}).
 -record(state, {num,
                verbose=false,
                flu_names,
                mgr_names,
                cr_count}).
 %% ETS table names
 -define(WRITTEN_TAB,  written).  % Successfully written data
 -define(ACCPT_TAB,    accpt).    % Errors with no harm, e.g. timeout
 -define(FAILED_TAB,   failed).   % Uncategorized errors, when happenes
                                 % it should be re-categorized to accept or critical
 -define(CRITICAL_TAB, critical). % Critical errors, e.g. double write to the same key
 -define(QC_OUT(P),
        eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)).
 %% EUNIT TEST DEFINITION
 prop_repair_test_() ->
    {PropTO, EUnitTO} = eqc_timeout(60),
    Verbose = eqc_verbose(),
    {spawn,
     [{timeout, EUnitTO,
       ?_assertEqual(
          true,
          eqc:quickcheck(eqc:testing_time(
                           PropTO, ?QC_OUT(noshrink(prop_repair(Verbose))))))}]}.
 prop_repair_par_test_() ->
    {PropTO, EUnitTO} = eqc_timeout(60),
    Verbose = eqc_verbose(),
    {spawn,
     [{timeout, EUnitTO,
       ?_assertEqual(
          true,
          eqc:quickcheck(eqc:testing_time(
                           PropTO, ?QC_OUT(noshrink(prop_repair_par(Verbose))))))}]}.
 %% Model
 weight(_S, change_partition) ->  20;
 weight(_S, _)                -> 100.
 %% Append
 append_args(#state{cr_count=CRCount}=S) ->
    [choose(1, CRCount), chunk(), S].
 append(CRIndex, Bin, #state{verbose=V}=S) ->
    CRList = cr_list(),
    {_SimSelfName, C} = lists:nth(CRIndex, CRList),
    Prefix = <<"pre">>,
    Len = byte_size(Bin),
    NSInfo = #ns_info{},
    NoCSum = <<>>,
    Opts1 = #append_opts{},
    Res = (catch machi_cr_client:append_chunk(C, NSInfo, Prefix, Bin, NoCSum, Opts1, sec(1))),
    case Res of
        {ok, {_Off, Len, _FileName}=Key} ->
            case ets:insert_new(?WRITTEN_TAB, {Key, Bin}) of
                true ->
                    [?V("<o>", []) || V],
                    ok;
                false ->
                    %% The Key is alread written, WHY!!!????
                    case ets:lookup(?WRITTEN_TAB, Key) of
                        [{Key, Bin}] ->
                            %% TODO: The identical binary is alread inserted in
                            %%  written table. Is this acceptable??? Hmm, maybe NO...
                            [?V("<dws:~w>", [Key]) || V],
                            true = ets:insert_new(?ACCPT_TAB,
                                                  {make_ref(), double_write_same, Key}),
                            {acceptable_error, doublewrite_the_same};
                        [{Key, OtherBin}] ->
                            [?V("<dwd:~w:~w>", [Key, {OtherBin, Bin}]) || V],
                            true = ets:insert_new(?CRITICAL_TAB,
                                                  {make_ref(), double_write_diff, Key}),
                            R = {critical_error,
                                 {doublewrite_diff, Key, {OtherBin, Bin}}},
                            %% TODO: when double write happens, it seems that
                            %% repair process got stack with endless loop. To
                            %% avoit it, return error here.
                            %% If this error/1 will be removed, one can possibly
                            %% know double write frequency/rate.
                            error(R)
                    end
            end;
        {error, partition} -> 
            [?V("<pt>", []) || V],
            true = ets:insert_new(?ACCPT_TAB, {make_ref(), timeout}),
            _ = tick(S),
            {acceptable_error, partition};
        {'EXIT', {timeout, _}} ->
            [?V("<to:~w:~w>", [_SimSelfName, C]) || V],
            true = ets:insert_new(?ACCPT_TAB, {make_ref(), timeout}),
            _ = tick(S),
            {acceptable_error, timeout};
        {ok, {_Off, UnexpectedLen, _FileName}=Key} ->
            [?V("<XX>", []) || V],
            true = ets:insert_new(?CRITICAL_TAB, {make_ref(), unexpected_len, Key}),
            {critical_error, {unexpected_len, Key, Len, UnexpectedLen}};
        {error, _Reason} = Error ->
            [?V("<er>", []) || V],
            true = ets:insert_new(?FAILED_TAB, {make_ref(), Error}),
            {other_error, Error};
        Other ->
            [?V("<er>", []) || V],
            true = ets:insert_new(?FAILED_TAB, {make_ref(), Other}),
            {other_error, Other}
    end.
 %% Change partition
 change_partition_args(#state{flu_names=FLUNames}=S) ->
    %% [partition(FLUNames), S].
    [partition_sym(FLUNames), S].
 change_partition(Partition,
                 #state{verbose=Verbose, flu_names=FLUNames}=S) ->
    [case Partition of
         [] -> ?V("## Turn OFF partition: ~w~n", [Partition]);
         _  -> ?V("## Turn ON  partition: ~w~n", [Partition])
     end || Verbose],
    machi_partition_simulator:always_these_partitions(Partition),
    _ = machi_partition_simulator:get(FLUNames),
    %% Don't wait for stable chain, tick will be executed on demand
    %% in append oprations
    _ = tick(S),
    ok.
 %% Generators
 num() ->
    choose(2, 5).
 cr_count(Num) ->
    Num * 3.
 %% Returns a list like
 %% `[{#p_srvr{name=a, port=7501, ..}, "./eqc/data.eqc.a/"}, ...]'
 all_list_extra(Num) ->
    {PortBase, DirBase} = get_port_dir_base(),
    [begin
         FLUNameStr = [$a + I - 1],
         FLUName = list_to_atom(FLUNameStr),
         MgrName = machi_flu_psup:make_mgr_supname(FLUName),
         {#p_srvr{name=FLUName, address="localhost", port=PortBase+I,
                  props=[{chmgr, MgrName}]},
          DirBase ++ "/data.eqc." ++ FLUNameStr}
     end || I <- lists:seq(1, Num)].
 sublist(L) ->
    ?LET(K, nat(),
    ?LET(L2, eqc_gen:vector(K, eqc_gen:oneof(L)),
         lists:usort(L2))).
 %% Generator for possibly assymmetric partition information
 partition(FLUNames) ->
    frequency([{10, return([])},
               {20, non_empty(sublist(flu_ordered_pairs(FLUNames)))}]).
 %% Generator for symmetric partition information
 partition_sym(FLUNames) ->
    ?LET(Pairs, non_empty(sublist(flu_pairs(FLUNames))),
         lists:flatmap(fun({One, Another}) -> [{One, Another}, {Another, One}] end,
                       Pairs)).
 flu_ordered_pairs(FLUNames) ->
    [{From, To} || From <- FLUNames, To <- FLUNames, From =/= To].
 flu_pairs(FLUNames) ->
    [{One, Another} || One <- FLUNames, Another <- FLUNames, One > Another].
 chunk() ->
    non_empty(binary(10)).
 %% Properties
 prop_repair(Verbose) ->
    error_logger:tty(false),
    application:load(sasl),
    application:set_env(sasl, sasl_error_logger, false),
    Seed = {1445,935441,287549},
    ?FORALL(Num, num(),
    ?FORALL(Cmds, commands(?MODULE, initial_state(Num, Verbose)),
            begin
                Target = setup_target(Num, Seed, Verbose),
                {H, S1, Res0} = run_commands(?MODULE, Cmds),
                %% ?V("S1=~w~n", [S1]),
                ?V("==== Start post operations, stabilize and confirm results~n", []),
                _ = stabilize(commands_len(Cmds), Target),
                {Dataloss, Critical} = confirm_result(Target),
                _ = cleanup(Target),
                pretty_commands(
                  ?MODULE, Cmds, {H, S1, Res0},
                  aggregate(with_title(cmds), command_names(Cmds),
                  collect(with_title(length5), (length(Cmds) div 5) * 5,
                          {Dataloss, Critical} =:= {0, 0})))
            end)).
 prop_repair_par(Verbose) ->
    error_logger:tty(false),
    application:load(sasl),
    application:set_env(sasl, sasl_error_logger, false),
    Seed = {1445,935441,287549},
    ?FORALL(Num, num(),
    ?FORALL(Cmds,
            %% Now try-and-err'ing, how to control command length and concurrency?
            ?SUCHTHAT(Cmds0, ?SIZED(Size, resize(Size,
                      parallel_commands(?MODULE, initial_state(Num, Verbose)))),
                      commands_len(Cmds0) > 20
                      andalso
                      concurrency(Cmds0) > 2),
            begin
                CmdsLen= commands_len(Cmds),
                Target = setup_target(Num, Seed, Verbose),
                {Seq, Par, Res0} = run_parallel_commands(?MODULE, Cmds),
                %% ?V("Seq=~w~n", [Seq]),
                %% ?V("Par=~w~n", [Par]),
                ?V("==== Start post operations, stabilize and confirm results~n", []),
                {FinalRes, {Dataloss, Critical}} =
                    case Res0 of
                        ok ->
                            Res1 = stabilize(CmdsLen, Target),
                            {Res1, confirm_result(Target)};
                        _ ->
                            ?V("Res0=~w~n", [Res0]),
                            {Res0, {undefined, undefined}}
                end,
                _ = cleanup(Target),
                %% Process is leaking? This log line can be removed after fix.
                [?V("process_count=~w~n", [erlang:system_info(process_count)]) || Verbose],
                pretty_commands(
                  ?MODULE, Cmds, {Seq, Par, Res0},
                  aggregate(with_title(cmds), command_names(Cmds),
                  collect(with_title(length5), (CmdsLen div 5) * 5,
                  collect(with_title(conc),    concurrency(Cmds),
                          {FinalRes, {Dataloss, Critical}} =:= {ok, {0, 0}})))
                 )
            end)).
 %% Initilization / setup
 %% Fake initialization function for debugging in shell like:
 %% > eqc_gen:sample(eqc_statem:commands(machi_ap_repair_eqc)).
 %% but not so helpful.
 initial_state() ->
    #state{cr_count=3}.
 initial_state(Num, Verbose) ->
    AllListE = all_list_extra(Num),
    FLUNames = [P#p_srvr.name || {P, _Dir} <- AllListE],
    MgrNames = [{Name, machi_flu_psup:make_mgr_supname(Name)} || Name <- FLUNames],
    #state{num=Num, verbose=Verbose,
           flu_names=FLUNames, mgr_names=MgrNames,
           cr_count=cr_count(Num)}.
 setup_target(Num, Seed, Verbose) ->
    %% ?V("setup_target(Num=~w, Seed=~w~nn", [Num, Seed]),
    AllListE = all_list_extra(Num),
    FLUNames = [P#p_srvr.name || {P, _Dir} <- AllListE],
    MgrNames = [{Name, machi_flu_psup:make_mgr_supname(Name)} || Name <- FLUNames],
    Dict = orddict:from_list([{P#p_srvr.name, P} || {P, _Dir} <- AllListE]),
    setup_chain(Seed, AllListE, FLUNames, MgrNames, Dict),
    _ = setup_cpool(AllListE, FLUNames, Dict),
    Target = #target{flu_names=FLUNames, mgr_names=MgrNames,
                     verbose=Verbose},
    %% Don't wait for complete chain. Even partialy completed, the chain
    %% should work fine. Right?
    wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames,
                      20, Verbose),
    Target.
 setup_chain(Seed, AllListE, FLUNames, MgrNames, Dict) ->
    ok = shutdown_hard(),
    [begin
         machi_test_util:clean_up_dir(Dir),
         filelib:ensure_dir(Dir ++ "/not-used")
     end || {_P, Dir} <- AllListE],
    [catch ets:delete(T) || T <- tabs()],
    [ets:new(T, [set, public, named_table,
                 {write_concurrency, true}, {read_concurrency, true}]) ||
        T <- tabs()],
    {ok, _} = application:ensure_all_started(machi),
    SimSpec = {part_sim,
               {machi_partition_simulator, start_link, [{0,0,0}, 0, 100]},
               permanent, 500, worker, []},
    {ok, _PSimPid} = supervisor:start_child(machi_sup, SimSpec),
    ok = machi_partition_simulator:set_seed(Seed),
    _Partitions = machi_partition_simulator:get(FLUNames),
    %% Start FLUs and setup the chain
    FLUOpts = [{use_partition_simulator, true},
               %% {private_write_verbose, true},
               {active_mode, false},
               {simulate_repair, false}],
    [{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, FLUOpts) ||
        {#p_srvr{name=Name, port=Port}, Dir} <- AllListE],
    [machi_chain_manager1:set_chain_members(MgrName, Dict) || {_, MgrName} <- MgrNames],
    ok.
 setup_cpool(AllListE, FLUNames, Dict) ->
    Num = length(AllListE),
    FCList = [begin
                  {ok, PCPid} = machi_proxy_flu1_client:start_link(P),
                  {Name, PCPid}
              end || {_, #p_srvr{name=Name}=P} <- Dict],
    %% CR clients are pooled, each has "name" which is interpreted "From"
    %% side of simulated partition.
    SimSelfNames = lists:append(lists:duplicate(cr_count(Num), FLUNames)),
    CRList = [begin
                  {ok, C} = machi_cr_client:start_link(
                              [P || {_, P} <- Dict],
                              [{use_partition_simulator, true},
                               {simulator_self_name, SimSelfName},
                               {simulator_members, FLUNames}]),
                  {SimSelfName, C}
              end || SimSelfName <- SimSelfNames],
    catch ets:delete(cpool),
    ets:new(cpool, [set, protected, named_table, {read_concurrency, true}]),
    ets:insert(cpool, {fc_list, FCList}),
    ets:insert(cpool, {cr_list, CRList}),
    {CRList, FCList}.
 fc_list() ->
    [{fc_list, FCList}] = ets:lookup(cpool, fc_list),
    FCList.
 cr_list() ->
    [{cr_list, CRList}] = ets:lookup(cpool, cr_list),
    CRList.
 %% Post run_commands
 stabilize(0, _T) ->
    ok;
 stabilize(_CmdsLen, #target{flu_names=FLUNames, mgr_names=MgrNames,
                            verbose=Verbose}) ->
    machi_partition_simulator:no_partitions(),
    true = wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames,
                             100, Verbose),
    ok.
 chain_state_all_ok(FLUNames) ->
    [{FLUName, {FLUNames, [], []}} || FLUName <- FLUNames].
 confirm_result(_T) ->
    [{_, C} | _] = cr_list(),
    [{written, _Written}, {accpt, Accpt},
     {failed, Failed}, {critical, Critical}] = tab_counts(),
    {OK, Dataloss} = confirm_written(C),
    ?V("  Written=~w, DATALOSS=~w, Acceptable=~w~n", [OK, Dataloss, Accpt]),
    ?V("  Failed=~w, Critical=~w~n~n", [Failed, Critical]),
    DirBase = get_dir_base(),
    Suffix = dump_file_suffix(),
    case Failed of
        0 -> ok;
        _ ->
            DumpFailed = filename:join(DirBase, "dump-failed-" ++ Suffix),
            ?V("Dump failed ETS tab to: ~s~n", [DumpFailed]),
            ets:tab2file(?FAILED_TAB, DumpFailed)
    end,
    case Critical of
        0 -> ok;
        _ ->
            DumpCritical = filename:join(DirBase, "dump-critical-" ++ Suffix),
            ?V("Dump critical ETS tab to: ~w~n", [DumpCritical]),
            ets:tab2file(?CRITICAL_TAB, DumpCritical)
    end,
    {Dataloss, Critical}.
 confirm_written(C) ->
    ets:foldl(
      fun({Key, Bin}, {OK, NG}) ->
              case assert_chunk(C, Key, Bin) of
                  ok -> {OK+1, NG};
                  {error, _} -> {OK, NG+1}
              end
      end, {0, 0}, ?WRITTEN_TAB).
 assert_chunk(C, {Off, Len, FileName}=Key, Bin) ->
    %% TODO: This probably a bug, read_chunk respnds with filename of `string()' type
    %% TODO : Use CSum instead of binary (after disuccsion about CSum is calmed down?)
    NSInfo = undefined,
    case (catch machi_cr_client:read_chunk(C, NSInfo, FileName, Off, Len, undefined, sec(3))) of
        {ok, {[{FileName, Off, Bin, _}], []}} ->
            ok;
        {ok, Got} ->
            ?V("read_chunk got different binary for Key=~p~n", [Key]),
            ?V("    Expected: ~p~n", [{[{FileName, Off, Bin, <<"CSum-NYI">>}], []}]),
            ?V("    Got:      ~p~n", [Got]),
            {error, different_binary};
        {error, Reason} ->
            ?V("read_chunk error for Key=~p: ~p~n", [Key, Reason]),
            {error, Reason};
        Other ->
            ?V("read_chunk other error for Key=~p: ~p~n", [Key, Other]),
            {error, Other}
    end.
 cleanup(_Target) ->
    [begin unlink(FC), catch exit(FC, kill) end || {_, FC} <- fc_list()],
    [begin unlink(CR), catch exit(CR, kill) end || {_, CR} <- cr_list()],
    _ = shutdown_hard().
 %% Internal misc utilities
 eqc_verbose() ->
    os:getenv("EQC_VERBOSE") =:= "true".
 eqc_timeout(Default) ->
    PropTimeout = case os:getenv("EQC_TIME") of
                      false -> Default;
                      V -> list_to_integer(V)
                  end,
    {PropTimeout, PropTimeout * 300}.
 get_port_dir_base() ->
    I = case os:getenv("EQC_BASE_PORT") of
            false -> 0;
            V -> list_to_integer(V)
        end,
    D = get_dir_base(),
    {7400 + (I * 100), D ++ "/" ++ integer_to_list(I)}.
 get_dir_base() ->
    case os:getenv("EQC_BASE_DIR") of
        false -> "./eqc";
        DD -> DD
    end.
 shutdown_hard() ->
    _STOP = application:stop(machi),
    timer:sleep(100).
 tick(#state{flu_names=FLUNames, mgr_names=MgrNames,
            verbose=Verbose}) ->
    tick(FLUNames, MgrNames, Verbose).
 tick(FLUNames, MgrNames, Verbose) ->
    tick(FLUNames, MgrNames, 2, 100, Verbose).
 tick(FLUNames, MgrNames, Iter, SleepMax, Verbose) ->
    TickFun = tick_fun(FLUNames, MgrNames, self()),
    TickFun(Iter, 0, SleepMax),
    FCList = fc_list(),
    [?V("## Chain state after tick()=~w~n", [chain_state(FCList)]) || Verbose].
 tick_fun(FLUNames, MgrNames, Parent) ->
    fun(Iters, SleepMin, SleepMax) ->
            %% ?V("^", []),
            Trigger =
                fun(FLUName, MgrName) ->
                        random:seed(now()),
                        [begin
                             erlang:yield(),
                             SleepMaxRand = random:uniform(SleepMax + 1),
                             %% io:format(user, "{t}", []),
                             Elapsed = machi_chain_manager1:sleep_ranked_order(
                                         SleepMin, SleepMaxRand,
                                         FLUName, FLUNames),
                             MgrName ! tick_check_environment,
                             %% Be more unfair by not sleeping here.
                             timer:sleep(max(SleepMax - Elapsed, 1)),
                             ok
                         end || _ <- lists:seq(1, Iters)],
                        Parent ! {done, self()}
                end,
            Pids = [{spawn(fun() -> Trigger(FLUName, MgrName) end), FLUName} ||
                       {FLUName, MgrName} <- MgrNames ],
            [receive
                 {done, ThePid} ->
                     ok
             after 120*1000 ->
                     exit({icky_timeout, M_name})
             end || {ThePid, M_name} <- Pids]
    end.
 wait_until_stable(ExpectedChainState, FLUNames, MgrNames, Verbose) ->
    wait_until_stable(ExpectedChainState, FLUNames, MgrNames, 20, Verbose).
 wait_until_stable(ExpectedChainState, FLUNames, MgrNames, Retries, Verbose) ->
    TickFun = tick_fun(FLUNames, MgrNames, self()),
    FCList = fc_list(),
    wait_until_stable1(ExpectedChainState, TickFun, FCList, Retries, Verbose).
 wait_until_stable1(ExpectedChainState, _TickFun, FCList, 0, _Verbose) ->
    ?V("  [ERROR] _ExpectedChainState ~p\n", [ExpectedChainState]),
    ?V("  [ERROR] wait_until_stable failed.... : ~p~n", [chain_state(FCList)]),
    ?V("  [ERROR] norm....                     : ~p~n", [normalize_chain_state(chain_state(FCList))]),
    false;
 wait_until_stable1(ExpectedChainState, TickFun, FCList, Reties, Verbose) ->
    [TickFun(3, 0, 100) || _ <- lists:seq(1, 3)],
    Normalized = normalize_chain_state(chain_state(FCList)),
    case Normalized of
        ExpectedChainState ->
            [?V("  Got stable chain: ~w~n", [chain_state(FCList)]) || Verbose],
            true;
        _ ->
            [?V("  NOT YET stable chain: ~w~n", [chain_state(FCList)]) || Verbose],
            wait_until_stable1(ExpectedChainState, TickFun, FCList, Reties-1, Verbose)
    end.
 normalize_chain_state(ChainState) ->
    lists:usort([{FLUName,
                  {lists:usort(UPI), lists:usort(Repairing), lists:usort(Down)}} ||
                    {FLUName, {_EpochNo, UPI, Repairing, Down}} <- ChainState]).
 chain_state(FCList) ->
    lists:usort(
      [case (catch machi_proxy_flu1_client:read_latest_projection(C, private, sec(5))) of
           {ok, #projection_v1{epoch_number=EpochNo, upi=UPI,
                               repairing=Repairing, down=Down}} ->
               {FLUName, {EpochNo, UPI, Repairing, Down}};
           Other ->
               {FLUName, Other}
       end || {FLUName, C} <- FCList]).
 tabs() -> [?WRITTEN_TAB, ?ACCPT_TAB, ?FAILED_TAB, ?CRITICAL_TAB].
 tab_counts() ->
    [{T, ets:info(T, size)} || T <- tabs()].
 sec(Sec) ->
    timer:seconds(Sec).
 commands_len({SeqCmds, ParCmdsList} = _Cmds) ->
    lists:sum([length(SeqCmds) | [length(P) || P <- ParCmdsList]]);
 commands_len(Cmds) ->
    length(Cmds).
 concurrency({_SeqCmds, ParCmdsList} = _Cmds) -> length(ParCmdsList);
 concurrency(_) -> 1.
 dump_file_suffix() ->
    {{Year, Month, Day}, {Hour, Min, Sec}} = calendar:local_time(),
    lists:flatten(
      io_lib:format("~4.10.0B-~2.10.0B-~2.10.0BT~2.10.0B:~2.10.0B:~2.10.0B.000Z",
                    [Year, Month, Day, Hour, Min, Sec])).
 -endif. % EQC
 -endif. % TEST
--- a/test/machi_chain_manager1_converge_demo.erl
+++ b/test/machi_chain_manager1_converge_demo.erl
@ -45,18 +45,14 @@
 -include_lib("eunit/include/eunit.hrl").
 help() ->
    io:format("~s\n", [short_doc()]).
 short_doc() ->
 "
 A visualization of the convergence behavior of the chain self-management
 algorithm for Machi.
-
+  1. Set up 4 FLUs and chain manager pairs.
  1. Set up some server and chain manager pairs.
  2. Create a number of different network partition scenarios, where
-     (simulated) partitions may be symmetric or asymmetric.  Then stop changing
+     (simulated) partitions may be symmetric or asymmetric.  Then halt changing
-     the partitions and keep the simulated network stable (and perhaps broken).
+     the partitions and keep the simulated network stable and broken.
  3. Run a number of iterations of the algorithm in parallel by poking each
     of the manager processes on a random'ish basis.
  4. Afterward, fetch the chain transition changes made by each FLU and
@ -65,65 +61,73 @@ algorithm for Machi.
 During the iteration periods, the following is a cheatsheet for the output.
 See the internal source for interpreting the rest of the output.
-    'SET partitions = '
+    'Let loose the dogs of war!'  Network instability
    'SET partitions = '           Network stability (but broken)
    'x uses:' The FLU x has made an internal state transition.  The rest of
              the line is a dump of internal state.
    '{t}'     This is a tick event which triggers one of the manager processes
              to evaluate its environment and perhaps make a state transition.
-        A pair-wise list of actors which cannot send messages.  The
+A long chain of '{t}{t}{t}{t}' means that the chain state has settled
-        list is uni-directional.  If there are three servers (a,b,c),
+to a stable configuration, which is the goal of the algorithm.
-        and if the partitions list is '[{a,b},{b,c}]' then all
+Press control-c to interrupt....".
        messages from a->b and b->c will be dropped, but any other
        sender->recipient messages will be delivered successfully.
-    'x uses:'
+long_doc() ->
    "
 'Let loose the dogs of war!'
-        The FLU x has made an internal state transition and is using
+    The simulated network is very unstable for a few seconds.
        this epoch's projection as operating chain configuration.  The
        rest of the line is a summary of the projection.
-    'CONFIRM epoch {N}'
+'x uses'
-        This message confirms that all of the servers listed in the
+    After a single iteration, server x has determined that the chain
-        UPI and repairing lists of the projection at epoch {N} have
+    should be defined by the upi, repair, and down list in this record.
-        agreed to use this projection because they all have written
+    If all participants reach the same conclusion at the same epoch
-        this projection to their respective private projection stores.
+    number (and checksum, see next item below), then the chain is
-        The chain is now usable by/available to all clients.
+    stable, fully configured, and can provide full service.
-    'Sweet, private projections are stable'
+'epoch,E'
-        This report announces that this iteration of the test cycle
+    The epoch number for this decision is E.  The checksum of the full
-        has passed successfully.  The report that follows briefly
+    record is not shown.  For purposes of the protocol, a server will
-        summarizes the latest private projection used by each
+    'wedge' itself and refuse service (until a new config is chosen)
-        participating server.  For example, when in strong consistency
+    whenever: a). it sees a bigger epoch number mentioned somewhere, or
-        mode with 'a' as a witness and 'b' and 'c' as real servers:
+    b). it sees the same epoch number but a different checksum.  In case
    of b), there was a network partition that has healed, and both sides
    had chosen to operate with an identical epoch number but different
    chain configs.
-        %% Legend:
+'upi', 'repair', and 'down'
        %% server name, epoch ID, UPI list, repairing list, down list, ...
        %%                         ... witness list, 'false' (a constant value)
-        [{a,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}},
+    Members in the chain that are fully in sync and thus preserving the
-         {b,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}}]
+    Update Propagation Invariant, up but under repair (simulated), and
    down, respectively.
-        Both servers 'a' and 'b' agree on epoch 1116 with epoch ID
+'ps,[some list]'
        {1116,<<23,143,246,55>>} where UPI=[a,b], repairing=[],
        down=[c], and witnesses=[a].
-        Server 'c' is not shown because 'c' has wedged itself OOS (out
+    The list of asymmetric network partitions.  {a,b} means that a
-        of service) by configuring a chain length of zero.
+    cannot send to b, but b can send to a.
-        If no servers are listed in the report (i.e. only '[]' is
+    This partition list is recorded for debugging purposes but is *not*
-        displayed), then all servers have wedged themselves OOS, and
+    used by the algorithm.  The algorithm only 'feels' its effects via
-        the chain is unavailable.
+    simulated timeout whenever there's a partition in one of the
    messaging directions.
-    'DoIt,' 
+'nodes_up,[list]'
-        This marks a group of tick events which trigger the manager
+    The best guess right now of which ndoes are up, relative to the
-        processes to evaluate their environment and perhaps make a
+    author node, specified by '{author,X}'
        state transition.
-A long chain of 'DoIt,DoIt,DoIt,' means that the chain state has
+'SET partitions = [some list]'
 (probably) settled to a stable configuration, which is the goal of the
 algorithm.
-Press control-c to interrupt the test....".
+    All subsequent iterations should have a stable list of partitions,
    i.e. the 'ps' list described should be stable.
 '{FLAP: x flaps n}!'
    Server x has detected that it's flapping/oscillating after iteration
    n of a naive/1st draft detection algorithm.
 ".
 %% ' silly Emacs syntax highlighting....
@ -134,7 +138,6 @@ Press control-c to interrupt the test....".
 %%     convergence_demo_testfun(3).
 -define(DEFAULT_MGR_OPTS, [{private_write_verbose, false},
                           {private_write_verbose_confirm, true},
                           {active_mode,false},
                           {use_partition_simulator, true}]).
@ -151,8 +154,7 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
    %% Faster test startup, commented: io:format(user, short_doc(), []),
    %% Faster test startup, commented: timer:sleep(3000),
-    Apps = [sasl, ranch],
+    application:start(sasl),
    [application:start(App) || App <- Apps],
    MgrOpts = MgrOpts0 ++ ?DEFAULT_MGR_OPTS,
    TcpPort = proplists:get_value(port_base, MgrOpts, 62877),
@ -189,18 +191,15 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
             end || #p_srvr{name=Name}=P <- Ps],
    MembersDict = machi_projection:make_members_dict(Ps),
    Witnesses = proplists:get_value(witnesses, MgrOpts, []),
    CMode = case {Witnesses, proplists:get_value(consistency_mode, MgrOpts,
                                                 ap_mode)} of
                {[_|_], _}   -> cp_mode;
                {_, cp_mode} -> cp_mode;
                {_, ap_mode} -> ap_mode
            end,
    MgrNamez = [begin
                    MgrName = machi_flu_psup:make_mgr_supname(Name),
-                    ok = ?MGR:set_chain_members(MgrName, ch_demo, 0, CMode,
+                    ok = ?MGR:set_chain_members(MgrName,MembersDict,Witnesses),
                                                MembersDict,Witnesses),
                    {Name, MgrName}
                end || #p_srvr{name=Name} <- Ps],
    CpApMode = case Witnesses /= [] of
                   true  -> cp_mode;
                   false -> ap_mode
               end,
    try
      [{_, Ma}|_] = MgrNamez,
@ -296,7 +295,7 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
                            private_projections_are_stable(Namez, DoIt)
                    end, false, lists:seq(0, MaxIters)),
           io:format(user, "\n~s Sweet, private projections are stable\n", [machi_util:pretty_time()]),
-           io:format(user, "\t~P\n", [get(stable), 24]),
+           io:format(user, "\t~P\n", [get(stable), 14]),
           io:format(user, "Rolling sanity check ... ", []),
           PrivProjs = [{Name, begin
                                   {ok, Ps8} = ?FLU_PC:get_all_projections(
@ -308,9 +307,9 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
               [{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane_retrospective(Psx, FLU)} ||
                   {FLU, Psx} <- PrivProjs]
           catch
-               _Err:_What when CMode == cp_mode ->
+               _Err:_What when CpApMode == cp_mode ->
                   io:format(user, "none proj skip detected, TODO? ", []);
-               _Err:_What when CMode == ap_mode ->
+               _Err:_What when CpApMode == ap_mode ->
                   io:format(user, "PrivProjs ~p\n", [PrivProjs]),
                   exit({line, ?LINE, _Err, _What})
           end,
@ -376,9 +375,9 @@ timer:sleep(1234),
              {FLU, Psx} <- PrivProjs],
          io:format(user, "\nAll sanity checks pass, hooray!\n", [])
      catch
-          _Err:_What when CMode == cp_mode ->
+          _Err:_What when CpApMode == cp_mode ->
              io:format(user, "none proj skip detected, TODO? ", []);
-          _Err:_What when CMode == ap_mode ->
+          _Err:_What when CpApMode == ap_mode ->
              io:format(user, "Report ~p\n", [Report]),
              io:format(user, "PrivProjs ~p\n", [PrivProjs]),
              exit({line, ?LINE, _Err, _What})
@ -395,8 +394,7 @@ timer:sleep(1234),
        exit(SupPid, normal),
        ok = machi_partition_simulator:stop(),
        [ok = ?FLU_PC:quit(PPid) || {_, PPid} <- Namez],
-        machi_util:wait_for_death(SupPid, 100),
+        machi_util:wait_for_death(SupPid, 100)
        [application:start(App) || App <- lists:reverse(Apps)]
    end.
 %% Many of the static partition lists below have been problematic at one
@ -721,7 +719,7 @@ private_projections_are_stable(Namez, PollFunc) ->
                true
        end,
-    %% io:format(user, "\nPriv1 ~p\nPriv2 ~p\n1==2 ~w ap_disjoint ~w u_all_peers ~w cp_mode_agree ~w\n", [lists:sort(Private1), lists:sort(Private2), Private1 == Private2, AP_mode_disjoint_test_p, Unanimous_with_all_peers_p, CP_mode_agree_test_p]),
+    io:format(user, "\nPriv1 ~p\nPriv2 ~p\n1==2 ~w ap_disjoint ~w u_all_peers ~w cp_mode_agree ~w\n", [lists:sort(Private1), lists:sort(Private2), Private1 == Private2, AP_mode_disjoint_test_p, Unanimous_with_all_peers_p, CP_mode_agree_test_p]),
    Private1 == Private2 andalso
        AP_mode_disjoint_test_p andalso
        (
--- a/test/machi_chain_manager1_test.erl
+++ b/test/machi_chain_manager1_test.erl
@ -273,43 +273,43 @@ make_prop_ets() ->
 -endif. % EQC
 make_advance_fun(FitList, FLUList, MgrList, Num) ->
    fun() ->
            [begin
                 [catch machi_fitness:trigger_early_adjustment(Fit, Tgt) ||
                     Fit <- FitList,
                     Tgt <- FLUList ],
                 [catch ?MGR:trigger_react_to_env(Mgr) || Mgr <- MgrList],
                 ok
             end || _ <- lists:seq(1, Num)]
    end.
 smoke0_test() ->
    {ok, _} = machi_partition_simulator:start_link({1,2,3}, 50, 50),
    Host = "localhost",
    TcpPort = 6623,
-    {[Pa], [M0], _Dirs} = machi_test_util:start_flu_packages(
+    {ok, FLUa} = machi_flu1:start_link([{a,TcpPort,"./data.a"}]),
-                            1, TcpPort, "./data.", []),
+    Pa = #p_srvr{name=a, address=Host, port=TcpPort},
    Members_Dict = machi_projection:make_members_dict([Pa]),
    %% Egadz, more racing on startup, yay.  TODO fix.
    timer:sleep(1),
    {ok, FLUaP} = ?FLU_PC:start_link(Pa),
    {ok, M0} = ?MGR:start_link(a, Members_Dict, [{active_mode, false}]),
    try
        pong = ?MGR:ping(M0)
    after
        ok = ?MGR:stop(M0),
        ok = machi_flu1:stop(FLUa),
        ok = ?FLU_PC:quit(FLUaP),
-        machi_test_util:stop_flu_packages()
+        ok = machi_partition_simulator:stop()
    end.
 smoke1_test_() ->
    {timeout, 1*60, fun() -> smoke1_test2() end}.
 smoke1_test2() ->
    machi_partition_simulator:start_link({1,2,3}, 100, 0),
    TcpPort = 62777,
-    MgrOpts = [{active_mode,false}],
+    FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}, {c,TcpPort+2,"./data.c"}],
-    try
+    P_s = [#p_srvr{name=Name, address="localhost", port=Port} ||
-        {Ps, MgrNames, _Dirs} = machi_test_util:start_flu_packages(
+              {Name,Port,_Dir} <- FluInfo],
                                  3, TcpPort, "./data.", MgrOpts),
        MembersDict = machi_projection:make_members_dict(Ps),
        [machi_chain_manager1:set_chain_members(M, MembersDict) || M <- MgrNames],
        Ma = hd(MgrNames),
-        {ok, P1} = ?MGR:test_calc_projection(Ma, false),
+    [machi_flu1_test:clean_up_data_dir(Dir) || {_,_,Dir} <- FluInfo],
    FLUs = [element(2, machi_flu1:start_link([{Name,Port,Dir}])) ||
               {Name,Port,Dir} <- FluInfo],
    MembersDict = machi_projection:make_members_dict(P_s),
    {ok, M0} = ?MGR:start_link(a, MembersDict, [{active_mode,false}]),
    try
        {ok, P1} = ?MGR:test_calc_projection(M0, false),
        % DERP! Check for race with manager's proxy vs. proj listener
        ok = lists:foldl(
                    fun(_, {_,{true,[{c,ok},{b,ok},{a,ok}]}}) ->
@ -318,35 +318,39 @@ smoke1_test2() ->
                            ok;             % Skip remaining!
                       (_, _Else) ->
                            timer:sleep(10),
-                            ?MGR:test_write_public_projection(Ma, P1)
+                            ?MGR:test_write_public_projection(M0, P1)
                    end, not_ok, lists:seq(1, 1000)),
        %% Writing the exact same projection multiple times returns ok:
        %% no change!
-        {_,{true,[{c,ok},{b,ok},{a,ok}]}} =  ?MGR:test_write_public_projection(Ma, P1),
+        {_,{true,[{c,ok},{b,ok},{a,ok}]}} =  ?MGR:test_write_public_projection(M0, P1),
-        {unanimous, P1, Extra1} = ?MGR:test_read_latest_public_projection(Ma, false),
+        {unanimous, P1, Extra1} = ?MGR:test_read_latest_public_projection(M0, false),
        ok
    after
-        machi_test_util:stop_flu_packages()
+        ok = ?MGR:stop(M0),
        [ok = machi_flu1:stop(X) || X <- FLUs],
        ok = machi_partition_simulator:stop()
    end.
-nonunanimous_setup_and_fix_test_() ->
+nonunanimous_setup_and_fix_test() ->
-    os:cmd("rm -f /tmp/moomoo.*"),
+    machi_partition_simulator:start_link({1,2,3}, 100, 0),
    {timeout, 1*60, fun() -> nonunanimous_setup_and_fix_test2() end}.
 nonunanimous_setup_and_fix_test2() ->
    TcpPort = 62877,
-    MgrOpts = [{active_mode,false}],
+    FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}],
-    {Ps, [Ma,Mb,Mc], Dirs} = machi_test_util:start_flu_packages(
+    P_s = [#p_srvr{name=Name, address="localhost", port=Port} ||
-                             3, TcpPort, "./data.", MgrOpts),
+              {Name,Port,_Dir} <- FluInfo],
    MembersDict = machi_projection:make_members_dict(Ps),
    ChainName = my_little_chain,
    [machi_chain_manager1:set_chain_members(M, ChainName, 0, ap_mode,
                                            MembersDict, []) || M <- [Ma, Mb]],
    [Proxy_a, Proxy_b, Proxy_c] = Proxies =
        [element(2, ?FLU_PC:start_link(P)) || P <- Ps],
    [machi_flu1_test:clean_up_data_dir(Dir) || {_,_,Dir} <- FluInfo],
    {ok, SupPid} = machi_flu_sup:start_link(),
    Opts = [{active_mode, false}],
    %% {ok, Mb} = ?MGR:start_link(b, MembersDict, [{active_mode, false}]++XX),
    [{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) ||
               {Name,Port,Dir} <- FluInfo],
    [Proxy_a, Proxy_b] = Proxies =
        [element(2,?FLU_PC:start_link(P)) || P <- P_s],
    MembersDict = machi_projection:make_members_dict(P_s),
    [Ma,Mb] = [a_chmgr, b_chmgr],
    ok = machi_chain_manager1:set_chain_members(Ma, MembersDict, []),
    ok = machi_chain_manager1:set_chain_members(Mb, MembersDict, []),
    try
        {ok, P1} = ?MGR:test_calc_projection(Ma, false),
@ -383,114 +387,16 @@ nonunanimous_setup_and_fix_test2() ->
        {ok, P2pb} = ?FLU_PC:read_latest_projection(Proxy_b, private),
        P2 = P2pb#projection_v1{dbg2=[]},
-        Mgrs = [a_chmgr, b_chmgr, c_chmgr],
+        %% Pspam = machi_projection:update_checksum(
-        Advance = make_advance_fun([a_fitness,b_fitness,c_fitness],
+        %%           P1b#projection_v1{epoch_number=?SPAM_PROJ_EPOCH,
-                                   [a,b,c],
+        %%                             dbg=[hello_spam]}),
-                                   Mgrs,
+        %% ok = ?FLU_PC:write_projection(Proxy_b, public, Pspam),
                                   3),
        Advance(),
        {_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Ma),
        {_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Mb),
        {_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Mc),
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("STEP: Remove 'a' from the chain.\n", []),
        MembersDict4 = machi_projection:make_members_dict(tl(Ps)),
        ok = machi_chain_manager1:set_chain_members(
               Mb, ChainName, TheEpoch_3, ap_mode, MembersDict4, []),
        Advance(),
        {ok, {true, _,_,_}} = ?FLU_PC:wedge_status(Proxy_a),
        {_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Mb),
        {_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Mc),
        [{ok, #projection_v1{upi=[b,c], repairing=[]}} =
             ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("STEP: Add a to the chain again (a is running).\n", []),
        MembersDict5 = machi_projection:make_members_dict(Ps),
        ok = machi_chain_manager1:set_chain_members(
               Mb, ChainName, TheEpoch_4, ap_mode, MembersDict5, []),
        Advance(),
        {_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Ma),
        {_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Mb),
        {_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Mc),
        [{ok, #projection_v1{upi=[b,c], repairing=[a]}} =
             ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies],
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("STEP: Stop a while a chain member, advance b&c.\n", []),
        ok = machi_flu_psup:stop_flu_package(a),
        Advance(),
        {_, _, TheEpoch_6} = ?MGR:trigger_react_to_env(Mb),
        {_, _, TheEpoch_6} = ?MGR:trigger_react_to_env(Mc),
        [{ok, #projection_v1{upi=[b,c], repairing=[]}} =
             ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("STEP: Remove 'a' from the chain.\n", []),
        MembersDict7 = machi_projection:make_members_dict(tl(Ps)),
        ok = machi_chain_manager1:set_chain_members(
               Mb, ChainName, TheEpoch_6, ap_mode, MembersDict7, []),
        Advance(),
        {_, _, TheEpoch_7} = ?MGR:trigger_react_to_env(Mb),
        {_, _, TheEpoch_7} = ?MGR:trigger_react_to_env(Mc),
        [{ok, #projection_v1{upi=[b,c], repairing=[]}} =
             ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("STEP: Start a, advance.\n", []),
        Opts = [{active_mode, false}, {initial_wedged, true}],
        #p_srvr{name=NameA} = hd(Ps),
        {ok,_}=machi_flu_psup:start_flu_package(NameA, TcpPort+1, hd(Dirs), Opts),
        Advance(),
        {ok, {true, _,_,_}} = ?FLU_PC:wedge_status(Proxy_a),
        {ok, {false, EpochID_8,_,_}} = ?FLU_PC:wedge_status(Proxy_b),
        {ok, {false, EpochID_8,_,_}} = ?FLU_PC:wedge_status(Proxy_c),
        [{ok, #projection_v1{upi=[b,c], repairing=[]}} =
             ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("STEP: Stop a, delete a's data, leave it stopped\n", []),
        ok = machi_flu_psup:stop_flu_package(a),
        Advance(),
        machi_flu1_test:clean_up_data_dir(hd(Dirs)),
        {ok, {false, _,_,_}} = ?FLU_PC:wedge_status(Proxy_b),
        {ok, {false, _,_,_}} = ?FLU_PC:wedge_status(Proxy_c),
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("STEP: Add a to the chain again (a is stopped).\n", []),
        MembersDict9 = machi_projection:make_members_dict(Ps),
        {_, _, TheEpoch_9} = ?MGR:trigger_react_to_env(Mb),
        ok = machi_chain_manager1:set_chain_members(
               Mb, ChainName, TheEpoch_9, ap_mode, MembersDict9, []),
        Advance(),
        {_, _, TheEpoch_9b} = ?MGR:trigger_react_to_env(Mb),
        true = (TheEpoch_9b > TheEpoch_9),
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("STEP: Start a, and it joins like it ought to\n", []),
        {ok,_}=machi_flu_psup:start_flu_package(NameA, TcpPort+1, hd(Dirs), Opts),
        Advance(),
        {ok, {false, {TheEpoch10,_},_,_}} = ?FLU_PC:wedge_status(Proxy_a),
        {ok, {false, {TheEpoch10,_},_,_}} = ?FLU_PC:wedge_status(Proxy_b),
        {ok, {false, {TheEpoch10,_},_,_}} = ?FLU_PC:wedge_status(Proxy_c),
        [{ok, #projection_v1{upi=[b,c], repairing=[a]}} =
             ?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies],
        ok
    after
        exit(SupPid, normal),
        [ok = ?FLU_PC:quit(X) || X <- Proxies],
-        machi_test_util:stop_flu_packages()
+        ok = machi_partition_simulator:stop()
    end.
 unanimous_report_test() ->
--- a/test/machi_cinfo_test.erl
+++ b/test/machi_cinfo_test.erl
@ -38,7 +38,6 @@ smoke_test_() ->
      fun() -> machi_cinfo:private_projection(a) end,
      fun() -> machi_cinfo:fitness(a) end,
      fun() -> machi_cinfo:chain_manager(a) end,
      fun() -> machi_cinfo:flu1(a) end,
      fun() -> machi_cinfo:dump() end
     ]}.
@ -49,7 +48,7 @@ setup() ->
          {c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}}
         ],
    [os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps],
-    {ok, SupPid} = machi_sup:start_link(),
+    {ok, SupPid} = machi_flu_sup:start_link(),
    %% Only run a, don't run b & c so we have 100% failures talking to them
    [begin
         #p_srvr{name=Name, port=Port, props=Dir} = P,
--- a/test/machi_cr_client_test.erl
+++ b/test/machi_cr_client_test.erl
@ -58,15 +58,9 @@ setup_smoke_test(Host, PortBase, Os, Witness_list) ->
    %% 4. Wait until all others are using epoch id from #3.
    %%
    %% Damn, this is a pain to make 100% deterministic, bleh.
-    CMode = if Witness_list == [] -> ap_mode;
+    ok = machi_chain_manager1:set_chain_members(a_chmgr, D, Witness_list),
-               Witness_list /= [] -> cp_mode
+    ok = machi_chain_manager1:set_chain_members(b_chmgr, D, Witness_list),
-            end,
+    ok = machi_chain_manager1:set_chain_members(c_chmgr, D, Witness_list),
    ok = machi_chain_manager1:set_chain_members(a_chmgr, ch0, 0, CMode,
                                                D, Witness_list),
    ok = machi_chain_manager1:set_chain_members(b_chmgr, ch0, 0, CMode,
                                                D, Witness_list),
    ok = machi_chain_manager1:set_chain_members(c_chmgr, ch0, 0, CMode,
                                                D, Witness_list),
    run_ticks([a_chmgr,b_chmgr,c_chmgr]),
    %% Everyone is settled on the same damn epoch id.
    {ok, EpochID} = machi_flu1_client:get_latest_epochid(Host, PortBase+0,
@ -102,13 +96,11 @@ run_ticks(MgrList) ->
    ok.
 smoke_test2() ->
-    {ok, SupPid} = machi_sup:start_link(),
+    {ok, SupPid} = machi_flu_sup:start_link(),
    error_logger:tty(false),
    try
        Prefix = <<"pre">>,
        Chunk1 = <<"yochunk">>,
        NSInfo = undefined,
        NoCSum = <<>>,
        Host = "localhost",
        PortBase = 64454,
        Os = [{ignore_stability_time, true}, {active_mode, false}],
@ -116,95 +108,91 @@ smoke_test2() ->
        %% Whew ... ok, now start some damn tests.
        {ok, C1} = machi_cr_client:start_link([P || {_,P}<-orddict:to_list(D)]),
-        machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum),
+        machi_cr_client:append_chunk(C1, Prefix, Chunk1),
        {ok, {Off1,Size1,File1}} =
-            machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum),
+            machi_cr_client:append_chunk(C1, Prefix, Chunk1),
-        BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, "foo")},
+        Chunk1_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, Chunk1},
        {error, bad_checksum} =
-            machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, BadCSum),
+            machi_cr_client:append_chunk(C1, Prefix, Chunk1_badcs),
        {ok, {[{_, Off1, Chunk1, _}], []}} =
-            machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, Size1, undefined),
+            machi_cr_client:read_chunk(C1, File1, Off1, Size1, []),
        {ok, PPP} = machi_flu1_client:read_latest_projection(Host, PortBase+0,
                                                             private),
        %% Verify that the client's CR wrote to all of them.
        [{ok, {[{_, Off1, Chunk1, _}], []}} =
             machi_flu1_client:read_chunk(
-               Host, PortBase+X, NSInfo, EpochID, File1, Off1, Size1, undefined) ||
+               Host, PortBase+X, EpochID, File1, Off1, Size1, []) ||
            X <- [0,1,2] ],
        %% Test read repair: Manually write to head, then verify that
        %% read-repair fixes all.
        FooOff1 = Off1 + (1024*1024),
        [{error, not_written} = machi_flu1_client:read_chunk(
-                                  Host, PortBase+X, NSInfo, EpochID,
+                                  Host, PortBase+X, EpochID,
-                                  File1, FooOff1, Size1, undefined) || X <- [0,1,2] ],
+                                  File1, FooOff1, Size1, []) || X <- [0,1,2] ],
-        ok = machi_flu1_client:write_chunk(Host, PortBase+0, NSInfo, EpochID,
+        ok = machi_flu1_client:write_chunk(Host, PortBase+0, EpochID,
-                                           File1, FooOff1, Chunk1, NoCSum),
+                                           File1, FooOff1, Chunk1),
-        {ok, {[{File1, FooOff1, Chunk1, _}=_YY], []}} =
+        {ok, {[{_, FooOff1, Chunk1, _}], []}} =
-            machi_flu1_client:read_chunk(Host, PortBase+0, NSInfo, EpochID,
+            machi_flu1_client:read_chunk(Host, PortBase+0, EpochID,
-                                         File1, FooOff1, Size1, undefined),
+                                         File1, FooOff1, Size1, []),
-        {ok, {[{File1, FooOff1, Chunk1, _}], []}} =
+        {ok, {[{_, FooOff1, Chunk1, _}], []}} =
-            machi_cr_client:read_chunk(C1, NSInfo, File1, FooOff1, Size1, undefined),
+            machi_cr_client:read_chunk(C1, File1, FooOff1, Size1, []),
        [?assertMatch({X,{ok, {[{_, FooOff1, Chunk1, _}], []}}},
                      {X,machi_flu1_client:read_chunk(
-                                 Host, PortBase+X, NSInfo, EpochID,
+                                 Host, PortBase+X, EpochID,
-                                 File1, FooOff1, Size1, undefined)})
+                                 File1, FooOff1, Size1, [])})
         || X <- [0,1,2] ],
        %% Test read repair: Manually write to middle, then same checking.
        FooOff2 = Off1 + (2*1024*1024),
        Chunk2 = <<"Middle repair chunk">>,
        Size2 = size(Chunk2),
-        ok = machi_flu1_client:write_chunk(Host, PortBase+1, NSInfo, EpochID,
+        ok = machi_flu1_client:write_chunk(Host, PortBase+1, EpochID,
-                                           File1, FooOff2, Chunk2, NoCSum),
+                                           File1, FooOff2, Chunk2),
-        {ok, {[{File1, FooOff2, Chunk2, _}], []}} =
+        {ok, {[{_, FooOff2, Chunk2, _}], []}} =
-            machi_cr_client:read_chunk(C1, NSInfo, File1, FooOff2, Size2, undefined),
+            machi_cr_client:read_chunk(C1, File1, FooOff2, Size2, []),
-        [{X,{ok, {[{File1, FooOff2, Chunk2, _}], []}}} =
+        [{X,{ok, {[{_, FooOff2, Chunk2, _}], []}}} =
             {X,machi_flu1_client:read_chunk(
-                  Host, PortBase+X, NSInfo, EpochID,
+                  Host, PortBase+X, EpochID,
-                  File1, FooOff2, Size2, undefined)} || X <- [0,1,2] ],
+                  File1, FooOff2, Size2, [])} || X <- [0,1,2] ],
        %% Misc API smoke & minor regression checks
-        {error, bad_arg} = machi_cr_client:read_chunk(C1, NSInfo, <<"no">>,
+        {error, bad_arg} = machi_cr_client:read_chunk(C1, <<"no">>,
-                                                      999999999, 1, undefined),
+                                                      999999999, 1, []),
-        {ok, {[{File1,Off1,Chunk1,_}, {File1,FooOff1,Chunk1,_}, {File1,FooOff2,Chunk2,_}],
+        {ok, {[{_,Off1,Chunk1,_}, {_,FooOff1,Chunk1,_}, {_,FooOff2,Chunk2,_}],
              []}} =
-            machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, 88888888, undefined),
+            machi_cr_client:read_chunk(C1, File1, Off1, 88888888, []),
        %% Checksum list return value is a primitive binary().
        {ok, KludgeBin} = machi_cr_client:checksum_list(C1, File1),
        true = is_binary(KludgeBin),
        {error, bad_arg} = machi_cr_client:checksum_list(C1, <<"!!!!">>),
-        io:format(user, "\nFiles = ~p\n", [machi_cr_client:list_files(C1)]),
+        %% Exactly one file right now
        %% Exactly one file right now, e.g.,
        %% {ok,[{2098202,<<"pre^b144ef13-db4d-4c9f-96e7-caff02dc754f^1">>}]}
        {ok, [_]} = machi_cr_client:list_files(C1),
-        %% Go back and test append_chunk() + extra and write_chunk()
+        %% Go back and test append_chunk_extra() and write_chunk()
        Chunk10 = <<"It's a different chunk!">>,
        Size10 = byte_size(Chunk10),
        Extra10 = 5,
        Opts1 = #append_opts{chunk_extra=Extra10*Size10},
        {ok, {Off10,Size10,File10}} =
-            machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk10,
+            machi_cr_client:append_chunk_extra(C1, Prefix, Chunk10,
-                                         NoCSum, Opts1),
+                                               Extra10 * Size10),
        {ok, {[{_, Off10, Chunk10, _}], []}} =
-            machi_cr_client:read_chunk(C1, NSInfo, File10, Off10, Size10, undefined),
+            machi_cr_client:read_chunk(C1, File10, Off10, Size10, []),
        [begin
             Offx = Off10 + (Seq * Size10),
             %% TODO: uncomment written/not_written enforcement is available.
-             %% {error,not_written} = machi_cr_client:read_chunk(C1, NSInfo, File10,
+             %% {error,not_written} = machi_cr_client:read_chunk(C1, File10,
             %%                                                  Offx, Size10),
             {ok, {Offx,Size10,File10}} =
-                 machi_cr_client:write_chunk(C1, NSInfo, File10, Offx, Chunk10, NoCSum),
+                 machi_cr_client:write_chunk(C1, File10, Offx, Chunk10),
             {ok, {[{_, Offx, Chunk10, _}], []}} =
-                 machi_cr_client:read_chunk(C1, NSInfo, File10, Offx, Size10, undefined)
+                 machi_cr_client:read_chunk(C1, File10, Offx, Size10, [])
         end || Seq <- lists:seq(1, Extra10)],
        {ok, {Off11,Size11,File11}} =
-            machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk10, NoCSum),
+            machi_cr_client:append_chunk(C1, Prefix, Chunk10),
-        %% %% Double-check that our reserved extra bytes were really honored!
+        %% Double-check that our reserved extra bytes were really honored!
-        %% true = (Off11 > (Off10 + (Extra10 * Size10))),
+        true = (Off11 > (Off10 + (Extra10 * Size10))),
 io:format(user, "\nFiles = ~p\n", [machi_cr_client:list_files(C1)]),
        ok
    after
@ -217,7 +205,7 @@ io:format(user, "\nFiles = ~p\n", [machi_cr_client:list_files(C1)]),
 witness_smoke_test_() -> {timeout, 1*60, fun() -> witness_smoke_test2() end}.
 witness_smoke_test2() ->
-    SupPid = case machi_sup:start_link() of
+    SupPid = case machi_flu_sup:start_link() of 
                {ok, P} -> P;
                {error, {already_started, P1}} -> P1;
                Other -> error(Other)
@ -227,8 +215,6 @@ witness_smoke_test2() ->
    try
        Prefix = <<"pre">>,
        Chunk1 = <<"yochunk">>,
        NSInfo = undefined,
        NoCSum = <<>>,
        Host = "localhost",
        PortBase = 64444,
        Os = [{ignore_stability_time, true}, {active_mode, false},
@ -238,15 +224,14 @@ witness_smoke_test2() ->
        %% Whew ... ok, now start some damn tests.
        {ok, C1} = machi_cr_client:start_link([P || {_,P}<-orddict:to_list(D)]),
-        {ok, _} = machi_cr_client:append_chunk(C1, NSInfo, Prefix,
+        {ok, _} = machi_cr_client:append_chunk(C1, Prefix, Chunk1),
                                               Chunk1, NoCSum),
        {ok, {Off1,Size1,File1}} =
-            machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum),
+            machi_cr_client:append_chunk(C1, Prefix, Chunk1),
-        BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, "foo")},
+        Chunk1_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, Chunk1},
        {error, bad_checksum} =
-            machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, BadCSum),
+            machi_cr_client:append_chunk(C1, Prefix, Chunk1_badcs),
        {ok, {[{_, Off1, Chunk1, _}], []}} =
-            machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, Size1, undefined),
+            machi_cr_client:read_chunk(C1, File1, Off1, Size1, []),
        %% Stop 'b' and let the chain reset.
        ok = machi_flu_psup:stop_flu_package(b),
@ -259,25 +244,24 @@ witness_smoke_test2() ->
        %% Let's wedge OurWitness and see what happens: timeout/partition.
        #p_srvr{name=WitName, address=WitA, port=WitP} =
            orddict:fetch(OurWitness, D),
-        {ok, {false, EpochID2,_,_}} = machi_flu1_client:wedge_status(WitA, WitP),
+        {ok, {false, EpochID2}} = machi_flu1_client:wedge_status(WitA, WitP),
        machi_flu1:wedge_myself(WitName, EpochID2),
        case machi_flu1_client:wedge_status(WitA, WitP) of
-            {ok, {true,  EpochID2,_,_}} ->
+            {ok, {true,  EpochID2}} ->
                ok;
-            {ok, {false,  EpochID2,_,_}} ->
+            {ok, {false,  EpochID2}} ->
                %% This is racy.  Work around it by sleeping a while.
                timer:sleep(6*1000),
-                {ok, {true,  EpochID2,_,_}} =
+                {ok, {true,  EpochID2}} =
                    machi_flu1_client:wedge_status(WitA, WitP)
        end,
        %% Chunk1 is still readable: not affected by wedged witness head.
        {ok, {[{_, Off1, Chunk1, _}], []}} =
-            machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, Size1, undefined),
+            machi_cr_client:read_chunk(C1, File1, Off1, Size1, []),
        %% But because the head is wedged, an append will fail.
        {error, partition} =
-            machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum,
+            machi_cr_client:append_chunk(C1, Prefix, Chunk1, 1*1000),
                                         #append_opts{}, 1*1000),
        %% The witness's wedge status should cause timeout/partition
        %% for write_chunk also.
@ -286,7 +270,7 @@ witness_smoke_test2() ->
        File10 = File1,
        Offx = Off1 + (1 * Size10),
        {error, partition} =
-            machi_cr_client:write_chunk(C1, NSInfo, File10, Offx, Chunk10, NoCSum, 1*1000),
+            machi_cr_client:write_chunk(C1, File10, Offx, Chunk10, 1*1000),
        ok
    after
--- a/test/machi_csum_table_test.erl
+++ b/test/machi_csum_table_test.erl
@ -2,31 +2,26 @@
 -compile(export_all).
 -include_lib("eunit/include/eunit.hrl").
-define(HDR, {0, 1024, none}).
+-define(HDR, {0, 1024, <<0>>}).
 cleanup(Dir) ->
    os:cmd("rm -rf " ++ Dir).
 smoke_test() ->
    Filename = "./temp-checksum-dumb-file",
-    _ = cleanup(Filename),
+    _ = file:delete(Filename),
    {ok, MC} = machi_csum_table:open(Filename, []),
-    ?assertEqual([{1024, infinity}],
+    [{1024, infinity}] = machi_csum_table:calc_unwritten_bytes(MC),
                 machi_csum_table:calc_unwritten_bytes(MC)),
    Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
    [] = machi_csum_table:find(MC, Offset, Size),
    ok = machi_csum_table:write(MC, Offset, Size, Checksum),
    [{1024, 40}, {1098, infinity}] = machi_csum_table:calc_unwritten_bytes(MC),
-    ?assertEqual([Entry], machi_csum_table:find(MC, Offset, Size)),
+    [Entry] = machi_csum_table:find(MC, Offset, Size),
-    ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined),
+    ok = machi_csum_table:trim(MC, Offset, Size),
-    ?assertEqual([{Offset, Size, trimmed}],
+    [{Offset, Size, trimmed}] = machi_csum_table:find(MC, Offset, Size),
                 machi_csum_table:find(MC, Offset, Size)),
    ok = machi_csum_table:close(MC),
    ok = machi_csum_table:delete(MC).
 close_test() ->
    Filename = "./temp-checksum-dumb-file-2",
-    _ = cleanup(Filename),
+    _ = file:delete(Filename),
    {ok, MC} = machi_csum_table:open(Filename, []),
    Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
    [] = machi_csum_table:find(MC, Offset, Size),
@ -36,33 +31,33 @@ close_test() ->
    {ok, MC2} = machi_csum_table:open(Filename, []),
    [Entry] = machi_csum_table:find(MC2, Offset, Size),
-    ok = machi_csum_table:trim(MC2, Offset, Size, undefined, undefined),
+    ok = machi_csum_table:trim(MC2, Offset, Size),
    [{Offset, Size, trimmed}] = machi_csum_table:find(MC2, Offset, Size),
    ok = machi_csum_table:delete(MC2).
 smoke2_test() ->
    Filename = "./temp-checksum-dumb-file-3",
-    _ = cleanup(Filename),
+    _ = file:delete(Filename),
    {ok, MC} = machi_csum_table:open(Filename, []),
    Entry = {Offset, Size, Checksum} = {1025, 10, <<"deadbeef">>},
    ok = machi_csum_table:write(MC, Offset, Size, Checksum),
-    ?assertEqual([], machi_csum_table:find(MC, 0, 0)),
+    [] = machi_csum_table:find(MC, 0, 0),
-    ?assertEqual([?HDR], machi_csum_table:find(MC, 0, 1)),
+    [?HDR] = machi_csum_table:find(MC, 0, 1),
    [Entry] = machi_csum_table:find(MC, Offset, Size),
    [?HDR] = machi_csum_table:find(MC, 1, 1024),
-    ?assertEqual([?HDR, Entry],
+    [?HDR, Entry] = machi_csum_table:find(MC, 1023, 1024),
                 machi_csum_table:find(MC, 1023, 1024)),
    [Entry] = machi_csum_table:find(MC, 1024, 1024),
    [Entry] = machi_csum_table:find(MC, 1025, 1024),
-    ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined),
+    ok = machi_csum_table:trim(MC, Offset, Size),
    [{Offset, Size, trimmed}] = machi_csum_table:find(MC, Offset, Size),
    ok = machi_csum_table:close(MC),
    ok = machi_csum_table:delete(MC).
 smoke3_test() ->
    Filename = "./temp-checksum-dumb-file-4",
-    _ = cleanup(Filename),
+    _ = file:delete(Filename),
    {ok, MC} = machi_csum_table:open(Filename, []),
    Scenario =
        [%% Command, {Offset, Size, Csum}, LeftNeighbor, RightNeibor
@ -112,20 +107,5 @@ smoke3_test() ->
    ok = machi_csum_table:close(MC),
    ok = machi_csum_table:delete(MC).
 %% TODO: add quickcheck test here
 %% Previous implementation
 -spec all_trimmed2(machi_csum_table:table(),
                   non_neg_integer(), non_neg_integer()) -> boolean().
 all_trimmed2(CsumT, Left, Right) ->
    Chunks = machi_csum_table:find(CsumT, Left, Right),
    runthru(Chunks, Left, Right).
 %% @doc make sure all trimmed chunks are continously chained
 %% TODO: test with EQC
 runthru([], Pos, Pos) -> true;
 runthru([], Pos0, Pos) when Pos0 < Pos -> false;
 runthru([{Offset0, Size0, trimmed}|T], Offset, Pos) when Offset0 =< Offset ->
    runthru(T, Offset0+Size0, Pos);
 runthru(_L, _O, _P) ->
    false.
--- a/test/machi_file_proxy_eqc.erl
+++ b/test/machi_file_proxy_eqc.erl
@ -31,18 +31,13 @@
 -define(QC_OUT(P),
        eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)).
 -define(TESTDIR, "./eqc").
 %% EUNIT TEST DEFINITION
 eqc_test_() ->
-    PropTimeout = case os:getenv("EQC_TIME") of
+    {timeout, 60,
                      false -> 30;
                      V     -> list_to_integer(V)
                  end,
    {timeout, PropTimeout*2 + 30,
     {spawn,
      [
-       ?_assertEqual(true, eqc:quickcheck(eqc:testing_time(PropTimeout, ?QC_OUT(prop_ok()))))
+        {timeout, 30, ?_assertEqual(true, eqc:quickcheck(eqc:testing_time(15, ?QC_OUT(prop_ok()))))}
      ]
     }}.
@ -93,14 +88,12 @@ data_with_csum(Limit) ->
 intervals([]) ->
    [];
 intervals([N]) ->
-    [{N, choose(1,1)}];
+    [{N, choose(1,150)}];
 intervals([A,B|T]) ->
-    [{A, oneof([choose(1, B-A), B-A])}|intervals([B|T])].
+    [{A, choose(1, B-A)}|intervals([B|T])].
 interval_list() ->
-    ?LET(L,
+    ?LET(L, list(choose(1024, 4096)), intervals(lists:usort(L))).
         oneof([list(choose(1025, 1033)), list(choose(1024, 4096))]),
         intervals(lists:usort(L))).
 shuffle_interval() ->
    ?LET(L, interval_list(), shuffle(L)).
@ -110,100 +103,40 @@ get_written_interval(L) ->
 %% INITIALIZATION
-record(state, {pid, prev_extra = 0,
+-record(state, {pid, prev_extra = 0, planned_writes=[], written=[]}).
                filename = undefined,
                planned_writes=[],
                planned_trims=[],
                written=[],
                trimmed=[]}).
-initial_state() ->
+initial_state() -> #state{written=[{0,1024}]}.
-    {_, _, MS} = os:timestamp(),
+initial_state(I) -> #state{written=[{0,1024}], planned_writes=I}.
    Filename = test_server:temp_name("eqc_data") ++ "." ++ integer_to_list(MS),
    #state{filename=Filename, written=[{0,1024}]}.
 initial_state(I, T) ->
    S=initial_state(),
    S#state{written=[{0,1024}],
            planned_writes=I,
            planned_trims=T}.
 weight(_S, rewrite) -> 1;
 weight(_S, _) -> 2.
 %% HELPERS
-get_overlaps(_Offset, _Len, [], Acc) -> lists:reverse(Acc);
+%% check if an operation is permitted based on whether a write has
-get_overlaps(Offset, Len, [{Pos, Sz} = Ck|T], Acc0)
+%% occurred
-%% Overlap judgement differnt from the one in machi_csum_table
+check_writes(_Op, [], _Off, _L) ->
-%% [a=Offset, b), [x=Pos, y) ...
+    false;
-  when
+check_writes(_Op, [{Pos, Sz}|_T], Off, L) when Pos == Off 
-      %% a =< x && x < b && b =< y
+                                          andalso Sz == L ->
-      (Offset =< Pos andalso Pos < Offset + Len andalso Offset + Len =< Pos + Sz) orelse
+    mostly_true;
-      %% a =< x          && y < b
+check_writes(read, [{Pos, Sz}|_T], Off, L) when Off >= Pos 
-      (Offset =< Pos andalso Pos + Sz < Offset + Len) orelse
+                                          andalso Off < (Pos + Sz) 
-      %% x < a && a < y  && y =< b
+                                          andalso Sz >= ( L - ( Off - Pos ) ) ->
-      (Pos < Offset andalso Offset < Pos + Sz andalso Pos + Sz =< Offset + Len) orelse
+    true;
-      %% x < a           && b < y
+check_writes(write, [{Pos, Sz}|_T], Off, L) when ( Off + L ) > Pos 
-      (Pos < Offset + Len andalso Offset + Len < Pos + Sz) ->
+                                                 andalso Off < (Pos + Sz) ->
-    get_overlaps(Offset, Len, T, [Ck|Acc0]);
+    true;
-get_overlaps(Offset, Len, [_Ck|T], Acc0) ->
+check_writes(Op, [_H|T], Off, L) ->
-    get_overlaps(Offset, Len, T, Acc0).
+    check_writes(Op, T, Off, L).
 %% Inefficient but simple easy code to verify by eyes - returns all
 %% bytes that fits in (Offset, Len)
 chop(Offset, Len, List) ->
    ChopLeft = fun({Pos, Sz}) when Pos < Offset andalso Offset =< Pos + Sz ->
                       {Offset, Sz + Pos - Offset};
                  ({Pos, Sz}) when Offset =< Pos andalso Pos + Sz < Offset + Len ->
                       {Pos, Sz};
                  ({Pos, _Sz}) when Offset =< Pos ->
                       {Pos, Offset + Len - Pos}
               end,
    ChopRight = fun({Pos, Sz}) when Offset + Len < Pos + Sz ->
                        {Pos, Offset + Len - Pos};
                   ({Pos, Sz}) ->
                        {Pos, Sz}
                end,
    Filter0 = fun({_, 0}) -> false;
                 (Other) -> {true, Other} end,
    lists:filtermap(fun(E) -> Filter0(ChopRight(ChopLeft(E))) end,
                    List).
 %% Returns all bytes that are at left side of the Offset
 chopped_left(_Offset, []) -> undefined;
 chopped_left(Offset, [{Pos,_Sz}|_]) when Pos < Offset ->
    {Pos, Offset - Pos};
 chopped_left(_, _) ->
    undefined.
 chopped_right(_Offset, []) -> undefined;
 chopped_right(Offset, List) ->
    {Pos, Sz} = lists:last(List),
    if Offset < Pos + Sz ->
            {Offset, Pos + Sz - Offset};
       true ->
            undefined
    end.
 cleanup_chunk(Offset, Length, ChunkList) ->
    Overlaps = get_overlaps(Offset, Length, ChunkList, []),
    NewCL0 = lists:foldl(fun lists:delete/2,
                         ChunkList, Overlaps),
    NewCL1 = case chopped_left(Offset, Overlaps) of
                 undefined -> NewCL0;
                 LeftRemain -> [LeftRemain|NewCL0]
             end,
    NewCL2 = case chopped_right(Offset+Length, Overlaps) of
                 undefined -> NewCL1;
                 RightRemain -> [RightRemain|NewCL1]
             end,
    lists:sort(NewCL2).
 is_error({error, _}) -> true;
 is_error({error, _, _}) -> true;
 is_error(Other) -> {expected_ERROR, Other}.
 probably_error(ok) -> true;
 probably_error(V) -> is_error(V).
 is_ok({ok, _, _}) -> true;
 is_ok(ok) -> true;
 is_ok(Other) -> {expected_OK, Other}.
@ -211,10 +144,11 @@ is_ok(Other) -> {expected_OK, Other}.
 get_offset({ok, _Filename, Offset}) -> Offset;
 get_offset(_) -> error(badarg).
-last_byte([]) -> 0;
+offset_valid(Offset, Extra, L) ->
-last_byte(L0) ->
+    {Pos, Sz} = lists:last(L),
-    L1 = lists:map(fun({Pos, Sz}) -> Pos + Sz end, L0),
+    Offset == Pos + Sz + Extra.
-    lists:last(lists:sort(L1)).
+
 -define(TESTDIR, "./eqc").
 cleanup() ->
    [begin
@ -228,17 +162,19 @@ cleanup() ->
 %% start
 start_pre(S) ->
-    S#state.pid =:= undefined.
+    S#state.pid == undefined.
 start_command(S) ->
    {call, ?MODULE, start, [S]}.
-start(#state{filename=File}) ->
+start(_S) ->
    {_, _, MS} = os:timestamp(),
    File = test_server:temp_name("eqc_data") ++ "." ++ integer_to_list(MS),
    {ok, Pid} = machi_file_proxy:start_link(some_flu, File, ?TESTDIR),
    unlink(Pid),
    Pid.
-start_next(S, Pid, _) ->
+start_next(S, Pid, _Args) ->
    S#state{pid = Pid}.
 %% read
@ -247,34 +183,31 @@ read_pre(S) ->
    S#state.pid /= undefined.
 read_args(S) ->
-    [S#state.pid, oneof([offset(), big_offset()]), len()].
+    [S#state.pid, offset(), len()].
 read_ok(S, Off, L) ->
    case S#state.written of
        [{0, 1024}] -> false;
        W -> check_writes(read, W, Off, L)
    end.
 read_post(S, [_Pid, Off, L], Res) ->
-    Written = get_overlaps(Off, L, S#state.written, []),
+    case read_ok(S, Off, L) of
-    Chopped = chop(Off, L, Written),
+        true -> is_ok(Res);
-    Trimmed = get_overlaps(Off, L, S#state.trimmed, []),
+        mostly_true -> is_ok(Res);
-    Eof = lists:max([Pos+Sz||{Pos,Sz}<-S#state.written]),
+        false -> is_error(Res)
    case Res of
        {ok, {Written0, Trimmed0}} ->
            Written1 = lists:map(fun({_, Pos, Chunk, _}) ->
                                         {Pos, iolist_size(Chunk)}
                                 end, Written0),
            Trimmed1 = lists:map(fun({_, Pos, Sz}) -> {Pos, Sz} end, Trimmed0),
            Chopped =:= Written1
                andalso Trimmed =:= Trimmed1;
        %% TODO: such response are ugly, rethink the SPEC
        {error, not_written} when Eof < Off + L ->
            true;
        {error, not_written} when Chopped =:= [] andalso Trimmed =:= [] ->
            true;
        _Other ->
            is_error(Res)
    end.
 read_next(S, _Res, _Args) -> S.
 read(Pid, Offset, Length) ->
-    machi_file_proxy:read(Pid, Offset, Length, [{needs_trimmed, true}]).
+    case machi_file_proxy:read(Pid, Offset, Length) of
        {ok, {Chunks, _}} ->
            [{_, Offset, Data, Csum}] = Chunks,
            {ok, Data, Csum};
        E ->
            E
    end.
 %% write
@ -283,7 +216,6 @@ write_pre(S) ->
 %% do not allow writes with empty data
 write_pre(_S, [_Pid, _Extra, {<<>>, _Tag, _Csum}]) ->
    ?assert(false),
    false;
 write_pre(_S, _Args) ->
    true.
@ -292,18 +224,28 @@ write_args(S) ->
    {Off, Len} = hd(S#state.planned_writes),
    [S#state.pid, Off, data_with_csum(Len)].
-write_post(S, [_Pid, Off, {Bin, _Tag, _Csum}] = _Args, Res) ->
+write_ok(_S, [_Pid, Off, _Data]) when Off < 1024 -> false;
 write_ok(S, [_Pid, Off, {Bin, _Tag, _Csum}]) ->
    Size = iolist_size(Bin),
-    case {get_overlaps(Off, Size, S#state.written, []),
+    %% Check writes checks if a byte range is *written*
-          get_overlaps(Off, Size, S#state.trimmed, [])} of
+    %% So writes are ok IFF they are NOT written, so
-        {[], []} ->
+    %% we want not check_writes/3 to be true.
-            %% No overlap neither with written ranges nor trimmed
+    check_writes(write, S#state.written, Off, Size).
-            %% ranges; OK to write things.
+
-            eq(Res, ok);
+write_post(S, Args, Res) ->
-        {_, _} ->
+    case write_ok(S, Args) of
-            %% overlap found in either or both at written or at
+        %% false means this range has NOT been written before, so
-            %% trimmed ranges; can't write.
+        %% it should succeed
-            is_error(Res)
+        false -> eq(Res, ok);
        %% mostly true means we've written this range before BUT
        %% as a special case if we get a call to write the EXACT
        %% same data that's already on the disk, we return "ok"
        %% instead of {error, written}.
        mostly_true -> probably_error(Res); 
        %% If we get true, then we've already written this section 
        %% or a portion of this range to disk and should return an 
        %% error.
        true -> is_error(Res)
    end.
 write_next(S, Res, [_Pid, Offset, {Bin, _Tag, _Csum}]) ->
@ -344,40 +286,24 @@ append_next(S, Res, [_Pid, Extra, {Bin, _Tag, _Csum}]) ->
    case is_ok(Res) of
        true -> 
            Offset = get_offset(Res),
-            S#state{prev_extra = Extra,
+            true = offset_valid(Offset, S#state.prev_extra, S#state.written),
-                    written = lists:sort(S#state.written ++ [{Offset, iolist_size(Bin)}])};
+            S#state{prev_extra = Extra, written = lists:sort(S#state.written ++ [{Offset, iolist_size(Bin)}])};
-        _Other ->
+        _ ->
            S
    end.
 %% appends should always succeed unless the disk is full 
 %% or there's a hardware failure.
-append_post(S, _Args, Res) ->
+append_post(_S, _Args, Res) ->
-    case is_ok(Res) of
+    true == is_ok(Res).
        true ->
            Offset = get_offset(Res),
            case erlang:max(last_byte(S#state.written),
                            last_byte(S#state.trimmed)) + S#state.prev_extra of
                Offset ->
                    true;
                UnexpectedByte ->
                    {wrong_offset_after_append,
                     {Offset, UnexpectedByte},
                     {S#state.written, S#state.prev_extra}}
            end;
        Error ->
            Error
    end.
 %% rewrite
 rewrite_pre(S) ->
-    S#state.pid /= undefined andalso
+    S#state.pid /= undefined andalso S#state.written /= [].
        (S#state.written ++ S#state.trimmed) /= [] .
 rewrite_args(S) ->
-    ?LET({Off, Len},
+    ?LET({Off, Len}, get_written_interval(S#state.written),
         get_written_interval(S#state.written ++ S#state.trimmed),
    [S#state.pid, Off, data_with_csum(Len)]).
 rewrite(Pid, Offset, {Bin, Tag, Csum}) ->
@ -391,88 +317,18 @@ rewrite_post(_S, _Args, Res) ->
 rewrite_next(S, _Res, _Args) ->
    S#state{prev_extra = 0}.
 %% trim
 trim_pre(S) ->
    S#state.pid /= undefined andalso S#state.planned_trims /= [].
 trim_args(S) ->
    {Offset, Length} = hd(S#state.planned_trims),
    [S#state.pid, Offset, Length].
 trim(Pid, Offset, Length) ->
    machi_file_proxy:trim(Pid, Offset, Length, false).
 trim_post(_S, [_Pid, _Offset, _Length], ok) ->
    true;
 trim_post(_S, [_Pid, _Offset, _Length], _Res) ->
    false.
 trim_next(S, Res, [_Pid, Offset, Length]) ->
    S1 = case is_ok(Res) of
             true ->
                 NewWritten = cleanup_chunk(Offset, Length, S#state.written),
                 Trimmed1 = cleanup_chunk(Offset, Length, S#state.trimmed),
                 NewTrimmed = lists:sort([{Offset, Length}|Trimmed1]),
                 S#state{trimmed=NewTrimmed,
                         written=NewWritten};
             _Other ->
                 S
         end,
    S1#state{prev_extra=0,
             planned_trims=tl(S#state.planned_trims)}.
 stop_pre(S) ->
    S#state.pid /= undefined.
 stop_args(S) ->
    [S#state.pid].
 stop(Pid) ->
    catch machi_file_proxy:stop(Pid).
 stop_post(_, _, _) -> true.
 stop_next(S, _, _) ->
    S#state{pid=undefined, prev_extra=0}.
 %% Property
 prop_ok() ->
  cleanup(),
-    ?FORALL({I, T},
+  ?FORALL(I, shuffle_interval(),
-            {shuffle_interval(), shuffle_interval()},
+    ?FORALL(Cmds, parallel_commands(?MODULE, initial_state(I)),
            ?FORALL(Cmds, parallel_commands(?MODULE, initial_state(I, T)),
    begin
      {H, S, Res} = run_parallel_commands(?MODULE, Cmds),
                        cleanup(),
        pretty_commands(?MODULE, Cmds, {H, S, Res},
        aggregate(command_names(Cmds), Res == ok))
-                    end)).
+    end)
-
+  ).
 %% Test for tester functions
 chopper_test_() ->
    [?_assertEqual([{0, 1024}],
                   get_overlaps(1, 1, [{0, 1024}], [])),
     ?_assertEqual([],
                   get_overlaps(10, 5, [{9, 1}, {15, 1}], [])),
     ?_assertEqual([{9,2},{14,1}],
                   get_overlaps(10, 5, [{9, 2}, {14, 1}], [])),
     ?_assertEqual([],       chop(0, 0, [{0,2}])),
     ?_assertEqual([{0, 1}], chop(0, 1, [{0,2}])),
     ?_assertEqual([],       chop(1, 0, [{0,2}])),
     ?_assertEqual([{1, 1}], chop(1, 1, [{0,2}])),
     ?_assertEqual([{1, 1}], chop(1, 2, [{0,2}])),
     ?_assertEqual([],       chop(2, 1, [{0,2}])),
     ?_assertEqual([],       chop(2, 2, [{0,2}])),
     ?_assertEqual([{1, 1}], chop(1, 3, [{0,2}])),
     ?_assertError(_,  chop(3, 1, [{0,2}])),
     ?_assertEqual([], chop(2, 3, [{0,2}])),
     ?_assertEqual({0, 1}, chopped_left(1, [{0, 1024}])),
     ?_assertEqual([{0, 1}, {2, 1022}], cleanup_chunk(1, 1, [{0, 1024}])),
     ?_assertEqual([{2, 1022}], cleanup_chunk(0, 2, [{0, 1}, {2, 1022}])),
     ?_assert(true)
    ].
 -endif. % EQC
 -endif. % TEST
--- a/test/machi_file_proxy_test.erl
+++ b/test/machi_file_proxy_test.erl
@ -38,7 +38,7 @@ clean_up_data_dir(DataDir) ->
 -ifndef(PULSE).
 -define(TESTDIR, "./t").
-define(HYOOGE, 75 * 1024 * 1024). % 75 MBytes
+-define(HYOOGE, 1 * 1024 * 1024 * 1024). % 1 long GB
 random_binary_single() ->
    %% OK, I guess it's not that random...
@ -76,50 +76,32 @@ random_binary(Start, End) ->
            binary:part(random_binary_single(), Start, End)
    end.
 setup() ->
    {ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR),
    Pid.
 teardown(Pid) ->
    catch machi_file_proxy:stop(Pid).
 machi_file_proxy_test_() ->
    clean_up_data_dir(?TESTDIR),
-    {setup,
+    {ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR),
     fun setup/0,
     fun teardown/1,
     fun(Pid) ->
    [
     ?_assertEqual({error, bad_arg}, machi_file_proxy:read(Pid, -1, -1)),
     ?_assertEqual({error, bad_arg}, machi_file_proxy:write(Pid, -1, <<"yo">>)),
     ?_assertEqual({error, bad_arg}, machi_file_proxy:append(Pid, [], -1, <<"krep">>)),
-              ?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1)),
+     ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1, 1)),
     ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, 1)),
-              ?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1024)),
+     ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1, 1024)),
     ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, ?HYOOGE)),
     ?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, ?HYOOGE, 1)),
-              {timeout, 10,
+     ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, random_binary(0, ?HYOOGE))),
               ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, random_binary(0, ?HYOOGE)))},
     ?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))),
     ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1024, <<"fail">>)),
     ?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, <<"fail">>)),
     ?_assertMatch({ok, {[{_, _, _, _}], []}}, machi_file_proxy:read(Pid, 1025, 1000)),
     ?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, [], 1024, <<"mind the gap">>)),
-              ?_assertEqual(ok, machi_file_proxy:write(Pid, 2060, [], random_binary(0, 1024)))
+     ?_assertEqual(ok, machi_file_proxy:write(Pid, 2060, [], random_binary(0, 1024))),
-             ]
+     ?_assertException(exit, {normal, _}, machi_file_proxy:stop(Pid))
-     end}.
+    ].
 multiple_chunks_read_test_() ->
    clean_up_data_dir(?TESTDIR),
-    {setup,
+    {ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR),
     fun setup/0,
     fun teardown/1,
     fun(Pid) ->
    [
              ?_assertEqual(ok, machi_file_proxy:trim(Pid, 0, 1, false)),
              ?_assertMatch({ok, {[], [{"test", 0, 1}]}},
                            machi_file_proxy:read(Pid, 0, 1,
                                                  #read_opts{needs_trimmed=true})),
     ?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))),
     ?_assertEqual(ok, machi_file_proxy:write(Pid, 10000, <<"fail">>)),
     ?_assertEqual(ok, machi_file_proxy:write(Pid, 20000, <<"fail">>)),
@ -132,11 +114,9 @@ multiple_chunks_read_test_() ->
                          {"test", 30000, <<"fail">>, _},
                          {"test", 530000, <<"fail">>, _}], []}},
                   machi_file_proxy:read(Pid, 1024, 530000)),
-              ?_assertMatch({ok, {[{"test", 1, _, _}], [{"test", 0, 1}]}},
+     ?_assertException(exit, {normal, _}, machi_file_proxy:stop(Pid))
-                            machi_file_proxy:read(Pid, 0, 1024,
+    ].
-                                                  #read_opts{needs_trimmed=true}))
+
             ]
     end}.
 -endif. % !PULSE
 -endif. % TEST.
--- a/test/machi_flu1_test.erl
+++ b/test/machi_flu1_test.erl
@ -30,22 +30,6 @@
 -define(FLU, machi_flu1).
 -define(FLU_C, machi_flu1_client).
 get_env_vars(App, Ks) ->
    Raw = [application:get_env(App, K) || K <- Ks],
    Old = lists:zip(Ks, Raw),
    {App, Old}.
 clean_up_env_vars({App, Old}) ->
    [case Res of
         undefined ->
             application:unset_env(App, K);
         {ok, V} ->
             application:set_env(App, K, V)
     end || {K, Res} <- Old].
 filter_env_var({ok, V}) -> V;
 filter_env_var(Else)    -> Else.
 clean_up_data_dir(DataDir) ->
    [begin
         Fs = filelib:wildcard(DataDir ++ Glob),
@ -85,46 +69,47 @@ maybe_start_sup() ->
        Pid -> Pid
    end.
 -ifndef(PULSE).
 flu_smoke_test() ->
    Host = "localhost",
-    TcpPort = 12957,
+    TcpPort = 32957,
    DataDir = "./data",
    NSInfo = undefined,
    NoCSum = <<>>,
    Prefix = <<"prefix!">>,
    BadPrefix = BadFile = "no/good",
    W_props = [{initial_wedged, false}],
-    {_, _, _} = machi_test_util:start_flu_package(smoke_flu, TcpPort, DataDir, W_props),
+    start_flu_package(smoke_flu, TcpPort, DataDir, W_props),
    try
        Msg = "Hello, world!",
        Msg = ?FLU_C:echo(Host, TcpPort, Msg),
-        {error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort,"does-not-exist"),
+        {error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort,
-        {error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort, BadFile),
+                                                     ?DUMMY_PV1_EPOCH,
                                                     "does-not-exist"),
        {error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort,
                                                ?DUMMY_PV1_EPOCH, BadFile),
        {ok, []} = ?FLU_C:list_files(Host, TcpPort, ?DUMMY_PV1_EPOCH),
-        {ok, {false, _,_,_}} = ?FLU_C:wedge_status(Host, TcpPort),
+        {ok, {false, _}} = ?FLU_C:wedge_status(Host, TcpPort),
        Chunk1 = <<"yo!">>,
-        {ok, {Off1,Len1,File1}} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
+        {ok, {Off1,Len1,File1}} = ?FLU_C:append_chunk(Host, TcpPort,
                                                      ?DUMMY_PV1_EPOCH,
-                                                      Prefix, Chunk1, NoCSum),
+                                                      Prefix, Chunk1),
-        {ok, {[{_, Off1, Chunk1, _}], _}} = ?FLU_C:read_chunk(Host, TcpPort,
+        {ok, {[{_, Off1, Chunk1, _}], _}} = ?FLU_C:read_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH,
-                                                       NSInfo, ?DUMMY_PV1_EPOCH,
+                                                         File1, Off1, Len1, []),
-                                                       File1, Off1, Len1,
+        {ok, KludgeBin} = ?FLU_C:checksum_list(Host, TcpPort,
-                                                       noopt),
+                                               ?DUMMY_PV1_EPOCH, File1),
        {ok, KludgeBin} = ?FLU_C:checksum_list(Host, TcpPort, File1),
        true = is_binary(KludgeBin),
-        {error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
+        {error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort,
                                               ?DUMMY_PV1_EPOCH,
-                                               BadPrefix, Chunk1, NoCSum),
+                                               BadPrefix, Chunk1),
        {ok, [{_,File1}]} = ?FLU_C:list_files(Host, TcpPort, ?DUMMY_PV1_EPOCH),
        Len1 = size(Chunk1),
        {error, not_written} = ?FLU_C:read_chunk(Host, TcpPort,
-                                                 NSInfo, ?DUMMY_PV1_EPOCH,
+                                                  ?DUMMY_PV1_EPOCH,
-                                                 File1, Off1*983829323, Len1,
+                                                  File1, Off1*983829323, Len1, []),
                                                 noopt),
        %% XXX FIXME
        %%
        %% This is failing because the read extends past the end of the file.
@ -133,22 +118,19 @@ flu_smoke_test() ->
        %% of the read will cause it to fail.
        %%
        %% {error, partial_read} = ?FLU_C:read_chunk(Host, TcpPort,
-        %%                                           NSInfo, ?DUMMY_PV1_EPOCH,
+        %%                                           ?DUMMY_PV1_EPOCH,
        %%                                           File1, Off1, Len1*9999),
-        {ok, {Off1b,Len1b,File1b}} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
+        {ok, {Off1b,Len1b,File1b}} = ?FLU_C:append_chunk(Host, TcpPort,
                                                         ?DUMMY_PV1_EPOCH,
-                                                         Prefix, Chunk1,NoCSum),
+                                                         Prefix, Chunk1),
        Extra = 42,
-        Opts1 = #append_opts{chunk_extra=Extra},
+        {ok, {Off1c,Len1c,File1c}} = ?FLU_C:append_chunk_extra(Host, TcpPort,
        {ok, {Off1c,Len1c,File1c}} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
                                                         ?DUMMY_PV1_EPOCH,
-                                                         Prefix, Chunk1, NoCSum,
+                                                         Prefix, Chunk1, Extra),
                                                         Opts1, infinity),
        {ok, {Off1d,Len1d,File1d}} = ?FLU_C:append_chunk(Host, TcpPort,
                                                         NSInfo,
                                                         ?DUMMY_PV1_EPOCH,
-                                                         Prefix, Chunk1,NoCSum),
+                                                         Prefix, Chunk1),
        if File1b == File1c, File1c == File1d ->
                true = (Off1c == Off1b + Len1b),
                true = (Off1d == Off1c + Len1c + Extra);
@ -156,44 +138,27 @@ flu_smoke_test() ->
                exit(not_mandatory_but_test_expected_same_file_fixme)
        end,
        Chunk1_cs = {<<?CSUM_TAG_NONE:8, 0:(8*20)>>, Chunk1},
        {ok, {Off1e,Len1e,File1e}} = ?FLU_C:append_chunk(Host, TcpPort,
                                                         ?DUMMY_PV1_EPOCH,
                                                         Prefix, Chunk1_cs),
        Chunk2 = <<"yo yo">>,
        Len2 = byte_size(Chunk2),
        Off2 = ?MINIMUM_OFFSET + 77,
-        File2 = "smoke-whole-file^^0^1^1",
+        File2 = "smoke-whole-file^1^1",
-        ok = ?FLU_C:write_chunk(Host, TcpPort, NSInfo, ?DUMMY_PV1_EPOCH,
+        ok = ?FLU_C:write_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH,
-                                File2, Off2, Chunk2, NoCSum),
+                                File2, Off2, Chunk2),
-        {error, bad_arg} = ?FLU_C:write_chunk(Host, TcpPort, NSInfo, ?DUMMY_PV1_EPOCH,
+        {error, bad_arg} = ?FLU_C:write_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH,
-                                              BadFile, Off2, Chunk2, NoCSum),
+                                              BadFile, Off2, Chunk2),
        {ok, {[{_, Off2, Chunk2, _}], _}} =
-            ?FLU_C:read_chunk(Host, TcpPort, NSInfo, ?DUMMY_PV1_EPOCH, File2, Off2, Len2, noopt),
+            ?FLU_C:read_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH, File2, Off2, Len2, []),
        {error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort,
-                                                 NSInfo, ?DUMMY_PV1_EPOCH,
+                                                 ?DUMMY_PV1_EPOCH,
-                                                 "no!!", Off2, Len2, noopt),
+                                                 "no!!", Off2, Len2, []),
        {error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort,
-                                             NSInfo, ?DUMMY_PV1_EPOCH,
+                                             ?DUMMY_PV1_EPOCH,
-                                             BadFile, Off2, Len2, noopt),
+                                             BadFile, Off2, Len2, []),
        %% Make a connected socket.
        Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}),
        %% Let's test some cluster version enforcement.
        Good_EpochNum = 0,
        Good_NSVersion = 0,
        Good_NS = <<>>,
        {ok, {false, {Good_EpochNum,_}, Good_NSVersion, GoodNS}} =
            ?FLU_C:wedge_status(Sock1),
        NS_good = #ns_info{version=Good_NSVersion, name=Good_NS},
        {ok, {[{_, Off2, Chunk2, _}], _}} =
            ?FLU_C:read_chunk(Sock1, NS_good, ?DUMMY_PV1_EPOCH,
                              File2, Off2, Len2, noopt),
        NS_bad_version = #ns_info{version=1, name=Good_NS},
        NS_bad_name = #ns_info{version=Good_NSVersion, name= <<"foons">>},
        {error, bad_epoch} =
            ?FLU_C:read_chunk(Sock1, NS_bad_version, ?DUMMY_PV1_EPOCH,
                              File2, Off2, Len2, noopt),
        {error, bad_arg} =
            ?FLU_C:read_chunk(Sock1, NS_bad_name, ?DUMMY_PV1_EPOCH,
                              File2, Off2, Len2, noopt),
        %% We know that File1 still exists.  Pretend that we've done a
        %% migration and exercise the delete_migration() API.
@ -210,23 +175,25 @@ flu_smoke_test() ->
        {error, bad_arg} = ?FLU_C:trunc_hack(Host, TcpPort,
                                             ?DUMMY_PV1_EPOCH, BadFile),
-        ok = ?FLU_C:quit(Sock1)
+        ok = ?FLU_C:quit(?FLU_C:connect(#p_srvr{address=Host,
                                                port=TcpPort}))
    after
-        machi_test_util:stop_flu_package()
+        stop_flu_package(smoke_flu)
    end.
 flu_projection_smoke_test() ->
    Host = "localhost",
-    TcpPort = 12959,
+    TcpPort = 32959,
    DataDir = "./data.projst",
-    {_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir),
+
    start_flu_package(projection_test_flu, TcpPort, DataDir),
    try
        [ok = flu_projection_common(Host, TcpPort, T) ||
            T <- [public, private] ]
-%% ,        {ok, {false, EpochID1,_,_}} = ?FLU_C:wedge_status(Host, TcpPort),
+%% ,        {ok, {false, EpochID1}} = ?FLU_C:wedge_status(Host, TcpPort),
 %% io:format(user, "EpochID1 ~p\n", [EpochID1])
    after
-        machi_test_util:stop_flu_package()
+        stop_flu_package(projection_test_flu)
    end.
 flu_projection_common(Host, TcpPort, T) ->
@ -254,32 +221,30 @@ flu_projection_common(Host, TcpPort, T) ->
 bad_checksum_test() ->
    Host = "localhost",
-    TcpPort = 12960,
+    TcpPort = 32960,
    DataDir = "./data.bct",
    Opts = [{initial_wedged, false}],
-    {_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts),
+    start_flu_package(projection_test_flu, TcpPort, DataDir, Opts),
    NSInfo = undefined,
    try
        Prefix = <<"some prefix">>,
        Chunk1 = <<"yo yo yo">>,
-        BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, ".................")},
+        Chunk1_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, Chunk1},
-        {error, bad_checksum} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
+        {error, bad_checksum} = ?FLU_C:append_chunk(Host, TcpPort,
                                                    ?DUMMY_PV1_EPOCH,
-                                                    Prefix,
+                                                    Prefix, Chunk1_badcs),
                                                    Chunk1, BadCSum),
        ok
    after
-        machi_test_util:stop_flu_package()
+        stop_flu_package(projection_test_flu)
    end.
 witness_test() ->
    Host = "localhost",
-    TcpPort = 12961,
+    TcpPort = 32961,
    DataDir = "./data.witness",
    Opts = [{initial_wedged, false}, {witness_mode, true}],
-    {_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts),
+    start_flu_package(projection_test_flu, TcpPort, DataDir, Opts),
    NSInfo = undefined,
    NoCSum = <<>>,
    try
        Prefix = <<"some prefix">>,
        Chunk1 = <<"yo yo yo">>,
@ -292,14 +257,15 @@ witness_test() ->
        {ok, EpochID1} = ?FLU_C:get_latest_epochid(Host, TcpPort, private),
        %% Witness-protected ops all fail
-        {error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo, EpochID1,
+        {error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort, EpochID1,
-                                               Prefix, Chunk1, NoCSum),
+                                               Prefix, Chunk1),
        File = <<"foofile">>,
-        {error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort, NSInfo, EpochID1,
+        {error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort, EpochID1,
-                                             File, 9999, 9999, noopt),
+                                             File, 9999, 9999, []),
-        {error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort, File),
+        {error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort, EpochID1,
                                                File),
        {error, bad_arg} = ?FLU_C:list_files(Host, TcpPort, EpochID1),
-        {ok, {false, EpochID1,_,_}} = ?FLU_C:wedge_status(Host, TcpPort),
+        {ok, {false, EpochID1}} = ?FLU_C:wedge_status(Host, TcpPort),
        {ok, _} = ?FLU_C:get_latest_epochid(Host, TcpPort, public),
        {ok, _} = ?FLU_C:read_latest_projection(Host, TcpPort, public),
        {error, not_written} = ?FLU_C:read_projection(Host, TcpPort,
@ -310,7 +276,7 @@ witness_test() ->
        ok
    after
-        machi_test_util:stop_flu_package()
+        stop_flu_package(projection_test_flu)
    end.
 %% The purpose of timing_pb_encoding_test_ and timing_bif_encoding_test_ is
--- a/test/machi_flu_psup_test.erl
+++ b/test/machi_flu_psup_test.erl
@ -38,12 +38,12 @@ smoke_test_() ->
    {timeout, 5*60, fun() -> smoke_test2() end}.
 smoke_test2() ->
-    Ps = [{a,#p_srvr{name=a, address="localhost", port=5550, props="./data.a"}},
+    Ps = [{a,#p_srvr{name=a, address="localhost", port=5555, props="./data.a"}},
-          {b,#p_srvr{name=b, address="localhost", port=5551, props="./data.b"}},
+          {b,#p_srvr{name=b, address="localhost", port=5556, props="./data.b"}},
-          {c,#p_srvr{name=c, address="localhost", port=5552, props="./data.c"}}
+          {c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}}
         ],
    [os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps],
-    {ok, SupPid} = machi_sup:start_link(),
+    {ok, SupPid} = machi_flu_sup:start_link(),
    try
        %% Only run a, don't run b & c so we have 100% failures talking to them
        [begin
@ -66,15 +66,15 @@ partial_stop_restart_test_() ->
    {timeout, 5*60, fun() -> partial_stop_restart2() end}.
 partial_stop_restart2() ->
-    Ps = [{a,#p_srvr{name=a, address="localhost", port=5560, props="./data.a"}},
+    Ps = [{a,#p_srvr{name=a, address="localhost", port=5555, props="./data.a"}},
-          {b,#p_srvr{name=b, address="localhost", port=5561, props="./data.b"}},
+          {b,#p_srvr{name=b, address="localhost", port=5556, props="./data.b"}},
-          {c,#p_srvr{name=c, address="localhost", port=5562, props="./data.c"}}
+          {c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}}
         ],
    ChMgrs = [machi_flu_psup:make_mgr_supname(P#p_srvr.name) || {_,P} <-Ps],
    PStores = [machi_flu_psup:make_proj_supname(P#p_srvr.name) || {_,P} <-Ps],
    Dict = orddict:from_list(Ps),
    [os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps],
-    {ok, SupPid} = machi_sup:start_link(),
+    {ok, SupPid} = machi_flu_sup:start_link(),
    DbgProps = [{initial_wedged, true}],
    Start = fun({_,P}) ->
                    #p_srvr{name=Name, port=Port, props=Dir} = P,
@ -84,23 +84,20 @@ partial_stop_restart2() ->
    WedgeStatus = fun({_,#p_srvr{address=Addr, port=TcpPort}}) ->
                          machi_flu1_client:wedge_status(Addr, TcpPort)
                  end,
    NSInfo = undefined,
    Append = fun({_,#p_srvr{address=Addr, port=TcpPort}}, EpochID) ->
                     NoCSum = <<>>,
                     machi_flu1_client:append_chunk(Addr, TcpPort,
-                                                    NSInfo, EpochID,
+                                                    EpochID,
-                                                    <<"prefix">>,
+                                                    <<"prefix">>, <<"data">>)
                                                    <<"data">>, NoCSum)
                  end,
    try
        [Start(P) || P <- Ps],
-        [{ok, {true, _,_,_}} = WedgeStatus(P) || P <- Ps], % all are wedged
+        [{ok, {true, _}} = WedgeStatus(P) || P <- Ps], % all are wedged
        [{error,wedged} = Append(P, ?DUMMY_PV1_EPOCH) || P <- Ps], % all are wedged
        [machi_chain_manager1:set_chain_members(ChMgr, Dict) ||
            ChMgr <- ChMgrs ],
-        {ok, {false, EpochID1,_,_}} = WedgeStatus(hd(Ps)),
+        {ok, {false, EpochID1}} = WedgeStatus(hd(Ps)),
-        [{ok, {false, EpochID1,_,_}} = WedgeStatus(P) || P <- Ps], % *not* wedged
+        [{ok, {false, EpochID1}} = WedgeStatus(P) || P <- Ps], % *not* wedged
        [{ok,_} = Append(P, EpochID1) || P <- Ps],             % *not* wedged
        {ok, {_,_,File1}} = Append(hd(Ps), EpochID1),
@ -126,9 +123,9 @@ partial_stop_restart2() ->
        Epoch_m = Proj_m#projection_v1.epoch_number,
        %% Confirm that all FLUs are *not* wedged, with correct proj & epoch
        Proj_mCSum = Proj_m#projection_v1.epoch_csum,
-        [{ok, {false, {Epoch_m, Proj_mCSum},_,_}} = WedgeStatus(P) || % *not* wedged
+        [{ok, {false, {Epoch_m, Proj_mCSum}}} = WedgeStatus(P) || % *not* wedged
             P <- Ps], 
-        {ok, {false, EpochID1,_,_}} = WedgeStatus(hd(Ps)),
+        {ok, {false, EpochID1}} = WedgeStatus(hd(Ps)),
        [{ok,_} = Append(P, EpochID1) || P <- Ps],            % *not* wedged
        %% Stop all but 'a'.
@ -148,10 +145,10 @@ partial_stop_restart2() ->
        {error, wedged} = Append(hd(Ps), EpochID1),
        {_, #p_srvr{address=Addr_a, port=TcpPort_a}} = hd(Ps),
        {error, wedged} = machi_flu1_client:read_chunk(
-                            Addr_a, TcpPort_a, NSInfo, ?DUMMY_PV1_EPOCH,
+                            Addr_a, TcpPort_a, ?DUMMY_PV1_EPOCH,
-                            <<>>, 99999999, 1, undefined),
+                            <<>>, 99999999, 1, []),
-        {error, bad_arg} = machi_flu1_client:checksum_list(
+        {error, wedged} = machi_flu1_client:checksum_list(
-                             Addr_a, TcpPort_a, <<>>),
+                            Addr_a, TcpPort_a, ?DUMMY_PV1_EPOCH, <<>>),
        %% list_files() is permitted despite wedged status
        {ok, _} = machi_flu1_client:list_files(
                            Addr_a, TcpPort_a, ?DUMMY_PV1_EPOCH),
@ -160,7 +157,7 @@ partial_stop_restart2() ->
        {now_using,_,Epoch_n} = machi_chain_manager1:trigger_react_to_env(
                                  hd(ChMgrs)),
        true = (Epoch_n > Epoch_m),
-        {ok, {false, EpochID3,_,_}} = WedgeStatus(hd(Ps)),
+        {ok, {false, EpochID3}} = WedgeStatus(hd(Ps)),
        %% The file we're assigned should be different with the epoch change.
        {ok, {_,_,File3}} = Append(hd(Ps), EpochID3),
        true = (File1 /= File3),
@ -176,19 +173,6 @@ partial_stop_restart2() ->
        ok
    end.
 p_srvr_rec_test() ->
    P = #p_srvr{name=a, address="localhost", port=1024, props=[yo]},
    [P] = machi_flu_sup:sanitize_p_srvr_records([P]),
    [P] = machi_flu_sup:sanitize_p_srvr_records([P,P]),
    [] = machi_flu_sup:sanitize_p_srvr_records([nope]),
    [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{proto_mod=does_not_exist}]),
    [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{proto_mod="lists"}]),
    [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{address=7}]),
    [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{port=5}]),
    [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{port=foo}]),
    [] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{props=foo}]),
    ok.
 -endif. % !PULSE
 -endif. % TEST
--- a/test/machi_lifecycle_mgr_test.erl
+++ b/test/machi_lifecycle_mgr_test.erl
@ -1,307 +0,0 @@
 %% -------------------------------------------------------------------
 %%
 %% Copyright (c) 2007-2014 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
 %% except in compliance with the License.  You may obtain
 %% a copy of the License at
 %%
 %%   http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing,
 %% software distributed under the License is distributed on an
 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 %% KIND, either express or implied.  See the License for the
 %% specific language governing permissions and limitations
 %% under the License.
 %%
 %% -------------------------------------------------------------------
 -module(machi_lifecycle_mgr_test).
 -compile(export_all).
 -ifdef(TEST).
 -ifndef(PULSE).
 -include_lib("eunit/include/eunit.hrl").
 -include("machi.hrl").
 -include("machi_projection.hrl").
 -define(MGR, machi_chain_manager1).
 setup() ->
    catch application:stop(machi),
    {ok, SupPid} = machi_sup:start_link(),
    error_logger:tty(false),
    Dir = "./" ++ atom_to_list(?MODULE) ++ ".datadir",
    machi_flu1_test:clean_up_data_dir(Dir ++ "/*/*"),
    machi_flu1_test:clean_up_data_dir(Dir),
    Envs = [{flu_data_dir, Dir ++ "/data/flu"},
            {flu_config_dir, Dir ++ "/etc/flu-config"},
            {chain_config_dir, Dir ++ "/etc/chain-config"},
            {platform_data_dir, Dir ++ "/data"},
            {platform_etc_dir, Dir ++ "/etc"},
            {not_used_pending, Dir ++ "/etc/pending"}
           ],
    EnvKeys = [K || {K,_V} <- Envs],
    undefined = application:get_env(machi, yo),
    Cleanup = machi_flu1_test:get_env_vars(machi, EnvKeys ++ [yo]),
    [begin
         filelib:ensure_dir(V ++ "/unused"),
         application:set_env(machi, K, V)
     end || {K, V} <- Envs],
    {SupPid, Dir, Cleanup}.
 cleanup({SupPid, Dir, Cleanup}) ->
    exit(SupPid, normal),
    machi_util:wait_for_death(SupPid, 100),
    error_logger:tty(true),
    catch application:stop(machi),
    machi_flu1_test:clean_up_data_dir(Dir ++ "/*/*"),
    machi_flu1_test:clean_up_data_dir(Dir),
    machi_flu1_test:clean_up_env_vars(Cleanup),
    undefined = application:get_env(machi, yo),
    ok.
 smoke_test_() ->
    {timeout, 60, fun() -> smoke_test2() end}.
 smoke_test2() ->
    YoCleanup = setup(),
    try
        Prefix = <<"pre">>,
        Chunk1 = <<"yochunk">>,
        Host = "localhost",
        PortBase = 60120,
        Pa = #p_srvr{name=a,address="localhost",port=PortBase+0},
        Pb = #p_srvr{name=b,address="localhost",port=PortBase+1},
        Pc = #p_srvr{name=c,address="localhost",port=PortBase+2},
        %% Pstore_a = machi_flu1:make_projection_server_regname(a),
        %% Pstore_b = machi_flu1:make_projection_server_regname(b),
        %% Pstore_c = machi_flu1:make_projection_server_regname(c),
        Pstores = [Pstore_a, Pstore_b, Pstore_c] =
            [machi_flu1:make_projection_server_regname(a),
             machi_flu1:make_projection_server_regname(b),
             machi_flu1:make_projection_server_regname(c)],
        ChMgrs = [ChMgr_a, ChMgr_b, ChMgr_c] =
            [machi_chain_manager1:make_chmgr_regname(a),
             machi_chain_manager1:make_chmgr_regname(b),
             machi_chain_manager1:make_chmgr_regname(c)],
        Fits = [Fit_a, Fit_b, Fit_c] =
            [machi_flu_psup:make_fitness_regname(a),
             machi_flu_psup:make_fitness_regname(b),
             machi_flu_psup:make_fitness_regname(c)],
        Advance = machi_chain_manager1_test:make_advance_fun(
                    Fits, [a,b,c], ChMgrs, 3),
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("\nSTEP: Start 3 FLUs, no chain.\n", []),
        [machi_lifecycle_mgr:make_pending_config(P) || P <- [Pa,Pb,Pc] ],
        {[_,_,_],[]} = machi_lifecycle_mgr:process_pending(),
        [{ok, #projection_v1{epoch_number=0}} =
             machi_projection_store:read_latest_projection(PSTORE, private)
         || PSTORE <- Pstores],
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("\nSTEP: Start chain = [a,b,c]\n", []),
        C1 = #chain_def_v1{name=cx, mode=ap_mode, full=[Pa,Pb,Pc],
                           local_run=[a,b,c]},
        machi_lifecycle_mgr:make_pending_config(C1),
        {[],[_]} = machi_lifecycle_mgr:process_pending(),
        Advance(),
        [{ok, #projection_v1{all_members=[a,b,c]}} =
             machi_projection_store:read_latest_projection(PSTORE, private)
         || PSTORE <- Pstores],
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("\nSTEP: Reset chain = [b,c]\n", []),
        C2 = #chain_def_v1{name=cx, mode=ap_mode, full=[Pb,Pc],
                           old_full=[a,b,c], old_witnesses=[],
                           local_stop=[a], local_run=[b,c]},
        machi_lifecycle_mgr:make_pending_config(C2),
        {[],[_]} = machi_lifecycle_mgr:process_pending(),
        Advance(),
        %% a should be down
        {'EXIT', _} = (catch machi_projection_store:read_latest_projection(
                               hd(Pstores), private)),
        [{ok, #projection_v1{all_members=[b,c]}} =
             machi_projection_store:read_latest_projection(PSTORE, private)
         || PSTORE <- tl(Pstores)],
        %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
        io:format("\nSTEP: Reset chain = []\n", []),
        C3 = #chain_def_v1{name=cx, mode=ap_mode, full=[],
                           old_full=[b,c], old_witnesses=[],
                           local_stop=[b,c], local_run=[]},
        machi_lifecycle_mgr:make_pending_config(C3),
        {[],[_]} = machi_lifecycle_mgr:process_pending(),
        Advance(),
        %% a,b,c should be down
        [{'EXIT', _} = (catch machi_projection_store:read_latest_projection(
                                PSTORE, private))
         || PSTORE <- Pstores],
        ok
    after
        cleanup(YoCleanup)
    end.
 ast_tuple_syntax_test() ->
    T = fun(L) -> machi_lifecycle_mgr:check_ast_tuple_syntax(L) end,
    Canon1 = [ {host, "localhost", []},
               {host, "localhost", [{client_interface, "1.2.3.4"},
                                    {admin_interface, "5.6.7.8"}]},
               {flu, 'fx', "foohost", 4000, []},
               switch_old_and_new,
               {chain, 'cy', ['fx', 'fy'], [{foo,"yay"},{bar,baz}]} ],
    {_Good,[]=_Bad} = T(Canon1),
    Canon1_norm = machi_lifecycle_mgr:normalize_ast_tuple_syntax(Canon1),
    true = (length(Canon1) == length(Canon1_norm)),
    {Canon1_norm_b, []} = T(Canon1_norm),
    true = (length(Canon1_norm) == length(Canon1_norm_b)),
    {[],[_,_,_,_]} =
        T([ {host, 'localhost', []},
            {host, 'localhost', yo},
            {host, "localhost", [{client_interface, 77.88293829832}]},
            {host, "localhost", [{client_interface, "1.2.3.4"},
                                 {bummer, "5.6.7.8"}]} ]),
    {[],[_,_,_,_,_,_]} =
        T([ {flu, 'fx', 'foohost', 4000, []},
            {flu, 'fx', <<"foohost">>, 4000, []},
            {flu, 'fx', "foohost", -4000, []},
            {flu, 'fx', "foohost", 40009999, []},
            {flu, 'fx', "foohost", 4000, gack},
            {flu, 'fx', "foohost", 4000, [22]} ]),
    {[],[_,_,_]} =
        T([ {chain, 'cy', ["fx", "fy"], [foo,{bar,baz}]},
            yoloyolo,
            {chain, "cy", ["fx", 27], oops,arity,way,way,way,too,big,x}
          ]).
 ast_run_test() ->
    PortBase = 20300,
    R1 = [
          {host, "localhost", "localhost", "localhost", []},
          {flu, 'f0', "localhost", PortBase+0, []},
          {flu, 'f1', "localhost", PortBase+1, []},
          {chain, 'ca', ['f0'], []},
          {chain, 'cb', ['f1'], []},
          switch_old_and_new,
          {flu, 'f2', "localhost", PortBase+2, []},
          {flu, 'f3', "localhost", PortBase+3, []},
          {flu, 'f4', "localhost", PortBase+4, []},
          {chain, 'ca', ['f0', 'f2'], []},
          {chain, 'cc', ['f3', 'f4'], []}
         ],
    {ok, Env1} = machi_lifecycle_mgr:run_ast(R1),
    %% Uncomment to examine the Env trees.
    %% Y1 = {lists:sort(gb_trees:to_list(element(1, Env1))),
    %%       lists:sort(gb_trees:to_list(element(2, Env1))),
    %%       element(3, Env1)},
    %% io:format(user, "\nY1 ~p\n", [Y1]),
    Negative_after_R1 =
        [
          {host, "localhost", "foo", "foo", []}, % dupe host
          {flu, 'f1', "other", PortBase+9999999, []}, % bogus port # (syntax)
          {flu, 'f1', "other", PortBase+888, []}, % dupe flu name
          {flu, 'f7', "localhost", PortBase+1, []}, % dupe host+port
          {chain, 'ca', ['f7'], []}, % unknown flu
          {chain, 'cc', ['f0'], []}, % flu previously assigned
          {chain, 'ca', cp_mode, ['f0', 'f1', 'f2'], [], []} % mode change
        ],
    [begin
         %% io:format(user, "dbg: Neg ~p\n", [Neg]),
         {error, _} = machi_lifecycle_mgr:run_ast(R1 ++ [Neg])
     end || Neg <- Negative_after_R1],
    %% The 'run' phase doesn't blow smoke.  What about 'diff'?
    {X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, "localhost"),
    %% There's only one host, "localhost", so 'all' should be exactly equal.
    {X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, all),
    %% io:format(user, "X1b: ~p\n", [X1b]),
    %% Append to the R1 scenario: for chain cc: add f5, remove f4
    %% Expect: see pattern matching below on X2b.
    R2 = (R1 -- [switch_old_and_new]) ++
        [switch_old_and_new,
         {flu, 'f5', "localhost", PortBase+5, []},
         {chain, 'cc', ['f3','f5'], []}],
    {ok, Env2} = machi_lifecycle_mgr:run_ast(R2),
    {_X2a, X2b} = machi_lifecycle_mgr:diff_env(Env2, "localhost"),
    %% io:format(user, "X2b: ~p\n", [X2b]),
    F5_port = PortBase+5,
    [#p_srvr{name='f5',address="localhost",port=F5_port},
     #chain_def_v1{name='cc',
                   full=[#p_srvr{name='f3'},#p_srvr{name='f5'}], witnesses=[],
                   old_full=[f3,f4], old_witnesses=[],
                   local_run=[f5], local_stop=[f4]}] = X2b,
    ok.
 ast_then_apply_test_() ->
    {timeout, 60, fun() -> ast_then_apply_test2() end}.
 ast_then_apply_test2() ->
    YoCleanup = setup(),
    try
        PortBase = 20400,
        NumChains = 4,
        ChainLen = 3,
        FLU_num = NumChains * ChainLen,
        FLU_defs = [{flu, list_to_atom("f"++integer_to_list(X)),
                     "localhost", PortBase+X, []} || X <- lists:seq(1,FLU_num)],
        FLU_names = [FLU || {flu,FLU,_,_,_} <- FLU_defs],
        Ch_defs = [{chain, list_to_atom("c"++integer_to_list(X)),
                    lists:sublist(FLU_names, X, 3),
                    []} || X <- lists:seq(1, FLU_num, 3)],
        R1 = [switch_old_and_new,
              {host, "localhost", "localhost", "localhost", []}]
             ++ FLU_defs ++ Ch_defs,
        {ok, Env1} = machi_lifecycle_mgr:run_ast(R1),
        {_X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, "localhost"),
        %% io:format(user, "X1b ~p\n", [X1b]),
        [machi_lifecycle_mgr:make_pending_config(X) || X <- X1b],
        {PassFLUs, PassChains} = machi_lifecycle_mgr:process_pending(),
        true = (length(PassFLUs) == length(FLU_defs)),
        true = (length(PassChains) == length(Ch_defs)),
        %% Kick the chain managers into doing something useful right now.
        Pstores = [list_to_atom(atom_to_list(X) ++ "_pstore") || X <- FLU_names],
        Fits = [list_to_atom(atom_to_list(X) ++ "_fitness") || X <- FLU_names],
        ChMgrs = [list_to_atom(atom_to_list(X) ++ "_chmgr") || X <- FLU_names],
        Advance = machi_chain_manager1_test:make_advance_fun(
                    Fits, FLU_names, ChMgrs, 3),
        Advance(),
        %% Sanity check: everyone is configured properly.
        [begin
             {ok, #projection_v1{epoch_number=Epoch, all_members=All,
                                 chain_name=ChainName, upi=UPI}} =
                 machi_projection_store:read_latest_projection(PStore, private),
             %% io:format(user, "~p: epoch ~p all ~p\n", [PStore, Epoch, All]),
             true = Epoch > 0,
             ChainLen = length(All),
             true = (length(UPI) > 0),
             {chain, _, Full, []} = lists:keyfind(ChainName, 2, Ch_defs),
             true = lists:sort(Full) == lists:sort(All)
         end || PStore <- Pstores],
        ok
    after
        cleanup(YoCleanup)
    end.
 -endif. % !PULSE
 -endif. % TEST
--- a/test/machi_merkle_tree_test.erl
+++ b/test/machi_merkle_tree_test.erl
@ -1,200 +0,0 @@
 %% -------------------------------------------------------------------
 %%
 %% Copyright (c) 2007-2015 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
 %% except in compliance with the License.  You may obtain
 %% a copy of the License at
 %%
 %%   http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing,
 %% software distributed under the License is distributed on an
 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 %% KIND, either express or implied.  See the License for the
 %% specific language governing permissions and limitations
 %% under the License.
 %%
 %% -------------------------------------------------------------------
 -module(machi_merkle_tree_test).
 -compile([export_all]).
 -include("machi_merkle_tree.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("kernel/include/file.hrl").
 -define(GAP_CHANCE, 0.10).
 %% unit tests
 basic_test() ->
    random:seed(os:timestamp()),
    Fsz = choose_size() * 1024,
    Filesize = max(Fsz, 10*1024*1024),
    ChunkSize = max(1048576, Filesize div 100),
    N = make_leaf_nodes(Filesize),
    D0 = #naive{ leaves = N, chunk_size = ChunkSize, recalc = true },
    T1 = machi_merkle_tree:build_tree(D0),
    D1 = #naive{ leaves = tl(N), chunk_size = ChunkSize, recalc = true },
    T2 = machi_merkle_tree:build_tree(D1),
    ?assertNotEqual(T1#naive.root, T2#naive.root),
    ?assertEqual(true, length(machi_merkle_tree:naive_diff(T1, T2)) == 1
                       orelse
                       Filesize > ChunkSize).
 make_leaf_nodes(Filesize) ->
    lists:reverse(
      lists:foldl(fun(T, Acc) -> machi_merkle_tree:update_acc(T, Acc) end, 
                  [], 
                  generate_offsets(Filesize, 1024, []))
     ).
 choose_int(Factor) ->
    random:uniform(1024*Factor).
 small_int() ->
    choose_int(10).
 medium_int() ->
    choose_int(1024).
 large_int() ->
    choose_int(4096).
 generate_offsets(Filesize, Current, Acc) when Current < Filesize ->
    Length0 = choose_size(),
    Length = case Length0 + Current > Filesize of
                 false -> Length0;
                  true -> Filesize - Current
    end,
    Data = term_to_binary(os:timestamp()),
    Checksum = machi_util:make_tagged_csum(client_sha, machi_util:checksum_chunk(Data)),
    Gap = maybe_gap(random:uniform()),
    generate_offsets(Filesize, Current + Length + Gap, [ {Current, Length, Checksum} | Acc ]);
 generate_offsets(_Filesize, _Current, Acc) ->
    lists:reverse(Acc).
 random_from_list(L) ->
    N = random:uniform(length(L)),
    lists:nth(N, L).
 choose_size() ->
    F = random_from_list([fun small_int/0, fun medium_int/0, fun large_int/0]),
    F().
 maybe_gap(Chance) when Chance < ?GAP_CHANCE ->
    choose_size();
 maybe_gap(_) -> 0.
 %% Define or remove these ifdefs if benchmarking is desired.
 -ifdef(BENCH).
 generate_offsets(FH, Filesize, Current, Acc) when Current < Filesize ->
    Length0 = choose_size(),
    Length = case Length0 + Current > Filesize of
                 false -> Length0;
                  true -> Filesize - Current
    end,
    {ok, Data} = file:pread(FH, Current, Length),
    Checksum = machi_util:make_tagged_csum(client_sha, machi_util:checksum_chunk(Data)),
    Gap = maybe_gap(random:uniform()),
    generate_offsets(FH, Filesize, Current + Length + Gap, [ {Current, Length, Checksum} | Acc ]);
 generate_offsets(_FH, _Filesize, _Current, Acc) ->
    lists:reverse(Acc).
 make_offsets_from_file(Filename) ->
    {ok, Info} = file:read_file_info(Filename),
    Filesize = Info#file_info.size,
    {ok, FH} = file:open(Filename, [read, raw, binary]),
    Offsets = generate_offsets(FH, Filesize, 1024, []),
    file:close(FH),
    Offsets.
 choose_filename() ->
    random_from_list([
        "def^c5ea7511-d649-47d6-a8c3-2b619379c237^1",
        "jkl^b077eff7-b2be-4773-a73f-fea4acb8a732^1",
        "stu^553fa47a-157c-4fac-b10f-2252c7d8c37a^1",
        "vwx^ae015d68-7689-4c9f-9677-926c6664f513^1",
        "yza^4c784dc2-19bf-4ac6-91f6-58bbe5aa88e0^1"
                     ]).
 make_csum_file(DataDir, Filename, Offsets) ->
    Path = machi_util:make_checksum_filename(DataDir, Filename),
    filelib:ensure_dir(Path),
    {ok, MC} = machi_csum_table:open(Path, []),
    lists:foreach(fun({Offset, Size, Checksum}) -> 
                    machi_csum_table:write(MC, Offset, Size, Checksum) end,
                  Offsets),
    machi_csum_table:close(MC).
 test() -> 
    test(100).
 test(N) ->
    {ok, F} = file:open("results.txt", [raw, write]),
    lists:foreach(fun(X) -> format_and_store(F, run_test(X)) end, lists:seq(1, N)).
 format_and_store(F, {OffsetNum, {MTime, MSize}, {NTime, NSize}}) ->
    S = io_lib:format("~w\t~w\t~w\t~w\t~w\n", [OffsetNum, MTime, MSize, NTime, NSize]),
    ok = file:write(F, S).
 run_test(C) ->
    random:seed(os:timestamp()),
    OffsetFn = "test/" ++ choose_filename(),
    O = make_offsets_from_file(OffsetFn),
    Fn = "csum_" ++ integer_to_list(C),
    make_csum_file(".", Fn, O),
    Osize = length(O),
    {MTime, {ok, M}} = timer:tc(fun() -> machi_merkle_tree:open(Fn, ".", merklet) end),
    {NTime, {ok, N}} = timer:tc(fun() -> machi_merkle_tree:open(Fn, ".", naive) end),
    ?assertEqual(Fn, machi_merkle_tree:filename(M)),
    ?assertEqual(Fn, machi_merkle_tree:filename(N)),
    MTree = machi_merkle_tree:tree(M),
    MSize = byte_size(term_to_binary(MTree)),
    NTree = machi_merkle_tree:tree(N),
    NSize = byte_size(term_to_binary(NTree)),
    ?assertEqual(same, machi_merkle_tree:diff(N, N)),
    ?assertEqual(same, machi_merkle_tree:diff(M, M)),
    {Osize, {MTime, MSize}, {NTime, NSize}}.
 torture_test(C) ->
    Results = [ run_torture_test() || _ <- lists:seq(1, C) ],
    {ok, F} = file:open("torture_results.txt", [raw, write]),
    lists:foreach(fun({MSize, MTime, NSize, NTime}) ->
                      file:write(F, io_lib:format("~p\t~p\t~p\t~p\n",
                                                [MSize, MTime, NSize, NTime]))
                  end, Results),
    ok = file:close(F).
 run_torture_test() ->
    {NTime, N} = timer:tc(fun() -> naive_torture() end), 
    MSize = byte_size(term_to_binary(M)),
    NSize = byte_size(term_to_binary(N)),
    {MSize, MTime, NSize, NTime}.
 naive_torture() ->
    N = lists:foldl(fun(T, Acc) -> machi_merkle_tree:update_acc(T, Acc) end, [], torture_generator()),
    T = #naive{ leaves = lists:reverse(N), chunk_size = 10010, recalc = true },
    machi_merkle_tree:build_tree(T).
 torture_generator() ->
    [ {O, 1, crypto:hash(sha, term_to_binary(now()))} || O <- lists:seq(1024, 1000000) ].
 -endif. % BENCH
--- a/test/machi_pb_high_client_test.erl
+++ b/test/machi_pb_high_client_test.erl
@ -24,7 +24,6 @@
 -ifdef(TEST).
 -ifndef(PULSE).
 -include("machi.hrl").
 -include("machi_pb.hrl").
 -include("machi_projection.hrl").
 -include_lib("eunit/include/eunit.hrl").
@ -35,15 +34,21 @@ smoke_test_() ->
    {timeout, 5*60, fun() -> smoke_test2() end}.
 smoke_test2() ->
-    PortBase = 5720,
+    Port = 5720,
-    ok = application:set_env(machi, max_file_size, 1024*1024),
+    Ps = [#p_srvr{name=a, address="localhost", port=Port, props="./data.a"}
-    try
+         ],
        {Ps, MgrNames, Dirs} = machi_test_util:start_flu_packages(
                                 1, PortBase, "./data.", []),
    D = orddict:from_list([{P#p_srvr.name, P} || P <- Ps]),
-        M0 = hd(MgrNames),
+    ok = application:set_env(machi, max_file_size, 1024*1024),
-        ok = machi_chain_manager1:set_chain_members(M0, D),
+    
-        [machi_chain_manager1:trigger_react_to_env(M0) || _ <-lists:seq(1,5)],
+    [os:cmd("rm -rf " ++ P#p_srvr.props) || P <- Ps],
    {ok, SupPid} = machi_flu_sup:start_link(),
    try
        [begin
             #p_srvr{name=Name, port=Port, props=Dir} = P,
             {ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, [])
         end || P <- Ps],
        ok = machi_chain_manager1:set_chain_members(a_chmgr, D),
        [machi_chain_manager1:trigger_react_to_env(a_chmgr) || _ <-lists:seq(1,5)],
        {ok, Clnt} = ?C:start_link(Ps),
        try
@ -56,18 +61,16 @@ smoke_test2() ->
            %% a separate test module?  Or separate test func?
            {error, _} = ?C:auth(Clnt, "foo", "bar"),
            PK = <<>>,
            Prefix = <<"prefix">>,
            Chunk1 = <<"Hello, chunk!">>,
            NS = "",
            NoCSum = <<>>,
            Opts1 = #append_opts{},
            {ok, {Off1, Size1, File1}} =
-                ?C:append_chunk(Clnt, NS, Prefix, Chunk1, NoCSum, Opts1),
+                ?C:append_chunk(Clnt, PK, Prefix, Chunk1, none, 0),
            true = is_binary(File1),
            Chunk2 = "It's another chunk",
            CSum2 = {client_sha, machi_util:checksum_chunk(Chunk2)},
            {ok, {Off2, Size2, File2}} =
-                ?C:append_chunk(Clnt, NS, Prefix, Chunk2, CSum2, Opts1),
+                ?C:append_chunk(Clnt, PK, Prefix, Chunk2, CSum2, 1024),
            Chunk3 = ["This is a ", <<"test,">>, 32, [["Hello, world!"]]],
            File3 = File2,
            Off3 = Off2 + iolist_size(Chunk2),
@ -78,9 +81,9 @@ smoke_test2() ->
                     {iolist_to_binary(Chunk2), File2, Off2, Size2},
                     {iolist_to_binary(Chunk3), File3, Off3, Size3}],
            [begin
-                 File = Fl,
+                 File = binary_to_list(Fl),
                 ?assertMatch({ok, {[{File, Off, Ch, _}], []}},
-                              ?C:read_chunk(Clnt, Fl, Off, Sz, undefined))
+                              ?C:read_chunk(Clnt, Fl, Off, Sz, []))
             end || {Ch, Fl, Off, Sz} <- Reads],
            {ok, KludgeBin} = ?C:checksum_list(Clnt, File1),
@ -90,8 +93,7 @@ smoke_test2() ->
            File1Bin = binary_to_list(File1),
            [begin
-                 #p_srvr{name=Name, props=Props} = P,
+                 #p_srvr{name=Name, port=Port, props=Dir} = P,
                 Dir = proplists:get_value(data_dir, Props),
                 ?assertEqual({ok, [File1Bin]},
                              file:list_dir(filename:join([Dir, "data"]))),
                 FileListFileName = filename:join([Dir, "known_files_" ++ atom_to_list(Name)]),
@ -104,23 +106,21 @@ smoke_test2() ->
             end || {_Ch, Fl, Off, Sz} <- Reads],
            [begin
                 {ok, {[], Trimmed}} =
-                        ?C:read_chunk(Clnt, Fl, Off, Sz, #read_opts{needs_trimmed=true}),
+                        ?C:read_chunk(Clnt, Fl, Off, Sz, [{needs_trimmed, true}]),
-                 Filename = Fl,
+                 Filename = binary_to_list(Fl),
                 ?assertEqual([{Filename, Off, Sz}], Trimmed)
             end || {_Ch, Fl, Off, Sz} <- Reads],
            LargeBytes = binary:copy(<<"x">>, 1024*1024),
            LBCsum = {client_sha, machi_util:checksum_chunk(LargeBytes)},
            {ok, {Offx, Sizex, Filex}} =
-                ?C:append_chunk(Clnt, NS,
+                ?C:append_chunk(Clnt, PK, Prefix, LargeBytes, LBCsum, 0),
                                Prefix, LargeBytes, LBCsum, Opts1),
            ok = ?C:trim_chunk(Clnt, Filex, Offx, Sizex),
            %% Make sure everything was trimmed
            File = binary_to_list(Filex),
            [begin
-                 #p_srvr{name=Name, props=Props} = P,
+                 #p_srvr{name=Name, port=_Port, props=Dir} = P,
                 Dir = proplists:get_value(data_dir, Props),
                 ?assertEqual({ok, []},
                              file:list_dir(filename:join([Dir, "data"]))),
                 FileListFileName = filename:join([Dir, "known_files_" ++ atom_to_list(Name)]),
@ -130,14 +130,17 @@ smoke_test2() ->
            [begin
                 {error, trimmed} =
-                        ?C:read_chunk(Clnt, Fl, Off, Sz, undefined)
+                        ?C:read_chunk(Clnt, Fl, Off, Sz, [])
             end || {_Ch, Fl, Off, Sz} <- Reads],
            ok
        after
            (catch ?C:quit(Clnt))
        end
    after
-        machi_test_util:stop_flu_packages()
+        exit(SupPid, normal),
        [os:cmd("rm -rf " ++ P#p_srvr.props) || P <- Ps],
        machi_util:wait_for_death(SupPid, 100),
        ok
    end.
 -endif. % !PULSE
--- a/test/machi_projection_store_test.erl
+++ b/test/machi_projection_store_test.erl
@ -33,7 +33,7 @@ smoke_test() ->
    Dir = "./data.a",
    Os = [{ignore_stability_time, true}, {active_mode, false}],
    os:cmd("rm -rf " ++ Dir),
-    machi_test_util:start_flu_package(a, PortBase, "./data.a", Os),
+    machi_flu1_test:start_flu_package(a, PortBase, "./data.a", Os),
    try
        P1 = machi_projection:new(1, a, [], [], [], [], []),
@ -58,7 +58,7 @@ smoke_test() ->
        ok
    after
-        machi_test_util:stop_flu_package()
+        machi_flu1_test:stop_flu_package(a)
    end.
 -endif. % !PULSE
--- a/test/machi_proxy_flu1_client_test.erl
+++ b/test/machi_proxy_flu1_client_test.erl
@ -32,55 +32,48 @@
 api_smoke_test() ->
    RegName = api_smoke_flu,
-    TcpPort = 17124,
+    Host = "localhost",
    TcpPort = 57124,
    DataDir = "./data.api_smoke_flu",
    W_props = [{active_mode, false},{initial_wedged, false}],
    Prefix = <<"prefix">>,
-    NSInfo = undefined,
+
-    NoCSum = <<>>,
+    machi_flu1_test:start_flu_package(RegName, TcpPort, DataDir, W_props),
    try
-        {[I], _, _} = machi_test_util:start_flu_package(
+        I = #p_srvr{name=RegName, address=Host, port=TcpPort},
                        RegName, TcpPort, DataDir, W_props),
        {ok, Prox1} = ?MUT:start_link(I),
        try
            FakeEpoch = ?DUMMY_PV1_EPOCH,
-            [{ok, {_,_,_}} = ?MUT:append_chunk(
+            [{ok, {_,_,_}} = ?MUT:append_chunk(Prox1,
-                                          Prox1, NSInfo, FakeEpoch,
+                                          FakeEpoch, Prefix, <<"data">>,
-                                          Prefix, <<"data">>, NoCSum) ||
+                                          infinity) || _ <- lists:seq(1,5)],
                _ <- lists:seq(1,5)],
            %% Stop the FLU, what happens?
-            machi_test_util:stop_flu_package(),
+            machi_flu1_test:stop_flu_package(RegName),
-            [{error,partition} = ?MUT:append_chunk(Prox1, NSInfo,
+            [{error,partition} = ?MUT:append_chunk(Prox1,
                                FakeEpoch, Prefix, <<"data-stopped1">>,
-                                NoCSum) || _ <- lists:seq(1,3)],
+                                infinity) || _ <- lists:seq(1,3)],
            %% Start the FLU again, we should be able to do stuff immediately
-            machi_test_util:start_flu_package(RegName, TcpPort, DataDir,
+            machi_flu1_test:start_flu_package(RegName, TcpPort, DataDir,
-                                              [no_cleanup|W_props]),
+                                                   [save_data_dir|W_props]),
            MyChunk = <<"my chunk data">>,
            {ok, {MyOff,MySize,MyFile}} =
-                ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch, Prefix, MyChunk,
+                ?MUT:append_chunk(Prox1, FakeEpoch, Prefix, MyChunk,
                                  NoCSum),
            {ok, {[{_, MyOff, MyChunk, _MyChunkCSUM}], []}} =
                ?MUT:read_chunk(Prox1, NSInfo, FakeEpoch, MyFile, MyOff, MySize, undefined),
            MyChunk2_parts = [<<"my chunk ">>, "data", <<", yeah, again">>],
            MyChunk2 = iolist_to_binary(MyChunk2_parts),
            Opts1 = #append_opts{chunk_extra=4242},
            {ok, {MyOff2,MySize2,MyFile2}} =
                ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch, Prefix,
                                  MyChunk2_parts, NoCSum, Opts1, infinity),
            [{ok, {[{_, MyOff2, MyChunk2, _}], []}} =
                ?MUT:read_chunk(Prox1, NSInfo, FakeEpoch, MyFile2, MyOff2, MySize2, DefaultOptions) ||
                DefaultOptions <- [undefined, noopt, none, any_atom_at_all] ],
            BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, "...................")},
            {error, bad_checksum} = ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
                                                      Prefix, MyChunk, BadCSum),
            {error, bad_checksum} = ?MUT:write_chunk(Prox1, NSInfo, FakeEpoch,
                                                     MyFile2,
                                                     MyOff2 + size(MyChunk2),
                                                     MyChunk, BadCSum,
                                  infinity),
            {ok, {[{_, MyOff, MyChunk, _}], []}} =
                ?MUT:read_chunk(Prox1, FakeEpoch, MyFile, MyOff, MySize, []),
            MyChunk2 = <<"my chunk data, yeah, again">>,
            {ok, {MyOff2,MySize2,MyFile2}} =
                ?MUT:append_chunk_extra(Prox1, FakeEpoch, Prefix,
                                        MyChunk2, 4242, infinity),
            {ok, {[{_, MyOff2, MyChunk2, _}], []}} =
                ?MUT:read_chunk(Prox1, FakeEpoch, MyFile2, MyOff2, MySize2, []),
            MyChunk_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, MyChunk},
            {error, bad_checksum} = ?MUT:append_chunk(Prox1, FakeEpoch,
                                                      Prefix, MyChunk_badcs),
            {error, bad_checksum} = ?MUT:write_chunk(Prox1, FakeEpoch,
                                                     <<"foo-file^1^1">>, 99832,
                                                     MyChunk_badcs),
            %% Put kick_projection_reaction() in the middle of the test so
            %% that any problems with its async nature will (hopefully)
@ -89,9 +82,9 @@ api_smoke_test() ->
            %% Alright, now for the rest of the API, whee
            BadFile = <<"no-such-file">>,
-            {error, bad_arg} = ?MUT:checksum_list(Prox1, BadFile),
+            {error, bad_arg} = ?MUT:checksum_list(Prox1, FakeEpoch, BadFile),
            {ok, [_|_]} = ?MUT:list_files(Prox1, FakeEpoch),
-            {ok, {false, _,_,_}} = ?MUT:wedge_status(Prox1),
+            {ok, {false, _}} = ?MUT:wedge_status(Prox1),
            {ok, {0, _SomeCSum}} = ?MUT:get_latest_epochid(Prox1, public),
            {ok, #projection_v1{epoch_number=0}} =
                 ?MUT:read_latest_projection(Prox1, public),
@ -109,7 +102,7 @@ api_smoke_test() ->
            _ = (catch ?MUT:quit(Prox1))
        end
    after
-        (catch machi_test_util:stop_flu_package())
+        (catch machi_flu1_test:stop_flu_package(RegName))
    end.
 flu_restart_test_() ->
@ -117,22 +110,22 @@ flu_restart_test_() ->
 flu_restart_test2() ->
    RegName = a,
-    TcpPort = 17125,
+    Host = "localhost",
    TcpPort = 57125,
    DataDir = "./data.api_smoke_flu2",
    W_props = [{initial_wedged, false}, {active_mode, false}],
-    NSInfo = undefined,
+    machi_flu1_test:start_flu_package(RegName, TcpPort, DataDir, W_props),
    NoCSum = <<>>,
    try
-        {[I], _, _} = machi_test_util:start_flu_package(
+        I = #p_srvr{name=RegName, address=Host, port=TcpPort},
                        RegName, TcpPort, DataDir, W_props),
        {ok, Prox1} = ?MUT:start_link(I),
        try
            FakeEpoch = ?DUMMY_PV1_EPOCH,
            Data   = <<"data!">>,
            Dataxx = <<"Fake!">>,
-            {ok, {Off1,Size1,File1}} = ?MUT:append_chunk(Prox1, NSInfo,
+            {ok, {Off1,Size1,File1}} = ?MUT:append_chunk(Prox1,
-                                         FakeEpoch, <<"prefix">>, Data, NoCSum),
+                                          FakeEpoch, <<"prefix">>, Data,
                                          infinity),
            P_a = #p_srvr{name=a, address="localhost", port=6622},
            P1   = machi_projection:new(1, RegName, [P_a], [], [RegName], [], []),
            P1xx = P1#projection_v1{dbg2=["dbg2 changes are ok"]},
@ -144,7 +137,7 @@ flu_restart_test2() ->
            {ok, EpochID} = ?MUT:get_epoch_id(Prox1),
            {ok, EpochID} = ?MUT:get_latest_epochid(Prox1, public),
            {ok, EpochID} = ?MUT:get_latest_epochid(Prox1, private),
-            ok = machi_test_util:stop_flu_package(), timer:sleep(50),
+            ok = machi_flu1_test:stop_flu_package(RegName), timer:sleep(50),
            %% Now that the last proxy op was successful and only
            %% after did we stop the FLU, let's check that both the
@ -156,10 +149,9 @@ flu_restart_test2() ->
            %% makes the code a bit convoluted.  (No LFE or
            %% Elixir macros here, alas, they'd be useful.)
            AppendOpts1 = #append_opts{chunk_extra=42},
            ExpectedOps = 
                [
-                 fun(run) -> ?assertEqual({ok, EpochID}, ?MUT:get_epoch_id(Prox1)),
+                 fun(run) -> {ok, EpochID} = ?MUT:get_epoch_id(Prox1),
                             ok;
                    (line) -> io:format("line ~p, ", [?LINE]);
                    (stop) -> ?MUT:get_epoch_id(Prox1) end,
@ -238,37 +230,35 @@ flu_restart_test2() ->
                    (stop) -> ?MUT:get_all_projections(Prox1, private)
                 end,
                 fun(run) -> {ok, {_,_,_}} =
-                                 ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
+                                 ?MUT:append_chunk(Prox1, FakeEpoch,
-                                                <<"prefix">>, Data, NoCSum),
+                                                <<"prefix">>, Data, infinity),
                             ok;
                    (line) -> io:format("line ~p, ", [?LINE]);
-                    (stop) -> ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
+                    (stop) -> ?MUT:append_chunk(Prox1, FakeEpoch,
-                                                <<"prefix">>, Data, NoCSum)
+                                                <<"prefix">>, Data, infinity)
                 end,
                 fun(run) -> {ok, {_,_,_}} =
-                                 ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
+                                 ?MUT:append_chunk_extra(Prox1, FakeEpoch,
-                                                   <<"prefix">>, Data, NoCSum,
+                                             <<"prefix">>, Data, 42, infinity),
                                                   AppendOpts1, infinity),
                             ok;
                    (line) -> io:format("line ~p, ", [?LINE]);
-                    (stop) -> ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
+                    (stop) -> ?MUT:append_chunk_extra(Prox1, FakeEpoch,
-                                                   <<"prefix">>, Data, NoCSum,
+                                             <<"prefix">>, Data, 42, infinity)
                                                   AppendOpts1, infinity)
                 end,
                 fun(run) -> {ok, {[{_, Off1, Data, _}], []}} =
-                                 ?MUT:read_chunk(Prox1, NSInfo, FakeEpoch,
+                                 ?MUT:read_chunk(Prox1, FakeEpoch,
-                                                 File1, Off1, Size1, undefined),
+                                                 File1, Off1, Size1, []),
                             ok;
                    (line) -> io:format("line ~p, ", [?LINE]);
-                    (stop) -> ?MUT:read_chunk(Prox1, NSInfo, FakeEpoch,
+                    (stop) -> ?MUT:read_chunk(Prox1, FakeEpoch,
-                                              File1, Off1, Size1, undefined)
+                                              File1, Off1, Size1, [])
                 end,
                 fun(run) -> {ok, KludgeBin} =
-                                 ?MUT:checksum_list(Prox1, File1),
+                                 ?MUT:checksum_list(Prox1, FakeEpoch, File1),
                             true = is_binary(KludgeBin),
                             ok;
                    (line) -> io:format("line ~p, ", [?LINE]);
-                    (stop) -> ?MUT:checksum_list(Prox1, File1)
+                    (stop) -> ?MUT:checksum_list(Prox1, FakeEpoch, File1)
                 end,
                 fun(run) -> {ok, _} =
                                 ?MUT:list_files(Prox1, FakeEpoch),
@ -284,32 +274,32 @@ flu_restart_test2() ->
                 end,
                 fun(run) ->
                             ok =
-                                 ?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
+                                 ?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
-                                                  Data, NoCSum, infinity),
+                                                  Data, infinity),
                             ok;
                    (line) -> io:format("line ~p, ", [?LINE]);
-                    (stop) -> ?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
+                    (stop) -> ?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
-                                               Data, NoCSum, infinity)
+                                               Data, infinity)
                 end,
                 fun(run) -> 
                         {error, written} =
-                                 ?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
+                                 ?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
-                                                  Dataxx, NoCSum, infinity),
+                                                  Dataxx, infinity),
                             ok;
                    (line) -> io:format("line ~p, ", [?LINE]);
-                    (stop) -> ?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
+                    (stop) -> ?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
-                                               Dataxx, NoCSum, infinity)
+                                               Dataxx, infinity)
                 end
                ],
            [begin
-                 machi_test_util:start_flu_package(
+                 machi_flu1_test:start_flu_package(
                          RegName, TcpPort, DataDir,
-                          [no_cleanup|W_props]),
+                          [save_data_dir|W_props]),
                 _ = Fun(line),
                 ok = Fun(run),
                 ok = Fun(run),
-                 ok = machi_test_util:stop_flu_package(),
+                 ok = machi_flu1_test:stop_flu_package(RegName),
                 {error, partition} = Fun(stop),
                 {error, partition} = Fun(stop),
                 ok
@ -319,7 +309,7 @@ flu_restart_test2() ->
            _ = (catch ?MUT:quit(Prox1))
        end
    after
-        (catch machi_test_util:stop_flu_package())
+        (catch machi_flu1_test:stop_flu_package(RegName))
    end.
 -endif. % !PULSE
--- a/test/machi_test_util.erl
+++ b/test/machi_test_util.erl
@ -1,111 +0,0 @@
 %% -------------------------------------------------------------------
 %%
 %% Copyright (c) 2007-2015 Basho Technologies, Inc.  All Rights Reserved.
 %%
 %% This file is provided to you under the Apache License,
 %% Version 2.0 (the "License"); you may not use this file
 %% except in compliance with the License.  You may obtain
 %% a copy of the License at
 %%
 %%   http://www.apache.org/licenses/LICENSE-2.0
 %%
 %% Unless required by applicable law or agreed to in writing,
 %% software distributed under the License is distributed on an
 %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 %% KIND, either express or implied.  See the License for the
 %% specific language governing permissions and limitations
 %% under the License.
 %%
 %% -------------------------------------------------------------------
 -module(machi_test_util).
 -compile(export_all).
 -ifdef(TEST).
 -ifndef(PULSE).
 -include_lib("eunit/include/eunit.hrl").
 -include("machi.hrl").
 -include("machi_projection.hrl").
 -define(FLU, machi_flu1).
 -define(FLU_C, machi_flu1_client).
 -spec start_flu_package(atom(), inet:port_number(), string()) ->
                               {Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}.
 start_flu_package(FluName, TcpPort, DataDir) ->
    start_flu_package(FluName, TcpPort, DataDir, []).
 -spec start_flu_package(atom(), inet:port_number(), string(), list()) ->
                               {Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}.
 start_flu_package(FluName, TcpPort, DataDir, Props) ->
    MgrName = machi_flu_psup:make_mgr_supname(FluName),
    FluInfo = [{#p_srvr{name=FluName, address="localhost", port=TcpPort,
                        props=[{chmgr, MgrName}, {data_dir, DataDir} | Props]},
                DataDir, MgrName}],
    start_flu_packages(FluInfo).
 -spec start_flu_packages(pos_integer(), inet:port_number(), string(), list()) ->
                                {Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}.
 start_flu_packages(FluCount, BaseTcpPort, DirPrefix, Props) ->
    FluInfo = flu_info(FluCount, BaseTcpPort, DirPrefix, Props),
    start_flu_packages(FluInfo).
 start_flu_packages(FluInfo) ->
    _ = stop_machi_sup(),
    clean_up(FluInfo),
    {ok, _SupPid} = machi_sup:start_link(),
    [{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, Props) ||
        {#p_srvr{name=Name, port=Port, props=Props}, Dir, _} <- FluInfo],
    {Ps, Dirs, MgrNames} = lists:unzip3(FluInfo),
    {Ps, MgrNames, Dirs}.
 stop_flu_package() ->
    stop_flu_packages().
 stop_flu_packages() ->
    stop_machi_sup().
 flu_info(FluCount, BaseTcpPort, DirPrefix, Props) ->
    [begin
         FLUNameStr = [$a + I - 1],
         FLUName = list_to_atom(FLUNameStr),
         MgrName = machi_flu_psup:make_mgr_supname(FLUName),
         DataDir = DirPrefix ++ "/data.eqc." ++ FLUNameStr,
         {#p_srvr{name=FLUName, address="localhost", port=BaseTcpPort + I,
                  props=[{chmgr, MgrName}, {data_dir, DataDir} | Props]},
          DataDir, MgrName}
     end || I <- lists:seq(1, FluCount)].
 stop_machi_sup() ->
    case whereis(machi_sup) of
        undefined -> ok;
        Pid ->
            catch exit(whereis(machi_sup), normal),
            machi_util:wait_for_death(Pid, 100)
    end.
 clean_up(FluInfo) ->
    _ = [begin
             case proplists:get_value(no_cleanup, Props) of
                 true -> ok;
                 _ ->
                     _ = machi_flu1:stop(FLUName),
                     clean_up_dir(Dir)
             end
         end || {#p_srvr{name=FLUName, props=Props}, Dir, _} <- FluInfo],
    ok.
 clean_up_dir(Dir) ->
    [begin
         Fs = filelib:wildcard(Dir ++ Glob),
         [file:delete(F) || F <- Fs],
         [file:del_dir(F) || F <- Fs]
     end || Glob <- ["*/*/*/*", "*/*/*", "*/*", "*"] ],
    _ = file:del_dir(Dir),
    ok.
 -endif. % !PULSE
 -endif. % TEST
--- a/tools.mk
+++ b/tools.mk
@ -27,7 +27,6 @@
 REBAR ?= ./rebar
 REVISION ?= $(shell git rev-parse --short HEAD)
 PROJECT ?= $(shell basename `find src -name "*.app.src"` .app.src)
 EUNIT_OPTS ?=
 .PHONY: compile-no-deps test docs xref dialyzer-run dialyzer-quick dialyzer \
 		cleanplt upload-docs
@ -36,7 +35,7 @@ compile-no-deps:
 	${REBAR} compile skip_deps=true
 test: compile
-	${REBAR} ${EUNIT_OPTS} eunit skip_deps=true
+	${REBAR} eunit skip_deps=true
 upload-docs: docs
 	@if [ -z "${BUCKET}" -o -z "${PROJECT}" -o -z "${REVISION}" ]; then \
@ -64,18 +63,10 @@ ERL_LIB_DIR = $(shell erl -eval '{io:format("~s\n", [code:lib_dir()]), erlang:ha
 native-ebin:
 	mkdir -p $(NATIVE_EBIN)
 	rm -f $(NATIVE_EBIN)/*.erl $(NATIVE_EBIN)/*.hrl $(NATIVE_EBIN)/*.beam
-	@for mod in lists dict digraph digraph_utils ets gb_sets gb_trees ordsets sets sofs; do \
+	cp $(ERL_LIB_DIR)/stdlib-*/src/{lists,dict,digraph,digraph_utils,ets,gb_sets,gb_trees,ordsets,sets,sofs}.erl $(NATIVE_EBIN)
-		cp $(ERL_LIB_DIR)/stdlib-*/src/"$$mod".erl $(NATIVE_EBIN); \
+	cp $(ERL_LIB_DIR)/compiler-*/src/{cerl,cerl_trees,core_parse}.?rl  $(NATIVE_EBIN)
-	done
+	cp $(ERL_LIB_DIR)/dialyzer-*/src/{dialyzer_analysis_callgraph,dialyzer,dialyzer_behaviours,dialyzer_codeserver,dialyzer_contracts,dialyzer_coordinator,dialyzer_dataflow,dialyzer_dep,dialyzer_plt,dialyzer_succ_typings,dialyzer_typesig,dialyzer_worker}.?rl  $(NATIVE_EBIN)
-	@for mod in cerl cerl_trees core_parse; do \
+	cp $(ERL_LIB_DIR)/hipe-*/*/{erl_types,erl_bif_types}.?rl  $(NATIVE_EBIN)
 		cp $(ERL_LIB_DIR)/compiler-*/src/"$$mod".?rl  $(NATIVE_EBIN); \
 	done
 	@for mod in dialyzer_analysis_callgraph dialyzer dialyzer_behaviours dialyzer_codeserver dialyzer_contracts dialyzer_coordinator dialyzer_dataflow dialyzer_dep dialyzer_plt dialyzer_succ_typings dialyzer_typesig dialyzer_worker; do \
 		cp $(ERL_LIB_DIR)/dialyzer-*/src/"$$mod".?rl  $(NATIVE_EBIN); \
 	done
 	@for mod in erl_types erl_bif_types; do \
 		cp $(ERL_LIB_DIR)/hipe-*/*/"$$mod".?rl  $(NATIVE_EBIN); \
 	done
 	erlc -o $(NATIVE_EBIN) -smp +native -DVSN='"$(DIALYZER_VERSION)"' $(NATIVE_EBIN)/*erl
 ${PLT}: compile