Compare commits
2 commits
master
...
tmp/ss/fil
| Author | SHA1 | Date | |
|---|---|---|---|
|
393132a1d8 | ||
|
|
0e21581e47 |
91 changed files with 4091 additions and 8017 deletions
5
.gitignore
vendored
5
.gitignore
vendored
|
|
@ -2,9 +2,7 @@ prototype/chain-manager/patch.*
|
|||
.eqc-info
|
||||
.eunit
|
||||
deps
|
||||
dev
|
||||
erl_crash.dump
|
||||
eqc
|
||||
.concrete/DEV_MODE
|
||||
.rebar
|
||||
edoc
|
||||
|
|
@ -22,12 +20,9 @@ include/machi_pb.hrl
|
|||
|
||||
# Release packaging
|
||||
rel/machi
|
||||
rel/vars/dev*vars.config
|
||||
|
||||
# Misc Scott cruft
|
||||
*.patch
|
||||
current_counterexample.eqc
|
||||
foo*
|
||||
RUNLOG*
|
||||
typescript*
|
||||
*.swp
|
||||
|
|
|
|||
|
|
@ -4,4 +4,4 @@ notifications:
|
|||
script: "priv/test-for-gh-pr.sh"
|
||||
otp_release:
|
||||
- 17.5
|
||||
## No, Dialyzer is too different between 17 & 18: - 18.1
|
||||
- 18.1
|
||||
|
|
|
|||
230
FAQ.md
230
FAQ.md
|
|
@ -11,14 +11,14 @@
|
|||
|
||||
+ [1 Questions about Machi in general](#n1)
|
||||
+ [1.1 What is Machi?](#n1.1)
|
||||
+ [1.2 What is a Machi chain?](#n1.2)
|
||||
+ [1.3 What is a Machi cluster?](#n1.3)
|
||||
+ [1.4 What is Machi like when operating in "eventually consistent" mode?](#n1.4)
|
||||
+ [1.5 What is Machi like when operating in "strongly consistent" mode?](#n1.5)
|
||||
+ [1.6 What does Machi's API look like?](#n1.6)
|
||||
+ [1.7 What licensing terms are used by Machi?](#n1.7)
|
||||
+ [1.8 Where can I find the Machi source code and documentation? Can I contribute?](#n1.8)
|
||||
+ [1.9 What is Machi's expected release schedule, packaging, and operating system/OS distribution support?](#n1.9)
|
||||
+ [1.2 What is a Machi "cluster of clusters"?](#n1.2)
|
||||
+ [1.2.1 This "cluster of clusters" idea needs a better name, don't you agree?](#n1.2.1)
|
||||
+ [1.3 What is Machi like when operating in "eventually consistent"/"AP mode"?](#n1.3)
|
||||
+ [1.4 What is Machi like when operating in "strongly consistent"/"CP mode"?](#n1.4)
|
||||
+ [1.5 What does Machi's API look like?](#n1.5)
|
||||
+ [1.6 What licensing terms are used by Machi?](#n1.6)
|
||||
+ [1.7 Where can I find the Machi source code and documentation? Can I contribute?](#n1.7)
|
||||
+ [1.8 What is Machi's expected release schedule, packaging, and operating system/OS distribution support?](#n1.8)
|
||||
+ [2 Questions about Machi relative to {{something else}}](#n2)
|
||||
+ [2.1 How is Machi better than Hadoop?](#n2.1)
|
||||
+ [2.2 How does Machi differ from HadoopFS/HDFS?](#n2.2)
|
||||
|
|
@ -28,15 +28,13 @@
|
|||
+ [3 Machi's specifics](#n3)
|
||||
+ [3.1 What technique is used to replicate Machi's files? Can other techniques be used?](#n3.1)
|
||||
+ [3.2 Does Machi have a reliance on a coordination service such as ZooKeeper or etcd?](#n3.2)
|
||||
+ [3.3 Are there any presentations available about Humming Consensus](#n3.3)
|
||||
+ [3.4 Is it true that there's an allegory written to describe Humming Consensus?](#n3.4)
|
||||
+ [3.5 How is Machi tested?](#n3.5)
|
||||
+ [3.6 Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks](#n3.6)
|
||||
+ [3.7 Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?](#n3.7)
|
||||
+ [3.8 What language(s) is Machi written in?](#n3.8)
|
||||
+ [3.9 Can Machi run on Windows? Can Machi run on 32-bit platforms?](#n3.9)
|
||||
+ [3.10 Does Machi use the Erlang/OTP network distribution system (aka "disterl")?](#n3.10)
|
||||
+ [3.11 Can I use HTTP to write/read stuff into/from Machi?](#n3.11)
|
||||
+ [3.3 Is it true that there's an allegory written to describe humming consensus?](#n3.3)
|
||||
+ [3.4 How is Machi tested?](#n3.4)
|
||||
+ [3.5 Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks](#n3.5)
|
||||
+ [3.6 Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?](#n3.6)
|
||||
+ [3.7 What language(s) is Machi written in?](#n3.7)
|
||||
+ [3.8 Does Machi use the Erlang/OTP network distribution system (aka "disterl")?](#n3.8)
|
||||
+ [3.9 Can I use HTTP to write/read stuff into/from Machi?](#n3.9)
|
||||
|
||||
<!-- ENDOUTLINE -->
|
||||
|
||||
|
|
@ -46,13 +44,13 @@
|
|||
<a name="n1.1">
|
||||
### 1.1. What is Machi?
|
||||
|
||||
Very briefly, Machi is a very simple append-only blob/file store.
|
||||
Very briefly, Machi is a very simple append-only file store.
|
||||
|
||||
Machi is
|
||||
"dumber" than many other file stores (i.e., lacking many features
|
||||
found in other file stores) such as HadoopFS or a simple NFS or CIFS file
|
||||
found in other file stores) such as HadoopFS or simple NFS or CIFS file
|
||||
server.
|
||||
However, Machi is a distributed blob/file store, which makes it different
|
||||
However, Machi is a distributed file store, which makes it different
|
||||
(and, in some ways, more complicated) than a simple NFS or CIFS file
|
||||
server.
|
||||
|
||||
|
|
@ -84,39 +82,45 @@ For a much longer answer, please see the
|
|||
[Machi high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-machi.pdf).
|
||||
|
||||
<a name="n1.2">
|
||||
### 1.2. What is a Machi chain?
|
||||
### 1.2. What is a Machi "cluster of clusters"?
|
||||
|
||||
A Machi chain is a small number of machines that maintain a common set
|
||||
of replicated files. A typical chain is of length 2 or 3. For
|
||||
critical data that must be available despite several simultaneous
|
||||
server failures, a chain length of 6 or 7 might be used.
|
||||
Machi's design is based on using small, well-understood and provable
|
||||
(mathematically) techniques to maintain multiple file copies without
|
||||
data loss or data corruption. At its lowest level, Machi contains no
|
||||
support for distribution/partitioning/sharding of files across many
|
||||
servers. A typical, fully-functional Machi cluster will likely be two
|
||||
or three machines.
|
||||
|
||||
<a name="n1.3">
|
||||
### 1.3. What is a Machi cluster?
|
||||
However, Machi is designed to be an excellent building block for
|
||||
building larger systems. A deployment of Machi "cluster of clusters"
|
||||
will use the "random slicing" technique for partitioning files across
|
||||
multiple Machi clusters that, as individuals, are unaware of the
|
||||
larger cluster-of-clusters scheme.
|
||||
|
||||
A Machi cluster is a collection of Machi chains that
|
||||
partitions/shards/distributes files (based on file name) across the
|
||||
collection of chains. Machi uses the "random slicing" algorithm (a
|
||||
variation of consistent hashing) to define the mapping of file name to
|
||||
chain name.
|
||||
|
||||
The cluster management service will be fully decentralized
|
||||
The cluster-of-clusters management service will be fully decentralized
|
||||
and run as a separate software service installed on each Machi
|
||||
cluster. This manager will appear to the local Machi server as simply
|
||||
another Machi file client. The cluster managers will take
|
||||
another Machi file client. The cluster-of-clusters managers will take
|
||||
care of file migration as the cluster grows and shrinks in capacity
|
||||
and in response to day-to-day changes in workload.
|
||||
|
||||
Though the cluster manager has not yet been implemented,
|
||||
Though the cluster-of-clusters manager has not yet been implemented,
|
||||
its design is fully decentralized and capable of operating despite
|
||||
multiple partial failure of its member chains. We expect this
|
||||
multiple partial failure of its member clusters. We expect this
|
||||
design to scale easily to at least one thousand servers.
|
||||
|
||||
Please see the
|
||||
[Machi source repository's 'doc' directory for more details](https://github.com/basho/machi/tree/master/doc/).
|
||||
|
||||
<a name="n1.4">
|
||||
### 1.4. What is Machi like when operating in "eventually consistent" mode?
|
||||
<a name="n1.2.1">
|
||||
#### 1.2.1. This "cluster of clusters" idea needs a better name, don't you agree?
|
||||
|
||||
Yes. Please help us: we are bad at naming things.
|
||||
For proof that naming things is hard, see
|
||||
[http://martinfowler.com/bliki/TwoHardThings.html](http://martinfowler.com/bliki/TwoHardThings.html)
|
||||
|
||||
<a name="n1.3">
|
||||
### 1.3. What is Machi like when operating in "eventually consistent"/"AP mode"?
|
||||
|
||||
Machi's operating mode dictates how a Machi cluster will react to
|
||||
network partitions. A network partition may be caused by:
|
||||
|
|
@ -126,30 +130,37 @@ network partitions. A network partition may be caused by:
|
|||
* An extreme server software "hang" or "pause", e.g. caused by OS
|
||||
scheduling problems such as a failing/stuttering disk device.
|
||||
|
||||
The consistency semantics of file operations while in eventual
|
||||
consistency mode during and after network partitions are:
|
||||
"AP mode" refers to the "A" and "P" properties of the "CAP
|
||||
conjecture", meaning that the cluster will be "Available" and
|
||||
"Partition tolerant".
|
||||
|
||||
The consistency semantics of file operations while in "AP mode" are
|
||||
eventually consistent during and after network partitions:
|
||||
|
||||
* File write operations are permitted by any client on the "same side"
|
||||
of the network partition.
|
||||
* File read operations are successful for any file contents where the
|
||||
client & server are on the "same side" of the network partition.
|
||||
* File read operations will probably fail for any file contents where the
|
||||
client & server are on "different sides" of the network partition.
|
||||
* After the network partition(s) is resolved, files are merged
|
||||
together from "all sides" of the partition(s).
|
||||
* Unique files are copied in their entirety.
|
||||
* Byte ranges within the same file are merged. This is possible
|
||||
due to Machi's restrictions on file naming and file offset
|
||||
assignment. Both file names and file offsets are always chosen
|
||||
by Machi servers according to rules which guarantee safe
|
||||
mergeability. Server-assigned names are a characteristic of a
|
||||
"blob store".
|
||||
due to Machi's restrictions on file naming (files names are
|
||||
alwoys assigned by Machi servers) and file offset assignments
|
||||
(byte offsets are also always chosen by Machi servers according
|
||||
to rules which guarantee safe mergeability.).
|
||||
|
||||
<a name="n1.5">
|
||||
### 1.5. What is Machi like when operating in "strongly consistent" mode?
|
||||
<a name="n1.4">
|
||||
### 1.4. What is Machi like when operating in "strongly consistent"/"CP mode"?
|
||||
|
||||
The consistency semantics of file operations while in strongly
|
||||
consistency mode during and after network partitions are:
|
||||
Machi's operating mode dictates how a Machi cluster will react to
|
||||
network partitions.
|
||||
"CP mode" refers to the "C" and "P" properties of the "CAP
|
||||
conjecture", meaning that the cluster will be "Consistent" and
|
||||
"Partition tolerant".
|
||||
|
||||
The consistency semantics of file operations while in "CP mode" are
|
||||
strongly consistent during and after network partitions:
|
||||
|
||||
* File write operations are permitted by any client on the "same side"
|
||||
of the network partition if and only if a quorum majority of Machi servers
|
||||
|
|
@ -164,19 +175,19 @@ consistency mode during and after network partitions are:
|
|||
|
||||
Machi's design can provide the illusion of quorum minority write
|
||||
availability if the cluster is configured to operate with "witness
|
||||
servers". (This feaure partially implemented, as of December 2015.)
|
||||
servers". (This feaure is not implemented yet, as of June 2015.)
|
||||
See Section 11 of
|
||||
[Machi chain manager high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-chain-mgr.pdf)
|
||||
for more details.
|
||||
|
||||
<a name="n1.6">
|
||||
### 1.6. What does Machi's API look like?
|
||||
<a name="n1.5">
|
||||
### 1.5. What does Machi's API look like?
|
||||
|
||||
The Machi API only contains a handful of API operations. The function
|
||||
arguments shown below (in simplifed form) use Erlang-style type annotations.
|
||||
arguments shown below use Erlang-style type annotations.
|
||||
|
||||
append_chunk(Prefix:binary(), Chunk:binary(), CheckSum:binary()).
|
||||
append_chunk_extra(Prefix:binary(), Chunk:binary(), CheckSum:binary(), ExtraSpace:non_neg_integer()).
|
||||
append_chunk(Prefix:binary(), Chunk:binary()).
|
||||
append_chunk_extra(Prefix:binary(), Chunk:binary(), ExtraSpace:non_neg_integer()).
|
||||
read_chunk(File:binary(), Offset:non_neg_integer(), Size:non_neg_integer()).
|
||||
|
||||
checksum_list(File:binary()).
|
||||
|
|
@ -194,22 +205,17 @@ Internally, there is a more complex protocol used by individual
|
|||
cluster members to manage file contents and to repair damaged/missing
|
||||
files. See Figure 3 in
|
||||
[Machi high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-machi.pdf)
|
||||
for more description.
|
||||
for more details.
|
||||
|
||||
The definitions of both the "high level" external protocol and "low
|
||||
level" internal protocol are in a
|
||||
[Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview)
|
||||
definition at [./src/machi.proto](./src/machi.proto).
|
||||
|
||||
<a name="n1.7">
|
||||
### 1.7. What licensing terms are used by Machi?
|
||||
<a name="n1.6">
|
||||
### 1.6. What licensing terms are used by Machi?
|
||||
|
||||
All Machi source code and documentation is licensed by
|
||||
[Basho Technologies, Inc.](http://www.basho.com/)
|
||||
under the [Apache Public License version 2](https://github.com/basho/machi/tree/master/LICENSE).
|
||||
|
||||
<a name="n1.8">
|
||||
### 1.8. Where can I find the Machi source code and documentation? Can I contribute?
|
||||
<a name="n1.7">
|
||||
### 1.7. Where can I find the Machi source code and documentation? Can I contribute?
|
||||
|
||||
All Machi source code and documentation can be found at GitHub:
|
||||
[https://github.com/basho/machi](https://github.com/basho/machi).
|
||||
|
|
@ -223,11 +229,11 @@ ideas for improvement, please see our contributing & collaboration
|
|||
guidelines at
|
||||
[https://github.com/basho/machi/blob/master/CONTRIBUTING.md](https://github.com/basho/machi/blob/master/CONTRIBUTING.md).
|
||||
|
||||
<a name="n1.9">
|
||||
### 1.9. What is Machi's expected release schedule, packaging, and operating system/OS distribution support?
|
||||
<a name="n1.8">
|
||||
### 1.8. What is Machi's expected release schedule, packaging, and operating system/OS distribution support?
|
||||
|
||||
Basho expects that Machi's first major product release will take place
|
||||
during the 2nd quarter of 2016.
|
||||
Basho expects that Machi's first release will take place near the end
|
||||
of calendar year 2015.
|
||||
|
||||
Basho's official support for operating systems (e.g. Linux, FreeBSD),
|
||||
operating system packaging (e.g. CentOS rpm/yum package management,
|
||||
|
|
@ -302,15 +308,15 @@ file's writable phase).
|
|||
|
||||
<tr>
|
||||
<td> Does not have any file distribution/partitioning/sharding across
|
||||
Machi chains: in a single Machi chain, all files are replicated by
|
||||
all servers in the chain. The "random slicing" technique is used
|
||||
Machi clusters: in a single Machi cluster, all files are replicated by
|
||||
all servers in the cluster. The "cluster of clusters" concept is used
|
||||
to distribute/partition/shard files across multiple Machi clusters.
|
||||
<td> File distribution/partitioning/sharding is performed
|
||||
automatically by the HDFS "name node".
|
||||
|
||||
<tr>
|
||||
<td> Machi requires no central "name node" for single chain use or
|
||||
for multi-chain cluster use.
|
||||
<td> Machi requires no central "name node" for single cluster use.
|
||||
Machi requires no central "name node" for "cluster of clusters" use
|
||||
<td> Requires a single "namenode" server to maintain file system contents
|
||||
and file content mapping. (May be deployed with a "secondary
|
||||
namenode" to reduce unavailability when the primary namenode fails.)
|
||||
|
|
@ -476,8 +482,8 @@ difficult to adapt to Machi's design goals:
|
|||
* Both protocols use quorum majority consensus, which requires a
|
||||
minimum of *2F + 1* working servers to tolerate *F* failures. For
|
||||
example, to tolerate 2 server failures, quorum majority protocols
|
||||
require a minimum of 5 servers. To tolerate the same number of
|
||||
failures, Chain Replication requires a minimum of only 3 servers.
|
||||
require a minium of 5 servers. To tolerate the same number of
|
||||
failures, Chain replication requires only 3 servers.
|
||||
* Machi's use of "humming consensus" to manage internal server
|
||||
metadata state would also (probably) require conversion to Paxos or
|
||||
Raft. (Or "outsourced" to a service such as ZooKeeper.)
|
||||
|
|
@ -494,17 +500,7 @@ Humming consensus is described in the
|
|||
[Machi chain manager high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-chain-mgr.pdf).
|
||||
|
||||
<a name="n3.3">
|
||||
### 3.3. Are there any presentations available about Humming Consensus
|
||||
|
||||
Scott recently (November 2015) gave a presentation at the
|
||||
[RICON 2015 conference](http://ricon.io) about one of the techniques
|
||||
used by Machi; "Managing Chain Replication Metadata with
|
||||
Humming Consensus" is available online now.
|
||||
* [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf)
|
||||
* [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q)
|
||||
|
||||
<a name="n3.4">
|
||||
### 3.4. Is it true that there's an allegory written to describe Humming Consensus?
|
||||
### 3.3. Is it true that there's an allegory written to describe humming consensus?
|
||||
|
||||
Yes. In homage to Leslie Lamport's original paper about the Paxos
|
||||
protocol, "The Part-time Parliamant", there is an allegorical story
|
||||
|
|
@ -515,8 +511,8 @@ The full story, full of wonder and mystery, is called
|
|||
There is also a
|
||||
[short followup blog posting](http://www.snookles.com/slf-blog/2015/03/20/on-humming-consensus-an-allegory-part-2/).
|
||||
|
||||
<a name="n3.5">
|
||||
### 3.5. How is Machi tested?
|
||||
<a name="n3.4">
|
||||
### 3.4. How is Machi tested?
|
||||
|
||||
While not formally proven yet, Machi's implementation of Chain
|
||||
Replication and of humming consensus have been extensively tested with
|
||||
|
|
@ -541,20 +537,16 @@ change several times during any single test case) and a random series
|
|||
of cluster operations, an event trace of all cluster activity is used
|
||||
to verify that no safety-critical rules have been violated.
|
||||
|
||||
All test code is available in the [./test](./test) subdirectory.
|
||||
Modules that use QuickCheck will use a file suffix of `_eqc`, for
|
||||
example, [./test/machi_ap_repair_eqc.erl](./test/machi_ap_repair_eqc.erl).
|
||||
|
||||
<a name="n3.6">
|
||||
### 3.6. Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks
|
||||
<a name="n3.5">
|
||||
### 3.5. Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks
|
||||
|
||||
No, Machi's design assumes that each Machi server is a fully
|
||||
independent hardware and assumes only standard local disks (Winchester
|
||||
and/or SSD style) with local-only interfaces (e.g. SATA, SCSI, PCI) in
|
||||
each machine.
|
||||
|
||||
<a name="n3.7">
|
||||
### 3.7. Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?
|
||||
<a name="n3.6">
|
||||
### 3.6. Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?
|
||||
|
||||
No. When used with servers with multiple disks, the intent is to
|
||||
deploy multiple Machi servers per machine: one Machi server per disk.
|
||||
|
|
@ -572,13 +564,10 @@ deploy multiple Machi servers per machine: one Machi server per disk.
|
|||
placement relative to 12 servers is smaller than a placement problem
|
||||
of managing 264 seprate disks (if each of 12 servers has 22 disks).
|
||||
|
||||
<a name="n3.8">
|
||||
### 3.8. What language(s) is Machi written in?
|
||||
<a name="n3.7">
|
||||
### 3.7. What language(s) is Machi written in?
|
||||
|
||||
So far, Machi is written in Erlang, mostly. Machi uses at least one
|
||||
library, [ELevelDB](https://github.com/basho/eleveldb), that is
|
||||
implemented both in C++ and in Erlang, using Erlang NIFs (Native
|
||||
Interface Functions) to allow Erlang code to call C++ functions.
|
||||
So far, Machi is written in 100% Erlang.
|
||||
|
||||
In the event that we encounter a performance problem that cannot be
|
||||
solved within the Erlang/OTP runtime environment, all of Machi's
|
||||
|
|
@ -587,16 +576,8 @@ in C, Java, or other "gotta go fast fast FAST!!" programming
|
|||
language. We expect that the Chain Replication manager and other
|
||||
critical "control plane" software will remain in Erlang.
|
||||
|
||||
<a name="n3.9">
|
||||
### 3.9. Can Machi run on Windows? Can Machi run on 32-bit platforms?
|
||||
|
||||
The ELevelDB NIF does not compile or run correctly on Erlang/OTP
|
||||
Windows platforms, nor does it compile correctly on 32-bit platforms.
|
||||
Machi should support all 64-bit UNIX-like platforms that are supported
|
||||
by Erlang/OTP and ELevelDB.
|
||||
|
||||
<a name="n3.10">
|
||||
### 3.10. Does Machi use the Erlang/OTP network distribution system (aka "disterl")?
|
||||
<a name="n3.8">
|
||||
### 3.8. Does Machi use the Erlang/OTP network distribution system (aka "disterl")?
|
||||
|
||||
No, Machi doesn't use Erlang/OTP's built-in distributed message
|
||||
passing system. The code would be *much* simpler if we did use
|
||||
|
|
@ -607,16 +588,19 @@ bit-twiddling magicSPEED ... without also having to find a replacement
|
|||
for disterl. (Or without having to re-invent disterl's features in
|
||||
another language.)
|
||||
|
||||
All wire protocols used by Machi are defined & implemented using
|
||||
[Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview).
|
||||
The definition file can be found at [./src/machi.proto](./src/machi.proto).
|
||||
<a name="artisanal-protocol">
|
||||
In the first drafts of the Machi code, the inter-node communication
|
||||
uses a hand-crafted, artisanal, mostly ASCII protocol as part of a
|
||||
"demo day" quick & dirty prototype. Work is underway (summer of 2015)
|
||||
to replace that protocol gradually with a well-structured,
|
||||
well-documented protocol based on Protocol Buffers data serialization.
|
||||
|
||||
<a name="n3.11">
|
||||
### 3.11. Can I use HTTP to write/read stuff into/from Machi?
|
||||
<a name="n3.9">
|
||||
### 3.9. Can I use HTTP to write/read stuff into/from Machi?
|
||||
|
||||
Short answer: No, not yet.
|
||||
|
||||
Longer answer: No, but it was possible as a hack, many months ago, see
|
||||
Yes, sort of. For as long as the legacy of
|
||||
Machi's first internal protocol & code still
|
||||
survives, it's possible to use a
|
||||
[primitive/hack'y HTTP interface that is described in this source code commit log](https://github.com/basho/machi/commit/6cebf397232cba8e63c5c9a0a8c02ba391b20fef).
|
||||
Please note that commit `6cebf397232cba8e63c5c9a0a8c02ba391b20fef` is
|
||||
required to try using this feature: the code has since bit-rotted and
|
||||
|
|
|
|||
36
Makefile
36
Makefile
|
|
@ -10,7 +10,7 @@ endif
|
|||
OVERLAY_VARS ?=
|
||||
EUNIT_OPTS = -v
|
||||
|
||||
.PHONY: rel stagedevrel deps package pkgclean edoc
|
||||
.PHONY: rel deps package pkgclean edoc
|
||||
|
||||
all: deps compile
|
||||
|
||||
|
|
@ -35,9 +35,6 @@ deps:
|
|||
clean:
|
||||
$(REBAR) -r clean
|
||||
|
||||
edoc: edoc-clean
|
||||
$(REBAR) skip_deps=true doc
|
||||
|
||||
edoc-clean:
|
||||
rm -f edoc/*.png edoc/*.html edoc/*.css edoc/edoc-info
|
||||
|
||||
|
|
@ -57,37 +54,6 @@ relclean:
|
|||
stage : rel
|
||||
$(foreach dep,$(wildcard deps/*), rm -rf rel/$(REPO)/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) rel/$(REPO)/lib;)
|
||||
|
||||
##
|
||||
## Developer targets
|
||||
##
|
||||
## devN - Make a dev build for node N
|
||||
## stagedevN - Make a stage dev build for node N (symlink libraries)
|
||||
## devrel - Make a dev build for 1..$DEVNODES
|
||||
## stagedevrel Make a stagedev build for 1..$DEVNODES
|
||||
##
|
||||
## Example, make a 68 node devrel cluster
|
||||
## make stagedevrel DEVNODES=68
|
||||
|
||||
.PHONY : stagedevrel devrel
|
||||
DEVNODES ?= 3
|
||||
|
||||
# 'seq' is not available on all *BSD, so using an alternate in awk
|
||||
SEQ = $(shell awk 'BEGIN { for (i = 1; i < '$(DEVNODES)'; i++) printf("%i ", i); print i ;exit(0);}')
|
||||
|
||||
$(eval stagedevrel : $(foreach n,$(SEQ),stagedev$(n)))
|
||||
$(eval devrel : $(foreach n,$(SEQ),dev$(n)))
|
||||
|
||||
dev% : all
|
||||
mkdir -p dev
|
||||
rel/gen_dev $@ rel/vars/dev_vars.config.src rel/vars/$@_vars.config
|
||||
(cd rel && ../rebar generate target_dir=../dev/$@ overlay_vars=vars/$@_vars.config)
|
||||
|
||||
stagedev% : dev%
|
||||
$(foreach dep,$(wildcard deps/*), rm -rf dev/$^/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) dev/$^/lib;)
|
||||
|
||||
devclean: clean
|
||||
rm -rf dev
|
||||
|
||||
DIALYZER_APPS = kernel stdlib sasl erts ssl compiler eunit crypto public_key syntax_tools
|
||||
PLT = $(HOME)/.machi_dialyzer_plt
|
||||
|
||||
|
|
|
|||
226
README.md
226
README.md
|
|
@ -1,136 +1,62 @@
|
|||
# Machi: a distributed, decentralized blob/large file store
|
||||
# Machi
|
||||
|
||||
[Travis-CI](http://travis-ci.org/basho/machi) :: 
|
||||
|
||||
Outline
|
||||
Our goal is a robust & reliable, distributed, highly available(*),
|
||||
large file store based upon write-once registers, append-only files,
|
||||
Chain Replication, and client-server style architecture. All members
|
||||
of the cluster store all of the files. Distributed load
|
||||
balancing/sharding of files is __outside__ of the scope of this
|
||||
system. However, it is a high priority that this system be able to
|
||||
integrate easily into systems that do provide distributed load
|
||||
balancing, e.g., Riak Core. Although strong consistency is a major
|
||||
feature of Chain Replication, first use cases will focus mainly on
|
||||
eventual consistency features --- strong consistency design will be
|
||||
discussed in a separate design document (read more below).
|
||||
|
||||
1. [Why another blob/file store?](#sec1)
|
||||
2. [Where to learn more about Machi](#sec2)
|
||||
3. [Development status summary](#sec3)
|
||||
4. [Contributing to Machi's development](#sec4)
|
||||
The ability for Machi to maintain strong consistency will make it
|
||||
attractive as a toolkit for building things like CORFU and Tango as
|
||||
well as better-known open source software such as Kafka's file
|
||||
replication. (See the bibliography of the [Machi high level design
|
||||
doc](./doc/high-level-machi.pdf) for further references.)
|
||||
|
||||
<a name="sec1">
|
||||
## 1. Why another blob/file store?
|
||||
(*) When operating in strong consistency mode (supporting
|
||||
sequential or linearizable semantics), the availability of the
|
||||
system is restricted to quorum majority availability. When in
|
||||
eventual consistency mode, service can be provided by any
|
||||
available server.
|
||||
|
||||
Our goal is a robust & reliable, distributed, highly available, large
|
||||
file and blob store. Such stores already exist, both in the open source world
|
||||
and in the commercial world. Why reinvent the wheel? We believe
|
||||
there are three reasons, ordered by decreasing rarity.
|
||||
## Status: mid-October 2015: work is underway
|
||||
|
||||
1. We want end-to-end checksums for all file data, from the initial
|
||||
file writer to every file reader, anywhere, all the time.
|
||||
2. We need flexibility to trade consistency for availability:
|
||||
e.g. weak consistency in exchange for being available in cases
|
||||
of partial system failure.
|
||||
3. We want to manage file replicas in a way that's provably correct
|
||||
and also easy to test.
|
||||
* The chain manager is ready for both eventual consistency use ("available
|
||||
mode") and strong consistency use ("consistent mode"). Both modes use a new
|
||||
consensus technique, Humming Consensus.
|
||||
* Scott will be
|
||||
[speaking about Humming Consensus](http://ricon.io/agenda/#managing-chain-replication-metadata-with-humming-consensus)
|
||||
at the [Ricon 2015 conference] (http://ricon.io) in San Francisco,
|
||||
CA, USA on Thursday, November 5th, 2015.
|
||||
* If you would like to run the network partition simulator
|
||||
mentioned in that Ricon presentation, please see the
|
||||
[partition simulator convergence test doc.](./doc/machi_chain_manager1_converge_demo.md)
|
||||
* Implementation of the file repair process for strong consistency
|
||||
is still in progress.
|
||||
|
||||
Criteria #3 is difficult to find in the open source world but perhaps
|
||||
not impossible.
|
||||
* All Machi client/server protocols are based on
|
||||
[Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview).
|
||||
* The current specification for Machi's protocols can be found at
|
||||
[https://github.com/basho/machi/blob/master/src/machi.proto](https://github.com/basho/machi/blob/master/src/machi.proto).
|
||||
* The Machi PB protocol is not yet stable. Expect change!
|
||||
* The Erlang language client implementation of the high-level
|
||||
protocol flavor is brittle (e.g., little error handling yet).
|
||||
|
||||
If we have app use cases where availability is more important than
|
||||
consistency, then systems that meet criteria #2 are also rare.
|
||||
Most file stores provide only strong consistency and therefore
|
||||
have unavoidable, unavailable behavior when parts of the system
|
||||
fail.
|
||||
What if we want a file store that is always available to write new
|
||||
file data and attempts best-effort file reads?
|
||||
|
||||
If we really do care about data loss and/or data corruption, then we
|
||||
really want both #3 and #1. Unfortunately, systems that meet
|
||||
criteria #1 are _very rare_. (Nonexistant?)
|
||||
Why? This is 2015. We have decades of research that shows
|
||||
that computer hardware can (and
|
||||
indeed does) corrupt data at nearly every level of the modern
|
||||
client/server application stack. Systems with end-to-end data
|
||||
corruption detection should be ubiquitous today. Alas, they are not.
|
||||
|
||||
Machi is an effort to change the deplorable state of the world, one
|
||||
Erlang function at a time.
|
||||
|
||||
<a name="sec2">
|
||||
## 2. Where to learn more about Machi
|
||||
If you'd like to work on a protocol such as Thrift, UBF,
|
||||
msgpack over UDP, or some other protocol, let us know by
|
||||
[opening an issue to discuss it](./issues/new).
|
||||
|
||||
The two major design documents for Machi are now mostly stable.
|
||||
Please see the [doc](./doc) directory's [README](./doc) for details.
|
||||
|
||||
We also have a
|
||||
[Frequently Asked Questions (FAQ) list](./FAQ.md).
|
||||
|
||||
Scott recently (November 2015) gave a presentation at the
|
||||
[RICON 2015 conference](http://ricon.io) about one of the techniques
|
||||
used by Machi; "Managing Chain Replication Metadata with
|
||||
Humming Consensus" is available online now.
|
||||
* [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf)
|
||||
* [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q)
|
||||
|
||||
See later in this document for how to run the Humming Consensus demos,
|
||||
including the network partition simulator.
|
||||
|
||||
<a name="sec3">
|
||||
## 3. Development status summary
|
||||
|
||||
Mid-March 2016: The Machi development team has been downsized in
|
||||
recent months, and the pace of development has slowed. Here is a
|
||||
summary of the status of Machi's major components.
|
||||
|
||||
* Humming Consensus and the chain manager
|
||||
* No new safety bugs have been found by model-checking tests.
|
||||
* A new document,
|
||||
[Hands-on experiments with Machi and Humming Consensus](doc/humming-consensus-demo.md)
|
||||
is now available. It is a tutorial for setting up a 3 virtual
|
||||
machine Machi cluster and how to demonstrate the chain manager's
|
||||
reactions to server stops & starts, crashes & restarts, and pauses
|
||||
(simulated by `SIGSTOP` and `SIGCONT`).
|
||||
* The chain manager can still make suboptimal-but-safe choices for
|
||||
chain transitions when a server hangs/pauses temporarily.
|
||||
* Recent chain manager changes have made the instability window
|
||||
much shorter when the slow/paused server resumes execution.
|
||||
* Scott believes that a modest change to the chain manager's
|
||||
calculation of a new projection can reduce flapping in this (and
|
||||
many other cases) less likely. Currently, the new local
|
||||
projection is calculated using only local state (i.e., the chain
|
||||
manager's internal state + the fitness server's state).
|
||||
However, if the "latest" projection read from the public
|
||||
projection stores were also input to the new projection
|
||||
calculation function, then many obviously bad projections can be
|
||||
avoided without needing rounds of Humming Consensus to
|
||||
demonstrate that a bad projection is bad.
|
||||
|
||||
* FLU/data server process
|
||||
* All known correctness bugs have been fixed.
|
||||
* Performance has not yet been measured. Performance measurement
|
||||
and enhancements are scheduled to start in the middle of March 2016.
|
||||
(This will include a much-needed update to the `basho_bench` driver.)
|
||||
|
||||
* Access protocols and client libraries
|
||||
* The protocol used by both external clients and internally (instead
|
||||
of using Erlang's native message passing mechanisms) is based on
|
||||
Protocol Buffers.
|
||||
* (Machi PB protocol specification: ./src/machi.proto)[./src/machi.proto]
|
||||
* At the moment, the PB specification contains two protocols.
|
||||
Sometime in the near future, the spec will be split to separate
|
||||
the external client API (the "high" protocol) from the internal
|
||||
communication API (the "low" protocol).
|
||||
|
||||
* Recent conference talks about Machi
|
||||
* Erlang Factory San Francisco 2016
|
||||
[the slides and video recording](http://www.erlang-factory.com/sfbay2016/scott-lystig-fritchie)
|
||||
will be available a few weeks after the conference ends on March
|
||||
11, 2016.
|
||||
* Ricon 2015
|
||||
* [The slides](http://ricon.io/archive/2015/slides/Scott_Fritchie_Ricon_2015.pdf)
|
||||
* and the [video recording](https://www.youtube.com/watch?v=yR5kHL1bu1Q&index=13&list=PL9Jh2HsAWHxIc7Tt2M6xez_TOP21GBH6M)
|
||||
are now available.
|
||||
* If you would like to run the Humming Consensus code (with or without
|
||||
the network partition simulator) as described in the RICON 2015
|
||||
presentation, please see the
|
||||
[Humming Consensus demo doc](./doc/humming_consensus_demo.md).
|
||||
|
||||
<a name="sec4">
|
||||
## 4. Contributing to Machi's development
|
||||
|
||||
### 4.1 License
|
||||
## Contributing to Machi: source code, documentation, etc.
|
||||
|
||||
Basho Technologies, Inc. as committed to licensing all work for Machi
|
||||
under the
|
||||
|
|
@ -146,29 +72,57 @@ We invite all contributors to review the
|
|||
[CONTRIBUTING.md](./CONTRIBUTING.md) document for guidelines for
|
||||
working with the Basho development team.
|
||||
|
||||
### 4.2 Development environment requirements
|
||||
## A brief survey of this directories in this repository
|
||||
|
||||
* A list of Frequently Asked Questions, a.k.a.
|
||||
[the Machi FAQ](./FAQ.md).
|
||||
|
||||
* The [doc](./doc/) directory: home for major documents about Machi:
|
||||
high level design documents as well as exploration of features still
|
||||
under design & review within Basho.
|
||||
|
||||
* The `ebin` directory: used for compiled application code
|
||||
|
||||
* The `include`, `src`, and `test` directories: contain the header
|
||||
files, source files, and test code for Machi, respectively.
|
||||
|
||||
* The [prototype](./prototype/) directory: contains proof of concept
|
||||
code, scaffolding libraries, and other exploratory code. Curious
|
||||
readers should see the [prototype/README.md](./prototype/README.md)
|
||||
file for more explanation of the small sub-projects found here.
|
||||
|
||||
## Development environment requirements
|
||||
|
||||
All development to date has been done with Erlang/OTP version 17 on OS
|
||||
X. The only known limitations for using R16 are minor type
|
||||
specification difference between R16 and 17, but we strongly suggest
|
||||
continuing development using version 17.
|
||||
|
||||
We also assume that you have the standard UNIX/Linux developer
|
||||
tool chain for C and C++ applications. Also, we assume
|
||||
that Git and GNU Make are available.
|
||||
The utility used to compile the Machi source code,
|
||||
We also assume that you have the standard UNIX/Linux developers
|
||||
tool chain for C and C++ applications. Specifically, we assume `make`
|
||||
is available. The utility used to compile the Machi source code,
|
||||
`rebar`, is pre-compiled and included in the repo.
|
||||
For more details, please see the
|
||||
[Machi development environment prerequisites doc](./doc/dev-prerequisites.md).
|
||||
|
||||
Machi has a dependency on the
|
||||
[ELevelDB](https://github.com/basho/eleveldb) library. ELevelDB only
|
||||
supports UNIX/Linux OSes and 64-bit versions of Erlang/OTP only; we
|
||||
apologize to Windows-based and 32-bit-based Erlang developers for this
|
||||
restriction.
|
||||
There are no known OS limits at this time: any platform that supports
|
||||
Erlang/OTP should be sufficient for Machi. This may change over time
|
||||
(e.g., adding NIFs which can make full portability to Windows OTP
|
||||
environments difficult), but it hasn't happened yet.
|
||||
|
||||
### 4.3 New protocols and features
|
||||
## Contributions
|
||||
|
||||
Basho encourages contributions to Riak from the community. Here’s how
|
||||
to get started.
|
||||
|
||||
* Fork the appropriate sub-projects that are affected by your change.
|
||||
* Create a topic branch for your change and checkout that branch.
|
||||
git checkout -b some-topic-branch
|
||||
* Make your changes and run the test suite if one is provided. (see below)
|
||||
* Commit your changes and push them to your fork.
|
||||
* Open pull-requests for the appropriate projects.
|
||||
* Contributors will review your pull request, suggest changes, and merge it when it’s ready and/or offer feedback.
|
||||
* To report a bug or issue, please open a new issue against this repository.
|
||||
|
||||
-The Machi team at Basho,
|
||||
[Scott Lystig Fritchie](mailto:scott@basho.com), technical lead, and
|
||||
[Matt Brender](mailto:mbrender@basho.com), your developer advocate.
|
||||
|
||||
If you'd like to work on a protocol such as Thrift, UBF,
|
||||
msgpack over UDP, or some other protocol, let us know by
|
||||
[opening an issue to discuss it](./issues/new).
|
||||
|
|
|
|||
|
|
@ -11,5 +11,6 @@ Unknown types:
|
|||
######## Specific messages #####################
|
||||
##################################################
|
||||
machi_chain_manager1.erl:2473: The created fun has no local return
|
||||
machi_chain_manager1.erl:2184: The pattern <_P1, P2, Else = {'expected_author2', UPI1_tail, _}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'>
|
||||
machi_chain_manager1.erl:2233: The pattern <_P1 = {'projection_v1', _, _, _, _, _, _, _, 'cp_mode', UPI1, Repairing1, _, _, _, _}, _P2 = {'projection_v1', _, _, _, _, _, _, _, 'cp_mode', UPI2, Repairing2, _, _, _, _}, Else = {'epoch_not_si', EpochX, 'not_gt', EpochY}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'>
|
||||
machi_chain_manager1.erl:2102: The pattern <_P1, P2, Else = {'expected_author2', UPI1_tail, _}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom() | binary(),all_members::'undefined' | [atom() | binary()],witnesses::[atom() | binary()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom() | binary()],repairing::'undefined' | [atom() | binary()],down::'undefined' | [atom() | binary()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom() | binary(),all_members::'undefined' | [atom() | binary()],witnesses::[atom() | binary()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom() | binary()],repairing::'undefined' | [atom() | binary()],down::'undefined' | [atom() | binary()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'>
|
||||
machi_chain_manager1.erl:2151: The pattern <_P1 = {'projection_v1', _, _, _, _, _, _, 'cp_mode', UPI1, Repairing1, _, _, _, _}, _P2 = {'projection_v1', _, _, _, _, _, _, 'cp_mode', UPI2, Repairing2, _, _, _, _}, Else = {'epoch_not_si', EpochX, 'not_gt', EpochY}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom() | binary(),all_members::'undefined' | [atom() | binary()],witnesses::[atom() | binary()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom() | binary()],repairing::'undefined' | [atom() | binary()],down::'undefined' | [atom() | binary()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom() | binary(),all_members::'undefined' | [atom() | binary()],witnesses::[atom() | binary()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom() | binary()],repairing::'undefined' | [atom() | binary()],down::'undefined' | [atom() | binary()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'>
|
||||
machi_flu1.erl:246: The created fun has no local return
|
||||
|
|
|
|||
|
|
@ -6,6 +6,20 @@ Erlang documentation, please use this link:
|
|||
|
||||
## Documents in this directory
|
||||
|
||||
### chain-self-management-sketch.org
|
||||
|
||||
[chain-self-management-sketch.org](chain-self-management-sketch.org)
|
||||
is a mostly-deprecated draft of
|
||||
an introduction to the
|
||||
self-management algorithm proposed for Machi. Most material has been
|
||||
moved to the [high-level-chain-mgr.pdf](high-level-chain-mgr.pdf) document.
|
||||
|
||||
### cluster-of-clusters (directory)
|
||||
|
||||
This directory contains the sketch of the "cluster of clusters" design
|
||||
strawman for partitioning/distributing/sharding files across a large
|
||||
number of independent Machi clusters.
|
||||
|
||||
### high-level-machi.pdf
|
||||
|
||||
[high-level-machi.pdf](high-level-machi.pdf)
|
||||
|
|
@ -36,9 +50,9 @@ introduction to the Humming Consensus algorithm. Its abstract:
|
|||
> of file updates to all replica servers in a Machi cluster. Chain
|
||||
> Replication is a variation of primary/backup replication where the
|
||||
> order of updates between the primary server and each of the backup
|
||||
> servers is strictly ordered into a single "chain". Management of
|
||||
> Chain Replication's metadata, e.g., "What is the current order of
|
||||
> servers in the chain?", remains an open research problem. The
|
||||
> servers is strictly ordered into a single ``chain''. Management of
|
||||
> Chain Replication's metadata, e.g., ``What is the current order of
|
||||
> servers in the chain?'', remains an open research problem. The
|
||||
> current state of the art for Chain Replication metadata management
|
||||
> relies on an external oracle (e.g., ZooKeeper) or the Elastic
|
||||
> Replication algorithm.
|
||||
|
|
@ -46,7 +60,7 @@ introduction to the Humming Consensus algorithm. Its abstract:
|
|||
> This document describes the Machi chain manager, the component
|
||||
> responsible for managing Chain Replication metadata state. The chain
|
||||
> manager uses a new technique, based on a variation of CORFU, called
|
||||
> "humming consensus".
|
||||
> ``humming consensus''.
|
||||
> Humming consensus does not require active participation by all or even
|
||||
> a majority of participants to make decisions. Machi's chain manager
|
||||
> bases its logic on humming consensus to make decisions about how to
|
||||
|
|
@ -57,18 +71,3 @@ introduction to the Humming Consensus algorithm. Its abstract:
|
|||
> decision during that epoch. When a differing decision is discovered,
|
||||
> new time epochs are proposed in which a new consensus is reached and
|
||||
> disseminated to all available participants.
|
||||
|
||||
### chain-self-management-sketch.org
|
||||
|
||||
[chain-self-management-sketch.org](chain-self-management-sketch.org)
|
||||
is a mostly-deprecated draft of
|
||||
an introduction to the
|
||||
self-management algorithm proposed for Machi. Most material has been
|
||||
moved to the [high-level-chain-mgr.pdf](high-level-chain-mgr.pdf) document.
|
||||
|
||||
### cluster (directory)
|
||||
|
||||
This directory contains the sketch of the cluster design
|
||||
strawman for partitioning/distributing/sharding files across a large
|
||||
number of independent Machi chains.
|
||||
|
||||
|
|
|
|||
|
|
@ -88,16 +88,16 @@ Single
|
|||
4 0 0 50 -1 2 14 0.0000 4 180 495 4425 3525 ~8%\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 240 1710 5025 3525 ~25% total keys\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 180 495 6825 3525 ~8%\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1485 600 600 Cluster1\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1485 3000 600 Cluster2\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1485 5400 600 Cluster3\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1485 300 2850 Cluster1\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1485 2700 2850 Cluster2\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1485 5175 2850 Cluster3\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 405 2100 2625 Cl\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 405 6900 2625 Cl\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 195 2175 3075 4\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 195 4575 3075 4\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 195 6975 3075 4\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 600 600 Chain1\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 3000 600 Chain2\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 5400 600 Chain3\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 285 2100 2625 C\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 285 4500 2625 C\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 285 6900 2625 C\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 525 2850 Chain1\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 2925 2850 Chain2\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 5325 2850 Chain3\001
|
||||
4 0 0 50 -1 2 18 0.0000 4 240 4350 1350 4875 Cluster locator, on the unit interval\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 405 4500 2625 Cl\001
|
||||
4 0 0 50 -1 2 18 0.0000 4 240 3990 1200 4875 CoC locator, on the unit interval\001
|
||||
BIN
doc/cluster-of-clusters/migration-3to4.png
Normal file
BIN
doc/cluster-of-clusters/migration-3to4.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 7.7 KiB |
BIN
doc/cluster-of-clusters/migration-4.png
Normal file
BIN
doc/cluster-of-clusters/migration-4.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 7.7 KiB |
427
doc/cluster-of-clusters/name-game-sketch.org
Normal file
427
doc/cluster-of-clusters/name-game-sketch.org
Normal file
|
|
@ -0,0 +1,427 @@
|
|||
-*- mode: org; -*-
|
||||
#+TITLE: Machi cluster-of-clusters "name game" sketch
|
||||
#+AUTHOR: Scott
|
||||
#+STARTUP: lognotedone hidestars indent showall inlineimages
|
||||
#+SEQ_TODO: TODO WORKING WAITING DONE
|
||||
#+COMMENT: M-x visual-line-mode
|
||||
#+COMMENT: Also, disable auto-fill-mode
|
||||
|
||||
* 1. "Name Games" with random-slicing style consistent hashing
|
||||
|
||||
Our goal: to distribute lots of files very evenly across a cluster of
|
||||
Machi clusters (hereafter called a "cluster of clusters" or "CoC").
|
||||
|
||||
* 2. Assumptions
|
||||
|
||||
** Basic familiarity with Machi high level design and Machi's "projection"
|
||||
|
||||
The [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] contains all of the basic
|
||||
background assumed by the rest of this document.
|
||||
|
||||
** Analogy: "neighborhood : city :: Machi : cluster-of-clusters"
|
||||
|
||||
Analogy: The word "machi" in Japanese means small town or
|
||||
neighborhood. As the Tokyo Metropolitan Area is built from many
|
||||
machis and smaller cities, therefore a big, partitioned file store can
|
||||
be built out of many small Machi clusters.
|
||||
|
||||
** Familiarity with the Machi cluster-of-clusters/CoC concept
|
||||
|
||||
It's clear (I hope!) from
|
||||
the [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] that Machi alone does not support
|
||||
any kind of file partitioning/distribution/sharding across multiple
|
||||
small Machi clusters. There must be another layer above a Machi cluster to
|
||||
provide such partitioning services.
|
||||
|
||||
The name "cluster of clusters" originated within Basho to avoid
|
||||
conflicting use of the word "cluster". A Machi cluster is usually
|
||||
synonymous with a single Chain Replication chain and a single set of
|
||||
machines (e.g. 2-5 machines). However, in the not-so-far future, we
|
||||
expect much more complicated patterns of Chain Replication to be used
|
||||
in real-world deployments.
|
||||
|
||||
"Cluster of clusters" is clunky and long, but we haven't found a good
|
||||
substitute yet. If you have a good suggestion, please contact us!
|
||||
~^_^~
|
||||
|
||||
Using the [[https://github.com/basho/machi/tree/master/prototype/demo-day-hack][cluster-of-clusters quick-and-dirty prototype]] as an
|
||||
architecture sketch, let's now assume that we have ~n~ independent Machi
|
||||
clusters. We assume that each of these clusters has roughly the same
|
||||
chain length in the nominal case, e.g. chain length of 3.
|
||||
We wish to provide partitioned/distributed file storage
|
||||
across all ~n~ clusters. We call the entire collection of ~n~ Machi
|
||||
clusters a "cluster of clusters", or abbreviated "CoC".
|
||||
|
||||
We may wish to have several types of Machi clusters, e.g. chain length
|
||||
of 3 for normal data, longer for cannot-afford-data-loss files, and
|
||||
shorter for don't-care-if-it-gets-lost files. Each of these types of
|
||||
chains will have a name ~N~ in the CoC namespace. The role of the CoC
|
||||
namespace will be demonstrated in Section 3 below.
|
||||
|
||||
** Continue CoC prototype's assumption: a Machi cluster is unaware of CoC
|
||||
|
||||
Let's continue with an assumption that an individual Machi cluster
|
||||
inside of the cluster-of-clusters is completely unaware of the
|
||||
cluster-of-clusters layer.
|
||||
|
||||
TODO: We may need to break this assumption sometime in the future?
|
||||
|
||||
** The reader is familiar with the random slicing technique
|
||||
|
||||
I'd done something very-very-nearly-identical for the Hibari database
|
||||
6 years ago. But the Hibari technique was based on stuff I did at
|
||||
Sendmail, Inc, so it felt old news to me. {shrug}
|
||||
|
||||
The Hibari documentation has a brief photo illustration of how random
|
||||
slicing works, see [[http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration][Hibari Sysadmin Guide, chain migration]]
|
||||
|
||||
For a comprehensive description, please see these two papers:
|
||||
|
||||
#+BEGIN_QUOTE
|
||||
Reliable and Randomized Data Distribution Strategies for Large Scale Storage Systems
|
||||
Alberto Miranda et al.
|
||||
http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.226.5609
|
||||
(short version, HIPC'11)
|
||||
|
||||
Random Slicing: Efficient and Scalable Data Placement for Large-Scale
|
||||
Storage Systems
|
||||
Alberto Miranda et al.
|
||||
DOI: http://dx.doi.org/10.1145/2632230 (long version, ACM Transactions
|
||||
on Storage, Vol. 10, No. 3, Article 9, 2014)
|
||||
#+END_QUOTE
|
||||
|
||||
** CoC locator: We borrow from random slicing but do not hash any strings!
|
||||
|
||||
We will use the general technique of random slicing, but we adapt the
|
||||
technique to fit our use case.
|
||||
|
||||
In general, random slicing says:
|
||||
|
||||
- Hash a string onto the unit interval [0.0, 1.0)
|
||||
- Calculate h(unit interval point, Map) -> bin, where ~Map~ partitions
|
||||
the unit interval into bins.
|
||||
|
||||
Our adaptation is in step 1: we do not hash any strings. Instead, we
|
||||
store & use the unit interval point as-is, without using a hash
|
||||
function in this step. This number is called the "CoC locator".
|
||||
|
||||
As described later in this doc, Machi file names are structured into
|
||||
several components. One component of the file name contains the "CoC
|
||||
locator"; we use the number as-is for step 2 above.
|
||||
|
||||
* 3. A simple illustration
|
||||
|
||||
We use a variation of the Random Slicing hash that we will call
|
||||
~rs_hash_with_float()~. The Erlang-style function type is shown
|
||||
below.
|
||||
|
||||
#+BEGIN_SRC erlang
|
||||
%% type specs, Erlang-style
|
||||
-spec rs_hash_with_float(float(), rs_hash:map()) -> rs_hash:cluster_id().
|
||||
#+END_SRC
|
||||
|
||||
I'm borrowing an illustration from the HibariDB documentation here,
|
||||
but it fits my purposes quite well. (I am the original creator of that
|
||||
image, and also the use license is compatible.)
|
||||
|
||||
#+CAPTION: Illustration of 'Map', using four Machi clusters
|
||||
|
||||
[[./migration-4.png]]
|
||||
|
||||
Assume that we have a random slicing map called ~Map~. This particular
|
||||
~Map~ maps the unit interval onto 4 Machi clusters:
|
||||
|
||||
| Hash range | Cluster ID |
|
||||
|-------------+------------|
|
||||
| 0.00 - 0.25 | Cluster1 |
|
||||
| 0.25 - 0.33 | Cluster4 |
|
||||
| 0.33 - 0.58 | Cluster2 |
|
||||
| 0.58 - 0.66 | Cluster4 |
|
||||
| 0.66 - 0.91 | Cluster3 |
|
||||
| 0.91 - 1.00 | Cluster4 |
|
||||
|
||||
Assume that the system chooses a CoC locator of 0.05.
|
||||
According to ~Map~, the value of
|
||||
~rs_hash_with_float(0.05,Map) = Cluster1~.
|
||||
Similarly, ~rs_hash_with_float(0.26,Map) = Cluster4~.
|
||||
|
||||
* 4. An additional assumption: clients will want some control over file location
|
||||
|
||||
We will continue to use the 4-cluster diagram from the previous
|
||||
section.
|
||||
|
||||
** Our new assumption: client control over initial file location
|
||||
|
||||
The CoC management scheme may decide that files need to migrate to
|
||||
other clusters. The reason could be for storage load or I/O load
|
||||
balancing reasons. It could be because a cluster is being
|
||||
decommissioned by its owners. There are many legitimate reasons why a
|
||||
file that is initially created on cluster ID X has been moved to
|
||||
cluster ID Y.
|
||||
|
||||
However, there are also legitimate reasons for why the client would want
|
||||
control over the choice of Machi cluster when the data is first
|
||||
written. The single biggest reason is load balancing. Assuming that
|
||||
the client (or the CoC management layer acting on behalf of the CoC
|
||||
client) knows the current utilization across the participating Machi
|
||||
clusters, then it may be very helpful to send new append() requests to
|
||||
under-utilized clusters.
|
||||
|
||||
* 5. Use of the CoC namespace: name separation plus chain type
|
||||
|
||||
Let us assume that the CoC framework provides several different types
|
||||
of chains:
|
||||
|
||||
| Chain length | CoC namespace | Mode | Comment |
|
||||
|--------------+---------------+------+----------------------------------|
|
||||
| 3 | normal | AP | Normal storage redundancy & cost |
|
||||
| 2 | cheap | AP | Reduced cost storage |
|
||||
| 1 | risky | AP | Really cheap storage |
|
||||
| 9 | paranoid | AP | Safety-critical storage |
|
||||
| 3 | sequential | CP | Strong consistency |
|
||||
|--------------+---------------+------+----------------------------------|
|
||||
|
||||
The client may want to choose the amount of redundancy that its
|
||||
application requires: normal, reduced cost, or perhaps even a single
|
||||
copy. The CoC namespace is used by the client to signal this
|
||||
intention.
|
||||
|
||||
Further, the CoC administrators may wish to use the namespace to
|
||||
provide separate storage for different applications. Jane's
|
||||
application may use the namespace "jane-normal" and Bob's app uses
|
||||
"bob-cheap". The CoC administrators may definite separate groups of
|
||||
chains on separate servers to serve these two applications.
|
||||
|
||||
* 6. Floating point is not required ... it is merely convenient for explanation
|
||||
|
||||
NOTE: Use of floating point terms is not required. For example,
|
||||
integer arithmetic could be used, if using a sufficiently large
|
||||
interval to create an even & smooth distribution of hashes across the
|
||||
expected maximum number of clusters.
|
||||
|
||||
For example, if the maximum CoC cluster size would be 4,000 individual
|
||||
Machi clusters, then a minimum of 12 bits of integer space is required
|
||||
to assign one integer per Machi cluster. However, for load balancing
|
||||
purposes, a finer grain of (for example) 100 integers per Machi
|
||||
cluster would permit file migration to move increments of
|
||||
approximately 1% of single Machi cluster's storage capacity. A
|
||||
minimum of 12+7=19 bits of hash space would be necessary to accommodate
|
||||
these constraints.
|
||||
|
||||
It is likely that Machi's final implementation will choose a 24 bit
|
||||
integer to represent the CoC locator.
|
||||
|
||||
* 7. Proposal: Break the opacity of Machi file names
|
||||
|
||||
Machi assigns file names based on:
|
||||
|
||||
~ClientSuppliedPrefix ++ "^" ++ SomeOpaqueFileNameSuffix~
|
||||
|
||||
What if the CoC client could peek inside of the opaque file name
|
||||
suffix in order to remove (or add) the CoC location information that
|
||||
we need?
|
||||
|
||||
** The notation we use
|
||||
|
||||
- ~T~ = the target CoC member/Cluster ID chosen by the CoC client at the time of ~append()~
|
||||
- ~p~ = file prefix, chosen by the CoC client.
|
||||
- ~L~ = the CoC locator
|
||||
- ~N~ = the CoC namespace
|
||||
- ~u~ = the Machi file server unique opaque file name suffix, e.g. a GUID string
|
||||
- ~F~ = a Machi file name, i.e., ~p^L^N^u~
|
||||
|
||||
** The details: CoC file write
|
||||
|
||||
1. CoC client chooses ~p~, ~T~, and ~N~ (i.e., the file prefix, target
|
||||
cluster, and target cluster namespace)
|
||||
2. CoC client knows the CoC ~Map~ for namespace ~N~.
|
||||
3. CoC client choose some CoC locator value ~L~ such that
|
||||
~rs_hash_with_float(L,Map) = T~ (see below).
|
||||
4. CoC client sends its request to cluster
|
||||
~T~: ~append_chunk(p,L,N,...) -> {ok,p^L^N^u,ByteOffset}~
|
||||
5. CoC stores/uses the file name ~F = p^L^N^u~.
|
||||
|
||||
** The details: CoC file read
|
||||
|
||||
1. CoC client knows the file name ~F~ and parses it to find
|
||||
the values of ~L~ and ~N~ (recall, ~F = p^L^N^u~).
|
||||
2. CoC client knows the CoC ~Map~ for type ~N~.
|
||||
3. CoC calculates ~rs_hash_with_float(L,Map) = T~
|
||||
4. CoC client sends request to cluster ~T~: ~read_chunk(F,...) ->~ ... success!
|
||||
|
||||
** The details: calculating 'L' (the CoC locator) to match a desired target cluster
|
||||
|
||||
1. We know ~Map~, the current CoC mapping for a CoC namespace ~N~.
|
||||
2. We look inside of ~Map~, and we find all of the unit interval ranges
|
||||
that map to our desired target cluster ~T~. Let's call this list
|
||||
~MapList = [Range1=(start,end],Range2=(start,end],...]~.
|
||||
3. In our example, ~T=Cluster2~. The example ~Map~ contains a single
|
||||
unit interval range for ~Cluster2~, ~[(0.33,0.58]]~.
|
||||
4. Choose a uniformly random number ~r~ on the unit interval.
|
||||
5. Calculate locator ~L~ by mapping ~r~ onto the concatenation
|
||||
of the CoC hash space range intervals in ~MapList~. For example,
|
||||
if ~r=0.5~, then ~L = 0.33 + 0.5*(0.58-0.33) = 0.455~, which is
|
||||
exactly in the middle of the ~(0.33,0.58]~ interval.
|
||||
|
||||
* 8. File migration (a.k.a. rebalancing/reparitioning/resharding/redistribution)
|
||||
|
||||
** What is "migration"?
|
||||
|
||||
This section describes Machi's file migration. Other storage systems
|
||||
call this process as "rebalancing", "repartitioning", "resharding" or
|
||||
"redistribution".
|
||||
For Riak Core applications, it is called "handoff" and "ring resizing"
|
||||
(depending on the context).
|
||||
See also the [[http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsUserGuide.html#Balancer][Hadoop file balancer]] for another example of a data
|
||||
migration process.
|
||||
|
||||
As discussed in section 5, the client can have good reason for wanting
|
||||
to have some control of the initial location of the file within the
|
||||
cluster. However, the cluster manager has an ongoing interest in
|
||||
balancing resources throughout the lifetime of the file. Disks will
|
||||
get full, hardware will change, read workload will fluctuate,
|
||||
etc etc.
|
||||
|
||||
This document uses the word "migration" to describe moving data from
|
||||
one Machi chain to another within a CoC system.
|
||||
|
||||
A simple variation of the Random Slicing hash algorithm can easily
|
||||
accommodate Machi's need to migrate files without interfering with
|
||||
availability. Machi's migration task is much simpler due to the
|
||||
immutable nature of Machi file data.
|
||||
|
||||
** Change to Random Slicing
|
||||
|
||||
The map used by the Random Slicing hash algorithm needs a few simple
|
||||
changes to make file migration straightforward.
|
||||
|
||||
- Add a "generation number", a strictly increasing number (similar to
|
||||
a Machi cluster's "epoch number") that reflects the history of
|
||||
changes made to the Random Slicing map
|
||||
- Use a list of Random Slicing maps instead of a single map, one map
|
||||
per chance that files may not have been migrated yet out of
|
||||
that map.
|
||||
|
||||
As an example:
|
||||
|
||||
#+CAPTION: Illustration of 'Map', using four Machi clusters
|
||||
|
||||
[[./migration-3to4.png]]
|
||||
|
||||
And the new Random Slicing map for some CoC namespace ~N~ might look
|
||||
like this:
|
||||
|
||||
| Generation number / Namespace | 7 / cheap |
|
||||
|-------------------------------+------------|
|
||||
| SubMap | 1 |
|
||||
|-------------------------------+------------|
|
||||
| Hash range | Cluster ID |
|
||||
|-------------------------------+------------|
|
||||
| 0.00 - 0.33 | Cluster1 |
|
||||
| 0.33 - 0.66 | Cluster2 |
|
||||
| 0.66 - 1.00 | Cluster3 |
|
||||
|-------------------------------+------------|
|
||||
| SubMap | 2 |
|
||||
|-------------------------------+------------|
|
||||
| Hash range | Cluster ID |
|
||||
|-------------------------------+------------|
|
||||
| 0.00 - 0.25 | Cluster1 |
|
||||
| 0.25 - 0.33 | Cluster4 |
|
||||
| 0.33 - 0.58 | Cluster2 |
|
||||
| 0.58 - 0.66 | Cluster4 |
|
||||
| 0.66 - 0.91 | Cluster3 |
|
||||
| 0.91 - 1.00 | Cluster4 |
|
||||
|
||||
When a new Random Slicing map contains a single submap, then its use
|
||||
is identical to the original Random Slicing algorithm. If the map
|
||||
contains multiple submaps, then the access rules change a bit:
|
||||
|
||||
- Write operations always go to the newest/largest submap.
|
||||
- Read operations attempt to read from all unique submaps.
|
||||
- Skip searching submaps that refer to the same cluster ID.
|
||||
- In this example, unit interval value 0.10 is mapped to Cluster1
|
||||
by both submaps.
|
||||
- Read from newest/largest submap to oldest/smallest submap.
|
||||
- If not found in any submap, search a second time (to handle races
|
||||
with file copying between submaps).
|
||||
- If the requested data is found, optionally copy it directly to the
|
||||
newest submap. (This is a variation of read repair (RR). RR here
|
||||
accelerates the migration process and can reduce the number of
|
||||
operations required to query servers in multiple submaps).
|
||||
|
||||
The cluster-of-clusters manager is responsible for:
|
||||
|
||||
- Managing the various generations of the CoC Random Slicing maps for
|
||||
all namespaces.
|
||||
- Distributing namespace maps to CoC clients.
|
||||
- Managing the processes that are responsible for copying "cold" data,
|
||||
i.e., files data that is not regularly accessed, to its new submap
|
||||
location.
|
||||
- When migration of a file to its new cluster is confirmed successful,
|
||||
delete it from the old cluster.
|
||||
|
||||
In example map #7, the CoC manager will copy files with unit interval
|
||||
assignments in ~(0.25,0.33]~, ~(0.58,0.66]~, and ~(0.91,1.00]~ from their
|
||||
old locations in cluster IDs Cluster1/2/3 to their new cluster,
|
||||
Cluster4. When the CoC manager is satisfied that all such files have
|
||||
been copied to Cluster4, then the CoC manager can create and
|
||||
distribute a new map, such as:
|
||||
|
||||
| Generation number / Namespace | 8 / cheap |
|
||||
|-------------------------------+------------|
|
||||
| SubMap | 1 |
|
||||
|-------------------------------+------------|
|
||||
| Hash range | Cluster ID |
|
||||
|-------------------------------+------------|
|
||||
| 0.00 - 0.25 | Cluster1 |
|
||||
| 0.25 - 0.33 | Cluster4 |
|
||||
| 0.33 - 0.58 | Cluster2 |
|
||||
| 0.58 - 0.66 | Cluster4 |
|
||||
| 0.66 - 0.91 | Cluster3 |
|
||||
| 0.91 - 1.00 | Cluster4 |
|
||||
|
||||
The HibariDB system performs data migrations in almost exactly this
|
||||
manner. However, one important
|
||||
limitation of HibariDB is not being able to
|
||||
perform more than one migration at a time. HibariDB's data is
|
||||
mutable, and mutation causes many problems already when migrating data
|
||||
across two submaps; three or more submaps was too complex to implement
|
||||
quickly.
|
||||
|
||||
Fortunately for Machi, its file data is immutable and therefore can
|
||||
easily manage many migrations in parallel, i.e., its submap list may
|
||||
be several maps long, each one for an in-progress file migration.
|
||||
|
||||
* 9. Other considerations for FLU/sequencer implementations
|
||||
|
||||
** Append to existing file when possible
|
||||
|
||||
In the earliest Machi FLU implementation, it was impossible to append
|
||||
to the same file after ~30 seconds. For example:
|
||||
|
||||
- Client: ~append(prefix="foo",...) -> {ok,"foo^suffix1",Offset1}~
|
||||
- Client: ~append(prefix="foo",...) -> {ok,"foo^suffix1",Offset2}~
|
||||
- Client: ~append(prefix="foo",...) -> {ok,"foo^suffix1",Offset3}~
|
||||
- Client: sleep 40 seconds
|
||||
- Server: after 30 seconds idle time, stop Erlang server process for
|
||||
the ~"foo^suffix1"~ file
|
||||
- Client: ...wakes up...
|
||||
- Client: ~append(prefix="foo",...) -> {ok,"foo^suffix2",Offset4}~
|
||||
|
||||
Our ideal append behavior is to always append to the same file. Why?
|
||||
It would be nice if Machi didn't create zillions of tiny files if the
|
||||
client appends to some prefix very infrequently. In general, it is
|
||||
better to create fewer & bigger files by re-using a Machi file name
|
||||
when possible.
|
||||
|
||||
The sequencer should always assign new offsets to the latest/newest
|
||||
file for any prefix, as long as all prerequisites are also true,
|
||||
|
||||
- The epoch has not changed. (In AP mode, epoch change -> mandatory file name suffix change.)
|
||||
- The latest file for prefix ~p~ is smaller than maximum file size for a FLU's configuration.
|
||||
|
||||
* 10. Acknowledgments
|
||||
|
||||
The source for the "migration-4.png" and "migration-3to4.png" images
|
||||
come from the [[http://hibari.github.io/hibari-doc/images/migration-3to4.png][HibariDB documentation]].
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 7.6 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 7.4 KiB |
|
|
@ -1,481 +0,0 @@
|
|||
-*- mode: org; -*-
|
||||
#+TITLE: Machi cluster "name game" sketch
|
||||
#+AUTHOR: Scott
|
||||
#+STARTUP: lognotedone hidestars indent showall inlineimages
|
||||
#+SEQ_TODO: TODO WORKING WAITING DONE
|
||||
#+COMMENT: M-x visual-line-mode
|
||||
#+COMMENT: Also, disable auto-fill-mode
|
||||
|
||||
* 1. "Name Games" with random-slicing style consistent hashing
|
||||
|
||||
Our goal: to distribute lots of files very evenly across a large
|
||||
collection of individual, small Machi chains.
|
||||
|
||||
* 2. Assumptions
|
||||
|
||||
** Basic familiarity with Machi high level design and Machi's "projection"
|
||||
|
||||
The [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] contains all of the basic
|
||||
background assumed by the rest of this document.
|
||||
|
||||
** Analogy: "neighborhood : city :: Machi chain : Machi cluster"
|
||||
|
||||
Analogy: The word "machi" in Japanese means small town or
|
||||
neighborhood. As the Tokyo Metropolitan Area is built from many
|
||||
machis and smaller cities, therefore a big, partitioned file store can
|
||||
be built out of many small Machi chains.
|
||||
|
||||
** Familiarity with the Machi chain concept
|
||||
|
||||
It's clear (I hope!) from
|
||||
the [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] that Machi alone does not support
|
||||
any kind of file partitioning/distribution/sharding across multiple
|
||||
small Machi chains. There must be another layer above a Machi chain to
|
||||
provide such partitioning services.
|
||||
|
||||
Using the [[https://github.com/basho/machi/tree/master/prototype/demo-day-hack][cluster quick-and-dirty prototype]] as an
|
||||
architecture sketch, let's now assume that we have ~n~ independent Machi
|
||||
chains. We assume that each of these chains has the same
|
||||
chain length in the nominal case, e.g. chain length of 3.
|
||||
We wish to provide partitioned/distributed file storage
|
||||
across all ~n~ chains. We call the entire collection of ~n~ Machi
|
||||
chains a "cluster".
|
||||
|
||||
We may wish to have several types of Machi clusters. For example:
|
||||
|
||||
+ Chain length of 1 for "don't care if it gets lost,
|
||||
store stuff very very cheaply" data.
|
||||
+ Chain length of 2 for normal data.
|
||||
+ Equivalent to quorum replication's reliability with 3 copies.
|
||||
+ Chain length of 7 for critical, unreplaceable data.
|
||||
+ Equivalent to quorum replication's reliability with 15 copies.
|
||||
|
||||
Each of these types of chains will have a name ~N~ in the
|
||||
namespace. The role of the cluster namespace will be demonstrated in
|
||||
Section 3 below.
|
||||
|
||||
** Continue an early assumption: a Machi chain is unaware of clustering
|
||||
|
||||
Let's continue with an assumption that an individual Machi chain
|
||||
inside of a cluster is completely unaware of the cluster layer.
|
||||
|
||||
** The reader is familiar with the random slicing technique
|
||||
|
||||
I'd done something very-very-nearly-like-this for the Hibari database
|
||||
6 years ago. But the Hibari technique was based on stuff I did at
|
||||
Sendmail, Inc, in 2000, so this technique feels like old news to me.
|
||||
{shrug}
|
||||
|
||||
The following section provides an illustrated example.
|
||||
Very quickly, the random slicing algorithm is:
|
||||
|
||||
- Hash a string onto the unit interval [0.0, 1.0)
|
||||
- Calculate h(unit interval point, Map) -> bin, where ~Map~ divides
|
||||
the unit interval into bins (or partitions or shards).
|
||||
|
||||
Machi's adaptation is in step 1: we do not hash any strings. Instead, we
|
||||
simply choose a number on the unit interval. This number is called
|
||||
the "cluster locator number".
|
||||
|
||||
As described later in this doc, Machi file names are structured into
|
||||
several components. One component of the file name contains the cluster
|
||||
locator number; we use the number as-is for step 2 above.
|
||||
|
||||
*** For more information about Random Slicing
|
||||
|
||||
For a comprehensive description of random slicing, please see the
|
||||
first two papers. For a quicker summary, please see the third
|
||||
reference.
|
||||
|
||||
#+BEGIN_QUOTE
|
||||
Reliable and Randomized Data Distribution Strategies for Large Scale Storage Systems
|
||||
Alberto Miranda et al.
|
||||
http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.226.5609
|
||||
(short version, HIPC'11)
|
||||
|
||||
Random Slicing: Efficient and Scalable Data Placement for Large-Scale
|
||||
Storage Systems
|
||||
Alberto Miranda et al.
|
||||
DOI: http://dx.doi.org/10.1145/2632230 (long version, ACM Transactions
|
||||
on Storage, Vol. 10, No. 3, Article 9, 2014)
|
||||
|
||||
[[http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration][Hibari Sysadmin Guide, chain migration section]].
|
||||
http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration
|
||||
#+END_QUOTE
|
||||
|
||||
* 3. A simple illustration
|
||||
|
||||
We use a variation of the Random Slicing hash that we will call
|
||||
~rs_hash_with_float()~. The Erlang-style function type is shown
|
||||
below.
|
||||
|
||||
#+BEGIN_SRC erlang
|
||||
%% type specs, Erlang-style
|
||||
-spec rs_hash_with_float(float(), rs_hash:map()) -> rs_hash:chain_id().
|
||||
#+END_SRC
|
||||
|
||||
I'm borrowing an illustration from the HibariDB documentation here,
|
||||
but it fits my purposes quite well. (I am the original creator of that
|
||||
image, and also the use license is compatible.)
|
||||
|
||||
#+CAPTION: Illustration of 'Map', using four Machi chains
|
||||
|
||||
[[./migration-4.png]]
|
||||
|
||||
Assume that we have a random slicing map called ~Map~. This particular
|
||||
~Map~ maps the unit interval onto 4 Machi chains:
|
||||
|
||||
| Hash range | Chain ID |
|
||||
|-------------+----------|
|
||||
| 0.00 - 0.25 | Chain1 |
|
||||
| 0.25 - 0.33 | Chain4 |
|
||||
| 0.33 - 0.58 | Chain2 |
|
||||
| 0.58 - 0.66 | Chain4 |
|
||||
| 0.66 - 0.91 | Chain3 |
|
||||
| 0.91 - 1.00 | Chain4 |
|
||||
|
||||
Assume that the system chooses a cluster locator of 0.05.
|
||||
According to ~Map~, the value of
|
||||
~rs_hash_with_float(0.05,Map) = Chain1~.
|
||||
Similarly, ~rs_hash_with_float(0.26,Map) = Chain4~.
|
||||
|
||||
This example should look very similar to Hibari's technique.
|
||||
The Hibari documentation has a brief photo illustration of how random
|
||||
slicing works, see [[http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration][Hibari Sysadmin Guide, chain migration]].
|
||||
|
||||
* 4. Use of the cluster namespace: name separation plus chain type
|
||||
|
||||
Let us assume that the cluster framework provides several different types
|
||||
of chains:
|
||||
|
||||
| Chain length | Namespace | Consistency Mode | Comment |
|
||||
|--------------+--------------+------------------+----------------------------------|
|
||||
| 3 | ~normal~ | eventual | Normal storage redundancy & cost |
|
||||
| 2 | ~reduced~ | eventual | Reduced cost storage |
|
||||
| 1 | ~risky~ | eventual | Really, really cheap storage |
|
||||
| 7 | ~paranoid~ | eventual | Safety-critical storage |
|
||||
| 3 | ~sequential~ | strong | Strong consistency |
|
||||
|--------------+--------------+------------------+----------------------------------|
|
||||
|
||||
The client may want to choose the amount of redundancy that its
|
||||
application requires: normal, reduced cost, or perhaps even a single
|
||||
copy. The cluster namespace is used by the client to signal this
|
||||
intention.
|
||||
|
||||
Further, the cluster administrators may wish to use the namespace to
|
||||
provide separate storage for different applications. Jane's
|
||||
application may use the namespace "jane-normal" and Bob's app uses
|
||||
"bob-reduced". Administrators may definine separate groups of
|
||||
chains on separate servers to serve these two applications.
|
||||
|
||||
* 5. In its lifetime, a file may be moved to different chains
|
||||
|
||||
The cluster management scheme may decide that files need to migrate to
|
||||
other chains -- i.e., file that is initially created on chain ID ~X~
|
||||
has been moved to chain ID ~Y~.
|
||||
|
||||
+ For storage load or I/O load balancing reasons.
|
||||
+ Because a chain is being decommissioned by the sysadmin.
|
||||
|
||||
* 6. Floating point is not required ... it is merely convenient for explanation
|
||||
|
||||
NOTE: Use of floating point terms is not required. For example,
|
||||
integer arithmetic could be used, if using a sufficiently large
|
||||
interval to create an even & smooth distribution of hashes across the
|
||||
expected maximum number of chains.
|
||||
|
||||
For example, if the maximum cluster size would be 4,000 individual
|
||||
Machi chains, then a minimum of 12 bits of integer space is required
|
||||
to assign one integer per Machi chain. However, for load balancing
|
||||
purposes, a finer grain of (for example) 100 integers per Machi
|
||||
chain would permit file migration to move increments of
|
||||
approximately 1% of single Machi chain's storage capacity. A
|
||||
minimum of 12+7=19 bits of hash space would be necessary to accommodate
|
||||
these constraints.
|
||||
|
||||
It is likely that Machi's final implementation will choose a 24 bit
|
||||
integer (or perhaps 32 bits) to represent the cluster locator.
|
||||
|
||||
* 7. Proposal: Break the opacity of Machi file names, slightly.
|
||||
|
||||
Machi assigns file names based on:
|
||||
|
||||
~ClientSuppliedPrefix ++ "^" ++ SomeOpaqueFileNameSuffix~
|
||||
|
||||
What if some parts of the system could peek inside of the opaque file name
|
||||
suffix in order to look at the cluster location information that we might
|
||||
code in the filename suffix?
|
||||
|
||||
We break the system into parts that speak two levels of protocols,
|
||||
"high" and "low".
|
||||
|
||||
+ The high level protocol is used outside of the Machi cluster
|
||||
+ The low level protocol is used inside of the Machi cluster
|
||||
|
||||
Both protocols are based on a Protocol Buffers specification and
|
||||
implementation. Other protocols, such as HTTP, will be added later.
|
||||
|
||||
#+BEGIN_SRC
|
||||
+-----------------------+
|
||||
| Machi external client |
|
||||
| e.g. Riak CS |
|
||||
+-----------------------+
|
||||
^
|
||||
| Machi "high" API
|
||||
| ProtoBuffs protocol Machi cluster boundary: outside
|
||||
.........................................................................
|
||||
| Machi cluster boundary: inside
|
||||
v
|
||||
+--------------------------+ +------------------------+
|
||||
| Machi "high" API service | | Machi HTTP API service |
|
||||
+--------------------------+ +------------------------+
|
||||
^ |
|
||||
| +------------------------+
|
||||
v v
|
||||
+------------------------+
|
||||
| Cluster bridge service |
|
||||
+------------------------+
|
||||
^
|
||||
| Machi "low" API
|
||||
| ProtoBuffs protocol
|
||||
+----------------------------------------+----+----+
|
||||
| | | |
|
||||
v v v v
|
||||
+-------------------------+ ... other chains...
|
||||
| Chain C1 (logical view) |
|
||||
| +--------------+ |
|
||||
| | FLU server 1 | |
|
||||
| | +--------------+ |
|
||||
| +--| FLU server 2 | |
|
||||
| +--------------+ | In reality, API bridge talks directly
|
||||
+-------------------------+ to each FLU server in a chain.
|
||||
#+END_SRC
|
||||
|
||||
** The notation we use
|
||||
|
||||
- ~N~ = the cluster namespace, chosen by the client.
|
||||
- ~p~ = file prefix, chosen by the client.
|
||||
- ~L~ = the cluster locator (a number, type is implementation-dependent)
|
||||
- ~Map~ = a mapping of cluster locators to chains
|
||||
- ~T~ = the target chain ID/name
|
||||
- ~u~ = a unique opaque file name suffix, e.g. a GUID string
|
||||
- ~F~ = a Machi file name, i.e., a concatenation of ~p^L^N^u~
|
||||
|
||||
** The details: cluster file append
|
||||
|
||||
0. Cluster client chooses ~N~ and ~p~ (i.e., cluster namespace and
|
||||
file prefix) and sends the append request to a Machi cluster member
|
||||
via the Protocol Buffers "high" API.
|
||||
1. Cluster bridge chooses ~T~ (i.e., target chain), based on criteria
|
||||
such as disk utilization percentage.
|
||||
2. Cluster bridge knows the cluster ~Map~ for namespace ~N~.
|
||||
3. Cluster bridge choose some cluster locator value ~L~ such that
|
||||
~rs_hash_with_float(L,Map) = T~ (see algorithm below).
|
||||
4. Cluster bridge sends its request to chain
|
||||
~T~: ~append_chunk(p,L,N,...) -> {ok,p^L^N^u,ByteOffset}~
|
||||
5. Cluster bridge forwards the reply tuple to the client.
|
||||
6. Client stores/uses the file name ~F = p^L^N^u~.
|
||||
|
||||
** The details: Cluster file read
|
||||
|
||||
0. Cluster client sends the read request to a Machi cluster member via
|
||||
the Protocol Buffers "high" API.
|
||||
1. Cluster bridge parses the file name ~F~ to find
|
||||
the values of ~L~ and ~N~ (recall, ~F = p^L^N^u~).
|
||||
2. Cluster bridge knows the Cluster ~Map~ for type ~N~.
|
||||
3. Cluster bridge calculates ~rs_hash_with_float(L,Map) = T~
|
||||
4. Cluster bridge sends request to chain ~T~:
|
||||
~read_chunk(F,...) ->~ ... reply
|
||||
5. Cluster bridge forwards the reply to the client.
|
||||
|
||||
** The details: calculating 'L' (the cluster locator number) to match a desired target chain
|
||||
|
||||
1. We know ~Map~, the current cluster mapping for a cluster namespace ~N~.
|
||||
2. We look inside of ~Map~, and we find all of the unit interval ranges
|
||||
that map to our desired target chain ~T~. Let's call this list
|
||||
~MapList = [Range1=(start,end],Range2=(start,end],...]~.
|
||||
3. In our example, ~T=Chain2~. The example ~Map~ contains a single
|
||||
unit interval range for ~Chain2~, ~[(0.33,0.58]]~.
|
||||
4. Choose a uniformly random number ~r~ on the unit interval.
|
||||
5. Calculate the cluster locator ~L~ by mapping ~r~ onto the concatenation
|
||||
of the cluster hash space range intervals in ~MapList~. For example,
|
||||
if ~r=0.5~, then ~L = 0.33 + 0.5*(0.58-0.33) = 0.455~, which is
|
||||
exactly in the middle of the ~(0.33,0.58]~ interval.
|
||||
|
||||
** A bit more about the cluster namespaces's meaning and use
|
||||
|
||||
For use by Riak CS, for example, we'd likely start with the following
|
||||
namespaces ... working our way down the list as we add new features
|
||||
and/or re-implement existing CS features.
|
||||
|
||||
- "standard" = Chain length = 3, eventually consistency mode
|
||||
- "reduced" = Chain length = 2, eventually consistency mode.
|
||||
- "stanchion7" = Chain length = 7, strong consistency mode. Perhaps
|
||||
use this namespace for the metadata required to re-implement the
|
||||
operations that are performed by today's Stanchion application.
|
||||
|
||||
We want the cluster framework to:
|
||||
|
||||
- provide means of creating and managing
|
||||
chains of different types, e.g., chain length, consistency mode.
|
||||
- manage the mapping of cluster namespace
|
||||
names to the chains in the system.
|
||||
- provide query functions to map a cluster
|
||||
namespace name to a cluster map,
|
||||
e.g. ~get_cluster_latest_map("reduced") -> Map{generation=7,...}~.
|
||||
|
||||
* 8. File migration (a.k.a. rebalancing/reparitioning/resharding/redistribution)
|
||||
|
||||
** What is "migration"?
|
||||
|
||||
This section describes Machi's file migration. Other storage systems
|
||||
call this process as "rebalancing", "repartitioning", "resharding" or
|
||||
"redistribution".
|
||||
For Riak Core applications, it is called "handoff" and "ring resizing"
|
||||
(depending on the context).
|
||||
See also the [[http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsUserGuide.html#Balancer][Hadoop file balancer]] for another example of a data
|
||||
migration process.
|
||||
|
||||
As discussed in section 5, the client can have good reason for wanting
|
||||
to have some control of the initial location of the file within the
|
||||
chain. However, the chain manager has an ongoing interest in
|
||||
balancing resources throughout the lifetime of the file. Disks will
|
||||
get full, hardware will change, read workload will fluctuate,
|
||||
etc etc.
|
||||
|
||||
This document uses the word "migration" to describe moving data from
|
||||
one Machi chain to another chain within a cluster system.
|
||||
|
||||
A simple variation of the Random Slicing hash algorithm can easily
|
||||
accommodate Machi's need to migrate files without interfering with
|
||||
availability. Machi's migration task is much simpler due to the
|
||||
immutable nature of Machi file data.
|
||||
|
||||
** Change to Random Slicing
|
||||
|
||||
The map used by the Random Slicing hash algorithm needs a few simple
|
||||
changes to make file migration straightforward.
|
||||
|
||||
- Add a "generation number", a strictly increasing number (similar to
|
||||
a Machi chain's "epoch number") that reflects the history of
|
||||
changes made to the Random Slicing map
|
||||
- Use a list of Random Slicing maps instead of a single map, one map
|
||||
per chance that files may not have been migrated yet out of
|
||||
that map.
|
||||
|
||||
As an example:
|
||||
|
||||
#+CAPTION: Illustration of 'Map', using four Machi chains
|
||||
|
||||
[[./migration-3to4.png]]
|
||||
|
||||
And the new Random Slicing map for some cluster namespace ~N~ might look
|
||||
like this:
|
||||
|
||||
| Generation number / Namespace | 7 / reduced |
|
||||
|-------------------------------+-------------|
|
||||
| SubMap | 1 |
|
||||
|-------------------------------+-------------|
|
||||
| Hash range | Chain ID |
|
||||
|-------------------------------+-------------|
|
||||
| 0.00 - 0.33 | Chain1 |
|
||||
| 0.33 - 0.66 | Chain2 |
|
||||
| 0.66 - 1.00 | Chain3 |
|
||||
|-------------------------------+-------------|
|
||||
| SubMap | 2 |
|
||||
|-------------------------------+-------------|
|
||||
| Hash range | Chain ID |
|
||||
|-------------------------------+-------------|
|
||||
| 0.00 - 0.25 | Chain1 |
|
||||
| 0.25 - 0.33 | Chain4 |
|
||||
| 0.33 - 0.58 | Chain2 |
|
||||
| 0.58 - 0.66 | Chain4 |
|
||||
| 0.66 - 0.91 | Chain3 |
|
||||
| 0.91 - 1.00 | Chain4 |
|
||||
|
||||
When a new Random Slicing map contains a single submap, then its use
|
||||
is identical to the original Random Slicing algorithm. If the map
|
||||
contains multiple submaps, then the access rules change a bit:
|
||||
|
||||
- Write operations always go to the newest/largest submap.
|
||||
- Read operations attempt to read from all unique submaps.
|
||||
- Skip searching submaps that refer to the same chain ID.
|
||||
- In this example, unit interval value 0.10 is mapped to Chain1
|
||||
by both submaps.
|
||||
- Read from newest/largest submap to oldest/smallest submap.
|
||||
- If not found in any submap, search a second time (to handle races
|
||||
with file copying between submaps).
|
||||
- If the requested data is found, optionally copy it directly to the
|
||||
newest submap. (This is a variation of read repair (RR). RR here
|
||||
accelerates the migration process and can reduce the number of
|
||||
operations required to query servers in multiple submaps).
|
||||
|
||||
The cluster manager is responsible for:
|
||||
|
||||
- Managing the various generations of the cluster Random Slicing maps for
|
||||
all namespaces.
|
||||
- Distributing namespace maps to cluster bridges.
|
||||
- Managing the processes that are responsible for copying "cold" data,
|
||||
i.e., files data that is not regularly accessed, to its new submap
|
||||
location.
|
||||
- When migration of a file to its new chain is confirmed successful,
|
||||
delete it from the old chain.
|
||||
|
||||
In example map #7, the cluster manager will copy files with unit interval
|
||||
assignments in ~(0.25,0.33]~, ~(0.58,0.66]~, and ~(0.91,1.00]~ from their
|
||||
old locations in chain IDs Chain1/2/3 to their new chain,
|
||||
Chain4. When the cluster manager is satisfied that all such files have
|
||||
been copied to Chain4, then the cluster manager can create and
|
||||
distribute a new map, such as:
|
||||
|
||||
| Generation number / Namespace | 8 / reduced |
|
||||
|-------------------------------+-------------|
|
||||
| SubMap | 1 |
|
||||
|-------------------------------+-------------|
|
||||
| Hash range | Chain ID |
|
||||
|-------------------------------+-------------|
|
||||
| 0.00 - 0.25 | Chain1 |
|
||||
| 0.25 - 0.33 | Chain4 |
|
||||
| 0.33 - 0.58 | Chain2 |
|
||||
| 0.58 - 0.66 | Chain4 |
|
||||
| 0.66 - 0.91 | Chain3 |
|
||||
| 0.91 - 1.00 | Chain4 |
|
||||
|
||||
The HibariDB system performs data migrations in almost exactly this
|
||||
manner. However, one important
|
||||
limitation of HibariDB is not being able to
|
||||
perform more than one migration at a time. HibariDB's data is
|
||||
mutable. Mutation causes many problems when migrating data
|
||||
across two submaps; three or more submaps was too complex to implement
|
||||
quickly and correctly.
|
||||
|
||||
Fortunately for Machi, its file data is immutable and therefore can
|
||||
easily manage many migrations in parallel, i.e., its submap list may
|
||||
be several maps long, each one for an in-progress file migration.
|
||||
|
||||
* 9. Other considerations for FLU/sequencer implementations
|
||||
|
||||
** Append to existing file when possible
|
||||
|
||||
The sequencer should always assign new offsets to the latest/newest
|
||||
file for any prefix, as long as all prerequisites are also true,
|
||||
|
||||
- The epoch has not changed. (In AP mode, epoch change -> mandatory
|
||||
file name suffix change.)
|
||||
- The cluster locator number is stable.
|
||||
- The latest file for prefix ~p~ is smaller than maximum file size for
|
||||
a FLU's configuration.
|
||||
|
||||
The stability of the cluster locator number is an implementation detail that
|
||||
must be managed by the cluster bridge.
|
||||
|
||||
Reuse of the same file is not possible if the bridge always chooses a
|
||||
different cluster locator number ~L~ or if the client always uses a unique
|
||||
file prefix ~p~. The latter is a sign of a misbehaved client; the
|
||||
former is a poorly-implemented bridge.
|
||||
|
||||
* 10. Acknowledgments
|
||||
|
||||
The original source for the "migration-4.png" and "migration-3to4.png" images
|
||||
come from the [[http://hibari.github.io/hibari-doc/images/migration-3to4.png][HibariDB documentation]].
|
||||
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
# Clone and compile Machi
|
||||
|
||||
Clone the Machi source repo and compile the source and test code. Run
|
||||
the following commands at your login shell:
|
||||
|
||||
cd /tmp
|
||||
git clone https://github.com/basho/machi.git
|
||||
cd machi
|
||||
git checkout master
|
||||
make # or 'gmake' if GNU make uses an alternate name
|
||||
|
||||
Then run the unit test suite. This may take up to two minutes or so
|
||||
to finish.
|
||||
|
||||
make test
|
||||
|
||||
At the end, the test suite should report that all tests passed. The
|
||||
actual number of tests shown in the "All `X` tests passed" line may be
|
||||
different than the example below.
|
||||
|
||||
[... many lines omitted ...]
|
||||
module 'event_logger'
|
||||
module 'chain_mgr_legacy'
|
||||
=======================================================
|
||||
All 90 tests passed.
|
||||
|
||||
If you had a test failure, a likely cause may be a limit on the number
|
||||
of file descriptors available to your user process. (Recent releases
|
||||
of OS X have a limit of 1024 file descriptors, which may be too slow.)
|
||||
The output of the `limit -n` will tell you your file descriptor limit.
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
## Machi developer environment prerequisites
|
||||
|
||||
1. Machi requires an 64-bit variant of UNIX: OS X, FreeBSD, Linux, or
|
||||
Solaris machine is a standard developer environment for C and C++
|
||||
applications (64-bit versions).
|
||||
2. You'll need the `git` source management utility.
|
||||
3. You'll need the 64-bit Erlang/OTP 17 runtime environment. Please
|
||||
don't use earlier or later versions until we have a chance to fix
|
||||
the compilation warnings that versions R16B and 18 will trigger.
|
||||
Also, please verify that you are not using a 32-bit Erlang/OTP
|
||||
runtime package.
|
||||
|
||||
For `git` and the Erlang runtime, please use your OS-specific
|
||||
package manager to install these. If your package manager doesn't
|
||||
have 64-bit Erlang/OTP version 17 available, then we recommend using the
|
||||
[precompiled packages available at Erlang Solutions](https://www.erlang-solutions.com/resources/download.html).
|
||||
|
||||
Also, please verify that you have enough file descriptors available to
|
||||
your user processes. The output of `ulimit -n` should report at least
|
||||
4,000 file descriptors available. If your limit is lower (a frequent
|
||||
problem for OS X users), please increase it to at least 4,000.
|
||||
|
||||
# Using Vagrant to set up a developer environment for Machi
|
||||
|
||||
The Machi source directory contains a `Vagrantfile` for creating an
|
||||
Ubuntu Linux-based virtual machine for compiling and running Machi.
|
||||
This file is in the
|
||||
[$SRC_TOP/priv/humming-consensus-demo.vagrant](../priv/humming-consensus-demo.vagrant)
|
||||
directory.
|
||||
|
||||
If used as-is, the virtual machine specification is modest.
|
||||
|
||||
* 1 virtual CPU
|
||||
* 512MB virtual memory
|
||||
* 768MB swap space
|
||||
* 79GB sparse virtual disk image. After installing prerequisites and
|
||||
compiling Machi, the root file system uses approximately 2.7 GBytes.
|
||||
|
||||
|
|
@ -1,617 +0,0 @@
|
|||
FLU and Chain Life Cycle Management -*- mode: org; -*-
|
||||
#+STARTUP: lognotedone hidestars indent showall inlineimages
|
||||
#+COMMENT: To generate the outline section: egrep '^\*[*]* ' doc/flu-and-chain-lifecycle.org | egrep -v '^\* Outline' | sed -e 's/^\*\*\* / + /' -e 's/^\*\* / + /' -e 's/^\* /+ /'
|
||||
|
||||
* FLU and Chain Life Cycle Management
|
||||
|
||||
In an ideal world, we (the Machi development team) would have a full
|
||||
vision of how Machi would be managed, down to the last detail of
|
||||
beautiful CLI character and network protocol bit. Our vision isn't
|
||||
complete yet, so we are working one small step at a time.
|
||||
|
||||
* Outline
|
||||
|
||||
+ FLU and Chain Life Cycle Management
|
||||
+ Terminology review
|
||||
+ Terminology: Machi run-time components/services/thingies
|
||||
+ Terminology: Machi chain data structures
|
||||
+ Terminology: Machi cluster data structures
|
||||
+ Overview of administrative life cycles
|
||||
+ Cluster administrative life cycle
|
||||
+ Chain administrative life cycle
|
||||
+ FLU server administrative life cycle
|
||||
+ Quick admin: declarative management of Machi FLU and chain life cycles
|
||||
+ Quick admin uses the "rc.d" config scheme for life cycle management
|
||||
+ Quick admin's declarative "language": an Erlang-flavored AST
|
||||
+ Term 'host': define a new host for FLU services
|
||||
+ Term 'flu': define a new FLU
|
||||
+ Term 'chain': define or reconfigure a chain
|
||||
+ Executing quick admin AST files via the 'machi-admin' utility
|
||||
+ Checking the syntax of an AST file
|
||||
+ Executing an AST file
|
||||
+ Using quick admin to manage multiple machines
|
||||
+ The "rc.d" style configuration file scheme
|
||||
+ Riak had a similar configuration file editing problem (and its solution)
|
||||
+ Machi's "rc.d" file scheme.
|
||||
+ FLU life cycle management using "rc.d" style files
|
||||
+ The key configuration components of a FLU
|
||||
+ Chain life cycle management using "rc.d" style files
|
||||
+ The key configuration components of a chain
|
||||
|
||||
* Terminology review
|
||||
|
||||
** Terminology: Machi run-time components/services/thingies
|
||||
|
||||
+ FLU: a basic Machi server, responsible for managing a collection of
|
||||
files.
|
||||
|
||||
+ Chain: a small collection of FLUs that maintain replicas of the same
|
||||
collection of files. A chain is usually small, 1-3 servers, where
|
||||
more than 3 would be used only in cases when availability of
|
||||
certain data is critical despite failures of several machines.
|
||||
+ The length of a chain is directly proportional to its
|
||||
replication factor, e.g., a chain length=3 will maintain
|
||||
(nominally) 3 replicas of each file.
|
||||
+ To maintain file availability when ~F~ failures have occurred, a
|
||||
chain must be at least ~F+1~ members long. (In comparison, the
|
||||
quorum replication technique requires ~2F+1~ members in the
|
||||
general case.)
|
||||
|
||||
+ Cluster: A collection of Machi chains that are used to store files
|
||||
in a horizontally partitioned/sharded/distributed manner.
|
||||
|
||||
** Terminology: Machi data structures
|
||||
|
||||
+ Projection: used to define a single chain: the chain's consistency
|
||||
mode (strong or eventual consistency), all members (from an
|
||||
administrative point of view), all active members (from a runtime,
|
||||
automatically-managed point of view), repairing/file-syncing members
|
||||
(also runtime, auto-managed), and so on
|
||||
|
||||
+ Epoch: A version number of a projection. The epoch number is used
|
||||
by both clients & servers to manage transitions from one projection
|
||||
to another, e.g., when the chain is temporarily shortened by the
|
||||
failure of a member FLU server.
|
||||
|
||||
** Terminology: Machi cluster data structures
|
||||
|
||||
+ Namespace: A collection of human-friendly names that are mapped to
|
||||
groups of Machi chains that provide the same type of storage
|
||||
service: consistency mode, replication policy, etc.
|
||||
+ A single namespace name, e.g. ~normal-ec~, is paired with a single
|
||||
cluster map (see below).
|
||||
+ Example: ~normal-ec~ might be a collection of Machi chains in
|
||||
eventually-consistent mode that are of length=3.
|
||||
+ Example: ~risky-ec~ might be a collection of Machi chains in
|
||||
eventually-consistent mode that are of length=1.
|
||||
+ Example: ~mgmt-critical~ might be a collection of Machi chains in
|
||||
strongly-consistent mode that are of length=7.
|
||||
|
||||
+ Cluster map: Encodes the rules which partition/shard/distribute
|
||||
the files stored in a particular namespace across a group of chains
|
||||
that collectively store the namespace's files.
|
||||
|
||||
+ Chain weight: A value assigned to each chain within a cluster map
|
||||
structure that defines the relative storage capacity of a chain
|
||||
within the namespace. For example, a chain weight=150 has 50% more
|
||||
capacity than a chain weight=100.
|
||||
|
||||
+ Cluster map epoch: The version number assigned to a cluster map.
|
||||
|
||||
* Overview of administrative life cycles
|
||||
|
||||
** Cluster administrative life cycle
|
||||
|
||||
+ Cluster is first created
|
||||
+ Adds namespaces (e.g. consistency policy + chain length policy) to
|
||||
the cluster
|
||||
+ Chains are added to/removed from a namespace to increase/decrease the
|
||||
namespace's storage capacity.
|
||||
+ Adjust chain weights within a namespace, e.g., to shift files
|
||||
within the namespace to chains with greater storage capacity
|
||||
resources and/or runtime I/O resources.
|
||||
|
||||
A cluster "file migration" is the process of moving files from one
|
||||
namespace member chain to another for purposes of shifting &
|
||||
re-balancing storage capacity and/or runtime I/O capacity.
|
||||
|
||||
** Chain administrative life cycle
|
||||
|
||||
+ A chain is created with an initial FLU membership list.
|
||||
+ Chain may be administratively modified zero or more times to
|
||||
add/remove member FLU servers.
|
||||
+ A chain may be decommissioned.
|
||||
|
||||
See also: http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html
|
||||
|
||||
** FLU server administrative life cycle
|
||||
|
||||
+ A FLU is created after an administrator chooses the FLU's runtime
|
||||
location is selected by the administrator: which machine/virtual
|
||||
machine, IP address and TCP port allocation, etc.
|
||||
+ An unassigned FLU may be added to a chain by chain administrative
|
||||
policy.
|
||||
+ A FLU that is assigned to a chain may be removed from that chain by
|
||||
chain administrative policy.
|
||||
+ In the current implementation, the FLU's Erlang processes will be
|
||||
halted. Then the FLU's data and metadata files will be moved to
|
||||
another area of the disk for safekeeping. Later, a "garbage
|
||||
collection" process can be used for reclaiming disk space used by
|
||||
halted FLU servers.
|
||||
|
||||
See also: http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html
|
||||
|
||||
* Quick admin: declarative management of Machi FLU and chain life cycles
|
||||
|
||||
The "quick admin" scheme is a temporary (?) tool for managing Machi
|
||||
FLU server and chain life cycles in a declarative manner. The API is
|
||||
described in this section.
|
||||
|
||||
** Quick admin uses the "rc.d" config scheme for life cycle management
|
||||
|
||||
As described at the top of
|
||||
http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html, the "rc.d"
|
||||
config files do not manage "policy". "Policy" is doing the right
|
||||
thing with a Machi cluster from a systems administrator's
|
||||
point of view. The "rc.d" config files can only implement decisions
|
||||
made according to policy.
|
||||
|
||||
The "quick admin" tool is a first attempt at automating policy
|
||||
decisions in a safe way (we hope) that is also easy to implement (we
|
||||
hope) with a variety of systems management tools, e.g. Chef, Puppet,
|
||||
Ansible, Saltstack, or plain-old-human-at-a-keyboard.
|
||||
|
||||
** Quick admin's declarative "language": an Erlang-flavored AST
|
||||
|
||||
The "language" that an administrator uses to express desired policy
|
||||
changes is not (yet) a true language. As a quick implementation hack,
|
||||
the current language is an Erlang-flavored abstract syntax tree
|
||||
(AST). The tree isn't very deep, either, frequently just one
|
||||
element tall. (Not much of a tree, is it?)
|
||||
|
||||
There are three terms in the language currently:
|
||||
|
||||
+ ~host~, define a new host that can execute FLU servers
|
||||
+ ~flu~, define a new FLU
|
||||
+ ~chain~, define a new chain or re-configure an existing chain with
|
||||
the same name
|
||||
|
||||
*** Term 'host': define a new host for FLU services
|
||||
|
||||
In this context, a host is a machine, virtual machine, or container
|
||||
that can execute the Machi application and can therefore provide FLU
|
||||
services, i.e. file service, Humming Consensus management.
|
||||
|
||||
Two formats may be used to define a new host:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{host, Name, Props}.
|
||||
{host, Name, AdminI, ClientI, Props}.
|
||||
#+END_SRC
|
||||
|
||||
The shorter tuple is shorthand notation for the latter. If the
|
||||
shorthand form is used, then it will be converted automatically to the
|
||||
long form as:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{host, Name, AdminI=Name, ClientI=Name, Props}.
|
||||
#+END_SRC
|
||||
|
||||
Type information, description, and restrictions:
|
||||
|
||||
+ ~Name::string()~ The ~Name~ attribute must be unique. Note that it
|
||||
is possible to define two different hosts, one using a DNS hostname
|
||||
and one using an IP address. The user must avoid this
|
||||
double-definition because it is not enforced by quick admin.
|
||||
+ The ~Name~ field is used for cross-reference purposes with other
|
||||
terms, e.g., ~flu~ and ~chain~.
|
||||
+ There is no syntax yet for removing a host definition.
|
||||
|
||||
+ ~AdminI::string()~ A DNS hostname or IP address for cluster
|
||||
administration purposes, e.g. SSH access.
|
||||
+ This field is unused at the present time.
|
||||
|
||||
+ ~ClientI::string()~ A DNS hostname or IP address for Machi's client
|
||||
protocol access, e.g., Protocol Buffers network API service.
|
||||
+ This field is unused at the present time.
|
||||
|
||||
+ ~props::proplist()~ is an Erlang-style property list for specifying
|
||||
additional configuration options, debugging information, sysadmin
|
||||
comments, etc.
|
||||
|
||||
+ A full-featured admin tool should also include managing several
|
||||
other aspects of configuration related to a "host". For example,
|
||||
for any single IP address, quick admin assumes that there will be
|
||||
exactly one Erlang VM that is running the Machi application. Of
|
||||
course, it is possible to have dozens of Erlang VMs on the same
|
||||
(let's assume for clarity) hardware machine and all running Machi
|
||||
... but there are additional aspects of such a machine that quick
|
||||
admin does not account for
|
||||
+ multiple IP addresses per machine
|
||||
+ multiple Machi package installation paths
|
||||
+ multiple Machi config files (e.g. cuttlefish config, ~etc.conf~,
|
||||
~vm.args~)
|
||||
+ multiple data directories/file system mount points
|
||||
+ This is also a management problem for quick admin for a single
|
||||
Machi package on a machine to take advantage of bulk data
|
||||
storage using multiple multiple file system mount points.
|
||||
+ multiple Erlang VM host names, required for distributed Erlang,
|
||||
which is used for communication with ~machi~ and ~machi-admin~
|
||||
command line utilities.
|
||||
+ and others....
|
||||
|
||||
*** Term 'flu': define a new FLU
|
||||
|
||||
A new FLU is defined relative to a previously-defined ~host~ entities;
|
||||
an exception will be thrown if the ~host~ cannot be cross-referenced.
|
||||
|
||||
#+BEGIN_SRC
|
||||
{flu, Name, HostName, Port, Props}
|
||||
#+END_SRC
|
||||
|
||||
Type information, description, and restrictions:
|
||||
|
||||
+ ~Name::atom()~ The name of the FLU, as a human-friendly name and
|
||||
also for internal management use; please note the ~atom()~ type.
|
||||
This name must be unique.
|
||||
+ The ~Name~ field is used for cross-reference purposes with the
|
||||
~chain~ term.
|
||||
+ There is no syntax yet for removing a FLU definition.
|
||||
|
||||
+ ~Hostname::string()~ The cross-reference name of the ~host~ that
|
||||
this FLU should run on.
|
||||
|
||||
+ ~Port::non_neg_integer()~ The TCP port used by this FLU server's
|
||||
Protocol Buffers network API listener service
|
||||
|
||||
+ ~props::proplist()~ is an Erlang-style property list for specifying
|
||||
additional configuration options, debugging information, sysadmin
|
||||
comments, etc.
|
||||
|
||||
*** Term 'chain': define or reconfigure a chain
|
||||
|
||||
A chain is defined relative to zero or more previously-defined ~flu~
|
||||
entities; an exception will be thrown if any ~flu~ cannot be
|
||||
cross-referenced.
|
||||
|
||||
Two formats may be used to define/reconfigure a chain:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{chain, Name, FullList, Props}.
|
||||
{chain, Name, CMode, FullList, Witnesses, Props}.
|
||||
#+END_SRC
|
||||
|
||||
The shorter tuple is shorthand notation for the latter. If the
|
||||
shorthand form is used, then it will be converted automatically to the
|
||||
long form as:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{chain, Name, ap_mode, FullList, [], Props}.
|
||||
#+END_SRC
|
||||
|
||||
Type information, description, and restrictions:
|
||||
|
||||
+ ~Name::atom()~ The name of the chain, as a human-friendly name and
|
||||
also for internal management use; please note the ~atom()~ type.
|
||||
This name must be unique.
|
||||
+ There is no syntax yet for removing a chain definition.
|
||||
|
||||
+ ~CMode::'ap_mode'|'cp_mode'~ Defines the consistency mode of the
|
||||
chain, either eventual consistency or strong consistency,
|
||||
respectively.
|
||||
+ A chain cannot change consistency mode, e.g., from
|
||||
strong~->~eventual consistency.
|
||||
|
||||
+ ~FullList::list(atom())~ Specifies the list of full-service FLU
|
||||
servers, i.e. servers that provide file data & metadata services as
|
||||
well as Humming Consensus. Each atom in the list must
|
||||
cross-reference with a previously defined ~chain~; an exception will
|
||||
be thrown if any ~flu~ cannot be cross-referenced.
|
||||
|
||||
+ ~Witnesses::list(atom())~ Specifies the list of witness-only
|
||||
servers, i.e. servers that only participate in Humming Consensus.
|
||||
Each atom in the list must cross-reference with a previously defined
|
||||
~chain~; an exception will be thrown if any ~flu~ cannot be
|
||||
cross-referenced.
|
||||
+ This list must be empty for eventual consistency chains.
|
||||
|
||||
+ ~props::proplist()~ is an Erlang-style property list for specifying
|
||||
additional configuration options, debugging information, sysadmin
|
||||
comments, etc.
|
||||
|
||||
+ If this term specifies a new ~chain~ name, then all of the member
|
||||
FLU servers (full & witness types) will be bootstrapped to a
|
||||
starting configuration.
|
||||
|
||||
+ If this term specifies a previously-defined ~chain~ name, then all
|
||||
of the member FLU servers (full & witness types, respectively) will
|
||||
be adjusted to add or remove members, as appropriate.
|
||||
+ Any FLU servers added to either list must not be assigned to any
|
||||
other chain, or they must be a member of this specific chain.
|
||||
+ Any FLU servers removed from either list will be halted.
|
||||
(See the "FLU server administrative life cycle" section above.)
|
||||
|
||||
** Executing quick admin AST files via the 'machi-admin' utility
|
||||
|
||||
Examples of quick admin AST files can be found in the
|
||||
~priv/quick-admin/examples~ directory. Below is an example that will
|
||||
define a new host ( ~"localhost"~ ), three new FLU servers ( ~f1~ & ~f2~
|
||||
and ~f3~ ), and an eventually consistent chain ( ~c1~ ) that uses the new
|
||||
FLU servers:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{host, "localhost", []}.
|
||||
{flu,f1,"localhost",20401,[]}.
|
||||
{flu,f2,"localhost",20402,[]}.
|
||||
{flu,f3,"localhost",20403,[]}.
|
||||
{chain,c1,[f1,f2,f3],[]}.
|
||||
#+END_SRC
|
||||
|
||||
*** Checking the syntax of an AST file
|
||||
|
||||
Given an AST config file, ~/path/to/ast/file~, its basic syntax and
|
||||
correctness can be checked without executing it.
|
||||
|
||||
#+BEGIN_SRC
|
||||
./rel/machi/bin/machi-admin quick-admin-check /path/to/ast/file
|
||||
#+END_SRC
|
||||
|
||||
+ The utility will exit with status zero and output ~ok~ if the syntax
|
||||
and proposed configuration appears to be correct.
|
||||
+ If there is an error, the utility will exit with status one, and an
|
||||
error message will be printed.
|
||||
|
||||
*** Executing an AST file
|
||||
|
||||
Given an AST config file, ~/path/to/ast/file~, it can be executed
|
||||
using the command:
|
||||
|
||||
#+BEGIN_SRC
|
||||
./rel/machi/bin/machi-admin quick-admin-apply /path/to/ast/file RelativeHost
|
||||
#+END_SRC
|
||||
|
||||
... where the last argument, ~RelativeHost~, should be the exact
|
||||
spelling of one of the previously defined AST ~host~ entities,
|
||||
*and also* is the same host that the ~machi-admin~ utility is being
|
||||
executed on.
|
||||
|
||||
Restrictions and warnings:
|
||||
|
||||
+ This is alpha quality software.
|
||||
|
||||
+ There is no "undo".
|
||||
+ Of course there is, but you need to resort to doing things like
|
||||
using ~machi attach~ to attach to the server's CLI to then execute
|
||||
magic Erlang incantations to stop FLUs, unconfigure chains, etc.
|
||||
+ Oh, and delete some files with magic paths, also.
|
||||
|
||||
** Using quick admin to manage multiple machines
|
||||
|
||||
A quick sketch follows:
|
||||
|
||||
1. Create the AST file to specify all of the changes that you wish to
|
||||
make to all hosts, FLUs, and/or chains, e.g., ~/tmp/ast.txt~.
|
||||
2. Check the basic syntax with the ~quick-admin-check~ argument to
|
||||
~machi-admin~.
|
||||
3. If the syntax is good, then copy ~/tmp/ast.txt~ to all hosts in the
|
||||
cluster, using the same path, ~/tmp/ast.txt~.
|
||||
4. For each machine in the cluster, run:
|
||||
#+BEGIN_SRC
|
||||
./rel/machi/bin/machi-admin quick-admin-apply /tmp/ast.txt RelativeHost
|
||||
#+END_SRC
|
||||
|
||||
... where RelativeHost is the AST ~host~ name of the machine that you
|
||||
are executing the ~machi-admin~ command on. The command should be
|
||||
successful, with exit status 0 and outputting the string ~ok~.
|
||||
|
||||
Finally, for each machine in the cluster, a listing of all files in
|
||||
the directory ~rel/machi/etc/quick-admin-archive~ should show exactly
|
||||
the same files, one for each time that ~quick-admin-apply~ has been
|
||||
run successfully on that machine.
|
||||
|
||||
* The "rc.d" style configuration file scheme
|
||||
|
||||
This configuration scheme is inspired by BSD UNIX's ~init(8)~ process
|
||||
manager's configuration style, called "rc.d" after the name of the
|
||||
directory where these files are stored, ~/etc/rc.d~. The ~init~
|
||||
process is responsible for (among other things) starting UNIX
|
||||
processes at machine boot time and stopping them when the machine is
|
||||
shut down.
|
||||
|
||||
The original scheme used by ~init~ to start processes at boot time was
|
||||
a single Bourne shell script called ~/etc/rc~. When a new software
|
||||
package was installed that required a daemon to be started at boot
|
||||
time, text was added to the ~/etc/rc~ file. Uninstalling packages was
|
||||
much trickier, because it meant removing lines from a file that
|
||||
*is a computer program (run by the Bourne shell, a Turing-complete
|
||||
programming language)*. Error-free editing of the ~/etc/rc~ script
|
||||
was impossible in all cases.
|
||||
|
||||
Later, ~init~'s configuration was split into a few master Bourne shell
|
||||
scripts and a subdirectory, ~/etc/rc.d~. The subdirectory contained
|
||||
shell scripts that were responsible for boot time starting of a single
|
||||
daemon or service, e.g. NFS or an HTTP server. When a new software
|
||||
package was added, a new file was added to the ~rc.d~ subdirectory.
|
||||
When a package was removed, the corresponding file in ~rc.d~ was
|
||||
removed. With this simple scheme, addition & removal of boot time
|
||||
scripts was vastly simplified.
|
||||
|
||||
** Riak had a similar configuration file editing problem (and its solution)
|
||||
|
||||
Another software product from Basho Technologies, Riak, had a similar
|
||||
configuration file editing problem. One file in particular,
|
||||
~app.config~, had a syntax that made it difficult both for human
|
||||
systems administrators and also computer programs to edit the file in
|
||||
a syntactically correct manner.
|
||||
|
||||
Later releases of Riak switched to an alternative configuration file
|
||||
format, one inspired by the BSD UNIX ~sysctl(8)~ utility and
|
||||
~sysctl.conf(5)~ file syntax. The ~sysctl.conf~ format is much easier
|
||||
to manage by computer programs to add items. Removing items is not
|
||||
100% simple, however: the correct lines must be identified and then
|
||||
removed (e.g. with Perl or a text editor or combination of ~grep -v~
|
||||
and ~mv~), but removing any comment lines that "belong" to the removed
|
||||
config item(s) is not any easy for a 1-line shell script to do 100%
|
||||
correctly.
|
||||
|
||||
Machi will use the ~sysctl.conf~ style configuration for some
|
||||
application configuration variables. However, adding & removing FLUs
|
||||
and chains will be managed using the "rc.d" style because of the
|
||||
"rc.d" scheme's simplicity and tolerance of mistakes by administrators
|
||||
(human or computer).
|
||||
|
||||
** Machi's "rc.d" file scheme.
|
||||
|
||||
Machi will use a single subdirectory that will contain configuration
|
||||
files for some life cycle management task, e.g. a single FLU or a
|
||||
single chain.
|
||||
|
||||
The contents of the file should be a single Erlang term, serialized in
|
||||
ASCII form as Erlang source code statement, i.e. a single Erlang term
|
||||
~T~ that is formatted by ~io:format("~w.",[T]).~. This file must be
|
||||
parseable by the Erlang function ~file:consult()~.
|
||||
|
||||
Later versions of Machi may change the file format to be more familiar
|
||||
to administrators who are unaccustomed to Erlang language syntax.
|
||||
|
||||
** FLU life cycle management using "rc.d" style files
|
||||
|
||||
*** The key configuration components of a FLU
|
||||
|
||||
1. The machine (or virtual machine) to run it on.
|
||||
2. The Machi software package's artifacts to execute.
|
||||
3. The disk device(s) used to store Machi file data & metadata, "rc.d"
|
||||
style config files, etc.
|
||||
4. The name, IP address and TCP port assigned to the FLU service.
|
||||
5. Its chain assignment.
|
||||
|
||||
Notes:
|
||||
|
||||
+ Items 1-3 are currently outside of the scope of this life cycle
|
||||
document. We assume that human administrators know how to do these
|
||||
things.
|
||||
+ Item 4's properties are explicitly managed by a FLU-defining "rc.d"
|
||||
style config file.
|
||||
+ Item 5 is managed by the chain life cycle management system.
|
||||
|
||||
Here is an example of a properly formatted FLU config file:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{p_srvr,f1,machi_flu1_client,"192.168.72.23",20401,[]}.
|
||||
#+END_SRC
|
||||
|
||||
... which corresponds to the following Erlang record definition:
|
||||
|
||||
#+BEGIN_SRC
|
||||
-record(p_srvr, {
|
||||
name :: atom(),
|
||||
proto_mod = 'machi_flu1_client' :: atom(), % Module name
|
||||
address :: term(), % Protocol-specific
|
||||
port :: term(), % Protocol-specific
|
||||
props = [] :: list() % proplist for other related info
|
||||
}).
|
||||
#+END_SRC
|
||||
|
||||
+ ~name~ is ~f1~. This is name of the FLU. This name should be
|
||||
unique over the lifetime of the administrative domain and thus
|
||||
managed by external policy. This name must be the same as the name
|
||||
of the config file that defines the FLU.
|
||||
+ ~proto_mod~ is used for internal management purposes and should be
|
||||
considered a mandatory constant.
|
||||
+ ~address~ is "192.168.72.23". The DNS hostname or IP address used
|
||||
by other servers to communicate with this FLU. This must be a valid
|
||||
IP address, previously assigned to this machine/VM using the
|
||||
appropriate operating system-specific procedure.
|
||||
+ ~port~ is TCP port 20401. The TCP port number that the FLU listens
|
||||
to for incoming Protocol Buffers-serialized communication. This TCP
|
||||
port must not be in use (now or in the future) by another Machi FLU
|
||||
or any other process running on this machine/VM.
|
||||
+ ~props~ is an Erlang-style property list for specifying additional
|
||||
configuration options, debugging information, sysadmin comments,
|
||||
etc.
|
||||
|
||||
** Chain life cycle management using "rc.d" style files
|
||||
|
||||
Unlike FLUs, chains have a self-management aspect that makes a chain
|
||||
life cycle different from a single FLU server. Machi's chains are
|
||||
self-managing, via Humming Consensus; see the
|
||||
https://github.com/basho/machi/tree/master/doc/ directory for much
|
||||
more detail about Humming Consensus. After FLUs have received their
|
||||
initial chain configuration for Humming Consensus, the FLUs will
|
||||
manage the chain (and each other) by themselves.
|
||||
|
||||
However, Humming Consensus does not handle three chain management
|
||||
problems:
|
||||
|
||||
1. Specifying the very first chain configuration,
|
||||
2. Altering the membership of the chain (i.e. adding/removing FLUs
|
||||
from the chain),
|
||||
3. Stopping the chain permanently.
|
||||
|
||||
A chain "rc.d" file will only be used to bootstrap a newly-defined FLU
|
||||
server. It's like a piece of glue information to introduce the new
|
||||
FLU to the Humming Consensus group that is managing the chain's
|
||||
dynamic state (e.g. which members are up or down). In all other
|
||||
respects, chain config files are ignored by life cycle management code.
|
||||
However, to mimic the life cycle of the FLU server's "rc.d" config
|
||||
files, a chain "rc.d" files is not deleted until the chain has been
|
||||
decommissioned (i.e. defined with length=0).
|
||||
|
||||
*** The key configuration components of a chain
|
||||
|
||||
1. The name of the chain.
|
||||
2. Consistency mode: eventually consistent or strongly consistent.
|
||||
3. The membership list of all FLU servers in the chain.
|
||||
+ Remember, all servers in a single chain will manage full replicas
|
||||
of the same collection of Machi files.
|
||||
4. If the chain is defined to use strongly consistent mode, then a
|
||||
list of "witness servers" may also be defined. See the
|
||||
[https://github.com/basho/machi/tree/master/doc/] documentation for
|
||||
more information on witness servers.
|
||||
+ The witness list must be empty for all chains in eventual
|
||||
consistency mode.
|
||||
|
||||
Here is an example of a properly formatted chain config file:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{chain_def_v1,c1,ap_mode,
|
||||
[{p_srvr,f1,machi_flu1_client,"localhost",20401,[]},
|
||||
{p_srvr,f2,machi_flu1_client,"localhost",20402,[]},
|
||||
{p_srvr,f3,machi_flu1_client,"localhost",20403,[]}],
|
||||
[],[],[],
|
||||
[f1,f2,f3],
|
||||
[],[]}.
|
||||
#+END_SRC
|
||||
|
||||
... which corresponds to the following Erlang record definition:
|
||||
|
||||
#+BEGIN_SRC
|
||||
-record(chain_def_v1, {
|
||||
name :: atom(), % chain name
|
||||
mode :: 'ap_mode' | 'cp_mode',
|
||||
full = [] :: [p_srvr()],
|
||||
witnesses = [] :: [p_srvr()],
|
||||
old_full = [] :: [atom()], % guard against some races
|
||||
old_witnesses=[] :: [atom()], % guard against some races
|
||||
local_run = [] :: [atom()], % must be tailored to each machine!
|
||||
local_stop = [] :: [atom()], % must be tailored to each machine!
|
||||
props = [] :: list() % proplist for other related info
|
||||
}).
|
||||
#+END_SRC
|
||||
|
||||
+ ~name~ is ~c1~, the name of the chain. This name should be unique
|
||||
over the lifetime of the administrative domain and thus managed by
|
||||
external policy. This name must be the same as the name of the
|
||||
config file that defines the chain.
|
||||
+ ~mode~ is ~ap_mode~, an internal code symbol for eventual
|
||||
consistency mode.
|
||||
+ ~full~ is a list of Erlang ~#p_srvr{}~ records for full-service
|
||||
members of the chain, i.e., providing Machi file data & metadata
|
||||
storage services.
|
||||
+ ~witnesses~ is a list of Erlang ~#p_srvr{}~ records for witness-only
|
||||
FLU servers, i.e., providing only Humming Consensus service.
|
||||
+ The next four fields are used for internal management only.
|
||||
+ ~props~ is an Erlang-style property list for specifying additional
|
||||
configuration options, debugging information, sysadmin comments,
|
||||
etc.
|
||||
|
||||
Binary file not shown.
|
|
@ -1,372 +0,0 @@
|
|||
|
||||
# Table of contents
|
||||
|
||||
* [Hands-on experiments with Machi and Humming Consensus](#hands-on)
|
||||
* [Using the network partition simulator and convergence demo test code](#partition-simulator)
|
||||
|
||||
<a name="hands-on">
|
||||
# Hands-on experiments with Machi and Humming Consensus
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Please refer to the
|
||||
[Machi development environment prerequisites doc](./dev-prerequisites.md)
|
||||
for Machi developer environment prerequisites.
|
||||
|
||||
If you do not have an Erlang/OTP runtime system available, but you do
|
||||
have [the Vagrant virtual machine](https://www.vagrantup.com/) manager
|
||||
available, then please refer to the instructions in the prerequisites
|
||||
doc for using Vagrant.
|
||||
|
||||
<a name="clone-compile">
|
||||
## Clone and compile the code
|
||||
|
||||
Please see the
|
||||
[Machi 'clone and compile' doc](./dev-clone-compile.md)
|
||||
for the short list of steps required to fetch the Machi source code
|
||||
from GitHub and to compile & test Machi.
|
||||
|
||||
## Running three Machi instances on a single machine
|
||||
|
||||
All of the commands that should be run at your login shell (e.g. Bash,
|
||||
c-shell) can be cut-and-pasted from this document directly to your
|
||||
login shell prompt.
|
||||
|
||||
Run the following command:
|
||||
|
||||
make stagedevrel
|
||||
|
||||
This will create a directory structure like this:
|
||||
|
||||
|-dev1-|... stand-alone Machi app + subdirectories
|
||||
|-dev-|-dev2-|... stand-alone Machi app + directories
|
||||
|-dev3-|... stand-alone Machi app + directories
|
||||
|
||||
Each of the `dev/dev1`, `dev/dev2`, and `dev/dev3` are stand-alone
|
||||
application instances of Machi and can be run independently of each
|
||||
other on the same machine. This demo will use all three.
|
||||
|
||||
The lifecycle management utilities for Machi are a bit immature,
|
||||
currently. They assume that each Machi server runs on a host with a
|
||||
unique hostname -- there is no flexibility built-in yet to easily run
|
||||
multiple Machi instances on the same machine. To continue with the
|
||||
demo, we need to use `sudo` or `su` to obtain superuser privileges to
|
||||
edit the `/etc/hosts` file.
|
||||
|
||||
Please add the following line to `/etc/hosts`, using this command:
|
||||
|
||||
sudo sh -c 'echo "127.0.0.1 machi1 machi2 machi3" >> /etc/hosts'
|
||||
|
||||
Next, we will use a shell script to finish setting up our cluster. It
|
||||
will do the following for us:
|
||||
|
||||
* Verify that the new line that was added to `/etc/hosts` is correct.
|
||||
* Modify the `etc/app.config` files to configure the Humming Consensus
|
||||
chain manager's actions logged to the `log/console.log` file.
|
||||
* Start the three application instances.
|
||||
* Verify that the three instances are running correctly.
|
||||
* Configure a single chain, with one FLU server per application
|
||||
instance.
|
||||
|
||||
Please run this script using this command:
|
||||
|
||||
./priv/humming-consensus-demo.setup.sh
|
||||
|
||||
If the output looks like this (and exits with status zero), then the
|
||||
script was successful.
|
||||
|
||||
Step: Verify that the required entries in /etc/hosts are present
|
||||
Step: add a verbose logging option to app.config
|
||||
Step: start three three Machi application instances
|
||||
pong
|
||||
pong
|
||||
pong
|
||||
Step: configure one chain to start a Humming Consensus group with three members
|
||||
Result: ok
|
||||
Result: ok
|
||||
Result: ok
|
||||
|
||||
We have now created a single replica chain, called `c1`, that has
|
||||
three file servers participating in the chain. Thanks to the
|
||||
hostnames that we added to `/etc/hosts`, all are using the localhost
|
||||
network interface.
|
||||
|
||||
| App instance | Pseudo | FLU name | TCP port |
|
||||
| directory | Hostname | | number |
|
||||
|--------------+----------+----------+----------|
|
||||
| dev1 | machi1 | flu1 | 20401 |
|
||||
| dev2 | machi2 | flu2 | 20402 |
|
||||
| dev3 | machi3 | flu3 | 20403 |
|
||||
|
||||
The log files for each application instance can be found in the
|
||||
`./dev/devN/log/console.log` file, where the `N` is the instance
|
||||
number: 1, 2, or 3.
|
||||
|
||||
## Understanding the chain manager's log file output
|
||||
|
||||
After running the `./priv/humming-consensus-demo.setup.sh` script,
|
||||
let's look at the last few lines of the `./dev/dev1/log/console.log`
|
||||
log file for Erlang VM process #1.
|
||||
|
||||
2016-03-09 10:16:35.676 [info] <0.105.0>@machi_lifecycle_mgr:process_pending_flu:422 Started FLU f1 with supervisor pid <0.128.0>
|
||||
2016-03-09 10:16:35.676 [info] <0.105.0>@machi_lifecycle_mgr:move_to_flu_config:540 Creating FLU config file f1
|
||||
2016-03-09 10:16:35.790 [info] <0.105.0>@machi_lifecycle_mgr:bootstrap_chain2:312 Configured chain c1 via FLU f1 to mode=ap_mode all=[f1,f2,f3] witnesses=[]
|
||||
2016-03-09 10:16:35.790 [info] <0.105.0>@machi_lifecycle_mgr:move_to_chain_config:546 Creating chain config file c1
|
||||
2016-03-09 10:16:44.139 [info] <0.132.0> CONFIRM epoch 1141 <<155,42,7,221>> upi [] rep [] auth f1 by f1
|
||||
2016-03-09 10:16:44.271 [info] <0.132.0> CONFIRM epoch 1148 <<57,213,154,16>> upi [f1] rep [] auth f1 by f1
|
||||
2016-03-09 10:16:44.864 [info] <0.132.0> CONFIRM epoch 1151 <<239,29,39,70>> upi [f1] rep [f3] auth f1 by f1
|
||||
2016-03-09 10:16:45.235 [info] <0.132.0> CONFIRM epoch 1152 <<173,17,66,225>> upi [f2] rep [f1,f3] auth f2 by f1
|
||||
2016-03-09 10:16:47.343 [info] <0.132.0> CONFIRM epoch 1154 <<154,231,224,149>> upi [f2,f1,f3] rep [] auth f2 by f1
|
||||
|
||||
Let's pick apart some of these lines. We have started all three
|
||||
servers at about the same time. We see some race conditions happen,
|
||||
and some jostling and readjustment happens pretty quickly in the first
|
||||
few seconds.
|
||||
|
||||
* `Started FLU f1 with supervisor pid <0.128.0>`
|
||||
* This VM, #1,
|
||||
started a FLU (Machi data server) with the name `f1`. In the Erlang
|
||||
process supervisor hierarchy, the process ID of the top supervisor
|
||||
is `<0.128.0>`.
|
||||
* `Configured chain c1 via FLU f1 to mode=ap_mode all=[f1,f2,f3] witnesses=[]`
|
||||
* A bootstrap configuration for a chain named `c1` has been created.
|
||||
* The FLUs/data servers that are eligible for participation in the
|
||||
chain have names `f1`, `f2`, and `f3`.
|
||||
* The chain will operate in eventual consistency mode (`ap_mode`)
|
||||
* The witness server list is empty. Witness servers are never used
|
||||
in eventual consistency mode.
|
||||
* `CONFIRM epoch 1141 <<155,42,7,221>> upi [] rep [] auth f1 by f1`
|
||||
* All participants in epoch 1141 are unanimous in adopting epoch
|
||||
1141's projection. All active membership lists are empty, so
|
||||
there is no functional chain replication yet, at least as far as
|
||||
server `f1` knows
|
||||
* The epoch's abbreviated checksum is `<<155,42,7,221>>`.
|
||||
* The UPI list, i.e. the replicas whose data is 100% in sync is
|
||||
`[]`, the empty list. (UPI = Update Propagation Invariant)
|
||||
* The list of servers that are under data repair (`rep`) is also
|
||||
empty, `[]`.
|
||||
* This projection was authored by server `f1`.
|
||||
* The log message was generated by server `f1`.
|
||||
* `CONFIRM epoch 1148 <<57,213,154,16>> upi [f1] rep [] auth f1 by f1`
|
||||
* Now the server `f1` has created a chain of length 1, `[f1]`.
|
||||
* Chain repair/file re-sync is not required when the UPI server list
|
||||
changes from length 0 -> 1.
|
||||
* `CONFIRM epoch 1151 <<239,29,39,70>> upi [f1] rep [f3] auth f1 by f1`
|
||||
* Server `f1` has noticed that server `f3` is alive. Apparently it
|
||||
has not yet noticed that server `f2` is also running.
|
||||
* Server `f3` is in the repair list.
|
||||
* `CONFIRM epoch 1152 <<173,17,66,225>> upi [f2] rep [f1,f3] auth f2 by f1`
|
||||
* Server `f2` is apparently now aware that all three servers are running.
|
||||
* The previous configuration used by `f2` was `upi [f2]`, i.e., `f2`
|
||||
was running in a chain of one. `f2` noticed that `f1` and `f3`
|
||||
were now available and has started adding them to the chain.
|
||||
* All new servers are always added to the tail of the chain in the
|
||||
repair list.
|
||||
* In eventual consistency mode, a UPI change like this is OK.
|
||||
* When performing a read, a client must read from both tail of the
|
||||
UPI list and also from all repairing servers.
|
||||
* When performing a write, the client writes to both the UPI
|
||||
server list and also the repairing list, in that order.
|
||||
* I.e., the client concatenates both lists,
|
||||
`UPI ++ Repairing`, for its chain configuration for the write.
|
||||
* Server `f2` will trigger file repair/re-sync shortly.
|
||||
* The waiting time for starting repair has been configured to be
|
||||
extremely short, 1 second. The default waiting time is 10
|
||||
seconds, in case Humming Consensus remains unstable.
|
||||
* `CONFIRM epoch 1154 <<154,231,224,149>> upi [f2,f1,f3] rep [] auth f2 by f1`
|
||||
* File repair/re-sync has finished. All file data on all servers
|
||||
are now in sync.
|
||||
* The UPI/in-sync part of the chain is now `[f2,f1,f3]`, and there
|
||||
are no servers under repair.
|
||||
|
||||
## Let's create some failures
|
||||
|
||||
Here are some suggestions for creating failures.
|
||||
|
||||
* Use the `./dev/devN/bin/machi stop` and `./dev/devN/bin/machi start`
|
||||
commands to stop & start VM #`N`.
|
||||
* Stop a VM abnormally by using `kill`. The OS process name to look
|
||||
for is `beam.smp`.
|
||||
* Suspend and resume a VM, using the `SIGSTOP` and `SIGCONT` signals.
|
||||
* E.g. `kill -STOP 9823` and `kill -CONT 9823`
|
||||
|
||||
The network partition simulator is not (yet) available when running
|
||||
Machi in this mode. Please see the next section for instructions on
|
||||
how to use partition simulator.
|
||||
|
||||
|
||||
<a name="partition-simulator">
|
||||
# Using the network partition simulator and convergence demo test code
|
||||
|
||||
This is the demo code mentioned in the presentation that Scott Lystig
|
||||
Fritchie gave at the
|
||||
[RICON 2015 conference](http://ricon.io).
|
||||
* [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf)
|
||||
* [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q)
|
||||
|
||||
## A complete example of all input and output
|
||||
|
||||
If you don't have an Erlang/OTP 17 runtime environment available,
|
||||
please see this file for full input and output of a strong consistency
|
||||
length=3 chain test:
|
||||
https://gist.github.com/slfritchie/8352efc88cc18e62c72c
|
||||
This file contains all commands input and all simulator output from a
|
||||
sample run of the simulator.
|
||||
|
||||
To help interpret the output of the test, please skip ahead to the
|
||||
"The test output is very verbose" section.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
If you don't have `git` and/or the Erlang 17 runtime system available
|
||||
on your OS X, FreeBSD, Linux, or Solaris machine, please take a look
|
||||
at the [Prerequisites section](#prerequisites) first. When you have
|
||||
installed the prerequisite software, please return back here.
|
||||
|
||||
## Clone and compile the code
|
||||
|
||||
Please briefly visit the [Clone and compile the code](#clone-compile)
|
||||
section. When finished, please return back here.
|
||||
|
||||
## Run an interactive Erlang CLI shell
|
||||
|
||||
Run the following command at your login shell:
|
||||
|
||||
erl -pz .eunit ebin deps/*/ebin
|
||||
|
||||
If you are using Erlang/OTP version 17, you should see some CLI output
|
||||
that looks like this:
|
||||
|
||||
Erlang/OTP 17 [erts-6.4] [source] [64-bit] [smp:8:8] [async-threads:10] [hipe] [kernel-poll:false] [dtrace]
|
||||
|
||||
Eshell V6.4 (abort with ^G)
|
||||
1>
|
||||
|
||||
## The test output is very verbose ... what are the important parts?
|
||||
|
||||
The output of the Erlang command
|
||||
`machi_chain_manager1_converge_demo:help()` will display the following
|
||||
guide to the output of the tests.
|
||||
|
||||
A visualization of the convergence behavior of the chain self-management
|
||||
algorithm for Machi.
|
||||
|
||||
1. Set up some server and chain manager pairs.
|
||||
2. Create a number of different network partition scenarios, where
|
||||
(simulated) partitions may be symmetric or asymmetric. Then stop changing
|
||||
the partitions and keep the simulated network stable (and perhaps broken).
|
||||
3. Run a number of iterations of the algorithm in parallel by poking each
|
||||
of the manager processes on a random'ish basis.
|
||||
4. Afterward, fetch the chain transition changes made by each FLU and
|
||||
verify that no transition was unsafe.
|
||||
|
||||
During the iteration periods, the following is a cheatsheet for the output.
|
||||
See the internal source for interpreting the rest of the output.
|
||||
|
||||
'SET partitions = '
|
||||
|
||||
A pair-wise list of actors which cannot send messages. The
|
||||
list is uni-directional. If there are three servers (a,b,c),
|
||||
and if the partitions list is '[{a,b},{b,c}]' then all
|
||||
messages from a->b and b->c will be dropped, but any other
|
||||
sender->recipient messages will be delivered successfully.
|
||||
|
||||
'x uses:'
|
||||
|
||||
The FLU x has made an internal state transition and is using
|
||||
this epoch's projection as operating chain configuration. The
|
||||
rest of the line is a summary of the projection.
|
||||
|
||||
'CONFIRM epoch {N}'
|
||||
|
||||
This message confirms that all of the servers listed in the
|
||||
UPI and repairing lists of the projection at epoch {N} have
|
||||
agreed to use this projection because they all have written
|
||||
this projection to their respective private projection stores.
|
||||
The chain is now usable by/available to all clients.
|
||||
|
||||
'Sweet, private projections are stable'
|
||||
|
||||
This report announces that this iteration of the test cycle
|
||||
has passed successfully. The report that follows briefly
|
||||
summarizes the latest private projection used by each
|
||||
participating server. For example, when in strong consistency
|
||||
mode with 'a' as a witness and 'b' and 'c' as real servers:
|
||||
|
||||
%% Legend:
|
||||
%% server name, epoch ID, UPI list, repairing list, down list, ...
|
||||
%% ... witness list, 'false' (a constant value)
|
||||
|
||||
[{a,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}},
|
||||
{b,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}}]
|
||||
|
||||
Both servers 'a' and 'b' agree on epoch 1116 with epoch ID
|
||||
{1116,<<23,143,246,55>>} where UPI=[a,b], repairing=[],
|
||||
down=[c], and witnesses=[a].
|
||||
|
||||
Server 'c' is not shown because 'c' has wedged itself OOS (out
|
||||
of service) by configuring a chain length of zero.
|
||||
|
||||
If no servers are listed in the report (i.e. only '[]' is
|
||||
displayed), then all servers have wedged themselves OOS, and
|
||||
the chain is unavailable.
|
||||
|
||||
'DoIt,'
|
||||
|
||||
This marks a group of tick events which trigger the manager
|
||||
processes to evaluate their environment and perhaps make a
|
||||
state transition.
|
||||
|
||||
A long chain of 'DoIt,DoIt,DoIt,' means that the chain state has
|
||||
(probably) settled to a stable configuration, which is the goal of the
|
||||
algorithm.
|
||||
|
||||
Press control-c to interrupt the test....".
|
||||
|
||||
## Run a test in eventual consistency mode
|
||||
|
||||
Run the following command at the Erlang CLI prompt:
|
||||
|
||||
machi_chain_manager1_converge_demo:t(3, [{private_write_verbose,true}]).
|
||||
|
||||
The first argument, `3`, is the number of servers to participate in
|
||||
the chain. Please note:
|
||||
|
||||
* Chain lengths as short as 1 or 2 are valid, but the results are a
|
||||
bit boring.
|
||||
* Chain lengths as long as 7 or 9 can be used, but they may
|
||||
suffer from longer periods of churn/instability before all chain
|
||||
managers reach agreement via humming consensus. (It is future work
|
||||
to shorten the worst of the unstable churn latencies.)
|
||||
* In eventual consistency mode, chain lengths may be even numbers,
|
||||
e.g. 2, 4, or 6.
|
||||
* The simulator will choose partition events from the permutations of
|
||||
all 1, 2, and 3 node partition pairs. The total runtime will
|
||||
increase *dramatically* with chain length.
|
||||
* Chain length 2: about 3 partition cases
|
||||
* Chain length 3: about 35 partition cases
|
||||
* Chain length 4: about 230 partition cases
|
||||
* Chain length 5: about 1100 partition cases
|
||||
|
||||
## Run a test in strong consistency mode (with witnesses):
|
||||
|
||||
*NOTE:* Due to a bug in the test code, please do not try to run the
|
||||
convergence test in strong consistency mode and also without the
|
||||
correct minority number of witness servers! If in doubt, please run
|
||||
the commands shown below exactly.
|
||||
|
||||
Run the following command at the Erlang CLI prompt:
|
||||
|
||||
machi_chain_manager1_converge_demo:t(3, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a]}]).
|
||||
|
||||
The first argument, `3`, is the number of servers to participate in
|
||||
the chain. Chain lengths as long as 7 or 9 can be used, but they may
|
||||
suffer from longer periods of churn/instability before all chain
|
||||
managers reach agreement via humming consensus.
|
||||
|
||||
Due to the bug mentioned above, please use the following
|
||||
commands when running with chain lengths of 5 or 7, respectively.
|
||||
|
||||
machi_chain_manager1_converge_demo:t(5, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a,b]}]).
|
||||
machi_chain_manager1_converge_demo:t(7, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a,b,c]}]).
|
||||
|
||||
185
doc/machi_chain_manager1_converge_demo.md
Normal file
185
doc/machi_chain_manager1_converge_demo.md
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
|
||||
# Using the network partition simulator and convergence demo test code
|
||||
|
||||
## A complete example of all input and output
|
||||
|
||||
If you don't have an Erlang/OTP 17 runtime environment available,
|
||||
please see this file for full input and output of a strong consistency
|
||||
length=3 chain test:
|
||||
https://gist.github.com/slfritchie/8352efc88cc18e62c72c
|
||||
This file contains all commands input and all simulator output from a
|
||||
sample run of the simulator.
|
||||
|
||||
To help interpret the output of the test, please skip ahead to the
|
||||
"The test output is very verbose" section.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. You'll need the `git` source management
|
||||
2. You'll need the Erlang/OTP 17 runtime environment. Please don't
|
||||
use earlier or later versions until we have a chance to fix the
|
||||
compilation warnings that versions R16B and 18 will trigger.
|
||||
|
||||
All of the commands that should be run at your login shell (e.g. Bash,
|
||||
c-shell) can be cut-and-pasted from this document directly to your
|
||||
login shell prompt.
|
||||
|
||||
## Clone and compile the code
|
||||
|
||||
Clone the Machi source repo and compile the source and test code. Run
|
||||
the following commands at your login shell:
|
||||
|
||||
cd /tmp
|
||||
git clone https://github.com/basho/machi.git
|
||||
cd machi
|
||||
git checkout master
|
||||
make
|
||||
|
||||
Then run the unit test suite. This may take up to two minutes or so
|
||||
to finish. Most of the tests will be silent; please be patient until
|
||||
the tests finish.
|
||||
|
||||
make test
|
||||
|
||||
## Run an interactive Erlang CLI shell
|
||||
|
||||
Run the following command at your login shell:
|
||||
|
||||
erl -pz .eunit ebin deps/*/ebin
|
||||
|
||||
If you are using Erlang/OTP version 17, you should see some CLI output
|
||||
that looks like this:
|
||||
|
||||
Erlang/OTP 17 [erts-6.4] [source] [64-bit] [smp:8:8] [async-threads:10] [hipe] [kernel-poll:false] [dtrace]
|
||||
|
||||
Eshell V6.4 (abort with ^G)
|
||||
1>
|
||||
|
||||
## The test output is very verbose ... what are the important parts?
|
||||
|
||||
The output of the Erlang command
|
||||
`machi_chain_manager1_converge_demo:help()` will display the following
|
||||
guide to the output of the tests.
|
||||
|
||||
A visualization of the convergence behavior of the chain self-management
|
||||
algorithm for Machi.
|
||||
|
||||
1. Set up some server and chain manager pairs.
|
||||
2. Create a number of different network partition scenarios, where
|
||||
(simulated) partitions may be symmetric or asymmetric. Then stop changing
|
||||
the partitions and keep the simulated network stable (and perhaps broken).
|
||||
3. Run a number of iterations of the algorithm in parallel by poking each
|
||||
of the manager processes on a random'ish basis.
|
||||
4. Afterward, fetch the chain transition changes made by each FLU and
|
||||
verify that no transition was unsafe.
|
||||
|
||||
During the iteration periods, the following is a cheatsheet for the output.
|
||||
See the internal source for interpreting the rest of the output.
|
||||
|
||||
'SET partitions = '
|
||||
|
||||
A pair-wise list of actors which cannot send messages. The
|
||||
list is uni-directional. If there are three servers (a,b,c),
|
||||
and if the partitions list is '[{a,b},{b,c}]' then all
|
||||
messages from a->b and b->c will be dropped, but any other
|
||||
sender->recipient messages will be delivered successfully.
|
||||
|
||||
'x uses:'
|
||||
|
||||
The FLU x has made an internal state transition and is using
|
||||
this epoch's projection as operating chain configuration. The
|
||||
rest of the line is a summary of the projection.
|
||||
|
||||
'CONFIRM epoch {N}'
|
||||
|
||||
This message confirms that all of the servers listed in the
|
||||
UPI and repairing lists of the projection at epoch {N} have
|
||||
agreed to use this projection because they all have written
|
||||
this projection to their respective private projection stores.
|
||||
The chain is now usable by/available to all clients.
|
||||
|
||||
'Sweet, private projections are stable'
|
||||
|
||||
This report announces that this iteration of the test cycle
|
||||
has passed successfully. The report that follows briefly
|
||||
summarizes the latest private projection used by each
|
||||
participating server. For example, when in strong consistency
|
||||
mode with 'a' as a witness and 'b' and 'c' as real servers:
|
||||
|
||||
%% Legend:
|
||||
%% server name, epoch ID, UPI list, repairing list, down list, ...
|
||||
%% ... witness list, 'false' (a constant value)
|
||||
|
||||
[{a,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}},
|
||||
{b,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}}]
|
||||
|
||||
Both servers 'a' and 'b' agree on epoch 1116 with epoch ID
|
||||
{1116,<<23,143,246,55>>} where UPI=[a,b], repairing=[],
|
||||
down=[c], and witnesses=[a].
|
||||
|
||||
Server 'c' is not shown because 'c' has wedged itself OOS (out
|
||||
of service) by configuring a chain length of zero.
|
||||
|
||||
If no servers are listed in the report (i.e. only '[]' is
|
||||
displayed), then all servers have wedged themselves OOS, and
|
||||
the chain is unavailable.
|
||||
|
||||
'DoIt,'
|
||||
|
||||
This marks a group of tick events which trigger the manager
|
||||
processes to evaluate their environment and perhaps make a
|
||||
state transition.
|
||||
|
||||
A long chain of 'DoIt,DoIt,DoIt,' means that the chain state has
|
||||
(probably) settled to a stable configuration, which is the goal of the
|
||||
algorithm.
|
||||
|
||||
Press control-c to interrupt the test....".
|
||||
|
||||
## Run a test in eventual consistency mode
|
||||
|
||||
Run the following command at the Erlang CLI prompt:
|
||||
|
||||
machi_chain_manager1_converge_demo:t(3, [{private_write_verbose,true}]).
|
||||
|
||||
The first argument, `3`, is the number of servers to participate in
|
||||
the chain. Please note:
|
||||
|
||||
* Chain lengths as short as 1 or 2 are valid, but the results are a
|
||||
bit boring.
|
||||
* Chain lengths as long as 7 or 9 can be used, but they may
|
||||
suffer from longer periods of churn/instability before all chain
|
||||
managers reach agreement via humming consensus. (It is future work
|
||||
to shorten the worst of the unstable churn latencies.)
|
||||
* In eventual consistency mode, chain lengths may be even numbers,
|
||||
e.g. 2, 4, or 6.
|
||||
* The simulator will choose partition events from the permutations of
|
||||
all 1, 2, and 3 node partition pairs. The total runtime will
|
||||
increase *dramatically* with chain length.
|
||||
* Chain length 2: about 3 partition cases
|
||||
* Chain length 3: about 35 partition cases
|
||||
* Chain length 4: about 230 partition cases
|
||||
* Chain length 5: about 1100 partition cases
|
||||
|
||||
## Run a test in strong consistency mode (with witnesses):
|
||||
|
||||
*NOTE:* Due to a bug in the test code, please do not try to run the
|
||||
convergence test in strong consistency mode and also without the
|
||||
correct minority number of witness servers! If in doubt, please run
|
||||
the commands shown below exactly.
|
||||
|
||||
Run the following command at the Erlang CLI prompt:
|
||||
|
||||
machi_chain_manager1_converge_demo:t(3, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a]}]).
|
||||
|
||||
The first argument, `3`, is the number of servers to participate in
|
||||
the chain. Chain lengths as long as 7 or 9 can be used, but they may
|
||||
suffer from longer periods of churn/instability before all chain
|
||||
managers reach agreement via humming consensus.
|
||||
|
||||
Due to the bug mentioned above, please use the following
|
||||
commands when running with chain lengths of 5 or 7, respectively.
|
||||
|
||||
machi_chain_manager1_converge_demo:t(5, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a,b]}]).
|
||||
machi_chain_manager1_converge_demo:t(7, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a,b,c]}]).
|
||||
|
||||
170
doc/overview.edoc
Normal file
170
doc/overview.edoc
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
|
||||
@title Machi: a small village of replicated files
|
||||
|
||||
@doc
|
||||
|
||||
== About This EDoc Documentation ==
|
||||
|
||||
This EDoc-style documentation will concern itself only with Erlang
|
||||
function APIs and function & data types. Higher-level design and
|
||||
commentary will remain outside of the Erlang EDoc system; please see
|
||||
the "Pointers to Other Machi Documentation" section below for more
|
||||
details.
|
||||
|
||||
Readers should beware that this documentation may be out-of-sync with
|
||||
the source code. When in doubt, use the `make edoc' command to
|
||||
regenerate all HTML pages.
|
||||
|
||||
It is the developer's responsibility to re-generate the documentation
|
||||
periodically and commit it to the Git repo.
|
||||
|
||||
== Machi Code Overview ==
|
||||
|
||||
=== Chain Manager ===
|
||||
|
||||
The Chain Manager is responsible for managing the state of Machi's
|
||||
"Chain Replication" state. This role is roughly analogous to the
|
||||
"Riak Core" application inside of Riak, which takes care of
|
||||
coordinating replica placement and replica repair.
|
||||
|
||||
For each primitive data server in the cluster, a Machi FLU, there is a
|
||||
Chain Manager process that manages its FLU's role within the Machi
|
||||
cluster's Chain Replication scheme. Each Chain Manager process
|
||||
executes locally and independently to manage the distributed state of
|
||||
a single Machi Chain Replication chain.
|
||||
|
||||
<ul>
|
||||
|
||||
<li> To contrast with Riak Core ... Riak Core's claimant process is
|
||||
solely responsible for managing certain critical aspects of
|
||||
Riak Core distributed state. Machi's Chain Manager process
|
||||
performs similar tasks as Riak Core's claimant. However, Machi
|
||||
has several active Chain Manager processes, one per FLU server,
|
||||
instead of a single active process like Core's claimant. Each
|
||||
Chain Manager process acts independently; each is constrained
|
||||
so that it will reach consensus via independent computation
|
||||
& action.
|
||||
|
||||
Full discussion of this distributed consensus is outside the
|
||||
scope of this document; see the "Pointers to Other Machi
|
||||
Documentation" section below for more information.
|
||||
</li>
|
||||
<li> Machi differs from a Riak Core application because Machi's
|
||||
replica placement policy is simply, "All Machi servers store
|
||||
replicas of all Machi files".
|
||||
Machi is intended to be a primitive building block for creating larger
|
||||
cluster-of-clusters where files are
|
||||
distributed/fragmented/sharded across a large pool of
|
||||
independent Machi clusters.
|
||||
</li>
|
||||
<li> See
|
||||
[https://www.usenix.org/legacy/events/osdi04/tech/renesse.html]
|
||||
for a copy of the paper, "Chain Replication for Supporting High
|
||||
Throughput and Availability" by Robbert van Renesse and Fred
|
||||
B. Schneider.
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
=== FLU ===
|
||||
|
||||
The FLU is the basic storage server for Machi.
|
||||
|
||||
<ul>
|
||||
<li> The name FLU is taken from "flash storage unit" from the paper
|
||||
"CORFU: A Shared Log Design for Flash Clusters" by
|
||||
Balakrishnan, Malkhi, Prabhakaran, and Wobber. See
|
||||
[https://www.usenix.org/conference/nsdi12/technical-sessions/presentation/balakrishnan]
|
||||
</li>
|
||||
<li> In CORFU, the sequencer step is a prerequisite step that is
|
||||
performed by a separate component, the Sequencer.
|
||||
In Machi, the `append_chunk()' protocol message has
|
||||
an implicit "sequencer" operation applied by the "head" of the
|
||||
Machi Chain Replication chain. If a client wishes to write
|
||||
data that has already been assigned a sequencer position, then
|
||||
the `write_chunk()' API function is used.
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
For each FLU, there are three independent tasks that are implemented
|
||||
using three different Erlang processes:
|
||||
|
||||
<ul>
|
||||
<li> A FLU server, implemented primarily by `machi_flu.erl'.
|
||||
</li>
|
||||
<li> A projection store server, implemented primarily by
|
||||
`machi_projection_store.erl'.
|
||||
</li>
|
||||
<li> A chain state manager server, implemented primarily by
|
||||
`machi_chain_manager1.erl'.
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
From the perspective of failure detection, it is very convenient that
|
||||
all three FLU-related services (file server, sequencer server, and
|
||||
projection server) are accessed using the same single TCP port.
|
||||
|
||||
=== Projection (data structure) ===
|
||||
|
||||
The projection is a data structure that specifies the current state
|
||||
of the Machi cluster: all FLUs, which FLUS are considered
|
||||
up/running or down/crashed/stopped, which FLUs are actively
|
||||
participants in the Chain Replication protocol, and which FLUs are
|
||||
under "repair" (i.e., having their data resyncronized when
|
||||
newly-added to a cluster or when restarting after a crash).
|
||||
|
||||
=== Projection Store (server) ===
|
||||
|
||||
The projection store is a storage service that is implemented by an
|
||||
Erlang/OTP `gen_server' process that is associated with each
|
||||
FLU. Conceptually, the projection store is an array of
|
||||
write-once registers. For each projection store register, the
|
||||
key is a 2-tuple of an epoch number (`non_neg_integer()' type)
|
||||
and a projection type (`public' or `private' type); the value is
|
||||
a projection data structure (`projection_v1()' type).
|
||||
|
||||
=== Client and Proxy Client ===
|
||||
|
||||
Machi is intentionally avoiding using distributed Erlang for Machi's
|
||||
communication. This design decision makes Erlang-side code more
|
||||
difficult & complex but allows us the freedom of implementing
|
||||
parts of Machi in other languages without major
|
||||
protocol&API&glue code changes later in the product's
|
||||
lifetime.
|
||||
|
||||
There are two layers of interface for Machi clients.
|
||||
|
||||
<ul>
|
||||
<li> The `machi_flu1_client' module implements an API that uses a
|
||||
TCP socket directly.
|
||||
</li>
|
||||
<li> The `machi_proxy_flu1_client' module implements an API that
|
||||
uses a local, long-lived `gen_server' process as a proxy for
|
||||
the remote, perhaps disconnected-or-crashed Machi FLU server.
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
The types for both modules ought to be the same. However, due to
|
||||
rapid code churn, some differences might exist. Any major difference
|
||||
is (almost by definition) a bug: please open a GitHub issue to request
|
||||
a correction.
|
||||
|
||||
== TODO notes ==
|
||||
|
||||
Any use of the string "TODO" in upper/lower/mixed case, anywhere in
|
||||
the code, is a reminder signal of unfinished work.
|
||||
|
||||
== Pointers to Other Machi Documentation ==
|
||||
|
||||
<ul>
|
||||
<li> If you are viewing this document locally, please look in the
|
||||
`../doc/' directory,
|
||||
</li>
|
||||
<li> If you are viewing this document via the Web, please find the
|
||||
documentation via this link:
|
||||
[http://github.com/basho/machi/tree/master/doc/]
|
||||
Please be aware that this link points to the `master' branch
|
||||
of the Machi source repository and therefore may be
|
||||
out-of-sync with non-`master' branch code.
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 115 KiB |
|
|
@ -23,8 +23,8 @@
|
|||
\copyrightdata{978-1-nnnn-nnnn-n/yy/mm}
|
||||
\doi{nnnnnnn.nnnnnnn}
|
||||
|
||||
\titlebanner{Draft \#0.92, October 2015}
|
||||
\preprintfooter{Draft \#0.92, October 2015}
|
||||
\titlebanner{Draft \#0.91, June 2015}
|
||||
\preprintfooter{Draft \#0.91, June 2015}
|
||||
|
||||
\title{Chain Replication metadata management in Machi, an immutable
|
||||
file store}
|
||||
|
|
@ -50,23 +50,19 @@ For an overview of the design of the larger Machi system, please see
|
|||
TODO Fix, after all of the recent changes to this document.
|
||||
|
||||
Machi is an immutable file store, now in active development by Basho
|
||||
Japan KK. Machi uses Chain Replication\footnote{Chain
|
||||
Japan KK. Machi uses Chain Replication to maintain strong consistency
|
||||
of file updates to all replica servers in a Machi cluster. Chain
|
||||
Replication is a variation of primary/backup replication where the
|
||||
order of updates between the primary server and each of the backup
|
||||
servers is strictly ordered into a single ``chain''.}
|
||||
to maintain strong consistency
|
||||
of file updates to all replica servers in a Machi cluster.
|
||||
|
||||
This document describes the Machi chain manager, the component
|
||||
responsible for managing Chain Replication metadata state.
|
||||
Management of
|
||||
chain metadata, e.g., ``What is the current order of
|
||||
servers is strictly ordered into a single ``chain''. Management of
|
||||
Chain Replication's metadata, e.g., ``What is the current order of
|
||||
servers in the chain?'', remains an open research problem. The
|
||||
current state of the art for Chain Replication metadata management
|
||||
relies on an external oracle (e.g., based on ZooKeeper) or the Elastic
|
||||
Replication \cite{elastic-chain-replication} algorithm.
|
||||
relies on an external oracle (e.g., ZooKeeper) or the Elastic
|
||||
Replication algorithm.
|
||||
|
||||
The chain
|
||||
This document describes the Machi chain manager, the component
|
||||
responsible for managing Chain Replication metadata state. The chain
|
||||
manager uses a new technique, based on a variation of CORFU, called
|
||||
``humming consensus''.
|
||||
Humming consensus does not require active participation by all or even
|
||||
|
|
@ -93,18 +89,20 @@ to perform these management tasks. Chain metadata state and state
|
|||
management tasks include:
|
||||
|
||||
\begin{itemize}
|
||||
\item Preserving data integrity of all metadata and data stored within
|
||||
the chain. Data loss is not an option.
|
||||
\item Preserving stable knowledge of chain membership (i.e. all nodes in
|
||||
the chain, regardless of operational status). We expect that a systems
|
||||
administrator will make all ``permanent'' decisions about
|
||||
the chain, regardless of operational status). A systems
|
||||
administrator is expected to make ``permanent'' decisions about
|
||||
chain membership.
|
||||
\item Using passive and/or active techniques to track operational
|
||||
state/status, e.g., up, down, restarting, full data sync in progress, partial
|
||||
data sync in progress, etc.
|
||||
state/status, e.g., up, down, restarting, full data sync, partial
|
||||
data sync, etc.
|
||||
\item Choosing the run-time replica ordering/state of the chain, based on
|
||||
current member status and past operational history. All chain
|
||||
state transitions must be done safely and without data loss or
|
||||
corruption.
|
||||
\item When a new node is added to the chain administratively or old node is
|
||||
\item As a new node is added to the chain administratively or old node is
|
||||
restarted, adding the node to the chain safely and perform any data
|
||||
synchronization/repair required to bring the node's data into
|
||||
full synchronization with the other nodes.
|
||||
|
|
@ -113,27 +111,39 @@ management tasks include:
|
|||
\subsection{Ultimate goal: Preserve data integrity of Chain Replicated data}
|
||||
|
||||
Preservation of data integrity is paramount to any chain state
|
||||
management technique for Machi. Loss or corruption of chain data must
|
||||
be avoided.
|
||||
management technique for Machi. Even when operating in an eventually
|
||||
consistent mode, Machi must not lose data without cause outside of all
|
||||
design, e.g., all particpants crash permanently.
|
||||
|
||||
\subsection{Goal: Contribute to Chain Replication metadata management research}
|
||||
|
||||
We believe that this new self-management algorithm, humming consensus,
|
||||
contributes a novel approach to Chain Replication metadata management.
|
||||
The ``monitor
|
||||
and mangage your neighbor'' technique proposed in Elastic Replication
|
||||
(Section \ref{ssec:elastic-replication}) appears to be the current
|
||||
state of the art in the distributed systems research community.
|
||||
Typical practice in the IT industry appears to favor using an external
|
||||
oracle, e.g., built on top of ZooKeeper as a trusted coordinator.
|
||||
oracle, e.g., using ZooKeeper as a trusted coordinator.
|
||||
|
||||
See Section~\ref{sec:cr-management-review} for a brief review of
|
||||
techniques used today.
|
||||
See Section~\ref{sec:cr-management-review} for a brief review.
|
||||
|
||||
\subsection{Goal: Support both eventually consistent \& strongly consistent modes of operation}
|
||||
|
||||
Chain Replication was originally designed by van Renesse and Schneider
|
||||
\cite{chain-replication} for applications that require strong
|
||||
consistency, e.g. sequential consistency. However, Machi has use
|
||||
cases where more relaxed eventual consistency semantics are
|
||||
sufficient. We wish to use the same Chain Replication management
|
||||
technique for both strong and eventual consistency environments.
|
||||
Machi's first use cases are all for use as a file store in an eventually
|
||||
consistent environment.
|
||||
In eventually consistent mode, humming consensus
|
||||
allows a Machi cluster to fragment into
|
||||
arbitrary islands of network partition, all the way down to 100\% of
|
||||
members running in complete network isolation from each other.
|
||||
Furthermore, it provides enough agreement to allow
|
||||
formerly-partitioned members to coordinate the reintegration and
|
||||
reconciliation of their data when partitions are healed.
|
||||
|
||||
Later, we wish the option of supporting strong consistency
|
||||
applications such as CORFU-style logging while reusing all (or most)
|
||||
of Machi's infrastructure. Such strongly consistent operation is the
|
||||
main focus of this document.
|
||||
|
||||
\subsection{Anti-goal: Minimize churn}
|
||||
|
||||
|
|
@ -194,18 +204,6 @@ would probably be preferable to add the feature to Riak Ensemble
|
|||
rather than to use ZooKeeper (and for Basho to document ZK, package
|
||||
ZK, provide commercial ZK support, etc.).
|
||||
|
||||
\subsection{An external management oracle, implemented by
|
||||
active/standby application failover}
|
||||
|
||||
This technique has been used in production of HibariDB. The customer
|
||||
very carefully deployed the oracle using the Erlang/OTP ``application
|
||||
controller'' on two machines to provide active/standby failover of the
|
||||
management oracle. The customer was willing to monitor this service
|
||||
very closely and was prepared to intervene manually during network
|
||||
partitions. (This controller is very susceptible to ``split brain
|
||||
syndrome''.) While this feature of Erlang/OTP is useful in other
|
||||
environments, we believe is it not sufficient for Machi's needs.
|
||||
|
||||
\section{Assumptions}
|
||||
\label{sec:assumptions}
|
||||
|
||||
|
|
@ -214,8 +212,8 @@ Paxos, Raft, et al.), why bother with a slightly different set of
|
|||
assumptions and a slightly different protocol?
|
||||
|
||||
The answer lies in one of our explicit goals: to have an option of
|
||||
running in an ``eventually consistent'' manner. We wish to be
|
||||
remain available, even if we are
|
||||
running in an ``eventually consistent'' manner. We wish to be able to
|
||||
make progress, i.e., remain available in the CAP sense, even if we are
|
||||
partitioned down to a single isolated node. VR, Paxos, and Raft
|
||||
alone are not sufficient to coordinate service availability at such
|
||||
small scale. The humming consensus algorithm can manage
|
||||
|
|
@ -249,15 +247,13 @@ synchronized by NTP.
|
|||
|
||||
The protocol and algorithm presented here do not specify or require any
|
||||
timestamps, physical or logical. Any mention of time inside of data
|
||||
structures are for human and/or diagnostic purposes only.
|
||||
structures are for human/historic/diagnostic purposes only.
|
||||
|
||||
Having said that, some notion of physical time is suggested
|
||||
occasionally for
|
||||
purposes of efficiency. For example, some ``sleep
|
||||
Having said that, some notion of physical time is suggested for
|
||||
purposes of efficiency. It's recommended that there be some ``sleep
|
||||
time'' between iterations of the algorithm: there is no need to ``busy
|
||||
wait'' by executing the algorithm as many times per minute as
|
||||
possible.
|
||||
See also Section~\ref{ssub:when-to-calc}.
|
||||
wait'' by executing the algorithm as quickly as possible. See also
|
||||
Section~\ref{ssub:when-to-calc}.
|
||||
|
||||
\subsection{Failure detector model}
|
||||
|
||||
|
|
@ -280,73 +276,55 @@ eventual consistency. Discussion of strongly consistent CP
|
|||
mode is always the default; exploration of AP mode features in this document
|
||||
will always be explictly noted.
|
||||
|
||||
%%\subsection{Use of the ``wedge state''}
|
||||
%%
|
||||
%%A participant in Chain Replication will enter ``wedge state'', as
|
||||
%%described by the Machi high level design \cite{machi-design} and by CORFU,
|
||||
%%when it receives information that
|
||||
%%a newer projection (i.e., run-time chain state reconfiguration) is
|
||||
%%available. The new projection may be created by a system
|
||||
%%administrator or calculated by the self-management algorithm.
|
||||
%%Notification may arrive via the projection store API or via the file
|
||||
%%I/O API.
|
||||
%%
|
||||
%%When in wedge state, the server will refuse all file write I/O API
|
||||
%%requests until the self-management algorithm has determined that
|
||||
%%humming consensus has been decided (see next bullet item). The server
|
||||
%%may also refuse file read I/O API requests, depending on its CP/AP
|
||||
%%operation mode.
|
||||
%%
|
||||
%%\subsection{Use of ``humming consensus''}
|
||||
%%
|
||||
%%CS literature uses the word ``consensus'' in the context of the problem
|
||||
%%description at \cite{wikipedia-consensus}
|
||||
%%.
|
||||
%%This traditional definition differs from what is described here as
|
||||
%%``humming consensus''.
|
||||
%%
|
||||
%%``Humming consensus'' describes
|
||||
%%consensus that is derived only from data that is visible/known at the current
|
||||
%%time.
|
||||
%%The algorithm will calculate
|
||||
%%a rough consensus despite not having input from a quorum majority
|
||||
%%of chain members. Humming consensus may proceed to make a
|
||||
%%decision based on data from only a single participant, i.e., only the local
|
||||
%%node.
|
||||
%%
|
||||
%%See Section~\ref{sec:humming-consensus} for detailed discussion.
|
||||
\subsection{Use of the ``wedge state''}
|
||||
|
||||
%%\subsection{Concurrent chain managers execute humming consensus independently}
|
||||
%%
|
||||
%%Each Machi file server has its own concurrent chain manager
|
||||
%%process embedded within it. Each chain manager process will
|
||||
%%execute the humming consensus algorithm using only local state (e.g.,
|
||||
%%the $P_{current}$ projection currently used by the local server) and
|
||||
%%values observed in everyone's projection stores
|
||||
%%(Section~\ref{sec:projection-store}).
|
||||
%%
|
||||
%%The chain manager communicates with the local Machi
|
||||
%%file server using the wedge and un-wedge request API. When humming
|
||||
%%consensus has chosen a projection $P_{new}$ to replace $P_{current}$,
|
||||
%%the value of $P_{new}$ is included in the un-wedge request.
|
||||
A participant in Chain Replication will enter ``wedge state'', as
|
||||
described by the Machi high level design \cite{machi-design} and by CORFU,
|
||||
when it receives information that
|
||||
a newer projection (i.e., run-time chain state reconfiguration) is
|
||||
available. The new projection may be created by a system
|
||||
administrator or calculated by the self-management algorithm.
|
||||
Notification may arrive via the projection store API or via the file
|
||||
I/O API.
|
||||
|
||||
\subsection{The reader is familiar with CORFU}
|
||||
When in wedge state, the server will refuse all file write I/O API
|
||||
requests until the self-management algorithm has determined that
|
||||
humming consensus has been decided (see next bullet item). The server
|
||||
may also refuse file read I/O API requests, depending on its CP/AP
|
||||
operation mode.
|
||||
|
||||
Machi borrows heavily from the techniques and data structures used by
|
||||
CORFU \cite[corfu1],\cite[corfu2]. We hope that the reader is
|
||||
familiar with CORFU's features, including:
|
||||
\subsection{Use of ``humming consensus''}
|
||||
|
||||
\begin{itemize}
|
||||
\item write-once registers for log data storage,
|
||||
\item the epoch, which defines a period of time when a cluster's configuration
|
||||
is stable,
|
||||
\item strictly increasing epoch numbers, which are identifiers
|
||||
for particular epochs,
|
||||
\item projections, which define the chain order and other details of
|
||||
data replication within the cluster, and
|
||||
\item the wedge state, used by servers to coordinate cluster changes
|
||||
during epoch transitions.
|
||||
\end{itemize}
|
||||
CS literature uses the word ``consensus'' in the context of the problem
|
||||
description at \cite{wikipedia-consensus}
|
||||
.
|
||||
This traditional definition differs from what is described here as
|
||||
``humming consensus''.
|
||||
|
||||
``Humming consensus'' describes
|
||||
consensus that is derived only from data that is visible/known at the current
|
||||
time.
|
||||
The algorithm will calculate
|
||||
a rough consensus despite not having input from all/majority
|
||||
of chain members. Humming consensus may proceed to make a
|
||||
decision based on data from only a single participant, i.e., only the local
|
||||
node.
|
||||
|
||||
See Section~\ref{sec:humming-consensus} for detailed discussion.
|
||||
|
||||
\subsection{Concurrent chain managers execute humming consensus independently}
|
||||
|
||||
Each Machi file server has its own concurrent chain manager
|
||||
process embedded within it. Each chain manager process will
|
||||
execute the humming consensus algorithm using only local state (e.g.,
|
||||
the $P_{current}$ projection currently used by the local server) and
|
||||
values observed in everyone's projection stores
|
||||
(Section~\ref{sec:projection-store}).
|
||||
|
||||
The chain manager communicates with the local Machi
|
||||
file server using the wedge and un-wedge request API. When humming
|
||||
consensus has chosen a projection $P_{new}$ to replace $P_{current}$,
|
||||
the value of $P_{new}$ is included in the un-wedge request.
|
||||
|
||||
\section{The projection store}
|
||||
\label{sec:projection-store}
|
||||
|
|
@ -365,15 +343,19 @@ this key. The
|
|||
store's value is either the special `unwritten' value\footnote{We use
|
||||
$\bot$ to denote the unwritten value.} or else a binary blob that is
|
||||
immutable thereafter; the projection data structure is
|
||||
serialized and stored in this binary blob. See
|
||||
\ref{sub:the-projection} for more detail.
|
||||
serialized and stored in this binary blob.
|
||||
|
||||
The projection store is vital for the correct implementation of humming
|
||||
consensus (Section~\ref{sec:humming-consensus}). The write-once
|
||||
register primitive allows us to reason about the store's behavior
|
||||
using the same logical tools and techniques as the CORFU ordered log.
|
||||
|
||||
\subsection{The publicly-writable half of the projection store}
|
||||
|
||||
The publicly-writable projection store is used to share information
|
||||
during the first half of humming consensus algorithm. Projections
|
||||
in the public half of the store form a log of
|
||||
suggestions\footnote{I hesitate to use the words ``propose'' or ``proposal''
|
||||
suggestions\footnote{I hesitate to use the word ``propose'' or ``proposal''
|
||||
anywhere in this document \ldots until I've done a more formal
|
||||
analysis of the protocol. Those words have too many connotations in
|
||||
the context of consensus protocols such as Paxos and Raft.}
|
||||
|
|
@ -387,9 +369,8 @@ Any chain member may read from the public half of the store.
|
|||
|
||||
The privately-writable projection store is used to store the
|
||||
Chain Replication metadata state (as chosen by humming consensus)
|
||||
that is in use now by the local Machi server. Earlier projections
|
||||
remain in the private half to keep a historical
|
||||
record of chain state transitions by the local server.
|
||||
that is in use now by the local Machi server as well as previous
|
||||
operation states.
|
||||
|
||||
Only the local server may write values into the private half of store.
|
||||
Any chain member may read from the private half of the store.
|
||||
|
|
@ -405,30 +386,35 @@ The private projection store serves multiple purposes, including:
|
|||
its sequence of $P_{current}$ projection changes.
|
||||
\end{itemize}
|
||||
|
||||
The private half of the projection store is not replicated.
|
||||
|
||||
\section{Projections: calculation, storage, and use}
|
||||
\label{sec:projections}
|
||||
|
||||
Machi uses a ``projection'' to determine how its Chain Replication replicas
|
||||
should operate; see \cite{machi-design} and \cite{corfu1}.
|
||||
should operate; see \cite{machi-design} and
|
||||
\cite{corfu1}. At runtime, a cluster must be able to respond both to
|
||||
administrative changes (e.g., substituting a failed server with
|
||||
replacement hardware) as well as local network conditions (e.g., is
|
||||
there a network partition?).
|
||||
|
||||
The projection defines the operational state of Chain Replication's
|
||||
chain order as well the (re-)synchronization of data managed by by
|
||||
newly-added/failed-and-now-recovering members of the chain. This
|
||||
chain metadata, together with computational processes that manage the
|
||||
chain, must be managed in a safe manner in order to avoid unintended
|
||||
data loss of data managed by the chain.
|
||||
|
||||
The concept of a projection is borrowed
|
||||
from CORFU but has a longer history, e.g., the Hibari key-value store
|
||||
\cite{cr-theory-and-practice} and goes back in research for decades,
|
||||
e.g., Porcupine \cite{porcupine}.
|
||||
|
||||
The projection defines the operational state of Chain Replication's
|
||||
chain order as well the (re-)synchronization of data managed by by
|
||||
newly-added/failed-and-now-recovering members of the chain.
|
||||
At runtime, a cluster must be able to respond both to
|
||||
administrative changes (e.g., substituting a failed server with
|
||||
replacement hardware) as well as local network conditions (e.g., is
|
||||
there a network partition?).
|
||||
|
||||
\subsection{The projection data structure}
|
||||
\label{sub:the-projection}
|
||||
|
||||
{\bf NOTE:} This section is a duplicate of the ``The Projection and
|
||||
the Projection Epoch Number'' section of the ``Machi: an immutable
|
||||
file store'' design doc \cite{machi-design}.
|
||||
the Projection Epoch Number'' section of \cite{machi-design}.
|
||||
|
||||
The projection data
|
||||
structure defines the current administration \& operational/runtime
|
||||
|
|
@ -459,7 +445,6 @@ Figure~\ref{fig:projection}. To summarize the major components:
|
|||
active_upi :: [m_server()],
|
||||
repairing :: [m_server()],
|
||||
down_members :: [m_server()],
|
||||
witness_servers :: [m_server()],
|
||||
dbg_annotations :: proplist()
|
||||
}).
|
||||
\end{verbatim}
|
||||
|
|
@ -469,12 +454,13 @@ Figure~\ref{fig:projection}. To summarize the major components:
|
|||
|
||||
\begin{itemize}
|
||||
\item {\tt epoch\_number} and {\tt epoch\_csum} The epoch number and
|
||||
projection checksum together form the unique identifier for this projection.
|
||||
projection checksum are unique identifiers for this projection.
|
||||
\item {\tt creation\_time} Wall-clock time, useful for humans and
|
||||
general debugging effort.
|
||||
\item {\tt author\_server} Name of the server that calculated the projection.
|
||||
\item {\tt all\_members} All servers in the chain, regardless of current
|
||||
operation status.
|
||||
operation status. If all operating conditions are perfect, the
|
||||
chain should operate in the order specified here.
|
||||
\item {\tt active\_upi} All active chain members that we know are
|
||||
fully repaired/in-sync with each other and therefore the Update
|
||||
Propagation Invariant (Section~\ref{sub:upi}) is always true.
|
||||
|
|
@ -482,10 +468,7 @@ Figure~\ref{fig:projection}. To summarize the major components:
|
|||
are in active data repair procedures.
|
||||
\item {\tt down\_members} All members that the {\tt author\_server}
|
||||
believes are currently down or partitioned.
|
||||
\item {\tt witness\_servers} If witness servers (Section~\ref{zzz})
|
||||
are used in strong consistency mode, then they are listed here. The
|
||||
set of {\tt witness\_servers} is a subset of {\tt all\_members}.
|
||||
\item {\tt dbg\_annotations} A ``kitchen sink'' property list, for code to
|
||||
\item {\tt dbg\_annotations} A ``kitchen sink'' proplist, for code to
|
||||
add any hints for why the projection change was made, delay/retry
|
||||
information, etc.
|
||||
\end{itemize}
|
||||
|
|
@ -495,8 +478,7 @@ Figure~\ref{fig:projection}. To summarize the major components:
|
|||
According to the CORFU research papers, if a server node $S$ or client
|
||||
node $C$ believes that epoch $E$ is the latest epoch, then any information
|
||||
that $S$ or $C$ receives from any source that an epoch $E+\delta$ (where
|
||||
$\delta > 0$) exists will push $S$ into the ``wedge'' state
|
||||
and force $C$ into a mode
|
||||
$\delta > 0$) exists will push $S$ into the ``wedge'' state and $C$ into a mode
|
||||
of searching for the projection definition for the newest epoch.
|
||||
|
||||
In the humming consensus description in
|
||||
|
|
@ -524,7 +506,7 @@ Humming consensus requires that any projection be identified by both
|
|||
the epoch number and the projection checksum, as described in
|
||||
Section~\ref{sub:the-projection}.
|
||||
|
||||
\section{Managing projection store replicas}
|
||||
\section{Managing multiple projection store replicas}
|
||||
\label{sec:managing-multiple-projection-stores}
|
||||
|
||||
An independent replica management technique very similar to the style
|
||||
|
|
@ -533,63 +515,11 @@ replicas of Machi's projection data structures.
|
|||
The major difference is that humming consensus
|
||||
{\em does not necessarily require}
|
||||
successful return status from a minimum number of participants (e.g.,
|
||||
a majority quorum).
|
||||
|
||||
\subsection{Writing to public projection stores}
|
||||
\label{sub:proj-store-writing}
|
||||
|
||||
Writing replicas of a projection $P_{new}$ to the cluster's public
|
||||
projection stores is similar to writing in a Dynamo-like system.
|
||||
The significant difference with Chain Replication is how we interpret
|
||||
the return status of each write operation.
|
||||
|
||||
In cases of {\tt error\_written} status,
|
||||
the process may be aborted and read repair
|
||||
triggered. The most common reason for {\tt error\_written} status
|
||||
is that another actor in the system has concurrently
|
||||
already calculated another
|
||||
(perhaps different\footnote{The {\tt error\_written} may also
|
||||
indicate that another server has performed read repair on the exact
|
||||
projection $P_{new}$ that the local server is trying to write!})
|
||||
projection using the same projection epoch number.
|
||||
|
||||
\subsection{Writing to private projection stores}
|
||||
|
||||
Only the local server/owner may write to the private half of a
|
||||
projection store. Private projection store values are never subject
|
||||
to read repair.
|
||||
|
||||
\subsection{Reading from public projection stores}
|
||||
\label{sub:proj-store-reading}
|
||||
|
||||
A read is simple: for an epoch $E$, send a public projection read API
|
||||
operation to all participants. Usually, the ``get latest epoch''
|
||||
variety is used.
|
||||
|
||||
The minimum number of non-error responses is only one.\footnote{The local
|
||||
projection store should always be available, even if no other remote
|
||||
replica projection stores are available.} If all available servers
|
||||
return a single, unanimous value $V_u, V_u \ne \bot$, then $V_u$ is
|
||||
the final result for epoch $E$.
|
||||
Any non-unanimous values are considered unresolvable for the
|
||||
epoch. This disagreement is resolved by newer
|
||||
writes to the public projection stores during subsequent iterations of
|
||||
humming consensus.
|
||||
|
||||
Unavailable servers may not necessarily interfere with making a decision.
|
||||
Humming consensus
|
||||
only uses as many public projections as are available at the present
|
||||
moment of time. Assume that some server $S$ is unavailable at time $t$ and
|
||||
becomes available at some later $t+\delta$.
|
||||
If at $t+\delta$ we
|
||||
discover that $S$'s public projection store for key $E$
|
||||
contains some disagreeing value $V_{weird}$, then the disagreement
|
||||
will be resolved in the exact same manner that would have been used as if we
|
||||
had seen the disagreeing values at the earlier time $t$.
|
||||
a quorum).
|
||||
|
||||
\subsection{Read repair: repair only unwritten values}
|
||||
|
||||
The ``read repair'' concept is also shared with Riak Core and Dynamo
|
||||
The idea of ``read repair'' is also shared with Riak Core and Dynamo
|
||||
systems. However, Machi has situations where read repair cannot truly
|
||||
``fix'' a key because two different values have been written by two
|
||||
different replicas.
|
||||
|
|
@ -600,24 +530,85 @@ values, all participants in humming consensus merely agree that there
|
|||
were multiple suggestions at that epoch which must be resolved by the
|
||||
creation and writing of newer projections with later epoch numbers.}
|
||||
Machi's projection store read repair can only repair values that are
|
||||
unwritten, i.e., currently storing $\bot$.
|
||||
unwritten, i.e., storing $\bot$.
|
||||
|
||||
The value used to repair unwritten $\bot$ values is the ``best'' projection that
|
||||
The value used to repair $\bot$ values is the ``best'' projection that
|
||||
is currently available for the current epoch $E$. If there is a single,
|
||||
unanimous value $V_{u}$ for the projection at epoch $E$, then $V_{u}$
|
||||
is used to repair all projections stores at $E$ that contain $\bot$
|
||||
is use to repair all projections stores at $E$ that contain $\bot$
|
||||
values. If the value of $K$ is not unanimous, then the ``highest
|
||||
ranked value'' $V_{best}$ is used for the repair; see
|
||||
Section~\ref{sub:ranking-projections} for a description of projection
|
||||
ranking.
|
||||
|
||||
If a non-$\bot$ value exists, then by definition\footnote{Definition
|
||||
of a write-once register} this value is immutable. The only
|
||||
conflict resolution path is to write a new projection with a newer and
|
||||
larger epoch number. Once a public projection with epoch number $E$ is
|
||||
written, projections with epochs smaller than $E$ are ignored by
|
||||
\subsection{Writing to public projection stores}
|
||||
\label{sub:proj-store-writing}
|
||||
|
||||
Writing replicas of a projection $P_{new}$ to the cluster's public
|
||||
projection stores is similar, in principle, to writing a Chain
|
||||
Replication-managed system or Dynamo-like system. But unlike Chain
|
||||
Replication, the order doesn't really matter.
|
||||
In fact, the two steps below may be performed in parallel.
|
||||
The significant difference with Chain Replication is how we interpret
|
||||
the return status of each write operation.
|
||||
|
||||
\begin{enumerate}
|
||||
\item Write $P_{new}$ to the local server's public projection store
|
||||
using $P_{new}$'s epoch number $E$ as the key.
|
||||
As a side effect, a successful write will trigger
|
||||
``wedge'' status in the local server, which will then cascade to other
|
||||
projection-related activity by the local chain manager.
|
||||
\item Write $P_{new}$ to key $E$ of each remote public projection store of
|
||||
all participants in the chain.
|
||||
\end{enumerate}
|
||||
|
||||
In cases of {\tt error\_written} status,
|
||||
the process may be aborted and read repair
|
||||
triggered. The most common reason for {\tt error\_written} status
|
||||
is that another actor in the system has
|
||||
already calculated another (perhaps different) projection using the
|
||||
same projection epoch number and that
|
||||
read repair is necessary. The {\tt error\_written} may also
|
||||
indicate that another server has performed read repair on the exact
|
||||
projection $P_{new}$ that the local server is trying to write!
|
||||
|
||||
\subsection{Writing to private projection stores}
|
||||
|
||||
Only the local server/owner may write to the private half of a
|
||||
projection store. Also, the private projection store is not replicated.
|
||||
|
||||
\subsection{Reading from public projection stores}
|
||||
\label{sub:proj-store-reading}
|
||||
|
||||
A read is simple: for an epoch $E$, send a public projection read API
|
||||
request to all participants. As when writing to the public projection
|
||||
stores, we can ignore any timeout/unavailable return
|
||||
status.\footnote{The success/failure status of projection reads and
|
||||
writes is {\em not} ignored with respect to the chain manager's
|
||||
internal liveness tracker. However, the liveness tracker's state is
|
||||
typically only used when calculating new projections.} If we
|
||||
discover any unwritten values $\bot$, the read repair protocol is
|
||||
followed.
|
||||
|
||||
The minimum number of non-error responses is only one.\footnote{The local
|
||||
projection store should always be available, even if no other remote
|
||||
replica projection stores are available.} If all available servers
|
||||
return a single, unanimous value $V_u, V_u \ne \bot$, then $V_u$ is
|
||||
the final result for epoch $E$.
|
||||
Any non-unanimous values are considered complete disagreement for the
|
||||
epoch. This disagreement is resolved by humming consensus by later
|
||||
writes to the public projection stores during subsequent iterations of
|
||||
humming consensus.
|
||||
|
||||
We are not concerned with unavailable servers. Humming consensus
|
||||
only uses as many public projections as are available at the present
|
||||
moment of time. If some server $S$ is unavailable at time $t$ and
|
||||
becomes available at some later $t+\delta$, and if at $t+\delta$ we
|
||||
discover that $S$'s public projection store for key $E$
|
||||
contains some disagreeing value $V_{weird}$, then the disagreement
|
||||
will be resolved in the exact same manner that would be used as if we
|
||||
had found the disagreeing values at the earlier time $t$.
|
||||
|
||||
\section{Phases of projection change, a prelude to Humming Consensus}
|
||||
\label{sec:phases-of-projection-change}
|
||||
|
||||
|
|
@ -680,7 +671,7 @@ straightforward; see
|
|||
Section~\ref{sub:proj-store-writing} for the technique for writing
|
||||
projections to all participating servers' projection stores.
|
||||
Humming Consensus does not care
|
||||
if the writes succeed or not. The next phase, adopting a
|
||||
if the writes succeed or not: its final phase, adopting a
|
||||
new projection, will determine which write operations are usable.
|
||||
|
||||
\subsection{Adoption a new projection}
|
||||
|
|
@ -694,8 +685,8 @@ to avoid direct parallels with protocols such as Raft and Paxos.)
|
|||
In general, a projection $P_{new}$ at epoch $E_{new}$ is adopted by a
|
||||
server only if
|
||||
the change in state from the local server's current projection to new
|
||||
projection, $P_{current} \rightarrow P_{new}$, will not cause data loss:
|
||||
the Update Propagation Invariant and all other safety checks
|
||||
projection, $P_{current} \rightarrow P_{new}$ will not cause data loss,
|
||||
e.g., the Update Propagation Invariant and all other safety checks
|
||||
required by chain repair in Section~\ref{sec:repair-entire-files}
|
||||
are correct. For example, any new epoch must be strictly larger than
|
||||
the current epoch, i.e., $E_{new} > E_{current}$.
|
||||
|
|
@ -705,12 +696,16 @@ available public projection stores. If the result is not a single
|
|||
unanmous projection, then we return to the step in
|
||||
Section~\ref{sub:projection-calculation}. If the result is a {\em
|
||||
unanimous} projection $P_{new}$ in epoch $E_{new}$, and if $P_{new}$
|
||||
does not violate chain safety checks, then the local node will:
|
||||
does not violate chain safety checks, then the local node may
|
||||
replace its local $P_{current}$ projection with $P_{new}$.
|
||||
|
||||
\begin{itemize}
|
||||
\item write $P_{current}$ to the local private projection store, and
|
||||
\item set its local operating state $P_{current} \leftarrow P_{new}$.
|
||||
\end{itemize}
|
||||
Not all safe projection transitions are useful, however. For example,
|
||||
it's trivally safe to suggest projection $P_{zero}$, where the chain
|
||||
length is zero. In an eventual consistency environment, projection
|
||||
$P_{one}$ where the chain length is exactly one is also trivially
|
||||
safe.\footnote{Although, if the total number of participants is more
|
||||
than one, eventual consistency would demand that $P_{self}$ cannot
|
||||
be used forever.}
|
||||
|
||||
\section{Humming Consensus}
|
||||
\label{sec:humming-consensus}
|
||||
|
|
@ -719,11 +714,13 @@ Humming consensus describes consensus that is derived only from data
|
|||
that is visible/available at the current time. It's OK if a network
|
||||
partition is in effect and not all chain members are available;
|
||||
the algorithm will calculate a rough consensus despite not
|
||||
having input from all chain members.
|
||||
having input from all chain members. Humming consensus
|
||||
may proceed to make a decision based on data from only one
|
||||
participant, i.e., only the local node.
|
||||
|
||||
\begin{itemize}
|
||||
|
||||
\item When operating in eventual consistency mode, humming
|
||||
\item When operating in AP mode, i.e., in eventual consistency mode, humming
|
||||
consensus may reconfigure a chain of length $N$ into $N$
|
||||
independent chains of length 1. When a network partition heals, the
|
||||
humming consensus is sufficient to manage the chain so that each
|
||||
|
|
@ -731,12 +728,11 @@ replica's data can be repaired/merged/reconciled safely.
|
|||
Other features of the Machi system are designed to assist such
|
||||
repair safely.
|
||||
|
||||
\item When operating in strong consistency mode, any
|
||||
chain shorter than the quorum majority of
|
||||
all members is invalid and therefore cannot be used. Any server with
|
||||
a too-short chain cannot not move itself out
|
||||
of wedged state and is therefore unavailable for general file service.
|
||||
In very general terms, this requirement for a quorum
|
||||
\item When operating in CP mode, i.e., in strong consistency mode, humming
|
||||
consensus would require additional restrictions. For example, any
|
||||
chain that didn't have a minimum length of the quorum majority size of
|
||||
all members would be invalid and therefore would not move itself out
|
||||
of wedged state. In very general terms, this requirement for a quorum
|
||||
majority of surviving participants is also a requirement for Paxos,
|
||||
Raft, and ZAB. See Section~\ref{sec:split-brain-management} for a
|
||||
proposal to handle ``split brain'' scenarios while in CP mode.
|
||||
|
|
@ -756,6 +752,8 @@ Section~\ref{sec:phases-of-projection-change}: network monitoring,
|
|||
calculating new projections, writing projections, then perhaps
|
||||
adopting the newest projection (which may or may not be the projection
|
||||
that we just wrote).
|
||||
Beginning with Section~\ref{sub:flapping-state}, we provide
|
||||
additional detail to the rough outline of humming consensus.
|
||||
|
||||
\begin{figure*}[htp]
|
||||
\resizebox{\textwidth}{!}{
|
||||
|
|
@ -803,15 +801,15 @@ is used by the flowchart and throughout this section.
|
|||
|
||||
\item[$\mathbf{P_{current}}$] The projection actively used by the local
|
||||
node right now. It is also the projection with largest
|
||||
epoch number in the local node's {\em private} projection store.
|
||||
epoch number in the local node's private projection store.
|
||||
|
||||
\item[$\mathbf{P_{newprop}}$] A new projection suggestion, as
|
||||
calculated by the local server
|
||||
(Section~\ref{sub:humming-projection-calculation}).
|
||||
|
||||
\item[$\mathbf{P_{latest}}$] The highest-ranked projection with the largest
|
||||
single epoch number that has been read from all available {\em public}
|
||||
projection stores.
|
||||
single epoch number that has been read from all available public
|
||||
projection stores, including the local node's public projection store.
|
||||
|
||||
\item[Unanimous] The $P_{latest}$ projection is unanimous if all
|
||||
replicas in all accessible public projection stores are effectively
|
||||
|
|
@ -830,7 +828,7 @@ is used by the flowchart and throughout this section.
|
|||
The flowchart has three columns, from left to right:
|
||||
|
||||
\begin{description}
|
||||
\item[Column A] Is there any reason to act?
|
||||
\item[Column A] Is there any reason to change?
|
||||
\item[Column B] Do I act?
|
||||
\item[Column C] How do I act?
|
||||
\begin{description}
|
||||
|
|
@ -865,12 +863,12 @@ In today's implementation, there is only a single criterion for
|
|||
determining the alive/perhaps-not-alive status of a remote server $S$:
|
||||
is $S$'s projection store available now? This question is answered by
|
||||
attemping to read the projection store on server $S$.
|
||||
If successful, then we assume that $S$ and all of $S$'s network services
|
||||
are available. If $S$'s projection store is not available for any
|
||||
reason (including timeout), we inform the local ``fitness server''
|
||||
that we have had a problem querying $S$. The fitness service may then
|
||||
take additional monitoring/querying actions before informing us (in a
|
||||
later iteration) that $S$ should be considered down.
|
||||
If successful, then we assume that all
|
||||
$S$ is available. If $S$'s projection store is not available for any
|
||||
reason (including timeout), we assume $S$ is entirely unavailable.
|
||||
This simple single
|
||||
criterion appears to be sufficient for humming consensus, according to
|
||||
simulations of arbitrary network partitions.
|
||||
|
||||
%% {\bf NOTE:} The projection store API is accessed via TCP. The network
|
||||
%% partition simulator, mentioned above and described at
|
||||
|
|
@ -885,10 +883,64 @@ Column~A of Figure~\ref{fig:flowchart}.
|
|||
See also, Section~\ref{sub:projection-calculation}.
|
||||
|
||||
Execution starts at ``Start'' state of Column~A of
|
||||
Figure~\ref{fig:flowchart}. Rule $A20$'s uses judgement from the
|
||||
local ``fitness server'' to select a definite
|
||||
Figure~\ref{fig:flowchart}. Rule $A20$'s uses recent success \&
|
||||
failures in accessing other public projection stores to select a hard
|
||||
boolean up/down status for each participating server.
|
||||
|
||||
\subsubsection{Calculating flapping state}
|
||||
|
||||
Also at this stage, the chain manager calculates its local
|
||||
``flapping'' state. The name ``flapping'' is borrowed from IP network
|
||||
engineer jargon ``route flapping'':
|
||||
|
||||
\begin{quotation}
|
||||
``Route flapping is caused by pathological conditions
|
||||
(hardware errors, software errors, configuration errors, intermittent
|
||||
errors in communications links, unreliable connections, etc.) within
|
||||
the network which cause certain reachability information to be
|
||||
repeatedly advertised and withdrawn.'' \cite{wikipedia-route-flapping}
|
||||
\end{quotation}
|
||||
|
||||
\paragraph{Flapping due to constantly changing network partitions and/or server crashes and restarts}
|
||||
|
||||
Currently, Machi does not attempt to dampen, smooth, or ignore recent
|
||||
history of constantly flapping peer servers. If necessary, a failure
|
||||
detector such as the $\phi$ accrual failure detector
|
||||
\cite{phi-accrual-failure-detector} can be used to help mange such
|
||||
situations.
|
||||
|
||||
\paragraph{Flapping due to asymmetric network partitions}
|
||||
|
||||
The simulator's behavior during stable periods where at least one node
|
||||
is the victim of an asymmetric network partition is \ldots weird,
|
||||
wonderful, and something I don't completely understand yet. This is
|
||||
another place where we need more eyes reviewing and trying to poke
|
||||
holes in the algorithm.
|
||||
|
||||
In cases where any node is a victim of an asymmetric network
|
||||
partition, the algorithm oscillates in a very predictable way: each
|
||||
server $S$ makes the same $P_{new}$ projection at epoch $E$ that $S$ made
|
||||
during a previous recent epoch $E-\delta$ (where $\delta$ is small, usually
|
||||
much less than 10). However, at least one node makes a suggestion that
|
||||
makes rough consensus impossible. When any epoch $E$ is not
|
||||
acceptable (because some node disagrees about something, e.g.,
|
||||
which nodes are down),
|
||||
the result is more new rounds of suggestions that create a repeating
|
||||
loop that lasts as long as the asymmetric partition lasts.
|
||||
|
||||
From the perspective of $S$'s chain manager, the pattern of this
|
||||
infinite loop is easy to detect: $S$ inspects the pattern of the last
|
||||
$L$ projections that it has suggested, e.g., the last 10.
|
||||
Tiny details such as the epoch number and creation timestamp will
|
||||
differ, but the major details such as UPI list and repairing list are
|
||||
the same.
|
||||
|
||||
If the major details of the last $L$ projections authored and
|
||||
suggested by $S$ are the same, then $S$ unilaterally decides that it
|
||||
is ``flapping'' and enters flapping state. See
|
||||
Section~\ref{sub:flapping-state} for additional disucssion of the
|
||||
flapping state.
|
||||
|
||||
\subsubsection{When to calculate a new projection}
|
||||
\label{ssub:when-to-calc}
|
||||
|
||||
|
|
@ -897,7 +949,7 @@ calculate a new projection. The timer interval is typically
|
|||
0.5--2.0 seconds, if the cluster has been stable. A client may call an
|
||||
external API call to trigger a new projection, e.g., if that client
|
||||
knows that an environment change has happened and wishes to trigger a
|
||||
response prior to the next timer firing (e.g.~at state $C200$).
|
||||
response prior to the next timer firing.
|
||||
|
||||
It's recommended that the timer interval be staggered according to the
|
||||
participant ranking rules in Section~\ref{sub:ranking-projections};
|
||||
|
|
@ -918,14 +970,15 @@ done by state $C110$ and that writing a public projection is done by
|
|||
states $C300$ and $C310$.
|
||||
|
||||
Broadly speaking, there are a number of decisions made in all three
|
||||
columns of Figure~\ref{fig:flowchart} to decide if and when a
|
||||
projection should be written at all. Sometimes, the best action is
|
||||
columns of Figure~\ref{fig:flowchart} to decide if and when any type
|
||||
of projection should be written at all. Sometimes, the best action is
|
||||
to do nothing.
|
||||
|
||||
\subsubsection{Column A: Is there any reason to change?}
|
||||
|
||||
The main tasks of the flowchart states in Column~A is to calculate a
|
||||
new projection $P_{new}$. Then we try to figure out which
|
||||
new projection $P_{new}$ and perhaps also the inner projection
|
||||
$P_{new2}$ if we're in flapping mode. Then we try to figure out which
|
||||
projection has the greatest merit: our current projection
|
||||
$P_{current}$, the new projection $P_{new}$, or the latest epoch
|
||||
$P_{latest}$. If our local $P_{current}$ projection is best, then
|
||||
|
|
@ -958,7 +1011,7 @@ The main decisions that states in Column B need to make are:
|
|||
|
||||
It's notable that if $P_{new}$ is truly the best projection available
|
||||
at the moment, it must always first be written to everyone's
|
||||
public projection stores and only afterward processed through another
|
||||
public projection stores and only then processed through another
|
||||
monitor \& calculate loop through the flowchart.
|
||||
|
||||
\subsubsection{Column C: How do I act?}
|
||||
|
|
@ -1000,14 +1053,14 @@ therefore the suggested projections at epoch $E$ are not unanimous.
|
|||
\paragraph{\#2: The transition from current $\rightarrow$ new projection is
|
||||
safe}
|
||||
|
||||
Given the current projection
|
||||
Given the projection that the server is currently using,
|
||||
$P_{current}$, the projection $P_{latest}$ is evaluated by numerous
|
||||
rules and invariants, relative to $P_{current}$.
|
||||
If such rule or invariant is
|
||||
violated/false, then the local server will discard $P_{latest}$.
|
||||
|
||||
The transition from $P_{current} \rightarrow P_{latest}$ is protected
|
||||
by rules and invariants that include:
|
||||
The transition from $P_{current} \rightarrow P_{latest}$ is checked
|
||||
for safety and sanity. The conditions used for the check include:
|
||||
|
||||
\begin{enumerate}
|
||||
\item The Erlang data types of all record members are correct.
|
||||
|
|
@ -1020,13 +1073,161 @@ by rules and invariants that include:
|
|||
The same re-reordering restriction applies to all
|
||||
servers in $P_{latest}$'s repairing list relative to
|
||||
$P_{current}$'s repairing list.
|
||||
\item Any server $S$ that is newly-added to $P_{latest}$'s UPI list must
|
||||
\item Any server $S$ that was added to $P_{latest}$'s UPI list must
|
||||
appear in the tail the UPI list. Furthermore, $S$ must have been in
|
||||
$P_{current}$'s repairing list and had successfully completed file
|
||||
repair prior to $S$'s promotion from the repairing list to the tail
|
||||
of the UPI list.
|
||||
repair prior to the transition.
|
||||
\end{enumerate}
|
||||
|
||||
\subsection{Additional discussion of flapping state}
|
||||
\label{sub:flapping-state}
|
||||
All $P_{new}$ projections
|
||||
calculated while in flapping state have additional diagnostic
|
||||
information added, including:
|
||||
|
||||
\begin{itemize}
|
||||
\item Flag: server $S$ is in flapping state.
|
||||
\item Epoch number \& wall clock timestamp when $S$ entered flapping state.
|
||||
\item The collection of all other known participants who are also
|
||||
flapping (with respective starting epoch numbers).
|
||||
\item A list of nodes that are suspected of being partitioned, called the
|
||||
``hosed list''. The hosed list is a union of all other hosed list
|
||||
members that are ever witnessed, directly or indirectly, by a server
|
||||
while in flapping state.
|
||||
\end{itemize}
|
||||
|
||||
\subsubsection{Flapping diagnostic data accumulates}
|
||||
|
||||
While in flapping state, this diagnostic data is gathered from
|
||||
all available participants and merged together in a CRDT-like manner.
|
||||
Once added to the diagnostic data list, a datum remains until
|
||||
$S$ drops out of flapping state. When flapping state stops, all
|
||||
accumulated diagnostic data is discarded.
|
||||
|
||||
This accumulation of diagnostic data in the projection data
|
||||
structure acts in part as a substitute for a separate gossip protocol.
|
||||
However, since all participants are already communicating with each
|
||||
other via read \& writes to each others' projection stores, the diagnostic
|
||||
data can propagate in a gossip-like manner via the projection stores.
|
||||
|
||||
\subsubsection{Flapping example (part 1)}
|
||||
\label{ssec:flapping-example}
|
||||
|
||||
Any server listed in the ``hosed list'' is suspected of having some
|
||||
kind of network communication problem with some other server. For
|
||||
example, let's examine a scenario involving a Machi cluster of servers
|
||||
$a$, $b$, $c$, $d$, and $e$. Assume there exists an asymmetric network
|
||||
partition such that messages from $a \rightarrow b$ are dropped, but
|
||||
messages from $b \rightarrow a$ are delivered.\footnote{If this
|
||||
partition were happening at or below the level of a reliable
|
||||
delivery network protocol like TCP, then communication in {\em both}
|
||||
directions would be affected by an asymmetric partition.
|
||||
However, in this model, we are
|
||||
assuming that a ``message'' lost during a network partition is a
|
||||
uni-directional projection API call or its response.}
|
||||
|
||||
Once a participant $S$ enters flapping state, it starts gathering the
|
||||
flapping starting epochs and hosed lists from all of the other
|
||||
projection stores that are available. The sum of this info is added
|
||||
to all projections calculated by $S$.
|
||||
For example, projections authored by $a$ will say that $a$ believes
|
||||
that $b$ is down.
|
||||
Likewise, projections authored by $b$ will say that $b$ believes
|
||||
that $a$ is down.
|
||||
|
||||
\subsubsection{The inner projection (flapping example, part 2)}
|
||||
\label{ssec:inner-projection}
|
||||
|
||||
\ldots We continue the example started in the previous subsection\ldots
|
||||
|
||||
Eventually, in a gossip-like manner, all other participants will
|
||||
eventually find that their hosed list is equal to $[a,b]$. Any other
|
||||
server, for example server $c$, will then calculate another
|
||||
projection, $P_{new2}$, using the assumption that both $a$ and $b$
|
||||
are down in addition to all other known unavailable servers.
|
||||
|
||||
\begin{itemize}
|
||||
\item If operating in the default CP mode, both $a$ and $b$ are down
|
||||
and therefore not eligible to participate in Chain Replication.
|
||||
%% The chain may continue service if a $c$, $d$, $e$ and/or witness
|
||||
%% servers can try to form a correct UPI list for the chain.
|
||||
This may cause an availability problem for the chain: we may not
|
||||
have a quorum of participants (real or witness-only) to form a
|
||||
correct UPI chain.
|
||||
\item If operating in AP mode, $a$ and $b$ can still form two separate
|
||||
chains of length one, using UPI lists of $[a]$ and $[b]$, respectively.
|
||||
\end{itemize}
|
||||
|
||||
This re-calculation, $P_{new2}$, of the new projection is called an
|
||||
``inner projection''. The inner projection definition is nested
|
||||
inside of its parent projection, using the same flapping disagnostic
|
||||
data used for other flapping status tracking.
|
||||
|
||||
When humming consensus has determined that a projection state change
|
||||
is necessary and is also safe (relative to both the outer and inner
|
||||
projections), then the outer projection\footnote{With the inner
|
||||
projection $P_{new2}$ nested inside of it.} is written to
|
||||
the local private projection store.
|
||||
With respect to future iterations of
|
||||
humming consensus, the innter projection is ignored.
|
||||
However, with respect to Chain Replication, the server's subsequent
|
||||
behavior
|
||||
{\em will consider the inner projection only}. The inner projection
|
||||
is used to order the UPI and repairing parts of the chain and trigger
|
||||
wedge/un-wedge behavior. The inner projection is also
|
||||
advertised to Machi clients.
|
||||
|
||||
The epoch of the inner projection, $E^{inner}$ is always less than or
|
||||
equal to the epoch of the outer projection, $E$. The $E^{inner}$
|
||||
epoch typically only changes when new servers are added to the hosed
|
||||
list.
|
||||
|
||||
To attempt a rough analogy, the outer projection is the carrier wave
|
||||
that is used to transmit the inner projection and its accompanying
|
||||
gossip of diagnostic data.
|
||||
|
||||
\subsubsection{Outer projection churn, inner projection stability}
|
||||
|
||||
One of the intriguing features of humming consensus's reaction to
|
||||
asymmetric partition: flapping behavior continues for as long as
|
||||
an any asymmetric partition exists.
|
||||
|
||||
\subsubsection{Stability in symmetric partition cases}
|
||||
|
||||
Although humming consensus hasn't been formally proven to handle all
|
||||
asymmetric and symmetric partition cases, the current implementation
|
||||
appears to converge rapidly to a single chain state in all symmetric
|
||||
partition cases. This is in contrast to asymmetric partition cases,
|
||||
where ``flapping'' will continue on every humming consensus iteration
|
||||
until all asymmetric partition disappears. A formal proof is an area of
|
||||
future work.
|
||||
|
||||
\subsubsection{Leaving flapping state and discarding inner projection}
|
||||
|
||||
There are two events that can trigger leaving flapping state.
|
||||
|
||||
\begin{itemize}
|
||||
|
||||
\item A server $S$ in flapping state notices that its long history of
|
||||
repeatedly suggesting the same projection will be broken:
|
||||
$S$ instead calculates some differing projection instead.
|
||||
This change in projection history happens whenever a perceived network
|
||||
partition changes in any way.
|
||||
|
||||
\item Server $S$ reads a public projection suggestion, $P_{noflap}$, that is
|
||||
authored by another server $S'$, and that $P_{noflap}$ no longer
|
||||
contains the flapping start epoch for $S'$ that is present in the
|
||||
history that $S$ has maintained while $S$ has been in
|
||||
flapping state.
|
||||
|
||||
\end{itemize}
|
||||
|
||||
When either trigger event happens, server $S$ will exit flapping state. All
|
||||
new projections authored by $S$ will have all flapping diagnostic data
|
||||
removed. This includes stopping use of the inner projection: the UPI
|
||||
list of the inner projection is copied to the outer projection's UPI
|
||||
list, to avoid a drastic change in UPI membership.
|
||||
|
||||
\subsection{Ranking projections}
|
||||
\label{sub:ranking-projections}
|
||||
|
||||
|
|
@ -1279,7 +1480,7 @@ as the foundation for Machi's data loss prevention techniques.
|
|||
\begin{figure}
|
||||
\centering
|
||||
$
|
||||
[\overbrace{\underbrace{H_1}_\textbf{Head}, M_{11}, \ldots, T_1,
|
||||
[\overbrace{\underbrace{H_1}_\textbf{Head}, M_{11}, T_1,
|
||||
H_2, M_{21},
|
||||
\ldots
|
||||
\underbrace{T_2}_\textbf{Tail}}^\textbf{Chain (U.P.~Invariant preserving)}
|
||||
|
|
@ -1588,7 +1789,7 @@ Manageability, availability and performance in Porcupine: a highly scalable, clu
|
|||
{\tt http://homes.cs.washington.edu/\%7Elevy/ porcupine.pdf}
|
||||
|
||||
\bibitem{chain-replication}
|
||||
van Renesse, Robbert and Schneider, Fred.
|
||||
van Renesse, Robbert et al.
|
||||
Chain Replication for Supporting High Throughput and Availability.
|
||||
Proceedings of the 6th Conference on Symposium on Operating Systems
|
||||
Design \& Implementation (OSDI'04) - Volume 6, 2004.
|
||||
|
|
|
|||
|
|
@ -1489,7 +1489,7 @@ In Usenix ATC 2009.
|
|||
{\tt https://www.usenix.org/legacy/event/usenix09/ tech/full\_papers/terrace/terrace.pdf}
|
||||
|
||||
\bibitem{chain-replication}
|
||||
van Renesse, Robbert and Schneider, Fred.
|
||||
van Renesse, Robbert et al.
|
||||
Chain Replication for Supporting High Throughput and Availability.
|
||||
Proceedings of the 6th Conference on Symposium on Operating Systems
|
||||
Design \& Implementation (OSDI'04) - Volume 6, 2004.
|
||||
|
|
|
|||
|
|
@ -40,24 +40,3 @@
|
|||
|
||||
%% TODO: it's used in flu_sup and elsewhere, change this to suitable name
|
||||
-define(TEST_ETS_TABLE, test_ets_table).
|
||||
|
||||
-define(DEFAULT_COC_NAMESPACE, "").
|
||||
-define(DEFAULT_COC_LOCATOR, 0).
|
||||
|
||||
-record(ns_info, {
|
||||
version = 0 :: machi_dt:namespace_version(),
|
||||
name = <<>> :: machi_dt:namespace(),
|
||||
locator = 0 :: machi_dt:locator()
|
||||
}).
|
||||
|
||||
-record(append_opts, {
|
||||
chunk_extra = 0 :: machi_dt:chunk_size(),
|
||||
preferred_file_name :: 'undefined' | machi_dt:file_name_s(),
|
||||
flag_fail_preferred = false :: boolean()
|
||||
}).
|
||||
|
||||
-record(read_opts, {
|
||||
no_checksum = false :: boolean(),
|
||||
no_chunk = false :: boolean(),
|
||||
needs_trimmed = false :: boolean()
|
||||
}).
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
%% machi merkle tree records
|
||||
|
||||
-record(naive, {
|
||||
chunk_size = 1048576 :: pos_integer(), %% default 1 MB
|
||||
recalc = true :: boolean(),
|
||||
root :: 'undefined' | binary(),
|
||||
lvl1 = [] :: [ binary() ],
|
||||
lvl2 = [] :: [ binary() ],
|
||||
lvl3 = [] :: [ binary() ],
|
||||
leaves = [] :: [ { Offset :: pos_integer(),
|
||||
Size :: pos_integer(),
|
||||
Csum :: binary()} ]
|
||||
}).
|
||||
|
||||
-record(mt, {
|
||||
filename :: string(),
|
||||
tree :: #naive{},
|
||||
backend = 'naive' :: 'naive'
|
||||
}).
|
||||
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
|
|
@ -22,11 +22,10 @@
|
|||
-define(MACHI_PROJECTION_HRL, true).
|
||||
|
||||
-type pv1_consistency_mode() :: 'ap_mode' | 'cp_mode'.
|
||||
-type pv1_chain_name():: atom().
|
||||
-type pv1_csum() :: binary().
|
||||
-type pv1_epoch() :: {pv1_epoch_n(), pv1_csum()}.
|
||||
-type pv1_epoch_n() :: non_neg_integer().
|
||||
-type pv1_server() :: atom().
|
||||
-type pv1_server() :: atom() | binary().
|
||||
-type pv1_timestamp() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}.
|
||||
|
||||
-record(p_srvr, {
|
||||
|
|
@ -56,7 +55,6 @@
|
|||
epoch_number :: pv1_epoch_n() | ?SPAM_PROJ_EPOCH,
|
||||
epoch_csum :: pv1_csum(),
|
||||
author_server :: pv1_server(),
|
||||
chain_name = ch_not_def_yet :: pv1_chain_name(),
|
||||
all_members :: [pv1_server()],
|
||||
witnesses = [] :: [pv1_server()],
|
||||
creation_time :: pv1_timestamp(),
|
||||
|
|
@ -77,16 +75,4 @@
|
|||
%% create a consistent projection ranking score.
|
||||
-define(MAX_CHAIN_LENGTH, 64).
|
||||
|
||||
-record(chain_def_v1, {
|
||||
name :: atom(), % chain name
|
||||
mode :: pv1_consistency_mode(),
|
||||
full = [] :: [p_srvr()],
|
||||
witnesses = [] :: [p_srvr()],
|
||||
old_full = [] :: [pv1_server()], % guard against some races
|
||||
old_witnesses=[] :: [pv1_server()], % guard against some races
|
||||
local_run = [] :: [pv1_server()], % must be tailored to each machine!
|
||||
local_stop = [] :: [pv1_server()], % must be tailored to each machine!
|
||||
props = [] :: list() % proplist for other related info
|
||||
}).
|
||||
|
||||
-endif. % !MACHI_PROJECTION_HRL
|
||||
|
|
|
|||
|
|
@ -1,56 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
echo "Step: Verify that the required entries in /etc/hosts are present"
|
||||
for i in 1 2 3; do
|
||||
grep machi$i /etc/hosts | egrep -s '^127.0.0.1' > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
echo ""
|
||||
echo "'grep -s machi$i' failed. Aborting, sorry."
|
||||
exit 1
|
||||
fi
|
||||
ping -c 1 machi$i > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
echo ""
|
||||
echo "Ping attempt on host machi$i failed. Aborting."
|
||||
echo ""
|
||||
ping -c 1 machi$i
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Step: add a verbose logging option to app.config"
|
||||
for i in 1 2 3; do
|
||||
ed ./dev/dev$i/etc/app.config <<EOF > /dev/null 2>&1
|
||||
/verbose_confirm
|
||||
a
|
||||
{chain_manager_opts, [{private_write_verbose_confirm,true}]},
|
||||
{stability_time, 1},
|
||||
.
|
||||
w
|
||||
q
|
||||
EOF
|
||||
done
|
||||
|
||||
echo "Step: start three three Machi application instances"
|
||||
for i in 1 2 3; do
|
||||
./dev/dev$i/bin/machi start
|
||||
./dev/dev$i/bin/machi ping
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Sorry, a 'ping' check for instance dev$i failed. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Step: configure one chain to start a Humming Consensus group with three members"
|
||||
|
||||
# Note: $CWD of each Machi proc is two levels below the source code root dir.
|
||||
LIFECYCLE000=../../priv/quick-admin-examples/demo-000
|
||||
for i in 3 2 1; do
|
||||
./dev/dev$i/bin/machi-admin quick-admin-apply $LIFECYCLE000 machi$i
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Sorry, 'machi-admin quick-admin-apply failed' on machi$i. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
exit 0
|
||||
93
priv/humming-consensus-demo.vagrant/Vagrantfile
vendored
93
priv/humming-consensus-demo.vagrant/Vagrantfile
vendored
|
|
@ -1,93 +0,0 @@
|
|||
# -*- mode: ruby -*-
|
||||
# vi: set ft=ruby :
|
||||
|
||||
# All Vagrant configuration is done below. The "2" in Vagrant.configure
|
||||
# configures the configuration version (we support older styles for
|
||||
# backwards compatibility). Please don't change it unless you know what
|
||||
# you're doing.
|
||||
Vagrant.configure(2) do |config|
|
||||
# The most common configuration options are documented and commented below.
|
||||
# For a complete reference, please see the online documentation at
|
||||
# https://docs.vagrantup.com.
|
||||
|
||||
# Every Vagrant development environment requires a box. You can search for
|
||||
# boxes at https://atlas.hashicorp.com/search.
|
||||
# If this Vagrant box has not been downloaded before (e.g. using "vagrant box add"),
|
||||
# then Vagrant will automatically download the VM image from HashiCorp.
|
||||
config.vm.box = "hashicorp/precise64"
|
||||
# If using a FreeBSD box, Bash may not be installed.
|
||||
# Use the config.ssh.shell setting to specify an alternate shell.
|
||||
# Note, however, that any code in the 'config.vm.provision' section
|
||||
# would then have to use this shell's syntax!
|
||||
# config.ssh.shell = "/bin/csh -l"
|
||||
|
||||
# Disable automatic box update checking. If you disable this, then
|
||||
# boxes will only be checked for updates when the user runs
|
||||
# `vagrant box outdated`. This is not recommended.
|
||||
# config.vm.box_check_update = false
|
||||
|
||||
# Create a forwarded port mapping which allows access to a specific port
|
||||
# within the machine from a port on the host machine. In the example below,
|
||||
# accessing "localhost:8080" will access port 80 on the guest machine.
|
||||
# config.vm.network "forwarded_port", guest: 80, host: 8080
|
||||
|
||||
# Create a private network, which allows host-only access to the machine
|
||||
# using a specific IP.
|
||||
# config.vm.network "private_network", ip: "192.168.33.10"
|
||||
|
||||
# Create a public network, which generally matched to bridged network.
|
||||
# Bridged networks make the machine appear as another physical device on
|
||||
# your network.
|
||||
# config.vm.network "public_network"
|
||||
|
||||
# Share an additional folder to the guest VM. The first argument is
|
||||
# the path on the host to the actual folder. The second argument is
|
||||
# the path on the guest to mount the folder. And the optional third
|
||||
# argument is a set of non-required options.
|
||||
# config.vm.synced_folder "../data", "/vagrant_data"
|
||||
|
||||
# Provider-specific configuration so you can fine-tune various
|
||||
# backing providers for Vagrant. These expose provider-specific options.
|
||||
# Example for VirtualBox:
|
||||
#
|
||||
config.vm.provider "virtualbox" do |vb|
|
||||
# Display the VirtualBox GUI when booting the machine
|
||||
# vb.gui = true
|
||||
|
||||
# Customize the amount of memory on the VM:
|
||||
vb.memory = "512"
|
||||
end
|
||||
#
|
||||
# View the documentation for the provider you are using for more
|
||||
# information on available options.
|
||||
|
||||
# Define a Vagrant Push strategy for pushing to Atlas. Other push strategies
|
||||
# such as FTP and Heroku are also available. See the documentation at
|
||||
# https://docs.vagrantup.com/v2/push/atlas.html for more information.
|
||||
# config.push.define "atlas" do |push|
|
||||
# push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME"
|
||||
# end
|
||||
|
||||
# Enable provisioning with a shell script. Additional provisioners such as
|
||||
# Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
|
||||
# documentation for more information about their specific syntax and use.
|
||||
config.vm.provision "shell", inline: <<-SHELL
|
||||
# Install prerequsites
|
||||
# Support here for FreeBSD is experimental
|
||||
apt-get update ; sudo apt-get install -y git sudo rsync ; # Ubuntu Linux
|
||||
env ASSUME_ALWAYS_YES=yes pkg install -f git sudo rsync ; # FreeBSD 10
|
||||
|
||||
# Install dependent packages, using slf-configurator
|
||||
git clone https://github.com/slfritchie/slf-configurator.git
|
||||
chown -R vagrant ./slf-configurator
|
||||
(cd slf-configurator ; sudo sh -x ./ALL.sh)
|
||||
echo 'export PATH=${PATH}:/usr/local/erlang/17.5/bin' >> ~vagrant/.bashrc
|
||||
export PATH=${PATH}:/usr/local/erlang/17.5/bin
|
||||
## echo 'set path = ( $path /usr/local/erlang/17.5/bin )' >> ~vagrant/.cshrc
|
||||
## setenv PATH /usr/local/erlang/17.5/bin:$PATH
|
||||
|
||||
git clone https://github.com/basho/machi.git
|
||||
(cd machi ; git checkout master ; make && make test )
|
||||
chown -R vagrant ./machi
|
||||
SHELL
|
||||
end
|
||||
|
|
@ -36,7 +36,7 @@ while (<I>) {
|
|||
$indent = " " x ($count * 4);
|
||||
s/^#*\s*[0-9. ]*//;
|
||||
$anchor = "n$label";
|
||||
printf T1 "%s+ [%s. %s](#%s)\n", $indent, $label, $_, $anchor;
|
||||
printf T1 "%s+ [%s %s](#%s)\n", $indent, $label, $_, $anchor;
|
||||
printf T2 "<a name=\"%s\">\n", $anchor;
|
||||
$line =~ s/(#+)\s*[0-9. ]*/$1 $label. /;
|
||||
print T2 $line;
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
{host, "localhost", []}.
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
{flu,f1,"localhost",20401,[]}.
|
||||
{flu,f2,"localhost",20402,[]}.
|
||||
{flu,f3,"localhost",20403,[]}.
|
||||
{chain,c1,[f1,f2,f3],[]}.
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
{flu,f4,"localhost",20404,[]}.
|
||||
{flu,f5,"localhost",20405,[]}.
|
||||
{flu,f6,"localhost",20406,[]}.
|
||||
{chain,c2,[f4,f5,f6],[]}.
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
{host, "machi1", []}.
|
||||
{host, "machi2", []}.
|
||||
{host, "machi3", []}.
|
||||
{flu,f1,"machi1",20401,[]}.
|
||||
{flu,f2,"machi2",20402,[]}.
|
||||
{flu,f3,"machi3",20403,[]}.
|
||||
{chain,c1,[f1,f2,f3],[]}.
|
||||
|
|
@ -5,13 +5,9 @@
|
|||
{edoc_opts, [{dir, "./edoc"}]}.
|
||||
|
||||
{deps, [
|
||||
{cuttlefish, ".*", {git, "git://github.com/basho/cuttlefish.git", {branch, "develop"}}},
|
||||
{sext, ".*", {git, "git://github.com/basho/sext.git", {branch, "master"}}},
|
||||
{eleveldb, ".*", {git, "git://github.com/basho/eleveldb.git", {branch, "develop"}}},
|
||||
{lager, ".*", {git, "git://github.com/basho/lager.git", {tag, "2.2.0"}}},
|
||||
{protobuffs, "0.8.*", {git, "git://github.com/basho/erlang_protobuffs.git", {tag, "0.8.1p4"}}},
|
||||
{riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {branch, "develop"}}},
|
||||
{ranch, ".*", {git, "git://github.com/ninenines/ranch.git", {branch, "master"}}},
|
||||
{node_package, ".*", {git, "git://github.com/basho/node_package.git", {branch, "develop"}}},
|
||||
{eper, ".*", {git, "git://github.com/basho/eper.git", {tag, "0.92-basho1"}}},
|
||||
{cluster_info, ".*", {git, "git://github.com/basho/cluster_info", {branch, "develop"}}}
|
||||
|
|
|
|||
|
|
@ -1,35 +1,25 @@
|
|||
[
|
||||
{machi, [
|
||||
%% Data directory for all FLUs.
|
||||
{flu_data_dir, "{{platform_data_dir}}/flu"},
|
||||
|
||||
%% FLU config directory
|
||||
{flu_config_dir, "{{platform_etc_dir}}/flu-config"},
|
||||
|
||||
%% Chain config directory
|
||||
{chain_config_dir, "{{platform_etc_dir}}/chain-config"},
|
||||
{flu_data_dir, "{{platform_data_dir}}"},
|
||||
|
||||
%% FLUs to start at app start.
|
||||
%% This task has moved to machi_flu_sup and machi_lifecycle_mgr.
|
||||
{initial_flus, [
|
||||
%% Remember, this is a list, so separate all tuples
|
||||
%% with a comma.
|
||||
%%
|
||||
%% {Name::atom(), Port::pos_integer(), proplist()}
|
||||
%%
|
||||
%% For example: {my_name_is_a, 12500, []}
|
||||
|
||||
]},
|
||||
|
||||
%% Number of metadata manager processes to run per FLU.
|
||||
%% Default = 10
|
||||
%% {metadata_manager_count, 2},
|
||||
|
||||
%% Default options for chain manager processes.
|
||||
%% {chain_manager_opts, [{private_write_verbose,true},
|
||||
%% {private_write_verbose_confirm,true}]},
|
||||
|
||||
%% Platform vars (mirror of reltool packaging)
|
||||
{platform_data_dir, "{{platform_data_dir}}"},
|
||||
{platform_etc_dir, "{{platform_etc_dir}}"},
|
||||
|
||||
%% Do not delete, do not put Machi config items after this line.
|
||||
{final_comma_stopper, do_not_delete}
|
||||
]
|
||||
},
|
||||
{lager, [
|
||||
{error_logger_hwm, 5000} % lager's default of 50/sec is too low
|
||||
]
|
||||
}
|
||||
].
|
||||
|
|
|
|||
|
|
@ -22,41 +22,23 @@ cd $RUNNER_BASE_DIR
|
|||
SCRIPT=`basename $0`
|
||||
|
||||
usage() {
|
||||
echo "Usage: $SCRIPT { quick-admin-check | quick-admin-apply | "
|
||||
echo "Usage: $SCRIPT { test | "
|
||||
echo " top }"
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
quick-admin-check)
|
||||
test)
|
||||
# Make sure the local node IS running
|
||||
node_up_check
|
||||
|
||||
shift
|
||||
|
||||
NODE_NAME=${NAME_ARG#* } # target machi server node name
|
||||
IN_FILE="$1"
|
||||
# Parse out the node name to pass to the client
|
||||
NODE_NAME=${NAME_ARG#* }
|
||||
|
||||
$ERTS_PATH/erl -noshell -noinput $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \
|
||||
-remsh $NODE_NAME \
|
||||
-eval "Me = self(), spawn('"$NODE_NAME"', fun() -> X = (catch(machi_lifecycle_mgr:quick_admin_sanity_check(\"$IN_FILE\"))), Me ! {res, X} end), XX = receive {res, Res} -> Res after 10*1000 -> timeout end, io:format(user, \"Result: ~p\n\", [XX]), case XX of \
|
||||
ok -> init:stop(); \
|
||||
_ -> init:stop(1) \
|
||||
end."
|
||||
|
||||
;;
|
||||
quick-admin-apply)
|
||||
# Make sure the local node IS running
|
||||
node_up_check
|
||||
|
||||
shift
|
||||
|
||||
NODE_NAME=${NAME_ARG#* } # target machi server node name
|
||||
IN_FILE="$1"
|
||||
RELATIVE_HOST="$2"
|
||||
|
||||
$ERTS_PATH/erl -noshell -noinput $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \
|
||||
-remsh $NODE_NAME \
|
||||
-eval "Me = self(), spawn('"$NODE_NAME"', fun() -> X = (catch(machi_lifecycle_mgr:quick_admin_apply(\"$IN_FILE\", \"$RELATIVE_HOST\"))), Me ! {res, X} end), XX = receive {res, Res} -> Res after 10*1000 -> timeout end, io:format(user, \"Result: ~p\n\", [XX]), case XX of \
|
||||
$ERTS_PATH/erl -noshell $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \
|
||||
-pa $RUNNER_LIB_DIR/basho-patches \
|
||||
-eval "case catch(machi:client_test(\"$NODE_NAME\")) of \
|
||||
ok -> init:stop(); \
|
||||
_ -> init:stop(1) \
|
||||
end."
|
||||
|
|
|
|||
16
rel/gen_dev
16
rel/gen_dev
|
|
@ -1,16 +0,0 @@
|
|||
#! /bin/sh
|
||||
#
|
||||
# Example usage: gen_dev dev4 vars.src vars
|
||||
#
|
||||
# Generate an overlay config for devNNN from vars.src and write to vars
|
||||
#
|
||||
|
||||
NAME=$1
|
||||
TEMPLATE=$2
|
||||
VARFILE=$3
|
||||
|
||||
NODE="$NAME@127.0.0.1"
|
||||
|
||||
echo "Generating $NAME - node='$NODE'"
|
||||
sed -e "s/@NODE@/$NODE/" \
|
||||
< $TEMPLATE > $VARFILE
|
||||
|
|
@ -47,7 +47,6 @@
|
|||
|
||||
{overlay, [
|
||||
{mkdir, "data"},
|
||||
{mkdir, "data/^PRESERVE"},
|
||||
{mkdir, "log"},
|
||||
|
||||
%% Copy base files for starting and interacting w/ node
|
||||
|
|
@ -94,20 +93,6 @@
|
|||
|
||||
{template, "files/vm.args", "etc/vm.args"},
|
||||
{template, "files/app.config", "etc/app.config"},
|
||||
{mkdir, "etc/chain-config"},
|
||||
{mkdir, "etc/flu-config"},
|
||||
{mkdir, "etc/pending"},
|
||||
{mkdir, "etc/rejected"},
|
||||
|
||||
%% Experiment: quick-admin
|
||||
{mkdir, "etc/quick-admin-archive"},
|
||||
{mkdir, "priv"},
|
||||
{mkdir, "priv/quick-admin-examples"},
|
||||
{copy, "../priv/quick-admin-examples/000", "priv/quick-admin-examples"},
|
||||
{copy, "../priv/quick-admin-examples/001", "priv/quick-admin-examples"},
|
||||
{copy, "../priv/quick-admin-examples/002", "priv/quick-admin-examples"},
|
||||
{copy, "../priv/quick-admin-examples/demo-000", "priv/quick-admin-examples/demo-000"},
|
||||
|
||||
{mkdir, "lib/basho-patches"}
|
||||
%% {copy, "../apps/machi/ebin/etop_txt.beam", "lib/basho-patches"}
|
||||
]}.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
|
||||
%% ex: ft=erlang ts=4 sw=4 et
|
||||
|
||||
%% NOTE: When modifying this file, also keep its near cousin
|
||||
%% config file rel/vars/dev_vars.config.src in sync!
|
||||
|
||||
%% Platform-specific installation paths
|
||||
{platform_bin_dir, "./bin"}.
|
||||
{platform_data_dir, "./data"}.
|
||||
|
|
|
|||
|
|
@ -1,48 +0,0 @@
|
|||
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
|
||||
%% ex: ft=erlang ts=4 sw=4 et
|
||||
|
||||
%% NOTE: When modifying this file, also keep its near cousin
|
||||
%% config file rel/vars/dev_vars.config.src in sync!
|
||||
|
||||
%% Platform-specific installation paths
|
||||
{platform_bin_dir, "./bin"}.
|
||||
{platform_data_dir, "./data"}.
|
||||
{platform_etc_dir, "./etc"}.
|
||||
{platform_lib_dir, "./lib"}.
|
||||
{platform_log_dir, "./log"}.
|
||||
|
||||
%%
|
||||
%% etc/app.config
|
||||
%%
|
||||
{sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}.
|
||||
{sasl_log_dir, "{{platform_log_dir}}/sasl"}.
|
||||
|
||||
%% lager
|
||||
{console_log_default, file}.
|
||||
|
||||
%%
|
||||
%% etc/vm.args
|
||||
%%
|
||||
{node, "@NODE@"}.
|
||||
{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}.
|
||||
|
||||
%%
|
||||
%% bin/machi
|
||||
%%
|
||||
{runner_script_dir, "\`cd \\`dirname $0\\` 1>/dev/null && /bin/pwd\`"}.
|
||||
{runner_base_dir, "{{runner_script_dir}}/.."}.
|
||||
{runner_etc_dir, "$RUNNER_BASE_DIR/etc"}.
|
||||
{runner_log_dir, "$RUNNER_BASE_DIR/log"}.
|
||||
{runner_lib_dir, "$RUNNER_BASE_DIR/lib"}.
|
||||
{runner_patch_dir, "$RUNNER_BASE_DIR/lib/basho-patches"}.
|
||||
{pipe_dir, "/tmp/$RUNNER_BASE_DIR/"}.
|
||||
{runner_user, ""}.
|
||||
{runner_wait_process, "machi_flu_sup"}.
|
||||
{runner_ulimit_warn, 65536}.
|
||||
|
||||
%%
|
||||
%% cuttlefish
|
||||
%%
|
||||
{cuttlefish, ""}. % blank = off
|
||||
{cuttlefish_conf, "machi.conf"}.
|
||||
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
{application, machi, [
|
||||
{description, "A village of write-once files."},
|
||||
{vsn, "0.0.1"},
|
||||
{applications, [kernel, stdlib, crypto, cluster_info, ranch]},
|
||||
{vsn, "0.0.0"},
|
||||
{applications, [kernel, stdlib, crypto, cluster_info]},
|
||||
{mod,{machi_app,[]}},
|
||||
{registered, []},
|
||||
{env, [
|
||||
|
|
|
|||
119
src/machi.proto
119
src/machi.proto
|
|
@ -170,18 +170,11 @@ message Mpb_AuthResp {
|
|||
// High level API: append_chunk() request & response
|
||||
|
||||
message Mpb_AppendChunkReq {
|
||||
// General namespace arguments
|
||||
/* In single chain/non-clustered environment, use namespace="" */
|
||||
required string namespace = 1;
|
||||
|
||||
required string prefix = 10;
|
||||
required bytes chunk = 11;
|
||||
required Mpb_ChunkCSum csum = 12;
|
||||
|
||||
optional uint32 chunk_extra = 20;
|
||||
optional string preferred_file_name = 21;
|
||||
/* Fail the operation if our preferred file name is not available */
|
||||
optional bool flag_fail_preferred = 22 [default=false];
|
||||
optional bytes placement_key = 1;
|
||||
required string prefix = 2;
|
||||
required bytes chunk = 3;
|
||||
required Mpb_ChunkCSum csum = 4;
|
||||
optional uint32 chunk_extra = 5;
|
||||
}
|
||||
|
||||
message Mpb_AppendChunkResp {
|
||||
|
|
@ -193,7 +186,7 @@ message Mpb_AppendChunkResp {
|
|||
// High level API: write_chunk() request & response
|
||||
|
||||
message Mpb_WriteChunkReq {
|
||||
required Mpb_Chunk chunk = 10;
|
||||
required Mpb_Chunk chunk = 1;
|
||||
}
|
||||
|
||||
message Mpb_WriteChunkResp {
|
||||
|
|
@ -203,22 +196,19 @@ message Mpb_WriteChunkResp {
|
|||
// High level API: read_chunk() request & response
|
||||
|
||||
message Mpb_ReadChunkReq {
|
||||
// No namespace arguments are required because NS is embedded
|
||||
// inside of the file name.
|
||||
|
||||
required Mpb_ChunkPos chunk_pos = 10;
|
||||
required Mpb_ChunkPos chunk_pos = 1;
|
||||
|
||||
// Use flag_no_checksum=non-zero to skip returning the chunk's checksum.
|
||||
// TODO: not implemented yet.
|
||||
optional bool flag_no_checksum = 20 [default=false];
|
||||
optional uint32 flag_no_checksum = 2 [default=0];
|
||||
|
||||
// Use flag_no_chunk=non-zero to skip returning the chunk (which
|
||||
// only makes sense if flag_no_checksum is not set).
|
||||
// TODO: not implemented yet.
|
||||
optional bool flag_no_chunk = 21 [default=false];
|
||||
optional uint32 flag_no_chunk = 3 [default=0];
|
||||
|
||||
// TODO: not implemented yet.
|
||||
optional bool flag_needs_trimmed = 22 [default=false];
|
||||
optional uint32 flag_needs_trimmed = 4 [default=0];
|
||||
}
|
||||
|
||||
message Mpb_ReadChunkResp {
|
||||
|
|
@ -254,8 +244,6 @@ message Mpb_ChecksumListResp {
|
|||
// High level API: list_files() request & response
|
||||
|
||||
message Mpb_ListFilesReq {
|
||||
// TODO: Add flag for file glob/regexp/other filter type
|
||||
// TODO: What else could go wrong?
|
||||
}
|
||||
|
||||
message Mpb_ListFilesResp {
|
||||
|
|
@ -342,17 +330,18 @@ message Mpb_ProjectionV1 {
|
|||
required uint32 epoch_number = 1;
|
||||
required bytes epoch_csum = 2;
|
||||
required string author_server = 3;
|
||||
required string chain_name = 4;
|
||||
repeated string all_members = 5;
|
||||
repeated string witnesses = 6;
|
||||
required Mpb_Now creation_time = 7;
|
||||
required Mpb_Mode mode = 8;
|
||||
repeated string upi = 9;
|
||||
repeated string repairing = 10;
|
||||
repeated string down = 11;
|
||||
required bytes opaque_dbg = 12;
|
||||
required bytes opaque_dbg2 = 13;
|
||||
repeated Mpb_MembersDictEntry members_dict = 14;
|
||||
repeated string all_members = 4;
|
||||
repeated string witnesses = 5;
|
||||
required Mpb_Now creation_time = 6;
|
||||
required Mpb_Mode mode = 7;
|
||||
repeated string upi = 8;
|
||||
repeated string repairing = 9;
|
||||
repeated string down = 10;
|
||||
optional bytes opaque_flap = 11;
|
||||
optional bytes opaque_inner = 12;
|
||||
required bytes opaque_dbg = 13;
|
||||
required bytes opaque_dbg2 = 14;
|
||||
repeated Mpb_MembersDictEntry members_dict = 15;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////
|
||||
|
|
@ -388,20 +377,12 @@ message Mpb_ProjectionV1 {
|
|||
// Low level API: append_chunk()
|
||||
|
||||
message Mpb_LL_AppendChunkReq {
|
||||
// General namespace arguments
|
||||
required uint32 namespace_version = 1;
|
||||
required string namespace = 2;
|
||||
required uint32 locator = 3;
|
||||
|
||||
required Mpb_EpochID epoch_id = 10;
|
||||
required string prefix = 11;
|
||||
required bytes chunk = 12;
|
||||
required Mpb_ChunkCSum csum = 13;
|
||||
|
||||
optional uint32 chunk_extra = 20;
|
||||
optional string preferred_file_name = 21;
|
||||
/* Fail the operation if our preferred file name is not available */
|
||||
optional bool flag_fail_preferred = 22 [default=false];
|
||||
required Mpb_EpochID epoch_id = 1;
|
||||
optional bytes placement_key = 2;
|
||||
required string prefix = 3;
|
||||
required bytes chunk = 4;
|
||||
required Mpb_ChunkCSum csum = 5;
|
||||
optional uint32 chunk_extra = 6;
|
||||
}
|
||||
|
||||
message Mpb_LL_AppendChunkResp {
|
||||
|
|
@ -413,12 +394,8 @@ message Mpb_LL_AppendChunkResp {
|
|||
// Low level API: write_chunk()
|
||||
|
||||
message Mpb_LL_WriteChunkReq {
|
||||
// General namespace arguments
|
||||
required uint32 namespace_version = 1;
|
||||
required string namespace = 2;
|
||||
|
||||
required Mpb_EpochID epoch_id = 10;
|
||||
required Mpb_Chunk chunk = 11;
|
||||
required Mpb_EpochID epoch_id = 1;
|
||||
required Mpb_Chunk chunk = 2;
|
||||
}
|
||||
|
||||
message Mpb_LL_WriteChunkResp {
|
||||
|
|
@ -428,23 +405,19 @@ message Mpb_LL_WriteChunkResp {
|
|||
// Low level API: read_chunk()
|
||||
|
||||
message Mpb_LL_ReadChunkReq {
|
||||
// General namespace arguments
|
||||
required uint32 namespace_version = 1;
|
||||
required string namespace = 2;
|
||||
|
||||
required Mpb_EpochID epoch_id = 10;
|
||||
required Mpb_ChunkPos chunk_pos = 11;
|
||||
required Mpb_EpochID epoch_id = 1;
|
||||
required Mpb_ChunkPos chunk_pos = 2;
|
||||
|
||||
// Use flag_no_checksum=non-zero to skip returning the chunk's checksum.
|
||||
// TODO: not implemented yet.
|
||||
optional bool flag_no_checksum = 20 [default=false];
|
||||
optional uint32 flag_no_checksum = 3 [default=0];
|
||||
|
||||
// Use flag_no_chunk=non-zero to skip returning the chunk (which
|
||||
// only makes sense if flag_checksum is not set).
|
||||
// TODO: not implemented yet.
|
||||
optional bool flag_no_chunk = 21 [default=false];
|
||||
optional uint32 flag_no_chunk = 4 [default=0];
|
||||
|
||||
optional bool flag_needs_trimmed = 22 [default=false];
|
||||
optional uint32 flag_needs_trimmed = 5 [default=0];
|
||||
}
|
||||
|
||||
message Mpb_LL_ReadChunkResp {
|
||||
|
|
@ -456,16 +429,11 @@ message Mpb_LL_ReadChunkResp {
|
|||
// Low level API: trim_chunk()
|
||||
|
||||
message Mpb_LL_TrimChunkReq {
|
||||
// General namespace arguments
|
||||
required uint32 namespace_version = 1;
|
||||
required string namespace = 2;
|
||||
|
||||
required Mpb_EpochID epoch_id = 10;
|
||||
required string file = 11;
|
||||
required uint64 offset = 12;
|
||||
required uint32 size = 13;
|
||||
|
||||
optional bool trigger_gc = 20 [default=false];
|
||||
required Mpb_EpochID epoch_id = 1;
|
||||
required string file = 2;
|
||||
required uint64 offset = 3;
|
||||
required uint32 size = 4;
|
||||
optional uint32 trigger_gc = 5 [default=0];
|
||||
}
|
||||
|
||||
message Mpb_LL_TrimChunkResp {
|
||||
|
|
@ -475,7 +443,8 @@ message Mpb_LL_TrimChunkResp {
|
|||
// Low level API: checksum_list()
|
||||
|
||||
message Mpb_LL_ChecksumListReq {
|
||||
required string file = 1;
|
||||
required Mpb_EpochID epoch_id = 1;
|
||||
required string file = 2;
|
||||
}
|
||||
|
||||
message Mpb_LL_ChecksumListResp {
|
||||
|
|
@ -506,9 +475,7 @@ message Mpb_LL_WedgeStatusReq {
|
|||
message Mpb_LL_WedgeStatusResp {
|
||||
required Mpb_GeneralStatusCode status = 1;
|
||||
optional Mpb_EpochID epoch_id = 2;
|
||||
optional bool wedged_flag = 3;
|
||||
optional uint32 namespace_version = 4;
|
||||
optional string namespace = 5;
|
||||
optional uint32 wedged_flag = 3;
|
||||
}
|
||||
|
||||
// Low level API: delete_migration()
|
||||
|
|
|
|||
|
|
@ -90,18 +90,17 @@ verify_file_checksums_local2(Sock1, EpochID, Path0) ->
|
|||
end.
|
||||
|
||||
verify_file_checksums_remote2(Sock1, EpochID, File) ->
|
||||
NSInfo = undefined,
|
||||
ReadChunk = fun(File_name, Offset, Size) ->
|
||||
?FLU_C:read_chunk(Sock1, NSInfo, EpochID,
|
||||
File_name, Offset, Size, undefined)
|
||||
?FLU_C:read_chunk(Sock1, EpochID,
|
||||
File_name, Offset, Size, [])
|
||||
end,
|
||||
verify_file_checksums_common(Sock1, EpochID, File, ReadChunk).
|
||||
|
||||
verify_file_checksums_common(Sock1, _EpochID, File, ReadChunk) ->
|
||||
verify_file_checksums_common(Sock1, EpochID, File, ReadChunk) ->
|
||||
try
|
||||
case ?FLU_C:checksum_list(Sock1, File) of
|
||||
case ?FLU_C:checksum_list(Sock1, EpochID, File) of
|
||||
{ok, InfoBin} ->
|
||||
Info = machi_csum_table:split_checksum_list_blob_decode(InfoBin),
|
||||
{Info, _} = machi_csum_table:split_checksum_list_blob_decode(InfoBin),
|
||||
Res = lists:foldl(verify_chunk_checksum(File, ReadChunk),
|
||||
[], Info),
|
||||
{ok, Res};
|
||||
|
|
@ -116,9 +115,7 @@ verify_file_checksums_common(Sock1, _EpochID, File, ReadChunk) ->
|
|||
end.
|
||||
|
||||
verify_chunk_checksum(File, ReadChunk) ->
|
||||
fun({0, ?MINIMUM_OFFSET, none}, []) ->
|
||||
[];
|
||||
({Offset, Size, <<_Tag:1/binary, CSum/binary>>}, Acc) ->
|
||||
fun({Offset, Size, <<_Tag:1/binary, CSum/binary>>}, Acc) ->
|
||||
case ReadChunk(File, Offset, Size) of
|
||||
{ok, {[{_, Offset, Chunk, _}], _}} ->
|
||||
CSum2 = machi_util:checksum_chunk(Chunk),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved.
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
|
|
@ -43,25 +43,23 @@
|
|||
%% could add new entries to this ETS table.
|
||||
%%
|
||||
%% Now we can use various integer-centric key generators that are
|
||||
%% already bundled with basho_bench. NOTE: this scheme does not allow
|
||||
%% mixing of 'append' and 'read' operations in the same config. Basho
|
||||
%% Bench does not support different key generators for different
|
||||
%% operations, unfortunately. The work-around is to run two different
|
||||
%% Basho Bench instances: on for 'append' ops with a key generator for
|
||||
%% the desired prefix(es), and the other for 'read' ops with an
|
||||
%% integer key generator.
|
||||
%% already bundled with basho_bench.
|
||||
%%
|
||||
%% TODO: The 'read' operator will always read chunks at exactly the
|
||||
%% byte offset & size as the original append/write ops. If reads are
|
||||
%% desired at any arbitrary offset & size, then a new strategy is
|
||||
%% required.
|
||||
%% TODO: Add CRC checking, when feasible and when supported on the
|
||||
%% server side.
|
||||
%%
|
||||
%% TODO: As an alternate idea, if we know that the chunks written are
|
||||
%% always the same size, and if we don't care about CRC checking, then
|
||||
%% all we need to know are the file names & file sizes on the server:
|
||||
%% we can then pick any valid offset within that file. That would
|
||||
%% certainly be more scalable than the zillion-row-ETS-table, which is
|
||||
%% definitely RAM-hungry.
|
||||
|
||||
-module(machi_basho_bench_driver).
|
||||
|
||||
-export([new/1, run/4]).
|
||||
|
||||
-record(m, {
|
||||
id,
|
||||
conn,
|
||||
max_key
|
||||
}).
|
||||
|
|
@ -83,7 +81,7 @@ new(Id) ->
|
|||
{read_concurrency, true}]),
|
||||
ets:insert(ETS, {max_key, 0}),
|
||||
ets:insert(ETS, {total_bytes, 0}),
|
||||
MaxKeys = load_ets_table_maybe(Conn, ETS),
|
||||
MaxKeys = load_ets_table(Conn, ETS),
|
||||
?INFO("Key preload: finished, ~w keys loaded", [MaxKeys]),
|
||||
Bytes = ets:lookup_element(ETS, total_bytes, 2),
|
||||
?INFO("Key preload: finished, chunk list specifies ~s MBytes of chunks",
|
||||
|
|
@ -92,14 +90,12 @@ new(Id) ->
|
|||
true ->
|
||||
ok
|
||||
end,
|
||||
{ok, #m{id=Id, conn=Conn}}.
|
||||
{ok, #m{conn=Conn}}.
|
||||
|
||||
run(append, KeyGen, ValueGen, #m{conn=Conn}=S) ->
|
||||
Prefix = KeyGen(),
|
||||
Value = ValueGen(),
|
||||
CSum = machi_util:make_client_csum(Value),
|
||||
AppendOpts = {append_opts,0,undefined,false}, % HACK FIXME
|
||||
case machi_cr_client:append_chunk(Conn, undefined, Prefix, Value, CSum, AppendOpts, ?THE_TIMEOUT) of
|
||||
case machi_cr_client:append_chunk(Conn, Prefix, Value, ?THE_TIMEOUT) of
|
||||
{ok, Pos} ->
|
||||
EtsKey = ets:update_counter(?ETS_TAB, max_key, 1),
|
||||
true = ets:insert(?ETS_TAB, {EtsKey, Pos}),
|
||||
|
|
@ -116,26 +112,9 @@ run(read, KeyGen, _ValueGen, #m{conn=Conn, max_key=MaxKey}=S) ->
|
|||
Idx = KeyGen() rem MaxKey,
|
||||
%% {File, Offset, Size, _CSum} = ets:lookup_element(?ETS_TAB, Idx, 2),
|
||||
{File, Offset, Size} = ets:lookup_element(?ETS_TAB, Idx, 2),
|
||||
ReadOpts = {read_opts,false,false,false}, % HACK FIXME
|
||||
case machi_cr_client:read_chunk(Conn, undefined, File, Offset, Size, ReadOpts, ?THE_TIMEOUT) of
|
||||
{ok, {Chunks, _Trimmed}} ->
|
||||
%% io:format(user, "Chunks ~P\n", [Chunks, 15]),
|
||||
%% {ok, S};
|
||||
case lists:all(fun({File2, Offset2, Chunk, CSum}) ->
|
||||
{_Tag, CS} = machi_util:unmake_tagged_csum(CSum),
|
||||
CS2 = machi_util:checksum_chunk(Chunk),
|
||||
if CS == CS2 ->
|
||||
true;
|
||||
CS /= CS2 ->
|
||||
?ERROR("Client-side checksum error for file ~p offset ~p expected ~p got ~p\n", [File2, Offset2, CS, CS2]),
|
||||
false
|
||||
end
|
||||
end, Chunks) of
|
||||
true ->
|
||||
{ok, S};
|
||||
false ->
|
||||
{error, bad_checksum, S}
|
||||
end;
|
||||
case machi_cr_client:read_chunk(Conn, File, Offset, Size, [], ?THE_TIMEOUT) of
|
||||
{ok, _Chunk} ->
|
||||
{ok, S};
|
||||
{error, _}=Err ->
|
||||
?ERROR("read file ~p offset ~w size ~w: ~w\n",
|
||||
[File, Offset, Size, Err]),
|
||||
|
|
@ -153,40 +132,21 @@ find_server_info(_Id) ->
|
|||
Ps
|
||||
end.
|
||||
|
||||
load_ets_table_maybe(Conn, ETS) ->
|
||||
case basho_bench_config:get(operations, undefined) of
|
||||
undefined ->
|
||||
?ERROR("The 'operations' key is missing from the config file, aborting", []),
|
||||
exit(bad_config);
|
||||
Ops when is_list(Ops) ->
|
||||
case lists:keyfind(read, 1, Ops) of
|
||||
{read,_} ->
|
||||
load_ets_table(Conn, ETS);
|
||||
false ->
|
||||
?INFO("No 'read' op in the 'operations' list ~p, skipping ETS table load.", [Ops]),
|
||||
0
|
||||
end
|
||||
end.
|
||||
|
||||
load_ets_table(Conn, ETS) ->
|
||||
{ok, Fs} = machi_cr_client:list_files(Conn),
|
||||
[begin
|
||||
{ok, InfoBin} = machi_cr_client:checksum_list(Conn, File, ?THE_TIMEOUT),
|
||||
PosList = machi_csum_table:split_checksum_list_blob_decode(InfoBin),
|
||||
?INFO("File ~s len PosList ~p\n", [File, length(PosList)]),
|
||||
{ok, InfoBin} = machi_cr_client:checksum_list(Conn, File),
|
||||
{PosList, _} = machi_csum_table:split_checksum_list_blob_decode(InfoBin),
|
||||
StartKey = ets:update_counter(ETS, max_key, 0),
|
||||
{_, C, Bytes} = lists:foldl(fun({_Off,0,_CSum}, {_K, _C, _Bs}=Acc) ->
|
||||
Acc;
|
||||
({0,_Sz,_CSum}, {_K, _C, _Bs}=Acc) ->
|
||||
Acc;
|
||||
({Off,Sz,_CSum}, {K, C, Bs}) ->
|
||||
V = {File, Off, Sz},
|
||||
ets:insert(ETS, {K, V}),
|
||||
{K + 1, C + 1, Bs + Sz}
|
||||
end, {StartKey, 0, 0}, PosList),
|
||||
_ = ets:update_counter(ETS, max_key, C),
|
||||
_ = ets:update_counter(ETS, total_bytes, Bytes),
|
||||
ok
|
||||
%% _EndKey = lists:foldl(fun({Off,Sz,CSum}, K) ->
|
||||
%% V = {File, Off, Sz, CSum},
|
||||
{_, Bytes} = lists:foldl(fun({Off,Sz,_CSum}, {K, Bs}) ->
|
||||
V = {File, Off, Sz},
|
||||
ets:insert(ETS, {K, V}),
|
||||
{K + 1, Bs + Sz}
|
||||
end, {StartKey, 0}, PosList),
|
||||
ets:update_counter(ETS, max_key, length(PosList)),
|
||||
ets:update_counter(ETS, total_bytes, Bytes)
|
||||
end || {_Size, File} <- Fs],
|
||||
ets:update_counter(?ETS_TAB, max_key, 0).
|
||||
|
||||
|
|
|
|||
|
|
@ -92,11 +92,8 @@
|
|||
-define(REPAIR_START_STABILITY_TIME, 10).
|
||||
-endif. % TEST
|
||||
|
||||
%% Maximum length of the history of adopted projections (via C120).
|
||||
-define(MAX_HISTORY_LENGTH, 8).
|
||||
|
||||
%% Magic constant for looping "too frequently" breaker.
|
||||
-define(TOO_FREQUENT_BREAKER, (?MAX_HISTORY_LENGTH+5)).
|
||||
%% Magic constant for looping "too frequently" breaker. TODO revisit & revise.
|
||||
-define(TOO_FREQUENT_BREAKER, 10).
|
||||
|
||||
-define(RETURN2(X), begin (catch put(why2, [?LINE|get(why2)])), X end).
|
||||
|
||||
|
|
@ -106,9 +103,12 @@
|
|||
%% Amount of epoch number skip-ahead for set_chain_members call
|
||||
-define(SET_CHAIN_MEMBERS_EPOCH_SKIP, 1111).
|
||||
|
||||
%% Maximum length of the history of adopted projections (via C120).
|
||||
-define(MAX_HISTORY_LENGTH, 30).
|
||||
|
||||
%% API
|
||||
-export([start_link/2, start_link/3, stop/1, ping/1,
|
||||
set_chain_members/2, set_chain_members/6, set_active/2,
|
||||
set_chain_members/2, set_chain_members/3, set_active/2,
|
||||
trigger_react_to_env/1]).
|
||||
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
|
||||
terminate/2, format_status/2, code_change/3]).
|
||||
|
|
@ -168,22 +168,13 @@ ping(Pid) ->
|
|||
%% with lowest rank, i.e. name z* first, name a* last.
|
||||
|
||||
set_chain_members(Pid, MembersDict) ->
|
||||
set_chain_members(Pid, ch0_name, 0, ap_mode, MembersDict, []).
|
||||
set_chain_members(Pid, MembersDict, []).
|
||||
|
||||
set_chain_members(Pid, ChainName, OldEpoch, CMode, MembersDict, Witness_list)
|
||||
when is_atom(ChainName) andalso
|
||||
is_integer(OldEpoch) andalso OldEpoch >= 0 andalso
|
||||
(CMode == ap_mode orelse CMode == cp_mode) andalso
|
||||
is_list(MembersDict) andalso
|
||||
is_list(Witness_list) ->
|
||||
case lists:all(fun({X, #p_srvr{name=X}}) -> true;
|
||||
(_) -> false
|
||||
end, MembersDict)
|
||||
andalso
|
||||
lists:all(fun(Witness) -> orddict:is_key(Witness, MembersDict) end,
|
||||
Witness_list) of
|
||||
set_chain_members(Pid, MembersDict, Witness_list) ->
|
||||
case lists:all(fun(Witness) -> orddict:is_key(Witness, MembersDict) end,
|
||||
Witness_list) of
|
||||
true ->
|
||||
Cmd = {set_chain_members, ChainName, OldEpoch, CMode, MembersDict, Witness_list},
|
||||
Cmd = {set_chain_members, MembersDict, Witness_list},
|
||||
gen_server:call(Pid, Cmd, infinity);
|
||||
false ->
|
||||
{error, bad_arg}
|
||||
|
|
@ -234,13 +225,11 @@ test_read_latest_public_projection(Pid, ReadRepairP) ->
|
|||
%% manager's pid in MgrOpts and use direct gen_server calls to the
|
||||
%% local projection store.
|
||||
|
||||
init({MyName, InitMembersDict, MgrOpts0}) ->
|
||||
init({MyName, InitMembersDict, MgrOpts}) ->
|
||||
put(ttt, [?LINE]),
|
||||
_ = random:seed(now()),
|
||||
init_remember_down_list(),
|
||||
MgrOpts = MgrOpts0 ++ application:get_env(machi, chain_manager_opts, []),
|
||||
Opt = fun(Key, Default) -> proplists:get_value(Key, MgrOpts, Default) end,
|
||||
|
||||
InitWitness_list = Opt(witnesses, []),
|
||||
ZeroAll_list = [P#p_srvr.name || {_,P} <- orddict:to_list(InitMembersDict)],
|
||||
ZeroProj = make_none_projection(0, MyName, ZeroAll_list,
|
||||
|
|
@ -292,7 +281,7 @@ init({MyName, InitMembersDict, MgrOpts0}) ->
|
|||
last_down=[no_such_server_initial_value_only],
|
||||
fitness_svr=machi_flu_psup:make_fitness_regname(MyName)
|
||||
}, Proj),
|
||||
S2 = do_set_chain_members_dict(MembersDict, S),
|
||||
{_, S2} = do_set_chain_members_dict(MembersDict, S),
|
||||
S3 = if ActiveP == false ->
|
||||
S2;
|
||||
ActiveP == true ->
|
||||
|
|
@ -302,17 +291,12 @@ init({MyName, InitMembersDict, MgrOpts0}) ->
|
|||
|
||||
handle_call({ping}, _From, S) ->
|
||||
{reply, pong, S};
|
||||
handle_call({set_chain_members, SetChainName, SetOldEpoch, CMode,
|
||||
MembersDict, Witness_list}, _From,
|
||||
handle_call({set_chain_members, MembersDict, Witness_list}, _From,
|
||||
#ch_mgr{name=MyName,
|
||||
proj=#projection_v1{all_members=OldAll_list,
|
||||
epoch_number=OldEpoch,
|
||||
chain_name=ChainName,
|
||||
upi=OldUPI}=OldProj}=S) ->
|
||||
true = (OldEpoch == 0) % in this case we want unconditional set of ch name
|
||||
orelse
|
||||
(SetOldEpoch == OldEpoch andalso SetChainName == ChainName),
|
||||
S2 = do_set_chain_members_dict(MembersDict, S),
|
||||
{Reply, S2} = do_set_chain_members_dict(MembersDict, S),
|
||||
%% TODO: should there be any additional sanity checks? Right now,
|
||||
%% if someone does something bad, then do_react_to_env() will
|
||||
%% crash, which will crash us, and we'll restart in a sane & old
|
||||
|
|
@ -326,10 +310,10 @@ handle_call({set_chain_members, SetChainName, SetOldEpoch, CMode,
|
|||
{NUPI, All_list -- NUPI}
|
||||
end,
|
||||
NewEpoch = OldEpoch + ?SET_CHAIN_MEMBERS_EPOCH_SKIP,
|
||||
CMode = calc_consistency_mode(Witness_list),
|
||||
ok = set_consistency_mode(machi_flu_psup:make_proj_supname(MyName), CMode),
|
||||
NewProj = machi_projection:update_checksum(
|
||||
OldProj#projection_v1{author_server=MyName,
|
||||
chain_name=SetChainName,
|
||||
creation_time=now(),
|
||||
mode=CMode,
|
||||
epoch_number=NewEpoch,
|
||||
|
|
@ -341,11 +325,7 @@ handle_call({set_chain_members, SetChainName, SetOldEpoch, CMode,
|
|||
members_dict=MembersDict}),
|
||||
S3 = set_proj(S2#ch_mgr{proj_history=queue:new(),
|
||||
consistency_mode=CMode}, NewProj),
|
||||
{Res, S4} = do_react_to_env(S3),
|
||||
Reply = case Res of
|
||||
{_,_,_} -> ok
|
||||
% Dialyzer says that all vals match with the 3-tuple pattern
|
||||
end,
|
||||
{_QQ, S4} = do_react_to_env(S3),
|
||||
{reply, Reply, S4};
|
||||
handle_call({set_active, Boolean}, _From, #ch_mgr{timer=TRef}=S) ->
|
||||
case {Boolean, TRef} of
|
||||
|
|
@ -377,8 +357,8 @@ handle_call({test_read_latest_public_projection, ReadRepairP}, _From, S) ->
|
|||
{reply, Res, S2};
|
||||
handle_call({trigger_react_to_env}=Call, _From, S) ->
|
||||
gobble_calls(Call),
|
||||
{Res, S2} = do_react_to_env(S),
|
||||
{reply, Res, S2};
|
||||
{TODOtodo, S2} = do_react_to_env(S),
|
||||
{reply, TODOtodo, S2};
|
||||
handle_call(_Call, _From, S) ->
|
||||
io:format(user, "\nBad call to ~p: ~p\n", [S#ch_mgr.name, _Call]),
|
||||
{reply, whaaaaaaaaaa, S}.
|
||||
|
|
@ -390,7 +370,6 @@ handle_cast(_Cast, S) ->
|
|||
handle_info(tick_check_environment, #ch_mgr{ignore_timer=true}=S) ->
|
||||
{noreply, S};
|
||||
handle_info(tick_check_environment, S) ->
|
||||
gobble_ticks(),
|
||||
{{_Delta, Props, _Epoch}, S1} = do_react_to_env(S),
|
||||
S2 = sanitize_repair_state(S1),
|
||||
S3 = perhaps_start_repair(S2),
|
||||
|
|
@ -463,7 +442,7 @@ get_my_proj_boot_info(MgrOpts, DefaultDict, DefaultProj, ProjType) ->
|
|||
{DefaultDict, DefaultProj};
|
||||
Store ->
|
||||
{ok, P} = machi_projection_store:read_latest_projection(Store,
|
||||
ProjType, 7789),
|
||||
ProjType),
|
||||
{P#projection_v1.members_dict, P}
|
||||
end.
|
||||
|
||||
|
|
@ -556,7 +535,6 @@ cl_write_public_proj2(FLUs, Partitions, Epoch, Proj, IgnoreWrittenErrorP, S) ->
|
|||
end
|
||||
end, {true, []}, FLUs),
|
||||
%% io:format(user, "\nWrite public ~w by ~w: ~w\n", [Epoch, S#ch_mgr.name, Rs]),
|
||||
%% io:format(user, "mgr ~w epoch ~w Rs ~p\n", [S#ch_mgr.name, Epoch, Rs]),
|
||||
{{remote_write_results, Rs}, S}.
|
||||
|
||||
do_cl_read_latest_public_projection(ReadRepairP,
|
||||
|
|
@ -578,41 +556,12 @@ do_cl_read_latest_public_projection(ReadRepairP,
|
|||
read_latest_projection_call_only(ProjectionType, AllHosed,
|
||||
#ch_mgr{proj=CurrentProj}=S) ->
|
||||
#projection_v1{all_members=All_list} = CurrentProj,
|
||||
All_queried_list = lists:sort(All_list -- AllHosed),
|
||||
read_latest_projection_call_only1(ProjectionType, AllHosed,
|
||||
All_queried_list, S).
|
||||
All_queried_list = All_list -- AllHosed,
|
||||
|
||||
read_latest_projection_call_only1(ProjectionType, AllHosed,
|
||||
All_queried_list, S) ->
|
||||
{Rs_tmp, S2} = read_latest_projection_call_only2(ProjectionType,
|
||||
All_queried_list, S),
|
||||
New_all_maybe =
|
||||
lists:usort(
|
||||
lists:flatten(
|
||||
[A_l || #projection_v1{all_members=A_l} <- Rs_tmp])) -- AllHosed,
|
||||
case New_all_maybe -- All_queried_list of
|
||||
[] ->
|
||||
FLUsRs = lists:zip(All_queried_list, Rs_tmp),
|
||||
{All_queried_list, FLUsRs, S2};
|
||||
[AnotherFLU|_] ->
|
||||
%% Stop AnotherFLU proxy, in unexpected case where it's open
|
||||
try
|
||||
Proxy = proxy_pid(AnotherFLU, S2),
|
||||
?FLU_PC:stop_proxies([Proxy])
|
||||
catch _:_ -> ok
|
||||
end,
|
||||
MD = orddict:from_list(
|
||||
lists:usort(
|
||||
lists:flatten(
|
||||
[orddict:to_list(D) || #projection_v1{members_dict=D} <- Rs_tmp]))),
|
||||
Another_P_srvr = orddict:fetch(AnotherFLU, MD),
|
||||
{ok, Proxy2} = ?FLU_PC:start_link(Another_P_srvr),
|
||||
S3 = S2#ch_mgr{proxies_dict=orddict:store(AnotherFLU, Proxy2,
|
||||
S2#ch_mgr.proxies_dict)},
|
||||
read_latest_projection_call_only1(
|
||||
ProjectionType, AllHosed,
|
||||
lists:usort([AnotherFLU|All_queried_list]), S3)
|
||||
end.
|
||||
{Rs, S2} = read_latest_projection_call_only2(ProjectionType,
|
||||
All_queried_list, S),
|
||||
FLUsRs = lists:zip(All_queried_list, Rs),
|
||||
{All_queried_list, FLUsRs, S2}.
|
||||
|
||||
read_latest_projection_call_only2(ProjectionType, All_queried_list, S) ->
|
||||
{_UpNodes, Partitions, S2} = calc_up_nodes(S),
|
||||
|
|
@ -652,8 +601,6 @@ rank_and_sort_projections_with_extra(All_queried_list, FLUsRs, ProjectionType,
|
|||
Witness_list = CurrentProj#projection_v1.witnesses,
|
||||
NoneProj = make_none_projection(0, MyName, [], Witness_list,
|
||||
orddict:new()),
|
||||
ChainName = CurrentProj#projection_v1.chain_name,
|
||||
NoneProj2 = NoneProj#projection_v1{chain_name=ChainName},
|
||||
Extra2 = [{all_members_replied, true},
|
||||
{all_queried_list, All_queried_list},
|
||||
{flus_rs, FLUsRs},
|
||||
|
|
@ -662,7 +609,7 @@ rank_and_sort_projections_with_extra(All_queried_list, FLUsRs, ProjectionType,
|
|||
{bad_answer_flus, BadAnswerFLUs},
|
||||
{bad_answers, BadAnswers},
|
||||
{not_unanimous_answers, []}],
|
||||
{not_unanimous, NoneProj2, Extra2, S};
|
||||
{not_unanimous, NoneProj, Extra2, S};
|
||||
ProjectionType == public, UnwrittenRs /= [] ->
|
||||
{needs_repair, FLUsRs, [flarfus], S};
|
||||
true ->
|
||||
|
|
@ -776,14 +723,13 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
runenv=RunEnv1,
|
||||
repair_final_status=RepairFS}=S) ->
|
||||
#projection_v1{epoch_number=OldEpochNum,
|
||||
chain_name=ChainName,
|
||||
members_dict=MembersDict,
|
||||
witnesses=OldWitness_list,
|
||||
upi=OldUPI_list,
|
||||
repairing=OldRepairing_list
|
||||
} = LastProj,
|
||||
LastUp = lists:usort(OldUPI_list ++ OldRepairing_list),
|
||||
AllMembers = CurrentProj#projection_v1.all_members,
|
||||
AllMembers = (S#ch_mgr.proj)#projection_v1.all_members,
|
||||
{Up0, Partitions, RunEnv2} = calc_up_nodes(MyName,
|
||||
AllMembers, RunEnv1),
|
||||
Up = Up0 -- AllHosed,
|
||||
|
|
@ -840,10 +786,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
D_foo=[{repair_done, {repair_final_status, ok, (S#ch_mgr.proj)#projection_v1.epoch_number}}],
|
||||
{NewUPI_list ++ Repairing_list2, [], RunEnv2};
|
||||
true ->
|
||||
D_foo=[d_foo2, {sim_p,Simulator_p},
|
||||
{simr_p,SimRepair_p}, {same_epoch,SameEpoch_p},
|
||||
{rel_to,RelativeToServer},
|
||||
{repch,RepChk_LastInUPI}, {repair_fs,RepairFS}],
|
||||
D_foo=[d_foo2],
|
||||
{NewUPI_list, OldRepairing_list, RunEnv2}
|
||||
end;
|
||||
{_ABC, _XYZ} ->
|
||||
|
|
@ -878,11 +821,10 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
end,
|
||||
?REACT({calc,?LINE,[{new_upi, NewUPI},{new_rep, NewRepairing}]}),
|
||||
|
||||
P0 = machi_projection:new(OldEpochNum + 1,
|
||||
MyName, MembersDict, Down, NewUPI, NewRepairing,
|
||||
D_foo ++
|
||||
Dbg ++ [{ps, Partitions},{nodes_up, Up}]),
|
||||
P1 = P0#projection_v1{chain_name=ChainName},
|
||||
P = machi_projection:new(OldEpochNum + 1,
|
||||
MyName, MembersDict, Down, NewUPI, NewRepairing,
|
||||
D_foo ++
|
||||
Dbg ++ [{ps, Partitions},{nodes_up, Up}]),
|
||||
P2 = if CMode == cp_mode ->
|
||||
UpWitnesses = [W || W <- Up, lists:member(W, OldWitness_list)],
|
||||
Majority = full_majority_size(AllMembers),
|
||||
|
|
@ -891,7 +833,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
SoFar = length(NewUPI ++ NewRepairing),
|
||||
if SoFar >= Majority ->
|
||||
?REACT({calc,?LINE,[]}),
|
||||
P1;
|
||||
P;
|
||||
true ->
|
||||
Need = Majority - SoFar,
|
||||
UpWitnesses = [W || W <- Up,
|
||||
|
|
@ -900,7 +842,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
Ws = lists:sublist(UpWitnesses, Need),
|
||||
?REACT({calc,?LINE,[{ws, Ws}]}),
|
||||
machi_projection:update_checksum(
|
||||
P1#projection_v1{upi=Ws++NewUPI});
|
||||
P#projection_v1{upi=Ws++NewUPI});
|
||||
true ->
|
||||
?REACT({calc,?LINE,[]}),
|
||||
P_none0 = make_none_projection(
|
||||
|
|
@ -913,7 +855,6 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
"Not enough witnesses are available now"
|
||||
end,
|
||||
P_none1 = P_none0#projection_v1{
|
||||
chain_name=ChainName,
|
||||
%% Stable creation time!
|
||||
creation_time={1,2,3},
|
||||
dbg=[{none_projection,true},
|
||||
|
|
@ -934,7 +875,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
end;
|
||||
CMode == ap_mode ->
|
||||
?REACT({calc,?LINE,[]}),
|
||||
P1
|
||||
P
|
||||
end,
|
||||
P3 = machi_projection:update_checksum(
|
||||
P2#projection_v1{mode=CMode, witnesses=OldWitness_list}),
|
||||
|
|
@ -1086,33 +1027,31 @@ rank_projection(#projection_v1{author_server=_Author,
|
|||
do_set_chain_members_dict(MembersDict, #ch_mgr{proxies_dict=OldProxiesDict}=S)->
|
||||
_ = ?FLU_PC:stop_proxies(OldProxiesDict),
|
||||
ProxiesDict = ?FLU_PC:start_proxies(MembersDict),
|
||||
S#ch_mgr{members_dict=MembersDict,
|
||||
proxies_dict=ProxiesDict}.
|
||||
{ok, S#ch_mgr{members_dict=MembersDict,
|
||||
proxies_dict=ProxiesDict}}.
|
||||
|
||||
do_react_to_env(#ch_mgr{name=MyName,
|
||||
proj=#projection_v1{epoch_number=Epoch,
|
||||
members_dict=[]=OldDict}=OldProj,
|
||||
opts=Opts}=S) ->
|
||||
put(ttt, [?LINE]),
|
||||
%% Read from our local *public* projection store. If some other
|
||||
%% chain member has written something there, and if we are a
|
||||
%% member of that chain, then we'll adopt that projection and then
|
||||
%% start actively humming in that chain.
|
||||
{NewMD, NewProj} =
|
||||
{NewMembersDict, NewProj} =
|
||||
get_my_public_proj_boot_info(Opts, OldDict, OldProj),
|
||||
case orddict:is_key(MyName, NewMD) of
|
||||
case orddict:is_key(MyName, NewMembersDict) of
|
||||
false ->
|
||||
{{empty_members_dict1, [], Epoch}, S};
|
||||
{{empty_members_dict, [], Epoch}, S};
|
||||
true ->
|
||||
CMode = NewProj#projection_v1.mode,
|
||||
S2 = do_set_chain_members_dict(NewMD, S),
|
||||
{Reply, S3} = react_to_env_C110(NewProj,
|
||||
S2#ch_mgr{members_dict=NewMD,
|
||||
consistency_mode=CMode}),
|
||||
{Reply, S3}
|
||||
{_, S2} = do_set_chain_members_dict(NewMembersDict, S),
|
||||
CMode = calc_consistency_mode(NewProj#projection_v1.witnesses),
|
||||
{{empty_members_dict, [], Epoch},
|
||||
set_proj(S2#ch_mgr{members_dict=NewMembersDict,
|
||||
consistency_mode=CMode}, NewProj)}
|
||||
end;
|
||||
do_react_to_env(S) ->
|
||||
put(ttt, [?LINE]),
|
||||
put(ttt, [?LINE]),
|
||||
%% The not_sanes manager counting dictionary is not strictly
|
||||
%% limited to flapping scenarios. (Though the mechanism first
|
||||
%% started as a way to deal with rare flapping scenarios.)
|
||||
|
|
@ -1211,7 +1150,7 @@ react_to_env_A10(S) ->
|
|||
?REACT(a10),
|
||||
react_to_env_A20(0, poll_private_proj_is_upi_unanimous(S)).
|
||||
|
||||
react_to_env_A20(Retries, #ch_mgr{name=MyName, proj=P_current}=S) ->
|
||||
react_to_env_A20(Retries, #ch_mgr{name=MyName}=S) ->
|
||||
?REACT(a20),
|
||||
init_remember_down_list(),
|
||||
{UnanimousTag, P_latest, ReadExtra, S2} =
|
||||
|
|
@ -1239,34 +1178,17 @@ react_to_env_A20(Retries, #ch_mgr{name=MyName, proj=P_current}=S) ->
|
|||
false when P_latest#projection_v1.epoch_number /= LastComplaint,
|
||||
P_latest#projection_v1.all_members /= [] ->
|
||||
put(rogue_server_epoch, P_latest#projection_v1.epoch_number),
|
||||
error_logger:info_msg("Chain manager ~w found latest public "
|
||||
"projection ~w with author ~w has a "
|
||||
"members list ~w that does not include me. "
|
||||
"We assume this is a result of administrator "
|
||||
"action and will thus wedge ourselves until "
|
||||
"we are re-added to the chain or shutdown.\n",
|
||||
error_logger:info_msg("Chain manager ~p found latest public "
|
||||
"projection ~p has author ~p has a "
|
||||
"members list ~p that does not include me.\n",
|
||||
[S#ch_mgr.name,
|
||||
P_latest#projection_v1.epoch_number,
|
||||
P_latest#projection_v1.author_server,
|
||||
P_latest#projection_v1.all_members]),
|
||||
EpochID = machi_projection:make_epoch_id(P_current),
|
||||
ProjStore = get_projection_store_pid_or_regname(S),
|
||||
{ok, NotifyPid} = machi_projection_store:get_wedge_notify_pid(ProjStore),
|
||||
_QQ = machi_flu1:update_wedge_state(NotifyPid, true, EpochID),
|
||||
#projection_v1{epoch_number=Epoch,
|
||||
chain_name=ChainName,
|
||||
all_members=All_list,
|
||||
witnesses=Witness_list,
|
||||
members_dict=MembersDict} = P_current,
|
||||
P_none0 = make_none_projection(Epoch,
|
||||
MyName, All_list, Witness_list, MembersDict),
|
||||
P_none = P_none0#projection_v1{chain_name=ChainName},
|
||||
{{now_using,[],Epoch}, set_proj(S2, P_none)};
|
||||
P_latest#projection_v1.all_members]);
|
||||
_ ->
|
||||
react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S2)
|
||||
end.
|
||||
ok
|
||||
end,
|
||||
|
||||
react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S) ->
|
||||
%% The UnanimousTag isn't quite sufficient for our needs. We need
|
||||
%% to determine if *all* of the UPI+Repairing FLUs are members of
|
||||
%% the unanimous server replies. All Repairing FLUs should be up
|
||||
|
|
@ -1311,7 +1233,7 @@ react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S) ->
|
|||
true ->
|
||||
exit({badbad, UnanimousTag})
|
||||
end,
|
||||
react_to_env_A29(Retries, P_latest, LatestUnanimousP, ReadExtra, S).
|
||||
react_to_env_A29(Retries, P_latest, LatestUnanimousP, ReadExtra, S2).
|
||||
|
||||
react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra,
|
||||
#ch_mgr{consistency_mode=CMode,
|
||||
|
|
@ -1345,6 +1267,7 @@ react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra,
|
|||
?REACT({a29, ?LINE,
|
||||
[{zerf_backstop, true},
|
||||
{zerf_in, machi_projection:make_summary(Zerf)}]}),
|
||||
%% io:format(user, "zerf_in: A29: ~p: ~w\n\t~p\n", [MyName, machi_projection:make_summary(Zerf), get(yyy_hack)]),
|
||||
#projection_v1{dbg=ZerfDbg} = Zerf,
|
||||
Backstop = if Zerf#projection_v1.upi == [] ->
|
||||
[];
|
||||
|
|
@ -1364,8 +1287,7 @@ react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra,
|
|||
end.
|
||||
|
||||
react_to_env_A30(Retries, P_latest, LatestUnanimousP, P_current_calc,
|
||||
#ch_mgr{name=MyName, proj=P_current,
|
||||
consistency_mode=CMode} = S) ->
|
||||
#ch_mgr{name=MyName, consistency_mode=CMode} = S) ->
|
||||
V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end,
|
||||
if V -> io:format(user, "A30: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
|
||||
?REACT(a30),
|
||||
|
|
@ -1385,17 +1307,15 @@ react_to_env_A30(Retries, P_latest, LatestUnanimousP, P_current_calc,
|
|||
P = #projection_v1{down=Down} =
|
||||
make_none_projection(Epoch + 1, MyName, All_list,
|
||||
Witness_list, MembersDict),
|
||||
ChainName = P_current#projection_v1.chain_name,
|
||||
P1 = P#projection_v1{chain_name=ChainName},
|
||||
P_newprop = if CMode == ap_mode ->
|
||||
%% Not really none proj: just myself, AP style
|
||||
machi_projection:update_checksum(
|
||||
P1#projection_v1{upi=[MyName],
|
||||
P#projection_v1{upi=[MyName],
|
||||
down=Down -- [MyName],
|
||||
dbg=[{hosed_list,AllHosed}]});
|
||||
CMode == cp_mode ->
|
||||
machi_projection:update_checksum(
|
||||
P1#projection_v1{dbg=[{hosed_list,AllHosed}]})
|
||||
P#projection_v1{dbg=[{hosed_list,AllHosed}]})
|
||||
end,
|
||||
react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP,
|
||||
P_current_calc, true, S);
|
||||
|
|
@ -1468,22 +1388,13 @@ react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP,
|
|||
%% we have a disagreement.
|
||||
not ordsets:is_disjoint(P_latest_s, Down_s)
|
||||
end,
|
||||
AmExcludedFromLatestAll_p =
|
||||
P_latest#projection_v1.epoch_number /= 0
|
||||
andalso
|
||||
(not lists:member(MyName, P_latest#projection_v1.all_members)),
|
||||
?REACT({a40, ?LINE,
|
||||
[{latest_author, P_latest#projection_v1.author_server},
|
||||
{am_excluded_from_latest_all_p, AmExcludedFromLatestAll_p},
|
||||
{author_is_down_p, LatestAuthorDownP},
|
||||
{rank_latest, Rank_latest},
|
||||
{rank_newprop, Rank_newprop}]}),
|
||||
|
||||
if
|
||||
AmExcludedFromLatestAll_p ->
|
||||
?REACT({a40, ?LINE, [{latest,machi_projection:make_summary(P_latest)}]}),
|
||||
react_to_env_A50(P_latest, [], S);
|
||||
|
||||
AmHosedP ->
|
||||
ExpectedUPI = if CMode == cp_mode -> [];
|
||||
CMode == ap_mode -> [MyName]
|
||||
|
|
@ -1649,10 +1560,12 @@ react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP,
|
|||
end,
|
||||
if GoTo50_p ->
|
||||
?REACT({a40, ?LINE, []}),
|
||||
%% io:format(user, "CONFIRM debug question line ~w\n", [?LINE]),
|
||||
FinalProps = [{throttle_seconds, 0}],
|
||||
react_to_env_A50(P_latest, FinalProps, S);
|
||||
true ->
|
||||
?REACT({a40, ?LINE, []}),
|
||||
io:format(user, "CONFIRM debug question line ~w\n", [?LINE]),
|
||||
react_to_env_C300(P_newprop, P_latest, S)
|
||||
end
|
||||
end.
|
||||
|
|
@ -1662,6 +1575,7 @@ react_to_env_A50(P_latest, FinalProps, #ch_mgr{proj=P_current}=S) ->
|
|||
?REACT({a50, ?LINE, [{current_epoch, P_current#projection_v1.epoch_number},
|
||||
{latest_epoch, P_latest#projection_v1.epoch_number},
|
||||
{final_props, FinalProps}]}),
|
||||
%% if S#ch_mgr.name == c -> io:format(user, "A50: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
|
||||
V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end,
|
||||
if V -> io:format(user, "A50: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
|
||||
{{no_change, FinalProps, P_current#projection_v1.epoch_number}, S}.
|
||||
|
|
@ -1915,7 +1829,7 @@ react_to_env_C100_inner(Author_latest, NotSanesDict0, _MyName,
|
|||
S2 = S#ch_mgr{not_sanes=NotSanesDict, sane_transitions=0},
|
||||
case orddict:fetch(Author_latest, NotSanesDict) of
|
||||
N when N > ?TOO_FREQUENT_BREAKER ->
|
||||
?V("\n\nYOYO ~w breaking the cycle insane-freq=~w by-author=~w of:\n current: ~w\n new : ~w\n", [_MyName, N, Author_latest, machi_projection:make_summary(S#ch_mgr.proj), machi_projection:make_summary(P_latest)]),
|
||||
%% ?V("\n\nYOYO ~w breaking the cycle of:\n current: ~w\n new : ~w\n", [_MyName, machi_projection:make_summary(S#ch_mgr.proj), machi_projection:make_summary(P_latest)]),
|
||||
?REACT({c100, ?LINE, [{not_sanes_author_count, N}]}),
|
||||
react_to_env_C103(P_newprop, P_latest, P_current_calc, S2);
|
||||
N ->
|
||||
|
|
@ -1936,14 +1850,12 @@ react_to_env_C103(#projection_v1{epoch_number=_Epoch_newprop} = _P_newprop,
|
|||
members_dict=MembersDict} = P_current,
|
||||
P_none0 = make_none_projection(Epoch_latest,
|
||||
MyName, All_list, Witness_list, MembersDict),
|
||||
ChainName = P_current#projection_v1.chain_name,
|
||||
P_none1 = P_none0#projection_v1{chain_name=ChainName,
|
||||
dbg=[{none_projection,true}]},
|
||||
P_none1 = P_none0#projection_v1{dbg=[{none_projection,true}]},
|
||||
P_none = machi_projection:update_checksum(P_none1),
|
||||
?REACT({c103, ?LINE,
|
||||
[{current_epoch, P_current#projection_v1.epoch_number},
|
||||
{none_projection_epoch, P_none#projection_v1.epoch_number}]}),
|
||||
io:format(user, "SET add_admin_down(~w) at ~w current_epoch ~w none_proj_epoch ~w =====================================\n", [MyName, time(), P_current#projection_v1.epoch_number, P_none#projection_v1.epoch_number]),
|
||||
io:format(user, "SET add_admin_down(~w) at ~w =====================================\n", [MyName, time()]),
|
||||
machi_fitness:add_admin_down(S#ch_mgr.fitness_svr, MyName, []),
|
||||
timer:sleep(5*1000),
|
||||
io:format(user, "SET delete_admin_down(~w) at ~w =====================================\n", [MyName, time()]),
|
||||
|
|
@ -1980,7 +1892,7 @@ react_to_env_C110(P_latest, #ch_mgr{name=MyName} = S) ->
|
|||
%% In contrast to the public projection store writes, Humming Consensus
|
||||
%% doesn't care about the status of writes to the public store: it's
|
||||
%% always relying only on successful reads of the public store.
|
||||
case {?FLU_PC:write_projection(MyStorePid, private, P_latest2,?TO*30+66),Goo} of
|
||||
case {?FLU_PC:write_projection(MyStorePid, private, P_latest2,?TO*30),Goo} of
|
||||
{ok, Goo} ->
|
||||
?REACT({c110, [{write, ok}]}),
|
||||
react_to_env_C111(P_latest, P_latest2, Extra1, MyStorePid, S);
|
||||
|
|
@ -2066,6 +1978,7 @@ react_to_env_C120(P_latest, FinalProps, #ch_mgr{proj_history=H,
|
|||
?REACT(c120),
|
||||
H2 = add_and_trunc_history(P_latest, H, ?MAX_HISTORY_LENGTH),
|
||||
|
||||
%% diversion_c120_verbose_goop(P_latest, S),
|
||||
?REACT({c120, [{latest, machi_projection:make_summary(P_latest)}]}),
|
||||
S2 = set_proj(S#ch_mgr{proj_history=H2,
|
||||
sane_transitions=Xtns + 1}, P_latest),
|
||||
|
|
@ -2073,21 +1986,20 @@ react_to_env_C120(P_latest, FinalProps, #ch_mgr{proj_history=H,
|
|||
false ->
|
||||
S2;
|
||||
{{_ConfEpoch, _ConfCSum}, ConfTime} ->
|
||||
P_latestEpoch = P_latest#projection_v1.epoch_number,
|
||||
io:format(user, "\nCONFIRM debug C120 ~w was annotated ~w\n", [S#ch_mgr.name, P_latestEpoch]),
|
||||
io:format(user, "\nCONFIRM debug C120 ~w was annotated ~w\n", [S#ch_mgr.name, P_latest#projection_v1.epoch_number]),
|
||||
S2#ch_mgr{proj_unanimous=ConfTime}
|
||||
end,
|
||||
V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end,
|
||||
if V -> io:format("C120: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
|
||||
{{now_using, FinalProps, P_latest#projection_v1.epoch_number}, S3}.
|
||||
|
||||
add_and_trunc_history(#projection_v1{epoch_number=0}, H, _MaxLength) ->
|
||||
H;
|
||||
add_and_trunc_history(#projection_v1{} = P_latest, H, MaxLength) ->
|
||||
add_and_trunc_history(P_latest, H, MaxLength) ->
|
||||
Latest_U_R = {P_latest#projection_v1.upi, P_latest#projection_v1.repairing},
|
||||
add_and_trunc_history(Latest_U_R, H, MaxLength);
|
||||
add_and_trunc_history(Item, H, MaxLength) ->
|
||||
H2 = queue:in(Item, H),
|
||||
H2 = if P_latest#projection_v1.epoch_number > 0 ->
|
||||
queue:in(Latest_U_R, H);
|
||||
true ->
|
||||
H
|
||||
end,
|
||||
case queue:len(H2) of
|
||||
X when X > MaxLength ->
|
||||
{_V, Hxx} = queue:out(H2),
|
||||
|
|
@ -2100,10 +2012,11 @@ react_to_env_C200(Retries, P_latest, S) ->
|
|||
?REACT(c200),
|
||||
try
|
||||
AuthorProxyPid = proxy_pid(P_latest#projection_v1.author_server, S),
|
||||
%% This is just advisory, we don't need a sync reply.
|
||||
?FLU_PC:kick_projection_reaction(AuthorProxyPid, [], 100)
|
||||
?FLU_PC:kick_projection_reaction(AuthorProxyPid, [])
|
||||
catch _Type:_Err ->
|
||||
ok
|
||||
%% ?V("TODO: tell_author_yo is broken: ~p ~p\n",
|
||||
%% [_Type, _Err]),
|
||||
ok
|
||||
end,
|
||||
react_to_env_C210(Retries, S).
|
||||
|
||||
|
|
@ -2293,7 +2206,6 @@ projection_transition_is_sane_except_si_epoch(
|
|||
creation_time=CreationTime1,
|
||||
mode=CMode1,
|
||||
author_server=AuthorServer1,
|
||||
chain_name=ChainName1,
|
||||
all_members=All_list1,
|
||||
witnesses=Witness_list1,
|
||||
down=Down_list1,
|
||||
|
|
@ -2305,7 +2217,6 @@ projection_transition_is_sane_except_si_epoch(
|
|||
creation_time=CreationTime2,
|
||||
mode=CMode2,
|
||||
author_server=AuthorServer2,
|
||||
chain_name=ChainName2,
|
||||
all_members=All_list2,
|
||||
witnesses=Witness_list2,
|
||||
down=Down_list2,
|
||||
|
|
@ -2326,8 +2237,7 @@ projection_transition_is_sane_except_si_epoch(
|
|||
true = is_binary(CSum1) andalso is_binary(CSum2),
|
||||
{_,_,_} = CreationTime1,
|
||||
{_,_,_} = CreationTime2,
|
||||
true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2),
|
||||
true = is_atom(ChainName1) andalso is_atom(ChainName2),
|
||||
true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2), % todo type may change?
|
||||
true = is_list(All_list1) andalso is_list(All_list2),
|
||||
true = is_list(Witness_list1) andalso is_list(Witness_list2),
|
||||
true = is_list(Down_list1) andalso is_list(Down_list2),
|
||||
|
|
@ -2339,9 +2249,6 @@ projection_transition_is_sane_except_si_epoch(
|
|||
%% projection_transition_is_sane_with_si_epoch().
|
||||
true = Epoch2 >= Epoch1,
|
||||
|
||||
%% Don't change chain names in the middle of the stream.
|
||||
true = (ChainName1 == ChainName2),
|
||||
|
||||
%% No duplicates
|
||||
true = lists:sort(Witness_list2) == lists:usort(Witness_list2),
|
||||
true = lists:sort(Down_list2) == lists:usort(Down_list2),
|
||||
|
|
@ -2349,7 +2256,7 @@ projection_transition_is_sane_except_si_epoch(
|
|||
true = lists:sort(Repairing_list2) == lists:usort(Repairing_list2),
|
||||
|
||||
%% Disjoint-ness
|
||||
%% %% %% %% %% %% %% %% All_list1 = All_list2, % todo will probably change
|
||||
All_list1 = All_list2, % todo will probably change
|
||||
%% true = lists:sort(All_list2) == lists:sort(Down_list2 ++ UPI_list2 ++
|
||||
%% Repairing_list2),
|
||||
[] = [X || X <- Witness_list2, not lists:member(X, All_list2)],
|
||||
|
|
@ -2454,7 +2361,8 @@ poll_private_proj_is_upi_unanimous_sleep(Count, #ch_mgr{runenv=RunEnv}=S) ->
|
|||
S2
|
||||
end.
|
||||
|
||||
poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current} = S) ->
|
||||
poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current,
|
||||
opts=MgrOpts} = S) ->
|
||||
UPI = P_current#projection_v1.upi,
|
||||
EpochID = machi_projection:make_epoch_id(P_current),
|
||||
{Rs, S2} = read_latest_projection_call_only2(private, UPI, S),
|
||||
|
|
@ -2487,30 +2395,33 @@ poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current} = S) ->
|
|||
Annotation = make_annotation(EpochID, Now),
|
||||
NewDbg2 = [Annotation|P_currentFull#projection_v1.dbg2],
|
||||
NewProj = P_currentFull#projection_v1{dbg2=NewDbg2},
|
||||
ProjStore = get_projection_store_pid_or_regname(S),
|
||||
ProjStore = case get_projection_store_regname(MgrOpts) of
|
||||
undefined ->
|
||||
machi_flu_psup:make_proj_supname(MyName);
|
||||
PStr ->
|
||||
PStr
|
||||
end,
|
||||
#projection_v1{epoch_number=_EpochRep,
|
||||
epoch_csum= <<_CSumRep:4/binary,_/binary>>,
|
||||
author_server=AuthRep,
|
||||
upi=_UPIRep,
|
||||
repairing=_RepairingRep} = NewProj,
|
||||
ok = machi_projection_store:write(ProjStore, private, NewProj),
|
||||
case proplists:get_value(private_write_verbose_confirm, S#ch_mgr.opts) of
|
||||
case proplists:get_value(private_write_verbose, S#ch_mgr.opts) of
|
||||
true ->
|
||||
error_logger:info_msg("CONFIRM epoch ~w ~w upi ~w rep ~w auth ~w by ~w\n", [_EpochRep, _CSumRep, _UPIRep, _RepairingRep, AuthRep, MyName]);
|
||||
io:format(user, "\n~s CONFIRM epoch ~w ~w upi ~w rep ~w by ~w\n", [machi_util:pretty_time(), _EpochRep, _CSumRep, _UPIRep, _RepairingRep, MyName]);
|
||||
_ ->
|
||||
ok
|
||||
end,
|
||||
%% Unwedge our FLU.
|
||||
{ok, NotifyPid} = machi_projection_store:get_wedge_notify_pid(ProjStore),
|
||||
_ = machi_flu1:update_wedge_state(NotifyPid, false, EpochID),
|
||||
#ch_mgr{proj_history=H} = S2,
|
||||
H2 = add_and_trunc_history({confirm, Epoch}, H,
|
||||
?MAX_HISTORY_LENGTH),
|
||||
S2#ch_mgr{proj_unanimous=Now, proj_history=H2};
|
||||
S2#ch_mgr{proj_unanimous=Now};
|
||||
_ ->
|
||||
S2
|
||||
end;
|
||||
_Else ->
|
||||
%% io:format(user, "poll by ~w: want ~W got ~W\n",
|
||||
%% [MyName, EpochID, 6, _Else, 8]),
|
||||
S2
|
||||
end.
|
||||
|
||||
|
|
@ -2546,14 +2457,6 @@ gobble_calls(StaticCall) ->
|
|||
ok
|
||||
end.
|
||||
|
||||
gobble_ticks() ->
|
||||
receive
|
||||
tick_check_environment ->
|
||||
gobble_ticks()
|
||||
after 0 ->
|
||||
ok
|
||||
end.
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
perhaps_start_repair(#ch_mgr{name=MyName,
|
||||
|
|
@ -2569,13 +2472,12 @@ perhaps_start_repair(#ch_mgr{name=MyName,
|
|||
%% RepairOpts = [{repair_mode, check}, verbose],
|
||||
RepairFun = fun() -> do_repair(S, RepairOpts, CMode) end,
|
||||
LastUPI = lists:last(UPI),
|
||||
StabilityTime = application:get_env(machi, stability_time, ?REPAIR_START_STABILITY_TIME),
|
||||
IgnoreStabilityTime_p = proplists:get_value(ignore_stability_time,
|
||||
S#ch_mgr.opts, false),
|
||||
case timer:now_diff(os:timestamp(), Start) div 1000000 of
|
||||
N when MyName == LastUPI andalso
|
||||
(IgnoreStabilityTime_p orelse
|
||||
N >= StabilityTime) ->
|
||||
N >= ?REPAIR_START_STABILITY_TIME) ->
|
||||
{WorkerPid, _Ref} = spawn_monitor(RepairFun),
|
||||
S#ch_mgr{repair_worker=WorkerPid,
|
||||
repair_start=os:timestamp(),
|
||||
|
|
@ -2616,8 +2518,8 @@ do_repair(#ch_mgr{name=MyName,
|
|||
T1 = os:timestamp(),
|
||||
RepairId = proplists:get_value(repair_id, Opts, id1),
|
||||
error_logger:info_msg(
|
||||
"Repair ~w start: tail ~p of ~p -> ~p, ~p\n",
|
||||
[RepairId, MyName, UPI0, Repairing, RepairMode]),
|
||||
"Repair start: tail ~p of ~p -> ~p, ~p ID ~w\n",
|
||||
[MyName, UPI0, Repairing, RepairMode, RepairId]),
|
||||
|
||||
UPI = UPI0 -- Witness_list,
|
||||
Res = machi_chain_repair:repair(RepairMode, MyName, Repairing, UPI,
|
||||
|
|
@ -2630,9 +2532,10 @@ do_repair(#ch_mgr{name=MyName,
|
|||
end,
|
||||
Stats = [{K, ets:lookup_element(ETS, K, 2)} || K <- ETS_T_Keys],
|
||||
error_logger:info_msg(
|
||||
"Repair ~w ~s: tail ~p of ~p finished ~p: "
|
||||
"~p Stats: ~p\n",
|
||||
[RepairId, Summary, MyName, UPI0, RepairMode, Res, Stats]),
|
||||
"Repair ~s: tail ~p of ~p finished ~p repair ID ~w: "
|
||||
"~p\nStats ~p\n",
|
||||
[Summary, MyName, UPI0, RepairMode, RepairId,
|
||||
Res, Stats]),
|
||||
ets:delete(ETS),
|
||||
exit({repair_final_status, Res});
|
||||
_ ->
|
||||
|
|
@ -2869,7 +2772,6 @@ full_majority_size(L) when is_list(L) ->
|
|||
full_majority_size(length(L)).
|
||||
|
||||
make_zerf(#projection_v1{epoch_number=OldEpochNum,
|
||||
chain_name=ChainName,
|
||||
all_members=AllMembers,
|
||||
members_dict=MembersDict,
|
||||
witnesses=OldWitness_list
|
||||
|
|
@ -2892,8 +2794,7 @@ make_zerf(#projection_v1{epoch_number=OldEpochNum,
|
|||
MyName, AllMembers, OldWitness_list,
|
||||
MembersDict),
|
||||
machi_projection:update_checksum(
|
||||
P#projection_v1{chain_name=ChainName,
|
||||
mode=cp_mode,
|
||||
P#projection_v1{mode=cp_mode,
|
||||
dbg2=[zerf_none,{up,Up},{maj,MajoritySize}]});
|
||||
true ->
|
||||
make_zerf2(OldEpochNum, Up, MajoritySize, MyName,
|
||||
|
|
@ -2908,6 +2809,7 @@ make_zerf2(OldEpochNum, Up, MajoritySize, MyName, AllMembers, OldWitness_list,
|
|||
Proj2 = Proj#projection_v1{dbg2=[{make_zerf,Epoch},
|
||||
{yyy_hack, get(yyy_hack)},
|
||||
{up,Up},{maj,MajoritySize}]},
|
||||
%% io:format(user, "ZERF ~w\n",[machi_projection:make_summary(Proj2)]),
|
||||
Proj2
|
||||
catch
|
||||
throw:{zerf,no_common} ->
|
||||
|
|
@ -2984,36 +2886,41 @@ zerf_find_last_annotated(FLU, MajoritySize, S) ->
|
|||
[] % lists:flatten() will destroy
|
||||
end.
|
||||
|
||||
perhaps_verbose_c111(P_latest2, #ch_mgr{name=MyName, opts=Opts}=S) ->
|
||||
PrivWriteVerb = proplists:get_value(private_write_verbose, Opts, false),
|
||||
PrivWriteVerbCONFIRM = proplists:get_value(private_write_verbose_confirm, Opts, false),
|
||||
if PrivWriteVerb orelse PrivWriteVerbCONFIRM ->
|
||||
perhaps_verbose_c111(P_latest2, S) ->
|
||||
case proplists:get_value(private_write_verbose, S#ch_mgr.opts) of
|
||||
true ->
|
||||
Dbg2X = lists:keydelete(react, 1,
|
||||
P_latest2#projection_v1.dbg2) ++
|
||||
[{is_annotated,is_annotated(P_latest2)}],
|
||||
P_latest2x = P_latest2#projection_v1{dbg2=Dbg2X}, % limit verbose len.
|
||||
Last2 = get(last_verbose),
|
||||
Summ2 = machi_projection:make_summary(P_latest2x),
|
||||
if PrivWriteVerb, Summ2 /= Last2 ->
|
||||
put(last_verbose, Summ2),
|
||||
error_logger:info_msg("~p uses plain: ~w \n",
|
||||
[MyName, Summ2]);
|
||||
true ->
|
||||
ok
|
||||
end,
|
||||
if PrivWriteVerbCONFIRM,
|
||||
P_latest2#projection_v1.upi == [],
|
||||
if P_latest2#projection_v1.upi == [],
|
||||
(S#ch_mgr.proj)#projection_v1.upi /= [] ->
|
||||
<<CSumRep:4/binary,_/binary>> =
|
||||
P_latest2#projection_v1.epoch_csum,
|
||||
error_logger:info_msg("CONFIRM epoch ~w ~w upi ~w rep ~w auth ~w by ~w\n", [(S#ch_mgr.proj)#projection_v1.epoch_number, CSumRep, P_latest2#projection_v1.upi, P_latest2#projection_v1.repairing, P_latest2#projection_v1.author_server, S#ch_mgr.name]);
|
||||
io:format(user, "\n~s CONFIRM epoch ~w ~w upi ~w rep ~w by ~w\n", [machi_util:pretty_time(), (S#ch_mgr.proj)#projection_v1.epoch_number, CSumRep, P_latest2#projection_v1.upi, P_latest2#projection_v1.repairing, S#ch_mgr.name]);
|
||||
true ->
|
||||
ok
|
||||
end,
|
||||
case proplists:get_value(private_write_verbose,
|
||||
S#ch_mgr.opts) of
|
||||
true when Summ2 /= Last2 ->
|
||||
put(last_verbose, Summ2),
|
||||
?V("\n~s ~p uses plain: ~w \n",
|
||||
[machi_util:pretty_time(), S#ch_mgr.name, Summ2]);
|
||||
_ ->
|
||||
ok
|
||||
end;
|
||||
true ->
|
||||
_ ->
|
||||
ok
|
||||
end.
|
||||
|
||||
calc_consistency_mode(_Witness_list = []) ->
|
||||
ap_mode;
|
||||
calc_consistency_mode(_Witness_list) ->
|
||||
cp_mode.
|
||||
|
||||
set_proj(S, Proj) ->
|
||||
S#ch_mgr{proj=Proj, proj_unanimous=false}.
|
||||
|
||||
|
|
@ -3046,10 +2953,3 @@ get_unfit_list(FitnessServer) ->
|
|||
[]
|
||||
end.
|
||||
|
||||
get_projection_store_pid_or_regname(#ch_mgr{name=MyName, opts=MgrOpts}) ->
|
||||
case get_projection_store_regname(MgrOpts) of
|
||||
undefined ->
|
||||
machi_flu_psup:make_proj_supname(MyName);
|
||||
PStr ->
|
||||
PStr
|
||||
end.
|
||||
|
|
|
|||
|
|
@ -103,8 +103,7 @@ repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
|
|||
Add = fun(Name, Pid) -> put(proxies_dict, orddict:store(Name, Pid, get(proxies_dict))) end,
|
||||
OurFLUs = lists:usort([Src] ++ Repairing ++ UPI), % AP assumption!
|
||||
RepairMode = proplists:get_value(repair_mode, Opts, repair),
|
||||
Verb = proplists:get_value(verbose, Opts, false),
|
||||
RepairId = proplists:get_value(repair_id, Opts, id1),
|
||||
Verb = proplists:get_value(verbose, Opts, true),
|
||||
Res = try
|
||||
_ = [begin
|
||||
{ok, Proxy} = machi_proxy_flu1_client:start_link(P),
|
||||
|
|
@ -117,39 +116,31 @@ repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
|
|||
get_file_lists(Proxy, FLU, Dict)
|
||||
end, D, ProxiesDict),
|
||||
MissingFileSummary = make_missing_file_summary(D2, OurFLUs),
|
||||
%% ?VERB("~w MissingFileSummary ~p\n",[RepairId,MissingFileSummary]),
|
||||
lager:info("Repair ~w MissingFileSummary ~p\n",
|
||||
[RepairId, MissingFileSummary]),
|
||||
?VERB("MissingFileSummary ~p\n", [MissingFileSummary]),
|
||||
|
||||
[ets:insert(ETS, {{directive_bytes, FLU}, 0}) || FLU <- OurFLUs],
|
||||
%% Repair files from perspective of Src, i.e. tail(UPI).
|
||||
SrcProxy = orddict:fetch(Src, ProxiesDict),
|
||||
{ok, EpochID} = machi_proxy_flu1_client:get_epoch_id(
|
||||
SrcProxy, ?SHORT_TIMEOUT),
|
||||
%% ?VERB("Make repair directives: "),
|
||||
?VERB("Make repair directives: "),
|
||||
Ds =
|
||||
[{File, make_repair_directives(
|
||||
ConsistencyMode, RepairMode, File, Size, EpochID,
|
||||
Verb,
|
||||
Src, OurFLUs, ProxiesDict, ETS)} ||
|
||||
{File, {Size, _MissingList}} <- MissingFileSummary],
|
||||
%% ?VERB(" done\n"),
|
||||
lager:info("Repair ~w repair directives finished\n", [RepairId]),
|
||||
?VERB(" done\n"),
|
||||
[begin
|
||||
[{_, Bytes}] = ets:lookup(ETS, {directive_bytes, FLU}),
|
||||
%% ?VERB("Out-of-sync data for FLU ~p: ~s MBytes\n",
|
||||
%% [FLU, mbytes(Bytes)]),
|
||||
lager:info("Repair ~w "
|
||||
"Out-of-sync data for FLU ~p: ~s MBytes\n",
|
||||
[RepairId, FLU, mbytes(Bytes)]),
|
||||
ok
|
||||
?VERB("Out-of-sync data for FLU ~p: ~s MBytes\n",
|
||||
[FLU, mbytes(Bytes)])
|
||||
end || FLU <- OurFLUs],
|
||||
|
||||
%% ?VERB("Execute repair directives: "),
|
||||
?VERB("Execute repair directives: "),
|
||||
ok = execute_repair_directives(ConsistencyMode, Ds, Src, EpochID,
|
||||
Verb, OurFLUs, ProxiesDict, ETS),
|
||||
%% ?VERB(" done\n"),
|
||||
lager:info("Repair ~w repair directives finished\n", [RepairId]),
|
||||
?VERB(" done\n"),
|
||||
ok
|
||||
catch
|
||||
What:Why ->
|
||||
|
|
@ -207,7 +198,7 @@ make_repair_compare_fun(SrcFLU) ->
|
|||
T_a =< T_b
|
||||
end.
|
||||
|
||||
make_repair_directives(ConsistencyMode, RepairMode, File, Size, _EpochID,
|
||||
make_repair_directives(ConsistencyMode, RepairMode, File, Size, EpochID,
|
||||
Verb, Src, FLUs0, ProxiesDict, ETS) ->
|
||||
true = (Size < ?MAX_OFFSET),
|
||||
FLUs = lists:usort(FLUs0),
|
||||
|
|
@ -216,9 +207,11 @@ make_repair_directives(ConsistencyMode, RepairMode, File, Size, _EpochID,
|
|||
Proxy = orddict:fetch(FLU, ProxiesDict),
|
||||
OffSzCs =
|
||||
case machi_proxy_flu1_client:checksum_list(
|
||||
Proxy, File, ?LONG_TIMEOUT) of
|
||||
Proxy, EpochID, File, ?LONG_TIMEOUT) of
|
||||
{ok, InfoBin} ->
|
||||
machi_csum_table:split_checksum_list_blob_decode(InfoBin);
|
||||
{Info, _} =
|
||||
machi_csum_table:split_checksum_list_blob_decode(InfoBin),
|
||||
Info;
|
||||
{error, no_such_file} ->
|
||||
[]
|
||||
end,
|
||||
|
|
@ -236,6 +229,7 @@ make_repair_directives(ConsistencyMode, RepairMode, File, Size, _EpochID,
|
|||
|
||||
make_repair_directives2(C2, ConsistencyMode, RepairMode,
|
||||
File, Verb, Src, FLUs, ProxiesDict, ETS) ->
|
||||
?VERB("."),
|
||||
make_repair_directives3(C2, ConsistencyMode, RepairMode,
|
||||
File, Verb, Src, FLUs, ProxiesDict, ETS, []).
|
||||
|
||||
|
|
@ -265,18 +259,7 @@ make_repair_directives3([{Offset, Size, CSum, _FLU}=A|Rest0],
|
|||
%% byte range from all FLUs
|
||||
%% 3b. Log big warning about data loss.
|
||||
%% 4. Log any other checksum discrepencies as they are found.
|
||||
QQ = [begin
|
||||
Pxy = orddict:fetch(FLU, ProxiesDict),
|
||||
{ok, EpochID} = machi_proxy_flu1_client:get_epoch_id(
|
||||
Pxy, ?SHORT_TIMEOUT),
|
||||
NSInfo = undefined,
|
||||
XX = machi_proxy_flu1_client:read_chunk(
|
||||
Pxy, NSInfo, EpochID, File, Offset, Size, undefined,
|
||||
?SHORT_TIMEOUT),
|
||||
{FLU, XX}
|
||||
end || {__Offset, __Size, __CSum, FLU} <- As],
|
||||
|
||||
exit({todo_repair_sanity_check, ?LINE, File, Offset, {as,As}, {qq,QQ}})
|
||||
exit({todo_repair_sanity_check, ?LINE, File, Offset, As})
|
||||
end,
|
||||
%% List construction guarantees us that there's at least one ?MAX_OFFSET
|
||||
%% item remains. Sort order + our "taking" of all exact Offset+Size
|
||||
|
|
@ -329,25 +312,23 @@ execute_repair_directives(ap_mode=_ConsistencyMode, Ds, _Src, EpochID, Verb,
|
|||
{ProxiesDict, EpochID, Verb, ETS}, Ds),
|
||||
ok.
|
||||
|
||||
execute_repair_directive({File, Cmds}, {ProxiesDict, EpochID, _Verb, ETS}=Acc) ->
|
||||
execute_repair_directive({File, Cmds}, {ProxiesDict, EpochID, Verb, ETS}=Acc) ->
|
||||
EtsKeys = [{in_files, t_in_files}, {in_chunks, t_in_chunks},
|
||||
{in_bytes, t_in_bytes}, {out_files, t_out_files},
|
||||
{out_chunks, t_out_chunks}, {out_bytes, t_out_bytes}],
|
||||
[ets:insert(ETS, {L_K, 0}) || {L_K, _T_K} <- EtsKeys],
|
||||
F = fun({copy, {Offset, Size, TaggedCSum, MySrc}, MyDsts}, Acc2) ->
|
||||
SrcP = orddict:fetch(MySrc, ProxiesDict),
|
||||
%% case ets:lookup_element(ETS, in_chunks, 2) rem 100 of
|
||||
%% 0 -> ?VERB(".2", []);
|
||||
%% _ -> ok
|
||||
%% end,
|
||||
case ets:lookup_element(ETS, in_chunks, 2) rem 100 of
|
||||
0 -> ?VERB(".", []);
|
||||
_ -> ok
|
||||
end,
|
||||
_T1 = os:timestamp(),
|
||||
%% TODO: support case multiple written or trimmed chunks returned
|
||||
NSInfo = undefined,
|
||||
{ok, {[{_, Offset, Chunk, _ReadCSum}|OtherChunks], []=_TrimmedList}} =
|
||||
{ok, {[{_, Offset, Chunk, _}], _}} =
|
||||
machi_proxy_flu1_client:read_chunk(
|
||||
SrcP, NSInfo, EpochID, File, Offset, Size, undefined,
|
||||
SrcP, EpochID, File, Offset, Size, [],
|
||||
?SHORT_TIMEOUT),
|
||||
[] = OtherChunks,
|
||||
_T2 = os:timestamp(),
|
||||
<<_Tag:1/binary, CSum/binary>> = TaggedCSum,
|
||||
case machi_util:checksum_chunk(Chunk) of
|
||||
|
|
@ -356,7 +337,7 @@ execute_repair_directive({File, Cmds}, {ProxiesDict, EpochID, _Verb, ETS}=Acc) -
|
|||
DstP = orddict:fetch(DstFLU, ProxiesDict),
|
||||
_T3 = os:timestamp(),
|
||||
ok = machi_proxy_flu1_client:write_chunk(
|
||||
DstP, NSInfo, EpochID, File, Offset, Chunk, TaggedCSum,
|
||||
DstP, EpochID, File, Offset, Chunk,
|
||||
?SHORT_TIMEOUT),
|
||||
_T4 = os:timestamp()
|
||||
end || DstFLU <- MyDsts],
|
||||
|
|
|
|||
|
|
@ -85,8 +85,8 @@ fitness(FluName) ->
|
|||
|
||||
-spec flu1(atom()) -> [{atom(), term()}].
|
||||
flu1(FluName) ->
|
||||
State = machi_flu1_append_server:current_state(FluName),
|
||||
machi_flu1_append_server:format_state(State).
|
||||
State = machi_flu1:current_state(FluName),
|
||||
machi_flu1:format_state(State).
|
||||
|
||||
%% Internal functions
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,53 +1,35 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_csum_table).
|
||||
|
||||
-export([open/2,
|
||||
find/3,
|
||||
write/6, write/4, trim/5,
|
||||
write/6, write/4, trim/5, trim/3,
|
||||
find_leftneighbor/2, find_rightneighbor/2,
|
||||
all_trimmed/3, any_trimmed/3,
|
||||
all_trimmed/2,
|
||||
sync/1,
|
||||
calc_unwritten_bytes/1,
|
||||
split_checksum_list_blob_decode/1,
|
||||
all/1,
|
||||
close/1, delete/1,
|
||||
foldl_chunks/3]).
|
||||
|
||||
-export([encode_csum_file_entry/3, encode_csum_file_entry_bin/3,
|
||||
decode_csum_file_entry/1]).
|
||||
|
||||
-include("machi.hrl").
|
||||
|
||||
-ifdef(TEST).
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-export([all/1]).
|
||||
-endif.
|
||||
|
||||
-record(machi_csum_table,
|
||||
{file :: string(),
|
||||
table :: eleveldb:db_ref()}).
|
||||
fd :: file:io_device(),
|
||||
table :: ets:tid()}).
|
||||
|
||||
-type table() :: #machi_csum_table{}.
|
||||
-type byte_sequence() :: { Offset :: non_neg_integer(),
|
||||
Size :: pos_integer()|infinity }.
|
||||
-type chunk() :: {Offset :: machi_dt:file_offset(),
|
||||
Size :: machi_dt:chunk_size(),
|
||||
machi_dt:chunk_csum() | trimmed | none}.
|
||||
|
||||
-export_type([table/0]).
|
||||
|
||||
|
|
@ -55,110 +37,100 @@
|
|||
{ok, table()} | {error, file:posix()}.
|
||||
|
||||
open(CSumFilename, _Opts) ->
|
||||
LevelDBOptions = [{create_if_missing, true},
|
||||
%% Keep this table small so as not to interfere
|
||||
%% operating system's file cache, which is for
|
||||
%% Machi's main read efficiency
|
||||
{total_leveldb_mem_percent, 10}],
|
||||
{ok, T} = eleveldb:open(CSumFilename, LevelDBOptions),
|
||||
%% Dummy entry for reserved headers
|
||||
ok = eleveldb:put(T,
|
||||
sext:encode({0, ?MINIMUM_OFFSET}),
|
||||
sext:encode(?CSUM_TAG_NONE_ATOM),
|
||||
[{sync, true}]),
|
||||
T = ets:new(?MODULE, [private, ordered_set]),
|
||||
CSum = machi_util:make_tagged_csum(none),
|
||||
%% Dummy entry for headers
|
||||
true = ets:insert_new(T, {0, ?MINIMUM_OFFSET, CSum}),
|
||||
C0 = #machi_csum_table{
|
||||
file=CSumFilename,
|
||||
table=T},
|
||||
{ok, C0}.
|
||||
case file:read_file(CSumFilename) of
|
||||
{ok, Bin} ->
|
||||
List = case split_checksum_list_blob_decode(Bin) of
|
||||
{List0, <<>>} ->
|
||||
List0;
|
||||
{List0, _Junk} ->
|
||||
%% Partially written, needs repair TODO
|
||||
List0
|
||||
end,
|
||||
%% assuming all entries are strictly ordered by offset,
|
||||
%% trim command should always come after checksum entry.
|
||||
%% *if* by any chance that order cound not be kept, we
|
||||
%% still can do ordering check and monotonic merge here.
|
||||
%% TODO: make some random injection tests?
|
||||
[begin %% Replay all file contents, Same logic as in write/6
|
||||
Chunks = find(C0, Offset, Size),
|
||||
lists:foreach(fun({O, _, _}) ->
|
||||
ets:delete(T, O)
|
||||
end, Chunks),
|
||||
true = ets:insert(T, {Offset, Size, CsumOrTrimmed})
|
||||
end
|
||||
|| {Offset, Size, CsumOrTrimmed} <- List],
|
||||
ok;
|
||||
{error, enoent} ->
|
||||
ok;
|
||||
Error ->
|
||||
throw(Error)
|
||||
end,
|
||||
{ok, Fd} = file:open(CSumFilename, [raw, binary, append]),
|
||||
{ok, C0#machi_csum_table{fd=Fd}}.
|
||||
|
||||
-spec split_checksum_list_blob_decode(binary())-> [chunk()].
|
||||
split_checksum_list_blob_decode(Bin) ->
|
||||
erlang:binary_to_term(Bin).
|
||||
|
||||
|
||||
-define(has_overlap(LeftOffset, LeftSize, RightOffset, RightSize),
|
||||
((LeftOffset - (RightOffset+RightSize)) * (LeftOffset+LeftSize - RightOffset) < 0)).
|
||||
|
||||
-spec find(table(), machi_dt:file_offset(), machi_dt:chunk_size())
|
||||
-> [chunk()].
|
||||
-spec find(table(), machi_dt:file_offset(), machi_dt:file_size()) ->
|
||||
list({machi_dt:file_offset(),
|
||||
machi_dt:file_size(),
|
||||
machi_dt:chunk_csum()|trimmed}).
|
||||
find(#machi_csum_table{table=T}, Offset, Size) ->
|
||||
{ok, I} = eleveldb:iterator(T, [], keys_only),
|
||||
EndKey = sext:encode({Offset+Size, 0}),
|
||||
StartKey = sext:encode({Offset, Size}),
|
||||
{ok, FirstKey} = case eleveldb:iterator_move(I, StartKey) of
|
||||
{error, invalid_iterator} ->
|
||||
try
|
||||
%% Assume that the invalid_iterator is because
|
||||
%% we tried to move to the end via StartKey.
|
||||
%% Instead, move there directly.
|
||||
{ok, _} = eleveldb:iterator_move(I, last),
|
||||
{ok, _} = eleveldb:iterator_move(I, prev)
|
||||
catch
|
||||
_:_ ->
|
||||
{ok, _} = eleveldb:iterator_move(I, first)
|
||||
end;
|
||||
{ok, _} = R0 ->
|
||||
case eleveldb:iterator_move(I, prev) of
|
||||
{error, invalid_iterator} ->
|
||||
R0;
|
||||
{ok, _} = R1 ->
|
||||
R1
|
||||
end
|
||||
end,
|
||||
_ = eleveldb:iterator_close(I),
|
||||
FoldFun = fun({K, V}, Acc) ->
|
||||
{TargetOffset, TargetSize} = sext:decode(K),
|
||||
case ?has_overlap(TargetOffset, TargetSize, Offset, Size) of
|
||||
true ->
|
||||
[{TargetOffset, TargetSize, sext:decode(V)}|Acc];
|
||||
false ->
|
||||
Acc
|
||||
end;
|
||||
(_K, Acc) ->
|
||||
lager:error("~p wrong option", [_K]),
|
||||
Acc
|
||||
end,
|
||||
lists:reverse(eleveldb_fold(T, FirstKey, EndKey, FoldFun, [])).
|
||||
ets:select(T, [{{'$1', '$2', '$3'},
|
||||
[inclusion_match_spec(Offset, Size)],
|
||||
['$_']}]).
|
||||
|
||||
%% @doc Updates all chunk info, by deleting existing entries if exists
|
||||
%% and putting new chunk info
|
||||
-spec write(table(),
|
||||
machi_dt:file_offset(), machi_dt:chunk_size(),
|
||||
machi_dt:chunk_csum()|'none'|'trimmed',
|
||||
undefined|chunk(), undefined|chunk()) ->
|
||||
ok | {error, term()}.
|
||||
write(#machi_csum_table{table=T} = CsumT, Offset, Size, CSum,
|
||||
-ifdef(TEST).
|
||||
all(#machi_csum_table{table=T}) ->
|
||||
ets:tab2list(T).
|
||||
-endif.
|
||||
|
||||
write(#machi_csum_table{fd=Fd, table=T} = CsumT,
|
||||
Offset, Size, CSum,
|
||||
LeftUpdate, RightUpdate) ->
|
||||
PutOps =
|
||||
[{put,
|
||||
sext:encode({Offset, Size}),
|
||||
sext:encode(CSum)}]
|
||||
++ case LeftUpdate of
|
||||
{LO, LS, LCsum} when LO + LS =:= Offset ->
|
||||
[{put,
|
||||
sext:encode({LO, LS}),
|
||||
sext:encode(LCsum)}];
|
||||
undefined ->
|
||||
[]
|
||||
end
|
||||
++ case RightUpdate of
|
||||
{RO, RS, RCsum} when RO =:= Offset + Size ->
|
||||
[{put,
|
||||
sext:encode({RO, RS}),
|
||||
sext:encode(RCsum)}];
|
||||
undefined ->
|
||||
[]
|
||||
end,
|
||||
Chunks = find(CsumT, Offset, Size),
|
||||
DeleteOps = lists:map(fun({O, L, _}) ->
|
||||
{delete, sext:encode({O, L})}
|
||||
Binary =
|
||||
[encode_csum_file_entry_bin(Offset, Size, CSum),
|
||||
case LeftUpdate of
|
||||
{LO, LS, LCsum} when LO + LS =:= Offset ->
|
||||
encode_csum_file_entry_bin(LO, LS, LCsum);
|
||||
undefined ->
|
||||
<<>>
|
||||
end,
|
||||
case RightUpdate of
|
||||
{RO, RS, RCsum} when RO =:= Offset + Size ->
|
||||
encode_csum_file_entry_bin(RO, RS, RCsum);
|
||||
undefined ->
|
||||
<<>>
|
||||
end],
|
||||
case file:write(Fd, Binary) of
|
||||
ok ->
|
||||
Chunks = find(CsumT, Offset, Size),
|
||||
lists:foreach(fun({O, _, _}) ->
|
||||
ets:delete(T, O)
|
||||
end, Chunks),
|
||||
%% io:format(user, "PutOps: ~P\n", [PutOps, 20]),
|
||||
%% io:format(user, "DelOps: ~P\n", [DeleteOps, 20]),
|
||||
eleveldb:write(T, DeleteOps ++ PutOps, [{sync, true}]).
|
||||
case LeftUpdate of
|
||||
{LO1, LS1, _} when LO1 + LS1 =:= Offset ->
|
||||
ets:insert(T, LeftUpdate);
|
||||
undefined -> noop
|
||||
end,
|
||||
case RightUpdate of
|
||||
{RO1, _, _} when RO1 =:= Offset + Size ->
|
||||
ets:insert(T, RightUpdate);
|
||||
undefined -> noop
|
||||
end,
|
||||
true = ets:insert(T, {Offset, Size, CSum}),
|
||||
ok;
|
||||
Error ->
|
||||
Error
|
||||
end.
|
||||
|
||||
-spec find_leftneighbor(table(), non_neg_integer()) ->
|
||||
undefined | chunk().
|
||||
undefined |
|
||||
{non_neg_integer(), machi_dt:chunk_size(), trimmed|machi_dt:chunk_csum()}.
|
||||
find_leftneighbor(CsumT, Offset) ->
|
||||
case find(CsumT, Offset, 1) of
|
||||
[] -> undefined;
|
||||
|
|
@ -167,7 +139,8 @@ find_leftneighbor(CsumT, Offset) ->
|
|||
end.
|
||||
|
||||
-spec find_rightneighbor(table(), non_neg_integer()) ->
|
||||
undefined | chunk().
|
||||
undefined |
|
||||
{non_neg_integer(), machi_dt:chunk_size(), trimmed|machi_dt:chunk_csum()}.
|
||||
find_rightneighbor(CsumT, Offset) ->
|
||||
case find(CsumT, Offset, 1) of
|
||||
[] -> undefined;
|
||||
|
|
@ -177,46 +150,40 @@ find_rightneighbor(CsumT, Offset) ->
|
|||
end.
|
||||
|
||||
-spec write(table(), machi_dt:file_offset(), machi_dt:file_size(),
|
||||
machi_dt:chunk_csum()|none|trimmed) ->
|
||||
machi_dt:chunk_csum()|trimmed) ->
|
||||
ok | {error, trimmed|file:posix()}.
|
||||
write(CsumT, Offset, Size, CSum) ->
|
||||
write(CsumT, Offset, Size, CSum, undefined, undefined).
|
||||
|
||||
trim(CsumT, Offset, Size, LeftUpdate, RightUpdate) ->
|
||||
write(CsumT, Offset, Size,
|
||||
trimmed, %% Should this be much smaller like $t or just 't'
|
||||
LeftUpdate, RightUpdate).
|
||||
write(CsumT, Offset, Size, trimmed, LeftUpdate, RightUpdate).
|
||||
|
||||
%% @doc returns whether all bytes in a specific window is continously
|
||||
%% trimmed or not
|
||||
-spec all_trimmed(table(), non_neg_integer(), non_neg_integer()) -> boolean().
|
||||
all_trimmed(#machi_csum_table{table=T}, Left, Right) ->
|
||||
FoldFun = fun({_, _}, false) ->
|
||||
false;
|
||||
({K, V}, Pos) when is_integer(Pos) andalso Pos =< Right ->
|
||||
case {sext:decode(K), sext:decode(V)} of
|
||||
{{Pos, Size}, trimmed} ->
|
||||
Pos + Size;
|
||||
{{Offset, Size}, _}
|
||||
when Offset + Size =< Left ->
|
||||
Left;
|
||||
_Eh ->
|
||||
false
|
||||
end
|
||||
end,
|
||||
case eleveldb:fold(T, FoldFun, Left, [{verify_checksums, true}]) of
|
||||
false -> false;
|
||||
Right -> true;
|
||||
LastTrimmed when LastTrimmed < Right -> false;
|
||||
_ -> %% LastTrimmed > Pos0, which is a irregular case but ok
|
||||
true
|
||||
-spec trim(table(), machi_dt:file_offset(), machi_dt:file_size()) ->
|
||||
ok | {error, file:posix()}.
|
||||
trim(#machi_csum_table{fd=Fd, table=T}, Offset, Size) ->
|
||||
Binary = encode_csum_file_entry_bin(Offset, Size, trimmed),
|
||||
case file:write(Fd, Binary) of
|
||||
ok ->
|
||||
true = ets:insert(T, {Offset, Size, trimmed}),
|
||||
ok;
|
||||
Error ->
|
||||
Error
|
||||
end.
|
||||
|
||||
%% @doc returns whether all bytes 0-Pos0 is continously trimmed or
|
||||
%% not, including header.
|
||||
-spec all_trimmed(table(), non_neg_integer(), non_neg_integer()) -> boolean().
|
||||
all_trimmed(#machi_csum_table{table=T}, Left, Right) ->
|
||||
runthru(ets:tab2list(T), Left, Right).
|
||||
|
||||
-spec all_trimmed(table(), non_neg_integer()) -> boolean().
|
||||
all_trimmed(CsumT, Pos0) ->
|
||||
all_trimmed(CsumT, 0, Pos0).
|
||||
all_trimmed(#machi_csum_table{table=T}, Pos) ->
|
||||
case ets:tab2list(T) of
|
||||
[{0, ?MINIMUM_OFFSET, _}|L] ->
|
||||
%% tl/1 to remove header space {0, 1024, <<0>>}
|
||||
runthru(L, ?MINIMUM_OFFSET, Pos);
|
||||
List ->
|
||||
%% In case a header is removed;
|
||||
runthru(List, 0, Pos)
|
||||
end.
|
||||
|
||||
-spec any_trimmed(table(),
|
||||
pos_integer(),
|
||||
|
|
@ -225,9 +192,13 @@ any_trimmed(CsumT, Offset, Size) ->
|
|||
Chunks = find(CsumT, Offset, Size),
|
||||
lists:any(fun({_, _, State}) -> State =:= trimmed end, Chunks).
|
||||
|
||||
-spec sync(table()) -> ok | {error, file:posix()}.
|
||||
sync(#machi_csum_table{fd=Fd}) ->
|
||||
file:sync(Fd).
|
||||
|
||||
-spec calc_unwritten_bytes(table()) -> [byte_sequence()].
|
||||
calc_unwritten_bytes(#machi_csum_table{table=_} = CsumT) ->
|
||||
case lists:sort(all(CsumT)) of
|
||||
calc_unwritten_bytes(#machi_csum_table{table=T}) ->
|
||||
case lists:sort(ets:tab2list(T)) of
|
||||
[] ->
|
||||
[{?MINIMUM_OFFSET, infinity}];
|
||||
Sorted ->
|
||||
|
|
@ -235,34 +206,101 @@ calc_unwritten_bytes(#machi_csum_table{table=_} = CsumT) ->
|
|||
build_unwritten_bytes_list(Sorted, LastOffset, [])
|
||||
end.
|
||||
|
||||
all(CsumT) ->
|
||||
FoldFun = fun(E, Acc) -> [E|Acc] end,
|
||||
lists:reverse(foldl_chunks(FoldFun, [], CsumT)).
|
||||
|
||||
-spec close(table()) -> ok.
|
||||
close(#machi_csum_table{table=T}) ->
|
||||
ok = eleveldb:close(T).
|
||||
close(#machi_csum_table{table=T, fd=Fd}) ->
|
||||
true = ets:delete(T),
|
||||
ok = file:close(Fd).
|
||||
|
||||
-spec delete(table()) -> ok.
|
||||
delete(#machi_csum_table{table=T, file=F}) ->
|
||||
catch eleveldb:close(T),
|
||||
%% TODO change this to directory walk
|
||||
case os:cmd("rm -rf " ++ F) of
|
||||
"" -> ok;
|
||||
delete(#machi_csum_table{file=F} = C) ->
|
||||
catch close(C),
|
||||
case file:delete(F) of
|
||||
ok -> ok;
|
||||
{error, enoent} -> ok;
|
||||
E -> E
|
||||
end.
|
||||
|
||||
-spec foldl_chunks(fun((chunk(), Acc0 :: term()) -> Acc :: term()),
|
||||
-spec foldl_chunks(fun(({non_neg_integer(), non_neg_integer(), term()},
|
||||
Acc0 :: term())
|
||||
-> Acc :: term()),
|
||||
Acc0 :: term(), table()) -> Acc :: term().
|
||||
foldl_chunks(Fun, Acc0, #machi_csum_table{table=T}) ->
|
||||
FoldFun = fun({K, V}, Acc) ->
|
||||
{Offset, Len} = sext:decode(K),
|
||||
Fun({Offset, Len, sext:decode(V)}, Acc);
|
||||
(_K, Acc) ->
|
||||
_ = lager:error("~p: wrong option?", [_K]),
|
||||
Acc
|
||||
end,
|
||||
eleveldb:fold(T, FoldFun, Acc0, [{verify_checksums, true}]).
|
||||
ets:foldl(Fun, Acc0, T).
|
||||
|
||||
%% @doc Encode `Offset + Size + TaggedCSum' into an `iolist()' type for
|
||||
%% internal storage by the FLU.
|
||||
|
||||
-spec encode_csum_file_entry(
|
||||
machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()) ->
|
||||
iolist().
|
||||
encode_csum_file_entry(Offset, Size, TaggedCSum) ->
|
||||
Len = 8 + 4 + byte_size(TaggedCSum),
|
||||
[<<$w, Len:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big>>,
|
||||
TaggedCSum].
|
||||
|
||||
%% @doc Encode `Offset + Size + TaggedCSum' into an `binary()' type for
|
||||
%% internal storage by the FLU.
|
||||
|
||||
-spec encode_csum_file_entry_bin(
|
||||
machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()) ->
|
||||
binary().
|
||||
encode_csum_file_entry_bin(Offset, Size, trimmed) ->
|
||||
<<$t, Offset:64/unsigned-big, Size:32/unsigned-big>>;
|
||||
encode_csum_file_entry_bin(Offset, Size, TaggedCSum) ->
|
||||
Len = 8 + 4 + byte_size(TaggedCSum),
|
||||
<<$w, Len:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big,
|
||||
TaggedCSum/binary>>.
|
||||
|
||||
%% @doc Decode a single `binary()' blob into an
|
||||
%% `{Offset,Size,TaggedCSum}' tuple.
|
||||
%%
|
||||
%% The internal encoding (which is currently exposed to the outside world
|
||||
%% via this function and related ones) is:
|
||||
%%
|
||||
%% <ul>
|
||||
%% <li> 1 byte: record length
|
||||
%% </li>
|
||||
%% <li> 8 bytes (unsigned big-endian): byte offset
|
||||
%% </li>
|
||||
%% <li> 4 bytes (unsigned big-endian): chunk size
|
||||
%% </li>
|
||||
%% <li> all remaining bytes: tagged checksum (1st byte = type tag)
|
||||
%% </li>
|
||||
%% </ul>
|
||||
%%
|
||||
%% See `machi.hrl' for the tagged checksum types, e.g.,
|
||||
%% `?CSUM_TAG_NONE'.
|
||||
|
||||
-spec decode_csum_file_entry(binary()) ->
|
||||
error |
|
||||
{machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()}.
|
||||
decode_csum_file_entry(<<_:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big, TaggedCSum/binary>>) ->
|
||||
{Offset, Size, TaggedCSum};
|
||||
decode_csum_file_entry(_Else) ->
|
||||
error.
|
||||
|
||||
%% @doc Split a `binary()' blob of `checksum_list' data into a list of
|
||||
%% `{Offset,Size,TaggedCSum}' tuples.
|
||||
|
||||
-spec split_checksum_list_blob_decode(binary()) ->
|
||||
{list({machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()}),
|
||||
TrailingJunk::binary()}.
|
||||
split_checksum_list_blob_decode(Bin) ->
|
||||
split_checksum_list_blob_decode(Bin, []).
|
||||
|
||||
split_checksum_list_blob_decode(<<$w, Len:8/unsigned-big, Part:Len/binary, Rest/binary>>, Acc)->
|
||||
One = <<Len:8/unsigned-big, Part/binary>>,
|
||||
case decode_csum_file_entry(One) of
|
||||
error ->
|
||||
split_checksum_list_blob_decode(Rest, Acc);
|
||||
DecOne ->
|
||||
split_checksum_list_blob_decode(Rest, [DecOne|Acc])
|
||||
end;
|
||||
split_checksum_list_blob_decode(<<$t, Offset:64/unsigned-big, Size:32/unsigned-big, Rest/binary>>, Acc) ->
|
||||
%% trimmed offset
|
||||
split_checksum_list_blob_decode(Rest, [{Offset, Size, trimmed}|Acc]);
|
||||
split_checksum_list_blob_decode(Rest, Acc) ->
|
||||
{lists:reverse(Acc), Rest}.
|
||||
|
||||
-spec build_unwritten_bytes_list( CsumData :: [{ Offset :: non_neg_integer(),
|
||||
Size :: pos_integer(),
|
||||
|
|
@ -284,46 +322,21 @@ build_unwritten_bytes_list([{CurrentOffset, CurrentSize, _Csum}|Rest], LastOffse
|
|||
build_unwritten_bytes_list([{CO, CS, _Ck}|Rest], _LastOffset, Acc) ->
|
||||
build_unwritten_bytes_list(Rest, CO + CS, Acc).
|
||||
|
||||
%% @doc make sure all trimmed chunks are continously chained
|
||||
%% TODO: test with EQC
|
||||
runthru([], Pos, Pos) -> true;
|
||||
runthru([], Pos0, Pos) when Pos0 < Pos -> false;
|
||||
runthru([{Offset0, Size0, trimmed}|T], Offset, Pos) when Offset0 =< Offset ->
|
||||
runthru(T, Offset0+Size0, Pos);
|
||||
runthru(_L, _O, _P) ->
|
||||
false.
|
||||
|
||||
%% @doc If you want to find an overlap among two areas [x, y] and [a,
|
||||
%% b] where x < y and a < b; if (a-y)*(b-x) < 0 then there's a
|
||||
%% b] where x < y and a < b; if (a-y)*(b-x) < 0 then there's a
|
||||
%% overlap, else, > 0 then there're no overlap. border condition = 0
|
||||
%% is not overlap in this offset-size case.
|
||||
%% inclusion_match_spec(Offset, Size) ->
|
||||
%% {'>', 0,
|
||||
%% {'*',
|
||||
%% {'-', Offset + Size, '$1'},
|
||||
%% {'-', Offset, {'+', '$1', '$2'}}}}.
|
||||
|
||||
-spec eleveldb_fold(eleveldb:db_ref(), binary(), binary(),
|
||||
fun(({binary(), binary()}, AccType::term()) -> AccType::term()),
|
||||
AccType0::term()) ->
|
||||
AccType::term().
|
||||
eleveldb_fold(Ref, Start, End, FoldFun, InitAcc) ->
|
||||
{ok, Iterator} = eleveldb:iterator(Ref, []),
|
||||
try
|
||||
eleveldb_do_fold(eleveldb:iterator_move(Iterator, Start),
|
||||
Iterator, End, FoldFun, InitAcc)
|
||||
catch throw:IteratorClosed ->
|
||||
{error, IteratorClosed}
|
||||
after
|
||||
eleveldb:iterator_close(Iterator)
|
||||
end.
|
||||
|
||||
-spec eleveldb_do_fold({ok, binary(), binary()}|{error, iterator_closed|invalid_iterator}|{ok,binary()},
|
||||
eleveldb:itr_ref(), binary(),
|
||||
fun(({binary(), binary()}, AccType::term()) -> AccType::term()),
|
||||
AccType::term()) ->
|
||||
AccType::term().
|
||||
eleveldb_do_fold({ok, Key, Value}, _, End, FoldFun, Acc)
|
||||
when End < Key ->
|
||||
FoldFun({Key, Value}, Acc);
|
||||
eleveldb_do_fold({ok, Key, Value}, Iterator, End, FoldFun, Acc) ->
|
||||
eleveldb_do_fold(eleveldb:iterator_move(Iterator, next),
|
||||
Iterator, End, FoldFun,
|
||||
FoldFun({Key, Value}, Acc));
|
||||
eleveldb_do_fold({error, iterator_closed}, _, _, _, Acc) ->
|
||||
%% It's really an error which is not expected
|
||||
throw({iterator_closed, Acc});
|
||||
eleveldb_do_fold({error, invalid_iterator}, _, _, _, Acc) ->
|
||||
%% Probably reached to end
|
||||
Acc.
|
||||
inclusion_match_spec(Offset, Size) ->
|
||||
{'>', 0,
|
||||
{'*',
|
||||
{'-', Offset + Size, '$1'},
|
||||
{'-', Offset, {'+', '$1', '$2'}}}}.
|
||||
|
|
|
|||
|
|
@ -20,24 +20,15 @@
|
|||
|
||||
-module(machi_dt).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-type append_opts() :: #append_opts{}.
|
||||
-type chunk() :: chunk_bin() | iolist(). % client can choose either rep.
|
||||
-type chunk_bin() :: binary(). % server returns binary() only.
|
||||
-type chunk_csum() :: <<>> | chunk_csum_bin() | {csum_tag(), binary()}.
|
||||
-type chunk_csum_bin() :: binary(). % 1 byte tag, N-1 bytes checksum
|
||||
-type chunk_cstrm() :: 'trimmed' | chunk_csum().
|
||||
-type chunk_summary() :: {file_offset(), chunk_size(), chunk_bin(), chunk_cstrm()}.
|
||||
-type chunk() :: chunk_bin() | {chunk_csum(), chunk_bin()}.
|
||||
-type chunk_bin() :: binary() | iolist(). % client can use either
|
||||
-type chunk_csum() :: binary(). % 1 byte tag, N-1 bytes checksum
|
||||
-type chunk_summary() :: {file_offset(), chunk_size(), binary()}.
|
||||
-type chunk_s() :: 'trimmed' | binary().
|
||||
-type chunk_pos() :: {file_offset(), chunk_size(), file_name_s()}.
|
||||
-type chunk_size() :: non_neg_integer().
|
||||
|
||||
%% Tags that stand for how that checksum was generated. See
|
||||
%% machi_util:make_tagged_csum/{1,2} for further documentation and
|
||||
%% implementation.
|
||||
-type csum_tag() :: none | client_sha | server_sha | server_regen_sha.
|
||||
|
||||
-type error_general() :: 'bad_arg' | 'wedged' | 'bad_checksum'.
|
||||
-type epoch_csum() :: binary().
|
||||
-type epoch_num() :: -1 | non_neg_integer().
|
||||
|
|
@ -50,26 +41,23 @@
|
|||
-type file_prefix() :: binary() | list().
|
||||
-type inet_host() :: inet:ip_address() | inet:hostname().
|
||||
-type inet_port() :: inet:port_number().
|
||||
-type locator() :: number().
|
||||
-type namespace() :: binary().
|
||||
-type namespace_version() :: non_neg_integer().
|
||||
-type ns_info() :: #ns_info{}.
|
||||
-type projection() :: #projection_v1{}.
|
||||
-type projection_type() :: 'public' | 'private'.
|
||||
-type read_opts() :: #read_opts{}.
|
||||
-type read_opts_x() :: 'undefined' | 'noopt' | 'none' | #read_opts{}.
|
||||
|
||||
%% @doc Tags that stand for how that checksum was generated. See
|
||||
%% machi_util:make_tagged_csum/{1,2} for further documentation and
|
||||
%% implementation.
|
||||
-type csum_tag() :: none | client_sha | server_sha | server_regen_sha.
|
||||
|
||||
-export_type([
|
||||
append_opts/0,
|
||||
chunk/0,
|
||||
chunk_bin/0,
|
||||
chunk_csum/0,
|
||||
chunk_csum_bin/0,
|
||||
chunk_cstrm/0,
|
||||
csum_tag/0,
|
||||
chunk_summary/0,
|
||||
chunk_s/0,
|
||||
chunk_pos/0,
|
||||
chunk_size/0,
|
||||
csum_tag/0,
|
||||
error_general/0,
|
||||
epoch_csum/0,
|
||||
epoch_num/0,
|
||||
|
|
@ -82,13 +70,7 @@
|
|||
file_prefix/0,
|
||||
inet_host/0,
|
||||
inet_port/0,
|
||||
locator/0,
|
||||
namespace/0,
|
||||
namespace_version/0,
|
||||
ns_info/0,
|
||||
projection/0,
|
||||
projection_type/0,
|
||||
read_opts/0,
|
||||
read_opts_x/0
|
||||
projection_type/0
|
||||
]).
|
||||
|
||||
|
|
|
|||
|
|
@ -57,8 +57,7 @@
|
|||
write/4,
|
||||
trim/4,
|
||||
append/2,
|
||||
append/4,
|
||||
checksum_list/1
|
||||
append/4
|
||||
]).
|
||||
|
||||
%% gen_server callbacks
|
||||
|
|
@ -71,7 +70,7 @@
|
|||
code_change/3
|
||||
]).
|
||||
|
||||
-define(TICK, 5*1000).
|
||||
-define(TICK, 30*1000). %% XXX FIXME Should be something like 5 seconds
|
||||
-define(TICK_THRESHOLD, 5). %% After this + 1 more quiescent ticks, shutdown
|
||||
-define(TIMEOUT, 10*1000).
|
||||
-define(TOO_MANY_ERRORS_RATIO, 50).
|
||||
|
|
@ -91,7 +90,6 @@
|
|||
csum_table :: machi_csum_table:table(),
|
||||
eof_position = 0 :: non_neg_integer(),
|
||||
max_file_size = ?DEFAULT_MAX_FILE_SIZE :: pos_integer(),
|
||||
rollover = false :: boolean(),
|
||||
tref :: reference(), %% timer ref
|
||||
ticks = 0 :: non_neg_integer(), %% ticks elapsed with no new operations
|
||||
ops = 0 :: non_neg_integer(), %% sum of all ops
|
||||
|
|
@ -142,18 +140,18 @@ sync(_Pid, Type) ->
|
|||
Data :: binary(), Checksum :: binary()}]} |
|
||||
{error, Reason :: term()}.
|
||||
read(Pid, Offset, Length) ->
|
||||
read(Pid, Offset, Length, #read_opts{}).
|
||||
read(Pid, Offset, Length, []).
|
||||
|
||||
-spec read(Pid :: pid(),
|
||||
Offset :: non_neg_integer(),
|
||||
Length :: non_neg_integer(),
|
||||
machi_dt:read_opts_x()) ->
|
||||
[{no_checksum|no_chunk|needs_trimmed, boolean()}]) ->
|
||||
{ok, [{Filename::string(), Offset :: non_neg_integer(),
|
||||
Data :: binary(), Checksum :: binary()}]} |
|
||||
{error, Reason :: term()}.
|
||||
read(Pid, Offset, Length, #read_opts{}=Opts)
|
||||
when is_pid(Pid) andalso is_integer(Offset) andalso Offset >= 0
|
||||
andalso is_integer(Length) andalso Length > 0 ->
|
||||
read(Pid, Offset, Length, Opts) when is_pid(Pid) andalso is_integer(Offset) andalso Offset >= 0
|
||||
andalso is_integer(Length) andalso Length > 0
|
||||
andalso is_list(Opts) ->
|
||||
gen_server:call(Pid, {read, Offset, Length, Opts}, ?TIMEOUT);
|
||||
read(_Pid, Offset, Length, Opts) ->
|
||||
lager:warning("Bad args to read: Offset ~p, Length ~p, Options ~p", [Offset, Length, Opts]),
|
||||
|
|
@ -212,10 +210,6 @@ append(_Pid, ClientMeta, Extra, _Data) ->
|
|||
lager:warning("Bad arg to append: ClientMeta ~p, Extra ~p", [ClientMeta, Extra]),
|
||||
{error, bad_arg}.
|
||||
|
||||
-spec checksum_list(pid()) -> {ok, list()}.
|
||||
checksum_list(Pid) ->
|
||||
gen_server:call(Pid, {checksum_list}, ?TIMEOUT).
|
||||
|
||||
%% gen_server callbacks
|
||||
|
||||
% @private
|
||||
|
|
@ -240,7 +234,7 @@ init({FluName, Filename, DataDir}) ->
|
|||
data_filehandle = FHd,
|
||||
csum_table = CsumTable,
|
||||
tref = Tref,
|
||||
eof_position = erlang:max(Eof, ?MINIMUM_OFFSET),
|
||||
eof_position = Eof,
|
||||
max_file_size = machi_config:max_file_size()},
|
||||
lager:debug("Starting file proxy ~p for filename ~p, state = ~p, Eof = ~p",
|
||||
[self(), Filename, St, Eof]),
|
||||
|
|
@ -255,22 +249,27 @@ handle_call({sync, data}, _From, State = #state{ data_filehandle = FHd }) ->
|
|||
R = file:sync(FHd),
|
||||
{reply, R, State};
|
||||
|
||||
handle_call({sync, csum}, _From, State) ->
|
||||
%% machi_csum_table always writes in {sync, true} option, so here
|
||||
%% explicit sync isn't actually needed.
|
||||
{reply, ok, State};
|
||||
handle_call({sync, csum}, _From, State = #state{ csum_table = T }) ->
|
||||
R = machi_csum_table:sync(T),
|
||||
{reply, R, State};
|
||||
|
||||
handle_call({sync, all}, _From, State = #state{filename = F,
|
||||
data_filehandle = FHd,
|
||||
csum_table = _T
|
||||
csum_table = T
|
||||
}) ->
|
||||
Resp = case file:sync(FHd) of
|
||||
ok ->
|
||||
ok;
|
||||
Error ->
|
||||
lager:error("Got ~p syncing all files for file ~p",
|
||||
[Error, F]),
|
||||
Error
|
||||
R = machi_csum_table:sync(T),
|
||||
R1 = file:sync(FHd),
|
||||
Resp = case {R, R1} of
|
||||
{ok, ok} -> ok;
|
||||
{ok, O1} ->
|
||||
lager:error("Got ~p during a data file sync on file ~p", [O1, F]),
|
||||
O1;
|
||||
{O2, ok} ->
|
||||
lager:error("Got ~p during a csum file sync on file ~p", [O2, F]),
|
||||
O2;
|
||||
{O3, O4} ->
|
||||
lager:error("Got ~p ~p syncing all files for file ~p", [O3, O4, F]),
|
||||
{O3, O4}
|
||||
end,
|
||||
{reply, Resp, State};
|
||||
|
||||
|
|
@ -299,15 +298,15 @@ handle_call({read, Offset, Length, Opts}, _From,
|
|||
}) ->
|
||||
%% TODO: use these options - NoChunk prevents reading from disks
|
||||
%% NoChecksum doesn't check checksums
|
||||
#read_opts{no_checksum=NoChecksum, no_chunk=NoChunk,
|
||||
needs_trimmed=NeedsTrimmed} = Opts,
|
||||
NoChecksum = proplists:get_value(no_checksum, Opts, false),
|
||||
NoChunk = proplists:get_value(no_chunk, Opts, false),
|
||||
{Resp, NewErr} =
|
||||
case do_read(FH, F, CsumTable, Offset, Length, NoChunk, NoChecksum) of
|
||||
{ok, {[], []}} ->
|
||||
{{error, not_written}, Err + 1};
|
||||
{ok, {Chunks0, Trimmed0}} ->
|
||||
Chunks = slice_both_side(Chunks0, Offset, Offset+Length),
|
||||
Trimmed = case NeedsTrimmed of
|
||||
Trimmed = case proplists:get_value(needs_trimmed, Opts, false) of
|
||||
true -> Trimmed0;
|
||||
false -> []
|
||||
end,
|
||||
|
|
@ -436,10 +435,6 @@ handle_call({append, ClientMeta, Extra, Data}, _From,
|
|||
{reply, Resp, State#state{appends = {T+1, NewErr},
|
||||
eof_position = NewEof}};
|
||||
|
||||
handle_call({checksum_list}, _FRom, State = #state{csum_table=T}) ->
|
||||
All = machi_csum_table:all(T),
|
||||
{reply, {ok, All}, State};
|
||||
|
||||
handle_call(Req, _From, State) ->
|
||||
lager:warning("Unknown call: ~p", [Req]),
|
||||
{reply, whoaaaaaaaaaaaa, State}.
|
||||
|
|
@ -450,23 +445,11 @@ handle_cast(Cast, State) ->
|
|||
{noreply, State}.
|
||||
|
||||
% @private
|
||||
handle_info(tick, State = #state{fluname = FluName,
|
||||
filename = F,
|
||||
eof_position = Eof,
|
||||
handle_info(tick, State = #state{eof_position = Eof,
|
||||
max_file_size = MaxFileSize}) when Eof >= MaxFileSize ->
|
||||
%% Older code halted here with {stop, file_rollover, State}.
|
||||
%% However, there may be other requests in our mailbox already
|
||||
%% and/or not yet delivered but in a race with the
|
||||
%% machi_flu_metadata_mgr. So we close our eleveldb instance (to
|
||||
%% avoid double-open attempt by a new file proxy proc), tell
|
||||
%% machi_flu_metadata_mgr that we request a rollover, then stop.
|
||||
%% terminate() will take care of forwarding messages that are
|
||||
%% caught in the race.
|
||||
lager:notice("Eof ~s position ~p >= max file size ~p. Shutting down.",
|
||||
[F, Eof, MaxFileSize]),
|
||||
State2 = close_files(State),
|
||||
machi_flu_metadata_mgr:stop_proxy_pid_rollover(FluName, {file, F}),
|
||||
{stop, normal, State2#state{rollover = true}};
|
||||
lager:notice("Eof position ~p >= max file size ~p. Shutting down.",
|
||||
[Eof, MaxFileSize]),
|
||||
{stop, file_rollover, State};
|
||||
|
||||
%% XXX Is this a good idea? Need to think this through a bit.
|
||||
handle_info(tick, State = #state{wedged = true}) ->
|
||||
|
|
@ -480,7 +463,7 @@ handle_info(tick, State = #state{
|
|||
writes = {WT, WE},
|
||||
appends = {AT, AE}
|
||||
}) when Ops > 100 andalso
|
||||
trunc(((RE+WE+AE) / (RT+WT+AT)) * 100) > ?TOO_MANY_ERRORS_RATIO ->
|
||||
trunc(((RE+WE+AE) / RT+WT+AT) * 100) > ?TOO_MANY_ERRORS_RATIO ->
|
||||
Errors = RE + WE + AE,
|
||||
lager:notice("Got ~p errors. Shutting down.", [Errors]),
|
||||
{stop, too_many_errors, State};
|
||||
|
|
@ -539,23 +522,31 @@ handle_info(Req, State) ->
|
|||
{noreply, State}.
|
||||
|
||||
% @private
|
||||
terminate(Reason, State = #state{fluname = FluName,
|
||||
filename = F,
|
||||
rollover = Rollover_p,
|
||||
reads = {RT, RE},
|
||||
writes = {WT, WE},
|
||||
appends = {AT, AE}
|
||||
}) ->
|
||||
terminate(Reason, #state{filename = F,
|
||||
data_filehandle = FHd,
|
||||
csum_table = T,
|
||||
reads = {RT, RE},
|
||||
writes = {WT, WE},
|
||||
appends = {AT, AE}
|
||||
}) ->
|
||||
lager:info("Shutting down proxy for file ~p because ~p", [F, Reason]),
|
||||
lager:info(" Op Tot/Error", []),
|
||||
lager:info(" Reads: ~p/~p", [RT, RE]),
|
||||
lager:info(" Writes: ~p/~p", [WT, WE]),
|
||||
lager:info("Appends: ~p/~p", [AT, AE]),
|
||||
close_files(State),
|
||||
if Rollover_p ->
|
||||
forward_late_messages(FluName, F, 500);
|
||||
true ->
|
||||
ok
|
||||
case FHd of
|
||||
undefined ->
|
||||
noop; %% file deleted
|
||||
_ ->
|
||||
ok = file:sync(FHd),
|
||||
ok = file:close(FHd)
|
||||
end,
|
||||
case T of
|
||||
undefined ->
|
||||
noop; %% file deleted
|
||||
_ ->
|
||||
ok = machi_csum_table:sync(T),
|
||||
ok = machi_csum_table:close(T)
|
||||
end,
|
||||
ok.
|
||||
|
||||
|
|
@ -606,8 +597,7 @@ check_or_make_tagged_csum(OtherTag, _ClientCsum, _Data) ->
|
|||
Size :: non_neg_integer(),
|
||||
NoChunk :: boolean(),
|
||||
NoChecksum :: boolean()
|
||||
) -> {ok, {Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum :: binary()}],
|
||||
Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]}} |
|
||||
) -> {ok, Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum :: binary()}]} |
|
||||
{error, bad_checksum} |
|
||||
{error, partial_read} |
|
||||
{error, file:posix()} |
|
||||
|
|
@ -631,14 +621,6 @@ do_read(FHd, Filename, CsumTable, Offset, Size, _, _) ->
|
|||
ChunkCsums = machi_csum_table:find(CsumTable, Offset, Size),
|
||||
read_all_ranges(FHd, Filename, ChunkCsums, [], []).
|
||||
|
||||
-spec read_all_ranges(file:io_device(), string(),
|
||||
[{non_neg_integer(),non_neg_integer(),trimmed|binary()}],
|
||||
Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum::binary()}],
|
||||
Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]) ->
|
||||
{ok, {
|
||||
Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum::binary()}],
|
||||
Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]}} |
|
||||
{erorr, term()|partial_read}.
|
||||
read_all_ranges(_, _, [], ReadChunks, TrimmedChunks) ->
|
||||
%% TODO: currently returns empty list of trimmed chunks
|
||||
{ok, {lists:reverse(ReadChunks), lists:reverse(TrimmedChunks)}};
|
||||
|
|
@ -650,11 +632,6 @@ read_all_ranges(FHd, Filename, [{Offset, Size, TaggedCsum}|T], ReadChunks, Trimm
|
|||
case file:pread(FHd, Offset, Size) of
|
||||
eof ->
|
||||
read_all_ranges(FHd, Filename, T, ReadChunks, TrimmedChunks);
|
||||
{ok, Bytes} when byte_size(Bytes) == Size, TaggedCsum =:= none ->
|
||||
read_all_ranges(FHd, Filename, T,
|
||||
[{Filename, Offset, Bytes,
|
||||
machi_util:make_tagged_csum(none, <<>>)}|ReadChunks],
|
||||
TrimmedChunks);
|
||||
{ok, Bytes} when byte_size(Bytes) == Size ->
|
||||
{Tag, Ck} = machi_util:unmake_tagged_csum(TaggedCsum),
|
||||
case check_or_make_tagged_csum(Tag, Ck, Bytes) of
|
||||
|
|
@ -850,7 +827,7 @@ maybe_gc(Reply, S = #state{fluname=FluName,
|
|||
filename = Filename,
|
||||
eof_position = Eof,
|
||||
csum_table=CsumTable}) ->
|
||||
case machi_csum_table:all_trimmed(CsumTable, ?MINIMUM_OFFSET, Eof) of
|
||||
case machi_csum_table:all_trimmed(CsumTable, Eof) of
|
||||
true ->
|
||||
lager:debug("GC? Let's do it: ~p.~n", [Filename]),
|
||||
%% Before unlinking a file, it should inform
|
||||
|
|
@ -873,36 +850,3 @@ maybe_gc(Reply, S = #state{fluname=FluName,
|
|||
false ->
|
||||
{reply, Reply, S}
|
||||
end.
|
||||
|
||||
close_files(State = #state{data_filehandle = FHd,
|
||||
csum_table = T}) ->
|
||||
case FHd of
|
||||
undefined ->
|
||||
noop; %% file deleted
|
||||
_ ->
|
||||
ok = file:sync(FHd),
|
||||
ok = file:close(FHd)
|
||||
end,
|
||||
case T of
|
||||
undefined ->
|
||||
noop; %% file deleted
|
||||
_ ->
|
||||
ok = machi_csum_table:close(T)
|
||||
end,
|
||||
State#state{data_filehandle = undefined, csum_table = undefined}.
|
||||
|
||||
forward_late_messages(FluName, F, Timeout) ->
|
||||
receive
|
||||
M ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, F}) of
|
||||
{ok, Pid} ->
|
||||
Pid ! M;
|
||||
{error, trimmed} ->
|
||||
lager:error("TODO: FLU ~p file ~p reports trimmed status "
|
||||
"when forwarding ~P\n",
|
||||
[FluName, F, M, 20])
|
||||
end,
|
||||
forward_late_messages(FluName, F, Timeout)
|
||||
after Timeout ->
|
||||
ok
|
||||
end.
|
||||
|
|
|
|||
|
|
@ -39,8 +39,7 @@
|
|||
get_unfit_list/1, update_local_down_list/3,
|
||||
add_admin_down/3, delete_admin_down/2,
|
||||
send_fitness_update_spam/3,
|
||||
send_spam_to_everyone/1,
|
||||
trigger_early_adjustment/2]).
|
||||
send_spam_to_everyone/1]).
|
||||
|
||||
%% gen_server callbacks
|
||||
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
|
||||
|
|
@ -82,13 +81,6 @@ send_fitness_update_spam(Pid, FromName, Dict) ->
|
|||
send_spam_to_everyone(Pid) ->
|
||||
gen_server:call(Pid, {send_spam_to_everyone}, infinity).
|
||||
|
||||
%% @doc For testing purposes, we don't want a test to wait for
|
||||
%% wall-clock time to elapse before the fitness server makes a
|
||||
%% down->up status decision.
|
||||
|
||||
trigger_early_adjustment(Pid, FLU) ->
|
||||
Pid ! {adjust_down_list, FLU}.
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
init([{MyFluName}|Args]) ->
|
||||
|
|
@ -108,7 +100,6 @@ handle_call({update_local_down_list, Down, MembersDict}, _From,
|
|||
#state{my_flu_name=MyFluName, pending_map=OldMap,
|
||||
local_down=OldDown, members_dict=OldMembersDict,
|
||||
admin_down=AdminDown}=S) ->
|
||||
verbose("FITNESS: ~w has down suspect ~w\n", [MyFluName, Down]),
|
||||
NewMap = store_in_map(OldMap, MyFluName, erlang:now(), Down,
|
||||
AdminDown, [props_yo]),
|
||||
S2 = if Down == OldDown, MembersDict == OldMembersDict ->
|
||||
|
|
@ -120,17 +111,13 @@ handle_call({update_local_down_list, Down, MembersDict}, _From,
|
|||
end,
|
||||
{reply, ok, S2#state{local_down=Down}};
|
||||
handle_call({add_admin_down, DownFLU, DownProps}, _From,
|
||||
#state{my_flu_name=MyFluName,
|
||||
local_down=OldDown, admin_down=AdminDown}=S) ->
|
||||
verbose("FITNESS: ~w add admin down ~w\n", [MyFluName, DownFLU]),
|
||||
#state{local_down=OldDown, admin_down=AdminDown}=S) ->
|
||||
NewAdminDown = [{DownFLU,DownProps}|lists:keydelete(DownFLU, 1, AdminDown)],
|
||||
S3 = finish_admin_down(erlang:now(), OldDown, NewAdminDown,
|
||||
[props_yo], S),
|
||||
{reply, ok, S3};
|
||||
handle_call({delete_admin_down, DownFLU}, _From,
|
||||
#state{my_flu_name=MyFluName,
|
||||
local_down=OldDown, admin_down=AdminDown}=S) ->
|
||||
verbose("FITNESS: ~w delete admin down ~w\n", [MyFluName, DownFLU]),
|
||||
#state{local_down=OldDown, admin_down=AdminDown}=S) ->
|
||||
NewAdminDown = lists:keydelete(DownFLU, 1, AdminDown),
|
||||
S3 = finish_admin_down(erlang:now(), OldDown, NewAdminDown,
|
||||
[props_yo], S),
|
||||
|
|
@ -148,8 +135,7 @@ handle_call(_Request, _From, S) ->
|
|||
handle_cast(_Msg, S) ->
|
||||
{noreply, S}.
|
||||
|
||||
handle_info({adjust_down_list, FLU}, #state{my_flu_name=MyFluName,
|
||||
active_unfit=ActiveUnfit}=S) ->
|
||||
handle_info({adjust_down_list, FLU}, #state{active_unfit=ActiveUnfit}=S) ->
|
||||
NewUnfit = make_unfit_list(S),
|
||||
Added_to_new = NewUnfit -- ActiveUnfit,
|
||||
Dropped_from_new = ActiveUnfit -- NewUnfit,
|
||||
|
|
@ -190,11 +176,9 @@ handle_info({adjust_down_list, FLU}, #state{my_flu_name=MyFluName,
|
|||
{true, true} ->
|
||||
error({bad, ?MODULE, ?LINE, FLU, ActiveUnfit, NewUnfit});
|
||||
{true, false} ->
|
||||
NewActive = wrap_active(MyFluName,lists:usort(ActiveUnfit++[FLU])),
|
||||
{noreply, S#state{active_unfit=NewActive}};
|
||||
{noreply, S#state{active_unfit=lists:usort(ActiveUnfit ++ [FLU])}};
|
||||
{false, true} ->
|
||||
NewActive = wrap_active(MyFluName,ActiveUnfit--[FLU]),
|
||||
{noreply, S#state{active_unfit=NewActive}};
|
||||
{noreply, S#state{active_unfit=ActiveUnfit -- [FLU]}};
|
||||
{false, false} ->
|
||||
{noreply, S}
|
||||
end;
|
||||
|
|
@ -432,18 +416,6 @@ map_value(Map) ->
|
|||
map_merge(Map1, Map2) ->
|
||||
?MAP:merge(Map1, Map2).
|
||||
|
||||
wrap_active(MyFluName, L) ->
|
||||
verbose("FITNESS: ~w has new down list ~w\n", [MyFluName, L]),
|
||||
L.
|
||||
|
||||
verbose(Fmt, Args) ->
|
||||
case application:get_env(machi, fitness_verbose) of
|
||||
{ok, true} ->
|
||||
error_logger:info_msg(Fmt, Args);
|
||||
_ ->
|
||||
ok
|
||||
end.
|
||||
|
||||
-ifdef(TEST).
|
||||
|
||||
dt_understanding_test() ->
|
||||
|
|
|
|||
|
|
@ -21,9 +21,7 @@
|
|||
%% @doc The Machi FLU file server + file location sequencer.
|
||||
%%
|
||||
%% This module implements only the Machi FLU file server and its
|
||||
%% implicit sequencer together with listener, append server,
|
||||
%% file management and file proxy processes.
|
||||
|
||||
%% implicit sequencer.
|
||||
%% Please see the EDoc "Overview" for details about the FLU as a
|
||||
%% primitive file server process vs. the larger Machi design of a FLU
|
||||
%% as a sequencer + file server + chain manager group of processes.
|
||||
|
|
@ -56,16 +54,32 @@
|
|||
|
||||
-ifdef(TEST).
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-export([timing_demo_test_COMMENTED_/0, sort_2lines/2]). % Just to suppress warning
|
||||
-endif. % TEST
|
||||
|
||||
-export([start_link/1, stop/1,
|
||||
update_wedge_state/3, wedge_myself/2]).
|
||||
-export([make_projection_server_regname/1,
|
||||
ets_table_name/1]).
|
||||
-export([make_listener_regname/1, make_projection_server_regname/1]).
|
||||
%% TODO: remove or replace in OTP way after gen_*'ified
|
||||
-export([main2/4]).
|
||||
-export([main2/4, run_append_server/2, run_listen_server/1,
|
||||
current_state/1, format_state/1]).
|
||||
|
||||
-record(state, {
|
||||
flu_name :: atom(),
|
||||
proj_store :: pid(),
|
||||
witness = false :: boolean(),
|
||||
append_pid :: pid(),
|
||||
tcp_port :: non_neg_integer(),
|
||||
data_dir :: string(),
|
||||
wedged = true :: boolean(),
|
||||
etstab :: ets:tid(),
|
||||
epoch_id :: 'undefined' | machi_dt:epoch_id(),
|
||||
pb_mode = undefined :: 'undefined' | 'high' | 'low',
|
||||
high_clnt :: 'undefined' | pid(),
|
||||
trim_table :: ets:tid(),
|
||||
props = [] :: list() % proplist
|
||||
}).
|
||||
|
||||
-define(SERVER_CMD_READ_TIMEOUT, 600*1000).
|
||||
-define(INIT_TIMEOUT, 60*1000).
|
||||
|
||||
start_link([{FluName, TcpPort, DataDir}|Rest])
|
||||
|
|
@ -73,12 +87,7 @@ start_link([{FluName, TcpPort, DataDir}|Rest])
|
|||
proc_lib:start_link(?MODULE, main2, [FluName, TcpPort, DataDir, Rest],
|
||||
?INIT_TIMEOUT).
|
||||
|
||||
stop(RegName) when is_atom(RegName) ->
|
||||
case whereis(RegName) of
|
||||
undefined -> ok;
|
||||
Pid -> stop(Pid)
|
||||
end;
|
||||
stop(Pid) when is_pid(Pid) ->
|
||||
stop(Pid) ->
|
||||
case erlang:is_process_alive(Pid) of
|
||||
true ->
|
||||
Pid ! killme,
|
||||
|
|
@ -87,14 +96,33 @@ stop(Pid) when is_pid(Pid) ->
|
|||
error
|
||||
end.
|
||||
|
||||
update_wedge_state(PidSpec, Boolean, EpochId) ->
|
||||
machi_flu1_append_server:int_update_wedge_state(PidSpec, Boolean, EpochId).
|
||||
update_wedge_state(PidSpec, Boolean, EpochId)
|
||||
when (Boolean == true orelse Boolean == false), is_tuple(EpochId) ->
|
||||
PidSpec ! {wedge_state_change, Boolean, EpochId}.
|
||||
|
||||
wedge_myself(PidSpec, EpochId) ->
|
||||
machi_flu1_append_server:int_wedge_myself(PidSpec, EpochId).
|
||||
wedge_myself(PidSpec, EpochId)
|
||||
when is_tuple(EpochId) ->
|
||||
PidSpec ! {wedge_myself, EpochId}.
|
||||
|
||||
current_state(PidSpec) ->
|
||||
PidSpec ! {current_state, self()},
|
||||
%% TODO: Not so rubust f(^^;)
|
||||
receive
|
||||
Res -> Res
|
||||
after
|
||||
60*1000 -> {error, timeout}
|
||||
end.
|
||||
|
||||
format_state(State) ->
|
||||
Fields = record_info(fields, state),
|
||||
[_Name | Values] = tuple_to_list(State),
|
||||
lists:zip(Fields, Values).
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
ets_table_name(FluName) when is_atom(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_epoch").
|
||||
|
||||
main2(FluName, TcpPort, DataDir, Props) ->
|
||||
{SendAppendPidToProj_p, ProjectionPid} =
|
||||
case proplists:get_value(projection_store_registered_name, Props) of
|
||||
|
|
@ -121,17 +149,25 @@ main2(FluName, TcpPort, DataDir, Props) ->
|
|||
{true, undefined}
|
||||
end,
|
||||
Witness_p = proplists:get_value(witness_mode, Props, false),
|
||||
|
||||
{ok, AppendPid} = start_append_server(FluName, Witness_p, Wedged_p, EpochId),
|
||||
|
||||
S0 = #state{flu_name=FluName,
|
||||
proj_store=ProjectionPid,
|
||||
tcp_port=TcpPort,
|
||||
data_dir=DataDir,
|
||||
wedged=Wedged_p,
|
||||
witness=Witness_p,
|
||||
etstab=ets_table_name(FluName),
|
||||
epoch_id=EpochId,
|
||||
props=Props},
|
||||
{ok, AppendPid} = start_append_server(S0, self()),
|
||||
if SendAppendPidToProj_p ->
|
||||
machi_projection_store:set_wedge_notify_pid(ProjectionPid, AppendPid);
|
||||
machi_projection_store:set_wedge_notify_pid(ProjectionPid,
|
||||
AppendPid);
|
||||
true ->
|
||||
ok
|
||||
end,
|
||||
{ok, ListenerPid} = start_listen_server(FluName, TcpPort, Witness_p, DataDir,
|
||||
ets_table_name(FluName), ProjectionPid,
|
||||
Props),
|
||||
%% io:format(user, "Listener started: ~w~n", [{FluName, ListenerPid}]),
|
||||
S1 = S0#state{append_pid=AppendPid},
|
||||
{ok, ListenPid} = start_listen_server(S1),
|
||||
|
||||
Config_e = machi_util:make_config_filename(DataDir, "unused"),
|
||||
ok = filelib:ensure_dir(Config_e),
|
||||
|
|
@ -141,24 +177,574 @@ main2(FluName, TcpPort, DataDir, Props) ->
|
|||
ok = filelib:ensure_dir(Projection_e),
|
||||
|
||||
put(flu_flu_name, FluName),
|
||||
put(flu_append_pid, AppendPid),
|
||||
put(flu_append_pid, S1#state.append_pid),
|
||||
put(flu_projection_pid, ProjectionPid),
|
||||
put(flu_listen_pid, ListenerPid),
|
||||
put(flu_listen_pid, ListenPid),
|
||||
proc_lib:init_ack({ok, self()}),
|
||||
|
||||
receive killme -> ok end,
|
||||
(catch exit(AppendPid, kill)),
|
||||
(catch exit(S1#state.append_pid, kill)),
|
||||
(catch exit(ProjectionPid, kill)),
|
||||
(catch exit(ListenerPid, kill)),
|
||||
(catch exit(ListenPid, kill)),
|
||||
ok.
|
||||
|
||||
start_append_server(FluName, Witness_p, Wedged_p, EpochId) ->
|
||||
machi_flu1_subsup:start_append_server(FluName, Witness_p, Wedged_p, EpochId).
|
||||
start_listen_server(S) ->
|
||||
proc_lib:start_link(?MODULE, run_listen_server, [S], ?INIT_TIMEOUT).
|
||||
|
||||
start_listen_server(FluName, TcpPort, Witness_p, DataDir, EtsTab, ProjectionPid,
|
||||
Props) ->
|
||||
machi_flu1_subsup:start_listener(FluName, TcpPort, Witness_p, DataDir,
|
||||
EtsTab, ProjectionPid, Props).
|
||||
start_append_server(S, AckPid) ->
|
||||
proc_lib:start_link(?MODULE, run_append_server, [AckPid, S], ?INIT_TIMEOUT).
|
||||
|
||||
run_listen_server(#state{flu_name=FluName, tcp_port=TcpPort}=S) ->
|
||||
register(make_listener_regname(FluName), self()),
|
||||
SockOpts = ?PB_PACKET_OPTS ++
|
||||
[{reuseaddr, true}, {mode, binary}, {active, false},
|
||||
{backlog,8192}],
|
||||
case gen_tcp:listen(TcpPort, SockOpts) of
|
||||
{ok, LSock} ->
|
||||
proc_lib:init_ack({ok, self()}),
|
||||
listen_server_loop(LSock, S);
|
||||
Else ->
|
||||
error_logger:warning_msg("~s:run_listen_server: "
|
||||
"listen to TCP port ~w: ~w\n",
|
||||
[?MODULE, TcpPort, Else]),
|
||||
exit({?MODULE, run_listen_server, tcp_port, TcpPort, Else})
|
||||
end.
|
||||
|
||||
run_append_server(FluPid, #state{flu_name=Name,
|
||||
wedged=Wedged_p,epoch_id=EpochId}=S) ->
|
||||
%% Reminder: Name is the "main" name of the FLU, i.e., no suffix
|
||||
register(Name, self()),
|
||||
TID = ets:new(ets_table_name(Name),
|
||||
[set, protected, named_table, {read_concurrency, true}]),
|
||||
ets:insert(TID, {epoch, {Wedged_p, EpochId}}),
|
||||
proc_lib:init_ack({ok, self()}),
|
||||
append_server_loop(FluPid, S#state{etstab=TID}).
|
||||
|
||||
listen_server_loop(LSock, S) ->
|
||||
{ok, Sock} = gen_tcp:accept(LSock),
|
||||
spawn_link(fun() -> net_server_loop(Sock, S) end),
|
||||
listen_server_loop(LSock, S).
|
||||
|
||||
append_server_loop(FluPid, #state{wedged=Wedged_p,
|
||||
witness=Witness_p,
|
||||
epoch_id=OldEpochId, flu_name=FluName}=S) ->
|
||||
receive
|
||||
{seq_append, From, _Prefix, _Chunk, _CSum, _Extra, _EpochID}
|
||||
when Witness_p ->
|
||||
%% The FLU's net_server_loop() process ought to filter all
|
||||
%% witness states, but we'll keep this clause for extra
|
||||
%% paranoia.
|
||||
From ! witness,
|
||||
append_server_loop(FluPid, S);
|
||||
{seq_append, From, _Prefix, _Chunk, _CSum, _Extra, _EpochID}
|
||||
when Wedged_p ->
|
||||
From ! wedged,
|
||||
append_server_loop(FluPid, S);
|
||||
{seq_append, From, Prefix, Chunk, CSum, Extra, EpochID} ->
|
||||
%% Old is the one from our state, plain old 'EpochID' comes
|
||||
%% from the client.
|
||||
_ = case OldEpochId == EpochID of
|
||||
true ->
|
||||
spawn(fun() ->
|
||||
append_server_dispatch(From, Prefix, Chunk, CSum, Extra, FluName, EpochID)
|
||||
end);
|
||||
false ->
|
||||
From ! {error, bad_epoch}
|
||||
end,
|
||||
append_server_loop(FluPid, S);
|
||||
{wedge_myself, WedgeEpochId} ->
|
||||
if not Wedged_p andalso WedgeEpochId == OldEpochId ->
|
||||
true = ets:insert(S#state.etstab,
|
||||
{epoch, {true, OldEpochId}}),
|
||||
%% Tell my chain manager that it might want to react to
|
||||
%% this new world.
|
||||
Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
|
||||
spawn(fun() ->
|
||||
catch machi_chain_manager1:trigger_react_to_env(Chmgr)
|
||||
end),
|
||||
append_server_loop(FluPid, S#state{wedged=true});
|
||||
true ->
|
||||
append_server_loop(FluPid, S)
|
||||
end;
|
||||
{wedge_state_change, Boolean, {NewEpoch, _}=NewEpochId} ->
|
||||
OldEpoch = case OldEpochId of {OldE, _} -> OldE;
|
||||
undefined -> -1
|
||||
end,
|
||||
if NewEpoch >= OldEpoch ->
|
||||
true = ets:insert(S#state.etstab,
|
||||
{epoch, {Boolean, NewEpochId}}),
|
||||
append_server_loop(FluPid, S#state{wedged=Boolean,
|
||||
epoch_id=NewEpochId});
|
||||
true ->
|
||||
append_server_loop(FluPid, S)
|
||||
end;
|
||||
{wedge_status, FromPid} ->
|
||||
#state{wedged=Wedged_p, epoch_id=EpochId} = S,
|
||||
FromPid ! {wedge_status_reply, Wedged_p, EpochId},
|
||||
append_server_loop(FluPid, S);
|
||||
{current_state, FromPid} ->
|
||||
FromPid ! S;
|
||||
Else ->
|
||||
io:format(user, "append_server_loop: WHA? ~p\n", [Else]),
|
||||
append_server_loop(FluPid, S)
|
||||
end.
|
||||
|
||||
net_server_loop(Sock, S) ->
|
||||
case gen_tcp:recv(Sock, 0, ?SERVER_CMD_READ_TIMEOUT) of
|
||||
{ok, Bin} ->
|
||||
{RespBin, S2} =
|
||||
case machi_pb:decode_mpb_ll_request(Bin) of
|
||||
LL_req when LL_req#mpb_ll_request.do_not_alter == 2 ->
|
||||
{R, NewS} = do_pb_ll_request(LL_req, S),
|
||||
{maybe_encode_response(R), mode(low, NewS)};
|
||||
_ ->
|
||||
HL_req = machi_pb:decode_mpb_request(Bin),
|
||||
1 = HL_req#mpb_request.do_not_alter,
|
||||
{R, NewS} = do_pb_hl_request(HL_req, make_high_clnt(S)),
|
||||
{machi_pb:encode_mpb_response(R), mode(high, NewS)}
|
||||
end,
|
||||
if RespBin == async_no_response ->
|
||||
net_server_loop(Sock, S2);
|
||||
true ->
|
||||
case gen_tcp:send(Sock, RespBin) of
|
||||
ok ->
|
||||
net_server_loop(Sock, S2);
|
||||
{error, _} ->
|
||||
(catch gen_tcp:close(Sock)),
|
||||
exit(normal)
|
||||
end
|
||||
end;
|
||||
{error, SockError} ->
|
||||
Msg = io_lib:format("Socket error ~w", [SockError]),
|
||||
R = #mpb_ll_response{req_id= <<>>,
|
||||
generic=#mpb_errorresp{code=1, msg=Msg}},
|
||||
_Resp = machi_pb:encode_mpb_ll_response(R),
|
||||
%% TODO: Weird that sometimes neither catch nor try/catch
|
||||
%% can prevent OTP's SASL from logging an error here.
|
||||
%% Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,.......
|
||||
%% TODO: is this what causes the intermittent PULSE deadlock errors?
|
||||
%% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000),
|
||||
(catch gen_tcp:close(Sock)),
|
||||
exit(normal)
|
||||
end.
|
||||
|
||||
maybe_encode_response(async_no_response=X) ->
|
||||
X;
|
||||
maybe_encode_response(R) ->
|
||||
machi_pb:encode_mpb_ll_response(R).
|
||||
|
||||
mode(Mode, #state{pb_mode=undefined}=S) ->
|
||||
S#state{pb_mode=Mode};
|
||||
mode(_, S) ->
|
||||
S.
|
||||
|
||||
make_high_clnt(#state{high_clnt=undefined}=S) ->
|
||||
{ok, Proj} = machi_projection_store:read_latest_projection(
|
||||
S#state.proj_store, private),
|
||||
Ps = [P_srvr || {_, P_srvr} <- orddict:to_list(
|
||||
Proj#projection_v1.members_dict)],
|
||||
{ok, Clnt} = machi_cr_client:start_link(Ps),
|
||||
S#state{high_clnt=Clnt};
|
||||
make_high_clnt(S) ->
|
||||
S.
|
||||
|
||||
do_pb_ll_request(#mpb_ll_request{req_id=ReqID}, #state{pb_mode=high}=S) ->
|
||||
Result = {high_error, 41, "Low protocol request while in high mode"},
|
||||
{machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
|
||||
do_pb_ll_request(PB_request, S) ->
|
||||
Req = machi_pb_translate:from_pb_request(PB_request),
|
||||
{ReqID, Cmd, Result, S2} =
|
||||
case Req of
|
||||
{RqID, {LowCmd, _}=CMD}
|
||||
when LowCmd == low_proj;
|
||||
LowCmd == low_wedge_status; LowCmd == low_list_files ->
|
||||
%% Skip wedge check for projection commands!
|
||||
%% Skip wedge check for these unprivileged commands
|
||||
{Rs, NewS} = do_pb_ll_request3(CMD, S),
|
||||
{RqID, CMD, Rs, NewS};
|
||||
{RqID, CMD} ->
|
||||
EpochID = element(2, CMD), % by common convention
|
||||
{Rs, NewS} = do_pb_ll_request2(EpochID, CMD, S),
|
||||
{RqID, CMD, Rs, NewS}
|
||||
end,
|
||||
{machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
|
||||
|
||||
do_pb_ll_request2(EpochID, CMD, S) ->
|
||||
{Wedged_p, CurrentEpochID} = ets:lookup_element(S#state.etstab, epoch, 2),
|
||||
if Wedged_p == true ->
|
||||
{{error, wedged}, S#state{epoch_id=CurrentEpochID}};
|
||||
is_tuple(EpochID)
|
||||
andalso
|
||||
EpochID /= CurrentEpochID ->
|
||||
{Epoch, _} = EpochID,
|
||||
{CurrentEpoch, _} = CurrentEpochID,
|
||||
if Epoch < CurrentEpoch ->
|
||||
ok;
|
||||
true ->
|
||||
%% We're at same epoch # but different checksum, or
|
||||
%% we're at a newer/bigger epoch #.
|
||||
_ = wedge_myself(S#state.flu_name, CurrentEpochID),
|
||||
ok
|
||||
end,
|
||||
{{error, bad_epoch}, S#state{epoch_id=CurrentEpochID}};
|
||||
true ->
|
||||
do_pb_ll_request3(CMD, S#state{epoch_id=CurrentEpochID})
|
||||
end.
|
||||
|
||||
%% Witness status does not matter below.
|
||||
do_pb_ll_request3({low_echo, _BogusEpochID, Msg}, S) ->
|
||||
{Msg, S};
|
||||
do_pb_ll_request3({low_auth, _BogusEpochID, _User, _Pass}, S) ->
|
||||
{-6, S};
|
||||
do_pb_ll_request3({low_wedge_status, _EpochID}, S) ->
|
||||
{do_server_wedge_status(S), S};
|
||||
do_pb_ll_request3({low_proj, PCMD}, S) ->
|
||||
{do_server_proj_request(PCMD, S), S};
|
||||
%% Witness status *matters* below
|
||||
do_pb_ll_request3({low_append_chunk, _EpochID, PKey, Prefix, Chunk, CSum_tag,
|
||||
CSum, ChunkExtra},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_append_chunk(PKey, Prefix, Chunk, CSum_tag, CSum,
|
||||
ChunkExtra, S), S};
|
||||
do_pb_ll_request3({low_write_chunk, _EpochID, File, Offset, Chunk, CSum_tag,
|
||||
CSum},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, S), S};
|
||||
do_pb_ll_request3({low_read_chunk, _EpochID, File, Offset, Size, Opts},
|
||||
#state{witness=false} = S) ->
|
||||
{do_server_read_chunk(File, Offset, Size, Opts, S), S};
|
||||
do_pb_ll_request3({low_trim_chunk, _EpochID, File, Offset, Size, TriggerGC},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_trim_chunk(File, Offset, Size, TriggerGC, S), S};
|
||||
do_pb_ll_request3({low_checksum_list, _EpochID, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_checksum_listing(File, S), S};
|
||||
do_pb_ll_request3({low_list_files, _EpochID},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_list_files(S), S};
|
||||
do_pb_ll_request3({low_delete_migration, _EpochID, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_delete_migration(File, S),
|
||||
#state{witness=false}=S};
|
||||
do_pb_ll_request3({low_trunc_hack, _EpochID, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_trunc_hack(File, S), S};
|
||||
do_pb_ll_request3(_, #state{witness=true}=S) ->
|
||||
{{error, bad_arg}, S}. % TODO: new status code??
|
||||
|
||||
do_pb_hl_request(#mpb_request{req_id=ReqID}, #state{pb_mode=low}=S) ->
|
||||
Result = {low_error, 41, "High protocol request while in low mode"},
|
||||
{machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
|
||||
do_pb_hl_request(PB_request, S) ->
|
||||
{ReqID, Cmd} = machi_pb_translate:from_pb_request(PB_request),
|
||||
{Result, S2} = do_pb_hl_request2(Cmd, S),
|
||||
{machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
|
||||
|
||||
do_pb_hl_request2({high_echo, Msg}, S) ->
|
||||
{Msg, S};
|
||||
do_pb_hl_request2({high_auth, _User, _Pass}, S) ->
|
||||
{-77, S};
|
||||
do_pb_hl_request2({high_append_chunk, _todoPK, Prefix, ChunkBin, TaggedCSum,
|
||||
ChunkExtra}, #state{high_clnt=Clnt}=S) ->
|
||||
Chunk = {TaggedCSum, ChunkBin},
|
||||
Res = machi_cr_client:append_chunk_extra(Clnt, Prefix, Chunk,
|
||||
ChunkExtra),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_write_chunk, File, Offset, ChunkBin, TaggedCSum},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
Chunk = {TaggedCSum, ChunkBin},
|
||||
Res = machi_cr_client:write_chunk(Clnt, File, Offset, Chunk),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_read_chunk, File, Offset, Size, Opts},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
Res = machi_cr_client:read_chunk(Clnt, File, Offset, Size, Opts),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_trim_chunk, File, Offset, Size},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
Res = machi_cr_client:trim_chunk(Clnt, File, Offset, Size),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_checksum_list, File}, #state{high_clnt=Clnt}=S) ->
|
||||
Res = machi_cr_client:checksum_list(Clnt, File),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_list_files}, #state{high_clnt=Clnt}=S) ->
|
||||
Res = machi_cr_client:list_files(Clnt),
|
||||
{Res, S}.
|
||||
|
||||
do_server_proj_request({get_latest_epochid, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:get_latest_epochid(ProjStore, ProjType);
|
||||
do_server_proj_request({read_latest_projection, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:read_latest_projection(ProjStore, ProjType);
|
||||
do_server_proj_request({read_projection, ProjType, Epoch},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:read(ProjStore, ProjType, Epoch);
|
||||
do_server_proj_request({write_projection, ProjType, Proj},
|
||||
#state{flu_name=FluName, proj_store=ProjStore}) ->
|
||||
if Proj#projection_v1.epoch_number == ?SPAM_PROJ_EPOCH ->
|
||||
%% io:format(user, "DBG ~s ~w ~P\n", [?MODULE, ?LINE, Proj, 5]),
|
||||
Chmgr = machi_flu_psup:make_fitness_regname(FluName),
|
||||
[Map] = Proj#projection_v1.dbg,
|
||||
catch machi_fitness:send_fitness_update_spam(
|
||||
Chmgr, Proj#projection_v1.author_server, Map);
|
||||
true ->
|
||||
catch machi_projection_store:write(ProjStore, ProjType, Proj)
|
||||
end;
|
||||
do_server_proj_request({get_all_projections, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:get_all_projections(ProjStore, ProjType);
|
||||
do_server_proj_request({list_all_projections, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:list_all_projections(ProjStore, ProjType);
|
||||
do_server_proj_request({kick_projection_reaction},
|
||||
#state{flu_name=FluName}) ->
|
||||
%% Tell my chain manager that it might want to react to
|
||||
%% this new world.
|
||||
Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
|
||||
spawn(fun() ->
|
||||
catch machi_chain_manager1:trigger_react_to_env(Chmgr)
|
||||
end),
|
||||
async_no_response.
|
||||
|
||||
do_server_append_chunk(PKey, Prefix, Chunk, CSum_tag, CSum,
|
||||
ChunkExtra, S) ->
|
||||
case sanitize_prefix(Prefix) of
|
||||
ok ->
|
||||
do_server_append_chunk2(PKey, Prefix, Chunk, CSum_tag, CSum,
|
||||
ChunkExtra, S);
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_append_chunk2(_PKey, Prefix, Chunk, CSum_tag, Client_CSum,
|
||||
ChunkExtra, #state{flu_name=FluName,
|
||||
epoch_id=EpochID}=_S) ->
|
||||
%% TODO: Do anything with PKey?
|
||||
try
|
||||
TaggedCSum = check_or_make_tagged_checksum(CSum_tag, Client_CSum,Chunk),
|
||||
R = {seq_append, self(), Prefix, Chunk, TaggedCSum, ChunkExtra, EpochID},
|
||||
FluName ! R,
|
||||
receive
|
||||
{assignment, Offset, File} ->
|
||||
Size = iolist_size(Chunk),
|
||||
{ok, {Offset, Size, File}};
|
||||
witness ->
|
||||
{error, bad_arg};
|
||||
wedged ->
|
||||
{error, wedged}
|
||||
after 10*1000 ->
|
||||
{error, partition}
|
||||
end
|
||||
catch
|
||||
throw:{bad_csum, _CS} ->
|
||||
{error, bad_checksum};
|
||||
error:badarg ->
|
||||
error_logger:error_msg("Message send to ~p gave badarg, make certain server is running with correct registered name\n", [?MODULE]),
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, #state{flu_name=FluName}) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
|
||||
{ok, Pid} ->
|
||||
Meta = [{client_csum_tag, CSum_tag}, {client_csum, CSum}],
|
||||
machi_file_proxy:write(Pid, Offset, Meta, Chunk);
|
||||
{error, trimmed} = Error ->
|
||||
Error
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_read_chunk(File, Offset, Size, Opts, #state{flu_name=FluName})->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
|
||||
{ok, Pid} ->
|
||||
case machi_file_proxy:read(Pid, Offset, Size, Opts) of
|
||||
%% XXX FIXME
|
||||
%% For now we are omiting the checksum data because it blows up
|
||||
%% protobufs.
|
||||
{ok, ChunksAndTrimmed} -> {ok, ChunksAndTrimmed};
|
||||
Other -> Other
|
||||
end;
|
||||
{error, trimmed} = Error ->
|
||||
Error
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_trim_chunk(File, Offset, Size, TriggerGC, #state{flu_name=FluName}) ->
|
||||
lager:debug("Hi there! I'm trimming this: ~s, (~p, ~p), ~p~n",
|
||||
[File, Offset, Size, TriggerGC]),
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
|
||||
{ok, Pid} ->
|
||||
machi_file_proxy:trim(Pid, Offset, Size, TriggerGC);
|
||||
{error, trimmed} = Trimmed ->
|
||||
%% Should be returned back to (maybe) trigger repair
|
||||
Trimmed
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_checksum_listing(File, #state{flu_name=FluName, data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
ok = sync_checksum_file(FluName, File),
|
||||
CSumPath = machi_util:make_checksum_filename(DataDir, File),
|
||||
%% TODO: If this file is legitimately bigger than our
|
||||
%% {packet_size,N} limit, then we'll have a difficult time, eh?
|
||||
case file:read_file(CSumPath) of
|
||||
{ok, Bin} ->
|
||||
if byte_size(Bin) > (?PB_MAX_MSG_SIZE - 1024) ->
|
||||
%% TODO: Fix this limitation by streaming the
|
||||
%% binary in multiple smaller PB messages.
|
||||
%% Also, don't read the file all at once. ^_^
|
||||
error_logger:error_msg("~s:~w oversize ~s\n",
|
||||
[?MODULE, ?LINE, CSumPath]),
|
||||
{error, bad_arg};
|
||||
true ->
|
||||
{ok, Bin}
|
||||
end;
|
||||
{error, enoent} ->
|
||||
{error, no_such_file};
|
||||
{error, _} ->
|
||||
{error, bad_arg}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_list_files(#state{data_dir=DataDir}=_S) ->
|
||||
{_, WildPath} = machi_util:make_data_filename(DataDir, ""),
|
||||
Files = filelib:wildcard("*", WildPath),
|
||||
{ok, [begin
|
||||
{ok, FI} = file:read_file_info(WildPath ++ "/" ++ File),
|
||||
Size = FI#file_info.size,
|
||||
{Size, File}
|
||||
end || File <- Files]}.
|
||||
|
||||
do_server_wedge_status(S) ->
|
||||
{Wedged_p, CurrentEpochID0} = ets:lookup_element(S#state.etstab, epoch, 2),
|
||||
CurrentEpochID = if CurrentEpochID0 == undefined ->
|
||||
?DUMMY_PV1_EPOCH;
|
||||
true ->
|
||||
CurrentEpochID0
|
||||
end,
|
||||
{Wedged_p, CurrentEpochID}.
|
||||
|
||||
do_server_delete_migration(File, #state{data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
{_, Path} = machi_util:make_data_filename(DataDir, File),
|
||||
case file:delete(Path) of
|
||||
ok ->
|
||||
ok;
|
||||
{error, enoent} ->
|
||||
{error, no_such_file};
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_trunc_hack(File, #state{data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
{_, Path} = machi_util:make_data_filename(DataDir, File),
|
||||
case file:open(Path, [read, write, binary, raw]) of
|
||||
{ok, FH} ->
|
||||
try
|
||||
{ok, ?MINIMUM_OFFSET} = file:position(FH,
|
||||
?MINIMUM_OFFSET),
|
||||
ok = file:truncate(FH),
|
||||
ok
|
||||
after
|
||||
file:close(FH)
|
||||
end;
|
||||
{error, enoent} ->
|
||||
{error, no_such_file};
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
append_server_dispatch(From, Prefix, Chunk, CSum, Extra, FluName, EpochId) ->
|
||||
Result = case handle_append(Prefix, Chunk, CSum, Extra, FluName, EpochId) of
|
||||
{ok, File, Offset} ->
|
||||
{assignment, Offset, File};
|
||||
Other ->
|
||||
Other
|
||||
end,
|
||||
From ! Result,
|
||||
exit(normal).
|
||||
|
||||
handle_append(_Prefix, <<>>, _Csum, _Extra, _FluName, _EpochId) ->
|
||||
{error, bad_arg};
|
||||
handle_append(Prefix, Chunk, Csum, Extra, FluName, EpochId) ->
|
||||
Res = machi_flu_filename_mgr:find_or_make_filename_from_prefix(FluName, EpochId, {prefix, Prefix}),
|
||||
case Res of
|
||||
{file, F} ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, F}) of
|
||||
{ok, Pid} ->
|
||||
{Tag, CS} = machi_util:unmake_tagged_csum(Csum),
|
||||
Meta = [{client_csum_tag, Tag}, {client_csum, CS}],
|
||||
machi_file_proxy:append(Pid, Meta, Extra, Chunk);
|
||||
{error, trimmed} = E ->
|
||||
E
|
||||
end;
|
||||
Error ->
|
||||
Error
|
||||
end.
|
||||
|
||||
sanitize_file_string(Str) ->
|
||||
case has_no_prohibited_chars(Str) andalso machi_util:is_valid_filename(Str) of
|
||||
true -> ok;
|
||||
false -> error
|
||||
end.
|
||||
|
||||
has_no_prohibited_chars(Str) ->
|
||||
case re:run(Str, "/") of
|
||||
nomatch ->
|
||||
true;
|
||||
_ ->
|
||||
true
|
||||
end.
|
||||
|
||||
sanitize_prefix(Prefix) ->
|
||||
%% We are using '^' as our component delimiter
|
||||
case re:run(Prefix, "/|\\^") of
|
||||
nomatch ->
|
||||
ok;
|
||||
_ ->
|
||||
error
|
||||
end.
|
||||
|
||||
sync_checksum_file(FluName, File) ->
|
||||
%% We just lookup the pid here - we don't start a proxy server. If
|
||||
%% there isn't a pid for this file, then we just return ok. The
|
||||
%% csum file was synced when the proxy was shutdown.
|
||||
%%
|
||||
%% If there *is* a pid, we call the sync function to ensure the
|
||||
%% csum file is sync'd before we return. (Or an error if we get
|
||||
%% an error).
|
||||
case machi_flu_metadata_mgr:lookup_proxy_pid(FluName, {file, File}) of
|
||||
undefined ->
|
||||
ok;
|
||||
Pid ->
|
||||
machi_file_proxy:sync(Pid, csum)
|
||||
end.
|
||||
|
||||
make_listener_regname(BaseName) ->
|
||||
list_to_atom(atom_to_list(BaseName) ++ "_listener").
|
||||
|
||||
%% This is the name of the projection store that is spawned by the
|
||||
%% *flu*, for use primarily in testing scenarios. In normal use, we
|
||||
|
|
@ -170,8 +756,26 @@ start_listen_server(FluName, TcpPort, Witness_p, DataDir, EtsTab, ProjectionPid,
|
|||
make_projection_server_regname(BaseName) ->
|
||||
list_to_atom(atom_to_list(BaseName) ++ "_pstore").
|
||||
|
||||
ets_table_name(FluName) when is_atom(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_epoch").
|
||||
check_or_make_tagged_checksum(?CSUM_TAG_NONE, _Client_CSum, Chunk) ->
|
||||
%% TODO: If the client was foolish enough to use
|
||||
%% this type of non-checksum, then the client gets
|
||||
%% what it deserves wrt data integrity, alas. In
|
||||
%% the client-side Chain Replication method, each
|
||||
%% server will calculated this independently, which
|
||||
%% isn't exactly what ought to happen for best data
|
||||
%% integrity checking. In server-side CR, the csum
|
||||
%% should be calculated by the head and passed down
|
||||
%% the chain together with the value.
|
||||
CS = machi_util:checksum_chunk(Chunk),
|
||||
machi_util:make_tagged_csum(server_sha, CS);
|
||||
check_or_make_tagged_checksum(?CSUM_TAG_CLIENT_SHA, Client_CSum, Chunk) ->
|
||||
CS = machi_util:checksum_chunk(Chunk),
|
||||
if CS == Client_CSum ->
|
||||
machi_util:make_tagged_csum(server_sha,
|
||||
Client_CSum);
|
||||
true ->
|
||||
throw({bad_csum, CS})
|
||||
end.
|
||||
|
||||
-ifdef(TEST).
|
||||
|
||||
|
|
@ -213,7 +817,7 @@ timing_demo_test2() ->
|
|||
lists:foldl(fun(X, _) ->
|
||||
B = machi_checksums:encode_csum_file_entry_hex(X, 100, CSum),
|
||||
%% file:write(ZZZ, [B, 10]),
|
||||
decode_csum_file_entry_hex(list_to_binary(B))
|
||||
machi_checksums:decode_csum_file_entry_hex(list_to_binary(B))
|
||||
end, x, Xs)
|
||||
end),
|
||||
io:format(user, "~.3f sec\n", [HexUSec / 1000000]),
|
||||
|
|
|
|||
|
|
@ -1,193 +0,0 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc Machi FLU1 append serialization server process
|
||||
|
||||
-module(machi_flu1_append_server).
|
||||
|
||||
-behavior(gen_server).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-ifdef(TEST).
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-endif. % TEST
|
||||
|
||||
-export([start_link/4]).
|
||||
|
||||
-export([init/1]).
|
||||
-export([handle_call/3, handle_cast/2, handle_info/2,
|
||||
terminate/2, code_change/3]).
|
||||
-export([int_update_wedge_state/3, int_wedge_myself/2]).
|
||||
-export([current_state/1, format_state/1]).
|
||||
|
||||
-record(state, {
|
||||
flu_name :: atom(),
|
||||
witness = false :: boolean(),
|
||||
wedged = true :: boolean(),
|
||||
etstab :: ets:tid(),
|
||||
epoch_id :: 'undefined' | machi_dt:epoch_id()
|
||||
}).
|
||||
|
||||
-define(INIT_TIMEOUT, 60*1000).
|
||||
-define(CALL_TIMEOUT, 60*1000).
|
||||
|
||||
-spec start_link(pv1_server(), boolean(), boolean(),
|
||||
undefined | machi_dt:epoch_id()) -> {ok, pid()}.
|
||||
start_link(Fluname, Witness_p, Wedged_p, EpochId) ->
|
||||
%% Reminder: Name is the "main" name of the FLU, i.e., no suffix
|
||||
gen_server:start_link({local, Fluname},
|
||||
?MODULE, [Fluname, Witness_p, Wedged_p, EpochId],
|
||||
[{timeout, ?INIT_TIMEOUT}]).
|
||||
|
||||
-spec current_state(atom() | pid()) -> term().
|
||||
current_state(PidSpec) ->
|
||||
gen_server:call(PidSpec, current_state, ?CALL_TIMEOUT).
|
||||
|
||||
format_state(State) ->
|
||||
Fields = record_info(fields, state),
|
||||
[_Name | Values] = tuple_to_list(State),
|
||||
lists:zip(Fields, Values).
|
||||
|
||||
int_update_wedge_state(PidSpec, Boolean, EpochId)
|
||||
when is_boolean(Boolean), is_tuple(EpochId) ->
|
||||
gen_server:cast(PidSpec, {wedge_state_change, Boolean, EpochId}).
|
||||
|
||||
int_wedge_myself(PidSpec, EpochId)
|
||||
when is_tuple(EpochId) ->
|
||||
gen_server:cast(PidSpec, {wedge_myself, EpochId}).
|
||||
|
||||
init([Fluname, Witness_p, Wedged_p, EpochId]) ->
|
||||
TID = ets:new(machi_flu1:ets_table_name(Fluname),
|
||||
[set, protected, named_table, {read_concurrency, true}]),
|
||||
ets:insert(TID, {epoch, {Wedged_p, EpochId}}),
|
||||
{ok, #state{flu_name=Fluname, witness=Witness_p, wedged=Wedged_p,
|
||||
etstab=TID, epoch_id=EpochId}}.
|
||||
|
||||
handle_call({seq_append, _From2, _NSInfo, _EpochID, _Prefix, _Chunk, _TCSum, _Opts},
|
||||
_From, #state{witness=true}=S) ->
|
||||
%% The FLU's machi_flu1_net_server process ought to filter all
|
||||
%% witness states, but we'll keep this clause for extra
|
||||
%% paranoia.
|
||||
{reply, witness, S};
|
||||
handle_call({seq_append, _From2, _NSInfo, _EpochID, _Prefix, _Chunk, _TCSum, _Opts},
|
||||
_From, #state{wedged=true}=S) ->
|
||||
{reply, wedged, S};
|
||||
handle_call({seq_append, _From2, NSInfo, EpochID,
|
||||
Prefix, Chunk, TCSum, Opts},
|
||||
From, #state{flu_name=FluName, epoch_id=OldEpochId}=S) ->
|
||||
%% Old is the one from our state, plain old 'EpochID' comes
|
||||
%% from the client.
|
||||
_ = case OldEpochId of
|
||||
EpochID ->
|
||||
spawn(fun() ->
|
||||
append_server_dispatch(From, NSInfo,
|
||||
Prefix, Chunk, TCSum, Opts,
|
||||
FluName, EpochID)
|
||||
end),
|
||||
{noreply, S};
|
||||
_ ->
|
||||
{reply, {error, bad_epoch}, S}
|
||||
end;
|
||||
%% TODO: Who sends this message?
|
||||
handle_call(wedge_status, _From,
|
||||
#state{wedged=Wedged_p, epoch_id=EpochId} = S) ->
|
||||
{reply, {wedge_status_reply, Wedged_p, EpochId}, S};
|
||||
handle_call(current_state, _From, S) ->
|
||||
{reply, S, S};
|
||||
handle_call(Else, From, S) ->
|
||||
io:format(user, "~s:handle_call: WHA? from=~w ~w\n", [?MODULE, From, Else]),
|
||||
{noreply, S}.
|
||||
|
||||
handle_cast({wedge_myself, WedgeEpochId},
|
||||
#state{flu_name=FluName, wedged=Wedged_p, epoch_id=OldEpochId}=S) ->
|
||||
if not Wedged_p andalso WedgeEpochId == OldEpochId ->
|
||||
true = ets:insert(S#state.etstab,
|
||||
{epoch, {true, OldEpochId}}),
|
||||
%% Tell my chain manager that it might want to react to
|
||||
%% this new world.
|
||||
Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
|
||||
spawn(fun() ->
|
||||
catch machi_chain_manager1:trigger_react_to_env(Chmgr)
|
||||
end),
|
||||
{noreply, S#state{wedged=true}};
|
||||
true ->
|
||||
{noreply, S}
|
||||
end;
|
||||
handle_cast({wedge_state_change, Boolean, {NewEpoch, _}=NewEpochId},
|
||||
#state{epoch_id=OldEpochId}=S) ->
|
||||
OldEpoch = case OldEpochId of {OldE, _} -> OldE;
|
||||
undefined -> -1
|
||||
end,
|
||||
if NewEpoch >= OldEpoch ->
|
||||
true = ets:insert(S#state.etstab,
|
||||
{epoch, {Boolean, NewEpochId}}),
|
||||
{noreply, S#state{wedged=Boolean, epoch_id=NewEpochId}};
|
||||
true ->
|
||||
{noreply, S}
|
||||
end;
|
||||
handle_cast(Else, S) ->
|
||||
io:format(user, "~s:handle_cast: WHA? ~p\n", [?MODULE, Else]),
|
||||
{noreply, S}.
|
||||
|
||||
handle_info(Else, S) ->
|
||||
io:format(user, "~s:handle_info: WHA? ~p\n", [?MODULE, Else]),
|
||||
{noreply, S}.
|
||||
|
||||
terminate(normal, _S) ->
|
||||
ok;
|
||||
terminate(Reason, _S) ->
|
||||
lager:warning("~s:terminate: ~w", [?MODULE, Reason]),
|
||||
ok.
|
||||
|
||||
code_change(_OldVsn, S, _Extra) ->
|
||||
{ok, S}.
|
||||
|
||||
append_server_dispatch(From, NSInfo,
|
||||
Prefix, Chunk, TCSum, Opts, FluName, EpochId) ->
|
||||
Result = case handle_append(NSInfo,
|
||||
Prefix, Chunk, TCSum, Opts, FluName, EpochId) of
|
||||
{ok, File, Offset} ->
|
||||
{assignment, Offset, File};
|
||||
Other ->
|
||||
Other
|
||||
end,
|
||||
_ = gen_server:reply(From, Result),
|
||||
ok.
|
||||
|
||||
handle_append(NSInfo,
|
||||
Prefix, Chunk, TCSum, Opts, FluName, EpochId) ->
|
||||
Res = machi_flu_filename_mgr:find_or_make_filename_from_prefix(
|
||||
FluName, EpochId, {prefix, Prefix}, NSInfo),
|
||||
case Res of
|
||||
{file, F} ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, F}) of
|
||||
{ok, Pid} ->
|
||||
{Tag, CS} = machi_util:unmake_tagged_csum(TCSum),
|
||||
Meta = [{client_csum_tag, Tag}, {client_csum, CS}],
|
||||
Extra = Opts#append_opts.chunk_extra,
|
||||
machi_file_proxy:append(Pid, Meta, Extra, Chunk);
|
||||
{error, trimmed} = E ->
|
||||
E
|
||||
end;
|
||||
Error ->
|
||||
Error
|
||||
end.
|
||||
|
|
@ -38,71 +38,6 @@
|
|||
%% TODO This EDoc was written first, and the EDoc and also `-type' and
|
||||
%% `-spec' definitions for {@link machi_proxy_flu1_client} and {@link
|
||||
%% machi_cr_client} must be improved.
|
||||
%%
|
||||
%% == Client API implementation notes ==
|
||||
%%
|
||||
%% At the moment, there are several modules that implement various
|
||||
%% subsets of the Machi API. The table below attempts to show how and
|
||||
%% why they differ.
|
||||
%%
|
||||
%% ```
|
||||
%% |--------------------------+-------+-----+------+------+-------+----------------|
|
||||
%% | | PB | | # | | Conn | Epoch & NS |
|
||||
%% | Module name | Level | CR? | FLUS | Impl | Life? | version aware? |
|
||||
%% |--------------------------+-------+-----+------+------+-------+----------------|
|
||||
%% | machi_pb_high_api_client | high | yes | many | proc | long | no |
|
||||
%% | machi_cr_client | low | yes | many | proc | long | no |
|
||||
%% | machi_proxy_flu1_client | low | no | 1 | proc | long | yes |
|
||||
%% | machi_flu1_client | low | no | 1 | lib | short | yes |
|
||||
%% |--------------------------+-------+-----+------+------+-------+----------------|
|
||||
%% '''
|
||||
%%
|
||||
%% In terms of use and API layering, the table rows are in highest`->'lowest
|
||||
%% order: each level calls the layer immediately below it.
|
||||
%%
|
||||
%% <dl>
|
||||
%% <dt> <b> PB Level</b> </dt>
|
||||
%% <dd> The Protocol Buffers API is divided logically into two levels,
|
||||
%% "low" and "high". The low-level protocol is used for intra-chain
|
||||
%% communication. The high-level protocol is used for clients outside
|
||||
%% of a Machi chain or Machi cluster of chains.
|
||||
%% </dd>
|
||||
%% <dt> <b> CR?</b> </dt>
|
||||
%% <dd> Does this API support (directly or indirectly) Chain
|
||||
%% Replication? If `no', then the API has no awareness of multiple
|
||||
%% replicas of any file or file chunk; unaware clients can only
|
||||
%% perform operations at a single Machi FLU's file service or
|
||||
%% projection store service.
|
||||
%% </dd>
|
||||
%% <dt> <b> # FLUs</b> </dt>
|
||||
%% <dd> Now many FLUs does this API layer communicate with
|
||||
%% simultaneously? Note that there is a one-to-one correspondence
|
||||
%% between this value and the "CR?" column's value.
|
||||
%% </dd>
|
||||
%% <dt> <b> Impl</b> </dt>
|
||||
%% <dd> Implementation: library-only or an Erlang process,
|
||||
%% e.g., `gen_server'.
|
||||
%% </dd>
|
||||
%% <dt> <b> Conn Life?</b> </dt>
|
||||
%% <dd> Expected TCP session connection life: short or long. At the
|
||||
%% lowest level, the {@link machi_flu1_client} API implementation takes
|
||||
%% no effort to reconnect to a remote FLU when its single TCP session
|
||||
%% is broken. For long-lived connection life APIs, the server side will
|
||||
%% automatically attempt to reconnect to remote FLUs when a TCP session
|
||||
%% is broken.
|
||||
%% </dd>
|
||||
%% <dt> <b> Epoch & NS version aware?</b> </dt>
|
||||
%% <dd> Are clients of this API responsible for knowing a chain's EpochID
|
||||
%% and namespace version numbers? If `no', then the server side of the
|
||||
%% API will automatically attempt to discover/re-discover the EpochID and
|
||||
%% namespace version numbers whenever they change.
|
||||
%% </dd>
|
||||
%% </dl>
|
||||
%%
|
||||
%% The only protocol that we expect to be used by entities outside of
|
||||
%% a single Machi chain or a multi-chain cluster is the "high"
|
||||
%% Protocol Buffers API. The {@link riak_pb_high_api_client} module
|
||||
%% is an Erlang reference implementation of this PB API.
|
||||
|
||||
-module(machi_flu1_client).
|
||||
|
||||
|
|
@ -115,15 +50,14 @@
|
|||
-include_lib("pulse_otp/include/pulse_otp.hrl").
|
||||
-endif.
|
||||
|
||||
-define(SHORT_TIMEOUT, 2500).
|
||||
-define(LONG_TIMEOUT, (60*1000)).
|
||||
-define(HARD_TIMEOUT, 2500).
|
||||
|
||||
-export([
|
||||
%% File API
|
||||
append_chunk/6, append_chunk/7,
|
||||
append_chunk/8, append_chunk/9,
|
||||
read_chunk/7, read_chunk/8,
|
||||
checksum_list/2, checksum_list/3,
|
||||
append_chunk/4, append_chunk/5,
|
||||
append_chunk_extra/5, append_chunk_extra/6,
|
||||
read_chunk/6, read_chunk/7,
|
||||
checksum_list/3, checksum_list/4,
|
||||
list_files/2, list_files/3,
|
||||
wedge_status/1, wedge_status/2,
|
||||
|
||||
|
|
@ -145,113 +79,106 @@
|
|||
]).
|
||||
%% For "internal" replication only.
|
||||
-export([
|
||||
write_chunk/7, write_chunk/8,
|
||||
trim_chunk/6,
|
||||
write_chunk/5, write_chunk/6,
|
||||
trim_chunk/5,
|
||||
delete_migration/3, delete_migration/4,
|
||||
trunc_hack/3, trunc_hack/4
|
||||
]).
|
||||
|
||||
-type port_wrap() :: {w,atom(),term()}.
|
||||
|
||||
-spec append_chunk(port_wrap(),
|
||||
'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
|
||||
machi_dt:file_prefix(), machi_dt:chunk(),
|
||||
machi_dt:chunk_csum()) ->
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix'.
|
||||
|
||||
-spec append_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk()) ->
|
||||
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
|
||||
append_chunk(Sock, NSInfo, EpochID, Prefix, Chunk, CSum) ->
|
||||
append_chunk(Sock, NSInfo, EpochID, Prefix, Chunk, CSum,
|
||||
#append_opts{}, ?LONG_TIMEOUT).
|
||||
append_chunk(Sock, EpochID, Prefix, Chunk) ->
|
||||
append_chunk2(Sock, EpochID, Prefix, Chunk, 0).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' and also request an additional `Extra' bytes.
|
||||
%%
|
||||
%% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
|
||||
%% the file offsets that follow `Chunk''s position for the following 4K will
|
||||
%% be reserved by the file sequencer for later write(s) by the
|
||||
%% `write_chunk()' API.
|
||||
%% with `Prefix'.
|
||||
|
||||
-spec append_chunk(machi_dt:inet_host(), machi_dt:inet_port(),
|
||||
'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
|
||||
machi_dt:file_prefix(), machi_dt:chunk(),
|
||||
machi_dt:chunk_csum()) ->
|
||||
machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk()) ->
|
||||
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
|
||||
append_chunk(Host, TcpPort, NSInfo, EpochID, Prefix, Chunk, CSum) ->
|
||||
append_chunk(Host, TcpPort, NSInfo, EpochID, Prefix, Chunk, CSum,
|
||||
#append_opts{}, ?LONG_TIMEOUT).
|
||||
|
||||
-spec append_chunk(port_wrap(),
|
||||
'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
|
||||
machi_dt:file_prefix(), machi_dt:chunk(),
|
||||
machi_dt:chunk_csum(), machi_dt:append_opts(), timeout()) ->
|
||||
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
|
||||
append_chunk(Sock, NSInfo0, EpochID, Prefix, Chunk, CSum, Opts, Timeout) ->
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
append_chunk2(Sock, NSInfo, EpochID, Prefix, Chunk, CSum, Opts, Timeout).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' and also request an additional `Extra' bytes.
|
||||
%%
|
||||
%% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
|
||||
%% the file offsets that follow `Chunk''s position for the following 4K will
|
||||
%% be reserved by the file sequencer for later write(s) by the
|
||||
%% `write_chunk()' API.
|
||||
|
||||
-spec append_chunk(machi_dt:inet_host(), machi_dt:inet_port(),
|
||||
'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
|
||||
machi_dt:file_prefix(), machi_dt:chunk(),
|
||||
machi_dt:chunk_csum(), machi_dt:append_opts(), timeout()) ->
|
||||
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
|
||||
append_chunk(Host, TcpPort, NSInfo0, EpochID,
|
||||
Prefix, Chunk, CSum, Opts, Timeout) ->
|
||||
append_chunk(Host, TcpPort, EpochID, Prefix, Chunk) ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
try
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
append_chunk2(Sock, NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum, Opts, Timeout)
|
||||
append_chunk2(Sock, EpochID, Prefix, Chunk, 0)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' and also request an additional `Extra' bytes.
|
||||
%%
|
||||
%% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
|
||||
%% the file offsets that follow `Chunk''s position for the following 4K will
|
||||
%% be reserved by the file sequencer for later write(s) by the
|
||||
%% `write_chunk()' API.
|
||||
|
||||
-spec append_chunk_extra(port_wrap(), machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) ->
|
||||
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
|
||||
append_chunk_extra(Sock, EpochID, Prefix, Chunk, ChunkExtra)
|
||||
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
|
||||
append_chunk2(Sock, EpochID, Prefix, Chunk, ChunkExtra).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' and also request an additional `Extra' bytes.
|
||||
%%
|
||||
%% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
|
||||
%% the file offsets that follow `Chunk''s position for the following 4K will
|
||||
%% be reserved by the file sequencer for later write(s) by the
|
||||
%% `write_chunk()' API.
|
||||
|
||||
-spec append_chunk_extra(machi_dt:inet_host(), machi_dt:inet_port(),
|
||||
machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) ->
|
||||
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
|
||||
append_chunk_extra(Host, TcpPort, EpochID, Prefix, Chunk, ChunkExtra)
|
||||
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
try
|
||||
append_chunk2(Sock, EpochID, Prefix, Chunk, ChunkExtra)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
|
||||
%% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
|
||||
|
||||
-spec read_chunk(port_wrap(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size(),
|
||||
machi_dt:read_opts_x()) ->
|
||||
{ok, {[machi_dt:chunk_summary()], [machi_dt:chunk_pos()]}} |
|
||||
-spec read_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size(),
|
||||
proplists:proplist()) ->
|
||||
{ok, machi_dt:chunk_s()} |
|
||||
{error, machi_dt:error_general() | 'not_written' | 'partial_read'} |
|
||||
{error, term()}.
|
||||
read_chunk(Sock, NSInfo0, EpochID, File, Offset, Size, Opts0)
|
||||
read_chunk(Sock, EpochID, File, Offset, Size, Opts)
|
||||
when Offset >= ?MINIMUM_OFFSET, Size >= 0 ->
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
Opts = machi_util:read_opts_default(Opts0),
|
||||
read_chunk2(Sock, NSInfo, EpochID, File, Offset, Size, Opts).
|
||||
read_chunk2(Sock, EpochID, File, Offset, Size, Opts).
|
||||
|
||||
%% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
|
||||
|
||||
-spec read_chunk(machi_dt:inet_host(), machi_dt:inet_port(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
|
||||
-spec read_chunk(machi_dt:inet_host(), machi_dt:inet_port(), machi_dt:epoch_id(),
|
||||
machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size(),
|
||||
machi_dt:read_opts_x()) ->
|
||||
{ok, [machi_dt:chunk_summary()]} |
|
||||
proplists:proplist()) ->
|
||||
{ok, machi_dt:chunk_s()} |
|
||||
{error, machi_dt:error_general() | 'not_written' | 'partial_read'} |
|
||||
{error, term()}.
|
||||
read_chunk(Host, TcpPort, NSInfo0, EpochID, File, Offset, Size, Opts0)
|
||||
read_chunk(Host, TcpPort, EpochID, File, Offset, Size, Opts)
|
||||
when Offset >= ?MINIMUM_OFFSET, Size >= 0 ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
Opts = machi_util:read_opts_default(Opts0),
|
||||
try
|
||||
read_chunk2(Sock, NSInfo, EpochID, File, Offset, Size, Opts)
|
||||
read_chunk2(Sock, EpochID, File, Offset, Size, Opts)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
|
||||
-spec checksum_list(port_wrap(), machi_dt:file_name()) ->
|
||||
-spec checksum_list(port_wrap(), machi_dt:epoch_id(), machi_dt:file_name()) ->
|
||||
{ok, binary()} |
|
||||
{error, machi_dt:error_general() | 'no_such_file' | 'partial_read'} |
|
||||
{error, term()}.
|
||||
checksum_list(Sock, File) ->
|
||||
checksum_list2(Sock, File).
|
||||
checksum_list(Sock, EpochID, File) ->
|
||||
checksum_list2(Sock, EpochID, File).
|
||||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
%%
|
||||
|
|
@ -275,13 +202,13 @@ checksum_list(Sock, File) ->
|
|||
%% Details of the encoding used inside the `binary()' blog can be found
|
||||
%% in the EDoc comments for {@link machi_flu1:decode_csum_file_entry/1}.
|
||||
|
||||
-spec checksum_list(machi_dt:inet_host(), machi_dt:inet_port(), machi_dt:file_name()) ->
|
||||
-spec checksum_list(machi_dt:inet_host(), machi_dt:inet_port(), machi_dt:epoch_id(), machi_dt:file_name()) ->
|
||||
{ok, binary()} |
|
||||
{error, machi_dt:error_general() | 'no_such_file'} | {error, term()}.
|
||||
checksum_list(Host, TcpPort, File) when is_integer(TcpPort) ->
|
||||
checksum_list(Host, TcpPort, EpochID, File) when is_integer(TcpPort) ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
try
|
||||
checksum_list2(Sock, File)
|
||||
checksum_list2(Sock, EpochID, File)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
|
|
@ -308,7 +235,7 @@ list_files(Host, TcpPort, EpochID) when is_integer(TcpPort) ->
|
|||
%% @doc Fetch the wedge status from the remote FLU.
|
||||
|
||||
-spec wedge_status(port_wrap()) ->
|
||||
{ok, {boolean(), machi_dt:epoch_id(), machi_dt:namespace_version(),machi_dt:namespace()}} | {error, term()}.
|
||||
{ok, {boolean(), machi_dt:epoch_id()}} | {error, term()}.
|
||||
|
||||
wedge_status(Sock) ->
|
||||
wedge_status2(Sock).
|
||||
|
|
@ -316,7 +243,7 @@ wedge_status(Sock) ->
|
|||
%% @doc Fetch the wedge status from the remote FLU.
|
||||
|
||||
-spec wedge_status(machi_dt:inet_host(), machi_dt:inet_port()) ->
|
||||
{ok, {boolean(), machi_dt:epoch_id(), machi_dt:namespace_version(),machi_dt:namespace()}} | {error, term()}.
|
||||
{ok, {boolean(), machi_dt:epoch_id()}} | {error, term()}.
|
||||
wedge_status(Host, TcpPort) when is_integer(TcpPort) ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
try
|
||||
|
|
@ -527,25 +454,23 @@ disconnect(_) ->
|
|||
%% @doc Restricted API: Write a chunk of already-sequenced data to
|
||||
%% `File' at `Offset'.
|
||||
|
||||
-spec write_chunk(port_wrap(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk(), machi_dt:chunk_csum()) ->
|
||||
-spec write_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk()) ->
|
||||
ok | {error, machi_dt:error_general()} | {error, term()}.
|
||||
write_chunk(Sock, NSInfo0, EpochID, File, Offset, Chunk, CSum)
|
||||
write_chunk(Sock, EpochID, File, Offset, Chunk)
|
||||
when Offset >= ?MINIMUM_OFFSET ->
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
write_chunk2(Sock, NSInfo, EpochID, File, Offset, Chunk, CSum).
|
||||
write_chunk2(Sock, EpochID, File, Offset, Chunk).
|
||||
|
||||
%% @doc Restricted API: Write a chunk of already-sequenced data to
|
||||
%% `File' at `Offset'.
|
||||
|
||||
-spec write_chunk(machi_dt:inet_host(), machi_dt:inet_port(),
|
||||
'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk(), machi_dt:chunk_csum()) ->
|
||||
machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk()) ->
|
||||
ok | {error, machi_dt:error_general()} | {error, term()}.
|
||||
write_chunk(Host, TcpPort, NSInfo0, EpochID, File, Offset, Chunk, CSum)
|
||||
write_chunk(Host, TcpPort, EpochID, File, Offset, Chunk)
|
||||
when Offset >= ?MINIMUM_OFFSET ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
try
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
write_chunk2(Sock, NSInfo, EpochID, File, Offset, Chunk, CSum)
|
||||
write_chunk2(Sock, EpochID, File, Offset, Chunk)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
|
|
@ -553,18 +478,16 @@ write_chunk(Host, TcpPort, NSInfo0, EpochID, File, Offset, Chunk, CSum)
|
|||
%% @doc Restricted API: Write a chunk of already-sequenced data to
|
||||
%% `File' at `Offset'.
|
||||
|
||||
-spec trim_chunk(port_wrap(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size()) ->
|
||||
-spec trim_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size()) ->
|
||||
ok | {error, machi_dt:error_general()} | {error, term()}.
|
||||
trim_chunk(Sock, NSInfo0, EpochID, File0, Offset, Size)
|
||||
trim_chunk(Sock, EpochID, File0, Offset, Size)
|
||||
when Offset >= ?MINIMUM_OFFSET ->
|
||||
ReqID = <<"id">>,
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
#ns_info{version=NSVersion, name=NS} = NSInfo,
|
||||
File = machi_util:make_binary(File0),
|
||||
true = (Offset >= ?MINIMUM_OFFSET),
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID,
|
||||
{low_trim_chunk, NSVersion, NS, EpochID, File, Offset, Size, 0}),
|
||||
{low_trim_chunk, EpochID, File, Offset, Size, 0}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
%% @doc Restricted API: Delete a file after it has been successfully
|
||||
|
|
@ -611,88 +534,83 @@ trunc_hack(Host, TcpPort, EpochID, File) when is_integer(TcpPort) ->
|
|||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
read_chunk2(Sock, NSInfo, EpochID, File0, Offset, Size, Opts) ->
|
||||
read_chunk2(Sock, EpochID, File0, Offset, Size, Opts) ->
|
||||
ReqID = <<"id">>,
|
||||
#ns_info{version=NSVersion, name=NS} = NSInfo,
|
||||
File = machi_util:make_binary(File0),
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID,
|
||||
{low_read_chunk, NSVersion, NS, EpochID, File, Offset, Size, Opts}),
|
||||
{low_read_chunk, EpochID, File, Offset, Size, Opts}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
append_chunk2(Sock, NSInfo, EpochID,
|
||||
Prefix0, Chunk, CSum0, Opts, Timeout) ->
|
||||
append_chunk2(Sock, EpochID, Prefix0, Chunk0, ChunkExtra) ->
|
||||
ReqID = <<"id">>,
|
||||
{Chunk, CSum_tag, CSum} =
|
||||
case Chunk0 of
|
||||
X when is_binary(X) ->
|
||||
{Chunk0, ?CSUM_TAG_NONE, <<>>};
|
||||
{ChunkCSum, Chk} ->
|
||||
{Tag, CS} = machi_util:unmake_tagged_csum(ChunkCSum),
|
||||
{Chk, Tag, CS}
|
||||
end,
|
||||
PKey = <<>>, % TODO
|
||||
Prefix = machi_util:make_binary(Prefix0),
|
||||
{CSum_tag, CSum} = case CSum0 of
|
||||
<<>> ->
|
||||
{?CSUM_TAG_NONE, <<>>};
|
||||
{_Tag, _CS} ->
|
||||
CSum0;
|
||||
B when is_binary(B) ->
|
||||
machi_util:unmake_tagged_csum(CSum0)
|
||||
end,
|
||||
#ns_info{version=NSVersion, name=NS, locator=NSLocator} = NSInfo,
|
||||
%% NOTE: The tuple position of NSLocator is a bit odd, because EpochID
|
||||
%% _must_ be in the 4th position (as NSV & NS must be in 2nd & 3rd).
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID,
|
||||
{low_append_chunk, NSVersion, NS, EpochID, NSLocator,
|
||||
Prefix, Chunk, CSum_tag, CSum, Opts}),
|
||||
do_pb_request_common(Sock, ReqID, Req, true, Timeout).
|
||||
{low_append_chunk, EpochID, PKey, Prefix, Chunk, CSum_tag, CSum,
|
||||
ChunkExtra}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
write_chunk2(Sock, NSInfo, EpochID, File0, Offset, Chunk, CSum0) ->
|
||||
write_chunk2(Sock, EpochID, File0, Offset, Chunk0) ->
|
||||
ReqID = <<"id">>,
|
||||
#ns_info{version=NSVersion, name=NS} = NSInfo,
|
||||
File = machi_util:make_binary(File0),
|
||||
true = (Offset >= ?MINIMUM_OFFSET),
|
||||
{CSum_tag, CSum} = case CSum0 of
|
||||
<<>> ->
|
||||
{?CSUM_TAG_NONE, <<>>};
|
||||
{_Tag, _CS} ->
|
||||
CSum0;
|
||||
B when is_binary(B) ->
|
||||
machi_util:unmake_tagged_csum(CSum0)
|
||||
end,
|
||||
{Chunk, CSum_tag, CSum} =
|
||||
case Chunk0 of
|
||||
X when is_binary(X) ->
|
||||
{Chunk0, ?CSUM_TAG_NONE, <<>>};
|
||||
{ChunkCSum, Chk} ->
|
||||
{Tag, CS} = machi_util:unmake_tagged_csum(ChunkCSum),
|
||||
{Chk, Tag, CS}
|
||||
end,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID,
|
||||
{low_write_chunk, NSVersion, NS, EpochID, File, Offset, Chunk, CSum_tag, CSum}),
|
||||
{low_write_chunk, EpochID, File, Offset, Chunk, CSum_tag, CSum}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
list2(Sock, EpochID) ->
|
||||
ReqID = <<"id">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_skip_wedge, {low_list_files, EpochID}}),
|
||||
ReqID, {low_list_files, EpochID}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
wedge_status2(Sock) ->
|
||||
ReqID = <<"id">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_skip_wedge, {low_wedge_status}}),
|
||||
ReqID, {low_wedge_status, undefined}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
echo2(Sock, Message) ->
|
||||
ReqID = <<"id">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_skip_wedge, {low_echo, Message}}),
|
||||
ReqID, {low_echo, undefined, Message}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
checksum_list2(Sock, File) ->
|
||||
checksum_list2(Sock, EpochID, File) ->
|
||||
ReqID = <<"id">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_skip_wedge, {low_checksum_list, File}}),
|
||||
ReqID, {low_checksum_list, EpochID, File}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
delete_migration2(Sock, EpochID, File) ->
|
||||
ReqID = <<"id">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_skip_wedge, {low_delete_migration, EpochID, File}}),
|
||||
ReqID, {low_delete_migration, EpochID, File}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
trunc_hack2(Sock, EpochID, File) ->
|
||||
ReqID = <<"id-trunc">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_skip_wedge, {low_trunc_hack, EpochID, File}}),
|
||||
ReqID, {low_trunc_hack, EpochID, File}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
get_latest_epochid2(Sock, ProjType) ->
|
||||
|
|
@ -735,18 +653,18 @@ kick_projection_reaction2(Sock, _Options) ->
|
|||
ReqID = <<42>>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_proj, {kick_projection_reaction}}),
|
||||
do_pb_request_common(Sock, ReqID, Req, false, ?LONG_TIMEOUT).
|
||||
do_pb_request_common(Sock, ReqID, Req, false).
|
||||
|
||||
do_pb_request_common(Sock, ReqID, Req) ->
|
||||
do_pb_request_common(Sock, ReqID, Req, true, ?LONG_TIMEOUT).
|
||||
do_pb_request_common(Sock, ReqID, Req, true).
|
||||
|
||||
do_pb_request_common(Sock, ReqID, Req, GetReply_p, Timeout) ->
|
||||
do_pb_request_common(Sock, ReqID, Req, GetReply_p) ->
|
||||
erase(bad_sock),
|
||||
try
|
||||
ReqBin = list_to_binary(machi_pb:encode_mpb_ll_request(Req)),
|
||||
ok = w_send(Sock, ReqBin),
|
||||
if GetReply_p ->
|
||||
case w_recv(Sock, 0, Timeout) of
|
||||
case w_recv(Sock, 0) of
|
||||
{ok, RespBin} ->
|
||||
Resp = machi_pb:decode_mpb_ll_response(RespBin),
|
||||
{ReqID2, Reply} = machi_pb_translate:from_pb_response(Resp),
|
||||
|
|
@ -792,7 +710,7 @@ w_connect(#p_srvr{proto_mod=?MODULE, address=Host, port=Port, props=Props}=_P)->
|
|||
case proplists:get_value(session_proto, Props, tcp) of
|
||||
tcp ->
|
||||
put(xxx, goofus),
|
||||
Sock = machi_util:connect(Host, Port, ?SHORT_TIMEOUT),
|
||||
Sock = machi_util:connect(Host, Port, ?HARD_TIMEOUT),
|
||||
put(xxx, Sock),
|
||||
ok = inet:setopts(Sock, ?PB_PACKET_OPTS),
|
||||
{w,tcp,Sock};
|
||||
|
|
@ -816,8 +734,8 @@ w_close({w,tcp,Sock}) ->
|
|||
catch gen_tcp:close(Sock),
|
||||
ok.
|
||||
|
||||
w_recv({w,tcp,Sock}, Amt, Timeout) ->
|
||||
gen_tcp:recv(Sock, Amt, Timeout).
|
||||
w_recv({w,tcp,Sock}, Amt) ->
|
||||
gen_tcp:recv(Sock, Amt, ?HARD_TIMEOUT).
|
||||
|
||||
w_send({w,tcp,Sock}, IoData) ->
|
||||
gen_tcp:send(Sock, IoData).
|
||||
|
|
|
|||
|
|
@ -1,634 +0,0 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc Ranch protocol callback module to handle PB protocol over
|
||||
%% transport, including both high and low modes.
|
||||
|
||||
%% TODO
|
||||
%% - Two modes, high and low should be separated at listener level?
|
||||
|
||||
-module(machi_flu1_net_server).
|
||||
|
||||
-behaviour(gen_server).
|
||||
-behaviour(ranch_protocol).
|
||||
|
||||
-export([start_link/4]).
|
||||
-export([init/1]).
|
||||
-export([handle_call/3, handle_cast/2, handle_info/2,
|
||||
terminate/2, code_change/3]).
|
||||
|
||||
-include_lib("kernel/include/file.hrl").
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_pb.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-ifdef(TEST).
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-endif. % TEST
|
||||
|
||||
-record(state, {
|
||||
%% Ranch's transport management stuff
|
||||
ref :: ranch:ref(),
|
||||
socket :: socket(),
|
||||
transport :: module(),
|
||||
|
||||
%% Machi FLU configurations, common for low and high
|
||||
data_dir :: string(),
|
||||
witness :: boolean(),
|
||||
pb_mode :: undefined | high | low,
|
||||
%% - Used in projection related requests in low mode
|
||||
%% - Used in spawning CR client in high mode
|
||||
proj_store :: pid(),
|
||||
|
||||
%% Low mode only items
|
||||
%% Current best knowledge, used for wedge_self / bad_epoch check
|
||||
epoch_id :: undefined | machi_dt:epoch_id(),
|
||||
%% Used in dispatching append_chunk* reqs to the
|
||||
%% append serializing process
|
||||
flu_name :: pv1_server(),
|
||||
%% Used in server_wedge_status to lookup the table
|
||||
epoch_tab :: ets:tab(),
|
||||
%% Clustering: cluster map version number
|
||||
namespace_version = 0 :: machi_dt:namespace_version(),
|
||||
%% Clustering: my (and my chain's) assignment to a specific namespace
|
||||
namespace = <<>> :: machi_dt:namespace(),
|
||||
|
||||
%% High mode only
|
||||
high_clnt :: pid(),
|
||||
|
||||
%% anything you want
|
||||
props = [] :: proplists:proplist()
|
||||
}).
|
||||
|
||||
-type socket() :: any().
|
||||
-type state() :: #state{}.
|
||||
|
||||
-spec start_link(ranch:ref(), socket(), module(), [term()]) -> {ok, pid()}.
|
||||
start_link(Ref, Socket, Transport, [FluName, Witness, DataDir, EpochTab, ProjStore, Props]) ->
|
||||
NS = proplists:get_value(namespace, Props, <<>>),
|
||||
true = is_binary(NS),
|
||||
proc_lib:start_link(?MODULE, init, [#state{ref=Ref,
|
||||
socket=Socket,
|
||||
transport=Transport,
|
||||
flu_name=FluName,
|
||||
witness=Witness,
|
||||
data_dir=DataDir,
|
||||
epoch_tab=EpochTab,
|
||||
proj_store=ProjStore,
|
||||
namespace=NS,
|
||||
props=Props}]).
|
||||
|
||||
-spec init(state()) -> no_return().
|
||||
init(#state{ref=Ref, socket=Socket, transport=Transport}=State) ->
|
||||
ok = proc_lib:init_ack({ok, self()}),
|
||||
ok = ranch:accept_ack(Ref),
|
||||
{_Wedged_p, CurrentEpochID} = lookup_epoch(State),
|
||||
ok = Transport:setopts(Socket, [{active, once}|?PB_PACKET_OPTS]),
|
||||
gen_server:enter_loop(?MODULE, [], State#state{epoch_id=CurrentEpochID}).
|
||||
|
||||
handle_call(Request, _From, S) ->
|
||||
lager:warning("~s:handle_call UNKNOWN message: ~w", [?MODULE, Request]),
|
||||
Reply = {error, {unknown_message, Request}},
|
||||
{reply, Reply, S}.
|
||||
|
||||
handle_cast(_Msg, S) ->
|
||||
lager:warning("~s:handle_cast UNKNOWN message: ~w", [?MODULE, _Msg]),
|
||||
{noreply, S}.
|
||||
|
||||
%% TODO: Other transport support needed?? TLS/SSL, SCTP
|
||||
handle_info({tcp, Socket, Data}=_Info, #state{socket=Socket}=S) ->
|
||||
lager:debug("~s:handle_info: ~w", [?MODULE, _Info]),
|
||||
transport_received(Socket, Data, S);
|
||||
handle_info({tcp_closed, Socket}=_Info, #state{socket=Socket}=S) ->
|
||||
lager:debug("~s:handle_info: ~w", [?MODULE, _Info]),
|
||||
transport_closed(Socket, S);
|
||||
handle_info({tcp_error, Socket, Reason}=_Info, #state{socket=Socket}=S) ->
|
||||
lager:warning("~s:handle_info (socket=~w) tcp_error: ~w", [?MODULE, Socket, Reason]),
|
||||
transport_error(Socket, Reason, S);
|
||||
handle_info(_Info, S) ->
|
||||
lager:warning("~s:handle_info UNKNOWN message: ~w", [?MODULE, _Info]),
|
||||
{noreply, S}.
|
||||
|
||||
terminate(normal, #state{socket=undefined}=_S) ->
|
||||
ok;
|
||||
terminate(Reason, #state{socket=undefined}=_S) ->
|
||||
lager:warning("~s:terminate (socket=undefined): ~w", [?MODULE, Reason]),
|
||||
ok;
|
||||
terminate(normal, #state{socket=Socket}=_S) ->
|
||||
(catch gen_tcp:close(Socket)),
|
||||
ok;
|
||||
terminate(Reason, #state{socket=Socket}=_S) ->
|
||||
lager:warning("~s:terminate (socket=Socket): ~w", [?MODULE, Reason]),
|
||||
(catch gen_tcp:close(Socket)),
|
||||
ok.
|
||||
|
||||
code_change(_OldVsn, S, _Extra) ->
|
||||
{ok, S}.
|
||||
|
||||
%% -- private
|
||||
|
||||
%%%% Common transport handling
|
||||
|
||||
-spec transport_received(socket(), machi_dt:chunk(), state()) ->
|
||||
{noreply, state()}.
|
||||
transport_received(Socket, <<"QUIT\n">>, #state{socket=Socket}=S) ->
|
||||
{stop, normal, S};
|
||||
transport_received(Socket, Bin, #state{transport=Transport}=S) ->
|
||||
{RespBin, S2} =
|
||||
case machi_pb:decode_mpb_ll_request(Bin) of
|
||||
LL_req when LL_req#mpb_ll_request.do_not_alter == 2 ->
|
||||
{R, NewS} = do_pb_ll_request(LL_req, S),
|
||||
{maybe_encode_response(R), set_mode(low, NewS)};
|
||||
_ ->
|
||||
HL_req = machi_pb:decode_mpb_request(Bin),
|
||||
1 = HL_req#mpb_request.do_not_alter,
|
||||
{R, NewS} = do_pb_hl_request(HL_req, make_high_clnt(S)),
|
||||
{machi_pb:encode_mpb_response(R), set_mode(high, NewS)}
|
||||
end,
|
||||
case RespBin of
|
||||
async_no_response ->
|
||||
Transport:setopts(Socket, [{active, once}]),
|
||||
{noreply, S2};
|
||||
_ ->
|
||||
case Transport:send(Socket, RespBin) of
|
||||
ok ->
|
||||
Transport:setopts(Socket, [{active, once}]),
|
||||
{noreply, S2};
|
||||
{error, Reason} ->
|
||||
transport_error(Socket, Reason, S2)
|
||||
end
|
||||
end.
|
||||
|
||||
-spec transport_closed(socket(), state()) -> {stop, term(), state()}.
|
||||
transport_closed(_Socket, S) ->
|
||||
{stop, normal, S}.
|
||||
|
||||
-spec transport_error(socket(), term(), state()) -> no_return().
|
||||
transport_error(Socket, Reason, #state{transport=Transport}=_S) ->
|
||||
Msg = io_lib:format("Socket error ~w", [Reason]),
|
||||
R = #mpb_ll_response{req_id= <<>>,
|
||||
generic=#mpb_errorresp{code=1, msg=Msg}},
|
||||
_Resp = machi_pb:encode_mpb_ll_response(R),
|
||||
%% TODO for TODO comments: comments below with four %s are copy-n-paste'd,
|
||||
%% then it should be considered they are still open and should be addressed.
|
||||
%%%% TODO: Weird that sometimes neither catch nor try/catch
|
||||
%%%% can prevent OTP's SASL from logging an error here.
|
||||
%%%% Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,.......
|
||||
%%%% TODO: is this what causes the intermittent PULSE deadlock errors?
|
||||
%%%% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000),
|
||||
(catch Transport:close(Socket)),
|
||||
_ = lager:warning("Socket error (~w -> ~w): ~w",
|
||||
[Transport:sockname(Socket), Transport:peername(Socket), Reason]),
|
||||
%% TODO: better to exit with `Reason' without logging?
|
||||
exit(normal).
|
||||
|
||||
maybe_encode_response(async_no_response=R) ->
|
||||
R;
|
||||
maybe_encode_response(R) ->
|
||||
machi_pb:encode_mpb_ll_response(R).
|
||||
|
||||
set_mode(Mode, #state{pb_mode=undefined}=S) ->
|
||||
S#state{pb_mode=Mode};
|
||||
set_mode(_, S) ->
|
||||
S.
|
||||
|
||||
%%%% Low PB mode %%%%
|
||||
|
||||
do_pb_ll_request(#mpb_ll_request{req_id=ReqID}, #state{pb_mode=high}=S) ->
|
||||
Result = {high_error, 41, "Low protocol request while in high mode"},
|
||||
{machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
|
||||
do_pb_ll_request(PB_request, S) ->
|
||||
Req = machi_pb_translate:from_pb_request(PB_request),
|
||||
{ReqID, Cmd, Result, S2} =
|
||||
case Req of
|
||||
{RqID, {low_skip_wedge, LowSubCmd}=Cmd0} ->
|
||||
%% Skip wedge check for these unprivileged commands
|
||||
{Rs, NewS} = do_pb_ll_request3(LowSubCmd, S),
|
||||
{RqID, Cmd0, Rs, NewS};
|
||||
{RqID, {low_proj, _LowSubCmd}=Cmd0} ->
|
||||
{Rs, NewS} = do_pb_ll_request3(Cmd0, S),
|
||||
{RqID, Cmd0, Rs, NewS};
|
||||
{RqID, Cmd0} ->
|
||||
%% All remaining must have NSVersion, NS, & EpochID at next pos
|
||||
NSVersion = element(2, Cmd0),
|
||||
NS = element(3, Cmd0),
|
||||
EpochID = element(4, Cmd0),
|
||||
{Rs, NewS} = do_pb_ll_request2(NSVersion, NS, EpochID, Cmd0, S),
|
||||
{RqID, Cmd0, Rs, NewS}
|
||||
end,
|
||||
{machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
|
||||
|
||||
%% do_pb_ll_request2(): Verification of epoch details & namespace details.
|
||||
|
||||
do_pb_ll_request2(NSVersion, NS, EpochID, CMD, S) ->
|
||||
{Wedged_p, CurrentEpochID} = lookup_epoch(S),
|
||||
if not is_tuple(EpochID) orelse tuple_size(EpochID) /= 2 ->
|
||||
exit({bad_epoch_id, EpochID, for, CMD});
|
||||
Wedged_p == true ->
|
||||
{{error, wedged}, S#state{epoch_id=CurrentEpochID}};
|
||||
EpochID /= CurrentEpochID ->
|
||||
{Epoch, _} = EpochID,
|
||||
{CurrentEpoch, _} = CurrentEpochID,
|
||||
if Epoch < CurrentEpoch ->
|
||||
{{error, bad_epoch}, S};
|
||||
true ->
|
||||
_ = machi_flu1:wedge_myself(S#state.flu_name, CurrentEpochID),
|
||||
{{error, wedged}, S#state{epoch_id=CurrentEpochID}}
|
||||
end;
|
||||
true ->
|
||||
#state{namespace_version=MyNSVersion, namespace=MyNS} = S,
|
||||
if NSVersion /= MyNSVersion ->
|
||||
{{error, bad_epoch}, S};
|
||||
NS /= MyNS ->
|
||||
{{error, bad_arg}, S};
|
||||
true ->
|
||||
do_pb_ll_request3(CMD, S)
|
||||
end
|
||||
end.
|
||||
|
||||
lookup_epoch(#state{epoch_tab=T}) ->
|
||||
%% TODO: race in shutdown to access ets table after owner dies
|
||||
ets:lookup_element(T, epoch, 2).
|
||||
|
||||
%% Witness status does not matter below.
|
||||
do_pb_ll_request3({low_echo, Msg}, S) ->
|
||||
{Msg, S};
|
||||
do_pb_ll_request3({low_auth, _User, _Pass}, S) ->
|
||||
{-6, S};
|
||||
do_pb_ll_request3({low_wedge_status}, S) ->
|
||||
{do_server_wedge_status(S), S};
|
||||
do_pb_ll_request3({low_proj, PCMD}, S) ->
|
||||
{do_server_proj_request(PCMD, S), S};
|
||||
|
||||
%% Witness status *matters* below
|
||||
do_pb_ll_request3({low_append_chunk, NSVersion, NS, EpochID, NSLocator,
|
||||
Prefix, Chunk, CSum_tag,
|
||||
CSum, Opts},
|
||||
#state{witness=false}=S) ->
|
||||
NSInfo = #ns_info{version=NSVersion, name=NS, locator=NSLocator},
|
||||
{do_server_append_chunk(NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum_tag, CSum,
|
||||
Opts, S), S};
|
||||
do_pb_ll_request3({low_write_chunk, _NSVersion, _NS, _EpochID, File, Offset, Chunk, CSum_tag,
|
||||
CSum},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, S), S};
|
||||
do_pb_ll_request3({low_read_chunk, _NSVersion, _NS, _EpochID, File, Offset, Size, Opts},
|
||||
#state{witness=false} = S) ->
|
||||
{do_server_read_chunk(File, Offset, Size, Opts, S), S};
|
||||
do_pb_ll_request3({low_trim_chunk, _NSVersion, _NS, _EpochID, File, Offset, Size, TriggerGC},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_trim_chunk(File, Offset, Size, TriggerGC, S), S};
|
||||
do_pb_ll_request3({low_checksum_list, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_checksum_listing(File, S), S};
|
||||
do_pb_ll_request3({low_list_files, _EpochID},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_list_files(S), S};
|
||||
do_pb_ll_request3({low_delete_migration, _EpochID, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_delete_migration(File, S),
|
||||
#state{witness=false}=S};
|
||||
do_pb_ll_request3({low_trunc_hack, _EpochID, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_trunc_hack(File, S), S};
|
||||
|
||||
do_pb_ll_request3(_, #state{witness=true}=S) ->
|
||||
{{error, bad_arg}, S}. % TODO: new status code??
|
||||
|
||||
do_server_proj_request({get_latest_epochid, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:get_latest_epochid(ProjStore, ProjType);
|
||||
do_server_proj_request({read_latest_projection, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:read_latest_projection(ProjStore, ProjType);
|
||||
do_server_proj_request({read_projection, ProjType, Epoch},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:read(ProjStore, ProjType, Epoch);
|
||||
do_server_proj_request({write_projection, ProjType, Proj},
|
||||
#state{flu_name=FluName, proj_store=ProjStore}) ->
|
||||
if Proj#projection_v1.epoch_number == ?SPAM_PROJ_EPOCH ->
|
||||
%% io:format(user, "DBG ~s ~w ~P\n", [?MODULE, ?LINE, Proj, 5]),
|
||||
Chmgr = machi_flu_psup:make_fitness_regname(FluName),
|
||||
[Map] = Proj#projection_v1.dbg,
|
||||
catch machi_fitness:send_fitness_update_spam(
|
||||
Chmgr, Proj#projection_v1.author_server, Map);
|
||||
true ->
|
||||
catch machi_projection_store:write(ProjStore, ProjType, Proj)
|
||||
end;
|
||||
do_server_proj_request({get_all_projections, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:get_all_projections(ProjStore, ProjType);
|
||||
do_server_proj_request({list_all_projections, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:list_all_projections(ProjStore, ProjType);
|
||||
do_server_proj_request({kick_projection_reaction},
|
||||
#state{flu_name=FluName}) ->
|
||||
%% Tell my chain manager that it might want to react to
|
||||
%% this new world.
|
||||
Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
|
||||
spawn(fun() ->
|
||||
catch machi_chain_manager1:trigger_react_to_env(Chmgr)
|
||||
end),
|
||||
async_no_response.
|
||||
|
||||
do_server_append_chunk(NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum_tag, CSum,
|
||||
Opts, S) ->
|
||||
case sanitize_prefix(Prefix) of
|
||||
ok ->
|
||||
do_server_append_chunk2(NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum_tag, CSum,
|
||||
Opts, S);
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_append_chunk2(NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum_tag, Client_CSum,
|
||||
Opts, #state{flu_name=FluName,
|
||||
epoch_id=EpochID}=_S) ->
|
||||
%% TODO: Do anything with PKey?
|
||||
try
|
||||
TaggedCSum = check_or_make_tagged_checksum(CSum_tag, Client_CSum,Chunk),
|
||||
R = {seq_append, self(), NSInfo, EpochID,
|
||||
Prefix, Chunk, TaggedCSum, Opts},
|
||||
case gen_server:call(FluName, R, 10*1000) of
|
||||
{assignment, Offset, File} ->
|
||||
Size = iolist_size(Chunk),
|
||||
{ok, {Offset, Size, File}};
|
||||
witness ->
|
||||
{error, bad_arg};
|
||||
wedged ->
|
||||
{error, wedged};
|
||||
{error, timeout} ->
|
||||
{error, partition}
|
||||
end
|
||||
catch
|
||||
throw:{bad_csum, _CS} ->
|
||||
{error, bad_checksum};
|
||||
error:badarg ->
|
||||
lager:error("badarg at ~w:do_server_append_chunk2:~w ~w",
|
||||
[?MODULE, ?LINE, erlang:get_stacktrace()]),
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, #state{flu_name=FluName}) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
|
||||
{ok, Pid} ->
|
||||
Meta = [{client_csum_tag, CSum_tag}, {client_csum, CSum}],
|
||||
machi_file_proxy:write(Pid, Offset, Meta, Chunk);
|
||||
{error, trimmed} = Error ->
|
||||
Error
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_read_chunk(File, Offset, Size, Opts, #state{flu_name=FluName})->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
|
||||
{ok, Pid} ->
|
||||
case machi_file_proxy:read(Pid, Offset, Size, Opts) of
|
||||
%% XXX FIXME
|
||||
%% For now we are omiting the checksum data because it blows up
|
||||
%% protobufs.
|
||||
{ok, ChunksAndTrimmed} -> {ok, ChunksAndTrimmed};
|
||||
Other -> Other
|
||||
end;
|
||||
{error, trimmed} = Error ->
|
||||
Error
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_trim_chunk(File, Offset, Size, TriggerGC, #state{flu_name=FluName}) ->
|
||||
lager:debug("Hi there! I'm trimming this: ~s, (~p, ~p), ~p~n",
|
||||
[File, Offset, Size, TriggerGC]),
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
|
||||
{ok, Pid} ->
|
||||
machi_file_proxy:trim(Pid, Offset, Size, TriggerGC);
|
||||
{error, trimmed} = Trimmed ->
|
||||
%% Should be returned back to (maybe) trigger repair
|
||||
Trimmed
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_checksum_listing(File, #state{flu_name=FluName, data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
|
||||
{ok, Pid} ->
|
||||
{ok, List} = machi_file_proxy:checksum_list(Pid),
|
||||
Bin = erlang:term_to_binary(List),
|
||||
if byte_size(Bin) > (?PB_MAX_MSG_SIZE - 1024) ->
|
||||
%% TODO: Fix this limitation by streaming the
|
||||
%% binary in multiple smaller PB messages.
|
||||
%% Also, don't read the file all at once. ^_^
|
||||
error_logger:error_msg("~s:~w oversize ~s\n",
|
||||
[?MODULE, ?LINE, DataDir]),
|
||||
{error, bad_arg};
|
||||
true ->
|
||||
{ok, Bin}
|
||||
end;
|
||||
{error, trimmed} ->
|
||||
{error, trimmed}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_list_files(#state{data_dir=DataDir}=_S) ->
|
||||
{_, WildPath} = machi_util:make_data_filename(DataDir, ""),
|
||||
Files = filelib:wildcard("*", WildPath),
|
||||
{ok, [begin
|
||||
{ok, FI} = file:read_file_info(WildPath ++ "/" ++ File),
|
||||
Size = FI#file_info.size,
|
||||
{Size, File}
|
||||
end || File <- Files]}.
|
||||
|
||||
do_server_wedge_status(#state{namespace_version=NSVersion, namespace=NS}=S) ->
|
||||
{Wedged_p, CurrentEpochID0} = lookup_epoch(S),
|
||||
CurrentEpochID = if CurrentEpochID0 == undefined ->
|
||||
?DUMMY_PV1_EPOCH;
|
||||
true ->
|
||||
CurrentEpochID0
|
||||
end,
|
||||
{Wedged_p, CurrentEpochID, NSVersion, NS}.
|
||||
|
||||
do_server_delete_migration(File, #state{data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
{_, Path} = machi_util:make_data_filename(DataDir, File),
|
||||
case file:delete(Path) of
|
||||
ok ->
|
||||
ok;
|
||||
{error, enoent} ->
|
||||
{error, no_such_file};
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_trunc_hack(File, #state{data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
{_, Path} = machi_util:make_data_filename(DataDir, File),
|
||||
case file:open(Path, [read, write, binary, raw]) of
|
||||
{ok, FH} ->
|
||||
try
|
||||
{ok, ?MINIMUM_OFFSET} = file:position(FH,
|
||||
?MINIMUM_OFFSET),
|
||||
ok = file:truncate(FH),
|
||||
ok
|
||||
after
|
||||
file:close(FH)
|
||||
end;
|
||||
{error, enoent} ->
|
||||
{error, no_such_file};
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
sanitize_file_string(Str) ->
|
||||
case has_no_prohibited_chars(Str) andalso machi_util:is_valid_filename(Str) of
|
||||
true -> ok;
|
||||
false -> error
|
||||
end.
|
||||
|
||||
has_no_prohibited_chars(Str) ->
|
||||
case re:run(Str, "/") of
|
||||
nomatch ->
|
||||
true;
|
||||
_ ->
|
||||
true
|
||||
end.
|
||||
|
||||
sanitize_prefix(Prefix) ->
|
||||
%% We are using '^' as our component delimiter
|
||||
case re:run(Prefix, "/|\\^") of
|
||||
nomatch ->
|
||||
ok;
|
||||
_ ->
|
||||
error
|
||||
end.
|
||||
|
||||
check_or_make_tagged_checksum(?CSUM_TAG_NONE, _Client_CSum, Chunk) ->
|
||||
%% TODO: If the client was foolish enough to use
|
||||
%% this type of non-checksum, then the client gets
|
||||
%% what it deserves wrt data integrity, alas. In
|
||||
%% the client-side Chain Replication method, each
|
||||
%% server will calculated this independently, which
|
||||
%% isn't exactly what ought to happen for best data
|
||||
%% integrity checking. In server-side CR, the csum
|
||||
%% should be calculated by the head and passed down
|
||||
%% the chain together with the value.
|
||||
CS = machi_util:checksum_chunk(Chunk),
|
||||
machi_util:make_tagged_csum(server_sha, CS);
|
||||
check_or_make_tagged_checksum(?CSUM_TAG_CLIENT_SHA, Client_CSum, Chunk) ->
|
||||
CS = machi_util:checksum_chunk(Chunk),
|
||||
if CS == Client_CSum ->
|
||||
machi_util:make_tagged_csum(server_sha,
|
||||
Client_CSum);
|
||||
true ->
|
||||
throw({bad_csum, CS})
|
||||
end.
|
||||
|
||||
%%%% High PB mode %%%%
|
||||
|
||||
do_pb_hl_request(#mpb_request{req_id=ReqID}, #state{pb_mode=low}=S) ->
|
||||
Result = {low_error, 41, "High protocol request while in low mode"},
|
||||
{machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
|
||||
do_pb_hl_request(PB_request, S) ->
|
||||
{ReqID, Cmd} = machi_pb_translate:from_pb_request(PB_request),
|
||||
{Result, S2} = do_pb_hl_request2(Cmd, S),
|
||||
{machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
|
||||
|
||||
do_pb_hl_request2({high_echo, Msg}, S) ->
|
||||
{Msg, S};
|
||||
do_pb_hl_request2({high_auth, _User, _Pass}, S) ->
|
||||
{-77, S};
|
||||
do_pb_hl_request2({high_append_chunk=Op, NS, Prefix, Chunk, TaggedCSum, Opts},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
NSInfo = #ns_info{name=NS}, % TODO populate other fields
|
||||
todo_perhaps_remind_ns_locator_not_chosen(Op),
|
||||
Res = machi_cr_client:append_chunk(Clnt, NSInfo,
|
||||
Prefix, Chunk, TaggedCSum, Opts),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_write_chunk=Op, File, Offset, Chunk, CSum},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
NSInfo = undefined,
|
||||
todo_perhaps_remind_ns_locator_not_chosen(Op),
|
||||
Res = machi_cr_client:write_chunk(Clnt, NSInfo, File, Offset, Chunk, CSum),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_read_chunk=Op, File, Offset, Size, Opts},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
NSInfo = undefined,
|
||||
todo_perhaps_remind_ns_locator_not_chosen(Op),
|
||||
Res = machi_cr_client:read_chunk(Clnt, NSInfo, File, Offset, Size, Opts),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_trim_chunk=Op, File, Offset, Size},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
NSInfo = undefined,
|
||||
todo_perhaps_remind_ns_locator_not_chosen(Op),
|
||||
Res = machi_cr_client:trim_chunk(Clnt, NSInfo, File, Offset, Size),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_checksum_list, File}, #state{high_clnt=Clnt}=S) ->
|
||||
Res = machi_cr_client:checksum_list(Clnt, File),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_list_files}, #state{high_clnt=Clnt}=S) ->
|
||||
Res = machi_cr_client:list_files(Clnt),
|
||||
{Res, S}.
|
||||
|
||||
make_high_clnt(#state{high_clnt=undefined}=S) ->
|
||||
{ok, Proj} = machi_projection_store:read_latest_projection(
|
||||
S#state.proj_store, private),
|
||||
Ps = [P_srvr || {_, P_srvr} <- orddict:to_list(
|
||||
Proj#projection_v1.members_dict)],
|
||||
{ok, Clnt} = machi_cr_client:start_link(Ps),
|
||||
S#state{high_clnt=Clnt};
|
||||
make_high_clnt(S) ->
|
||||
S.
|
||||
|
||||
todo_perhaps_remind_ns_locator_not_chosen(Op) ->
|
||||
Key = {?MODULE, Op},
|
||||
case get(Key) of
|
||||
undefined ->
|
||||
io:format(user, "TODO op ~w is using default locator value\n",
|
||||
[Op]),
|
||||
put(Key, true);
|
||||
_ ->
|
||||
ok
|
||||
end.
|
||||
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc A supervisor to hold dynamic processes inside single
|
||||
%% FLU service, ranch listener and append server.
|
||||
|
||||
%% TODO: This supervisor is maybe useless. First introduced for
|
||||
%% workaround to start listener dynamically in flu1 initialization
|
||||
%% phase. Because `machi_flu_psup' is being blocked in flu1
|
||||
%% initialization time, adding a child to the supervisor leads to
|
||||
%% deadlock. If initialization can be done only by static arguments,
|
||||
%% then this supervisor should be removed and added as a direct child
|
||||
%% of `machi_flu_psup'.
|
||||
|
||||
-module(machi_flu1_subsup).
|
||||
-behaviour(supervisor).
|
||||
|
||||
%% public API
|
||||
-export([start_link/1,
|
||||
start_append_server/4,
|
||||
stop_append_server/1,
|
||||
start_listener/7,
|
||||
stop_listener/1,
|
||||
subsup_name/1,
|
||||
listener_name/1]).
|
||||
|
||||
%% supervisor callback
|
||||
-export([init/1]).
|
||||
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-define(SHUTDOWN, 5000).
|
||||
-define(BACKLOG, 8192).
|
||||
|
||||
-spec start_link(pv1_server()) -> {ok, pid()}.
|
||||
start_link(FluName) ->
|
||||
supervisor:start_link({local, subsup_name(FluName)}, ?MODULE, []).
|
||||
|
||||
-spec start_append_server(pv1_server(), boolean(), boolean(),
|
||||
undefined | machi_dt:epoch_id()) ->
|
||||
{ok, pid()}.
|
||||
start_append_server(FluName, Witness_p, Wedged_p, EpochId) ->
|
||||
supervisor:start_child(subsup_name(FluName),
|
||||
append_server_spec(FluName, Witness_p, Wedged_p, EpochId)).
|
||||
|
||||
-spec stop_append_server(pv1_server()) -> ok.
|
||||
stop_append_server(FluName) ->
|
||||
SubSup = listener_name(FluName),
|
||||
ok = supervisor:terminate_child(SubSup, FluName),
|
||||
ok = supervisor:delete_child(SubSup, FluName).
|
||||
|
||||
-spec start_listener(pv1_server(), inet:port_number(), boolean(),
|
||||
string(), ets:tab(), atom() | pid(),
|
||||
proplists:proplist()) -> {ok, pid()}.
|
||||
start_listener(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore,
|
||||
Props) ->
|
||||
supervisor:start_child(subsup_name(FluName),
|
||||
listener_spec(FluName, TcpPort, Witness, DataDir,
|
||||
EpochTab, ProjStore, Props)).
|
||||
|
||||
-spec stop_listener(pv1_server()) -> ok.
|
||||
stop_listener(FluName) ->
|
||||
SupName = subsup_name(FluName),
|
||||
ListenerName = listener_name(FluName),
|
||||
ok = supervisor:terminate_child(SupName, ListenerName),
|
||||
ok = supervisor:delete_child(SupName, ListenerName).
|
||||
|
||||
-spec subsup_name(pv1_server()) -> atom().
|
||||
subsup_name(FluName) when is_atom(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_flu1_subsup").
|
||||
|
||||
-spec listener_name(pv1_server()) -> atom().
|
||||
listener_name(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_listener").
|
||||
|
||||
%% Supervisor callback
|
||||
|
||||
init([]) ->
|
||||
SupFlags = {one_for_all, 1000, 10},
|
||||
{ok, {SupFlags, []}}.
|
||||
|
||||
%% private
|
||||
|
||||
-spec listener_spec(pv1_server(), inet:port_number(), boolean(),
|
||||
string(), ets:tab(), atom() | pid(),
|
||||
proplists:proplist()) -> supervisor:child_spec().
|
||||
listener_spec(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore, Props) ->
|
||||
ListenerName = listener_name(FluName),
|
||||
NbAcceptors = 10,
|
||||
TcpOpts = [{port, TcpPort}, {backlog, ?BACKLOG}],
|
||||
NetServerOpts = [FluName, Witness, DataDir, EpochTab, ProjStore, Props],
|
||||
ranch:child_spec(ListenerName, NbAcceptors,
|
||||
ranch_tcp, TcpOpts,
|
||||
machi_flu1_net_server, NetServerOpts).
|
||||
|
||||
-spec append_server_spec(pv1_server(), boolean(), boolean(),
|
||||
undefined | machi_dt:epoch_id()) -> supervisor:child_spec().
|
||||
append_server_spec(FluName, Witness_p, Wedged_p, EpochId) ->
|
||||
{FluName, {machi_flu1_append_server, start_link,
|
||||
[FluName, Witness_p, Wedged_p, EpochId]},
|
||||
permanent, ?SHUTDOWN, worker, [machi_flu1_append_server]}.
|
||||
|
|
@ -51,8 +51,8 @@
|
|||
-export([
|
||||
child_spec/2,
|
||||
start_link/2,
|
||||
find_or_make_filename_from_prefix/4,
|
||||
increment_prefix_sequence/3,
|
||||
find_or_make_filename_from_prefix/3,
|
||||
increment_prefix_sequence/2,
|
||||
list_files_by_prefix/2
|
||||
]).
|
||||
|
||||
|
|
@ -67,7 +67,6 @@
|
|||
]).
|
||||
|
||||
-define(TIMEOUT, 10 * 1000).
|
||||
-include("machi.hrl"). %% included for #ns_info record
|
||||
-include("machi_projection.hrl"). %% included for pv1_epoch type
|
||||
|
||||
-record(state, {fluname :: atom(),
|
||||
|
|
@ -89,30 +88,26 @@ start_link(FluName, DataDir) when is_atom(FluName) andalso is_list(DataDir) ->
|
|||
gen_server:start_link({local, N}, ?MODULE, [FluName, DataDir], []).
|
||||
|
||||
-spec find_or_make_filename_from_prefix( FluName :: atom(),
|
||||
EpochId :: pv1_epoch(),
|
||||
Prefix :: {prefix, string()},
|
||||
machi_dt:ns_info()) ->
|
||||
EpochId :: pv1_epoch_n(),
|
||||
Prefix :: {prefix, string()} ) ->
|
||||
{file, Filename :: string()} | {error, Reason :: term() } | timeout.
|
||||
% @doc Find the latest available or make a filename from a prefix. A prefix
|
||||
% should be in the form of a tagged tuple `{prefix, P}'. Returns a tagged
|
||||
% tuple in the form of `{file, F}' or an `{error, Reason}'
|
||||
find_or_make_filename_from_prefix(FluName, EpochId,
|
||||
{prefix, Prefix},
|
||||
#ns_info{}=NSInfo)
|
||||
when is_atom(FluName) ->
|
||||
find_or_make_filename_from_prefix(FluName, EpochId, {prefix, Prefix}) when is_atom(FluName) ->
|
||||
N = make_filename_mgr_name(FluName),
|
||||
gen_server:call(N, {find_filename, FluName, EpochId, NSInfo, Prefix}, ?TIMEOUT);
|
||||
find_or_make_filename_from_prefix(_FluName, _EpochId, Other, Other2) ->
|
||||
lager:error("~p is not a valid prefix/locator ~p", [Other, Other2]),
|
||||
gen_server:call(N, {find_filename, EpochId, Prefix}, ?TIMEOUT);
|
||||
find_or_make_filename_from_prefix(_FluName, _EpochId, Other) ->
|
||||
lager:error("~p is not a valid prefix.", [Other]),
|
||||
error(badarg).
|
||||
|
||||
-spec increment_prefix_sequence( FluName :: atom(), NSInfo :: machi_dt:ns_info(), Prefix :: {prefix, string()} ) ->
|
||||
-spec increment_prefix_sequence( FluName :: atom(), Prefix :: {prefix, string()} ) ->
|
||||
ok | {error, Reason :: term() } | timeout.
|
||||
% @doc Increment the sequence counter for a given prefix. Prefix should
|
||||
% be in the form of `{prefix, P}'.
|
||||
increment_prefix_sequence(FluName, #ns_info{}=NSInfo, {prefix, Prefix}) when is_atom(FluName) ->
|
||||
gen_server:call(make_filename_mgr_name(FluName), {increment_sequence, NSInfo, Prefix}, ?TIMEOUT);
|
||||
increment_prefix_sequence(_FluName, _NSInfo, Other) ->
|
||||
increment_prefix_sequence(FluName, {prefix, Prefix}) when is_atom(FluName) ->
|
||||
gen_server:call(make_filename_mgr_name(FluName), {increment_sequence, Prefix}, ?TIMEOUT);
|
||||
increment_prefix_sequence(_FluName, Other) ->
|
||||
lager:error("~p is not a valid prefix.", [Other]),
|
||||
error(badarg).
|
||||
|
||||
|
|
@ -131,7 +126,7 @@ list_files_by_prefix(_FluName, Other) ->
|
|||
init([FluName, DataDir]) ->
|
||||
Tid = ets:new(make_filename_mgr_name(FluName), [named_table, {read_concurrency, true}]),
|
||||
{ok, #state{fluname = FluName,
|
||||
epoch = ?DUMMY_PV1_EPOCH,
|
||||
epoch = {0, <<"NONE">>},
|
||||
datadir = DataDir,
|
||||
tid = Tid}}.
|
||||
|
||||
|
|
@ -143,23 +138,26 @@ handle_cast(Req, State) ->
|
|||
%% the FLU has already validated that the caller's epoch id and the FLU's epoch id
|
||||
%% are the same. So we *assume* that remains the case here - that is to say, we
|
||||
%% are not wedged.
|
||||
handle_call({find_filename, FluName, EpochId, NSInfo, Prefix}, _From,
|
||||
S = #state{ datadir = DataDir, epoch = EpochId, tid = Tid }) ->
|
||||
handle_call({find_filename, EpochId, Prefix}, _From, S = #state{ fluname = FluName,
|
||||
datadir = DataDir,
|
||||
epoch = EpochId,
|
||||
tid = Tid}) ->
|
||||
%% Our state and the caller's epoch ids are the same. Business as usual.
|
||||
File = handle_find_file(FluName, Tid, NSInfo, Prefix, DataDir),
|
||||
File = handle_find_file(Tid, Prefix, DataDir, FluName),
|
||||
{reply, {file, File}, S};
|
||||
|
||||
handle_call({find_filename, _FluName, EpochId, NSInfo, Prefix}, _From, S = #state{ datadir = DataDir, tid = Tid }) ->
|
||||
handle_call({find_filename, EpochId, Prefix}, _From, S = #state{ fluname = FluName,
|
||||
datadir = DataDir,
|
||||
tid = Tid }) ->
|
||||
%% If the epoch id in our state and the caller's epoch id were the same, it would've
|
||||
%% matched the above clause. Since we're here, we know that they are different.
|
||||
%% If epoch ids between our state and the caller's are different, we must increment the
|
||||
%% sequence number, generate a filename and then cache it.
|
||||
File = increment_and_cache_filename(Tid, DataDir, NSInfo, Prefix),
|
||||
File = increment_and_cache_filename(Tid, DataDir, Prefix, FluName),
|
||||
{reply, {file, File}, S#state{epoch = EpochId}};
|
||||
|
||||
handle_call({increment_sequence, #ns_info{name=NS, locator=NSLocator}, Prefix}, _From, S = #state{ datadir = DataDir, tid=Tid }) ->
|
||||
NSInfo = #ns_info{name=NS, locator=NSLocator},
|
||||
_File = increment_and_cache_filename(Tid, DataDir, NSInfo, Prefix),
|
||||
handle_call({increment_sequence, Prefix}, _From, S = #state{ datadir = DataDir }) ->
|
||||
ok = machi_util:increment_max_filenum(DataDir, Prefix),
|
||||
{reply, ok, S};
|
||||
handle_call({list_files, Prefix}, From, S = #state{ datadir = DataDir }) ->
|
||||
spawn(fun() ->
|
||||
|
|
@ -192,38 +190,64 @@ generate_uuid_v4_str() ->
|
|||
io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b",
|
||||
[A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]).
|
||||
|
||||
find_file(DataDir, Prefix, FluName, N) ->
|
||||
{_Filename, Path} = machi_util:make_data_filename(DataDir, Prefix, "*", FluName, N),
|
||||
filelib:wildcard(Path).
|
||||
|
||||
list_files(DataDir, Prefix) ->
|
||||
{F_bin, Path} = machi_util:make_data_filename(DataDir, "*^" ++ Prefix ++ "^*"),
|
||||
{F_bin, Path} = machi_util:make_data_filename(DataDir, Prefix, "*", "*"),
|
||||
filelib:wildcard(binary_to_list(F_bin), filename:dirname(Path)).
|
||||
|
||||
make_filename_mgr_name(FluName) when is_atom(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_filename_mgr").
|
||||
|
||||
handle_find_file(_FluName, Tid, #ns_info{name=NS, locator=NSLocator}, Prefix, DataDir) ->
|
||||
case ets:lookup(Tid, {NS, NSLocator, Prefix}) of
|
||||
[] ->
|
||||
N = machi_util:read_max_filenum(DataDir, NS, NSLocator, Prefix),
|
||||
F = generate_filename(DataDir, NS, NSLocator, Prefix, N),
|
||||
true = ets:insert(Tid, {{NS, NSLocator, Prefix}, F}),
|
||||
handle_find_file(Tid, Prefix, DataDir, FluName) ->
|
||||
N = machi_util:read_max_filenum(DataDir, Prefix),
|
||||
{File, Cleanup} =
|
||||
case find_file(DataDir, Prefix, FluName, N) of
|
||||
[] ->
|
||||
{find_or_make_filename(Tid, DataDir, Prefix, FluName, N), false};
|
||||
[H] -> {H, true};
|
||||
[_ | _] = L ->
|
||||
lager:error(
|
||||
"Searching for a matching file to prefix ~p "
|
||||
"with flu name ~w and sequence number ~p gave multiples: ~p",
|
||||
[Prefix, FluName, N, L]),
|
||||
exit({bad_file_enties, {Prefix, FluName, N, L}})
|
||||
end,
|
||||
maybe_cleanup(Tid, {Prefix, N}, Cleanup),
|
||||
filename:basename(File).
|
||||
|
||||
find_or_make_filename(Tid, DataDir, Prefix, FluName, N) ->
|
||||
case ets:lookup(Tid, {Prefix, N}) of
|
||||
[] ->
|
||||
F = generate_filename(DataDir, Prefix, FluName, N),
|
||||
true = ets:insert_new(Tid, {{Prefix, N}, F}),
|
||||
F;
|
||||
[{_Key, File}] ->
|
||||
File
|
||||
end.
|
||||
|
||||
generate_filename(DataDir, NS, NSLocator, Prefix, N) ->
|
||||
{F, _Q} = machi_util:make_data_filename(
|
||||
generate_filename(DataDir, Prefix, FluName, N) ->
|
||||
{F, _} = machi_util:make_data_filename(
|
||||
DataDir,
|
||||
NS, NSLocator, Prefix,
|
||||
Prefix,
|
||||
generate_uuid_v4_str(),
|
||||
FluName,
|
||||
N),
|
||||
binary_to_list(F).
|
||||
|
||||
increment_and_cache_filename(Tid, DataDir, #ns_info{name=NS,locator=NSLocator}, Prefix) ->
|
||||
ok = machi_util:increment_max_filenum(DataDir, NS, NSLocator, Prefix),
|
||||
N = machi_util:read_max_filenum(DataDir, NS, NSLocator, Prefix),
|
||||
F = generate_filename(DataDir, NS, NSLocator, Prefix, N),
|
||||
true = ets:insert(Tid, {{NS, NSLocator, Prefix}, F}),
|
||||
F.
|
||||
maybe_cleanup(_Tid, _Key, false) ->
|
||||
ok;
|
||||
maybe_cleanup(Tid, Key, true) ->
|
||||
true = ets:delete(Tid, Key).
|
||||
|
||||
increment_and_cache_filename(Tid, DataDir, Prefix, FluName) ->
|
||||
ok = machi_util:increment_max_filenum(DataDir, Prefix),
|
||||
N = machi_util:read_max_filenum(DataDir, Prefix),
|
||||
F = generate_filename(DataDir, Prefix, FluName, N),
|
||||
true = ets:insert_new(Tid, {{Prefix, N}, F}),
|
||||
filename:basename(F).
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@
|
|||
-module(machi_flu_metadata_mgr).
|
||||
-behaviour(gen_server).
|
||||
|
||||
-include("machi.hrl").
|
||||
|
||||
-define(MAX_MGRS, 10). %% number of managers to start by default.
|
||||
-define(HASH(X), erlang:phash2(X)). %% hash algorithm to use
|
||||
|
|
@ -63,7 +62,6 @@
|
|||
lookup_proxy_pid/2,
|
||||
start_proxy_pid/2,
|
||||
stop_proxy_pid/2,
|
||||
stop_proxy_pid_rollover/2,
|
||||
build_metadata_mgr_name/2,
|
||||
trim_file/2
|
||||
]).
|
||||
|
|
@ -101,10 +99,7 @@ start_proxy_pid(FluName, {file, Filename}) ->
|
|||
gen_server:call(get_manager_atom(FluName, Filename), {start_proxy_pid, Filename}, ?TIMEOUT).
|
||||
|
||||
stop_proxy_pid(FluName, {file, Filename}) ->
|
||||
gen_server:call(get_manager_atom(FluName, Filename), {stop_proxy_pid, false, Filename}, ?TIMEOUT).
|
||||
|
||||
stop_proxy_pid_rollover(FluName, {file, Filename}) ->
|
||||
gen_server:call(get_manager_atom(FluName, Filename), {stop_proxy_pid, true, Filename}, ?TIMEOUT).
|
||||
gen_server:call(get_manager_atom(FluName, Filename), {stop_proxy_pid, Filename}, ?TIMEOUT).
|
||||
|
||||
trim_file(FluName, {file, Filename}) ->
|
||||
gen_server:call(get_manager_atom(FluName, Filename), {trim_file, Filename}, ?TIMEOUT).
|
||||
|
|
@ -155,7 +150,7 @@ handle_call({start_proxy_pid, Filename}, _From,
|
|||
{reply, {error, trimmed}, State}
|
||||
end;
|
||||
|
||||
handle_call({stop_proxy_pid, Rollover_p, Filename}, _From, State = #state{ tid = Tid }) ->
|
||||
handle_call({stop_proxy_pid, Filename}, _From, State = #state{ tid = Tid }) ->
|
||||
case lookup_md(Tid, Filename) of
|
||||
not_found ->
|
||||
ok;
|
||||
|
|
@ -163,13 +158,8 @@ handle_call({stop_proxy_pid, Rollover_p, Filename}, _From, State = #state{ tid =
|
|||
ok;
|
||||
#md{ proxy_pid = Pid, mref = M } = R ->
|
||||
demonitor(M, [flush]),
|
||||
if Rollover_p ->
|
||||
do_rollover(Filename, State);
|
||||
true ->
|
||||
machi_file_proxy:stop(Pid),
|
||||
update_ets(Tid, R#md{ proxy_pid = undefined,
|
||||
mref = undefined })
|
||||
end
|
||||
machi_file_proxy:stop(Pid),
|
||||
update_ets(Tid, R#md{ proxy_pid = undefined, mref = undefined })
|
||||
end,
|
||||
{reply, ok, State};
|
||||
|
||||
|
|
@ -191,6 +181,25 @@ handle_info({'DOWN', Mref, process, Pid, normal}, State = #state{ tid = Tid }) -
|
|||
clear_ets(Tid, Mref),
|
||||
{noreply, State};
|
||||
|
||||
handle_info({'DOWN', Mref, process, Pid, file_rollover}, State = #state{ fluname = FluName,
|
||||
tid = Tid }) ->
|
||||
lager:info("file proxy ~p shutdown because of file rollover", [Pid]),
|
||||
R = get_md_record_by_mref(Tid, Mref),
|
||||
[Prefix | _Rest] = machi_util:parse_filename(R#md.filename),
|
||||
|
||||
%% We only increment the counter here. The filename will be generated on the
|
||||
%% next append request to that prefix and since the filename will have a new
|
||||
%% sequence number it probably will be associated with a different metadata
|
||||
%% manager. That's why we don't want to generate a new file name immediately
|
||||
%% and use it to start a new file proxy.
|
||||
ok = machi_flu_filename_mgr:increment_prefix_sequence(FluName, {prefix, Prefix}),
|
||||
|
||||
%% purge our ets table of this entry completely since it is likely the
|
||||
%% new filename (whenever it comes) will be in a different manager than
|
||||
%% us.
|
||||
purge_ets(Tid, R),
|
||||
{noreply, State};
|
||||
|
||||
handle_info({'DOWN', Mref, process, Pid, wedged}, State = #state{ tid = Tid }) ->
|
||||
lager:error("file proxy ~p shutdown because it's wedged", [Pid]),
|
||||
clear_ets(Tid, Mref),
|
||||
|
|
@ -263,35 +272,8 @@ get_md_record_by_mref(Tid, Mref) ->
|
|||
[R] = ets:match_object(Tid, {md, '_', '_', Mref}),
|
||||
R.
|
||||
|
||||
get_md_record_by_filename(Tid, Filename) ->
|
||||
[R] = ets:lookup(Tid, Filename),
|
||||
R.
|
||||
|
||||
get_env(Setting, Default) ->
|
||||
case application:get_env(machi, Setting) of
|
||||
undefined -> Default;
|
||||
{ok, V} -> V
|
||||
end.
|
||||
|
||||
do_rollover(Filename, _State = #state{ fluname = FluName,
|
||||
tid = Tid }) ->
|
||||
R = get_md_record_by_filename(Tid, Filename),
|
||||
lager:info("file ~p proxy ~p shutdown because of file rollover",
|
||||
[Filename, R#md.proxy_pid]),
|
||||
{Prefix, NS, NSLocator, _, _} =
|
||||
machi_util:parse_filename(R#md.filename),
|
||||
|
||||
%% We only increment the counter here. The filename will be generated on the
|
||||
%% next append request to that prefix and since the filename will have a new
|
||||
%% sequence number it probably will be associated with a different metadata
|
||||
%% manager. That's why we don't want to generate a new file name immediately
|
||||
%% and use it to start a new file proxy.
|
||||
NSInfo = #ns_info{name=NS, locator=NSLocator},
|
||||
lager:warning("INCR: ~p ~p\n", [FluName, Prefix]),
|
||||
ok = machi_flu_filename_mgr:increment_prefix_sequence(FluName, NSInfo, {prefix, Prefix}),
|
||||
|
||||
%% purge our ets table of this entry completely since it is likely the
|
||||
%% new filename (whenever it comes) will be in a different manager than
|
||||
%% us.
|
||||
purge_ets(Tid, R),
|
||||
ok.
|
||||
|
|
|
|||
|
|
@ -83,8 +83,6 @@
|
|||
%% Supervisor callbacks
|
||||
-export([init/1]).
|
||||
|
||||
make_package_spec(#p_srvr{name=FluName, port=TcpPort, props=Props}) when is_list(Props) ->
|
||||
make_package_spec({FluName, TcpPort, Props});
|
||||
make_package_spec({FluName, TcpPort, Props}) when is_list(Props) ->
|
||||
FluDataDir = get_env(flu_data_dir, undefined_is_invalid),
|
||||
MyDataDir = filename:join(FluDataDir, atom_to_list(FluName)),
|
||||
|
|
@ -96,7 +94,7 @@ make_package_spec(FluName, TcpPort, DataDir, Props) ->
|
|||
permanent, ?SHUTDOWN, supervisor, []}.
|
||||
|
||||
start_flu_package(#p_srvr{name=FluName, port=TcpPort, props=Props}) ->
|
||||
DataDir = get_data_dir(FluName, Props),
|
||||
DataDir = get_data_dir(Props),
|
||||
start_flu_package(FluName, TcpPort, DataDir, Props).
|
||||
|
||||
start_flu_package(FluName, TcpPort, DataDir, Props) ->
|
||||
|
|
@ -145,19 +143,16 @@ init([FluName, TcpPort, DataDir, Props0]) ->
|
|||
|
||||
FProxySupSpec = machi_file_proxy_sup:child_spec(FluName),
|
||||
|
||||
Flu1SubSupSpec = {machi_flu1_subsup:subsup_name(FluName),
|
||||
{machi_flu1_subsup, start_link, [FluName]},
|
||||
permanent, ?SHUTDOWN, supervisor, []},
|
||||
|
||||
FluSpec = {FluName,
|
||||
{machi_flu1, start_link,
|
||||
[ [{FluName, TcpPort, DataDir}|Props] ]},
|
||||
permanent, ?SHUTDOWN, worker, []},
|
||||
|
||||
|
||||
{ok, {SupFlags, [
|
||||
ProjSpec, FitnessSpec, MgrSpec,
|
||||
FProxySupSpec, FNameMgrSpec, MetaSupSpec,
|
||||
Flu1SubSupSpec, FluSpec]}}.
|
||||
ProjSpec, FitnessSpec, MgrSpec,
|
||||
FProxySupSpec, FNameMgrSpec, MetaSupSpec,
|
||||
FluSpec]}}.
|
||||
|
||||
make_flu_regname(FluName) when is_atom(FluName) ->
|
||||
FluName.
|
||||
|
|
@ -180,11 +175,8 @@ get_env(Setting, Default) ->
|
|||
{ok, V} -> V
|
||||
end.
|
||||
|
||||
get_data_dir(FluName, Props) ->
|
||||
get_data_dir(Props) ->
|
||||
case proplists:get_value(data_dir, Props) of
|
||||
Path when is_list(Path) ->
|
||||
Path;
|
||||
undefined ->
|
||||
{ok, Dir} = application:get_env(machi, flu_data_dir),
|
||||
Dir ++ "/" ++ atom_to_list(FluName)
|
||||
Path
|
||||
end.
|
||||
|
|
|
|||
|
|
@ -21,9 +21,6 @@
|
|||
%% @doc Supervisor for Machi FLU servers and their related support
|
||||
%% servers.
|
||||
%%
|
||||
%% Responsibility for managing FLU and chain lifecycle after the initial
|
||||
%% application startup is delegated to {@link machi_lifecycle_mgr}.
|
||||
%%
|
||||
%% See {@link machi_flu_psup} for an illustration of the entire Machi
|
||||
%% application process structure.
|
||||
|
||||
|
|
@ -32,11 +29,8 @@
|
|||
-behaviour(supervisor).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
-include("machi_verbose.hrl").
|
||||
|
||||
-ifdef(TEST).
|
||||
-compile(export_all).
|
||||
-ifdef(PULSE).
|
||||
-compile({parse_transform, pulse_instrument}).
|
||||
-include_lib("pulse_otp/include/pulse_otp.hrl").
|
||||
|
|
@ -44,12 +38,9 @@
|
|||
-else.
|
||||
-define(SHUTDOWN, 5000).
|
||||
-endif.
|
||||
-endif. %TEST
|
||||
|
||||
%% API
|
||||
-export([start_link/0,
|
||||
get_initial_flus/0, load_rc_d_files_from_dir/1,
|
||||
sanitize_p_srvr_records/1]).
|
||||
-export([start_link/0]).
|
||||
|
||||
%% Supervisor callbacks
|
||||
-export([init/1]).
|
||||
|
|
@ -78,66 +69,5 @@ get_initial_flus() ->
|
|||
[].
|
||||
-else. % PULSE
|
||||
get_initial_flus() ->
|
||||
DoesNotExist = "/tmp/does/not/exist",
|
||||
ConfigDir = case application:get_env(machi, flu_config_dir, DoesNotExist) of
|
||||
DoesNotExist ->
|
||||
DoesNotExist;
|
||||
Dir ->
|
||||
Dir
|
||||
end,
|
||||
Ps = [P || {_File, P} <- load_rc_d_files_from_dir(ConfigDir)],
|
||||
sanitize_p_srvr_records(Ps).
|
||||
application:get_env(machi, initial_flus, []).
|
||||
-endif. % PULSE
|
||||
|
||||
load_rc_d_files_from_dir(Dir) ->
|
||||
Files = filelib:wildcard(Dir ++ "/*"),
|
||||
[case file:consult(File) of
|
||||
{ok, [X]} ->
|
||||
{File, X};
|
||||
_ ->
|
||||
lager:warning("Error parsing file '~s', ignoring",
|
||||
[File]),
|
||||
{File, []}
|
||||
end || File <- Files].
|
||||
|
||||
sanitize_p_srvr_records(Ps) ->
|
||||
{Sane, _} = lists:foldl(fun sanitize_p_srvr_rec/2, {[], dict:new()}, Ps),
|
||||
Sane.
|
||||
|
||||
sanitize_p_srvr_rec(Whole, {Acc, D}) ->
|
||||
try
|
||||
#p_srvr{name=Name,
|
||||
proto_mod=PMod,
|
||||
address=Address,
|
||||
port=Port,
|
||||
props=Props} = Whole,
|
||||
true = is_atom(Name),
|
||||
NameK = {name, Name},
|
||||
error = dict:find(NameK, D),
|
||||
true = is_atom(PMod),
|
||||
case code:is_loaded(PMod) of
|
||||
{file, _} ->
|
||||
ok;
|
||||
_ ->
|
||||
{module, _} = code:load_file(PMod),
|
||||
ok
|
||||
end,
|
||||
if is_list(Address) -> ok;
|
||||
is_tuple(Address) -> ok % Erlang-style IPv4 or IPv6
|
||||
end,
|
||||
true = is_integer(Port) andalso Port >= 1024 andalso Port =< 65534,
|
||||
PortK = {port, Port},
|
||||
error = dict:find(PortK, D),
|
||||
true = is_list(Props),
|
||||
|
||||
%% All is sane enough.
|
||||
D2 = dict:store(NameK, Name,
|
||||
dict:store(PortK, Port, D)),
|
||||
{[Whole|Acc], D2}
|
||||
catch _:_ ->
|
||||
_ = lager:log(error, self(),
|
||||
"~s: Bad (or duplicate name/port) p_srvr record, "
|
||||
"skipping: ~P\n",
|
||||
[?MODULE, Whole, 15]),
|
||||
{Acc, D}
|
||||
end.
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,156 +0,0 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc Creates a Merkle tree per file based on the checksum data for
|
||||
%% a given data file.
|
||||
%%
|
||||
%% The `naive' implementation representation is:
|
||||
%%
|
||||
%% `<<Length:64, Offset:32, 0>>' for unwritten bytes
|
||||
%% `<<Length:64, Offset:32, 1>>' for trimmed bytes
|
||||
%% `<<Length:64, Offset:32, Csum/binary>>' for written bytes
|
||||
%%
|
||||
%% The tree feeds these leaf nodes into hashes representing chunks of a minimum
|
||||
%% size of at least 1024 KB (1 MB), but if the file size is larger, we will try
|
||||
%% to get about 100 chunks for the first rollup "Level 1." We aim for around 10
|
||||
%% hashes at level 2, and then 2 hashes level 3 and finally the root.
|
||||
|
||||
-module(machi_merkle_tree).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_merkle_tree.hrl").
|
||||
|
||||
-ifdef(TEST).
|
||||
-compile(export_all).
|
||||
-else.
|
||||
-export([
|
||||
open/2,
|
||||
open/3,
|
||||
tree/1,
|
||||
filename/1,
|
||||
diff/2
|
||||
]).
|
||||
-endif.
|
||||
|
||||
-define(TRIMMED, <<1>>).
|
||||
-define(UNWRITTEN, <<0>>).
|
||||
-define(NAIVE_ENCODE(Offset, Size, Data), <<Offset:64/unsigned-big, Size:32/unsigned-big, Data/binary>>).
|
||||
|
||||
-define(MINIMUM_CHUNK, 1048576). %% 1024 * 1024
|
||||
-define(LEVEL_SIZE, 10).
|
||||
-define(H, sha).
|
||||
|
||||
%% public API
|
||||
|
||||
open(Filename, DataDir) ->
|
||||
open(Filename, DataDir, naive).
|
||||
|
||||
open(Filename, DataDir, Type) ->
|
||||
Tree = load_filename(Filename, DataDir, Type),
|
||||
{ok, #mt{ filename = Filename, tree = Tree, backend = Type}}.
|
||||
|
||||
tree(#mt{ tree = T, backend = naive }) ->
|
||||
case T#naive.recalc of
|
||||
true -> build_tree(T);
|
||||
false -> T
|
||||
end.
|
||||
|
||||
filename(#mt{ filename = F }) -> F.
|
||||
|
||||
diff(#mt{backend = naive, tree = T1}, #mt{backend = naive, tree = T2}) ->
|
||||
case T1#naive.root == T2#naive.root of
|
||||
true -> same;
|
||||
false -> naive_diff(T1, T2)
|
||||
end;
|
||||
diff(_, _) -> error(badarg).
|
||||
|
||||
%% private
|
||||
|
||||
% @private
|
||||
load_filename(Filename, DataDir, naive) ->
|
||||
{Last, M} = do_load(Filename, DataDir, fun insert_csum_naive/2, []),
|
||||
ChunkSize = max(?MINIMUM_CHUNK, Last div 100),
|
||||
T = #naive{ leaves = lists:reverse(M), chunk_size = ChunkSize, recalc = true },
|
||||
build_tree(T).
|
||||
|
||||
do_load(Filename, DataDir, FoldFun, AccInit) ->
|
||||
CsumFile = machi_util:make_checksum_filename(DataDir, Filename),
|
||||
{ok, T} = machi_csum_table:open(CsumFile, []),
|
||||
Acc = machi_csum_table:foldl_chunks(FoldFun, {0, AccInit}, T),
|
||||
ok = machi_csum_table:close(T),
|
||||
Acc.
|
||||
|
||||
% @private
|
||||
insert_csum_naive({Last, Size, _Csum}=In, {Last, MT}) ->
|
||||
%% no gap
|
||||
{Last+Size, update_acc(In, MT)};
|
||||
insert_csum_naive({Offset, Size, _Csum}=In, {Last, MT}) ->
|
||||
Hole = Offset - Last,
|
||||
MT0 = update_acc({Last, Hole, unwritten}, MT),
|
||||
{Offset+Size, update_acc(In, MT0)}.
|
||||
|
||||
% @private
|
||||
update_acc({Offset, Size, unwritten}, MT) ->
|
||||
[ {Offset, Size, ?NAIVE_ENCODE(Offset, Size, ?UNWRITTEN)} | MT ];
|
||||
update_acc({Offset, Size, trimmed}, MT) ->
|
||||
[ {Offset, Size, ?NAIVE_ENCODE(Offset, Size, ?TRIMMED)} | MT ];
|
||||
update_acc({Offset, Size, <<_Tag:8, Csum/binary>>}, MT) ->
|
||||
[ {Offset, Size, ?NAIVE_ENCODE(Offset, Size, Csum)} | MT ].
|
||||
|
||||
build_tree(MT = #naive{ leaves = L, chunk_size = ChunkSize }) ->
|
||||
Lvl1s = build_level_1(ChunkSize, L, 1, [ crypto:hash_init(?H) ]),
|
||||
Mod2 = length(Lvl1s) div ?LEVEL_SIZE,
|
||||
Lvl2s = build_int_level(Mod2, Lvl1s, 1, [ crypto:hash_init(?H) ]),
|
||||
Mod3 = length(Lvl2s) div 2,
|
||||
Lvl3s = build_int_level(Mod3, Lvl2s, 1, [ crypto:hash_init(?H) ]),
|
||||
Root = build_root(Lvl3s, crypto:hash_init(?H)),
|
||||
MT#naive{ root = Root, lvl1 = Lvl1s, lvl2 = Lvl2s, lvl3 = Lvl3s, recalc = false }.
|
||||
|
||||
build_root([], Ctx) ->
|
||||
crypto:hash_final(Ctx);
|
||||
build_root([H|T], Ctx) ->
|
||||
build_root(T, crypto:hash_update(Ctx, H)).
|
||||
|
||||
build_int_level(_Mod, [], _Cnt, [ Ctx | Rest ]) ->
|
||||
lists:reverse( [ crypto:hash_final(Ctx) | Rest ] );
|
||||
build_int_level(Mod, [H|T], Cnt, [ Ctx | Rest ]) when Cnt rem Mod == 0 ->
|
||||
NewCtx = crypto:hash_init(?H),
|
||||
build_int_level(Mod, T, Cnt + 1, [ crypto:hash_update(NewCtx, H), crypto:hash_final(Ctx) | Rest ]);
|
||||
build_int_level(Mod, [H|T], Cnt, [ Ctx | Rest ]) ->
|
||||
build_int_level(Mod, T, Cnt+1, [ crypto:hash_update(Ctx, H) | Rest ]).
|
||||
|
||||
build_level_1(_Size, [], _Multiple, [ Ctx | Rest ]) ->
|
||||
lists:reverse([ crypto:hash_final(Ctx) | Rest ]);
|
||||
build_level_1(Size, [{Pos, Len, Hash}|T], Multiple, [ Ctx | Rest ])
|
||||
when ( Pos + Len ) > ( Size * Multiple ) ->
|
||||
NewCtx = crypto:hash_init(?H),
|
||||
build_level_1(Size, T, Multiple+1,
|
||||
[ crypto:hash_update(NewCtx, Hash), crypto:hash_final(Ctx) | Rest ]);
|
||||
build_level_1(Size, [{Pos, Len, Hash}|T], Multiple, [ Ctx | Rest ])
|
||||
when ( Pos + Len ) =< ( Size * Multiple ) ->
|
||||
build_level_1(Size, T, Multiple, [ crypto:hash_update(Ctx, Hash) | Rest ]).
|
||||
|
||||
naive_diff(#naive{lvl1 = L1}, #naive{lvl1=L2, chunk_size=CS2}) ->
|
||||
Set1 = gb_sets:from_list(lists:zip(lists:seq(1, length(L1)), L1)),
|
||||
Set2 = gb_sets:from_list(lists:zip(lists:seq(1, length(L2)), L2)),
|
||||
|
||||
%% The byte ranges in list 2 that do not match in list 1
|
||||
%% Or should we do something else?
|
||||
[ {(X-1)*CS2, CS2, SHA} || {X, SHA} <- gb_sets:to_list(gb_sets:subtract(Set1, Set2)) ].
|
||||
|
|
@ -25,10 +25,6 @@
|
|||
%% to a single socket connection, and there is no code to deal with
|
||||
%% multiple connections/load balancing/error handling to several/all
|
||||
%% Machi cluster servers.
|
||||
%%
|
||||
%% Please see {@link machi_flu1_client} the "Client API implemntation notes"
|
||||
%% section for how this module relates to the rest of the client API
|
||||
%% implementation.
|
||||
|
||||
-module(machi_pb_high_client).
|
||||
|
||||
|
|
@ -62,7 +58,7 @@
|
|||
count=0 :: non_neg_integer()
|
||||
}).
|
||||
|
||||
%% Official error types that is specific in Machi
|
||||
%% @doc official error types that is specific in Machi
|
||||
-type machi_client_error_reason() :: bad_arg | wedged | bad_checksum |
|
||||
partition | not_written | written |
|
||||
trimmed | no_such_file | partial_read |
|
||||
|
|
@ -100,33 +96,30 @@ auth(PidSpec, User, Pass) ->
|
|||
auth(PidSpec, User, Pass, Timeout) ->
|
||||
send_sync(PidSpec, {auth, User, Pass}, Timeout).
|
||||
|
||||
-spec append_chunk(pid(),
|
||||
NS::machi_dt:namespace(), Prefix::machi_dt:file_prefix(),
|
||||
Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum(),
|
||||
Opts::machi_dt:append_opts()) ->
|
||||
-spec append_chunk(pid(), PlacementKey::binary(), Prefix::binary(), Chunk::binary(),
|
||||
CSum::binary(), ChunkExtra::non_neg_integer()) ->
|
||||
{ok, Filename::string(), Offset::machi_dt:file_offset()} |
|
||||
{error, machi_client_error_reason()}.
|
||||
append_chunk(PidSpec, NS, Prefix, Chunk, CSum, Opts) ->
|
||||
append_chunk(PidSpec, NS, Prefix, Chunk, CSum, Opts, ?DEFAULT_TIMEOUT).
|
||||
append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra) ->
|
||||
append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra, ?DEFAULT_TIMEOUT).
|
||||
|
||||
-spec append_chunk(pid(),
|
||||
NS::machi_dt:namespace(), Prefix::machi_dt:file_prefix(),
|
||||
Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum(),
|
||||
Opts::machi_dt:append_opts(),
|
||||
-spec append_chunk(pid(), PlacementKey::binary(), Prefix::binary(),
|
||||
Chunk::binary(), CSum::binary(),
|
||||
ChunkExtra::non_neg_integer(),
|
||||
Timeout::non_neg_integer()) ->
|
||||
{ok, Filename::string(), Offset::machi_dt:file_offset()} |
|
||||
{error, machi_client_error_reason()}.
|
||||
append_chunk(PidSpec, NS, Prefix, Chunk, CSum, Opts, Timeout) ->
|
||||
send_sync(PidSpec, {append_chunk, NS, Prefix, Chunk, CSum, Opts}, Timeout).
|
||||
append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra, Timeout) ->
|
||||
send_sync(PidSpec, {append_chunk, PlacementKey, Prefix, Chunk, CSum, ChunkExtra}, Timeout).
|
||||
|
||||
-spec write_chunk(pid(), File::string(), machi_dt:file_offset(),
|
||||
Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum()) ->
|
||||
Chunk::binary(), CSum::binary()) ->
|
||||
ok | {error, machi_client_error_reason()}.
|
||||
write_chunk(PidSpec, File, Offset, Chunk, CSum) ->
|
||||
write_chunk(PidSpec, File, Offset, Chunk, CSum, ?DEFAULT_TIMEOUT).
|
||||
|
||||
-spec write_chunk(pid(), File::string(), machi_dt:file_offset(),
|
||||
Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum(), Timeout::non_neg_integer()) ->
|
||||
Chunk::binary(), CSum::binary(), Timeout::non_neg_integer()) ->
|
||||
ok | {error, machi_client_error_reason()}.
|
||||
write_chunk(PidSpec, File, Offset, Chunk, CSum, Timeout) ->
|
||||
send_sync(PidSpec, {write_chunk, File, Offset, Chunk, CSum}, Timeout).
|
||||
|
|
@ -135,25 +128,24 @@ write_chunk(PidSpec, File, Offset, Chunk, CSum, Timeout) ->
|
|||
%% {Chunks, TrimmedChunks}}' for live file while it returns `{error,
|
||||
%% trimmed}' if all bytes of the file was trimmed.
|
||||
-spec read_chunk(pid(), File::string(), machi_dt:file_offset(), machi_dt:chunk_size(),
|
||||
machi_dt:read_opts_x()) ->
|
||||
[{flag_no_checksum | flag_no_chunk | needs_trimmed, boolean()}]) ->
|
||||
{ok, {Chunks::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size(), binary()}],
|
||||
Trimmed::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size()}]}} |
|
||||
{error, machi_client_error_reason()}.
|
||||
read_chunk(PidSpec, File, Offset, Size, Opts) ->
|
||||
read_chunk(PidSpec, File, Offset, Size, Opts, ?DEFAULT_TIMEOUT).
|
||||
read_chunk(PidSpec, File, Offset, Size, Options) ->
|
||||
read_chunk(PidSpec, File, Offset, Size, Options, ?DEFAULT_TIMEOUT).
|
||||
|
||||
-spec read_chunk(pid(), File::string(), machi_dt:file_offset(), machi_dt:chunk_size(),
|
||||
machi_dt:read_opts_x(),
|
||||
[{flag_no_checksum | flag_no_chunk | needs_trimmed, boolean()}],
|
||||
Timeout::non_neg_integer()) ->
|
||||
{ok, {Chunks::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size(), binary()}],
|
||||
Trimmed::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size()}]}} |
|
||||
{error, machi_client_error_reason()}.
|
||||
read_chunk(PidSpec, File, Offset, Size, Opts0, Timeout) ->
|
||||
Opts = machi_util:read_opts_default(Opts0),
|
||||
send_sync(PidSpec, {read_chunk, File, Offset, Size, Opts}, Timeout).
|
||||
read_chunk(PidSpec, File, Offset, Size, Options, Timeout) ->
|
||||
send_sync(PidSpec, {read_chunk, File, Offset, Size, Options}, Timeout).
|
||||
|
||||
%% @doc Trims arbitrary binary range of any file. If a specified range
|
||||
%% has any byte trimmed, it fails and returns `{error, trimmed}'.
|
||||
%% has any byte trimmed, it fails and returns `{error, trimmed}`.
|
||||
%% Otherwise it trims all bytes in that range. If there are
|
||||
%% overlapping chunks with client-specified checksum, they will cut
|
||||
%% off and checksum are re-calculated in server side. TODO: Add
|
||||
|
|
@ -289,19 +281,19 @@ do_send_sync2({auth, User, Pass}, #state{sock=Sock}=S) ->
|
|||
Res = {bummer, {X, Y, erlang:get_stacktrace()}},
|
||||
{Res, S}
|
||||
end;
|
||||
do_send_sync2({append_chunk, NS, Prefix, Chunk, CSum, Opts},
|
||||
do_send_sync2({append_chunk, PlacementKey, Prefix, Chunk, CSum, ChunkExtra},
|
||||
#state{sock=Sock, sock_id=Index, count=Count}=S) ->
|
||||
try
|
||||
ReqID = <<Index:64/big, Count:64/big>>,
|
||||
PK = if PlacementKey == <<>> -> undefined;
|
||||
true -> PlacementKey
|
||||
end,
|
||||
CSumT = convert_csum_req(CSum, Chunk),
|
||||
{ChunkExtra, Pref, FailPref} = machi_pb_translate:conv_from_append_opts(Opts),
|
||||
Req = #mpb_appendchunkreq{namespace=NS,
|
||||
Req = #mpb_appendchunkreq{placement_key=PK,
|
||||
prefix=Prefix,
|
||||
chunk=Chunk,
|
||||
csum=CSumT,
|
||||
chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref},
|
||||
chunk_extra=ChunkExtra},
|
||||
R1a = #mpb_request{req_id=ReqID, do_not_alter=1,
|
||||
append_chunk=Req},
|
||||
Bin1a = machi_pb:encode_mpb_request(R1a),
|
||||
|
|
@ -346,13 +338,13 @@ do_send_sync2({write_chunk, File, Offset, Chunk, CSum},
|
|||
Res = {bummer, {X, Y, erlang:get_stacktrace()}},
|
||||
{Res, S#state{count=Count+1}}
|
||||
end;
|
||||
do_send_sync2({read_chunk, File, Offset, Size, Opts},
|
||||
do_send_sync2({read_chunk, File, Offset, Size, Options},
|
||||
#state{sock=Sock, sock_id=Index, count=Count}=S) ->
|
||||
try
|
||||
ReqID = <<Index:64/big, Count:64/big>>,
|
||||
#read_opts{no_checksum=FlagNoChecksum,
|
||||
no_chunk=FlagNoChunk,
|
||||
needs_trimmed=NeedsTrimmed} = Opts,
|
||||
FlagNoChecksum = proplists:get_value(no_checksum, Options, false),
|
||||
FlagNoChunk = proplists:get_value(no_chunk, Options, false),
|
||||
NeedsTrimmed = proplists:get_value(needs_trimmed, Options, false),
|
||||
Req = #mpb_readchunkreq{chunk_pos=#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size},
|
||||
|
|
@ -445,15 +437,9 @@ do_send_sync2({list_files},
|
|||
{Res, S#state{count=Count+1}}
|
||||
end.
|
||||
|
||||
%% We only convert the checksum types that make sense here:
|
||||
%% none or client_sha. None of the other types should be sent
|
||||
%% to us via the PB high protocol.
|
||||
|
||||
convert_csum_req(none, Chunk) ->
|
||||
#mpb_chunkcsum{type='CSUM_TAG_CLIENT_SHA',
|
||||
csum=machi_util:checksum_chunk(Chunk)};
|
||||
convert_csum_req(<<>>, Chunk) ->
|
||||
convert_csum_req(none, Chunk);
|
||||
convert_csum_req({client_sha, CSumBin}, _Chunk) ->
|
||||
#mpb_chunkcsum{type='CSUM_TAG_CLIENT_SHA',
|
||||
csum=CSumBin}.
|
||||
|
|
@ -501,12 +487,12 @@ convert_read_chunk_resp(#mpb_readchunkresp{status='OK', chunks=PB_Chunks, trimme
|
|||
csum=#mpb_chunkcsum{type=T, csum=Ck}}) ->
|
||||
%% TODO: cleanup export
|
||||
Csum = <<(machi_pb_translate:conv_to_csum_tag(T)):8, Ck/binary>>,
|
||||
{list_to_binary(File), Offset, Chunk, Csum}
|
||||
{File, Offset, Chunk, Csum}
|
||||
end, PB_Chunks),
|
||||
Trimmed = lists:map(fun(#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size}) ->
|
||||
{list_to_binary(File), Offset, Size}
|
||||
{File, Offset, Size}
|
||||
end, PB_Trimmed),
|
||||
{ok, {Chunks, Trimmed}};
|
||||
convert_read_chunk_resp(#mpb_readchunkresp{status=Status}) ->
|
||||
|
|
|
|||
|
|
@ -34,9 +34,7 @@
|
|||
-export([from_pb_request/1,
|
||||
from_pb_response/1,
|
||||
to_pb_request/2,
|
||||
to_pb_response/3,
|
||||
conv_from_append_opts/1,
|
||||
conv_to_append_opts/1
|
||||
to_pb_response/3
|
||||
]).
|
||||
|
||||
%% TODO: fixme cleanup
|
||||
|
|
@ -45,104 +43,93 @@
|
|||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
echo=#mpb_echoreq{message=Msg}}) ->
|
||||
{ReqID, {low_skip_wedge, {low_echo, Msg}}};
|
||||
{ReqID, {low_echo, undefined, Msg}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
auth=#mpb_authreq{user=User, password=Pass}}) ->
|
||||
{ReqID, {low_skip_wedge, {low_auth, User, Pass}}};
|
||||
{ReqID, {low_auth, undefined, User, Pass}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
append_chunk=IR=#mpb_ll_appendchunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS_str,
|
||||
locator=NSLocator,
|
||||
append_chunk=#mpb_ll_appendchunkreq{
|
||||
epoch_id=PB_EpochID,
|
||||
placement_key=PKey,
|
||||
prefix=Prefix,
|
||||
chunk=Chunk,
|
||||
csum=#mpb_chunkcsum{type=CSum_type, csum=CSum}}}) ->
|
||||
NS = list_to_binary(NS_str),
|
||||
csum=#mpb_chunkcsum{type=CSum_type, csum=CSum},
|
||||
chunk_extra=ChunkExtra}}) ->
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
CSum_tag = conv_to_csum_tag(CSum_type),
|
||||
Opts = conv_to_append_opts(IR),
|
||||
%% NOTE: The tuple position of NSLocator is a bit odd, because EpochID
|
||||
%% _must_ be in the 4th position (as NSV & NS must be in 2nd & 3rd).
|
||||
{ReqID, {low_append_chunk, NSVersion, NS, EpochID, NSLocator,
|
||||
Prefix, Chunk, CSum_tag, CSum, Opts}};
|
||||
{ReqID, {low_append_chunk, EpochID, PKey, Prefix, Chunk, CSum_tag, CSum,
|
||||
ChunkExtra}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
write_chunk=#mpb_ll_writechunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS_str,
|
||||
epoch_id=PB_EpochID,
|
||||
chunk=#mpb_chunk{file_name=File,
|
||||
offset=Offset,
|
||||
chunk=Chunk,
|
||||
csum=#mpb_chunkcsum{type=CSum_type, csum=CSum}}}}) ->
|
||||
NS = list_to_binary(NS_str),
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
CSum_tag = conv_to_csum_tag(CSum_type),
|
||||
{ReqID, {low_write_chunk, NSVersion, NS, EpochID, File, Offset, Chunk, CSum_tag, CSum}};
|
||||
{ReqID, {low_write_chunk, EpochID, File, Offset, Chunk, CSum_tag, CSum}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
read_chunk=#mpb_ll_readchunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS_str,
|
||||
epoch_id=PB_EpochID,
|
||||
chunk_pos=ChunkPos,
|
||||
flag_no_checksum=PB_GetNoChecksum,
|
||||
flag_no_chunk=PB_GetNoChunk,
|
||||
flag_needs_trimmed=PB_NeedsTrimmed}}) ->
|
||||
NS = list_to_binary(NS_str),
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
Opts = #read_opts{no_checksum=PB_GetNoChecksum,
|
||||
no_chunk=PB_GetNoChunk,
|
||||
needs_trimmed=PB_NeedsTrimmed},
|
||||
Opts = [{no_checksum, conv_to_boolean(PB_GetNoChecksum)},
|
||||
{no_chunk, conv_to_boolean(PB_GetNoChunk)},
|
||||
{needs_trimmed, conv_to_boolean(PB_NeedsTrimmed)}],
|
||||
#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size} = ChunkPos,
|
||||
{ReqID, {low_read_chunk, NSVersion, NS, EpochID, File, Offset, Size, Opts}};
|
||||
{ReqID, {low_read_chunk, EpochID, File, Offset, Size, Opts}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
trim_chunk=#mpb_ll_trimchunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS_str,
|
||||
epoch_id=PB_EpochID,
|
||||
file=File,
|
||||
offset=Offset,
|
||||
size=Size,
|
||||
trigger_gc=TriggerGC}}) ->
|
||||
NS = list_to_binary(NS_str),
|
||||
trigger_gc=PB_TriggerGC}}) ->
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
{ReqID, {low_trim_chunk, NSVersion, NS, EpochID, File, Offset, Size, TriggerGC}};
|
||||
TriggerGC = conv_to_boolean(PB_TriggerGC),
|
||||
{ReqID, {low_trim_chunk, EpochID, File, Offset, Size, TriggerGC}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
checksum_list=#mpb_ll_checksumlistreq{
|
||||
epoch_id=PB_EpochID,
|
||||
file=File}}) ->
|
||||
{ReqID, {low_skip_wedge, {low_checksum_list, File}}};
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
{ReqID, {low_checksum_list, EpochID, File}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
list_files=#mpb_ll_listfilesreq{
|
||||
epoch_id=PB_EpochID}}) ->
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
{ReqID, {low_skip_wedge, {low_list_files, EpochID}}};
|
||||
{ReqID, {low_list_files, EpochID}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
wedge_status=#mpb_ll_wedgestatusreq{}}) ->
|
||||
{ReqID, {low_skip_wedge, {low_wedge_status}}};
|
||||
{ReqID, {low_wedge_status, undefined}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
delete_migration=#mpb_ll_deletemigrationreq{
|
||||
epoch_id=PB_EpochID,
|
||||
file=File}}) ->
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
{ReqID, {low_skip_wedge, {low_delete_migration, EpochID, File}}};
|
||||
{ReqID, {low_delete_migration, EpochID, File}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
trunc_hack=#mpb_ll_trunchackreq{
|
||||
epoch_id=PB_EpochID,
|
||||
file=File}}) ->
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
{ReqID, {low_skip_wedge, {low_trunc_hack, EpochID, File}}};
|
||||
{ReqID, {low_trunc_hack, EpochID, File}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
proj_gl=#mpb_ll_getlatestepochidreq{type=ProjType}}) ->
|
||||
|
|
@ -183,22 +170,22 @@ from_pb_request(#mpb_request{req_id=ReqID,
|
|||
{ReqID, {high_auth, User, Pass}};
|
||||
from_pb_request(#mpb_request{req_id=ReqID,
|
||||
append_chunk=IR=#mpb_appendchunkreq{}}) ->
|
||||
#mpb_appendchunkreq{namespace=NS_str,
|
||||
#mpb_appendchunkreq{placement_key=__todoPK,
|
||||
prefix=Prefix,
|
||||
chunk=Chunk,
|
||||
csum=CSum} = IR,
|
||||
NS = list_to_binary(NS_str),
|
||||
csum=CSum,
|
||||
chunk_extra=ChunkExtra} = IR,
|
||||
TaggedCSum = make_tagged_csum(CSum, Chunk),
|
||||
Opts = conv_to_append_opts(IR),
|
||||
{ReqID, {high_append_chunk, NS, Prefix, Chunk, TaggedCSum, Opts}};
|
||||
{ReqID, {high_append_chunk, __todoPK, Prefix, Chunk, TaggedCSum,
|
||||
ChunkExtra}};
|
||||
from_pb_request(#mpb_request{req_id=ReqID,
|
||||
write_chunk=IR=#mpb_writechunkreq{}}) ->
|
||||
#mpb_writechunkreq{chunk=#mpb_chunk{file_name=File,
|
||||
offset=Offset,
|
||||
chunk=Chunk,
|
||||
csum=CSumRec}} = IR,
|
||||
CSum = make_tagged_csum(CSumRec, Chunk),
|
||||
{ReqID, {high_write_chunk, File, Offset, Chunk, CSum}};
|
||||
csum=CSum}} = IR,
|
||||
TaggedCSum = make_tagged_csum(CSum, Chunk),
|
||||
{ReqID, {high_write_chunk, File, Offset, Chunk, TaggedCSum}};
|
||||
from_pb_request(#mpb_request{req_id=ReqID,
|
||||
read_chunk=IR=#mpb_readchunkreq{}}) ->
|
||||
#mpb_readchunkreq{chunk_pos=#mpb_chunkpos{file_name=File,
|
||||
|
|
@ -207,10 +194,11 @@ from_pb_request(#mpb_request{req_id=ReqID,
|
|||
flag_no_checksum=FlagNoChecksum,
|
||||
flag_no_chunk=FlagNoChunk,
|
||||
flag_needs_trimmed=NeedsTrimmed} = IR,
|
||||
Opts = #read_opts{no_checksum=FlagNoChecksum,
|
||||
no_chunk=FlagNoChunk,
|
||||
needs_trimmed=NeedsTrimmed},
|
||||
{ReqID, {high_read_chunk, File, Offset, Size, Opts}};
|
||||
%% I want MAPS
|
||||
Options = [{no_checksum, machi_util:int2bool(FlagNoChecksum)},
|
||||
{no_chunk, machi_util:int2bool(FlagNoChunk)},
|
||||
{needs_trimmed, machi_util:int2bool(NeedsTrimmed)}],
|
||||
{ReqID, {high_read_chunk, File, Offset, Size, Options}};
|
||||
from_pb_request(#mpb_request{req_id=ReqID,
|
||||
trim_chunk=IR=#mpb_trimchunkreq{}}) ->
|
||||
#mpb_trimchunkreq{chunk_pos=#mpb_chunkpos{file_name=File,
|
||||
|
|
@ -274,12 +262,12 @@ from_pb_response(#mpb_ll_response{
|
|||
chunk=Bytes,
|
||||
csum=#mpb_chunkcsum{type=T,csum=Ck}}) ->
|
||||
Csum = <<(conv_to_csum_tag(T)):8, Ck/binary>>,
|
||||
{list_to_binary(File), Offset, Bytes, Csum}
|
||||
{File, Offset, Bytes, Csum}
|
||||
end, PB_Chunks),
|
||||
Trimmed = lists:map(fun(#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size}) ->
|
||||
{list_to_binary(File), Offset, Size}
|
||||
{File, Offset, Size}
|
||||
end, PB_Trimmed),
|
||||
{ReqID, {ok, {Chunks, Trimmed}}};
|
||||
_ ->
|
||||
|
|
@ -315,16 +303,12 @@ from_pb_response(#mpb_ll_response{
|
|||
from_pb_response(#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
wedge_status=#mpb_ll_wedgestatusresp{
|
||||
status=Status,
|
||||
epoch_id=PB_EpochID, wedged_flag=Wedged_p,
|
||||
namespace_version=NSVersion, namespace=NS_str}}) ->
|
||||
GeneralStatus = case machi_pb_high_client:convert_general_status_code(Status) of
|
||||
ok -> ok;
|
||||
_Else -> {yukky, _Else}
|
||||
end,
|
||||
epoch_id=PB_EpochID, wedged_flag=PB_Wedged}}) ->
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
NS = list_to_binary(NS_str),
|
||||
{ReqID, {GeneralStatus, {Wedged_p, EpochID, NSVersion, NS}}};
|
||||
Wedged_p = if PB_Wedged == 1 -> true;
|
||||
PB_Wedged == 0 -> false
|
||||
end,
|
||||
{ReqID, {ok, {Wedged_p, EpochID}}};
|
||||
from_pb_response(#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
delete_migration=#mpb_ll_deletemigrationresp{
|
||||
|
|
@ -390,100 +374,89 @@ from_pb_response(#mpb_ll_response{
|
|||
'OK' ->
|
||||
{ReqID, {ok, Epochs}};
|
||||
_ ->
|
||||
{ReqID, machi_pb_high_client:convert_general_status_code(Status)}
|
||||
{ReqID< machi_pb_high_client:convert_general_status_code(Status)}
|
||||
end.
|
||||
%% No response for proj_kp/kick_projection_reaction
|
||||
|
||||
%% TODO: move the #mbp_* record making code from
|
||||
%% machi_pb_high_client:do_send_sync() clauses into to_pb_request().
|
||||
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_echo, Msg}}) ->
|
||||
to_pb_request(ReqID, {low_echo, _BogusEpochID, Msg}) ->
|
||||
#mpb_ll_request{
|
||||
req_id=ReqID, do_not_alter=2,
|
||||
echo=#mpb_echoreq{message=Msg}};
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_auth, User, Pass}}) ->
|
||||
to_pb_request(ReqID, {low_auth, _BogusEpochID, User, Pass}) ->
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
auth=#mpb_authreq{user=User, password=Pass}};
|
||||
%% NOTE: The tuple position of NSLocator is a bit odd, because EpochID
|
||||
%% _must_ be in the 4th position (as NSV & NS must be in 2nd & 3rd).
|
||||
to_pb_request(ReqID, {low_append_chunk, NSVersion, NS, EpochID, NSLocator,
|
||||
Prefix, Chunk, CSum_tag, CSum, Opts}) ->
|
||||
to_pb_request(ReqID, {low_append_chunk, EpochID, PKey, Prefix, Chunk,
|
||||
CSum_tag, CSum, ChunkExtra}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
CSum_type = conv_from_csum_tag(CSum_tag),
|
||||
PB_CSum = #mpb_chunkcsum{type=CSum_type, csum=CSum},
|
||||
{ChunkExtra, Pref, FailPref} = conv_from_append_opts(Opts),
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
append_chunk=#mpb_ll_appendchunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS,
|
||||
locator=NSLocator,
|
||||
epoch_id=PB_EpochID,
|
||||
placement_key=PKey,
|
||||
prefix=Prefix,
|
||||
chunk=Chunk,
|
||||
csum=PB_CSum,
|
||||
chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref}};
|
||||
to_pb_request(ReqID, {low_write_chunk, NSVersion, NS, EpochID, File, Offset, Chunk, CSum_tag, CSum}) ->
|
||||
chunk_extra=ChunkExtra}};
|
||||
to_pb_request(ReqID, {low_write_chunk, EpochID, File, Offset, Chunk, CSum_tag, CSum}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
CSum_type = conv_from_csum_tag(CSum_tag),
|
||||
PB_CSum = #mpb_chunkcsum{type=CSum_type, csum=CSum},
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
write_chunk=#mpb_ll_writechunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS,
|
||||
epoch_id=PB_EpochID,
|
||||
chunk=#mpb_chunk{file_name=File,
|
||||
offset=Offset,
|
||||
chunk=Chunk,
|
||||
csum=PB_CSum}}};
|
||||
to_pb_request(ReqID, {low_read_chunk, NSVersion, NS, EpochID, File, Offset, Size, Opts}) ->
|
||||
to_pb_request(ReqID, {low_read_chunk, EpochID, File, Offset, Size, Opts}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#read_opts{no_checksum=FNChecksum,
|
||||
no_chunk=FNChunk,
|
||||
needs_trimmed=NeedsTrimmed} = Opts,
|
||||
FNChecksum = proplists:get_value(no_checksum, Opts, false),
|
||||
FNChunk = proplists:get_value(no_chunk, Opts, false),
|
||||
NeedsTrimmed = proplists:get_value(needs_trimmed, Opts, false),
|
||||
#mpb_ll_request{
|
||||
req_id=ReqID, do_not_alter=2,
|
||||
read_chunk=#mpb_ll_readchunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS,
|
||||
epoch_id=PB_EpochID,
|
||||
chunk_pos=#mpb_chunkpos{
|
||||
epoch_id=PB_EpochID,
|
||||
chunk_pos=#mpb_chunkpos{
|
||||
file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size},
|
||||
flag_no_checksum=FNChecksum,
|
||||
flag_no_chunk=FNChunk,
|
||||
flag_needs_trimmed=NeedsTrimmed}};
|
||||
to_pb_request(ReqID, {low_trim_chunk, NSVersion, NS, EpochID, File, Offset, Size, TriggerGC}) ->
|
||||
flag_no_checksum=machi_util:bool2int(FNChecksum),
|
||||
flag_no_chunk=machi_util:bool2int(FNChunk),
|
||||
flag_needs_trimmed=machi_util:bool2int(NeedsTrimmed)}};
|
||||
to_pb_request(ReqID, {low_trim_chunk, EpochID, File, Offset, Size, TriggerGC}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
trim_chunk=#mpb_ll_trimchunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS,
|
||||
epoch_id=PB_EpochID,
|
||||
file=File,
|
||||
offset=Offset,
|
||||
size=Size,
|
||||
trigger_gc=TriggerGC}};
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_checksum_list, File}}) ->
|
||||
to_pb_request(ReqID, {low_checksum_list, EpochID, File}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
checksum_list=#mpb_ll_checksumlistreq{
|
||||
epoch_id=PB_EpochID,
|
||||
file=File}};
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_list_files, EpochID}}) ->
|
||||
to_pb_request(ReqID, {low_list_files, EpochID}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
list_files=#mpb_ll_listfilesreq{epoch_id=PB_EpochID}};
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_wedge_status}}) ->
|
||||
to_pb_request(ReqID, {low_wedge_status, _BogusEpochID}) ->
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
wedge_status=#mpb_ll_wedgestatusreq{}};
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_delete_migration, EpochID, File}}) ->
|
||||
to_pb_request(ReqID, {low_delete_migration, EpochID, File}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
delete_migration=#mpb_ll_deletemigrationreq{
|
||||
epoch_id=PB_EpochID,
|
||||
file=File}};
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_trunc_hack, EpochID, File}}) ->
|
||||
to_pb_request(ReqID, {low_trunc_hack, EpochID, File}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
trunc_hack=#mpb_ll_trunchackreq{
|
||||
|
|
@ -519,15 +492,15 @@ to_pb_response(_ReqID, _, async_no_response=X) ->
|
|||
X;
|
||||
to_pb_response(ReqID, _, {low_error, ErrCode, ErrMsg}) ->
|
||||
make_ll_error_resp(ReqID, ErrCode, ErrMsg);
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_echo, _Msg}}, Resp) ->
|
||||
to_pb_response(ReqID, {low_echo, _BogusEpochID, _Msg}, Resp) ->
|
||||
#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
echo=#mpb_echoresp{message=Resp}};
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_auth, _, _}}, __TODO_Resp) ->
|
||||
to_pb_response(ReqID, {low_auth, _, _, _}, __TODO_Resp) ->
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
generic=#mpb_errorresp{code=1,
|
||||
msg="AUTH not implemented"}};
|
||||
to_pb_response(ReqID, {low_append_chunk, _NSV, _NS, _EID, _NSL, _Pfx, _Ch, _CST, _CS, _O}, Resp)->
|
||||
to_pb_response(ReqID, {low_append_chunk, _EID, _PKey, _Pfx, _Ch, _CST, _CS, _CE}, Resp)->
|
||||
case Resp of
|
||||
{ok, {Offset, Size, File}} ->
|
||||
Where = #mpb_chunkpos{offset=Offset,
|
||||
|
|
@ -543,11 +516,11 @@ to_pb_response(ReqID, {low_append_chunk, _NSV, _NS, _EID, _NSL, _Pfx, _Ch, _CST,
|
|||
_Else ->
|
||||
make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {low_write_chunk, _NSV, _NS, _EID, _Fl, _Off, _Ch, _CST, _CS},Resp)->
|
||||
to_pb_response(ReqID, {low_write_chunk, _EID, _Fl, _Off, _Ch, _CST, _CS},Resp)->
|
||||
Status = conv_from_status(Resp),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
write_chunk=#mpb_ll_writechunkresp{status=Status}};
|
||||
to_pb_response(ReqID, {low_read_chunk, _NSV, _NS, _EID, _Fl, _Off, _Sz, _Opts}, Resp)->
|
||||
to_pb_response(ReqID, {low_read_chunk, _EID, _Fl, _Off, _Sz, _Opts}, Resp)->
|
||||
case Resp of
|
||||
{ok, {Chunks, Trimmed}} ->
|
||||
PB_Chunks = lists:map(fun({File, Offset, Bytes, Csum}) ->
|
||||
|
|
@ -574,7 +547,7 @@ to_pb_response(ReqID, {low_read_chunk, _NSV, _NS, _EID, _Fl, _Off, _Sz, _Opts},
|
|||
_Else ->
|
||||
make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {low_trim_chunk, _, _, _, _, _, _, _}, Resp) ->
|
||||
to_pb_response(ReqID, {low_trim_chunk, _, _, _, _, _}, Resp) ->
|
||||
case Resp of
|
||||
ok ->
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
|
|
@ -582,11 +555,11 @@ to_pb_response(ReqID, {low_trim_chunk, _, _, _, _, _, _, _}, Resp) ->
|
|||
{error, _}=Error ->
|
||||
Status = conv_from_status(Error),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
trim_chunk=#mpb_ll_trimchunkresp{status=Status}};
|
||||
read_chunk=#mpb_ll_trimchunkresp{status=Status}};
|
||||
_Else ->
|
||||
make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_checksum_list, _File}}, Resp) ->
|
||||
to_pb_response(ReqID, {low_checksum_list, _EpochID, _File}, Resp) ->
|
||||
case Resp of
|
||||
{ok, Chunk} ->
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
|
|
@ -599,7 +572,7 @@ to_pb_response(ReqID, {low_skip_wedge, {low_checksum_list, _File}}, Resp) ->
|
|||
_Else ->
|
||||
make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_list_files, _EpochID}}, Resp) ->
|
||||
to_pb_response(ReqID, {low_list_files, _EpochID}, Resp) ->
|
||||
case Resp of
|
||||
{ok, FileInfo} ->
|
||||
PB_Files = [#mpb_fileinfo{file_size=Size, file_name=Name} ||
|
||||
|
|
@ -614,28 +587,26 @@ to_pb_response(ReqID, {low_skip_wedge, {low_list_files, _EpochID}}, Resp) ->
|
|||
_Else ->
|
||||
make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_wedge_status}}, Resp) ->
|
||||
to_pb_response(ReqID, {low_wedge_status, _BogusEpochID}, Resp) ->
|
||||
case Resp of
|
||||
{error, _}=Error ->
|
||||
Status = conv_from_status(Error),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
wedge_status=#mpb_ll_wedgestatusresp{status=Status}};
|
||||
{Wedged_p, EpochID, NSVersion, NS} ->
|
||||
{Wedged_p, EpochID} ->
|
||||
PB_Wedged = conv_from_boolean(Wedged_p),
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
wedge_status=#mpb_ll_wedgestatusresp{
|
||||
status='OK',
|
||||
epoch_id=PB_EpochID,
|
||||
wedged_flag=Wedged_p,
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS
|
||||
}}
|
||||
wedged_flag=PB_Wedged}}
|
||||
end;
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_delete_migration, _EID, _Fl}}, Resp)->
|
||||
to_pb_response(ReqID, {low_delete_migration, _EID, _Fl}, Resp)->
|
||||
Status = conv_from_status(Resp),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
delete_migration=#mpb_ll_deletemigrationresp{status=Status}};
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_trunc_hack, _EID, _Fl}}, Resp)->
|
||||
to_pb_response(ReqID, {low_trunc_hack, _EID, _Fl}, Resp)->
|
||||
Status = conv_from_status(Resp),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
trunc_hack=#mpb_ll_trunchackresp{status=Status}};
|
||||
|
|
@ -716,7 +687,7 @@ to_pb_response(ReqID, {high_auth, _User, _Pass}, _Resp) ->
|
|||
#mpb_response{req_id=ReqID,
|
||||
generic=#mpb_errorresp{code=1,
|
||||
msg="AUTH not implemented"}};
|
||||
to_pb_response(ReqID, {high_append_chunk, _NS, _Prefix, _Chunk, _TSum, _O}, Resp)->
|
||||
to_pb_response(ReqID, {high_append_chunk, _TODO, _Prefix, _Chunk, _TSum, _CE}, Resp)->
|
||||
case Resp of
|
||||
{ok, {Offset, Size, File}} ->
|
||||
Where = #mpb_chunkpos{offset=Offset,
|
||||
|
|
@ -732,7 +703,7 @@ to_pb_response(ReqID, {high_append_chunk, _NS, _Prefix, _Chunk, _TSum, _O}, Resp
|
|||
_Else ->
|
||||
make_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {high_write_chunk, _File, _Offset, _Chunk, _CSum}, Resp) ->
|
||||
to_pb_response(ReqID, {high_write_chunk, _File, _Offset, _Chunk, _TaggedCSum}, Resp) ->
|
||||
case Resp of
|
||||
{ok, {_,_,_}} ->
|
||||
%% machi_cr_client returns ok 2-tuple, convert to simple ok.
|
||||
|
|
@ -822,12 +793,12 @@ make_tagged_csum(#mpb_chunkcsum{type='CSUM_TAG_CLIENT_SHA', csum=CSum}, _CB) ->
|
|||
make_ll_error_resp(ReqID, Code, Msg) ->
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
generic=#mpb_errorresp{code=Code,
|
||||
msg=Msg}}.
|
||||
msg=Msg}}.
|
||||
|
||||
make_error_resp(ReqID, Code, Msg) ->
|
||||
#mpb_response{req_id=ReqID,
|
||||
generic=#mpb_errorresp{code=Code,
|
||||
msg=Msg}}.
|
||||
msg=Msg}}.
|
||||
|
||||
conv_from_epoch_id({Epoch, EpochCSum}) ->
|
||||
#mpb_epochid{epoch_number=Epoch,
|
||||
|
|
@ -840,7 +811,6 @@ conv_to_epoch_id(#mpb_epochid{epoch_number=Epoch,
|
|||
conv_to_projection_v1(#mpb_projectionv1{epoch_number=Epoch,
|
||||
epoch_csum=CSum,
|
||||
author_server=Author,
|
||||
chain_name=ChainName,
|
||||
all_members=AllMembers,
|
||||
witnesses=Witnesses,
|
||||
creation_time=CTime,
|
||||
|
|
@ -854,7 +824,6 @@ conv_to_projection_v1(#mpb_projectionv1{epoch_number=Epoch,
|
|||
#projection_v1{epoch_number=Epoch,
|
||||
epoch_csum=CSum,
|
||||
author_server=to_atom(Author),
|
||||
chain_name=to_atom(ChainName),
|
||||
all_members=[to_atom(X) || X <- AllMembers],
|
||||
witnesses=[to_atom(X) || X <- Witnesses],
|
||||
creation_time=conv_to_now(CTime),
|
||||
|
|
@ -984,34 +953,24 @@ conv_from_status({error, partial_read}) ->
|
|||
conv_from_status({error, bad_epoch}) ->
|
||||
'BAD_EPOCH';
|
||||
conv_from_status(_OOPS) ->
|
||||
io:format(user, "HEY, ~s:~w got ~p\n", [?MODULE, ?LINE, _OOPS]),
|
||||
io:format(user, "HEY, ~s:~w got ~w\n", [?MODULE, ?LINE, _OOPS]),
|
||||
'BAD_JOSS'.
|
||||
|
||||
conv_from_append_opts(#append_opts{chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref}) ->
|
||||
{ChunkExtra, Pref, FailPref}.
|
||||
conv_to_boolean(undefined) ->
|
||||
false;
|
||||
conv_to_boolean(0) ->
|
||||
false;
|
||||
conv_to_boolean(N) when is_integer(N) ->
|
||||
true.
|
||||
|
||||
|
||||
conv_to_append_opts(#mpb_appendchunkreq{
|
||||
chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref}) ->
|
||||
#append_opts{chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref};
|
||||
conv_to_append_opts(#mpb_ll_appendchunkreq{
|
||||
chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref}) ->
|
||||
#append_opts{chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref}.
|
||||
conv_from_boolean(false) ->
|
||||
0;
|
||||
conv_from_boolean(true) ->
|
||||
1.
|
||||
|
||||
conv_from_projection_v1(#projection_v1{epoch_number=Epoch,
|
||||
epoch_csum=CSum,
|
||||
author_server=Author,
|
||||
chain_name=ChainName,
|
||||
all_members=AllMembers,
|
||||
witnesses=Witnesses,
|
||||
creation_time=CTime,
|
||||
|
|
@ -1025,7 +984,6 @@ conv_from_projection_v1(#projection_v1{epoch_number=Epoch,
|
|||
#mpb_projectionv1{epoch_number=Epoch,
|
||||
epoch_csum=CSum,
|
||||
author_server=to_list(Author),
|
||||
chain_name=to_list(ChainName),
|
||||
all_members=[to_list(X) || X <- AllMembers],
|
||||
witnesses=[to_list(X) || X <- Witnesses],
|
||||
creation_time=conv_from_now(CTime),
|
||||
|
|
|
|||
|
|
@ -1,23 +1,3 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_plist).
|
||||
|
||||
%%% @doc persistent list of binaries
|
||||
|
|
|
|||
|
|
@ -174,7 +174,6 @@ make_summary(#projection_v1{epoch_number=EpochNum,
|
|||
repairing=Repairing_list,
|
||||
dbg=Dbg, dbg2=Dbg2}) ->
|
||||
[{epoch,EpochNum}, {csum,_CSum4},
|
||||
{all, _All_list},
|
||||
{author,Author}, {mode,CMode},{witnesses, Witness_list},
|
||||
{upi,UPI_list},{repair,Repairing_list},{down,Down_list}] ++
|
||||
[{d,Dbg}, {d2,Dbg2}].
|
||||
|
|
|
|||
|
|
@ -321,7 +321,7 @@ do_proj_write3(ProjType, #projection_v1{epoch_number=Epoch,
|
|||
end.
|
||||
|
||||
do_proj_write4(ProjType, Proj, Path, Epoch, #state{consistency_mode=CMode}=S) ->
|
||||
{{ok, FH}, Epoch, Path} = {file:open(Path, [write, raw, binary]), Epoch, Path},
|
||||
{ok, FH} = file:open(Path, [write, raw, binary]),
|
||||
ok = file:write(FH, term_to_binary(Proj)),
|
||||
ok = file:sync(FH),
|
||||
ok = file:close(FH),
|
||||
|
|
@ -387,6 +387,7 @@ wait_for_liveness(PidSpec, StartTime, WaitTime) ->
|
|||
undefined ->
|
||||
case timer:now_diff(os:timestamp(), StartTime) div 1000 of
|
||||
X when X < WaitTime ->
|
||||
io:format(user, "\nYOO ~p ~p\n", [PidSpec, lists:sort(registered())]),
|
||||
timer:sleep(1),
|
||||
wait_for_liveness(PidSpec, StartTime, WaitTime)
|
||||
end;
|
||||
|
|
|
|||
|
|
@ -22,10 +22,6 @@
|
|||
%% proxy-process style API for hiding messy details such as TCP
|
||||
%% connection/disconnection with the remote Machi server.
|
||||
%%
|
||||
%% Please see {@link machi_flu1_client} the "Client API implemntation notes"
|
||||
%% section for how this module relates to the rest of the client API
|
||||
%% implementation.
|
||||
%%
|
||||
%% Machi is intentionally avoiding using distributed Erlang for
|
||||
%% Machi's communication. This design decision makes Erlang-side code
|
||||
%% more difficult & complex, but it's the price to pay for some
|
||||
|
|
@ -61,9 +57,10 @@
|
|||
%% FLU1 API
|
||||
-export([
|
||||
%% File API
|
||||
append_chunk/6, append_chunk/8,
|
||||
read_chunk/7, read_chunk/8,
|
||||
checksum_list/2, checksum_list/3,
|
||||
append_chunk/4, append_chunk/5,
|
||||
append_chunk_extra/5, append_chunk_extra/6,
|
||||
read_chunk/6, read_chunk/7,
|
||||
checksum_list/3, checksum_list/4,
|
||||
list_files/2, list_files/3,
|
||||
wedge_status/1, wedge_status/2,
|
||||
|
||||
|
|
@ -81,8 +78,8 @@
|
|||
quit/1,
|
||||
|
||||
%% Internal API
|
||||
write_chunk/7, write_chunk/8,
|
||||
trim_chunk/6, trim_chunk/7,
|
||||
write_chunk/5, write_chunk/6,
|
||||
trim_chunk/5, trim_chunk/6,
|
||||
|
||||
%% Helpers
|
||||
stop_proxies/1, start_proxies/1
|
||||
|
|
@ -107,39 +104,51 @@ start_link(#p_srvr{}=I) ->
|
|||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix'.
|
||||
|
||||
append_chunk(PidSpec, NSInfo, EpochID, Prefix, Chunk, CSum) ->
|
||||
append_chunk(PidSpec, NSInfo, EpochID, Prefix, Chunk, CSum,
|
||||
#append_opts{}, infinity).
|
||||
append_chunk(PidSpec, EpochID, Prefix, Chunk) ->
|
||||
append_chunk(PidSpec, EpochID, Prefix, Chunk, infinity).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix'.
|
||||
|
||||
append_chunk(PidSpec, NSInfo, EpochID, Prefix, Chunk, CSum, Opts,
|
||||
Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {append_chunk, NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum, Opts, Timeout}},
|
||||
append_chunk(PidSpec, EpochID, Prefix, Chunk, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {append_chunk, EpochID, Prefix, Chunk}},
|
||||
Timeout).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix'.
|
||||
|
||||
append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra)
|
||||
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
|
||||
append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra, infinity).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix'.
|
||||
|
||||
append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {append_chunk_extra, EpochID, Prefix,
|
||||
Chunk, ChunkExtra}},
|
||||
Timeout).
|
||||
|
||||
%% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
|
||||
|
||||
read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, Opts) ->
|
||||
read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, Opts, infinity).
|
||||
read_chunk(PidSpec, EpochID, File, Offset, Size, Opts) ->
|
||||
read_chunk(PidSpec, EpochID, File, Offset, Size, Opts, infinity).
|
||||
|
||||
%% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
|
||||
|
||||
read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, Opts, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {read_chunk, NSInfo, EpochID, File, Offset, Size, Opts}},
|
||||
read_chunk(PidSpec, EpochID, File, Offset, Size, Opts, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {read_chunk, EpochID, File, Offset, Size, Opts}},
|
||||
Timeout).
|
||||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
|
||||
checksum_list(PidSpec, File) ->
|
||||
checksum_list(PidSpec, File, infinity).
|
||||
checksum_list(PidSpec, EpochID, File) ->
|
||||
checksum_list(PidSpec, EpochID, File, infinity).
|
||||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
|
||||
checksum_list(PidSpec, File, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {checksum_list, File}},
|
||||
checksum_list(PidSpec, EpochID, File, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {checksum_list, EpochID, File}},
|
||||
Timeout).
|
||||
|
||||
%% @doc Fetch the list of all files on the remote FLU.
|
||||
|
|
@ -280,18 +289,18 @@ quit(PidSpec) ->
|
|||
%% @doc Write a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' at `Offset'.
|
||||
|
||||
write_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, CSum) ->
|
||||
write_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, CSum, infinity).
|
||||
write_chunk(PidSpec, EpochID, File, Offset, Chunk) ->
|
||||
write_chunk(PidSpec, EpochID, File, Offset, Chunk, infinity).
|
||||
|
||||
%% @doc Write a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' at `Offset'.
|
||||
|
||||
write_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, CSum, Timeout) ->
|
||||
case gen_server:call(PidSpec, {req, {write_chunk, NSInfo, EpochID, File, Offset, Chunk, CSum}},
|
||||
write_chunk(PidSpec, EpochID, File, Offset, Chunk, Timeout) ->
|
||||
case gen_server:call(PidSpec, {req, {write_chunk, EpochID, File, Offset, Chunk}},
|
||||
Timeout) of
|
||||
{error, written}=Err ->
|
||||
Size = byte_size(Chunk),
|
||||
case read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, undefined, Timeout) of
|
||||
case read_chunk(PidSpec, EpochID, File, Offset, Size, [], Timeout) of
|
||||
{ok, {[{File, Offset, Chunk2, _}], []}} when Chunk2 == Chunk ->
|
||||
%% See equivalent comment inside write_projection().
|
||||
ok;
|
||||
|
|
@ -303,15 +312,15 @@ write_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, CSum, Timeout) ->
|
|||
end.
|
||||
|
||||
|
||||
trim_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size) ->
|
||||
trim_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, infinity).
|
||||
trim_chunk(PidSpec, EpochID, File, Offset, Size) ->
|
||||
trim_chunk(PidSpec, EpochID, File, Offset, Size, infinity).
|
||||
|
||||
%% @doc Write a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' at `Offset'.
|
||||
|
||||
trim_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, Timeout) ->
|
||||
trim_chunk(PidSpec, EpochID, File, Offset, Chunk, Timeout) ->
|
||||
gen_server:call(PidSpec,
|
||||
{req, {trim_chunk, NSInfo, EpochID, File, Offset, Chunk}},
|
||||
{req, {trim_chunk, EpochID, File, Offset, Chunk}},
|
||||
Timeout).
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
|
@ -375,24 +384,24 @@ do_req_retry(_Req, 2, Err, S) ->
|
|||
do_req_retry(Req, Depth, _Err, S) ->
|
||||
do_req(Req, Depth + 1, try_connect(disconnect(S))).
|
||||
|
||||
make_req_fun({append_chunk, NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum, Opts, Timeout},
|
||||
make_req_fun({append_chunk, EpochID, Prefix, Chunk},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:append_chunk(Sock, NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum, Opts, Timeout)
|
||||
end;
|
||||
make_req_fun({read_chunk, NSInfo, EpochID, File, Offset, Size, Opts},
|
||||
fun() -> Mod:append_chunk(Sock, EpochID, Prefix, Chunk) end;
|
||||
make_req_fun({append_chunk_extra, EpochID, Prefix, Chunk, ChunkExtra},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:read_chunk(Sock, NSInfo, EpochID, File, Offset, Size, Opts) end;
|
||||
make_req_fun({write_chunk, NSInfo, EpochID, File, Offset, Chunk, CSum},
|
||||
fun() -> Mod:append_chunk_extra(Sock, EpochID, Prefix, Chunk, ChunkExtra) end;
|
||||
make_req_fun({read_chunk, EpochID, File, Offset, Size, Opts},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:write_chunk(Sock, NSInfo, EpochID, File, Offset, Chunk, CSum) end;
|
||||
make_req_fun({trim_chunk, NSInfo, EpochID, File, Offset, Size},
|
||||
fun() -> Mod:read_chunk(Sock, EpochID, File, Offset, Size, Opts) end;
|
||||
make_req_fun({write_chunk, EpochID, File, Offset, Chunk},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:trim_chunk(Sock, NSInfo, EpochID, File, Offset, Size) end;
|
||||
make_req_fun({checksum_list, File},
|
||||
fun() -> Mod:write_chunk(Sock, EpochID, File, Offset, Chunk) end;
|
||||
make_req_fun({trim_chunk, EpochID, File, Offset, Size},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:checksum_list(Sock, File) end;
|
||||
fun() -> Mod:trim_chunk(Sock, EpochID, File, Offset, Size) end;
|
||||
make_req_fun({checksum_list, EpochID, File},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:checksum_list(Sock, EpochID, File) end;
|
||||
make_req_fun({list_files, EpochID},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:list_files(Sock, EpochID) end;
|
||||
|
|
|
|||
|
|
@ -47,6 +47,8 @@ start_link() ->
|
|||
supervisor:start_link({local, ?SERVER}, ?MODULE, []).
|
||||
|
||||
init([]) ->
|
||||
%% {_, Ps} = process_info(self(), links),
|
||||
%% [unlink(P) || P <- Ps],
|
||||
RestartStrategy = one_for_one,
|
||||
MaxRestarts = 1000,
|
||||
MaxSecondsBetweenRestarts = 3600,
|
||||
|
|
@ -60,16 +62,9 @@ init([]) ->
|
|||
ServerSup =
|
||||
{machi_flu_sup, {machi_flu_sup, start_link, []},
|
||||
Restart, Shutdown, Type, []},
|
||||
RanchSup = {ranch_sup, {ranch_sup, start_link, []},
|
||||
Restart, Shutdown, supervisor, [ranch_sup]},
|
||||
LifecycleMgr =
|
||||
{machi_lifecycle_mgr, {machi_lifecycle_mgr, start_link, []},
|
||||
Restart, Shutdown, worker, []},
|
||||
RunningApps = [A || {A,_D,_V} <- application:which_applications()],
|
||||
Specs = case lists:member(ranch, RunningApps) of
|
||||
true ->
|
||||
[ServerSup, LifecycleMgr];
|
||||
false ->
|
||||
[ServerSup, RanchSup, LifecycleMgr]
|
||||
end,
|
||||
{ok, {SupFlags, Specs}}.
|
||||
|
||||
{ok, {SupFlags, [ServerSup]}}.
|
||||
|
||||
%% AChild = {'AName', {'AModule', start_link, []},
|
||||
%% Restart, Shutdown, Type, ['AModule']},
|
||||
%% {ok, {SupFlags, [AChild]}}.
|
||||
|
|
|
|||
|
|
@ -25,19 +25,18 @@
|
|||
-export([
|
||||
checksum_chunk/1,
|
||||
make_tagged_csum/1, make_tagged_csum/2,
|
||||
make_client_csum/1,
|
||||
unmake_tagged_csum/1,
|
||||
hexstr_to_bin/1, bin_to_hexstr/1,
|
||||
hexstr_to_int/1, int_to_hexstr/2, int_to_hexbin/2,
|
||||
make_binary/1, make_string/1,
|
||||
make_regname/1,
|
||||
make_config_filename/4, make_config_filename/2,
|
||||
make_config_filename/2,
|
||||
make_checksum_filename/4, make_checksum_filename/2,
|
||||
make_data_filename/6, make_data_filename/2,
|
||||
make_data_filename/5, make_data_filename/4, make_data_filename/2,
|
||||
make_projection_filename/2,
|
||||
is_valid_filename/1,
|
||||
parse_filename/1,
|
||||
read_max_filenum/4, increment_max_filenum/4,
|
||||
read_max_filenum/2, increment_max_filenum/2,
|
||||
info_msg/2, verb/1, verb/2,
|
||||
mbytes/1,
|
||||
pretty_time/0, pretty_time/2,
|
||||
|
|
@ -50,9 +49,7 @@
|
|||
%% Other
|
||||
wait_for_death/2, wait_for_life/2,
|
||||
bool2int/1,
|
||||
int2bool/1,
|
||||
read_opts_default/1,
|
||||
ns_info_default/1
|
||||
int2bool/1
|
||||
]).
|
||||
|
||||
-include("machi.hrl").
|
||||
|
|
@ -71,20 +68,10 @@ make_regname(Prefix) when is_list(Prefix) ->
|
|||
|
||||
%% @doc Calculate a config file path, by common convention.
|
||||
|
||||
-spec make_config_filename(string(), machi_dt:namespace(), machi_dt:locator(), string()) ->
|
||||
string().
|
||||
make_config_filename(DataDir, NS, NSLocator, Prefix) ->
|
||||
NSLocator_str = int_to_hexstr(NSLocator, 32),
|
||||
lists:flatten(io_lib:format("~s/config/~s^~s^~s",
|
||||
[DataDir, Prefix, NS, NSLocator_str])).
|
||||
|
||||
%% @doc Calculate a config file path, by common convention.
|
||||
|
||||
-spec make_config_filename(string(), string()) ->
|
||||
string().
|
||||
make_config_filename(DataDir, Filename) ->
|
||||
lists:flatten(io_lib:format("~s/config/~s",
|
||||
[DataDir, Filename])).
|
||||
make_config_filename(DataDir, Prefix) ->
|
||||
lists:flatten(io_lib:format("~s/config/~s", [DataDir, Prefix])).
|
||||
|
||||
%% @doc Calculate a checksum file path, by common convention.
|
||||
|
||||
|
|
@ -105,19 +92,24 @@ make_checksum_filename(DataDir, FileName) ->
|
|||
|
||||
%% @doc Calculate a file data file path, by common convention.
|
||||
|
||||
-spec make_data_filename(string(), machi_dt:namespace(), machi_dt:locator(), string(), atom()|string()|binary(), integer()|string()) ->
|
||||
-spec make_data_filename(string(), string(), atom()|string()|binary(), atom(), integer()|string()) ->
|
||||
{binary(), string()}.
|
||||
make_data_filename(DataDir, NS, NSLocator, Prefix, SequencerName, FileNum)
|
||||
make_data_filename(DataDir, Prefix, SequencerName, FluName, FileNum)
|
||||
when is_atom(FluName) andalso is_integer(FileNum) ->
|
||||
make_data_filename(DataDir, Prefix, SequencerName,
|
||||
atom_to_list(FluName) ++ "-" ++ integer_to_list(FileNum)).
|
||||
|
||||
-spec make_data_filename(string(), string(), atom()|string()|binary(), integer()|string()) ->
|
||||
{binary(), string()}.
|
||||
make_data_filename(DataDir, Prefix, SequencerName, FileNum)
|
||||
when is_integer(FileNum) ->
|
||||
NSLocator_str = int_to_hexstr(NSLocator, 32),
|
||||
File = erlang:iolist_to_binary(io_lib:format("~s^~s^~s^~s^~w",
|
||||
[Prefix, NS, NSLocator_str, SequencerName, FileNum])),
|
||||
File = erlang:iolist_to_binary(io_lib:format("~s^~s^~w",
|
||||
[Prefix, SequencerName, FileNum])),
|
||||
make_data_filename2(DataDir, File);
|
||||
make_data_filename(DataDir, NS, NSLocator, Prefix, SequencerName, String)
|
||||
make_data_filename(DataDir, Prefix, SequencerName, String)
|
||||
when is_list(String) ->
|
||||
NSLocator_str = int_to_hexstr(NSLocator, 32),
|
||||
File = erlang:iolist_to_binary(io_lib:format("~s^~s^~s^~s^~s",
|
||||
[Prefix, NS, NSLocator_str, SequencerName, string])),
|
||||
File = erlang:iolist_to_binary(io_lib:format("~s^~s^~s",
|
||||
[Prefix, SequencerName, String])),
|
||||
make_data_filename2(DataDir, File).
|
||||
|
||||
make_data_filename2(DataDir, File) ->
|
||||
|
|
@ -149,44 +141,34 @@ make_projection_filename(DataDir, File) ->
|
|||
-spec is_valid_filename( Filename :: string() ) -> true | false.
|
||||
is_valid_filename(Filename) ->
|
||||
case parse_filename(Filename) of
|
||||
{} -> false;
|
||||
{_,_,_,_,_} -> true
|
||||
[] -> false;
|
||||
_ -> true
|
||||
end.
|
||||
|
||||
%% @doc Given a machi filename, return a set of components in a list.
|
||||
%% The components will be:
|
||||
%% <ul>
|
||||
%% <li>Prefix</li>
|
||||
%% <li>Cluster namespace</li>
|
||||
%% <li>Cluster locator</li>
|
||||
%% <li>UUID</li>
|
||||
%% <li>Sequence number</li>
|
||||
%% </ul>
|
||||
%%
|
||||
%% Invalid filenames will return an empty list.
|
||||
-spec parse_filename( Filename :: string() ) -> {} | {string(), machi_dt:namespace(), machi_dt:locator(), string(), string() }.
|
||||
-spec parse_filename( Filename :: string() ) -> [ string() ].
|
||||
parse_filename(Filename) ->
|
||||
case string:tokens(Filename, "^") of
|
||||
[Prefix, NS, NSLocator, UUID, SeqNo] ->
|
||||
{Prefix, NS, list_to_integer(NSLocator), UUID, SeqNo};
|
||||
[Prefix, NSLocator, UUID, SeqNo] ->
|
||||
%% string:tokens() doesn't consider "foo^^bar" as 3 tokens {sigh}
|
||||
case re:replace(Filename, "[^^]+", "x", [global,{return,binary}]) of
|
||||
<<"x^^x^x^x">> ->
|
||||
{Prefix, <<"">>, list_to_integer(NSLocator), UUID, SeqNo};
|
||||
_ ->
|
||||
{}
|
||||
end;
|
||||
_ -> {}
|
||||
[_Prefix, _UUID, _SeqNo] = L -> L;
|
||||
_ -> []
|
||||
end.
|
||||
|
||||
|
||||
%% @doc Read the file size of a config file, which is used as the
|
||||
%% basis for a minimum sequence number.
|
||||
|
||||
-spec read_max_filenum(string(), machi_dt:namespace(), machi_dt:locator(), string()) ->
|
||||
-spec read_max_filenum(string(), string()) ->
|
||||
non_neg_integer().
|
||||
read_max_filenum(DataDir, NS, NSLocator, Prefix) ->
|
||||
case file:read_file_info(make_config_filename(DataDir, NS, NSLocator, Prefix)) of
|
||||
read_max_filenum(DataDir, Prefix) ->
|
||||
case file:read_file_info(make_config_filename(DataDir, Prefix)) of
|
||||
{error, enoent} ->
|
||||
0;
|
||||
{ok, FI} ->
|
||||
|
|
@ -196,11 +178,11 @@ read_max_filenum(DataDir, NS, NSLocator, Prefix) ->
|
|||
%% @doc Increase the file size of a config file, which is used as the
|
||||
%% basis for a minimum sequence number.
|
||||
|
||||
-spec increment_max_filenum(string(), machi_dt:namespace(), machi_dt:locator(), string()) ->
|
||||
-spec increment_max_filenum(string(), string()) ->
|
||||
ok | {error, term()}.
|
||||
increment_max_filenum(DataDir, NS, NSLocator, Prefix) ->
|
||||
increment_max_filenum(DataDir, Prefix) ->
|
||||
try
|
||||
{ok, FH} = file:open(make_config_filename(DataDir, NS, NSLocator, Prefix), [append]),
|
||||
{ok, FH} = file:open(make_config_filename(DataDir, Prefix), [append]),
|
||||
ok = file:write(FH, "x"),
|
||||
ok = file:sync(FH),
|
||||
ok = file:close(FH)
|
||||
|
|
@ -289,25 +271,12 @@ int_to_hexbin(I, I_size) ->
|
|||
checksum_chunk(Chunk) when is_binary(Chunk); is_list(Chunk) ->
|
||||
crypto:hash(sha, Chunk).
|
||||
|
||||
convert_csum_tag(A) when is_atom(A)->
|
||||
A;
|
||||
convert_csum_tag(?CSUM_TAG_NONE) ->
|
||||
?CSUM_TAG_NONE_ATOM;
|
||||
convert_csum_tag(?CSUM_TAG_CLIENT_SHA) ->
|
||||
?CSUM_TAG_CLIENT_SHA_ATOM;
|
||||
convert_csum_tag(?CSUM_TAG_SERVER_SHA) ->
|
||||
?CSUM_TAG_SERVER_SHA_ATOM;
|
||||
convert_csum_tag(?CSUM_TAG_SERVER_REGEN_SHA) ->
|
||||
?CSUM_TAG_SERVER_REGEN_SHA_ATOM.
|
||||
|
||||
%% @doc Create a tagged checksum
|
||||
|
||||
make_tagged_csum(none) ->
|
||||
<<?CSUM_TAG_NONE:8>>;
|
||||
make_tagged_csum(<<>>) ->
|
||||
<<?CSUM_TAG_NONE:8>>;
|
||||
make_tagged_csum({Tag, CSum}) ->
|
||||
make_tagged_csum(convert_csum_tag(Tag), CSum).
|
||||
make_tagged_csum(Tag, CSum).
|
||||
|
||||
%% @doc Makes tagged csum. Each meanings are:
|
||||
%% none / ?CSUM_TAG_NONE
|
||||
|
|
@ -328,9 +297,6 @@ make_tagged_csum(?CSUM_TAG_SERVER_SHA_ATOM, SHA) ->
|
|||
make_tagged_csum(?CSUM_TAG_SERVER_REGEN_SHA_ATOM, SHA) ->
|
||||
<<?CSUM_TAG_SERVER_REGEN_SHA:8, SHA/binary>>.
|
||||
|
||||
make_client_csum(BinOrList) ->
|
||||
make_tagged_csum(?CSUM_TAG_CLIENT_SHA_ATOM, checksum_chunk(BinOrList)).
|
||||
|
||||
unmake_tagged_csum(<<Tag:8, Rest/binary>>) ->
|
||||
{Tag, Rest}.
|
||||
|
||||
|
|
@ -378,7 +344,7 @@ wait_for_death(Pid, Iters) when is_pid(Pid) ->
|
|||
false ->
|
||||
ok;
|
||||
true ->
|
||||
timer:sleep(10),
|
||||
timer:sleep(1),
|
||||
wait_for_death(Pid, Iters-1)
|
||||
end.
|
||||
|
||||
|
|
@ -449,17 +415,3 @@ bool2int(true) -> 1;
|
|||
bool2int(false) -> 0.
|
||||
int2bool(0) -> false;
|
||||
int2bool(I) when is_integer(I) -> true.
|
||||
|
||||
read_opts_default(#read_opts{}=NSInfo) ->
|
||||
NSInfo;
|
||||
read_opts_default(A) when A == 'undefined'; A == 'noopt'; A == 'none' ->
|
||||
#read_opts{};
|
||||
read_opts_default(A) when is_atom(A) ->
|
||||
#read_opts{}.
|
||||
|
||||
ns_info_default(#ns_info{}=NSInfo) ->
|
||||
NSInfo;
|
||||
ns_info_default(A) when is_atom(A) ->
|
||||
#ns_info{}.
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -22,8 +22,6 @@
|
|||
|
||||
-module(machi_yessir_client).
|
||||
|
||||
-ifdef(TODO_refactoring_deferred).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
|
|
@ -32,7 +30,7 @@
|
|||
append_chunk/4, append_chunk/5,
|
||||
append_chunk_extra/5, append_chunk_extra/6,
|
||||
read_chunk/5, read_chunk/6,
|
||||
checksum_list/2, checksum_list/3,
|
||||
checksum_list/3, checksum_list/4,
|
||||
list_files/2, list_files/3,
|
||||
wedge_status/1, wedge_status/2,
|
||||
|
||||
|
|
@ -175,24 +173,24 @@ read_chunk(_Host, _TcpPort, EpochID, File, Offset, Size)
|
|||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
|
||||
checksum_list(#yessir{name=Name,chunk_size=ChunkSize}, File) ->
|
||||
checksum_list(#yessir{name=Name,chunk_size=ChunkSize}, _EpochID, File) ->
|
||||
case get({Name,offset,File}) of
|
||||
undefined ->
|
||||
{error, no_such_file};
|
||||
MaxOffset ->
|
||||
C = machi_util:make_tagged_csum(client_sha,
|
||||
make_csum(Name, ChunkSize)),
|
||||
Cs = [{Offset, ChunkSize, C} ||
|
||||
Offset <- lists:seq(?MINIMUM_OFFSET, MaxOffset, ChunkSize)],
|
||||
{ok, term_to_binary(Cs)}
|
||||
Cs = [machi_csum_table:encode_csum_file_entry_bin(Offset, ChunkSize, C) ||
|
||||
Offset <- lists:seq(?MINIMUM_OFFSET, MaxOffset, ChunkSize)],
|
||||
{ok, Cs}
|
||||
end.
|
||||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
|
||||
checksum_list(_Host, _TcpPort, File) ->
|
||||
checksum_list(_Host, _TcpPort, EpochID, File) ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE}),
|
||||
try
|
||||
checksum_list(Sock, File)
|
||||
checksum_list(Sock, EpochID, File)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
|
|
@ -511,5 +509,3 @@ disconnect(#yessir{name=Name}) ->
|
|||
%% =INFO REPORT==== 17-May-2015::18:57:52 ===
|
||||
%% Repair success: tail a of [a] finished ap_mode repair ID {a,{1431,856671,140404}}: ok
|
||||
%% Stats [{t_in_files,0},{t_in_chunks,10413},{t_in_bytes,682426368},{t_out_files,0},{t_out_chunks,10413},{t_out_bytes,682426368},{t_bad_chunks,0},{t_elapsed_seconds,1.591}]
|
||||
|
||||
-endif. % TODO_refactoring_deferred
|
||||
|
|
|
|||
|
|
@ -33,59 +33,49 @@
|
|||
-define(FLU_C, machi_flu1_client).
|
||||
|
||||
verify_file_checksums_test_() ->
|
||||
{setup,
|
||||
fun() -> os:cmd("rm -rf ./data") end,
|
||||
fun(_) -> os:cmd("rm -rf ./data") end,
|
||||
{timeout, 60, fun() -> verify_file_checksums_test2() end}
|
||||
}.
|
||||
{timeout, 60, fun() -> verify_file_checksums_test2() end}.
|
||||
|
||||
verify_file_checksums_test2() ->
|
||||
Host = "localhost",
|
||||
TcpPort = 32958,
|
||||
DataDir = "./data",
|
||||
W_props = [{initial_wedged, false}],
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
try
|
||||
machi_test_util:start_flu_package(verify1_flu, TcpPort, DataDir,
|
||||
machi_flu1_test:start_flu_package(verify1_flu, TcpPort, DataDir,
|
||||
W_props),
|
||||
Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}),
|
||||
try
|
||||
Prefix = <<"verify_prefix">>,
|
||||
NumChunks = 10,
|
||||
[{ok, _} = ?FLU_C:append_chunk(Sock1, NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
Prefix, <<X:(X*8)/big>>, NoCSum) ||
|
||||
X <- lists:seq(1, NumChunks)],
|
||||
{ok, [{_FileSize,File}]} = ?FLU_C:list_files(Sock1, ?DUMMY_PV1_EPOCH),
|
||||
?assertEqual({ok, []},
|
||||
machi_admin_util:verify_file_checksums_remote(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, File)),
|
||||
Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}),
|
||||
try
|
||||
Prefix = <<"verify_prefix">>,
|
||||
NumChunks = 10,
|
||||
[{ok, _} = ?FLU_C:append_chunk(Sock1, ?DUMMY_PV1_EPOCH,
|
||||
Prefix, <<X:(X*8)/big>>) ||
|
||||
X <- lists:seq(1, NumChunks)],
|
||||
{ok, [{_FileSize,File}]} = ?FLU_C:list_files(Sock1, ?DUMMY_PV1_EPOCH),
|
||||
{ok, []} = machi_admin_util:verify_file_checksums_remote(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, File),
|
||||
|
||||
%% Clobber the first 3 chunks, which are sizes 1/2/3.
|
||||
{_, Path} = machi_util:make_data_filename(DataDir,binary_to_list(File)),
|
||||
{ok, FH} = file:open(Path, [read,write]),
|
||||
{ok, _} = file:position(FH, ?MINIMUM_OFFSET),
|
||||
ok = file:write(FH, "y"),
|
||||
ok = file:write(FH, "yo"),
|
||||
ok = file:write(FH, "yo!"),
|
||||
ok = file:close(FH),
|
||||
%% Clobber the first 3 chunks, which are sizes 1/2/3.
|
||||
{_, Path} = machi_util:make_data_filename(DataDir,binary_to_list(File)),
|
||||
{ok, FH} = file:open(Path, [read,write]),
|
||||
{ok, _} = file:position(FH, ?MINIMUM_OFFSET),
|
||||
ok = file:write(FH, "y"),
|
||||
ok = file:write(FH, "yo"),
|
||||
ok = file:write(FH, "yo!"),
|
||||
ok = file:close(FH),
|
||||
|
||||
%% Check the local flavor of the API: should be 3 bad checksums
|
||||
{ok, Res1} = machi_admin_util:verify_file_checksums_local(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, Path),
|
||||
3 = length(Res1),
|
||||
%% Check the local flavor of the API: should be 3 bad checksums
|
||||
{ok, Res1} = machi_admin_util:verify_file_checksums_local(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, Path),
|
||||
3 = length(Res1),
|
||||
|
||||
%% Check the remote flavor of the API: should be 3 bad checksums
|
||||
{ok, Res2} = machi_admin_util:verify_file_checksums_remote(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, File),
|
||||
3 = length(Res2),
|
||||
%% Check the remote flavor of the API: should be 3 bad checksums
|
||||
{ok, Res2} = machi_admin_util:verify_file_checksums_remote(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, File),
|
||||
3 = length(Res2),
|
||||
|
||||
ok
|
||||
after
|
||||
catch ?FLU_C:quit(Sock1)
|
||||
end
|
||||
ok
|
||||
after
|
||||
catch machi_test_util:stop_flu_package()
|
||||
catch ?FLU_C:quit(Sock1),
|
||||
catch machi_flu1_test:stop_flu_package(verify1_flu)
|
||||
end.
|
||||
|
||||
-endif. % !PULSE
|
||||
|
|
|
|||
|
|
@ -47,8 +47,6 @@
|
|||
%% - Operations other than append, write, trim
|
||||
%% - Use checksum instead of binary to save memory
|
||||
%% - More variety for partitioning pattern: non-constant failure
|
||||
%% - Stop and restart
|
||||
%% - Suspend and resume of some erlang processes
|
||||
|
||||
-module(machi_ap_repair_eqc).
|
||||
|
||||
|
|
@ -62,16 +60,6 @@
|
|||
-include_lib("eqc/include/eqc_statem.hrl").
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
-record(target, {verbose=false,
|
||||
flu_names,
|
||||
mgr_names}).
|
||||
|
||||
-record(state, {num,
|
||||
verbose=false,
|
||||
flu_names,
|
||||
mgr_names,
|
||||
cr_count}).
|
||||
|
||||
%% ETS table names
|
||||
-define(WRITTEN_TAB, written). % Successfully written data
|
||||
-define(ACCPT_TAB, accpt). % Errors with no harm, e.g. timeout
|
||||
|
|
@ -103,25 +91,32 @@ prop_repair_par_test_() ->
|
|||
eqc:quickcheck(eqc:testing_time(
|
||||
PropTO, ?QC_OUT(noshrink(prop_repair_par(Verbose))))))}]}.
|
||||
|
||||
%% Model
|
||||
%% SHELL HELPERS
|
||||
test() -> test(100).
|
||||
test(N) -> test(N, true).
|
||||
test(N, Verbose) -> quickcheck(numtests(N, noshrink(prop_repair_par(Verbose)))).
|
||||
check() -> check(prop_repair_par(true), current_counterexample()).
|
||||
|
||||
-record(state, {num, % Number of FLU servers
|
||||
seed, % Seed for partition simulator
|
||||
verbose=false, % Verbose output for debugging
|
||||
flu_names, % List of FLU names
|
||||
mgr_names, % List of chain manager names
|
||||
fc_list, % List of FLU1 proxy clients
|
||||
cr_list}). % List of CR clients
|
||||
|
||||
weight(_S, change_partition) -> 20;
|
||||
weight(_S, _) -> 100.
|
||||
|
||||
%% Append
|
||||
%% append
|
||||
|
||||
append_args(#state{cr_count=CRCount}=S) ->
|
||||
[choose(1, CRCount), chunk(), S].
|
||||
append_args(#state{cr_list=CRList}=S) ->
|
||||
[elements(CRList), chunk(), S].
|
||||
|
||||
append(CRIndex, Bin, #state{verbose=V}=S) ->
|
||||
CRList = cr_list(),
|
||||
{_SimSelfName, C} = lists:nth(CRIndex, CRList),
|
||||
append({_SimSelfName, C}, Bin, #state{verbose=V}=S) ->
|
||||
Prefix = <<"pre">>,
|
||||
Len = byte_size(Bin),
|
||||
NSInfo = #ns_info{},
|
||||
NoCSum = <<>>,
|
||||
Opts1 = #append_opts{},
|
||||
Res = (catch machi_cr_client:append_chunk(C, NSInfo, Prefix, Bin, NoCSum, Opts1, sec(1))),
|
||||
Res = (catch machi_cr_client:append_chunk(C, Prefix, Bin, {sec(1), sec(1)})),
|
||||
case Res of
|
||||
{ok, {_Off, Len, _FileName}=Key} ->
|
||||
case ets:insert_new(?WRITTEN_TAB, {Key, Bin}) of
|
||||
|
|
@ -176,7 +171,7 @@ append(CRIndex, Bin, #state{verbose=V}=S) ->
|
|||
{other_error, Other}
|
||||
end.
|
||||
|
||||
%% Change partition
|
||||
%% change partition
|
||||
|
||||
change_partition_args(#state{flu_names=FLUNames}=S) ->
|
||||
%% [partition(FLUNames), S].
|
||||
|
|
@ -193,34 +188,13 @@ change_partition(Partition,
|
|||
%% Don't wait for stable chain, tick will be executed on demand
|
||||
%% in append oprations
|
||||
_ = tick(S),
|
||||
|
||||
ok.
|
||||
|
||||
%% Generators
|
||||
|
||||
num() ->
|
||||
choose(2, 5).
|
||||
|
||||
cr_count(Num) ->
|
||||
Num * 3.
|
||||
|
||||
%% Returns a list like
|
||||
%% `[{#p_srvr{name=a, port=7501, ..}, "./eqc/data.eqc.a/"}, ...]'
|
||||
all_list_extra(Num) ->
|
||||
{PortBase, DirBase} = get_port_dir_base(),
|
||||
[begin
|
||||
FLUNameStr = [$a + I - 1],
|
||||
FLUName = list_to_atom(FLUNameStr),
|
||||
MgrName = machi_flu_psup:make_mgr_supname(FLUName),
|
||||
{#p_srvr{name=FLUName, address="localhost", port=PortBase+I,
|
||||
props=[{chmgr, MgrName}]},
|
||||
DirBase ++ "/data.eqc." ++ FLUNameStr}
|
||||
end || I <- lists:seq(1, Num)].
|
||||
|
||||
sublist(L) ->
|
||||
?LET(K, nat(),
|
||||
?LET(L2, eqc_gen:vector(K, eqc_gen:oneof(L)),
|
||||
lists:usort(L2))).
|
||||
%% return(3).
|
||||
|
||||
%% Generator for possibly assymmetric partition information
|
||||
partition(FLUNames) ->
|
||||
|
|
@ -242,7 +216,7 @@ flu_pairs(FLUNames) ->
|
|||
chunk() ->
|
||||
non_empty(binary(10)).
|
||||
|
||||
%% Properties
|
||||
%% Property
|
||||
|
||||
prop_repair(Verbose) ->
|
||||
error_logger:tty(false),
|
||||
|
|
@ -251,17 +225,17 @@ prop_repair(Verbose) ->
|
|||
|
||||
Seed = {1445,935441,287549},
|
||||
?FORALL(Num, num(),
|
||||
?FORALL(Cmds, commands(?MODULE, initial_state(Num, Verbose)),
|
||||
?FORALL(Cmds, commands(?MODULE, initial_state(Num, Seed, Verbose)),
|
||||
begin
|
||||
Target = setup_target(Num, Seed, Verbose),
|
||||
{H, S1, Res0} = run_commands(?MODULE, Cmds),
|
||||
SetupState = setup_chain(Num, Seed, Verbose),
|
||||
{H, S1, Res} = run_commands(?MODULE, Cmds),
|
||||
%% ?V("S1=~w~n", [S1]),
|
||||
?V("==== Start post operations, stabilize and confirm results~n", []),
|
||||
_ = stabilize(commands_len(Cmds), Target),
|
||||
{Dataloss, Critical} = confirm_result(Target),
|
||||
_ = cleanup(Target),
|
||||
{_Res2, S2} = stabilize(commands_len(Cmds), SetupState),
|
||||
{Dataloss, Critical} = confirm_result(S2),
|
||||
_ = cleanup(SetupState),
|
||||
pretty_commands(
|
||||
?MODULE, Cmds, {H, S1, Res0},
|
||||
?MODULE, Cmds, {H, S1, Res},
|
||||
aggregate(with_title(cmds), command_names(Cmds),
|
||||
collect(with_title(length5), (length(Cmds) div 5) * 5,
|
||||
{Dataloss, Critical} =:= {0, 0})))
|
||||
|
|
@ -277,35 +251,35 @@ prop_repair_par(Verbose) ->
|
|||
?FORALL(Cmds,
|
||||
%% Now try-and-err'ing, how to control command length and concurrency?
|
||||
?SUCHTHAT(Cmds0, ?SIZED(Size, resize(Size,
|
||||
parallel_commands(?MODULE, initial_state(Num, Verbose)))),
|
||||
parallel_commands(?MODULE, initial_state(Num, Seed, Verbose)))),
|
||||
commands_len(Cmds0) > 20
|
||||
andalso
|
||||
concurrency(Cmds0) > 2),
|
||||
begin
|
||||
CmdsLen= commands_len(Cmds),
|
||||
Target = setup_target(Num, Seed, Verbose),
|
||||
{Seq, Par, Res0} = run_parallel_commands(?MODULE, Cmds),
|
||||
SetupState = setup_chain(Num, Seed, Verbose),
|
||||
{Seq, Par, Res} = run_parallel_commands(?MODULE, Cmds),
|
||||
%% ?V("Seq=~w~n", [Seq]),
|
||||
%% ?V("Par=~w~n", [Par]),
|
||||
?V("==== Start post operations, stabilize and confirm results~n", []),
|
||||
{FinalRes, {Dataloss, Critical}} =
|
||||
case Res0 of
|
||||
{Dataloss, Critical} =
|
||||
case Res of
|
||||
ok ->
|
||||
Res1 = stabilize(CmdsLen, Target),
|
||||
{Res1, confirm_result(Target)};
|
||||
{_Res2, S2} = stabilize(CmdsLen, SetupState),
|
||||
confirm_result(S2);
|
||||
_ ->
|
||||
?V("Res0=~w~n", [Res0]),
|
||||
{Res0, {undefined, undefined}}
|
||||
?V("Res=~w~n", [Res]),
|
||||
{undefined, undefined}
|
||||
end,
|
||||
_ = cleanup(Target),
|
||||
_ = cleanup(SetupState),
|
||||
%% Process is leaking? This log line can be removed after fix.
|
||||
[?V("process_count=~w~n", [erlang:system_info(process_count)]) || Verbose],
|
||||
?V("process_count=~w~n", [erlang:system_info(process_count)]),
|
||||
pretty_commands(
|
||||
?MODULE, Cmds, {Seq, Par, Res0},
|
||||
?MODULE, Cmds, {Seq, Par, Res},
|
||||
aggregate(with_title(cmds), command_names(Cmds),
|
||||
collect(with_title(length5), (CmdsLen div 5) * 5,
|
||||
collect(with_title(conc), concurrency(Cmds),
|
||||
{FinalRes, {Dataloss, Critical}} =:= {ok, {0, 0}})))
|
||||
{Dataloss, Critical} =:= {0, 0})))
|
||||
)
|
||||
end)).
|
||||
|
||||
|
|
@ -315,38 +289,44 @@ prop_repair_par(Verbose) ->
|
|||
%% > eqc_gen:sample(eqc_statem:commands(machi_ap_repair_eqc)).
|
||||
%% but not so helpful.
|
||||
initial_state() ->
|
||||
#state{cr_count=3}.
|
||||
#state{cr_list=[a,b,c]}.
|
||||
|
||||
initial_state(Num, Verbose) ->
|
||||
AllListE = all_list_extra(Num),
|
||||
FLUNames = [P#p_srvr.name || {P, _Dir} <- AllListE],
|
||||
MgrNames = [{Name, machi_flu_psup:make_mgr_supname(Name)} || Name <- FLUNames],
|
||||
#state{num=Num, verbose=Verbose,
|
||||
flu_names=FLUNames, mgr_names=MgrNames,
|
||||
cr_count=cr_count(Num)}.
|
||||
|
||||
setup_target(Num, Seed, Verbose) ->
|
||||
%% ?V("setup_target(Num=~w, Seed=~w~nn", [Num, Seed]),
|
||||
initial_state(Num, Seed, Verbose) ->
|
||||
AllListE = all_list_extra(Num),
|
||||
FLUNames = [P#p_srvr.name || {P, _Dir} <- AllListE],
|
||||
MgrNames = [{Name, machi_flu_psup:make_mgr_supname(Name)} || Name <- FLUNames],
|
||||
Dict = orddict:from_list([{P#p_srvr.name, P} || {P, _Dir} <- AllListE]),
|
||||
|
||||
setup_chain(Seed, AllListE, FLUNames, MgrNames, Dict),
|
||||
_ = setup_cpool(AllListE, FLUNames, Dict),
|
||||
FCList = [begin
|
||||
{ok, PCPid} = machi_proxy_flu1_client:start_link(P),
|
||||
{Name, PCPid}
|
||||
end || {_, #p_srvr{name=Name}=P} <- Dict],
|
||||
%% CR clients are pooled, each has "name" which is interpreted "From"
|
||||
%% side of simulated partition.
|
||||
CRListCount = 10, % ad-hoc
|
||||
SimSelfNames = lists:append(lists:duplicate(CRListCount div Num +1, FLUNames)),
|
||||
CRList = [begin
|
||||
{ok, C} = machi_cr_client:start_link(
|
||||
[P || {_, P} <- Dict],
|
||||
[{use_partition_simulator, true},
|
||||
{simulator_self_name, SimSelfName},
|
||||
{simulator_members, FLUNames}]),
|
||||
{SimSelfName, C}
|
||||
end || SimSelfName <- SimSelfNames],
|
||||
#state{num=Num, seed=Seed, verbose=Verbose,
|
||||
flu_names=FLUNames, mgr_names=MgrNames,
|
||||
cr_list=CRList, fc_list=FCList}.
|
||||
|
||||
Target = #target{flu_names=FLUNames, mgr_names=MgrNames,
|
||||
verbose=Verbose},
|
||||
%% Don't wait for complete chain. Even partialy completed, the chain
|
||||
%% should work fine. Right?
|
||||
wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames,
|
||||
20, Verbose),
|
||||
Target.
|
||||
setup_chain(Num, Seed, Verbose) ->
|
||||
%% ?V("setup_chain(Num=~w, Seed=~w~nn", [Num, Seed]),
|
||||
AllListE = all_list_extra(Num),
|
||||
FLUNames = [P#p_srvr.name || {P, _Dir} <- AllListE],
|
||||
MgrNames = [{Name, machi_flu_psup:make_mgr_supname(Name)} || Name <- FLUNames],
|
||||
Dict = orddict:from_list([{P#p_srvr.name, P} || {P, _Dir} <- AllListE]),
|
||||
|
||||
setup_chain(Seed, AllListE, FLUNames, MgrNames, Dict) ->
|
||||
ok = shutdown_hard(),
|
||||
[begin
|
||||
machi_test_util:clean_up_dir(Dir),
|
||||
machi_flu1_test:clean_up_data_dir(Dir),
|
||||
filelib:ensure_dir(Dir ++ "/not-used")
|
||||
end || {_P, Dir} <- AllListE],
|
||||
[catch ets:delete(T) || T <- tabs()],
|
||||
|
|
@ -371,55 +351,29 @@ setup_chain(Seed, AllListE, FLUNames, MgrNames, Dict) ->
|
|||
[{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, FLUOpts) ||
|
||||
{#p_srvr{name=Name, port=Port}, Dir} <- AllListE],
|
||||
[machi_chain_manager1:set_chain_members(MgrName, Dict) || {_, MgrName} <- MgrNames],
|
||||
ok.
|
||||
|
||||
setup_cpool(AllListE, FLUNames, Dict) ->
|
||||
Num = length(AllListE),
|
||||
FCList = [begin
|
||||
{ok, PCPid} = machi_proxy_flu1_client:start_link(P),
|
||||
{Name, PCPid}
|
||||
end || {_, #p_srvr{name=Name}=P} <- Dict],
|
||||
%% CR clients are pooled, each has "name" which is interpreted "From"
|
||||
%% side of simulated partition.
|
||||
SimSelfNames = lists:append(lists:duplicate(cr_count(Num), FLUNames)),
|
||||
CRList = [begin
|
||||
{ok, C} = machi_cr_client:start_link(
|
||||
[P || {_, P} <- Dict],
|
||||
[{use_partition_simulator, true},
|
||||
{simulator_self_name, SimSelfName},
|
||||
{simulator_members, FLUNames}]),
|
||||
{SimSelfName, C}
|
||||
end || SimSelfName <- SimSelfNames],
|
||||
catch ets:delete(cpool),
|
||||
ets:new(cpool, [set, protected, named_table, {read_concurrency, true}]),
|
||||
ets:insert(cpool, {fc_list, FCList}),
|
||||
ets:insert(cpool, {cr_list, CRList}),
|
||||
{CRList, FCList}.
|
||||
State = initial_state(Num, Seed, Verbose),
|
||||
%% Don't wait for complete chain. Even partialy completed, the chain
|
||||
%% should work fine. Right?
|
||||
wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames,
|
||||
State#state.fc_list, 20, Verbose),
|
||||
State.
|
||||
|
||||
fc_list() ->
|
||||
[{fc_list, FCList}] = ets:lookup(cpool, fc_list),
|
||||
FCList.
|
||||
%% Post commands
|
||||
|
||||
cr_list() ->
|
||||
[{cr_list, CRList}] = ets:lookup(cpool, cr_list),
|
||||
CRList.
|
||||
|
||||
%% Post run_commands
|
||||
|
||||
stabilize(0, _T) ->
|
||||
ok;
|
||||
stabilize(_CmdsLen, #target{flu_names=FLUNames, mgr_names=MgrNames,
|
||||
verbose=Verbose}) ->
|
||||
stabilize(0, S) ->
|
||||
{ok, S};
|
||||
stabilize(_CmdsLen, #state{flu_names=FLUNames, mgr_names=MgrNames,
|
||||
fc_list=FCList, verbose=Verbose}=S) ->
|
||||
machi_partition_simulator:no_partitions(),
|
||||
true = wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames,
|
||||
100, Verbose),
|
||||
ok.
|
||||
wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames,
|
||||
FCList, 100, Verbose),
|
||||
{ok, S}.
|
||||
|
||||
chain_state_all_ok(FLUNames) ->
|
||||
[{FLUName, {FLUNames, [], []}} || FLUName <- FLUNames].
|
||||
|
||||
confirm_result(_T) ->
|
||||
[{_, C} | _] = cr_list(),
|
||||
confirm_result(#state{cr_list=[{_, C}|_]}=_S) ->
|
||||
[{written, _Written}, {accpt, Accpt},
|
||||
{failed, Failed}, {critical, Critical}] = tab_counts(),
|
||||
{OK, Dataloss} = confirm_written(C),
|
||||
|
|
@ -431,7 +385,7 @@ confirm_result(_T) ->
|
|||
0 -> ok;
|
||||
_ ->
|
||||
DumpFailed = filename:join(DirBase, "dump-failed-" ++ Suffix),
|
||||
?V("Dump failed ETS tab to: ~s~n", [DumpFailed]),
|
||||
?V("Dump failed ETS tab to: ~w~n", [DumpFailed]),
|
||||
ets:tab2file(?FAILED_TAB, DumpFailed)
|
||||
end,
|
||||
case Critical of
|
||||
|
|
@ -454,14 +408,14 @@ confirm_written(C) ->
|
|||
|
||||
assert_chunk(C, {Off, Len, FileName}=Key, Bin) ->
|
||||
%% TODO: This probably a bug, read_chunk respnds with filename of `string()' type
|
||||
FileNameStr = binary_to_list(FileName),
|
||||
%% TODO : Use CSum instead of binary (after disuccsion about CSum is calmed down?)
|
||||
NSInfo = undefined,
|
||||
case (catch machi_cr_client:read_chunk(C, NSInfo, FileName, Off, Len, undefined, sec(3))) of
|
||||
{ok, {[{FileName, Off, Bin, _}], []}} ->
|
||||
case (catch machi_cr_client:read_chunk(C, FileName, Off, Len, [], sec(3))) of
|
||||
{ok, {[{FileNameStr, Off, Bin, _}], []}} ->
|
||||
ok;
|
||||
{ok, Got} ->
|
||||
?V("read_chunk got different binary for Key=~p~n", [Key]),
|
||||
?V(" Expected: ~p~n", [{[{FileName, Off, Bin, <<"CSum-NYI">>}], []}]),
|
||||
?V(" Expected: ~p~n", [{[{FileNameStr, Off, Bin, <<"CSum-NYI">>}], []}]),
|
||||
?V(" Got: ~p~n", [Got]),
|
||||
{error, different_binary};
|
||||
{error, Reason} ->
|
||||
|
|
@ -472,23 +426,36 @@ assert_chunk(C, {Off, Len, FileName}=Key, Bin) ->
|
|||
{error, Other}
|
||||
end.
|
||||
|
||||
cleanup(_Target) ->
|
||||
[begin unlink(FC), catch exit(FC, kill) end || {_, FC} <- fc_list()],
|
||||
[begin unlink(CR), catch exit(CR, kill) end || {_, CR} <- cr_list()],
|
||||
cleanup(#state{fc_list=FCList, cr_list=CRList}=_S) ->
|
||||
[catch machi_proxy_flu1_client:quit(FC) || FC <- FCList],
|
||||
[catch machi_cr_client:quit(CR) || CR <- CRList],
|
||||
_ = shutdown_hard().
|
||||
|
||||
%% Internal misc utilities
|
||||
%% Internal utilities
|
||||
|
||||
eqc_verbose() ->
|
||||
os:getenv("EQC_VERBOSE") =:= "true".
|
||||
|
||||
eqc_timeout(Default) ->
|
||||
PropTimeout = case os:getenv("EQC_TIME") of
|
||||
PropTimeout = case os:getenv("EQC_TIMEOUT") of
|
||||
false -> Default;
|
||||
V -> list_to_integer(V)
|
||||
end,
|
||||
{PropTimeout, PropTimeout * 300}.
|
||||
|
||||
%% Returns a list like
|
||||
%% `[{#p_srvr{name=a, port=7501, ..}, "./eqc/data.eqc.a/"}, ...]'
|
||||
all_list_extra(Num) ->
|
||||
{PortBase, DirBase} = get_port_dir_base(),
|
||||
[begin
|
||||
FLUNameStr = [$a + I - 1],
|
||||
FLUName = list_to_atom(FLUNameStr),
|
||||
MgrName = machi_flu_psup:make_mgr_supname(FLUName),
|
||||
{#p_srvr{name=FLUName, address="localhost", port=PortBase+I,
|
||||
props=[{chmgr, MgrName}]},
|
||||
DirBase ++ "/data.eqc." ++ FLUNameStr}
|
||||
end || I <- lists:seq(1, Num)].
|
||||
|
||||
get_port_dir_base() ->
|
||||
I = case os:getenv("EQC_BASE_PORT") of
|
||||
false -> 0;
|
||||
|
|
@ -508,16 +475,15 @@ shutdown_hard() ->
|
|||
timer:sleep(100).
|
||||
|
||||
tick(#state{flu_names=FLUNames, mgr_names=MgrNames,
|
||||
verbose=Verbose}) ->
|
||||
tick(FLUNames, MgrNames, Verbose).
|
||||
fc_list=FCList, verbose=Verbose}) ->
|
||||
tick(FLUNames, MgrNames, FCList, Verbose).
|
||||
|
||||
tick(FLUNames, MgrNames, Verbose) ->
|
||||
tick(FLUNames, MgrNames, 2, 100, Verbose).
|
||||
tick(FLUNames, MgrNames, FCList, Verbose) ->
|
||||
tick(FLUNames, MgrNames, 2, 100, FCList, Verbose).
|
||||
|
||||
tick(FLUNames, MgrNames, Iter, SleepMax, Verbose) ->
|
||||
tick(FLUNames, MgrNames, Iter, SleepMax, FCList, Verbose) ->
|
||||
TickFun = tick_fun(FLUNames, MgrNames, self()),
|
||||
TickFun(Iter, 0, SleepMax),
|
||||
FCList = fc_list(),
|
||||
[?V("## Chain state after tick()=~w~n", [chain_state(FCList)]) || Verbose].
|
||||
|
||||
tick_fun(FLUNames, MgrNames, Parent) ->
|
||||
|
|
@ -550,18 +516,15 @@ tick_fun(FLUNames, MgrNames, Parent) ->
|
|||
end || {ThePid, M_name} <- Pids]
|
||||
end.
|
||||
|
||||
wait_until_stable(ExpectedChainState, FLUNames, MgrNames, Verbose) ->
|
||||
wait_until_stable(ExpectedChainState, FLUNames, MgrNames, 20, Verbose).
|
||||
wait_until_stable(ExpectedChainState, FLUNames, MgrNames, FCList, Verbose) ->
|
||||
wait_until_stable(ExpectedChainState, FLUNames, MgrNames, FCList, 20, Verbose).
|
||||
|
||||
wait_until_stable(ExpectedChainState, FLUNames, MgrNames, Retries, Verbose) ->
|
||||
wait_until_stable(ExpectedChainState, FLUNames, MgrNames, FCList, Retries, Verbose) ->
|
||||
TickFun = tick_fun(FLUNames, MgrNames, self()),
|
||||
FCList = fc_list(),
|
||||
wait_until_stable1(ExpectedChainState, TickFun, FCList, Retries, Verbose).
|
||||
|
||||
wait_until_stable1(ExpectedChainState, _TickFun, FCList, 0, _Verbose) ->
|
||||
?V(" [ERROR] _ExpectedChainState ~p\n", [ExpectedChainState]),
|
||||
wait_until_stable1(_ExpectedChainState, _TickFun, FCList, 0, _Verbose) ->
|
||||
?V(" [ERROR] wait_until_stable failed.... : ~p~n", [chain_state(FCList)]),
|
||||
?V(" [ERROR] norm.... : ~p~n", [normalize_chain_state(chain_state(FCList))]),
|
||||
false;
|
||||
wait_until_stable1(ExpectedChainState, TickFun, FCList, Reties, Verbose) ->
|
||||
[TickFun(3, 0, 100) || _ <- lists:seq(1, 3)],
|
||||
|
|
|
|||
|
|
@ -134,7 +134,6 @@ Press control-c to interrupt the test....".
|
|||
%% convergence_demo_testfun(3).
|
||||
|
||||
-define(DEFAULT_MGR_OPTS, [{private_write_verbose, false},
|
||||
{private_write_verbose_confirm, true},
|
||||
{active_mode,false},
|
||||
{use_partition_simulator, true}]).
|
||||
|
||||
|
|
@ -151,8 +150,7 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
|
|||
%% Faster test startup, commented: io:format(user, short_doc(), []),
|
||||
%% Faster test startup, commented: timer:sleep(3000),
|
||||
|
||||
Apps = [sasl, ranch],
|
||||
[application:start(App) || App <- Apps],
|
||||
application:start(sasl),
|
||||
|
||||
MgrOpts = MgrOpts0 ++ ?DEFAULT_MGR_OPTS,
|
||||
TcpPort = proplists:get_value(port_base, MgrOpts, 62877),
|
||||
|
|
@ -189,18 +187,15 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
|
|||
end || #p_srvr{name=Name}=P <- Ps],
|
||||
MembersDict = machi_projection:make_members_dict(Ps),
|
||||
Witnesses = proplists:get_value(witnesses, MgrOpts, []),
|
||||
CMode = case {Witnesses, proplists:get_value(consistency_mode, MgrOpts,
|
||||
ap_mode)} of
|
||||
{[_|_], _} -> cp_mode;
|
||||
{_, cp_mode} -> cp_mode;
|
||||
{_, ap_mode} -> ap_mode
|
||||
end,
|
||||
MgrNamez = [begin
|
||||
MgrName = machi_flu_psup:make_mgr_supname(Name),
|
||||
ok = ?MGR:set_chain_members(MgrName, ch_demo, 0, CMode,
|
||||
MembersDict,Witnesses),
|
||||
ok = ?MGR:set_chain_members(MgrName,MembersDict,Witnesses),
|
||||
{Name, MgrName}
|
||||
end || #p_srvr{name=Name} <- Ps],
|
||||
CpApMode = case Witnesses /= [] of
|
||||
true -> cp_mode;
|
||||
false -> ap_mode
|
||||
end,
|
||||
|
||||
try
|
||||
[{_, Ma}|_] = MgrNamez,
|
||||
|
|
@ -308,9 +303,9 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
|
|||
[{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane_retrospective(Psx, FLU)} ||
|
||||
{FLU, Psx} <- PrivProjs]
|
||||
catch
|
||||
_Err:_What when CMode == cp_mode ->
|
||||
_Err:_What when CpApMode == cp_mode ->
|
||||
io:format(user, "none proj skip detected, TODO? ", []);
|
||||
_Err:_What when CMode == ap_mode ->
|
||||
_Err:_What when CpApMode == ap_mode ->
|
||||
io:format(user, "PrivProjs ~p\n", [PrivProjs]),
|
||||
exit({line, ?LINE, _Err, _What})
|
||||
end,
|
||||
|
|
@ -376,9 +371,9 @@ timer:sleep(1234),
|
|||
{FLU, Psx} <- PrivProjs],
|
||||
io:format(user, "\nAll sanity checks pass, hooray!\n", [])
|
||||
catch
|
||||
_Err:_What when CMode == cp_mode ->
|
||||
_Err:_What when CpApMode == cp_mode ->
|
||||
io:format(user, "none proj skip detected, TODO? ", []);
|
||||
_Err:_What when CMode == ap_mode ->
|
||||
_Err:_What when CpApMode == ap_mode ->
|
||||
io:format(user, "Report ~p\n", [Report]),
|
||||
io:format(user, "PrivProjs ~p\n", [PrivProjs]),
|
||||
exit({line, ?LINE, _Err, _What})
|
||||
|
|
@ -395,8 +390,7 @@ timer:sleep(1234),
|
|||
exit(SupPid, normal),
|
||||
ok = machi_partition_simulator:stop(),
|
||||
[ok = ?FLU_PC:quit(PPid) || {_, PPid} <- Namez],
|
||||
machi_util:wait_for_death(SupPid, 100),
|
||||
[application:start(App) || App <- lists:reverse(Apps)]
|
||||
machi_util:wait_for_death(SupPid, 100)
|
||||
end.
|
||||
|
||||
%% Many of the static partition lists below have been problematic at one
|
||||
|
|
|
|||
|
|
@ -273,43 +273,43 @@ make_prop_ets() ->
|
|||
|
||||
-endif. % EQC
|
||||
|
||||
make_advance_fun(FitList, FLUList, MgrList, Num) ->
|
||||
fun() ->
|
||||
[begin
|
||||
[catch machi_fitness:trigger_early_adjustment(Fit, Tgt) ||
|
||||
Fit <- FitList,
|
||||
Tgt <- FLUList ],
|
||||
[catch ?MGR:trigger_react_to_env(Mgr) || Mgr <- MgrList],
|
||||
ok
|
||||
end || _ <- lists:seq(1, Num)]
|
||||
end.
|
||||
|
||||
smoke0_test() ->
|
||||
{ok, _} = machi_partition_simulator:start_link({1,2,3}, 50, 50),
|
||||
Host = "localhost",
|
||||
TcpPort = 6623,
|
||||
{[Pa], [M0], _Dirs} = machi_test_util:start_flu_packages(
|
||||
1, TcpPort, "./data.", []),
|
||||
{ok, FLUa} = machi_flu1:start_link([{a,TcpPort,"./data.a"}]),
|
||||
Pa = #p_srvr{name=a, address=Host, port=TcpPort},
|
||||
Members_Dict = machi_projection:make_members_dict([Pa]),
|
||||
%% Egadz, more racing on startup, yay. TODO fix.
|
||||
timer:sleep(1),
|
||||
{ok, FLUaP} = ?FLU_PC:start_link(Pa),
|
||||
{ok, M0} = ?MGR:start_link(a, Members_Dict, [{active_mode, false}]),
|
||||
try
|
||||
pong = ?MGR:ping(M0)
|
||||
after
|
||||
ok = ?MGR:stop(M0),
|
||||
ok = machi_flu1:stop(FLUa),
|
||||
ok = ?FLU_PC:quit(FLUaP),
|
||||
machi_test_util:stop_flu_packages()
|
||||
ok = machi_partition_simulator:stop()
|
||||
end.
|
||||
|
||||
smoke1_test_() ->
|
||||
{timeout, 1*60, fun() -> smoke1_test2() end}.
|
||||
|
||||
smoke1_test2() ->
|
||||
machi_partition_simulator:start_link({1,2,3}, 100, 0),
|
||||
TcpPort = 62777,
|
||||
MgrOpts = [{active_mode,false}],
|
||||
try
|
||||
{Ps, MgrNames, _Dirs} = machi_test_util:start_flu_packages(
|
||||
3, TcpPort, "./data.", MgrOpts),
|
||||
MembersDict = machi_projection:make_members_dict(Ps),
|
||||
[machi_chain_manager1:set_chain_members(M, MembersDict) || M <- MgrNames],
|
||||
Ma = hd(MgrNames),
|
||||
FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}, {c,TcpPort+2,"./data.c"}],
|
||||
P_s = [#p_srvr{name=Name, address="localhost", port=Port} ||
|
||||
{Name,Port,_Dir} <- FluInfo],
|
||||
|
||||
{ok, P1} = ?MGR:test_calc_projection(Ma, false),
|
||||
[machi_flu1_test:clean_up_data_dir(Dir) || {_,_,Dir} <- FluInfo],
|
||||
FLUs = [element(2, machi_flu1:start_link([{Name,Port,Dir}])) ||
|
||||
{Name,Port,Dir} <- FluInfo],
|
||||
MembersDict = machi_projection:make_members_dict(P_s),
|
||||
{ok, M0} = ?MGR:start_link(a, MembersDict, [{active_mode,false}]),
|
||||
try
|
||||
{ok, P1} = ?MGR:test_calc_projection(M0, false),
|
||||
% DERP! Check for race with manager's proxy vs. proj listener
|
||||
ok = lists:foldl(
|
||||
fun(_, {_,{true,[{c,ok},{b,ok},{a,ok}]}}) ->
|
||||
|
|
@ -318,35 +318,39 @@ smoke1_test2() ->
|
|||
ok; % Skip remaining!
|
||||
(_, _Else) ->
|
||||
timer:sleep(10),
|
||||
?MGR:test_write_public_projection(Ma, P1)
|
||||
?MGR:test_write_public_projection(M0, P1)
|
||||
end, not_ok, lists:seq(1, 1000)),
|
||||
%% Writing the exact same projection multiple times returns ok:
|
||||
%% no change!
|
||||
{_,{true,[{c,ok},{b,ok},{a,ok}]}} = ?MGR:test_write_public_projection(Ma, P1),
|
||||
{unanimous, P1, Extra1} = ?MGR:test_read_latest_public_projection(Ma, false),
|
||||
{_,{true,[{c,ok},{b,ok},{a,ok}]}} = ?MGR:test_write_public_projection(M0, P1),
|
||||
{unanimous, P1, Extra1} = ?MGR:test_read_latest_public_projection(M0, false),
|
||||
|
||||
ok
|
||||
after
|
||||
machi_test_util:stop_flu_packages()
|
||||
ok = ?MGR:stop(M0),
|
||||
[ok = machi_flu1:stop(X) || X <- FLUs],
|
||||
ok = machi_partition_simulator:stop()
|
||||
end.
|
||||
|
||||
nonunanimous_setup_and_fix_test_() ->
|
||||
os:cmd("rm -f /tmp/moomoo.*"),
|
||||
{timeout, 1*60, fun() -> nonunanimous_setup_and_fix_test2() end}.
|
||||
|
||||
nonunanimous_setup_and_fix_test2() ->
|
||||
nonunanimous_setup_and_fix_test() ->
|
||||
machi_partition_simulator:start_link({1,2,3}, 100, 0),
|
||||
TcpPort = 62877,
|
||||
MgrOpts = [{active_mode,false}],
|
||||
{Ps, [Ma,Mb,Mc], Dirs} = machi_test_util:start_flu_packages(
|
||||
3, TcpPort, "./data.", MgrOpts),
|
||||
MembersDict = machi_projection:make_members_dict(Ps),
|
||||
ChainName = my_little_chain,
|
||||
[machi_chain_manager1:set_chain_members(M, ChainName, 0, ap_mode,
|
||||
MembersDict, []) || M <- [Ma, Mb]],
|
||||
|
||||
[Proxy_a, Proxy_b, Proxy_c] = Proxies =
|
||||
[element(2, ?FLU_PC:start_link(P)) || P <- Ps],
|
||||
|
||||
FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}],
|
||||
P_s = [#p_srvr{name=Name, address="localhost", port=Port} ||
|
||||
{Name,Port,_Dir} <- FluInfo],
|
||||
|
||||
[machi_flu1_test:clean_up_data_dir(Dir) || {_,_,Dir} <- FluInfo],
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
Opts = [{active_mode, false}],
|
||||
%% {ok, Mb} = ?MGR:start_link(b, MembersDict, [{active_mode, false}]++XX),
|
||||
[{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) ||
|
||||
{Name,Port,Dir} <- FluInfo],
|
||||
[Proxy_a, Proxy_b] = Proxies =
|
||||
[element(2,?FLU_PC:start_link(P)) || P <- P_s],
|
||||
MembersDict = machi_projection:make_members_dict(P_s),
|
||||
[Ma,Mb] = [a_chmgr, b_chmgr],
|
||||
ok = machi_chain_manager1:set_chain_members(Ma, MembersDict, []),
|
||||
ok = machi_chain_manager1:set_chain_members(Mb, MembersDict, []),
|
||||
try
|
||||
{ok, P1} = ?MGR:test_calc_projection(Ma, false),
|
||||
|
||||
|
|
@ -383,114 +387,16 @@ nonunanimous_setup_and_fix_test2() ->
|
|||
{ok, P2pb} = ?FLU_PC:read_latest_projection(Proxy_b, private),
|
||||
P2 = P2pb#projection_v1{dbg2=[]},
|
||||
|
||||
Mgrs = [a_chmgr, b_chmgr, c_chmgr],
|
||||
Advance = make_advance_fun([a_fitness,b_fitness,c_fitness],
|
||||
[a,b,c],
|
||||
Mgrs,
|
||||
3),
|
||||
Advance(),
|
||||
{_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Ma),
|
||||
{_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Mb),
|
||||
{_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Mc),
|
||||
%% Pspam = machi_projection:update_checksum(
|
||||
%% P1b#projection_v1{epoch_number=?SPAM_PROJ_EPOCH,
|
||||
%% dbg=[hello_spam]}),
|
||||
%% ok = ?FLU_PC:write_projection(Proxy_b, public, Pspam),
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Remove 'a' from the chain.\n", []),
|
||||
|
||||
MembersDict4 = machi_projection:make_members_dict(tl(Ps)),
|
||||
ok = machi_chain_manager1:set_chain_members(
|
||||
Mb, ChainName, TheEpoch_3, ap_mode, MembersDict4, []),
|
||||
|
||||
Advance(),
|
||||
{ok, {true, _,_,_}} = ?FLU_PC:wedge_status(Proxy_a),
|
||||
{_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Mb),
|
||||
{_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Mc),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Add a to the chain again (a is running).\n", []),
|
||||
|
||||
MembersDict5 = machi_projection:make_members_dict(Ps),
|
||||
ok = machi_chain_manager1:set_chain_members(
|
||||
Mb, ChainName, TheEpoch_4, ap_mode, MembersDict5, []),
|
||||
|
||||
Advance(),
|
||||
{_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Ma),
|
||||
{_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Mb),
|
||||
{_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Mc),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[a]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Stop a while a chain member, advance b&c.\n", []),
|
||||
|
||||
ok = machi_flu_psup:stop_flu_package(a),
|
||||
Advance(),
|
||||
{_, _, TheEpoch_6} = ?MGR:trigger_react_to_env(Mb),
|
||||
{_, _, TheEpoch_6} = ?MGR:trigger_react_to_env(Mc),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Remove 'a' from the chain.\n", []),
|
||||
|
||||
MembersDict7 = machi_projection:make_members_dict(tl(Ps)),
|
||||
ok = machi_chain_manager1:set_chain_members(
|
||||
Mb, ChainName, TheEpoch_6, ap_mode, MembersDict7, []),
|
||||
|
||||
Advance(),
|
||||
{_, _, TheEpoch_7} = ?MGR:trigger_react_to_env(Mb),
|
||||
{_, _, TheEpoch_7} = ?MGR:trigger_react_to_env(Mc),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Start a, advance.\n", []),
|
||||
|
||||
Opts = [{active_mode, false}, {initial_wedged, true}],
|
||||
#p_srvr{name=NameA} = hd(Ps),
|
||||
{ok,_}=machi_flu_psup:start_flu_package(NameA, TcpPort+1, hd(Dirs), Opts),
|
||||
Advance(),
|
||||
{ok, {true, _,_,_}} = ?FLU_PC:wedge_status(Proxy_a),
|
||||
{ok, {false, EpochID_8,_,_}} = ?FLU_PC:wedge_status(Proxy_b),
|
||||
{ok, {false, EpochID_8,_,_}} = ?FLU_PC:wedge_status(Proxy_c),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Stop a, delete a's data, leave it stopped\n", []),
|
||||
|
||||
ok = machi_flu_psup:stop_flu_package(a),
|
||||
Advance(),
|
||||
machi_flu1_test:clean_up_data_dir(hd(Dirs)),
|
||||
{ok, {false, _,_,_}} = ?FLU_PC:wedge_status(Proxy_b),
|
||||
{ok, {false, _,_,_}} = ?FLU_PC:wedge_status(Proxy_c),
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Add a to the chain again (a is stopped).\n", []),
|
||||
|
||||
MembersDict9 = machi_projection:make_members_dict(Ps),
|
||||
{_, _, TheEpoch_9} = ?MGR:trigger_react_to_env(Mb),
|
||||
ok = machi_chain_manager1:set_chain_members(
|
||||
Mb, ChainName, TheEpoch_9, ap_mode, MembersDict9, []),
|
||||
Advance(),
|
||||
{_, _, TheEpoch_9b} = ?MGR:trigger_react_to_env(Mb),
|
||||
true = (TheEpoch_9b > TheEpoch_9),
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Start a, and it joins like it ought to\n", []),
|
||||
|
||||
{ok,_}=machi_flu_psup:start_flu_package(NameA, TcpPort+1, hd(Dirs), Opts),
|
||||
Advance(),
|
||||
{ok, {false, {TheEpoch10,_},_,_}} = ?FLU_PC:wedge_status(Proxy_a),
|
||||
{ok, {false, {TheEpoch10,_},_,_}} = ?FLU_PC:wedge_status(Proxy_b),
|
||||
{ok, {false, {TheEpoch10,_},_,_}} = ?FLU_PC:wedge_status(Proxy_c),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[a]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies],
|
||||
ok
|
||||
after
|
||||
exit(SupPid, normal),
|
||||
[ok = ?FLU_PC:quit(X) || X <- Proxies],
|
||||
machi_test_util:stop_flu_packages()
|
||||
ok = machi_partition_simulator:stop()
|
||||
end.
|
||||
|
||||
unanimous_report_test() ->
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ smoke_test_() ->
|
|||
fun() -> machi_cinfo:private_projection(a) end,
|
||||
fun() -> machi_cinfo:fitness(a) end,
|
||||
fun() -> machi_cinfo:chain_manager(a) end,
|
||||
fun() -> machi_cinfo:flu1(a) end,
|
||||
fun() -> machi_cinfo:dump() end
|
||||
]}.
|
||||
|
||||
|
|
@ -49,7 +48,7 @@ setup() ->
|
|||
{c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}}
|
||||
],
|
||||
[os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps],
|
||||
{ok, SupPid} = machi_sup:start_link(),
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
%% Only run a, don't run b & c so we have 100% failures talking to them
|
||||
[begin
|
||||
#p_srvr{name=Name, port=Port, props=Dir} = P,
|
||||
|
|
|
|||
|
|
@ -58,15 +58,9 @@ setup_smoke_test(Host, PortBase, Os, Witness_list) ->
|
|||
%% 4. Wait until all others are using epoch id from #3.
|
||||
%%
|
||||
%% Damn, this is a pain to make 100% deterministic, bleh.
|
||||
CMode = if Witness_list == [] -> ap_mode;
|
||||
Witness_list /= [] -> cp_mode
|
||||
end,
|
||||
ok = machi_chain_manager1:set_chain_members(a_chmgr, ch0, 0, CMode,
|
||||
D, Witness_list),
|
||||
ok = machi_chain_manager1:set_chain_members(b_chmgr, ch0, 0, CMode,
|
||||
D, Witness_list),
|
||||
ok = machi_chain_manager1:set_chain_members(c_chmgr, ch0, 0, CMode,
|
||||
D, Witness_list),
|
||||
ok = machi_chain_manager1:set_chain_members(a_chmgr, D, Witness_list),
|
||||
ok = machi_chain_manager1:set_chain_members(b_chmgr, D, Witness_list),
|
||||
ok = machi_chain_manager1:set_chain_members(c_chmgr, D, Witness_list),
|
||||
run_ticks([a_chmgr,b_chmgr,c_chmgr]),
|
||||
%% Everyone is settled on the same damn epoch id.
|
||||
{ok, EpochID} = machi_flu1_client:get_latest_epochid(Host, PortBase+0,
|
||||
|
|
@ -102,13 +96,11 @@ run_ticks(MgrList) ->
|
|||
ok.
|
||||
|
||||
smoke_test2() ->
|
||||
{ok, SupPid} = machi_sup:start_link(),
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
error_logger:tty(false),
|
||||
try
|
||||
Prefix = <<"pre">>,
|
||||
Chunk1 = <<"yochunk">>,
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
Host = "localhost",
|
||||
PortBase = 64454,
|
||||
Os = [{ignore_stability_time, true}, {active_mode, false}],
|
||||
|
|
@ -116,95 +108,91 @@ smoke_test2() ->
|
|||
|
||||
%% Whew ... ok, now start some damn tests.
|
||||
{ok, C1} = machi_cr_client:start_link([P || {_,P}<-orddict:to_list(D)]),
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum),
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1),
|
||||
{ok, {Off1,Size1,File1}} =
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum),
|
||||
BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, "foo")},
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1),
|
||||
Chunk1_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, Chunk1},
|
||||
{error, bad_checksum} =
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, BadCSum),
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1_badcs),
|
||||
{ok, {[{_, Off1, Chunk1, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, Size1, undefined),
|
||||
machi_cr_client:read_chunk(C1, File1, Off1, Size1, []),
|
||||
{ok, PPP} = machi_flu1_client:read_latest_projection(Host, PortBase+0,
|
||||
private),
|
||||
%% Verify that the client's CR wrote to all of them.
|
||||
[{ok, {[{_, Off1, Chunk1, _}], []}} =
|
||||
machi_flu1_client:read_chunk(
|
||||
Host, PortBase+X, NSInfo, EpochID, File1, Off1, Size1, undefined) ||
|
||||
Host, PortBase+X, EpochID, File1, Off1, Size1, []) ||
|
||||
X <- [0,1,2] ],
|
||||
|
||||
%% Test read repair: Manually write to head, then verify that
|
||||
%% read-repair fixes all.
|
||||
FooOff1 = Off1 + (1024*1024),
|
||||
[{error, not_written} = machi_flu1_client:read_chunk(
|
||||
Host, PortBase+X, NSInfo, EpochID,
|
||||
File1, FooOff1, Size1, undefined) || X <- [0,1,2] ],
|
||||
ok = machi_flu1_client:write_chunk(Host, PortBase+0, NSInfo, EpochID,
|
||||
File1, FooOff1, Chunk1, NoCSum),
|
||||
{ok, {[{File1, FooOff1, Chunk1, _}=_YY], []}} =
|
||||
machi_flu1_client:read_chunk(Host, PortBase+0, NSInfo, EpochID,
|
||||
File1, FooOff1, Size1, undefined),
|
||||
{ok, {[{File1, FooOff1, Chunk1, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, FooOff1, Size1, undefined),
|
||||
Host, PortBase+X, EpochID,
|
||||
File1, FooOff1, Size1, []) || X <- [0,1,2] ],
|
||||
ok = machi_flu1_client:write_chunk(Host, PortBase+0, EpochID,
|
||||
File1, FooOff1, Chunk1),
|
||||
{ok, {[{_, FooOff1, Chunk1, _}], []}} =
|
||||
machi_flu1_client:read_chunk(Host, PortBase+0, EpochID,
|
||||
File1, FooOff1, Size1, []),
|
||||
{ok, {[{_, FooOff1, Chunk1, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, File1, FooOff1, Size1, []),
|
||||
[?assertMatch({X,{ok, {[{_, FooOff1, Chunk1, _}], []}}},
|
||||
{X,machi_flu1_client:read_chunk(
|
||||
Host, PortBase+X, NSInfo, EpochID,
|
||||
File1, FooOff1, Size1, undefined)})
|
||||
Host, PortBase+X, EpochID,
|
||||
File1, FooOff1, Size1, [])})
|
||||
|| X <- [0,1,2] ],
|
||||
|
||||
%% Test read repair: Manually write to middle, then same checking.
|
||||
FooOff2 = Off1 + (2*1024*1024),
|
||||
Chunk2 = <<"Middle repair chunk">>,
|
||||
Size2 = size(Chunk2),
|
||||
ok = machi_flu1_client:write_chunk(Host, PortBase+1, NSInfo, EpochID,
|
||||
File1, FooOff2, Chunk2, NoCSum),
|
||||
{ok, {[{File1, FooOff2, Chunk2, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, FooOff2, Size2, undefined),
|
||||
[{X,{ok, {[{File1, FooOff2, Chunk2, _}], []}}} =
|
||||
ok = machi_flu1_client:write_chunk(Host, PortBase+1, EpochID,
|
||||
File1, FooOff2, Chunk2),
|
||||
{ok, {[{_, FooOff2, Chunk2, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, File1, FooOff2, Size2, []),
|
||||
[{X,{ok, {[{_, FooOff2, Chunk2, _}], []}}} =
|
||||
{X,machi_flu1_client:read_chunk(
|
||||
Host, PortBase+X, NSInfo, EpochID,
|
||||
File1, FooOff2, Size2, undefined)} || X <- [0,1,2] ],
|
||||
Host, PortBase+X, EpochID,
|
||||
File1, FooOff2, Size2, [])} || X <- [0,1,2] ],
|
||||
|
||||
%% Misc API smoke & minor regression checks
|
||||
{error, bad_arg} = machi_cr_client:read_chunk(C1, NSInfo, <<"no">>,
|
||||
999999999, 1, undefined),
|
||||
{ok, {[{File1,Off1,Chunk1,_}, {File1,FooOff1,Chunk1,_}, {File1,FooOff2,Chunk2,_}],
|
||||
{error, bad_arg} = machi_cr_client:read_chunk(C1, <<"no">>,
|
||||
999999999, 1, []),
|
||||
{ok, {[{_,Off1,Chunk1,_}, {_,FooOff1,Chunk1,_}, {_,FooOff2,Chunk2,_}],
|
||||
[]}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, 88888888, undefined),
|
||||
machi_cr_client:read_chunk(C1, File1, Off1, 88888888, []),
|
||||
%% Checksum list return value is a primitive binary().
|
||||
{ok, KludgeBin} = machi_cr_client:checksum_list(C1, File1),
|
||||
true = is_binary(KludgeBin),
|
||||
|
||||
{error, bad_arg} = machi_cr_client:checksum_list(C1, <<"!!!!">>),
|
||||
io:format(user, "\nFiles = ~p\n", [machi_cr_client:list_files(C1)]),
|
||||
%% Exactly one file right now, e.g.,
|
||||
%% {ok,[{2098202,<<"pre^b144ef13-db4d-4c9f-96e7-caff02dc754f^1">>}]}
|
||||
%% Exactly one file right now
|
||||
{ok, [_]} = machi_cr_client:list_files(C1),
|
||||
|
||||
%% Go back and test append_chunk() + extra and write_chunk()
|
||||
%% Go back and test append_chunk_extra() and write_chunk()
|
||||
Chunk10 = <<"It's a different chunk!">>,
|
||||
Size10 = byte_size(Chunk10),
|
||||
Extra10 = 5,
|
||||
Opts1 = #append_opts{chunk_extra=Extra10*Size10},
|
||||
{ok, {Off10,Size10,File10}} =
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk10,
|
||||
NoCSum, Opts1),
|
||||
machi_cr_client:append_chunk_extra(C1, Prefix, Chunk10,
|
||||
Extra10 * Size10),
|
||||
{ok, {[{_, Off10, Chunk10, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File10, Off10, Size10, undefined),
|
||||
machi_cr_client:read_chunk(C1, File10, Off10, Size10, []),
|
||||
[begin
|
||||
Offx = Off10 + (Seq * Size10),
|
||||
%% TODO: uncomment written/not_written enforcement is available.
|
||||
%% {error,not_written} = machi_cr_client:read_chunk(C1, NSInfo, File10,
|
||||
%% {error,not_written} = machi_cr_client:read_chunk(C1, File10,
|
||||
%% Offx, Size10),
|
||||
{ok, {Offx,Size10,File10}} =
|
||||
machi_cr_client:write_chunk(C1, NSInfo, File10, Offx, Chunk10, NoCSum),
|
||||
machi_cr_client:write_chunk(C1, File10, Offx, Chunk10),
|
||||
{ok, {[{_, Offx, Chunk10, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File10, Offx, Size10, undefined)
|
||||
machi_cr_client:read_chunk(C1, File10, Offx, Size10, [])
|
||||
end || Seq <- lists:seq(1, Extra10)],
|
||||
{ok, {Off11,Size11,File11}} =
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk10, NoCSum),
|
||||
%% %% Double-check that our reserved extra bytes were really honored!
|
||||
%% true = (Off11 > (Off10 + (Extra10 * Size10))),
|
||||
io:format(user, "\nFiles = ~p\n", [machi_cr_client:list_files(C1)]),
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk10),
|
||||
%% Double-check that our reserved extra bytes were really honored!
|
||||
true = (Off11 > (Off10 + (Extra10 * Size10))),
|
||||
|
||||
ok
|
||||
after
|
||||
|
|
@ -217,7 +205,7 @@ io:format(user, "\nFiles = ~p\n", [machi_cr_client:list_files(C1)]),
|
|||
witness_smoke_test_() -> {timeout, 1*60, fun() -> witness_smoke_test2() end}.
|
||||
|
||||
witness_smoke_test2() ->
|
||||
SupPid = case machi_sup:start_link() of
|
||||
SupPid = case machi_flu_sup:start_link() of
|
||||
{ok, P} -> P;
|
||||
{error, {already_started, P1}} -> P1;
|
||||
Other -> error(Other)
|
||||
|
|
@ -227,8 +215,6 @@ witness_smoke_test2() ->
|
|||
try
|
||||
Prefix = <<"pre">>,
|
||||
Chunk1 = <<"yochunk">>,
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
Host = "localhost",
|
||||
PortBase = 64444,
|
||||
Os = [{ignore_stability_time, true}, {active_mode, false},
|
||||
|
|
@ -238,15 +224,14 @@ witness_smoke_test2() ->
|
|||
|
||||
%% Whew ... ok, now start some damn tests.
|
||||
{ok, C1} = machi_cr_client:start_link([P || {_,P}<-orddict:to_list(D)]),
|
||||
{ok, _} = machi_cr_client:append_chunk(C1, NSInfo, Prefix,
|
||||
Chunk1, NoCSum),
|
||||
{ok, _} = machi_cr_client:append_chunk(C1, Prefix, Chunk1),
|
||||
{ok, {Off1,Size1,File1}} =
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum),
|
||||
BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, "foo")},
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1),
|
||||
Chunk1_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, Chunk1},
|
||||
{error, bad_checksum} =
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, BadCSum),
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1_badcs),
|
||||
{ok, {[{_, Off1, Chunk1, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, Size1, undefined),
|
||||
machi_cr_client:read_chunk(C1, File1, Off1, Size1, []),
|
||||
|
||||
%% Stop 'b' and let the chain reset.
|
||||
ok = machi_flu_psup:stop_flu_package(b),
|
||||
|
|
@ -259,25 +244,24 @@ witness_smoke_test2() ->
|
|||
%% Let's wedge OurWitness and see what happens: timeout/partition.
|
||||
#p_srvr{name=WitName, address=WitA, port=WitP} =
|
||||
orddict:fetch(OurWitness, D),
|
||||
{ok, {false, EpochID2,_,_}} = machi_flu1_client:wedge_status(WitA, WitP),
|
||||
{ok, {false, EpochID2}} = machi_flu1_client:wedge_status(WitA, WitP),
|
||||
machi_flu1:wedge_myself(WitName, EpochID2),
|
||||
case machi_flu1_client:wedge_status(WitA, WitP) of
|
||||
{ok, {true, EpochID2,_,_}} ->
|
||||
{ok, {true, EpochID2}} ->
|
||||
ok;
|
||||
{ok, {false, EpochID2,_,_}} ->
|
||||
{ok, {false, EpochID2}} ->
|
||||
%% This is racy. Work around it by sleeping a while.
|
||||
timer:sleep(6*1000),
|
||||
{ok, {true, EpochID2,_,_}} =
|
||||
{ok, {true, EpochID2}} =
|
||||
machi_flu1_client:wedge_status(WitA, WitP)
|
||||
end,
|
||||
|
||||
%% Chunk1 is still readable: not affected by wedged witness head.
|
||||
{ok, {[{_, Off1, Chunk1, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, Size1, undefined),
|
||||
machi_cr_client:read_chunk(C1, File1, Off1, Size1, []),
|
||||
%% But because the head is wedged, an append will fail.
|
||||
{error, partition} =
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum,
|
||||
#append_opts{}, 1*1000),
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1, 1*1000),
|
||||
|
||||
%% The witness's wedge status should cause timeout/partition
|
||||
%% for write_chunk also.
|
||||
|
|
@ -286,7 +270,7 @@ witness_smoke_test2() ->
|
|||
File10 = File1,
|
||||
Offx = Off1 + (1 * Size10),
|
||||
{error, partition} =
|
||||
machi_cr_client:write_chunk(C1, NSInfo, File10, Offx, Chunk10, NoCSum, 1*1000),
|
||||
machi_cr_client:write_chunk(C1, File10, Offx, Chunk10, 1*1000),
|
||||
|
||||
ok
|
||||
after
|
||||
|
|
|
|||
|
|
@ -2,31 +2,26 @@
|
|||
-compile(export_all).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-define(HDR, {0, 1024, none}).
|
||||
|
||||
cleanup(Dir) ->
|
||||
os:cmd("rm -rf " ++ Dir).
|
||||
-define(HDR, {0, 1024, <<0>>}).
|
||||
|
||||
smoke_test() ->
|
||||
Filename = "./temp-checksum-dumb-file",
|
||||
_ = cleanup(Filename),
|
||||
_ = file:delete(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
?assertEqual([{1024, infinity}],
|
||||
machi_csum_table:calc_unwritten_bytes(MC)),
|
||||
[{1024, infinity}] = machi_csum_table:calc_unwritten_bytes(MC),
|
||||
Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
|
||||
[] = machi_csum_table:find(MC, Offset, Size),
|
||||
ok = machi_csum_table:write(MC, Offset, Size, Checksum),
|
||||
[{1024, 40}, {1098, infinity}] = machi_csum_table:calc_unwritten_bytes(MC),
|
||||
?assertEqual([Entry], machi_csum_table:find(MC, Offset, Size)),
|
||||
ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined),
|
||||
?assertEqual([{Offset, Size, trimmed}],
|
||||
machi_csum_table:find(MC, Offset, Size)),
|
||||
[Entry] = machi_csum_table:find(MC, Offset, Size),
|
||||
ok = machi_csum_table:trim(MC, Offset, Size),
|
||||
[{Offset, Size, trimmed}] = machi_csum_table:find(MC, Offset, Size),
|
||||
ok = machi_csum_table:close(MC),
|
||||
ok = machi_csum_table:delete(MC).
|
||||
|
||||
close_test() ->
|
||||
Filename = "./temp-checksum-dumb-file-2",
|
||||
_ = cleanup(Filename),
|
||||
_ = file:delete(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
|
||||
[] = machi_csum_table:find(MC, Offset, Size),
|
||||
|
|
@ -36,33 +31,32 @@ close_test() ->
|
|||
|
||||
{ok, MC2} = machi_csum_table:open(Filename, []),
|
||||
[Entry] = machi_csum_table:find(MC2, Offset, Size),
|
||||
ok = machi_csum_table:trim(MC2, Offset, Size, undefined, undefined),
|
||||
ok = machi_csum_table:trim(MC2, Offset, Size),
|
||||
[{Offset, Size, trimmed}] = machi_csum_table:find(MC2, Offset, Size),
|
||||
ok = machi_csum_table:delete(MC2).
|
||||
|
||||
smoke2_test() ->
|
||||
Filename = "./temp-checksum-dumb-file-3",
|
||||
_ = cleanup(Filename),
|
||||
_ = file:delete(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
Entry = {Offset, Size, Checksum} = {1025, 10, <<"deadbeef">>},
|
||||
ok = machi_csum_table:write(MC, Offset, Size, Checksum),
|
||||
?assertEqual([], machi_csum_table:find(MC, 0, 0)),
|
||||
?assertEqual([?HDR], machi_csum_table:find(MC, 0, 1)),
|
||||
[] = machi_csum_table:find(MC, 0, 0),
|
||||
[?HDR] = machi_csum_table:find(MC, 0, 1),
|
||||
[Entry] = machi_csum_table:find(MC, Offset, Size),
|
||||
[?HDR] = machi_csum_table:find(MC, 1, 1024),
|
||||
?assertEqual([?HDR, Entry],
|
||||
machi_csum_table:find(MC, 1023, 1024)),
|
||||
[?HDR, Entry] = machi_csum_table:find(MC, 1023, 1024),
|
||||
[Entry] = machi_csum_table:find(MC, 1024, 1024),
|
||||
[Entry] = machi_csum_table:find(MC, 1025, 1024),
|
||||
|
||||
ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined),
|
||||
ok = machi_csum_table:trim(MC, Offset, Size),
|
||||
[{Offset, Size, trimmed}] = machi_csum_table:find(MC, Offset, Size),
|
||||
ok = machi_csum_table:close(MC),
|
||||
ok = machi_csum_table:delete(MC).
|
||||
|
||||
smoke3_test() ->
|
||||
Filename = "./temp-checksum-dumb-file-4",
|
||||
_ = cleanup(Filename),
|
||||
_ = file:delete(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
Scenario =
|
||||
[%% Command, {Offset, Size, Csum}, LeftNeighbor, RightNeibor
|
||||
|
|
@ -113,19 +107,3 @@ smoke3_test() ->
|
|||
ok = machi_csum_table:delete(MC).
|
||||
|
||||
%% TODO: add quickcheck test here
|
||||
|
||||
%% Previous implementation
|
||||
-spec all_trimmed2(machi_csum_table:table(),
|
||||
non_neg_integer(), non_neg_integer()) -> boolean().
|
||||
all_trimmed2(CsumT, Left, Right) ->
|
||||
Chunks = machi_csum_table:find(CsumT, Left, Right),
|
||||
runthru(Chunks, Left, Right).
|
||||
|
||||
%% @doc make sure all trimmed chunks are continously chained
|
||||
%% TODO: test with EQC
|
||||
runthru([], Pos, Pos) -> true;
|
||||
runthru([], Pos0, Pos) when Pos0 < Pos -> false;
|
||||
runthru([{Offset0, Size0, trimmed}|T], Offset, Pos) when Offset0 =< Offset ->
|
||||
runthru(T, Offset0+Size0, Pos);
|
||||
runthru(_L, _O, _P) ->
|
||||
false.
|
||||
|
|
|
|||
|
|
@ -31,18 +31,13 @@
|
|||
-define(QC_OUT(P),
|
||||
eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)).
|
||||
|
||||
-define(TESTDIR, "./eqc").
|
||||
|
||||
%% EUNIT TEST DEFINITION
|
||||
eqc_test_() ->
|
||||
PropTimeout = case os:getenv("EQC_TIME") of
|
||||
false -> 30;
|
||||
V -> list_to_integer(V)
|
||||
end,
|
||||
{timeout, PropTimeout*2 + 30,
|
||||
{timeout, 60,
|
||||
{spawn,
|
||||
[
|
||||
?_assertEqual(true, eqc:quickcheck(eqc:testing_time(PropTimeout, ?QC_OUT(prop_ok()))))
|
||||
{timeout, 30, ?_assertEqual(true, eqc:quickcheck(eqc:testing_time(15, ?QC_OUT(prop_ok()))))}
|
||||
]
|
||||
}}.
|
||||
|
||||
|
|
@ -107,26 +102,19 @@ shuffle_interval() ->
|
|||
|
||||
get_written_interval(L) ->
|
||||
?LET({O, Ln}, elements(L), {O+1, Ln-1}).
|
||||
|
||||
|
||||
%% INITIALIZATION
|
||||
|
||||
-record(state, {pid, prev_extra = 0,
|
||||
filename = undefined,
|
||||
planned_writes=[],
|
||||
planned_trims=[],
|
||||
written=[],
|
||||
trimmed=[]}).
|
||||
|
||||
initial_state() ->
|
||||
{_, _, MS} = os:timestamp(),
|
||||
Filename = test_server:temp_name("eqc_data") ++ "." ++ integer_to_list(MS),
|
||||
#state{filename=Filename, written=[{0,1024}]}.
|
||||
|
||||
initial_state(I, T) ->
|
||||
S=initial_state(),
|
||||
S#state{written=[{0,1024}],
|
||||
planned_writes=I,
|
||||
planned_trims=T}.
|
||||
initial_state() -> #state{written=[{0,1024}]}.
|
||||
initial_state(I, T) -> #state{written=[{0,1024}],
|
||||
planned_writes=I,
|
||||
planned_trims=T}.
|
||||
|
||||
weight(_S, rewrite) -> 1;
|
||||
weight(_S, _) -> 2.
|
||||
|
|
@ -148,6 +136,11 @@ get_overlaps(Offset, Len, [{Pos, Sz} = Ck|T], Acc0)
|
|||
(Pos < Offset + Len andalso Offset + Len < Pos + Sz) ->
|
||||
get_overlaps(Offset, Len, T, [Ck|Acc0]);
|
||||
get_overlaps(Offset, Len, [_Ck|T], Acc0) ->
|
||||
%% ?debugVal({Offset, Len, _Ck}),
|
||||
%% ?debugVal(Offset =< Pos andalso Pos < Offset + Len andalso Offset + Len =< Pos + Sz),
|
||||
%% ?debugVal(Offset =< Pos andalso Pos + Sz < Offset + Len),
|
||||
%% ?debugVal(Pos < Offset andalso Offset < Pos + Sz andalso Pos + Sz < Offset + Len),
|
||||
%% ?debugVal(Pos < Offset + Len andalso Offset + Len < Pos + Sz),
|
||||
get_overlaps(Offset, Len, T, Acc0).
|
||||
|
||||
%% Inefficient but simple easy code to verify by eyes - returns all
|
||||
|
|
@ -216,6 +209,8 @@ last_byte(L0) ->
|
|||
L1 = lists:map(fun({Pos, Sz}) -> Pos + Sz end, L0),
|
||||
lists:last(lists:sort(L1)).
|
||||
|
||||
-define(TESTDIR, "./eqc").
|
||||
|
||||
cleanup() ->
|
||||
[begin
|
||||
Fs = filelib:wildcard(?TESTDIR ++ Glob),
|
||||
|
|
@ -233,12 +228,14 @@ start_pre(S) ->
|
|||
start_command(S) ->
|
||||
{call, ?MODULE, start, [S]}.
|
||||
|
||||
start(#state{filename=File}) ->
|
||||
start(_S) ->
|
||||
{_, _, MS} = os:timestamp(),
|
||||
File = test_server:temp_name("eqc_data") ++ "." ++ integer_to_list(MS),
|
||||
{ok, Pid} = machi_file_proxy:start_link(some_flu, File, ?TESTDIR),
|
||||
unlink(Pid),
|
||||
Pid.
|
||||
|
||||
start_next(S, Pid, _) ->
|
||||
start_next(S, Pid, _Args) ->
|
||||
S#state{pid = Pid}.
|
||||
|
||||
%% read
|
||||
|
|
@ -247,19 +244,25 @@ read_pre(S) ->
|
|||
S#state.pid /= undefined.
|
||||
|
||||
read_args(S) ->
|
||||
[S#state.pid, oneof([offset(), big_offset()]), len()].
|
||||
[S#state.pid, offset(), len()].
|
||||
|
||||
read_post(S, [_Pid, Off, L], Res) ->
|
||||
Written = get_overlaps(Off, L, S#state.written, []),
|
||||
Chopped = chop(Off, L, Written),
|
||||
Trimmed = get_overlaps(Off, L, S#state.trimmed, []),
|
||||
Eof = lists:max([Pos+Sz||{Pos,Sz}<-S#state.written]),
|
||||
%% ?debugVal({Off, L}),
|
||||
%% ?debugVal(S),
|
||||
case Res of
|
||||
{ok, {Written0, Trimmed0}} ->
|
||||
Written1 = lists:map(fun({_, Pos, Chunk, _}) ->
|
||||
{Pos, iolist_size(Chunk)}
|
||||
end, Written0),
|
||||
Trimmed1 = lists:map(fun({_, Pos, Sz}) -> {Pos, Sz} end, Trimmed0),
|
||||
%% ?debugVal({Written, Chopped, Written1}),
|
||||
%% ?debugVal({Trimmed, Trimmed1}),
|
||||
%% ?assertEqual(Chopped, Written1),
|
||||
%% ?assertEqual(Trimmed, Trimmed1),
|
||||
Chopped =:= Written1
|
||||
andalso Trimmed =:= Trimmed1;
|
||||
%% TODO: such response are ugly, rethink the SPEC
|
||||
|
|
@ -267,7 +270,8 @@ read_post(S, [_Pid, Off, L], Res) ->
|
|||
true;
|
||||
{error, not_written} when Chopped =:= [] andalso Trimmed =:= [] ->
|
||||
true;
|
||||
_Other ->
|
||||
Other ->
|
||||
?debugVal(Other),
|
||||
is_error(Res)
|
||||
end.
|
||||
|
||||
|
|
@ -308,13 +312,13 @@ write_post(S, [_Pid, Off, {Bin, _Tag, _Csum}] = _Args, Res) ->
|
|||
|
||||
write_next(S, Res, [_Pid, Offset, {Bin, _Tag, _Csum}]) ->
|
||||
S0 = case is_ok(Res) of
|
||||
true ->
|
||||
true ->
|
||||
S#state{written = lists:sort(S#state.written ++ [{Offset, iolist_size(Bin)}]) };
|
||||
_ ->
|
||||
_ ->
|
||||
S
|
||||
end,
|
||||
S0#state{prev_extra = 0, planned_writes=tl(S0#state.planned_writes)}.
|
||||
|
||||
|
||||
|
||||
write(Pid, Offset, {Bin, Tag, Csum}) ->
|
||||
Meta = [{client_csum_tag, Tag},
|
||||
|
|
@ -324,6 +328,7 @@ write(Pid, Offset, {Bin, Tag, Csum}) ->
|
|||
%% append
|
||||
|
||||
append_pre(S) ->
|
||||
?assert(undefined =/= S#state.written),
|
||||
S#state.pid /= undefined.
|
||||
|
||||
%% do not allow appends with empty binary data
|
||||
|
|
@ -342,32 +347,20 @@ append(Pid, Extra, {Bin, Tag, Csum}) ->
|
|||
|
||||
append_next(S, Res, [_Pid, Extra, {Bin, _Tag, _Csum}]) ->
|
||||
case is_ok(Res) of
|
||||
true ->
|
||||
true ->
|
||||
Offset = get_offset(Res),
|
||||
S#state{prev_extra = Extra,
|
||||
written = lists:sort(S#state.written ++ [{Offset, iolist_size(Bin)}])};
|
||||
_Other ->
|
||||
Expected = erlang:max(last_byte(S#state.written) + S#state.prev_extra,
|
||||
last_byte(S#state.trimmed)),
|
||||
?assertEqual(Expected, Offset),
|
||||
S#state{prev_extra = Extra, written = lists:sort(S#state.written ++ [{Offset, iolist_size(Bin)}])};
|
||||
_ ->
|
||||
S
|
||||
end.
|
||||
|
||||
%% appends should always succeed unless the disk is full
|
||||
%% appends should always succeed unless the disk is full
|
||||
%% or there's a hardware failure.
|
||||
append_post(S, _Args, Res) ->
|
||||
case is_ok(Res) of
|
||||
true ->
|
||||
Offset = get_offset(Res),
|
||||
case erlang:max(last_byte(S#state.written),
|
||||
last_byte(S#state.trimmed)) + S#state.prev_extra of
|
||||
Offset ->
|
||||
true;
|
||||
UnexpectedByte ->
|
||||
{wrong_offset_after_append,
|
||||
{Offset, UnexpectedByte},
|
||||
{S#state.written, S#state.prev_extra}}
|
||||
end;
|
||||
Error ->
|
||||
Error
|
||||
end.
|
||||
append_post(_S, _Args, Res) ->
|
||||
true == is_ok(Res).
|
||||
|
||||
%% rewrite
|
||||
|
||||
|
|
@ -422,33 +415,24 @@ trim_next(S, Res, [_Pid, Offset, Length]) ->
|
|||
S1#state{prev_extra=0,
|
||||
planned_trims=tl(S#state.planned_trims)}.
|
||||
|
||||
stop_pre(S) ->
|
||||
S#state.pid /= undefined.
|
||||
|
||||
stop_args(S) ->
|
||||
[S#state.pid].
|
||||
|
||||
stop(Pid) ->
|
||||
catch machi_file_proxy:stop(Pid).
|
||||
|
||||
stop_post(_, _, _) -> true.
|
||||
|
||||
stop_next(S, _, _) ->
|
||||
S#state{pid=undefined, prev_extra=0}.
|
||||
|
||||
%% Property
|
||||
|
||||
prop_ok() ->
|
||||
cleanup(),
|
||||
?FORALL({I, T},
|
||||
{shuffle_interval(), shuffle_interval()},
|
||||
?FORALL(Cmds, parallel_commands(?MODULE, initial_state(I, T)),
|
||||
begin
|
||||
{H, S, Res} = run_parallel_commands(?MODULE, Cmds),
|
||||
cleanup(),
|
||||
pretty_commands(?MODULE, Cmds, {H, S, Res},
|
||||
aggregate(command_names(Cmds), Res == ok))
|
||||
end)).
|
||||
cleanup(),
|
||||
?FORALL({I, T},
|
||||
{shuffle_interval(), shuffle_interval()},
|
||||
?FORALL(Cmds, parallel_commands(?MODULE, initial_state(I, T)),
|
||||
begin
|
||||
{H, S, Res} = run_parallel_commands(?MODULE, Cmds),
|
||||
%% case S#state.pid of
|
||||
%% undefined -> noop;
|
||||
%% Pid ->
|
||||
%% machi_file_proxy:stop(Pid)
|
||||
%% end,
|
||||
pretty_commands(?MODULE, Cmds, {H, S, Res},
|
||||
aggregate(command_names(Cmds), Res == ok))
|
||||
end)
|
||||
).
|
||||
|
||||
%% Test for tester functions
|
||||
chopper_test_() ->
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ clean_up_data_dir(DataDir) ->
|
|||
-ifndef(PULSE).
|
||||
|
||||
-define(TESTDIR, "./t").
|
||||
-define(HYOOGE, 75 * 1024 * 1024). % 75 MBytes
|
||||
-define(HYOOGE, 1 * 1024 * 1024 * 1024). % 1 long GB
|
||||
|
||||
random_binary_single() ->
|
||||
%% OK, I guess it's not that random...
|
||||
|
|
@ -76,67 +76,54 @@ random_binary(Start, End) ->
|
|||
binary:part(random_binary_single(), Start, End)
|
||||
end.
|
||||
|
||||
setup() ->
|
||||
{ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR),
|
||||
Pid.
|
||||
|
||||
teardown(Pid) ->
|
||||
catch machi_file_proxy:stop(Pid).
|
||||
|
||||
machi_file_proxy_test_() ->
|
||||
clean_up_data_dir(?TESTDIR),
|
||||
{setup,
|
||||
fun setup/0,
|
||||
fun teardown/1,
|
||||
fun(Pid) ->
|
||||
[
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:read(Pid, -1, -1)),
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:write(Pid, -1, <<"yo">>)),
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:append(Pid, [], -1, <<"krep">>)),
|
||||
?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, 1)),
|
||||
?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1024)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, ?HYOOGE)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, ?HYOOGE, 1)),
|
||||
{timeout, 10,
|
||||
?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, random_binary(0, ?HYOOGE)))},
|
||||
?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))),
|
||||
?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1024, <<"fail">>)),
|
||||
?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, <<"fail">>)),
|
||||
?_assertMatch({ok, {[{_, _, _, _}], []}}, machi_file_proxy:read(Pid, 1025, 1000)),
|
||||
?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, [], 1024, <<"mind the gap">>)),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 2060, [], random_binary(0, 1024)))
|
||||
]
|
||||
end}.
|
||||
{ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR),
|
||||
[
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:read(Pid, -1, -1)),
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:write(Pid, -1, <<"yo">>)),
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:append(Pid, [], -1, <<"krep">>)),
|
||||
?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, 1)),
|
||||
?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1024)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, ?HYOOGE)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, ?HYOOGE, 1)),
|
||||
?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, random_binary(0, ?HYOOGE))),
|
||||
?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))),
|
||||
?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1024, <<"fail">>)),
|
||||
?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, <<"fail">>)),
|
||||
?_assertMatch({ok, {[{_, _, _, _}], []}}, machi_file_proxy:read(Pid, 1025, 1000)),
|
||||
?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, [], 1024, <<"mind the gap">>)),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 2060, [], random_binary(0, 1024))),
|
||||
?_assertException(exit, {normal, _}, machi_file_proxy:stop(Pid))
|
||||
].
|
||||
|
||||
multiple_chunks_read_test_() ->
|
||||
clean_up_data_dir(?TESTDIR),
|
||||
{setup,
|
||||
fun setup/0,
|
||||
fun teardown/1,
|
||||
fun(Pid) ->
|
||||
[
|
||||
?_assertEqual(ok, machi_file_proxy:trim(Pid, 0, 1, false)),
|
||||
?_assertMatch({ok, {[], [{"test", 0, 1}]}},
|
||||
machi_file_proxy:read(Pid, 0, 1,
|
||||
#read_opts{needs_trimmed=true})),
|
||||
?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 10000, <<"fail">>)),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 20000, <<"fail">>)),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 30000, <<"fail">>)),
|
||||
%% Freeza
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 530000, <<"fail">>)),
|
||||
?_assertMatch({ok, {[{"test", 1024, _, _},
|
||||
{"test", 10000, <<"fail">>, _},
|
||||
{"test", 20000, <<"fail">>, _},
|
||||
{"test", 30000, <<"fail">>, _},
|
||||
{"test", 530000, <<"fail">>, _}], []}},
|
||||
machi_file_proxy:read(Pid, 1024, 530000)),
|
||||
?_assertMatch({ok, {[{"test", 1, _, _}], [{"test", 0, 1}]}},
|
||||
machi_file_proxy:read(Pid, 0, 1024,
|
||||
#read_opts{needs_trimmed=true}))
|
||||
]
|
||||
end}.
|
||||
{ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR),
|
||||
[
|
||||
?_assertEqual(ok, machi_file_proxy:trim(Pid, 0, 1, false)),
|
||||
?_assertMatch({ok, {[], [{"test", 0, 1}]}},
|
||||
machi_file_proxy:read(Pid, 0, 1,
|
||||
[{needs_trimmed, true}])),
|
||||
?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 10000, <<"fail">>)),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 20000, <<"fail">>)),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 30000, <<"fail">>)),
|
||||
%% Freeza
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 530000, <<"fail">>)),
|
||||
?_assertMatch({ok, {[{"test", 1024, _, _},
|
||||
{"test", 10000, <<"fail">>, _},
|
||||
{"test", 20000, <<"fail">>, _},
|
||||
{"test", 30000, <<"fail">>, _},
|
||||
{"test", 530000, <<"fail">>, _}], []}},
|
||||
machi_file_proxy:read(Pid, 1024, 530000)),
|
||||
?_assertMatch({ok, {[{"test", 1, _, _}], [{"test", 0, 1}]}},
|
||||
machi_file_proxy:read(Pid, 0, 1024,
|
||||
[{needs_trimmed, true}])),
|
||||
?_assertException(exit, {normal, _}, machi_file_proxy:stop(Pid))
|
||||
].
|
||||
|
||||
|
||||
-endif. % !PULSE
|
||||
-endif. % TEST.
|
||||
|
|
|
|||
|
|
@ -30,22 +30,6 @@
|
|||
-define(FLU, machi_flu1).
|
||||
-define(FLU_C, machi_flu1_client).
|
||||
|
||||
get_env_vars(App, Ks) ->
|
||||
Raw = [application:get_env(App, K) || K <- Ks],
|
||||
Old = lists:zip(Ks, Raw),
|
||||
{App, Old}.
|
||||
|
||||
clean_up_env_vars({App, Old}) ->
|
||||
[case Res of
|
||||
undefined ->
|
||||
application:unset_env(App, K);
|
||||
{ok, V} ->
|
||||
application:set_env(App, K, V)
|
||||
end || {K, Res} <- Old].
|
||||
|
||||
filter_env_var({ok, V}) -> V;
|
||||
filter_env_var(Else) -> Else.
|
||||
|
||||
clean_up_data_dir(DataDir) ->
|
||||
[begin
|
||||
Fs = filelib:wildcard(DataDir ++ Glob),
|
||||
|
|
@ -85,46 +69,47 @@ maybe_start_sup() ->
|
|||
Pid -> Pid
|
||||
end.
|
||||
|
||||
|
||||
-ifndef(PULSE).
|
||||
|
||||
flu_smoke_test() ->
|
||||
Host = "localhost",
|
||||
TcpPort = 12957,
|
||||
TcpPort = 32957,
|
||||
DataDir = "./data",
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
Prefix = <<"prefix!">>,
|
||||
BadPrefix = BadFile = "no/good",
|
||||
|
||||
W_props = [{initial_wedged, false}],
|
||||
{_, _, _} = machi_test_util:start_flu_package(smoke_flu, TcpPort, DataDir, W_props),
|
||||
start_flu_package(smoke_flu, TcpPort, DataDir, W_props),
|
||||
try
|
||||
Msg = "Hello, world!",
|
||||
Msg = ?FLU_C:echo(Host, TcpPort, Msg),
|
||||
{error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort,"does-not-exist"),
|
||||
{error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort, BadFile),
|
||||
{error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
"does-not-exist"),
|
||||
{error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH, BadFile),
|
||||
|
||||
{ok, []} = ?FLU_C:list_files(Host, TcpPort, ?DUMMY_PV1_EPOCH),
|
||||
{ok, {false, _,_,_}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
{ok, {false, _}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
|
||||
Chunk1 = <<"yo!">>,
|
||||
{ok, {Off1,Len1,File1}} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
|
||||
{ok, {Off1,Len1,File1}} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix, Chunk1, NoCSum),
|
||||
{ok, {[{_, Off1, Chunk1, _}], _}} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
File1, Off1, Len1,
|
||||
noopt),
|
||||
{ok, KludgeBin} = ?FLU_C:checksum_list(Host, TcpPort, File1),
|
||||
Prefix, Chunk1),
|
||||
{ok, {[{_, Off1, Chunk1, _}], _}} = ?FLU_C:read_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH,
|
||||
File1, Off1, Len1, []),
|
||||
{ok, KludgeBin} = ?FLU_C:checksum_list(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH, File1),
|
||||
true = is_binary(KludgeBin),
|
||||
{error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
|
||||
{error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
BadPrefix, Chunk1, NoCSum),
|
||||
BadPrefix, Chunk1),
|
||||
{ok, [{_,File1}]} = ?FLU_C:list_files(Host, TcpPort, ?DUMMY_PV1_EPOCH),
|
||||
Len1 = size(Chunk1),
|
||||
{error, not_written} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
File1, Off1*983829323, Len1,
|
||||
noopt),
|
||||
?DUMMY_PV1_EPOCH,
|
||||
File1, Off1*983829323, Len1, []),
|
||||
%% XXX FIXME
|
||||
%%
|
||||
%% This is failing because the read extends past the end of the file.
|
||||
|
|
@ -133,22 +118,19 @@ flu_smoke_test() ->
|
|||
%% of the read will cause it to fail.
|
||||
%%
|
||||
%% {error, partial_read} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
%% NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
%% ?DUMMY_PV1_EPOCH,
|
||||
%% File1, Off1, Len1*9999),
|
||||
|
||||
{ok, {Off1b,Len1b,File1b}} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
|
||||
{ok, {Off1b,Len1b,File1b}} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix, Chunk1,NoCSum),
|
||||
Prefix, Chunk1),
|
||||
Extra = 42,
|
||||
Opts1 = #append_opts{chunk_extra=Extra},
|
||||
{ok, {Off1c,Len1c,File1c}} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
|
||||
{ok, {Off1c,Len1c,File1c}} = ?FLU_C:append_chunk_extra(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix, Chunk1, NoCSum,
|
||||
Opts1, infinity),
|
||||
Prefix, Chunk1, Extra),
|
||||
{ok, {Off1d,Len1d,File1d}} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
NSInfo,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix, Chunk1,NoCSum),
|
||||
Prefix, Chunk1),
|
||||
if File1b == File1c, File1c == File1d ->
|
||||
true = (Off1c == Off1b + Len1b),
|
||||
true = (Off1d == Off1c + Len1c + Extra);
|
||||
|
|
@ -156,44 +138,27 @@ flu_smoke_test() ->
|
|||
exit(not_mandatory_but_test_expected_same_file_fixme)
|
||||
end,
|
||||
|
||||
Chunk1_cs = {<<?CSUM_TAG_NONE:8, 0:(8*20)>>, Chunk1},
|
||||
{ok, {Off1e,Len1e,File1e}} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix, Chunk1_cs),
|
||||
|
||||
Chunk2 = <<"yo yo">>,
|
||||
Len2 = byte_size(Chunk2),
|
||||
Off2 = ?MINIMUM_OFFSET + 77,
|
||||
File2 = "smoke-whole-file^^0^1^1",
|
||||
ok = ?FLU_C:write_chunk(Host, TcpPort, NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
File2, Off2, Chunk2, NoCSum),
|
||||
{error, bad_arg} = ?FLU_C:write_chunk(Host, TcpPort, NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
BadFile, Off2, Chunk2, NoCSum),
|
||||
File2 = "smoke-whole-file^1^1",
|
||||
ok = ?FLU_C:write_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH,
|
||||
File2, Off2, Chunk2),
|
||||
{error, bad_arg} = ?FLU_C:write_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH,
|
||||
BadFile, Off2, Chunk2),
|
||||
{ok, {[{_, Off2, Chunk2, _}], _}} =
|
||||
?FLU_C:read_chunk(Host, TcpPort, NSInfo, ?DUMMY_PV1_EPOCH, File2, Off2, Len2, noopt),
|
||||
?FLU_C:read_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH, File2, Off2, Len2, []),
|
||||
{error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
"no!!", Off2, Len2, noopt),
|
||||
?DUMMY_PV1_EPOCH,
|
||||
"no!!", Off2, Len2, []),
|
||||
{error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
BadFile, Off2, Len2, noopt),
|
||||
|
||||
%% Make a connected socket.
|
||||
Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}),
|
||||
|
||||
%% Let's test some cluster version enforcement.
|
||||
Good_EpochNum = 0,
|
||||
Good_NSVersion = 0,
|
||||
Good_NS = <<>>,
|
||||
{ok, {false, {Good_EpochNum,_}, Good_NSVersion, GoodNS}} =
|
||||
?FLU_C:wedge_status(Sock1),
|
||||
NS_good = #ns_info{version=Good_NSVersion, name=Good_NS},
|
||||
{ok, {[{_, Off2, Chunk2, _}], _}} =
|
||||
?FLU_C:read_chunk(Sock1, NS_good, ?DUMMY_PV1_EPOCH,
|
||||
File2, Off2, Len2, noopt),
|
||||
NS_bad_version = #ns_info{version=1, name=Good_NS},
|
||||
NS_bad_name = #ns_info{version=Good_NSVersion, name= <<"foons">>},
|
||||
{error, bad_epoch} =
|
||||
?FLU_C:read_chunk(Sock1, NS_bad_version, ?DUMMY_PV1_EPOCH,
|
||||
File2, Off2, Len2, noopt),
|
||||
{error, bad_arg} =
|
||||
?FLU_C:read_chunk(Sock1, NS_bad_name, ?DUMMY_PV1_EPOCH,
|
||||
File2, Off2, Len2, noopt),
|
||||
?DUMMY_PV1_EPOCH,
|
||||
BadFile, Off2, Len2, []),
|
||||
|
||||
%% We know that File1 still exists. Pretend that we've done a
|
||||
%% migration and exercise the delete_migration() API.
|
||||
|
|
@ -210,23 +175,25 @@ flu_smoke_test() ->
|
|||
{error, bad_arg} = ?FLU_C:trunc_hack(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH, BadFile),
|
||||
|
||||
ok = ?FLU_C:quit(Sock1)
|
||||
ok = ?FLU_C:quit(?FLU_C:connect(#p_srvr{address=Host,
|
||||
port=TcpPort}))
|
||||
after
|
||||
machi_test_util:stop_flu_package()
|
||||
stop_flu_package(smoke_flu)
|
||||
end.
|
||||
|
||||
flu_projection_smoke_test() ->
|
||||
Host = "localhost",
|
||||
TcpPort = 12959,
|
||||
TcpPort = 32959,
|
||||
DataDir = "./data.projst",
|
||||
{_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir),
|
||||
|
||||
start_flu_package(projection_test_flu, TcpPort, DataDir),
|
||||
try
|
||||
[ok = flu_projection_common(Host, TcpPort, T) ||
|
||||
T <- [public, private] ]
|
||||
%% , {ok, {false, EpochID1,_,_}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
%% , {ok, {false, EpochID1}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
%% io:format(user, "EpochID1 ~p\n", [EpochID1])
|
||||
after
|
||||
machi_test_util:stop_flu_package()
|
||||
stop_flu_package(projection_test_flu)
|
||||
end.
|
||||
|
||||
flu_projection_common(Host, TcpPort, T) ->
|
||||
|
|
@ -254,32 +221,30 @@ flu_projection_common(Host, TcpPort, T) ->
|
|||
|
||||
bad_checksum_test() ->
|
||||
Host = "localhost",
|
||||
TcpPort = 12960,
|
||||
TcpPort = 32960,
|
||||
DataDir = "./data.bct",
|
||||
|
||||
Opts = [{initial_wedged, false}],
|
||||
{_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts),
|
||||
NSInfo = undefined,
|
||||
start_flu_package(projection_test_flu, TcpPort, DataDir, Opts),
|
||||
try
|
||||
Prefix = <<"some prefix">>,
|
||||
Chunk1 = <<"yo yo yo">>,
|
||||
BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, ".................")},
|
||||
{error, bad_checksum} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
|
||||
Chunk1_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, Chunk1},
|
||||
{error, bad_checksum} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix,
|
||||
Chunk1, BadCSum),
|
||||
Prefix, Chunk1_badcs),
|
||||
ok
|
||||
after
|
||||
machi_test_util:stop_flu_package()
|
||||
stop_flu_package(projection_test_flu)
|
||||
end.
|
||||
|
||||
witness_test() ->
|
||||
Host = "localhost",
|
||||
TcpPort = 12961,
|
||||
TcpPort = 32961,
|
||||
DataDir = "./data.witness",
|
||||
|
||||
Opts = [{initial_wedged, false}, {witness_mode, true}],
|
||||
{_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts),
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
start_flu_package(projection_test_flu, TcpPort, DataDir, Opts),
|
||||
try
|
||||
Prefix = <<"some prefix">>,
|
||||
Chunk1 = <<"yo yo yo">>,
|
||||
|
|
@ -292,14 +257,15 @@ witness_test() ->
|
|||
{ok, EpochID1} = ?FLU_C:get_latest_epochid(Host, TcpPort, private),
|
||||
|
||||
%% Witness-protected ops all fail
|
||||
{error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo, EpochID1,
|
||||
Prefix, Chunk1, NoCSum),
|
||||
{error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort, EpochID1,
|
||||
Prefix, Chunk1),
|
||||
File = <<"foofile">>,
|
||||
{error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort, NSInfo, EpochID1,
|
||||
File, 9999, 9999, noopt),
|
||||
{error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort, File),
|
||||
{error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort, EpochID1,
|
||||
File, 9999, 9999, []),
|
||||
{error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort, EpochID1,
|
||||
File),
|
||||
{error, bad_arg} = ?FLU_C:list_files(Host, TcpPort, EpochID1),
|
||||
{ok, {false, EpochID1,_,_}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
{ok, {false, EpochID1}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
{ok, _} = ?FLU_C:get_latest_epochid(Host, TcpPort, public),
|
||||
{ok, _} = ?FLU_C:read_latest_projection(Host, TcpPort, public),
|
||||
{error, not_written} = ?FLU_C:read_projection(Host, TcpPort,
|
||||
|
|
@ -310,7 +276,7 @@ witness_test() ->
|
|||
|
||||
ok
|
||||
after
|
||||
machi_test_util:stop_flu_package()
|
||||
stop_flu_package(projection_test_flu)
|
||||
end.
|
||||
|
||||
%% The purpose of timing_pb_encoding_test_ and timing_bif_encoding_test_ is
|
||||
|
|
|
|||
|
|
@ -38,12 +38,12 @@ smoke_test_() ->
|
|||
{timeout, 5*60, fun() -> smoke_test2() end}.
|
||||
|
||||
smoke_test2() ->
|
||||
Ps = [{a,#p_srvr{name=a, address="localhost", port=5550, props="./data.a"}},
|
||||
{b,#p_srvr{name=b, address="localhost", port=5551, props="./data.b"}},
|
||||
{c,#p_srvr{name=c, address="localhost", port=5552, props="./data.c"}}
|
||||
Ps = [{a,#p_srvr{name=a, address="localhost", port=5555, props="./data.a"}},
|
||||
{b,#p_srvr{name=b, address="localhost", port=5556, props="./data.b"}},
|
||||
{c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}}
|
||||
],
|
||||
[os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps],
|
||||
{ok, SupPid} = machi_sup:start_link(),
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
try
|
||||
%% Only run a, don't run b & c so we have 100% failures talking to them
|
||||
[begin
|
||||
|
|
@ -66,15 +66,15 @@ partial_stop_restart_test_() ->
|
|||
{timeout, 5*60, fun() -> partial_stop_restart2() end}.
|
||||
|
||||
partial_stop_restart2() ->
|
||||
Ps = [{a,#p_srvr{name=a, address="localhost", port=5560, props="./data.a"}},
|
||||
{b,#p_srvr{name=b, address="localhost", port=5561, props="./data.b"}},
|
||||
{c,#p_srvr{name=c, address="localhost", port=5562, props="./data.c"}}
|
||||
Ps = [{a,#p_srvr{name=a, address="localhost", port=5555, props="./data.a"}},
|
||||
{b,#p_srvr{name=b, address="localhost", port=5556, props="./data.b"}},
|
||||
{c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}}
|
||||
],
|
||||
ChMgrs = [machi_flu_psup:make_mgr_supname(P#p_srvr.name) || {_,P} <-Ps],
|
||||
PStores = [machi_flu_psup:make_proj_supname(P#p_srvr.name) || {_,P} <-Ps],
|
||||
Dict = orddict:from_list(Ps),
|
||||
[os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps],
|
||||
{ok, SupPid} = machi_sup:start_link(),
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
DbgProps = [{initial_wedged, true}],
|
||||
Start = fun({_,P}) ->
|
||||
#p_srvr{name=Name, port=Port, props=Dir} = P,
|
||||
|
|
@ -84,23 +84,20 @@ partial_stop_restart2() ->
|
|||
WedgeStatus = fun({_,#p_srvr{address=Addr, port=TcpPort}}) ->
|
||||
machi_flu1_client:wedge_status(Addr, TcpPort)
|
||||
end,
|
||||
NSInfo = undefined,
|
||||
Append = fun({_,#p_srvr{address=Addr, port=TcpPort}}, EpochID) ->
|
||||
NoCSum = <<>>,
|
||||
machi_flu1_client:append_chunk(Addr, TcpPort,
|
||||
NSInfo, EpochID,
|
||||
<<"prefix">>,
|
||||
<<"data">>, NoCSum)
|
||||
EpochID,
|
||||
<<"prefix">>, <<"data">>)
|
||||
end,
|
||||
try
|
||||
[Start(P) || P <- Ps],
|
||||
[{ok, {true, _,_,_}} = WedgeStatus(P) || P <- Ps], % all are wedged
|
||||
[{ok, {true, _}} = WedgeStatus(P) || P <- Ps], % all are wedged
|
||||
[{error,wedged} = Append(P, ?DUMMY_PV1_EPOCH) || P <- Ps], % all are wedged
|
||||
|
||||
[machi_chain_manager1:set_chain_members(ChMgr, Dict) ||
|
||||
ChMgr <- ChMgrs ],
|
||||
{ok, {false, EpochID1,_,_}} = WedgeStatus(hd(Ps)),
|
||||
[{ok, {false, EpochID1,_,_}} = WedgeStatus(P) || P <- Ps], % *not* wedged
|
||||
{ok, {false, EpochID1}} = WedgeStatus(hd(Ps)),
|
||||
[{ok, {false, EpochID1}} = WedgeStatus(P) || P <- Ps], % *not* wedged
|
||||
[{ok,_} = Append(P, EpochID1) || P <- Ps], % *not* wedged
|
||||
{ok, {_,_,File1}} = Append(hd(Ps), EpochID1),
|
||||
|
||||
|
|
@ -126,9 +123,9 @@ partial_stop_restart2() ->
|
|||
Epoch_m = Proj_m#projection_v1.epoch_number,
|
||||
%% Confirm that all FLUs are *not* wedged, with correct proj & epoch
|
||||
Proj_mCSum = Proj_m#projection_v1.epoch_csum,
|
||||
[{ok, {false, {Epoch_m, Proj_mCSum},_,_}} = WedgeStatus(P) || % *not* wedged
|
||||
[{ok, {false, {Epoch_m, Proj_mCSum}}} = WedgeStatus(P) || % *not* wedged
|
||||
P <- Ps],
|
||||
{ok, {false, EpochID1,_,_}} = WedgeStatus(hd(Ps)),
|
||||
{ok, {false, EpochID1}} = WedgeStatus(hd(Ps)),
|
||||
[{ok,_} = Append(P, EpochID1) || P <- Ps], % *not* wedged
|
||||
|
||||
%% Stop all but 'a'.
|
||||
|
|
@ -148,10 +145,10 @@ partial_stop_restart2() ->
|
|||
{error, wedged} = Append(hd(Ps), EpochID1),
|
||||
{_, #p_srvr{address=Addr_a, port=TcpPort_a}} = hd(Ps),
|
||||
{error, wedged} = machi_flu1_client:read_chunk(
|
||||
Addr_a, TcpPort_a, NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
<<>>, 99999999, 1, undefined),
|
||||
{error, bad_arg} = machi_flu1_client:checksum_list(
|
||||
Addr_a, TcpPort_a, <<>>),
|
||||
Addr_a, TcpPort_a, ?DUMMY_PV1_EPOCH,
|
||||
<<>>, 99999999, 1, []),
|
||||
{error, wedged} = machi_flu1_client:checksum_list(
|
||||
Addr_a, TcpPort_a, ?DUMMY_PV1_EPOCH, <<>>),
|
||||
%% list_files() is permitted despite wedged status
|
||||
{ok, _} = machi_flu1_client:list_files(
|
||||
Addr_a, TcpPort_a, ?DUMMY_PV1_EPOCH),
|
||||
|
|
@ -160,7 +157,7 @@ partial_stop_restart2() ->
|
|||
{now_using,_,Epoch_n} = machi_chain_manager1:trigger_react_to_env(
|
||||
hd(ChMgrs)),
|
||||
true = (Epoch_n > Epoch_m),
|
||||
{ok, {false, EpochID3,_,_}} = WedgeStatus(hd(Ps)),
|
||||
{ok, {false, EpochID3}} = WedgeStatus(hd(Ps)),
|
||||
%% The file we're assigned should be different with the epoch change.
|
||||
{ok, {_,_,File3}} = Append(hd(Ps), EpochID3),
|
||||
true = (File1 /= File3),
|
||||
|
|
@ -176,19 +173,6 @@ partial_stop_restart2() ->
|
|||
ok
|
||||
end.
|
||||
|
||||
p_srvr_rec_test() ->
|
||||
P = #p_srvr{name=a, address="localhost", port=1024, props=[yo]},
|
||||
[P] = machi_flu_sup:sanitize_p_srvr_records([P]),
|
||||
[P] = machi_flu_sup:sanitize_p_srvr_records([P,P]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([nope]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{proto_mod=does_not_exist}]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{proto_mod="lists"}]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{address=7}]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{port=5}]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{port=foo}]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{props=foo}]),
|
||||
ok.
|
||||
|
||||
-endif. % !PULSE
|
||||
-endif. % TEST
|
||||
|
||||
|
|
|
|||
|
|
@ -1,307 +0,0 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_lifecycle_mgr_test).
|
||||
-compile(export_all).
|
||||
|
||||
-ifdef(TEST).
|
||||
-ifndef(PULSE).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-define(MGR, machi_chain_manager1).
|
||||
|
||||
setup() ->
|
||||
catch application:stop(machi),
|
||||
{ok, SupPid} = machi_sup:start_link(),
|
||||
error_logger:tty(false),
|
||||
Dir = "./" ++ atom_to_list(?MODULE) ++ ".datadir",
|
||||
machi_flu1_test:clean_up_data_dir(Dir ++ "/*/*"),
|
||||
machi_flu1_test:clean_up_data_dir(Dir),
|
||||
Envs = [{flu_data_dir, Dir ++ "/data/flu"},
|
||||
{flu_config_dir, Dir ++ "/etc/flu-config"},
|
||||
{chain_config_dir, Dir ++ "/etc/chain-config"},
|
||||
{platform_data_dir, Dir ++ "/data"},
|
||||
{platform_etc_dir, Dir ++ "/etc"},
|
||||
{not_used_pending, Dir ++ "/etc/pending"}
|
||||
],
|
||||
EnvKeys = [K || {K,_V} <- Envs],
|
||||
undefined = application:get_env(machi, yo),
|
||||
Cleanup = machi_flu1_test:get_env_vars(machi, EnvKeys ++ [yo]),
|
||||
[begin
|
||||
filelib:ensure_dir(V ++ "/unused"),
|
||||
application:set_env(machi, K, V)
|
||||
end || {K, V} <- Envs],
|
||||
{SupPid, Dir, Cleanup}.
|
||||
|
||||
cleanup({SupPid, Dir, Cleanup}) ->
|
||||
exit(SupPid, normal),
|
||||
machi_util:wait_for_death(SupPid, 100),
|
||||
error_logger:tty(true),
|
||||
catch application:stop(machi),
|
||||
machi_flu1_test:clean_up_data_dir(Dir ++ "/*/*"),
|
||||
machi_flu1_test:clean_up_data_dir(Dir),
|
||||
machi_flu1_test:clean_up_env_vars(Cleanup),
|
||||
undefined = application:get_env(machi, yo),
|
||||
ok.
|
||||
|
||||
smoke_test_() ->
|
||||
{timeout, 60, fun() -> smoke_test2() end}.
|
||||
|
||||
smoke_test2() ->
|
||||
YoCleanup = setup(),
|
||||
try
|
||||
Prefix = <<"pre">>,
|
||||
Chunk1 = <<"yochunk">>,
|
||||
Host = "localhost",
|
||||
PortBase = 60120,
|
||||
|
||||
Pa = #p_srvr{name=a,address="localhost",port=PortBase+0},
|
||||
Pb = #p_srvr{name=b,address="localhost",port=PortBase+1},
|
||||
Pc = #p_srvr{name=c,address="localhost",port=PortBase+2},
|
||||
%% Pstore_a = machi_flu1:make_projection_server_regname(a),
|
||||
%% Pstore_b = machi_flu1:make_projection_server_regname(b),
|
||||
%% Pstore_c = machi_flu1:make_projection_server_regname(c),
|
||||
Pstores = [Pstore_a, Pstore_b, Pstore_c] =
|
||||
[machi_flu1:make_projection_server_regname(a),
|
||||
machi_flu1:make_projection_server_regname(b),
|
||||
machi_flu1:make_projection_server_regname(c)],
|
||||
ChMgrs = [ChMgr_a, ChMgr_b, ChMgr_c] =
|
||||
[machi_chain_manager1:make_chmgr_regname(a),
|
||||
machi_chain_manager1:make_chmgr_regname(b),
|
||||
machi_chain_manager1:make_chmgr_regname(c)],
|
||||
Fits = [Fit_a, Fit_b, Fit_c] =
|
||||
[machi_flu_psup:make_fitness_regname(a),
|
||||
machi_flu_psup:make_fitness_regname(b),
|
||||
machi_flu_psup:make_fitness_regname(c)],
|
||||
Advance = machi_chain_manager1_test:make_advance_fun(
|
||||
Fits, [a,b,c], ChMgrs, 3),
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("\nSTEP: Start 3 FLUs, no chain.\n", []),
|
||||
|
||||
[machi_lifecycle_mgr:make_pending_config(P) || P <- [Pa,Pb,Pc] ],
|
||||
{[_,_,_],[]} = machi_lifecycle_mgr:process_pending(),
|
||||
[{ok, #projection_v1{epoch_number=0}} =
|
||||
machi_projection_store:read_latest_projection(PSTORE, private)
|
||||
|| PSTORE <- Pstores],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("\nSTEP: Start chain = [a,b,c]\n", []),
|
||||
|
||||
C1 = #chain_def_v1{name=cx, mode=ap_mode, full=[Pa,Pb,Pc],
|
||||
local_run=[a,b,c]},
|
||||
machi_lifecycle_mgr:make_pending_config(C1),
|
||||
{[],[_]} = machi_lifecycle_mgr:process_pending(),
|
||||
Advance(),
|
||||
[{ok, #projection_v1{all_members=[a,b,c]}} =
|
||||
machi_projection_store:read_latest_projection(PSTORE, private)
|
||||
|| PSTORE <- Pstores],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("\nSTEP: Reset chain = [b,c]\n", []),
|
||||
|
||||
C2 = #chain_def_v1{name=cx, mode=ap_mode, full=[Pb,Pc],
|
||||
old_full=[a,b,c], old_witnesses=[],
|
||||
local_stop=[a], local_run=[b,c]},
|
||||
machi_lifecycle_mgr:make_pending_config(C2),
|
||||
{[],[_]} = machi_lifecycle_mgr:process_pending(),
|
||||
Advance(),
|
||||
%% a should be down
|
||||
{'EXIT', _} = (catch machi_projection_store:read_latest_projection(
|
||||
hd(Pstores), private)),
|
||||
[{ok, #projection_v1{all_members=[b,c]}} =
|
||||
machi_projection_store:read_latest_projection(PSTORE, private)
|
||||
|| PSTORE <- tl(Pstores)],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("\nSTEP: Reset chain = []\n", []),
|
||||
|
||||
C3 = #chain_def_v1{name=cx, mode=ap_mode, full=[],
|
||||
old_full=[b,c], old_witnesses=[],
|
||||
local_stop=[b,c], local_run=[]},
|
||||
machi_lifecycle_mgr:make_pending_config(C3),
|
||||
{[],[_]} = machi_lifecycle_mgr:process_pending(),
|
||||
Advance(),
|
||||
%% a,b,c should be down
|
||||
[{'EXIT', _} = (catch machi_projection_store:read_latest_projection(
|
||||
PSTORE, private))
|
||||
|| PSTORE <- Pstores],
|
||||
|
||||
ok
|
||||
after
|
||||
cleanup(YoCleanup)
|
||||
end.
|
||||
|
||||
ast_tuple_syntax_test() ->
|
||||
T = fun(L) -> machi_lifecycle_mgr:check_ast_tuple_syntax(L) end,
|
||||
Canon1 = [ {host, "localhost", []},
|
||||
{host, "localhost", [{client_interface, "1.2.3.4"},
|
||||
{admin_interface, "5.6.7.8"}]},
|
||||
{flu, 'fx', "foohost", 4000, []},
|
||||
switch_old_and_new,
|
||||
{chain, 'cy', ['fx', 'fy'], [{foo,"yay"},{bar,baz}]} ],
|
||||
|
||||
{_Good,[]=_Bad} = T(Canon1),
|
||||
Canon1_norm = machi_lifecycle_mgr:normalize_ast_tuple_syntax(Canon1),
|
||||
true = (length(Canon1) == length(Canon1_norm)),
|
||||
{Canon1_norm_b, []} = T(Canon1_norm),
|
||||
true = (length(Canon1_norm) == length(Canon1_norm_b)),
|
||||
|
||||
{[],[_,_,_,_]} =
|
||||
T([ {host, 'localhost', []},
|
||||
{host, 'localhost', yo},
|
||||
{host, "localhost", [{client_interface, 77.88293829832}]},
|
||||
{host, "localhost", [{client_interface, "1.2.3.4"},
|
||||
{bummer, "5.6.7.8"}]} ]),
|
||||
{[],[_,_,_,_,_,_]} =
|
||||
T([ {flu, 'fx', 'foohost', 4000, []},
|
||||
{flu, 'fx', <<"foohost">>, 4000, []},
|
||||
{flu, 'fx', "foohost", -4000, []},
|
||||
{flu, 'fx', "foohost", 40009999, []},
|
||||
{flu, 'fx', "foohost", 4000, gack},
|
||||
{flu, 'fx', "foohost", 4000, [22]} ]),
|
||||
{[],[_,_,_]} =
|
||||
T([ {chain, 'cy', ["fx", "fy"], [foo,{bar,baz}]},
|
||||
yoloyolo,
|
||||
{chain, "cy", ["fx", 27], oops,arity,way,way,way,too,big,x}
|
||||
]).
|
||||
|
||||
ast_run_test() ->
|
||||
PortBase = 20300,
|
||||
R1 = [
|
||||
{host, "localhost", "localhost", "localhost", []},
|
||||
{flu, 'f0', "localhost", PortBase+0, []},
|
||||
{flu, 'f1', "localhost", PortBase+1, []},
|
||||
{chain, 'ca', ['f0'], []},
|
||||
{chain, 'cb', ['f1'], []},
|
||||
switch_old_and_new,
|
||||
{flu, 'f2', "localhost", PortBase+2, []},
|
||||
{flu, 'f3', "localhost", PortBase+3, []},
|
||||
{flu, 'f4', "localhost", PortBase+4, []},
|
||||
{chain, 'ca', ['f0', 'f2'], []},
|
||||
{chain, 'cc', ['f3', 'f4'], []}
|
||||
],
|
||||
|
||||
{ok, Env1} = machi_lifecycle_mgr:run_ast(R1),
|
||||
%% Uncomment to examine the Env trees.
|
||||
%% Y1 = {lists:sort(gb_trees:to_list(element(1, Env1))),
|
||||
%% lists:sort(gb_trees:to_list(element(2, Env1))),
|
||||
%% element(3, Env1)},
|
||||
%% io:format(user, "\nY1 ~p\n", [Y1]),
|
||||
|
||||
Negative_after_R1 =
|
||||
[
|
||||
{host, "localhost", "foo", "foo", []}, % dupe host
|
||||
{flu, 'f1', "other", PortBase+9999999, []}, % bogus port # (syntax)
|
||||
{flu, 'f1', "other", PortBase+888, []}, % dupe flu name
|
||||
{flu, 'f7', "localhost", PortBase+1, []}, % dupe host+port
|
||||
{chain, 'ca', ['f7'], []}, % unknown flu
|
||||
{chain, 'cc', ['f0'], []}, % flu previously assigned
|
||||
{chain, 'ca', cp_mode, ['f0', 'f1', 'f2'], [], []} % mode change
|
||||
],
|
||||
[begin
|
||||
%% io:format(user, "dbg: Neg ~p\n", [Neg]),
|
||||
{error, _} = machi_lifecycle_mgr:run_ast(R1 ++ [Neg])
|
||||
end || Neg <- Negative_after_R1],
|
||||
|
||||
%% The 'run' phase doesn't blow smoke. What about 'diff'?
|
||||
{X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, "localhost"),
|
||||
%% There's only one host, "localhost", so 'all' should be exactly equal.
|
||||
{X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, all),
|
||||
%% io:format(user, "X1b: ~p\n", [X1b]),
|
||||
|
||||
%% Append to the R1 scenario: for chain cc: add f5, remove f4
|
||||
%% Expect: see pattern matching below on X2b.
|
||||
R2 = (R1 -- [switch_old_and_new]) ++
|
||||
[switch_old_and_new,
|
||||
{flu, 'f5', "localhost", PortBase+5, []},
|
||||
{chain, 'cc', ['f3','f5'], []}],
|
||||
{ok, Env2} = machi_lifecycle_mgr:run_ast(R2),
|
||||
{_X2a, X2b} = machi_lifecycle_mgr:diff_env(Env2, "localhost"),
|
||||
%% io:format(user, "X2b: ~p\n", [X2b]),
|
||||
F5_port = PortBase+5,
|
||||
[#p_srvr{name='f5',address="localhost",port=F5_port},
|
||||
#chain_def_v1{name='cc',
|
||||
full=[#p_srvr{name='f3'},#p_srvr{name='f5'}], witnesses=[],
|
||||
old_full=[f3,f4], old_witnesses=[],
|
||||
local_run=[f5], local_stop=[f4]}] = X2b,
|
||||
|
||||
ok.
|
||||
|
||||
ast_then_apply_test_() ->
|
||||
{timeout, 60, fun() -> ast_then_apply_test2() end}.
|
||||
|
||||
ast_then_apply_test2() ->
|
||||
YoCleanup = setup(),
|
||||
try
|
||||
PortBase = 20400,
|
||||
NumChains = 4,
|
||||
ChainLen = 3,
|
||||
FLU_num = NumChains * ChainLen,
|
||||
FLU_defs = [{flu, list_to_atom("f"++integer_to_list(X)),
|
||||
"localhost", PortBase+X, []} || X <- lists:seq(1,FLU_num)],
|
||||
FLU_names = [FLU || {flu,FLU,_,_,_} <- FLU_defs],
|
||||
Ch_defs = [{chain, list_to_atom("c"++integer_to_list(X)),
|
||||
lists:sublist(FLU_names, X, 3),
|
||||
[]} || X <- lists:seq(1, FLU_num, 3)],
|
||||
|
||||
R1 = [switch_old_and_new,
|
||||
{host, "localhost", "localhost", "localhost", []}]
|
||||
++ FLU_defs ++ Ch_defs,
|
||||
{ok, Env1} = machi_lifecycle_mgr:run_ast(R1),
|
||||
{_X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, "localhost"),
|
||||
%% io:format(user, "X1b ~p\n", [X1b]),
|
||||
[machi_lifecycle_mgr:make_pending_config(X) || X <- X1b],
|
||||
{PassFLUs, PassChains} = machi_lifecycle_mgr:process_pending(),
|
||||
true = (length(PassFLUs) == length(FLU_defs)),
|
||||
true = (length(PassChains) == length(Ch_defs)),
|
||||
|
||||
%% Kick the chain managers into doing something useful right now.
|
||||
Pstores = [list_to_atom(atom_to_list(X) ++ "_pstore") || X <- FLU_names],
|
||||
Fits = [list_to_atom(atom_to_list(X) ++ "_fitness") || X <- FLU_names],
|
||||
ChMgrs = [list_to_atom(atom_to_list(X) ++ "_chmgr") || X <- FLU_names],
|
||||
Advance = machi_chain_manager1_test:make_advance_fun(
|
||||
Fits, FLU_names, ChMgrs, 3),
|
||||
Advance(),
|
||||
|
||||
%% Sanity check: everyone is configured properly.
|
||||
[begin
|
||||
{ok, #projection_v1{epoch_number=Epoch, all_members=All,
|
||||
chain_name=ChainName, upi=UPI}} =
|
||||
machi_projection_store:read_latest_projection(PStore, private),
|
||||
%% io:format(user, "~p: epoch ~p all ~p\n", [PStore, Epoch, All]),
|
||||
true = Epoch > 0,
|
||||
ChainLen = length(All),
|
||||
true = (length(UPI) > 0),
|
||||
{chain, _, Full, []} = lists:keyfind(ChainName, 2, Ch_defs),
|
||||
true = lists:sort(Full) == lists:sort(All)
|
||||
end || PStore <- Pstores],
|
||||
|
||||
ok
|
||||
after
|
||||
cleanup(YoCleanup)
|
||||
end.
|
||||
|
||||
-endif. % !PULSE
|
||||
-endif. % TEST
|
||||
|
|
@ -1,200 +0,0 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_merkle_tree_test).
|
||||
-compile([export_all]).
|
||||
|
||||
-include("machi_merkle_tree.hrl").
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-include_lib("kernel/include/file.hrl").
|
||||
|
||||
-define(GAP_CHANCE, 0.10).
|
||||
|
||||
%% unit tests
|
||||
basic_test() ->
|
||||
random:seed(os:timestamp()),
|
||||
Fsz = choose_size() * 1024,
|
||||
Filesize = max(Fsz, 10*1024*1024),
|
||||
ChunkSize = max(1048576, Filesize div 100),
|
||||
N = make_leaf_nodes(Filesize),
|
||||
D0 = #naive{ leaves = N, chunk_size = ChunkSize, recalc = true },
|
||||
T1 = machi_merkle_tree:build_tree(D0),
|
||||
|
||||
D1 = #naive{ leaves = tl(N), chunk_size = ChunkSize, recalc = true },
|
||||
T2 = machi_merkle_tree:build_tree(D1),
|
||||
|
||||
?assertNotEqual(T1#naive.root, T2#naive.root),
|
||||
?assertEqual(true, length(machi_merkle_tree:naive_diff(T1, T2)) == 1
|
||||
orelse
|
||||
Filesize > ChunkSize).
|
||||
|
||||
|
||||
make_leaf_nodes(Filesize) ->
|
||||
lists:reverse(
|
||||
lists:foldl(fun(T, Acc) -> machi_merkle_tree:update_acc(T, Acc) end,
|
||||
[],
|
||||
generate_offsets(Filesize, 1024, []))
|
||||
).
|
||||
|
||||
choose_int(Factor) ->
|
||||
random:uniform(1024*Factor).
|
||||
|
||||
small_int() ->
|
||||
choose_int(10).
|
||||
|
||||
medium_int() ->
|
||||
choose_int(1024).
|
||||
|
||||
large_int() ->
|
||||
choose_int(4096).
|
||||
|
||||
generate_offsets(Filesize, Current, Acc) when Current < Filesize ->
|
||||
Length0 = choose_size(),
|
||||
|
||||
Length = case Length0 + Current > Filesize of
|
||||
false -> Length0;
|
||||
true -> Filesize - Current
|
||||
end,
|
||||
Data = term_to_binary(os:timestamp()),
|
||||
Checksum = machi_util:make_tagged_csum(client_sha, machi_util:checksum_chunk(Data)),
|
||||
Gap = maybe_gap(random:uniform()),
|
||||
generate_offsets(Filesize, Current + Length + Gap, [ {Current, Length, Checksum} | Acc ]);
|
||||
generate_offsets(_Filesize, _Current, Acc) ->
|
||||
lists:reverse(Acc).
|
||||
|
||||
|
||||
random_from_list(L) ->
|
||||
N = random:uniform(length(L)),
|
||||
lists:nth(N, L).
|
||||
|
||||
choose_size() ->
|
||||
F = random_from_list([fun small_int/0, fun medium_int/0, fun large_int/0]),
|
||||
F().
|
||||
|
||||
maybe_gap(Chance) when Chance < ?GAP_CHANCE ->
|
||||
choose_size();
|
||||
maybe_gap(_) -> 0.
|
||||
|
||||
%% Define or remove these ifdefs if benchmarking is desired.
|
||||
-ifdef(BENCH).
|
||||
generate_offsets(FH, Filesize, Current, Acc) when Current < Filesize ->
|
||||
Length0 = choose_size(),
|
||||
|
||||
Length = case Length0 + Current > Filesize of
|
||||
false -> Length0;
|
||||
true -> Filesize - Current
|
||||
end,
|
||||
{ok, Data} = file:pread(FH, Current, Length),
|
||||
Checksum = machi_util:make_tagged_csum(client_sha, machi_util:checksum_chunk(Data)),
|
||||
Gap = maybe_gap(random:uniform()),
|
||||
generate_offsets(FH, Filesize, Current + Length + Gap, [ {Current, Length, Checksum} | Acc ]);
|
||||
generate_offsets(_FH, _Filesize, _Current, Acc) ->
|
||||
lists:reverse(Acc).
|
||||
|
||||
make_offsets_from_file(Filename) ->
|
||||
{ok, Info} = file:read_file_info(Filename),
|
||||
Filesize = Info#file_info.size,
|
||||
{ok, FH} = file:open(Filename, [read, raw, binary]),
|
||||
Offsets = generate_offsets(FH, Filesize, 1024, []),
|
||||
file:close(FH),
|
||||
Offsets.
|
||||
|
||||
choose_filename() ->
|
||||
random_from_list([
|
||||
"def^c5ea7511-d649-47d6-a8c3-2b619379c237^1",
|
||||
"jkl^b077eff7-b2be-4773-a73f-fea4acb8a732^1",
|
||||
"stu^553fa47a-157c-4fac-b10f-2252c7d8c37a^1",
|
||||
"vwx^ae015d68-7689-4c9f-9677-926c6664f513^1",
|
||||
"yza^4c784dc2-19bf-4ac6-91f6-58bbe5aa88e0^1"
|
||||
]).
|
||||
|
||||
|
||||
make_csum_file(DataDir, Filename, Offsets) ->
|
||||
Path = machi_util:make_checksum_filename(DataDir, Filename),
|
||||
filelib:ensure_dir(Path),
|
||||
{ok, MC} = machi_csum_table:open(Path, []),
|
||||
lists:foreach(fun({Offset, Size, Checksum}) ->
|
||||
machi_csum_table:write(MC, Offset, Size, Checksum) end,
|
||||
Offsets),
|
||||
machi_csum_table:close(MC).
|
||||
|
||||
|
||||
test() ->
|
||||
test(100).
|
||||
|
||||
test(N) ->
|
||||
{ok, F} = file:open("results.txt", [raw, write]),
|
||||
lists:foreach(fun(X) -> format_and_store(F, run_test(X)) end, lists:seq(1, N)).
|
||||
|
||||
format_and_store(F, {OffsetNum, {MTime, MSize}, {NTime, NSize}}) ->
|
||||
S = io_lib:format("~w\t~w\t~w\t~w\t~w\n", [OffsetNum, MTime, MSize, NTime, NSize]),
|
||||
ok = file:write(F, S).
|
||||
|
||||
run_test(C) ->
|
||||
random:seed(os:timestamp()),
|
||||
OffsetFn = "test/" ++ choose_filename(),
|
||||
O = make_offsets_from_file(OffsetFn),
|
||||
Fn = "csum_" ++ integer_to_list(C),
|
||||
make_csum_file(".", Fn, O),
|
||||
|
||||
Osize = length(O),
|
||||
|
||||
{MTime, {ok, M}} = timer:tc(fun() -> machi_merkle_tree:open(Fn, ".", merklet) end),
|
||||
{NTime, {ok, N}} = timer:tc(fun() -> machi_merkle_tree:open(Fn, ".", naive) end),
|
||||
|
||||
?assertEqual(Fn, machi_merkle_tree:filename(M)),
|
||||
?assertEqual(Fn, machi_merkle_tree:filename(N)),
|
||||
|
||||
MTree = machi_merkle_tree:tree(M),
|
||||
MSize = byte_size(term_to_binary(MTree)),
|
||||
|
||||
NTree = machi_merkle_tree:tree(N),
|
||||
NSize = byte_size(term_to_binary(NTree)),
|
||||
|
||||
?assertEqual(same, machi_merkle_tree:diff(N, N)),
|
||||
?assertEqual(same, machi_merkle_tree:diff(M, M)),
|
||||
{Osize, {MTime, MSize}, {NTime, NSize}}.
|
||||
|
||||
torture_test(C) ->
|
||||
Results = [ run_torture_test() || _ <- lists:seq(1, C) ],
|
||||
{ok, F} = file:open("torture_results.txt", [raw, write]),
|
||||
lists:foreach(fun({MSize, MTime, NSize, NTime}) ->
|
||||
file:write(F, io_lib:format("~p\t~p\t~p\t~p\n",
|
||||
[MSize, MTime, NSize, NTime]))
|
||||
end, Results),
|
||||
ok = file:close(F).
|
||||
|
||||
run_torture_test() ->
|
||||
{NTime, N} = timer:tc(fun() -> naive_torture() end),
|
||||
|
||||
MSize = byte_size(term_to_binary(M)),
|
||||
NSize = byte_size(term_to_binary(N)),
|
||||
|
||||
{MSize, MTime, NSize, NTime}.
|
||||
|
||||
naive_torture() ->
|
||||
N = lists:foldl(fun(T, Acc) -> machi_merkle_tree:update_acc(T, Acc) end, [], torture_generator()),
|
||||
T = #naive{ leaves = lists:reverse(N), chunk_size = 10010, recalc = true },
|
||||
machi_merkle_tree:build_tree(T).
|
||||
|
||||
torture_generator() ->
|
||||
[ {O, 1, crypto:hash(sha, term_to_binary(now()))} || O <- lists:seq(1024, 1000000) ].
|
||||
-endif. % BENCH
|
||||
|
|
@ -24,7 +24,6 @@
|
|||
-ifdef(TEST).
|
||||
-ifndef(PULSE).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_pb.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
|
@ -35,15 +34,21 @@ smoke_test_() ->
|
|||
{timeout, 5*60, fun() -> smoke_test2() end}.
|
||||
|
||||
smoke_test2() ->
|
||||
PortBase = 5720,
|
||||
Port = 5720,
|
||||
Ps = [#p_srvr{name=a, address="localhost", port=Port, props="./data.a"}
|
||||
],
|
||||
D = orddict:from_list([{P#p_srvr.name, P} || P <- Ps]),
|
||||
ok = application:set_env(machi, max_file_size, 1024*1024),
|
||||
|
||||
[os:cmd("rm -rf " ++ P#p_srvr.props) || P <- Ps],
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
try
|
||||
{Ps, MgrNames, Dirs} = machi_test_util:start_flu_packages(
|
||||
1, PortBase, "./data.", []),
|
||||
D = orddict:from_list([{P#p_srvr.name, P} || P <- Ps]),
|
||||
M0 = hd(MgrNames),
|
||||
ok = machi_chain_manager1:set_chain_members(M0, D),
|
||||
[machi_chain_manager1:trigger_react_to_env(M0) || _ <-lists:seq(1,5)],
|
||||
[begin
|
||||
#p_srvr{name=Name, port=Port, props=Dir} = P,
|
||||
{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, [])
|
||||
end || P <- Ps],
|
||||
ok = machi_chain_manager1:set_chain_members(a_chmgr, D),
|
||||
[machi_chain_manager1:trigger_react_to_env(a_chmgr) || _ <-lists:seq(1,5)],
|
||||
|
||||
{ok, Clnt} = ?C:start_link(Ps),
|
||||
try
|
||||
|
|
@ -56,18 +61,16 @@ smoke_test2() ->
|
|||
%% a separate test module? Or separate test func?
|
||||
{error, _} = ?C:auth(Clnt, "foo", "bar"),
|
||||
|
||||
PK = <<>>,
|
||||
Prefix = <<"prefix">>,
|
||||
Chunk1 = <<"Hello, chunk!">>,
|
||||
NS = "",
|
||||
NoCSum = <<>>,
|
||||
Opts1 = #append_opts{},
|
||||
{ok, {Off1, Size1, File1}} =
|
||||
?C:append_chunk(Clnt, NS, Prefix, Chunk1, NoCSum, Opts1),
|
||||
?C:append_chunk(Clnt, PK, Prefix, Chunk1, none, 0),
|
||||
true = is_binary(File1),
|
||||
Chunk2 = "It's another chunk",
|
||||
CSum2 = {client_sha, machi_util:checksum_chunk(Chunk2)},
|
||||
{ok, {Off2, Size2, File2}} =
|
||||
?C:append_chunk(Clnt, NS, Prefix, Chunk2, CSum2, Opts1),
|
||||
?C:append_chunk(Clnt, PK, Prefix, Chunk2, CSum2, 1024),
|
||||
Chunk3 = ["This is a ", <<"test,">>, 32, [["Hello, world!"]]],
|
||||
File3 = File2,
|
||||
Off3 = Off2 + iolist_size(Chunk2),
|
||||
|
|
@ -78,9 +81,9 @@ smoke_test2() ->
|
|||
{iolist_to_binary(Chunk2), File2, Off2, Size2},
|
||||
{iolist_to_binary(Chunk3), File3, Off3, Size3}],
|
||||
[begin
|
||||
File = Fl,
|
||||
File = binary_to_list(Fl),
|
||||
?assertMatch({ok, {[{File, Off, Ch, _}], []}},
|
||||
?C:read_chunk(Clnt, Fl, Off, Sz, undefined))
|
||||
?C:read_chunk(Clnt, Fl, Off, Sz, []))
|
||||
end || {Ch, Fl, Off, Sz} <- Reads],
|
||||
|
||||
{ok, KludgeBin} = ?C:checksum_list(Clnt, File1),
|
||||
|
|
@ -90,8 +93,7 @@ smoke_test2() ->
|
|||
|
||||
File1Bin = binary_to_list(File1),
|
||||
[begin
|
||||
#p_srvr{name=Name, props=Props} = P,
|
||||
Dir = proplists:get_value(data_dir, Props),
|
||||
#p_srvr{name=Name, port=Port, props=Dir} = P,
|
||||
?assertEqual({ok, [File1Bin]},
|
||||
file:list_dir(filename:join([Dir, "data"]))),
|
||||
FileListFileName = filename:join([Dir, "known_files_" ++ atom_to_list(Name)]),
|
||||
|
|
@ -104,23 +106,21 @@ smoke_test2() ->
|
|||
end || {_Ch, Fl, Off, Sz} <- Reads],
|
||||
[begin
|
||||
{ok, {[], Trimmed}} =
|
||||
?C:read_chunk(Clnt, Fl, Off, Sz, #read_opts{needs_trimmed=true}),
|
||||
Filename = Fl,
|
||||
?C:read_chunk(Clnt, Fl, Off, Sz, [{needs_trimmed, true}]),
|
||||
Filename = binary_to_list(Fl),
|
||||
?assertEqual([{Filename, Off, Sz}], Trimmed)
|
||||
end || {_Ch, Fl, Off, Sz} <- Reads],
|
||||
|
||||
LargeBytes = binary:copy(<<"x">>, 1024*1024),
|
||||
LBCsum = {client_sha, machi_util:checksum_chunk(LargeBytes)},
|
||||
{ok, {Offx, Sizex, Filex}} =
|
||||
?C:append_chunk(Clnt, NS,
|
||||
Prefix, LargeBytes, LBCsum, Opts1),
|
||||
?C:append_chunk(Clnt, PK, Prefix, LargeBytes, LBCsum, 0),
|
||||
ok = ?C:trim_chunk(Clnt, Filex, Offx, Sizex),
|
||||
|
||||
%% Make sure everything was trimmed
|
||||
File = binary_to_list(Filex),
|
||||
[begin
|
||||
#p_srvr{name=Name, props=Props} = P,
|
||||
Dir = proplists:get_value(data_dir, Props),
|
||||
#p_srvr{name=Name, port=_Port, props=Dir} = P,
|
||||
?assertEqual({ok, []},
|
||||
file:list_dir(filename:join([Dir, "data"]))),
|
||||
FileListFileName = filename:join([Dir, "known_files_" ++ atom_to_list(Name)]),
|
||||
|
|
@ -130,14 +130,17 @@ smoke_test2() ->
|
|||
|
||||
[begin
|
||||
{error, trimmed} =
|
||||
?C:read_chunk(Clnt, Fl, Off, Sz, undefined)
|
||||
?C:read_chunk(Clnt, Fl, Off, Sz, [])
|
||||
end || {_Ch, Fl, Off, Sz} <- Reads],
|
||||
ok
|
||||
after
|
||||
(catch ?C:quit(Clnt))
|
||||
end
|
||||
after
|
||||
machi_test_util:stop_flu_packages()
|
||||
exit(SupPid, normal),
|
||||
[os:cmd("rm -rf " ++ P#p_srvr.props) || P <- Ps],
|
||||
machi_util:wait_for_death(SupPid, 100),
|
||||
ok
|
||||
end.
|
||||
|
||||
-endif. % !PULSE
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ smoke_test() ->
|
|||
Dir = "./data.a",
|
||||
Os = [{ignore_stability_time, true}, {active_mode, false}],
|
||||
os:cmd("rm -rf " ++ Dir),
|
||||
machi_test_util:start_flu_package(a, PortBase, "./data.a", Os),
|
||||
machi_flu1_test:start_flu_package(a, PortBase, "./data.a", Os),
|
||||
|
||||
try
|
||||
P1 = machi_projection:new(1, a, [], [], [], [], []),
|
||||
|
|
@ -58,7 +58,7 @@ smoke_test() ->
|
|||
|
||||
ok
|
||||
after
|
||||
machi_test_util:stop_flu_package()
|
||||
machi_flu1_test:stop_flu_package(a)
|
||||
end.
|
||||
|
||||
-endif. % !PULSE
|
||||
|
|
|
|||
|
|
@ -32,55 +32,48 @@
|
|||
|
||||
api_smoke_test() ->
|
||||
RegName = api_smoke_flu,
|
||||
TcpPort = 17124,
|
||||
Host = "localhost",
|
||||
TcpPort = 57124,
|
||||
DataDir = "./data.api_smoke_flu",
|
||||
W_props = [{active_mode, false},{initial_wedged, false}],
|
||||
Prefix = <<"prefix">>,
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
|
||||
machi_flu1_test:start_flu_package(RegName, TcpPort, DataDir, W_props),
|
||||
|
||||
try
|
||||
{[I], _, _} = machi_test_util:start_flu_package(
|
||||
RegName, TcpPort, DataDir, W_props),
|
||||
I = #p_srvr{name=RegName, address=Host, port=TcpPort},
|
||||
{ok, Prox1} = ?MUT:start_link(I),
|
||||
try
|
||||
FakeEpoch = ?DUMMY_PV1_EPOCH,
|
||||
[{ok, {_,_,_}} = ?MUT:append_chunk(
|
||||
Prox1, NSInfo, FakeEpoch,
|
||||
Prefix, <<"data">>, NoCSum) ||
|
||||
_ <- lists:seq(1,5)],
|
||||
[{ok, {_,_,_}} = ?MUT:append_chunk(Prox1,
|
||||
FakeEpoch, Prefix, <<"data">>,
|
||||
infinity) || _ <- lists:seq(1,5)],
|
||||
%% Stop the FLU, what happens?
|
||||
machi_test_util:stop_flu_package(),
|
||||
[{error,partition} = ?MUT:append_chunk(Prox1, NSInfo,
|
||||
machi_flu1_test:stop_flu_package(RegName),
|
||||
[{error,partition} = ?MUT:append_chunk(Prox1,
|
||||
FakeEpoch, Prefix, <<"data-stopped1">>,
|
||||
NoCSum) || _ <- lists:seq(1,3)],
|
||||
infinity) || _ <- lists:seq(1,3)],
|
||||
%% Start the FLU again, we should be able to do stuff immediately
|
||||
machi_test_util:start_flu_package(RegName, TcpPort, DataDir,
|
||||
[no_cleanup|W_props]),
|
||||
machi_flu1_test:start_flu_package(RegName, TcpPort, DataDir,
|
||||
[save_data_dir|W_props]),
|
||||
MyChunk = <<"my chunk data">>,
|
||||
{ok, {MyOff,MySize,MyFile}} =
|
||||
?MUT:append_chunk(Prox1, NSInfo, FakeEpoch, Prefix, MyChunk,
|
||||
NoCSum),
|
||||
{ok, {[{_, MyOff, MyChunk, _MyChunkCSUM}], []}} =
|
||||
?MUT:read_chunk(Prox1, NSInfo, FakeEpoch, MyFile, MyOff, MySize, undefined),
|
||||
MyChunk2_parts = [<<"my chunk ">>, "data", <<", yeah, again">>],
|
||||
MyChunk2 = iolist_to_binary(MyChunk2_parts),
|
||||
Opts1 = #append_opts{chunk_extra=4242},
|
||||
?MUT:append_chunk(Prox1, FakeEpoch, Prefix, MyChunk,
|
||||
infinity),
|
||||
{ok, {[{_, MyOff, MyChunk, _}], []}} =
|
||||
?MUT:read_chunk(Prox1, FakeEpoch, MyFile, MyOff, MySize, []),
|
||||
MyChunk2 = <<"my chunk data, yeah, again">>,
|
||||
{ok, {MyOff2,MySize2,MyFile2}} =
|
||||
?MUT:append_chunk(Prox1, NSInfo, FakeEpoch, Prefix,
|
||||
MyChunk2_parts, NoCSum, Opts1, infinity),
|
||||
[{ok, {[{_, MyOff2, MyChunk2, _}], []}} =
|
||||
?MUT:read_chunk(Prox1, NSInfo, FakeEpoch, MyFile2, MyOff2, MySize2, DefaultOptions) ||
|
||||
DefaultOptions <- [undefined, noopt, none, any_atom_at_all] ],
|
||||
|
||||
BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, "...................")},
|
||||
{error, bad_checksum} = ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
Prefix, MyChunk, BadCSum),
|
||||
{error, bad_checksum} = ?MUT:write_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
MyFile2,
|
||||
MyOff2 + size(MyChunk2),
|
||||
MyChunk, BadCSum,
|
||||
infinity),
|
||||
?MUT:append_chunk_extra(Prox1, FakeEpoch, Prefix,
|
||||
MyChunk2, 4242, infinity),
|
||||
{ok, {[{_, MyOff2, MyChunk2, _}], []}} =
|
||||
?MUT:read_chunk(Prox1, FakeEpoch, MyFile2, MyOff2, MySize2, []),
|
||||
MyChunk_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, MyChunk},
|
||||
{error, bad_checksum} = ?MUT:append_chunk(Prox1, FakeEpoch,
|
||||
Prefix, MyChunk_badcs),
|
||||
{error, bad_checksum} = ?MUT:write_chunk(Prox1, FakeEpoch,
|
||||
<<"foo-file^1^1">>, 99832,
|
||||
MyChunk_badcs),
|
||||
|
||||
%% Put kick_projection_reaction() in the middle of the test so
|
||||
%% that any problems with its async nature will (hopefully)
|
||||
|
|
@ -89,9 +82,9 @@ api_smoke_test() ->
|
|||
|
||||
%% Alright, now for the rest of the API, whee
|
||||
BadFile = <<"no-such-file">>,
|
||||
{error, bad_arg} = ?MUT:checksum_list(Prox1, BadFile),
|
||||
{error, bad_arg} = ?MUT:checksum_list(Prox1, FakeEpoch, BadFile),
|
||||
{ok, [_|_]} = ?MUT:list_files(Prox1, FakeEpoch),
|
||||
{ok, {false, _,_,_}} = ?MUT:wedge_status(Prox1),
|
||||
{ok, {false, _}} = ?MUT:wedge_status(Prox1),
|
||||
{ok, {0, _SomeCSum}} = ?MUT:get_latest_epochid(Prox1, public),
|
||||
{ok, #projection_v1{epoch_number=0}} =
|
||||
?MUT:read_latest_projection(Prox1, public),
|
||||
|
|
@ -109,7 +102,7 @@ api_smoke_test() ->
|
|||
_ = (catch ?MUT:quit(Prox1))
|
||||
end
|
||||
after
|
||||
(catch machi_test_util:stop_flu_package())
|
||||
(catch machi_flu1_test:stop_flu_package(RegName))
|
||||
end.
|
||||
|
||||
flu_restart_test_() ->
|
||||
|
|
@ -117,22 +110,22 @@ flu_restart_test_() ->
|
|||
|
||||
flu_restart_test2() ->
|
||||
RegName = a,
|
||||
TcpPort = 17125,
|
||||
Host = "localhost",
|
||||
TcpPort = 57125,
|
||||
DataDir = "./data.api_smoke_flu2",
|
||||
W_props = [{initial_wedged, false}, {active_mode, false}],
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
machi_flu1_test:start_flu_package(RegName, TcpPort, DataDir, W_props),
|
||||
|
||||
try
|
||||
{[I], _, _} = machi_test_util:start_flu_package(
|
||||
RegName, TcpPort, DataDir, W_props),
|
||||
I = #p_srvr{name=RegName, address=Host, port=TcpPort},
|
||||
{ok, Prox1} = ?MUT:start_link(I),
|
||||
try
|
||||
FakeEpoch = ?DUMMY_PV1_EPOCH,
|
||||
Data = <<"data!">>,
|
||||
Dataxx = <<"Fake!">>,
|
||||
{ok, {Off1,Size1,File1}} = ?MUT:append_chunk(Prox1, NSInfo,
|
||||
FakeEpoch, <<"prefix">>, Data, NoCSum),
|
||||
{ok, {Off1,Size1,File1}} = ?MUT:append_chunk(Prox1,
|
||||
FakeEpoch, <<"prefix">>, Data,
|
||||
infinity),
|
||||
P_a = #p_srvr{name=a, address="localhost", port=6622},
|
||||
P1 = machi_projection:new(1, RegName, [P_a], [], [RegName], [], []),
|
||||
P1xx = P1#projection_v1{dbg2=["dbg2 changes are ok"]},
|
||||
|
|
@ -144,7 +137,7 @@ flu_restart_test2() ->
|
|||
{ok, EpochID} = ?MUT:get_epoch_id(Prox1),
|
||||
{ok, EpochID} = ?MUT:get_latest_epochid(Prox1, public),
|
||||
{ok, EpochID} = ?MUT:get_latest_epochid(Prox1, private),
|
||||
ok = machi_test_util:stop_flu_package(), timer:sleep(50),
|
||||
ok = machi_flu1_test:stop_flu_package(RegName), timer:sleep(50),
|
||||
|
||||
%% Now that the last proxy op was successful and only
|
||||
%% after did we stop the FLU, let's check that both the
|
||||
|
|
@ -156,10 +149,9 @@ flu_restart_test2() ->
|
|||
%% makes the code a bit convoluted. (No LFE or
|
||||
%% Elixir macros here, alas, they'd be useful.)
|
||||
|
||||
AppendOpts1 = #append_opts{chunk_extra=42},
|
||||
ExpectedOps =
|
||||
[
|
||||
fun(run) -> ?assertEqual({ok, EpochID}, ?MUT:get_epoch_id(Prox1)),
|
||||
fun(run) -> {ok, EpochID} = ?MUT:get_epoch_id(Prox1),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:get_epoch_id(Prox1) end,
|
||||
|
|
@ -238,37 +230,35 @@ flu_restart_test2() ->
|
|||
(stop) -> ?MUT:get_all_projections(Prox1, private)
|
||||
end,
|
||||
fun(run) -> {ok, {_,_,_}} =
|
||||
?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
<<"prefix">>, Data, NoCSum),
|
||||
?MUT:append_chunk(Prox1, FakeEpoch,
|
||||
<<"prefix">>, Data, infinity),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
<<"prefix">>, Data, NoCSum)
|
||||
(stop) -> ?MUT:append_chunk(Prox1, FakeEpoch,
|
||||
<<"prefix">>, Data, infinity)
|
||||
end,
|
||||
fun(run) -> {ok, {_,_,_}} =
|
||||
?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
<<"prefix">>, Data, NoCSum,
|
||||
AppendOpts1, infinity),
|
||||
?MUT:append_chunk_extra(Prox1, FakeEpoch,
|
||||
<<"prefix">>, Data, 42, infinity),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
<<"prefix">>, Data, NoCSum,
|
||||
AppendOpts1, infinity)
|
||||
(stop) -> ?MUT:append_chunk_extra(Prox1, FakeEpoch,
|
||||
<<"prefix">>, Data, 42, infinity)
|
||||
end,
|
||||
fun(run) -> {ok, {[{_, Off1, Data, _}], []}} =
|
||||
?MUT:read_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
File1, Off1, Size1, undefined),
|
||||
?MUT:read_chunk(Prox1, FakeEpoch,
|
||||
File1, Off1, Size1, []),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:read_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
File1, Off1, Size1, undefined)
|
||||
(stop) -> ?MUT:read_chunk(Prox1, FakeEpoch,
|
||||
File1, Off1, Size1, [])
|
||||
end,
|
||||
fun(run) -> {ok, KludgeBin} =
|
||||
?MUT:checksum_list(Prox1, File1),
|
||||
?MUT:checksum_list(Prox1, FakeEpoch, File1),
|
||||
true = is_binary(KludgeBin),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:checksum_list(Prox1, File1)
|
||||
(stop) -> ?MUT:checksum_list(Prox1, FakeEpoch, File1)
|
||||
end,
|
||||
fun(run) -> {ok, _} =
|
||||
?MUT:list_files(Prox1, FakeEpoch),
|
||||
|
|
@ -284,32 +274,32 @@ flu_restart_test2() ->
|
|||
end,
|
||||
fun(run) ->
|
||||
ok =
|
||||
?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
|
||||
Data, NoCSum, infinity),
|
||||
?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
|
||||
Data, infinity),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
|
||||
Data, NoCSum, infinity)
|
||||
(stop) -> ?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
|
||||
Data, infinity)
|
||||
end,
|
||||
fun(run) ->
|
||||
{error, written} =
|
||||
?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
|
||||
Dataxx, NoCSum, infinity),
|
||||
?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
|
||||
Dataxx, infinity),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
|
||||
Dataxx, NoCSum, infinity)
|
||||
(stop) -> ?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
|
||||
Dataxx, infinity)
|
||||
end
|
||||
],
|
||||
|
||||
[begin
|
||||
machi_test_util:start_flu_package(
|
||||
machi_flu1_test:start_flu_package(
|
||||
RegName, TcpPort, DataDir,
|
||||
[no_cleanup|W_props]),
|
||||
[save_data_dir|W_props]),
|
||||
_ = Fun(line),
|
||||
ok = Fun(run),
|
||||
ok = Fun(run),
|
||||
ok = machi_test_util:stop_flu_package(),
|
||||
ok = machi_flu1_test:stop_flu_package(RegName),
|
||||
{error, partition} = Fun(stop),
|
||||
{error, partition} = Fun(stop),
|
||||
ok
|
||||
|
|
@ -319,8 +309,8 @@ flu_restart_test2() ->
|
|||
_ = (catch ?MUT:quit(Prox1))
|
||||
end
|
||||
after
|
||||
(catch machi_test_util:stop_flu_package())
|
||||
(catch machi_flu1_test:stop_flu_package(RegName))
|
||||
end.
|
||||
|
||||
|
||||
-endif. % !PULSE
|
||||
-endif. % TEST
|
||||
|
|
|
|||
|
|
@ -1,111 +0,0 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_test_util).
|
||||
-compile(export_all).
|
||||
|
||||
-ifdef(TEST).
|
||||
-ifndef(PULSE).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-define(FLU, machi_flu1).
|
||||
-define(FLU_C, machi_flu1_client).
|
||||
|
||||
-spec start_flu_package(atom(), inet:port_number(), string()) ->
|
||||
{Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}.
|
||||
start_flu_package(FluName, TcpPort, DataDir) ->
|
||||
start_flu_package(FluName, TcpPort, DataDir, []).
|
||||
|
||||
-spec start_flu_package(atom(), inet:port_number(), string(), list()) ->
|
||||
{Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}.
|
||||
start_flu_package(FluName, TcpPort, DataDir, Props) ->
|
||||
MgrName = machi_flu_psup:make_mgr_supname(FluName),
|
||||
FluInfo = [{#p_srvr{name=FluName, address="localhost", port=TcpPort,
|
||||
props=[{chmgr, MgrName}, {data_dir, DataDir} | Props]},
|
||||
DataDir, MgrName}],
|
||||
start_flu_packages(FluInfo).
|
||||
|
||||
-spec start_flu_packages(pos_integer(), inet:port_number(), string(), list()) ->
|
||||
{Ps::[#p_srvr{}], MgrNames::[atom()], Dirs::[string()]}.
|
||||
start_flu_packages(FluCount, BaseTcpPort, DirPrefix, Props) ->
|
||||
FluInfo = flu_info(FluCount, BaseTcpPort, DirPrefix, Props),
|
||||
start_flu_packages(FluInfo).
|
||||
|
||||
start_flu_packages(FluInfo) ->
|
||||
_ = stop_machi_sup(),
|
||||
clean_up(FluInfo),
|
||||
{ok, _SupPid} = machi_sup:start_link(),
|
||||
[{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, Props) ||
|
||||
{#p_srvr{name=Name, port=Port, props=Props}, Dir, _} <- FluInfo],
|
||||
{Ps, Dirs, MgrNames} = lists:unzip3(FluInfo),
|
||||
{Ps, MgrNames, Dirs}.
|
||||
|
||||
stop_flu_package() ->
|
||||
stop_flu_packages().
|
||||
|
||||
stop_flu_packages() ->
|
||||
stop_machi_sup().
|
||||
|
||||
flu_info(FluCount, BaseTcpPort, DirPrefix, Props) ->
|
||||
[begin
|
||||
FLUNameStr = [$a + I - 1],
|
||||
FLUName = list_to_atom(FLUNameStr),
|
||||
MgrName = machi_flu_psup:make_mgr_supname(FLUName),
|
||||
DataDir = DirPrefix ++ "/data.eqc." ++ FLUNameStr,
|
||||
{#p_srvr{name=FLUName, address="localhost", port=BaseTcpPort + I,
|
||||
props=[{chmgr, MgrName}, {data_dir, DataDir} | Props]},
|
||||
DataDir, MgrName}
|
||||
end || I <- lists:seq(1, FluCount)].
|
||||
|
||||
stop_machi_sup() ->
|
||||
case whereis(machi_sup) of
|
||||
undefined -> ok;
|
||||
Pid ->
|
||||
catch exit(whereis(machi_sup), normal),
|
||||
machi_util:wait_for_death(Pid, 100)
|
||||
end.
|
||||
|
||||
clean_up(FluInfo) ->
|
||||
_ = [begin
|
||||
case proplists:get_value(no_cleanup, Props) of
|
||||
true -> ok;
|
||||
_ ->
|
||||
_ = machi_flu1:stop(FLUName),
|
||||
clean_up_dir(Dir)
|
||||
end
|
||||
end || {#p_srvr{name=FLUName, props=Props}, Dir, _} <- FluInfo],
|
||||
ok.
|
||||
|
||||
clean_up_dir(Dir) ->
|
||||
[begin
|
||||
Fs = filelib:wildcard(Dir ++ Glob),
|
||||
[file:delete(F) || F <- Fs],
|
||||
[file:del_dir(F) || F <- Fs]
|
||||
end || Glob <- ["*/*/*/*", "*/*/*", "*/*", "*"] ],
|
||||
_ = file:del_dir(Dir),
|
||||
ok.
|
||||
|
||||
-endif. % !PULSE
|
||||
-endif. % TEST
|
||||
|
||||
16
tools.mk
16
tools.mk
|
|
@ -64,18 +64,10 @@ ERL_LIB_DIR = $(shell erl -eval '{io:format("~s\n", [code:lib_dir()]), erlang:ha
|
|||
native-ebin:
|
||||
mkdir -p $(NATIVE_EBIN)
|
||||
rm -f $(NATIVE_EBIN)/*.erl $(NATIVE_EBIN)/*.hrl $(NATIVE_EBIN)/*.beam
|
||||
@for mod in lists dict digraph digraph_utils ets gb_sets gb_trees ordsets sets sofs; do \
|
||||
cp $(ERL_LIB_DIR)/stdlib-*/src/"$$mod".erl $(NATIVE_EBIN); \
|
||||
done
|
||||
@for mod in cerl cerl_trees core_parse; do \
|
||||
cp $(ERL_LIB_DIR)/compiler-*/src/"$$mod".?rl $(NATIVE_EBIN); \
|
||||
done
|
||||
@for mod in dialyzer_analysis_callgraph dialyzer dialyzer_behaviours dialyzer_codeserver dialyzer_contracts dialyzer_coordinator dialyzer_dataflow dialyzer_dep dialyzer_plt dialyzer_succ_typings dialyzer_typesig dialyzer_worker; do \
|
||||
cp $(ERL_LIB_DIR)/dialyzer-*/src/"$$mod".?rl $(NATIVE_EBIN); \
|
||||
done
|
||||
@for mod in erl_types erl_bif_types; do \
|
||||
cp $(ERL_LIB_DIR)/hipe-*/*/"$$mod".?rl $(NATIVE_EBIN); \
|
||||
done
|
||||
cp $(ERL_LIB_DIR)/stdlib-*/src/{lists,dict,digraph,digraph_utils,ets,gb_sets,gb_trees,ordsets,sets,sofs}.erl $(NATIVE_EBIN)
|
||||
cp $(ERL_LIB_DIR)/compiler-*/src/{cerl,cerl_trees,core_parse}.?rl $(NATIVE_EBIN)
|
||||
cp $(ERL_LIB_DIR)/dialyzer-*/src/{dialyzer_analysis_callgraph,dialyzer,dialyzer_behaviours,dialyzer_codeserver,dialyzer_contracts,dialyzer_coordinator,dialyzer_dataflow,dialyzer_dep,dialyzer_plt,dialyzer_succ_typings,dialyzer_typesig,dialyzer_worker}.?rl $(NATIVE_EBIN)
|
||||
cp $(ERL_LIB_DIR)/hipe-*/*/{erl_types,erl_bif_types}.?rl $(NATIVE_EBIN)
|
||||
erlc -o $(NATIVE_EBIN) -smp +native -DVSN='"$(DIALYZER_VERSION)"' $(NATIVE_EBIN)/*erl
|
||||
|
||||
${PLT}: compile
|
||||
|
|
|
|||
Loading…
Reference in a new issue