Compare commits
354 commits
slf/cp-mod
...
master
Author | SHA1 | Date | |
---|---|---|---|
|
e87bd59a97 | ||
|
1e0bb4c404 | ||
|
549963545f | ||
|
2aa8917875 | ||
|
27cbf1e38c | ||
|
d2fa79e037 | ||
|
57ba204210 | ||
|
24f8cb21a2 | ||
|
e63db8dedc | ||
|
a739b5265c | ||
|
767f5d9e60 | ||
|
8c21539fcb | ||
|
0f24b69378 | ||
|
ec9d391047 | ||
|
fa71a918b8 | ||
|
6cddfcf988 | ||
|
6b000f6e7c | ||
|
96c46ec5aa | ||
|
cd166361aa | ||
|
4e5c16f5e2 | ||
|
16153a5d31 | ||
|
84f522f865 | ||
|
fc46cd1b25 | ||
|
184a54ebbd | ||
|
4cb166368a | ||
|
f433e84fab | ||
|
a3fbe2c8bb | ||
|
bdf47da10c | ||
|
6c03f5c1a6 | ||
|
11921d82bf | ||
|
a27425147d | ||
|
34f8632f19 | ||
|
c02a0bed70 | ||
|
1d8bc19891 | ||
|
53ce6d89dd | ||
|
2e46d199c8 | ||
|
0f543b4c4d | ||
|
d5c3da78fb | ||
|
affad6b1d3 | ||
|
ed56a2c6cf | ||
|
c2e9a83372 | ||
|
67dad7fb8a | ||
|
9d4483ae68 | ||
|
12ebf4390d | ||
|
b246ebc376 | ||
|
943e23e050 | ||
|
ecfad4726b | ||
|
7c39af5bb7 | ||
|
3bd575899f | ||
|
a7f42d636e | ||
|
fbb0203f67 | ||
|
6e17988ac7 | ||
|
202ace33d3 | ||
|
2fddf2ec2d | ||
|
3b82dc2e38 | ||
|
3b594504fe | ||
|
a3fc1c3d68 | ||
|
f09eef14eb | ||
|
c65424569d | ||
|
3c6f1be5d0 | ||
|
76ae4247cd | ||
|
e24acb7246 | ||
|
5a65a164c3 | ||
|
0a8c4156c2 | ||
|
3d730ea215 | ||
|
6089ee6851 | ||
|
2932a17ea6 | ||
|
03b118b52c | ||
|
546901ef49 | ||
|
de5d5e88dd | ||
|
70d42a3fb5 | ||
|
7d262fd3ec | ||
|
b8297afc18 | ||
|
c49ccafdc6 | ||
|
d602663060 | ||
|
0922def0d6 | ||
|
bb0e67f6e0 | ||
|
51a05ba770 | ||
|
1d1bfadb96 | ||
|
f98b4da45b | ||
|
dffb73330d | ||
|
3a0086afb2 | ||
|
dcb2464cb8 | ||
|
463d20a9fd | ||
|
b8b3e872e4 | ||
|
9bd885ccb4 | ||
|
ec56164bd1 | ||
|
e5f7f3ba9a | ||
|
d196dbcee5 | ||
|
e4a784d3dd | ||
|
6f077fbb62 | ||
|
e55115fdba | ||
|
009bad230f | ||
|
6b7d871ada | ||
|
1db232db1b | ||
|
3826af8ee2 | ||
|
df8eea8c10 | ||
|
61eae1300f | ||
|
436c308db2 | ||
|
3ee3de1aaf | ||
|
6a5c590ad1 | ||
|
c37f23d97a | ||
|
9cec53eea6 | ||
|
eef00e4f8f | ||
|
9472bad37b | ||
|
cb706f0d23 | ||
|
61ef7739cd | ||
|
b0a9e65ca2 | ||
|
95e2df304e | ||
|
7f25fcc8f8 | ||
|
cd9bf9eeab | ||
|
2e2d282afc | ||
|
2871f8397c | ||
|
65eec61f82 | ||
|
7301c8308e | ||
|
b243a9b863 | ||
|
f23e500993 | ||
|
ade4430d30 | ||
|
aa0a0413d1 | ||
|
14765a7279 | ||
|
a1f5a6ce62 | ||
|
7614910f36 | ||
|
9579b1b8b2 | ||
|
a8785e44b1 | ||
|
83b4466a85 | ||
|
69280bfb4f | ||
|
0fc7bc74b7 | ||
|
8285899dba | ||
|
37ac09a680 | ||
|
3391c89818 | ||
|
e27a59e20f | ||
|
27e8a31307 | ||
|
ef10ebed22 | ||
|
16acda3c7e | ||
|
1bc9033076 | ||
|
38e63e8181 | ||
|
293eb4810f | ||
|
5aeaf872d9 | ||
|
89e80a8862 | ||
|
07c2b97918 | ||
|
1d3d121d83 | ||
|
3c880dc437 | ||
|
befa776685 | ||
|
a7ffef6b8e | ||
|
8528567954 | ||
|
cf0829b934 | ||
|
35c48300a5 | ||
|
66de92490c | ||
|
e9b1134cd9 | ||
|
37f33fae7b | ||
|
d44e9dd542 | ||
|
0d517d2377 | ||
|
2f95305292 | ||
|
29c2ede275 | ||
|
916ac754d7 | ||
|
5477f3b6f8 | ||
|
a1c834518d | ||
|
c3002d9852 | ||
|
10a27ce7dd | ||
|
014ba89e3a | ||
|
ac10f97220 | ||
|
1f56850c2b | ||
|
a5a0369905 | ||
|
4ce7a87d56 | ||
|
d5c56980b3 | ||
|
fd649c00d2 | ||
|
9f6b53fc15 | ||
|
e11cdfe95c | ||
|
8d3f631d84 | ||
|
84058f8c9c | ||
|
ad419ada50 | ||
|
049311614f | ||
|
3a35fe38c8 | ||
|
73890171ba | ||
|
919a408e17 | ||
|
6786820401 | ||
|
81fae32539 | ||
|
2e6e6dd9e8 | ||
|
ce41f9005e | ||
|
922baaf433 | ||
|
9e4dc83f2a | ||
|
1b0711f151 | ||
|
39a937db9b | ||
|
f56037240e | ||
|
d0e6417f5d | ||
|
3f6f9e2c6b | ||
|
c1e5426034 | ||
|
3b087c0388 | ||
|
62c8dacc65 | ||
|
b956f9e1f0 | ||
|
850a8786b6 | ||
|
557525af05 | ||
|
30000d6602 | ||
|
059f591d3f | ||
|
3122f2cf54 | ||
|
3c5a9e6f53 | ||
|
72a4fab49d | ||
|
b5005c3526 | ||
|
6fa2de28cd | ||
|
93b168415d | ||
|
bf5768eb47 | ||
|
447c8c8d48 | ||
|
b859e23a37 | ||
|
44497d5af8 | ||
|
611f33e81b | ||
|
028135d927 | ||
|
7086899941 | ||
|
170b3cd797 | ||
|
f7358424e4 | ||
|
5e571f6009 | ||
|
7f561f34e0 | ||
|
77096c5f82 | ||
|
b710517c64 | ||
|
1b8401e7de | ||
|
48fabdcd5f | ||
|
61f02dfc9f | ||
|
5ec35773e3 | ||
|
d2b1c7512a | ||
|
cfaed63fa7 | ||
|
b500d5f449 | ||
|
dd17b1de0a | ||
|
7377624579 | ||
|
6e3347f727 | ||
|
8a61055f55 | ||
|
d59c1fae31 | ||
|
39ac71048b | ||
|
9fb19aa8ee | ||
|
bbbd9748f3 | ||
|
b2eb3e089c | ||
|
5913531e32 | ||
|
60364fe0ca | ||
|
0e4ae818af | ||
|
3d6d4d8be3 | ||
|
c5661571e3 | ||
|
5fd225bcdb | ||
|
0f688d6279 | ||
|
41bd8fa64e | ||
|
51c97da133 | ||
|
49b4b1c304 | ||
|
058de6dc9c | ||
|
a0588cbaed | ||
|
3751ca14b5 | ||
|
3bb5ffa50e | ||
|
3d3d26b3af | ||
|
224a293c03 | ||
|
8a51230760 | ||
|
b8c5d21876 | ||
|
5008cbd2d0 | ||
|
9c31139b62 | ||
|
b3b24b1178 | ||
|
5122ee00a9 | ||
|
0c4c42cc52 | ||
|
d2ac5b0583 | ||
|
028ddc79ff | ||
|
edce9d6463 | ||
|
bd91167093 | ||
|
595f9a463e | ||
|
177aca0a68 | ||
|
478107915b | ||
|
30d7e592a3 | ||
|
79e0ae9fe5 | ||
|
1c8e436a64 | ||
|
9d177c6b54 | ||
|
976a701e0c | ||
|
981b55c070 | ||
|
a43397a7b8 | ||
|
84f9ccc4f5 | ||
|
ebb9bc3f5a | ||
|
a369f0666a | ||
|
1193fb8510 | ||
|
6f9814ffb4 | ||
|
bf61b2900b | ||
|
ecd7eb195a | ||
|
980d646d64 | ||
|
3e975f53b8 | ||
|
208c02853f | ||
|
c407ee23f2 | ||
|
96c037e799 | ||
|
6961930b0f | ||
|
8ab9d3ee50 | ||
|
1485fa416f | ||
|
39774bc70f | ||
|
19d935051f | ||
|
f04e7f7132 | ||
|
ae7eba9dcf | ||
|
5f953bc5dd | ||
|
cf9d56e9bf | ||
|
cb67764273 | ||
|
aac302a289 | ||
|
de364d3a99 | ||
|
00ac0f4cd3 | ||
|
29eccd5bee | ||
|
de68973e46 | ||
|
bc3c08cf37 | ||
|
a6a49ca0fb | ||
|
c48513d131 | ||
|
bc45b8e37c | ||
|
299016cafb | ||
|
ab6c23a335 | ||
|
6f790527f5 | ||
|
58d2a43b37 | ||
|
e45469b5ce | ||
|
baeffbab0b | ||
|
e344ee42ff | ||
|
d6a3180ecd | ||
|
71cbf019f5 | ||
|
7439a2738d | ||
|
07ceff095a | ||
|
8eb9cc9700 | ||
|
ed112bfb52 | ||
|
6dbf52db6f | ||
|
8cd41a7bf2 | ||
|
63612c8823 | ||
|
1b612bd969 | ||
|
ec9682520a | ||
|
fe71b72494 | ||
|
f8707c61c0 | ||
|
161e6cd9f9 | ||
|
85e1e5a26d | ||
|
e113f6ffdd | ||
|
dfe953b7d8 | ||
|
2724960eaf | ||
|
5131ebdd16 | ||
|
777909b0f5 | ||
|
cbf773215e | ||
|
8a8c4dcede | ||
|
f3e6d46e36 | ||
|
da0b331936 | ||
|
855f94925c | ||
|
8187e01fe0 | ||
|
289b2bcc7c | ||
|
5926cef44a | ||
|
d9ede473dd | ||
|
c1b9038447 | ||
|
aca3759e45 | ||
|
1ecbb5cffe | ||
|
303aad97e9 | ||
|
4d0019f141 | ||
|
952d2fa508 | ||
|
2bfc199294 | ||
|
7912e77e9e | ||
|
679046600f | ||
|
ed5dec1cd6 | ||
|
796937fe75 | ||
|
0054445f13 | ||
|
d627f238bf | ||
|
f83b0973f2 | ||
|
7a6999465a | ||
|
2d0c03ef35 | ||
|
36c11e7d08 | ||
|
d3fe7ee181 | ||
|
d7daf203fb | ||
|
3fb3890788 | ||
|
6d5b61f747 |
102 changed files with 12496 additions and 3858 deletions
17
.gitignore
vendored
17
.gitignore
vendored
|
@ -2,17 +2,32 @@ prototype/chain-manager/patch.*
|
|||
.eqc-info
|
||||
.eunit
|
||||
deps
|
||||
*.plt
|
||||
dev
|
||||
erl_crash.dump
|
||||
eqc
|
||||
.concrete/DEV_MODE
|
||||
.rebar
|
||||
edoc
|
||||
|
||||
# Dialyzer stuff
|
||||
.dialyzer-last-run.txt
|
||||
.ebin.native
|
||||
.local_dialyzer_plt
|
||||
dialyzer_unhandled_warnings
|
||||
dialyzer_warnings
|
||||
*.plt
|
||||
|
||||
# PB artifacts for Erlang
|
||||
include/machi_pb.hrl
|
||||
|
||||
# Release packaging
|
||||
rel/machi
|
||||
rel/vars/dev*vars.config
|
||||
|
||||
# Misc Scott cruft
|
||||
*.patch
|
||||
current_counterexample.eqc
|
||||
foo*
|
||||
RUNLOG*
|
||||
typescript*
|
||||
*.swp
|
||||
|
|
7
.travis.yml
Normal file
7
.travis.yml
Normal file
|
@ -0,0 +1,7 @@
|
|||
language: erlang
|
||||
notifications:
|
||||
email: scott@basho.com
|
||||
script: "priv/test-for-gh-pr.sh"
|
||||
otp_release:
|
||||
- 17.5
|
||||
## No, Dialyzer is too different between 17 & 18: - 18.1
|
230
FAQ.md
230
FAQ.md
|
@ -11,14 +11,14 @@
|
|||
|
||||
+ [1 Questions about Machi in general](#n1)
|
||||
+ [1.1 What is Machi?](#n1.1)
|
||||
+ [1.2 What is a Machi "cluster of clusters"?](#n1.2)
|
||||
+ [1.2.1 This "cluster of clusters" idea needs a better name, don't you agree?](#n1.2.1)
|
||||
+ [1.3 What is Machi like when operating in "eventually consistent"/"AP mode"?](#n1.3)
|
||||
+ [1.4 What is Machi like when operating in "strongly consistent"/"CP mode"?](#n1.4)
|
||||
+ [1.5 What does Machi's API look like?](#n1.5)
|
||||
+ [1.6 What licensing terms are used by Machi?](#n1.6)
|
||||
+ [1.7 Where can I find the Machi source code and documentation? Can I contribute?](#n1.7)
|
||||
+ [1.8 What is Machi's expected release schedule, packaging, and operating system/OS distribution support?](#n1.8)
|
||||
+ [1.2 What is a Machi chain?](#n1.2)
|
||||
+ [1.3 What is a Machi cluster?](#n1.3)
|
||||
+ [1.4 What is Machi like when operating in "eventually consistent" mode?](#n1.4)
|
||||
+ [1.5 What is Machi like when operating in "strongly consistent" mode?](#n1.5)
|
||||
+ [1.6 What does Machi's API look like?](#n1.6)
|
||||
+ [1.7 What licensing terms are used by Machi?](#n1.7)
|
||||
+ [1.8 Where can I find the Machi source code and documentation? Can I contribute?](#n1.8)
|
||||
+ [1.9 What is Machi's expected release schedule, packaging, and operating system/OS distribution support?](#n1.9)
|
||||
+ [2 Questions about Machi relative to {{something else}}](#n2)
|
||||
+ [2.1 How is Machi better than Hadoop?](#n2.1)
|
||||
+ [2.2 How does Machi differ from HadoopFS/HDFS?](#n2.2)
|
||||
|
@ -28,13 +28,15 @@
|
|||
+ [3 Machi's specifics](#n3)
|
||||
+ [3.1 What technique is used to replicate Machi's files? Can other techniques be used?](#n3.1)
|
||||
+ [3.2 Does Machi have a reliance on a coordination service such as ZooKeeper or etcd?](#n3.2)
|
||||
+ [3.3 Is it true that there's an allegory written to describe humming consensus?](#n3.3)
|
||||
+ [3.4 How is Machi tested?](#n3.4)
|
||||
+ [3.5 Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks](#n3.5)
|
||||
+ [3.6 Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?](#n3.6)
|
||||
+ [3.7 What language(s) is Machi written in?](#n3.7)
|
||||
+ [3.8 Does Machi use the Erlang/OTP network distribution system (aka "disterl")?](#n3.8)
|
||||
+ [3.9 Can I use HTTP to write/read stuff into/from Machi?](#n3.9)
|
||||
+ [3.3 Are there any presentations available about Humming Consensus](#n3.3)
|
||||
+ [3.4 Is it true that there's an allegory written to describe Humming Consensus?](#n3.4)
|
||||
+ [3.5 How is Machi tested?](#n3.5)
|
||||
+ [3.6 Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks](#n3.6)
|
||||
+ [3.7 Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?](#n3.7)
|
||||
+ [3.8 What language(s) is Machi written in?](#n3.8)
|
||||
+ [3.9 Can Machi run on Windows? Can Machi run on 32-bit platforms?](#n3.9)
|
||||
+ [3.10 Does Machi use the Erlang/OTP network distribution system (aka "disterl")?](#n3.10)
|
||||
+ [3.11 Can I use HTTP to write/read stuff into/from Machi?](#n3.11)
|
||||
|
||||
<!-- ENDOUTLINE -->
|
||||
|
||||
|
@ -44,13 +46,13 @@
|
|||
<a name="n1.1">
|
||||
### 1.1. What is Machi?
|
||||
|
||||
Very briefly, Machi is a very simple append-only file store.
|
||||
Very briefly, Machi is a very simple append-only blob/file store.
|
||||
|
||||
Machi is
|
||||
"dumber" than many other file stores (i.e., lacking many features
|
||||
found in other file stores) such as HadoopFS or simple NFS or CIFS file
|
||||
found in other file stores) such as HadoopFS or a simple NFS or CIFS file
|
||||
server.
|
||||
However, Machi is a distributed file store, which makes it different
|
||||
However, Machi is a distributed blob/file store, which makes it different
|
||||
(and, in some ways, more complicated) than a simple NFS or CIFS file
|
||||
server.
|
||||
|
||||
|
@ -82,45 +84,39 @@ For a much longer answer, please see the
|
|||
[Machi high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-machi.pdf).
|
||||
|
||||
<a name="n1.2">
|
||||
### 1.2. What is a Machi "cluster of clusters"?
|
||||
### 1.2. What is a Machi chain?
|
||||
|
||||
Machi's design is based on using small, well-understood and provable
|
||||
(mathematically) techniques to maintain multiple file copies without
|
||||
data loss or data corruption. At its lowest level, Machi contains no
|
||||
support for distribution/partitioning/sharding of files across many
|
||||
servers. A typical, fully-functional Machi cluster will likely be two
|
||||
or three machines.
|
||||
A Machi chain is a small number of machines that maintain a common set
|
||||
of replicated files. A typical chain is of length 2 or 3. For
|
||||
critical data that must be available despite several simultaneous
|
||||
server failures, a chain length of 6 or 7 might be used.
|
||||
|
||||
However, Machi is designed to be an excellent building block for
|
||||
building larger systems. A deployment of Machi "cluster of clusters"
|
||||
will use the "random slicing" technique for partitioning files across
|
||||
multiple Machi clusters that, as individuals, are unaware of the
|
||||
larger cluster-of-clusters scheme.
|
||||
<a name="n1.3">
|
||||
### 1.3. What is a Machi cluster?
|
||||
|
||||
The cluster-of-clusters management service will be fully decentralized
|
||||
A Machi cluster is a collection of Machi chains that
|
||||
partitions/shards/distributes files (based on file name) across the
|
||||
collection of chains. Machi uses the "random slicing" algorithm (a
|
||||
variation of consistent hashing) to define the mapping of file name to
|
||||
chain name.
|
||||
|
||||
The cluster management service will be fully decentralized
|
||||
and run as a separate software service installed on each Machi
|
||||
cluster. This manager will appear to the local Machi server as simply
|
||||
another Machi file client. The cluster-of-clusters managers will take
|
||||
another Machi file client. The cluster managers will take
|
||||
care of file migration as the cluster grows and shrinks in capacity
|
||||
and in response to day-to-day changes in workload.
|
||||
|
||||
Though the cluster-of-clusters manager has not yet been implemented,
|
||||
Though the cluster manager has not yet been implemented,
|
||||
its design is fully decentralized and capable of operating despite
|
||||
multiple partial failure of its member clusters. We expect this
|
||||
multiple partial failure of its member chains. We expect this
|
||||
design to scale easily to at least one thousand servers.
|
||||
|
||||
Please see the
|
||||
[Machi source repository's 'doc' directory for more details](https://github.com/basho/machi/tree/master/doc/).
|
||||
|
||||
<a name="n1.2.1">
|
||||
#### 1.2.1. This "cluster of clusters" idea needs a better name, don't you agree?
|
||||
|
||||
Yes. Please help us: we are bad at naming things.
|
||||
For proof that naming things is hard, see
|
||||
[http://martinfowler.com/bliki/TwoHardThings.html](http://martinfowler.com/bliki/TwoHardThings.html)
|
||||
|
||||
<a name="n1.3">
|
||||
### 1.3. What is Machi like when operating in "eventually consistent"/"AP mode"?
|
||||
<a name="n1.4">
|
||||
### 1.4. What is Machi like when operating in "eventually consistent" mode?
|
||||
|
||||
Machi's operating mode dictates how a Machi cluster will react to
|
||||
network partitions. A network partition may be caused by:
|
||||
|
@ -130,37 +126,30 @@ network partitions. A network partition may be caused by:
|
|||
* An extreme server software "hang" or "pause", e.g. caused by OS
|
||||
scheduling problems such as a failing/stuttering disk device.
|
||||
|
||||
"AP mode" refers to the "A" and "P" properties of the "CAP
|
||||
conjecture", meaning that the cluster will be "Available" and
|
||||
"Partition tolerant".
|
||||
|
||||
The consistency semantics of file operations while in "AP mode" are
|
||||
eventually consistent during and after network partitions:
|
||||
The consistency semantics of file operations while in eventual
|
||||
consistency mode during and after network partitions are:
|
||||
|
||||
* File write operations are permitted by any client on the "same side"
|
||||
of the network partition.
|
||||
* File read operations are successful for any file contents where the
|
||||
client & server are on the "same side" of the network partition.
|
||||
* File read operations will probably fail for any file contents where the
|
||||
client & server are on "different sides" of the network partition.
|
||||
* After the network partition(s) is resolved, files are merged
|
||||
together from "all sides" of the partition(s).
|
||||
* Unique files are copied in their entirety.
|
||||
* Byte ranges within the same file are merged. This is possible
|
||||
due to Machi's restrictions on file naming (files names are
|
||||
alwoys assigned by Machi servers) and file offset assignments
|
||||
(byte offsets are also always chosen by Machi servers according
|
||||
to rules which guarantee safe mergeability.).
|
||||
due to Machi's restrictions on file naming and file offset
|
||||
assignment. Both file names and file offsets are always chosen
|
||||
by Machi servers according to rules which guarantee safe
|
||||
mergeability. Server-assigned names are a characteristic of a
|
||||
"blob store".
|
||||
|
||||
<a name="n1.4">
|
||||
### 1.4. What is Machi like when operating in "strongly consistent"/"CP mode"?
|
||||
<a name="n1.5">
|
||||
### 1.5. What is Machi like when operating in "strongly consistent" mode?
|
||||
|
||||
Machi's operating mode dictates how a Machi cluster will react to
|
||||
network partitions.
|
||||
"CP mode" refers to the "C" and "P" properties of the "CAP
|
||||
conjecture", meaning that the cluster will be "Consistent" and
|
||||
"Partition tolerant".
|
||||
|
||||
The consistency semantics of file operations while in "CP mode" are
|
||||
strongly consistent during and after network partitions:
|
||||
The consistency semantics of file operations while in strongly
|
||||
consistency mode during and after network partitions are:
|
||||
|
||||
* File write operations are permitted by any client on the "same side"
|
||||
of the network partition if and only if a quorum majority of Machi servers
|
||||
|
@ -175,19 +164,19 @@ strongly consistent during and after network partitions:
|
|||
|
||||
Machi's design can provide the illusion of quorum minority write
|
||||
availability if the cluster is configured to operate with "witness
|
||||
servers". (This feaure is not implemented yet, as of June 2015.)
|
||||
servers". (This feaure partially implemented, as of December 2015.)
|
||||
See Section 11 of
|
||||
[Machi chain manager high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-chain-mgr.pdf)
|
||||
for more details.
|
||||
|
||||
<a name="n1.5">
|
||||
### 1.5. What does Machi's API look like?
|
||||
<a name="n1.6">
|
||||
### 1.6. What does Machi's API look like?
|
||||
|
||||
The Machi API only contains a handful of API operations. The function
|
||||
arguments shown below use Erlang-style type annotations.
|
||||
arguments shown below (in simplifed form) use Erlang-style type annotations.
|
||||
|
||||
append_chunk(Prefix:binary(), Chunk:binary()).
|
||||
append_chunk_extra(Prefix:binary(), Chunk:binary(), ExtraSpace:non_neg_integer()).
|
||||
append_chunk(Prefix:binary(), Chunk:binary(), CheckSum:binary()).
|
||||
append_chunk_extra(Prefix:binary(), Chunk:binary(), CheckSum:binary(), ExtraSpace:non_neg_integer()).
|
||||
read_chunk(File:binary(), Offset:non_neg_integer(), Size:non_neg_integer()).
|
||||
|
||||
checksum_list(File:binary()).
|
||||
|
@ -205,17 +194,22 @@ Internally, there is a more complex protocol used by individual
|
|||
cluster members to manage file contents and to repair damaged/missing
|
||||
files. See Figure 3 in
|
||||
[Machi high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-machi.pdf)
|
||||
for more details.
|
||||
for more description.
|
||||
|
||||
<a name="n1.6">
|
||||
### 1.6. What licensing terms are used by Machi?
|
||||
The definitions of both the "high level" external protocol and "low
|
||||
level" internal protocol are in a
|
||||
[Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview)
|
||||
definition at [./src/machi.proto](./src/machi.proto).
|
||||
|
||||
<a name="n1.7">
|
||||
### 1.7. What licensing terms are used by Machi?
|
||||
|
||||
All Machi source code and documentation is licensed by
|
||||
[Basho Technologies, Inc.](http://www.basho.com/)
|
||||
under the [Apache Public License version 2](https://github.com/basho/machi/tree/master/LICENSE).
|
||||
|
||||
<a name="n1.7">
|
||||
### 1.7. Where can I find the Machi source code and documentation? Can I contribute?
|
||||
<a name="n1.8">
|
||||
### 1.8. Where can I find the Machi source code and documentation? Can I contribute?
|
||||
|
||||
All Machi source code and documentation can be found at GitHub:
|
||||
[https://github.com/basho/machi](https://github.com/basho/machi).
|
||||
|
@ -229,11 +223,11 @@ ideas for improvement, please see our contributing & collaboration
|
|||
guidelines at
|
||||
[https://github.com/basho/machi/blob/master/CONTRIBUTING.md](https://github.com/basho/machi/blob/master/CONTRIBUTING.md).
|
||||
|
||||
<a name="n1.8">
|
||||
### 1.8. What is Machi's expected release schedule, packaging, and operating system/OS distribution support?
|
||||
<a name="n1.9">
|
||||
### 1.9. What is Machi's expected release schedule, packaging, and operating system/OS distribution support?
|
||||
|
||||
Basho expects that Machi's first release will take place near the end
|
||||
of calendar year 2015.
|
||||
Basho expects that Machi's first major product release will take place
|
||||
during the 2nd quarter of 2016.
|
||||
|
||||
Basho's official support for operating systems (e.g. Linux, FreeBSD),
|
||||
operating system packaging (e.g. CentOS rpm/yum package management,
|
||||
|
@ -308,15 +302,15 @@ file's writable phase).
|
|||
|
||||
<tr>
|
||||
<td> Does not have any file distribution/partitioning/sharding across
|
||||
Machi clusters: in a single Machi cluster, all files are replicated by
|
||||
all servers in the cluster. The "cluster of clusters" concept is used
|
||||
Machi chains: in a single Machi chain, all files are replicated by
|
||||
all servers in the chain. The "random slicing" technique is used
|
||||
to distribute/partition/shard files across multiple Machi clusters.
|
||||
<td> File distribution/partitioning/sharding is performed
|
||||
automatically by the HDFS "name node".
|
||||
|
||||
<tr>
|
||||
<td> Machi requires no central "name node" for single cluster use.
|
||||
Machi requires no central "name node" for "cluster of clusters" use
|
||||
<td> Machi requires no central "name node" for single chain use or
|
||||
for multi-chain cluster use.
|
||||
<td> Requires a single "namenode" server to maintain file system contents
|
||||
and file content mapping. (May be deployed with a "secondary
|
||||
namenode" to reduce unavailability when the primary namenode fails.)
|
||||
|
@ -482,8 +476,8 @@ difficult to adapt to Machi's design goals:
|
|||
* Both protocols use quorum majority consensus, which requires a
|
||||
minimum of *2F + 1* working servers to tolerate *F* failures. For
|
||||
example, to tolerate 2 server failures, quorum majority protocols
|
||||
require a minium of 5 servers. To tolerate the same number of
|
||||
failures, Chain replication requires only 3 servers.
|
||||
require a minimum of 5 servers. To tolerate the same number of
|
||||
failures, Chain Replication requires a minimum of only 3 servers.
|
||||
* Machi's use of "humming consensus" to manage internal server
|
||||
metadata state would also (probably) require conversion to Paxos or
|
||||
Raft. (Or "outsourced" to a service such as ZooKeeper.)
|
||||
|
@ -500,7 +494,17 @@ Humming consensus is described in the
|
|||
[Machi chain manager high level design doc](https://github.com/basho/machi/tree/master/doc/high-level-chain-mgr.pdf).
|
||||
|
||||
<a name="n3.3">
|
||||
### 3.3. Is it true that there's an allegory written to describe humming consensus?
|
||||
### 3.3. Are there any presentations available about Humming Consensus
|
||||
|
||||
Scott recently (November 2015) gave a presentation at the
|
||||
[RICON 2015 conference](http://ricon.io) about one of the techniques
|
||||
used by Machi; "Managing Chain Replication Metadata with
|
||||
Humming Consensus" is available online now.
|
||||
* [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf)
|
||||
* [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q)
|
||||
|
||||
<a name="n3.4">
|
||||
### 3.4. Is it true that there's an allegory written to describe Humming Consensus?
|
||||
|
||||
Yes. In homage to Leslie Lamport's original paper about the Paxos
|
||||
protocol, "The Part-time Parliamant", there is an allegorical story
|
||||
|
@ -511,8 +515,8 @@ The full story, full of wonder and mystery, is called
|
|||
There is also a
|
||||
[short followup blog posting](http://www.snookles.com/slf-blog/2015/03/20/on-humming-consensus-an-allegory-part-2/).
|
||||
|
||||
<a name="n3.4">
|
||||
### 3.4. How is Machi tested?
|
||||
<a name="n3.5">
|
||||
### 3.5. How is Machi tested?
|
||||
|
||||
While not formally proven yet, Machi's implementation of Chain
|
||||
Replication and of humming consensus have been extensively tested with
|
||||
|
@ -537,16 +541,20 @@ change several times during any single test case) and a random series
|
|||
of cluster operations, an event trace of all cluster activity is used
|
||||
to verify that no safety-critical rules have been violated.
|
||||
|
||||
<a name="n3.5">
|
||||
### 3.5. Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks
|
||||
All test code is available in the [./test](./test) subdirectory.
|
||||
Modules that use QuickCheck will use a file suffix of `_eqc`, for
|
||||
example, [./test/machi_ap_repair_eqc.erl](./test/machi_ap_repair_eqc.erl).
|
||||
|
||||
<a name="n3.6">
|
||||
### 3.6. Does Machi require shared disk storage? e.g. iSCSI, NBD (Network Block Device), Fibre Channel disks
|
||||
|
||||
No, Machi's design assumes that each Machi server is a fully
|
||||
independent hardware and assumes only standard local disks (Winchester
|
||||
and/or SSD style) with local-only interfaces (e.g. SATA, SCSI, PCI) in
|
||||
each machine.
|
||||
|
||||
<a name="n3.6">
|
||||
### 3.6. Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?
|
||||
<a name="n3.7">
|
||||
### 3.7. Does Machi require or assume that servers with large numbers of disks must use RAID-0/1/5/6/10/50/60 to create a single block device?
|
||||
|
||||
No. When used with servers with multiple disks, the intent is to
|
||||
deploy multiple Machi servers per machine: one Machi server per disk.
|
||||
|
@ -564,10 +572,13 @@ deploy multiple Machi servers per machine: one Machi server per disk.
|
|||
placement relative to 12 servers is smaller than a placement problem
|
||||
of managing 264 seprate disks (if each of 12 servers has 22 disks).
|
||||
|
||||
<a name="n3.7">
|
||||
### 3.7. What language(s) is Machi written in?
|
||||
<a name="n3.8">
|
||||
### 3.8. What language(s) is Machi written in?
|
||||
|
||||
So far, Machi is written in 100% Erlang.
|
||||
So far, Machi is written in Erlang, mostly. Machi uses at least one
|
||||
library, [ELevelDB](https://github.com/basho/eleveldb), that is
|
||||
implemented both in C++ and in Erlang, using Erlang NIFs (Native
|
||||
Interface Functions) to allow Erlang code to call C++ functions.
|
||||
|
||||
In the event that we encounter a performance problem that cannot be
|
||||
solved within the Erlang/OTP runtime environment, all of Machi's
|
||||
|
@ -576,8 +587,16 @@ in C, Java, or other "gotta go fast fast FAST!!" programming
|
|||
language. We expect that the Chain Replication manager and other
|
||||
critical "control plane" software will remain in Erlang.
|
||||
|
||||
<a name="n3.8">
|
||||
### 3.8. Does Machi use the Erlang/OTP network distribution system (aka "disterl")?
|
||||
<a name="n3.9">
|
||||
### 3.9. Can Machi run on Windows? Can Machi run on 32-bit platforms?
|
||||
|
||||
The ELevelDB NIF does not compile or run correctly on Erlang/OTP
|
||||
Windows platforms, nor does it compile correctly on 32-bit platforms.
|
||||
Machi should support all 64-bit UNIX-like platforms that are supported
|
||||
by Erlang/OTP and ELevelDB.
|
||||
|
||||
<a name="n3.10">
|
||||
### 3.10. Does Machi use the Erlang/OTP network distribution system (aka "disterl")?
|
||||
|
||||
No, Machi doesn't use Erlang/OTP's built-in distributed message
|
||||
passing system. The code would be *much* simpler if we did use
|
||||
|
@ -588,19 +607,16 @@ bit-twiddling magicSPEED ... without also having to find a replacement
|
|||
for disterl. (Or without having to re-invent disterl's features in
|
||||
another language.)
|
||||
|
||||
<a name="artisanal-protocol">
|
||||
In the first drafts of the Machi code, the inter-node communication
|
||||
uses a hand-crafted, artisanal, mostly ASCII protocol as part of a
|
||||
"demo day" quick & dirty prototype. Work is underway (summer of 2015)
|
||||
to replace that protocol gradually with a well-structured,
|
||||
well-documented protocol based on Protocol Buffers data serialization.
|
||||
All wire protocols used by Machi are defined & implemented using
|
||||
[Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview).
|
||||
The definition file can be found at [./src/machi.proto](./src/machi.proto).
|
||||
|
||||
<a name="n3.9">
|
||||
### 3.9. Can I use HTTP to write/read stuff into/from Machi?
|
||||
<a name="n3.11">
|
||||
### 3.11. Can I use HTTP to write/read stuff into/from Machi?
|
||||
|
||||
Yes, sort of. For as long as the legacy of
|
||||
Machi's first internal protocol & code still
|
||||
survives, it's possible to use a
|
||||
Short answer: No, not yet.
|
||||
|
||||
Longer answer: No, but it was possible as a hack, many months ago, see
|
||||
[primitive/hack'y HTTP interface that is described in this source code commit log](https://github.com/basho/machi/commit/6cebf397232cba8e63c5c9a0a8c02ba391b20fef).
|
||||
Please note that commit `6cebf397232cba8e63c5c9a0a8c02ba391b20fef` is
|
||||
required to try using this feature: the code has since bit-rotted and
|
||||
|
|
108
Makefile
108
Makefile
|
@ -1,54 +1,94 @@
|
|||
REBAR_BIN := $(shell which rebar)
|
||||
ifeq ($(REBAR_BIN),)
|
||||
REBAR_BIN = ./rebar
|
||||
REPO ?= machi
|
||||
PKG_REVISION ?= $(shell git describe --tags)
|
||||
PKG_BUILD = 1
|
||||
BASE_DIR = $(shell pwd)
|
||||
ERLANG_BIN = $(shell dirname $(shell which erl))
|
||||
REBAR := $(shell which rebar)
|
||||
ifeq ($(REBAR),)
|
||||
REBAR = $(BASE_DIR)/rebar
|
||||
endif
|
||||
OVERLAY_VARS ?=
|
||||
EUNIT_OPTS = -v
|
||||
|
||||
.PHONY: rel deps package pkgclean edoc
|
||||
.PHONY: rel stagedevrel deps package pkgclean edoc
|
||||
|
||||
all: deps compile
|
||||
|
||||
compile:
|
||||
$(REBAR_BIN) compile
|
||||
$(REBAR) compile
|
||||
|
||||
## Make reltool happy by creating a fake entry in the deps dir for
|
||||
## machi, because reltool really wants to have a path with
|
||||
## "machi/ebin" at the end, but we also don't want infinite recursion
|
||||
## if we just symlink "deps/machi" -> ".."
|
||||
|
||||
generate:
|
||||
rm -rf deps/machi
|
||||
mkdir deps/machi
|
||||
ln -s ../../ebin deps/machi
|
||||
ln -s ../../src deps/machi
|
||||
$(REBAR) generate $(OVERLAY_VARS) 2>&1 | grep -v 'command does not apply to directory'
|
||||
|
||||
deps:
|
||||
$(REBAR_BIN) get-deps
|
||||
$(REBAR) get-deps
|
||||
|
||||
clean:
|
||||
$(REBAR_BIN) -r clean
|
||||
|
||||
test: deps compile eunit
|
||||
|
||||
eunit:
|
||||
$(REBAR_BIN) -v skip_deps=true eunit
|
||||
$(REBAR) -r clean
|
||||
|
||||
edoc: edoc-clean
|
||||
$(REBAR_BIN) skip_deps=true doc
|
||||
$(REBAR) skip_deps=true doc
|
||||
|
||||
edoc-clean:
|
||||
rm -f edoc/*.png edoc/*.html edoc/*.css edoc/edoc-info
|
||||
|
||||
pulse: compile
|
||||
env USE_PULSE=1 $(REBAR_BIN) skip_deps=true clean compile
|
||||
env USE_PULSE=1 $(REBAR_BIN) skip_deps=true -D PULSE eunit -v
|
||||
@echo Sorry, PULSE test needs maintenance. -SLF
|
||||
#env USE_PULSE=1 $(REBAR) skip_deps=true clean compile
|
||||
#env USE_PULSE=1 $(REBAR) skip_deps=true -D PULSE eunit -v
|
||||
|
||||
APPS = kernel stdlib sasl erts ssl compiler eunit crypto
|
||||
##
|
||||
## Release targets
|
||||
##
|
||||
rel: deps compile generate
|
||||
|
||||
relclean:
|
||||
rm -rf rel/$(REPO)
|
||||
|
||||
stage : rel
|
||||
$(foreach dep,$(wildcard deps/*), rm -rf rel/$(REPO)/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) rel/$(REPO)/lib;)
|
||||
|
||||
##
|
||||
## Developer targets
|
||||
##
|
||||
## devN - Make a dev build for node N
|
||||
## stagedevN - Make a stage dev build for node N (symlink libraries)
|
||||
## devrel - Make a dev build for 1..$DEVNODES
|
||||
## stagedevrel Make a stagedev build for 1..$DEVNODES
|
||||
##
|
||||
## Example, make a 68 node devrel cluster
|
||||
## make stagedevrel DEVNODES=68
|
||||
|
||||
.PHONY : stagedevrel devrel
|
||||
DEVNODES ?= 3
|
||||
|
||||
# 'seq' is not available on all *BSD, so using an alternate in awk
|
||||
SEQ = $(shell awk 'BEGIN { for (i = 1; i < '$(DEVNODES)'; i++) printf("%i ", i); print i ;exit(0);}')
|
||||
|
||||
$(eval stagedevrel : $(foreach n,$(SEQ),stagedev$(n)))
|
||||
$(eval devrel : $(foreach n,$(SEQ),dev$(n)))
|
||||
|
||||
dev% : all
|
||||
mkdir -p dev
|
||||
rel/gen_dev $@ rel/vars/dev_vars.config.src rel/vars/$@_vars.config
|
||||
(cd rel && ../rebar generate target_dir=../dev/$@ overlay_vars=vars/$@_vars.config)
|
||||
|
||||
stagedev% : dev%
|
||||
$(foreach dep,$(wildcard deps/*), rm -rf dev/$^/lib/$(shell basename $(dep))* && ln -sf $(abspath $(dep)) dev/$^/lib;)
|
||||
|
||||
devclean: clean
|
||||
rm -rf dev
|
||||
|
||||
DIALYZER_APPS = kernel stdlib sasl erts ssl compiler eunit crypto public_key syntax_tools
|
||||
PLT = $(HOME)/.machi_dialyzer_plt
|
||||
|
||||
build_plt: deps compile
|
||||
dialyzer --build_plt --output_plt $(PLT) --apps $(APPS) deps/*/ebin
|
||||
|
||||
DIALYZER_DEP_APPS = ebin/machi_pb.beam deps/protobuffs/ebin
|
||||
DIALYZER_FLAGS = -Wno_return -Wrace_conditions -Wunderspecs
|
||||
|
||||
dialyzer: deps compile
|
||||
dialyzer $(DIALYZER_FLAGS) --plt $(PLT) ebin $(DIALYZER_DEP_APPS) | \
|
||||
egrep -v -f ./filter-dialyzer-dep-warnings
|
||||
|
||||
dialyzer-test: deps compile
|
||||
echo Force rebar to recompile .eunit dir w/o running tests > /dev/null
|
||||
rebar skip_deps=true eunit suite=lamport_clock
|
||||
dialyzer $(DIALYZER_FLAGS) --plt $(PLT) .eunit $(DIALYZER_DEP_APPS) | \
|
||||
egrep -v -f ./filter-dialyzer-dep-warnings
|
||||
|
||||
clean_plt:
|
||||
rm $(PLT)
|
||||
include tools.mk
|
||||
|
|
227
README.md
227
README.md
|
@ -1,59 +1,136 @@
|
|||
# Machi
|
||||
# Machi: a distributed, decentralized blob/large file store
|
||||
|
||||
Our goal is a robust & reliable, distributed, highly available(*),
|
||||
large file store based upon write-once registers, append-only files,
|
||||
Chain Replication, and client-server style architecture. All members
|
||||
of the cluster store all of the files. Distributed load
|
||||
balancing/sharding of files is __outside__ of the scope of this
|
||||
system. However, it is a high priority that this system be able to
|
||||
integrate easily into systems that do provide distributed load
|
||||
balancing, e.g., Riak Core. Although strong consistency is a major
|
||||
feature of Chain Replication, first use cases will focus mainly on
|
||||
eventual consistency features --- strong consistency design will be
|
||||
discussed in a separate design document (read more below).
|
||||
[Travis-CI](http://travis-ci.org/basho/machi) :: ![Travis-CI](https://secure.travis-ci.org/basho/machi.png)
|
||||
|
||||
The ability for Machi to maintain strong consistency will make it
|
||||
attractive as a toolkit for building things like CORFU and Tango as
|
||||
well as better-known open source software such as Kafka's file
|
||||
replication. (See the bibliography of the [Machi high level design
|
||||
doc](./doc/high-level-machi.pdf) for further references.)
|
||||
Outline
|
||||
|
||||
(*) Capable of operating in "AP mode" or "CP mode" relative to the
|
||||
CAP Theorem.
|
||||
1. [Why another blob/file store?](#sec1)
|
||||
2. [Where to learn more about Machi](#sec2)
|
||||
3. [Development status summary](#sec3)
|
||||
4. [Contributing to Machi's development](#sec4)
|
||||
|
||||
## Status: mid-June 2015: work is underway
|
||||
<a name="sec1">
|
||||
## 1. Why another blob/file store?
|
||||
|
||||
The two major design documents for Machi are now ready or nearly ready
|
||||
for internal Basho and external party review. Please see the
|
||||
[doc](./doc) directory's [README](./doc) for details
|
||||
Our goal is a robust & reliable, distributed, highly available, large
|
||||
file and blob store. Such stores already exist, both in the open source world
|
||||
and in the commercial world. Why reinvent the wheel? We believe
|
||||
there are three reasons, ordered by decreasing rarity.
|
||||
|
||||
* Machi high level design
|
||||
* Machi chain self-management design
|
||||
1. We want end-to-end checksums for all file data, from the initial
|
||||
file writer to every file reader, anywhere, all the time.
|
||||
2. We need flexibility to trade consistency for availability:
|
||||
e.g. weak consistency in exchange for being available in cases
|
||||
of partial system failure.
|
||||
3. We want to manage file replicas in a way that's provably correct
|
||||
and also easy to test.
|
||||
|
||||
The work of implementing first draft of Machi is now underway. The
|
||||
code from the [prototype/demo-day-hack](prototype/demo-day-hack/) directory is
|
||||
being used as the initial scaffolding.
|
||||
Criteria #3 is difficult to find in the open source world but perhaps
|
||||
not impossible.
|
||||
|
||||
* The chain manager is ready for "AP mode" use in eventual
|
||||
consistency use cases.
|
||||
If we have app use cases where availability is more important than
|
||||
consistency, then systems that meet criteria #2 are also rare.
|
||||
Most file stores provide only strong consistency and therefore
|
||||
have unavoidable, unavailable behavior when parts of the system
|
||||
fail.
|
||||
What if we want a file store that is always available to write new
|
||||
file data and attempts best-effort file reads?
|
||||
|
||||
* All Machi client/server protocols are based on
|
||||
[Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview).
|
||||
* The current specification for Machi's protocols can be found at
|
||||
[https://github.com/basho/machi/blob/master/src/machi.proto](https://github.com/basho/machi/blob/master/src/machi.proto).
|
||||
* The Machi PB protocol is not yet stable. Expect change!
|
||||
* The Erlang language client implementation of the high-level
|
||||
protocol flavor is very brittle (e.g., very little error
|
||||
handling yet).
|
||||
* The Erlang language client implementation of the low-level
|
||||
protocol flavor are still a work-in-progress ... but they are
|
||||
more robust than the high-level library's implementation.
|
||||
If we really do care about data loss and/or data corruption, then we
|
||||
really want both #3 and #1. Unfortunately, systems that meet
|
||||
criteria #1 are _very rare_. (Nonexistant?)
|
||||
Why? This is 2015. We have decades of research that shows
|
||||
that computer hardware can (and
|
||||
indeed does) corrupt data at nearly every level of the modern
|
||||
client/server application stack. Systems with end-to-end data
|
||||
corruption detection should be ubiquitous today. Alas, they are not.
|
||||
|
||||
If you'd like to work on a protocol such as Thrift, UBF,
|
||||
msgpack over UDP, or some other protocol, let us know by
|
||||
[opening an issue](./issues/new) to discuss it.
|
||||
Machi is an effort to change the deplorable state of the world, one
|
||||
Erlang function at a time.
|
||||
|
||||
## Contributing to Machi: source code, documentation, etc.
|
||||
<a name="sec2">
|
||||
## 2. Where to learn more about Machi
|
||||
|
||||
The two major design documents for Machi are now mostly stable.
|
||||
Please see the [doc](./doc) directory's [README](./doc) for details.
|
||||
|
||||
We also have a
|
||||
[Frequently Asked Questions (FAQ) list](./FAQ.md).
|
||||
|
||||
Scott recently (November 2015) gave a presentation at the
|
||||
[RICON 2015 conference](http://ricon.io) about one of the techniques
|
||||
used by Machi; "Managing Chain Replication Metadata with
|
||||
Humming Consensus" is available online now.
|
||||
* [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf)
|
||||
* [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q)
|
||||
|
||||
See later in this document for how to run the Humming Consensus demos,
|
||||
including the network partition simulator.
|
||||
|
||||
<a name="sec3">
|
||||
## 3. Development status summary
|
||||
|
||||
Mid-March 2016: The Machi development team has been downsized in
|
||||
recent months, and the pace of development has slowed. Here is a
|
||||
summary of the status of Machi's major components.
|
||||
|
||||
* Humming Consensus and the chain manager
|
||||
* No new safety bugs have been found by model-checking tests.
|
||||
* A new document,
|
||||
[Hands-on experiments with Machi and Humming Consensus](doc/humming-consensus-demo.md)
|
||||
is now available. It is a tutorial for setting up a 3 virtual
|
||||
machine Machi cluster and how to demonstrate the chain manager's
|
||||
reactions to server stops & starts, crashes & restarts, and pauses
|
||||
(simulated by `SIGSTOP` and `SIGCONT`).
|
||||
* The chain manager can still make suboptimal-but-safe choices for
|
||||
chain transitions when a server hangs/pauses temporarily.
|
||||
* Recent chain manager changes have made the instability window
|
||||
much shorter when the slow/paused server resumes execution.
|
||||
* Scott believes that a modest change to the chain manager's
|
||||
calculation of a new projection can reduce flapping in this (and
|
||||
many other cases) less likely. Currently, the new local
|
||||
projection is calculated using only local state (i.e., the chain
|
||||
manager's internal state + the fitness server's state).
|
||||
However, if the "latest" projection read from the public
|
||||
projection stores were also input to the new projection
|
||||
calculation function, then many obviously bad projections can be
|
||||
avoided without needing rounds of Humming Consensus to
|
||||
demonstrate that a bad projection is bad.
|
||||
|
||||
* FLU/data server process
|
||||
* All known correctness bugs have been fixed.
|
||||
* Performance has not yet been measured. Performance measurement
|
||||
and enhancements are scheduled to start in the middle of March 2016.
|
||||
(This will include a much-needed update to the `basho_bench` driver.)
|
||||
|
||||
* Access protocols and client libraries
|
||||
* The protocol used by both external clients and internally (instead
|
||||
of using Erlang's native message passing mechanisms) is based on
|
||||
Protocol Buffers.
|
||||
* (Machi PB protocol specification: ./src/machi.proto)[./src/machi.proto]
|
||||
* At the moment, the PB specification contains two protocols.
|
||||
Sometime in the near future, the spec will be split to separate
|
||||
the external client API (the "high" protocol) from the internal
|
||||
communication API (the "low" protocol).
|
||||
|
||||
* Recent conference talks about Machi
|
||||
* Erlang Factory San Francisco 2016
|
||||
[the slides and video recording](http://www.erlang-factory.com/sfbay2016/scott-lystig-fritchie)
|
||||
will be available a few weeks after the conference ends on March
|
||||
11, 2016.
|
||||
* Ricon 2015
|
||||
* [The slides](http://ricon.io/archive/2015/slides/Scott_Fritchie_Ricon_2015.pdf)
|
||||
* and the [video recording](https://www.youtube.com/watch?v=yR5kHL1bu1Q&index=13&list=PL9Jh2HsAWHxIc7Tt2M6xez_TOP21GBH6M)
|
||||
are now available.
|
||||
* If you would like to run the Humming Consensus code (with or without
|
||||
the network partition simulator) as described in the RICON 2015
|
||||
presentation, please see the
|
||||
[Humming Consensus demo doc](./doc/humming_consensus_demo.md).
|
||||
|
||||
<a name="sec4">
|
||||
## 4. Contributing to Machi's development
|
||||
|
||||
### 4.1 License
|
||||
|
||||
Basho Technologies, Inc. as committed to licensing all work for Machi
|
||||
under the
|
||||
|
@ -69,57 +146,29 @@ We invite all contributors to review the
|
|||
[CONTRIBUTING.md](./CONTRIBUTING.md) document for guidelines for
|
||||
working with the Basho development team.
|
||||
|
||||
## A brief survey of this directories in this repository
|
||||
|
||||
* A list of Frequently Asked Questions, a.k.a.
|
||||
[the Machi FAQ](./FAQ.md).
|
||||
|
||||
* The [doc](./doc/) directory: home for major documents about Machi:
|
||||
high level design documents as well as exploration of features still
|
||||
under design & review within Basho.
|
||||
|
||||
* The `ebin` directory: used for compiled application code
|
||||
|
||||
* The `include`, `src`, and `test` directories: contain the header
|
||||
files, source files, and test code for Machi, respectively.
|
||||
|
||||
* The [prototype](./prototype/) directory: contains proof of concept
|
||||
code, scaffolding libraries, and other exploratory code. Curious
|
||||
readers should see the [prototype/README.md](./prototype/README.md)
|
||||
file for more explanation of the small sub-projects found here.
|
||||
|
||||
## Development environment requirements
|
||||
### 4.2 Development environment requirements
|
||||
|
||||
All development to date has been done with Erlang/OTP version 17 on OS
|
||||
X. The only known limitations for using R16 are minor type
|
||||
specification difference between R16 and 17, but we strongly suggest
|
||||
continuing development using version 17.
|
||||
|
||||
We also assume that you have the standard UNIX/Linux developers
|
||||
tool chain for C and C++ applications. Specifically, we assume `make`
|
||||
is available. The utility used to compile the Machi source code,
|
||||
We also assume that you have the standard UNIX/Linux developer
|
||||
tool chain for C and C++ applications. Also, we assume
|
||||
that Git and GNU Make are available.
|
||||
The utility used to compile the Machi source code,
|
||||
`rebar`, is pre-compiled and included in the repo.
|
||||
For more details, please see the
|
||||
[Machi development environment prerequisites doc](./doc/dev-prerequisites.md).
|
||||
|
||||
There are no known OS limits at this time: any platform that supports
|
||||
Erlang/OTP should be sufficient for Machi. This may change over time
|
||||
(e.g., adding NIFs which can make full portability to Windows OTP
|
||||
environments difficult), but it hasn't happened yet.
|
||||
Machi has a dependency on the
|
||||
[ELevelDB](https://github.com/basho/eleveldb) library. ELevelDB only
|
||||
supports UNIX/Linux OSes and 64-bit versions of Erlang/OTP only; we
|
||||
apologize to Windows-based and 32-bit-based Erlang developers for this
|
||||
restriction.
|
||||
|
||||
## Contributions
|
||||
|
||||
Basho encourages contributions to Riak from the community. Here’s how
|
||||
to get started.
|
||||
|
||||
* Fork the appropriate sub-projects that are affected by your change.
|
||||
* Create a topic branch for your change and checkout that branch.
|
||||
git checkout -b some-topic-branch
|
||||
* Make your changes and run the test suite if one is provided. (see below)
|
||||
* Commit your changes and push them to your fork.
|
||||
* Open pull-requests for the appropriate projects.
|
||||
* Contributors will review your pull request, suggest changes, and merge it when it’s ready and/or offer feedback.
|
||||
* To report a bug or issue, please open a new issue against this repository.
|
||||
|
||||
-The Machi team at Basho,
|
||||
[Scott Lystig Fritchie](mailto:scott@basho.com), technical lead, and
|
||||
[Matt Brender](mailto:mbrender@basho.com), your developer advocate.
|
||||
### 4.3 New protocols and features
|
||||
|
||||
If you'd like to work on a protocol such as Thrift, UBF,
|
||||
msgpack over UDP, or some other protocol, let us know by
|
||||
[opening an issue to discuss it](./issues/new).
|
||||
|
|
|
@ -71,7 +71,7 @@ func, and pattern match Erlang style in that func.
|
|||
*** DONE Any new errors are added to UpNodes/DownNodes tracking as used today
|
||||
*** DONE At end of react loop, if UpNodes list differs, inform hosed mgr.
|
||||
|
||||
* TODO fitness_mon, the fitness monitor
|
||||
* DONE fitness_mon, the fitness monitor
|
||||
** DONE Map key & val sketch
|
||||
|
||||
Logical sketch:
|
||||
|
@ -91,24 +91,24 @@ Implementation sketch:
|
|||
|
||||
** DONE Investigate riak_dt data structure definition, manipulating, etc.
|
||||
** DONE Add dependency on riak_dt
|
||||
** WORKING Update is an entire dict from Observer O
|
||||
*** TODO Merge my pending map + update map + my last mod time + my unfit list
|
||||
*** TODO if merged /= pending:
|
||||
**** TODO Schedule async tick (more)
|
||||
** DONE Update is an entire dict from Observer O
|
||||
*** DONE Merge my pending map + update map + my last mod time + my unfit list
|
||||
*** DONE if merged /= pending:
|
||||
**** DONE Schedule async tick (more)
|
||||
|
||||
Tick message contains list of servers with differing state as of this
|
||||
instant in time... we want to avoid triggering decisions about
|
||||
fitness/unfitness for other servers where we might have received less
|
||||
than a full time period's worth of waiting.
|
||||
|
||||
**** TODO Spam merged map to All_list -- [Me, O]
|
||||
**** TODO Set pending <- merged
|
||||
**** DONE Spam merged map to All_list -- [Me]
|
||||
**** DONE Set pending <- merged
|
||||
|
||||
*** TODO When we receive an async tick
|
||||
**** TODO set active map <- pending map for all servers in ticks list
|
||||
**** TODO Send ch_mgr a react_to_env tick trigger
|
||||
*** TODO react_to_env tick trigger actions
|
||||
**** TODO Filter active map to remove stale entries (i.e. no update in 1 hour)
|
||||
**** TODO If time since last map spam is too long, spam our *pending* map
|
||||
**** TODO Proceed with normal react processing, using *active* map for AllHosed!
|
||||
*** DONE When we receive an async tick
|
||||
**** DONE set active map <- pending map for all servers in ticks list
|
||||
**** DONE Send ch_mgr a react_to_env tick trigger
|
||||
*** DONE react_to_env tick trigger actions
|
||||
**** DONE Filter active map to remove stale entries (i.e. no update in 1 hour)
|
||||
**** DONE If time since last map spam is too long, spam our *pending* map
|
||||
**** DONE Proceed with normal react processing, using *active* map for AllHosed!
|
||||
|
||||
|
|
15
dialyzer.ignore-warnings
Normal file
15
dialyzer.ignore-warnings
Normal file
|
@ -0,0 +1,15 @@
|
|||
### The auto-generated code of machi_pb.beam has some complaints, not fixed yet.
|
||||
machi_pb.erl:0:
|
||||
##################################################
|
||||
######## Specific types #####################
|
||||
##################################################
|
||||
Unknown types:
|
||||
basho_bench_config:get/2
|
||||
machi_partition_simulator:get/1
|
||||
hamcrest:matchspec/0
|
||||
##################################################
|
||||
######## Specific messages #####################
|
||||
##################################################
|
||||
machi_chain_manager1.erl:2473: The created fun has no local return
|
||||
machi_chain_manager1.erl:2184: The pattern <_P1, P2, Else = {'expected_author2', UPI1_tail, _}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'>
|
||||
machi_chain_manager1.erl:2233: The pattern <_P1 = {'projection_v1', _, _, _, _, _, _, _, 'cp_mode', UPI1, Repairing1, _, _, _, _}, _P2 = {'projection_v1', _, _, _, _, _, _, _, 'cp_mode', UPI2, Repairing2, _, _, _, _}, Else = {'epoch_not_si', EpochX, 'not_gt', EpochY}> can never match the type <#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::'undefined' | binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},#projection_v1{epoch_number::'undefined' | non_neg_integer(),epoch_csum::binary(),author_server::atom(),chain_name::atom(),all_members::'undefined' | [atom()],witnesses::[atom()],creation_time::'undefined' | {non_neg_integer(),non_neg_integer(),non_neg_integer()},mode::'ap_mode' | 'cp_mode',upi::'undefined' | [atom()],repairing::'undefined' | [atom()],down::'undefined' | [atom()],dbg::'undefined' | [any()],dbg2::'undefined' | [any()],members_dict::'undefined' | [{_,_}]},'true'>
|
|
@ -6,20 +6,6 @@ Erlang documentation, please use this link:
|
|||
|
||||
## Documents in this directory
|
||||
|
||||
### chain-self-management-sketch.org
|
||||
|
||||
[chain-self-management-sketch.org](chain-self-management-sketch.org)
|
||||
is a mostly-deprecated draft of
|
||||
an introduction to the
|
||||
self-management algorithm proposed for Machi. Most material has been
|
||||
moved to the [high-level-chain-mgr.pdf](high-level-chain-mgr.pdf) document.
|
||||
|
||||
### cluster-of-clusters (directory)
|
||||
|
||||
This directory contains the sketch of the "cluster of clusters" design
|
||||
strawman for partitioning/distributing/sharding files across a large
|
||||
number of independent Machi clusters.
|
||||
|
||||
### high-level-machi.pdf
|
||||
|
||||
[high-level-machi.pdf](high-level-machi.pdf)
|
||||
|
@ -50,9 +36,9 @@ introduction to the Humming Consensus algorithm. Its abstract:
|
|||
> of file updates to all replica servers in a Machi cluster. Chain
|
||||
> Replication is a variation of primary/backup replication where the
|
||||
> order of updates between the primary server and each of the backup
|
||||
> servers is strictly ordered into a single ``chain''. Management of
|
||||
> Chain Replication's metadata, e.g., ``What is the current order of
|
||||
> servers in the chain?'', remains an open research problem. The
|
||||
> servers is strictly ordered into a single "chain". Management of
|
||||
> Chain Replication's metadata, e.g., "What is the current order of
|
||||
> servers in the chain?", remains an open research problem. The
|
||||
> current state of the art for Chain Replication metadata management
|
||||
> relies on an external oracle (e.g., ZooKeeper) or the Elastic
|
||||
> Replication algorithm.
|
||||
|
@ -60,7 +46,7 @@ introduction to the Humming Consensus algorithm. Its abstract:
|
|||
> This document describes the Machi chain manager, the component
|
||||
> responsible for managing Chain Replication metadata state. The chain
|
||||
> manager uses a new technique, based on a variation of CORFU, called
|
||||
> ``humming consensus''.
|
||||
> "humming consensus".
|
||||
> Humming consensus does not require active participation by all or even
|
||||
> a majority of participants to make decisions. Machi's chain manager
|
||||
> bases its logic on humming consensus to make decisions about how to
|
||||
|
@ -71,3 +57,18 @@ introduction to the Humming Consensus algorithm. Its abstract:
|
|||
> decision during that epoch. When a differing decision is discovered,
|
||||
> new time epochs are proposed in which a new consensus is reached and
|
||||
> disseminated to all available participants.
|
||||
|
||||
### chain-self-management-sketch.org
|
||||
|
||||
[chain-self-management-sketch.org](chain-self-management-sketch.org)
|
||||
is a mostly-deprecated draft of
|
||||
an introduction to the
|
||||
self-management algorithm proposed for Machi. Most material has been
|
||||
moved to the [high-level-chain-mgr.pdf](high-level-chain-mgr.pdf) document.
|
||||
|
||||
### cluster (directory)
|
||||
|
||||
This directory contains the sketch of the cluster design
|
||||
strawman for partitioning/distributing/sharding files across a large
|
||||
number of independent Machi chains.
|
||||
|
||||
|
|
Binary file not shown.
Before Width: | Height: | Size: 8.7 KiB |
Binary file not shown.
Before Width: | Height: | Size: 7.8 KiB |
|
@ -1,435 +0,0 @@
|
|||
-*- mode: org; -*-
|
||||
#+TITLE: Machi cluster-of-clusters "name game" sketch
|
||||
#+AUTHOR: Scott
|
||||
#+STARTUP: lognotedone hidestars indent showall inlineimages
|
||||
#+SEQ_TODO: TODO WORKING WAITING DONE
|
||||
#+COMMENT: M-x visual-line-mode
|
||||
#+COMMENT: Also, disable auto-fill-mode
|
||||
|
||||
* 1. "Name Games" with random-slicing style consistent hashing
|
||||
|
||||
Our goal: to distribute lots of files very evenly across a cluster of
|
||||
Machi clusters (hereafter called a "cluster of clusters" or "CoC").
|
||||
|
||||
* 2. Assumptions
|
||||
|
||||
** Basic familiarity with Machi high level design and Machi's "projection"
|
||||
|
||||
The [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] contains all of the basic
|
||||
background assumed by the rest of this document.
|
||||
|
||||
** Familiarity with the Machi cluster-of-clusters/CoC concept
|
||||
|
||||
This isn't yet well-defined (April 2015). However, it's clear from
|
||||
the [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] that Machi alone does not support
|
||||
any kind of file partitioning/distribution/sharding across multiple
|
||||
small Machi clusters. There must be another layer above a Machi cluster to
|
||||
provide such partitioning services.
|
||||
|
||||
The name "cluster of clusters" orignated within Basho to avoid
|
||||
conflicting use of the word "cluster". A Machi cluster is usually
|
||||
synonymous with a single Chain Replication chain and a single set of
|
||||
machines (e.g. 2-5 machines). However, in the not-so-far future, we
|
||||
expect much more complicated patterns of Chain Replication to be used
|
||||
in real-world deployments.
|
||||
|
||||
"Cluster of clusters" is clunky and long, but we haven't found a good
|
||||
substitute yet. If you have a good suggestion, please contact us!
|
||||
~^_^~
|
||||
|
||||
Using the [[https://github.com/basho/machi/tree/master/prototype/demo-day-hack][cluster-of-clusters quick-and-dirty prototype]] as an
|
||||
architecture sketch, let's now assume that we have ~N~ independent Machi
|
||||
clusters. We wish to provide partitioned/distributed file storage
|
||||
across all ~N~ clusters. We call the entire collection of ~N~ Machi
|
||||
clusters a "cluster of clusters", or abbreviated "CoC".
|
||||
|
||||
** Continue CoC prototype's assumption: a Machi cluster is unaware of CoC
|
||||
|
||||
Let's continue with an assumption that an individual Machi cluster
|
||||
inside of the cluster-of-clusters is completely unaware of the
|
||||
cluster-of-clusters layer.
|
||||
|
||||
We may need to break this assumption sometime in the future? It isn't
|
||||
quite clear yet, sorry.
|
||||
|
||||
** Analogy: "neighborhood : city :: Machi : cluster-of-clusters"
|
||||
|
||||
Analogy: The word "machi" in Japanese means small town or
|
||||
neighborhood. As the Tokyo Metropolitan Area is built from many
|
||||
machis and smaller cities, therefore a big, partitioned file store can
|
||||
be built out of many small Machi clusters.
|
||||
|
||||
** The reader is familiar with the random slicing technique
|
||||
|
||||
I'd done something very-very-nearly-identical for the Hibari database
|
||||
6 years ago. But the Hibari technique was based on stuff I did at
|
||||
Sendmail, Inc, so it felt old news to me. {shrug}
|
||||
|
||||
The Hibari documentation has a brief photo illustration of how random
|
||||
slicing works, see [[http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration][Hibari Sysadmin Guide, chain migration]]
|
||||
|
||||
For a comprehensive description, please see these two papers:
|
||||
|
||||
#+BEGIN_QUOTE
|
||||
Reliable and Randomized Data Distribution Strategies for Large Scale Storage Systems
|
||||
Alberto Miranda et al.
|
||||
http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.226.5609
|
||||
(short version, HIPC'11)
|
||||
|
||||
Random Slicing: Efficient and Scalable Data Placement for Large-Scale
|
||||
Storage Systems
|
||||
Alberto Miranda et al.
|
||||
DOI: http://dx.doi.org/10.1145/2632230 (long version, ACM Transactions
|
||||
on Storage, Vol. 10, No. 3, Article 9, 2014)
|
||||
#+END_QUOTE
|
||||
|
||||
** We use random slicing to map CoC file names -> Machi cluster ID/name
|
||||
|
||||
We will use a single random slicing map. This map (called ~Map~ in
|
||||
the descriptions below), together with the random slicing hash
|
||||
function (called ~rs_hash()~ below), will be used to map:
|
||||
|
||||
#+BEGIN_QUOTE
|
||||
CoC client-visible file name -> Machi cluster ID/name/thingie
|
||||
#+END_QUOTE
|
||||
|
||||
** Machi cluster ID/name management: TBD, but, really, should be simple
|
||||
|
||||
The mapping from:
|
||||
|
||||
#+BEGIN_QUOTE
|
||||
Machi CoC member ID/name/thingie -> ???
|
||||
#+END_QUOTE
|
||||
|
||||
... remains To Be Determined. But, really, this is going to be pretty
|
||||
simple. The ID/name/thingie will probably be a human-friendly,
|
||||
printable ASCII string, and the "???" will probably be a single Machi
|
||||
cluster projection data structure.
|
||||
|
||||
The Machi projection is enough information to contact any member of
|
||||
that cluster and, if necessary, request the most up-to-date projection
|
||||
information required to use that cluster.
|
||||
|
||||
It's likely that the projection given by this map will be out-of-date,
|
||||
so the client must be ready to use the standard Machi procedure to
|
||||
request the cluster's current projection, in any case.
|
||||
|
||||
* 3. A simple illustration
|
||||
|
||||
I'm borrowing an illustration from the HibariDB documentation here,
|
||||
but it fits my purposes quite well. (And I originally created that
|
||||
image, and the use license is OK.)
|
||||
|
||||
#+CAPTION: Illustration of 'Map', using four Machi clusters
|
||||
|
||||
[[./migration-4.png]]
|
||||
|
||||
Assume that we have a random slicing map called ~Map~. This particular
|
||||
~Map~ maps the unit interval onto 4 Machi clusters:
|
||||
|
||||
| Hash range | Cluster ID |
|
||||
|-------------+------------|
|
||||
| 0.00 - 0.25 | Cluster1 |
|
||||
| 0.25 - 0.33 | Cluster4 |
|
||||
| 0.33 - 0.58 | Cluster2 |
|
||||
| 0.58 - 0.66 | Cluster4 |
|
||||
| 0.66 - 0.91 | Cluster3 |
|
||||
| 0.91 - 1.00 | Cluster4 |
|
||||
|
||||
Then, if we had CoC file name "~foo~", the hash ~SHA("foo")~ maps to about
|
||||
0.05 on the unit interval. So, according to ~Map~, the value of
|
||||
~rs_hash("foo",Map) = Cluster1~. Similarly, ~SHA("hello")~ is about
|
||||
0.67 on the unit interval, so ~rs_hash("hello",Map) = Cluster3~.
|
||||
|
||||
* 4. An additional assumption: clients will want some control over file placement
|
||||
|
||||
We will continue to use the 4-cluster diagram from the previous
|
||||
section.
|
||||
|
||||
When a client wishes to append data to a Machi file, the Machi server
|
||||
chooses the file name & byte offset for storing that data. This
|
||||
feature is why Machi's eventual consistency operating mode is so
|
||||
nifty: it allows us to merge together files safely at any time because
|
||||
any two client append operations will always write to different files
|
||||
& different offsets.
|
||||
|
||||
** Our new assumption: client control over initial file placement
|
||||
|
||||
The CoC management scheme may decide that files need to migrate to
|
||||
other clusters. The reason could be for storage load or I/O load
|
||||
balancing reasons. It could be because a cluster is being
|
||||
decomissioned by its owners. There are many legitimate reasons why a
|
||||
file that is initially created on cluster ID X has been moved to
|
||||
cluster ID Y.
|
||||
|
||||
However, there are also legitimate reasons for why the client would want
|
||||
control over the choice of Machi cluster when the data is first
|
||||
written. The single biggest reason is load balancing. Assuming that
|
||||
the client (or the CoC management layer acting on behalf of the CoC
|
||||
client) knows the current utilization across the participating Machi
|
||||
clusters, then it may be very helpful to send new append() requests to
|
||||
under-utilized clusters.
|
||||
|
||||
** Cool! Except for a couple of problems...
|
||||
|
||||
If the client wants to store some data
|
||||
on Cluster2 and therefore sends an ~append("foo",CoolData)~ request to
|
||||
the head of Cluster2 (which the client magically knows how to
|
||||
contact), then the result will look something like
|
||||
~{ok,"foo.s923.z47",ByteOffset}~.
|
||||
|
||||
Therefore, the file name "~foo.s923.z47~" must be used by any Machi
|
||||
CoC client in order to retrieve the CoolData bytes.
|
||||
|
||||
*** Problem #1: "foo.s923.z47" doesn't always map via random slicing to Cluster2
|
||||
|
||||
... if we ignore the problem of "CoC files may be redistributed in the
|
||||
future", then we still have a problem.
|
||||
|
||||
In fact, the value of ~ps_hash("foo.s923.z47",Map)~ is Cluster1.
|
||||
|
||||
*** Problem #2: We want CoC files to move around automatically
|
||||
|
||||
If the CoC client stores two pieces of information, the file name
|
||||
"~foo.s923.z47~" and the Cluster ID Cluster2, then what happens when the
|
||||
cluster-of-clusters system decides to rebalance files across all
|
||||
machines? The CoC manager may decide to move our file to Cluster66.
|
||||
|
||||
How will a future CoC client wishes to retrieve CoolData when Cluster2
|
||||
no longer stores the required file?
|
||||
|
||||
**** When migrating the file, we could put a "pointer" on Cluster2 that points to the new location, Cluster66.
|
||||
|
||||
This scheme is a bit brittle, even if all of the pointers are always
|
||||
created 100% correctly. Also, if Cluster2 is ever unavailable, then
|
||||
we cannot fetch our CoolData, even though the file moved away from
|
||||
Cluster2 several years ago.
|
||||
|
||||
The scheme would also introduce extra round-trips to the servers
|
||||
whenever we try to read a file where we do not know the most
|
||||
up-to-date cluster ID for.
|
||||
|
||||
**** We could store a pointer to file "foo.s923.z47"'s location in an LDAP database!
|
||||
|
||||
Or we could store it in Riak. Or in another, external database. We'd
|
||||
rather not create such an external dependency, however. Furthermore,
|
||||
we would also have the same problem of updating this external database
|
||||
each time that a file is moved/rebalanced across the CoC.
|
||||
|
||||
* 5. Proposal: Break the opacity of Machi file names, slightly
|
||||
|
||||
Assuming that Machi keeps the scheme of creating file names (in
|
||||
response to ~append()~ and ~sequencer_new_range()~ calls) based on a
|
||||
predictable client-supplied prefix and an opaque suffix, e.g.,
|
||||
|
||||
~append("foo",CoolData) -> {ok,"foo.s923.z47",ByteOffset}.~
|
||||
|
||||
... then we propose that all CoC and Machi parties be aware of this
|
||||
naming scheme, i.e. that Machi assigns file names based on:
|
||||
|
||||
~ClientSuppliedPrefix ++ "." ++ SomeOpaqueFileNameSuffix~
|
||||
|
||||
The Machi system doesn't care about the file name -- a Machi server
|
||||
will treat the entire file name as an opaque thing. But this document
|
||||
is called the "Name Game" for a reason!
|
||||
|
||||
What if the CoC client could peek inside of the opaque file name
|
||||
suffix in order to remove (or add) the CoC location information that
|
||||
we need?
|
||||
|
||||
** The details: legend
|
||||
|
||||
- ~T~ = the target CoC member/Cluster ID chosen at the time of ~append()~
|
||||
- ~p~ = file prefix, chosen by the CoC client (This is exactly the Machi client-chosen file prefix).
|
||||
- ~s.z~ = the Machi file server opaque file name suffix (Which we
|
||||
happen to know is a combination of sequencer ID plus file serial
|
||||
number. This implementation may change, for example, to use a
|
||||
standard GUID string (rendered into ASCII hexadecimal digits) instead.)
|
||||
- ~K~ = the CoC placement key
|
||||
|
||||
We use a variation of ~rs_hash()~, called ~rs_hash_with_float()~. The
|
||||
former uses a string as its 1st argument; the latter uses a floating
|
||||
point number as its 1st argument. Both return a cluster ID name
|
||||
thingie.
|
||||
|
||||
#+BEGIN_SRC erlang
|
||||
%% type specs, Erlang style
|
||||
-spec rs_hash(string(), rs_hash:map()) -> rs_hash:cluster_id().
|
||||
-spec rs_hash_with_float(float(), rs_hash:map()) -> rs_hash:cluster_id().
|
||||
#+END_SRC
|
||||
|
||||
NOTE: Use of floating point terms is not required. For example,
|
||||
integer arithmetic could be used, if using a sufficiently large
|
||||
interval to create an even & smooth distribution of hashes across the
|
||||
expected maximum number of clusters.
|
||||
|
||||
For example, if the maximum CoC cluster size would be 4,000 individual
|
||||
Machi clusters, then a minimum of 12 bits of integer space is required
|
||||
to assign one integer per Machi cluster. However, for load balancing
|
||||
purposes, a finer grain of (for example) 100 integers per Machi
|
||||
cluster would permit file migration to move increments of
|
||||
approximately 1% of single Machi cluster's storage capacity. A
|
||||
minimum of 19 bits of hash space would be necessary to accomodate
|
||||
these constraints.
|
||||
|
||||
** The details: CoC file write
|
||||
|
||||
1. CoC client chooses ~p~ and ~T~ (i.e., the file prefix & target cluster)
|
||||
2. CoC client knows the CoC ~Map~
|
||||
3. CoC client calculates a value ~K~ such that ~rs_hash_with_float(K,Map) = T~, using the method described below.
|
||||
4. CoC client requests @ cluster ~T~: ~append_chunk(p,K,...) -> {ok,p.K.s.z,ByteOffset}~
|
||||
5. CoC stores/uses the file name ~p.K.s.z~.
|
||||
|
||||
** The details: CoC file read
|
||||
|
||||
1. CoC client knows the file name ~p.K.s.z~ and parses it to find
|
||||
~K~'s value.
|
||||
2. CoC client knows the CoC ~Map~
|
||||
3. Coc calculates ~rs_hash_with_float(K,Map) = T~
|
||||
4. CoC client requests @ cluster ~T~: ~read_chunk(p.K.s.z,...) ->~ ... success!
|
||||
|
||||
** The details: calculating 'K', the CoC placement key
|
||||
|
||||
1. We know ~Map~, the current CoC mapping.
|
||||
2. We look inside of ~Map~, and we find all of the unit interval ranges
|
||||
that map to our desired target cluster ~T~. Let's call this list
|
||||
~MapList = [Range1=(start,end],Range2=(start,end],...]~.
|
||||
3. In our example, ~T=Cluster2~. The example ~Map~ contains a single
|
||||
unit interval range for ~Cluster2~, ~[(0.33,0.58]]~.
|
||||
4. Choose a uniformally random number ~r~ on the unit interval.
|
||||
5. Calculate placement key ~K~ by mapping ~r~ onto the concatenation
|
||||
of the CoC hash space range intervals in ~MapList~. For example,
|
||||
if ~r=0.5~, then ~K = 0.33 + 0.5*(0.58-0.33) = 0.455~, which is
|
||||
exactly in the middle of the ~(0.33,0.58]~ interval.
|
||||
6. If necessary, encode ~K~ in a file name-friendly manner, e.g., convert it to hexadecimal ASCII digits to create file name ~p.K.s.z~.
|
||||
|
||||
** The details: calculating 'K', an alternative method
|
||||
|
||||
If the Law of Large Numbers and our random number generator do not create the kind of smooth & even distribution of files across the CoC as we wish, an alternative method of calculating ~K~ follows.
|
||||
|
||||
If each server in each Machi cluster keeps track of the CoC ~Map~ and also of all values of ~K~ for all files that it stores, then we can simply ask a cluster member to recommend a value of ~K~ that is least represented by existing files.
|
||||
|
||||
* 6. File migration (aka rebalancing/reparitioning/redistribution)
|
||||
|
||||
** What is "file migration"?
|
||||
|
||||
As discussed in section 5, the client can have good reason for wanting
|
||||
to have some control of the initial location of the file within the
|
||||
cluster. However, the cluster manager has an ongoing interest in
|
||||
balancing resources throughout the lifetime of the file. Disks will
|
||||
get full, hardware will change, read workload will fluctuate,
|
||||
etc etc.
|
||||
|
||||
This document uses the word "migration" to describe moving data from
|
||||
one CoC cluster to another. In other systems, this process is
|
||||
described with words such as rebalancing, repartitioning, and
|
||||
resharding. For Riak Core applications, the mechanisms are "handoff"
|
||||
and "ring resizing". See the [[http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsUserGuide.html#Balancer][Hadoop file balancer]] for another example.
|
||||
|
||||
A simple variation of the Random Slicing hash algorithm can easily
|
||||
accomodate Machi's need to migrate files without interfering with
|
||||
availability. Machi's migration task is much simpler due to the
|
||||
immutable nature of Machi file data.
|
||||
|
||||
** Change to Random Slicing
|
||||
|
||||
The map used by the Random Slicing hash algorithm needs a few simple
|
||||
changes to make file migration straightforward.
|
||||
|
||||
- Add a "generation number", a strictly increasing number (similar to
|
||||
a Machi cluster's "epoch number") that reflects the history of
|
||||
changes made to the Random Slicing map
|
||||
- Use a list of Random Slicing maps instead of a single map, one map
|
||||
per possibility that files may not have been migrated yet out of
|
||||
that map.
|
||||
|
||||
As an example:
|
||||
|
||||
#+CAPTION: Illustration of 'Map', using four Machi clusters
|
||||
|
||||
[[./migration-3to4.png]]
|
||||
|
||||
And the new Random Slicing map might look like this:
|
||||
|
||||
| Generation number | 7 |
|
||||
|-------------------+------------|
|
||||
| SubMap | 1 |
|
||||
|-------------------+------------|
|
||||
| Hash range | Cluster ID |
|
||||
|-------------------+------------|
|
||||
| 0.00 - 0.33 | Cluster1 |
|
||||
| 0.33 - 0.66 | Cluster2 |
|
||||
| 0.66 - 1.00 | Cluster3 |
|
||||
|-------------------+------------|
|
||||
| SubMap | 2 |
|
||||
|-------------------+------------|
|
||||
| Hash range | Cluster ID |
|
||||
|-------------------+------------|
|
||||
| 0.00 - 0.25 | Cluster1 |
|
||||
| 0.25 - 0.33 | Cluster4 |
|
||||
| 0.33 - 0.58 | Cluster2 |
|
||||
| 0.58 - 0.66 | Cluster4 |
|
||||
| 0.66 - 0.91 | Cluster3 |
|
||||
| 0.91 - 1.00 | Cluster4 |
|
||||
|
||||
When a new Random Slicing map contains a single submap, then its use
|
||||
is identical to the original Random Slicing algorithm. If the map
|
||||
contains multiple submaps, then the access rules change a bit:
|
||||
|
||||
- Write operations always go to the latest/largest submap.
|
||||
- Read operations attempt to read from all unique submaps.
|
||||
- Skip searching submaps that refer to the same cluster ID.
|
||||
- In this example, unit interval value 0.10 is mapped to Cluster1
|
||||
by both submaps.
|
||||
- Read from latest/largest submap to oldest/smallest submap.
|
||||
- If not found in any submap, search a second time (to handle races
|
||||
with file copying between submaps).
|
||||
- If the requested data is found, optionally copy it directly to the
|
||||
latest submap (as a variation of read repair which really simply
|
||||
accelerates the migration process and can reduce the number of
|
||||
operations required to query servers in multiple submaps).
|
||||
|
||||
The cluster-of-clusters manager is responsible for:
|
||||
|
||||
- Managing the various generations of the CoC Random Slicing maps,
|
||||
including distributing them to CoC clients.
|
||||
- Managing the processes that are responsible for copying "cold" data,
|
||||
i.e., files data that is not regularly accessed, to its new submap
|
||||
location.
|
||||
- When migration of a file to its new cluster is confirmed successful,
|
||||
delete it from the old cluster.
|
||||
|
||||
In example map #7, the CoC manager will copy files with unit interval
|
||||
assignments in ~(0.25,0.33]~, ~(0.58,0.66]~, and ~(0.91,1.00]~ from their
|
||||
old locations in cluster IDs Cluster1/2/3 to their new cluster,
|
||||
Cluster4. When the CoC manager is satisfied that all such files have
|
||||
been copied to Cluster4, then the CoC manager can create and
|
||||
distribute a new map, such as:
|
||||
|
||||
| Generation number | 8 |
|
||||
|-------------------+------------|
|
||||
| SubMap | 1 |
|
||||
|-------------------+------------|
|
||||
| Hash range | Cluster ID |
|
||||
|-------------------+------------|
|
||||
| 0.00 - 0.25 | Cluster1 |
|
||||
| 0.25 - 0.33 | Cluster4 |
|
||||
| 0.33 - 0.58 | Cluster2 |
|
||||
| 0.58 - 0.66 | Cluster4 |
|
||||
| 0.66 - 0.91 | Cluster3 |
|
||||
| 0.91 - 1.00 | Cluster4 |
|
||||
|
||||
One limitation of HibariDB that I haven't fixed is not being able to
|
||||
perform more than one migration at a time. The trade-off is that such
|
||||
migration is difficult enough across two submaps; three or more
|
||||
submaps becomes even more complicated.
|
||||
|
||||
Fortunately for Machi, its file data is immutable and therefore can
|
||||
easily manage many migrations in parallel, i.e., its submap list may
|
||||
be several maps long, each one for an in-progress file migration.
|
||||
|
||||
* Acknowledgements
|
||||
|
||||
The source for the "migration-4.png" and "migration-3to4.png" images
|
||||
come from the [[http://hibari.github.io/hibari-doc/images/migration-3to4.png][HibariDB documentation]].
|
||||
|
103
doc/cluster/migration-3to4.fig
Normal file
103
doc/cluster/migration-3to4.fig
Normal file
|
@ -0,0 +1,103 @@
|
|||
#FIG 3.2 Produced by xfig version 3.2.5b
|
||||
Landscape
|
||||
Center
|
||||
Inches
|
||||
Letter
|
||||
94.00
|
||||
Single
|
||||
-2
|
||||
1200 2
|
||||
6 7425 2700 8700 3300
|
||||
4 0 0 50 -1 2 18 0.0000 4 195 645 7425 2895 After\001
|
||||
4 0 0 50 -1 2 18 0.0000 4 255 1215 7425 3210 Migration\001
|
||||
-6
|
||||
6 7425 450 8700 1050
|
||||
4 0 0 50 -1 2 18 0.0000 4 195 780 7425 675 Before\001
|
||||
4 0 0 50 -1 2 18 0.0000 4 255 1215 7425 990 Migration\001
|
||||
-6
|
||||
6 75 1425 6900 2325
|
||||
6 4875 1425 6900 2325
|
||||
6 5400 1575 6375 2175
|
||||
4 0 0 50 -1 2 14 0.0000 4 165 390 5400 1800 Not\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 225 945 5400 2100 migrated\001
|
||||
-6
|
||||
2 2 1 2 0 7 50 -1 -1 6.000 0 0 -1 0 0 5
|
||||
4950 1500 6825 1500 6825 2250 4950 2250 4950 1500
|
||||
-6
|
||||
6 2475 1425 4500 2325
|
||||
6 3000 1575 3975 2175
|
||||
4 0 0 50 -1 2 14 0.0000 4 165 390 3000 1800 Not\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 225 945 3000 2100 migrated\001
|
||||
-6
|
||||
2 2 1 2 0 7 50 -1 -1 6.000 0 0 -1 0 0 5
|
||||
2550 1500 4425 1500 4425 2250 2550 2250 2550 1500
|
||||
-6
|
||||
6 75 1425 2100 2325
|
||||
6 600 1575 1575 2175
|
||||
4 0 0 50 -1 2 14 0.0000 4 165 390 600 1800 Not\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 225 945 600 2100 migrated\001
|
||||
-6
|
||||
2 2 1 2 0 7 50 -1 -1 6.000 0 0 -1 0 0 5
|
||||
150 1500 2025 1500 2025 2250 150 2250 150 1500
|
||||
-6
|
||||
-6
|
||||
2 1 0 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2
|
||||
1 1 3.00 60.00 120.00
|
||||
150 4200 150 3750
|
||||
2 1 0 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2
|
||||
1 1 3.00 60.00 120.00
|
||||
3750 4200 3750 3750
|
||||
2 1 0 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2
|
||||
1 1 3.00 60.00 120.00
|
||||
2025 4200 2025 3750
|
||||
2 1 0 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2
|
||||
1 1 3.00 60.00 120.00
|
||||
7350 4200 7350 3750
|
||||
2 1 0 2 0 7 50 -1 -1 6.000 0 0 -1 1 0 2
|
||||
1 1 3.00 60.00 120.00
|
||||
5550 4200 5550 3750
|
||||
2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
|
||||
2550 0 2550 1500 150 1500 150 0 2550 0
|
||||
2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
|
||||
4950 0 4950 1500 2550 1500 2550 0 4950 0
|
||||
2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
|
||||
7350 0 7350 1500 4950 1500 4950 0 7350 0
|
||||
2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
|
||||
150 2250 2025 2250 2025 3750 150 3750 150 2250
|
||||
2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
|
||||
4425 2250 4950 2250 4950 3750 4425 3750 4425 2250
|
||||
2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
|
||||
4950 2250 6825 2250 6825 3750 4950 3750 4950 2250
|
||||
2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
|
||||
6825 2250 7350 2250 7350 3750 6825 3750 6825 2250
|
||||
2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
|
||||
2025 2250 2550 2250 2550 3750 2025 3750 2025 2250
|
||||
2 2 0 3 0 7 50 -1 -1 0.000 0 0 -1 0 0 5
|
||||
2550 2250 4425 2250 4425 3750 2550 3750 2550 2250
|
||||
4 0 0 50 -1 2 18 0.0000 4 195 480 75 4500 0.00\001
|
||||
4 0 0 50 -1 2 18 0.0000 4 195 480 6825 4500 1.00\001
|
||||
4 0 0 50 -1 2 18 0.0000 4 195 480 1725 4500 0.25\001
|
||||
4 0 0 50 -1 2 18 0.0000 4 195 480 3525 4500 0.50\001
|
||||
4 0 0 50 -1 2 18 0.0000 4 195 480 5250 4500 0.75\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 240 1710 450 1275 ~33% total keys\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 240 1710 2925 1275 ~33% total keys\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 240 1710 5250 1275 ~33% total keys\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 180 495 2025 3525 ~8%\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 240 1710 300 3525 ~25% total keys\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 240 1710 2625 3525 ~25% total keys\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 180 495 4425 3525 ~8%\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 240 1710 5025 3525 ~25% total keys\001
|
||||
4 0 0 50 -1 2 14 0.0000 4 180 495 6825 3525 ~8%\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 195 2175 3075 4\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 195 4575 3075 4\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 195 6975 3075 4\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 600 600 Chain1\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 3000 600 Chain2\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 5400 600 Chain3\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 285 2100 2625 C\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 285 4500 2625 C\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 285 6900 2625 C\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 525 2850 Chain1\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 2925 2850 Chain2\001
|
||||
4 0 0 50 -1 2 24 0.0000 4 270 1245 5325 2850 Chain3\001
|
||||
4 0 0 50 -1 2 18 0.0000 4 240 4350 1350 4875 Cluster locator, on the unit interval\001
|
BIN
doc/cluster/migration-3to4.png
Normal file
BIN
doc/cluster/migration-3to4.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 7.6 KiB |
BIN
doc/cluster/migration-4.png
Normal file
BIN
doc/cluster/migration-4.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 7.4 KiB |
481
doc/cluster/name-game-sketch.org
Normal file
481
doc/cluster/name-game-sketch.org
Normal file
|
@ -0,0 +1,481 @@
|
|||
-*- mode: org; -*-
|
||||
#+TITLE: Machi cluster "name game" sketch
|
||||
#+AUTHOR: Scott
|
||||
#+STARTUP: lognotedone hidestars indent showall inlineimages
|
||||
#+SEQ_TODO: TODO WORKING WAITING DONE
|
||||
#+COMMENT: M-x visual-line-mode
|
||||
#+COMMENT: Also, disable auto-fill-mode
|
||||
|
||||
* 1. "Name Games" with random-slicing style consistent hashing
|
||||
|
||||
Our goal: to distribute lots of files very evenly across a large
|
||||
collection of individual, small Machi chains.
|
||||
|
||||
* 2. Assumptions
|
||||
|
||||
** Basic familiarity with Machi high level design and Machi's "projection"
|
||||
|
||||
The [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] contains all of the basic
|
||||
background assumed by the rest of this document.
|
||||
|
||||
** Analogy: "neighborhood : city :: Machi chain : Machi cluster"
|
||||
|
||||
Analogy: The word "machi" in Japanese means small town or
|
||||
neighborhood. As the Tokyo Metropolitan Area is built from many
|
||||
machis and smaller cities, therefore a big, partitioned file store can
|
||||
be built out of many small Machi chains.
|
||||
|
||||
** Familiarity with the Machi chain concept
|
||||
|
||||
It's clear (I hope!) from
|
||||
the [[https://github.com/basho/machi/blob/master/doc/high-level-machi.pdf][Machi high level design document]] that Machi alone does not support
|
||||
any kind of file partitioning/distribution/sharding across multiple
|
||||
small Machi chains. There must be another layer above a Machi chain to
|
||||
provide such partitioning services.
|
||||
|
||||
Using the [[https://github.com/basho/machi/tree/master/prototype/demo-day-hack][cluster quick-and-dirty prototype]] as an
|
||||
architecture sketch, let's now assume that we have ~n~ independent Machi
|
||||
chains. We assume that each of these chains has the same
|
||||
chain length in the nominal case, e.g. chain length of 3.
|
||||
We wish to provide partitioned/distributed file storage
|
||||
across all ~n~ chains. We call the entire collection of ~n~ Machi
|
||||
chains a "cluster".
|
||||
|
||||
We may wish to have several types of Machi clusters. For example:
|
||||
|
||||
+ Chain length of 1 for "don't care if it gets lost,
|
||||
store stuff very very cheaply" data.
|
||||
+ Chain length of 2 for normal data.
|
||||
+ Equivalent to quorum replication's reliability with 3 copies.
|
||||
+ Chain length of 7 for critical, unreplaceable data.
|
||||
+ Equivalent to quorum replication's reliability with 15 copies.
|
||||
|
||||
Each of these types of chains will have a name ~N~ in the
|
||||
namespace. The role of the cluster namespace will be demonstrated in
|
||||
Section 3 below.
|
||||
|
||||
** Continue an early assumption: a Machi chain is unaware of clustering
|
||||
|
||||
Let's continue with an assumption that an individual Machi chain
|
||||
inside of a cluster is completely unaware of the cluster layer.
|
||||
|
||||
** The reader is familiar with the random slicing technique
|
||||
|
||||
I'd done something very-very-nearly-like-this for the Hibari database
|
||||
6 years ago. But the Hibari technique was based on stuff I did at
|
||||
Sendmail, Inc, in 2000, so this technique feels like old news to me.
|
||||
{shrug}
|
||||
|
||||
The following section provides an illustrated example.
|
||||
Very quickly, the random slicing algorithm is:
|
||||
|
||||
- Hash a string onto the unit interval [0.0, 1.0)
|
||||
- Calculate h(unit interval point, Map) -> bin, where ~Map~ divides
|
||||
the unit interval into bins (or partitions or shards).
|
||||
|
||||
Machi's adaptation is in step 1: we do not hash any strings. Instead, we
|
||||
simply choose a number on the unit interval. This number is called
|
||||
the "cluster locator number".
|
||||
|
||||
As described later in this doc, Machi file names are structured into
|
||||
several components. One component of the file name contains the cluster
|
||||
locator number; we use the number as-is for step 2 above.
|
||||
|
||||
*** For more information about Random Slicing
|
||||
|
||||
For a comprehensive description of random slicing, please see the
|
||||
first two papers. For a quicker summary, please see the third
|
||||
reference.
|
||||
|
||||
#+BEGIN_QUOTE
|
||||
Reliable and Randomized Data Distribution Strategies for Large Scale Storage Systems
|
||||
Alberto Miranda et al.
|
||||
http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.226.5609
|
||||
(short version, HIPC'11)
|
||||
|
||||
Random Slicing: Efficient and Scalable Data Placement for Large-Scale
|
||||
Storage Systems
|
||||
Alberto Miranda et al.
|
||||
DOI: http://dx.doi.org/10.1145/2632230 (long version, ACM Transactions
|
||||
on Storage, Vol. 10, No. 3, Article 9, 2014)
|
||||
|
||||
[[http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration][Hibari Sysadmin Guide, chain migration section]].
|
||||
http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration
|
||||
#+END_QUOTE
|
||||
|
||||
* 3. A simple illustration
|
||||
|
||||
We use a variation of the Random Slicing hash that we will call
|
||||
~rs_hash_with_float()~. The Erlang-style function type is shown
|
||||
below.
|
||||
|
||||
#+BEGIN_SRC erlang
|
||||
%% type specs, Erlang-style
|
||||
-spec rs_hash_with_float(float(), rs_hash:map()) -> rs_hash:chain_id().
|
||||
#+END_SRC
|
||||
|
||||
I'm borrowing an illustration from the HibariDB documentation here,
|
||||
but it fits my purposes quite well. (I am the original creator of that
|
||||
image, and also the use license is compatible.)
|
||||
|
||||
#+CAPTION: Illustration of 'Map', using four Machi chains
|
||||
|
||||
[[./migration-4.png]]
|
||||
|
||||
Assume that we have a random slicing map called ~Map~. This particular
|
||||
~Map~ maps the unit interval onto 4 Machi chains:
|
||||
|
||||
| Hash range | Chain ID |
|
||||
|-------------+----------|
|
||||
| 0.00 - 0.25 | Chain1 |
|
||||
| 0.25 - 0.33 | Chain4 |
|
||||
| 0.33 - 0.58 | Chain2 |
|
||||
| 0.58 - 0.66 | Chain4 |
|
||||
| 0.66 - 0.91 | Chain3 |
|
||||
| 0.91 - 1.00 | Chain4 |
|
||||
|
||||
Assume that the system chooses a cluster locator of 0.05.
|
||||
According to ~Map~, the value of
|
||||
~rs_hash_with_float(0.05,Map) = Chain1~.
|
||||
Similarly, ~rs_hash_with_float(0.26,Map) = Chain4~.
|
||||
|
||||
This example should look very similar to Hibari's technique.
|
||||
The Hibari documentation has a brief photo illustration of how random
|
||||
slicing works, see [[http://hibari.github.io/hibari-doc/hibari-sysadmin-guide.en.html#chain-migration][Hibari Sysadmin Guide, chain migration]].
|
||||
|
||||
* 4. Use of the cluster namespace: name separation plus chain type
|
||||
|
||||
Let us assume that the cluster framework provides several different types
|
||||
of chains:
|
||||
|
||||
| Chain length | Namespace | Consistency Mode | Comment |
|
||||
|--------------+--------------+------------------+----------------------------------|
|
||||
| 3 | ~normal~ | eventual | Normal storage redundancy & cost |
|
||||
| 2 | ~reduced~ | eventual | Reduced cost storage |
|
||||
| 1 | ~risky~ | eventual | Really, really cheap storage |
|
||||
| 7 | ~paranoid~ | eventual | Safety-critical storage |
|
||||
| 3 | ~sequential~ | strong | Strong consistency |
|
||||
|--------------+--------------+------------------+----------------------------------|
|
||||
|
||||
The client may want to choose the amount of redundancy that its
|
||||
application requires: normal, reduced cost, or perhaps even a single
|
||||
copy. The cluster namespace is used by the client to signal this
|
||||
intention.
|
||||
|
||||
Further, the cluster administrators may wish to use the namespace to
|
||||
provide separate storage for different applications. Jane's
|
||||
application may use the namespace "jane-normal" and Bob's app uses
|
||||
"bob-reduced". Administrators may definine separate groups of
|
||||
chains on separate servers to serve these two applications.
|
||||
|
||||
* 5. In its lifetime, a file may be moved to different chains
|
||||
|
||||
The cluster management scheme may decide that files need to migrate to
|
||||
other chains -- i.e., file that is initially created on chain ID ~X~
|
||||
has been moved to chain ID ~Y~.
|
||||
|
||||
+ For storage load or I/O load balancing reasons.
|
||||
+ Because a chain is being decommissioned by the sysadmin.
|
||||
|
||||
* 6. Floating point is not required ... it is merely convenient for explanation
|
||||
|
||||
NOTE: Use of floating point terms is not required. For example,
|
||||
integer arithmetic could be used, if using a sufficiently large
|
||||
interval to create an even & smooth distribution of hashes across the
|
||||
expected maximum number of chains.
|
||||
|
||||
For example, if the maximum cluster size would be 4,000 individual
|
||||
Machi chains, then a minimum of 12 bits of integer space is required
|
||||
to assign one integer per Machi chain. However, for load balancing
|
||||
purposes, a finer grain of (for example) 100 integers per Machi
|
||||
chain would permit file migration to move increments of
|
||||
approximately 1% of single Machi chain's storage capacity. A
|
||||
minimum of 12+7=19 bits of hash space would be necessary to accommodate
|
||||
these constraints.
|
||||
|
||||
It is likely that Machi's final implementation will choose a 24 bit
|
||||
integer (or perhaps 32 bits) to represent the cluster locator.
|
||||
|
||||
* 7. Proposal: Break the opacity of Machi file names, slightly.
|
||||
|
||||
Machi assigns file names based on:
|
||||
|
||||
~ClientSuppliedPrefix ++ "^" ++ SomeOpaqueFileNameSuffix~
|
||||
|
||||
What if some parts of the system could peek inside of the opaque file name
|
||||
suffix in order to look at the cluster location information that we might
|
||||
code in the filename suffix?
|
||||
|
||||
We break the system into parts that speak two levels of protocols,
|
||||
"high" and "low".
|
||||
|
||||
+ The high level protocol is used outside of the Machi cluster
|
||||
+ The low level protocol is used inside of the Machi cluster
|
||||
|
||||
Both protocols are based on a Protocol Buffers specification and
|
||||
implementation. Other protocols, such as HTTP, will be added later.
|
||||
|
||||
#+BEGIN_SRC
|
||||
+-----------------------+
|
||||
| Machi external client |
|
||||
| e.g. Riak CS |
|
||||
+-----------------------+
|
||||
^
|
||||
| Machi "high" API
|
||||
| ProtoBuffs protocol Machi cluster boundary: outside
|
||||
.........................................................................
|
||||
| Machi cluster boundary: inside
|
||||
v
|
||||
+--------------------------+ +------------------------+
|
||||
| Machi "high" API service | | Machi HTTP API service |
|
||||
+--------------------------+ +------------------------+
|
||||
^ |
|
||||
| +------------------------+
|
||||
v v
|
||||
+------------------------+
|
||||
| Cluster bridge service |
|
||||
+------------------------+
|
||||
^
|
||||
| Machi "low" API
|
||||
| ProtoBuffs protocol
|
||||
+----------------------------------------+----+----+
|
||||
| | | |
|
||||
v v v v
|
||||
+-------------------------+ ... other chains...
|
||||
| Chain C1 (logical view) |
|
||||
| +--------------+ |
|
||||
| | FLU server 1 | |
|
||||
| | +--------------+ |
|
||||
| +--| FLU server 2 | |
|
||||
| +--------------+ | In reality, API bridge talks directly
|
||||
+-------------------------+ to each FLU server in a chain.
|
||||
#+END_SRC
|
||||
|
||||
** The notation we use
|
||||
|
||||
- ~N~ = the cluster namespace, chosen by the client.
|
||||
- ~p~ = file prefix, chosen by the client.
|
||||
- ~L~ = the cluster locator (a number, type is implementation-dependent)
|
||||
- ~Map~ = a mapping of cluster locators to chains
|
||||
- ~T~ = the target chain ID/name
|
||||
- ~u~ = a unique opaque file name suffix, e.g. a GUID string
|
||||
- ~F~ = a Machi file name, i.e., a concatenation of ~p^L^N^u~
|
||||
|
||||
** The details: cluster file append
|
||||
|
||||
0. Cluster client chooses ~N~ and ~p~ (i.e., cluster namespace and
|
||||
file prefix) and sends the append request to a Machi cluster member
|
||||
via the Protocol Buffers "high" API.
|
||||
1. Cluster bridge chooses ~T~ (i.e., target chain), based on criteria
|
||||
such as disk utilization percentage.
|
||||
2. Cluster bridge knows the cluster ~Map~ for namespace ~N~.
|
||||
3. Cluster bridge choose some cluster locator value ~L~ such that
|
||||
~rs_hash_with_float(L,Map) = T~ (see algorithm below).
|
||||
4. Cluster bridge sends its request to chain
|
||||
~T~: ~append_chunk(p,L,N,...) -> {ok,p^L^N^u,ByteOffset}~
|
||||
5. Cluster bridge forwards the reply tuple to the client.
|
||||
6. Client stores/uses the file name ~F = p^L^N^u~.
|
||||
|
||||
** The details: Cluster file read
|
||||
|
||||
0. Cluster client sends the read request to a Machi cluster member via
|
||||
the Protocol Buffers "high" API.
|
||||
1. Cluster bridge parses the file name ~F~ to find
|
||||
the values of ~L~ and ~N~ (recall, ~F = p^L^N^u~).
|
||||
2. Cluster bridge knows the Cluster ~Map~ for type ~N~.
|
||||
3. Cluster bridge calculates ~rs_hash_with_float(L,Map) = T~
|
||||
4. Cluster bridge sends request to chain ~T~:
|
||||
~read_chunk(F,...) ->~ ... reply
|
||||
5. Cluster bridge forwards the reply to the client.
|
||||
|
||||
** The details: calculating 'L' (the cluster locator number) to match a desired target chain
|
||||
|
||||
1. We know ~Map~, the current cluster mapping for a cluster namespace ~N~.
|
||||
2. We look inside of ~Map~, and we find all of the unit interval ranges
|
||||
that map to our desired target chain ~T~. Let's call this list
|
||||
~MapList = [Range1=(start,end],Range2=(start,end],...]~.
|
||||
3. In our example, ~T=Chain2~. The example ~Map~ contains a single
|
||||
unit interval range for ~Chain2~, ~[(0.33,0.58]]~.
|
||||
4. Choose a uniformly random number ~r~ on the unit interval.
|
||||
5. Calculate the cluster locator ~L~ by mapping ~r~ onto the concatenation
|
||||
of the cluster hash space range intervals in ~MapList~. For example,
|
||||
if ~r=0.5~, then ~L = 0.33 + 0.5*(0.58-0.33) = 0.455~, which is
|
||||
exactly in the middle of the ~(0.33,0.58]~ interval.
|
||||
|
||||
** A bit more about the cluster namespaces's meaning and use
|
||||
|
||||
For use by Riak CS, for example, we'd likely start with the following
|
||||
namespaces ... working our way down the list as we add new features
|
||||
and/or re-implement existing CS features.
|
||||
|
||||
- "standard" = Chain length = 3, eventually consistency mode
|
||||
- "reduced" = Chain length = 2, eventually consistency mode.
|
||||
- "stanchion7" = Chain length = 7, strong consistency mode. Perhaps
|
||||
use this namespace for the metadata required to re-implement the
|
||||
operations that are performed by today's Stanchion application.
|
||||
|
||||
We want the cluster framework to:
|
||||
|
||||
- provide means of creating and managing
|
||||
chains of different types, e.g., chain length, consistency mode.
|
||||
- manage the mapping of cluster namespace
|
||||
names to the chains in the system.
|
||||
- provide query functions to map a cluster
|
||||
namespace name to a cluster map,
|
||||
e.g. ~get_cluster_latest_map("reduced") -> Map{generation=7,...}~.
|
||||
|
||||
* 8. File migration (a.k.a. rebalancing/reparitioning/resharding/redistribution)
|
||||
|
||||
** What is "migration"?
|
||||
|
||||
This section describes Machi's file migration. Other storage systems
|
||||
call this process as "rebalancing", "repartitioning", "resharding" or
|
||||
"redistribution".
|
||||
For Riak Core applications, it is called "handoff" and "ring resizing"
|
||||
(depending on the context).
|
||||
See also the [[http://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsUserGuide.html#Balancer][Hadoop file balancer]] for another example of a data
|
||||
migration process.
|
||||
|
||||
As discussed in section 5, the client can have good reason for wanting
|
||||
to have some control of the initial location of the file within the
|
||||
chain. However, the chain manager has an ongoing interest in
|
||||
balancing resources throughout the lifetime of the file. Disks will
|
||||
get full, hardware will change, read workload will fluctuate,
|
||||
etc etc.
|
||||
|
||||
This document uses the word "migration" to describe moving data from
|
||||
one Machi chain to another chain within a cluster system.
|
||||
|
||||
A simple variation of the Random Slicing hash algorithm can easily
|
||||
accommodate Machi's need to migrate files without interfering with
|
||||
availability. Machi's migration task is much simpler due to the
|
||||
immutable nature of Machi file data.
|
||||
|
||||
** Change to Random Slicing
|
||||
|
||||
The map used by the Random Slicing hash algorithm needs a few simple
|
||||
changes to make file migration straightforward.
|
||||
|
||||
- Add a "generation number", a strictly increasing number (similar to
|
||||
a Machi chain's "epoch number") that reflects the history of
|
||||
changes made to the Random Slicing map
|
||||
- Use a list of Random Slicing maps instead of a single map, one map
|
||||
per chance that files may not have been migrated yet out of
|
||||
that map.
|
||||
|
||||
As an example:
|
||||
|
||||
#+CAPTION: Illustration of 'Map', using four Machi chains
|
||||
|
||||
[[./migration-3to4.png]]
|
||||
|
||||
And the new Random Slicing map for some cluster namespace ~N~ might look
|
||||
like this:
|
||||
|
||||
| Generation number / Namespace | 7 / reduced |
|
||||
|-------------------------------+-------------|
|
||||
| SubMap | 1 |
|
||||
|-------------------------------+-------------|
|
||||
| Hash range | Chain ID |
|
||||
|-------------------------------+-------------|
|
||||
| 0.00 - 0.33 | Chain1 |
|
||||
| 0.33 - 0.66 | Chain2 |
|
||||
| 0.66 - 1.00 | Chain3 |
|
||||
|-------------------------------+-------------|
|
||||
| SubMap | 2 |
|
||||
|-------------------------------+-------------|
|
||||
| Hash range | Chain ID |
|
||||
|-------------------------------+-------------|
|
||||
| 0.00 - 0.25 | Chain1 |
|
||||
| 0.25 - 0.33 | Chain4 |
|
||||
| 0.33 - 0.58 | Chain2 |
|
||||
| 0.58 - 0.66 | Chain4 |
|
||||
| 0.66 - 0.91 | Chain3 |
|
||||
| 0.91 - 1.00 | Chain4 |
|
||||
|
||||
When a new Random Slicing map contains a single submap, then its use
|
||||
is identical to the original Random Slicing algorithm. If the map
|
||||
contains multiple submaps, then the access rules change a bit:
|
||||
|
||||
- Write operations always go to the newest/largest submap.
|
||||
- Read operations attempt to read from all unique submaps.
|
||||
- Skip searching submaps that refer to the same chain ID.
|
||||
- In this example, unit interval value 0.10 is mapped to Chain1
|
||||
by both submaps.
|
||||
- Read from newest/largest submap to oldest/smallest submap.
|
||||
- If not found in any submap, search a second time (to handle races
|
||||
with file copying between submaps).
|
||||
- If the requested data is found, optionally copy it directly to the
|
||||
newest submap. (This is a variation of read repair (RR). RR here
|
||||
accelerates the migration process and can reduce the number of
|
||||
operations required to query servers in multiple submaps).
|
||||
|
||||
The cluster manager is responsible for:
|
||||
|
||||
- Managing the various generations of the cluster Random Slicing maps for
|
||||
all namespaces.
|
||||
- Distributing namespace maps to cluster bridges.
|
||||
- Managing the processes that are responsible for copying "cold" data,
|
||||
i.e., files data that is not regularly accessed, to its new submap
|
||||
location.
|
||||
- When migration of a file to its new chain is confirmed successful,
|
||||
delete it from the old chain.
|
||||
|
||||
In example map #7, the cluster manager will copy files with unit interval
|
||||
assignments in ~(0.25,0.33]~, ~(0.58,0.66]~, and ~(0.91,1.00]~ from their
|
||||
old locations in chain IDs Chain1/2/3 to their new chain,
|
||||
Chain4. When the cluster manager is satisfied that all such files have
|
||||
been copied to Chain4, then the cluster manager can create and
|
||||
distribute a new map, such as:
|
||||
|
||||
| Generation number / Namespace | 8 / reduced |
|
||||
|-------------------------------+-------------|
|
||||
| SubMap | 1 |
|
||||
|-------------------------------+-------------|
|
||||
| Hash range | Chain ID |
|
||||
|-------------------------------+-------------|
|
||||
| 0.00 - 0.25 | Chain1 |
|
||||
| 0.25 - 0.33 | Chain4 |
|
||||
| 0.33 - 0.58 | Chain2 |
|
||||
| 0.58 - 0.66 | Chain4 |
|
||||
| 0.66 - 0.91 | Chain3 |
|
||||
| 0.91 - 1.00 | Chain4 |
|
||||
|
||||
The HibariDB system performs data migrations in almost exactly this
|
||||
manner. However, one important
|
||||
limitation of HibariDB is not being able to
|
||||
perform more than one migration at a time. HibariDB's data is
|
||||
mutable. Mutation causes many problems when migrating data
|
||||
across two submaps; three or more submaps was too complex to implement
|
||||
quickly and correctly.
|
||||
|
||||
Fortunately for Machi, its file data is immutable and therefore can
|
||||
easily manage many migrations in parallel, i.e., its submap list may
|
||||
be several maps long, each one for an in-progress file migration.
|
||||
|
||||
* 9. Other considerations for FLU/sequencer implementations
|
||||
|
||||
** Append to existing file when possible
|
||||
|
||||
The sequencer should always assign new offsets to the latest/newest
|
||||
file for any prefix, as long as all prerequisites are also true,
|
||||
|
||||
- The epoch has not changed. (In AP mode, epoch change -> mandatory
|
||||
file name suffix change.)
|
||||
- The cluster locator number is stable.
|
||||
- The latest file for prefix ~p~ is smaller than maximum file size for
|
||||
a FLU's configuration.
|
||||
|
||||
The stability of the cluster locator number is an implementation detail that
|
||||
must be managed by the cluster bridge.
|
||||
|
||||
Reuse of the same file is not possible if the bridge always chooses a
|
||||
different cluster locator number ~L~ or if the client always uses a unique
|
||||
file prefix ~p~. The latter is a sign of a misbehaved client; the
|
||||
former is a poorly-implemented bridge.
|
||||
|
||||
* 10. Acknowledgments
|
||||
|
||||
The original source for the "migration-4.png" and "migration-3to4.png" images
|
||||
come from the [[http://hibari.github.io/hibari-doc/images/migration-3to4.png][HibariDB documentation]].
|
||||
|
30
doc/dev-clone-compile.md
Normal file
30
doc/dev-clone-compile.md
Normal file
|
@ -0,0 +1,30 @@
|
|||
# Clone and compile Machi
|
||||
|
||||
Clone the Machi source repo and compile the source and test code. Run
|
||||
the following commands at your login shell:
|
||||
|
||||
cd /tmp
|
||||
git clone https://github.com/basho/machi.git
|
||||
cd machi
|
||||
git checkout master
|
||||
make # or 'gmake' if GNU make uses an alternate name
|
||||
|
||||
Then run the unit test suite. This may take up to two minutes or so
|
||||
to finish.
|
||||
|
||||
make test
|
||||
|
||||
At the end, the test suite should report that all tests passed. The
|
||||
actual number of tests shown in the "All `X` tests passed" line may be
|
||||
different than the example below.
|
||||
|
||||
[... many lines omitted ...]
|
||||
module 'event_logger'
|
||||
module 'chain_mgr_legacy'
|
||||
=======================================================
|
||||
All 90 tests passed.
|
||||
|
||||
If you had a test failure, a likely cause may be a limit on the number
|
||||
of file descriptors available to your user process. (Recent releases
|
||||
of OS X have a limit of 1024 file descriptors, which may be too slow.)
|
||||
The output of the `limit -n` will tell you your file descriptor limit.
|
38
doc/dev-prerequisites.md
Normal file
38
doc/dev-prerequisites.md
Normal file
|
@ -0,0 +1,38 @@
|
|||
## Machi developer environment prerequisites
|
||||
|
||||
1. Machi requires an 64-bit variant of UNIX: OS X, FreeBSD, Linux, or
|
||||
Solaris machine is a standard developer environment for C and C++
|
||||
applications (64-bit versions).
|
||||
2. You'll need the `git` source management utility.
|
||||
3. You'll need the 64-bit Erlang/OTP 17 runtime environment. Please
|
||||
don't use earlier or later versions until we have a chance to fix
|
||||
the compilation warnings that versions R16B and 18 will trigger.
|
||||
Also, please verify that you are not using a 32-bit Erlang/OTP
|
||||
runtime package.
|
||||
|
||||
For `git` and the Erlang runtime, please use your OS-specific
|
||||
package manager to install these. If your package manager doesn't
|
||||
have 64-bit Erlang/OTP version 17 available, then we recommend using the
|
||||
[precompiled packages available at Erlang Solutions](https://www.erlang-solutions.com/resources/download.html).
|
||||
|
||||
Also, please verify that you have enough file descriptors available to
|
||||
your user processes. The output of `ulimit -n` should report at least
|
||||
4,000 file descriptors available. If your limit is lower (a frequent
|
||||
problem for OS X users), please increase it to at least 4,000.
|
||||
|
||||
# Using Vagrant to set up a developer environment for Machi
|
||||
|
||||
The Machi source directory contains a `Vagrantfile` for creating an
|
||||
Ubuntu Linux-based virtual machine for compiling and running Machi.
|
||||
This file is in the
|
||||
[$SRC_TOP/priv/humming-consensus-demo.vagrant](../priv/humming-consensus-demo.vagrant)
|
||||
directory.
|
||||
|
||||
If used as-is, the virtual machine specification is modest.
|
||||
|
||||
* 1 virtual CPU
|
||||
* 512MB virtual memory
|
||||
* 768MB swap space
|
||||
* 79GB sparse virtual disk image. After installing prerequisites and
|
||||
compiling Machi, the root file system uses approximately 2.7 GBytes.
|
||||
|
617
doc/flu-and-chain-lifecycle.org
Normal file
617
doc/flu-and-chain-lifecycle.org
Normal file
|
@ -0,0 +1,617 @@
|
|||
FLU and Chain Life Cycle Management -*- mode: org; -*-
|
||||
#+STARTUP: lognotedone hidestars indent showall inlineimages
|
||||
#+COMMENT: To generate the outline section: egrep '^\*[*]* ' doc/flu-and-chain-lifecycle.org | egrep -v '^\* Outline' | sed -e 's/^\*\*\* / + /' -e 's/^\*\* / + /' -e 's/^\* /+ /'
|
||||
|
||||
* FLU and Chain Life Cycle Management
|
||||
|
||||
In an ideal world, we (the Machi development team) would have a full
|
||||
vision of how Machi would be managed, down to the last detail of
|
||||
beautiful CLI character and network protocol bit. Our vision isn't
|
||||
complete yet, so we are working one small step at a time.
|
||||
|
||||
* Outline
|
||||
|
||||
+ FLU and Chain Life Cycle Management
|
||||
+ Terminology review
|
||||
+ Terminology: Machi run-time components/services/thingies
|
||||
+ Terminology: Machi chain data structures
|
||||
+ Terminology: Machi cluster data structures
|
||||
+ Overview of administrative life cycles
|
||||
+ Cluster administrative life cycle
|
||||
+ Chain administrative life cycle
|
||||
+ FLU server administrative life cycle
|
||||
+ Quick admin: declarative management of Machi FLU and chain life cycles
|
||||
+ Quick admin uses the "rc.d" config scheme for life cycle management
|
||||
+ Quick admin's declarative "language": an Erlang-flavored AST
|
||||
+ Term 'host': define a new host for FLU services
|
||||
+ Term 'flu': define a new FLU
|
||||
+ Term 'chain': define or reconfigure a chain
|
||||
+ Executing quick admin AST files via the 'machi-admin' utility
|
||||
+ Checking the syntax of an AST file
|
||||
+ Executing an AST file
|
||||
+ Using quick admin to manage multiple machines
|
||||
+ The "rc.d" style configuration file scheme
|
||||
+ Riak had a similar configuration file editing problem (and its solution)
|
||||
+ Machi's "rc.d" file scheme.
|
||||
+ FLU life cycle management using "rc.d" style files
|
||||
+ The key configuration components of a FLU
|
||||
+ Chain life cycle management using "rc.d" style files
|
||||
+ The key configuration components of a chain
|
||||
|
||||
* Terminology review
|
||||
|
||||
** Terminology: Machi run-time components/services/thingies
|
||||
|
||||
+ FLU: a basic Machi server, responsible for managing a collection of
|
||||
files.
|
||||
|
||||
+ Chain: a small collection of FLUs that maintain replicas of the same
|
||||
collection of files. A chain is usually small, 1-3 servers, where
|
||||
more than 3 would be used only in cases when availability of
|
||||
certain data is critical despite failures of several machines.
|
||||
+ The length of a chain is directly proportional to its
|
||||
replication factor, e.g., a chain length=3 will maintain
|
||||
(nominally) 3 replicas of each file.
|
||||
+ To maintain file availability when ~F~ failures have occurred, a
|
||||
chain must be at least ~F+1~ members long. (In comparison, the
|
||||
quorum replication technique requires ~2F+1~ members in the
|
||||
general case.)
|
||||
|
||||
+ Cluster: A collection of Machi chains that are used to store files
|
||||
in a horizontally partitioned/sharded/distributed manner.
|
||||
|
||||
** Terminology: Machi data structures
|
||||
|
||||
+ Projection: used to define a single chain: the chain's consistency
|
||||
mode (strong or eventual consistency), all members (from an
|
||||
administrative point of view), all active members (from a runtime,
|
||||
automatically-managed point of view), repairing/file-syncing members
|
||||
(also runtime, auto-managed), and so on
|
||||
|
||||
+ Epoch: A version number of a projection. The epoch number is used
|
||||
by both clients & servers to manage transitions from one projection
|
||||
to another, e.g., when the chain is temporarily shortened by the
|
||||
failure of a member FLU server.
|
||||
|
||||
** Terminology: Machi cluster data structures
|
||||
|
||||
+ Namespace: A collection of human-friendly names that are mapped to
|
||||
groups of Machi chains that provide the same type of storage
|
||||
service: consistency mode, replication policy, etc.
|
||||
+ A single namespace name, e.g. ~normal-ec~, is paired with a single
|
||||
cluster map (see below).
|
||||
+ Example: ~normal-ec~ might be a collection of Machi chains in
|
||||
eventually-consistent mode that are of length=3.
|
||||
+ Example: ~risky-ec~ might be a collection of Machi chains in
|
||||
eventually-consistent mode that are of length=1.
|
||||
+ Example: ~mgmt-critical~ might be a collection of Machi chains in
|
||||
strongly-consistent mode that are of length=7.
|
||||
|
||||
+ Cluster map: Encodes the rules which partition/shard/distribute
|
||||
the files stored in a particular namespace across a group of chains
|
||||
that collectively store the namespace's files.
|
||||
|
||||
+ Chain weight: A value assigned to each chain within a cluster map
|
||||
structure that defines the relative storage capacity of a chain
|
||||
within the namespace. For example, a chain weight=150 has 50% more
|
||||
capacity than a chain weight=100.
|
||||
|
||||
+ Cluster map epoch: The version number assigned to a cluster map.
|
||||
|
||||
* Overview of administrative life cycles
|
||||
|
||||
** Cluster administrative life cycle
|
||||
|
||||
+ Cluster is first created
|
||||
+ Adds namespaces (e.g. consistency policy + chain length policy) to
|
||||
the cluster
|
||||
+ Chains are added to/removed from a namespace to increase/decrease the
|
||||
namespace's storage capacity.
|
||||
+ Adjust chain weights within a namespace, e.g., to shift files
|
||||
within the namespace to chains with greater storage capacity
|
||||
resources and/or runtime I/O resources.
|
||||
|
||||
A cluster "file migration" is the process of moving files from one
|
||||
namespace member chain to another for purposes of shifting &
|
||||
re-balancing storage capacity and/or runtime I/O capacity.
|
||||
|
||||
** Chain administrative life cycle
|
||||
|
||||
+ A chain is created with an initial FLU membership list.
|
||||
+ Chain may be administratively modified zero or more times to
|
||||
add/remove member FLU servers.
|
||||
+ A chain may be decommissioned.
|
||||
|
||||
See also: http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html
|
||||
|
||||
** FLU server administrative life cycle
|
||||
|
||||
+ A FLU is created after an administrator chooses the FLU's runtime
|
||||
location is selected by the administrator: which machine/virtual
|
||||
machine, IP address and TCP port allocation, etc.
|
||||
+ An unassigned FLU may be added to a chain by chain administrative
|
||||
policy.
|
||||
+ A FLU that is assigned to a chain may be removed from that chain by
|
||||
chain administrative policy.
|
||||
+ In the current implementation, the FLU's Erlang processes will be
|
||||
halted. Then the FLU's data and metadata files will be moved to
|
||||
another area of the disk for safekeeping. Later, a "garbage
|
||||
collection" process can be used for reclaiming disk space used by
|
||||
halted FLU servers.
|
||||
|
||||
See also: http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html
|
||||
|
||||
* Quick admin: declarative management of Machi FLU and chain life cycles
|
||||
|
||||
The "quick admin" scheme is a temporary (?) tool for managing Machi
|
||||
FLU server and chain life cycles in a declarative manner. The API is
|
||||
described in this section.
|
||||
|
||||
** Quick admin uses the "rc.d" config scheme for life cycle management
|
||||
|
||||
As described at the top of
|
||||
http://basho.github.io/machi/edoc/machi_lifecycle_mgr.html, the "rc.d"
|
||||
config files do not manage "policy". "Policy" is doing the right
|
||||
thing with a Machi cluster from a systems administrator's
|
||||
point of view. The "rc.d" config files can only implement decisions
|
||||
made according to policy.
|
||||
|
||||
The "quick admin" tool is a first attempt at automating policy
|
||||
decisions in a safe way (we hope) that is also easy to implement (we
|
||||
hope) with a variety of systems management tools, e.g. Chef, Puppet,
|
||||
Ansible, Saltstack, or plain-old-human-at-a-keyboard.
|
||||
|
||||
** Quick admin's declarative "language": an Erlang-flavored AST
|
||||
|
||||
The "language" that an administrator uses to express desired policy
|
||||
changes is not (yet) a true language. As a quick implementation hack,
|
||||
the current language is an Erlang-flavored abstract syntax tree
|
||||
(AST). The tree isn't very deep, either, frequently just one
|
||||
element tall. (Not much of a tree, is it?)
|
||||
|
||||
There are three terms in the language currently:
|
||||
|
||||
+ ~host~, define a new host that can execute FLU servers
|
||||
+ ~flu~, define a new FLU
|
||||
+ ~chain~, define a new chain or re-configure an existing chain with
|
||||
the same name
|
||||
|
||||
*** Term 'host': define a new host for FLU services
|
||||
|
||||
In this context, a host is a machine, virtual machine, or container
|
||||
that can execute the Machi application and can therefore provide FLU
|
||||
services, i.e. file service, Humming Consensus management.
|
||||
|
||||
Two formats may be used to define a new host:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{host, Name, Props}.
|
||||
{host, Name, AdminI, ClientI, Props}.
|
||||
#+END_SRC
|
||||
|
||||
The shorter tuple is shorthand notation for the latter. If the
|
||||
shorthand form is used, then it will be converted automatically to the
|
||||
long form as:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{host, Name, AdminI=Name, ClientI=Name, Props}.
|
||||
#+END_SRC
|
||||
|
||||
Type information, description, and restrictions:
|
||||
|
||||
+ ~Name::string()~ The ~Name~ attribute must be unique. Note that it
|
||||
is possible to define two different hosts, one using a DNS hostname
|
||||
and one using an IP address. The user must avoid this
|
||||
double-definition because it is not enforced by quick admin.
|
||||
+ The ~Name~ field is used for cross-reference purposes with other
|
||||
terms, e.g., ~flu~ and ~chain~.
|
||||
+ There is no syntax yet for removing a host definition.
|
||||
|
||||
+ ~AdminI::string()~ A DNS hostname or IP address for cluster
|
||||
administration purposes, e.g. SSH access.
|
||||
+ This field is unused at the present time.
|
||||
|
||||
+ ~ClientI::string()~ A DNS hostname or IP address for Machi's client
|
||||
protocol access, e.g., Protocol Buffers network API service.
|
||||
+ This field is unused at the present time.
|
||||
|
||||
+ ~props::proplist()~ is an Erlang-style property list for specifying
|
||||
additional configuration options, debugging information, sysadmin
|
||||
comments, etc.
|
||||
|
||||
+ A full-featured admin tool should also include managing several
|
||||
other aspects of configuration related to a "host". For example,
|
||||
for any single IP address, quick admin assumes that there will be
|
||||
exactly one Erlang VM that is running the Machi application. Of
|
||||
course, it is possible to have dozens of Erlang VMs on the same
|
||||
(let's assume for clarity) hardware machine and all running Machi
|
||||
... but there are additional aspects of such a machine that quick
|
||||
admin does not account for
|
||||
+ multiple IP addresses per machine
|
||||
+ multiple Machi package installation paths
|
||||
+ multiple Machi config files (e.g. cuttlefish config, ~etc.conf~,
|
||||
~vm.args~)
|
||||
+ multiple data directories/file system mount points
|
||||
+ This is also a management problem for quick admin for a single
|
||||
Machi package on a machine to take advantage of bulk data
|
||||
storage using multiple multiple file system mount points.
|
||||
+ multiple Erlang VM host names, required for distributed Erlang,
|
||||
which is used for communication with ~machi~ and ~machi-admin~
|
||||
command line utilities.
|
||||
+ and others....
|
||||
|
||||
*** Term 'flu': define a new FLU
|
||||
|
||||
A new FLU is defined relative to a previously-defined ~host~ entities;
|
||||
an exception will be thrown if the ~host~ cannot be cross-referenced.
|
||||
|
||||
#+BEGIN_SRC
|
||||
{flu, Name, HostName, Port, Props}
|
||||
#+END_SRC
|
||||
|
||||
Type information, description, and restrictions:
|
||||
|
||||
+ ~Name::atom()~ The name of the FLU, as a human-friendly name and
|
||||
also for internal management use; please note the ~atom()~ type.
|
||||
This name must be unique.
|
||||
+ The ~Name~ field is used for cross-reference purposes with the
|
||||
~chain~ term.
|
||||
+ There is no syntax yet for removing a FLU definition.
|
||||
|
||||
+ ~Hostname::string()~ The cross-reference name of the ~host~ that
|
||||
this FLU should run on.
|
||||
|
||||
+ ~Port::non_neg_integer()~ The TCP port used by this FLU server's
|
||||
Protocol Buffers network API listener service
|
||||
|
||||
+ ~props::proplist()~ is an Erlang-style property list for specifying
|
||||
additional configuration options, debugging information, sysadmin
|
||||
comments, etc.
|
||||
|
||||
*** Term 'chain': define or reconfigure a chain
|
||||
|
||||
A chain is defined relative to zero or more previously-defined ~flu~
|
||||
entities; an exception will be thrown if any ~flu~ cannot be
|
||||
cross-referenced.
|
||||
|
||||
Two formats may be used to define/reconfigure a chain:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{chain, Name, FullList, Props}.
|
||||
{chain, Name, CMode, FullList, Witnesses, Props}.
|
||||
#+END_SRC
|
||||
|
||||
The shorter tuple is shorthand notation for the latter. If the
|
||||
shorthand form is used, then it will be converted automatically to the
|
||||
long form as:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{chain, Name, ap_mode, FullList, [], Props}.
|
||||
#+END_SRC
|
||||
|
||||
Type information, description, and restrictions:
|
||||
|
||||
+ ~Name::atom()~ The name of the chain, as a human-friendly name and
|
||||
also for internal management use; please note the ~atom()~ type.
|
||||
This name must be unique.
|
||||
+ There is no syntax yet for removing a chain definition.
|
||||
|
||||
+ ~CMode::'ap_mode'|'cp_mode'~ Defines the consistency mode of the
|
||||
chain, either eventual consistency or strong consistency,
|
||||
respectively.
|
||||
+ A chain cannot change consistency mode, e.g., from
|
||||
strong~->~eventual consistency.
|
||||
|
||||
+ ~FullList::list(atom())~ Specifies the list of full-service FLU
|
||||
servers, i.e. servers that provide file data & metadata services as
|
||||
well as Humming Consensus. Each atom in the list must
|
||||
cross-reference with a previously defined ~chain~; an exception will
|
||||
be thrown if any ~flu~ cannot be cross-referenced.
|
||||
|
||||
+ ~Witnesses::list(atom())~ Specifies the list of witness-only
|
||||
servers, i.e. servers that only participate in Humming Consensus.
|
||||
Each atom in the list must cross-reference with a previously defined
|
||||
~chain~; an exception will be thrown if any ~flu~ cannot be
|
||||
cross-referenced.
|
||||
+ This list must be empty for eventual consistency chains.
|
||||
|
||||
+ ~props::proplist()~ is an Erlang-style property list for specifying
|
||||
additional configuration options, debugging information, sysadmin
|
||||
comments, etc.
|
||||
|
||||
+ If this term specifies a new ~chain~ name, then all of the member
|
||||
FLU servers (full & witness types) will be bootstrapped to a
|
||||
starting configuration.
|
||||
|
||||
+ If this term specifies a previously-defined ~chain~ name, then all
|
||||
of the member FLU servers (full & witness types, respectively) will
|
||||
be adjusted to add or remove members, as appropriate.
|
||||
+ Any FLU servers added to either list must not be assigned to any
|
||||
other chain, or they must be a member of this specific chain.
|
||||
+ Any FLU servers removed from either list will be halted.
|
||||
(See the "FLU server administrative life cycle" section above.)
|
||||
|
||||
** Executing quick admin AST files via the 'machi-admin' utility
|
||||
|
||||
Examples of quick admin AST files can be found in the
|
||||
~priv/quick-admin/examples~ directory. Below is an example that will
|
||||
define a new host ( ~"localhost"~ ), three new FLU servers ( ~f1~ & ~f2~
|
||||
and ~f3~ ), and an eventually consistent chain ( ~c1~ ) that uses the new
|
||||
FLU servers:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{host, "localhost", []}.
|
||||
{flu,f1,"localhost",20401,[]}.
|
||||
{flu,f2,"localhost",20402,[]}.
|
||||
{flu,f3,"localhost",20403,[]}.
|
||||
{chain,c1,[f1,f2,f3],[]}.
|
||||
#+END_SRC
|
||||
|
||||
*** Checking the syntax of an AST file
|
||||
|
||||
Given an AST config file, ~/path/to/ast/file~, its basic syntax and
|
||||
correctness can be checked without executing it.
|
||||
|
||||
#+BEGIN_SRC
|
||||
./rel/machi/bin/machi-admin quick-admin-check /path/to/ast/file
|
||||
#+END_SRC
|
||||
|
||||
+ The utility will exit with status zero and output ~ok~ if the syntax
|
||||
and proposed configuration appears to be correct.
|
||||
+ If there is an error, the utility will exit with status one, and an
|
||||
error message will be printed.
|
||||
|
||||
*** Executing an AST file
|
||||
|
||||
Given an AST config file, ~/path/to/ast/file~, it can be executed
|
||||
using the command:
|
||||
|
||||
#+BEGIN_SRC
|
||||
./rel/machi/bin/machi-admin quick-admin-apply /path/to/ast/file RelativeHost
|
||||
#+END_SRC
|
||||
|
||||
... where the last argument, ~RelativeHost~, should be the exact
|
||||
spelling of one of the previously defined AST ~host~ entities,
|
||||
*and also* is the same host that the ~machi-admin~ utility is being
|
||||
executed on.
|
||||
|
||||
Restrictions and warnings:
|
||||
|
||||
+ This is alpha quality software.
|
||||
|
||||
+ There is no "undo".
|
||||
+ Of course there is, but you need to resort to doing things like
|
||||
using ~machi attach~ to attach to the server's CLI to then execute
|
||||
magic Erlang incantations to stop FLUs, unconfigure chains, etc.
|
||||
+ Oh, and delete some files with magic paths, also.
|
||||
|
||||
** Using quick admin to manage multiple machines
|
||||
|
||||
A quick sketch follows:
|
||||
|
||||
1. Create the AST file to specify all of the changes that you wish to
|
||||
make to all hosts, FLUs, and/or chains, e.g., ~/tmp/ast.txt~.
|
||||
2. Check the basic syntax with the ~quick-admin-check~ argument to
|
||||
~machi-admin~.
|
||||
3. If the syntax is good, then copy ~/tmp/ast.txt~ to all hosts in the
|
||||
cluster, using the same path, ~/tmp/ast.txt~.
|
||||
4. For each machine in the cluster, run:
|
||||
#+BEGIN_SRC
|
||||
./rel/machi/bin/machi-admin quick-admin-apply /tmp/ast.txt RelativeHost
|
||||
#+END_SRC
|
||||
|
||||
... where RelativeHost is the AST ~host~ name of the machine that you
|
||||
are executing the ~machi-admin~ command on. The command should be
|
||||
successful, with exit status 0 and outputting the string ~ok~.
|
||||
|
||||
Finally, for each machine in the cluster, a listing of all files in
|
||||
the directory ~rel/machi/etc/quick-admin-archive~ should show exactly
|
||||
the same files, one for each time that ~quick-admin-apply~ has been
|
||||
run successfully on that machine.
|
||||
|
||||
* The "rc.d" style configuration file scheme
|
||||
|
||||
This configuration scheme is inspired by BSD UNIX's ~init(8)~ process
|
||||
manager's configuration style, called "rc.d" after the name of the
|
||||
directory where these files are stored, ~/etc/rc.d~. The ~init~
|
||||
process is responsible for (among other things) starting UNIX
|
||||
processes at machine boot time and stopping them when the machine is
|
||||
shut down.
|
||||
|
||||
The original scheme used by ~init~ to start processes at boot time was
|
||||
a single Bourne shell script called ~/etc/rc~. When a new software
|
||||
package was installed that required a daemon to be started at boot
|
||||
time, text was added to the ~/etc/rc~ file. Uninstalling packages was
|
||||
much trickier, because it meant removing lines from a file that
|
||||
*is a computer program (run by the Bourne shell, a Turing-complete
|
||||
programming language)*. Error-free editing of the ~/etc/rc~ script
|
||||
was impossible in all cases.
|
||||
|
||||
Later, ~init~'s configuration was split into a few master Bourne shell
|
||||
scripts and a subdirectory, ~/etc/rc.d~. The subdirectory contained
|
||||
shell scripts that were responsible for boot time starting of a single
|
||||
daemon or service, e.g. NFS or an HTTP server. When a new software
|
||||
package was added, a new file was added to the ~rc.d~ subdirectory.
|
||||
When a package was removed, the corresponding file in ~rc.d~ was
|
||||
removed. With this simple scheme, addition & removal of boot time
|
||||
scripts was vastly simplified.
|
||||
|
||||
** Riak had a similar configuration file editing problem (and its solution)
|
||||
|
||||
Another software product from Basho Technologies, Riak, had a similar
|
||||
configuration file editing problem. One file in particular,
|
||||
~app.config~, had a syntax that made it difficult both for human
|
||||
systems administrators and also computer programs to edit the file in
|
||||
a syntactically correct manner.
|
||||
|
||||
Later releases of Riak switched to an alternative configuration file
|
||||
format, one inspired by the BSD UNIX ~sysctl(8)~ utility and
|
||||
~sysctl.conf(5)~ file syntax. The ~sysctl.conf~ format is much easier
|
||||
to manage by computer programs to add items. Removing items is not
|
||||
100% simple, however: the correct lines must be identified and then
|
||||
removed (e.g. with Perl or a text editor or combination of ~grep -v~
|
||||
and ~mv~), but removing any comment lines that "belong" to the removed
|
||||
config item(s) is not any easy for a 1-line shell script to do 100%
|
||||
correctly.
|
||||
|
||||
Machi will use the ~sysctl.conf~ style configuration for some
|
||||
application configuration variables. However, adding & removing FLUs
|
||||
and chains will be managed using the "rc.d" style because of the
|
||||
"rc.d" scheme's simplicity and tolerance of mistakes by administrators
|
||||
(human or computer).
|
||||
|
||||
** Machi's "rc.d" file scheme.
|
||||
|
||||
Machi will use a single subdirectory that will contain configuration
|
||||
files for some life cycle management task, e.g. a single FLU or a
|
||||
single chain.
|
||||
|
||||
The contents of the file should be a single Erlang term, serialized in
|
||||
ASCII form as Erlang source code statement, i.e. a single Erlang term
|
||||
~T~ that is formatted by ~io:format("~w.",[T]).~. This file must be
|
||||
parseable by the Erlang function ~file:consult()~.
|
||||
|
||||
Later versions of Machi may change the file format to be more familiar
|
||||
to administrators who are unaccustomed to Erlang language syntax.
|
||||
|
||||
** FLU life cycle management using "rc.d" style files
|
||||
|
||||
*** The key configuration components of a FLU
|
||||
|
||||
1. The machine (or virtual machine) to run it on.
|
||||
2. The Machi software package's artifacts to execute.
|
||||
3. The disk device(s) used to store Machi file data & metadata, "rc.d"
|
||||
style config files, etc.
|
||||
4. The name, IP address and TCP port assigned to the FLU service.
|
||||
5. Its chain assignment.
|
||||
|
||||
Notes:
|
||||
|
||||
+ Items 1-3 are currently outside of the scope of this life cycle
|
||||
document. We assume that human administrators know how to do these
|
||||
things.
|
||||
+ Item 4's properties are explicitly managed by a FLU-defining "rc.d"
|
||||
style config file.
|
||||
+ Item 5 is managed by the chain life cycle management system.
|
||||
|
||||
Here is an example of a properly formatted FLU config file:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{p_srvr,f1,machi_flu1_client,"192.168.72.23",20401,[]}.
|
||||
#+END_SRC
|
||||
|
||||
... which corresponds to the following Erlang record definition:
|
||||
|
||||
#+BEGIN_SRC
|
||||
-record(p_srvr, {
|
||||
name :: atom(),
|
||||
proto_mod = 'machi_flu1_client' :: atom(), % Module name
|
||||
address :: term(), % Protocol-specific
|
||||
port :: term(), % Protocol-specific
|
||||
props = [] :: list() % proplist for other related info
|
||||
}).
|
||||
#+END_SRC
|
||||
|
||||
+ ~name~ is ~f1~. This is name of the FLU. This name should be
|
||||
unique over the lifetime of the administrative domain and thus
|
||||
managed by external policy. This name must be the same as the name
|
||||
of the config file that defines the FLU.
|
||||
+ ~proto_mod~ is used for internal management purposes and should be
|
||||
considered a mandatory constant.
|
||||
+ ~address~ is "192.168.72.23". The DNS hostname or IP address used
|
||||
by other servers to communicate with this FLU. This must be a valid
|
||||
IP address, previously assigned to this machine/VM using the
|
||||
appropriate operating system-specific procedure.
|
||||
+ ~port~ is TCP port 20401. The TCP port number that the FLU listens
|
||||
to for incoming Protocol Buffers-serialized communication. This TCP
|
||||
port must not be in use (now or in the future) by another Machi FLU
|
||||
or any other process running on this machine/VM.
|
||||
+ ~props~ is an Erlang-style property list for specifying additional
|
||||
configuration options, debugging information, sysadmin comments,
|
||||
etc.
|
||||
|
||||
** Chain life cycle management using "rc.d" style files
|
||||
|
||||
Unlike FLUs, chains have a self-management aspect that makes a chain
|
||||
life cycle different from a single FLU server. Machi's chains are
|
||||
self-managing, via Humming Consensus; see the
|
||||
https://github.com/basho/machi/tree/master/doc/ directory for much
|
||||
more detail about Humming Consensus. After FLUs have received their
|
||||
initial chain configuration for Humming Consensus, the FLUs will
|
||||
manage the chain (and each other) by themselves.
|
||||
|
||||
However, Humming Consensus does not handle three chain management
|
||||
problems:
|
||||
|
||||
1. Specifying the very first chain configuration,
|
||||
2. Altering the membership of the chain (i.e. adding/removing FLUs
|
||||
from the chain),
|
||||
3. Stopping the chain permanently.
|
||||
|
||||
A chain "rc.d" file will only be used to bootstrap a newly-defined FLU
|
||||
server. It's like a piece of glue information to introduce the new
|
||||
FLU to the Humming Consensus group that is managing the chain's
|
||||
dynamic state (e.g. which members are up or down). In all other
|
||||
respects, chain config files are ignored by life cycle management code.
|
||||
However, to mimic the life cycle of the FLU server's "rc.d" config
|
||||
files, a chain "rc.d" files is not deleted until the chain has been
|
||||
decommissioned (i.e. defined with length=0).
|
||||
|
||||
*** The key configuration components of a chain
|
||||
|
||||
1. The name of the chain.
|
||||
2. Consistency mode: eventually consistent or strongly consistent.
|
||||
3. The membership list of all FLU servers in the chain.
|
||||
+ Remember, all servers in a single chain will manage full replicas
|
||||
of the same collection of Machi files.
|
||||
4. If the chain is defined to use strongly consistent mode, then a
|
||||
list of "witness servers" may also be defined. See the
|
||||
[https://github.com/basho/machi/tree/master/doc/] documentation for
|
||||
more information on witness servers.
|
||||
+ The witness list must be empty for all chains in eventual
|
||||
consistency mode.
|
||||
|
||||
Here is an example of a properly formatted chain config file:
|
||||
|
||||
#+BEGIN_SRC
|
||||
{chain_def_v1,c1,ap_mode,
|
||||
[{p_srvr,f1,machi_flu1_client,"localhost",20401,[]},
|
||||
{p_srvr,f2,machi_flu1_client,"localhost",20402,[]},
|
||||
{p_srvr,f3,machi_flu1_client,"localhost",20403,[]}],
|
||||
[],[],[],
|
||||
[f1,f2,f3],
|
||||
[],[]}.
|
||||
#+END_SRC
|
||||
|
||||
... which corresponds to the following Erlang record definition:
|
||||
|
||||
#+BEGIN_SRC
|
||||
-record(chain_def_v1, {
|
||||
name :: atom(), % chain name
|
||||
mode :: 'ap_mode' | 'cp_mode',
|
||||
full = [] :: [p_srvr()],
|
||||
witnesses = [] :: [p_srvr()],
|
||||
old_full = [] :: [atom()], % guard against some races
|
||||
old_witnesses=[] :: [atom()], % guard against some races
|
||||
local_run = [] :: [atom()], % must be tailored to each machine!
|
||||
local_stop = [] :: [atom()], % must be tailored to each machine!
|
||||
props = [] :: list() % proplist for other related info
|
||||
}).
|
||||
#+END_SRC
|
||||
|
||||
+ ~name~ is ~c1~, the name of the chain. This name should be unique
|
||||
over the lifetime of the administrative domain and thus managed by
|
||||
external policy. This name must be the same as the name of the
|
||||
config file that defines the chain.
|
||||
+ ~mode~ is ~ap_mode~, an internal code symbol for eventual
|
||||
consistency mode.
|
||||
+ ~full~ is a list of Erlang ~#p_srvr{}~ records for full-service
|
||||
members of the chain, i.e., providing Machi file data & metadata
|
||||
storage services.
|
||||
+ ~witnesses~ is a list of Erlang ~#p_srvr{}~ records for witness-only
|
||||
FLU servers, i.e., providing only Humming Consensus service.
|
||||
+ The next four fields are used for internal management only.
|
||||
+ ~props~ is an Erlang-style property list for specifying additional
|
||||
configuration options, debugging information, sysadmin comments,
|
||||
etc.
|
||||
|
Binary file not shown.
372
doc/humming-consensus-demo.md
Normal file
372
doc/humming-consensus-demo.md
Normal file
|
@ -0,0 +1,372 @@
|
|||
|
||||
# Table of contents
|
||||
|
||||
* [Hands-on experiments with Machi and Humming Consensus](#hands-on)
|
||||
* [Using the network partition simulator and convergence demo test code](#partition-simulator)
|
||||
|
||||
<a name="hands-on">
|
||||
# Hands-on experiments with Machi and Humming Consensus
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Please refer to the
|
||||
[Machi development environment prerequisites doc](./dev-prerequisites.md)
|
||||
for Machi developer environment prerequisites.
|
||||
|
||||
If you do not have an Erlang/OTP runtime system available, but you do
|
||||
have [the Vagrant virtual machine](https://www.vagrantup.com/) manager
|
||||
available, then please refer to the instructions in the prerequisites
|
||||
doc for using Vagrant.
|
||||
|
||||
<a name="clone-compile">
|
||||
## Clone and compile the code
|
||||
|
||||
Please see the
|
||||
[Machi 'clone and compile' doc](./dev-clone-compile.md)
|
||||
for the short list of steps required to fetch the Machi source code
|
||||
from GitHub and to compile & test Machi.
|
||||
|
||||
## Running three Machi instances on a single machine
|
||||
|
||||
All of the commands that should be run at your login shell (e.g. Bash,
|
||||
c-shell) can be cut-and-pasted from this document directly to your
|
||||
login shell prompt.
|
||||
|
||||
Run the following command:
|
||||
|
||||
make stagedevrel
|
||||
|
||||
This will create a directory structure like this:
|
||||
|
||||
|-dev1-|... stand-alone Machi app + subdirectories
|
||||
|-dev-|-dev2-|... stand-alone Machi app + directories
|
||||
|-dev3-|... stand-alone Machi app + directories
|
||||
|
||||
Each of the `dev/dev1`, `dev/dev2`, and `dev/dev3` are stand-alone
|
||||
application instances of Machi and can be run independently of each
|
||||
other on the same machine. This demo will use all three.
|
||||
|
||||
The lifecycle management utilities for Machi are a bit immature,
|
||||
currently. They assume that each Machi server runs on a host with a
|
||||
unique hostname -- there is no flexibility built-in yet to easily run
|
||||
multiple Machi instances on the same machine. To continue with the
|
||||
demo, we need to use `sudo` or `su` to obtain superuser privileges to
|
||||
edit the `/etc/hosts` file.
|
||||
|
||||
Please add the following line to `/etc/hosts`, using this command:
|
||||
|
||||
sudo sh -c 'echo "127.0.0.1 machi1 machi2 machi3" >> /etc/hosts'
|
||||
|
||||
Next, we will use a shell script to finish setting up our cluster. It
|
||||
will do the following for us:
|
||||
|
||||
* Verify that the new line that was added to `/etc/hosts` is correct.
|
||||
* Modify the `etc/app.config` files to configure the Humming Consensus
|
||||
chain manager's actions logged to the `log/console.log` file.
|
||||
* Start the three application instances.
|
||||
* Verify that the three instances are running correctly.
|
||||
* Configure a single chain, with one FLU server per application
|
||||
instance.
|
||||
|
||||
Please run this script using this command:
|
||||
|
||||
./priv/humming-consensus-demo.setup.sh
|
||||
|
||||
If the output looks like this (and exits with status zero), then the
|
||||
script was successful.
|
||||
|
||||
Step: Verify that the required entries in /etc/hosts are present
|
||||
Step: add a verbose logging option to app.config
|
||||
Step: start three three Machi application instances
|
||||
pong
|
||||
pong
|
||||
pong
|
||||
Step: configure one chain to start a Humming Consensus group with three members
|
||||
Result: ok
|
||||
Result: ok
|
||||
Result: ok
|
||||
|
||||
We have now created a single replica chain, called `c1`, that has
|
||||
three file servers participating in the chain. Thanks to the
|
||||
hostnames that we added to `/etc/hosts`, all are using the localhost
|
||||
network interface.
|
||||
|
||||
| App instance | Pseudo | FLU name | TCP port |
|
||||
| directory | Hostname | | number |
|
||||
|--------------+----------+----------+----------|
|
||||
| dev1 | machi1 | flu1 | 20401 |
|
||||
| dev2 | machi2 | flu2 | 20402 |
|
||||
| dev3 | machi3 | flu3 | 20403 |
|
||||
|
||||
The log files for each application instance can be found in the
|
||||
`./dev/devN/log/console.log` file, where the `N` is the instance
|
||||
number: 1, 2, or 3.
|
||||
|
||||
## Understanding the chain manager's log file output
|
||||
|
||||
After running the `./priv/humming-consensus-demo.setup.sh` script,
|
||||
let's look at the last few lines of the `./dev/dev1/log/console.log`
|
||||
log file for Erlang VM process #1.
|
||||
|
||||
2016-03-09 10:16:35.676 [info] <0.105.0>@machi_lifecycle_mgr:process_pending_flu:422 Started FLU f1 with supervisor pid <0.128.0>
|
||||
2016-03-09 10:16:35.676 [info] <0.105.0>@machi_lifecycle_mgr:move_to_flu_config:540 Creating FLU config file f1
|
||||
2016-03-09 10:16:35.790 [info] <0.105.0>@machi_lifecycle_mgr:bootstrap_chain2:312 Configured chain c1 via FLU f1 to mode=ap_mode all=[f1,f2,f3] witnesses=[]
|
||||
2016-03-09 10:16:35.790 [info] <0.105.0>@machi_lifecycle_mgr:move_to_chain_config:546 Creating chain config file c1
|
||||
2016-03-09 10:16:44.139 [info] <0.132.0> CONFIRM epoch 1141 <<155,42,7,221>> upi [] rep [] auth f1 by f1
|
||||
2016-03-09 10:16:44.271 [info] <0.132.0> CONFIRM epoch 1148 <<57,213,154,16>> upi [f1] rep [] auth f1 by f1
|
||||
2016-03-09 10:16:44.864 [info] <0.132.0> CONFIRM epoch 1151 <<239,29,39,70>> upi [f1] rep [f3] auth f1 by f1
|
||||
2016-03-09 10:16:45.235 [info] <0.132.0> CONFIRM epoch 1152 <<173,17,66,225>> upi [f2] rep [f1,f3] auth f2 by f1
|
||||
2016-03-09 10:16:47.343 [info] <0.132.0> CONFIRM epoch 1154 <<154,231,224,149>> upi [f2,f1,f3] rep [] auth f2 by f1
|
||||
|
||||
Let's pick apart some of these lines. We have started all three
|
||||
servers at about the same time. We see some race conditions happen,
|
||||
and some jostling and readjustment happens pretty quickly in the first
|
||||
few seconds.
|
||||
|
||||
* `Started FLU f1 with supervisor pid <0.128.0>`
|
||||
* This VM, #1,
|
||||
started a FLU (Machi data server) with the name `f1`. In the Erlang
|
||||
process supervisor hierarchy, the process ID of the top supervisor
|
||||
is `<0.128.0>`.
|
||||
* `Configured chain c1 via FLU f1 to mode=ap_mode all=[f1,f2,f3] witnesses=[]`
|
||||
* A bootstrap configuration for a chain named `c1` has been created.
|
||||
* The FLUs/data servers that are eligible for participation in the
|
||||
chain have names `f1`, `f2`, and `f3`.
|
||||
* The chain will operate in eventual consistency mode (`ap_mode`)
|
||||
* The witness server list is empty. Witness servers are never used
|
||||
in eventual consistency mode.
|
||||
* `CONFIRM epoch 1141 <<155,42,7,221>> upi [] rep [] auth f1 by f1`
|
||||
* All participants in epoch 1141 are unanimous in adopting epoch
|
||||
1141's projection. All active membership lists are empty, so
|
||||
there is no functional chain replication yet, at least as far as
|
||||
server `f1` knows
|
||||
* The epoch's abbreviated checksum is `<<155,42,7,221>>`.
|
||||
* The UPI list, i.e. the replicas whose data is 100% in sync is
|
||||
`[]`, the empty list. (UPI = Update Propagation Invariant)
|
||||
* The list of servers that are under data repair (`rep`) is also
|
||||
empty, `[]`.
|
||||
* This projection was authored by server `f1`.
|
||||
* The log message was generated by server `f1`.
|
||||
* `CONFIRM epoch 1148 <<57,213,154,16>> upi [f1] rep [] auth f1 by f1`
|
||||
* Now the server `f1` has created a chain of length 1, `[f1]`.
|
||||
* Chain repair/file re-sync is not required when the UPI server list
|
||||
changes from length 0 -> 1.
|
||||
* `CONFIRM epoch 1151 <<239,29,39,70>> upi [f1] rep [f3] auth f1 by f1`
|
||||
* Server `f1` has noticed that server `f3` is alive. Apparently it
|
||||
has not yet noticed that server `f2` is also running.
|
||||
* Server `f3` is in the repair list.
|
||||
* `CONFIRM epoch 1152 <<173,17,66,225>> upi [f2] rep [f1,f3] auth f2 by f1`
|
||||
* Server `f2` is apparently now aware that all three servers are running.
|
||||
* The previous configuration used by `f2` was `upi [f2]`, i.e., `f2`
|
||||
was running in a chain of one. `f2` noticed that `f1` and `f3`
|
||||
were now available and has started adding them to the chain.
|
||||
* All new servers are always added to the tail of the chain in the
|
||||
repair list.
|
||||
* In eventual consistency mode, a UPI change like this is OK.
|
||||
* When performing a read, a client must read from both tail of the
|
||||
UPI list and also from all repairing servers.
|
||||
* When performing a write, the client writes to both the UPI
|
||||
server list and also the repairing list, in that order.
|
||||
* I.e., the client concatenates both lists,
|
||||
`UPI ++ Repairing`, for its chain configuration for the write.
|
||||
* Server `f2` will trigger file repair/re-sync shortly.
|
||||
* The waiting time for starting repair has been configured to be
|
||||
extremely short, 1 second. The default waiting time is 10
|
||||
seconds, in case Humming Consensus remains unstable.
|
||||
* `CONFIRM epoch 1154 <<154,231,224,149>> upi [f2,f1,f3] rep [] auth f2 by f1`
|
||||
* File repair/re-sync has finished. All file data on all servers
|
||||
are now in sync.
|
||||
* The UPI/in-sync part of the chain is now `[f2,f1,f3]`, and there
|
||||
are no servers under repair.
|
||||
|
||||
## Let's create some failures
|
||||
|
||||
Here are some suggestions for creating failures.
|
||||
|
||||
* Use the `./dev/devN/bin/machi stop` and `./dev/devN/bin/machi start`
|
||||
commands to stop & start VM #`N`.
|
||||
* Stop a VM abnormally by using `kill`. The OS process name to look
|
||||
for is `beam.smp`.
|
||||
* Suspend and resume a VM, using the `SIGSTOP` and `SIGCONT` signals.
|
||||
* E.g. `kill -STOP 9823` and `kill -CONT 9823`
|
||||
|
||||
The network partition simulator is not (yet) available when running
|
||||
Machi in this mode. Please see the next section for instructions on
|
||||
how to use partition simulator.
|
||||
|
||||
|
||||
<a name="partition-simulator">
|
||||
# Using the network partition simulator and convergence demo test code
|
||||
|
||||
This is the demo code mentioned in the presentation that Scott Lystig
|
||||
Fritchie gave at the
|
||||
[RICON 2015 conference](http://ricon.io).
|
||||
* [slides (PDF format)](http://ricon.io/speakers/slides/Scott_Fritchie_Ricon_2015.pdf)
|
||||
* [video](https://www.youtube.com/watch?v=yR5kHL1bu1Q)
|
||||
|
||||
## A complete example of all input and output
|
||||
|
||||
If you don't have an Erlang/OTP 17 runtime environment available,
|
||||
please see this file for full input and output of a strong consistency
|
||||
length=3 chain test:
|
||||
https://gist.github.com/slfritchie/8352efc88cc18e62c72c
|
||||
This file contains all commands input and all simulator output from a
|
||||
sample run of the simulator.
|
||||
|
||||
To help interpret the output of the test, please skip ahead to the
|
||||
"The test output is very verbose" section.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
If you don't have `git` and/or the Erlang 17 runtime system available
|
||||
on your OS X, FreeBSD, Linux, or Solaris machine, please take a look
|
||||
at the [Prerequisites section](#prerequisites) first. When you have
|
||||
installed the prerequisite software, please return back here.
|
||||
|
||||
## Clone and compile the code
|
||||
|
||||
Please briefly visit the [Clone and compile the code](#clone-compile)
|
||||
section. When finished, please return back here.
|
||||
|
||||
## Run an interactive Erlang CLI shell
|
||||
|
||||
Run the following command at your login shell:
|
||||
|
||||
erl -pz .eunit ebin deps/*/ebin
|
||||
|
||||
If you are using Erlang/OTP version 17, you should see some CLI output
|
||||
that looks like this:
|
||||
|
||||
Erlang/OTP 17 [erts-6.4] [source] [64-bit] [smp:8:8] [async-threads:10] [hipe] [kernel-poll:false] [dtrace]
|
||||
|
||||
Eshell V6.4 (abort with ^G)
|
||||
1>
|
||||
|
||||
## The test output is very verbose ... what are the important parts?
|
||||
|
||||
The output of the Erlang command
|
||||
`machi_chain_manager1_converge_demo:help()` will display the following
|
||||
guide to the output of the tests.
|
||||
|
||||
A visualization of the convergence behavior of the chain self-management
|
||||
algorithm for Machi.
|
||||
|
||||
1. Set up some server and chain manager pairs.
|
||||
2. Create a number of different network partition scenarios, where
|
||||
(simulated) partitions may be symmetric or asymmetric. Then stop changing
|
||||
the partitions and keep the simulated network stable (and perhaps broken).
|
||||
3. Run a number of iterations of the algorithm in parallel by poking each
|
||||
of the manager processes on a random'ish basis.
|
||||
4. Afterward, fetch the chain transition changes made by each FLU and
|
||||
verify that no transition was unsafe.
|
||||
|
||||
During the iteration periods, the following is a cheatsheet for the output.
|
||||
See the internal source for interpreting the rest of the output.
|
||||
|
||||
'SET partitions = '
|
||||
|
||||
A pair-wise list of actors which cannot send messages. The
|
||||
list is uni-directional. If there are three servers (a,b,c),
|
||||
and if the partitions list is '[{a,b},{b,c}]' then all
|
||||
messages from a->b and b->c will be dropped, but any other
|
||||
sender->recipient messages will be delivered successfully.
|
||||
|
||||
'x uses:'
|
||||
|
||||
The FLU x has made an internal state transition and is using
|
||||
this epoch's projection as operating chain configuration. The
|
||||
rest of the line is a summary of the projection.
|
||||
|
||||
'CONFIRM epoch {N}'
|
||||
|
||||
This message confirms that all of the servers listed in the
|
||||
UPI and repairing lists of the projection at epoch {N} have
|
||||
agreed to use this projection because they all have written
|
||||
this projection to their respective private projection stores.
|
||||
The chain is now usable by/available to all clients.
|
||||
|
||||
'Sweet, private projections are stable'
|
||||
|
||||
This report announces that this iteration of the test cycle
|
||||
has passed successfully. The report that follows briefly
|
||||
summarizes the latest private projection used by each
|
||||
participating server. For example, when in strong consistency
|
||||
mode with 'a' as a witness and 'b' and 'c' as real servers:
|
||||
|
||||
%% Legend:
|
||||
%% server name, epoch ID, UPI list, repairing list, down list, ...
|
||||
%% ... witness list, 'false' (a constant value)
|
||||
|
||||
[{a,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}},
|
||||
{b,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}}]
|
||||
|
||||
Both servers 'a' and 'b' agree on epoch 1116 with epoch ID
|
||||
{1116,<<23,143,246,55>>} where UPI=[a,b], repairing=[],
|
||||
down=[c], and witnesses=[a].
|
||||
|
||||
Server 'c' is not shown because 'c' has wedged itself OOS (out
|
||||
of service) by configuring a chain length of zero.
|
||||
|
||||
If no servers are listed in the report (i.e. only '[]' is
|
||||
displayed), then all servers have wedged themselves OOS, and
|
||||
the chain is unavailable.
|
||||
|
||||
'DoIt,'
|
||||
|
||||
This marks a group of tick events which trigger the manager
|
||||
processes to evaluate their environment and perhaps make a
|
||||
state transition.
|
||||
|
||||
A long chain of 'DoIt,DoIt,DoIt,' means that the chain state has
|
||||
(probably) settled to a stable configuration, which is the goal of the
|
||||
algorithm.
|
||||
|
||||
Press control-c to interrupt the test....".
|
||||
|
||||
## Run a test in eventual consistency mode
|
||||
|
||||
Run the following command at the Erlang CLI prompt:
|
||||
|
||||
machi_chain_manager1_converge_demo:t(3, [{private_write_verbose,true}]).
|
||||
|
||||
The first argument, `3`, is the number of servers to participate in
|
||||
the chain. Please note:
|
||||
|
||||
* Chain lengths as short as 1 or 2 are valid, but the results are a
|
||||
bit boring.
|
||||
* Chain lengths as long as 7 or 9 can be used, but they may
|
||||
suffer from longer periods of churn/instability before all chain
|
||||
managers reach agreement via humming consensus. (It is future work
|
||||
to shorten the worst of the unstable churn latencies.)
|
||||
* In eventual consistency mode, chain lengths may be even numbers,
|
||||
e.g. 2, 4, or 6.
|
||||
* The simulator will choose partition events from the permutations of
|
||||
all 1, 2, and 3 node partition pairs. The total runtime will
|
||||
increase *dramatically* with chain length.
|
||||
* Chain length 2: about 3 partition cases
|
||||
* Chain length 3: about 35 partition cases
|
||||
* Chain length 4: about 230 partition cases
|
||||
* Chain length 5: about 1100 partition cases
|
||||
|
||||
## Run a test in strong consistency mode (with witnesses):
|
||||
|
||||
*NOTE:* Due to a bug in the test code, please do not try to run the
|
||||
convergence test in strong consistency mode and also without the
|
||||
correct minority number of witness servers! If in doubt, please run
|
||||
the commands shown below exactly.
|
||||
|
||||
Run the following command at the Erlang CLI prompt:
|
||||
|
||||
machi_chain_manager1_converge_demo:t(3, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a]}]).
|
||||
|
||||
The first argument, `3`, is the number of servers to participate in
|
||||
the chain. Chain lengths as long as 7 or 9 can be used, but they may
|
||||
suffer from longer periods of churn/instability before all chain
|
||||
managers reach agreement via humming consensus.
|
||||
|
||||
Due to the bug mentioned above, please use the following
|
||||
commands when running with chain lengths of 5 or 7, respectively.
|
||||
|
||||
machi_chain_manager1_converge_demo:t(5, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a,b]}]).
|
||||
machi_chain_manager1_converge_demo:t(7, [{private_write_verbose,true}, {consistency_mode, cp_mode}, {witnesses, [a,b,c]}]).
|
||||
|
|
@ -1,170 +0,0 @@
|
|||
|
||||
@title Machi: a small village of replicated files
|
||||
|
||||
@doc
|
||||
|
||||
== About This EDoc Documentation ==
|
||||
|
||||
This EDoc-style documentation will concern itself only with Erlang
|
||||
function APIs and function & data types. Higher-level design and
|
||||
commentary will remain outside of the Erlang EDoc system; please see
|
||||
the "Pointers to Other Machi Documentation" section below for more
|
||||
details.
|
||||
|
||||
Readers should beware that this documentation may be out-of-sync with
|
||||
the source code. When in doubt, use the `make edoc' command to
|
||||
regenerate all HTML pages.
|
||||
|
||||
It is the developer's responsibility to re-generate the documentation
|
||||
periodically and commit it to the Git repo.
|
||||
|
||||
== Machi Code Overview ==
|
||||
|
||||
=== Chain Manager ===
|
||||
|
||||
The Chain Manager is responsible for managing the state of Machi's
|
||||
"Chain Replication" state. This role is roughly analogous to the
|
||||
"Riak Core" application inside of Riak, which takes care of
|
||||
coordinating replica placement and replica repair.
|
||||
|
||||
For each primitive data server in the cluster, a Machi FLU, there is a
|
||||
Chain Manager process that manages its FLU's role within the Machi
|
||||
cluster's Chain Replication scheme. Each Chain Manager process
|
||||
executes locally and independently to manage the distributed state of
|
||||
a single Machi Chain Replication chain.
|
||||
|
||||
<ul>
|
||||
|
||||
<li> To contrast with Riak Core ... Riak Core's claimant process is
|
||||
solely responsible for managing certain critical aspects of
|
||||
Riak Core distributed state. Machi's Chain Manager process
|
||||
performs similar tasks as Riak Core's claimant. However, Machi
|
||||
has several active Chain Manager processes, one per FLU server,
|
||||
instead of a single active process like Core's claimant. Each
|
||||
Chain Manager process acts independently; each is constrained
|
||||
so that it will reach consensus via independent computation
|
||||
& action.
|
||||
|
||||
Full discussion of this distributed consensus is outside the
|
||||
scope of this document; see the "Pointers to Other Machi
|
||||
Documentation" section below for more information.
|
||||
</li>
|
||||
<li> Machi differs from a Riak Core application because Machi's
|
||||
replica placement policy is simply, "All Machi servers store
|
||||
replicas of all Machi files".
|
||||
Machi is intended to be a primitive building block for creating larger
|
||||
cluster-of-clusters where files are
|
||||
distributed/fragmented/sharded across a large pool of
|
||||
independent Machi clusters.
|
||||
</li>
|
||||
<li> See
|
||||
[https://www.usenix.org/legacy/events/osdi04/tech/renesse.html]
|
||||
for a copy of the paper, "Chain Replication for Supporting High
|
||||
Throughput and Availability" by Robbert van Renesse and Fred
|
||||
B. Schneider.
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
=== FLU ===
|
||||
|
||||
The FLU is the basic storage server for Machi.
|
||||
|
||||
<ul>
|
||||
<li> The name FLU is taken from "flash storage unit" from the paper
|
||||
"CORFU: A Shared Log Design for Flash Clusters" by
|
||||
Balakrishnan, Malkhi, Prabhakaran, and Wobber. See
|
||||
[https://www.usenix.org/conference/nsdi12/technical-sessions/presentation/balakrishnan]
|
||||
</li>
|
||||
<li> In CORFU, the sequencer step is a prerequisite step that is
|
||||
performed by a separate component, the Sequencer.
|
||||
In Machi, the `append_chunk()' protocol message has
|
||||
an implicit "sequencer" operation applied by the "head" of the
|
||||
Machi Chain Replication chain. If a client wishes to write
|
||||
data that has already been assigned a sequencer position, then
|
||||
the `write_chunk()' API function is used.
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
For each FLU, there are three independent tasks that are implemented
|
||||
using three different Erlang processes:
|
||||
|
||||
<ul>
|
||||
<li> A FLU server, implemented primarily by `machi_flu.erl'.
|
||||
</li>
|
||||
<li> A projection store server, implemented primarily by
|
||||
`machi_projection_store.erl'.
|
||||
</li>
|
||||
<li> A chain state manager server, implemented primarily by
|
||||
`machi_chain_manager1.erl'.
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
From the perspective of failure detection, it is very convenient that
|
||||
all three FLU-related services (file server, sequencer server, and
|
||||
projection server) are accessed using the same single TCP port.
|
||||
|
||||
=== Projection (data structure) ===
|
||||
|
||||
The projection is a data structure that specifies the current state
|
||||
of the Machi cluster: all FLUs, which FLUS are considered
|
||||
up/running or down/crashed/stopped, which FLUs are actively
|
||||
participants in the Chain Replication protocol, and which FLUs are
|
||||
under "repair" (i.e., having their data resyncronized when
|
||||
newly-added to a cluster or when restarting after a crash).
|
||||
|
||||
=== Projection Store (server) ===
|
||||
|
||||
The projection store is a storage service that is implemented by an
|
||||
Erlang/OTP `gen_server' process that is associated with each
|
||||
FLU. Conceptually, the projection store is an array of
|
||||
write-once registers. For each projection store register, the
|
||||
key is a 2-tuple of an epoch number (`non_neg_integer()' type)
|
||||
and a projection type (`public' or `private' type); the value is
|
||||
a projection data structure (`projection_v1()' type).
|
||||
|
||||
=== Client and Proxy Client ===
|
||||
|
||||
Machi is intentionally avoiding using distributed Erlang for Machi's
|
||||
communication. This design decision makes Erlang-side code more
|
||||
difficult & complex but allows us the freedom of implementing
|
||||
parts of Machi in other languages without major
|
||||
protocol&API&glue code changes later in the product's
|
||||
lifetime.
|
||||
|
||||
There are two layers of interface for Machi clients.
|
||||
|
||||
<ul>
|
||||
<li> The `machi_flu1_client' module implements an API that uses a
|
||||
TCP socket directly.
|
||||
</li>
|
||||
<li> The `machi_proxy_flu1_client' module implements an API that
|
||||
uses a local, long-lived `gen_server' process as a proxy for
|
||||
the remote, perhaps disconnected-or-crashed Machi FLU server.
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
The types for both modules ought to be the same. However, due to
|
||||
rapid code churn, some differences might exist. Any major difference
|
||||
is (almost by definition) a bug: please open a GitHub issue to request
|
||||
a correction.
|
||||
|
||||
== TODO notes ==
|
||||
|
||||
Any use of the string "TODO" in upper/lower/mixed case, anywhere in
|
||||
the code, is a reminder signal of unfinished work.
|
||||
|
||||
== Pointers to Other Machi Documentation ==
|
||||
|
||||
<ul>
|
||||
<li> If you are viewing this document locally, please look in the
|
||||
`../doc/' directory,
|
||||
</li>
|
||||
<li> If you are viewing this document via the Web, please find the
|
||||
documentation via this link:
|
||||
[http://github.com/basho/machi/tree/master/doc/]
|
||||
Please be aware that this link points to the `master' branch
|
||||
of the Machi source repository and therefore may be
|
||||
out-of-sync with non-`master' branch code.
|
||||
</li>
|
||||
|
||||
</ul>
|
BIN
doc/process-protocol-module-overview.jpg
Normal file
BIN
doc/process-protocol-module-overview.jpg
Normal file
Binary file not shown.
After Width: | Height: | Size: 115 KiB |
|
@ -23,8 +23,8 @@
|
|||
\copyrightdata{978-1-nnnn-nnnn-n/yy/mm}
|
||||
\doi{nnnnnnn.nnnnnnn}
|
||||
|
||||
\titlebanner{Draft \#0.91, June 2015}
|
||||
\preprintfooter{Draft \#0.91, June 2015}
|
||||
\titlebanner{Draft \#0.92, October 2015}
|
||||
\preprintfooter{Draft \#0.92, October 2015}
|
||||
|
||||
\title{Chain Replication metadata management in Machi, an immutable
|
||||
file store}
|
||||
|
@ -50,19 +50,23 @@ For an overview of the design of the larger Machi system, please see
|
|||
TODO Fix, after all of the recent changes to this document.
|
||||
|
||||
Machi is an immutable file store, now in active development by Basho
|
||||
Japan KK. Machi uses Chain Replication to maintain strong consistency
|
||||
of file updates to all replica servers in a Machi cluster. Chain
|
||||
Japan KK. Machi uses Chain Replication\footnote{Chain
|
||||
Replication is a variation of primary/backup replication where the
|
||||
order of updates between the primary server and each of the backup
|
||||
servers is strictly ordered into a single ``chain''. Management of
|
||||
Chain Replication's metadata, e.g., ``What is the current order of
|
||||
servers in the chain?'', remains an open research problem. The
|
||||
current state of the art for Chain Replication metadata management
|
||||
relies on an external oracle (e.g., ZooKeeper) or the Elastic
|
||||
Replication algorithm.
|
||||
servers is strictly ordered into a single ``chain''.}
|
||||
to maintain strong consistency
|
||||
of file updates to all replica servers in a Machi cluster.
|
||||
|
||||
This document describes the Machi chain manager, the component
|
||||
responsible for managing Chain Replication metadata state. The chain
|
||||
responsible for managing Chain Replication metadata state.
|
||||
Management of
|
||||
chain metadata, e.g., ``What is the current order of
|
||||
servers in the chain?'', remains an open research problem. The
|
||||
current state of the art for Chain Replication metadata management
|
||||
relies on an external oracle (e.g., based on ZooKeeper) or the Elastic
|
||||
Replication \cite{elastic-chain-replication} algorithm.
|
||||
|
||||
The chain
|
||||
manager uses a new technique, based on a variation of CORFU, called
|
||||
``humming consensus''.
|
||||
Humming consensus does not require active participation by all or even
|
||||
|
@ -89,20 +93,18 @@ to perform these management tasks. Chain metadata state and state
|
|||
management tasks include:
|
||||
|
||||
\begin{itemize}
|
||||
\item Preserving data integrity of all metadata and data stored within
|
||||
the chain. Data loss is not an option.
|
||||
\item Preserving stable knowledge of chain membership (i.e. all nodes in
|
||||
the chain, regardless of operational status). A systems
|
||||
administrator is expected to make ``permanent'' decisions about
|
||||
the chain, regardless of operational status). We expect that a systems
|
||||
administrator will make all ``permanent'' decisions about
|
||||
chain membership.
|
||||
\item Using passive and/or active techniques to track operational
|
||||
state/status, e.g., up, down, restarting, full data sync, partial
|
||||
data sync, etc.
|
||||
state/status, e.g., up, down, restarting, full data sync in progress, partial
|
||||
data sync in progress, etc.
|
||||
\item Choosing the run-time replica ordering/state of the chain, based on
|
||||
current member status and past operational history. All chain
|
||||
state transitions must be done safely and without data loss or
|
||||
corruption.
|
||||
\item As a new node is added to the chain administratively or old node is
|
||||
\item When a new node is added to the chain administratively or old node is
|
||||
restarted, adding the node to the chain safely and perform any data
|
||||
synchronization/repair required to bring the node's data into
|
||||
full synchronization with the other nodes.
|
||||
|
@ -111,39 +113,27 @@ management tasks include:
|
|||
\subsection{Ultimate goal: Preserve data integrity of Chain Replicated data}
|
||||
|
||||
Preservation of data integrity is paramount to any chain state
|
||||
management technique for Machi. Even when operating in an eventually
|
||||
consistent mode, Machi must not lose data without cause outside of all
|
||||
design, e.g., all particpants crash permanently.
|
||||
management technique for Machi. Loss or corruption of chain data must
|
||||
be avoided.
|
||||
|
||||
\subsection{Goal: Contribute to Chain Replication metadata management research}
|
||||
|
||||
We believe that this new self-management algorithm, humming consensus,
|
||||
contributes a novel approach to Chain Replication metadata management.
|
||||
The ``monitor
|
||||
and mangage your neighbor'' technique proposed in Elastic Replication
|
||||
(Section \ref{ssec:elastic-replication}) appears to be the current
|
||||
state of the art in the distributed systems research community.
|
||||
Typical practice in the IT industry appears to favor using an external
|
||||
oracle, e.g., using ZooKeeper as a trusted coordinator.
|
||||
oracle, e.g., built on top of ZooKeeper as a trusted coordinator.
|
||||
|
||||
See Section~\ref{sec:cr-management-review} for a brief review.
|
||||
See Section~\ref{sec:cr-management-review} for a brief review of
|
||||
techniques used today.
|
||||
|
||||
\subsection{Goal: Support both eventually consistent \& strongly consistent modes of operation}
|
||||
|
||||
Machi's first use cases are all for use as a file store in an eventually
|
||||
consistent environment.
|
||||
In eventually consistent mode, humming consensus
|
||||
allows a Machi cluster to fragment into
|
||||
arbitrary islands of network partition, all the way down to 100\% of
|
||||
members running in complete network isolation from each other.
|
||||
Furthermore, it provides enough agreement to allow
|
||||
formerly-partitioned members to coordinate the reintegration and
|
||||
reconciliation of their data when partitions are healed.
|
||||
|
||||
Later, we wish the option of supporting strong consistency
|
||||
applications such as CORFU-style logging while reusing all (or most)
|
||||
of Machi's infrastructure. Such strongly consistent operation is the
|
||||
main focus of this document.
|
||||
Chain Replication was originally designed by van Renesse and Schneider
|
||||
\cite{chain-replication} for applications that require strong
|
||||
consistency, e.g. sequential consistency. However, Machi has use
|
||||
cases where more relaxed eventual consistency semantics are
|
||||
sufficient. We wish to use the same Chain Replication management
|
||||
technique for both strong and eventual consistency environments.
|
||||
|
||||
\subsection{Anti-goal: Minimize churn}
|
||||
|
||||
|
@ -204,6 +194,18 @@ would probably be preferable to add the feature to Riak Ensemble
|
|||
rather than to use ZooKeeper (and for Basho to document ZK, package
|
||||
ZK, provide commercial ZK support, etc.).
|
||||
|
||||
\subsection{An external management oracle, implemented by
|
||||
active/standby application failover}
|
||||
|
||||
This technique has been used in production of HibariDB. The customer
|
||||
very carefully deployed the oracle using the Erlang/OTP ``application
|
||||
controller'' on two machines to provide active/standby failover of the
|
||||
management oracle. The customer was willing to monitor this service
|
||||
very closely and was prepared to intervene manually during network
|
||||
partitions. (This controller is very susceptible to ``split brain
|
||||
syndrome''.) While this feature of Erlang/OTP is useful in other
|
||||
environments, we believe is it not sufficient for Machi's needs.
|
||||
|
||||
\section{Assumptions}
|
||||
\label{sec:assumptions}
|
||||
|
||||
|
@ -212,8 +214,8 @@ Paxos, Raft, et al.), why bother with a slightly different set of
|
|||
assumptions and a slightly different protocol?
|
||||
|
||||
The answer lies in one of our explicit goals: to have an option of
|
||||
running in an ``eventually consistent'' manner. We wish to be able to
|
||||
make progress, i.e., remain available in the CAP sense, even if we are
|
||||
running in an ``eventually consistent'' manner. We wish to be
|
||||
remain available, even if we are
|
||||
partitioned down to a single isolated node. VR, Paxos, and Raft
|
||||
alone are not sufficient to coordinate service availability at such
|
||||
small scale. The humming consensus algorithm can manage
|
||||
|
@ -247,13 +249,15 @@ synchronized by NTP.
|
|||
|
||||
The protocol and algorithm presented here do not specify or require any
|
||||
timestamps, physical or logical. Any mention of time inside of data
|
||||
structures are for human/historic/diagnostic purposes only.
|
||||
structures are for human and/or diagnostic purposes only.
|
||||
|
||||
Having said that, some notion of physical time is suggested for
|
||||
purposes of efficiency. It's recommended that there be some ``sleep
|
||||
Having said that, some notion of physical time is suggested
|
||||
occasionally for
|
||||
purposes of efficiency. For example, some ``sleep
|
||||
time'' between iterations of the algorithm: there is no need to ``busy
|
||||
wait'' by executing the algorithm as quickly as possible. See also
|
||||
Section~\ref{ssub:when-to-calc}.
|
||||
wait'' by executing the algorithm as many times per minute as
|
||||
possible.
|
||||
See also Section~\ref{ssub:when-to-calc}.
|
||||
|
||||
\subsection{Failure detector model}
|
||||
|
||||
|
@ -276,55 +280,73 @@ eventual consistency. Discussion of strongly consistent CP
|
|||
mode is always the default; exploration of AP mode features in this document
|
||||
will always be explictly noted.
|
||||
|
||||
\subsection{Use of the ``wedge state''}
|
||||
%%\subsection{Use of the ``wedge state''}
|
||||
%%
|
||||
%%A participant in Chain Replication will enter ``wedge state'', as
|
||||
%%described by the Machi high level design \cite{machi-design} and by CORFU,
|
||||
%%when it receives information that
|
||||
%%a newer projection (i.e., run-time chain state reconfiguration) is
|
||||
%%available. The new projection may be created by a system
|
||||
%%administrator or calculated by the self-management algorithm.
|
||||
%%Notification may arrive via the projection store API or via the file
|
||||
%%I/O API.
|
||||
%%
|
||||
%%When in wedge state, the server will refuse all file write I/O API
|
||||
%%requests until the self-management algorithm has determined that
|
||||
%%humming consensus has been decided (see next bullet item). The server
|
||||
%%may also refuse file read I/O API requests, depending on its CP/AP
|
||||
%%operation mode.
|
||||
%%
|
||||
%%\subsection{Use of ``humming consensus''}
|
||||
%%
|
||||
%%CS literature uses the word ``consensus'' in the context of the problem
|
||||
%%description at \cite{wikipedia-consensus}
|
||||
%%.
|
||||
%%This traditional definition differs from what is described here as
|
||||
%%``humming consensus''.
|
||||
%%
|
||||
%%``Humming consensus'' describes
|
||||
%%consensus that is derived only from data that is visible/known at the current
|
||||
%%time.
|
||||
%%The algorithm will calculate
|
||||
%%a rough consensus despite not having input from a quorum majority
|
||||
%%of chain members. Humming consensus may proceed to make a
|
||||
%%decision based on data from only a single participant, i.e., only the local
|
||||
%%node.
|
||||
%%
|
||||
%%See Section~\ref{sec:humming-consensus} for detailed discussion.
|
||||
|
||||
A participant in Chain Replication will enter ``wedge state'', as
|
||||
described by the Machi high level design \cite{machi-design} and by CORFU,
|
||||
when it receives information that
|
||||
a newer projection (i.e., run-time chain state reconfiguration) is
|
||||
available. The new projection may be created by a system
|
||||
administrator or calculated by the self-management algorithm.
|
||||
Notification may arrive via the projection store API or via the file
|
||||
I/O API.
|
||||
%%\subsection{Concurrent chain managers execute humming consensus independently}
|
||||
%%
|
||||
%%Each Machi file server has its own concurrent chain manager
|
||||
%%process embedded within it. Each chain manager process will
|
||||
%%execute the humming consensus algorithm using only local state (e.g.,
|
||||
%%the $P_{current}$ projection currently used by the local server) and
|
||||
%%values observed in everyone's projection stores
|
||||
%%(Section~\ref{sec:projection-store}).
|
||||
%%
|
||||
%%The chain manager communicates with the local Machi
|
||||
%%file server using the wedge and un-wedge request API. When humming
|
||||
%%consensus has chosen a projection $P_{new}$ to replace $P_{current}$,
|
||||
%%the value of $P_{new}$ is included in the un-wedge request.
|
||||
|
||||
When in wedge state, the server will refuse all file write I/O API
|
||||
requests until the self-management algorithm has determined that
|
||||
humming consensus has been decided (see next bullet item). The server
|
||||
may also refuse file read I/O API requests, depending on its CP/AP
|
||||
operation mode.
|
||||
\subsection{The reader is familiar with CORFU}
|
||||
|
||||
\subsection{Use of ``humming consensus''}
|
||||
Machi borrows heavily from the techniques and data structures used by
|
||||
CORFU \cite[corfu1],\cite[corfu2]. We hope that the reader is
|
||||
familiar with CORFU's features, including:
|
||||
|
||||
CS literature uses the word ``consensus'' in the context of the problem
|
||||
description at \cite{wikipedia-consensus}
|
||||
.
|
||||
This traditional definition differs from what is described here as
|
||||
``humming consensus''.
|
||||
|
||||
``Humming consensus'' describes
|
||||
consensus that is derived only from data that is visible/known at the current
|
||||
time.
|
||||
The algorithm will calculate
|
||||
a rough consensus despite not having input from all/majority
|
||||
of chain members. Humming consensus may proceed to make a
|
||||
decision based on data from only a single participant, i.e., only the local
|
||||
node.
|
||||
|
||||
See Section~\ref{sec:humming-consensus} for detailed discussion.
|
||||
|
||||
\subsection{Concurrent chain managers execute humming consensus independently}
|
||||
|
||||
Each Machi file server has its own concurrent chain manager
|
||||
process embedded within it. Each chain manager process will
|
||||
execute the humming consensus algorithm using only local state (e.g.,
|
||||
the $P_{current}$ projection currently used by the local server) and
|
||||
values observed in everyone's projection stores
|
||||
(Section~\ref{sec:projection-store}).
|
||||
|
||||
The chain manager communicates with the local Machi
|
||||
file server using the wedge and un-wedge request API. When humming
|
||||
consensus has chosen a projection $P_{new}$ to replace $P_{current}$,
|
||||
the value of $P_{new}$ is included in the un-wedge request.
|
||||
\begin{itemize}
|
||||
\item write-once registers for log data storage,
|
||||
\item the epoch, which defines a period of time when a cluster's configuration
|
||||
is stable,
|
||||
\item strictly increasing epoch numbers, which are identifiers
|
||||
for particular epochs,
|
||||
\item projections, which define the chain order and other details of
|
||||
data replication within the cluster, and
|
||||
\item the wedge state, used by servers to coordinate cluster changes
|
||||
during epoch transitions.
|
||||
\end{itemize}
|
||||
|
||||
\section{The projection store}
|
||||
\label{sec:projection-store}
|
||||
|
@ -343,19 +365,15 @@ this key. The
|
|||
store's value is either the special `unwritten' value\footnote{We use
|
||||
$\bot$ to denote the unwritten value.} or else a binary blob that is
|
||||
immutable thereafter; the projection data structure is
|
||||
serialized and stored in this binary blob.
|
||||
|
||||
The projection store is vital for the correct implementation of humming
|
||||
consensus (Section~\ref{sec:humming-consensus}). The write-once
|
||||
register primitive allows us to reason about the store's behavior
|
||||
using the same logical tools and techniques as the CORFU ordered log.
|
||||
serialized and stored in this binary blob. See
|
||||
\ref{sub:the-projection} for more detail.
|
||||
|
||||
\subsection{The publicly-writable half of the projection store}
|
||||
|
||||
The publicly-writable projection store is used to share information
|
||||
during the first half of humming consensus algorithm. Projections
|
||||
in the public half of the store form a log of
|
||||
suggestions\footnote{I hesitate to use the word ``propose'' or ``proposal''
|
||||
suggestions\footnote{I hesitate to use the words ``propose'' or ``proposal''
|
||||
anywhere in this document \ldots until I've done a more formal
|
||||
analysis of the protocol. Those words have too many connotations in
|
||||
the context of consensus protocols such as Paxos and Raft.}
|
||||
|
@ -369,8 +387,9 @@ Any chain member may read from the public half of the store.
|
|||
|
||||
The privately-writable projection store is used to store the
|
||||
Chain Replication metadata state (as chosen by humming consensus)
|
||||
that is in use now by the local Machi server as well as previous
|
||||
operation states.
|
||||
that is in use now by the local Machi server. Earlier projections
|
||||
remain in the private half to keep a historical
|
||||
record of chain state transitions by the local server.
|
||||
|
||||
Only the local server may write values into the private half of store.
|
||||
Any chain member may read from the private half of the store.
|
||||
|
@ -386,35 +405,30 @@ The private projection store serves multiple purposes, including:
|
|||
its sequence of $P_{current}$ projection changes.
|
||||
\end{itemize}
|
||||
|
||||
The private half of the projection store is not replicated.
|
||||
|
||||
\section{Projections: calculation, storage, and use}
|
||||
\label{sec:projections}
|
||||
|
||||
Machi uses a ``projection'' to determine how its Chain Replication replicas
|
||||
should operate; see \cite{machi-design} and
|
||||
\cite{corfu1}. At runtime, a cluster must be able to respond both to
|
||||
administrative changes (e.g., substituting a failed server with
|
||||
replacement hardware) as well as local network conditions (e.g., is
|
||||
there a network partition?).
|
||||
|
||||
The projection defines the operational state of Chain Replication's
|
||||
chain order as well the (re-)synchronization of data managed by by
|
||||
newly-added/failed-and-now-recovering members of the chain. This
|
||||
chain metadata, together with computational processes that manage the
|
||||
chain, must be managed in a safe manner in order to avoid unintended
|
||||
data loss of data managed by the chain.
|
||||
|
||||
should operate; see \cite{machi-design} and \cite{corfu1}.
|
||||
The concept of a projection is borrowed
|
||||
from CORFU but has a longer history, e.g., the Hibari key-value store
|
||||
\cite{cr-theory-and-practice} and goes back in research for decades,
|
||||
e.g., Porcupine \cite{porcupine}.
|
||||
|
||||
The projection defines the operational state of Chain Replication's
|
||||
chain order as well the (re-)synchronization of data managed by by
|
||||
newly-added/failed-and-now-recovering members of the chain.
|
||||
At runtime, a cluster must be able to respond both to
|
||||
administrative changes (e.g., substituting a failed server with
|
||||
replacement hardware) as well as local network conditions (e.g., is
|
||||
there a network partition?).
|
||||
|
||||
\subsection{The projection data structure}
|
||||
\label{sub:the-projection}
|
||||
|
||||
{\bf NOTE:} This section is a duplicate of the ``The Projection and
|
||||
the Projection Epoch Number'' section of \cite{machi-design}.
|
||||
the Projection Epoch Number'' section of the ``Machi: an immutable
|
||||
file store'' design doc \cite{machi-design}.
|
||||
|
||||
The projection data
|
||||
structure defines the current administration \& operational/runtime
|
||||
|
@ -445,6 +459,7 @@ Figure~\ref{fig:projection}. To summarize the major components:
|
|||
active_upi :: [m_server()],
|
||||
repairing :: [m_server()],
|
||||
down_members :: [m_server()],
|
||||
witness_servers :: [m_server()],
|
||||
dbg_annotations :: proplist()
|
||||
}).
|
||||
\end{verbatim}
|
||||
|
@ -454,13 +469,12 @@ Figure~\ref{fig:projection}. To summarize the major components:
|
|||
|
||||
\begin{itemize}
|
||||
\item {\tt epoch\_number} and {\tt epoch\_csum} The epoch number and
|
||||
projection checksum are unique identifiers for this projection.
|
||||
projection checksum together form the unique identifier for this projection.
|
||||
\item {\tt creation\_time} Wall-clock time, useful for humans and
|
||||
general debugging effort.
|
||||
\item {\tt author\_server} Name of the server that calculated the projection.
|
||||
\item {\tt all\_members} All servers in the chain, regardless of current
|
||||
operation status. If all operating conditions are perfect, the
|
||||
chain should operate in the order specified here.
|
||||
operation status.
|
||||
\item {\tt active\_upi} All active chain members that we know are
|
||||
fully repaired/in-sync with each other and therefore the Update
|
||||
Propagation Invariant (Section~\ref{sub:upi}) is always true.
|
||||
|
@ -468,7 +482,10 @@ Figure~\ref{fig:projection}. To summarize the major components:
|
|||
are in active data repair procedures.
|
||||
\item {\tt down\_members} All members that the {\tt author\_server}
|
||||
believes are currently down or partitioned.
|
||||
\item {\tt dbg\_annotations} A ``kitchen sink'' proplist, for code to
|
||||
\item {\tt witness\_servers} If witness servers (Section~\ref{zzz})
|
||||
are used in strong consistency mode, then they are listed here. The
|
||||
set of {\tt witness\_servers} is a subset of {\tt all\_members}.
|
||||
\item {\tt dbg\_annotations} A ``kitchen sink'' property list, for code to
|
||||
add any hints for why the projection change was made, delay/retry
|
||||
information, etc.
|
||||
\end{itemize}
|
||||
|
@ -478,7 +495,8 @@ Figure~\ref{fig:projection}. To summarize the major components:
|
|||
According to the CORFU research papers, if a server node $S$ or client
|
||||
node $C$ believes that epoch $E$ is the latest epoch, then any information
|
||||
that $S$ or $C$ receives from any source that an epoch $E+\delta$ (where
|
||||
$\delta > 0$) exists will push $S$ into the ``wedge'' state and $C$ into a mode
|
||||
$\delta > 0$) exists will push $S$ into the ``wedge'' state
|
||||
and force $C$ into a mode
|
||||
of searching for the projection definition for the newest epoch.
|
||||
|
||||
In the humming consensus description in
|
||||
|
@ -506,7 +524,7 @@ Humming consensus requires that any projection be identified by both
|
|||
the epoch number and the projection checksum, as described in
|
||||
Section~\ref{sub:the-projection}.
|
||||
|
||||
\section{Managing multiple projection store replicas}
|
||||
\section{Managing projection store replicas}
|
||||
\label{sec:managing-multiple-projection-stores}
|
||||
|
||||
An independent replica management technique very similar to the style
|
||||
|
@ -515,11 +533,63 @@ replicas of Machi's projection data structures.
|
|||
The major difference is that humming consensus
|
||||
{\em does not necessarily require}
|
||||
successful return status from a minimum number of participants (e.g.,
|
||||
a quorum).
|
||||
a majority quorum).
|
||||
|
||||
\subsection{Writing to public projection stores}
|
||||
\label{sub:proj-store-writing}
|
||||
|
||||
Writing replicas of a projection $P_{new}$ to the cluster's public
|
||||
projection stores is similar to writing in a Dynamo-like system.
|
||||
The significant difference with Chain Replication is how we interpret
|
||||
the return status of each write operation.
|
||||
|
||||
In cases of {\tt error\_written} status,
|
||||
the process may be aborted and read repair
|
||||
triggered. The most common reason for {\tt error\_written} status
|
||||
is that another actor in the system has concurrently
|
||||
already calculated another
|
||||
(perhaps different\footnote{The {\tt error\_written} may also
|
||||
indicate that another server has performed read repair on the exact
|
||||
projection $P_{new}$ that the local server is trying to write!})
|
||||
projection using the same projection epoch number.
|
||||
|
||||
\subsection{Writing to private projection stores}
|
||||
|
||||
Only the local server/owner may write to the private half of a
|
||||
projection store. Private projection store values are never subject
|
||||
to read repair.
|
||||
|
||||
\subsection{Reading from public projection stores}
|
||||
\label{sub:proj-store-reading}
|
||||
|
||||
A read is simple: for an epoch $E$, send a public projection read API
|
||||
operation to all participants. Usually, the ``get latest epoch''
|
||||
variety is used.
|
||||
|
||||
The minimum number of non-error responses is only one.\footnote{The local
|
||||
projection store should always be available, even if no other remote
|
||||
replica projection stores are available.} If all available servers
|
||||
return a single, unanimous value $V_u, V_u \ne \bot$, then $V_u$ is
|
||||
the final result for epoch $E$.
|
||||
Any non-unanimous values are considered unresolvable for the
|
||||
epoch. This disagreement is resolved by newer
|
||||
writes to the public projection stores during subsequent iterations of
|
||||
humming consensus.
|
||||
|
||||
Unavailable servers may not necessarily interfere with making a decision.
|
||||
Humming consensus
|
||||
only uses as many public projections as are available at the present
|
||||
moment of time. Assume that some server $S$ is unavailable at time $t$ and
|
||||
becomes available at some later $t+\delta$.
|
||||
If at $t+\delta$ we
|
||||
discover that $S$'s public projection store for key $E$
|
||||
contains some disagreeing value $V_{weird}$, then the disagreement
|
||||
will be resolved in the exact same manner that would have been used as if we
|
||||
had seen the disagreeing values at the earlier time $t$.
|
||||
|
||||
\subsection{Read repair: repair only unwritten values}
|
||||
|
||||
The idea of ``read repair'' is also shared with Riak Core and Dynamo
|
||||
The ``read repair'' concept is also shared with Riak Core and Dynamo
|
||||
systems. However, Machi has situations where read repair cannot truly
|
||||
``fix'' a key because two different values have been written by two
|
||||
different replicas.
|
||||
|
@ -530,85 +600,24 @@ values, all participants in humming consensus merely agree that there
|
|||
were multiple suggestions at that epoch which must be resolved by the
|
||||
creation and writing of newer projections with later epoch numbers.}
|
||||
Machi's projection store read repair can only repair values that are
|
||||
unwritten, i.e., storing $\bot$.
|
||||
unwritten, i.e., currently storing $\bot$.
|
||||
|
||||
The value used to repair $\bot$ values is the ``best'' projection that
|
||||
The value used to repair unwritten $\bot$ values is the ``best'' projection that
|
||||
is currently available for the current epoch $E$. If there is a single,
|
||||
unanimous value $V_{u}$ for the projection at epoch $E$, then $V_{u}$
|
||||
is use to repair all projections stores at $E$ that contain $\bot$
|
||||
is used to repair all projections stores at $E$ that contain $\bot$
|
||||
values. If the value of $K$ is not unanimous, then the ``highest
|
||||
ranked value'' $V_{best}$ is used for the repair; see
|
||||
Section~\ref{sub:ranking-projections} for a description of projection
|
||||
ranking.
|
||||
|
||||
\subsection{Writing to public projection stores}
|
||||
\label{sub:proj-store-writing}
|
||||
|
||||
Writing replicas of a projection $P_{new}$ to the cluster's public
|
||||
projection stores is similar, in principle, to writing a Chain
|
||||
Replication-managed system or Dynamo-like system. But unlike Chain
|
||||
Replication, the order doesn't really matter.
|
||||
In fact, the two steps below may be performed in parallel.
|
||||
The significant difference with Chain Replication is how we interpret
|
||||
the return status of each write operation.
|
||||
|
||||
\begin{enumerate}
|
||||
\item Write $P_{new}$ to the local server's public projection store
|
||||
using $P_{new}$'s epoch number $E$ as the key.
|
||||
As a side effect, a successful write will trigger
|
||||
``wedge'' status in the local server, which will then cascade to other
|
||||
projection-related activity by the local chain manager.
|
||||
\item Write $P_{new}$ to key $E$ of each remote public projection store of
|
||||
all participants in the chain.
|
||||
\end{enumerate}
|
||||
|
||||
In cases of {\tt error\_written} status,
|
||||
the process may be aborted and read repair
|
||||
triggered. The most common reason for {\tt error\_written} status
|
||||
is that another actor in the system has
|
||||
already calculated another (perhaps different) projection using the
|
||||
same projection epoch number and that
|
||||
read repair is necessary. The {\tt error\_written} may also
|
||||
indicate that another server has performed read repair on the exact
|
||||
projection $P_{new}$ that the local server is trying to write!
|
||||
|
||||
\subsection{Writing to private projection stores}
|
||||
|
||||
Only the local server/owner may write to the private half of a
|
||||
projection store. Also, the private projection store is not replicated.
|
||||
|
||||
\subsection{Reading from public projection stores}
|
||||
\label{sub:proj-store-reading}
|
||||
|
||||
A read is simple: for an epoch $E$, send a public projection read API
|
||||
request to all participants. As when writing to the public projection
|
||||
stores, we can ignore any timeout/unavailable return
|
||||
status.\footnote{The success/failure status of projection reads and
|
||||
writes is {\em not} ignored with respect to the chain manager's
|
||||
internal liveness tracker. However, the liveness tracker's state is
|
||||
typically only used when calculating new projections.} If we
|
||||
discover any unwritten values $\bot$, the read repair protocol is
|
||||
followed.
|
||||
|
||||
The minimum number of non-error responses is only one.\footnote{The local
|
||||
projection store should always be available, even if no other remote
|
||||
replica projection stores are available.} If all available servers
|
||||
return a single, unanimous value $V_u, V_u \ne \bot$, then $V_u$ is
|
||||
the final result for epoch $E$.
|
||||
Any non-unanimous values are considered complete disagreement for the
|
||||
epoch. This disagreement is resolved by humming consensus by later
|
||||
writes to the public projection stores during subsequent iterations of
|
||||
If a non-$\bot$ value exists, then by definition\footnote{Definition
|
||||
of a write-once register} this value is immutable. The only
|
||||
conflict resolution path is to write a new projection with a newer and
|
||||
larger epoch number. Once a public projection with epoch number $E$ is
|
||||
written, projections with epochs smaller than $E$ are ignored by
|
||||
humming consensus.
|
||||
|
||||
We are not concerned with unavailable servers. Humming consensus
|
||||
only uses as many public projections as are available at the present
|
||||
moment of time. If some server $S$ is unavailable at time $t$ and
|
||||
becomes available at some later $t+\delta$, and if at $t+\delta$ we
|
||||
discover that $S$'s public projection store for key $E$
|
||||
contains some disagreeing value $V_{weird}$, then the disagreement
|
||||
will be resolved in the exact same manner that would be used as if we
|
||||
had found the disagreeing values at the earlier time $t$.
|
||||
|
||||
\section{Phases of projection change, a prelude to Humming Consensus}
|
||||
\label{sec:phases-of-projection-change}
|
||||
|
||||
|
@ -671,7 +680,7 @@ straightforward; see
|
|||
Section~\ref{sub:proj-store-writing} for the technique for writing
|
||||
projections to all participating servers' projection stores.
|
||||
Humming Consensus does not care
|
||||
if the writes succeed or not: its final phase, adopting a
|
||||
if the writes succeed or not. The next phase, adopting a
|
||||
new projection, will determine which write operations are usable.
|
||||
|
||||
\subsection{Adoption a new projection}
|
||||
|
@ -685,8 +694,8 @@ to avoid direct parallels with protocols such as Raft and Paxos.)
|
|||
In general, a projection $P_{new}$ at epoch $E_{new}$ is adopted by a
|
||||
server only if
|
||||
the change in state from the local server's current projection to new
|
||||
projection, $P_{current} \rightarrow P_{new}$ will not cause data loss,
|
||||
e.g., the Update Propagation Invariant and all other safety checks
|
||||
projection, $P_{current} \rightarrow P_{new}$, will not cause data loss:
|
||||
the Update Propagation Invariant and all other safety checks
|
||||
required by chain repair in Section~\ref{sec:repair-entire-files}
|
||||
are correct. For example, any new epoch must be strictly larger than
|
||||
the current epoch, i.e., $E_{new} > E_{current}$.
|
||||
|
@ -696,16 +705,12 @@ available public projection stores. If the result is not a single
|
|||
unanmous projection, then we return to the step in
|
||||
Section~\ref{sub:projection-calculation}. If the result is a {\em
|
||||
unanimous} projection $P_{new}$ in epoch $E_{new}$, and if $P_{new}$
|
||||
does not violate chain safety checks, then the local node may
|
||||
replace its local $P_{current}$ projection with $P_{new}$.
|
||||
does not violate chain safety checks, then the local node will:
|
||||
|
||||
Not all safe projection transitions are useful, however. For example,
|
||||
it's trivally safe to suggest projection $P_{zero}$, where the chain
|
||||
length is zero. In an eventual consistency environment, projection
|
||||
$P_{one}$ where the chain length is exactly one is also trivially
|
||||
safe.\footnote{Although, if the total number of participants is more
|
||||
than one, eventual consistency would demand that $P_{self}$ cannot
|
||||
be used forever.}
|
||||
\begin{itemize}
|
||||
\item write $P_{current}$ to the local private projection store, and
|
||||
\item set its local operating state $P_{current} \leftarrow P_{new}$.
|
||||
\end{itemize}
|
||||
|
||||
\section{Humming Consensus}
|
||||
\label{sec:humming-consensus}
|
||||
|
@ -714,13 +719,11 @@ Humming consensus describes consensus that is derived only from data
|
|||
that is visible/available at the current time. It's OK if a network
|
||||
partition is in effect and not all chain members are available;
|
||||
the algorithm will calculate a rough consensus despite not
|
||||
having input from all chain members. Humming consensus
|
||||
may proceed to make a decision based on data from only one
|
||||
participant, i.e., only the local node.
|
||||
having input from all chain members.
|
||||
|
||||
\begin{itemize}
|
||||
|
||||
\item When operating in AP mode, i.e., in eventual consistency mode, humming
|
||||
\item When operating in eventual consistency mode, humming
|
||||
consensus may reconfigure a chain of length $N$ into $N$
|
||||
independent chains of length 1. When a network partition heals, the
|
||||
humming consensus is sufficient to manage the chain so that each
|
||||
|
@ -728,11 +731,12 @@ replica's data can be repaired/merged/reconciled safely.
|
|||
Other features of the Machi system are designed to assist such
|
||||
repair safely.
|
||||
|
||||
\item When operating in CP mode, i.e., in strong consistency mode, humming
|
||||
consensus would require additional restrictions. For example, any
|
||||
chain that didn't have a minimum length of the quorum majority size of
|
||||
all members would be invalid and therefore would not move itself out
|
||||
of wedged state. In very general terms, this requirement for a quorum
|
||||
\item When operating in strong consistency mode, any
|
||||
chain shorter than the quorum majority of
|
||||
all members is invalid and therefore cannot be used. Any server with
|
||||
a too-short chain cannot not move itself out
|
||||
of wedged state and is therefore unavailable for general file service.
|
||||
In very general terms, this requirement for a quorum
|
||||
majority of surviving participants is also a requirement for Paxos,
|
||||
Raft, and ZAB. See Section~\ref{sec:split-brain-management} for a
|
||||
proposal to handle ``split brain'' scenarios while in CP mode.
|
||||
|
@ -752,8 +756,6 @@ Section~\ref{sec:phases-of-projection-change}: network monitoring,
|
|||
calculating new projections, writing projections, then perhaps
|
||||
adopting the newest projection (which may or may not be the projection
|
||||
that we just wrote).
|
||||
Beginning with Section~\ref{sub:flapping-state}, we provide
|
||||
additional detail to the rough outline of humming consensus.
|
||||
|
||||
\begin{figure*}[htp]
|
||||
\resizebox{\textwidth}{!}{
|
||||
|
@ -801,15 +803,15 @@ is used by the flowchart and throughout this section.
|
|||
|
||||
\item[$\mathbf{P_{current}}$] The projection actively used by the local
|
||||
node right now. It is also the projection with largest
|
||||
epoch number in the local node's private projection store.
|
||||
epoch number in the local node's {\em private} projection store.
|
||||
|
||||
\item[$\mathbf{P_{newprop}}$] A new projection suggestion, as
|
||||
calculated by the local server
|
||||
(Section~\ref{sub:humming-projection-calculation}).
|
||||
|
||||
\item[$\mathbf{P_{latest}}$] The highest-ranked projection with the largest
|
||||
single epoch number that has been read from all available public
|
||||
projection stores, including the local node's public projection store.
|
||||
single epoch number that has been read from all available {\em public}
|
||||
projection stores.
|
||||
|
||||
\item[Unanimous] The $P_{latest}$ projection is unanimous if all
|
||||
replicas in all accessible public projection stores are effectively
|
||||
|
@ -828,7 +830,7 @@ is used by the flowchart and throughout this section.
|
|||
The flowchart has three columns, from left to right:
|
||||
|
||||
\begin{description}
|
||||
\item[Column A] Is there any reason to change?
|
||||
\item[Column A] Is there any reason to act?
|
||||
\item[Column B] Do I act?
|
||||
\item[Column C] How do I act?
|
||||
\begin{description}
|
||||
|
@ -863,12 +865,12 @@ In today's implementation, there is only a single criterion for
|
|||
determining the alive/perhaps-not-alive status of a remote server $S$:
|
||||
is $S$'s projection store available now? This question is answered by
|
||||
attemping to read the projection store on server $S$.
|
||||
If successful, then we assume that all
|
||||
$S$ is available. If $S$'s projection store is not available for any
|
||||
reason (including timeout), we assume $S$ is entirely unavailable.
|
||||
This simple single
|
||||
criterion appears to be sufficient for humming consensus, according to
|
||||
simulations of arbitrary network partitions.
|
||||
If successful, then we assume that $S$ and all of $S$'s network services
|
||||
are available. If $S$'s projection store is not available for any
|
||||
reason (including timeout), we inform the local ``fitness server''
|
||||
that we have had a problem querying $S$. The fitness service may then
|
||||
take additional monitoring/querying actions before informing us (in a
|
||||
later iteration) that $S$ should be considered down.
|
||||
|
||||
%% {\bf NOTE:} The projection store API is accessed via TCP. The network
|
||||
%% partition simulator, mentioned above and described at
|
||||
|
@ -883,64 +885,10 @@ Column~A of Figure~\ref{fig:flowchart}.
|
|||
See also, Section~\ref{sub:projection-calculation}.
|
||||
|
||||
Execution starts at ``Start'' state of Column~A of
|
||||
Figure~\ref{fig:flowchart}. Rule $A20$'s uses recent success \&
|
||||
failures in accessing other public projection stores to select a hard
|
||||
Figure~\ref{fig:flowchart}. Rule $A20$'s uses judgement from the
|
||||
local ``fitness server'' to select a definite
|
||||
boolean up/down status for each participating server.
|
||||
|
||||
\subsubsection{Calculating flapping state}
|
||||
|
||||
Also at this stage, the chain manager calculates its local
|
||||
``flapping'' state. The name ``flapping'' is borrowed from IP network
|
||||
engineer jargon ``route flapping'':
|
||||
|
||||
\begin{quotation}
|
||||
``Route flapping is caused by pathological conditions
|
||||
(hardware errors, software errors, configuration errors, intermittent
|
||||
errors in communications links, unreliable connections, etc.) within
|
||||
the network which cause certain reachability information to be
|
||||
repeatedly advertised and withdrawn.'' \cite{wikipedia-route-flapping}
|
||||
\end{quotation}
|
||||
|
||||
\paragraph{Flapping due to constantly changing network partitions and/or server crashes and restarts}
|
||||
|
||||
Currently, Machi does not attempt to dampen, smooth, or ignore recent
|
||||
history of constantly flapping peer servers. If necessary, a failure
|
||||
detector such as the $\phi$ accrual failure detector
|
||||
\cite{phi-accrual-failure-detector} can be used to help mange such
|
||||
situations.
|
||||
|
||||
\paragraph{Flapping due to asymmetric network partitions}
|
||||
|
||||
The simulator's behavior during stable periods where at least one node
|
||||
is the victim of an asymmetric network partition is \ldots weird,
|
||||
wonderful, and something I don't completely understand yet. This is
|
||||
another place where we need more eyes reviewing and trying to poke
|
||||
holes in the algorithm.
|
||||
|
||||
In cases where any node is a victim of an asymmetric network
|
||||
partition, the algorithm oscillates in a very predictable way: each
|
||||
server $S$ makes the same $P_{new}$ projection at epoch $E$ that $S$ made
|
||||
during a previous recent epoch $E-\delta$ (where $\delta$ is small, usually
|
||||
much less than 10). However, at least one node makes a suggestion that
|
||||
makes rough consensus impossible. When any epoch $E$ is not
|
||||
acceptable (because some node disagrees about something, e.g.,
|
||||
which nodes are down),
|
||||
the result is more new rounds of suggestions that create a repeating
|
||||
loop that lasts as long as the asymmetric partition lasts.
|
||||
|
||||
From the perspective of $S$'s chain manager, the pattern of this
|
||||
infinite loop is easy to detect: $S$ inspects the pattern of the last
|
||||
$L$ projections that it has suggested, e.g., the last 10.
|
||||
Tiny details such as the epoch number and creation timestamp will
|
||||
differ, but the major details such as UPI list and repairing list are
|
||||
the same.
|
||||
|
||||
If the major details of the last $L$ projections authored and
|
||||
suggested by $S$ are the same, then $S$ unilaterally decides that it
|
||||
is ``flapping'' and enters flapping state. See
|
||||
Section~\ref{sub:flapping-state} for additional disucssion of the
|
||||
flapping state.
|
||||
|
||||
\subsubsection{When to calculate a new projection}
|
||||
\label{ssub:when-to-calc}
|
||||
|
||||
|
@ -949,7 +897,7 @@ calculate a new projection. The timer interval is typically
|
|||
0.5--2.0 seconds, if the cluster has been stable. A client may call an
|
||||
external API call to trigger a new projection, e.g., if that client
|
||||
knows that an environment change has happened and wishes to trigger a
|
||||
response prior to the next timer firing.
|
||||
response prior to the next timer firing (e.g.~at state $C200$).
|
||||
|
||||
It's recommended that the timer interval be staggered according to the
|
||||
participant ranking rules in Section~\ref{sub:ranking-projections};
|
||||
|
@ -970,15 +918,14 @@ done by state $C110$ and that writing a public projection is done by
|
|||
states $C300$ and $C310$.
|
||||
|
||||
Broadly speaking, there are a number of decisions made in all three
|
||||
columns of Figure~\ref{fig:flowchart} to decide if and when any type
|
||||
of projection should be written at all. Sometimes, the best action is
|
||||
columns of Figure~\ref{fig:flowchart} to decide if and when a
|
||||
projection should be written at all. Sometimes, the best action is
|
||||
to do nothing.
|
||||
|
||||
\subsubsection{Column A: Is there any reason to change?}
|
||||
|
||||
The main tasks of the flowchart states in Column~A is to calculate a
|
||||
new projection $P_{new}$ and perhaps also the inner projection
|
||||
$P_{new2}$ if we're in flapping mode. Then we try to figure out which
|
||||
new projection $P_{new}$. Then we try to figure out which
|
||||
projection has the greatest merit: our current projection
|
||||
$P_{current}$, the new projection $P_{new}$, or the latest epoch
|
||||
$P_{latest}$. If our local $P_{current}$ projection is best, then
|
||||
|
@ -1011,7 +958,7 @@ The main decisions that states in Column B need to make are:
|
|||
|
||||
It's notable that if $P_{new}$ is truly the best projection available
|
||||
at the moment, it must always first be written to everyone's
|
||||
public projection stores and only then processed through another
|
||||
public projection stores and only afterward processed through another
|
||||
monitor \& calculate loop through the flowchart.
|
||||
|
||||
\subsubsection{Column C: How do I act?}
|
||||
|
@ -1053,14 +1000,14 @@ therefore the suggested projections at epoch $E$ are not unanimous.
|
|||
\paragraph{\#2: The transition from current $\rightarrow$ new projection is
|
||||
safe}
|
||||
|
||||
Given the projection that the server is currently using,
|
||||
Given the current projection
|
||||
$P_{current}$, the projection $P_{latest}$ is evaluated by numerous
|
||||
rules and invariants, relative to $P_{current}$.
|
||||
If such rule or invariant is
|
||||
violated/false, then the local server will discard $P_{latest}$.
|
||||
|
||||
The transition from $P_{current} \rightarrow P_{latest}$ is checked
|
||||
for safety and sanity. The conditions used for the check include:
|
||||
The transition from $P_{current} \rightarrow P_{latest}$ is protected
|
||||
by rules and invariants that include:
|
||||
|
||||
\begin{enumerate}
|
||||
\item The Erlang data types of all record members are correct.
|
||||
|
@ -1073,161 +1020,13 @@ for safety and sanity. The conditions used for the check include:
|
|||
The same re-reordering restriction applies to all
|
||||
servers in $P_{latest}$'s repairing list relative to
|
||||
$P_{current}$'s repairing list.
|
||||
\item Any server $S$ that was added to $P_{latest}$'s UPI list must
|
||||
\item Any server $S$ that is newly-added to $P_{latest}$'s UPI list must
|
||||
appear in the tail the UPI list. Furthermore, $S$ must have been in
|
||||
$P_{current}$'s repairing list and had successfully completed file
|
||||
repair prior to the transition.
|
||||
repair prior to $S$'s promotion from the repairing list to the tail
|
||||
of the UPI list.
|
||||
\end{enumerate}
|
||||
|
||||
\subsection{Additional discussion of flapping state}
|
||||
\label{sub:flapping-state}
|
||||
All $P_{new}$ projections
|
||||
calculated while in flapping state have additional diagnostic
|
||||
information added, including:
|
||||
|
||||
\begin{itemize}
|
||||
\item Flag: server $S$ is in flapping state.
|
||||
\item Epoch number \& wall clock timestamp when $S$ entered flapping state.
|
||||
\item The collection of all other known participants who are also
|
||||
flapping (with respective starting epoch numbers).
|
||||
\item A list of nodes that are suspected of being partitioned, called the
|
||||
``hosed list''. The hosed list is a union of all other hosed list
|
||||
members that are ever witnessed, directly or indirectly, by a server
|
||||
while in flapping state.
|
||||
\end{itemize}
|
||||
|
||||
\subsubsection{Flapping diagnostic data accumulates}
|
||||
|
||||
While in flapping state, this diagnostic data is gathered from
|
||||
all available participants and merged together in a CRDT-like manner.
|
||||
Once added to the diagnostic data list, a datum remains until
|
||||
$S$ drops out of flapping state. When flapping state stops, all
|
||||
accumulated diagnostic data is discarded.
|
||||
|
||||
This accumulation of diagnostic data in the projection data
|
||||
structure acts in part as a substitute for a separate gossip protocol.
|
||||
However, since all participants are already communicating with each
|
||||
other via read \& writes to each others' projection stores, the diagnostic
|
||||
data can propagate in a gossip-like manner via the projection stores.
|
||||
|
||||
\subsubsection{Flapping example (part 1)}
|
||||
\label{ssec:flapping-example}
|
||||
|
||||
Any server listed in the ``hosed list'' is suspected of having some
|
||||
kind of network communication problem with some other server. For
|
||||
example, let's examine a scenario involving a Machi cluster of servers
|
||||
$a$, $b$, $c$, $d$, and $e$. Assume there exists an asymmetric network
|
||||
partition such that messages from $a \rightarrow b$ are dropped, but
|
||||
messages from $b \rightarrow a$ are delivered.\footnote{If this
|
||||
partition were happening at or below the level of a reliable
|
||||
delivery network protocol like TCP, then communication in {\em both}
|
||||
directions would be affected by an asymmetric partition.
|
||||
However, in this model, we are
|
||||
assuming that a ``message'' lost during a network partition is a
|
||||
uni-directional projection API call or its response.}
|
||||
|
||||
Once a participant $S$ enters flapping state, it starts gathering the
|
||||
flapping starting epochs and hosed lists from all of the other
|
||||
projection stores that are available. The sum of this info is added
|
||||
to all projections calculated by $S$.
|
||||
For example, projections authored by $a$ will say that $a$ believes
|
||||
that $b$ is down.
|
||||
Likewise, projections authored by $b$ will say that $b$ believes
|
||||
that $a$ is down.
|
||||
|
||||
\subsubsection{The inner projection (flapping example, part 2)}
|
||||
\label{ssec:inner-projection}
|
||||
|
||||
\ldots We continue the example started in the previous subsection\ldots
|
||||
|
||||
Eventually, in a gossip-like manner, all other participants will
|
||||
eventually find that their hosed list is equal to $[a,b]$. Any other
|
||||
server, for example server $c$, will then calculate another
|
||||
projection, $P_{new2}$, using the assumption that both $a$ and $b$
|
||||
are down in addition to all other known unavailable servers.
|
||||
|
||||
\begin{itemize}
|
||||
\item If operating in the default CP mode, both $a$ and $b$ are down
|
||||
and therefore not eligible to participate in Chain Replication.
|
||||
%% The chain may continue service if a $c$, $d$, $e$ and/or witness
|
||||
%% servers can try to form a correct UPI list for the chain.
|
||||
This may cause an availability problem for the chain: we may not
|
||||
have a quorum of participants (real or witness-only) to form a
|
||||
correct UPI chain.
|
||||
\item If operating in AP mode, $a$ and $b$ can still form two separate
|
||||
chains of length one, using UPI lists of $[a]$ and $[b]$, respectively.
|
||||
\end{itemize}
|
||||
|
||||
This re-calculation, $P_{new2}$, of the new projection is called an
|
||||
``inner projection''. The inner projection definition is nested
|
||||
inside of its parent projection, using the same flapping disagnostic
|
||||
data used for other flapping status tracking.
|
||||
|
||||
When humming consensus has determined that a projection state change
|
||||
is necessary and is also safe (relative to both the outer and inner
|
||||
projections), then the outer projection\footnote{With the inner
|
||||
projection $P_{new2}$ nested inside of it.} is written to
|
||||
the local private projection store.
|
||||
With respect to future iterations of
|
||||
humming consensus, the innter projection is ignored.
|
||||
However, with respect to Chain Replication, the server's subsequent
|
||||
behavior
|
||||
{\em will consider the inner projection only}. The inner projection
|
||||
is used to order the UPI and repairing parts of the chain and trigger
|
||||
wedge/un-wedge behavior. The inner projection is also
|
||||
advertised to Machi clients.
|
||||
|
||||
The epoch of the inner projection, $E^{inner}$ is always less than or
|
||||
equal to the epoch of the outer projection, $E$. The $E^{inner}$
|
||||
epoch typically only changes when new servers are added to the hosed
|
||||
list.
|
||||
|
||||
To attempt a rough analogy, the outer projection is the carrier wave
|
||||
that is used to transmit the inner projection and its accompanying
|
||||
gossip of diagnostic data.
|
||||
|
||||
\subsubsection{Outer projection churn, inner projection stability}
|
||||
|
||||
One of the intriguing features of humming consensus's reaction to
|
||||
asymmetric partition: flapping behavior continues for as long as
|
||||
an any asymmetric partition exists.
|
||||
|
||||
\subsubsection{Stability in symmetric partition cases}
|
||||
|
||||
Although humming consensus hasn't been formally proven to handle all
|
||||
asymmetric and symmetric partition cases, the current implementation
|
||||
appears to converge rapidly to a single chain state in all symmetric
|
||||
partition cases. This is in contrast to asymmetric partition cases,
|
||||
where ``flapping'' will continue on every humming consensus iteration
|
||||
until all asymmetric partition disappears. A formal proof is an area of
|
||||
future work.
|
||||
|
||||
\subsubsection{Leaving flapping state and discarding inner projection}
|
||||
|
||||
There are two events that can trigger leaving flapping state.
|
||||
|
||||
\begin{itemize}
|
||||
|
||||
\item A server $S$ in flapping state notices that its long history of
|
||||
repeatedly suggesting the same projection will be broken:
|
||||
$S$ instead calculates some differing projection instead.
|
||||
This change in projection history happens whenever a perceived network
|
||||
partition changes in any way.
|
||||
|
||||
\item Server $S$ reads a public projection suggestion, $P_{noflap}$, that is
|
||||
authored by another server $S'$, and that $P_{noflap}$ no longer
|
||||
contains the flapping start epoch for $S'$ that is present in the
|
||||
history that $S$ has maintained while $S$ has been in
|
||||
flapping state.
|
||||
|
||||
\end{itemize}
|
||||
|
||||
When either trigger event happens, server $S$ will exit flapping state. All
|
||||
new projections authored by $S$ will have all flapping diagnostic data
|
||||
removed. This includes stopping use of the inner projection: the UPI
|
||||
list of the inner projection is copied to the outer projection's UPI
|
||||
list, to avoid a drastic change in UPI membership.
|
||||
|
||||
\subsection{Ranking projections}
|
||||
\label{sub:ranking-projections}
|
||||
|
||||
|
@ -1480,7 +1279,7 @@ as the foundation for Machi's data loss prevention techniques.
|
|||
\begin{figure}
|
||||
\centering
|
||||
$
|
||||
[\overbrace{\underbrace{H_1}_\textbf{Head}, M_{11}, T_1,
|
||||
[\overbrace{\underbrace{H_1}_\textbf{Head}, M_{11}, \ldots, T_1,
|
||||
H_2, M_{21},
|
||||
\ldots
|
||||
\underbrace{T_2}_\textbf{Tail}}^\textbf{Chain (U.P.~Invariant preserving)}
|
||||
|
@ -1789,7 +1588,7 @@ Manageability, availability and performance in Porcupine: a highly scalable, clu
|
|||
{\tt http://homes.cs.washington.edu/\%7Elevy/ porcupine.pdf}
|
||||
|
||||
\bibitem{chain-replication}
|
||||
van Renesse, Robbert et al.
|
||||
van Renesse, Robbert and Schneider, Fred.
|
||||
Chain Replication for Supporting High Throughput and Availability.
|
||||
Proceedings of the 6th Conference on Symposium on Operating Systems
|
||||
Design \& Implementation (OSDI'04) - Volume 6, 2004.
|
||||
|
|
|
@ -1489,7 +1489,7 @@ In Usenix ATC 2009.
|
|||
{\tt https://www.usenix.org/legacy/event/usenix09/ tech/full\_papers/terrace/terrace.pdf}
|
||||
|
||||
\bibitem{chain-replication}
|
||||
van Renesse, Robbert et al.
|
||||
van Renesse, Robbert and Schneider, Fred.
|
||||
Chain Replication for Supporting High Throughput and Availability.
|
||||
Proceedings of the 6th Conference on Symposium on Operating Systems
|
||||
Design \& Implementation (OSDI'04) - Volume 6, 2004.
|
||||
|
|
|
@ -1,12 +0,0 @@
|
|||
####################### patterns for general errors in dep modules:
|
||||
^protobuffs\.erl:
|
||||
^protobuffs_[a-z_]*\.erl:
|
||||
^leexinc\.hrl:[0-9][0-9]*:
|
||||
^machi_chain_manager1.erl:[0-9][0-9]*: Guard test RetrospectiveP::'false' =:= 'true' can never succeed
|
||||
^machi_pb\.erl:[0-9][0-9]*:
|
||||
^pokemon_pb\.erl:[0-9][0-9]*:
|
||||
####################### patterns for unknown functions:
|
||||
^ basho_bench_config:get/2
|
||||
^ erl_prettypr:format/1
|
||||
^ erl_syntax:form_list/1
|
||||
^ machi_partition_simulator:get/1
|
|
@ -18,10 +18,9 @@
|
|||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-define(MAX_FILE_SIZE, 256*1024*1024). % 256 MBytes
|
||||
-define(MAX_CHUNK_SIZE, ((1 bsl 32) - 1)).
|
||||
%% -define(DATA_DIR, "/Volumes/SAM1/seq-tests/data").
|
||||
-define(DATA_DIR, "./data").
|
||||
%% @doc Now 4GiBytes, could be up to 64bit due to PB message limit of
|
||||
%% chunk size
|
||||
-define(DEFAULT_MAX_FILE_SIZE, ((1 bsl 32) - 1)).
|
||||
-define(MINIMUM_OFFSET, 1024).
|
||||
|
||||
%% 0th draft of checksum typing with 1st byte.
|
||||
|
@ -30,8 +29,35 @@
|
|||
-define(CSUM_TAG_SERVER_SHA, 2). % Server-genereated SHA1
|
||||
-define(CSUM_TAG_SERVER_REGEN_SHA, 3). % Server-regenerated SHA1
|
||||
|
||||
-define(CSUM_TAG_NONE_ATOM, none).
|
||||
-define(CSUM_TAG_CLIENT_SHA_ATOM, client_sha).
|
||||
-define(CSUM_TAG_SERVER_SHA_ATOM, server_sha).
|
||||
-define(CSUM_TAG_SERVER_REGEN_SHA_ATOM, server_regen_sha).
|
||||
|
||||
%% Protocol Buffers goop
|
||||
-define(PB_MAX_MSG_SIZE, (33*1024*1024)).
|
||||
-define(PB_PACKET_OPTS, [{packet, 4}, {packet_size, ?PB_MAX_MSG_SIZE}]).
|
||||
|
||||
%% TODO: it's used in flu_sup and elsewhere, change this to suitable name
|
||||
-define(TEST_ETS_TABLE, test_ets_table).
|
||||
|
||||
-define(DEFAULT_COC_NAMESPACE, "").
|
||||
-define(DEFAULT_COC_LOCATOR, 0).
|
||||
|
||||
-record(ns_info, {
|
||||
version = 0 :: machi_dt:namespace_version(),
|
||||
name = <<>> :: machi_dt:namespace(),
|
||||
locator = 0 :: machi_dt:locator()
|
||||
}).
|
||||
|
||||
-record(append_opts, {
|
||||
chunk_extra = 0 :: machi_dt:chunk_size(),
|
||||
preferred_file_name :: 'undefined' | machi_dt:file_name_s(),
|
||||
flag_fail_preferred = false :: boolean()
|
||||
}).
|
||||
|
||||
-record(read_opts, {
|
||||
no_checksum = false :: boolean(),
|
||||
no_chunk = false :: boolean(),
|
||||
needs_trimmed = false :: boolean()
|
||||
}).
|
||||
|
|
20
include/machi_merkle_tree.hrl
Normal file
20
include/machi_merkle_tree.hrl
Normal file
|
@ -0,0 +1,20 @@
|
|||
%% machi merkle tree records
|
||||
|
||||
-record(naive, {
|
||||
chunk_size = 1048576 :: pos_integer(), %% default 1 MB
|
||||
recalc = true :: boolean(),
|
||||
root :: 'undefined' | binary(),
|
||||
lvl1 = [] :: [ binary() ],
|
||||
lvl2 = [] :: [ binary() ],
|
||||
lvl3 = [] :: [ binary() ],
|
||||
leaves = [] :: [ { Offset :: pos_integer(),
|
||||
Size :: pos_integer(),
|
||||
Csum :: binary()} ]
|
||||
}).
|
||||
|
||||
-record(mt, {
|
||||
filename :: string(),
|
||||
tree :: #naive{},
|
||||
backend = 'naive' :: 'naive'
|
||||
}).
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved.
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
|
@ -22,10 +22,11 @@
|
|||
-define(MACHI_PROJECTION_HRL, true).
|
||||
|
||||
-type pv1_consistency_mode() :: 'ap_mode' | 'cp_mode'.
|
||||
-type pv1_chain_name():: atom().
|
||||
-type pv1_csum() :: binary().
|
||||
-type pv1_epoch() :: {pv1_epoch_n(), pv1_csum()}.
|
||||
-type pv1_epoch_n() :: non_neg_integer().
|
||||
-type pv1_server() :: atom() | binary().
|
||||
-type pv1_server() :: atom().
|
||||
-type pv1_timestamp() :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}.
|
||||
|
||||
-record(p_srvr, {
|
||||
|
@ -55,6 +56,7 @@
|
|||
epoch_number :: pv1_epoch_n() | ?SPAM_PROJ_EPOCH,
|
||||
epoch_csum :: pv1_csum(),
|
||||
author_server :: pv1_server(),
|
||||
chain_name = ch_not_def_yet :: pv1_chain_name(),
|
||||
all_members :: [pv1_server()],
|
||||
witnesses = [] :: [pv1_server()],
|
||||
creation_time :: pv1_timestamp(),
|
||||
|
@ -75,4 +77,16 @@
|
|||
%% create a consistent projection ranking score.
|
||||
-define(MAX_CHAIN_LENGTH, 64).
|
||||
|
||||
-record(chain_def_v1, {
|
||||
name :: atom(), % chain name
|
||||
mode :: pv1_consistency_mode(),
|
||||
full = [] :: [p_srvr()],
|
||||
witnesses = [] :: [p_srvr()],
|
||||
old_full = [] :: [pv1_server()], % guard against some races
|
||||
old_witnesses=[] :: [pv1_server()], % guard against some races
|
||||
local_run = [] :: [pv1_server()], % must be tailored to each machine!
|
||||
local_stop = [] :: [pv1_server()], % must be tailored to each machine!
|
||||
props = [] :: list() % proplist for other related info
|
||||
}).
|
||||
|
||||
-endif. % !MACHI_PROJECTION_HRL
|
||||
|
|
56
priv/humming-consensus-demo.setup.sh
Executable file
56
priv/humming-consensus-demo.setup.sh
Executable file
|
@ -0,0 +1,56 @@
|
|||
#!/bin/sh
|
||||
|
||||
echo "Step: Verify that the required entries in /etc/hosts are present"
|
||||
for i in 1 2 3; do
|
||||
grep machi$i /etc/hosts | egrep -s '^127.0.0.1' > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
echo ""
|
||||
echo "'grep -s machi$i' failed. Aborting, sorry."
|
||||
exit 1
|
||||
fi
|
||||
ping -c 1 machi$i > /dev/null 2>&1
|
||||
if [ $? -ne 0 ]; then
|
||||
echo ""
|
||||
echo "Ping attempt on host machi$i failed. Aborting."
|
||||
echo ""
|
||||
ping -c 1 machi$i
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Step: add a verbose logging option to app.config"
|
||||
for i in 1 2 3; do
|
||||
ed ./dev/dev$i/etc/app.config <<EOF > /dev/null 2>&1
|
||||
/verbose_confirm
|
||||
a
|
||||
{chain_manager_opts, [{private_write_verbose_confirm,true}]},
|
||||
{stability_time, 1},
|
||||
.
|
||||
w
|
||||
q
|
||||
EOF
|
||||
done
|
||||
|
||||
echo "Step: start three three Machi application instances"
|
||||
for i in 1 2 3; do
|
||||
./dev/dev$i/bin/machi start
|
||||
./dev/dev$i/bin/machi ping
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Sorry, a 'ping' check for instance dev$i failed. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Step: configure one chain to start a Humming Consensus group with three members"
|
||||
|
||||
# Note: $CWD of each Machi proc is two levels below the source code root dir.
|
||||
LIFECYCLE000=../../priv/quick-admin-examples/demo-000
|
||||
for i in 3 2 1; do
|
||||
./dev/dev$i/bin/machi-admin quick-admin-apply $LIFECYCLE000 machi$i
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Sorry, 'machi-admin quick-admin-apply failed' on machi$i. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
exit 0
|
93
priv/humming-consensus-demo.vagrant/Vagrantfile
vendored
Normal file
93
priv/humming-consensus-demo.vagrant/Vagrantfile
vendored
Normal file
|
@ -0,0 +1,93 @@
|
|||
# -*- mode: ruby -*-
|
||||
# vi: set ft=ruby :
|
||||
|
||||
# All Vagrant configuration is done below. The "2" in Vagrant.configure
|
||||
# configures the configuration version (we support older styles for
|
||||
# backwards compatibility). Please don't change it unless you know what
|
||||
# you're doing.
|
||||
Vagrant.configure(2) do |config|
|
||||
# The most common configuration options are documented and commented below.
|
||||
# For a complete reference, please see the online documentation at
|
||||
# https://docs.vagrantup.com.
|
||||
|
||||
# Every Vagrant development environment requires a box. You can search for
|
||||
# boxes at https://atlas.hashicorp.com/search.
|
||||
# If this Vagrant box has not been downloaded before (e.g. using "vagrant box add"),
|
||||
# then Vagrant will automatically download the VM image from HashiCorp.
|
||||
config.vm.box = "hashicorp/precise64"
|
||||
# If using a FreeBSD box, Bash may not be installed.
|
||||
# Use the config.ssh.shell setting to specify an alternate shell.
|
||||
# Note, however, that any code in the 'config.vm.provision' section
|
||||
# would then have to use this shell's syntax!
|
||||
# config.ssh.shell = "/bin/csh -l"
|
||||
|
||||
# Disable automatic box update checking. If you disable this, then
|
||||
# boxes will only be checked for updates when the user runs
|
||||
# `vagrant box outdated`. This is not recommended.
|
||||
# config.vm.box_check_update = false
|
||||
|
||||
# Create a forwarded port mapping which allows access to a specific port
|
||||
# within the machine from a port on the host machine. In the example below,
|
||||
# accessing "localhost:8080" will access port 80 on the guest machine.
|
||||
# config.vm.network "forwarded_port", guest: 80, host: 8080
|
||||
|
||||
# Create a private network, which allows host-only access to the machine
|
||||
# using a specific IP.
|
||||
# config.vm.network "private_network", ip: "192.168.33.10"
|
||||
|
||||
# Create a public network, which generally matched to bridged network.
|
||||
# Bridged networks make the machine appear as another physical device on
|
||||
# your network.
|
||||
# config.vm.network "public_network"
|
||||
|
||||
# Share an additional folder to the guest VM. The first argument is
|
||||
# the path on the host to the actual folder. The second argument is
|
||||
# the path on the guest to mount the folder. And the optional third
|
||||
# argument is a set of non-required options.
|
||||
# config.vm.synced_folder "../data", "/vagrant_data"
|
||||
|
||||
# Provider-specific configuration so you can fine-tune various
|
||||
# backing providers for Vagrant. These expose provider-specific options.
|
||||
# Example for VirtualBox:
|
||||
#
|
||||
config.vm.provider "virtualbox" do |vb|
|
||||
# Display the VirtualBox GUI when booting the machine
|
||||
# vb.gui = true
|
||||
|
||||
# Customize the amount of memory on the VM:
|
||||
vb.memory = "512"
|
||||
end
|
||||
#
|
||||
# View the documentation for the provider you are using for more
|
||||
# information on available options.
|
||||
|
||||
# Define a Vagrant Push strategy for pushing to Atlas. Other push strategies
|
||||
# such as FTP and Heroku are also available. See the documentation at
|
||||
# https://docs.vagrantup.com/v2/push/atlas.html for more information.
|
||||
# config.push.define "atlas" do |push|
|
||||
# push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME"
|
||||
# end
|
||||
|
||||
# Enable provisioning with a shell script. Additional provisioners such as
|
||||
# Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
|
||||
# documentation for more information about their specific syntax and use.
|
||||
config.vm.provision "shell", inline: <<-SHELL
|
||||
# Install prerequsites
|
||||
# Support here for FreeBSD is experimental
|
||||
apt-get update ; sudo apt-get install -y git sudo rsync ; # Ubuntu Linux
|
||||
env ASSUME_ALWAYS_YES=yes pkg install -f git sudo rsync ; # FreeBSD 10
|
||||
|
||||
# Install dependent packages, using slf-configurator
|
||||
git clone https://github.com/slfritchie/slf-configurator.git
|
||||
chown -R vagrant ./slf-configurator
|
||||
(cd slf-configurator ; sudo sh -x ./ALL.sh)
|
||||
echo 'export PATH=${PATH}:/usr/local/erlang/17.5/bin' >> ~vagrant/.bashrc
|
||||
export PATH=${PATH}:/usr/local/erlang/17.5/bin
|
||||
## echo 'set path = ( $path /usr/local/erlang/17.5/bin )' >> ~vagrant/.cshrc
|
||||
## setenv PATH /usr/local/erlang/17.5/bin:$PATH
|
||||
|
||||
git clone https://github.com/basho/machi.git
|
||||
(cd machi ; git checkout master ; make && make test )
|
||||
chown -R vagrant ./machi
|
||||
SHELL
|
||||
end
|
|
@ -36,7 +36,7 @@ while (<I>) {
|
|||
$indent = " " x ($count * 4);
|
||||
s/^#*\s*[0-9. ]*//;
|
||||
$anchor = "n$label";
|
||||
printf T1 "%s+ [%s %s](#%s)\n", $indent, $label, $_, $anchor;
|
||||
printf T1 "%s+ [%s. %s](#%s)\n", $indent, $label, $_, $anchor;
|
||||
printf T2 "<a name=\"%s\">\n", $anchor;
|
||||
$line =~ s/(#+)\s*[0-9. ]*/$1 $label. /;
|
||||
print T2 $line;
|
||||
|
|
1
priv/quick-admin-examples/000
Normal file
1
priv/quick-admin-examples/000
Normal file
|
@ -0,0 +1 @@
|
|||
{host, "localhost", []}.
|
4
priv/quick-admin-examples/001
Normal file
4
priv/quick-admin-examples/001
Normal file
|
@ -0,0 +1,4 @@
|
|||
{flu,f1,"localhost",20401,[]}.
|
||||
{flu,f2,"localhost",20402,[]}.
|
||||
{flu,f3,"localhost",20403,[]}.
|
||||
{chain,c1,[f1,f2,f3],[]}.
|
4
priv/quick-admin-examples/002
Normal file
4
priv/quick-admin-examples/002
Normal file
|
@ -0,0 +1,4 @@
|
|||
{flu,f4,"localhost",20404,[]}.
|
||||
{flu,f5,"localhost",20405,[]}.
|
||||
{flu,f6,"localhost",20406,[]}.
|
||||
{chain,c2,[f4,f5,f6],[]}.
|
7
priv/quick-admin-examples/demo-000
Normal file
7
priv/quick-admin-examples/demo-000
Normal file
|
@ -0,0 +1,7 @@
|
|||
{host, "machi1", []}.
|
||||
{host, "machi2", []}.
|
||||
{host, "machi3", []}.
|
||||
{flu,f1,"machi1",20401,[]}.
|
||||
{flu,f2,"machi2",20402,[]}.
|
||||
{flu,f3,"machi3",20403,[]}.
|
||||
{chain,c1,[f1,f2,f3],[]}.
|
10
priv/test-for-gh-pr.sh
Executable file
10
priv/test-for-gh-pr.sh
Executable file
|
@ -0,0 +1,10 @@
|
|||
#!/bin/sh
|
||||
|
||||
if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then
|
||||
echo '$TRAVIS_PULL_REQUEST is false, skipping tests'
|
||||
exit 0
|
||||
else
|
||||
echo '$TRAVIS_PULL_REQUEST is not false ($TRAVIS_PULL_REQUEST), running tests'
|
||||
make test
|
||||
make dialyzer
|
||||
fi
|
15
rebar.config
15
rebar.config
|
@ -1,12 +1,21 @@
|
|||
{require_otp_vsn, "17"}.
|
||||
{require_otp_vsn, "17|18"}.
|
||||
|
||||
%%% {erl_opts, [warnings_as_errors, {parse_transform, lager_transform}, debug_info]}.
|
||||
{erl_opts, [{parse_transform, lager_transform}, debug_info]}.
|
||||
{edoc_opts, [{dir, "./edoc"}]}.
|
||||
|
||||
{deps, [
|
||||
{lager, ".*", {git, "git://github.com/basho/lager.git", {tag, "2.0.1"}}},
|
||||
{cuttlefish, ".*", {git, "git://github.com/basho/cuttlefish.git", {branch, "develop"}}},
|
||||
{sext, ".*", {git, "git://github.com/basho/sext.git", {branch, "master"}}},
|
||||
{eleveldb, ".*", {git, "git://github.com/basho/eleveldb.git", {branch, "develop"}}},
|
||||
{lager, ".*", {git, "git://github.com/basho/lager.git", {tag, "2.2.0"}}},
|
||||
{protobuffs, "0.8.*", {git, "git://github.com/basho/erlang_protobuffs.git", {tag, "0.8.1p4"}}},
|
||||
{riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {branch, "develop"}}}
|
||||
{riak_dt, ".*", {git, "git://github.com/basho/riak_dt.git", {branch, "develop"}}},
|
||||
{ranch, ".*", {git, "git://github.com/ninenines/ranch.git", {branch, "master"}}},
|
||||
{node_package, ".*", {git, "git://github.com/basho/node_package.git", {branch, "develop"}}},
|
||||
{eper, ".*", {git, "git://github.com/basho/eper.git", {tag, "0.92-basho1"}}},
|
||||
{cluster_info, ".*", {git, "git://github.com/basho/cluster_info", {branch, "develop"}}}
|
||||
]}.
|
||||
|
||||
{sub_dirs, ["rel", "apps/machi"]}.
|
||||
{lib_dirs, ["apps/machi"]}.
|
||||
|
|
35
rel/files/app.config
Normal file
35
rel/files/app.config
Normal file
|
@ -0,0 +1,35 @@
|
|||
[
|
||||
{machi, [
|
||||
%% Data directory for all FLUs.
|
||||
{flu_data_dir, "{{platform_data_dir}}/flu"},
|
||||
|
||||
%% FLU config directory
|
||||
{flu_config_dir, "{{platform_etc_dir}}/flu-config"},
|
||||
|
||||
%% Chain config directory
|
||||
{chain_config_dir, "{{platform_etc_dir}}/chain-config"},
|
||||
|
||||
%% FLUs to start at app start.
|
||||
%% This task has moved to machi_flu_sup and machi_lifecycle_mgr.
|
||||
|
||||
%% Number of metadata manager processes to run per FLU.
|
||||
%% Default = 10
|
||||
%% {metadata_manager_count, 2},
|
||||
|
||||
%% Default options for chain manager processes.
|
||||
%% {chain_manager_opts, [{private_write_verbose,true},
|
||||
%% {private_write_verbose_confirm,true}]},
|
||||
|
||||
%% Platform vars (mirror of reltool packaging)
|
||||
{platform_data_dir, "{{platform_data_dir}}"},
|
||||
{platform_etc_dir, "{{platform_etc_dir}}"},
|
||||
|
||||
%% Do not delete, do not put Machi config items after this line.
|
||||
{final_comma_stopper, do_not_delete}
|
||||
]
|
||||
},
|
||||
{lager, [
|
||||
{error_logger_hwm, 5000} % lager's default of 50/sec is too low
|
||||
]
|
||||
}
|
||||
].
|
84
rel/files/machi-admin
Executable file
84
rel/files/machi-admin
Executable file
|
@ -0,0 +1,84 @@
|
|||
#!/bin/sh
|
||||
# -*- tab-width:4;indent-tabs-mode:nil -*-
|
||||
# ex: ts=4 sw=4 et
|
||||
|
||||
# Pull environment for this install
|
||||
. "{{runner_base_dir}}/lib/env.sh"
|
||||
|
||||
# Make sure the user running this script is the owner and/or su to that user
|
||||
check_user "$@"
|
||||
ES=$?
|
||||
if [ "$ES" -ne 0 ]; then
|
||||
exit $ES
|
||||
fi
|
||||
|
||||
# Keep track of where script was invoked
|
||||
ORIGINAL_DIR=$(pwd)
|
||||
|
||||
# Make sure CWD is set to runner run dir
|
||||
cd $RUNNER_BASE_DIR
|
||||
|
||||
# Identify the script name
|
||||
SCRIPT=`basename $0`
|
||||
|
||||
usage() {
|
||||
echo "Usage: $SCRIPT { quick-admin-check | quick-admin-apply | "
|
||||
echo " top }"
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
quick-admin-check)
|
||||
# Make sure the local node IS running
|
||||
node_up_check
|
||||
|
||||
shift
|
||||
|
||||
NODE_NAME=${NAME_ARG#* } # target machi server node name
|
||||
IN_FILE="$1"
|
||||
|
||||
$ERTS_PATH/erl -noshell -noinput $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \
|
||||
-remsh $NODE_NAME \
|
||||
-eval "Me = self(), spawn('"$NODE_NAME"', fun() -> X = (catch(machi_lifecycle_mgr:quick_admin_sanity_check(\"$IN_FILE\"))), Me ! {res, X} end), XX = receive {res, Res} -> Res after 10*1000 -> timeout end, io:format(user, \"Result: ~p\n\", [XX]), case XX of \
|
||||
ok -> init:stop(); \
|
||||
_ -> init:stop(1) \
|
||||
end."
|
||||
|
||||
;;
|
||||
quick-admin-apply)
|
||||
# Make sure the local node IS running
|
||||
node_up_check
|
||||
|
||||
shift
|
||||
|
||||
NODE_NAME=${NAME_ARG#* } # target machi server node name
|
||||
IN_FILE="$1"
|
||||
RELATIVE_HOST="$2"
|
||||
|
||||
$ERTS_PATH/erl -noshell -noinput $NAME_PARAM machi_test$NAME_HOST $COOKIE_ARG \
|
||||
-remsh $NODE_NAME \
|
||||
-eval "Me = self(), spawn('"$NODE_NAME"', fun() -> X = (catch(machi_lifecycle_mgr:quick_admin_apply(\"$IN_FILE\", \"$RELATIVE_HOST\"))), Me ! {res, X} end), XX = receive {res, Res} -> Res after 10*1000 -> timeout end, io:format(user, \"Result: ~p\n\", [XX]), case XX of \
|
||||
ok -> init:stop(); \
|
||||
_ -> init:stop(1) \
|
||||
end."
|
||||
|
||||
;;
|
||||
top)
|
||||
# Make sure the local node IS running
|
||||
node_up_check
|
||||
|
||||
shift
|
||||
|
||||
MYPID=$$
|
||||
NODE_NAME=${NAME_ARG#* }
|
||||
$ERTS_PATH/erl -noshell -noinput \
|
||||
-pa $RUNNER_LIB_DIR/basho-patches \
|
||||
-hidden $NAME_PARAM machi_etop$MYPID$NAME_HOST $COOKIE_ARG \
|
||||
-s etop -s erlang halt -output text \
|
||||
-node $NODE_NAME \
|
||||
$* -tracing off
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
220
rel/files/machi.schema
Normal file
220
rel/files/machi.schema
Normal file
|
@ -0,0 +1,220 @@
|
|||
%%-*- mode: erlang -*-
|
||||
|
||||
%% @doc Where to emit the default log messages (typically at 'info'
|
||||
%% severity):
|
||||
%% off: disabled
|
||||
%% file: the file specified by log.console.file
|
||||
%% console: to standard output (seen when using `machi attach-direct`)
|
||||
%% both: log.console.file and standard out.
|
||||
{mapping, "log.console", "lager.handlers", [
|
||||
{default, {{console_log_default}} },
|
||||
{datatype, {enum, [off, file, console, both]}}
|
||||
]}.
|
||||
|
||||
%% @doc The severity level of the console log, default is 'info'.
|
||||
{mapping, "log.console.level", "lager.handlers", [
|
||||
{default, info},
|
||||
{datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}}
|
||||
]}.
|
||||
|
||||
%% @doc When 'log.console' is set to 'file' or 'both', the file where
|
||||
%% console messages will be logged.
|
||||
{mapping, "log.console.file", "lager.handlers", [
|
||||
{default, "$(platform_log_dir)/console.log"},
|
||||
{datatype, file}
|
||||
]}.
|
||||
|
||||
%% @doc The file where error messages will be logged.
|
||||
{mapping, "log.error.file", "lager.handlers", [
|
||||
{default, "$(platform_log_dir)/error.log"},
|
||||
{datatype, file}
|
||||
]}.
|
||||
|
||||
%% @doc When set to 'on', enables log output to syslog.
|
||||
{mapping, "log.syslog", "lager.handlers", [
|
||||
{default, off},
|
||||
{datatype, flag}
|
||||
]}.
|
||||
|
||||
%% @doc When set to 'on', enables log output to syslog.
|
||||
{mapping, "log.syslog.ident", "lager.handlers", [
|
||||
{default, "machi"},
|
||||
hidden
|
||||
]}.
|
||||
|
||||
%% @doc Syslog facility to log entries from Riak.
|
||||
{mapping, "log.syslog.facility", "lager.handlers", [
|
||||
{default, daemon},
|
||||
{datatype, {enum,[kern, user, mail, daemon, auth, syslog,
|
||||
lpr, news, uucp, clock, authpriv, ftp,
|
||||
cron, local0, local1, local2, local3,
|
||||
local4, local5, local6, local7]}},
|
||||
hidden
|
||||
]}.
|
||||
|
||||
%% @doc The severity level at which to log entries to syslog, default is 'info'.
|
||||
{mapping, "log.syslog.level", "lager.handlers", [
|
||||
{default, info},
|
||||
{datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, none]}},
|
||||
hidden
|
||||
]}.
|
||||
|
||||
{translation,
|
||||
"lager.handlers",
|
||||
fun(Conf) ->
|
||||
SyslogHandler = case cuttlefish:conf_get("log.syslog", Conf) of
|
||||
true ->
|
||||
Ident = cuttlefish:conf_get("log.syslog.ident", Conf),
|
||||
Facility = cuttlefish:conf_get("log.syslog.facility", Conf),
|
||||
LogLevel = cuttlefish:conf_get("log.syslog.level", Conf),
|
||||
[{lager_syslog_backend, [Ident, Facility, LogLevel]}];
|
||||
_ -> []
|
||||
end,
|
||||
ErrorHandler = case cuttlefish:conf_get("log.error.file", Conf) of
|
||||
undefined -> [];
|
||||
ErrorFilename -> [{lager_file_backend, [{file, ErrorFilename},
|
||||
{level, error},
|
||||
{size, 10485760},
|
||||
{date, "$D0"},
|
||||
{count, 5}]}]
|
||||
end,
|
||||
|
||||
ConsoleLogLevel = cuttlefish:conf_get("log.console.level", Conf),
|
||||
ConsoleLogFile = cuttlefish:conf_get("log.console.file", Conf),
|
||||
|
||||
ConsoleHandler = {lager_console_backend, ConsoleLogLevel},
|
||||
ConsoleFileHandler = {lager_file_backend, [{file, ConsoleLogFile},
|
||||
{level, ConsoleLogLevel},
|
||||
{size, 10485760},
|
||||
{date, "$D0"},
|
||||
{count, 5}]},
|
||||
|
||||
ConsoleHandlers = case cuttlefish:conf_get("log.console", Conf) of
|
||||
off -> [];
|
||||
file -> [ConsoleFileHandler];
|
||||
console -> [ConsoleHandler];
|
||||
both -> [ConsoleHandler, ConsoleFileHandler];
|
||||
_ -> []
|
||||
end,
|
||||
SyslogHandler ++ ConsoleHandlers ++ ErrorHandler
|
||||
end
|
||||
}.
|
||||
|
||||
|
||||
%% @doc Whether to enable Erlang's built-in error logger.
|
||||
{mapping, "sasl", "sasl.sasl_error_logger", [
|
||||
{default, off},
|
||||
{datatype, flag},
|
||||
hidden
|
||||
]}.
|
||||
|
||||
%% @doc Whether to enable the crash log.
|
||||
{mapping, "log.crash", "lager.crash_log", [
|
||||
{default, on},
|
||||
{datatype, flag}
|
||||
]}.
|
||||
|
||||
%% @doc If the crash log is enabled, the file where its messages will
|
||||
%% be written.
|
||||
{mapping, "log.crash.file", "lager.crash_log", [
|
||||
{default, "$(platform_log_dir)/crash.log"},
|
||||
{datatype, file}
|
||||
]}.
|
||||
|
||||
{translation,
|
||||
"lager.crash_log",
|
||||
fun(Conf) ->
|
||||
case cuttlefish:conf_get("log.crash", Conf) of
|
||||
false -> undefined;
|
||||
_ ->
|
||||
cuttlefish:conf_get("log.crash.file", Conf, "{{platform_log_dir}}/crash.log")
|
||||
end
|
||||
end}.
|
||||
|
||||
%% @doc Maximum size in bytes of individual messages in the crash log
|
||||
{mapping, "log.crash.maximum_message_size", "lager.crash_log_msg_size", [
|
||||
{default, "64KB"},
|
||||
{datatype, bytesize}
|
||||
]}.
|
||||
|
||||
%% @doc Maximum size of the crash log in bytes, before it is rotated
|
||||
{mapping, "log.crash.size", "lager.crash_log_size", [
|
||||
{default, "10MB"},
|
||||
{datatype, bytesize}
|
||||
]}.
|
||||
|
||||
%% @doc The schedule on which to rotate the crash log. For more
|
||||
%% information see:
|
||||
%% https://github.com/basho/lager/blob/master/README.md#internal-log-rotation
|
||||
{mapping, "log.crash.rotation", "lager.crash_log_date", [
|
||||
{default, "$D0"}
|
||||
]}.
|
||||
|
||||
%% @doc The number of rotated crash logs to keep. When set to
|
||||
%% 'current', only the current open log file is kept.
|
||||
{mapping, "log.crash.rotation.keep", "lager.crash_log_count", [
|
||||
{default, 5},
|
||||
{datatype, [integer, {atom, current}]},
|
||||
{validators, ["rotation_count"]}
|
||||
]}.
|
||||
|
||||
{validator,
|
||||
"rotation_count",
|
||||
"must be 'current' or a positive integer",
|
||||
fun(current) -> true;
|
||||
(Int) when is_integer(Int) andalso Int >= 0 -> true;
|
||||
(_) -> false
|
||||
end}.
|
||||
|
||||
{translation,
|
||||
"lager.crash_log_count",
|
||||
fun(Conf) ->
|
||||
case cuttlefish:conf_get("log.crash.rotation.keep", Conf) of
|
||||
current -> 0;
|
||||
Int -> Int
|
||||
end
|
||||
end}.
|
||||
|
||||
%% @doc Whether to redirect error_logger messages into lager -
|
||||
%% defaults to true
|
||||
{mapping, "log.error.redirect", "lager.error_logger_redirect", [
|
||||
{default, on},
|
||||
{datatype, flag},
|
||||
hidden
|
||||
]}.
|
||||
|
||||
%% @doc Maximum number of error_logger messages to handle in a second
|
||||
{mapping, "log.error.messages_per_second", "lager.error_logger_hwm", [
|
||||
{default, 100},
|
||||
{datatype, integer},
|
||||
hidden
|
||||
]}.
|
||||
|
||||
|
||||
%% @doc Cookie for distributed node communication. All nodes in the
|
||||
%% same cluster should use the same cookie or they will not be able to
|
||||
%% communicate.
|
||||
{mapping, "distributed_cookie", "vm_args.-setcookie", [
|
||||
{default, "machi"}
|
||||
]}.
|
||||
|
||||
|
||||
%% override zdbbl from 1mb to 32mb
|
||||
{mapping, "erlang.distribution_buffer_size", "vm_args.+zdbbl", [
|
||||
{default, "32MB"},
|
||||
merge
|
||||
]}.
|
||||
|
||||
%% VM scheduler collapse, part 1 of 2
|
||||
{mapping, "erlang.schedulers.force_wakeup_interval", "vm_args.+sfwi", [
|
||||
{default, 500},
|
||||
{datatype, integer},
|
||||
merge
|
||||
]}.
|
||||
|
||||
%% VM scheduler collapse, part 2 of 2
|
||||
{mapping, "erlang.schedulers.compaction_of_load", "vm_args.+scl", [
|
||||
{default, false},
|
||||
merge
|
||||
]}.
|
||||
|
27
rel/files/vm.args
Normal file
27
rel/files/vm.args
Normal file
|
@ -0,0 +1,27 @@
|
|||
## Name of the riak node
|
||||
-name {{node}}
|
||||
|
||||
## Cookie for distributed erlang. All nodes in the same cluster
|
||||
## should use the same cookie or they will not be able to communicate.
|
||||
-setcookie machi
|
||||
|
||||
## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive
|
||||
## (Disabled by default..use with caution!)
|
||||
##-heart
|
||||
|
||||
## Enable kernel poll and a few async threads
|
||||
+K true
|
||||
+A 64
|
||||
|
||||
## Treat error_logger warnings as warnings
|
||||
+W w
|
||||
|
||||
## Increase number of concurrent ports/sockets
|
||||
-env ERL_MAX_PORTS 4096
|
||||
|
||||
## ## Tweak GC to run more often
|
||||
## -env ERL_FULLSWEEP_AFTER 0
|
||||
|
||||
## Set the location of crash dumps
|
||||
-env ERL_CRASH_DUMP {{crash_dump}}
|
||||
|
16
rel/gen_dev
Executable file
16
rel/gen_dev
Executable file
|
@ -0,0 +1,16 @@
|
|||
#! /bin/sh
|
||||
#
|
||||
# Example usage: gen_dev dev4 vars.src vars
|
||||
#
|
||||
# Generate an overlay config for devNNN from vars.src and write to vars
|
||||
#
|
||||
|
||||
NAME=$1
|
||||
TEMPLATE=$2
|
||||
VARFILE=$3
|
||||
|
||||
NODE="$NAME@127.0.0.1"
|
||||
|
||||
echo "Generating $NAME - node='$NODE'"
|
||||
sed -e "s/@NODE@/$NODE/" \
|
||||
< $TEMPLATE > $VARFILE
|
4
rel/rebar.config
Normal file
4
rel/rebar.config
Normal file
|
@ -0,0 +1,4 @@
|
|||
{lib_dirs, ["../deps"]}.
|
||||
%% {plugin_dir, "../deps/cuttlefish/src"}.
|
||||
%% {plugins, [cuttlefish_rebar_plugin]}.
|
||||
%% {cuttlefish_filename, "machi.conf"}.
|
113
rel/reltool.config
Normal file
113
rel/reltool.config
Normal file
|
@ -0,0 +1,113 @@
|
|||
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
|
||||
%% ex: ft=erlang ts=4 sw=4 et
|
||||
{sys, [
|
||||
{lib_dirs, ["../deps"]},
|
||||
{rel, "machi", "0.0.0",
|
||||
[
|
||||
kernel,
|
||||
stdlib,
|
||||
lager,
|
||||
sasl,
|
||||
public_key,
|
||||
ssl,
|
||||
%% riak_sysmon,
|
||||
%% os_mon,
|
||||
crypto,
|
||||
runtime_tools,
|
||||
machi
|
||||
%% cluster_info,
|
||||
%% exometer_core,
|
||||
]},
|
||||
{rel, "start_clean", "",
|
||||
[
|
||||
kernel,
|
||||
stdlib
|
||||
]},
|
||||
{boot_rel, "machi"},
|
||||
{profile, embedded},
|
||||
{excl_sys_filters, ["^bin/.*",
|
||||
"^erts.*/bin/(dialyzer|typer)",
|
||||
"^erts.*/doc",
|
||||
"^erts.*/man"]},
|
||||
{excl_archive_filters, [".*"]},
|
||||
%% {app, cuttlefish, [{incl_cond, include}]},
|
||||
%% {app, cluster_info, [{incl_cond, include}]},
|
||||
{app, eper, [{incl_cond, include}]},
|
||||
{app, sasl, [{incl_cond, include}]},
|
||||
%% {app, syslog, [{incl_cond, include}]},
|
||||
%% {app, lager_syslog, [{incl_cond, include}]},
|
||||
{app, lager, [{incl_cond, include}]}
|
||||
%% {app, exometer_core, [{incl_cond, include}]},
|
||||
]}.
|
||||
|
||||
|
||||
{target_dir, "machi"}.
|
||||
|
||||
{overlay_vars, "vars.config"}.
|
||||
|
||||
{overlay, [
|
||||
{mkdir, "data"},
|
||||
{mkdir, "data/^PRESERVE"},
|
||||
{mkdir, "log"},
|
||||
|
||||
%% Copy base files for starting and interacting w/ node
|
||||
{copy, "../deps/node_package/priv/base/erl",
|
||||
"{{erts_vsn}}/bin/erl"},
|
||||
{copy, "../deps/node_package/priv/base/nodetool",
|
||||
"{{erts_vsn}}/bin/nodetool"},
|
||||
%% {copy, "../deps/cuttlefish/cuttlefish",
|
||||
%% "{{erts_vsn}}/bin/cuttlefish"},
|
||||
{template, "../deps/node_package/priv/base/runner",
|
||||
"bin/machi"},
|
||||
{template, "../deps/node_package/priv/base/env.sh",
|
||||
"lib/env.sh"},
|
||||
{template, "../deps/node_package/priv/base/app_epath.sh",
|
||||
"lib/app_epath.sh"},
|
||||
|
||||
%% Copy config files
|
||||
|
||||
%% Cuttlefish Schema Files have a priority order.
|
||||
%% Anything in a file prefixed with 00- will override
|
||||
%% anything in a file with a higher numbered prefix.
|
||||
|
||||
%% Please only use 0[0-9]-*.schema for development purposes
|
||||
%% NOTHING PERMANENT
|
||||
|
||||
%% {template, "files/riak.schema", "lib/10-riak.schema"},
|
||||
%% {template, "../deps/cuttlefish/priv/erlang_vm.schema", "lib/11-erlang_vm.schema"},
|
||||
|
||||
%% {template, "../deps/riak_core/priv/riak_core.schema", "lib/12-riak_core.schema"},
|
||||
%% {template, "../deps/riak_api/priv/riak_api.schema", "lib/13-riak_api.schema"},
|
||||
%% {template, "../deps/riak_kv/priv/riak_kv.schema", "lib/14-riak_kv.schema"},
|
||||
%% {template, "../deps/riak_sysmon/priv/riak_sysmon.schema", "lib/15-riak_sysmon.schema"},
|
||||
%% {template, "../deps/bitcask/priv/bitcask.schema", "lib/16-bitcask.schema"},
|
||||
%% {template, "../deps/bitcask/priv/bitcask_multi.schema", "lib/17-bitcask_multi.schema"},
|
||||
%% {template, "../deps/riak_control/priv/riak_control.schema", "lib/18-riak_control.schema"},
|
||||
|
||||
%% {template, "../deps/riak_kv/priv/multi_backend.schema", "lib/20-multi_backend.schema"},
|
||||
%% {template, "../deps/eleveldb/priv/eleveldb.schema", "lib/21-leveldb.schema"},
|
||||
%% {template, "../deps/eleveldb/priv/eleveldb_multi.schema", "lib/22-leveldb_multi.schema"},
|
||||
%% {template, "../deps/yokozuna/priv/yokozuna.schema", "lib/30-yokozuna.schema"},
|
||||
|
||||
%% Copy additional bin scripts
|
||||
{template, "files/machi-admin", "bin/machi-admin"},
|
||||
|
||||
{template, "files/vm.args", "etc/vm.args"},
|
||||
{template, "files/app.config", "etc/app.config"},
|
||||
{mkdir, "etc/chain-config"},
|
||||
{mkdir, "etc/flu-config"},
|
||||
{mkdir, "etc/pending"},
|
||||
{mkdir, "etc/rejected"},
|
||||
|
||||
%% Experiment: quick-admin
|
||||
{mkdir, "etc/quick-admin-archive"},
|
||||
{mkdir, "priv"},
|
||||
{mkdir, "priv/quick-admin-examples"},
|
||||
{copy, "../priv/quick-admin-examples/000", "priv/quick-admin-examples"},
|
||||
{copy, "../priv/quick-admin-examples/001", "priv/quick-admin-examples"},
|
||||
{copy, "../priv/quick-admin-examples/002", "priv/quick-admin-examples"},
|
||||
{copy, "../priv/quick-admin-examples/demo-000", "priv/quick-admin-examples/demo-000"},
|
||||
|
||||
{mkdir, "lib/basho-patches"}
|
||||
%% {copy, "../apps/machi/ebin/etop_txt.beam", "lib/basho-patches"}
|
||||
]}.
|
48
rel/vars.config
Normal file
48
rel/vars.config
Normal file
|
@ -0,0 +1,48 @@
|
|||
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
|
||||
%% ex: ft=erlang ts=4 sw=4 et
|
||||
|
||||
%% NOTE: When modifying this file, also keep its near cousin
|
||||
%% config file rel/vars/dev_vars.config.src in sync!
|
||||
|
||||
%% Platform-specific installation paths
|
||||
{platform_bin_dir, "./bin"}.
|
||||
{platform_data_dir, "./data"}.
|
||||
{platform_etc_dir, "./etc"}.
|
||||
{platform_lib_dir, "./lib"}.
|
||||
{platform_log_dir, "./log"}.
|
||||
|
||||
%%
|
||||
%% etc/app.config
|
||||
%%
|
||||
{sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}.
|
||||
{sasl_log_dir, "{{platform_log_dir}}/sasl"}.
|
||||
|
||||
%% lager
|
||||
{console_log_default, file}.
|
||||
|
||||
%%
|
||||
%% etc/vm.args
|
||||
%%
|
||||
{node, "machi@127.0.0.1"}.
|
||||
{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}.
|
||||
|
||||
%%
|
||||
%% bin/machi
|
||||
%%
|
||||
{runner_script_dir, "\`cd \\`dirname $0\\` 1>/dev/null && /bin/pwd\`"}.
|
||||
{runner_base_dir, "{{runner_script_dir}}/.."}.
|
||||
{runner_etc_dir, "$RUNNER_BASE_DIR/etc"}.
|
||||
{runner_log_dir, "$RUNNER_BASE_DIR/log"}.
|
||||
{runner_lib_dir, "$RUNNER_BASE_DIR/lib"}.
|
||||
{runner_patch_dir, "$RUNNER_BASE_DIR/lib/basho-patches"}.
|
||||
{pipe_dir, "/tmp/$RUNNER_BASE_DIR/"}.
|
||||
{runner_user, ""}.
|
||||
{runner_wait_process, "machi_flu_sup"}.
|
||||
{runner_ulimit_warn, 65536}.
|
||||
|
||||
%%
|
||||
%% cuttlefish
|
||||
%%
|
||||
{cuttlefish, ""}. % blank = off
|
||||
{cuttlefish_conf, "machi.conf"}.
|
||||
|
48
rel/vars/dev_vars.config.src
Normal file
48
rel/vars/dev_vars.config.src
Normal file
|
@ -0,0 +1,48 @@
|
|||
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
|
||||
%% ex: ft=erlang ts=4 sw=4 et
|
||||
|
||||
%% NOTE: When modifying this file, also keep its near cousin
|
||||
%% config file rel/vars/dev_vars.config.src in sync!
|
||||
|
||||
%% Platform-specific installation paths
|
||||
{platform_bin_dir, "./bin"}.
|
||||
{platform_data_dir, "./data"}.
|
||||
{platform_etc_dir, "./etc"}.
|
||||
{platform_lib_dir, "./lib"}.
|
||||
{platform_log_dir, "./log"}.
|
||||
|
||||
%%
|
||||
%% etc/app.config
|
||||
%%
|
||||
{sasl_error_log, "{{platform_log_dir}}/sasl-error.log"}.
|
||||
{sasl_log_dir, "{{platform_log_dir}}/sasl"}.
|
||||
|
||||
%% lager
|
||||
{console_log_default, file}.
|
||||
|
||||
%%
|
||||
%% etc/vm.args
|
||||
%%
|
||||
{node, "@NODE@"}.
|
||||
{crash_dump, "{{platform_log_dir}}/erl_crash.dump"}.
|
||||
|
||||
%%
|
||||
%% bin/machi
|
||||
%%
|
||||
{runner_script_dir, "\`cd \\`dirname $0\\` 1>/dev/null && /bin/pwd\`"}.
|
||||
{runner_base_dir, "{{runner_script_dir}}/.."}.
|
||||
{runner_etc_dir, "$RUNNER_BASE_DIR/etc"}.
|
||||
{runner_log_dir, "$RUNNER_BASE_DIR/log"}.
|
||||
{runner_lib_dir, "$RUNNER_BASE_DIR/lib"}.
|
||||
{runner_patch_dir, "$RUNNER_BASE_DIR/lib/basho-patches"}.
|
||||
{pipe_dir, "/tmp/$RUNNER_BASE_DIR/"}.
|
||||
{runner_user, ""}.
|
||||
{runner_wait_process, "machi_flu_sup"}.
|
||||
{runner_ulimit_warn, 65536}.
|
||||
|
||||
%%
|
||||
%% cuttlefish
|
||||
%%
|
||||
{cuttlefish, ""}. % blank = off
|
||||
{cuttlefish_conf, "machi.conf"}.
|
||||
|
|
@ -1,13 +1,10 @@
|
|||
{application, machi, [
|
||||
{description, "A village of write-once files."},
|
||||
{vsn, "0.0.0"},
|
||||
{applications, [kernel, stdlib, crypto]},
|
||||
{vsn, "0.0.1"},
|
||||
{applications, [kernel, stdlib, crypto, cluster_info, ranch]},
|
||||
{mod,{machi_app,[]}},
|
||||
{registered, []},
|
||||
{env, [
|
||||
{flu_list,
|
||||
[
|
||||
%%%%%% {flu_a, 32900, "./data.flu_a"}
|
||||
]}
|
||||
%% Don't use this static env for defaults, or we will fall into config hell.
|
||||
]}
|
||||
]}.
|
||||
|
|
218
src/machi.proto
218
src/machi.proto
|
@ -40,12 +40,18 @@ enum Mpb_GeneralStatusCode {
|
|||
BAD_ARG = 1;
|
||||
WEDGED = 2;
|
||||
BAD_CHECKSUM = 3;
|
||||
/*
|
||||
** There is no timout error code, only PARTITION. If the client
|
||||
** wants to know if a lot of time has elapsed, then the client
|
||||
** can do its own timekeeping.
|
||||
*/
|
||||
PARTITION = 4;
|
||||
NOT_WRITTEN = 5;
|
||||
WRITTEN = 6;
|
||||
NO_SUCH_FILE = 7;
|
||||
PARTIAL_READ = 8;
|
||||
BAD_EPOCH = 9;
|
||||
TRIMMED = 7; // The whole file was trimmed
|
||||
NO_SUCH_FILE = 8;
|
||||
PARTIAL_READ = 9;
|
||||
BAD_EPOCH = 10;
|
||||
BAD_JOSS = 255; // Only for testing by the Taipan
|
||||
}
|
||||
|
||||
|
@ -81,6 +87,14 @@ message Mpb_ChunkCSum {
|
|||
optional bytes csum = 2;
|
||||
}
|
||||
|
||||
message Mpb_Chunk {
|
||||
required uint64 offset = 1;
|
||||
required string file_name = 2;
|
||||
required bytes chunk = 3;
|
||||
// TODO: must be required, in future?
|
||||
optional Mpb_ChunkCSum csum = 4;
|
||||
}
|
||||
|
||||
// epoch_id() type
|
||||
message Mpb_EpochID {
|
||||
required uint32 epoch_number = 1;
|
||||
|
@ -125,6 +139,7 @@ message Mpb_ErrorResp {
|
|||
// append_chunk() : Mpb_AppendChunkReq and Mpb_AppendChunkResp
|
||||
// write_chunk() : Mpb_WriteChunkReq and Mpb_WriteChunkResp
|
||||
// read_chunk() : Mpb_ReadChunkReq and Mpb_ReadChunkResp
|
||||
// trim_chunk() : Mpb_TrimChunkReq and Mpb_TrimChunkResp
|
||||
// checksum_list() : Mpb_ChecksumListReq and Mpb_ChecksumListResp
|
||||
// list_files() : Mpb_ListFilesReq and Mpb_ListFilesResp
|
||||
//
|
||||
|
@ -155,11 +170,18 @@ message Mpb_AuthResp {
|
|||
// High level API: append_chunk() request & response
|
||||
|
||||
message Mpb_AppendChunkReq {
|
||||
optional bytes placement_key = 1;
|
||||
required string prefix = 2;
|
||||
required bytes chunk = 3;
|
||||
required Mpb_ChunkCSum csum = 4;
|
||||
optional uint32 chunk_extra = 5;
|
||||
// General namespace arguments
|
||||
/* In single chain/non-clustered environment, use namespace="" */
|
||||
required string namespace = 1;
|
||||
|
||||
required string prefix = 10;
|
||||
required bytes chunk = 11;
|
||||
required Mpb_ChunkCSum csum = 12;
|
||||
|
||||
optional uint32 chunk_extra = 20;
|
||||
optional string preferred_file_name = 21;
|
||||
/* Fail the operation if our preferred file name is not available */
|
||||
optional bool flag_fail_preferred = 22 [default=false];
|
||||
}
|
||||
|
||||
message Mpb_AppendChunkResp {
|
||||
|
@ -171,10 +193,7 @@ message Mpb_AppendChunkResp {
|
|||
// High level API: write_chunk() request & response
|
||||
|
||||
message Mpb_WriteChunkReq {
|
||||
required string file = 1;
|
||||
required uint64 offset = 2;
|
||||
required bytes chunk = 3;
|
||||
required Mpb_ChunkCSum csum = 4;
|
||||
required Mpb_Chunk chunk = 10;
|
||||
}
|
||||
|
||||
message Mpb_WriteChunkResp {
|
||||
|
@ -184,22 +203,38 @@ message Mpb_WriteChunkResp {
|
|||
// High level API: read_chunk() request & response
|
||||
|
||||
message Mpb_ReadChunkReq {
|
||||
required string file = 1;
|
||||
required uint64 offset = 2;
|
||||
required uint32 size = 3;
|
||||
// No namespace arguments are required because NS is embedded
|
||||
// inside of the file name.
|
||||
|
||||
// Use flag_checksum=non-zero to request the chunk's checksum also
|
||||
optional uint32 flag_checksum = 4 [default=0];
|
||||
required Mpb_ChunkPos chunk_pos = 10;
|
||||
|
||||
// Use flag_no_checksum=non-zero to skip returning the chunk's checksum.
|
||||
// TODO: not implemented yet.
|
||||
optional bool flag_no_checksum = 20 [default=false];
|
||||
|
||||
// Use flag_no_chunk=non-zero to skip returning the chunk (which
|
||||
// only makes sense if flag_checksum is set).
|
||||
optional uint32 flag_no_chunk = 5 [default=0];
|
||||
// only makes sense if flag_no_checksum is not set).
|
||||
// TODO: not implemented yet.
|
||||
optional bool flag_no_chunk = 21 [default=false];
|
||||
|
||||
// TODO: not implemented yet.
|
||||
optional bool flag_needs_trimmed = 22 [default=false];
|
||||
}
|
||||
|
||||
message Mpb_ReadChunkResp {
|
||||
required Mpb_GeneralStatusCode status = 1;
|
||||
optional bytes chunk = 2;
|
||||
optional Mpb_ChunkCSum csum = 3;
|
||||
repeated Mpb_Chunk chunks = 2;
|
||||
repeated Mpb_ChunkPos trimmed = 3;
|
||||
}
|
||||
|
||||
// High level API: trim_chunk() request & response
|
||||
|
||||
message Mpb_TrimChunkReq {
|
||||
required Mpb_ChunkPos chunk_pos = 1;
|
||||
}
|
||||
|
||||
message Mpb_TrimChunkResp {
|
||||
required Mpb_GeneralStatusCode status = 1;
|
||||
}
|
||||
|
||||
// High level API: checksum_list() request & response
|
||||
|
@ -219,6 +254,8 @@ message Mpb_ChecksumListResp {
|
|||
// High level API: list_files() request & response
|
||||
|
||||
message Mpb_ListFilesReq {
|
||||
// TODO: Add flag for file glob/regexp/other filter type
|
||||
// TODO: What else could go wrong?
|
||||
}
|
||||
|
||||
message Mpb_ListFilesResp {
|
||||
|
@ -251,8 +288,9 @@ message Mpb_Request {
|
|||
optional Mpb_AppendChunkReq append_chunk = 112;
|
||||
optional Mpb_WriteChunkReq write_chunk = 113;
|
||||
optional Mpb_ReadChunkReq read_chunk = 114;
|
||||
optional Mpb_ChecksumListReq checksum_list = 115;
|
||||
optional Mpb_ListFilesReq list_files = 116;
|
||||
optional Mpb_TrimChunkReq trim_chunk = 115;
|
||||
optional Mpb_ChecksumListReq checksum_list = 116;
|
||||
optional Mpb_ListFilesReq list_files = 117;
|
||||
}
|
||||
|
||||
message Mpb_Response {
|
||||
|
@ -274,8 +312,9 @@ message Mpb_Response {
|
|||
optional Mpb_AppendChunkResp append_chunk = 12;
|
||||
optional Mpb_WriteChunkResp write_chunk = 13;
|
||||
optional Mpb_ReadChunkResp read_chunk = 14;
|
||||
optional Mpb_ChecksumListResp checksum_list = 15;
|
||||
optional Mpb_ListFilesResp list_files = 16;
|
||||
optional Mpb_TrimChunkResp trim_chunk = 15;
|
||||
optional Mpb_ChecksumListResp checksum_list = 16;
|
||||
optional Mpb_ListFilesResp list_files = 17;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////
|
||||
|
@ -303,18 +342,17 @@ message Mpb_ProjectionV1 {
|
|||
required uint32 epoch_number = 1;
|
||||
required bytes epoch_csum = 2;
|
||||
required string author_server = 3;
|
||||
repeated string all_members = 4;
|
||||
repeated string witnesses = 5;
|
||||
required Mpb_Now creation_time = 6;
|
||||
required Mpb_Mode mode = 7;
|
||||
repeated string upi = 8;
|
||||
repeated string repairing = 9;
|
||||
repeated string down = 10;
|
||||
optional bytes opaque_flap = 11;
|
||||
optional bytes opaque_inner = 12;
|
||||
required bytes opaque_dbg = 13;
|
||||
required bytes opaque_dbg2 = 14;
|
||||
repeated Mpb_MembersDictEntry members_dict = 15;
|
||||
required string chain_name = 4;
|
||||
repeated string all_members = 5;
|
||||
repeated string witnesses = 6;
|
||||
required Mpb_Now creation_time = 7;
|
||||
required Mpb_Mode mode = 8;
|
||||
repeated string upi = 9;
|
||||
repeated string repairing = 10;
|
||||
repeated string down = 11;
|
||||
required bytes opaque_dbg = 12;
|
||||
required bytes opaque_dbg2 = 13;
|
||||
repeated Mpb_MembersDictEntry members_dict = 14;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////
|
||||
|
@ -329,6 +367,7 @@ message Mpb_ProjectionV1 {
|
|||
// append_chunk()
|
||||
// write_chunk()
|
||||
// read_chunk()
|
||||
// trim_chunk()
|
||||
// checksum_list()
|
||||
// list_files()
|
||||
// wedge_status()
|
||||
|
@ -349,12 +388,20 @@ message Mpb_ProjectionV1 {
|
|||
// Low level API: append_chunk()
|
||||
|
||||
message Mpb_LL_AppendChunkReq {
|
||||
required Mpb_EpochID epoch_id = 1;
|
||||
optional bytes placement_key = 2;
|
||||
required string prefix = 3;
|
||||
required bytes chunk = 4;
|
||||
required Mpb_ChunkCSum csum = 5;
|
||||
optional uint32 chunk_extra = 6;
|
||||
// General namespace arguments
|
||||
required uint32 namespace_version = 1;
|
||||
required string namespace = 2;
|
||||
required uint32 locator = 3;
|
||||
|
||||
required Mpb_EpochID epoch_id = 10;
|
||||
required string prefix = 11;
|
||||
required bytes chunk = 12;
|
||||
required Mpb_ChunkCSum csum = 13;
|
||||
|
||||
optional uint32 chunk_extra = 20;
|
||||
optional string preferred_file_name = 21;
|
||||
/* Fail the operation if our preferred file name is not available */
|
||||
optional bool flag_fail_preferred = 22 [default=false];
|
||||
}
|
||||
|
||||
message Mpb_LL_AppendChunkResp {
|
||||
|
@ -366,11 +413,12 @@ message Mpb_LL_AppendChunkResp {
|
|||
// Low level API: write_chunk()
|
||||
|
||||
message Mpb_LL_WriteChunkReq {
|
||||
required Mpb_EpochID epoch_id = 1;
|
||||
required string file = 2;
|
||||
required uint64 offset = 3;
|
||||
required bytes chunk = 4;
|
||||
required Mpb_ChunkCSum csum = 5;
|
||||
// General namespace arguments
|
||||
required uint32 namespace_version = 1;
|
||||
required string namespace = 2;
|
||||
|
||||
required Mpb_EpochID epoch_id = 10;
|
||||
required Mpb_Chunk chunk = 11;
|
||||
}
|
||||
|
||||
message Mpb_LL_WriteChunkResp {
|
||||
|
@ -380,30 +428,54 @@ message Mpb_LL_WriteChunkResp {
|
|||
// Low level API: read_chunk()
|
||||
|
||||
message Mpb_LL_ReadChunkReq {
|
||||
required Mpb_EpochID epoch_id = 1;
|
||||
required string file = 2;
|
||||
required uint64 offset = 3;
|
||||
required uint32 size = 4;
|
||||
// General namespace arguments
|
||||
required uint32 namespace_version = 1;
|
||||
required string namespace = 2;
|
||||
|
||||
// Use flag_checksum=non-zero to request the chunk's checksum also
|
||||
optional uint32 flag_get_checksum = 5 [default=0];
|
||||
required Mpb_EpochID epoch_id = 10;
|
||||
required Mpb_ChunkPos chunk_pos = 11;
|
||||
|
||||
// Use flag_no_checksum=non-zero to skip returning the chunk's checksum.
|
||||
// TODO: not implemented yet.
|
||||
optional bool flag_no_checksum = 20 [default=false];
|
||||
|
||||
// Use flag_no_chunk=non-zero to skip returning the chunk (which
|
||||
// only makes sense if flag_checksum is set).
|
||||
optional uint32 flag_no_chunk = 6 [default=0];
|
||||
// only makes sense if flag_checksum is not set).
|
||||
// TODO: not implemented yet.
|
||||
optional bool flag_no_chunk = 21 [default=false];
|
||||
|
||||
optional bool flag_needs_trimmed = 22 [default=false];
|
||||
}
|
||||
|
||||
message Mpb_LL_ReadChunkResp {
|
||||
required Mpb_GeneralStatusCode status = 1;
|
||||
optional bytes chunk = 2;
|
||||
optional Mpb_ChunkCSum csum = 3;
|
||||
repeated Mpb_Chunk chunks = 2;
|
||||
repeated Mpb_ChunkPos trimmed = 3;
|
||||
}
|
||||
|
||||
// Low level API: trim_chunk()
|
||||
|
||||
message Mpb_LL_TrimChunkReq {
|
||||
// General namespace arguments
|
||||
required uint32 namespace_version = 1;
|
||||
required string namespace = 2;
|
||||
|
||||
required Mpb_EpochID epoch_id = 10;
|
||||
required string file = 11;
|
||||
required uint64 offset = 12;
|
||||
required uint32 size = 13;
|
||||
|
||||
optional bool trigger_gc = 20 [default=false];
|
||||
}
|
||||
|
||||
message Mpb_LL_TrimChunkResp {
|
||||
required Mpb_GeneralStatusCode status = 1;
|
||||
}
|
||||
|
||||
// Low level API: checksum_list()
|
||||
|
||||
message Mpb_LL_ChecksumListReq {
|
||||
required Mpb_EpochID epoch_id = 1;
|
||||
required string file = 2;
|
||||
required string file = 1;
|
||||
}
|
||||
|
||||
message Mpb_LL_ChecksumListResp {
|
||||
|
@ -434,7 +506,9 @@ message Mpb_LL_WedgeStatusReq {
|
|||
message Mpb_LL_WedgeStatusResp {
|
||||
required Mpb_GeneralStatusCode status = 1;
|
||||
optional Mpb_EpochID epoch_id = 2;
|
||||
optional uint32 wedged_flag = 3;
|
||||
optional bool wedged_flag = 3;
|
||||
optional uint32 namespace_version = 4;
|
||||
optional string namespace = 5;
|
||||
}
|
||||
|
||||
// Low level API: delete_migration()
|
||||
|
@ -563,11 +637,12 @@ message Mpb_LL_Request {
|
|||
optional Mpb_LL_AppendChunkReq append_chunk = 30;
|
||||
optional Mpb_LL_WriteChunkReq write_chunk = 31;
|
||||
optional Mpb_LL_ReadChunkReq read_chunk = 32;
|
||||
optional Mpb_LL_ChecksumListReq checksum_list = 33;
|
||||
optional Mpb_LL_ListFilesReq list_files = 34;
|
||||
optional Mpb_LL_WedgeStatusReq wedge_status = 35;
|
||||
optional Mpb_LL_DeleteMigrationReq delete_migration = 36;
|
||||
optional Mpb_LL_TruncHackReq trunc_hack = 37;
|
||||
optional Mpb_LL_TrimChunkReq trim_chunk = 33;
|
||||
optional Mpb_LL_ChecksumListReq checksum_list = 34;
|
||||
optional Mpb_LL_ListFilesReq list_files = 35;
|
||||
optional Mpb_LL_WedgeStatusReq wedge_status = 36;
|
||||
optional Mpb_LL_DeleteMigrationReq delete_migration = 37;
|
||||
optional Mpb_LL_TruncHackReq trunc_hack = 38;
|
||||
}
|
||||
|
||||
message Mpb_LL_Response {
|
||||
|
@ -597,9 +672,10 @@ message Mpb_LL_Response {
|
|||
optional Mpb_LL_AppendChunkResp append_chunk = 30;
|
||||
optional Mpb_LL_WriteChunkResp write_chunk = 31;
|
||||
optional Mpb_LL_ReadChunkResp read_chunk = 32;
|
||||
optional Mpb_LL_ChecksumListResp checksum_list = 33;
|
||||
optional Mpb_LL_ListFilesResp list_files = 34;
|
||||
optional Mpb_LL_WedgeStatusResp wedge_status = 35;
|
||||
optional Mpb_LL_DeleteMigrationResp delete_migration = 36;
|
||||
optional Mpb_LL_TruncHackResp trunc_hack = 37;
|
||||
optional Mpb_LL_TrimChunkResp trim_chunk = 33;
|
||||
optional Mpb_LL_ChecksumListResp checksum_list = 34;
|
||||
optional Mpb_LL_ListFilesResp list_files = 35;
|
||||
optional Mpb_LL_WedgeStatusResp wedge_status = 36;
|
||||
optional Mpb_LL_DeleteMigrationResp delete_migration = 37;
|
||||
optional Mpb_LL_TruncHackResp trunc_hack = 38;
|
||||
}
|
||||
|
|
|
@ -73,8 +73,13 @@ verify_file_checksums_local2(Sock1, EpochID, Path0) ->
|
|||
{ok, FH} ->
|
||||
File = re:replace(Path, ".*/", "", [{return, binary}]),
|
||||
try
|
||||
ReadChunk = fun(_File, Offset, Size) ->
|
||||
file:pread(FH, Offset, Size)
|
||||
ReadChunk = fun(F, Offset, Size) ->
|
||||
case file:pread(FH, Offset, Size) of
|
||||
{ok, Bin} ->
|
||||
{ok, {[{F, Offset, Bin, undefined}], []}};
|
||||
Err ->
|
||||
Err
|
||||
end
|
||||
end,
|
||||
verify_file_checksums_common(Sock1, EpochID, File, ReadChunk)
|
||||
after
|
||||
|
@ -85,17 +90,18 @@ verify_file_checksums_local2(Sock1, EpochID, Path0) ->
|
|||
end.
|
||||
|
||||
verify_file_checksums_remote2(Sock1, EpochID, File) ->
|
||||
NSInfo = undefined,
|
||||
ReadChunk = fun(File_name, Offset, Size) ->
|
||||
?FLU_C:read_chunk(Sock1, EpochID,
|
||||
File_name, Offset, Size)
|
||||
?FLU_C:read_chunk(Sock1, NSInfo, EpochID,
|
||||
File_name, Offset, Size, undefined)
|
||||
end,
|
||||
verify_file_checksums_common(Sock1, EpochID, File, ReadChunk).
|
||||
|
||||
verify_file_checksums_common(Sock1, EpochID, File, ReadChunk) ->
|
||||
verify_file_checksums_common(Sock1, _EpochID, File, ReadChunk) ->
|
||||
try
|
||||
case ?FLU_C:checksum_list(Sock1, EpochID, File) of
|
||||
case ?FLU_C:checksum_list(Sock1, File) of
|
||||
{ok, InfoBin} ->
|
||||
{Info, _} = machi_flu1:split_checksum_list_blob_decode(InfoBin),
|
||||
Info = machi_csum_table:split_checksum_list_blob_decode(InfoBin),
|
||||
Res = lists:foldl(verify_chunk_checksum(File, ReadChunk),
|
||||
[], Info),
|
||||
{ok, Res};
|
||||
|
@ -110,9 +116,11 @@ verify_file_checksums_common(Sock1, EpochID, File, ReadChunk) ->
|
|||
end.
|
||||
|
||||
verify_chunk_checksum(File, ReadChunk) ->
|
||||
fun({Offset, Size, <<_Tag:1/binary, CSum/binary>>}, Acc) ->
|
||||
fun({0, ?MINIMUM_OFFSET, none}, []) ->
|
||||
[];
|
||||
({Offset, Size, <<_Tag:1/binary, CSum/binary>>}, Acc) ->
|
||||
case ReadChunk(File, Offset, Size) of
|
||||
{ok, Chunk} ->
|
||||
{ok, {[{_, Offset, Chunk, _}], _}} ->
|
||||
CSum2 = machi_util:checksum_chunk(Chunk),
|
||||
if CSum == CSum2 ->
|
||||
Acc;
|
||||
|
|
|
@ -36,12 +36,8 @@
|
|||
-export([start/2, stop/1]).
|
||||
|
||||
start(_StartType, _StartArgs) ->
|
||||
case machi_sup:start_link() of
|
||||
{ok, Pid} ->
|
||||
{ok, Pid};
|
||||
Error ->
|
||||
Error
|
||||
end.
|
||||
machi_cinfo:register(),
|
||||
machi_sup:start_link().
|
||||
|
||||
stop(_State) ->
|
||||
ok.
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
|
@ -43,23 +43,25 @@
|
|||
%% could add new entries to this ETS table.
|
||||
%%
|
||||
%% Now we can use various integer-centric key generators that are
|
||||
%% already bundled with basho_bench.
|
||||
%% already bundled with basho_bench. NOTE: this scheme does not allow
|
||||
%% mixing of 'append' and 'read' operations in the same config. Basho
|
||||
%% Bench does not support different key generators for different
|
||||
%% operations, unfortunately. The work-around is to run two different
|
||||
%% Basho Bench instances: on for 'append' ops with a key generator for
|
||||
%% the desired prefix(es), and the other for 'read' ops with an
|
||||
%% integer key generator.
|
||||
%%
|
||||
%% TODO: Add CRC checking, when feasible and when supported on the
|
||||
%% server side.
|
||||
%%
|
||||
%% TODO: As an alternate idea, if we know that the chunks written are
|
||||
%% always the same size, and if we don't care about CRC checking, then
|
||||
%% all we need to know are the file names & file sizes on the server:
|
||||
%% we can then pick any valid offset within that file. That would
|
||||
%% certainly be more scalable than the zillion-row-ETS-table, which is
|
||||
%% definitely RAM-hungry.
|
||||
%% TODO: The 'read' operator will always read chunks at exactly the
|
||||
%% byte offset & size as the original append/write ops. If reads are
|
||||
%% desired at any arbitrary offset & size, then a new strategy is
|
||||
%% required.
|
||||
|
||||
-module(machi_basho_bench_driver).
|
||||
|
||||
-export([new/1, run/4]).
|
||||
|
||||
-record(m, {
|
||||
id,
|
||||
conn,
|
||||
max_key
|
||||
}).
|
||||
|
@ -81,7 +83,7 @@ new(Id) ->
|
|||
{read_concurrency, true}]),
|
||||
ets:insert(ETS, {max_key, 0}),
|
||||
ets:insert(ETS, {total_bytes, 0}),
|
||||
MaxKeys = load_ets_table(Conn, ETS),
|
||||
MaxKeys = load_ets_table_maybe(Conn, ETS),
|
||||
?INFO("Key preload: finished, ~w keys loaded", [MaxKeys]),
|
||||
Bytes = ets:lookup_element(ETS, total_bytes, 2),
|
||||
?INFO("Key preload: finished, chunk list specifies ~s MBytes of chunks",
|
||||
|
@ -90,12 +92,14 @@ new(Id) ->
|
|||
true ->
|
||||
ok
|
||||
end,
|
||||
{ok, #m{conn=Conn}}.
|
||||
{ok, #m{id=Id, conn=Conn}}.
|
||||
|
||||
run(append, KeyGen, ValueGen, #m{conn=Conn}=S) ->
|
||||
Prefix = KeyGen(),
|
||||
Value = ValueGen(),
|
||||
case machi_cr_client:append_chunk(Conn, Prefix, Value, ?THE_TIMEOUT) of
|
||||
CSum = machi_util:make_client_csum(Value),
|
||||
AppendOpts = {append_opts,0,undefined,false}, % HACK FIXME
|
||||
case machi_cr_client:append_chunk(Conn, undefined, Prefix, Value, CSum, AppendOpts, ?THE_TIMEOUT) of
|
||||
{ok, Pos} ->
|
||||
EtsKey = ets:update_counter(?ETS_TAB, max_key, 1),
|
||||
true = ets:insert(?ETS_TAB, {EtsKey, Pos}),
|
||||
|
@ -112,9 +116,26 @@ run(read, KeyGen, _ValueGen, #m{conn=Conn, max_key=MaxKey}=S) ->
|
|||
Idx = KeyGen() rem MaxKey,
|
||||
%% {File, Offset, Size, _CSum} = ets:lookup_element(?ETS_TAB, Idx, 2),
|
||||
{File, Offset, Size} = ets:lookup_element(?ETS_TAB, Idx, 2),
|
||||
case machi_cr_client:read_chunk(Conn, File, Offset, Size, ?THE_TIMEOUT) of
|
||||
{ok, _Chunk} ->
|
||||
{ok, S};
|
||||
ReadOpts = {read_opts,false,false,false}, % HACK FIXME
|
||||
case machi_cr_client:read_chunk(Conn, undefined, File, Offset, Size, ReadOpts, ?THE_TIMEOUT) of
|
||||
{ok, {Chunks, _Trimmed}} ->
|
||||
%% io:format(user, "Chunks ~P\n", [Chunks, 15]),
|
||||
%% {ok, S};
|
||||
case lists:all(fun({File2, Offset2, Chunk, CSum}) ->
|
||||
{_Tag, CS} = machi_util:unmake_tagged_csum(CSum),
|
||||
CS2 = machi_util:checksum_chunk(Chunk),
|
||||
if CS == CS2 ->
|
||||
true;
|
||||
CS /= CS2 ->
|
||||
?ERROR("Client-side checksum error for file ~p offset ~p expected ~p got ~p\n", [File2, Offset2, CS, CS2]),
|
||||
false
|
||||
end
|
||||
end, Chunks) of
|
||||
true ->
|
||||
{ok, S};
|
||||
false ->
|
||||
{error, bad_checksum, S}
|
||||
end;
|
||||
{error, _}=Err ->
|
||||
?ERROR("read file ~p offset ~w size ~w: ~w\n",
|
||||
[File, Offset, Size, Err]),
|
||||
|
@ -132,21 +153,40 @@ find_server_info(_Id) ->
|
|||
Ps
|
||||
end.
|
||||
|
||||
load_ets_table_maybe(Conn, ETS) ->
|
||||
case basho_bench_config:get(operations, undefined) of
|
||||
undefined ->
|
||||
?ERROR("The 'operations' key is missing from the config file, aborting", []),
|
||||
exit(bad_config);
|
||||
Ops when is_list(Ops) ->
|
||||
case lists:keyfind(read, 1, Ops) of
|
||||
{read,_} ->
|
||||
load_ets_table(Conn, ETS);
|
||||
false ->
|
||||
?INFO("No 'read' op in the 'operations' list ~p, skipping ETS table load.", [Ops]),
|
||||
0
|
||||
end
|
||||
end.
|
||||
|
||||
load_ets_table(Conn, ETS) ->
|
||||
{ok, Fs} = machi_cr_client:list_files(Conn),
|
||||
[begin
|
||||
{ok, InfoBin} = machi_cr_client:checksum_list(Conn, File),
|
||||
{PosList, _} = machi_flu1:split_checksum_list_blob_decode(InfoBin),
|
||||
{ok, InfoBin} = machi_cr_client:checksum_list(Conn, File, ?THE_TIMEOUT),
|
||||
PosList = machi_csum_table:split_checksum_list_blob_decode(InfoBin),
|
||||
?INFO("File ~s len PosList ~p\n", [File, length(PosList)]),
|
||||
StartKey = ets:update_counter(ETS, max_key, 0),
|
||||
%% _EndKey = lists:foldl(fun({Off,Sz,CSum}, K) ->
|
||||
%% V = {File, Off, Sz, CSum},
|
||||
{_, Bytes} = lists:foldl(fun({Off,Sz,_CSum}, {K, Bs}) ->
|
||||
V = {File, Off, Sz},
|
||||
ets:insert(ETS, {K, V}),
|
||||
{K + 1, Bs + Sz}
|
||||
end, {StartKey, 0}, PosList),
|
||||
ets:update_counter(ETS, max_key, length(PosList)),
|
||||
ets:update_counter(ETS, total_bytes, Bytes)
|
||||
{_, C, Bytes} = lists:foldl(fun({_Off,0,_CSum}, {_K, _C, _Bs}=Acc) ->
|
||||
Acc;
|
||||
({0,_Sz,_CSum}, {_K, _C, _Bs}=Acc) ->
|
||||
Acc;
|
||||
({Off,Sz,_CSum}, {K, C, Bs}) ->
|
||||
V = {File, Off, Sz},
|
||||
ets:insert(ETS, {K, V}),
|
||||
{K + 1, C + 1, Bs + Sz}
|
||||
end, {StartKey, 0, 0}, PosList),
|
||||
_ = ets:update_counter(ETS, max_key, C),
|
||||
_ = ets:update_counter(ETS, total_bytes, Bytes),
|
||||
ok
|
||||
end || {_Size, File} <- Fs],
|
||||
ets:update_counter(?ETS_TAB, max_key, 0).
|
||||
|
||||
|
|
|
@ -92,8 +92,11 @@
|
|||
-define(REPAIR_START_STABILITY_TIME, 10).
|
||||
-endif. % TEST
|
||||
|
||||
%% Magic constant for looping "too frequently" breaker. TODO revisit & revise.
|
||||
-define(TOO_FREQUENT_BREAKER, 10).
|
||||
%% Maximum length of the history of adopted projections (via C120).
|
||||
-define(MAX_HISTORY_LENGTH, 8).
|
||||
|
||||
%% Magic constant for looping "too frequently" breaker.
|
||||
-define(TOO_FREQUENT_BREAKER, (?MAX_HISTORY_LENGTH+5)).
|
||||
|
||||
-define(RETURN2(X), begin (catch put(why2, [?LINE|get(why2)])), X end).
|
||||
|
||||
|
@ -103,20 +106,19 @@
|
|||
%% Amount of epoch number skip-ahead for set_chain_members call
|
||||
-define(SET_CHAIN_MEMBERS_EPOCH_SKIP, 1111).
|
||||
|
||||
%% Maximum length of the history of adopted projections (via C120).
|
||||
-define(MAX_HISTORY_LENGTH, 30).
|
||||
|
||||
%% API
|
||||
-export([start_link/2, start_link/3, stop/1, ping/1,
|
||||
set_chain_members/2, set_chain_members/3, set_active/2,
|
||||
set_chain_members/2, set_chain_members/6, set_active/2,
|
||||
trigger_react_to_env/1]).
|
||||
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
|
||||
terminate/2, code_change/3]).
|
||||
terminate/2, format_status/2, code_change/3]).
|
||||
|
||||
-export([make_chmgr_regname/1, projection_transitions_are_sane/2,
|
||||
simple_chain_state_transition_is_sane/3,
|
||||
simple_chain_state_transition_is_sane/5,
|
||||
chain_state_transition_is_sane/6]).
|
||||
-export([perhaps_call/5, % for partition simulator use w/machi_fitness
|
||||
init_remember_down_list/0]).
|
||||
%% Exports so that EDoc docs generated for these internal funcs.
|
||||
-export([mk/3]).
|
||||
|
||||
|
@ -129,8 +131,7 @@
|
|||
-export([test_calc_projection/2,
|
||||
test_write_public_projection/2,
|
||||
test_read_latest_public_projection/2]).
|
||||
-export([perhaps_call/5, % for partition simulator use w/machi_fitness
|
||||
init_remember_down_list/0, update_remember_down_list/1,
|
||||
-export([update_remember_down_list/1,
|
||||
get_remember_down_list/0]).
|
||||
|
||||
-ifdef(EQC).
|
||||
|
@ -167,13 +168,22 @@ ping(Pid) ->
|
|||
%% with lowest rank, i.e. name z* first, name a* last.
|
||||
|
||||
set_chain_members(Pid, MembersDict) ->
|
||||
set_chain_members(Pid, MembersDict, []).
|
||||
set_chain_members(Pid, ch0_name, 0, ap_mode, MembersDict, []).
|
||||
|
||||
set_chain_members(Pid, MembersDict, Witness_list) ->
|
||||
case lists:all(fun(Witness) -> orddict:is_key(Witness, MembersDict) end,
|
||||
Witness_list) of
|
||||
set_chain_members(Pid, ChainName, OldEpoch, CMode, MembersDict, Witness_list)
|
||||
when is_atom(ChainName) andalso
|
||||
is_integer(OldEpoch) andalso OldEpoch >= 0 andalso
|
||||
(CMode == ap_mode orelse CMode == cp_mode) andalso
|
||||
is_list(MembersDict) andalso
|
||||
is_list(Witness_list) ->
|
||||
case lists:all(fun({X, #p_srvr{name=X}}) -> true;
|
||||
(_) -> false
|
||||
end, MembersDict)
|
||||
andalso
|
||||
lists:all(fun(Witness) -> orddict:is_key(Witness, MembersDict) end,
|
||||
Witness_list) of
|
||||
true ->
|
||||
Cmd = {set_chain_members, MembersDict, Witness_list},
|
||||
Cmd = {set_chain_members, ChainName, OldEpoch, CMode, MembersDict, Witness_list},
|
||||
gen_server:call(Pid, Cmd, infinity);
|
||||
false ->
|
||||
{error, bad_arg}
|
||||
|
@ -224,11 +234,13 @@ test_read_latest_public_projection(Pid, ReadRepairP) ->
|
|||
%% manager's pid in MgrOpts and use direct gen_server calls to the
|
||||
%% local projection store.
|
||||
|
||||
init({MyName, InitMembersDict, MgrOpts}) ->
|
||||
put(ttt, [?LINE]),
|
||||
random:seed(now()),
|
||||
init({MyName, InitMembersDict, MgrOpts0}) ->
|
||||
put(ttt, [?LINE]),
|
||||
_ = random:seed(now()),
|
||||
init_remember_down_list(),
|
||||
MgrOpts = MgrOpts0 ++ application:get_env(machi, chain_manager_opts, []),
|
||||
Opt = fun(Key, Default) -> proplists:get_value(Key, MgrOpts, Default) end,
|
||||
|
||||
InitWitness_list = Opt(witnesses, []),
|
||||
ZeroAll_list = [P#p_srvr.name || {_,P} <- orddict:to_list(InitMembersDict)],
|
||||
ZeroProj = make_none_projection(0, MyName, ZeroAll_list,
|
||||
|
@ -280,7 +292,7 @@ put(ttt, [?LINE]),
|
|||
last_down=[no_such_server_initial_value_only],
|
||||
fitness_svr=machi_flu_psup:make_fitness_regname(MyName)
|
||||
}, Proj),
|
||||
{_, S2} = do_set_chain_members_dict(MembersDict, S),
|
||||
S2 = do_set_chain_members_dict(MembersDict, S),
|
||||
S3 = if ActiveP == false ->
|
||||
S2;
|
||||
ActiveP == true ->
|
||||
|
@ -290,12 +302,17 @@ put(ttt, [?LINE]),
|
|||
|
||||
handle_call({ping}, _From, S) ->
|
||||
{reply, pong, S};
|
||||
handle_call({set_chain_members, MembersDict, Witness_list}, _From,
|
||||
handle_call({set_chain_members, SetChainName, SetOldEpoch, CMode,
|
||||
MembersDict, Witness_list}, _From,
|
||||
#ch_mgr{name=MyName,
|
||||
proj=#projection_v1{all_members=OldAll_list,
|
||||
epoch_number=OldEpoch,
|
||||
chain_name=ChainName,
|
||||
upi=OldUPI}=OldProj}=S) ->
|
||||
{Reply, S2} = do_set_chain_members_dict(MembersDict, S),
|
||||
true = (OldEpoch == 0) % in this case we want unconditional set of ch name
|
||||
orelse
|
||||
(SetOldEpoch == OldEpoch andalso SetChainName == ChainName),
|
||||
S2 = do_set_chain_members_dict(MembersDict, S),
|
||||
%% TODO: should there be any additional sanity checks? Right now,
|
||||
%% if someone does something bad, then do_react_to_env() will
|
||||
%% crash, which will crash us, and we'll restart in a sane & old
|
||||
|
@ -309,10 +326,10 @@ handle_call({set_chain_members, MembersDict, Witness_list}, _From,
|
|||
{NUPI, All_list -- NUPI}
|
||||
end,
|
||||
NewEpoch = OldEpoch + ?SET_CHAIN_MEMBERS_EPOCH_SKIP,
|
||||
CMode = calc_consistency_mode(Witness_list),
|
||||
ok = set_consistency_mode(machi_flu_psup:make_proj_supname(MyName), CMode),
|
||||
NewProj = machi_projection:update_checksum(
|
||||
OldProj#projection_v1{author_server=MyName,
|
||||
chain_name=SetChainName,
|
||||
creation_time=now(),
|
||||
mode=CMode,
|
||||
epoch_number=NewEpoch,
|
||||
|
@ -324,7 +341,11 @@ handle_call({set_chain_members, MembersDict, Witness_list}, _From,
|
|||
members_dict=MembersDict}),
|
||||
S3 = set_proj(S2#ch_mgr{proj_history=queue:new(),
|
||||
consistency_mode=CMode}, NewProj),
|
||||
{_QQ, S4} = do_react_to_env(S3),
|
||||
{Res, S4} = do_react_to_env(S3),
|
||||
Reply = case Res of
|
||||
{_,_,_} -> ok
|
||||
% Dialyzer says that all vals match with the 3-tuple pattern
|
||||
end,
|
||||
{reply, Reply, S4};
|
||||
handle_call({set_active, Boolean}, _From, #ch_mgr{timer=TRef}=S) ->
|
||||
case {Boolean, TRef} of
|
||||
|
@ -356,8 +377,8 @@ handle_call({test_read_latest_public_projection, ReadRepairP}, _From, S) ->
|
|||
{reply, Res, S2};
|
||||
handle_call({trigger_react_to_env}=Call, _From, S) ->
|
||||
gobble_calls(Call),
|
||||
{TODOtodo, S2} = do_react_to_env(S),
|
||||
{reply, TODOtodo, S2};
|
||||
{Res, S2} = do_react_to_env(S),
|
||||
{reply, Res, S2};
|
||||
handle_call(_Call, _From, S) ->
|
||||
io:format(user, "\nBad call to ~p: ~p\n", [S#ch_mgr.name, _Call]),
|
||||
{reply, whaaaaaaaaaa, S}.
|
||||
|
@ -369,6 +390,7 @@ handle_cast(_Cast, S) ->
|
|||
handle_info(tick_check_environment, #ch_mgr{ignore_timer=true}=S) ->
|
||||
{noreply, S};
|
||||
handle_info(tick_check_environment, S) ->
|
||||
gobble_ticks(),
|
||||
{{_Delta, Props, _Epoch}, S1} = do_react_to_env(S),
|
||||
S2 = sanitize_repair_state(S1),
|
||||
S3 = perhaps_start_repair(S2),
|
||||
|
@ -400,6 +422,11 @@ handle_info(Msg, #ch_mgr{name=MyName}=S) ->
|
|||
terminate(_Reason, _S) ->
|
||||
ok.
|
||||
|
||||
format_status(_Opt, [_PDict, Status]) ->
|
||||
Fields = record_info(fields, ch_mgr),
|
||||
[_Name | Values] = tuple_to_list(Status),
|
||||
lists:zip(Fields, Values).
|
||||
|
||||
code_change(_OldVsn, S, _Extra) ->
|
||||
{ok, S}.
|
||||
|
||||
|
@ -436,7 +463,7 @@ get_my_proj_boot_info(MgrOpts, DefaultDict, DefaultProj, ProjType) ->
|
|||
{DefaultDict, DefaultProj};
|
||||
Store ->
|
||||
{ok, P} = machi_projection_store:read_latest_projection(Store,
|
||||
ProjType),
|
||||
ProjType, 7789),
|
||||
{P#projection_v1.members_dict, P}
|
||||
end.
|
||||
|
||||
|
@ -529,6 +556,7 @@ cl_write_public_proj2(FLUs, Partitions, Epoch, Proj, IgnoreWrittenErrorP, S) ->
|
|||
end
|
||||
end, {true, []}, FLUs),
|
||||
%% io:format(user, "\nWrite public ~w by ~w: ~w\n", [Epoch, S#ch_mgr.name, Rs]),
|
||||
%% io:format(user, "mgr ~w epoch ~w Rs ~p\n", [S#ch_mgr.name, Epoch, Rs]),
|
||||
{{remote_write_results, Rs}, S}.
|
||||
|
||||
do_cl_read_latest_public_projection(ReadRepairP,
|
||||
|
@ -550,12 +578,41 @@ do_cl_read_latest_public_projection(ReadRepairP,
|
|||
read_latest_projection_call_only(ProjectionType, AllHosed,
|
||||
#ch_mgr{proj=CurrentProj}=S) ->
|
||||
#projection_v1{all_members=All_list} = CurrentProj,
|
||||
All_queried_list = All_list -- AllHosed,
|
||||
All_queried_list = lists:sort(All_list -- AllHosed),
|
||||
read_latest_projection_call_only1(ProjectionType, AllHosed,
|
||||
All_queried_list, S).
|
||||
|
||||
{Rs, S2} = read_latest_projection_call_only2(ProjectionType,
|
||||
All_queried_list, S),
|
||||
FLUsRs = lists:zip(All_queried_list, Rs),
|
||||
{All_queried_list, FLUsRs, S2}.
|
||||
read_latest_projection_call_only1(ProjectionType, AllHosed,
|
||||
All_queried_list, S) ->
|
||||
{Rs_tmp, S2} = read_latest_projection_call_only2(ProjectionType,
|
||||
All_queried_list, S),
|
||||
New_all_maybe =
|
||||
lists:usort(
|
||||
lists:flatten(
|
||||
[A_l || #projection_v1{all_members=A_l} <- Rs_tmp])) -- AllHosed,
|
||||
case New_all_maybe -- All_queried_list of
|
||||
[] ->
|
||||
FLUsRs = lists:zip(All_queried_list, Rs_tmp),
|
||||
{All_queried_list, FLUsRs, S2};
|
||||
[AnotherFLU|_] ->
|
||||
%% Stop AnotherFLU proxy, in unexpected case where it's open
|
||||
try
|
||||
Proxy = proxy_pid(AnotherFLU, S2),
|
||||
?FLU_PC:stop_proxies([Proxy])
|
||||
catch _:_ -> ok
|
||||
end,
|
||||
MD = orddict:from_list(
|
||||
lists:usort(
|
||||
lists:flatten(
|
||||
[orddict:to_list(D) || #projection_v1{members_dict=D} <- Rs_tmp]))),
|
||||
Another_P_srvr = orddict:fetch(AnotherFLU, MD),
|
||||
{ok, Proxy2} = ?FLU_PC:start_link(Another_P_srvr),
|
||||
S3 = S2#ch_mgr{proxies_dict=orddict:store(AnotherFLU, Proxy2,
|
||||
S2#ch_mgr.proxies_dict)},
|
||||
read_latest_projection_call_only1(
|
||||
ProjectionType, AllHosed,
|
||||
lists:usort([AnotherFLU|All_queried_list]), S3)
|
||||
end.
|
||||
|
||||
read_latest_projection_call_only2(ProjectionType, All_queried_list, S) ->
|
||||
{_UpNodes, Partitions, S2} = calc_up_nodes(S),
|
||||
|
@ -595,6 +652,8 @@ rank_and_sort_projections_with_extra(All_queried_list, FLUsRs, ProjectionType,
|
|||
Witness_list = CurrentProj#projection_v1.witnesses,
|
||||
NoneProj = make_none_projection(0, MyName, [], Witness_list,
|
||||
orddict:new()),
|
||||
ChainName = CurrentProj#projection_v1.chain_name,
|
||||
NoneProj2 = NoneProj#projection_v1{chain_name=ChainName},
|
||||
Extra2 = [{all_members_replied, true},
|
||||
{all_queried_list, All_queried_list},
|
||||
{flus_rs, FLUsRs},
|
||||
|
@ -603,7 +662,7 @@ rank_and_sort_projections_with_extra(All_queried_list, FLUsRs, ProjectionType,
|
|||
{bad_answer_flus, BadAnswerFLUs},
|
||||
{bad_answers, BadAnswers},
|
||||
{not_unanimous_answers, []}],
|
||||
{not_unanimous, NoneProj, Extra2, S};
|
||||
{not_unanimous, NoneProj2, Extra2, S};
|
||||
ProjectionType == public, UnwrittenRs /= [] ->
|
||||
{needs_repair, FLUsRs, [flarfus], S};
|
||||
true ->
|
||||
|
@ -672,12 +731,12 @@ calc_projection(S, RelativeToServer) ->
|
|||
calc_projection(#ch_mgr{proj=P_current}=S, RelativeToServer, AllHosed) ->
|
||||
calc_projection(S, RelativeToServer, AllHosed, P_current).
|
||||
|
||||
calc_projection(#ch_mgr{name=MyName, consistency_mode=CMode,
|
||||
runenv=RunEnv}=S,
|
||||
calc_projection(#ch_mgr{name=_MyName, consistency_mode=CMode,
|
||||
runenv=_RunEnv}=S,
|
||||
RelativeToServer, AllHosed, P_current) ->
|
||||
Dbg = [],
|
||||
%% OldThreshold = proplists:get_value(old_threshold, RunEnv),
|
||||
%% NoPartitionThreshold = proplists:get_value(no_partition_threshold, RunEnv),
|
||||
%% OldThreshold = proplists:get_value(old_threshold, _RunEnv),
|
||||
%% NoPartitionThreshold = proplists:get_value(no_partition_threshold, _RunEnv),
|
||||
if CMode == ap_mode ->
|
||||
calc_projection2(P_current, RelativeToServer, AllHosed, Dbg, S);
|
||||
CMode == cp_mode ->
|
||||
|
@ -700,7 +759,7 @@ calc_projection(#ch_mgr{name=MyName, consistency_mode=CMode,
|
|||
%% Dbg, S);
|
||||
%% {_, false} ->
|
||||
%% {Up, _Partitions, RunEnv2} = calc_up_nodes(
|
||||
%% MyName, AllMembers, RunEnv),
|
||||
%% _MyName, AllMembers, _RunEnv),
|
||||
%% %% We can't improve on the current projection.
|
||||
%% {P_current, S#ch_mgr{runenv=RunEnv2}, Up}
|
||||
%% end
|
||||
|
@ -717,13 +776,14 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
runenv=RunEnv1,
|
||||
repair_final_status=RepairFS}=S) ->
|
||||
#projection_v1{epoch_number=OldEpochNum,
|
||||
chain_name=ChainName,
|
||||
members_dict=MembersDict,
|
||||
witnesses=OldWitness_list,
|
||||
upi=OldUPI_list,
|
||||
repairing=OldRepairing_list
|
||||
} = LastProj,
|
||||
LastUp = lists:usort(OldUPI_list ++ OldRepairing_list),
|
||||
AllMembers = (S#ch_mgr.proj)#projection_v1.all_members,
|
||||
AllMembers = CurrentProj#projection_v1.all_members,
|
||||
{Up0, Partitions, RunEnv2} = calc_up_nodes(MyName,
|
||||
AllMembers, RunEnv1),
|
||||
Up = Up0 -- AllHosed,
|
||||
|
@ -780,7 +840,10 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
D_foo=[{repair_done, {repair_final_status, ok, (S#ch_mgr.proj)#projection_v1.epoch_number}}],
|
||||
{NewUPI_list ++ Repairing_list2, [], RunEnv2};
|
||||
true ->
|
||||
D_foo=[d_foo2],
|
||||
D_foo=[d_foo2, {sim_p,Simulator_p},
|
||||
{simr_p,SimRepair_p}, {same_epoch,SameEpoch_p},
|
||||
{rel_to,RelativeToServer},
|
||||
{repch,RepChk_LastInUPI}, {repair_fs,RepairFS}],
|
||||
{NewUPI_list, OldRepairing_list, RunEnv2}
|
||||
end;
|
||||
{_ABC, _XYZ} ->
|
||||
|
@ -815,10 +878,11 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
end,
|
||||
?REACT({calc,?LINE,[{new_upi, NewUPI},{new_rep, NewRepairing}]}),
|
||||
|
||||
P = machi_projection:new(OldEpochNum + 1,
|
||||
MyName, MembersDict, Down, NewUPI, NewRepairing,
|
||||
D_foo ++
|
||||
Dbg ++ [{ps, Partitions},{nodes_up, Up}]),
|
||||
P0 = machi_projection:new(OldEpochNum + 1,
|
||||
MyName, MembersDict, Down, NewUPI, NewRepairing,
|
||||
D_foo ++
|
||||
Dbg ++ [{ps, Partitions},{nodes_up, Up}]),
|
||||
P1 = P0#projection_v1{chain_name=ChainName},
|
||||
P2 = if CMode == cp_mode ->
|
||||
UpWitnesses = [W || W <- Up, lists:member(W, OldWitness_list)],
|
||||
Majority = full_majority_size(AllMembers),
|
||||
|
@ -827,7 +891,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
SoFar = length(NewUPI ++ NewRepairing),
|
||||
if SoFar >= Majority ->
|
||||
?REACT({calc,?LINE,[]}),
|
||||
P;
|
||||
P1;
|
||||
true ->
|
||||
Need = Majority - SoFar,
|
||||
UpWitnesses = [W || W <- Up,
|
||||
|
@ -836,7 +900,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
Ws = lists:sublist(UpWitnesses, Need),
|
||||
?REACT({calc,?LINE,[{ws, Ws}]}),
|
||||
machi_projection:update_checksum(
|
||||
P#projection_v1{upi=Ws++NewUPI});
|
||||
P1#projection_v1{upi=Ws++NewUPI});
|
||||
true ->
|
||||
?REACT({calc,?LINE,[]}),
|
||||
P_none0 = make_none_projection(
|
||||
|
@ -849,6 +913,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
"Not enough witnesses are available now"
|
||||
end,
|
||||
P_none1 = P_none0#projection_v1{
|
||||
chain_name=ChainName,
|
||||
%% Stable creation time!
|
||||
creation_time={1,2,3},
|
||||
dbg=[{none_projection,true},
|
||||
|
@ -869,7 +934,7 @@ calc_projection2(LastProj, RelativeToServer, AllHosed, Dbg,
|
|||
end;
|
||||
CMode == ap_mode ->
|
||||
?REACT({calc,?LINE,[]}),
|
||||
P
|
||||
P1
|
||||
end,
|
||||
P3 = machi_projection:update_checksum(
|
||||
P2#projection_v1{mode=CMode, witnesses=OldWitness_list}),
|
||||
|
@ -1021,31 +1086,33 @@ rank_projection(#projection_v1{author_server=_Author,
|
|||
do_set_chain_members_dict(MembersDict, #ch_mgr{proxies_dict=OldProxiesDict}=S)->
|
||||
_ = ?FLU_PC:stop_proxies(OldProxiesDict),
|
||||
ProxiesDict = ?FLU_PC:start_proxies(MembersDict),
|
||||
{ok, S#ch_mgr{members_dict=MembersDict,
|
||||
proxies_dict=ProxiesDict}}.
|
||||
S#ch_mgr{members_dict=MembersDict,
|
||||
proxies_dict=ProxiesDict}.
|
||||
|
||||
do_react_to_env(#ch_mgr{name=MyName,
|
||||
proj=#projection_v1{epoch_number=Epoch,
|
||||
members_dict=[]=OldDict}=OldProj,
|
||||
opts=Opts}=S) ->
|
||||
put(ttt, [?LINE]),
|
||||
%% Read from our local *public* projection store. If some other
|
||||
%% chain member has written something there, and if we are a
|
||||
%% member of that chain, then we'll adopt that projection and then
|
||||
%% start actively humming in that chain.
|
||||
{NewMembersDict, NewProj} =
|
||||
{NewMD, NewProj} =
|
||||
get_my_public_proj_boot_info(Opts, OldDict, OldProj),
|
||||
case orddict:is_key(MyName, NewMembersDict) of
|
||||
case orddict:is_key(MyName, NewMD) of
|
||||
false ->
|
||||
{{empty_members_dict, [], Epoch}, S};
|
||||
{{empty_members_dict1, [], Epoch}, S};
|
||||
true ->
|
||||
{_, S2} = do_set_chain_members_dict(NewMembersDict, S),
|
||||
CMode = calc_consistency_mode(NewProj#projection_v1.witnesses),
|
||||
{{empty_members_dict, [], Epoch},
|
||||
set_proj(S2#ch_mgr{members_dict=NewMembersDict,
|
||||
consistency_mode=CMode}, NewProj)}
|
||||
CMode = NewProj#projection_v1.mode,
|
||||
S2 = do_set_chain_members_dict(NewMD, S),
|
||||
{Reply, S3} = react_to_env_C110(NewProj,
|
||||
S2#ch_mgr{members_dict=NewMD,
|
||||
consistency_mode=CMode}),
|
||||
{Reply, S3}
|
||||
end;
|
||||
do_react_to_env(S) ->
|
||||
put(ttt, [?LINE]),
|
||||
put(ttt, [?LINE]),
|
||||
%% The not_sanes manager counting dictionary is not strictly
|
||||
%% limited to flapping scenarios. (Though the mechanism first
|
||||
%% started as a way to deal with rare flapping scenarios.)
|
||||
|
@ -1144,7 +1211,7 @@ react_to_env_A10(S) ->
|
|||
?REACT(a10),
|
||||
react_to_env_A20(0, poll_private_proj_is_upi_unanimous(S)).
|
||||
|
||||
react_to_env_A20(Retries, #ch_mgr{name=MyName}=S) ->
|
||||
react_to_env_A20(Retries, #ch_mgr{name=MyName, proj=P_current}=S) ->
|
||||
?REACT(a20),
|
||||
init_remember_down_list(),
|
||||
{UnanimousTag, P_latest, ReadExtra, S2} =
|
||||
|
@ -1172,17 +1239,34 @@ react_to_env_A20(Retries, #ch_mgr{name=MyName}=S) ->
|
|||
false when P_latest#projection_v1.epoch_number /= LastComplaint,
|
||||
P_latest#projection_v1.all_members /= [] ->
|
||||
put(rogue_server_epoch, P_latest#projection_v1.epoch_number),
|
||||
error_logger:info_msg("Chain manager ~p found latest public "
|
||||
"projection ~p has author ~p has a "
|
||||
"members list ~p that does not include me.\n",
|
||||
error_logger:info_msg("Chain manager ~w found latest public "
|
||||
"projection ~w with author ~w has a "
|
||||
"members list ~w that does not include me. "
|
||||
"We assume this is a result of administrator "
|
||||
"action and will thus wedge ourselves until "
|
||||
"we are re-added to the chain or shutdown.\n",
|
||||
[S#ch_mgr.name,
|
||||
P_latest#projection_v1.epoch_number,
|
||||
P_latest#projection_v1.author_server,
|
||||
P_latest#projection_v1.all_members]);
|
||||
P_latest#projection_v1.all_members]),
|
||||
EpochID = machi_projection:make_epoch_id(P_current),
|
||||
ProjStore = get_projection_store_pid_or_regname(S),
|
||||
{ok, NotifyPid} = machi_projection_store:get_wedge_notify_pid(ProjStore),
|
||||
_QQ = machi_flu1:update_wedge_state(NotifyPid, true, EpochID),
|
||||
#projection_v1{epoch_number=Epoch,
|
||||
chain_name=ChainName,
|
||||
all_members=All_list,
|
||||
witnesses=Witness_list,
|
||||
members_dict=MembersDict} = P_current,
|
||||
P_none0 = make_none_projection(Epoch,
|
||||
MyName, All_list, Witness_list, MembersDict),
|
||||
P_none = P_none0#projection_v1{chain_name=ChainName},
|
||||
{{now_using,[],Epoch}, set_proj(S2, P_none)};
|
||||
_ ->
|
||||
ok
|
||||
end,
|
||||
react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S2)
|
||||
end.
|
||||
|
||||
react_to_env_A21(Retries, UnanimousTag, P_latest, ReadExtra, S) ->
|
||||
%% The UnanimousTag isn't quite sufficient for our needs. We need
|
||||
%% to determine if *all* of the UPI+Repairing FLUs are members of
|
||||
%% the unanimous server replies. All Repairing FLUs should be up
|
||||
|
@ -1227,7 +1311,7 @@ react_to_env_A20(Retries, #ch_mgr{name=MyName}=S) ->
|
|||
true ->
|
||||
exit({badbad, UnanimousTag})
|
||||
end,
|
||||
react_to_env_A29(Retries, P_latest, LatestUnanimousP, ReadExtra, S2).
|
||||
react_to_env_A29(Retries, P_latest, LatestUnanimousP, ReadExtra, S).
|
||||
|
||||
react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra,
|
||||
#ch_mgr{consistency_mode=CMode,
|
||||
|
@ -1261,7 +1345,6 @@ react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra,
|
|||
?REACT({a29, ?LINE,
|
||||
[{zerf_backstop, true},
|
||||
{zerf_in, machi_projection:make_summary(Zerf)}]}),
|
||||
%% io:format(user, "zerf_in: A29: ~p: ~w\n\t~p\n", [MyName, machi_projection:make_summary(Zerf), get(yyy_hack)]),
|
||||
#projection_v1{dbg=ZerfDbg} = Zerf,
|
||||
Backstop = if Zerf#projection_v1.upi == [] ->
|
||||
[];
|
||||
|
@ -1281,7 +1364,8 @@ react_to_env_A29(Retries, P_latest, LatestUnanimousP, _ReadExtra,
|
|||
end.
|
||||
|
||||
react_to_env_A30(Retries, P_latest, LatestUnanimousP, P_current_calc,
|
||||
#ch_mgr{name=MyName, consistency_mode=CMode} = S) ->
|
||||
#ch_mgr{name=MyName, proj=P_current,
|
||||
consistency_mode=CMode} = S) ->
|
||||
V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end,
|
||||
if V -> io:format(user, "A30: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
|
||||
?REACT(a30),
|
||||
|
@ -1301,15 +1385,17 @@ react_to_env_A30(Retries, P_latest, LatestUnanimousP, P_current_calc,
|
|||
P = #projection_v1{down=Down} =
|
||||
make_none_projection(Epoch + 1, MyName, All_list,
|
||||
Witness_list, MembersDict),
|
||||
ChainName = P_current#projection_v1.chain_name,
|
||||
P1 = P#projection_v1{chain_name=ChainName},
|
||||
P_newprop = if CMode == ap_mode ->
|
||||
%% Not really none proj: just myself, AP style
|
||||
machi_projection:update_checksum(
|
||||
P#projection_v1{upi=[MyName],
|
||||
P1#projection_v1{upi=[MyName],
|
||||
down=Down -- [MyName],
|
||||
dbg=[{hosed_list,AllHosed}]});
|
||||
CMode == cp_mode ->
|
||||
machi_projection:update_checksum(
|
||||
P#projection_v1{dbg=[{hosed_list,AllHosed}]})
|
||||
P1#projection_v1{dbg=[{hosed_list,AllHosed}]})
|
||||
end,
|
||||
react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP,
|
||||
P_current_calc, true, S);
|
||||
|
@ -1382,13 +1468,22 @@ react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP,
|
|||
%% we have a disagreement.
|
||||
not ordsets:is_disjoint(P_latest_s, Down_s)
|
||||
end,
|
||||
AmExcludedFromLatestAll_p =
|
||||
P_latest#projection_v1.epoch_number /= 0
|
||||
andalso
|
||||
(not lists:member(MyName, P_latest#projection_v1.all_members)),
|
||||
?REACT({a40, ?LINE,
|
||||
[{latest_author, P_latest#projection_v1.author_server},
|
||||
{am_excluded_from_latest_all_p, AmExcludedFromLatestAll_p},
|
||||
{author_is_down_p, LatestAuthorDownP},
|
||||
{rank_latest, Rank_latest},
|
||||
{rank_newprop, Rank_newprop}]}),
|
||||
|
||||
if
|
||||
AmExcludedFromLatestAll_p ->
|
||||
?REACT({a40, ?LINE, [{latest,machi_projection:make_summary(P_latest)}]}),
|
||||
react_to_env_A50(P_latest, [], S);
|
||||
|
||||
AmHosedP ->
|
||||
ExpectedUPI = if CMode == cp_mode -> [];
|
||||
CMode == ap_mode -> [MyName]
|
||||
|
@ -1554,12 +1649,10 @@ react_to_env_A40(Retries, P_newprop, P_latest, LatestUnanimousP,
|
|||
end,
|
||||
if GoTo50_p ->
|
||||
?REACT({a40, ?LINE, []}),
|
||||
%% io:format(user, "CONFIRM debug question line ~w\n", [?LINE]),
|
||||
FinalProps = [{throttle_seconds, 0}],
|
||||
react_to_env_A50(P_latest, FinalProps, S);
|
||||
true ->
|
||||
?REACT({a40, ?LINE, []}),
|
||||
io:format(user, "CONFIRM debug question line ~w\n", [?LINE]),
|
||||
react_to_env_C300(P_newprop, P_latest, S)
|
||||
end
|
||||
end.
|
||||
|
@ -1569,7 +1662,6 @@ react_to_env_A50(P_latest, FinalProps, #ch_mgr{proj=P_current}=S) ->
|
|||
?REACT({a50, ?LINE, [{current_epoch, P_current#projection_v1.epoch_number},
|
||||
{latest_epoch, P_latest#projection_v1.epoch_number},
|
||||
{final_props, FinalProps}]}),
|
||||
%% if S#ch_mgr.name == c -> io:format(user, "A50: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
|
||||
V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end,
|
||||
if V -> io:format(user, "A50: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
|
||||
{{no_change, FinalProps, P_current#projection_v1.epoch_number}, S}.
|
||||
|
@ -1779,9 +1871,10 @@ react_to_env_C100(P_newprop,
|
|||
if Sane == true ->
|
||||
ok;
|
||||
true ->
|
||||
QQ_current = lists:flatten(io_lib:format("cur=~w:~w,~w/calc=~w:~w,~w", [P_current#projection_v1.epoch_number, P_current#projection_v1.upi, P_current#projection_v1.repairing, P_current_calc#projection_v1.epoch_number, P_current_calc#projection_v1.upi, P_current_calc#projection_v1.repairing])),
|
||||
QQ_latest = lists:flatten(io_lib:format("~w:~w,~w", [P_latest#projection_v1.epoch_number, P_latest#projection_v1.upi, P_latest#projection_v1.repairing])),
|
||||
?V("\n~w-insane-~w-auth=~w ~s -> ~s ~w\n ~p\n ~p\n", [?LINE, MyName, P_newprop#projection_v1.author_server, QQ_current, QQ_latest, Sane, get(why2), get(react)])
|
||||
%% QQ_current = lists:flatten(io_lib:format("cur=~w:~w,~w/calc=~w:~w,~w", [P_current#projection_v1.epoch_number, P_current#projection_v1.upi, P_current#projection_v1.repairing, P_current_calc#projection_v1.epoch_number, P_current_calc#projection_v1.upi, P_current_calc#projection_v1.repairing])),
|
||||
%% QQ_latest = lists:flatten(io_lib:format("~w:~w,~w", [P_latest#projection_v1.epoch_number, P_latest#projection_v1.upi, P_latest#projection_v1.repairing])),
|
||||
%% ?V("\n~w-insane-~w-auth=~w ~s -> ~s ~w\n ~p\n ~p\n", [?LINE, MyName, P_newprop#projection_v1.author_server, QQ_current, QQ_latest, Sane, get(why2), get(react)]),
|
||||
ok
|
||||
end,
|
||||
?REACT({c100, ?LINE, [zoo, {me,MyName},
|
||||
{author_latest,Author_latest},
|
||||
|
@ -1797,11 +1890,11 @@ react_to_env_C100(P_newprop,
|
|||
%% or for when we got stuck in an insane projection transition
|
||||
%% and were forced to the none projection to recover.
|
||||
?REACT({c100, ?LINE, [first_write]}),
|
||||
if Sane == true -> ok; true -> ?V("~w-insane-~w-~w:~w:~w,", [?LINE, MyName, P_newprop#projection_v1.epoch_number, P_newprop#projection_v1.upi, P_newprop#projection_v1.repairing]) end, %%% DELME!!!
|
||||
%% if Sane == true -> ok; true -> ?V("~w-insane-~w-~w:~w:~w,", [?LINE, MyName, P_newprop#projection_v1.epoch_number, P_newprop#projection_v1.upi, P_newprop#projection_v1.repairing]) end, %%% DELME!!!
|
||||
react_to_env_C110(P_latest, S);
|
||||
true ->
|
||||
?REACT({c100, ?LINE, []}),
|
||||
if Sane == true -> ok; true -> ?V("~w-insane-~w-~w:~w:~w@~w,", [?LINE, MyName, P_newprop#projection_v1.epoch_number, P_newprop#projection_v1.upi, P_newprop#projection_v1.repairing, ?LINE]) end, %%% DELME!!!
|
||||
%% if Sane == true -> ok; true -> ?V("~w-insane-~w-~w:~w:~w@~w,", [?LINE, MyName, P_newprop#projection_v1.epoch_number, P_newprop#projection_v1.upi, P_newprop#projection_v1.repairing, ?LINE]) end, %%% DELME!!!
|
||||
|
||||
V = case file:read_file("/tmp/bugbug."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end,
|
||||
if V ->
|
||||
|
@ -1816,17 +1909,17 @@ react_to_env_C100(P_newprop,
|
|||
P_newprop, P_latest, P_current_calc, S)
|
||||
end.
|
||||
|
||||
react_to_env_C100_inner(Author_latest, NotSanesDict0, MyName,
|
||||
react_to_env_C100_inner(Author_latest, NotSanesDict0, _MyName,
|
||||
P_newprop, P_latest, P_current_calc, S) ->
|
||||
NotSanesDict = orddict:update_counter(Author_latest, 1, NotSanesDict0),
|
||||
S2 = S#ch_mgr{not_sanes=NotSanesDict, sane_transitions=0},
|
||||
case orddict:fetch(Author_latest, NotSanesDict) of
|
||||
N when N > ?TOO_FREQUENT_BREAKER ->
|
||||
?V("\n\nYOYO ~w breaking the cycle of:\n current: ~w\n new : ~w\n", [MyName, machi_projection:make_summary(S#ch_mgr.proj), machi_projection:make_summary(P_latest)]),
|
||||
?V("\n\nYOYO ~w breaking the cycle insane-freq=~w by-author=~w of:\n current: ~w\n new : ~w\n", [_MyName, N, Author_latest, machi_projection:make_summary(S#ch_mgr.proj), machi_projection:make_summary(P_latest)]),
|
||||
?REACT({c100, ?LINE, [{not_sanes_author_count, N}]}),
|
||||
react_to_env_C103(P_newprop, P_latest, P_current_calc, S2);
|
||||
N ->
|
||||
?V("YOYO,~w,~w,~w,",[MyName, P_latest#projection_v1.epoch_number,N]),
|
||||
%% ?V("YOYO,~w,~w,~w,",[_MyName, P_latest#projection_v1.epoch_number,N]),
|
||||
?REACT({c100, ?LINE, [{not_sanes_author_count, N}]}),
|
||||
%% P_latest is not sane.
|
||||
%% By process of elimination, P_newprop is best,
|
||||
|
@ -1843,12 +1936,14 @@ react_to_env_C103(#projection_v1{epoch_number=_Epoch_newprop} = _P_newprop,
|
|||
members_dict=MembersDict} = P_current,
|
||||
P_none0 = make_none_projection(Epoch_latest,
|
||||
MyName, All_list, Witness_list, MembersDict),
|
||||
P_none1 = P_none0#projection_v1{dbg=[{none_projection,true}]},
|
||||
ChainName = P_current#projection_v1.chain_name,
|
||||
P_none1 = P_none0#projection_v1{chain_name=ChainName,
|
||||
dbg=[{none_projection,true}]},
|
||||
P_none = machi_projection:update_checksum(P_none1),
|
||||
?REACT({c103, ?LINE,
|
||||
[{current_epoch, P_current#projection_v1.epoch_number},
|
||||
{none_projection_epoch, P_none#projection_v1.epoch_number}]}),
|
||||
io:format(user, "SET add_admin_down(~w) at ~w =====================================\n", [MyName, time()]),
|
||||
io:format(user, "SET add_admin_down(~w) at ~w current_epoch ~w none_proj_epoch ~w =====================================\n", [MyName, time(), P_current#projection_v1.epoch_number, P_none#projection_v1.epoch_number]),
|
||||
machi_fitness:add_admin_down(S#ch_mgr.fitness_svr, MyName, []),
|
||||
timer:sleep(5*1000),
|
||||
io:format(user, "SET delete_admin_down(~w) at ~w =====================================\n", [MyName, time()]),
|
||||
|
@ -1885,7 +1980,7 @@ react_to_env_C110(P_latest, #ch_mgr{name=MyName} = S) ->
|
|||
%% In contrast to the public projection store writes, Humming Consensus
|
||||
%% doesn't care about the status of writes to the public store: it's
|
||||
%% always relying only on successful reads of the public store.
|
||||
case {?FLU_PC:write_projection(MyStorePid, private, P_latest2,?TO*30),Goo} of
|
||||
case {?FLU_PC:write_projection(MyStorePid, private, P_latest2,?TO*30+66),Goo} of
|
||||
{ok, Goo} ->
|
||||
?REACT({c110, [{write, ok}]}),
|
||||
react_to_env_C111(P_latest, P_latest2, Extra1, MyStorePid, S);
|
||||
|
@ -1971,7 +2066,6 @@ react_to_env_C120(P_latest, FinalProps, #ch_mgr{proj_history=H,
|
|||
?REACT(c120),
|
||||
H2 = add_and_trunc_history(P_latest, H, ?MAX_HISTORY_LENGTH),
|
||||
|
||||
%% diversion_c120_verbose_goop(P_latest, S),
|
||||
?REACT({c120, [{latest, machi_projection:make_summary(P_latest)}]}),
|
||||
S2 = set_proj(S#ch_mgr{proj_history=H2,
|
||||
sane_transitions=Xtns + 1}, P_latest),
|
||||
|
@ -1979,20 +2073,21 @@ react_to_env_C120(P_latest, FinalProps, #ch_mgr{proj_history=H,
|
|||
false ->
|
||||
S2;
|
||||
{{_ConfEpoch, _ConfCSum}, ConfTime} ->
|
||||
io:format(user, "\nCONFIRM debug C120 ~w was annotated ~w\n", [S#ch_mgr.name, P_latest#projection_v1.epoch_number]),
|
||||
P_latestEpoch = P_latest#projection_v1.epoch_number,
|
||||
io:format(user, "\nCONFIRM debug C120 ~w was annotated ~w\n", [S#ch_mgr.name, P_latestEpoch]),
|
||||
S2#ch_mgr{proj_unanimous=ConfTime}
|
||||
end,
|
||||
V = case file:read_file("/tmp/moomoo."++atom_to_list(S#ch_mgr.name)) of {ok,_} -> true; _ -> false end,
|
||||
if V -> io:format("C120: ~w: ~p\n", [S#ch_mgr.name, get(react)]); true -> ok end,
|
||||
{{now_using, FinalProps, P_latest#projection_v1.epoch_number}, S3}.
|
||||
|
||||
add_and_trunc_history(P_latest, H, MaxLength) ->
|
||||
add_and_trunc_history(#projection_v1{epoch_number=0}, H, _MaxLength) ->
|
||||
H;
|
||||
add_and_trunc_history(#projection_v1{} = P_latest, H, MaxLength) ->
|
||||
Latest_U_R = {P_latest#projection_v1.upi, P_latest#projection_v1.repairing},
|
||||
H2 = if P_latest#projection_v1.epoch_number > 0 ->
|
||||
queue:in(Latest_U_R, H);
|
||||
true ->
|
||||
H
|
||||
end,
|
||||
add_and_trunc_history(Latest_U_R, H, MaxLength);
|
||||
add_and_trunc_history(Item, H, MaxLength) ->
|
||||
H2 = queue:in(Item, H),
|
||||
case queue:len(H2) of
|
||||
X when X > MaxLength ->
|
||||
{_V, Hxx} = queue:out(H2),
|
||||
|
@ -2005,17 +2100,16 @@ react_to_env_C200(Retries, P_latest, S) ->
|
|||
?REACT(c200),
|
||||
try
|
||||
AuthorProxyPid = proxy_pid(P_latest#projection_v1.author_server, S),
|
||||
?FLU_PC:kick_projection_reaction(AuthorProxyPid, [])
|
||||
%% This is just advisory, we don't need a sync reply.
|
||||
?FLU_PC:kick_projection_reaction(AuthorProxyPid, [], 100)
|
||||
catch _Type:_Err ->
|
||||
%% ?V("TODO: tell_author_yo is broken: ~p ~p\n",
|
||||
%% [_Type, _Err]),
|
||||
ok
|
||||
ok
|
||||
end,
|
||||
react_to_env_C210(Retries, S).
|
||||
|
||||
react_to_env_C210(Retries, #ch_mgr{name=MyName, proj=Proj} = S) ->
|
||||
?REACT(c210),
|
||||
sleep_ranked_order(10, 100, MyName, Proj#projection_v1.all_members),
|
||||
sleep_ranked_order(250, 500, MyName, Proj#projection_v1.all_members),
|
||||
react_to_env_C220(Retries, S).
|
||||
|
||||
react_to_env_C220(Retries, S) ->
|
||||
|
@ -2199,6 +2293,7 @@ projection_transition_is_sane_except_si_epoch(
|
|||
creation_time=CreationTime1,
|
||||
mode=CMode1,
|
||||
author_server=AuthorServer1,
|
||||
chain_name=ChainName1,
|
||||
all_members=All_list1,
|
||||
witnesses=Witness_list1,
|
||||
down=Down_list1,
|
||||
|
@ -2210,6 +2305,7 @@ projection_transition_is_sane_except_si_epoch(
|
|||
creation_time=CreationTime2,
|
||||
mode=CMode2,
|
||||
author_server=AuthorServer2,
|
||||
chain_name=ChainName2,
|
||||
all_members=All_list2,
|
||||
witnesses=Witness_list2,
|
||||
down=Down_list2,
|
||||
|
@ -2230,7 +2326,8 @@ projection_transition_is_sane_except_si_epoch(
|
|||
true = is_binary(CSum1) andalso is_binary(CSum2),
|
||||
{_,_,_} = CreationTime1,
|
||||
{_,_,_} = CreationTime2,
|
||||
true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2), % todo type may change?
|
||||
true = is_atom(AuthorServer1) andalso is_atom(AuthorServer2),
|
||||
true = is_atom(ChainName1) andalso is_atom(ChainName2),
|
||||
true = is_list(All_list1) andalso is_list(All_list2),
|
||||
true = is_list(Witness_list1) andalso is_list(Witness_list2),
|
||||
true = is_list(Down_list1) andalso is_list(Down_list2),
|
||||
|
@ -2242,6 +2339,9 @@ projection_transition_is_sane_except_si_epoch(
|
|||
%% projection_transition_is_sane_with_si_epoch().
|
||||
true = Epoch2 >= Epoch1,
|
||||
|
||||
%% Don't change chain names in the middle of the stream.
|
||||
true = (ChainName1 == ChainName2),
|
||||
|
||||
%% No duplicates
|
||||
true = lists:sort(Witness_list2) == lists:usort(Witness_list2),
|
||||
true = lists:sort(Down_list2) == lists:usort(Down_list2),
|
||||
|
@ -2249,7 +2349,7 @@ projection_transition_is_sane_except_si_epoch(
|
|||
true = lists:sort(Repairing_list2) == lists:usort(Repairing_list2),
|
||||
|
||||
%% Disjoint-ness
|
||||
All_list1 = All_list2, % todo will probably change
|
||||
%% %% %% %% %% %% %% %% All_list1 = All_list2, % todo will probably change
|
||||
%% true = lists:sort(All_list2) == lists:sort(Down_list2 ++ UPI_list2 ++
|
||||
%% Repairing_list2),
|
||||
[] = [X || X <- Witness_list2, not lists:member(X, All_list2)],
|
||||
|
@ -2354,8 +2454,7 @@ poll_private_proj_is_upi_unanimous_sleep(Count, #ch_mgr{runenv=RunEnv}=S) ->
|
|||
S2
|
||||
end.
|
||||
|
||||
poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current,
|
||||
opts=MgrOpts} = S) ->
|
||||
poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current} = S) ->
|
||||
UPI = P_current#projection_v1.upi,
|
||||
EpochID = machi_projection:make_epoch_id(P_current),
|
||||
{Rs, S2} = read_latest_projection_call_only2(private, UPI, S),
|
||||
|
@ -2388,33 +2487,30 @@ poll_private_proj_is_upi_unanimous3(#ch_mgr{name=MyName, proj=P_current,
|
|||
Annotation = make_annotation(EpochID, Now),
|
||||
NewDbg2 = [Annotation|P_currentFull#projection_v1.dbg2],
|
||||
NewProj = P_currentFull#projection_v1{dbg2=NewDbg2},
|
||||
ProjStore = case get_projection_store_regname(MgrOpts) of
|
||||
undefined ->
|
||||
machi_flu_psup:make_proj_supname(MyName);
|
||||
PStr ->
|
||||
PStr
|
||||
end,
|
||||
ProjStore = get_projection_store_pid_or_regname(S),
|
||||
#projection_v1{epoch_number=_EpochRep,
|
||||
epoch_csum= <<_CSumRep:4/binary,_/binary>>,
|
||||
author_server=AuthRep,
|
||||
upi=_UPIRep,
|
||||
repairing=_RepairingRep} = NewProj,
|
||||
ok = machi_projection_store:write(ProjStore, private, NewProj),
|
||||
case proplists:get_value(private_write_verbose, S#ch_mgr.opts) of
|
||||
case proplists:get_value(private_write_verbose_confirm, S#ch_mgr.opts) of
|
||||
true ->
|
||||
io:format(user, "\n~s CONFIRM epoch ~w ~w upi ~w rep ~w by ~w\n", [machi_util:pretty_time(), _EpochRep, _CSumRep, _UPIRep, _RepairingRep, MyName]);
|
||||
error_logger:info_msg("CONFIRM epoch ~w ~w upi ~w rep ~w auth ~w by ~w\n", [_EpochRep, _CSumRep, _UPIRep, _RepairingRep, AuthRep, MyName]);
|
||||
_ ->
|
||||
ok
|
||||
end,
|
||||
%% Unwedge our FLU.
|
||||
{ok, NotifyPid} = machi_projection_store:get_wedge_notify_pid(ProjStore),
|
||||
_ = machi_flu1:update_wedge_state(NotifyPid, false, EpochID),
|
||||
S2#ch_mgr{proj_unanimous=Now};
|
||||
#ch_mgr{proj_history=H} = S2,
|
||||
H2 = add_and_trunc_history({confirm, Epoch}, H,
|
||||
?MAX_HISTORY_LENGTH),
|
||||
S2#ch_mgr{proj_unanimous=Now, proj_history=H2};
|
||||
_ ->
|
||||
S2
|
||||
end;
|
||||
_Else ->
|
||||
%% io:format(user, "poll by ~w: want ~W got ~W\n",
|
||||
%% [MyName, EpochID, 6, _Else, 8]),
|
||||
S2
|
||||
end.
|
||||
|
||||
|
@ -2450,6 +2546,14 @@ gobble_calls(StaticCall) ->
|
|||
ok
|
||||
end.
|
||||
|
||||
gobble_ticks() ->
|
||||
receive
|
||||
tick_check_environment ->
|
||||
gobble_ticks()
|
||||
after 0 ->
|
||||
ok
|
||||
end.
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
perhaps_start_repair(#ch_mgr{name=MyName,
|
||||
|
@ -2465,12 +2569,13 @@ perhaps_start_repair(#ch_mgr{name=MyName,
|
|||
%% RepairOpts = [{repair_mode, check}, verbose],
|
||||
RepairFun = fun() -> do_repair(S, RepairOpts, CMode) end,
|
||||
LastUPI = lists:last(UPI),
|
||||
StabilityTime = application:get_env(machi, stability_time, ?REPAIR_START_STABILITY_TIME),
|
||||
IgnoreStabilityTime_p = proplists:get_value(ignore_stability_time,
|
||||
S#ch_mgr.opts, false),
|
||||
case timer:now_diff(os:timestamp(), Start) div 1000000 of
|
||||
N when MyName == LastUPI andalso
|
||||
(IgnoreStabilityTime_p orelse
|
||||
N >= ?REPAIR_START_STABILITY_TIME) ->
|
||||
N >= StabilityTime) ->
|
||||
{WorkerPid, _Ref} = spawn_monitor(RepairFun),
|
||||
S#ch_mgr{repair_worker=WorkerPid,
|
||||
repair_start=os:timestamp(),
|
||||
|
@ -2511,8 +2616,8 @@ do_repair(#ch_mgr{name=MyName,
|
|||
T1 = os:timestamp(),
|
||||
RepairId = proplists:get_value(repair_id, Opts, id1),
|
||||
error_logger:info_msg(
|
||||
"Repair start: tail ~p of ~p -> ~p, ~p ID ~w\n",
|
||||
[MyName, UPI0, Repairing, RepairMode, RepairId]),
|
||||
"Repair ~w start: tail ~p of ~p -> ~p, ~p\n",
|
||||
[RepairId, MyName, UPI0, Repairing, RepairMode]),
|
||||
|
||||
UPI = UPI0 -- Witness_list,
|
||||
Res = machi_chain_repair:repair(RepairMode, MyName, Repairing, UPI,
|
||||
|
@ -2525,10 +2630,9 @@ do_repair(#ch_mgr{name=MyName,
|
|||
end,
|
||||
Stats = [{K, ets:lookup_element(ETS, K, 2)} || K <- ETS_T_Keys],
|
||||
error_logger:info_msg(
|
||||
"Repair ~s: tail ~p of ~p finished ~p repair ID ~w: "
|
||||
"~p\nStats ~p\n",
|
||||
[Summary, MyName, UPI0, RepairMode, RepairId,
|
||||
Res, Stats]),
|
||||
"Repair ~w ~s: tail ~p of ~p finished ~p: "
|
||||
"~p Stats: ~p\n",
|
||||
[RepairId, Summary, MyName, UPI0, RepairMode, Res, Stats]),
|
||||
ets:delete(ETS),
|
||||
exit({repair_final_status, Res});
|
||||
_ ->
|
||||
|
@ -2671,10 +2775,10 @@ simple_chain_state_transition_is_sane(_Author1, UPI1, Repair1, Author2, UPI2) ->
|
|||
?RETURN2(true);
|
||||
UPI1_tail ->
|
||||
?RETURN2({expected_author2,UPI1_tail,
|
||||
[{upi1,UPI1},
|
||||
{repair1,Repair1},
|
||||
{author2,Author2},
|
||||
{upi2,UPI2}]})
|
||||
[{upi1,UPI1},
|
||||
{repair1,Repair1},
|
||||
{author2,Author2},
|
||||
{upi2,UPI2}]})
|
||||
end
|
||||
end
|
||||
end.
|
||||
|
@ -2765,6 +2869,7 @@ full_majority_size(L) when is_list(L) ->
|
|||
full_majority_size(length(L)).
|
||||
|
||||
make_zerf(#projection_v1{epoch_number=OldEpochNum,
|
||||
chain_name=ChainName,
|
||||
all_members=AllMembers,
|
||||
members_dict=MembersDict,
|
||||
witnesses=OldWitness_list
|
||||
|
@ -2787,7 +2892,8 @@ make_zerf(#projection_v1{epoch_number=OldEpochNum,
|
|||
MyName, AllMembers, OldWitness_list,
|
||||
MembersDict),
|
||||
machi_projection:update_checksum(
|
||||
P#projection_v1{mode=cp_mode,
|
||||
P#projection_v1{chain_name=ChainName,
|
||||
mode=cp_mode,
|
||||
dbg2=[zerf_none,{up,Up},{maj,MajoritySize}]});
|
||||
true ->
|
||||
make_zerf2(OldEpochNum, Up, MajoritySize, MyName,
|
||||
|
@ -2802,7 +2908,6 @@ make_zerf2(OldEpochNum, Up, MajoritySize, MyName, AllMembers, OldWitness_list,
|
|||
Proj2 = Proj#projection_v1{dbg2=[{make_zerf,Epoch},
|
||||
{yyy_hack, get(yyy_hack)},
|
||||
{up,Up},{maj,MajoritySize}]},
|
||||
%% io:format(user, "ZERF ~w\n",[machi_projection:make_summary(Proj2)]),
|
||||
Proj2
|
||||
catch
|
||||
throw:{zerf,no_common} ->
|
||||
|
@ -2879,41 +2984,36 @@ zerf_find_last_annotated(FLU, MajoritySize, S) ->
|
|||
[] % lists:flatten() will destroy
|
||||
end.
|
||||
|
||||
perhaps_verbose_c111(P_latest2, S) ->
|
||||
case proplists:get_value(private_write_verbose, S#ch_mgr.opts) of
|
||||
true ->
|
||||
perhaps_verbose_c111(P_latest2, #ch_mgr{name=MyName, opts=Opts}=S) ->
|
||||
PrivWriteVerb = proplists:get_value(private_write_verbose, Opts, false),
|
||||
PrivWriteVerbCONFIRM = proplists:get_value(private_write_verbose_confirm, Opts, false),
|
||||
if PrivWriteVerb orelse PrivWriteVerbCONFIRM ->
|
||||
Dbg2X = lists:keydelete(react, 1,
|
||||
P_latest2#projection_v1.dbg2) ++
|
||||
[{is_annotated,is_annotated(P_latest2)}],
|
||||
P_latest2x = P_latest2#projection_v1{dbg2=Dbg2X}, % limit verbose len.
|
||||
Last2 = get(last_verbose),
|
||||
Summ2 = machi_projection:make_summary(P_latest2x),
|
||||
if P_latest2#projection_v1.upi == [],
|
||||
(S#ch_mgr.proj)#projection_v1.upi /= [] ->
|
||||
<<CSumRep:4/binary,_/binary>> =
|
||||
P_latest2#projection_v1.epoch_csum,
|
||||
io:format(user, "\n~s CONFIRM epoch ~w ~w upi ~w rep ~w by ~w\n", [machi_util:pretty_time(), (S#ch_mgr.proj)#projection_v1.epoch_number, CSumRep, P_latest2#projection_v1.upi, P_latest2#projection_v1.repairing, S#ch_mgr.name]);
|
||||
if PrivWriteVerb, Summ2 /= Last2 ->
|
||||
put(last_verbose, Summ2),
|
||||
error_logger:info_msg("~p uses plain: ~w \n",
|
||||
[MyName, Summ2]);
|
||||
true ->
|
||||
ok
|
||||
end,
|
||||
case proplists:get_value(private_write_verbose,
|
||||
S#ch_mgr.opts) of
|
||||
true when Summ2 /= Last2 ->
|
||||
put(last_verbose, Summ2),
|
||||
?V("\n~s ~p uses plain: ~w \n",
|
||||
[machi_util:pretty_time(), S#ch_mgr.name, Summ2]);
|
||||
_ ->
|
||||
if PrivWriteVerbCONFIRM,
|
||||
P_latest2#projection_v1.upi == [],
|
||||
(S#ch_mgr.proj)#projection_v1.upi /= [] ->
|
||||
<<CSumRep:4/binary,_/binary>> =
|
||||
P_latest2#projection_v1.epoch_csum,
|
||||
error_logger:info_msg("CONFIRM epoch ~w ~w upi ~w rep ~w auth ~w by ~w\n", [(S#ch_mgr.proj)#projection_v1.epoch_number, CSumRep, P_latest2#projection_v1.upi, P_latest2#projection_v1.repairing, P_latest2#projection_v1.author_server, S#ch_mgr.name]);
|
||||
true ->
|
||||
ok
|
||||
end;
|
||||
_ ->
|
||||
true ->
|
||||
ok
|
||||
end.
|
||||
|
||||
calc_consistency_mode(_Witness_list = []) ->
|
||||
ap_mode;
|
||||
calc_consistency_mode(_Witness_list) ->
|
||||
cp_mode.
|
||||
|
||||
set_proj(S, Proj) ->
|
||||
S#ch_mgr{proj=Proj, proj_unanimous=false}.
|
||||
|
||||
|
@ -2946,3 +3046,10 @@ get_unfit_list(FitnessServer) ->
|
|||
[]
|
||||
end.
|
||||
|
||||
get_projection_store_pid_or_regname(#ch_mgr{name=MyName, opts=MgrOpts}) ->
|
||||
case get_projection_store_regname(MgrOpts) of
|
||||
undefined ->
|
||||
machi_flu_psup:make_proj_supname(MyName);
|
||||
PStr ->
|
||||
PStr
|
||||
end.
|
||||
|
|
|
@ -96,11 +96,6 @@
|
|||
|
||||
-export([repair/7]).
|
||||
|
||||
repair_cp(_Src, _Dst, _MembersDict, _Opts) ->
|
||||
%% TODO: add missing function: wipe away any trace of chunks
|
||||
%% are present on Dst but missing on Src.
|
||||
exit(todo_cp_mode).
|
||||
|
||||
repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
|
||||
%% Use process dict so that 'after' clause can always quit all
|
||||
%% proxy pids.
|
||||
|
@ -108,12 +103,13 @@ repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
|
|||
Add = fun(Name, Pid) -> put(proxies_dict, orddict:store(Name, Pid, get(proxies_dict))) end,
|
||||
OurFLUs = lists:usort([Src] ++ Repairing ++ UPI), % AP assumption!
|
||||
RepairMode = proplists:get_value(repair_mode, Opts, repair),
|
||||
Verb = proplists:get_value(verbose, Opts, true),
|
||||
Verb = proplists:get_value(verbose, Opts, false),
|
||||
RepairId = proplists:get_value(repair_id, Opts, id1),
|
||||
Res = try
|
||||
[begin
|
||||
{ok, Proxy} = machi_proxy_flu1_client:start_link(P),
|
||||
Add(FLU, Proxy)
|
||||
end || {FLU,P} <- MembersDict, lists:member(FLU, OurFLUs)],
|
||||
_ = [begin
|
||||
{ok, Proxy} = machi_proxy_flu1_client:start_link(P),
|
||||
Add(FLU, Proxy)
|
||||
end || {FLU,P} <- MembersDict, lists:member(FLU, OurFLUs)],
|
||||
ProxiesDict = get(proxies_dict),
|
||||
|
||||
D = dict:new(),
|
||||
|
@ -121,31 +117,39 @@ repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
|
|||
get_file_lists(Proxy, FLU, Dict)
|
||||
end, D, ProxiesDict),
|
||||
MissingFileSummary = make_missing_file_summary(D2, OurFLUs),
|
||||
?VERB("MissingFileSummary ~p\n", [MissingFileSummary]),
|
||||
%% ?VERB("~w MissingFileSummary ~p\n",[RepairId,MissingFileSummary]),
|
||||
lager:info("Repair ~w MissingFileSummary ~p\n",
|
||||
[RepairId, MissingFileSummary]),
|
||||
|
||||
[ets:insert(ETS, {{directive_bytes, FLU}, 0}) || FLU <- OurFLUs],
|
||||
%% Repair files from perspective of Src, i.e. tail(UPI).
|
||||
SrcProxy = orddict:fetch(Src, ProxiesDict),
|
||||
{ok, EpochID} = machi_proxy_flu1_client:get_epoch_id(
|
||||
SrcProxy, ?SHORT_TIMEOUT),
|
||||
?VERB("Make repair directives: "),
|
||||
%% ?VERB("Make repair directives: "),
|
||||
Ds =
|
||||
[{File, make_repair_directives(
|
||||
ConsistencyMode, RepairMode, File, Size, EpochID,
|
||||
Verb,
|
||||
Src, OurFLUs, ProxiesDict, ETS)} ||
|
||||
{File, {Size, _MissingList}} <- MissingFileSummary],
|
||||
?VERB(" done\n"),
|
||||
%% ?VERB(" done\n"),
|
||||
lager:info("Repair ~w repair directives finished\n", [RepairId]),
|
||||
[begin
|
||||
[{_, Bytes}] = ets:lookup(ETS, {directive_bytes, FLU}),
|
||||
?VERB("Out-of-sync data for FLU ~p: ~s MBytes\n",
|
||||
[FLU, mbytes(Bytes)])
|
||||
%% ?VERB("Out-of-sync data for FLU ~p: ~s MBytes\n",
|
||||
%% [FLU, mbytes(Bytes)]),
|
||||
lager:info("Repair ~w "
|
||||
"Out-of-sync data for FLU ~p: ~s MBytes\n",
|
||||
[RepairId, FLU, mbytes(Bytes)]),
|
||||
ok
|
||||
end || FLU <- OurFLUs],
|
||||
|
||||
?VERB("Execute repair directives: "),
|
||||
%% ?VERB("Execute repair directives: "),
|
||||
ok = execute_repair_directives(ConsistencyMode, Ds, Src, EpochID,
|
||||
Verb, OurFLUs, ProxiesDict, ETS),
|
||||
?VERB(" done\n"),
|
||||
%% ?VERB(" done\n"),
|
||||
lager:info("Repair ~w repair directives finished\n", [RepairId]),
|
||||
ok
|
||||
catch
|
||||
What:Why ->
|
||||
|
@ -156,7 +160,7 @@ repair(ap_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
|
|||
Pid <- orddict:to_list(get(proxies_dict))]
|
||||
end,
|
||||
Res;
|
||||
repair(cp_mode=ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
|
||||
repair(cp_mode=_ConsistencyMode, Src, Repairing, UPI, MembersDict, ETS, Opts) ->
|
||||
io:format(user, "\n\nTODO! cp_mode repair is not fully implemented!\n\n", []),
|
||||
repair(ap_mode, Src, Repairing, UPI, MembersDict, ETS, Opts).
|
||||
|
||||
|
@ -203,7 +207,7 @@ make_repair_compare_fun(SrcFLU) ->
|
|||
T_a =< T_b
|
||||
end.
|
||||
|
||||
make_repair_directives(ConsistencyMode, RepairMode, File, Size, EpochID,
|
||||
make_repair_directives(ConsistencyMode, RepairMode, File, Size, _EpochID,
|
||||
Verb, Src, FLUs0, ProxiesDict, ETS) ->
|
||||
true = (Size < ?MAX_OFFSET),
|
||||
FLUs = lists:usort(FLUs0),
|
||||
|
@ -212,11 +216,9 @@ make_repair_directives(ConsistencyMode, RepairMode, File, Size, EpochID,
|
|||
Proxy = orddict:fetch(FLU, ProxiesDict),
|
||||
OffSzCs =
|
||||
case machi_proxy_flu1_client:checksum_list(
|
||||
Proxy, EpochID, File, ?LONG_TIMEOUT) of
|
||||
Proxy, File, ?LONG_TIMEOUT) of
|
||||
{ok, InfoBin} ->
|
||||
{Info, _} =
|
||||
machi_flu1:split_checksum_list_blob_decode(InfoBin),
|
||||
Info;
|
||||
machi_csum_table:split_checksum_list_blob_decode(InfoBin);
|
||||
{error, no_such_file} ->
|
||||
[]
|
||||
end,
|
||||
|
@ -234,7 +236,6 @@ make_repair_directives(ConsistencyMode, RepairMode, File, Size, EpochID,
|
|||
|
||||
make_repair_directives2(C2, ConsistencyMode, RepairMode,
|
||||
File, Verb, Src, FLUs, ProxiesDict, ETS) ->
|
||||
?VERB("."),
|
||||
make_repair_directives3(C2, ConsistencyMode, RepairMode,
|
||||
File, Verb, Src, FLUs, ProxiesDict, ETS, []).
|
||||
|
||||
|
@ -264,7 +265,18 @@ make_repair_directives3([{Offset, Size, CSum, _FLU}=A|Rest0],
|
|||
%% byte range from all FLUs
|
||||
%% 3b. Log big warning about data loss.
|
||||
%% 4. Log any other checksum discrepencies as they are found.
|
||||
exit({todo_repair_sanity_check, ?LINE, File, Offset, As})
|
||||
QQ = [begin
|
||||
Pxy = orddict:fetch(FLU, ProxiesDict),
|
||||
{ok, EpochID} = machi_proxy_flu1_client:get_epoch_id(
|
||||
Pxy, ?SHORT_TIMEOUT),
|
||||
NSInfo = undefined,
|
||||
XX = machi_proxy_flu1_client:read_chunk(
|
||||
Pxy, NSInfo, EpochID, File, Offset, Size, undefined,
|
||||
?SHORT_TIMEOUT),
|
||||
{FLU, XX}
|
||||
end || {__Offset, __Size, __CSum, FLU} <- As],
|
||||
|
||||
exit({todo_repair_sanity_check, ?LINE, File, Offset, {as,As}, {qq,QQ}})
|
||||
end,
|
||||
%% List construction guarantees us that there's at least one ?MAX_OFFSET
|
||||
%% item remains. Sort order + our "taking" of all exact Offset+Size
|
||||
|
@ -285,15 +297,16 @@ make_repair_directives3([{Offset, Size, CSum, _FLU}=A|Rest0],
|
|||
true -> Src;
|
||||
false -> hd(Gots)
|
||||
end,
|
||||
[ets:update_counter(ETS, {directive_bytes, FLU_m}, Size) ||
|
||||
FLU_m <- Missing],
|
||||
_ = [ets:update_counter(ETS, {directive_bytes, FLU_m}, Size) ||
|
||||
FLU_m <- Missing],
|
||||
if Missing == [] ->
|
||||
noop;
|
||||
true ->
|
||||
{copy, A, Missing}
|
||||
end;
|
||||
ConsistencyMode == cp_mode ->
|
||||
exit({todo_cp_mode, ?MODULE, ?LINE})
|
||||
end
|
||||
%% end;
|
||||
%% ConsistencyMode == cp_mode ->
|
||||
%% exit({todo_cp_mode, ?MODULE, ?LINE})
|
||||
end,
|
||||
Acc2 = if Do == noop -> Acc;
|
||||
true -> [Do|Acc]
|
||||
|
@ -316,38 +329,42 @@ execute_repair_directives(ap_mode=_ConsistencyMode, Ds, _Src, EpochID, Verb,
|
|||
{ProxiesDict, EpochID, Verb, ETS}, Ds),
|
||||
ok.
|
||||
|
||||
execute_repair_directive({File, Cmds}, {ProxiesDict, EpochID, Verb, ETS}=Acc) ->
|
||||
execute_repair_directive({File, Cmds}, {ProxiesDict, EpochID, _Verb, ETS}=Acc) ->
|
||||
EtsKeys = [{in_files, t_in_files}, {in_chunks, t_in_chunks},
|
||||
{in_bytes, t_in_bytes}, {out_files, t_out_files},
|
||||
{out_chunks, t_out_chunks}, {out_bytes, t_out_bytes}],
|
||||
[ets:insert(ETS, {L_K, 0}) || {L_K, _T_K} <- EtsKeys],
|
||||
F = fun({copy, {Offset, Size, TaggedCSum, MySrc}, MyDsts}, Acc2) ->
|
||||
SrcP = orddict:fetch(MySrc, ProxiesDict),
|
||||
case ets:lookup_element(ETS, in_chunks, 2) rem 100 of
|
||||
0 -> ?VERB(".", []);
|
||||
_ -> ok
|
||||
end,
|
||||
%% case ets:lookup_element(ETS, in_chunks, 2) rem 100 of
|
||||
%% 0 -> ?VERB(".2", []);
|
||||
%% _ -> ok
|
||||
%% end,
|
||||
_T1 = os:timestamp(),
|
||||
{ok, Chunk} = machi_proxy_flu1_client:read_chunk(
|
||||
SrcP, EpochID, File, Offset, Size,
|
||||
?SHORT_TIMEOUT),
|
||||
%% TODO: support case multiple written or trimmed chunks returned
|
||||
NSInfo = undefined,
|
||||
{ok, {[{_, Offset, Chunk, _ReadCSum}|OtherChunks], []=_TrimmedList}} =
|
||||
machi_proxy_flu1_client:read_chunk(
|
||||
SrcP, NSInfo, EpochID, File, Offset, Size, undefined,
|
||||
?SHORT_TIMEOUT),
|
||||
[] = OtherChunks,
|
||||
_T2 = os:timestamp(),
|
||||
<<_Tag:1/binary, CSum/binary>> = TaggedCSum,
|
||||
case machi_util:checksum_chunk(Chunk) of
|
||||
CSum_now when CSum_now == CSum ->
|
||||
[begin
|
||||
DstP = orddict:fetch(DstFLU, ProxiesDict),
|
||||
_T3 = os:timestamp(),
|
||||
ok = machi_proxy_flu1_client:write_chunk(
|
||||
DstP, EpochID, File, Offset, Chunk,
|
||||
?SHORT_TIMEOUT),
|
||||
_T4 = os:timestamp()
|
||||
end || DstFLU <- MyDsts],
|
||||
ets:update_counter(ETS, in_chunks, 1),
|
||||
ets:update_counter(ETS, in_bytes, Size),
|
||||
_ = [begin
|
||||
DstP = orddict:fetch(DstFLU, ProxiesDict),
|
||||
_T3 = os:timestamp(),
|
||||
ok = machi_proxy_flu1_client:write_chunk(
|
||||
DstP, NSInfo, EpochID, File, Offset, Chunk, TaggedCSum,
|
||||
?SHORT_TIMEOUT),
|
||||
_T4 = os:timestamp()
|
||||
end || DstFLU <- MyDsts],
|
||||
_ = ets:update_counter(ETS, in_chunks, 1),
|
||||
_ = ets:update_counter(ETS, in_bytes, Size),
|
||||
N = length(MyDsts),
|
||||
ets:update_counter(ETS, out_chunks, N),
|
||||
ets:update_counter(ETS, out_bytes, N*Size),
|
||||
_ = ets:update_counter(ETS, out_chunks, N),
|
||||
_ = ets:update_counter(ETS, out_bytes, N*Size),
|
||||
Acc2;
|
||||
CSum_now ->
|
||||
error_logger:error_msg(
|
||||
|
@ -366,8 +383,8 @@ execute_repair_directive({File, Cmds}, {ProxiesDict, EpochID, Verb, ETS}=Acc) ->
|
|||
end,
|
||||
ok = lists:foldl(F, ok, Cmds),
|
||||
%% Copy this file's stats to the total counts.
|
||||
[ets:update_counter(ETS, T_K, ets:lookup_element(ETS, L_K, 2)) ||
|
||||
{L_K, T_K} <- EtsKeys],
|
||||
_ = [ets:update_counter(ETS, T_K, ets:lookup_element(ETS, L_K, 2)) ||
|
||||
{L_K, T_K} <- EtsKeys],
|
||||
Acc.
|
||||
|
||||
mbytes(N) ->
|
||||
|
|
104
src/machi_cinfo.erl
Normal file
104
src/machi_cinfo.erl
Normal file
|
@ -0,0 +1,104 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc cluster_info callback module for machi specific information
|
||||
%% gathering.
|
||||
|
||||
-module(machi_cinfo).
|
||||
|
||||
%% cluster_info callbacks
|
||||
-export([register/0, cluster_info_init/0, cluster_info_generator_funs/0]).
|
||||
|
||||
%% for debug in interactive shell
|
||||
-export([dump/0,
|
||||
public_projection/1, private_projection/1,
|
||||
chain_manager/1, fitness/1, flu1/1]).
|
||||
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-spec register() -> ok.
|
||||
register() ->
|
||||
ok = cluster_info:register_app(?MODULE).
|
||||
|
||||
-spec cluster_info_init() -> ok.
|
||||
cluster_info_init() ->
|
||||
ok.
|
||||
|
||||
-spec cluster_info_generator_funs() -> [{string(), fun((pid()) -> ok)}].
|
||||
cluster_info_generator_funs() ->
|
||||
FluNames = [Name || {Name, _, _, _} <- supervisor:which_children(machi_flu_sup)],
|
||||
lists:flatten([generator_funs_package(Name) || Name <- FluNames]).
|
||||
|
||||
generator_funs_package(FluName) ->
|
||||
[{"Public projection of FLU " ++ atom_to_list(FluName),
|
||||
cinfo_wrapper(fun public_projection/1, FluName)},
|
||||
{"Private projection of FLU " ++ atom_to_list(FluName),
|
||||
cinfo_wrapper(fun private_projection/1, FluName)},
|
||||
{"Chain manager status of FLU " ++ atom_to_list(FluName),
|
||||
cinfo_wrapper(fun chain_manager/1, FluName)},
|
||||
{"Fitness server status of FLU " ++ atom_to_list(FluName),
|
||||
cinfo_wrapper(fun fitness/1, FluName)},
|
||||
{"FLU1 status of FLU " ++ atom_to_list(FluName),
|
||||
cinfo_wrapper(fun flu1/1, FluName)}].
|
||||
|
||||
dump() ->
|
||||
{{Y,M,D},{HH,MM,SS}} = calendar:local_time(),
|
||||
Filename = lists:flatten(io_lib:format(
|
||||
"machi-ci-~4..0B~2..0B~2..0B-~2..0B~2..0B~2..0B.html",
|
||||
[Y,M,D,HH,MM,SS])),
|
||||
cluster_info:dump_local_node(Filename).
|
||||
|
||||
-spec public_projection(atom()) -> [{atom(), term()}].
|
||||
public_projection(FluName) ->
|
||||
projection(FluName, public).
|
||||
|
||||
-spec private_projection(atom()) -> [{atom(), term()}].
|
||||
private_projection(FluName) ->
|
||||
projection(FluName, private).
|
||||
|
||||
-spec chain_manager(atom()) -> term().
|
||||
chain_manager(FluName) ->
|
||||
Mgr = machi_flu_psup:make_mgr_supname(FluName),
|
||||
sys:get_status(Mgr).
|
||||
|
||||
-spec fitness(atom()) -> term().
|
||||
fitness(FluName) ->
|
||||
Fitness = machi_flu_psup:make_fitness_regname(FluName),
|
||||
sys:get_status(Fitness).
|
||||
|
||||
-spec flu1(atom()) -> [{atom(), term()}].
|
||||
flu1(FluName) ->
|
||||
State = machi_flu1_append_server:current_state(FluName),
|
||||
machi_flu1_append_server:format_state(State).
|
||||
|
||||
%% Internal functions
|
||||
|
||||
projection(FluName, Kind) ->
|
||||
ProjStore = machi_flu1:make_projection_server_regname(FluName),
|
||||
{ok, Projection} = machi_projection_store:read_latest_projection(
|
||||
whereis(ProjStore), Kind),
|
||||
Fields = record_info(fields, projection_v1),
|
||||
[_Name | Values] = tuple_to_list(Projection),
|
||||
lists:zip(Fields, Values).
|
||||
|
||||
cinfo_wrapper(Fun, FluName) ->
|
||||
fun(C) ->
|
||||
cluster_info:format(C, "~p", [Fun(FluName)])
|
||||
end.
|
43
src/machi_config.erl
Normal file
43
src/machi_config.erl
Normal file
|
@ -0,0 +1,43 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc Configuration consulting utilities. Some conventions:
|
||||
%% - The function name should match with exact configuration
|
||||
%% name in `app.config' or `advanced.config' of `machi' section.
|
||||
%% - The default value of that configuration is expected to be in
|
||||
%% cuttlefish schema file. Otherwise some macro in headers may
|
||||
%% be chosen.
|
||||
%% - Documentation of the configuration is supposed to be written
|
||||
%% in cuttlefish schema file, rather than @doc section of the function.
|
||||
%% - spec of the function should be written.
|
||||
%% - Returning `undefined' is strongly discouraged. Return some default
|
||||
%% value instead.
|
||||
%% - `application:get_env/3' is recommended. See `max_file_size/0' for
|
||||
%% example.
|
||||
|
||||
-module(machi_config).
|
||||
|
||||
-include("machi.hrl").
|
||||
|
||||
-export([max_file_size/0]).
|
||||
|
||||
-spec max_file_size() -> pos_integer().
|
||||
max_file_size() ->
|
||||
application:get_env(machi, max_file_size, ?DEFAULT_MAX_FILE_SIZE).
|
File diff suppressed because it is too large
Load diff
329
src/machi_csum_table.erl
Normal file
329
src/machi_csum_table.erl
Normal file
|
@ -0,0 +1,329 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_csum_table).
|
||||
|
||||
-export([open/2,
|
||||
find/3,
|
||||
write/6, write/4, trim/5,
|
||||
find_leftneighbor/2, find_rightneighbor/2,
|
||||
all_trimmed/3, any_trimmed/3,
|
||||
all_trimmed/2,
|
||||
calc_unwritten_bytes/1,
|
||||
split_checksum_list_blob_decode/1,
|
||||
all/1,
|
||||
close/1, delete/1,
|
||||
foldl_chunks/3]).
|
||||
|
||||
-include("machi.hrl").
|
||||
|
||||
-ifdef(TEST).
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-endif.
|
||||
|
||||
-record(machi_csum_table,
|
||||
{file :: string(),
|
||||
table :: eleveldb:db_ref()}).
|
||||
|
||||
-type table() :: #machi_csum_table{}.
|
||||
-type byte_sequence() :: { Offset :: non_neg_integer(),
|
||||
Size :: pos_integer()|infinity }.
|
||||
-type chunk() :: {Offset :: machi_dt:file_offset(),
|
||||
Size :: machi_dt:chunk_size(),
|
||||
machi_dt:chunk_csum() | trimmed | none}.
|
||||
|
||||
-export_type([table/0]).
|
||||
|
||||
-spec open(string(), proplists:proplist()) ->
|
||||
{ok, table()} | {error, file:posix()}.
|
||||
|
||||
open(CSumFilename, _Opts) ->
|
||||
LevelDBOptions = [{create_if_missing, true},
|
||||
%% Keep this table small so as not to interfere
|
||||
%% operating system's file cache, which is for
|
||||
%% Machi's main read efficiency
|
||||
{total_leveldb_mem_percent, 10}],
|
||||
{ok, T} = eleveldb:open(CSumFilename, LevelDBOptions),
|
||||
%% Dummy entry for reserved headers
|
||||
ok = eleveldb:put(T,
|
||||
sext:encode({0, ?MINIMUM_OFFSET}),
|
||||
sext:encode(?CSUM_TAG_NONE_ATOM),
|
||||
[{sync, true}]),
|
||||
C0 = #machi_csum_table{
|
||||
file=CSumFilename,
|
||||
table=T},
|
||||
{ok, C0}.
|
||||
|
||||
-spec split_checksum_list_blob_decode(binary())-> [chunk()].
|
||||
split_checksum_list_blob_decode(Bin) ->
|
||||
erlang:binary_to_term(Bin).
|
||||
|
||||
|
||||
-define(has_overlap(LeftOffset, LeftSize, RightOffset, RightSize),
|
||||
((LeftOffset - (RightOffset+RightSize)) * (LeftOffset+LeftSize - RightOffset) < 0)).
|
||||
|
||||
-spec find(table(), machi_dt:file_offset(), machi_dt:chunk_size())
|
||||
-> [chunk()].
|
||||
find(#machi_csum_table{table=T}, Offset, Size) ->
|
||||
{ok, I} = eleveldb:iterator(T, [], keys_only),
|
||||
EndKey = sext:encode({Offset+Size, 0}),
|
||||
StartKey = sext:encode({Offset, Size}),
|
||||
{ok, FirstKey} = case eleveldb:iterator_move(I, StartKey) of
|
||||
{error, invalid_iterator} ->
|
||||
try
|
||||
%% Assume that the invalid_iterator is because
|
||||
%% we tried to move to the end via StartKey.
|
||||
%% Instead, move there directly.
|
||||
{ok, _} = eleveldb:iterator_move(I, last),
|
||||
{ok, _} = eleveldb:iterator_move(I, prev)
|
||||
catch
|
||||
_:_ ->
|
||||
{ok, _} = eleveldb:iterator_move(I, first)
|
||||
end;
|
||||
{ok, _} = R0 ->
|
||||
case eleveldb:iterator_move(I, prev) of
|
||||
{error, invalid_iterator} ->
|
||||
R0;
|
||||
{ok, _} = R1 ->
|
||||
R1
|
||||
end
|
||||
end,
|
||||
_ = eleveldb:iterator_close(I),
|
||||
FoldFun = fun({K, V}, Acc) ->
|
||||
{TargetOffset, TargetSize} = sext:decode(K),
|
||||
case ?has_overlap(TargetOffset, TargetSize, Offset, Size) of
|
||||
true ->
|
||||
[{TargetOffset, TargetSize, sext:decode(V)}|Acc];
|
||||
false ->
|
||||
Acc
|
||||
end;
|
||||
(_K, Acc) ->
|
||||
lager:error("~p wrong option", [_K]),
|
||||
Acc
|
||||
end,
|
||||
lists:reverse(eleveldb_fold(T, FirstKey, EndKey, FoldFun, [])).
|
||||
|
||||
%% @doc Updates all chunk info, by deleting existing entries if exists
|
||||
%% and putting new chunk info
|
||||
-spec write(table(),
|
||||
machi_dt:file_offset(), machi_dt:chunk_size(),
|
||||
machi_dt:chunk_csum()|'none'|'trimmed',
|
||||
undefined|chunk(), undefined|chunk()) ->
|
||||
ok | {error, term()}.
|
||||
write(#machi_csum_table{table=T} = CsumT, Offset, Size, CSum,
|
||||
LeftUpdate, RightUpdate) ->
|
||||
PutOps =
|
||||
[{put,
|
||||
sext:encode({Offset, Size}),
|
||||
sext:encode(CSum)}]
|
||||
++ case LeftUpdate of
|
||||
{LO, LS, LCsum} when LO + LS =:= Offset ->
|
||||
[{put,
|
||||
sext:encode({LO, LS}),
|
||||
sext:encode(LCsum)}];
|
||||
undefined ->
|
||||
[]
|
||||
end
|
||||
++ case RightUpdate of
|
||||
{RO, RS, RCsum} when RO =:= Offset + Size ->
|
||||
[{put,
|
||||
sext:encode({RO, RS}),
|
||||
sext:encode(RCsum)}];
|
||||
undefined ->
|
||||
[]
|
||||
end,
|
||||
Chunks = find(CsumT, Offset, Size),
|
||||
DeleteOps = lists:map(fun({O, L, _}) ->
|
||||
{delete, sext:encode({O, L})}
|
||||
end, Chunks),
|
||||
%% io:format(user, "PutOps: ~P\n", [PutOps, 20]),
|
||||
%% io:format(user, "DelOps: ~P\n", [DeleteOps, 20]),
|
||||
eleveldb:write(T, DeleteOps ++ PutOps, [{sync, true}]).
|
||||
|
||||
-spec find_leftneighbor(table(), non_neg_integer()) ->
|
||||
undefined | chunk().
|
||||
find_leftneighbor(CsumT, Offset) ->
|
||||
case find(CsumT, Offset, 1) of
|
||||
[] -> undefined;
|
||||
[{Offset, _, _}] -> undefined;
|
||||
[{LOffset, _, CsumOrTrimmed}] -> {LOffset, Offset - LOffset, CsumOrTrimmed}
|
||||
end.
|
||||
|
||||
-spec find_rightneighbor(table(), non_neg_integer()) ->
|
||||
undefined | chunk().
|
||||
find_rightneighbor(CsumT, Offset) ->
|
||||
case find(CsumT, Offset, 1) of
|
||||
[] -> undefined;
|
||||
[{Offset, _, _}] -> undefined;
|
||||
[{ROffset, RSize, CsumOrTrimmed}] ->
|
||||
{Offset, ROffset + RSize - Offset, CsumOrTrimmed}
|
||||
end.
|
||||
|
||||
-spec write(table(), machi_dt:file_offset(), machi_dt:file_size(),
|
||||
machi_dt:chunk_csum()|none|trimmed) ->
|
||||
ok | {error, trimmed|file:posix()}.
|
||||
write(CsumT, Offset, Size, CSum) ->
|
||||
write(CsumT, Offset, Size, CSum, undefined, undefined).
|
||||
|
||||
trim(CsumT, Offset, Size, LeftUpdate, RightUpdate) ->
|
||||
write(CsumT, Offset, Size,
|
||||
trimmed, %% Should this be much smaller like $t or just 't'
|
||||
LeftUpdate, RightUpdate).
|
||||
|
||||
%% @doc returns whether all bytes in a specific window is continously
|
||||
%% trimmed or not
|
||||
-spec all_trimmed(table(), non_neg_integer(), non_neg_integer()) -> boolean().
|
||||
all_trimmed(#machi_csum_table{table=T}, Left, Right) ->
|
||||
FoldFun = fun({_, _}, false) ->
|
||||
false;
|
||||
({K, V}, Pos) when is_integer(Pos) andalso Pos =< Right ->
|
||||
case {sext:decode(K), sext:decode(V)} of
|
||||
{{Pos, Size}, trimmed} ->
|
||||
Pos + Size;
|
||||
{{Offset, Size}, _}
|
||||
when Offset + Size =< Left ->
|
||||
Left;
|
||||
_Eh ->
|
||||
false
|
||||
end
|
||||
end,
|
||||
case eleveldb:fold(T, FoldFun, Left, [{verify_checksums, true}]) of
|
||||
false -> false;
|
||||
Right -> true;
|
||||
LastTrimmed when LastTrimmed < Right -> false;
|
||||
_ -> %% LastTrimmed > Pos0, which is a irregular case but ok
|
||||
true
|
||||
end.
|
||||
|
||||
%% @doc returns whether all bytes 0-Pos0 is continously trimmed or
|
||||
%% not, including header.
|
||||
-spec all_trimmed(table(), non_neg_integer()) -> boolean().
|
||||
all_trimmed(CsumT, Pos0) ->
|
||||
all_trimmed(CsumT, 0, Pos0).
|
||||
|
||||
-spec any_trimmed(table(),
|
||||
pos_integer(),
|
||||
machi_dt:chunk_size()) -> boolean().
|
||||
any_trimmed(CsumT, Offset, Size) ->
|
||||
Chunks = find(CsumT, Offset, Size),
|
||||
lists:any(fun({_, _, State}) -> State =:= trimmed end, Chunks).
|
||||
|
||||
-spec calc_unwritten_bytes(table()) -> [byte_sequence()].
|
||||
calc_unwritten_bytes(#machi_csum_table{table=_} = CsumT) ->
|
||||
case lists:sort(all(CsumT)) of
|
||||
[] ->
|
||||
[{?MINIMUM_OFFSET, infinity}];
|
||||
Sorted ->
|
||||
{LastOffset, _, _} = hd(Sorted),
|
||||
build_unwritten_bytes_list(Sorted, LastOffset, [])
|
||||
end.
|
||||
|
||||
all(CsumT) ->
|
||||
FoldFun = fun(E, Acc) -> [E|Acc] end,
|
||||
lists:reverse(foldl_chunks(FoldFun, [], CsumT)).
|
||||
|
||||
-spec close(table()) -> ok.
|
||||
close(#machi_csum_table{table=T}) ->
|
||||
ok = eleveldb:close(T).
|
||||
|
||||
-spec delete(table()) -> ok.
|
||||
delete(#machi_csum_table{table=T, file=F}) ->
|
||||
catch eleveldb:close(T),
|
||||
%% TODO change this to directory walk
|
||||
case os:cmd("rm -rf " ++ F) of
|
||||
"" -> ok;
|
||||
E -> E
|
||||
end.
|
||||
|
||||
-spec foldl_chunks(fun((chunk(), Acc0 :: term()) -> Acc :: term()),
|
||||
Acc0 :: term(), table()) -> Acc :: term().
|
||||
foldl_chunks(Fun, Acc0, #machi_csum_table{table=T}) ->
|
||||
FoldFun = fun({K, V}, Acc) ->
|
||||
{Offset, Len} = sext:decode(K),
|
||||
Fun({Offset, Len, sext:decode(V)}, Acc);
|
||||
(_K, Acc) ->
|
||||
_ = lager:error("~p: wrong option?", [_K]),
|
||||
Acc
|
||||
end,
|
||||
eleveldb:fold(T, FoldFun, Acc0, [{verify_checksums, true}]).
|
||||
|
||||
-spec build_unwritten_bytes_list( CsumData :: [{ Offset :: non_neg_integer(),
|
||||
Size :: pos_integer(),
|
||||
Checksum :: binary() }],
|
||||
LastOffset :: non_neg_integer(),
|
||||
Acc :: list() ) -> [byte_sequence()].
|
||||
% @private Given a <b>sorted</b> list of checksum data tuples, return a sorted
|
||||
% list of unwritten byte ranges. The output list <b>always</b> has at least one
|
||||
% entry: the last tuple in the list is guaranteed to be the current end of
|
||||
% bytes written to a particular file with the special space moniker
|
||||
% `infinity'.
|
||||
build_unwritten_bytes_list([], Last, Acc) ->
|
||||
NewAcc = [ {Last, infinity} | Acc ],
|
||||
lists:reverse(NewAcc);
|
||||
build_unwritten_bytes_list([{CurrentOffset, CurrentSize, _Csum}|Rest], LastOffset, Acc) when
|
||||
CurrentOffset /= LastOffset ->
|
||||
Hole = CurrentOffset - LastOffset,
|
||||
build_unwritten_bytes_list(Rest, (CurrentOffset+CurrentSize), [{LastOffset, Hole}|Acc]);
|
||||
build_unwritten_bytes_list([{CO, CS, _Ck}|Rest], _LastOffset, Acc) ->
|
||||
build_unwritten_bytes_list(Rest, CO + CS, Acc).
|
||||
|
||||
%% @doc If you want to find an overlap among two areas [x, y] and [a,
|
||||
%% b] where x < y and a < b; if (a-y)*(b-x) < 0 then there's a
|
||||
%% overlap, else, > 0 then there're no overlap. border condition = 0
|
||||
%% is not overlap in this offset-size case.
|
||||
%% inclusion_match_spec(Offset, Size) ->
|
||||
%% {'>', 0,
|
||||
%% {'*',
|
||||
%% {'-', Offset + Size, '$1'},
|
||||
%% {'-', Offset, {'+', '$1', '$2'}}}}.
|
||||
|
||||
-spec eleveldb_fold(eleveldb:db_ref(), binary(), binary(),
|
||||
fun(({binary(), binary()}, AccType::term()) -> AccType::term()),
|
||||
AccType0::term()) ->
|
||||
AccType::term().
|
||||
eleveldb_fold(Ref, Start, End, FoldFun, InitAcc) ->
|
||||
{ok, Iterator} = eleveldb:iterator(Ref, []),
|
||||
try
|
||||
eleveldb_do_fold(eleveldb:iterator_move(Iterator, Start),
|
||||
Iterator, End, FoldFun, InitAcc)
|
||||
catch throw:IteratorClosed ->
|
||||
{error, IteratorClosed}
|
||||
after
|
||||
eleveldb:iterator_close(Iterator)
|
||||
end.
|
||||
|
||||
-spec eleveldb_do_fold({ok, binary(), binary()}|{error, iterator_closed|invalid_iterator}|{ok,binary()},
|
||||
eleveldb:itr_ref(), binary(),
|
||||
fun(({binary(), binary()}, AccType::term()) -> AccType::term()),
|
||||
AccType::term()) ->
|
||||
AccType::term().
|
||||
eleveldb_do_fold({ok, Key, Value}, _, End, FoldFun, Acc)
|
||||
when End < Key ->
|
||||
FoldFun({Key, Value}, Acc);
|
||||
eleveldb_do_fold({ok, Key, Value}, Iterator, End, FoldFun, Acc) ->
|
||||
eleveldb_do_fold(eleveldb:iterator_move(Iterator, next),
|
||||
Iterator, End, FoldFun,
|
||||
FoldFun({Key, Value}, Acc));
|
||||
eleveldb_do_fold({error, iterator_closed}, _, _, _, Acc) ->
|
||||
%% It's really an error which is not expected
|
||||
throw({iterator_closed, Acc});
|
||||
eleveldb_do_fold({error, invalid_iterator}, _, _, _, Acc) ->
|
||||
%% Probably reached to end
|
||||
Acc.
|
|
@ -20,15 +20,24 @@
|
|||
|
||||
-module(machi_dt).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-type chunk() :: chunk_bin() | {chunk_csum(), chunk_bin()}.
|
||||
-type chunk_bin() :: binary() | iolist(). % client can use either
|
||||
-type chunk_csum() :: binary(). % 1 byte tag, N-1 bytes checksum
|
||||
-type chunk_summary() :: {file_offset(), chunk_size(), binary()}.
|
||||
-type chunk_s() :: binary(). % server always uses binary()
|
||||
-type append_opts() :: #append_opts{}.
|
||||
-type chunk() :: chunk_bin() | iolist(). % client can choose either rep.
|
||||
-type chunk_bin() :: binary(). % server returns binary() only.
|
||||
-type chunk_csum() :: <<>> | chunk_csum_bin() | {csum_tag(), binary()}.
|
||||
-type chunk_csum_bin() :: binary(). % 1 byte tag, N-1 bytes checksum
|
||||
-type chunk_cstrm() :: 'trimmed' | chunk_csum().
|
||||
-type chunk_summary() :: {file_offset(), chunk_size(), chunk_bin(), chunk_cstrm()}.
|
||||
-type chunk_pos() :: {file_offset(), chunk_size(), file_name_s()}.
|
||||
-type chunk_size() :: non_neg_integer().
|
||||
|
||||
%% Tags that stand for how that checksum was generated. See
|
||||
%% machi_util:make_tagged_csum/{1,2} for further documentation and
|
||||
%% implementation.
|
||||
-type csum_tag() :: none | client_sha | server_sha | server_regen_sha.
|
||||
|
||||
-type error_general() :: 'bad_arg' | 'wedged' | 'bad_checksum'.
|
||||
-type epoch_csum() :: binary().
|
||||
-type epoch_num() :: -1 | non_neg_integer().
|
||||
|
@ -41,17 +50,26 @@
|
|||
-type file_prefix() :: binary() | list().
|
||||
-type inet_host() :: inet:ip_address() | inet:hostname().
|
||||
-type inet_port() :: inet:port_number().
|
||||
-type locator() :: number().
|
||||
-type namespace() :: binary().
|
||||
-type namespace_version() :: non_neg_integer().
|
||||
-type ns_info() :: #ns_info{}.
|
||||
-type projection() :: #projection_v1{}.
|
||||
-type projection_type() :: 'public' | 'private'.
|
||||
-type read_opts() :: #read_opts{}.
|
||||
-type read_opts_x() :: 'undefined' | 'noopt' | 'none' | #read_opts{}.
|
||||
|
||||
-export_type([
|
||||
append_opts/0,
|
||||
chunk/0,
|
||||
chunk_bin/0,
|
||||
chunk_csum/0,
|
||||
chunk_csum_bin/0,
|
||||
chunk_cstrm/0,
|
||||
chunk_summary/0,
|
||||
chunk_s/0,
|
||||
chunk_pos/0,
|
||||
chunk_size/0,
|
||||
csum_tag/0,
|
||||
error_general/0,
|
||||
epoch_csum/0,
|
||||
epoch_num/0,
|
||||
|
@ -64,7 +82,13 @@
|
|||
file_prefix/0,
|
||||
inet_host/0,
|
||||
inet_port/0,
|
||||
locator/0,
|
||||
namespace/0,
|
||||
namespace_version/0,
|
||||
ns_info/0,
|
||||
projection/0,
|
||||
projection_type/0
|
||||
projection_type/0,
|
||||
read_opts/0,
|
||||
read_opts_x/0
|
||||
]).
|
||||
|
||||
|
|
908
src/machi_file_proxy.erl
Normal file
908
src/machi_file_proxy.erl
Normal file
|
@ -0,0 +1,908 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc This is a proxy process which mediates access to Machi FLU
|
||||
%% controlled files. In particular, it manages the "write-once register"
|
||||
%% conceit at the heart of Machi's design.
|
||||
%%
|
||||
%% Read, write and append requests for a single file will be managed
|
||||
%% through this proxy. Clients can also request syncs for specific
|
||||
%% types of filehandles.
|
||||
%%
|
||||
%% As operations are requested, the proxy keeps track of how many
|
||||
%% operations it has performed (and how many errors were generated.)
|
||||
%% After a sufficient number of inactivity, the server terminates
|
||||
%% itself.
|
||||
%%
|
||||
%% TODO:
|
||||
%% 1. Some way to transition the proxy into a wedged state that
|
||||
%% doesn't rely on message delivery.
|
||||
%%
|
||||
%% 2. Check max file size on appends. Writes we take on faith we can
|
||||
%% and should handle.
|
||||
%%
|
||||
%% 3. Async checksum reads on startup.
|
||||
|
||||
-module(machi_file_proxy).
|
||||
-behaviour(gen_server).
|
||||
|
||||
-include("machi.hrl").
|
||||
|
||||
%% public API
|
||||
-export([
|
||||
start_link/3,
|
||||
stop/1,
|
||||
sync/1,
|
||||
sync/2,
|
||||
read/3,
|
||||
read/4,
|
||||
write/3,
|
||||
write/4,
|
||||
trim/4,
|
||||
append/2,
|
||||
append/4,
|
||||
checksum_list/1
|
||||
]).
|
||||
|
||||
%% gen_server callbacks
|
||||
-export([
|
||||
init/1,
|
||||
handle_call/3,
|
||||
handle_cast/2,
|
||||
handle_info/2,
|
||||
terminate/2,
|
||||
code_change/3
|
||||
]).
|
||||
|
||||
-define(TICK, 5*1000).
|
||||
-define(TICK_THRESHOLD, 5). %% After this + 1 more quiescent ticks, shutdown
|
||||
-define(TIMEOUT, 10*1000).
|
||||
-define(TOO_MANY_ERRORS_RATIO, 50).
|
||||
|
||||
-type op_stats() :: { Total :: non_neg_integer(),
|
||||
Errors :: non_neg_integer() }.
|
||||
|
||||
-record(state, {
|
||||
fluname :: atom(),
|
||||
data_dir :: string() | undefined,
|
||||
filename :: string() | undefined,
|
||||
data_path :: string() | undefined,
|
||||
wedged = false :: boolean(),
|
||||
csum_file :: string()|undefined,
|
||||
csum_path :: string()|undefined,
|
||||
data_filehandle :: file:io_device(),
|
||||
csum_table :: machi_csum_table:table(),
|
||||
eof_position = 0 :: non_neg_integer(),
|
||||
max_file_size = ?DEFAULT_MAX_FILE_SIZE :: pos_integer(),
|
||||
rollover = false :: boolean(),
|
||||
tref :: reference(), %% timer ref
|
||||
ticks = 0 :: non_neg_integer(), %% ticks elapsed with no new operations
|
||||
ops = 0 :: non_neg_integer(), %% sum of all ops
|
||||
reads = {0, 0} :: op_stats(),
|
||||
writes = {0, 0} :: op_stats(),
|
||||
appends = {0, 0} :: op_stats(),
|
||||
trims = {0, 0} :: op_stats()
|
||||
}).
|
||||
|
||||
%% Public API
|
||||
|
||||
% @doc Start a new instance of the file proxy service. Takes the filename
|
||||
% and data directory as arguments. This function is typically called by the
|
||||
% `machi_file_proxy_sup:start_proxy/2' function.
|
||||
-spec start_link(FluName :: atom(), Filename :: string(), DataDir :: string()) -> any().
|
||||
start_link(FluName, Filename, DataDir) ->
|
||||
gen_server:start_link(?MODULE, {FluName, Filename, DataDir}, []).
|
||||
|
||||
% @doc Request to stop an instance of the file proxy service.
|
||||
-spec stop(Pid :: pid()) -> ok.
|
||||
stop(Pid) when is_pid(Pid) ->
|
||||
gen_server:call(Pid, {stop}, ?TIMEOUT).
|
||||
|
||||
% @doc Force a sync of all filehandles
|
||||
-spec sync(Pid :: pid()) -> ok|{error, term()}.
|
||||
sync(Pid) when is_pid(Pid) ->
|
||||
sync(Pid, all);
|
||||
sync(_Pid) ->
|
||||
lager:warning("Bad pid to sync"),
|
||||
{error, bad_arg}.
|
||||
|
||||
% @doc Force a sync of a specific filehandle type. Valid types are `all', `csum' and `data'.
|
||||
-spec sync(Pid :: pid(), Type :: all|data|csum) -> ok|{error, term()}.
|
||||
sync(Pid, Type) when is_pid(Pid) andalso
|
||||
( Type =:= all orelse Type =:= csum orelse Type =:= data ) ->
|
||||
gen_server:call(Pid, {sync, Type}, ?TIMEOUT);
|
||||
sync(_Pid, Type) ->
|
||||
lager:warning("Bad arg to sync: Type ~p", [Type]),
|
||||
{error, bad_arg}.
|
||||
|
||||
% @doc Read file at offset for length. This returns a sequence of all
|
||||
% written and trimmed (optional) bytes that overlaps with requested
|
||||
% offset and length. Borders are not aligned.
|
||||
-spec read(Pid :: pid(),
|
||||
Offset :: non_neg_integer(),
|
||||
Length :: non_neg_integer()) ->
|
||||
{ok, [{Filename::string(), Offset :: non_neg_integer(),
|
||||
Data :: binary(), Checksum :: binary()}]} |
|
||||
{error, Reason :: term()}.
|
||||
read(Pid, Offset, Length) ->
|
||||
read(Pid, Offset, Length, #read_opts{}).
|
||||
|
||||
-spec read(Pid :: pid(),
|
||||
Offset :: non_neg_integer(),
|
||||
Length :: non_neg_integer(),
|
||||
machi_dt:read_opts_x()) ->
|
||||
{ok, [{Filename::string(), Offset :: non_neg_integer(),
|
||||
Data :: binary(), Checksum :: binary()}]} |
|
||||
{error, Reason :: term()}.
|
||||
read(Pid, Offset, Length, #read_opts{}=Opts)
|
||||
when is_pid(Pid) andalso is_integer(Offset) andalso Offset >= 0
|
||||
andalso is_integer(Length) andalso Length > 0 ->
|
||||
gen_server:call(Pid, {read, Offset, Length, Opts}, ?TIMEOUT);
|
||||
read(_Pid, Offset, Length, Opts) ->
|
||||
lager:warning("Bad args to read: Offset ~p, Length ~p, Options ~p", [Offset, Length, Opts]),
|
||||
{error, bad_arg}.
|
||||
|
||||
% @doc Write data at offset
|
||||
-spec write(Pid :: pid(), Offset :: non_neg_integer(), Data :: binary()) -> ok|{error, term()}.
|
||||
write(Pid, Offset, Data) when is_pid(Pid) andalso is_integer(Offset) andalso Offset >= 0
|
||||
andalso is_binary(Data) ->
|
||||
write(Pid, Offset, [], Data);
|
||||
write(_Pid, Offset, _Data) ->
|
||||
lager:warning("Bad arg to write: Offset ~p", [Offset]),
|
||||
{error, bad_arg}.
|
||||
|
||||
% @doc Write data at offset, including the client metadata. ClientMeta is a proplist
|
||||
% that expects the following keys and values:
|
||||
% <ul>
|
||||
% <li>`client_csum_tag' - the type of checksum from the client as defined in the machi.hrl file</li>
|
||||
% <li>`client_csum' - the checksum value from the client</li>
|
||||
% </ul>
|
||||
-spec write(Pid :: pid(), Offset :: non_neg_integer(), ClientMeta :: proplists:proplist(),
|
||||
Data :: binary()) -> ok|{error, term()}.
|
||||
write(Pid, Offset, ClientMeta, Data) when is_pid(Pid) andalso is_integer(Offset) andalso Offset >= 0
|
||||
andalso is_list(ClientMeta) andalso is_binary(Data) ->
|
||||
gen_server:call(Pid, {write, Offset, ClientMeta, Data}, ?TIMEOUT);
|
||||
write(_Pid, Offset, ClientMeta, _Data) ->
|
||||
lager:warning("Bad arg to write: Offset ~p, ClientMeta: ~p", [Offset, ClientMeta]),
|
||||
{error, bad_arg}.
|
||||
|
||||
trim(Pid, Offset, Size, TriggerGC) when is_pid(Pid),
|
||||
is_integer(Offset) andalso Offset >= 0,
|
||||
is_integer(Size) andalso Size > 0,
|
||||
is_boolean(TriggerGC) ->
|
||||
gen_server:call(Pid, {trim ,Offset, Size, TriggerGC}, ?TIMEOUT).
|
||||
|
||||
% @doc Append data
|
||||
-spec append(Pid :: pid(), Data :: binary()) -> {ok, File :: string(), Offset :: non_neg_integer()}
|
||||
|{error, term()}.
|
||||
append(Pid, Data) when is_pid(Pid) andalso is_binary(Data) ->
|
||||
append(Pid, [], 0, Data);
|
||||
append(_Pid, _Data) ->
|
||||
lager:warning("Bad arguments to append/2"),
|
||||
{error, bad_arg}.
|
||||
|
||||
% @doc Append data to file, supplying client metadata and (if desired) a
|
||||
% reservation for additional space. ClientMeta is a proplist and expects the
|
||||
% same keys as write/4.
|
||||
-spec append(Pid :: pid(), ClientMeta :: proplists:proplist(),
|
||||
Extra :: non_neg_integer(), Data :: binary()) -> {ok, File :: string(), Offset :: non_neg_integer()}
|
||||
|{error, term()}.
|
||||
append(Pid, ClientMeta, Extra, Data) when is_pid(Pid) andalso is_list(ClientMeta)
|
||||
andalso is_integer(Extra) andalso Extra >= 0
|
||||
andalso is_binary(Data) ->
|
||||
gen_server:call(Pid, {append, ClientMeta, Extra, Data}, ?TIMEOUT);
|
||||
append(_Pid, ClientMeta, Extra, _Data) ->
|
||||
lager:warning("Bad arg to append: ClientMeta ~p, Extra ~p", [ClientMeta, Extra]),
|
||||
{error, bad_arg}.
|
||||
|
||||
-spec checksum_list(pid()) -> {ok, list()}.
|
||||
checksum_list(Pid) ->
|
||||
gen_server:call(Pid, {checksum_list}, ?TIMEOUT).
|
||||
|
||||
%% gen_server callbacks
|
||||
|
||||
% @private
|
||||
init({FluName, Filename, DataDir}) ->
|
||||
CsumFile = machi_util:make_checksum_filename(DataDir, Filename),
|
||||
{_, DPath} = machi_util:make_data_filename(DataDir, Filename),
|
||||
ok = filelib:ensure_dir(CsumFile),
|
||||
ok = filelib:ensure_dir(DPath),
|
||||
{ok, CsumTable} = machi_csum_table:open(CsumFile, []),
|
||||
UnwrittenBytes = machi_csum_table:calc_unwritten_bytes(CsumTable),
|
||||
{Eof, infinity} = lists:last(UnwrittenBytes),
|
||||
{ok, FHd} = file:open(DPath, [read, write, binary, raw]),
|
||||
%% Reserve for EC and stuff, to prevent eof when read
|
||||
ok = file:pwrite(FHd, 0, binary:copy(<<"so what?">>, ?MINIMUM_OFFSET div 8)),
|
||||
Tref = schedule_tick(),
|
||||
St = #state{
|
||||
fluname = FluName,
|
||||
filename = Filename,
|
||||
data_dir = DataDir,
|
||||
data_path = DPath,
|
||||
csum_file = CsumFile,
|
||||
data_filehandle = FHd,
|
||||
csum_table = CsumTable,
|
||||
tref = Tref,
|
||||
eof_position = erlang:max(Eof, ?MINIMUM_OFFSET),
|
||||
max_file_size = machi_config:max_file_size()},
|
||||
lager:debug("Starting file proxy ~p for filename ~p, state = ~p, Eof = ~p",
|
||||
[self(), Filename, St, Eof]),
|
||||
{ok, St}.
|
||||
|
||||
% @private
|
||||
handle_call({stop}, _From, State) ->
|
||||
lager:debug("Requested to stop."),
|
||||
{stop, normal, State};
|
||||
|
||||
handle_call({sync, data}, _From, State = #state{ data_filehandle = FHd }) ->
|
||||
R = file:sync(FHd),
|
||||
{reply, R, State};
|
||||
|
||||
handle_call({sync, csum}, _From, State) ->
|
||||
%% machi_csum_table always writes in {sync, true} option, so here
|
||||
%% explicit sync isn't actually needed.
|
||||
{reply, ok, State};
|
||||
|
||||
handle_call({sync, all}, _From, State = #state{filename = F,
|
||||
data_filehandle = FHd,
|
||||
csum_table = _T
|
||||
}) ->
|
||||
Resp = case file:sync(FHd) of
|
||||
ok ->
|
||||
ok;
|
||||
Error ->
|
||||
lager:error("Got ~p syncing all files for file ~p",
|
||||
[Error, F]),
|
||||
Error
|
||||
end,
|
||||
{reply, Resp, State};
|
||||
|
||||
%%% READS
|
||||
|
||||
handle_call({read, _Offset, _Length, _}, _From,
|
||||
State = #state{wedged = true,
|
||||
reads = {T, Err}
|
||||
}) ->
|
||||
{reply, {error, wedged}, State#state{writes = {T + 1, Err + 1}}};
|
||||
|
||||
handle_call({read, Offset, Length, _Opts}, _From,
|
||||
State = #state{eof_position = Eof,
|
||||
reads = {T, Err}
|
||||
}) when Offset > Eof ->
|
||||
%% make sure [Offset, Offset+Length) has an overlap with file range
|
||||
lager:error("Read request at offset ~p for ~p bytes is past the last write offset of ~p",
|
||||
[Offset, Length, Eof]),
|
||||
{reply, {error, not_written}, State#state{reads = {T + 1, Err + 1}}};
|
||||
|
||||
handle_call({read, Offset, Length, Opts}, _From,
|
||||
State = #state{filename = F,
|
||||
data_filehandle = FH,
|
||||
csum_table = CsumTable,
|
||||
reads = {T, Err}
|
||||
}) ->
|
||||
%% TODO: use these options - NoChunk prevents reading from disks
|
||||
%% NoChecksum doesn't check checksums
|
||||
#read_opts{no_checksum=NoChecksum, no_chunk=NoChunk,
|
||||
needs_trimmed=NeedsTrimmed} = Opts,
|
||||
{Resp, NewErr} =
|
||||
case do_read(FH, F, CsumTable, Offset, Length, NoChunk, NoChecksum) of
|
||||
{ok, {[], []}} ->
|
||||
{{error, not_written}, Err + 1};
|
||||
{ok, {Chunks0, Trimmed0}} ->
|
||||
Chunks = slice_both_side(Chunks0, Offset, Offset+Length),
|
||||
Trimmed = case NeedsTrimmed of
|
||||
true -> Trimmed0;
|
||||
false -> []
|
||||
end,
|
||||
{{ok, {Chunks, Trimmed}}, Err};
|
||||
Error ->
|
||||
lager:error("Can't read ~p, ~p at File ~p", [Offset, Length, F]),
|
||||
{Error, Err + 1}
|
||||
end,
|
||||
{reply, Resp, State#state{reads = {T+1, NewErr}}};
|
||||
|
||||
%%% WRITES
|
||||
|
||||
handle_call({write, _Offset, _ClientMeta, _Data}, _From,
|
||||
State = #state{wedged = true,
|
||||
writes = {T, Err}
|
||||
}) ->
|
||||
{reply, {error, wedged}, State#state{writes = {T + 1, Err + 1}}};
|
||||
|
||||
handle_call({write, Offset, ClientMeta, Data}, _From,
|
||||
State = #state{filename = F,
|
||||
writes = {T, Err},
|
||||
data_filehandle = FHd,
|
||||
csum_table = CsumTable}) ->
|
||||
|
||||
ClientCsumTag = proplists:get_value(client_csum_tag, ClientMeta, ?CSUM_TAG_NONE),
|
||||
ClientCsum = proplists:get_value(client_csum, ClientMeta, <<>>),
|
||||
|
||||
{Resp, NewErr} =
|
||||
case check_or_make_tagged_csum(ClientCsumTag, ClientCsum, Data) of
|
||||
{error, {bad_csum, Bad}} ->
|
||||
lager:error("Bad checksum on write; client sent ~p, we computed ~p",
|
||||
[ClientCsum, Bad]),
|
||||
{{error, bad_checksum}, Err + 1};
|
||||
TaggedCsum ->
|
||||
case handle_write(FHd, CsumTable, F, TaggedCsum, Offset, Data) of
|
||||
ok ->
|
||||
{ok, Err};
|
||||
Error ->
|
||||
{Error, Err + 1}
|
||||
end
|
||||
end,
|
||||
{NewEof, infinity} = lists:last(machi_csum_table:calc_unwritten_bytes(CsumTable)),
|
||||
lager:debug("Wrote ~p bytes at ~p of file ~p, NewEOF = ~p~n",
|
||||
[iolist_size(Data), Offset, F, NewEof]),
|
||||
{reply, Resp, State#state{writes = {T+1, NewErr},
|
||||
eof_position = NewEof}};
|
||||
|
||||
|
||||
%%% TRIMS
|
||||
|
||||
handle_call({trim, _Offset, _ClientMeta, _Data}, _From,
|
||||
State = #state{wedged = true,
|
||||
writes = {T, Err}
|
||||
}) ->
|
||||
{reply, {error, wedged}, State#state{writes = {T + 1, Err + 1}}};
|
||||
|
||||
handle_call({trim, Offset, Size, _TriggerGC}, _From,
|
||||
State = #state{data_filehandle=FHd,
|
||||
ops = Ops,
|
||||
trims = {T, Err},
|
||||
csum_table = CsumTable}) ->
|
||||
|
||||
case machi_csum_table:all_trimmed(CsumTable, Offset, Offset+Size) of
|
||||
true ->
|
||||
NewState = State#state{ops=Ops+1, trims={T, Err+1}},
|
||||
%% All bytes of that range was already trimmed returns ok
|
||||
%% here, not {error, trimmed}, which means the whole file
|
||||
%% was trimmed
|
||||
maybe_gc(ok, NewState);
|
||||
false ->
|
||||
LUpdate = maybe_regenerate_checksum(
|
||||
FHd,
|
||||
machi_csum_table:find_leftneighbor(CsumTable, Offset)),
|
||||
RUpdate = maybe_regenerate_checksum(
|
||||
FHd,
|
||||
machi_csum_table:find_rightneighbor(CsumTable, Offset+Size)),
|
||||
|
||||
case machi_csum_table:trim(CsumTable, Offset, Size, LUpdate, RUpdate) of
|
||||
ok ->
|
||||
{NewEof, infinity} = lists:last(machi_csum_table:calc_unwritten_bytes(CsumTable)),
|
||||
NewState = State#state{ops=Ops+1,
|
||||
trims={T+1, Err},
|
||||
eof_position=NewEof},
|
||||
maybe_gc(ok, NewState);
|
||||
Error ->
|
||||
{reply, Error, State#state{ops=Ops+1, trims={T, Err+1}}}
|
||||
end
|
||||
end;
|
||||
|
||||
%% APPENDS
|
||||
|
||||
handle_call({append, _ClientMeta, _Extra, _Data}, _From,
|
||||
State = #state{wedged = true,
|
||||
appends = {T, Err}
|
||||
}) ->
|
||||
{reply, {error, wedged}, State#state{appends = {T+1, Err+1}}};
|
||||
|
||||
handle_call({append, ClientMeta, Extra, Data}, _From,
|
||||
State = #state{eof_position = EofP,
|
||||
filename = F,
|
||||
appends = {T, Err},
|
||||
data_filehandle = FHd,
|
||||
csum_table = CsumTable
|
||||
}) ->
|
||||
|
||||
ClientCsumTag = proplists:get_value(client_csum_tag, ClientMeta, ?CSUM_TAG_NONE),
|
||||
ClientCsum = proplists:get_value(client_csum, ClientMeta, <<>>),
|
||||
|
||||
{Resp, NewErr} =
|
||||
case check_or_make_tagged_csum(ClientCsumTag, ClientCsum, Data) of
|
||||
{error, {bad_csum, Bad}} ->
|
||||
lager:error("Bad checksum; client sent ~p, we computed ~p",
|
||||
[ClientCsum, Bad]),
|
||||
{{error, bad_checksum}, Err + 1};
|
||||
TaggedCsum ->
|
||||
case handle_write(FHd, CsumTable, F, TaggedCsum, EofP, Data) of
|
||||
ok ->
|
||||
{{ok, F, EofP}, Err};
|
||||
Error ->
|
||||
{Error, Err + 1}
|
||||
end
|
||||
end,
|
||||
NewEof = EofP + byte_size(Data) + Extra,
|
||||
lager:debug("appended ~p bytes at ~p file ~p. NewEofP = ~p",
|
||||
[iolist_size(Data), EofP, F, NewEof]),
|
||||
{reply, Resp, State#state{appends = {T+1, NewErr},
|
||||
eof_position = NewEof}};
|
||||
|
||||
handle_call({checksum_list}, _FRom, State = #state{csum_table=T}) ->
|
||||
All = machi_csum_table:all(T),
|
||||
{reply, {ok, All}, State};
|
||||
|
||||
handle_call(Req, _From, State) ->
|
||||
lager:warning("Unknown call: ~p", [Req]),
|
||||
{reply, whoaaaaaaaaaaaa, State}.
|
||||
|
||||
% @private
|
||||
handle_cast(Cast, State) ->
|
||||
lager:warning("Unknown cast: ~p", [Cast]),
|
||||
{noreply, State}.
|
||||
|
||||
% @private
|
||||
handle_info(tick, State = #state{fluname = FluName,
|
||||
filename = F,
|
||||
eof_position = Eof,
|
||||
max_file_size = MaxFileSize}) when Eof >= MaxFileSize ->
|
||||
%% Older code halted here with {stop, file_rollover, State}.
|
||||
%% However, there may be other requests in our mailbox already
|
||||
%% and/or not yet delivered but in a race with the
|
||||
%% machi_flu_metadata_mgr. So we close our eleveldb instance (to
|
||||
%% avoid double-open attempt by a new file proxy proc), tell
|
||||
%% machi_flu_metadata_mgr that we request a rollover, then stop.
|
||||
%% terminate() will take care of forwarding messages that are
|
||||
%% caught in the race.
|
||||
lager:notice("Eof ~s position ~p >= max file size ~p. Shutting down.",
|
||||
[F, Eof, MaxFileSize]),
|
||||
State2 = close_files(State),
|
||||
machi_flu_metadata_mgr:stop_proxy_pid_rollover(FluName, {file, F}),
|
||||
{stop, normal, State2#state{rollover = true}};
|
||||
|
||||
%% XXX Is this a good idea? Need to think this through a bit.
|
||||
handle_info(tick, State = #state{wedged = true}) ->
|
||||
{stop, wedged, State};
|
||||
|
||||
%% I dunno. This may not be a good idea, but it seems like if we're throwing lots of
|
||||
%% errors, we ought to shut down and give up our file descriptors.
|
||||
handle_info(tick, State = #state{
|
||||
ops = Ops,
|
||||
reads = {RT, RE},
|
||||
writes = {WT, WE},
|
||||
appends = {AT, AE}
|
||||
}) when Ops > 100 andalso
|
||||
trunc(((RE+WE+AE) / (RT+WT+AT)) * 100) > ?TOO_MANY_ERRORS_RATIO ->
|
||||
Errors = RE + WE + AE,
|
||||
lager:notice("Got ~p errors. Shutting down.", [Errors]),
|
||||
{stop, too_many_errors, State};
|
||||
|
||||
handle_info(tick, State = #state{
|
||||
ticks = Ticks,
|
||||
ops = Ops,
|
||||
reads = {RT, _RE},
|
||||
writes = {WT, _WE},
|
||||
appends = {AT, _AE}}) when Ops == RT + WT + AT, Ticks == ?TICK_THRESHOLD ->
|
||||
lager:debug("Got 5 ticks with no new activity. Shutting down."),
|
||||
{stop, normal, State};
|
||||
|
||||
handle_info(tick, State = #state{
|
||||
ticks = Ticks,
|
||||
ops = Ops,
|
||||
reads = {RT, _RE},
|
||||
writes = {WT, _WE},
|
||||
appends = {AT, _AE}}) when Ops == RT + WT + AT ->
|
||||
lager:debug("No new activity since last tick. Incrementing tick counter."),
|
||||
Tref = schedule_tick(),
|
||||
{noreply, State#state{tref = Tref, ticks = Ticks + 1}};
|
||||
|
||||
handle_info(tick, State = #state{
|
||||
reads = {RT, _RE},
|
||||
writes = {WT, _WE},
|
||||
appends = {AT, _AE}
|
||||
}) ->
|
||||
Ops = RT + WT + AT,
|
||||
lager:debug("Setting ops counter to ~p", [Ops]),
|
||||
Tref = schedule_tick(),
|
||||
{noreply, State#state{tref = Tref, ops = Ops}};
|
||||
|
||||
%handle_info({wedged, EpochId} State = #state{epoch = E}) when E /= EpochId ->
|
||||
% lager:notice("Wedge epoch ~p but ignoring because our epoch id is ~p", [EpochId, E]),
|
||||
% {noreply, State};
|
||||
|
||||
%handle_info({wedged, EpochId}, State = #state{epoch = E}) when E == EpochId ->
|
||||
% lager:notice("Wedge epoch ~p same as our epoch id ~p; we are wedged. Bummer.", [EpochId, E]),
|
||||
% {noreply, State#state{wedged = true}};
|
||||
|
||||
% flu1.erl:
|
||||
% ProxyPid = get_proxy_pid(Filename),
|
||||
% Are we wedged? if not
|
||||
% machi_file_proxy:read(Pid, Offset, Length)
|
||||
% otherwise -> error,wedged
|
||||
%
|
||||
% get_proxy_pid(Filename) ->
|
||||
% Pid = lookup_pid(Filename)
|
||||
% is_pid_alive(Pid)
|
||||
% Pid
|
||||
% if not alive then start one
|
||||
|
||||
handle_info(Req, State) ->
|
||||
lager:warning("Unknown info message: ~p", [Req]),
|
||||
{noreply, State}.
|
||||
|
||||
% @private
|
||||
terminate(Reason, State = #state{fluname = FluName,
|
||||
filename = F,
|
||||
rollover = Rollover_p,
|
||||
reads = {RT, RE},
|
||||
writes = {WT, WE},
|
||||
appends = {AT, AE}
|
||||
}) ->
|
||||
lager:info("Shutting down proxy for file ~p because ~p", [F, Reason]),
|
||||
lager:info(" Op Tot/Error", []),
|
||||
lager:info(" Reads: ~p/~p", [RT, RE]),
|
||||
lager:info(" Writes: ~p/~p", [WT, WE]),
|
||||
lager:info("Appends: ~p/~p", [AT, AE]),
|
||||
close_files(State),
|
||||
if Rollover_p ->
|
||||
forward_late_messages(FluName, F, 500);
|
||||
true ->
|
||||
ok
|
||||
end,
|
||||
ok.
|
||||
|
||||
% @private
|
||||
code_change(_OldVsn, State, _Extra) ->
|
||||
{ok, State}.
|
||||
|
||||
%% Private functions
|
||||
|
||||
-spec schedule_tick() -> reference().
|
||||
schedule_tick() ->
|
||||
erlang:send_after(?TICK, self(), tick).
|
||||
|
||||
-spec check_or_make_tagged_csum(Type :: non_neg_integer(),
|
||||
Checksum :: binary(),
|
||||
Data :: binary() ) -> binary() |
|
||||
{error, {bad_csum, Bad :: binary()}}.
|
||||
check_or_make_tagged_csum(?CSUM_TAG_NONE, _Csum, Data) ->
|
||||
%% We are making a checksum here
|
||||
Csum = machi_util:checksum_chunk(Data),
|
||||
machi_util:make_tagged_csum(server_sha, Csum);
|
||||
check_or_make_tagged_csum(Tag, InCsum, Data) when Tag == ?CSUM_TAG_CLIENT_SHA;
|
||||
Tag == ?CSUM_TAG_SERVER_SHA ->
|
||||
Csum = machi_util:checksum_chunk(Data),
|
||||
case Csum =:= InCsum of
|
||||
true ->
|
||||
machi_util:make_tagged_csum(server_sha, Csum);
|
||||
false ->
|
||||
{error, {bad_csum, Csum}}
|
||||
end;
|
||||
check_or_make_tagged_csum(?CSUM_TAG_SERVER_REGEN_SHA,
|
||||
InCsum, Data) ->
|
||||
Csum = machi_util:checksum_chunk(Data),
|
||||
case Csum =:= InCsum of
|
||||
true ->
|
||||
machi_util:make_tagged_csum(server_regen_sha, Csum);
|
||||
false ->
|
||||
{error, {bad_csum, Csum}}
|
||||
end;
|
||||
check_or_make_tagged_csum(OtherTag, _ClientCsum, _Data) ->
|
||||
lager:warning("Unknown checksum tag ~p", [OtherTag]),
|
||||
{error, bad_checksum}.
|
||||
|
||||
-spec do_read(FHd :: file:io_device(),
|
||||
Filename :: string(),
|
||||
CsumTable :: machi_csum_table:table(),
|
||||
Offset :: non_neg_integer(),
|
||||
Size :: non_neg_integer(),
|
||||
NoChunk :: boolean(),
|
||||
NoChecksum :: boolean()
|
||||
) -> {ok, {Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum :: binary()}],
|
||||
Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]}} |
|
||||
{error, bad_checksum} |
|
||||
{error, partial_read} |
|
||||
{error, file:posix()} |
|
||||
{error, Other :: term() }.
|
||||
% @private Attempt a read operation on the given offset and length.
|
||||
% <li>
|
||||
% <ul> If the byte range is not yet written, `{error, not_written}' is
|
||||
% returned.</ul>
|
||||
% <ul> If the checksum given does not match what comes off the disk,
|
||||
% `{error, bad_checksum}' is returned.</ul>
|
||||
% <ul> If the number of bytes that comes off the disk is not the requested length,
|
||||
% `{error, partial_read}' is returned.</ul>
|
||||
% <ul> If the offset is at or beyond the current file boundary, `eof' is returned.</ul>
|
||||
% <ul> If some kind of POSIX error occurs, the OTP version of that POSIX error
|
||||
% tuple is returned.</ul>
|
||||
% </li>
|
||||
%
|
||||
do_read(FHd, Filename, CsumTable, Offset, Size, _, _) ->
|
||||
%% Note that find/3 only returns overlapping chunks, both borders
|
||||
%% are not aligned to original Offset and Size.
|
||||
ChunkCsums = machi_csum_table:find(CsumTable, Offset, Size),
|
||||
read_all_ranges(FHd, Filename, ChunkCsums, [], []).
|
||||
|
||||
-spec read_all_ranges(file:io_device(), string(),
|
||||
[{non_neg_integer(),non_neg_integer(),trimmed|binary()}],
|
||||
Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum::binary()}],
|
||||
Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]) ->
|
||||
{ok, {
|
||||
Chunks :: [{string(), Offset::non_neg_integer(), binary(), Csum::binary()}],
|
||||
Trimmed :: [{string(), Offset::non_neg_integer(), Size::non_neg_integer()}]}} |
|
||||
{erorr, term()|partial_read}.
|
||||
read_all_ranges(_, _, [], ReadChunks, TrimmedChunks) ->
|
||||
%% TODO: currently returns empty list of trimmed chunks
|
||||
{ok, {lists:reverse(ReadChunks), lists:reverse(TrimmedChunks)}};
|
||||
|
||||
read_all_ranges(FHd, Filename, [{Offset, Size, trimmed}|T], ReadChunks, TrimmedChunks) ->
|
||||
read_all_ranges(FHd, Filename, T, ReadChunks, [{Filename, Offset, Size}|TrimmedChunks]);
|
||||
|
||||
read_all_ranges(FHd, Filename, [{Offset, Size, TaggedCsum}|T], ReadChunks, TrimmedChunks) ->
|
||||
case file:pread(FHd, Offset, Size) of
|
||||
eof ->
|
||||
read_all_ranges(FHd, Filename, T, ReadChunks, TrimmedChunks);
|
||||
{ok, Bytes} when byte_size(Bytes) == Size, TaggedCsum =:= none ->
|
||||
read_all_ranges(FHd, Filename, T,
|
||||
[{Filename, Offset, Bytes,
|
||||
machi_util:make_tagged_csum(none, <<>>)}|ReadChunks],
|
||||
TrimmedChunks);
|
||||
{ok, Bytes} when byte_size(Bytes) == Size ->
|
||||
{Tag, Ck} = machi_util:unmake_tagged_csum(TaggedCsum),
|
||||
case check_or_make_tagged_csum(Tag, Ck, Bytes) of
|
||||
{error, Bad} ->
|
||||
lager:error("Bad checksum; got ~p, expected ~p",
|
||||
[Bad, Ck]),
|
||||
{error, bad_checksum};
|
||||
TaggedCsum ->
|
||||
read_all_ranges(FHd, Filename, T,
|
||||
[{Filename, Offset, Bytes, TaggedCsum}|ReadChunks],
|
||||
TrimmedChunks);
|
||||
OtherCsum when Tag =:= ?CSUM_TAG_NONE ->
|
||||
%% XXX FIXME: Should we return something other than
|
||||
%% {ok, ....} in this case?
|
||||
read_all_ranges(FHd, Filename, T,
|
||||
[{Filename, Offset, Bytes, OtherCsum}|ReadChunks],
|
||||
TrimmedChunks)
|
||||
end;
|
||||
{ok, Partial} ->
|
||||
lager:error("In file ~p, offset ~p, wanted to read ~p bytes, but got ~p",
|
||||
[Filename, Offset, Size, byte_size(Partial)]),
|
||||
{error, partial_read};
|
||||
Other ->
|
||||
lager:error("While reading file ~p, offset ~p, length ~p, got ~p",
|
||||
[Filename, Offset, Size, Other]),
|
||||
{error, Other}
|
||||
end.
|
||||
|
||||
-spec handle_write( FHd :: file:io_device(),
|
||||
CsumTable :: machi_csum_table:table(),
|
||||
Filename :: string(),
|
||||
TaggedCsum :: binary(),
|
||||
Offset :: non_neg_integer(),
|
||||
Data :: binary()
|
||||
) -> ok |
|
||||
{error, written} |
|
||||
{error, Reason :: term()}.
|
||||
% @private Implements the write and append operation. The first task is to
|
||||
% determine if the offset and data size has been written. If not, the write
|
||||
% is allowed proceed. A special case is made when an offset and data size
|
||||
% match a checksum. In that case we read the data off the disk, validate the
|
||||
% checksum and return a "fake" ok response as if the write had been performed
|
||||
% when it hasn't really.
|
||||
%
|
||||
% If a write proceeds, the offset, size and checksum are written to a
|
||||
% metadata file, and the internal list of unwritten bytes is modified
|
||||
% to reflect the just-performed write. This is then returned to the
|
||||
% caller as `ok'
|
||||
handle_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Data) ->
|
||||
Size = iolist_size(Data),
|
||||
|
||||
case machi_csum_table:find(CsumTable, Offset, Size) of
|
||||
[] -> %% Nothing should be there
|
||||
try
|
||||
do_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Size, Data)
|
||||
catch
|
||||
%% XXX FIXME: be more specific on badmatch that might
|
||||
%% occur around line 593 when we write the checksum
|
||||
%% file entry for the data blob we just put on the disk
|
||||
error:Reason ->
|
||||
{error, Reason}
|
||||
end;
|
||||
[{Offset, Size, TaggedCsum}] ->
|
||||
case do_read(FHd, Filename, CsumTable, Offset, Size, false, false) of
|
||||
{error, _} = E ->
|
||||
lager:warning("This should never happen: got ~p while reading"
|
||||
" at offset ~p in file ~p that's supposedly written",
|
||||
[E, Offset, Filename]),
|
||||
{error, server_insanity};
|
||||
{ok, {[{_, Offset, Data, TaggedCsum}], _}} ->
|
||||
%% TODO: what if different checksum got from do_read()?
|
||||
ok;
|
||||
{ok, _Other} ->
|
||||
%% TODO: leave some debug/warning message here?
|
||||
{error, written}
|
||||
end;
|
||||
[{Offset, Size, OtherCsum}] ->
|
||||
%% Got a checksum, but it doesn't match the data block's
|
||||
lager:error("During a potential write at offset ~p in file ~p,"
|
||||
" a check for unwritten bytes gave us checksum ~p"
|
||||
" but the data we were trying to write has checksum ~p",
|
||||
[Offset, Filename, OtherCsum, TaggedCsum]),
|
||||
{error, written};
|
||||
_Chunks ->
|
||||
%% TODO: Do we try to read all continuous chunks to see
|
||||
%% wether its total checksum matches client-provided checksum?
|
||||
case machi_csum_table:any_trimmed(CsumTable, Offset, Size) of
|
||||
true ->
|
||||
%% More than a byte is trimmed, besides, do we
|
||||
%% have to return exact written bytes? No. Clients
|
||||
%% must issue read_chunk() with needs_trimmed
|
||||
%% option as true
|
||||
{error, trimmed};
|
||||
false ->
|
||||
%% No byte is trimmed, but at least one byte is written
|
||||
{error, written}
|
||||
end
|
||||
end.
|
||||
|
||||
% @private Implements the disk writes for both the write and append
|
||||
% operation.
|
||||
-spec do_write( FHd :: file:io_device(),
|
||||
CsumTable :: machi_csum_table:table(),
|
||||
Filename :: string(),
|
||||
TaggedCsum :: binary(),
|
||||
Offset :: non_neg_integer(),
|
||||
Size :: non_neg_integer(),
|
||||
Data :: binary()
|
||||
) -> ok | {error, Reason :: term()}.
|
||||
do_write(FHd, CsumTable, Filename, TaggedCsum, Offset, Size, Data) ->
|
||||
case file:pwrite(FHd, Offset, Data) of
|
||||
ok ->
|
||||
lager:debug("Successful write in file ~p at offset ~p, length ~p",
|
||||
[Filename, Offset, Size]),
|
||||
|
||||
%% Overlapping chunk; calculate checksum
|
||||
%% read {LOffset, Offset - LOffset} and make csum
|
||||
%% as server_sha
|
||||
LUpdate = maybe_regenerate_checksum(
|
||||
FHd,
|
||||
machi_csum_table:find_leftneighbor(CsumTable, Offset)),
|
||||
RUpdate = maybe_regenerate_checksum(
|
||||
FHd,
|
||||
machi_csum_table:find_rightneighbor(CsumTable, Offset+Size)),
|
||||
ok = machi_csum_table:write(CsumTable, Offset, Size,
|
||||
TaggedCsum, LUpdate, RUpdate),
|
||||
lager:debug("Successful write to checksum file for ~p",
|
||||
[Filename]),
|
||||
ok;
|
||||
Other ->
|
||||
lager:error("Got ~p during write to file ~p at offset ~p, length ~p",
|
||||
[Other, Filename, Offset, Size]),
|
||||
{error, Other}
|
||||
end.
|
||||
|
||||
%% @doc Trim both right and left border of chunks to fit in to given
|
||||
%% range [LeftPos, RightPos]. TODO: write unit tests for this function.
|
||||
|
||||
%% Dialyzer 'can never match': slice_both_side([], _, _) ->
|
||||
%% [];
|
||||
slice_both_side([], _, _) ->
|
||||
[];
|
||||
slice_both_side([{F, Offset, Chunk, _Csum}|L], LeftPos, RightPos)
|
||||
when Offset < LeftPos andalso LeftPos < RightPos ->
|
||||
TrashLen = (LeftPos - Offset),
|
||||
<<_:TrashLen/binary, NewChunk/binary>> = Chunk,
|
||||
NewChecksum = machi_util:make_tagged_csum(?CSUM_TAG_SERVER_REGEN_SHA_ATOM, Chunk),
|
||||
NewH = {F, LeftPos, NewChunk, NewChecksum},
|
||||
slice_both_side([NewH|L], LeftPos, RightPos);
|
||||
slice_both_side(Chunks, LeftPos, RightPos) when LeftPos =< RightPos ->
|
||||
%% TODO: optimize
|
||||
[{F, Offset, Chunk, _Csum}|L] = lists:reverse(Chunks),
|
||||
Size = iolist_size(Chunk),
|
||||
if RightPos < Offset + Size ->
|
||||
NewSize = RightPos - Offset,
|
||||
<<NewChunk:NewSize/binary, _/binary>> = Chunk,
|
||||
NewChecksum = machi_util:make_tagged_csum(?CSUM_TAG_SERVER_REGEN_SHA_ATOM, Chunk),
|
||||
lists:reverse([{F, Offset, NewChunk, NewChecksum}|L]);
|
||||
true ->
|
||||
Chunks
|
||||
end.
|
||||
|
||||
maybe_regenerate_checksum(_, undefined) ->
|
||||
undefined;
|
||||
maybe_regenerate_checksum(_, {_, _, trimmed} = Change) ->
|
||||
Change;
|
||||
maybe_regenerate_checksum(FHd, {Offset, Size, _Csum}) ->
|
||||
case file:pread(FHd, Offset, Size) of
|
||||
eof ->
|
||||
error({eof, Offset, Size});
|
||||
{ok, Bytes} when byte_size(Bytes) =:= Size ->
|
||||
|
||||
TaggedCsum = machi_util:make_tagged_csum(server_regen_sha,
|
||||
machi_util:checksum_chunk(Bytes)),
|
||||
{Offset, Size, TaggedCsum};
|
||||
Error ->
|
||||
throw(Error)
|
||||
end.
|
||||
|
||||
%% GC: make sure unwritten bytes = [{Eof, infinity}] and Eof is > max
|
||||
%% file size walk through the checksum table and make sure all chunks
|
||||
%% trimmed Then unlink the file
|
||||
-spec maybe_gc(term(), #state{}) ->
|
||||
{reply, term(), #state{}} | {stop, normal, term(), #state{}}.
|
||||
maybe_gc(Reply, S = #state{eof_position = Eof,
|
||||
max_file_size = MaxFileSize}) when Eof < MaxFileSize ->
|
||||
lager:debug("The file is still small; not trying GC (Eof, MaxFileSize) = (~p, ~p)~n",
|
||||
[Eof, MaxFileSize]),
|
||||
{reply, Reply, S};
|
||||
maybe_gc(Reply, S = #state{fluname=FluName,
|
||||
data_filehandle = FHd,
|
||||
data_dir = DataDir,
|
||||
filename = Filename,
|
||||
eof_position = Eof,
|
||||
csum_table=CsumTable}) ->
|
||||
case machi_csum_table:all_trimmed(CsumTable, ?MINIMUM_OFFSET, Eof) of
|
||||
true ->
|
||||
lager:debug("GC? Let's do it: ~p.~n", [Filename]),
|
||||
%% Before unlinking a file, it should inform
|
||||
%% machi_flu_filename_mgr that this file is
|
||||
%% deleted and mark it as "trimmed" to avoid
|
||||
%% filename reuse and resurrection. Maybe garbage
|
||||
%% will remain if a process crashed but it also
|
||||
%% should be recovered at filename_mgr startup.
|
||||
|
||||
%% Also, this should be informed *before* file proxy
|
||||
%% deletes files.
|
||||
ok = machi_flu_metadata_mgr:trim_file(FluName, {file, Filename}),
|
||||
ok = file:close(FHd),
|
||||
{_, DPath} = machi_util:make_data_filename(DataDir, Filename),
|
||||
ok = file:delete(DPath),
|
||||
machi_csum_table:delete(CsumTable),
|
||||
{stop, normal, Reply,
|
||||
S#state{data_filehandle=undefined,
|
||||
csum_table=undefined}};
|
||||
false ->
|
||||
{reply, Reply, S}
|
||||
end.
|
||||
|
||||
close_files(State = #state{data_filehandle = FHd,
|
||||
csum_table = T}) ->
|
||||
case FHd of
|
||||
undefined ->
|
||||
noop; %% file deleted
|
||||
_ ->
|
||||
ok = file:sync(FHd),
|
||||
ok = file:close(FHd)
|
||||
end,
|
||||
case T of
|
||||
undefined ->
|
||||
noop; %% file deleted
|
||||
_ ->
|
||||
ok = machi_csum_table:close(T)
|
||||
end,
|
||||
State#state{data_filehandle = undefined, csum_table = undefined}.
|
||||
|
||||
forward_late_messages(FluName, F, Timeout) ->
|
||||
receive
|
||||
M ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, F}) of
|
||||
{ok, Pid} ->
|
||||
Pid ! M;
|
||||
{error, trimmed} ->
|
||||
lager:error("TODO: FLU ~p file ~p reports trimmed status "
|
||||
"when forwarding ~P\n",
|
||||
[FluName, F, M, 20])
|
||||
end,
|
||||
forward_late_messages(FluName, F, Timeout)
|
||||
after Timeout ->
|
||||
ok
|
||||
end.
|
57
src/machi_file_proxy_sup.erl
Normal file
57
src/machi_file_proxy_sup.erl
Normal file
|
@ -0,0 +1,57 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc This is the main supervisor for the file proxies.
|
||||
-module(machi_file_proxy_sup).
|
||||
-behaviour(supervisor).
|
||||
|
||||
%% public API
|
||||
-export([
|
||||
child_spec/1,
|
||||
start_link/1,
|
||||
start_proxy/3
|
||||
]).
|
||||
|
||||
%% supervisor callback
|
||||
-export([
|
||||
init/1
|
||||
]).
|
||||
|
||||
child_spec(FluName) ->
|
||||
Name = make_proxy_name(FluName),
|
||||
{Name,
|
||||
{?MODULE, start_link, [FluName]},
|
||||
permanent, 5000, supervisor, [?MODULE]}.
|
||||
|
||||
start_link(FluName) ->
|
||||
supervisor:start_link({local, make_proxy_name(FluName)}, ?MODULE, []).
|
||||
|
||||
start_proxy(FluName, DataDir, Filename) ->
|
||||
supervisor:start_child(make_proxy_name(FluName),
|
||||
[FluName, Filename, DataDir]).
|
||||
|
||||
init([]) ->
|
||||
SupFlags = {simple_one_for_one, 1000, 10},
|
||||
ChildSpec = {unused, {machi_file_proxy, start_link, []},
|
||||
temporary, 2000, worker, [machi_file_proxy]},
|
||||
{ok, {SupFlags, [ChildSpec]}}.
|
||||
|
||||
make_proxy_name(FluName) when is_atom(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_file_proxy_sup").
|
|
@ -39,11 +39,12 @@
|
|||
get_unfit_list/1, update_local_down_list/3,
|
||||
add_admin_down/3, delete_admin_down/2,
|
||||
send_fitness_update_spam/3,
|
||||
send_spam_to_everyone/1]).
|
||||
send_spam_to_everyone/1,
|
||||
trigger_early_adjustment/2]).
|
||||
|
||||
%% gen_server callbacks
|
||||
-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
|
||||
terminate/2, code_change/3]).
|
||||
terminate/2, code_change/3, format_status/2]).
|
||||
|
||||
-record(state, {
|
||||
my_flu_name :: atom() | binary(),
|
||||
|
@ -81,12 +82,19 @@ send_fitness_update_spam(Pid, FromName, Dict) ->
|
|||
send_spam_to_everyone(Pid) ->
|
||||
gen_server:call(Pid, {send_spam_to_everyone}, infinity).
|
||||
|
||||
%% @doc For testing purposes, we don't want a test to wait for
|
||||
%% wall-clock time to elapse before the fitness server makes a
|
||||
%% down->up status decision.
|
||||
|
||||
trigger_early_adjustment(Pid, FLU) ->
|
||||
Pid ! {adjust_down_list, FLU}.
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
init([{MyFluName}|Args]) ->
|
||||
RegName = machi_flu_psup:make_fitness_regname(MyFluName),
|
||||
register(RegName, self()),
|
||||
timer:send_interval(5000, dump),
|
||||
{ok, _} = timer:send_interval(5000, debug_dump),
|
||||
UseSimulatorP = proplists:get_value(use_partition_simulator, Args, false),
|
||||
{ok, #state{my_flu_name=MyFluName, reg_name=RegName,
|
||||
partition_simulator_p=UseSimulatorP,
|
||||
|
@ -100,24 +108,29 @@ handle_call({update_local_down_list, Down, MembersDict}, _From,
|
|||
#state{my_flu_name=MyFluName, pending_map=OldMap,
|
||||
local_down=OldDown, members_dict=OldMembersDict,
|
||||
admin_down=AdminDown}=S) ->
|
||||
verbose("FITNESS: ~w has down suspect ~w\n", [MyFluName, Down]),
|
||||
NewMap = store_in_map(OldMap, MyFluName, erlang:now(), Down,
|
||||
AdminDown, [props_yo]),
|
||||
S2 = if Down == OldDown, MembersDict == OldMembersDict ->
|
||||
%% Do nothing only if both are equal. If members_dict is
|
||||
%% changing, that's sufficient reason to spam.
|
||||
ok;
|
||||
S;
|
||||
true ->
|
||||
do_map_change(NewMap, [MyFluName], MembersDict, S)
|
||||
end,
|
||||
{reply, ok, S2#state{local_down=Down}};
|
||||
handle_call({add_admin_down, DownFLU, DownProps}, _From,
|
||||
#state{local_down=OldDown, admin_down=AdminDown}=S) ->
|
||||
#state{my_flu_name=MyFluName,
|
||||
local_down=OldDown, admin_down=AdminDown}=S) ->
|
||||
verbose("FITNESS: ~w add admin down ~w\n", [MyFluName, DownFLU]),
|
||||
NewAdminDown = [{DownFLU,DownProps}|lists:keydelete(DownFLU, 1, AdminDown)],
|
||||
S3 = finish_admin_down(erlang:now(), OldDown, NewAdminDown,
|
||||
[props_yo], S),
|
||||
{reply, ok, S3};
|
||||
handle_call({delete_admin_down, DownFLU}, _From,
|
||||
#state{local_down=OldDown, admin_down=AdminDown}=S) ->
|
||||
#state{my_flu_name=MyFluName,
|
||||
local_down=OldDown, admin_down=AdminDown}=S) ->
|
||||
verbose("FITNESS: ~w delete admin down ~w\n", [MyFluName, DownFLU]),
|
||||
NewAdminDown = lists:keydelete(DownFLU, 1, AdminDown),
|
||||
S3 = finish_admin_down(erlang:now(), OldDown, NewAdminDown,
|
||||
[props_yo], S),
|
||||
|
@ -135,11 +148,12 @@ handle_call(_Request, _From, S) ->
|
|||
handle_cast(_Msg, S) ->
|
||||
{noreply, S}.
|
||||
|
||||
handle_info({adjust_down_list, FLU}, #state{active_unfit=ActiveUnfit}=S) ->
|
||||
handle_info({adjust_down_list, FLU}, #state{my_flu_name=MyFluName,
|
||||
active_unfit=ActiveUnfit}=S) ->
|
||||
NewUnfit = make_unfit_list(S),
|
||||
Added_to_new = NewUnfit -- ActiveUnfit,
|
||||
Dropped_from_new = ActiveUnfit -- NewUnfit,
|
||||
io:format(user, "adjust_down_list: ~w: adjust ~w: add ~p drop ~p\n", [S#state.my_flu_name, FLU, Added_to_new, Dropped_from_new]),
|
||||
%% io:format(user, "adjust_down_list: ~w: adjust ~w: add ~p drop ~p\n", [S#state.my_flu_name, FLU, Added_to_new, Dropped_from_new]),
|
||||
%% We need to schedule a new round of adjustment messages. They might
|
||||
%% be redundant, or they might not. Here's a case where the current
|
||||
%% code needs the extra:
|
||||
|
@ -171,21 +185,23 @@ handle_info({adjust_down_list, FLU}, #state{active_unfit=ActiveUnfit}=S) ->
|
|||
%% hiding where we need this extra round of messages to *remove* a
|
||||
%% FLU from the active_unfit list?
|
||||
|
||||
schedule_adjust_messages(lists:usort(Added_to_new ++ Dropped_from_new)),
|
||||
_ = schedule_adjust_messages(lists:usort(Added_to_new ++ Dropped_from_new)),
|
||||
case {lists:member(FLU,Added_to_new), lists:member(FLU,Dropped_from_new)} of
|
||||
{true, true} ->
|
||||
error({bad, ?MODULE, ?LINE, FLU, ActiveUnfit, NewUnfit});
|
||||
{true, false} ->
|
||||
{noreply, S#state{active_unfit=lists:usort(ActiveUnfit ++ [FLU])}};
|
||||
NewActive = wrap_active(MyFluName,lists:usort(ActiveUnfit++[FLU])),
|
||||
{noreply, S#state{active_unfit=NewActive}};
|
||||
{false, true} ->
|
||||
{noreply, S#state{active_unfit=ActiveUnfit -- [FLU]}};
|
||||
NewActive = wrap_active(MyFluName,ActiveUnfit--[FLU]),
|
||||
{noreply, S#state{active_unfit=NewActive}};
|
||||
{false, false} ->
|
||||
{noreply, S}
|
||||
end;
|
||||
handle_info(dump, #state{my_flu_name=MyFluName,active_unfit=ActiveUnfit,
|
||||
pending_map=Map}=S) ->
|
||||
%% io:format(user, "DUMP: ~w/~w: ~p ~W\n", [MyFluName, self(), ActiveUnfit, map_value(Map), 13]),
|
||||
io:format(user, "DUMP ~w: ~w, ", [MyFluName, ActiveUnfit]),
|
||||
handle_info(debug_dump, #state{my_flu_name=_MyFluName,active_unfit=_ActiveUnfit,
|
||||
pending_map=_Map}=S) ->
|
||||
%% io:format(user, "DUMP: ~w/~w: ~p ~W\n", [_MyFluName, self(), _ActiveUnfit, map_value(_Map), 13]),
|
||||
%% io:format(user, "DUMP ~w: ~w, ", [MyFluName, ActiveUnfit]),
|
||||
{noreply, S};
|
||||
handle_info(_Info, S) ->
|
||||
{noreply, S}.
|
||||
|
@ -193,6 +209,11 @@ handle_info(_Info, S) ->
|
|||
terminate(_Reason, _S) ->
|
||||
ok.
|
||||
|
||||
format_status(_Opt, [_PDict, Status]) ->
|
||||
Fields = record_info(fields, state),
|
||||
[_Name | Values] = tuple_to_list(Status),
|
||||
lists:zip(Fields, Values).
|
||||
|
||||
code_change(_OldVsn, S, _Extra) ->
|
||||
{ok, S}.
|
||||
|
||||
|
@ -290,8 +311,8 @@ proxy_pid(Name, #state{proxies_dict=ProxiesDict}) ->
|
|||
|
||||
calc_unfit(All_list, HosedAnnotations) ->
|
||||
G = digraph:new(),
|
||||
[digraph:add_vertex(G, V) || V <- All_list],
|
||||
[digraph:add_edge(G, V1, V2) || {V1, problem_with, V2} <- HosedAnnotations],
|
||||
_ = [digraph:add_vertex(G, V) || V <- All_list],
|
||||
_ = [digraph:add_edge(G, V1, V2) || {V1, problem_with, V2} <- HosedAnnotations],
|
||||
calc_unfit2(lists:sort(digraph:vertices(G)), G).
|
||||
|
||||
calc_unfit2([], G) ->
|
||||
|
@ -345,7 +366,7 @@ do_map_change(NewMap, DontSendList, MembersDict,
|
|||
#state{my_flu_name=_MyFluName, pending_map=OldMap}=S) ->
|
||||
send_spam(NewMap, DontSendList, MembersDict, S),
|
||||
ChangedServers = find_changed_servers(OldMap, NewMap, _MyFluName),
|
||||
schedule_adjust_messages(ChangedServers),
|
||||
_ = schedule_adjust_messages(ChangedServers),
|
||||
%% _OldMapV = map_value(OldMap),
|
||||
%% _MapV = map_value(NewMap),
|
||||
%% io:format(user, "TODO: ~w async tick trigger/scheduling... ~w for:\n"
|
||||
|
@ -390,6 +411,7 @@ map_set(Actor, Map, Key, ValTerm) ->
|
|||
Actor, Map),
|
||||
Map2.
|
||||
|
||||
-ifdef(TEST).
|
||||
map_get(Map, Key) ->
|
||||
Field = {Key, ?LWWREG},
|
||||
case lists:keyfind(Field, 1, ?MAP:value(Map)) of
|
||||
|
@ -398,6 +420,7 @@ map_get(Map, Key) ->
|
|||
{Field, ValBin} ->
|
||||
{ok, binary_to_term(ValBin)}
|
||||
end.
|
||||
-endif. % TEST
|
||||
|
||||
map_fold(Fun, Acc, Map) ->
|
||||
Vs = map_value(Map),
|
||||
|
@ -409,23 +432,17 @@ map_value(Map) ->
|
|||
map_merge(Map1, Map2) ->
|
||||
?MAP:merge(Map1, Map2).
|
||||
|
||||
-ifdef(TEST).
|
||||
wrap_active(MyFluName, L) ->
|
||||
verbose("FITNESS: ~w has new down list ~w\n", [MyFluName, L]),
|
||||
L.
|
||||
|
||||
testing_sleep_perhaps() ->
|
||||
try
|
||||
[{_,Max}] = ets:lookup(?TEST_ETS_TABLE, projection_store_sleep_time),
|
||||
MSec = random:uniform(Max),
|
||||
timer:sleep(MSec),
|
||||
ok
|
||||
catch _X:_Y ->
|
||||
verbose(Fmt, Args) ->
|
||||
case application:get_env(machi, fitness_verbose) of
|
||||
{ok, true} ->
|
||||
error_logger:info_msg(Fmt, Args);
|
||||
_ ->
|
||||
ok
|
||||
end.
|
||||
-else. % TEST
|
||||
|
||||
testing_sleep_perhaps() ->
|
||||
ok.
|
||||
|
||||
-endif. % TEST
|
||||
|
||||
-ifdef(TEST).
|
||||
|
||||
|
|
|
@ -21,7 +21,9 @@
|
|||
%% @doc The Machi FLU file server + file location sequencer.
|
||||
%%
|
||||
%% This module implements only the Machi FLU file server and its
|
||||
%% implicit sequencer.
|
||||
%% implicit sequencer together with listener, append server,
|
||||
%% file management and file proxy processes.
|
||||
|
||||
%% Please see the EDoc "Overview" for details about the FLU as a
|
||||
%% primitive file server process vs. the larger Machi design of a FLU
|
||||
%% as a sequencer + file server + chain manager group of processes.
|
||||
|
@ -35,14 +37,6 @@
|
|||
%%
|
||||
%% === TODO items ===
|
||||
%%
|
||||
%% TODO There is a major missing feature in this FLU implementation:
|
||||
%% there is no "write-once" enforcement for any position in a Machi
|
||||
%% file. At the moment, we rely on correct behavior of the client
|
||||
%% & the sequencer to avoid overwriting data. In the Real World,
|
||||
%% however, all Machi file data is supposed to be exactly write-once
|
||||
%% to avoid problems with bugs, wire protocol corruption, malicious
|
||||
%% clients, etc.
|
||||
%%
|
||||
%% TODO The per-file metadata tuple store is missing from this implementation.
|
||||
%%
|
||||
%% TODO Section 4.1 ("The FLU") of the Machi design doc suggests that
|
||||
|
@ -62,37 +56,29 @@
|
|||
|
||||
-ifdef(TEST).
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-export([timing_demo_test_COMMENTED_/0, sort_2lines/2]). % Just to suppress warning
|
||||
-endif. % TEST
|
||||
|
||||
-define(SERVER_CMD_READ_TIMEOUT, 600*1000).
|
||||
|
||||
-export([start_link/1, stop/1,
|
||||
update_wedge_state/3, wedge_myself/2]).
|
||||
-export([make_listener_regname/1, make_projection_server_regname/1]).
|
||||
-export([encode_csum_file_entry/3, encode_csum_file_entry_bin/3,
|
||||
decode_csum_file_entry/1,
|
||||
split_checksum_list_blob/1, split_checksum_list_blob_decode/1]).
|
||||
-export([make_projection_server_regname/1,
|
||||
ets_table_name/1]).
|
||||
%% TODO: remove or replace in OTP way after gen_*'ified
|
||||
-export([main2/4]).
|
||||
|
||||
-record(state, {
|
||||
flu_name :: atom(),
|
||||
proj_store :: pid(),
|
||||
witness = false :: boolean(),
|
||||
append_pid :: pid(),
|
||||
tcp_port :: non_neg_integer(),
|
||||
data_dir :: string(),
|
||||
wedged = true :: boolean(),
|
||||
etstab :: ets:tid(),
|
||||
epoch_id :: 'undefined' | machi_dt:epoch_id(),
|
||||
pb_mode = undefined :: 'undefined' | 'high' | 'low',
|
||||
high_clnt :: 'undefined' | pid(),
|
||||
props = [] :: list() % proplist
|
||||
}).
|
||||
-define(INIT_TIMEOUT, 60*1000).
|
||||
|
||||
start_link([{FluName, TcpPort, DataDir}|Rest])
|
||||
when is_atom(FluName), is_integer(TcpPort), is_list(DataDir) ->
|
||||
{ok, spawn_link(fun() -> main2(FluName, TcpPort, DataDir, Rest) end)}.
|
||||
proc_lib:start_link(?MODULE, main2, [FluName, TcpPort, DataDir, Rest],
|
||||
?INIT_TIMEOUT).
|
||||
|
||||
stop(Pid) ->
|
||||
stop(RegName) when is_atom(RegName) ->
|
||||
case whereis(RegName) of
|
||||
undefined -> ok;
|
||||
Pid -> stop(Pid)
|
||||
end;
|
||||
stop(Pid) when is_pid(Pid) ->
|
||||
case erlang:is_process_alive(Pid) of
|
||||
true ->
|
||||
Pid ! killme,
|
||||
|
@ -101,19 +87,14 @@ stop(Pid) ->
|
|||
error
|
||||
end.
|
||||
|
||||
update_wedge_state(PidSpec, Boolean, EpochId)
|
||||
when (Boolean == true orelse Boolean == false), is_tuple(EpochId) ->
|
||||
PidSpec ! {wedge_state_change, Boolean, EpochId}.
|
||||
update_wedge_state(PidSpec, Boolean, EpochId) ->
|
||||
machi_flu1_append_server:int_update_wedge_state(PidSpec, Boolean, EpochId).
|
||||
|
||||
wedge_myself(PidSpec, EpochId)
|
||||
when is_tuple(EpochId) ->
|
||||
PidSpec ! {wedge_myself, EpochId}.
|
||||
wedge_myself(PidSpec, EpochId) ->
|
||||
machi_flu1_append_server:int_wedge_myself(PidSpec, EpochId).
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
ets_table_name(FluName) when is_atom(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_epoch").
|
||||
|
||||
main2(FluName, TcpPort, DataDir, Props) ->
|
||||
{SendAppendPidToProj_p, ProjectionPid} =
|
||||
case proplists:get_value(projection_store_registered_name, Props) of
|
||||
|
@ -140,27 +121,17 @@ main2(FluName, TcpPort, DataDir, Props) ->
|
|||
{true, undefined}
|
||||
end,
|
||||
Witness_p = proplists:get_value(witness_mode, Props, false),
|
||||
S0 = #state{flu_name=FluName,
|
||||
proj_store=ProjectionPid,
|
||||
tcp_port=TcpPort,
|
||||
data_dir=DataDir,
|
||||
wedged=Wedged_p,
|
||||
witness=Witness_p,
|
||||
etstab=ets_table_name(FluName),
|
||||
epoch_id=EpochId,
|
||||
props=Props},
|
||||
AppendPid = start_append_server(S0, self()),
|
||||
receive
|
||||
append_server_ack -> ok
|
||||
end,
|
||||
|
||||
{ok, AppendPid} = start_append_server(FluName, Witness_p, Wedged_p, EpochId),
|
||||
if SendAppendPidToProj_p ->
|
||||
machi_projection_store:set_wedge_notify_pid(ProjectionPid,
|
||||
AppendPid);
|
||||
machi_projection_store:set_wedge_notify_pid(ProjectionPid, AppendPid);
|
||||
true ->
|
||||
ok
|
||||
end,
|
||||
S1 = S0#state{append_pid=AppendPid},
|
||||
ListenPid = start_listen_server(S1),
|
||||
{ok, ListenerPid} = start_listen_server(FluName, TcpPort, Witness_p, DataDir,
|
||||
ets_table_name(FluName), ProjectionPid,
|
||||
Props),
|
||||
%% io:format(user, "Listener started: ~w~n", [{FluName, ListenerPid}]),
|
||||
|
||||
Config_e = machi_util:make_config_filename(DataDir, "unused"),
|
||||
ok = filelib:ensure_dir(Config_e),
|
||||
|
@ -170,673 +141,24 @@ main2(FluName, TcpPort, DataDir, Props) ->
|
|||
ok = filelib:ensure_dir(Projection_e),
|
||||
|
||||
put(flu_flu_name, FluName),
|
||||
put(flu_append_pid, S1#state.append_pid),
|
||||
put(flu_append_pid, AppendPid),
|
||||
put(flu_projection_pid, ProjectionPid),
|
||||
put(flu_listen_pid, ListenPid),
|
||||
put(flu_listen_pid, ListenerPid),
|
||||
proc_lib:init_ack({ok, self()}),
|
||||
|
||||
receive killme -> ok end,
|
||||
(catch exit(S1#state.append_pid, kill)),
|
||||
(catch exit(AppendPid, kill)),
|
||||
(catch exit(ProjectionPid, kill)),
|
||||
(catch exit(ListenPid, kill)),
|
||||
(catch exit(ListenerPid, kill)),
|
||||
ok.
|
||||
|
||||
start_listen_server(S) ->
|
||||
proc_lib:spawn_link(fun() -> run_listen_server(S) end).
|
||||
start_append_server(FluName, Witness_p, Wedged_p, EpochId) ->
|
||||
machi_flu1_subsup:start_append_server(FluName, Witness_p, Wedged_p, EpochId).
|
||||
|
||||
start_append_server(S, AckPid) ->
|
||||
FluPid = self(),
|
||||
proc_lib:spawn_link(fun() -> run_append_server(FluPid, AckPid, S) end).
|
||||
|
||||
run_listen_server(#state{flu_name=FluName, tcp_port=TcpPort}=S) ->
|
||||
register(make_listener_regname(FluName), self()),
|
||||
SockOpts = ?PB_PACKET_OPTS ++
|
||||
[{reuseaddr, true}, {mode, binary}, {active, false},
|
||||
{backlog,8192}],
|
||||
case gen_tcp:listen(TcpPort, SockOpts) of
|
||||
{ok, LSock} ->
|
||||
listen_server_loop(LSock, S);
|
||||
Else ->
|
||||
error_logger:warning_msg("~s:run_listen_server: "
|
||||
"listen to TCP port ~w: ~w\n",
|
||||
[?MODULE, TcpPort, Else]),
|
||||
exit({?MODULE, run_listen_server, tcp_port, TcpPort, Else})
|
||||
end.
|
||||
|
||||
run_append_server(FluPid, AckPid, #state{flu_name=Name,
|
||||
wedged=Wedged_p,epoch_id=EpochId}=S) ->
|
||||
%% Reminder: Name is the "main" name of the FLU, i.e., no suffix
|
||||
register(Name, self()),
|
||||
TID = ets:new(ets_table_name(Name),
|
||||
[set, protected, named_table, {read_concurrency, true}]),
|
||||
ets:insert(TID, {epoch, {Wedged_p, EpochId}}),
|
||||
AckPid ! append_server_ack,
|
||||
append_server_loop(FluPid, S#state{etstab=TID}).
|
||||
|
||||
listen_server_loop(LSock, S) ->
|
||||
{ok, Sock} = gen_tcp:accept(LSock),
|
||||
spawn_link(fun() -> net_server_loop(Sock, S) end),
|
||||
listen_server_loop(LSock, S).
|
||||
|
||||
append_server_loop(FluPid, #state{data_dir=DataDir, wedged=Wedged_p,
|
||||
witness=Witness_p,
|
||||
epoch_id=OldEpochId, flu_name=FluName}=S) ->
|
||||
AppendServerPid = self(),
|
||||
receive
|
||||
{seq_append, From, _Prefix, _Chunk, _CSum, _Extra, _EpochID}
|
||||
when Witness_p ->
|
||||
%% The FLU's net_server_loop() process ought to filter all
|
||||
%% witness states, but we'll keep this clause for extra
|
||||
%% paranoia.
|
||||
From ! witness,
|
||||
append_server_loop(FluPid, S);
|
||||
{seq_append, From, _Prefix, _Chunk, _CSum, _Extra, _EpochID}
|
||||
when Wedged_p ->
|
||||
From ! wedged,
|
||||
append_server_loop(FluPid, S);
|
||||
{seq_append, From, Prefix, Chunk, CSum, Extra, EpochID} ->
|
||||
spawn(fun() -> append_server_dispatch(From, Prefix,
|
||||
Chunk, CSum, Extra, EpochID,
|
||||
DataDir, AppendServerPid) end),
|
||||
append_server_loop(FluPid, S);
|
||||
{wedge_myself, WedgeEpochId} ->
|
||||
if not Wedged_p andalso WedgeEpochId == OldEpochId ->
|
||||
true = ets:insert(S#state.etstab,
|
||||
{epoch, {true, OldEpochId}}),
|
||||
%% Tell my chain manager that it might want to react to
|
||||
%% this new world.
|
||||
Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
|
||||
spawn(fun() ->
|
||||
catch machi_chain_manager1:trigger_react_to_env(Chmgr)
|
||||
end),
|
||||
append_server_loop(FluPid, S#state{wedged=true});
|
||||
true ->
|
||||
append_server_loop(FluPid, S)
|
||||
end;
|
||||
{wedge_state_change, Boolean, {NewEpoch, _}=NewEpochId} ->
|
||||
OldEpoch = case OldEpochId of {OldE, _} -> OldE;
|
||||
undefined -> -1
|
||||
end,
|
||||
if NewEpoch >= OldEpoch ->
|
||||
true = ets:insert(S#state.etstab,
|
||||
{epoch, {Boolean, NewEpochId}}),
|
||||
append_server_loop(FluPid, S#state{wedged=Boolean,
|
||||
epoch_id=NewEpochId});
|
||||
true ->
|
||||
append_server_loop(FluPid, S)
|
||||
end;
|
||||
{wedge_status, FromPid} ->
|
||||
#state{wedged=Wedged_p, epoch_id=EpochId} = S,
|
||||
FromPid ! {wedge_status_reply, Wedged_p, EpochId},
|
||||
append_server_loop(FluPid, S);
|
||||
Else ->
|
||||
io:format(user, "append_server_loop: WHA? ~p\n", [Else]),
|
||||
append_server_loop(FluPid, S)
|
||||
end.
|
||||
|
||||
net_server_loop(Sock, S) ->
|
||||
case gen_tcp:recv(Sock, 0, ?SERVER_CMD_READ_TIMEOUT) of
|
||||
{ok, Bin} ->
|
||||
{RespBin, S2} =
|
||||
case machi_pb:decode_mpb_ll_request(Bin) of
|
||||
LL_req when LL_req#mpb_ll_request.do_not_alter == 2 ->
|
||||
{R, NewS} = do_pb_ll_request(LL_req, S),
|
||||
{maybe_encode_response(R), mode(low, NewS)};
|
||||
_ ->
|
||||
HL_req = machi_pb:decode_mpb_request(Bin),
|
||||
1 = HL_req#mpb_request.do_not_alter,
|
||||
{R, NewS} = do_pb_hl_request(HL_req, make_high_clnt(S)),
|
||||
{machi_pb:encode_mpb_response(R), mode(high, NewS)}
|
||||
end,
|
||||
if RespBin == async_no_response ->
|
||||
net_server_loop(Sock, S2);
|
||||
true ->
|
||||
case gen_tcp:send(Sock, RespBin) of
|
||||
ok ->
|
||||
net_server_loop(Sock, S2);
|
||||
{error, _} ->
|
||||
(catch gen_tcp:close(Sock)),
|
||||
exit(normal)
|
||||
end
|
||||
end;
|
||||
{error, SockError} ->
|
||||
Msg = io_lib:format("Socket error ~w", [SockError]),
|
||||
R = #mpb_ll_response{req_id= <<>>,
|
||||
generic=#mpb_errorresp{code=1, msg=Msg}},
|
||||
_Resp = machi_pb:encode_mpb_ll_response(R),
|
||||
%% TODO: Weird that sometimes neither catch nor try/catch
|
||||
%% can prevent OTP's SASL from logging an error here.
|
||||
%% Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,.......
|
||||
%% TODO: is this what causes the intermittent PULSE deadlock errors?
|
||||
%% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000),
|
||||
(catch gen_tcp:close(Sock)),
|
||||
exit(normal)
|
||||
end.
|
||||
|
||||
maybe_encode_response(async_no_response=X) ->
|
||||
X;
|
||||
maybe_encode_response(R) ->
|
||||
machi_pb:encode_mpb_ll_response(R).
|
||||
|
||||
mode(Mode, #state{pb_mode=undefined}=S) ->
|
||||
S#state{pb_mode=Mode};
|
||||
mode(_, S) ->
|
||||
S.
|
||||
|
||||
make_high_clnt(#state{high_clnt=undefined}=S) ->
|
||||
{ok, Proj} = machi_projection_store:read_latest_projection(
|
||||
S#state.proj_store, private),
|
||||
Ps = [P_srvr || {_, P_srvr} <- orddict:to_list(
|
||||
Proj#projection_v1.members_dict)],
|
||||
{ok, Clnt} = machi_cr_client:start_link(Ps),
|
||||
S#state{high_clnt=Clnt};
|
||||
make_high_clnt(S) ->
|
||||
S.
|
||||
|
||||
do_pb_ll_request(#mpb_ll_request{req_id=ReqID}, #state{pb_mode=high}=S) ->
|
||||
Result = {high_error, 41, "Low protocol request while in high mode"},
|
||||
{machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
|
||||
do_pb_ll_request(PB_request, S) ->
|
||||
Req = machi_pb_translate:from_pb_request(PB_request),
|
||||
{ReqID, Cmd, Result, S2} =
|
||||
case Req of
|
||||
{RqID, {LowCmd, _}=CMD}
|
||||
when LowCmd == low_proj;
|
||||
LowCmd == low_wedge_status; LowCmd == low_list_files ->
|
||||
%% Skip wedge check for projection commands!
|
||||
%% Skip wedge check for these unprivileged commands
|
||||
{Rs, NewS} = do_pb_ll_request3(CMD, S),
|
||||
{RqID, CMD, Rs, NewS};
|
||||
{RqID, CMD} ->
|
||||
EpochID = element(2, CMD), % by common convention
|
||||
{Rs, NewS} = do_pb_ll_request2(EpochID, CMD, S),
|
||||
{RqID, CMD, Rs, NewS}
|
||||
end,
|
||||
{machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
|
||||
|
||||
do_pb_ll_request2(EpochID, CMD, S) ->
|
||||
{Wedged_p, CurrentEpochID} = ets:lookup_element(S#state.etstab, epoch, 2),
|
||||
if Wedged_p == true ->
|
||||
{{error, wedged}, S#state{epoch_id=CurrentEpochID}};
|
||||
is_tuple(EpochID)
|
||||
andalso
|
||||
EpochID /= CurrentEpochID ->
|
||||
{Epoch, _} = EpochID,
|
||||
{CurrentEpoch, _} = CurrentEpochID,
|
||||
if Epoch < CurrentEpoch ->
|
||||
ok;
|
||||
true ->
|
||||
%% We're at same epoch # but different checksum, or
|
||||
%% we're at a newer/bigger epoch #.
|
||||
wedge_myself(S#state.flu_name, CurrentEpochID),
|
||||
ok
|
||||
end,
|
||||
{{error, bad_epoch}, S#state{epoch_id=CurrentEpochID}};
|
||||
true ->
|
||||
do_pb_ll_request3(CMD, S#state{epoch_id=CurrentEpochID})
|
||||
end.
|
||||
|
||||
%% Witness status does not matter below.
|
||||
do_pb_ll_request3({low_echo, _BogusEpochID, Msg}, S) ->
|
||||
{Msg, S};
|
||||
do_pb_ll_request3({low_auth, _BogusEpochID, _User, _Pass}, S) ->
|
||||
{-6, S};
|
||||
do_pb_ll_request3({low_wedge_status, _EpochID}, S) ->
|
||||
{do_server_wedge_status(S), S};
|
||||
do_pb_ll_request3({low_proj, PCMD}, S) ->
|
||||
{do_server_proj_request(PCMD, S), S};
|
||||
%% Witness status *matters* below
|
||||
do_pb_ll_request3({low_append_chunk, _EpochID, PKey, Prefix, Chunk, CSum_tag,
|
||||
CSum, ChunkExtra},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_append_chunk(PKey, Prefix, Chunk, CSum_tag, CSum,
|
||||
ChunkExtra, S), S};
|
||||
do_pb_ll_request3({low_write_chunk, _EpochID, File, Offset, Chunk, CSum_tag,
|
||||
CSum},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, S), S};
|
||||
do_pb_ll_request3({low_read_chunk, _EpochID, File, Offset, Size, Opts},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_read_chunk(File, Offset, Size, Opts, S), S};
|
||||
do_pb_ll_request3({low_checksum_list, _EpochID, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_checksum_listing(File, S), S};
|
||||
do_pb_ll_request3({low_list_files, _EpochID},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_list_files(S), S};
|
||||
do_pb_ll_request3({low_delete_migration, _EpochID, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_delete_migration(File, S),
|
||||
#state{witness=false}=S};
|
||||
do_pb_ll_request3({low_trunc_hack, _EpochID, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_trunc_hack(File, S), S};
|
||||
do_pb_ll_request3(_, #state{witness=true}=S) ->
|
||||
{{error, bad_arg}, S}. % TODO: new status code??
|
||||
|
||||
do_pb_hl_request(#mpb_request{req_id=ReqID}, #state{pb_mode=low}=S) ->
|
||||
Result = {low_error, 41, "High protocol request while in low mode"},
|
||||
{machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
|
||||
do_pb_hl_request(PB_request, S) ->
|
||||
{ReqID, Cmd} = machi_pb_translate:from_pb_request(PB_request),
|
||||
{Result, S2} = do_pb_hl_request2(Cmd, S),
|
||||
{machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
|
||||
|
||||
do_pb_hl_request2({high_echo, Msg}, S) ->
|
||||
{Msg, S};
|
||||
do_pb_hl_request2({high_auth, _User, _Pass}, S) ->
|
||||
{-77, S};
|
||||
do_pb_hl_request2({high_append_chunk, _todoPK, Prefix, ChunkBin, TaggedCSum,
|
||||
ChunkExtra}, #state{high_clnt=Clnt}=S) ->
|
||||
Chunk = {TaggedCSum, ChunkBin},
|
||||
Res = machi_cr_client:append_chunk_extra(Clnt, Prefix, Chunk,
|
||||
ChunkExtra),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_write_chunk, File, Offset, ChunkBin, TaggedCSum},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
Chunk = {TaggedCSum, ChunkBin},
|
||||
Res = machi_cr_client:write_chunk(Clnt, File, Offset, Chunk),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_read_chunk, File, Offset, Size},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
Res = machi_cr_client:read_chunk(Clnt, File, Offset, Size),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_checksum_list, File}, #state{high_clnt=Clnt}=S) ->
|
||||
Res = machi_cr_client:checksum_list(Clnt, File),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_list_files}, #state{high_clnt=Clnt}=S) ->
|
||||
Res = machi_cr_client:list_files(Clnt),
|
||||
{Res, S}.
|
||||
|
||||
do_server_proj_request({get_latest_epochid, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:get_latest_epochid(ProjStore, ProjType);
|
||||
do_server_proj_request({read_latest_projection, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:read_latest_projection(ProjStore, ProjType);
|
||||
do_server_proj_request({read_projection, ProjType, Epoch},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:read(ProjStore, ProjType, Epoch);
|
||||
do_server_proj_request({write_projection, ProjType, Proj},
|
||||
#state{flu_name=FluName, proj_store=ProjStore}) ->
|
||||
if Proj#projection_v1.epoch_number == ?SPAM_PROJ_EPOCH ->
|
||||
%% io:format(user, "DBG ~s ~w ~P\n", [?MODULE, ?LINE, Proj, 5]),
|
||||
Chmgr = machi_flu_psup:make_fitness_regname(FluName),
|
||||
[Map] = Proj#projection_v1.dbg,
|
||||
catch machi_fitness:send_fitness_update_spam(
|
||||
Chmgr, Proj#projection_v1.author_server, Map);
|
||||
true ->
|
||||
catch machi_projection_store:write(ProjStore, ProjType, Proj)
|
||||
end;
|
||||
do_server_proj_request({get_all_projections, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:get_all_projections(ProjStore, ProjType);
|
||||
do_server_proj_request({list_all_projections, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:list_all_projections(ProjStore, ProjType);
|
||||
do_server_proj_request({kick_projection_reaction},
|
||||
#state{flu_name=FluName}) ->
|
||||
%% Tell my chain manager that it might want to react to
|
||||
%% this new world.
|
||||
Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
|
||||
spawn(fun() ->
|
||||
catch machi_chain_manager1:trigger_react_to_env(Chmgr)
|
||||
end),
|
||||
async_no_response.
|
||||
|
||||
do_server_append_chunk(PKey, Prefix, Chunk, CSum_tag, CSum,
|
||||
ChunkExtra, S) ->
|
||||
case sanitize_file_string(Prefix) of
|
||||
ok ->
|
||||
do_server_append_chunk2(PKey, Prefix, Chunk, CSum_tag, CSum,
|
||||
ChunkExtra, S);
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_append_chunk2(_PKey, Prefix, Chunk, CSum_tag, Client_CSum,
|
||||
ChunkExtra, #state{flu_name=FluName,
|
||||
epoch_id=EpochID}=_S) ->
|
||||
%% TODO: Do anything with PKey?
|
||||
try
|
||||
TaggedCSum = check_or_make_tagged_checksum(CSum_tag, Client_CSum,Chunk),
|
||||
FluName ! {seq_append, self(), Prefix, Chunk, TaggedCSum,
|
||||
ChunkExtra, EpochID},
|
||||
receive
|
||||
{assignment, Offset, File} ->
|
||||
Size = iolist_size(Chunk),
|
||||
{ok, {Offset, Size, File}};
|
||||
witness ->
|
||||
{error, bad_arg};
|
||||
wedged ->
|
||||
{error, wedged}
|
||||
after 10*1000 ->
|
||||
{error, partition}
|
||||
end
|
||||
catch
|
||||
throw:{bad_csum, _CS} ->
|
||||
{error, bad_checksum};
|
||||
error:badarg ->
|
||||
error_logger:error_msg("Message send to ~p gave badarg, make certain server is running with correct registered name\n", [?MODULE]),
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum,
|
||||
#state{data_dir=DataDir}=S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
CSumPath = machi_util:make_checksum_filename(DataDir, File),
|
||||
case file:open(CSumPath, [append, raw, binary]) of
|
||||
{ok, FHc} ->
|
||||
Path = DataDir ++ "/data/" ++
|
||||
machi_util:make_string(File),
|
||||
{ok, FHd} = file:open(Path, [read, write, raw, binary]),
|
||||
try
|
||||
do_server_write_chunk2(
|
||||
File, Offset, Chunk, CSum_tag, CSum, DataDir,
|
||||
FHc, FHd)
|
||||
after
|
||||
(catch file:close(FHc)),
|
||||
(catch file:close(FHd))
|
||||
end;
|
||||
{error, enoent} ->
|
||||
ok = filelib:ensure_dir(CSumPath),
|
||||
do_server_write_chunk(File, Offset, Chunk, CSum_tag,
|
||||
CSum, S)
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_write_chunk2(_File, Offset, Chunk, CSum_tag,
|
||||
Client_CSum, _DataDir, FHc, FHd) ->
|
||||
try
|
||||
TaggedCSum = check_or_make_tagged_checksum(CSum_tag, Client_CSum,Chunk),
|
||||
Size = iolist_size(Chunk),
|
||||
case file:pwrite(FHd, Offset, Chunk) of
|
||||
ok ->
|
||||
CSum_info = encode_csum_file_entry(Offset, Size, TaggedCSum),
|
||||
ok = file:write(FHc, CSum_info),
|
||||
ok;
|
||||
_Else3 ->
|
||||
machi_util:verb("Else3 ~p ~p ~p\n",
|
||||
[Offset, Size, _Else3]),
|
||||
{error, bad_arg}
|
||||
end
|
||||
catch
|
||||
throw:{bad_csum, _CS} ->
|
||||
{error, bad_checksum};
|
||||
error:badarg ->
|
||||
error_logger:error_msg("Message send to ~p gave badarg, make certain server is running with correct registered name\n", [?MODULE]),
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_read_chunk(File, Offset, Size, _Opts, #state{data_dir=DataDir})->
|
||||
%% TODO: Look inside Opts someday.
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
{_, Path} = machi_util:make_data_filename(DataDir, File),
|
||||
case file:open(Path, [read, binary, raw]) of
|
||||
{ok, FH} ->
|
||||
try
|
||||
case file:pread(FH, Offset, Size) of
|
||||
{ok, Bytes} when byte_size(Bytes) == Size ->
|
||||
{ok, Bytes};
|
||||
{ok, Bytes} ->
|
||||
machi_util:verb("ok read but wanted ~p got ~p: ~p @ offset ~p\n",
|
||||
[Size,size(Bytes),File,Offset]),
|
||||
io:format(user, "ok read but wanted ~p got ~p: ~p @ offset ~p\n",
|
||||
[Size,size(Bytes),File,Offset]),
|
||||
{error, partial_read};
|
||||
eof ->
|
||||
{error, not_written}; %% TODO perhaps_do_net_server_ec_read(Sock, FH);
|
||||
_Else2 ->
|
||||
machi_util:verb("Else2 ~p ~p ~P\n",
|
||||
[Offset, Size, _Else2, 20]),
|
||||
{error, bad_read}
|
||||
end
|
||||
after
|
||||
file:close(FH)
|
||||
end;
|
||||
{error, enoent} ->
|
||||
{error, not_written};
|
||||
{error, _Else} ->
|
||||
io:format(user, "Unexpected ~p at ~p ~p\n",
|
||||
[_Else, ?MODULE, ?LINE]),
|
||||
{error, bad_arg}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_checksum_listing(File, #state{data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
ok = sync_checksum_file(File),
|
||||
CSumPath = machi_util:make_checksum_filename(DataDir, File),
|
||||
%% TODO: If this file is legitimately bigger than our
|
||||
%% {packet_size,N} limit, then we'll have a difficult time, eh?
|
||||
case file:read_file(CSumPath) of
|
||||
{ok, Bin} ->
|
||||
if byte_size(Bin) > (?PB_MAX_MSG_SIZE - 1024) ->
|
||||
%% TODO: Fix this limitation by streaming the
|
||||
%% binary in multiple smaller PB messages.
|
||||
%% Also, don't read the file all at once. ^_^
|
||||
error_logger:error_msg("~s:~w oversize ~s\n",
|
||||
[?MODULE, ?LINE, CSumPath]),
|
||||
{error, bad_arg};
|
||||
true ->
|
||||
{ok, Bin}
|
||||
end;
|
||||
{error, enoent} ->
|
||||
{error, no_such_file};
|
||||
{error, _} ->
|
||||
{error, bad_arg}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_list_files(#state{data_dir=DataDir}=_S) ->
|
||||
{_, WildPath} = machi_util:make_data_filename(DataDir, ""),
|
||||
Files = filelib:wildcard("*", WildPath),
|
||||
{ok, [begin
|
||||
{ok, FI} = file:read_file_info(WildPath ++ "/" ++ File),
|
||||
Size = FI#file_info.size,
|
||||
{Size, File}
|
||||
end || File <- Files]}.
|
||||
|
||||
do_server_wedge_status(S) ->
|
||||
{Wedged_p, CurrentEpochID0} = ets:lookup_element(S#state.etstab, epoch, 2),
|
||||
CurrentEpochID = if CurrentEpochID0 == undefined ->
|
||||
?DUMMY_PV1_EPOCH;
|
||||
true ->
|
||||
CurrentEpochID0
|
||||
end,
|
||||
{Wedged_p, CurrentEpochID}.
|
||||
|
||||
do_server_delete_migration(File, #state{data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
{_, Path} = machi_util:make_data_filename(DataDir, File),
|
||||
case file:delete(Path) of
|
||||
ok ->
|
||||
ok;
|
||||
{error, enoent} ->
|
||||
{error, no_such_file};
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_trunc_hack(File, #state{data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
{_, Path} = machi_util:make_data_filename(DataDir, File),
|
||||
case file:open(Path, [read, write, binary, raw]) of
|
||||
{ok, FH} ->
|
||||
try
|
||||
{ok, ?MINIMUM_OFFSET} = file:position(FH,
|
||||
?MINIMUM_OFFSET),
|
||||
ok = file:truncate(FH),
|
||||
ok
|
||||
after
|
||||
file:close(FH)
|
||||
end;
|
||||
{error, enoent} ->
|
||||
{error, no_such_file};
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
append_server_dispatch(From, Prefix, Chunk, CSum, Extra, EpochID,
|
||||
DataDir, LinkPid) ->
|
||||
Pid = write_server_get_pid(Prefix, EpochID, DataDir, LinkPid),
|
||||
Pid ! {seq_append, From, Prefix, Chunk, CSum, Extra, EpochID},
|
||||
exit(normal).
|
||||
|
||||
sanitize_file_string(Str) ->
|
||||
case re:run(Str, "/") of
|
||||
nomatch ->
|
||||
ok;
|
||||
_ ->
|
||||
error
|
||||
end.
|
||||
|
||||
sync_checksum_file(File) ->
|
||||
Prefix = re:replace(File, "\\..*", "", [{return, binary}]),
|
||||
case write_server_find_pid(Prefix) of
|
||||
undefined ->
|
||||
ok;
|
||||
Pid ->
|
||||
Ref = make_ref(),
|
||||
Pid ! {sync_stuff, self(), Ref},
|
||||
receive
|
||||
{sync_finished, Ref} ->
|
||||
ok
|
||||
after 5000 ->
|
||||
case write_server_find_pid(Prefix) of
|
||||
undefined ->
|
||||
ok;
|
||||
Pid2 when Pid2 /= Pid ->
|
||||
ok;
|
||||
_Pid2 ->
|
||||
error
|
||||
end
|
||||
end
|
||||
end.
|
||||
|
||||
write_server_get_pid(Prefix, EpochID, DataDir, LinkPid) ->
|
||||
case write_server_find_pid(Prefix) of
|
||||
undefined ->
|
||||
start_seq_append_server(Prefix, EpochID, DataDir, LinkPid),
|
||||
timer:sleep(1),
|
||||
write_server_get_pid(Prefix, EpochID, DataDir, LinkPid);
|
||||
Pid ->
|
||||
Pid
|
||||
end.
|
||||
|
||||
write_server_find_pid(Prefix) ->
|
||||
FluName = machi_util:make_regname(Prefix),
|
||||
whereis(FluName).
|
||||
|
||||
start_seq_append_server(Prefix, EpochID, DataDir, AppendServerPid) ->
|
||||
proc_lib:spawn_link(fun() ->
|
||||
%% The following is only necessary to
|
||||
%% make nice process relationships in
|
||||
%% 'appmon' and related tools.
|
||||
put('$ancestors', [AppendServerPid]),
|
||||
put('$initial_call', {x,y,3}),
|
||||
link(AppendServerPid),
|
||||
run_seq_append_server(Prefix, EpochID, DataDir)
|
||||
end).
|
||||
|
||||
run_seq_append_server(Prefix, EpochID, DataDir) ->
|
||||
true = register(machi_util:make_regname(Prefix), self()),
|
||||
run_seq_append_server2(Prefix, EpochID, DataDir).
|
||||
|
||||
run_seq_append_server2(Prefix, EpochID, DataDir) ->
|
||||
FileNum = machi_util:read_max_filenum(DataDir, Prefix) + 1,
|
||||
case machi_util:increment_max_filenum(DataDir, Prefix) of
|
||||
ok ->
|
||||
machi_util:info_msg("start: ~p server at file ~w\n",
|
||||
[Prefix, FileNum]),
|
||||
seq_append_server_loop(DataDir, Prefix, EpochID, FileNum);
|
||||
Else ->
|
||||
error_logger:error_msg("start: ~p server at file ~w: ~p\n",
|
||||
[Prefix, FileNum, Else]),
|
||||
exit(Else)
|
||||
|
||||
end.
|
||||
|
||||
-spec seq_name_hack() -> string().
|
||||
seq_name_hack() ->
|
||||
lists:flatten(io_lib:format("~.36B~.36B",
|
||||
[element(3,now()),
|
||||
list_to_integer(os:getpid())])).
|
||||
|
||||
seq_append_server_loop(DataDir, Prefix, EpochID, FileNum) ->
|
||||
SequencerNameHack = seq_name_hack(),
|
||||
{File, FullPath} = machi_util:make_data_filename(
|
||||
DataDir, Prefix, SequencerNameHack, FileNum),
|
||||
{ok, FHd} = file:open(FullPath,
|
||||
[read, write, raw, binary]),
|
||||
CSumPath = machi_util:make_checksum_filename(
|
||||
DataDir, Prefix, SequencerNameHack, FileNum),
|
||||
{ok, FHc} = file:open(CSumPath, [append, raw, binary]),
|
||||
seq_append_server_loop(DataDir, Prefix, File, {FHd,FHc}, EpochID, FileNum,
|
||||
?MINIMUM_OFFSET).
|
||||
|
||||
seq_append_server_loop(DataDir, Prefix, _File, {FHd,FHc}, EpochID,
|
||||
FileNum, Offset)
|
||||
when Offset > ?MAX_FILE_SIZE ->
|
||||
ok = file:close(FHd),
|
||||
ok = file:close(FHc),
|
||||
machi_util:info_msg("rollover: ~p server at file ~w offset ~w\n",
|
||||
[Prefix, FileNum, Offset]),
|
||||
run_seq_append_server2(Prefix, EpochID, DataDir);
|
||||
seq_append_server_loop(DataDir, Prefix, File, {FHd,FHc}=FH_, EpochID,
|
||||
FileNum, Offset) ->
|
||||
receive
|
||||
{seq_append, From, Prefix, Chunk, TaggedCSum, Extra, R_EpochID}
|
||||
when R_EpochID == EpochID ->
|
||||
if Chunk /= <<>> ->
|
||||
ok = file:pwrite(FHd, Offset, Chunk);
|
||||
true ->
|
||||
ok
|
||||
end,
|
||||
From ! {assignment, Offset, File},
|
||||
Size = iolist_size(Chunk),
|
||||
CSum_info = encode_csum_file_entry(Offset, Size, TaggedCSum),
|
||||
ok = file:write(FHc, CSum_info),
|
||||
seq_append_server_loop(DataDir, Prefix, File, FH_, EpochID,
|
||||
FileNum, Offset + Size + Extra);
|
||||
{seq_append, _From, _Prefix, _Chunk, _TCSum, _Extra, R_EpochID}=MSG ->
|
||||
%% Rare'ish event: send MSG to myself so it doesn't get lost
|
||||
%% while we recurse around to pick up a new FileNum.
|
||||
self() ! MSG,
|
||||
machi_util:info_msg("rollover: ~p server at file ~w offset ~w "
|
||||
"by new epoch_id ~W\n",
|
||||
[Prefix, FileNum, Offset, R_EpochID, 8]),
|
||||
run_seq_append_server2(Prefix, R_EpochID, DataDir);
|
||||
{sync_stuff, FromPid, Ref} ->
|
||||
file:sync(FHc),
|
||||
FromPid ! {sync_finished, Ref},
|
||||
seq_append_server_loop(DataDir, Prefix, File, FH_, EpochID,
|
||||
FileNum, Offset)
|
||||
after 30*1000 ->
|
||||
ok = file:close(FHd),
|
||||
ok = file:close(FHc),
|
||||
machi_util:info_msg("stop: ~p server ~p at file ~w offset ~w\n",
|
||||
[Prefix, self(), FileNum, Offset]),
|
||||
exit(normal)
|
||||
end.
|
||||
|
||||
make_listener_regname(BaseName) ->
|
||||
list_to_atom(atom_to_list(BaseName) ++ "_listener").
|
||||
start_listen_server(FluName, TcpPort, Witness_p, DataDir, EtsTab, ProjectionPid,
|
||||
Props) ->
|
||||
machi_flu1_subsup:start_listener(FluName, TcpPort, Witness_p, DataDir,
|
||||
EtsTab, ProjectionPid, Props).
|
||||
|
||||
%% This is the name of the projection store that is spawned by the
|
||||
%% *flu*, for use primarily in testing scenarios. In normal use, we
|
||||
|
@ -848,119 +170,8 @@ make_listener_regname(BaseName) ->
|
|||
make_projection_server_regname(BaseName) ->
|
||||
list_to_atom(atom_to_list(BaseName) ++ "_pstore").
|
||||
|
||||
%% @doc Encode `Offset + Size + TaggedCSum' into an `iolist()' type for
|
||||
%% internal storage by the FLU.
|
||||
|
||||
-spec encode_csum_file_entry(
|
||||
machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()) ->
|
||||
iolist().
|
||||
encode_csum_file_entry(Offset, Size, TaggedCSum) ->
|
||||
Len = 8 + 4 + byte_size(TaggedCSum),
|
||||
[<<Len:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big>>,
|
||||
TaggedCSum].
|
||||
|
||||
%% @doc Encode `Offset + Size + TaggedCSum' into an `binary()' type for
|
||||
%% internal storage by the FLU.
|
||||
|
||||
-spec encode_csum_file_entry_bin(
|
||||
machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()) ->
|
||||
binary().
|
||||
encode_csum_file_entry_bin(Offset, Size, TaggedCSum) ->
|
||||
Len = 8 + 4 + byte_size(TaggedCSum),
|
||||
<<Len:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big,
|
||||
TaggedCSum/binary>>.
|
||||
|
||||
%% @doc Decode a single `binary()' blob into an
|
||||
%% `{Offset,Size,TaggedCSum}' tuple.
|
||||
%%
|
||||
%% The internal encoding (which is currently exposed to the outside world
|
||||
%% via this function and related ones) is:
|
||||
%%
|
||||
%% <ul>
|
||||
%% <li> 1 byte: record length
|
||||
%% </li>
|
||||
%% <li> 8 bytes (unsigned big-endian): byte offset
|
||||
%% </li>
|
||||
%% <li> 4 bytes (unsigned big-endian): chunk size
|
||||
%% </li>
|
||||
%% <li> all remaining bytes: tagged checksum (1st byte = type tag)
|
||||
%% </li>
|
||||
%% </ul>
|
||||
%%
|
||||
%% See `machi.hrl' for the tagged checksum types, e.g.,
|
||||
%% `?CSUM_TAG_NONE'.
|
||||
|
||||
-spec decode_csum_file_entry(binary()) ->
|
||||
error |
|
||||
{machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()}.
|
||||
decode_csum_file_entry(<<_:8/unsigned-big, Offset:64/unsigned-big, Size:32/unsigned-big, TaggedCSum/binary>>) ->
|
||||
{Offset, Size, TaggedCSum};
|
||||
decode_csum_file_entry(_Else) ->
|
||||
error.
|
||||
|
||||
%% @doc Split a `binary()' blob of `checksum_list' data into a list of
|
||||
%% unparsed `binary()' blobs, one per entry.
|
||||
%%
|
||||
%% Decode the unparsed blobs with {@link decode_csum_file_entry/1}, if
|
||||
%% desired.
|
||||
%%
|
||||
%% The return value `TrailingJunk' is unparseable bytes at the end of
|
||||
%% the checksum list blob.
|
||||
|
||||
-spec split_checksum_list_blob(binary()) ->
|
||||
{list(binary()), TrailingJunk::binary()}.
|
||||
split_checksum_list_blob(Bin) ->
|
||||
split_checksum_list_blob(Bin, []).
|
||||
|
||||
split_checksum_list_blob(<<Len:8/unsigned-big, Part:Len/binary, Rest/binary>>, Acc)->
|
||||
case get(hack_length) of
|
||||
Len -> ok;
|
||||
_ -> put(hack_different, true)
|
||||
end,
|
||||
split_checksum_list_blob(Rest, [<<Len:8/unsigned-big, Part/binary>>|Acc]);
|
||||
split_checksum_list_blob(Rest, Acc) ->
|
||||
{lists:reverse(Acc), Rest}.
|
||||
|
||||
%% @doc Split a `binary()' blob of `checksum_list' data into a list of
|
||||
%% `{Offset,Size,TaggedCSum}' tuples.
|
||||
|
||||
-spec split_checksum_list_blob_decode(binary()) ->
|
||||
{list({machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()}),
|
||||
TrailingJunk::binary()}.
|
||||
split_checksum_list_blob_decode(Bin) ->
|
||||
split_checksum_list_blob_decode(Bin, []).
|
||||
|
||||
split_checksum_list_blob_decode(<<Len:8/unsigned-big, Part:Len/binary, Rest/binary>>, Acc)->
|
||||
One = <<Len:8/unsigned-big, Part/binary>>,
|
||||
case decode_csum_file_entry(One) of
|
||||
error ->
|
||||
split_checksum_list_blob_decode(Rest, Acc);
|
||||
DecOne ->
|
||||
split_checksum_list_blob_decode(Rest, [DecOne|Acc])
|
||||
end;
|
||||
split_checksum_list_blob_decode(Rest, Acc) ->
|
||||
{lists:reverse(Acc), Rest}.
|
||||
|
||||
check_or_make_tagged_checksum(?CSUM_TAG_NONE, _Client_CSum, Chunk) ->
|
||||
%% TODO: If the client was foolish enough to use
|
||||
%% this type of non-checksum, then the client gets
|
||||
%% what it deserves wrt data integrity, alas. In
|
||||
%% the client-side Chain Replication method, each
|
||||
%% server will calculated this independently, which
|
||||
%% isn't exactly what ought to happen for best data
|
||||
%% integrity checking. In server-side CR, the csum
|
||||
%% should be calculated by the head and passed down
|
||||
%% the chain together with the value.
|
||||
CS = machi_util:checksum_chunk(Chunk),
|
||||
machi_util:make_tagged_csum(server_sha, CS);
|
||||
check_or_make_tagged_checksum(?CSUM_TAG_CLIENT_SHA, Client_CSum, Chunk) ->
|
||||
CS = machi_util:checksum_chunk(Chunk),
|
||||
if CS == Client_CSum ->
|
||||
machi_util:make_tagged_csum(server_sha,
|
||||
Client_CSum);
|
||||
true ->
|
||||
throw({bad_csum, CS})
|
||||
end.
|
||||
ets_table_name(FluName) when is_atom(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_epoch").
|
||||
|
||||
-ifdef(TEST).
|
||||
|
||||
|
@ -1000,7 +211,7 @@ timing_demo_test2() ->
|
|||
{HexUSec, _} =
|
||||
timer:tc(fun() ->
|
||||
lists:foldl(fun(X, _) ->
|
||||
B = encode_csum_file_entry_hex(X, 100, CSum),
|
||||
B = machi_checksums:encode_csum_file_entry_hex(X, 100, CSum),
|
||||
%% file:write(ZZZ, [B, 10]),
|
||||
decode_csum_file_entry_hex(list_to_binary(B))
|
||||
end, x, Xs)
|
||||
|
@ -1013,14 +224,14 @@ timing_demo_test2() ->
|
|||
{NotSortedUSec, _} =
|
||||
timer:tc(fun() ->
|
||||
lists:foldl(fun(X, _) ->
|
||||
B = encode_csum_file_entry(X, 100, CSum),
|
||||
decode_csum_file_entry(list_to_binary(B))
|
||||
B = machi_checksums:encode_csum_file_entry(X, 100, CSum),
|
||||
machi_checksums:decode_csum_file_entry(list_to_binary(B))
|
||||
end, x, Xs)
|
||||
end),
|
||||
io:format(user, "~.3f sec\n", [NotSortedUSec / 1000000]),
|
||||
|
||||
NotHexList = lists:foldl(fun(X, Acc) ->
|
||||
B = encode_csum_file_entry(X, 100, CSum),
|
||||
B = machi_checksums:encode_csum_file_entry(X, 100, CSum),
|
||||
[B|Acc]
|
||||
end, [], Xs),
|
||||
NotHexBin = iolist_to_binary(NotHexList),
|
||||
|
@ -1031,7 +242,7 @@ timing_demo_test2() ->
|
|||
timer:tc(fun() ->
|
||||
put(hack_length, 29),
|
||||
put(hack_different, false),
|
||||
{Sorted, _Leftover} = split_checksum_list_blob(NotHexBin),
|
||||
{Sorted, _Leftover} = machi_checksums:split_checksum_list_blob(NotHexBin),
|
||||
io:format(user, " Leftover ~p (hack_different ~p) ", [_Leftover, get(hack_different)]),
|
||||
Sorted
|
||||
end),
|
||||
|
@ -1105,7 +316,7 @@ sort_input_fun(FH, PrevStuff) ->
|
|||
true ->
|
||||
<<PrevStuff/binary, NewStuff/binary>>
|
||||
end,
|
||||
{SplitRes, Leftover} = split_checksum_list_blob(AllStuff),
|
||||
{SplitRes, Leftover} = machi_checksums:split_checksum_list_blob(AllStuff),
|
||||
{SplitRes, sort_input_fun(FH, Leftover)};
|
||||
eof ->
|
||||
end_of_input
|
||||
|
|
193
src/machi_flu1_append_server.erl
Normal file
193
src/machi_flu1_append_server.erl
Normal file
|
@ -0,0 +1,193 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc Machi FLU1 append serialization server process
|
||||
|
||||
-module(machi_flu1_append_server).
|
||||
|
||||
-behavior(gen_server).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-ifdef(TEST).
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-endif. % TEST
|
||||
|
||||
-export([start_link/4]).
|
||||
|
||||
-export([init/1]).
|
||||
-export([handle_call/3, handle_cast/2, handle_info/2,
|
||||
terminate/2, code_change/3]).
|
||||
-export([int_update_wedge_state/3, int_wedge_myself/2]).
|
||||
-export([current_state/1, format_state/1]).
|
||||
|
||||
-record(state, {
|
||||
flu_name :: atom(),
|
||||
witness = false :: boolean(),
|
||||
wedged = true :: boolean(),
|
||||
etstab :: ets:tid(),
|
||||
epoch_id :: 'undefined' | machi_dt:epoch_id()
|
||||
}).
|
||||
|
||||
-define(INIT_TIMEOUT, 60*1000).
|
||||
-define(CALL_TIMEOUT, 60*1000).
|
||||
|
||||
-spec start_link(pv1_server(), boolean(), boolean(),
|
||||
undefined | machi_dt:epoch_id()) -> {ok, pid()}.
|
||||
start_link(Fluname, Witness_p, Wedged_p, EpochId) ->
|
||||
%% Reminder: Name is the "main" name of the FLU, i.e., no suffix
|
||||
gen_server:start_link({local, Fluname},
|
||||
?MODULE, [Fluname, Witness_p, Wedged_p, EpochId],
|
||||
[{timeout, ?INIT_TIMEOUT}]).
|
||||
|
||||
-spec current_state(atom() | pid()) -> term().
|
||||
current_state(PidSpec) ->
|
||||
gen_server:call(PidSpec, current_state, ?CALL_TIMEOUT).
|
||||
|
||||
format_state(State) ->
|
||||
Fields = record_info(fields, state),
|
||||
[_Name | Values] = tuple_to_list(State),
|
||||
lists:zip(Fields, Values).
|
||||
|
||||
int_update_wedge_state(PidSpec, Boolean, EpochId)
|
||||
when is_boolean(Boolean), is_tuple(EpochId) ->
|
||||
gen_server:cast(PidSpec, {wedge_state_change, Boolean, EpochId}).
|
||||
|
||||
int_wedge_myself(PidSpec, EpochId)
|
||||
when is_tuple(EpochId) ->
|
||||
gen_server:cast(PidSpec, {wedge_myself, EpochId}).
|
||||
|
||||
init([Fluname, Witness_p, Wedged_p, EpochId]) ->
|
||||
TID = ets:new(machi_flu1:ets_table_name(Fluname),
|
||||
[set, protected, named_table, {read_concurrency, true}]),
|
||||
ets:insert(TID, {epoch, {Wedged_p, EpochId}}),
|
||||
{ok, #state{flu_name=Fluname, witness=Witness_p, wedged=Wedged_p,
|
||||
etstab=TID, epoch_id=EpochId}}.
|
||||
|
||||
handle_call({seq_append, _From2, _NSInfo, _EpochID, _Prefix, _Chunk, _TCSum, _Opts},
|
||||
_From, #state{witness=true}=S) ->
|
||||
%% The FLU's machi_flu1_net_server process ought to filter all
|
||||
%% witness states, but we'll keep this clause for extra
|
||||
%% paranoia.
|
||||
{reply, witness, S};
|
||||
handle_call({seq_append, _From2, _NSInfo, _EpochID, _Prefix, _Chunk, _TCSum, _Opts},
|
||||
_From, #state{wedged=true}=S) ->
|
||||
{reply, wedged, S};
|
||||
handle_call({seq_append, _From2, NSInfo, EpochID,
|
||||
Prefix, Chunk, TCSum, Opts},
|
||||
From, #state{flu_name=FluName, epoch_id=OldEpochId}=S) ->
|
||||
%% Old is the one from our state, plain old 'EpochID' comes
|
||||
%% from the client.
|
||||
_ = case OldEpochId of
|
||||
EpochID ->
|
||||
spawn(fun() ->
|
||||
append_server_dispatch(From, NSInfo,
|
||||
Prefix, Chunk, TCSum, Opts,
|
||||
FluName, EpochID)
|
||||
end),
|
||||
{noreply, S};
|
||||
_ ->
|
||||
{reply, {error, bad_epoch}, S}
|
||||
end;
|
||||
%% TODO: Who sends this message?
|
||||
handle_call(wedge_status, _From,
|
||||
#state{wedged=Wedged_p, epoch_id=EpochId} = S) ->
|
||||
{reply, {wedge_status_reply, Wedged_p, EpochId}, S};
|
||||
handle_call(current_state, _From, S) ->
|
||||
{reply, S, S};
|
||||
handle_call(Else, From, S) ->
|
||||
io:format(user, "~s:handle_call: WHA? from=~w ~w\n", [?MODULE, From, Else]),
|
||||
{noreply, S}.
|
||||
|
||||
handle_cast({wedge_myself, WedgeEpochId},
|
||||
#state{flu_name=FluName, wedged=Wedged_p, epoch_id=OldEpochId}=S) ->
|
||||
if not Wedged_p andalso WedgeEpochId == OldEpochId ->
|
||||
true = ets:insert(S#state.etstab,
|
||||
{epoch, {true, OldEpochId}}),
|
||||
%% Tell my chain manager that it might want to react to
|
||||
%% this new world.
|
||||
Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
|
||||
spawn(fun() ->
|
||||
catch machi_chain_manager1:trigger_react_to_env(Chmgr)
|
||||
end),
|
||||
{noreply, S#state{wedged=true}};
|
||||
true ->
|
||||
{noreply, S}
|
||||
end;
|
||||
handle_cast({wedge_state_change, Boolean, {NewEpoch, _}=NewEpochId},
|
||||
#state{epoch_id=OldEpochId}=S) ->
|
||||
OldEpoch = case OldEpochId of {OldE, _} -> OldE;
|
||||
undefined -> -1
|
||||
end,
|
||||
if NewEpoch >= OldEpoch ->
|
||||
true = ets:insert(S#state.etstab,
|
||||
{epoch, {Boolean, NewEpochId}}),
|
||||
{noreply, S#state{wedged=Boolean, epoch_id=NewEpochId}};
|
||||
true ->
|
||||
{noreply, S}
|
||||
end;
|
||||
handle_cast(Else, S) ->
|
||||
io:format(user, "~s:handle_cast: WHA? ~p\n", [?MODULE, Else]),
|
||||
{noreply, S}.
|
||||
|
||||
handle_info(Else, S) ->
|
||||
io:format(user, "~s:handle_info: WHA? ~p\n", [?MODULE, Else]),
|
||||
{noreply, S}.
|
||||
|
||||
terminate(normal, _S) ->
|
||||
ok;
|
||||
terminate(Reason, _S) ->
|
||||
lager:warning("~s:terminate: ~w", [?MODULE, Reason]),
|
||||
ok.
|
||||
|
||||
code_change(_OldVsn, S, _Extra) ->
|
||||
{ok, S}.
|
||||
|
||||
append_server_dispatch(From, NSInfo,
|
||||
Prefix, Chunk, TCSum, Opts, FluName, EpochId) ->
|
||||
Result = case handle_append(NSInfo,
|
||||
Prefix, Chunk, TCSum, Opts, FluName, EpochId) of
|
||||
{ok, File, Offset} ->
|
||||
{assignment, Offset, File};
|
||||
Other ->
|
||||
Other
|
||||
end,
|
||||
_ = gen_server:reply(From, Result),
|
||||
ok.
|
||||
|
||||
handle_append(NSInfo,
|
||||
Prefix, Chunk, TCSum, Opts, FluName, EpochId) ->
|
||||
Res = machi_flu_filename_mgr:find_or_make_filename_from_prefix(
|
||||
FluName, EpochId, {prefix, Prefix}, NSInfo),
|
||||
case Res of
|
||||
{file, F} ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, F}) of
|
||||
{ok, Pid} ->
|
||||
{Tag, CS} = machi_util:unmake_tagged_csum(TCSum),
|
||||
Meta = [{client_csum_tag, Tag}, {client_csum, CS}],
|
||||
Extra = Opts#append_opts.chunk_extra,
|
||||
machi_file_proxy:append(Pid, Meta, Extra, Chunk);
|
||||
{error, trimmed} = E ->
|
||||
E
|
||||
end;
|
||||
Error ->
|
||||
Error
|
||||
end.
|
|
@ -38,6 +38,71 @@
|
|||
%% TODO This EDoc was written first, and the EDoc and also `-type' and
|
||||
%% `-spec' definitions for {@link machi_proxy_flu1_client} and {@link
|
||||
%% machi_cr_client} must be improved.
|
||||
%%
|
||||
%% == Client API implementation notes ==
|
||||
%%
|
||||
%% At the moment, there are several modules that implement various
|
||||
%% subsets of the Machi API. The table below attempts to show how and
|
||||
%% why they differ.
|
||||
%%
|
||||
%% ```
|
||||
%% |--------------------------+-------+-----+------+------+-------+----------------|
|
||||
%% | | PB | | # | | Conn | Epoch & NS |
|
||||
%% | Module name | Level | CR? | FLUS | Impl | Life? | version aware? |
|
||||
%% |--------------------------+-------+-----+------+------+-------+----------------|
|
||||
%% | machi_pb_high_api_client | high | yes | many | proc | long | no |
|
||||
%% | machi_cr_client | low | yes | many | proc | long | no |
|
||||
%% | machi_proxy_flu1_client | low | no | 1 | proc | long | yes |
|
||||
%% | machi_flu1_client | low | no | 1 | lib | short | yes |
|
||||
%% |--------------------------+-------+-----+------+------+-------+----------------|
|
||||
%% '''
|
||||
%%
|
||||
%% In terms of use and API layering, the table rows are in highest`->'lowest
|
||||
%% order: each level calls the layer immediately below it.
|
||||
%%
|
||||
%% <dl>
|
||||
%% <dt> <b> PB Level</b> </dt>
|
||||
%% <dd> The Protocol Buffers API is divided logically into two levels,
|
||||
%% "low" and "high". The low-level protocol is used for intra-chain
|
||||
%% communication. The high-level protocol is used for clients outside
|
||||
%% of a Machi chain or Machi cluster of chains.
|
||||
%% </dd>
|
||||
%% <dt> <b> CR?</b> </dt>
|
||||
%% <dd> Does this API support (directly or indirectly) Chain
|
||||
%% Replication? If `no', then the API has no awareness of multiple
|
||||
%% replicas of any file or file chunk; unaware clients can only
|
||||
%% perform operations at a single Machi FLU's file service or
|
||||
%% projection store service.
|
||||
%% </dd>
|
||||
%% <dt> <b> # FLUs</b> </dt>
|
||||
%% <dd> Now many FLUs does this API layer communicate with
|
||||
%% simultaneously? Note that there is a one-to-one correspondence
|
||||
%% between this value and the "CR?" column's value.
|
||||
%% </dd>
|
||||
%% <dt> <b> Impl</b> </dt>
|
||||
%% <dd> Implementation: library-only or an Erlang process,
|
||||
%% e.g., `gen_server'.
|
||||
%% </dd>
|
||||
%% <dt> <b> Conn Life?</b> </dt>
|
||||
%% <dd> Expected TCP session connection life: short or long. At the
|
||||
%% lowest level, the {@link machi_flu1_client} API implementation takes
|
||||
%% no effort to reconnect to a remote FLU when its single TCP session
|
||||
%% is broken. For long-lived connection life APIs, the server side will
|
||||
%% automatically attempt to reconnect to remote FLUs when a TCP session
|
||||
%% is broken.
|
||||
%% </dd>
|
||||
%% <dt> <b> Epoch & NS version aware?</b> </dt>
|
||||
%% <dd> Are clients of this API responsible for knowing a chain's EpochID
|
||||
%% and namespace version numbers? If `no', then the server side of the
|
||||
%% API will automatically attempt to discover/re-discover the EpochID and
|
||||
%% namespace version numbers whenever they change.
|
||||
%% </dd>
|
||||
%% </dl>
|
||||
%%
|
||||
%% The only protocol that we expect to be used by entities outside of
|
||||
%% a single Machi chain or a multi-chain cluster is the "high"
|
||||
%% Protocol Buffers API. The {@link riak_pb_high_api_client} module
|
||||
%% is an Erlang reference implementation of this PB API.
|
||||
|
||||
-module(machi_flu1_client).
|
||||
|
||||
|
@ -50,14 +115,15 @@
|
|||
-include_lib("pulse_otp/include/pulse_otp.hrl").
|
||||
-endif.
|
||||
|
||||
-define(HARD_TIMEOUT, 2500).
|
||||
-define(SHORT_TIMEOUT, 2500).
|
||||
-define(LONG_TIMEOUT, (60*1000)).
|
||||
|
||||
-export([
|
||||
%% File API
|
||||
append_chunk/4, append_chunk/5,
|
||||
append_chunk_extra/5, append_chunk_extra/6,
|
||||
read_chunk/5, read_chunk/6,
|
||||
checksum_list/3, checksum_list/4,
|
||||
append_chunk/6, append_chunk/7,
|
||||
append_chunk/8, append_chunk/9,
|
||||
read_chunk/7, read_chunk/8,
|
||||
checksum_list/2, checksum_list/3,
|
||||
list_files/2, list_files/3,
|
||||
wedge_status/1, wedge_status/2,
|
||||
|
||||
|
@ -79,34 +145,48 @@
|
|||
]).
|
||||
%% For "internal" replication only.
|
||||
-export([
|
||||
write_chunk/5, write_chunk/6,
|
||||
write_chunk/7, write_chunk/8,
|
||||
trim_chunk/6,
|
||||
delete_migration/3, delete_migration/4,
|
||||
trunc_hack/3, trunc_hack/4
|
||||
]).
|
||||
|
||||
-type port_wrap() :: {w,atom(),term()}.
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix'.
|
||||
|
||||
-spec append_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk()) ->
|
||||
-spec append_chunk(port_wrap(),
|
||||
'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
|
||||
machi_dt:file_prefix(), machi_dt:chunk(),
|
||||
machi_dt:chunk_csum()) ->
|
||||
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
|
||||
append_chunk(Sock, EpochID, Prefix, Chunk) ->
|
||||
append_chunk2(Sock, EpochID, Prefix, Chunk, 0).
|
||||
append_chunk(Sock, NSInfo, EpochID, Prefix, Chunk, CSum) ->
|
||||
append_chunk(Sock, NSInfo, EpochID, Prefix, Chunk, CSum,
|
||||
#append_opts{}, ?LONG_TIMEOUT).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix'.
|
||||
%% with `Prefix' and also request an additional `Extra' bytes.
|
||||
%%
|
||||
%% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
|
||||
%% the file offsets that follow `Chunk''s position for the following 4K will
|
||||
%% be reserved by the file sequencer for later write(s) by the
|
||||
%% `write_chunk()' API.
|
||||
|
||||
-spec append_chunk(machi_dt:inet_host(), machi_dt:inet_port(),
|
||||
machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk()) ->
|
||||
'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
|
||||
machi_dt:file_prefix(), machi_dt:chunk(),
|
||||
machi_dt:chunk_csum()) ->
|
||||
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
|
||||
append_chunk(Host, TcpPort, EpochID, Prefix, Chunk) ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
try
|
||||
append_chunk2(Sock, EpochID, Prefix, Chunk, 0)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
append_chunk(Host, TcpPort, NSInfo, EpochID, Prefix, Chunk, CSum) ->
|
||||
append_chunk(Host, TcpPort, NSInfo, EpochID, Prefix, Chunk, CSum,
|
||||
#append_opts{}, ?LONG_TIMEOUT).
|
||||
|
||||
-spec append_chunk(port_wrap(),
|
||||
'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
|
||||
machi_dt:file_prefix(), machi_dt:chunk(),
|
||||
machi_dt:chunk_csum(), machi_dt:append_opts(), timeout()) ->
|
||||
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
|
||||
append_chunk(Sock, NSInfo0, EpochID, Prefix, Chunk, CSum, Opts, Timeout) ->
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
append_chunk2(Sock, NSInfo, EpochID, Prefix, Chunk, CSum, Opts, Timeout).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' and also request an additional `Extra' bytes.
|
||||
|
@ -116,66 +196,62 @@ append_chunk(Host, TcpPort, EpochID, Prefix, Chunk) ->
|
|||
%% be reserved by the file sequencer for later write(s) by the
|
||||
%% `write_chunk()' API.
|
||||
|
||||
-spec append_chunk_extra(port_wrap(), machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) ->
|
||||
-spec append_chunk(machi_dt:inet_host(), machi_dt:inet_port(),
|
||||
'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
|
||||
machi_dt:file_prefix(), machi_dt:chunk(),
|
||||
machi_dt:chunk_csum(), machi_dt:append_opts(), timeout()) ->
|
||||
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
|
||||
append_chunk_extra(Sock, EpochID, Prefix, Chunk, ChunkExtra)
|
||||
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
|
||||
append_chunk2(Sock, EpochID, Prefix, Chunk, ChunkExtra).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' and also request an additional `Extra' bytes.
|
||||
%%
|
||||
%% For example, if the `Chunk' size is 1 KByte and `Extra' is 4K Bytes, then
|
||||
%% the file offsets that follow `Chunk''s position for the following 4K will
|
||||
%% be reserved by the file sequencer for later write(s) by the
|
||||
%% `write_chunk()' API.
|
||||
|
||||
-spec append_chunk_extra(machi_dt:inet_host(), machi_dt:inet_port(),
|
||||
machi_dt:epoch_id(), machi_dt:file_prefix(), machi_dt:chunk(), machi_dt:chunk_size()) ->
|
||||
{ok, machi_dt:chunk_pos()} | {error, machi_dt:error_general()} | {error, term()}.
|
||||
append_chunk_extra(Host, TcpPort, EpochID, Prefix, Chunk, ChunkExtra)
|
||||
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
|
||||
append_chunk(Host, TcpPort, NSInfo0, EpochID,
|
||||
Prefix, Chunk, CSum, Opts, Timeout) ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
try
|
||||
append_chunk2(Sock, EpochID, Prefix, Chunk, ChunkExtra)
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
append_chunk2(Sock, NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum, Opts, Timeout)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
|
||||
%% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
|
||||
|
||||
-spec read_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size()) ->
|
||||
{ok, machi_dt:chunk_s()} |
|
||||
-spec read_chunk(port_wrap(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size(),
|
||||
machi_dt:read_opts_x()) ->
|
||||
{ok, {[machi_dt:chunk_summary()], [machi_dt:chunk_pos()]}} |
|
||||
{error, machi_dt:error_general() | 'not_written' | 'partial_read'} |
|
||||
{error, term()}.
|
||||
read_chunk(Sock, EpochID, File, Offset, Size)
|
||||
read_chunk(Sock, NSInfo0, EpochID, File, Offset, Size, Opts0)
|
||||
when Offset >= ?MINIMUM_OFFSET, Size >= 0 ->
|
||||
read_chunk2(Sock, EpochID, File, Offset, Size).
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
Opts = machi_util:read_opts_default(Opts0),
|
||||
read_chunk2(Sock, NSInfo, EpochID, File, Offset, Size, Opts).
|
||||
|
||||
%% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
|
||||
|
||||
-spec read_chunk(machi_dt:inet_host(), machi_dt:inet_port(), machi_dt:epoch_id(),
|
||||
machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size()) ->
|
||||
{ok, machi_dt:chunk_s()} |
|
||||
-spec read_chunk(machi_dt:inet_host(), machi_dt:inet_port(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(),
|
||||
machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size(),
|
||||
machi_dt:read_opts_x()) ->
|
||||
{ok, [machi_dt:chunk_summary()]} |
|
||||
{error, machi_dt:error_general() | 'not_written' | 'partial_read'} |
|
||||
{error, term()}.
|
||||
read_chunk(Host, TcpPort, EpochID, File, Offset, Size)
|
||||
read_chunk(Host, TcpPort, NSInfo0, EpochID, File, Offset, Size, Opts0)
|
||||
when Offset >= ?MINIMUM_OFFSET, Size >= 0 ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
Opts = machi_util:read_opts_default(Opts0),
|
||||
try
|
||||
read_chunk2(Sock, EpochID, File, Offset, Size)
|
||||
read_chunk2(Sock, NSInfo, EpochID, File, Offset, Size, Opts)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
|
||||
-spec checksum_list(port_wrap(), machi_dt:epoch_id(), machi_dt:file_name()) ->
|
||||
-spec checksum_list(port_wrap(), machi_dt:file_name()) ->
|
||||
{ok, binary()} |
|
||||
{error, machi_dt:error_general() | 'no_such_file' | 'partial_read'} |
|
||||
{error, term()}.
|
||||
checksum_list(Sock, EpochID, File) ->
|
||||
checksum_list2(Sock, EpochID, File).
|
||||
checksum_list(Sock, File) ->
|
||||
checksum_list2(Sock, File).
|
||||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
%%
|
||||
|
@ -199,13 +275,13 @@ checksum_list(Sock, EpochID, File) ->
|
|||
%% Details of the encoding used inside the `binary()' blog can be found
|
||||
%% in the EDoc comments for {@link machi_flu1:decode_csum_file_entry/1}.
|
||||
|
||||
-spec checksum_list(machi_dt:inet_host(), machi_dt:inet_port(), machi_dt:epoch_id(), machi_dt:file_name()) ->
|
||||
-spec checksum_list(machi_dt:inet_host(), machi_dt:inet_port(), machi_dt:file_name()) ->
|
||||
{ok, binary()} |
|
||||
{error, machi_dt:error_general() | 'no_such_file'} | {error, term()}.
|
||||
checksum_list(Host, TcpPort, EpochID, File) when is_integer(TcpPort) ->
|
||||
checksum_list(Host, TcpPort, File) when is_integer(TcpPort) ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
try
|
||||
checksum_list2(Sock, EpochID, File)
|
||||
checksum_list2(Sock, File)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
|
@ -232,7 +308,7 @@ list_files(Host, TcpPort, EpochID) when is_integer(TcpPort) ->
|
|||
%% @doc Fetch the wedge status from the remote FLU.
|
||||
|
||||
-spec wedge_status(port_wrap()) ->
|
||||
{ok, {boolean(), machi_dt:epoch_id()}} | {error, term()}.
|
||||
{ok, {boolean(), machi_dt:epoch_id(), machi_dt:namespace_version(),machi_dt:namespace()}} | {error, term()}.
|
||||
|
||||
wedge_status(Sock) ->
|
||||
wedge_status2(Sock).
|
||||
|
@ -240,7 +316,7 @@ wedge_status(Sock) ->
|
|||
%% @doc Fetch the wedge status from the remote FLU.
|
||||
|
||||
-spec wedge_status(machi_dt:inet_host(), machi_dt:inet_port()) ->
|
||||
{ok, {boolean(), machi_dt:epoch_id()}} | {error, term()}.
|
||||
{ok, {boolean(), machi_dt:epoch_id(), machi_dt:namespace_version(),machi_dt:namespace()}} | {error, term()}.
|
||||
wedge_status(Host, TcpPort) when is_integer(TcpPort) ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
try
|
||||
|
@ -451,27 +527,46 @@ disconnect(_) ->
|
|||
%% @doc Restricted API: Write a chunk of already-sequenced data to
|
||||
%% `File' at `Offset'.
|
||||
|
||||
-spec write_chunk(port_wrap(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk()) ->
|
||||
-spec write_chunk(port_wrap(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk(), machi_dt:chunk_csum()) ->
|
||||
ok | {error, machi_dt:error_general()} | {error, term()}.
|
||||
write_chunk(Sock, EpochID, File, Offset, Chunk)
|
||||
write_chunk(Sock, NSInfo0, EpochID, File, Offset, Chunk, CSum)
|
||||
when Offset >= ?MINIMUM_OFFSET ->
|
||||
write_chunk2(Sock, EpochID, File, Offset, Chunk).
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
write_chunk2(Sock, NSInfo, EpochID, File, Offset, Chunk, CSum).
|
||||
|
||||
%% @doc Restricted API: Write a chunk of already-sequenced data to
|
||||
%% `File' at `Offset'.
|
||||
|
||||
-spec write_chunk(machi_dt:inet_host(), machi_dt:inet_port(),
|
||||
machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk()) ->
|
||||
'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk(), machi_dt:chunk_csum()) ->
|
||||
ok | {error, machi_dt:error_general()} | {error, term()}.
|
||||
write_chunk(Host, TcpPort, EpochID, File, Offset, Chunk)
|
||||
write_chunk(Host, TcpPort, NSInfo0, EpochID, File, Offset, Chunk, CSum)
|
||||
when Offset >= ?MINIMUM_OFFSET ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE, address=Host, port=TcpPort}),
|
||||
try
|
||||
write_chunk2(Sock, EpochID, File, Offset, Chunk)
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
write_chunk2(Sock, NSInfo, EpochID, File, Offset, Chunk, CSum)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
|
||||
%% @doc Restricted API: Write a chunk of already-sequenced data to
|
||||
%% `File' at `Offset'.
|
||||
|
||||
-spec trim_chunk(port_wrap(), 'undefined' | machi_dt:ns_info(), machi_dt:epoch_id(), machi_dt:file_name(), machi_dt:file_offset(), machi_dt:chunk_size()) ->
|
||||
ok | {error, machi_dt:error_general()} | {error, term()}.
|
||||
trim_chunk(Sock, NSInfo0, EpochID, File0, Offset, Size)
|
||||
when Offset >= ?MINIMUM_OFFSET ->
|
||||
ReqID = <<"id">>,
|
||||
NSInfo = machi_util:ns_info_default(NSInfo0),
|
||||
#ns_info{version=NSVersion, name=NS} = NSInfo,
|
||||
File = machi_util:make_binary(File0),
|
||||
true = (Offset >= ?MINIMUM_OFFSET),
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID,
|
||||
{low_trim_chunk, NSVersion, NS, EpochID, File, Offset, Size, 0}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
%% @doc Restricted API: Delete a file after it has been successfully
|
||||
%% migrated.
|
||||
|
||||
|
@ -516,83 +611,88 @@ trunc_hack(Host, TcpPort, EpochID, File) when is_integer(TcpPort) ->
|
|||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
read_chunk2(Sock, EpochID, File0, Offset, Size) ->
|
||||
read_chunk2(Sock, NSInfo, EpochID, File0, Offset, Size, Opts) ->
|
||||
ReqID = <<"id">>,
|
||||
#ns_info{version=NSVersion, name=NS} = NSInfo,
|
||||
File = machi_util:make_binary(File0),
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID,
|
||||
{low_read_chunk, EpochID, File, Offset, Size, []}),
|
||||
{low_read_chunk, NSVersion, NS, EpochID, File, Offset, Size, Opts}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
append_chunk2(Sock, EpochID, Prefix0, Chunk0, ChunkExtra) ->
|
||||
append_chunk2(Sock, NSInfo, EpochID,
|
||||
Prefix0, Chunk, CSum0, Opts, Timeout) ->
|
||||
ReqID = <<"id">>,
|
||||
{Chunk, CSum_tag, CSum} =
|
||||
case Chunk0 of
|
||||
X when is_binary(X) ->
|
||||
{Chunk0, ?CSUM_TAG_NONE, <<>>};
|
||||
{ChunkCSum, Chk} ->
|
||||
{Tag, CS} = machi_util:unmake_tagged_csum(ChunkCSum),
|
||||
{Chk, Tag, CS}
|
||||
end,
|
||||
PKey = <<>>, % TODO
|
||||
Prefix = machi_util:make_binary(Prefix0),
|
||||
{CSum_tag, CSum} = case CSum0 of
|
||||
<<>> ->
|
||||
{?CSUM_TAG_NONE, <<>>};
|
||||
{_Tag, _CS} ->
|
||||
CSum0;
|
||||
B when is_binary(B) ->
|
||||
machi_util:unmake_tagged_csum(CSum0)
|
||||
end,
|
||||
#ns_info{version=NSVersion, name=NS, locator=NSLocator} = NSInfo,
|
||||
%% NOTE: The tuple position of NSLocator is a bit odd, because EpochID
|
||||
%% _must_ be in the 4th position (as NSV & NS must be in 2nd & 3rd).
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID,
|
||||
{low_append_chunk, EpochID, PKey, Prefix, Chunk, CSum_tag, CSum,
|
||||
ChunkExtra}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
{low_append_chunk, NSVersion, NS, EpochID, NSLocator,
|
||||
Prefix, Chunk, CSum_tag, CSum, Opts}),
|
||||
do_pb_request_common(Sock, ReqID, Req, true, Timeout).
|
||||
|
||||
write_chunk2(Sock, EpochID, File0, Offset, Chunk0) ->
|
||||
write_chunk2(Sock, NSInfo, EpochID, File0, Offset, Chunk, CSum0) ->
|
||||
ReqID = <<"id">>,
|
||||
#ns_info{version=NSVersion, name=NS} = NSInfo,
|
||||
File = machi_util:make_binary(File0),
|
||||
true = (Offset >= ?MINIMUM_OFFSET),
|
||||
{Chunk, CSum_tag, CSum} =
|
||||
case Chunk0 of
|
||||
X when is_binary(X) ->
|
||||
{Chunk0, ?CSUM_TAG_NONE, <<>>};
|
||||
{ChunkCSum, Chk} ->
|
||||
{Tag, CS} = machi_util:unmake_tagged_csum(ChunkCSum),
|
||||
{Chk, Tag, CS}
|
||||
end,
|
||||
{CSum_tag, CSum} = case CSum0 of
|
||||
<<>> ->
|
||||
{?CSUM_TAG_NONE, <<>>};
|
||||
{_Tag, _CS} ->
|
||||
CSum0;
|
||||
B when is_binary(B) ->
|
||||
machi_util:unmake_tagged_csum(CSum0)
|
||||
end,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID,
|
||||
{low_write_chunk, EpochID, File, Offset, Chunk, CSum_tag, CSum}),
|
||||
{low_write_chunk, NSVersion, NS, EpochID, File, Offset, Chunk, CSum_tag, CSum}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
list2(Sock, EpochID) ->
|
||||
ReqID = <<"id">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_list_files, EpochID}),
|
||||
ReqID, {low_skip_wedge, {low_list_files, EpochID}}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
wedge_status2(Sock) ->
|
||||
ReqID = <<"id">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_wedge_status, undefined}),
|
||||
ReqID, {low_skip_wedge, {low_wedge_status}}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
echo2(Sock, Message) ->
|
||||
ReqID = <<"id">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_echo, undefined, Message}),
|
||||
ReqID, {low_skip_wedge, {low_echo, Message}}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
checksum_list2(Sock, EpochID, File) ->
|
||||
checksum_list2(Sock, File) ->
|
||||
ReqID = <<"id">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_checksum_list, EpochID, File}),
|
||||
ReqID, {low_skip_wedge, {low_checksum_list, File}}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
delete_migration2(Sock, EpochID, File) ->
|
||||
ReqID = <<"id">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_delete_migration, EpochID, File}),
|
||||
ReqID, {low_skip_wedge, {low_delete_migration, EpochID, File}}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
trunc_hack2(Sock, EpochID, File) ->
|
||||
ReqID = <<"id-trunc">>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_trunc_hack, EpochID, File}),
|
||||
ReqID, {low_skip_wedge, {low_trunc_hack, EpochID, File}}),
|
||||
do_pb_request_common(Sock, ReqID, Req).
|
||||
|
||||
get_latest_epochid2(Sock, ProjType) ->
|
||||
|
@ -635,18 +735,18 @@ kick_projection_reaction2(Sock, _Options) ->
|
|||
ReqID = <<42>>,
|
||||
Req = machi_pb_translate:to_pb_request(
|
||||
ReqID, {low_proj, {kick_projection_reaction}}),
|
||||
do_pb_request_common(Sock, ReqID, Req, false).
|
||||
do_pb_request_common(Sock, ReqID, Req, false, ?LONG_TIMEOUT).
|
||||
|
||||
do_pb_request_common(Sock, ReqID, Req) ->
|
||||
do_pb_request_common(Sock, ReqID, Req, true).
|
||||
do_pb_request_common(Sock, ReqID, Req, true, ?LONG_TIMEOUT).
|
||||
|
||||
do_pb_request_common(Sock, ReqID, Req, GetReply_p) ->
|
||||
do_pb_request_common(Sock, ReqID, Req, GetReply_p, Timeout) ->
|
||||
erase(bad_sock),
|
||||
try
|
||||
ReqBin = list_to_binary(machi_pb:encode_mpb_ll_request(Req)),
|
||||
ok = w_send(Sock, ReqBin),
|
||||
if GetReply_p ->
|
||||
case w_recv(Sock, 0) of
|
||||
case w_recv(Sock, 0, Timeout) of
|
||||
{ok, RespBin} ->
|
||||
Resp = machi_pb:decode_mpb_ll_response(RespBin),
|
||||
{ReqID2, Reply} = machi_pb_translate:from_pb_response(Resp),
|
||||
|
@ -692,7 +792,7 @@ w_connect(#p_srvr{proto_mod=?MODULE, address=Host, port=Port, props=Props}=_P)->
|
|||
case proplists:get_value(session_proto, Props, tcp) of
|
||||
tcp ->
|
||||
put(xxx, goofus),
|
||||
Sock = machi_util:connect(Host, Port, ?HARD_TIMEOUT),
|
||||
Sock = machi_util:connect(Host, Port, ?SHORT_TIMEOUT),
|
||||
put(xxx, Sock),
|
||||
ok = inet:setopts(Sock, ?PB_PACKET_OPTS),
|
||||
{w,tcp,Sock};
|
||||
|
@ -708,7 +808,7 @@ put(xxx, Sock),
|
|||
end
|
||||
catch
|
||||
_X:_Y ->
|
||||
io:format(user, "DBG Whoa ~w w_connect port ~w sock ~w err ~w ~w\n", [time(), Port, get(xxx), _X, _Y]),
|
||||
%% io:format(user, "DBG Whoa ~w w_connect port ~w sock ~w err ~w ~w\n", [time(), Port, get(xxx), _X, _Y]),
|
||||
undefined
|
||||
end.
|
||||
|
||||
|
@ -716,8 +816,8 @@ w_close({w,tcp,Sock}) ->
|
|||
catch gen_tcp:close(Sock),
|
||||
ok.
|
||||
|
||||
w_recv({w,tcp,Sock}, Amt) ->
|
||||
gen_tcp:recv(Sock, Amt, ?HARD_TIMEOUT).
|
||||
w_recv({w,tcp,Sock}, Amt, Timeout) ->
|
||||
gen_tcp:recv(Sock, Amt, Timeout).
|
||||
|
||||
w_send({w,tcp,Sock}, IoData) ->
|
||||
gen_tcp:send(Sock, IoData).
|
||||
|
|
634
src/machi_flu1_net_server.erl
Normal file
634
src/machi_flu1_net_server.erl
Normal file
|
@ -0,0 +1,634 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc Ranch protocol callback module to handle PB protocol over
|
||||
%% transport, including both high and low modes.
|
||||
|
||||
%% TODO
|
||||
%% - Two modes, high and low should be separated at listener level?
|
||||
|
||||
-module(machi_flu1_net_server).
|
||||
|
||||
-behaviour(gen_server).
|
||||
-behaviour(ranch_protocol).
|
||||
|
||||
-export([start_link/4]).
|
||||
-export([init/1]).
|
||||
-export([handle_call/3, handle_cast/2, handle_info/2,
|
||||
terminate/2, code_change/3]).
|
||||
|
||||
-include_lib("kernel/include/file.hrl").
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_pb.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-ifdef(TEST).
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-endif. % TEST
|
||||
|
||||
-record(state, {
|
||||
%% Ranch's transport management stuff
|
||||
ref :: ranch:ref(),
|
||||
socket :: socket(),
|
||||
transport :: module(),
|
||||
|
||||
%% Machi FLU configurations, common for low and high
|
||||
data_dir :: string(),
|
||||
witness :: boolean(),
|
||||
pb_mode :: undefined | high | low,
|
||||
%% - Used in projection related requests in low mode
|
||||
%% - Used in spawning CR client in high mode
|
||||
proj_store :: pid(),
|
||||
|
||||
%% Low mode only items
|
||||
%% Current best knowledge, used for wedge_self / bad_epoch check
|
||||
epoch_id :: undefined | machi_dt:epoch_id(),
|
||||
%% Used in dispatching append_chunk* reqs to the
|
||||
%% append serializing process
|
||||
flu_name :: pv1_server(),
|
||||
%% Used in server_wedge_status to lookup the table
|
||||
epoch_tab :: ets:tab(),
|
||||
%% Clustering: cluster map version number
|
||||
namespace_version = 0 :: machi_dt:namespace_version(),
|
||||
%% Clustering: my (and my chain's) assignment to a specific namespace
|
||||
namespace = <<>> :: machi_dt:namespace(),
|
||||
|
||||
%% High mode only
|
||||
high_clnt :: pid(),
|
||||
|
||||
%% anything you want
|
||||
props = [] :: proplists:proplist()
|
||||
}).
|
||||
|
||||
-type socket() :: any().
|
||||
-type state() :: #state{}.
|
||||
|
||||
-spec start_link(ranch:ref(), socket(), module(), [term()]) -> {ok, pid()}.
|
||||
start_link(Ref, Socket, Transport, [FluName, Witness, DataDir, EpochTab, ProjStore, Props]) ->
|
||||
NS = proplists:get_value(namespace, Props, <<>>),
|
||||
true = is_binary(NS),
|
||||
proc_lib:start_link(?MODULE, init, [#state{ref=Ref,
|
||||
socket=Socket,
|
||||
transport=Transport,
|
||||
flu_name=FluName,
|
||||
witness=Witness,
|
||||
data_dir=DataDir,
|
||||
epoch_tab=EpochTab,
|
||||
proj_store=ProjStore,
|
||||
namespace=NS,
|
||||
props=Props}]).
|
||||
|
||||
-spec init(state()) -> no_return().
|
||||
init(#state{ref=Ref, socket=Socket, transport=Transport}=State) ->
|
||||
ok = proc_lib:init_ack({ok, self()}),
|
||||
ok = ranch:accept_ack(Ref),
|
||||
{_Wedged_p, CurrentEpochID} = lookup_epoch(State),
|
||||
ok = Transport:setopts(Socket, [{active, once}|?PB_PACKET_OPTS]),
|
||||
gen_server:enter_loop(?MODULE, [], State#state{epoch_id=CurrentEpochID}).
|
||||
|
||||
handle_call(Request, _From, S) ->
|
||||
lager:warning("~s:handle_call UNKNOWN message: ~w", [?MODULE, Request]),
|
||||
Reply = {error, {unknown_message, Request}},
|
||||
{reply, Reply, S}.
|
||||
|
||||
handle_cast(_Msg, S) ->
|
||||
lager:warning("~s:handle_cast UNKNOWN message: ~w", [?MODULE, _Msg]),
|
||||
{noreply, S}.
|
||||
|
||||
%% TODO: Other transport support needed?? TLS/SSL, SCTP
|
||||
handle_info({tcp, Socket, Data}=_Info, #state{socket=Socket}=S) ->
|
||||
lager:debug("~s:handle_info: ~w", [?MODULE, _Info]),
|
||||
transport_received(Socket, Data, S);
|
||||
handle_info({tcp_closed, Socket}=_Info, #state{socket=Socket}=S) ->
|
||||
lager:debug("~s:handle_info: ~w", [?MODULE, _Info]),
|
||||
transport_closed(Socket, S);
|
||||
handle_info({tcp_error, Socket, Reason}=_Info, #state{socket=Socket}=S) ->
|
||||
lager:warning("~s:handle_info (socket=~w) tcp_error: ~w", [?MODULE, Socket, Reason]),
|
||||
transport_error(Socket, Reason, S);
|
||||
handle_info(_Info, S) ->
|
||||
lager:warning("~s:handle_info UNKNOWN message: ~w", [?MODULE, _Info]),
|
||||
{noreply, S}.
|
||||
|
||||
terminate(normal, #state{socket=undefined}=_S) ->
|
||||
ok;
|
||||
terminate(Reason, #state{socket=undefined}=_S) ->
|
||||
lager:warning("~s:terminate (socket=undefined): ~w", [?MODULE, Reason]),
|
||||
ok;
|
||||
terminate(normal, #state{socket=Socket}=_S) ->
|
||||
(catch gen_tcp:close(Socket)),
|
||||
ok;
|
||||
terminate(Reason, #state{socket=Socket}=_S) ->
|
||||
lager:warning("~s:terminate (socket=Socket): ~w", [?MODULE, Reason]),
|
||||
(catch gen_tcp:close(Socket)),
|
||||
ok.
|
||||
|
||||
code_change(_OldVsn, S, _Extra) ->
|
||||
{ok, S}.
|
||||
|
||||
%% -- private
|
||||
|
||||
%%%% Common transport handling
|
||||
|
||||
-spec transport_received(socket(), machi_dt:chunk(), state()) ->
|
||||
{noreply, state()}.
|
||||
transport_received(Socket, <<"QUIT\n">>, #state{socket=Socket}=S) ->
|
||||
{stop, normal, S};
|
||||
transport_received(Socket, Bin, #state{transport=Transport}=S) ->
|
||||
{RespBin, S2} =
|
||||
case machi_pb:decode_mpb_ll_request(Bin) of
|
||||
LL_req when LL_req#mpb_ll_request.do_not_alter == 2 ->
|
||||
{R, NewS} = do_pb_ll_request(LL_req, S),
|
||||
{maybe_encode_response(R), set_mode(low, NewS)};
|
||||
_ ->
|
||||
HL_req = machi_pb:decode_mpb_request(Bin),
|
||||
1 = HL_req#mpb_request.do_not_alter,
|
||||
{R, NewS} = do_pb_hl_request(HL_req, make_high_clnt(S)),
|
||||
{machi_pb:encode_mpb_response(R), set_mode(high, NewS)}
|
||||
end,
|
||||
case RespBin of
|
||||
async_no_response ->
|
||||
Transport:setopts(Socket, [{active, once}]),
|
||||
{noreply, S2};
|
||||
_ ->
|
||||
case Transport:send(Socket, RespBin) of
|
||||
ok ->
|
||||
Transport:setopts(Socket, [{active, once}]),
|
||||
{noreply, S2};
|
||||
{error, Reason} ->
|
||||
transport_error(Socket, Reason, S2)
|
||||
end
|
||||
end.
|
||||
|
||||
-spec transport_closed(socket(), state()) -> {stop, term(), state()}.
|
||||
transport_closed(_Socket, S) ->
|
||||
{stop, normal, S}.
|
||||
|
||||
-spec transport_error(socket(), term(), state()) -> no_return().
|
||||
transport_error(Socket, Reason, #state{transport=Transport}=_S) ->
|
||||
Msg = io_lib:format("Socket error ~w", [Reason]),
|
||||
R = #mpb_ll_response{req_id= <<>>,
|
||||
generic=#mpb_errorresp{code=1, msg=Msg}},
|
||||
_Resp = machi_pb:encode_mpb_ll_response(R),
|
||||
%% TODO for TODO comments: comments below with four %s are copy-n-paste'd,
|
||||
%% then it should be considered they are still open and should be addressed.
|
||||
%%%% TODO: Weird that sometimes neither catch nor try/catch
|
||||
%%%% can prevent OTP's SASL from logging an error here.
|
||||
%%%% Error in process <0.545.0> with exit value: {badarg,[{erlang,port_command,.......
|
||||
%%%% TODO: is this what causes the intermittent PULSE deadlock errors?
|
||||
%%%% _ = (catch gen_tcp:send(Sock, _Resp)), timer:sleep(1000),
|
||||
(catch Transport:close(Socket)),
|
||||
_ = lager:warning("Socket error (~w -> ~w): ~w",
|
||||
[Transport:sockname(Socket), Transport:peername(Socket), Reason]),
|
||||
%% TODO: better to exit with `Reason' without logging?
|
||||
exit(normal).
|
||||
|
||||
maybe_encode_response(async_no_response=R) ->
|
||||
R;
|
||||
maybe_encode_response(R) ->
|
||||
machi_pb:encode_mpb_ll_response(R).
|
||||
|
||||
set_mode(Mode, #state{pb_mode=undefined}=S) ->
|
||||
S#state{pb_mode=Mode};
|
||||
set_mode(_, S) ->
|
||||
S.
|
||||
|
||||
%%%% Low PB mode %%%%
|
||||
|
||||
do_pb_ll_request(#mpb_ll_request{req_id=ReqID}, #state{pb_mode=high}=S) ->
|
||||
Result = {high_error, 41, "Low protocol request while in high mode"},
|
||||
{machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
|
||||
do_pb_ll_request(PB_request, S) ->
|
||||
Req = machi_pb_translate:from_pb_request(PB_request),
|
||||
{ReqID, Cmd, Result, S2} =
|
||||
case Req of
|
||||
{RqID, {low_skip_wedge, LowSubCmd}=Cmd0} ->
|
||||
%% Skip wedge check for these unprivileged commands
|
||||
{Rs, NewS} = do_pb_ll_request3(LowSubCmd, S),
|
||||
{RqID, Cmd0, Rs, NewS};
|
||||
{RqID, {low_proj, _LowSubCmd}=Cmd0} ->
|
||||
{Rs, NewS} = do_pb_ll_request3(Cmd0, S),
|
||||
{RqID, Cmd0, Rs, NewS};
|
||||
{RqID, Cmd0} ->
|
||||
%% All remaining must have NSVersion, NS, & EpochID at next pos
|
||||
NSVersion = element(2, Cmd0),
|
||||
NS = element(3, Cmd0),
|
||||
EpochID = element(4, Cmd0),
|
||||
{Rs, NewS} = do_pb_ll_request2(NSVersion, NS, EpochID, Cmd0, S),
|
||||
{RqID, Cmd0, Rs, NewS}
|
||||
end,
|
||||
{machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
|
||||
|
||||
%% do_pb_ll_request2(): Verification of epoch details & namespace details.
|
||||
|
||||
do_pb_ll_request2(NSVersion, NS, EpochID, CMD, S) ->
|
||||
{Wedged_p, CurrentEpochID} = lookup_epoch(S),
|
||||
if not is_tuple(EpochID) orelse tuple_size(EpochID) /= 2 ->
|
||||
exit({bad_epoch_id, EpochID, for, CMD});
|
||||
Wedged_p == true ->
|
||||
{{error, wedged}, S#state{epoch_id=CurrentEpochID}};
|
||||
EpochID /= CurrentEpochID ->
|
||||
{Epoch, _} = EpochID,
|
||||
{CurrentEpoch, _} = CurrentEpochID,
|
||||
if Epoch < CurrentEpoch ->
|
||||
{{error, bad_epoch}, S};
|
||||
true ->
|
||||
_ = machi_flu1:wedge_myself(S#state.flu_name, CurrentEpochID),
|
||||
{{error, wedged}, S#state{epoch_id=CurrentEpochID}}
|
||||
end;
|
||||
true ->
|
||||
#state{namespace_version=MyNSVersion, namespace=MyNS} = S,
|
||||
if NSVersion /= MyNSVersion ->
|
||||
{{error, bad_epoch}, S};
|
||||
NS /= MyNS ->
|
||||
{{error, bad_arg}, S};
|
||||
true ->
|
||||
do_pb_ll_request3(CMD, S)
|
||||
end
|
||||
end.
|
||||
|
||||
lookup_epoch(#state{epoch_tab=T}) ->
|
||||
%% TODO: race in shutdown to access ets table after owner dies
|
||||
ets:lookup_element(T, epoch, 2).
|
||||
|
||||
%% Witness status does not matter below.
|
||||
do_pb_ll_request3({low_echo, Msg}, S) ->
|
||||
{Msg, S};
|
||||
do_pb_ll_request3({low_auth, _User, _Pass}, S) ->
|
||||
{-6, S};
|
||||
do_pb_ll_request3({low_wedge_status}, S) ->
|
||||
{do_server_wedge_status(S), S};
|
||||
do_pb_ll_request3({low_proj, PCMD}, S) ->
|
||||
{do_server_proj_request(PCMD, S), S};
|
||||
|
||||
%% Witness status *matters* below
|
||||
do_pb_ll_request3({low_append_chunk, NSVersion, NS, EpochID, NSLocator,
|
||||
Prefix, Chunk, CSum_tag,
|
||||
CSum, Opts},
|
||||
#state{witness=false}=S) ->
|
||||
NSInfo = #ns_info{version=NSVersion, name=NS, locator=NSLocator},
|
||||
{do_server_append_chunk(NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum_tag, CSum,
|
||||
Opts, S), S};
|
||||
do_pb_ll_request3({low_write_chunk, _NSVersion, _NS, _EpochID, File, Offset, Chunk, CSum_tag,
|
||||
CSum},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, S), S};
|
||||
do_pb_ll_request3({low_read_chunk, _NSVersion, _NS, _EpochID, File, Offset, Size, Opts},
|
||||
#state{witness=false} = S) ->
|
||||
{do_server_read_chunk(File, Offset, Size, Opts, S), S};
|
||||
do_pb_ll_request3({low_trim_chunk, _NSVersion, _NS, _EpochID, File, Offset, Size, TriggerGC},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_trim_chunk(File, Offset, Size, TriggerGC, S), S};
|
||||
do_pb_ll_request3({low_checksum_list, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_checksum_listing(File, S), S};
|
||||
do_pb_ll_request3({low_list_files, _EpochID},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_list_files(S), S};
|
||||
do_pb_ll_request3({low_delete_migration, _EpochID, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_delete_migration(File, S),
|
||||
#state{witness=false}=S};
|
||||
do_pb_ll_request3({low_trunc_hack, _EpochID, File},
|
||||
#state{witness=false}=S) ->
|
||||
{do_server_trunc_hack(File, S), S};
|
||||
|
||||
do_pb_ll_request3(_, #state{witness=true}=S) ->
|
||||
{{error, bad_arg}, S}. % TODO: new status code??
|
||||
|
||||
do_server_proj_request({get_latest_epochid, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:get_latest_epochid(ProjStore, ProjType);
|
||||
do_server_proj_request({read_latest_projection, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:read_latest_projection(ProjStore, ProjType);
|
||||
do_server_proj_request({read_projection, ProjType, Epoch},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:read(ProjStore, ProjType, Epoch);
|
||||
do_server_proj_request({write_projection, ProjType, Proj},
|
||||
#state{flu_name=FluName, proj_store=ProjStore}) ->
|
||||
if Proj#projection_v1.epoch_number == ?SPAM_PROJ_EPOCH ->
|
||||
%% io:format(user, "DBG ~s ~w ~P\n", [?MODULE, ?LINE, Proj, 5]),
|
||||
Chmgr = machi_flu_psup:make_fitness_regname(FluName),
|
||||
[Map] = Proj#projection_v1.dbg,
|
||||
catch machi_fitness:send_fitness_update_spam(
|
||||
Chmgr, Proj#projection_v1.author_server, Map);
|
||||
true ->
|
||||
catch machi_projection_store:write(ProjStore, ProjType, Proj)
|
||||
end;
|
||||
do_server_proj_request({get_all_projections, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:get_all_projections(ProjStore, ProjType);
|
||||
do_server_proj_request({list_all_projections, ProjType},
|
||||
#state{proj_store=ProjStore}) ->
|
||||
machi_projection_store:list_all_projections(ProjStore, ProjType);
|
||||
do_server_proj_request({kick_projection_reaction},
|
||||
#state{flu_name=FluName}) ->
|
||||
%% Tell my chain manager that it might want to react to
|
||||
%% this new world.
|
||||
Chmgr = machi_chain_manager1:make_chmgr_regname(FluName),
|
||||
spawn(fun() ->
|
||||
catch machi_chain_manager1:trigger_react_to_env(Chmgr)
|
||||
end),
|
||||
async_no_response.
|
||||
|
||||
do_server_append_chunk(NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum_tag, CSum,
|
||||
Opts, S) ->
|
||||
case sanitize_prefix(Prefix) of
|
||||
ok ->
|
||||
do_server_append_chunk2(NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum_tag, CSum,
|
||||
Opts, S);
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_append_chunk2(NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum_tag, Client_CSum,
|
||||
Opts, #state{flu_name=FluName,
|
||||
epoch_id=EpochID}=_S) ->
|
||||
%% TODO: Do anything with PKey?
|
||||
try
|
||||
TaggedCSum = check_or_make_tagged_checksum(CSum_tag, Client_CSum,Chunk),
|
||||
R = {seq_append, self(), NSInfo, EpochID,
|
||||
Prefix, Chunk, TaggedCSum, Opts},
|
||||
case gen_server:call(FluName, R, 10*1000) of
|
||||
{assignment, Offset, File} ->
|
||||
Size = iolist_size(Chunk),
|
||||
{ok, {Offset, Size, File}};
|
||||
witness ->
|
||||
{error, bad_arg};
|
||||
wedged ->
|
||||
{error, wedged};
|
||||
{error, timeout} ->
|
||||
{error, partition}
|
||||
end
|
||||
catch
|
||||
throw:{bad_csum, _CS} ->
|
||||
{error, bad_checksum};
|
||||
error:badarg ->
|
||||
lager:error("badarg at ~w:do_server_append_chunk2:~w ~w",
|
||||
[?MODULE, ?LINE, erlang:get_stacktrace()]),
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_write_chunk(File, Offset, Chunk, CSum_tag, CSum, #state{flu_name=FluName}) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
|
||||
{ok, Pid} ->
|
||||
Meta = [{client_csum_tag, CSum_tag}, {client_csum, CSum}],
|
||||
machi_file_proxy:write(Pid, Offset, Meta, Chunk);
|
||||
{error, trimmed} = Error ->
|
||||
Error
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_read_chunk(File, Offset, Size, Opts, #state{flu_name=FluName})->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
|
||||
{ok, Pid} ->
|
||||
case machi_file_proxy:read(Pid, Offset, Size, Opts) of
|
||||
%% XXX FIXME
|
||||
%% For now we are omiting the checksum data because it blows up
|
||||
%% protobufs.
|
||||
{ok, ChunksAndTrimmed} -> {ok, ChunksAndTrimmed};
|
||||
Other -> Other
|
||||
end;
|
||||
{error, trimmed} = Error ->
|
||||
Error
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_trim_chunk(File, Offset, Size, TriggerGC, #state{flu_name=FluName}) ->
|
||||
lager:debug("Hi there! I'm trimming this: ~s, (~p, ~p), ~p~n",
|
||||
[File, Offset, Size, TriggerGC]),
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
|
||||
{ok, Pid} ->
|
||||
machi_file_proxy:trim(Pid, Offset, Size, TriggerGC);
|
||||
{error, trimmed} = Trimmed ->
|
||||
%% Should be returned back to (maybe) trigger repair
|
||||
Trimmed
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_checksum_listing(File, #state{flu_name=FluName, data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
case machi_flu_metadata_mgr:start_proxy_pid(FluName, {file, File}) of
|
||||
{ok, Pid} ->
|
||||
{ok, List} = machi_file_proxy:checksum_list(Pid),
|
||||
Bin = erlang:term_to_binary(List),
|
||||
if byte_size(Bin) > (?PB_MAX_MSG_SIZE - 1024) ->
|
||||
%% TODO: Fix this limitation by streaming the
|
||||
%% binary in multiple smaller PB messages.
|
||||
%% Also, don't read the file all at once. ^_^
|
||||
error_logger:error_msg("~s:~w oversize ~s\n",
|
||||
[?MODULE, ?LINE, DataDir]),
|
||||
{error, bad_arg};
|
||||
true ->
|
||||
{ok, Bin}
|
||||
end;
|
||||
{error, trimmed} ->
|
||||
{error, trimmed}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_list_files(#state{data_dir=DataDir}=_S) ->
|
||||
{_, WildPath} = machi_util:make_data_filename(DataDir, ""),
|
||||
Files = filelib:wildcard("*", WildPath),
|
||||
{ok, [begin
|
||||
{ok, FI} = file:read_file_info(WildPath ++ "/" ++ File),
|
||||
Size = FI#file_info.size,
|
||||
{Size, File}
|
||||
end || File <- Files]}.
|
||||
|
||||
do_server_wedge_status(#state{namespace_version=NSVersion, namespace=NS}=S) ->
|
||||
{Wedged_p, CurrentEpochID0} = lookup_epoch(S),
|
||||
CurrentEpochID = if CurrentEpochID0 == undefined ->
|
||||
?DUMMY_PV1_EPOCH;
|
||||
true ->
|
||||
CurrentEpochID0
|
||||
end,
|
||||
{Wedged_p, CurrentEpochID, NSVersion, NS}.
|
||||
|
||||
do_server_delete_migration(File, #state{data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
{_, Path} = machi_util:make_data_filename(DataDir, File),
|
||||
case file:delete(Path) of
|
||||
ok ->
|
||||
ok;
|
||||
{error, enoent} ->
|
||||
{error, no_such_file};
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
do_server_trunc_hack(File, #state{data_dir=DataDir}=_S) ->
|
||||
case sanitize_file_string(File) of
|
||||
ok ->
|
||||
{_, Path} = machi_util:make_data_filename(DataDir, File),
|
||||
case file:open(Path, [read, write, binary, raw]) of
|
||||
{ok, FH} ->
|
||||
try
|
||||
{ok, ?MINIMUM_OFFSET} = file:position(FH,
|
||||
?MINIMUM_OFFSET),
|
||||
ok = file:truncate(FH),
|
||||
ok
|
||||
after
|
||||
file:close(FH)
|
||||
end;
|
||||
{error, enoent} ->
|
||||
{error, no_such_file};
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end;
|
||||
_ ->
|
||||
{error, bad_arg}
|
||||
end.
|
||||
|
||||
sanitize_file_string(Str) ->
|
||||
case has_no_prohibited_chars(Str) andalso machi_util:is_valid_filename(Str) of
|
||||
true -> ok;
|
||||
false -> error
|
||||
end.
|
||||
|
||||
has_no_prohibited_chars(Str) ->
|
||||
case re:run(Str, "/") of
|
||||
nomatch ->
|
||||
true;
|
||||
_ ->
|
||||
true
|
||||
end.
|
||||
|
||||
sanitize_prefix(Prefix) ->
|
||||
%% We are using '^' as our component delimiter
|
||||
case re:run(Prefix, "/|\\^") of
|
||||
nomatch ->
|
||||
ok;
|
||||
_ ->
|
||||
error
|
||||
end.
|
||||
|
||||
check_or_make_tagged_checksum(?CSUM_TAG_NONE, _Client_CSum, Chunk) ->
|
||||
%% TODO: If the client was foolish enough to use
|
||||
%% this type of non-checksum, then the client gets
|
||||
%% what it deserves wrt data integrity, alas. In
|
||||
%% the client-side Chain Replication method, each
|
||||
%% server will calculated this independently, which
|
||||
%% isn't exactly what ought to happen for best data
|
||||
%% integrity checking. In server-side CR, the csum
|
||||
%% should be calculated by the head and passed down
|
||||
%% the chain together with the value.
|
||||
CS = machi_util:checksum_chunk(Chunk),
|
||||
machi_util:make_tagged_csum(server_sha, CS);
|
||||
check_or_make_tagged_checksum(?CSUM_TAG_CLIENT_SHA, Client_CSum, Chunk) ->
|
||||
CS = machi_util:checksum_chunk(Chunk),
|
||||
if CS == Client_CSum ->
|
||||
machi_util:make_tagged_csum(server_sha,
|
||||
Client_CSum);
|
||||
true ->
|
||||
throw({bad_csum, CS})
|
||||
end.
|
||||
|
||||
%%%% High PB mode %%%%
|
||||
|
||||
do_pb_hl_request(#mpb_request{req_id=ReqID}, #state{pb_mode=low}=S) ->
|
||||
Result = {low_error, 41, "High protocol request while in low mode"},
|
||||
{machi_pb_translate:to_pb_response(ReqID, unused, Result), S};
|
||||
do_pb_hl_request(PB_request, S) ->
|
||||
{ReqID, Cmd} = machi_pb_translate:from_pb_request(PB_request),
|
||||
{Result, S2} = do_pb_hl_request2(Cmd, S),
|
||||
{machi_pb_translate:to_pb_response(ReqID, Cmd, Result), S2}.
|
||||
|
||||
do_pb_hl_request2({high_echo, Msg}, S) ->
|
||||
{Msg, S};
|
||||
do_pb_hl_request2({high_auth, _User, _Pass}, S) ->
|
||||
{-77, S};
|
||||
do_pb_hl_request2({high_append_chunk=Op, NS, Prefix, Chunk, TaggedCSum, Opts},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
NSInfo = #ns_info{name=NS}, % TODO populate other fields
|
||||
todo_perhaps_remind_ns_locator_not_chosen(Op),
|
||||
Res = machi_cr_client:append_chunk(Clnt, NSInfo,
|
||||
Prefix, Chunk, TaggedCSum, Opts),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_write_chunk=Op, File, Offset, Chunk, CSum},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
NSInfo = undefined,
|
||||
todo_perhaps_remind_ns_locator_not_chosen(Op),
|
||||
Res = machi_cr_client:write_chunk(Clnt, NSInfo, File, Offset, Chunk, CSum),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_read_chunk=Op, File, Offset, Size, Opts},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
NSInfo = undefined,
|
||||
todo_perhaps_remind_ns_locator_not_chosen(Op),
|
||||
Res = machi_cr_client:read_chunk(Clnt, NSInfo, File, Offset, Size, Opts),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_trim_chunk=Op, File, Offset, Size},
|
||||
#state{high_clnt=Clnt}=S) ->
|
||||
NSInfo = undefined,
|
||||
todo_perhaps_remind_ns_locator_not_chosen(Op),
|
||||
Res = machi_cr_client:trim_chunk(Clnt, NSInfo, File, Offset, Size),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_checksum_list, File}, #state{high_clnt=Clnt}=S) ->
|
||||
Res = machi_cr_client:checksum_list(Clnt, File),
|
||||
{Res, S};
|
||||
do_pb_hl_request2({high_list_files}, #state{high_clnt=Clnt}=S) ->
|
||||
Res = machi_cr_client:list_files(Clnt),
|
||||
{Res, S}.
|
||||
|
||||
make_high_clnt(#state{high_clnt=undefined}=S) ->
|
||||
{ok, Proj} = machi_projection_store:read_latest_projection(
|
||||
S#state.proj_store, private),
|
||||
Ps = [P_srvr || {_, P_srvr} <- orddict:to_list(
|
||||
Proj#projection_v1.members_dict)],
|
||||
{ok, Clnt} = machi_cr_client:start_link(Ps),
|
||||
S#state{high_clnt=Clnt};
|
||||
make_high_clnt(S) ->
|
||||
S.
|
||||
|
||||
todo_perhaps_remind_ns_locator_not_chosen(Op) ->
|
||||
Key = {?MODULE, Op},
|
||||
case get(Key) of
|
||||
undefined ->
|
||||
io:format(user, "TODO op ~w is using default locator value\n",
|
||||
[Op]),
|
||||
put(Key, true);
|
||||
_ ->
|
||||
ok
|
||||
end.
|
||||
|
118
src/machi_flu1_subsup.erl
Normal file
118
src/machi_flu1_subsup.erl
Normal file
|
@ -0,0 +1,118 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc A supervisor to hold dynamic processes inside single
|
||||
%% FLU service, ranch listener and append server.
|
||||
|
||||
%% TODO: This supervisor is maybe useless. First introduced for
|
||||
%% workaround to start listener dynamically in flu1 initialization
|
||||
%% phase. Because `machi_flu_psup' is being blocked in flu1
|
||||
%% initialization time, adding a child to the supervisor leads to
|
||||
%% deadlock. If initialization can be done only by static arguments,
|
||||
%% then this supervisor should be removed and added as a direct child
|
||||
%% of `machi_flu_psup'.
|
||||
|
||||
-module(machi_flu1_subsup).
|
||||
-behaviour(supervisor).
|
||||
|
||||
%% public API
|
||||
-export([start_link/1,
|
||||
start_append_server/4,
|
||||
stop_append_server/1,
|
||||
start_listener/7,
|
||||
stop_listener/1,
|
||||
subsup_name/1,
|
||||
listener_name/1]).
|
||||
|
||||
%% supervisor callback
|
||||
-export([init/1]).
|
||||
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-define(SHUTDOWN, 5000).
|
||||
-define(BACKLOG, 8192).
|
||||
|
||||
-spec start_link(pv1_server()) -> {ok, pid()}.
|
||||
start_link(FluName) ->
|
||||
supervisor:start_link({local, subsup_name(FluName)}, ?MODULE, []).
|
||||
|
||||
-spec start_append_server(pv1_server(), boolean(), boolean(),
|
||||
undefined | machi_dt:epoch_id()) ->
|
||||
{ok, pid()}.
|
||||
start_append_server(FluName, Witness_p, Wedged_p, EpochId) ->
|
||||
supervisor:start_child(subsup_name(FluName),
|
||||
append_server_spec(FluName, Witness_p, Wedged_p, EpochId)).
|
||||
|
||||
-spec stop_append_server(pv1_server()) -> ok.
|
||||
stop_append_server(FluName) ->
|
||||
SubSup = listener_name(FluName),
|
||||
ok = supervisor:terminate_child(SubSup, FluName),
|
||||
ok = supervisor:delete_child(SubSup, FluName).
|
||||
|
||||
-spec start_listener(pv1_server(), inet:port_number(), boolean(),
|
||||
string(), ets:tab(), atom() | pid(),
|
||||
proplists:proplist()) -> {ok, pid()}.
|
||||
start_listener(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore,
|
||||
Props) ->
|
||||
supervisor:start_child(subsup_name(FluName),
|
||||
listener_spec(FluName, TcpPort, Witness, DataDir,
|
||||
EpochTab, ProjStore, Props)).
|
||||
|
||||
-spec stop_listener(pv1_server()) -> ok.
|
||||
stop_listener(FluName) ->
|
||||
SupName = subsup_name(FluName),
|
||||
ListenerName = listener_name(FluName),
|
||||
ok = supervisor:terminate_child(SupName, ListenerName),
|
||||
ok = supervisor:delete_child(SupName, ListenerName).
|
||||
|
||||
-spec subsup_name(pv1_server()) -> atom().
|
||||
subsup_name(FluName) when is_atom(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_flu1_subsup").
|
||||
|
||||
-spec listener_name(pv1_server()) -> atom().
|
||||
listener_name(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_listener").
|
||||
|
||||
%% Supervisor callback
|
||||
|
||||
init([]) ->
|
||||
SupFlags = {one_for_all, 1000, 10},
|
||||
{ok, {SupFlags, []}}.
|
||||
|
||||
%% private
|
||||
|
||||
-spec listener_spec(pv1_server(), inet:port_number(), boolean(),
|
||||
string(), ets:tab(), atom() | pid(),
|
||||
proplists:proplist()) -> supervisor:child_spec().
|
||||
listener_spec(FluName, TcpPort, Witness, DataDir, EpochTab, ProjStore, Props) ->
|
||||
ListenerName = listener_name(FluName),
|
||||
NbAcceptors = 10,
|
||||
TcpOpts = [{port, TcpPort}, {backlog, ?BACKLOG}],
|
||||
NetServerOpts = [FluName, Witness, DataDir, EpochTab, ProjStore, Props],
|
||||
ranch:child_spec(ListenerName, NbAcceptors,
|
||||
ranch_tcp, TcpOpts,
|
||||
machi_flu1_net_server, NetServerOpts).
|
||||
|
||||
-spec append_server_spec(pv1_server(), boolean(), boolean(),
|
||||
undefined | machi_dt:epoch_id()) -> supervisor:child_spec().
|
||||
append_server_spec(FluName, Witness_p, Wedged_p, EpochId) ->
|
||||
{FluName, {machi_flu1_append_server, start_link,
|
||||
[FluName, Witness_p, Wedged_p, EpochId]},
|
||||
permanent, ?SHUTDOWN, worker, [machi_flu1_append_server]}.
|
232
src/machi_flu_filename_mgr.erl
Normal file
232
src/machi_flu_filename_mgr.erl
Normal file
|
@ -0,0 +1,232 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% @doc This process is responsible for managing filenames assigned to
|
||||
%% prefixes. It's started out of `machi_flu_psup'.
|
||||
%%
|
||||
%% Supported operations include finding the "current" filename assigned to
|
||||
%% a prefix. Incrementing the sequence number and returning a new file name
|
||||
%% and listing all data files assigned to a given prefix.
|
||||
%%
|
||||
%% All prefixes should have the form of `{prefix, P}'. Single filename
|
||||
%% return values have the form of `{file, F}'.
|
||||
%%
|
||||
%% <h2>Finding the current file associated with a sequence</h2>
|
||||
%% First it looks up the sequence number from the prefix name. If
|
||||
%% no sequence file is found, it uses 0 as the sequence number and searches
|
||||
%% for a matching file with the prefix and 0 as the sequence number.
|
||||
%% If no file is found, the it generates a new filename by incorporating
|
||||
%% the given prefix, a randomly generated (v4) UUID and 0 as the
|
||||
%% sequence number.
|
||||
%%
|
||||
%% If the sequence number is > 0, then the process scans the filesystem
|
||||
%% looking for a filename which matches the prefix and given sequence number and
|
||||
%% returns that.
|
||||
|
||||
-module(machi_flu_filename_mgr).
|
||||
-behavior(gen_server).
|
||||
|
||||
-ifdef(TEST).
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-compile(export_all).
|
||||
-endif.
|
||||
|
||||
-export([
|
||||
child_spec/2,
|
||||
start_link/2,
|
||||
find_or_make_filename_from_prefix/4,
|
||||
increment_prefix_sequence/3,
|
||||
list_files_by_prefix/2
|
||||
]).
|
||||
|
||||
%% gen_server callbacks
|
||||
-export([
|
||||
init/1,
|
||||
handle_cast/2,
|
||||
handle_call/3,
|
||||
handle_info/2,
|
||||
terminate/2,
|
||||
code_change/3
|
||||
]).
|
||||
|
||||
-define(TIMEOUT, 10 * 1000).
|
||||
-include("machi.hrl"). %% included for #ns_info record
|
||||
-include("machi_projection.hrl"). %% included for pv1_epoch type
|
||||
|
||||
-record(state, {fluname :: atom(),
|
||||
tid :: ets:tid(),
|
||||
datadir :: string(),
|
||||
epoch :: pv1_epoch()
|
||||
}).
|
||||
|
||||
%% public API
|
||||
|
||||
child_spec(FluName, DataDir) ->
|
||||
Name = make_filename_mgr_name(FluName),
|
||||
{Name,
|
||||
{?MODULE, start_link, [FluName, DataDir]},
|
||||
permanent, 5000, worker, [?MODULE]}.
|
||||
|
||||
start_link(FluName, DataDir) when is_atom(FluName) andalso is_list(DataDir) ->
|
||||
N = make_filename_mgr_name(FluName),
|
||||
gen_server:start_link({local, N}, ?MODULE, [FluName, DataDir], []).
|
||||
|
||||
-spec find_or_make_filename_from_prefix( FluName :: atom(),
|
||||
EpochId :: pv1_epoch(),
|
||||
Prefix :: {prefix, string()},
|
||||
machi_dt:ns_info()) ->
|
||||
{file, Filename :: string()} | {error, Reason :: term() } | timeout.
|
||||
% @doc Find the latest available or make a filename from a prefix. A prefix
|
||||
% should be in the form of a tagged tuple `{prefix, P}'. Returns a tagged
|
||||
% tuple in the form of `{file, F}' or an `{error, Reason}'
|
||||
find_or_make_filename_from_prefix(FluName, EpochId,
|
||||
{prefix, Prefix},
|
||||
#ns_info{}=NSInfo)
|
||||
when is_atom(FluName) ->
|
||||
N = make_filename_mgr_name(FluName),
|
||||
gen_server:call(N, {find_filename, FluName, EpochId, NSInfo, Prefix}, ?TIMEOUT);
|
||||
find_or_make_filename_from_prefix(_FluName, _EpochId, Other, Other2) ->
|
||||
lager:error("~p is not a valid prefix/locator ~p", [Other, Other2]),
|
||||
error(badarg).
|
||||
|
||||
-spec increment_prefix_sequence( FluName :: atom(), NSInfo :: machi_dt:ns_info(), Prefix :: {prefix, string()} ) ->
|
||||
ok | {error, Reason :: term() } | timeout.
|
||||
% @doc Increment the sequence counter for a given prefix. Prefix should
|
||||
% be in the form of `{prefix, P}'.
|
||||
increment_prefix_sequence(FluName, #ns_info{}=NSInfo, {prefix, Prefix}) when is_atom(FluName) ->
|
||||
gen_server:call(make_filename_mgr_name(FluName), {increment_sequence, NSInfo, Prefix}, ?TIMEOUT);
|
||||
increment_prefix_sequence(_FluName, _NSInfo, Other) ->
|
||||
lager:error("~p is not a valid prefix.", [Other]),
|
||||
error(badarg).
|
||||
|
||||
-spec list_files_by_prefix( FluName :: atom(), Prefix :: {prefix, string()} ) ->
|
||||
[ file:name() ] | timeout | {error, Reason :: term() }.
|
||||
% @doc Given a prefix in the form of `{prefix, P}' return
|
||||
% all the data files associated with that prefix. Returns
|
||||
% a list.
|
||||
list_files_by_prefix(FluName, {prefix, Prefix}) when is_atom(FluName) ->
|
||||
gen_server:call(make_filename_mgr_name(FluName), {list_files, Prefix}, ?TIMEOUT);
|
||||
list_files_by_prefix(_FluName, Other) ->
|
||||
lager:error("~p is not a valid prefix.", [Other]),
|
||||
error(badarg).
|
||||
|
||||
%% gen_server API
|
||||
init([FluName, DataDir]) ->
|
||||
Tid = ets:new(make_filename_mgr_name(FluName), [named_table, {read_concurrency, true}]),
|
||||
{ok, #state{fluname = FluName,
|
||||
epoch = ?DUMMY_PV1_EPOCH,
|
||||
datadir = DataDir,
|
||||
tid = Tid}}.
|
||||
|
||||
handle_cast(Req, State) ->
|
||||
lager:warning("Got unknown cast ~p", [Req]),
|
||||
{noreply, State}.
|
||||
|
||||
%% Important assumption: by the time we reach here the EpochId is kosher.
|
||||
%% the FLU has already validated that the caller's epoch id and the FLU's epoch id
|
||||
%% are the same. So we *assume* that remains the case here - that is to say, we
|
||||
%% are not wedged.
|
||||
handle_call({find_filename, FluName, EpochId, NSInfo, Prefix}, _From,
|
||||
S = #state{ datadir = DataDir, epoch = EpochId, tid = Tid }) ->
|
||||
%% Our state and the caller's epoch ids are the same. Business as usual.
|
||||
File = handle_find_file(FluName, Tid, NSInfo, Prefix, DataDir),
|
||||
{reply, {file, File}, S};
|
||||
|
||||
handle_call({find_filename, _FluName, EpochId, NSInfo, Prefix}, _From, S = #state{ datadir = DataDir, tid = Tid }) ->
|
||||
%% If the epoch id in our state and the caller's epoch id were the same, it would've
|
||||
%% matched the above clause. Since we're here, we know that they are different.
|
||||
%% If epoch ids between our state and the caller's are different, we must increment the
|
||||
%% sequence number, generate a filename and then cache it.
|
||||
File = increment_and_cache_filename(Tid, DataDir, NSInfo, Prefix),
|
||||
{reply, {file, File}, S#state{epoch = EpochId}};
|
||||
|
||||
handle_call({increment_sequence, #ns_info{name=NS, locator=NSLocator}, Prefix}, _From, S = #state{ datadir = DataDir, tid=Tid }) ->
|
||||
NSInfo = #ns_info{name=NS, locator=NSLocator},
|
||||
_File = increment_and_cache_filename(Tid, DataDir, NSInfo, Prefix),
|
||||
{reply, ok, S};
|
||||
handle_call({list_files, Prefix}, From, S = #state{ datadir = DataDir }) ->
|
||||
spawn(fun() ->
|
||||
L = list_files(DataDir, Prefix),
|
||||
gen_server:reply(From, L)
|
||||
end),
|
||||
{noreply, S};
|
||||
|
||||
handle_call(Req, From, State) ->
|
||||
lager:warning("Got unknown call ~p from ~p", [Req, From]),
|
||||
{reply, hoge, State}.
|
||||
|
||||
handle_info(Info, State) ->
|
||||
lager:warning("Got unknown info ~p", [Info]),
|
||||
{noreply, State}.
|
||||
|
||||
terminate(Reason, _State) ->
|
||||
lager:info("Shutting down because ~p", [Reason]),
|
||||
ok.
|
||||
|
||||
code_change(_OldVsn, State, _Extra) ->
|
||||
{ok, State}.
|
||||
|
||||
%% private
|
||||
|
||||
%% Quoted from https://github.com/afiskon/erlang-uuid-v4/blob/master/src/uuid.erl
|
||||
%% MIT License
|
||||
generate_uuid_v4_str() ->
|
||||
<<A:32, B:16, C:16, D:16, E:48>> = crypto:strong_rand_bytes(16),
|
||||
io_lib:format("~8.16.0b-~4.16.0b-4~3.16.0b-~4.16.0b-~12.16.0b",
|
||||
[A, B, C band 16#0fff, D band 16#3fff bor 16#8000, E]).
|
||||
|
||||
list_files(DataDir, Prefix) ->
|
||||
{F_bin, Path} = machi_util:make_data_filename(DataDir, "*^" ++ Prefix ++ "^*"),
|
||||
filelib:wildcard(binary_to_list(F_bin), filename:dirname(Path)).
|
||||
|
||||
make_filename_mgr_name(FluName) when is_atom(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_filename_mgr").
|
||||
|
||||
handle_find_file(_FluName, Tid, #ns_info{name=NS, locator=NSLocator}, Prefix, DataDir) ->
|
||||
case ets:lookup(Tid, {NS, NSLocator, Prefix}) of
|
||||
[] ->
|
||||
N = machi_util:read_max_filenum(DataDir, NS, NSLocator, Prefix),
|
||||
F = generate_filename(DataDir, NS, NSLocator, Prefix, N),
|
||||
true = ets:insert(Tid, {{NS, NSLocator, Prefix}, F}),
|
||||
F;
|
||||
[{_Key, File}] ->
|
||||
File
|
||||
end.
|
||||
|
||||
generate_filename(DataDir, NS, NSLocator, Prefix, N) ->
|
||||
{F, _Q} = machi_util:make_data_filename(
|
||||
DataDir,
|
||||
NS, NSLocator, Prefix,
|
||||
generate_uuid_v4_str(),
|
||||
N),
|
||||
binary_to_list(F).
|
||||
|
||||
increment_and_cache_filename(Tid, DataDir, #ns_info{name=NS,locator=NSLocator}, Prefix) ->
|
||||
ok = machi_util:increment_max_filenum(DataDir, NS, NSLocator, Prefix),
|
||||
N = machi_util:read_max_filenum(DataDir, NS, NSLocator, Prefix),
|
||||
F = generate_filename(DataDir, NS, NSLocator, Prefix, N),
|
||||
true = ets:insert(Tid, {{NS, NSLocator, Prefix}, F}),
|
||||
F.
|
||||
|
||||
|
||||
|
||||
-ifdef(TEST).
|
||||
|
||||
-endif.
|
297
src/machi_flu_metadata_mgr.erl
Normal file
297
src/machi_flu_metadata_mgr.erl
Normal file
|
@ -0,0 +1,297 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc This is a metadata service for the machi FLU which currently
|
||||
%% tracks the mappings between filenames and file proxies.
|
||||
%%
|
||||
%% The service takes a given hash space and spreads it out over a
|
||||
%% pool of N processes which are responsible for 1/Nth the hash
|
||||
%% space. When a user requests an operation on a particular file
|
||||
%% the filename is hashed into the hash space and the request
|
||||
%% forwarded to a particular manager responsible for that slice
|
||||
%% of the hash space.
|
||||
%%
|
||||
%% The current hash implementation is `erlang:phash2/1' which has
|
||||
%% a range between 0..2^27-1 or 134,217,727.
|
||||
|
||||
-module(machi_flu_metadata_mgr).
|
||||
-behaviour(gen_server).
|
||||
|
||||
-include("machi.hrl").
|
||||
|
||||
-define(MAX_MGRS, 10). %% number of managers to start by default.
|
||||
-define(HASH(X), erlang:phash2(X)). %% hash algorithm to use
|
||||
-define(TIMEOUT, 10 * 1000). %% 10 second timeout
|
||||
|
||||
-define(KNOWN_FILES_LIST_PREFIX, "known_files_").
|
||||
|
||||
-record(state, {fluname :: atom(),
|
||||
datadir :: string(),
|
||||
tid :: ets:tid(),
|
||||
cnt :: non_neg_integer(),
|
||||
trimmed_files :: machi_plist:plist()
|
||||
}).
|
||||
|
||||
%% This record goes in the ets table where filename is the key
|
||||
-record(md, {filename :: string(),
|
||||
proxy_pid :: undefined|pid(),
|
||||
mref :: undefined|reference() %% monitor ref for file proxy
|
||||
}).
|
||||
|
||||
%% public api
|
||||
-export([
|
||||
child_spec/4,
|
||||
start_link/4,
|
||||
lookup_manager_pid/2,
|
||||
lookup_proxy_pid/2,
|
||||
start_proxy_pid/2,
|
||||
stop_proxy_pid/2,
|
||||
stop_proxy_pid_rollover/2,
|
||||
build_metadata_mgr_name/2,
|
||||
trim_file/2
|
||||
]).
|
||||
|
||||
%% gen_server callbacks
|
||||
-export([
|
||||
init/1,
|
||||
handle_cast/2,
|
||||
handle_call/3,
|
||||
handle_info/2,
|
||||
terminate/2,
|
||||
code_change/3
|
||||
]).
|
||||
|
||||
%% Public API
|
||||
build_metadata_mgr_name(FluName, N) when is_atom(FluName) andalso is_integer(N) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_metadata_mgr_" ++ integer_to_list(N)).
|
||||
|
||||
child_spec(FluName, C, DataDir, N) ->
|
||||
Name = build_metadata_mgr_name(FluName, C),
|
||||
{Name,
|
||||
{?MODULE, start_link, [FluName, Name, DataDir, N]},
|
||||
permanent, 5000, worker, [?MODULE]}.
|
||||
|
||||
start_link(FluName, Name, DataDir, Num) when is_atom(Name) andalso is_list(DataDir) ->
|
||||
gen_server:start_link({local, Name}, ?MODULE, [FluName, Name, DataDir, Num], []).
|
||||
|
||||
lookup_manager_pid(FluName, {file, Filename}) ->
|
||||
whereis(get_manager_atom(FluName, Filename)).
|
||||
|
||||
lookup_proxy_pid(FluName, {file, Filename}) ->
|
||||
gen_server:call(get_manager_atom(FluName, Filename), {proxy_pid, Filename}, ?TIMEOUT).
|
||||
|
||||
start_proxy_pid(FluName, {file, Filename}) ->
|
||||
gen_server:call(get_manager_atom(FluName, Filename), {start_proxy_pid, Filename}, ?TIMEOUT).
|
||||
|
||||
stop_proxy_pid(FluName, {file, Filename}) ->
|
||||
gen_server:call(get_manager_atom(FluName, Filename), {stop_proxy_pid, false, Filename}, ?TIMEOUT).
|
||||
|
||||
stop_proxy_pid_rollover(FluName, {file, Filename}) ->
|
||||
gen_server:call(get_manager_atom(FluName, Filename), {stop_proxy_pid, true, Filename}, ?TIMEOUT).
|
||||
|
||||
trim_file(FluName, {file, Filename}) ->
|
||||
gen_server:call(get_manager_atom(FluName, Filename), {trim_file, Filename}, ?TIMEOUT).
|
||||
|
||||
%% gen_server callbacks
|
||||
init([FluName, Name, DataDir, Num]) ->
|
||||
%% important: we'll need another persistent storage to
|
||||
%% remember deleted (trimmed) file, to prevent resurrection after
|
||||
%% flu restart and append.
|
||||
FileListFileName =
|
||||
filename:join([DataDir, ?KNOWN_FILES_LIST_PREFIX ++ atom_to_list(FluName)]),
|
||||
{ok, PList} = machi_plist:open(FileListFileName, []),
|
||||
%% TODO make sure all files non-existent, if any remaining files
|
||||
%% here, just delete it. They're in the list *because* they're all
|
||||
%% trimmed.
|
||||
|
||||
Tid = ets:new(Name, [{keypos, 2}, {read_concurrency, true}, {write_concurrency, true}]),
|
||||
{ok, #state{fluname = FluName, datadir = DataDir, tid = Tid, cnt = Num,
|
||||
trimmed_files=PList}}.
|
||||
|
||||
handle_cast(Req, State) ->
|
||||
lager:warning("Got unknown cast ~p", [Req]),
|
||||
{noreply, State}.
|
||||
|
||||
handle_call({proxy_pid, Filename}, _From, State = #state{ tid = Tid }) ->
|
||||
Reply = case lookup_md(Tid, Filename) of
|
||||
not_found -> undefined;
|
||||
R -> R#md.proxy_pid
|
||||
end,
|
||||
{reply, Reply, State};
|
||||
|
||||
handle_call({start_proxy_pid, Filename}, _From,
|
||||
State = #state{ fluname = N, tid = Tid, datadir = D,
|
||||
trimmed_files=TrimmedFiles}) ->
|
||||
case machi_plist:find(TrimmedFiles, Filename) of
|
||||
false ->
|
||||
NewR = case lookup_md(Tid, Filename) of
|
||||
not_found ->
|
||||
start_file_proxy(N, D, Filename);
|
||||
#md{ proxy_pid = undefined } = R0 ->
|
||||
start_file_proxy(N, D, R0);
|
||||
#md{ proxy_pid = _Pid } = R1 ->
|
||||
R1
|
||||
end,
|
||||
update_ets(Tid, NewR),
|
||||
{reply, {ok, NewR#md.proxy_pid}, State};
|
||||
true ->
|
||||
{reply, {error, trimmed}, State}
|
||||
end;
|
||||
|
||||
handle_call({stop_proxy_pid, Rollover_p, Filename}, _From, State = #state{ tid = Tid }) ->
|
||||
case lookup_md(Tid, Filename) of
|
||||
not_found ->
|
||||
ok;
|
||||
#md{ proxy_pid = undefined } ->
|
||||
ok;
|
||||
#md{ proxy_pid = Pid, mref = M } = R ->
|
||||
demonitor(M, [flush]),
|
||||
if Rollover_p ->
|
||||
do_rollover(Filename, State);
|
||||
true ->
|
||||
machi_file_proxy:stop(Pid),
|
||||
update_ets(Tid, R#md{ proxy_pid = undefined,
|
||||
mref = undefined })
|
||||
end
|
||||
end,
|
||||
{reply, ok, State};
|
||||
|
||||
handle_call({trim_file, Filename}, _,
|
||||
S = #state{trimmed_files = TrimmedFiles }) ->
|
||||
case machi_plist:add(TrimmedFiles, Filename) of
|
||||
{ok, TrimmedFiles2} ->
|
||||
{reply, ok, S#state{trimmed_files=TrimmedFiles2}};
|
||||
Error ->
|
||||
{reply, Error, S}
|
||||
end;
|
||||
|
||||
handle_call(Req, From, State) ->
|
||||
lager:warning("Got unknown call ~p from ~p", [Req, From]),
|
||||
{reply, hoge, State}.
|
||||
|
||||
handle_info({'DOWN', Mref, process, Pid, normal}, State = #state{ tid = Tid }) ->
|
||||
lager:debug("file proxy ~p shutdown normally", [Pid]),
|
||||
clear_ets(Tid, Mref),
|
||||
{noreply, State};
|
||||
|
||||
handle_info({'DOWN', Mref, process, Pid, wedged}, State = #state{ tid = Tid }) ->
|
||||
lager:error("file proxy ~p shutdown because it's wedged", [Pid]),
|
||||
clear_ets(Tid, Mref),
|
||||
{noreply, State};
|
||||
handle_info({'DOWN', _Mref, process, Pid, trimmed}, State = #state{ tid = _Tid }) ->
|
||||
lager:debug("file proxy ~p shutdown because the file was trimmed", [Pid]),
|
||||
{noreply, State};
|
||||
handle_info({'DOWN', Mref, process, Pid, Error}, State = #state{ tid = Tid }) ->
|
||||
lager:error("file proxy ~p shutdown because ~p", [Pid, Error]),
|
||||
clear_ets(Tid, Mref),
|
||||
{noreply, State};
|
||||
|
||||
handle_info(Info, State) ->
|
||||
lager:warning("Got unknown info ~p", [Info]),
|
||||
{noreply, State}.
|
||||
|
||||
terminate(Reason, _State = #state{trimmed_files=TrimmedFiles}) ->
|
||||
lager:info("Shutting down because ~p", [Reason]),
|
||||
machi_plist:close(TrimmedFiles),
|
||||
ok.
|
||||
|
||||
code_change(_OldVsn, State, _Extra) ->
|
||||
{ok, State}.
|
||||
|
||||
%% Private functions
|
||||
|
||||
compute_hash(Data) ->
|
||||
?HASH(Data).
|
||||
|
||||
compute_worker(Hash) ->
|
||||
MgrCount = get_env(metadata_manager_count, ?MAX_MGRS),
|
||||
(Hash rem MgrCount) + 1.
|
||||
%% TODO MARK: Hrm, the intermittent failures of both of the tests
|
||||
%% in machi_cr_client_test.erl were due to this func returning 0.
|
||||
%% But machi_flu_metadata_mgr_sup:init() doesn't make a child with N=0.
|
||||
%%
|
||||
%% The remaining puzzle for me, which I'm now punting to you, sorry,
|
||||
%% is why those two tests would sometimes pass, which implies to me that
|
||||
%% sometimes the code path used by those tests never chooses worker 0
|
||||
%% and occasionally it does choose worker 0.
|
||||
|
||||
get_manager_atom(FluName, Data) ->
|
||||
build_metadata_mgr_name(FluName, compute_worker(compute_hash(Data))).
|
||||
|
||||
lookup_md(Tid, Data) ->
|
||||
case ets:lookup(Tid, Data) of
|
||||
[] -> not_found;
|
||||
[R] -> R
|
||||
end.
|
||||
|
||||
start_file_proxy(FluName, D, R = #md{filename = F} ) ->
|
||||
{ok, Pid} = machi_file_proxy_sup:start_proxy(FluName, D, F),
|
||||
Mref = monitor(process, Pid),
|
||||
R#md{ proxy_pid = Pid, mref = Mref };
|
||||
|
||||
start_file_proxy(FluName, D, Filename) ->
|
||||
start_file_proxy(FluName, D, #md{ filename = Filename }).
|
||||
|
||||
update_ets(Tid, R) ->
|
||||
ets:insert(Tid, R).
|
||||
|
||||
clear_ets(Tid, Mref) ->
|
||||
R = get_md_record_by_mref(Tid, Mref),
|
||||
update_ets(Tid, R#md{ proxy_pid = undefined, mref = undefined }).
|
||||
|
||||
purge_ets(Tid, R) ->
|
||||
true = ets:delete_object(Tid, R).
|
||||
|
||||
get_md_record_by_mref(Tid, Mref) ->
|
||||
[R] = ets:match_object(Tid, {md, '_', '_', Mref}),
|
||||
R.
|
||||
|
||||
get_md_record_by_filename(Tid, Filename) ->
|
||||
[R] = ets:lookup(Tid, Filename),
|
||||
R.
|
||||
|
||||
get_env(Setting, Default) ->
|
||||
case application:get_env(machi, Setting) of
|
||||
undefined -> Default;
|
||||
{ok, V} -> V
|
||||
end.
|
||||
|
||||
do_rollover(Filename, _State = #state{ fluname = FluName,
|
||||
tid = Tid }) ->
|
||||
R = get_md_record_by_filename(Tid, Filename),
|
||||
lager:info("file ~p proxy ~p shutdown because of file rollover",
|
||||
[Filename, R#md.proxy_pid]),
|
||||
{Prefix, NS, NSLocator, _, _} =
|
||||
machi_util:parse_filename(R#md.filename),
|
||||
|
||||
%% We only increment the counter here. The filename will be generated on the
|
||||
%% next append request to that prefix and since the filename will have a new
|
||||
%% sequence number it probably will be associated with a different metadata
|
||||
%% manager. That's why we don't want to generate a new file name immediately
|
||||
%% and use it to start a new file proxy.
|
||||
NSInfo = #ns_info{name=NS, locator=NSLocator},
|
||||
lager:warning("INCR: ~p ~p\n", [FluName, Prefix]),
|
||||
ok = machi_flu_filename_mgr:increment_prefix_sequence(FluName, NSInfo, {prefix, Prefix}),
|
||||
|
||||
%% purge our ets table of this entry completely since it is likely the
|
||||
%% new filename (whenever it comes) will be in a different manager than
|
||||
%% us.
|
||||
purge_ets(Tid, R),
|
||||
ok.
|
59
src/machi_flu_metadata_mgr_sup.erl
Normal file
59
src/machi_flu_metadata_mgr_sup.erl
Normal file
|
@ -0,0 +1,59 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc This is the supervisor for the collection of metadata
|
||||
%% managers. It's started out of `machi_flu_psup'. It reads an
|
||||
%% application environment variable named `metadata_manager_count'
|
||||
%% with a default of 10 if it is not set.
|
||||
|
||||
-module(machi_flu_metadata_mgr_sup).
|
||||
-behaviour(supervisor).
|
||||
|
||||
%% public API
|
||||
-export([
|
||||
child_spec/3,
|
||||
start_link/3
|
||||
]).
|
||||
|
||||
%% supervisor callback
|
||||
-export([init/1]).
|
||||
|
||||
child_spec(FluName, DataDir, N) ->
|
||||
{make_sup_name(FluName),
|
||||
{?MODULE, start_link, [FluName, DataDir, N]},
|
||||
permanent, 5000, supervisor, [?MODULE]}.
|
||||
|
||||
start_link(FluName, DataDir, N) ->
|
||||
supervisor:start_link({local, make_sup_name(FluName)}, ?MODULE, [FluName, DataDir, N]).
|
||||
|
||||
init([FluName, DataDir, N]) ->
|
||||
Restart = one_for_one,
|
||||
MaxRestarts = 1000,
|
||||
SecondsBetween = 3600,
|
||||
SupFlags = {Restart, MaxRestarts, SecondsBetween},
|
||||
|
||||
Children =
|
||||
[ machi_flu_metadata_mgr:child_spec(FluName, C, DataDir, N) || C <- lists:seq(1,N) ],
|
||||
|
||||
{ok, {SupFlags, Children}}.
|
||||
|
||||
make_sup_name(FluName) when is_atom(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_metadata_mgr_sup").
|
||||
|
|
@ -73,7 +73,7 @@
|
|||
-endif.
|
||||
|
||||
%% External API
|
||||
-export([make_package_spec/4,
|
||||
-export([make_package_spec/1, make_package_spec/4,
|
||||
start_flu_package/1, start_flu_package/4, stop_flu_package/1]).
|
||||
%% Internal API
|
||||
-export([start_link/4,
|
||||
|
@ -83,13 +83,20 @@
|
|||
%% Supervisor callbacks
|
||||
-export([init/1]).
|
||||
|
||||
make_package_spec(#p_srvr{name=FluName, port=TcpPort, props=Props}) when is_list(Props) ->
|
||||
make_package_spec({FluName, TcpPort, Props});
|
||||
make_package_spec({FluName, TcpPort, Props}) when is_list(Props) ->
|
||||
FluDataDir = get_env(flu_data_dir, undefined_is_invalid),
|
||||
MyDataDir = filename:join(FluDataDir, atom_to_list(FluName)),
|
||||
make_package_spec(FluName, TcpPort, MyDataDir, Props).
|
||||
|
||||
make_package_spec(FluName, TcpPort, DataDir, Props) ->
|
||||
{FluName, {machi_flu_psup, start_link,
|
||||
[FluName, TcpPort, DataDir, Props]},
|
||||
permanent, ?SHUTDOWN, supervisor, []}.
|
||||
|
||||
start_flu_package(#p_srvr{name=FluName, port=TcpPort, props=Props}) ->
|
||||
DataDir = proplists:get_value(data_dir, Props),
|
||||
DataDir = get_data_dir(FluName, Props),
|
||||
start_flu_package(FluName, TcpPort, DataDir, Props).
|
||||
|
||||
start_flu_package(FluName, TcpPort, DataDir, Props) ->
|
||||
|
@ -130,11 +137,27 @@ init([FluName, TcpPort, DataDir, Props0]) ->
|
|||
{machi_chain_manager1, start_link,
|
||||
[FluName, [], Props]},
|
||||
permanent, ?SHUTDOWN, worker, []},
|
||||
|
||||
FNameMgrSpec = machi_flu_filename_mgr:child_spec(FluName, DataDir),
|
||||
|
||||
MetaMgrCnt = get_env(metadata_manager_count, 10),
|
||||
MetaSupSpec = machi_flu_metadata_mgr_sup:child_spec(FluName, DataDir, MetaMgrCnt),
|
||||
|
||||
FProxySupSpec = machi_file_proxy_sup:child_spec(FluName),
|
||||
|
||||
Flu1SubSupSpec = {machi_flu1_subsup:subsup_name(FluName),
|
||||
{machi_flu1_subsup, start_link, [FluName]},
|
||||
permanent, ?SHUTDOWN, supervisor, []},
|
||||
|
||||
FluSpec = {FluName,
|
||||
{machi_flu1, start_link,
|
||||
[ [{FluName, TcpPort, DataDir}|Props] ]},
|
||||
permanent, ?SHUTDOWN, worker, []},
|
||||
{ok, {SupFlags, [ProjSpec, FitnessSpec, MgrSpec, FluSpec]}}.
|
||||
|
||||
{ok, {SupFlags, [
|
||||
ProjSpec, FitnessSpec, MgrSpec,
|
||||
FProxySupSpec, FNameMgrSpec, MetaSupSpec,
|
||||
Flu1SubSupSpec, FluSpec]}}.
|
||||
|
||||
make_flu_regname(FluName) when is_atom(FluName) ->
|
||||
FluName.
|
||||
|
@ -150,3 +173,18 @@ make_proj_supname(ProjName) when is_atom(ProjName) ->
|
|||
|
||||
make_fitness_regname(FluName) when is_atom(FluName) ->
|
||||
list_to_atom(atom_to_list(FluName) ++ "_fitness").
|
||||
|
||||
get_env(Setting, Default) ->
|
||||
case application:get_env(machi, Setting) of
|
||||
undefined -> Default;
|
||||
{ok, V} -> V
|
||||
end.
|
||||
|
||||
get_data_dir(FluName, Props) ->
|
||||
case proplists:get_value(data_dir, Props) of
|
||||
Path when is_list(Path) ->
|
||||
Path;
|
||||
undefined ->
|
||||
{ok, Dir} = application:get_env(machi, flu_data_dir),
|
||||
Dir ++ "/" ++ atom_to_list(FluName)
|
||||
end.
|
||||
|
|
|
@ -21,6 +21,9 @@
|
|||
%% @doc Supervisor for Machi FLU servers and their related support
|
||||
%% servers.
|
||||
%%
|
||||
%% Responsibility for managing FLU and chain lifecycle after the initial
|
||||
%% application startup is delegated to {@link machi_lifecycle_mgr}.
|
||||
%%
|
||||
%% See {@link machi_flu_psup} for an illustration of the entire Machi
|
||||
%% application process structure.
|
||||
|
||||
|
@ -29,8 +32,11 @@
|
|||
-behaviour(supervisor).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
-include("machi_verbose.hrl").
|
||||
|
||||
-ifdef(TEST).
|
||||
-compile(export_all).
|
||||
-ifdef(PULSE).
|
||||
-compile({parse_transform, pulse_instrument}).
|
||||
-include_lib("pulse_otp/include/pulse_otp.hrl").
|
||||
|
@ -38,9 +44,12 @@
|
|||
-else.
|
||||
-define(SHUTDOWN, 5000).
|
||||
-endif.
|
||||
-endif. %TEST
|
||||
|
||||
%% API
|
||||
-export([start_link/0]).
|
||||
-export([start_link/0,
|
||||
get_initial_flus/0, load_rc_d_files_from_dir/1,
|
||||
sanitize_p_srvr_records/1]).
|
||||
|
||||
%% Supervisor callbacks
|
||||
-export([init/1]).
|
||||
|
@ -56,13 +65,11 @@ init([]) ->
|
|||
MaxSecondsBetweenRestarts = 3600,
|
||||
SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts},
|
||||
|
||||
Tab = ets:new(?TEST_ETS_TABLE, [named_table, public, ordered_set,
|
||||
{read_concurrency,true}]),
|
||||
_Tab = ets:new(?TEST_ETS_TABLE, [named_table, public, ordered_set,
|
||||
{read_concurrency,true}]),
|
||||
|
||||
Ps = get_initial_flus(),
|
||||
FLU_specs = [machi_flu_psup:make_package_spec(FluName, TcpPort,
|
||||
DataDir, Props) ||
|
||||
{FluName, TcpPort, DataDir, Props} <- Ps],
|
||||
FLU_specs = [machi_flu_psup:make_package_spec(P) || P <- Ps],
|
||||
|
||||
{ok, {SupFlags, FLU_specs}}.
|
||||
|
||||
|
@ -71,5 +78,66 @@ get_initial_flus() ->
|
|||
[].
|
||||
-else. % PULSE
|
||||
get_initial_flus() ->
|
||||
application:get_env(machi, initial_flus, []).
|
||||
DoesNotExist = "/tmp/does/not/exist",
|
||||
ConfigDir = case application:get_env(machi, flu_config_dir, DoesNotExist) of
|
||||
DoesNotExist ->
|
||||
DoesNotExist;
|
||||
Dir ->
|
||||
Dir
|
||||
end,
|
||||
Ps = [P || {_File, P} <- load_rc_d_files_from_dir(ConfigDir)],
|
||||
sanitize_p_srvr_records(Ps).
|
||||
-endif. % PULSE
|
||||
|
||||
load_rc_d_files_from_dir(Dir) ->
|
||||
Files = filelib:wildcard(Dir ++ "/*"),
|
||||
[case file:consult(File) of
|
||||
{ok, [X]} ->
|
||||
{File, X};
|
||||
_ ->
|
||||
lager:warning("Error parsing file '~s', ignoring",
|
||||
[File]),
|
||||
{File, []}
|
||||
end || File <- Files].
|
||||
|
||||
sanitize_p_srvr_records(Ps) ->
|
||||
{Sane, _} = lists:foldl(fun sanitize_p_srvr_rec/2, {[], dict:new()}, Ps),
|
||||
Sane.
|
||||
|
||||
sanitize_p_srvr_rec(Whole, {Acc, D}) ->
|
||||
try
|
||||
#p_srvr{name=Name,
|
||||
proto_mod=PMod,
|
||||
address=Address,
|
||||
port=Port,
|
||||
props=Props} = Whole,
|
||||
true = is_atom(Name),
|
||||
NameK = {name, Name},
|
||||
error = dict:find(NameK, D),
|
||||
true = is_atom(PMod),
|
||||
case code:is_loaded(PMod) of
|
||||
{file, _} ->
|
||||
ok;
|
||||
_ ->
|
||||
{module, _} = code:load_file(PMod),
|
||||
ok
|
||||
end,
|
||||
if is_list(Address) -> ok;
|
||||
is_tuple(Address) -> ok % Erlang-style IPv4 or IPv6
|
||||
end,
|
||||
true = is_integer(Port) andalso Port >= 1024 andalso Port =< 65534,
|
||||
PortK = {port, Port},
|
||||
error = dict:find(PortK, D),
|
||||
true = is_list(Props),
|
||||
|
||||
%% All is sane enough.
|
||||
D2 = dict:store(NameK, Name,
|
||||
dict:store(PortK, Port, D)),
|
||||
{[Whole|Acc], D2}
|
||||
catch _:_ ->
|
||||
_ = lager:log(error, self(),
|
||||
"~s: Bad (or duplicate name/port) p_srvr record, "
|
||||
"skipping: ~P\n",
|
||||
[?MODULE, Whole, 15]),
|
||||
{Acc, D}
|
||||
end.
|
||||
|
|
1015
src/machi_lifecycle_mgr.erl
Normal file
1015
src/machi_lifecycle_mgr.erl
Normal file
File diff suppressed because it is too large
Load diff
156
src/machi_merkle_tree.erl
Normal file
156
src/machi_merkle_tree.erl
Normal file
|
@ -0,0 +1,156 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc Creates a Merkle tree per file based on the checksum data for
|
||||
%% a given data file.
|
||||
%%
|
||||
%% The `naive' implementation representation is:
|
||||
%%
|
||||
%% `<<Length:64, Offset:32, 0>>' for unwritten bytes
|
||||
%% `<<Length:64, Offset:32, 1>>' for trimmed bytes
|
||||
%% `<<Length:64, Offset:32, Csum/binary>>' for written bytes
|
||||
%%
|
||||
%% The tree feeds these leaf nodes into hashes representing chunks of a minimum
|
||||
%% size of at least 1024 KB (1 MB), but if the file size is larger, we will try
|
||||
%% to get about 100 chunks for the first rollup "Level 1." We aim for around 10
|
||||
%% hashes at level 2, and then 2 hashes level 3 and finally the root.
|
||||
|
||||
-module(machi_merkle_tree).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_merkle_tree.hrl").
|
||||
|
||||
-ifdef(TEST).
|
||||
-compile(export_all).
|
||||
-else.
|
||||
-export([
|
||||
open/2,
|
||||
open/3,
|
||||
tree/1,
|
||||
filename/1,
|
||||
diff/2
|
||||
]).
|
||||
-endif.
|
||||
|
||||
-define(TRIMMED, <<1>>).
|
||||
-define(UNWRITTEN, <<0>>).
|
||||
-define(NAIVE_ENCODE(Offset, Size, Data), <<Offset:64/unsigned-big, Size:32/unsigned-big, Data/binary>>).
|
||||
|
||||
-define(MINIMUM_CHUNK, 1048576). %% 1024 * 1024
|
||||
-define(LEVEL_SIZE, 10).
|
||||
-define(H, sha).
|
||||
|
||||
%% public API
|
||||
|
||||
open(Filename, DataDir) ->
|
||||
open(Filename, DataDir, naive).
|
||||
|
||||
open(Filename, DataDir, Type) ->
|
||||
Tree = load_filename(Filename, DataDir, Type),
|
||||
{ok, #mt{ filename = Filename, tree = Tree, backend = Type}}.
|
||||
|
||||
tree(#mt{ tree = T, backend = naive }) ->
|
||||
case T#naive.recalc of
|
||||
true -> build_tree(T);
|
||||
false -> T
|
||||
end.
|
||||
|
||||
filename(#mt{ filename = F }) -> F.
|
||||
|
||||
diff(#mt{backend = naive, tree = T1}, #mt{backend = naive, tree = T2}) ->
|
||||
case T1#naive.root == T2#naive.root of
|
||||
true -> same;
|
||||
false -> naive_diff(T1, T2)
|
||||
end;
|
||||
diff(_, _) -> error(badarg).
|
||||
|
||||
%% private
|
||||
|
||||
% @private
|
||||
load_filename(Filename, DataDir, naive) ->
|
||||
{Last, M} = do_load(Filename, DataDir, fun insert_csum_naive/2, []),
|
||||
ChunkSize = max(?MINIMUM_CHUNK, Last div 100),
|
||||
T = #naive{ leaves = lists:reverse(M), chunk_size = ChunkSize, recalc = true },
|
||||
build_tree(T).
|
||||
|
||||
do_load(Filename, DataDir, FoldFun, AccInit) ->
|
||||
CsumFile = machi_util:make_checksum_filename(DataDir, Filename),
|
||||
{ok, T} = machi_csum_table:open(CsumFile, []),
|
||||
Acc = machi_csum_table:foldl_chunks(FoldFun, {0, AccInit}, T),
|
||||
ok = machi_csum_table:close(T),
|
||||
Acc.
|
||||
|
||||
% @private
|
||||
insert_csum_naive({Last, Size, _Csum}=In, {Last, MT}) ->
|
||||
%% no gap
|
||||
{Last+Size, update_acc(In, MT)};
|
||||
insert_csum_naive({Offset, Size, _Csum}=In, {Last, MT}) ->
|
||||
Hole = Offset - Last,
|
||||
MT0 = update_acc({Last, Hole, unwritten}, MT),
|
||||
{Offset+Size, update_acc(In, MT0)}.
|
||||
|
||||
% @private
|
||||
update_acc({Offset, Size, unwritten}, MT) ->
|
||||
[ {Offset, Size, ?NAIVE_ENCODE(Offset, Size, ?UNWRITTEN)} | MT ];
|
||||
update_acc({Offset, Size, trimmed}, MT) ->
|
||||
[ {Offset, Size, ?NAIVE_ENCODE(Offset, Size, ?TRIMMED)} | MT ];
|
||||
update_acc({Offset, Size, <<_Tag:8, Csum/binary>>}, MT) ->
|
||||
[ {Offset, Size, ?NAIVE_ENCODE(Offset, Size, Csum)} | MT ].
|
||||
|
||||
build_tree(MT = #naive{ leaves = L, chunk_size = ChunkSize }) ->
|
||||
Lvl1s = build_level_1(ChunkSize, L, 1, [ crypto:hash_init(?H) ]),
|
||||
Mod2 = length(Lvl1s) div ?LEVEL_SIZE,
|
||||
Lvl2s = build_int_level(Mod2, Lvl1s, 1, [ crypto:hash_init(?H) ]),
|
||||
Mod3 = length(Lvl2s) div 2,
|
||||
Lvl3s = build_int_level(Mod3, Lvl2s, 1, [ crypto:hash_init(?H) ]),
|
||||
Root = build_root(Lvl3s, crypto:hash_init(?H)),
|
||||
MT#naive{ root = Root, lvl1 = Lvl1s, lvl2 = Lvl2s, lvl3 = Lvl3s, recalc = false }.
|
||||
|
||||
build_root([], Ctx) ->
|
||||
crypto:hash_final(Ctx);
|
||||
build_root([H|T], Ctx) ->
|
||||
build_root(T, crypto:hash_update(Ctx, H)).
|
||||
|
||||
build_int_level(_Mod, [], _Cnt, [ Ctx | Rest ]) ->
|
||||
lists:reverse( [ crypto:hash_final(Ctx) | Rest ] );
|
||||
build_int_level(Mod, [H|T], Cnt, [ Ctx | Rest ]) when Cnt rem Mod == 0 ->
|
||||
NewCtx = crypto:hash_init(?H),
|
||||
build_int_level(Mod, T, Cnt + 1, [ crypto:hash_update(NewCtx, H), crypto:hash_final(Ctx) | Rest ]);
|
||||
build_int_level(Mod, [H|T], Cnt, [ Ctx | Rest ]) ->
|
||||
build_int_level(Mod, T, Cnt+1, [ crypto:hash_update(Ctx, H) | Rest ]).
|
||||
|
||||
build_level_1(_Size, [], _Multiple, [ Ctx | Rest ]) ->
|
||||
lists:reverse([ crypto:hash_final(Ctx) | Rest ]);
|
||||
build_level_1(Size, [{Pos, Len, Hash}|T], Multiple, [ Ctx | Rest ])
|
||||
when ( Pos + Len ) > ( Size * Multiple ) ->
|
||||
NewCtx = crypto:hash_init(?H),
|
||||
build_level_1(Size, T, Multiple+1,
|
||||
[ crypto:hash_update(NewCtx, Hash), crypto:hash_final(Ctx) | Rest ]);
|
||||
build_level_1(Size, [{Pos, Len, Hash}|T], Multiple, [ Ctx | Rest ])
|
||||
when ( Pos + Len ) =< ( Size * Multiple ) ->
|
||||
build_level_1(Size, T, Multiple, [ crypto:hash_update(Ctx, Hash) | Rest ]).
|
||||
|
||||
naive_diff(#naive{lvl1 = L1}, #naive{lvl1=L2, chunk_size=CS2}) ->
|
||||
Set1 = gb_sets:from_list(lists:zip(lists:seq(1, length(L1)), L1)),
|
||||
Set2 = gb_sets:from_list(lists:zip(lists:seq(1, length(L2)), L2)),
|
||||
|
||||
%% The byte ranges in list 2 that do not match in list 1
|
||||
%% Or should we do something else?
|
||||
[ {(X-1)*CS2, CS2, SHA} || {X, SHA} <- gb_sets:to_list(gb_sets:subtract(Set1, Set2)) ].
|
|
@ -25,6 +25,10 @@
|
|||
%% to a single socket connection, and there is no code to deal with
|
||||
%% multiple connections/load balancing/error handling to several/all
|
||||
%% Machi cluster servers.
|
||||
%%
|
||||
%% Please see {@link machi_flu1_client} the "Client API implemntation notes"
|
||||
%% section for how this module relates to the rest of the client API
|
||||
%% implementation.
|
||||
|
||||
-module(machi_pb_high_client).
|
||||
|
||||
|
@ -40,7 +44,8 @@
|
|||
auth/3, auth/4,
|
||||
append_chunk/6, append_chunk/7,
|
||||
write_chunk/5, write_chunk/6,
|
||||
read_chunk/4, read_chunk/5,
|
||||
read_chunk/5, read_chunk/6,
|
||||
trim_chunk/4, trim_chunk/5,
|
||||
checksum_list/2, checksum_list/3,
|
||||
list_files/1, list_files/2
|
||||
]).
|
||||
|
@ -57,48 +62,114 @@
|
|||
count=0 :: non_neg_integer()
|
||||
}).
|
||||
|
||||
%% Official error types that is specific in Machi
|
||||
-type machi_client_error_reason() :: bad_arg | wedged | bad_checksum |
|
||||
partition | not_written | written |
|
||||
trimmed | no_such_file | partial_read |
|
||||
bad_epoch | inet:posix().
|
||||
|
||||
%% @doc Creates a client process
|
||||
-spec start_link(p_srvr_dict()) -> {ok, pid()} | {error, machi_client_error_reason()}.
|
||||
start_link(P_srvr_list) ->
|
||||
gen_server:start_link(?MODULE, [P_srvr_list], []).
|
||||
|
||||
%% @doc Stops a client process.
|
||||
-spec quit(pid()) -> ok.
|
||||
quit(PidSpec) ->
|
||||
gen_server:call(PidSpec, quit, infinity).
|
||||
|
||||
connected_p(PidSpec) ->
|
||||
gen_server:call(PidSpec, connected_p, infinity).
|
||||
|
||||
-spec echo(pid(), string()) -> {ok, string()} | {error, machi_client_error_reason()}.
|
||||
echo(PidSpec, String) ->
|
||||
echo(PidSpec, String, ?DEFAULT_TIMEOUT).
|
||||
|
||||
-spec echo(pid(), string(), non_neg_integer()) -> {ok, string()} | {error, machi_client_error_reason()}.
|
||||
echo(PidSpec, String, Timeout) ->
|
||||
send_sync(PidSpec, {echo, String}, Timeout).
|
||||
|
||||
%% TODO: auth() is not implemented. Auth requires SSL, and this client
|
||||
%% doesn't support SSL yet. This is just a placeholder and reminder.
|
||||
|
||||
-spec auth(pid(), string(), string()) -> ok | {error, machi_client_error_reason()}.
|
||||
auth(PidSpec, User, Pass) ->
|
||||
auth(PidSpec, User, Pass, ?DEFAULT_TIMEOUT).
|
||||
|
||||
-spec auth(pid(), string(), string(), non_neg_integer()) -> ok | {error, machi_client_error_reason()}.
|
||||
auth(PidSpec, User, Pass, Timeout) ->
|
||||
send_sync(PidSpec, {auth, User, Pass}, Timeout).
|
||||
|
||||
append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra) ->
|
||||
append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra, ?DEFAULT_TIMEOUT).
|
||||
-spec append_chunk(pid(),
|
||||
NS::machi_dt:namespace(), Prefix::machi_dt:file_prefix(),
|
||||
Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum(),
|
||||
Opts::machi_dt:append_opts()) ->
|
||||
{ok, Filename::string(), Offset::machi_dt:file_offset()} |
|
||||
{error, machi_client_error_reason()}.
|
||||
append_chunk(PidSpec, NS, Prefix, Chunk, CSum, Opts) ->
|
||||
append_chunk(PidSpec, NS, Prefix, Chunk, CSum, Opts, ?DEFAULT_TIMEOUT).
|
||||
|
||||
append_chunk(PidSpec, PlacementKey, Prefix, Chunk, CSum, ChunkExtra, Timeout) ->
|
||||
send_sync(PidSpec, {append_chunk, PlacementKey, Prefix, Chunk, CSum, ChunkExtra}, Timeout).
|
||||
-spec append_chunk(pid(),
|
||||
NS::machi_dt:namespace(), Prefix::machi_dt:file_prefix(),
|
||||
Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum(),
|
||||
Opts::machi_dt:append_opts(),
|
||||
Timeout::non_neg_integer()) ->
|
||||
{ok, Filename::string(), Offset::machi_dt:file_offset()} |
|
||||
{error, machi_client_error_reason()}.
|
||||
append_chunk(PidSpec, NS, Prefix, Chunk, CSum, Opts, Timeout) ->
|
||||
send_sync(PidSpec, {append_chunk, NS, Prefix, Chunk, CSum, Opts}, Timeout).
|
||||
|
||||
-spec write_chunk(pid(), File::string(), machi_dt:file_offset(),
|
||||
Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum()) ->
|
||||
ok | {error, machi_client_error_reason()}.
|
||||
write_chunk(PidSpec, File, Offset, Chunk, CSum) ->
|
||||
write_chunk(PidSpec, File, Offset, Chunk, CSum, ?DEFAULT_TIMEOUT).
|
||||
|
||||
-spec write_chunk(pid(), File::string(), machi_dt:file_offset(),
|
||||
Chunk::machi_dt:chunk(), CSum::machi_dt:chunk_csum(), Timeout::non_neg_integer()) ->
|
||||
ok | {error, machi_client_error_reason()}.
|
||||
write_chunk(PidSpec, File, Offset, Chunk, CSum, Timeout) ->
|
||||
send_sync(PidSpec, {write_chunk, File, Offset, Chunk, CSum}, Timeout).
|
||||
|
||||
read_chunk(PidSpec, File, Offset, Size) ->
|
||||
read_chunk(PidSpec, File, Offset, Size, ?DEFAULT_TIMEOUT).
|
||||
%% @doc Tries to read a chunk of a specified file. It returns `{ok,
|
||||
%% {Chunks, TrimmedChunks}}' for live file while it returns `{error,
|
||||
%% trimmed}' if all bytes of the file was trimmed.
|
||||
-spec read_chunk(pid(), File::string(), machi_dt:file_offset(), machi_dt:chunk_size(),
|
||||
machi_dt:read_opts_x()) ->
|
||||
{ok, {Chunks::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size(), binary()}],
|
||||
Trimmed::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size()}]}} |
|
||||
{error, machi_client_error_reason()}.
|
||||
read_chunk(PidSpec, File, Offset, Size, Opts) ->
|
||||
read_chunk(PidSpec, File, Offset, Size, Opts, ?DEFAULT_TIMEOUT).
|
||||
|
||||
read_chunk(PidSpec, File, Offset, Size, Timeout) ->
|
||||
send_sync(PidSpec, {read_chunk, File, Offset, Size}, Timeout).
|
||||
-spec read_chunk(pid(), File::string(), machi_dt:file_offset(), machi_dt:chunk_size(),
|
||||
machi_dt:read_opts_x(),
|
||||
Timeout::non_neg_integer()) ->
|
||||
{ok, {Chunks::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size(), binary()}],
|
||||
Trimmed::[{File::string(), machi_dt:file_offset(), machi_dt:chunk_size()}]}} |
|
||||
{error, machi_client_error_reason()}.
|
||||
read_chunk(PidSpec, File, Offset, Size, Opts0, Timeout) ->
|
||||
Opts = machi_util:read_opts_default(Opts0),
|
||||
send_sync(PidSpec, {read_chunk, File, Offset, Size, Opts}, Timeout).
|
||||
|
||||
%% @doc Trims arbitrary binary range of any file. If a specified range
|
||||
%% has any byte trimmed, it fails and returns `{error, trimmed}'.
|
||||
%% Otherwise it trims all bytes in that range. If there are
|
||||
%% overlapping chunks with client-specified checksum, they will cut
|
||||
%% off and checksum are re-calculated in server side. TODO: Add
|
||||
%% option specifying whether to trigger GC.
|
||||
-spec trim_chunk(pid(), string(), non_neg_integer(), machi_dt:chunk_size()) ->
|
||||
ok | {error, machi_client_error_reason()}.
|
||||
trim_chunk(PidSpec, File, Offset, Size) ->
|
||||
trim_chunk(PidSpec, File, Offset, Size, ?DEFAULT_TIMEOUT).
|
||||
|
||||
trim_chunk(PidSpec, File, Offset, Size, Timeout) ->
|
||||
send_sync(PidSpec, {trim_chunk, File, Offset, Size}, Timeout).
|
||||
|
||||
%% @doc Returns a binary that has checksums and chunks encoded inside
|
||||
%% (This is because encoding-decoding them are inefficient). TODO:
|
||||
%% return a structured list of them.
|
||||
-spec checksum_list(pid(), string()) -> {ok, binary()} | {error, machi_client_error_reason()}.
|
||||
checksum_list(PidSpec, File) ->
|
||||
checksum_list(PidSpec, File, ?DEFAULT_TIMEOUT).
|
||||
|
||||
|
@ -218,19 +289,19 @@ do_send_sync2({auth, User, Pass}, #state{sock=Sock}=S) ->
|
|||
Res = {bummer, {X, Y, erlang:get_stacktrace()}},
|
||||
{Res, S}
|
||||
end;
|
||||
do_send_sync2({append_chunk, PlacementKey, Prefix, Chunk, CSum, ChunkExtra},
|
||||
do_send_sync2({append_chunk, NS, Prefix, Chunk, CSum, Opts},
|
||||
#state{sock=Sock, sock_id=Index, count=Count}=S) ->
|
||||
try
|
||||
ReqID = <<Index:64/big, Count:64/big>>,
|
||||
PK = if PlacementKey == <<>> -> undefined;
|
||||
true -> PlacementKey
|
||||
end,
|
||||
CSumT = convert_csum_req(CSum, Chunk),
|
||||
Req = #mpb_appendchunkreq{placement_key=PK,
|
||||
{ChunkExtra, Pref, FailPref} = machi_pb_translate:conv_from_append_opts(Opts),
|
||||
Req = #mpb_appendchunkreq{namespace=NS,
|
||||
prefix=Prefix,
|
||||
chunk=Chunk,
|
||||
csum=CSumT,
|
||||
chunk_extra=ChunkExtra},
|
||||
chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref},
|
||||
R1a = #mpb_request{req_id=ReqID, do_not_alter=1,
|
||||
append_chunk=Req},
|
||||
Bin1a = machi_pb:encode_mpb_request(R1a),
|
||||
|
@ -253,10 +324,11 @@ do_send_sync2({write_chunk, File, Offset, Chunk, CSum},
|
|||
try
|
||||
ReqID = <<Index:64/big, Count:64/big>>,
|
||||
CSumT = convert_csum_req(CSum, Chunk),
|
||||
Req = #mpb_writechunkreq{file=File,
|
||||
offset=Offset,
|
||||
chunk=Chunk,
|
||||
csum=CSumT},
|
||||
Req = #mpb_writechunkreq{chunk=
|
||||
#mpb_chunk{chunk=Chunk,
|
||||
file_name=File,
|
||||
offset=Offset,
|
||||
csum=CSumT}},
|
||||
R1a = #mpb_request{req_id=ReqID, do_not_alter=1,
|
||||
write_chunk=Req},
|
||||
Bin1a = machi_pb:encode_mpb_request(R1a),
|
||||
|
@ -274,13 +346,19 @@ do_send_sync2({write_chunk, File, Offset, Chunk, CSum},
|
|||
Res = {bummer, {X, Y, erlang:get_stacktrace()}},
|
||||
{Res, S#state{count=Count+1}}
|
||||
end;
|
||||
do_send_sync2({read_chunk, File, Offset, Size},
|
||||
do_send_sync2({read_chunk, File, Offset, Size, Opts},
|
||||
#state{sock=Sock, sock_id=Index, count=Count}=S) ->
|
||||
try
|
||||
ReqID = <<Index:64/big, Count:64/big>>,
|
||||
Req = #mpb_readchunkreq{file=File,
|
||||
offset=Offset,
|
||||
size=Size},
|
||||
#read_opts{no_checksum=FlagNoChecksum,
|
||||
no_chunk=FlagNoChunk,
|
||||
needs_trimmed=NeedsTrimmed} = Opts,
|
||||
Req = #mpb_readchunkreq{chunk_pos=#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size},
|
||||
flag_no_checksum=machi_util:bool2int(FlagNoChecksum),
|
||||
flag_no_chunk=machi_util:bool2int(FlagNoChunk),
|
||||
flag_needs_trimmed=machi_util:bool2int(NeedsTrimmed)},
|
||||
R1a = #mpb_request{req_id=ReqID, do_not_alter=1,
|
||||
read_chunk=Req},
|
||||
Bin1a = machi_pb:encode_mpb_request(R1a),
|
||||
|
@ -298,6 +376,30 @@ do_send_sync2({read_chunk, File, Offset, Size},
|
|||
Res = {bummer, {X, Y, erlang:get_stacktrace()}},
|
||||
{Res, S#state{count=Count+1}}
|
||||
end;
|
||||
do_send_sync2({trim_chunk, File, Offset, Size},
|
||||
#state{sock=Sock, sock_id=Index, count=Count}=S) ->
|
||||
try
|
||||
ReqID = <<Index:64/big, Count:64/big>>,
|
||||
Req = #mpb_trimchunkreq{chunk_pos=#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size}},
|
||||
R1a = #mpb_request{req_id=ReqID, do_not_alter=1,
|
||||
trim_chunk=Req},
|
||||
Bin1a = machi_pb:encode_mpb_request(R1a),
|
||||
ok = gen_tcp:send(Sock, Bin1a),
|
||||
{ok, Bin1B} = gen_tcp:recv(Sock, 0),
|
||||
case (catch machi_pb:decode_mpb_response(Bin1B)) of
|
||||
#mpb_response{req_id=ReqID, trim_chunk=R} when R /= undefined ->
|
||||
Result = convert_trim_chunk_resp(R),
|
||||
{Result, S#state{count=Count+1}};
|
||||
#mpb_response{req_id=ReqID, generic=G} when G /= undefined ->
|
||||
#mpb_errorresp{code=Code, msg=Msg, extra=Extra} = G,
|
||||
{{error, {Code, Msg, Extra}}, S#state{count=Count+1}}
|
||||
end
|
||||
catch X:Y ->
|
||||
Res = {bummer, {X, Y, erlang:get_stacktrace()}},
|
||||
{Res, S#state{count=Count+1}}
|
||||
end;
|
||||
do_send_sync2({checksum_list, File},
|
||||
#state{sock=Sock, sock_id=Index, count=Count}=S) ->
|
||||
try
|
||||
|
@ -343,9 +445,15 @@ do_send_sync2({list_files},
|
|||
{Res, S#state{count=Count+1}}
|
||||
end.
|
||||
|
||||
%% We only convert the checksum types that make sense here:
|
||||
%% none or client_sha. None of the other types should be sent
|
||||
%% to us via the PB high protocol.
|
||||
|
||||
convert_csum_req(none, Chunk) ->
|
||||
#mpb_chunkcsum{type='CSUM_TAG_CLIENT_SHA',
|
||||
csum=machi_util:checksum_chunk(Chunk)};
|
||||
convert_csum_req(<<>>, Chunk) ->
|
||||
convert_csum_req(none, Chunk);
|
||||
convert_csum_req({client_sha, CSumBin}, _Chunk) ->
|
||||
#mpb_chunkcsum{type='CSUM_TAG_CLIENT_SHA',
|
||||
csum=CSumBin}.
|
||||
|
@ -370,6 +478,8 @@ convert_general_status_code('NOT_WRITTEN') ->
|
|||
{error, not_written};
|
||||
convert_general_status_code('WRITTEN') ->
|
||||
{error, written};
|
||||
convert_general_status_code('TRIMMED') ->
|
||||
{error, trimmed};
|
||||
convert_general_status_code('NO_SUCH_FILE') ->
|
||||
{error, no_such_file};
|
||||
convert_general_status_code('PARTIAL_READ') ->
|
||||
|
@ -384,11 +494,29 @@ convert_write_chunk_resp(#mpb_writechunkresp{status='OK'}) ->
|
|||
convert_write_chunk_resp(#mpb_writechunkresp{status=Status}) ->
|
||||
convert_general_status_code(Status).
|
||||
|
||||
convert_read_chunk_resp(#mpb_readchunkresp{status='OK', chunk=Chunk}) ->
|
||||
{ok, Chunk};
|
||||
convert_read_chunk_resp(#mpb_readchunkresp{status='OK', chunks=PB_Chunks, trimmed=PB_Trimmed}) ->
|
||||
Chunks = lists:map(fun(#mpb_chunk{offset=Offset,
|
||||
file_name=File,
|
||||
chunk=Chunk,
|
||||
csum=#mpb_chunkcsum{type=T, csum=Ck}}) ->
|
||||
%% TODO: cleanup export
|
||||
Csum = <<(machi_pb_translate:conv_to_csum_tag(T)):8, Ck/binary>>,
|
||||
{list_to_binary(File), Offset, Chunk, Csum}
|
||||
end, PB_Chunks),
|
||||
Trimmed = lists:map(fun(#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size}) ->
|
||||
{list_to_binary(File), Offset, Size}
|
||||
end, PB_Trimmed),
|
||||
{ok, {Chunks, Trimmed}};
|
||||
convert_read_chunk_resp(#mpb_readchunkresp{status=Status}) ->
|
||||
convert_general_status_code(Status).
|
||||
|
||||
convert_trim_chunk_resp(#mpb_trimchunkresp{status='OK'}) ->
|
||||
ok;
|
||||
convert_trim_chunk_resp(#mpb_trimchunkresp{status=Status}) ->
|
||||
convert_general_status_code(Status).
|
||||
|
||||
convert_checksum_list_resp(#mpb_checksumlistresp{status='OK', chunk=Chunk}) ->
|
||||
{ok, Chunk};
|
||||
convert_checksum_list_resp(#mpb_checksumlistresp{status=Status}) ->
|
||||
|
|
|
@ -15,8 +15,8 @@
|
|||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
|
||||
-module(machi_pb_translate).
|
||||
|
||||
|
@ -34,85 +34,115 @@
|
|||
-export([from_pb_request/1,
|
||||
from_pb_response/1,
|
||||
to_pb_request/2,
|
||||
to_pb_response/3
|
||||
to_pb_response/3,
|
||||
conv_from_append_opts/1,
|
||||
conv_to_append_opts/1
|
||||
]).
|
||||
|
||||
%% TODO: fixme cleanup
|
||||
-export([conv_to_csum_tag/1]).
|
||||
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
echo=#mpb_echoreq{message=Msg}}) ->
|
||||
{ReqID, {low_echo, undefined, Msg}};
|
||||
{ReqID, {low_skip_wedge, {low_echo, Msg}}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
auth=#mpb_authreq{user=User, password=Pass}}) ->
|
||||
{ReqID, {low_auth, undefined, User, Pass}};
|
||||
{ReqID, {low_skip_wedge, {low_auth, User, Pass}}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
append_chunk=#mpb_ll_appendchunkreq{
|
||||
append_chunk=IR=#mpb_ll_appendchunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS_str,
|
||||
locator=NSLocator,
|
||||
epoch_id=PB_EpochID,
|
||||
placement_key=PKey,
|
||||
prefix=Prefix,
|
||||
chunk=Chunk,
|
||||
csum=#mpb_chunkcsum{type=CSum_type, csum=CSum},
|
||||
chunk_extra=ChunkExtra}}) ->
|
||||
csum=#mpb_chunkcsum{type=CSum_type, csum=CSum}}}) ->
|
||||
NS = list_to_binary(NS_str),
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
CSum_tag = conv_to_csum_tag(CSum_type),
|
||||
{ReqID, {low_append_chunk, EpochID, PKey, Prefix, Chunk, CSum_tag, CSum,
|
||||
ChunkExtra}};
|
||||
Opts = conv_to_append_opts(IR),
|
||||
%% NOTE: The tuple position of NSLocator is a bit odd, because EpochID
|
||||
%% _must_ be in the 4th position (as NSV & NS must be in 2nd & 3rd).
|
||||
{ReqID, {low_append_chunk, NSVersion, NS, EpochID, NSLocator,
|
||||
Prefix, Chunk, CSum_tag, CSum, Opts}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
write_chunk=#mpb_ll_writechunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS_str,
|
||||
epoch_id=PB_EpochID,
|
||||
file=File,
|
||||
offset=Offset,
|
||||
chunk=Chunk,
|
||||
csum=#mpb_chunkcsum{type=CSum_type, csum=CSum}}}) ->
|
||||
chunk=#mpb_chunk{file_name=File,
|
||||
offset=Offset,
|
||||
chunk=Chunk,
|
||||
csum=#mpb_chunkcsum{type=CSum_type, csum=CSum}}}}) ->
|
||||
NS = list_to_binary(NS_str),
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
CSum_tag = conv_to_csum_tag(CSum_type),
|
||||
{ReqID, {low_write_chunk, EpochID, File, Offset, Chunk, CSum_tag, CSum}};
|
||||
{ReqID, {low_write_chunk, NSVersion, NS, EpochID, File, Offset, Chunk, CSum_tag, CSum}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
read_chunk=#mpb_ll_readchunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS_str,
|
||||
epoch_id=PB_EpochID,
|
||||
chunk_pos=ChunkPos,
|
||||
flag_no_checksum=PB_GetNoChecksum,
|
||||
flag_no_chunk=PB_GetNoChunk,
|
||||
flag_needs_trimmed=PB_NeedsTrimmed}}) ->
|
||||
NS = list_to_binary(NS_str),
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
Opts = #read_opts{no_checksum=PB_GetNoChecksum,
|
||||
no_chunk=PB_GetNoChunk,
|
||||
needs_trimmed=PB_NeedsTrimmed},
|
||||
#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size} = ChunkPos,
|
||||
{ReqID, {low_read_chunk, NSVersion, NS, EpochID, File, Offset, Size, Opts}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
trim_chunk=#mpb_ll_trimchunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS_str,
|
||||
epoch_id=PB_EpochID,
|
||||
file=File,
|
||||
offset=Offset,
|
||||
size=Size,
|
||||
flag_get_checksum=PB_GetChecksum,
|
||||
flag_no_chunk=PB_GetNoChunk}}) ->
|
||||
trigger_gc=TriggerGC}}) ->
|
||||
NS = list_to_binary(NS_str),
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
Opts = [{get_checksum, conv_to_boolean(PB_GetChecksum)},
|
||||
{no_chunk, conv_to_boolean(PB_GetNoChunk)}],
|
||||
{ReqID, {low_read_chunk, EpochID, File, Offset, Size, Opts}};
|
||||
{ReqID, {low_trim_chunk, NSVersion, NS, EpochID, File, Offset, Size, TriggerGC}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
checksum_list=#mpb_ll_checksumlistreq{
|
||||
epoch_id=PB_EpochID,
|
||||
file=File}}) ->
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
{ReqID, {low_checksum_list, EpochID, File}};
|
||||
{ReqID, {low_skip_wedge, {low_checksum_list, File}}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
list_files=#mpb_ll_listfilesreq{
|
||||
epoch_id=PB_EpochID}}) ->
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
{ReqID, {low_list_files, EpochID}};
|
||||
{ReqID, {low_skip_wedge, {low_list_files, EpochID}}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
wedge_status=#mpb_ll_wedgestatusreq{}}) ->
|
||||
{ReqID, {low_wedge_status, undefined}};
|
||||
{ReqID, {low_skip_wedge, {low_wedge_status}}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
delete_migration=#mpb_ll_deletemigrationreq{
|
||||
epoch_id=PB_EpochID,
|
||||
file=File}}) ->
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
{ReqID, {low_delete_migration, EpochID, File}};
|
||||
{ReqID, {low_skip_wedge, {low_delete_migration, EpochID, File}}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
trunc_hack=#mpb_ll_trunchackreq{
|
||||
epoch_id=PB_EpochID,
|
||||
file=File}}) ->
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
{ReqID, {low_trunc_hack, EpochID, File}};
|
||||
{ReqID, {low_skip_wedge, {low_trunc_hack, EpochID, File}}};
|
||||
from_pb_request(#mpb_ll_request{
|
||||
req_id=ReqID,
|
||||
proj_gl=#mpb_ll_getlatestepochidreq{type=ProjType}}) ->
|
||||
|
@ -153,28 +183,40 @@ from_pb_request(#mpb_request{req_id=ReqID,
|
|||
{ReqID, {high_auth, User, Pass}};
|
||||
from_pb_request(#mpb_request{req_id=ReqID,
|
||||
append_chunk=IR=#mpb_appendchunkreq{}}) ->
|
||||
#mpb_appendchunkreq{placement_key=__todoPK,
|
||||
#mpb_appendchunkreq{namespace=NS_str,
|
||||
prefix=Prefix,
|
||||
chunk=Chunk,
|
||||
csum=CSum,
|
||||
chunk_extra=ChunkExtra} = IR,
|
||||
csum=CSum} = IR,
|
||||
NS = list_to_binary(NS_str),
|
||||
TaggedCSum = make_tagged_csum(CSum, Chunk),
|
||||
{ReqID, {high_append_chunk, __todoPK, Prefix, Chunk, TaggedCSum,
|
||||
ChunkExtra}};
|
||||
Opts = conv_to_append_opts(IR),
|
||||
{ReqID, {high_append_chunk, NS, Prefix, Chunk, TaggedCSum, Opts}};
|
||||
from_pb_request(#mpb_request{req_id=ReqID,
|
||||
write_chunk=IR=#mpb_writechunkreq{}}) ->
|
||||
#mpb_writechunkreq{file=File,
|
||||
offset=Offset,
|
||||
chunk=Chunk,
|
||||
csum=CSum} = IR,
|
||||
TaggedCSum = make_tagged_csum(CSum, Chunk),
|
||||
{ReqID, {high_write_chunk, File, Offset, Chunk, TaggedCSum}};
|
||||
#mpb_writechunkreq{chunk=#mpb_chunk{file_name=File,
|
||||
offset=Offset,
|
||||
chunk=Chunk,
|
||||
csum=CSumRec}} = IR,
|
||||
CSum = make_tagged_csum(CSumRec, Chunk),
|
||||
{ReqID, {high_write_chunk, File, Offset, Chunk, CSum}};
|
||||
from_pb_request(#mpb_request{req_id=ReqID,
|
||||
read_chunk=IR=#mpb_readchunkreq{}}) ->
|
||||
#mpb_readchunkreq{file=File,
|
||||
offset=Offset,
|
||||
size=Size} = IR,
|
||||
{ReqID, {high_read_chunk, File, Offset, Size}};
|
||||
#mpb_readchunkreq{chunk_pos=#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size},
|
||||
flag_no_checksum=FlagNoChecksum,
|
||||
flag_no_chunk=FlagNoChunk,
|
||||
flag_needs_trimmed=NeedsTrimmed} = IR,
|
||||
Opts = #read_opts{no_checksum=FlagNoChecksum,
|
||||
no_chunk=FlagNoChunk,
|
||||
needs_trimmed=NeedsTrimmed},
|
||||
{ReqID, {high_read_chunk, File, Offset, Size, Opts}};
|
||||
from_pb_request(#mpb_request{req_id=ReqID,
|
||||
trim_chunk=IR=#mpb_trimchunkreq{}}) ->
|
||||
#mpb_trimchunkreq{chunk_pos=#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size}} = IR,
|
||||
{ReqID, {high_trim_chunk, File, Offset, Size}};
|
||||
from_pb_request(#mpb_request{req_id=ReqID,
|
||||
checksum_list=IR=#mpb_checksumlistreq{}}) ->
|
||||
#mpb_checksumlistreq{file=File} = IR,
|
||||
|
@ -190,10 +232,17 @@ from_pb_request(_Else) ->
|
|||
|
||||
from_pb_response(#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
% There is no separate LL error response record
|
||||
generic=#mpb_errorresp{code=Code, msg=Msg}}) ->
|
||||
{ReqID, {error, {Code, Msg}}};
|
||||
from_pb_response(#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
% There is no separate LL echo response record
|
||||
echo=#mpb_echoresp{message=Msg}}) ->
|
||||
{ReqID, Msg};
|
||||
from_pb_response(#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
% There is no separate LL auth response record
|
||||
auth=#mpb_authresp{code=Code}}) ->
|
||||
{ReqID, Code};
|
||||
from_pb_response(#mpb_ll_response{
|
||||
|
@ -216,13 +265,30 @@ from_pb_response(#mpb_ll_response{
|
|||
from_pb_response(#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
read_chunk=#mpb_ll_readchunkresp{status=Status,
|
||||
chunk=Chunk}}) ->
|
||||
chunks=PB_Chunks,
|
||||
trimmed=PB_Trimmed}}) ->
|
||||
case Status of
|
||||
'OK' ->
|
||||
{ReqID, {ok, Chunk}};
|
||||
Chunks = lists:map(fun(#mpb_chunk{file_name=File,
|
||||
offset=Offset,
|
||||
chunk=Bytes,
|
||||
csum=#mpb_chunkcsum{type=T,csum=Ck}}) ->
|
||||
Csum = <<(conv_to_csum_tag(T)):8, Ck/binary>>,
|
||||
{list_to_binary(File), Offset, Bytes, Csum}
|
||||
end, PB_Chunks),
|
||||
Trimmed = lists:map(fun(#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size}) ->
|
||||
{list_to_binary(File), Offset, Size}
|
||||
end, PB_Trimmed),
|
||||
{ReqID, {ok, {Chunks, Trimmed}}};
|
||||
_ ->
|
||||
{ReqID, machi_pb_high_client:convert_general_status_code(Status)}
|
||||
end;
|
||||
from_pb_response(#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
trim_chunk=#mpb_ll_trimchunkresp{status=Status}}) ->
|
||||
{ReqID, machi_pb_high_client:convert_general_status_code(Status)};
|
||||
from_pb_response(#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
checksum_list=#mpb_ll_checksumlistresp{
|
||||
|
@ -249,12 +315,16 @@ from_pb_response(#mpb_ll_response{
|
|||
from_pb_response(#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
wedge_status=#mpb_ll_wedgestatusresp{
|
||||
epoch_id=PB_EpochID, wedged_flag=PB_Wedged}}) ->
|
||||
status=Status,
|
||||
epoch_id=PB_EpochID, wedged_flag=Wedged_p,
|
||||
namespace_version=NSVersion, namespace=NS_str}}) ->
|
||||
GeneralStatus = case machi_pb_high_client:convert_general_status_code(Status) of
|
||||
ok -> ok;
|
||||
_Else -> {yukky, _Else}
|
||||
end,
|
||||
EpochID = conv_to_epoch_id(PB_EpochID),
|
||||
Wedged_p = if PB_Wedged == 1 -> true;
|
||||
PB_Wedged == 0 -> false
|
||||
end,
|
||||
{ReqID, {ok, {Wedged_p, EpochID}}};
|
||||
NS = list_to_binary(NS_str),
|
||||
{ReqID, {GeneralStatus, {Wedged_p, EpochID, NSVersion, NS}}};
|
||||
from_pb_response(#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
delete_migration=#mpb_ll_deletemigrationresp{
|
||||
|
@ -320,74 +390,100 @@ from_pb_response(#mpb_ll_response{
|
|||
'OK' ->
|
||||
{ReqID, {ok, Epochs}};
|
||||
_ ->
|
||||
{ReqID< machi_pb_high_client:convert_general_status_code(Status)}
|
||||
{ReqID, machi_pb_high_client:convert_general_status_code(Status)}
|
||||
end.
|
||||
%% No response for proj_kp/kick_projection_reaction
|
||||
|
||||
%% TODO: move the #mbp_* record making code from
|
||||
%% machi_pb_high_client:do_send_sync() clauses into to_pb_request().
|
||||
|
||||
to_pb_request(ReqID, {low_echo, _BogusEpochID, Msg}) ->
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_echo, Msg}}) ->
|
||||
#mpb_ll_request{
|
||||
req_id=ReqID, do_not_alter=2,
|
||||
echo=#mpb_echoreq{message=Msg}};
|
||||
to_pb_request(ReqID, {low_auth, _BogusEpochID, User, Pass}) ->
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_auth, User, Pass}}) ->
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
auth=#mpb_authreq{user=User, password=Pass}};
|
||||
to_pb_request(ReqID, {low_append_chunk, EpochID, PKey, Prefix, Chunk,
|
||||
CSum_tag, CSum, ChunkExtra}) ->
|
||||
%% NOTE: The tuple position of NSLocator is a bit odd, because EpochID
|
||||
%% _must_ be in the 4th position (as NSV & NS must be in 2nd & 3rd).
|
||||
to_pb_request(ReqID, {low_append_chunk, NSVersion, NS, EpochID, NSLocator,
|
||||
Prefix, Chunk, CSum_tag, CSum, Opts}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
CSum_type = conv_from_csum_tag(CSum_tag),
|
||||
PB_CSum = #mpb_chunkcsum{type=CSum_type, csum=CSum},
|
||||
{ChunkExtra, Pref, FailPref} = conv_from_append_opts(Opts),
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
append_chunk=#mpb_ll_appendchunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS,
|
||||
locator=NSLocator,
|
||||
epoch_id=PB_EpochID,
|
||||
placement_key=PKey,
|
||||
prefix=Prefix,
|
||||
chunk=Chunk,
|
||||
csum=PB_CSum,
|
||||
chunk_extra=ChunkExtra}};
|
||||
to_pb_request(ReqID, {low_write_chunk, EpochID, File, Offset, Chunk, CSum_tag, CSum}) ->
|
||||
chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref}};
|
||||
to_pb_request(ReqID, {low_write_chunk, NSVersion, NS, EpochID, File, Offset, Chunk, CSum_tag, CSum}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
CSum_type = conv_from_csum_tag(CSum_tag),
|
||||
PB_CSum = #mpb_chunkcsum{type=CSum_type, csum=CSum},
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
write_chunk=#mpb_ll_writechunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS,
|
||||
epoch_id=PB_EpochID,
|
||||
file=File,
|
||||
offset=Offset,
|
||||
chunk=Chunk,
|
||||
csum=PB_CSum}};
|
||||
to_pb_request(ReqID, {low_read_chunk, EpochID, File, Offset, Size, _Opts}) ->
|
||||
%% TODO: stop ignoring Opts ^_^
|
||||
chunk=#mpb_chunk{file_name=File,
|
||||
offset=Offset,
|
||||
chunk=Chunk,
|
||||
csum=PB_CSum}}};
|
||||
to_pb_request(ReqID, {low_read_chunk, NSVersion, NS, EpochID, File, Offset, Size, Opts}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#read_opts{no_checksum=FNChecksum,
|
||||
no_chunk=FNChunk,
|
||||
needs_trimmed=NeedsTrimmed} = Opts,
|
||||
#mpb_ll_request{
|
||||
req_id=ReqID, do_not_alter=2,
|
||||
read_chunk=#mpb_ll_readchunkreq{
|
||||
epoch_id=PB_EpochID,
|
||||
file=File,
|
||||
offset=Offset,
|
||||
size=Size}};
|
||||
to_pb_request(ReqID, {low_checksum_list, EpochID, File}) ->
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS,
|
||||
epoch_id=PB_EpochID,
|
||||
chunk_pos=#mpb_chunkpos{
|
||||
file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size},
|
||||
flag_no_checksum=FNChecksum,
|
||||
flag_no_chunk=FNChunk,
|
||||
flag_needs_trimmed=NeedsTrimmed}};
|
||||
to_pb_request(ReqID, {low_trim_chunk, NSVersion, NS, EpochID, File, Offset, Size, TriggerGC}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
trim_chunk=#mpb_ll_trimchunkreq{
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS,
|
||||
epoch_id=PB_EpochID,
|
||||
file=File,
|
||||
offset=Offset,
|
||||
size=Size,
|
||||
trigger_gc=TriggerGC}};
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_checksum_list, File}}) ->
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
checksum_list=#mpb_ll_checksumlistreq{
|
||||
epoch_id=PB_EpochID,
|
||||
file=File}};
|
||||
to_pb_request(ReqID, {low_list_files, EpochID}) ->
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_list_files, EpochID}}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
list_files=#mpb_ll_listfilesreq{epoch_id=PB_EpochID}};
|
||||
to_pb_request(ReqID, {low_wedge_status, _BogusEpochID}) ->
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_wedge_status}}) ->
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
wedge_status=#mpb_ll_wedgestatusreq{}};
|
||||
to_pb_request(ReqID, {low_delete_migration, EpochID, File}) ->
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_delete_migration, EpochID, File}}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
delete_migration=#mpb_ll_deletemigrationreq{
|
||||
epoch_id=PB_EpochID,
|
||||
file=File}};
|
||||
to_pb_request(ReqID, {low_trunc_hack, EpochID, File}) ->
|
||||
to_pb_request(ReqID, {low_skip_wedge, {low_trunc_hack, EpochID, File}}) ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#mpb_ll_request{req_id=ReqID, do_not_alter=2,
|
||||
trunc_hack=#mpb_ll_trunchackreq{
|
||||
|
@ -423,15 +519,15 @@ to_pb_response(_ReqID, _, async_no_response=X) ->
|
|||
X;
|
||||
to_pb_response(ReqID, _, {low_error, ErrCode, ErrMsg}) ->
|
||||
make_ll_error_resp(ReqID, ErrCode, ErrMsg);
|
||||
to_pb_response(ReqID, {low_echo, _BogusEpochID, _Msg}, Resp) ->
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_echo, _Msg}}, Resp) ->
|
||||
#mpb_ll_response{
|
||||
req_id=ReqID,
|
||||
echo=#mpb_echoresp{message=Resp}};
|
||||
to_pb_response(ReqID, {low_auth, _, _, _}, __TODO_Resp) ->
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_auth, _, _}}, __TODO_Resp) ->
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
generic=#mpb_errorresp{code=1,
|
||||
msg="AUTH not implemented"}};
|
||||
to_pb_response(ReqID, {low_append_chunk, _EID, _PKey, _Pfx, _Ch, _CST, _CS, _CE}, Resp)->
|
||||
to_pb_response(ReqID, {low_append_chunk, _NSV, _NS, _EID, _NSL, _Pfx, _Ch, _CST, _CS, _O}, Resp)->
|
||||
case Resp of
|
||||
{ok, {Offset, Size, File}} ->
|
||||
Where = #mpb_chunkpos{offset=Offset,
|
||||
|
@ -447,18 +543,30 @@ to_pb_response(ReqID, {low_append_chunk, _EID, _PKey, _Pfx, _Ch, _CST, _CS, _CE}
|
|||
_Else ->
|
||||
make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {low_write_chunk, _EID, _Fl, _Off, _Ch, _CST, _CS},Resp)->
|
||||
to_pb_response(ReqID, {low_write_chunk, _NSV, _NS, _EID, _Fl, _Off, _Ch, _CST, _CS},Resp)->
|
||||
Status = conv_from_status(Resp),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
write_chunk=#mpb_ll_writechunkresp{status=Status}};
|
||||
to_pb_response(ReqID, {low_read_chunk, _EID, _Fl, _Off, _Sz, _Opts}, Resp)->
|
||||
to_pb_response(ReqID, {low_read_chunk, _NSV, _NS, _EID, _Fl, _Off, _Sz, _Opts}, Resp)->
|
||||
case Resp of
|
||||
{ok, Chunk} ->
|
||||
CSum = undefined, % TODO not implemented
|
||||
{ok, {Chunks, Trimmed}} ->
|
||||
PB_Chunks = lists:map(fun({File, Offset, Bytes, Csum}) ->
|
||||
{Tag, Ck} = machi_util:unmake_tagged_csum(Csum),
|
||||
#mpb_chunk{file_name=File,
|
||||
offset=Offset,
|
||||
chunk=Bytes,
|
||||
csum=#mpb_chunkcsum{type=conv_from_csum_tag(Tag),
|
||||
csum=Ck}}
|
||||
end, Chunks),
|
||||
PB_Trimmed = lists:map(fun({File, Offset, Size}) ->
|
||||
#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size}
|
||||
end, Trimmed),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
read_chunk=#mpb_ll_readchunkresp{status='OK',
|
||||
chunk=Chunk,
|
||||
csum=CSum}};
|
||||
chunks=PB_Chunks,
|
||||
trimmed=PB_Trimmed}};
|
||||
{error, _}=Error ->
|
||||
Status = conv_from_status(Error),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
|
@ -466,7 +574,19 @@ to_pb_response(ReqID, {low_read_chunk, _EID, _Fl, _Off, _Sz, _Opts}, Resp)->
|
|||
_Else ->
|
||||
make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {low_checksum_list, _EpochID, _File}, Resp) ->
|
||||
to_pb_response(ReqID, {low_trim_chunk, _, _, _, _, _, _, _}, Resp) ->
|
||||
case Resp of
|
||||
ok ->
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
trim_chunk=#mpb_ll_trimchunkresp{status='OK'}};
|
||||
{error, _}=Error ->
|
||||
Status = conv_from_status(Error),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
trim_chunk=#mpb_ll_trimchunkresp{status=Status}};
|
||||
_Else ->
|
||||
make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_checksum_list, _File}}, Resp) ->
|
||||
case Resp of
|
||||
{ok, Chunk} ->
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
|
@ -479,7 +599,7 @@ to_pb_response(ReqID, {low_checksum_list, _EpochID, _File}, Resp) ->
|
|||
_Else ->
|
||||
make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {low_list_files, _EpochID}, Resp) ->
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_list_files, _EpochID}}, Resp) ->
|
||||
case Resp of
|
||||
{ok, FileInfo} ->
|
||||
PB_Files = [#mpb_fileinfo{file_size=Size, file_name=Name} ||
|
||||
|
@ -494,26 +614,28 @@ to_pb_response(ReqID, {low_list_files, _EpochID}, Resp) ->
|
|||
_Else ->
|
||||
make_ll_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {low_wedge_status, _BogusEpochID}, Resp) ->
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_wedge_status}}, Resp) ->
|
||||
case Resp of
|
||||
{error, _}=Error ->
|
||||
Status = conv_from_status(Error),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
wedge_status=#mpb_ll_wedgestatusresp{status=Status}};
|
||||
{Wedged_p, EpochID} ->
|
||||
PB_Wedged = conv_from_boolean(Wedged_p),
|
||||
{Wedged_p, EpochID, NSVersion, NS} ->
|
||||
PB_EpochID = conv_from_epoch_id(EpochID),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
wedge_status=#mpb_ll_wedgestatusresp{
|
||||
status='OK',
|
||||
epoch_id=PB_EpochID,
|
||||
wedged_flag=PB_Wedged}}
|
||||
wedged_flag=Wedged_p,
|
||||
namespace_version=NSVersion,
|
||||
namespace=NS
|
||||
}}
|
||||
end;
|
||||
to_pb_response(ReqID, {low_delete_migration, _EID, _Fl}, Resp)->
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_delete_migration, _EID, _Fl}}, Resp)->
|
||||
Status = conv_from_status(Resp),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
delete_migration=#mpb_ll_deletemigrationresp{status=Status}};
|
||||
to_pb_response(ReqID, {low_trunc_hack, _EID, _Fl}, Resp)->
|
||||
to_pb_response(ReqID, {low_skip_wedge, {low_trunc_hack, _EID, _Fl}}, Resp)->
|
||||
Status = conv_from_status(Resp),
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
trunc_hack=#mpb_ll_trunchackresp{status=Status}};
|
||||
|
@ -594,7 +716,7 @@ to_pb_response(ReqID, {high_auth, _User, _Pass}, _Resp) ->
|
|||
#mpb_response{req_id=ReqID,
|
||||
generic=#mpb_errorresp{code=1,
|
||||
msg="AUTH not implemented"}};
|
||||
to_pb_response(ReqID, {high_append_chunk, _TODO, _Prefix, _Chunk, _TSum, _CE}, Resp)->
|
||||
to_pb_response(ReqID, {high_append_chunk, _NS, _Prefix, _Chunk, _TSum, _O}, Resp)->
|
||||
case Resp of
|
||||
{ok, {Offset, Size, File}} ->
|
||||
Where = #mpb_chunkpos{offset=Offset,
|
||||
|
@ -610,7 +732,7 @@ to_pb_response(ReqID, {high_append_chunk, _TODO, _Prefix, _Chunk, _TSum, _CE}, R
|
|||
_Else ->
|
||||
make_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {high_write_chunk, _File, _Offset, _Chunk, _TaggedCSum}, Resp) ->
|
||||
to_pb_response(ReqID, {high_write_chunk, _File, _Offset, _Chunk, _CSum}, Resp) ->
|
||||
case Resp of
|
||||
{ok, {_,_,_}} ->
|
||||
%% machi_cr_client returns ok 2-tuple, convert to simple ok.
|
||||
|
@ -623,12 +745,26 @@ to_pb_response(ReqID, {high_write_chunk, _File, _Offset, _Chunk, _TaggedCSum}, R
|
|||
_Else ->
|
||||
make_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {high_read_chunk, _File, _Offset, _Size}, Resp) ->
|
||||
to_pb_response(ReqID, {high_read_chunk, _File, _Offset, _Size, _}, Resp) ->
|
||||
case Resp of
|
||||
{ok, Chunk} ->
|
||||
{ok, {Chunks, Trimmed}} ->
|
||||
PB_Chunks = lists:map(fun({File, Offset, Bytes, Csum}) ->
|
||||
{Tag, Ck} = machi_util:unmake_tagged_csum(Csum),
|
||||
#mpb_chunk{
|
||||
offset=Offset,
|
||||
file_name=File,
|
||||
chunk=Bytes,
|
||||
csum=#mpb_chunkcsum{type=conv_from_csum_tag(Tag), csum=Ck}}
|
||||
end, Chunks),
|
||||
PB_Trimmed = lists:map(fun({File, Offset, Size}) ->
|
||||
#mpb_chunkpos{file_name=File,
|
||||
offset=Offset,
|
||||
chunk_size=Size}
|
||||
end, Trimmed),
|
||||
#mpb_response{req_id=ReqID,
|
||||
read_chunk=#mpb_readchunkresp{status='OK',
|
||||
chunk=Chunk}};
|
||||
chunks=PB_Chunks,
|
||||
trimmed=PB_Trimmed}};
|
||||
{error, _}=Error ->
|
||||
Status = conv_from_status(Error),
|
||||
#mpb_response{req_id=ReqID,
|
||||
|
@ -636,6 +772,18 @@ to_pb_response(ReqID, {high_read_chunk, _File, _Offset, _Size}, Resp) ->
|
|||
_Else ->
|
||||
make_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {high_trim_chunk, _File, _Offset, _Size}, Resp) ->
|
||||
case Resp of
|
||||
ok ->
|
||||
#mpb_response{req_id=ReqID,
|
||||
trim_chunk=#mpb_trimchunkresp{status='OK'}};
|
||||
{error, _}=Error ->
|
||||
Status = conv_from_status(Error),
|
||||
#mpb_response{req_id=ReqID,
|
||||
trim_chunk=#mpb_trimchunkresp{status=Status}};
|
||||
_Else ->
|
||||
make_error_resp(ReqID, 66, io_lib:format("err ~p", [_Else]))
|
||||
end;
|
||||
to_pb_response(ReqID, {high_checksum_list, _File}, Resp) ->
|
||||
case Resp of
|
||||
{ok, Chunk} ->
|
||||
|
@ -674,12 +822,12 @@ make_tagged_csum(#mpb_chunkcsum{type='CSUM_TAG_CLIENT_SHA', csum=CSum}, _CB) ->
|
|||
make_ll_error_resp(ReqID, Code, Msg) ->
|
||||
#mpb_ll_response{req_id=ReqID,
|
||||
generic=#mpb_errorresp{code=Code,
|
||||
msg=Msg}}.
|
||||
msg=Msg}}.
|
||||
|
||||
make_error_resp(ReqID, Code, Msg) ->
|
||||
#mpb_response{req_id=ReqID,
|
||||
generic=#mpb_errorresp{code=Code,
|
||||
msg=Msg}}.
|
||||
msg=Msg}}.
|
||||
|
||||
conv_from_epoch_id({Epoch, EpochCSum}) ->
|
||||
#mpb_epochid{epoch_number=Epoch,
|
||||
|
@ -692,6 +840,7 @@ conv_to_epoch_id(#mpb_epochid{epoch_number=Epoch,
|
|||
conv_to_projection_v1(#mpb_projectionv1{epoch_number=Epoch,
|
||||
epoch_csum=CSum,
|
||||
author_server=Author,
|
||||
chain_name=ChainName,
|
||||
all_members=AllMembers,
|
||||
witnesses=Witnesses,
|
||||
creation_time=CTime,
|
||||
|
@ -705,6 +854,7 @@ conv_to_projection_v1(#mpb_projectionv1{epoch_number=Epoch,
|
|||
#projection_v1{epoch_number=Epoch,
|
||||
epoch_csum=CSum,
|
||||
author_server=to_atom(Author),
|
||||
chain_name=to_atom(ChainName),
|
||||
all_members=[to_atom(X) || X <- AllMembers],
|
||||
witnesses=[to_atom(X) || X <- Witnesses],
|
||||
creation_time=conv_to_now(CTime),
|
||||
|
@ -722,16 +872,6 @@ enc_sexp(T) ->
|
|||
dec_sexp(Bin) when is_binary(Bin) ->
|
||||
binary_to_term(Bin).
|
||||
|
||||
enc_optional_sexp(undefined) ->
|
||||
undefined;
|
||||
enc_optional_sexp(T) ->
|
||||
enc_sexp(T).
|
||||
|
||||
dec_optional_sexp(undefined) ->
|
||||
undefined;
|
||||
dec_optional_sexp(T) ->
|
||||
dec_sexp(T).
|
||||
|
||||
conv_from_members_dict(D) ->
|
||||
%% Use list_to_binary() here to "flatten" the serialized #p_srvr{}
|
||||
[#mpb_membersdictentry{key=to_list(K), val=conv_from_p_srvr(V)} ||
|
||||
|
@ -835,6 +975,8 @@ conv_from_status({error, not_written}) ->
|
|||
'NOT_WRITTEN';
|
||||
conv_from_status({error, written}) ->
|
||||
'WRITTEN';
|
||||
conv_from_status({error, trimmed}) ->
|
||||
'TRIMMED';
|
||||
conv_from_status({error, no_such_file}) ->
|
||||
'NO_SUCH_FILE';
|
||||
conv_from_status({error, partial_read}) ->
|
||||
|
@ -842,24 +984,34 @@ conv_from_status({error, partial_read}) ->
|
|||
conv_from_status({error, bad_epoch}) ->
|
||||
'BAD_EPOCH';
|
||||
conv_from_status(_OOPS) ->
|
||||
io:format(user, "HEY, ~s:~w got ~w\n", [?MODULE, ?LINE, _OOPS]),
|
||||
io:format(user, "HEY, ~s:~w got ~p\n", [?MODULE, ?LINE, _OOPS]),
|
||||
'BAD_JOSS'.
|
||||
|
||||
conv_to_boolean(undefined) ->
|
||||
false;
|
||||
conv_to_boolean(0) ->
|
||||
false;
|
||||
conv_to_boolean(N) when is_integer(N) ->
|
||||
true.
|
||||
conv_from_append_opts(#append_opts{chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref}) ->
|
||||
{ChunkExtra, Pref, FailPref}.
|
||||
|
||||
conv_from_boolean(false) ->
|
||||
0;
|
||||
conv_from_boolean(true) ->
|
||||
1.
|
||||
|
||||
conv_to_append_opts(#mpb_appendchunkreq{
|
||||
chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref}) ->
|
||||
#append_opts{chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref};
|
||||
conv_to_append_opts(#mpb_ll_appendchunkreq{
|
||||
chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref}) ->
|
||||
#append_opts{chunk_extra=ChunkExtra,
|
||||
preferred_file_name=Pref,
|
||||
flag_fail_preferred=FailPref}.
|
||||
|
||||
conv_from_projection_v1(#projection_v1{epoch_number=Epoch,
|
||||
epoch_csum=CSum,
|
||||
author_server=Author,
|
||||
chain_name=ChainName,
|
||||
all_members=AllMembers,
|
||||
witnesses=Witnesses,
|
||||
creation_time=CTime,
|
||||
|
@ -873,6 +1025,7 @@ conv_from_projection_v1(#projection_v1{epoch_number=Epoch,
|
|||
#mpb_projectionv1{epoch_number=Epoch,
|
||||
epoch_csum=CSum,
|
||||
author_server=to_list(Author),
|
||||
chain_name=to_list(ChainName),
|
||||
all_members=[to_list(X) || X <- AllMembers],
|
||||
witnesses=[to_list(X) || X <- Witnesses],
|
||||
creation_time=conv_from_now(CTime),
|
||||
|
|
89
src/machi_plist.erl
Normal file
89
src/machi_plist.erl
Normal file
|
@ -0,0 +1,89 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2016 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_plist).
|
||||
|
||||
%%% @doc persistent list of binaries
|
||||
|
||||
-export([open/2, close/1, find/2, add/2]).
|
||||
|
||||
-ifdef(TEST).
|
||||
-export([all/1]).
|
||||
-endif.
|
||||
|
||||
-record(machi_plist,
|
||||
{filename :: file:filename_all(),
|
||||
fd :: file:io_device(),
|
||||
list = [] :: list(string)}).
|
||||
|
||||
-type plist() :: #machi_plist{}.
|
||||
-export_type([plist/0]).
|
||||
|
||||
-spec open(file:filename_all(), proplists:proplist()) ->
|
||||
{ok, plist()} | {error, file:posix()}.
|
||||
open(Filename, _Opt) ->
|
||||
%% TODO: This decode could fail if the file didn't finish writing
|
||||
%% whole contents, which should be fixed by some persistent
|
||||
%% solution.
|
||||
List = case file:read_file(Filename) of
|
||||
{ok, <<>>} -> [];
|
||||
{ok, Bin} -> binary_to_term(Bin);
|
||||
{error, enoent} -> []
|
||||
end,
|
||||
case file:open(Filename, [read, write, raw, binary, sync]) of
|
||||
{ok, Fd} ->
|
||||
{ok, #machi_plist{filename=Filename,
|
||||
fd=Fd,
|
||||
list=List}};
|
||||
Error ->
|
||||
Error
|
||||
end.
|
||||
|
||||
-spec close(plist()) -> ok.
|
||||
close(#machi_plist{fd=Fd}) ->
|
||||
_ = file:close(Fd).
|
||||
|
||||
-spec find(plist(), string()) -> boolean().
|
||||
find(#machi_plist{list=List}, Name) ->
|
||||
lists:member(Name, List).
|
||||
|
||||
-spec add(plist(), string()) -> {ok, plist()} | {error, file:posix()}.
|
||||
add(Plist = #machi_plist{list=List0, fd=Fd}, Name) ->
|
||||
case find(Plist, Name) of
|
||||
true ->
|
||||
{ok, Plist};
|
||||
false ->
|
||||
List = lists:append(List0, [Name]),
|
||||
%% TODO: partial write could break the file with other
|
||||
%% persistent info (even lose data of trimmed states);
|
||||
%% needs a solution.
|
||||
case file:pwrite(Fd, 0, term_to_binary(List)) of
|
||||
ok ->
|
||||
{ok, Plist#machi_plist{list=List}};
|
||||
Error ->
|
||||
Error
|
||||
end
|
||||
end.
|
||||
|
||||
-ifdef(TEST).
|
||||
-spec all(plist()) -> [file:filename()].
|
||||
all(#machi_plist{list=List}) ->
|
||||
List.
|
||||
-endif.
|
|
@ -174,6 +174,7 @@ make_summary(#projection_v1{epoch_number=EpochNum,
|
|||
repairing=Repairing_list,
|
||||
dbg=Dbg, dbg2=Dbg2}) ->
|
||||
[{epoch,EpochNum}, {csum,_CSum4},
|
||||
{all, _All_list},
|
||||
{author,Author}, {mode,CMode},{witnesses, Witness_list},
|
||||
{upi,UPI_list},{repair,Repairing_list},{down,Down_list}] ++
|
||||
[{d,Dbg}, {d2,Dbg2}].
|
||||
|
|
|
@ -321,7 +321,7 @@ do_proj_write3(ProjType, #projection_v1{epoch_number=Epoch,
|
|||
end.
|
||||
|
||||
do_proj_write4(ProjType, Proj, Path, Epoch, #state{consistency_mode=CMode}=S) ->
|
||||
{ok, FH} = file:open(Path, [write, raw, binary]),
|
||||
{{ok, FH}, Epoch, Path} = {file:open(Path, [write, raw, binary]), Epoch, Path},
|
||||
ok = file:write(FH, term_to_binary(Proj)),
|
||||
ok = file:sync(FH),
|
||||
ok = file:close(FH),
|
||||
|
@ -387,7 +387,6 @@ wait_for_liveness(PidSpec, StartTime, WaitTime) ->
|
|||
undefined ->
|
||||
case timer:now_diff(os:timestamp(), StartTime) div 1000 of
|
||||
X when X < WaitTime ->
|
||||
io:format(user, "\nYOO ~p ~p\n", [PidSpec, lists:sort(registered())]),
|
||||
timer:sleep(1),
|
||||
wait_for_liveness(PidSpec, StartTime, WaitTime)
|
||||
end;
|
||||
|
|
|
@ -22,6 +22,10 @@
|
|||
%% proxy-process style API for hiding messy details such as TCP
|
||||
%% connection/disconnection with the remote Machi server.
|
||||
%%
|
||||
%% Please see {@link machi_flu1_client} the "Client API implemntation notes"
|
||||
%% section for how this module relates to the rest of the client API
|
||||
%% implementation.
|
||||
%%
|
||||
%% Machi is intentionally avoiding using distributed Erlang for
|
||||
%% Machi's communication. This design decision makes Erlang-side code
|
||||
%% more difficult & complex, but it's the price to pay for some
|
||||
|
@ -57,10 +61,9 @@
|
|||
%% FLU1 API
|
||||
-export([
|
||||
%% File API
|
||||
append_chunk/4, append_chunk/5,
|
||||
append_chunk_extra/5, append_chunk_extra/6,
|
||||
read_chunk/5, read_chunk/6,
|
||||
checksum_list/3, checksum_list/4,
|
||||
append_chunk/6, append_chunk/8,
|
||||
read_chunk/7, read_chunk/8,
|
||||
checksum_list/2, checksum_list/3,
|
||||
list_files/2, list_files/3,
|
||||
wedge_status/1, wedge_status/2,
|
||||
|
||||
|
@ -78,7 +81,8 @@
|
|||
quit/1,
|
||||
|
||||
%% Internal API
|
||||
write_chunk/5, write_chunk/6,
|
||||
write_chunk/7, write_chunk/8,
|
||||
trim_chunk/6, trim_chunk/7,
|
||||
|
||||
%% Helpers
|
||||
stop_proxies/1, start_proxies/1
|
||||
|
@ -103,51 +107,39 @@ start_link(#p_srvr{}=I) ->
|
|||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix'.
|
||||
|
||||
append_chunk(PidSpec, EpochID, Prefix, Chunk) ->
|
||||
append_chunk(PidSpec, EpochID, Prefix, Chunk, infinity).
|
||||
append_chunk(PidSpec, NSInfo, EpochID, Prefix, Chunk, CSum) ->
|
||||
append_chunk(PidSpec, NSInfo, EpochID, Prefix, Chunk, CSum,
|
||||
#append_opts{}, infinity).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix'.
|
||||
|
||||
append_chunk(PidSpec, EpochID, Prefix, Chunk, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {append_chunk, EpochID, Prefix, Chunk}},
|
||||
Timeout).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix'.
|
||||
|
||||
append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra)
|
||||
when is_integer(ChunkExtra), ChunkExtra >= 0 ->
|
||||
append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra, infinity).
|
||||
|
||||
%% @doc Append a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix'.
|
||||
|
||||
append_chunk_extra(PidSpec, EpochID, Prefix, Chunk, ChunkExtra, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {append_chunk_extra, EpochID, Prefix,
|
||||
Chunk, ChunkExtra}},
|
||||
append_chunk(PidSpec, NSInfo, EpochID, Prefix, Chunk, CSum, Opts,
|
||||
Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {append_chunk, NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum, Opts, Timeout}},
|
||||
Timeout).
|
||||
|
||||
%% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
|
||||
|
||||
read_chunk(PidSpec, EpochID, File, Offset, Size) ->
|
||||
read_chunk(PidSpec, EpochID, File, Offset, Size, infinity).
|
||||
read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, Opts) ->
|
||||
read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, Opts, infinity).
|
||||
|
||||
%% @doc Read a chunk of data of size `Size' from `File' at `Offset'.
|
||||
|
||||
read_chunk(PidSpec, EpochID, File, Offset, Size, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {read_chunk, EpochID, File, Offset, Size}},
|
||||
read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, Opts, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {read_chunk, NSInfo, EpochID, File, Offset, Size, Opts}},
|
||||
Timeout).
|
||||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
|
||||
checksum_list(PidSpec, EpochID, File) ->
|
||||
checksum_list(PidSpec, EpochID, File, infinity).
|
||||
checksum_list(PidSpec, File) ->
|
||||
checksum_list(PidSpec, File, infinity).
|
||||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
|
||||
checksum_list(PidSpec, EpochID, File, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {checksum_list, EpochID, File}},
|
||||
checksum_list(PidSpec, File, Timeout) ->
|
||||
gen_server:call(PidSpec, {req, {checksum_list, File}},
|
||||
Timeout).
|
||||
|
||||
%% @doc Fetch the list of all files on the remote FLU.
|
||||
|
@ -288,18 +280,19 @@ quit(PidSpec) ->
|
|||
%% @doc Write a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' at `Offset'.
|
||||
|
||||
write_chunk(PidSpec, EpochID, File, Offset, Chunk) ->
|
||||
write_chunk(PidSpec, EpochID, File, Offset, Chunk, infinity).
|
||||
write_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, CSum) ->
|
||||
write_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, CSum, infinity).
|
||||
|
||||
%% @doc Write a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' at `Offset'.
|
||||
|
||||
write_chunk(PidSpec, EpochID, File, Offset, Chunk, Timeout) ->
|
||||
case gen_server:call(PidSpec, {req, {write_chunk, EpochID, File, Offset, Chunk}},
|
||||
write_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, CSum, Timeout) ->
|
||||
case gen_server:call(PidSpec, {req, {write_chunk, NSInfo, EpochID, File, Offset, Chunk, CSum}},
|
||||
Timeout) of
|
||||
{error, written}=Err ->
|
||||
case read_chunk(PidSpec, EpochID, File, Offset, Chunk, Timeout) of
|
||||
{ok, Chunk2} when Chunk2 == Chunk ->
|
||||
Size = byte_size(Chunk),
|
||||
case read_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, undefined, Timeout) of
|
||||
{ok, {[{File, Offset, Chunk2, _}], []}} when Chunk2 == Chunk ->
|
||||
%% See equivalent comment inside write_projection().
|
||||
ok;
|
||||
_ ->
|
||||
|
@ -309,6 +302,18 @@ write_chunk(PidSpec, EpochID, File, Offset, Chunk, Timeout) ->
|
|||
Else
|
||||
end.
|
||||
|
||||
|
||||
trim_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size) ->
|
||||
trim_chunk(PidSpec, NSInfo, EpochID, File, Offset, Size, infinity).
|
||||
|
||||
%% @doc Write a chunk (binary- or iolist-style) of data to a file
|
||||
%% with `Prefix' at `Offset'.
|
||||
|
||||
trim_chunk(PidSpec, NSInfo, EpochID, File, Offset, Chunk, Timeout) ->
|
||||
gen_server:call(PidSpec,
|
||||
{req, {trim_chunk, NSInfo, EpochID, File, Offset, Chunk}},
|
||||
Timeout).
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
|
||||
init([I]) ->
|
||||
|
@ -370,21 +375,24 @@ do_req_retry(_Req, 2, Err, S) ->
|
|||
do_req_retry(Req, Depth, _Err, S) ->
|
||||
do_req(Req, Depth + 1, try_connect(disconnect(S))).
|
||||
|
||||
make_req_fun({append_chunk, EpochID, Prefix, Chunk},
|
||||
make_req_fun({append_chunk, NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum, Opts, Timeout},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:append_chunk(Sock, EpochID, Prefix, Chunk) end;
|
||||
make_req_fun({append_chunk_extra, EpochID, Prefix, Chunk, ChunkExtra},
|
||||
fun() -> Mod:append_chunk(Sock, NSInfo, EpochID,
|
||||
Prefix, Chunk, CSum, Opts, Timeout)
|
||||
end;
|
||||
make_req_fun({read_chunk, NSInfo, EpochID, File, Offset, Size, Opts},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:append_chunk_extra(Sock, EpochID, Prefix, Chunk, ChunkExtra) end;
|
||||
make_req_fun({read_chunk, EpochID, File, Offset, Size},
|
||||
fun() -> Mod:read_chunk(Sock, NSInfo, EpochID, File, Offset, Size, Opts) end;
|
||||
make_req_fun({write_chunk, NSInfo, EpochID, File, Offset, Chunk, CSum},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:read_chunk(Sock, EpochID, File, Offset, Size) end;
|
||||
make_req_fun({write_chunk, EpochID, File, Offset, Chunk},
|
||||
fun() -> Mod:write_chunk(Sock, NSInfo, EpochID, File, Offset, Chunk, CSum) end;
|
||||
make_req_fun({trim_chunk, NSInfo, EpochID, File, Offset, Size},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:write_chunk(Sock, EpochID, File, Offset, Chunk) end;
|
||||
make_req_fun({checksum_list, EpochID, File},
|
||||
fun() -> Mod:trim_chunk(Sock, NSInfo, EpochID, File, Offset, Size) end;
|
||||
make_req_fun({checksum_list, File},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:checksum_list(Sock, EpochID, File) end;
|
||||
fun() -> Mod:checksum_list(Sock, File) end;
|
||||
make_req_fun({list_files, EpochID},
|
||||
#state{sock=Sock,i=#p_srvr{proto_mod=Mod}}) ->
|
||||
fun() -> Mod:list_files(Sock, EpochID) end;
|
||||
|
|
|
@ -1,194 +0,0 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% @doc "Mothballed" sequencer code, perhaps to be reused sometime in
|
||||
%% the future?
|
||||
|
||||
-module(machi_sequencer).
|
||||
|
||||
-compile(export_all).
|
||||
|
||||
-include_lib("kernel/include/file.hrl").
|
||||
|
||||
-define(CONFIG_DIR, "./config").
|
||||
-define(DATA_DIR, "./data").
|
||||
|
||||
seq(Server, Prefix, Size) when is_binary(Prefix), is_integer(Size), Size > -1 ->
|
||||
Server ! {seq, self(), Prefix, Size},
|
||||
receive
|
||||
{assignment, File, Offset} ->
|
||||
{File, Offset}
|
||||
after 1*1000 ->
|
||||
bummer
|
||||
end.
|
||||
|
||||
seq_direct(Prefix, Size) when is_binary(Prefix), is_integer(Size), Size > -1 ->
|
||||
RegName = make_regname(Prefix),
|
||||
seq(RegName, Prefix, Size).
|
||||
|
||||
start_server() ->
|
||||
start_server(?MODULE).
|
||||
|
||||
start_server(Name) ->
|
||||
spawn_link(fun() -> run_server(Name) end).
|
||||
|
||||
run_server(Name) ->
|
||||
register(Name, self()),
|
||||
ets:new(?MODULE, [named_table, public, {write_concurrency, true}]),
|
||||
server_loop().
|
||||
|
||||
server_loop() ->
|
||||
receive
|
||||
{seq, From, Prefix, Size} ->
|
||||
spawn(fun() -> server_dispatch(From, Prefix, Size) end),
|
||||
server_loop()
|
||||
end.
|
||||
|
||||
server_dispatch(From, Prefix, Size) ->
|
||||
RegName = make_regname(Prefix),
|
||||
case whereis(RegName) of
|
||||
undefined ->
|
||||
start_prefix_server(Prefix),
|
||||
timer:sleep(1),
|
||||
server_dispatch(From, Prefix, Size);
|
||||
Pid ->
|
||||
Pid ! {seq, From, Prefix, Size}
|
||||
end,
|
||||
exit(normal).
|
||||
|
||||
start_prefix_server(Prefix) ->
|
||||
spawn(fun() -> run_prefix_server(Prefix) end).
|
||||
|
||||
run_prefix_server(Prefix) ->
|
||||
true = register(make_regname(Prefix), self()),
|
||||
ok = filelib:ensure_dir(?CONFIG_DIR ++ "/unused"),
|
||||
ok = filelib:ensure_dir(?DATA_DIR ++ "/unused"),
|
||||
FileNum = read_max_filenum(Prefix) + 1,
|
||||
ok = increment_max_filenum(Prefix),
|
||||
prefix_server_loop(Prefix, FileNum).
|
||||
|
||||
prefix_server_loop(Prefix, FileNum) ->
|
||||
File = make_data_filename(Prefix, FileNum),
|
||||
prefix_server_loop(Prefix, File, FileNum, 0).
|
||||
|
||||
prefix_server_loop(Prefix, File, FileNum, Offset) ->
|
||||
receive
|
||||
{seq, From, Prefix, Size} ->
|
||||
From ! {assignment, File, Offset},
|
||||
prefix_server_loop(Prefix, File, FileNum, Offset + Size)
|
||||
after 30*1000 ->
|
||||
io:format("timeout: ~p server stopping\n", [Prefix]),
|
||||
exit(normal)
|
||||
end.
|
||||
|
||||
make_regname(Prefix) ->
|
||||
erlang:binary_to_atom(Prefix, latin1).
|
||||
|
||||
make_config_filename(Prefix) ->
|
||||
lists:flatten(io_lib:format("~s/~s", [?CONFIG_DIR, Prefix])).
|
||||
|
||||
make_data_filename(Prefix, FileNum) ->
|
||||
erlang:iolist_to_binary(io_lib:format("~s/~s.~w",
|
||||
[?DATA_DIR, Prefix, FileNum])).
|
||||
|
||||
read_max_filenum(Prefix) ->
|
||||
case file:read_file_info(make_config_filename(Prefix)) of
|
||||
{error, enoent} ->
|
||||
0;
|
||||
{ok, FI} ->
|
||||
FI#file_info.size
|
||||
end.
|
||||
|
||||
increment_max_filenum(Prefix) ->
|
||||
{ok, FH} = file:open(make_config_filename(Prefix), [append]),
|
||||
ok = file:write(FH, "x"),
|
||||
%% ok = file:sync(FH),
|
||||
ok = file:close(FH).
|
||||
|
||||
%%%%%%%%%%%%%%%%%
|
||||
|
||||
%% basho_bench callbacks
|
||||
|
||||
-define(SEQ, ?MODULE).
|
||||
|
||||
new(1) ->
|
||||
start_server(),
|
||||
timer:sleep(100),
|
||||
{ok, unused};
|
||||
new(_Id) ->
|
||||
{ok, unused}.
|
||||
|
||||
run(null, _KeyGen, _ValgueGen, State) ->
|
||||
{ok, State};
|
||||
run(keygen_then_null, KeyGen, _ValgueGen, State) ->
|
||||
_Prefix = KeyGen(),
|
||||
{ok, State};
|
||||
run(seq, KeyGen, _ValgueGen, State) ->
|
||||
Prefix = KeyGen(),
|
||||
{_, _} = ?SEQ:seq(?SEQ, Prefix, 1),
|
||||
{ok, State};
|
||||
run(seq_direct, KeyGen, _ValgueGen, State) ->
|
||||
Prefix = KeyGen(),
|
||||
Name = ?SEQ:make_regname(Prefix),
|
||||
case get(Name) of
|
||||
undefined ->
|
||||
case whereis(Name) of
|
||||
undefined ->
|
||||
{_, _} = ?SEQ:seq(?SEQ, Prefix, 1);
|
||||
Pid ->
|
||||
put(Name, Pid),
|
||||
{_, _} = ?SEQ:seq(Pid, Prefix, 1)
|
||||
end;
|
||||
Pid ->
|
||||
{_, _} = ?SEQ:seq(Pid, Prefix, 1)
|
||||
end,
|
||||
{ok, State};
|
||||
run(seq_ets, KeyGen, _ValgueGen, State) ->
|
||||
Tab = ?MODULE,
|
||||
Prefix = KeyGen(),
|
||||
Res = try
|
||||
BigNum = ets:update_counter(Tab, Prefix, 1),
|
||||
BigBin = <<BigNum:80/big>>,
|
||||
<<FileNum:32/big, Offset:48/big>> = BigBin,
|
||||
%% if Offset rem 1000 == 0 ->
|
||||
%% io:format("~p,~p ", [FileNum, Offset]);
|
||||
%% true ->
|
||||
%% ok
|
||||
%% end,
|
||||
{fakefake, FileNum, Offset}
|
||||
catch error:badarg ->
|
||||
FileNum2 = 1, Offset2 = 0,
|
||||
FileBin = <<FileNum2:32/big>>,
|
||||
OffsetBin = <<Offset2:48/big>>,
|
||||
Glop = <<FileBin/binary, OffsetBin/binary>>,
|
||||
<<Base:80/big>> = Glop,
|
||||
%% if Prefix == <<"42">> -> io:format("base:~w\n", [Base]); true -> ok end,
|
||||
%% Base = 0,
|
||||
case ets:insert_new(Tab, {Prefix, Base}) of
|
||||
true ->
|
||||
{<<"fakefakefake">>, Base};
|
||||
false ->
|
||||
Result2 = ets:update_counter(Tab, Prefix, 1),
|
||||
{<<"fakefakefake">>, Result2}
|
||||
end
|
||||
end,
|
||||
Res = Res,
|
||||
{ok, State}.
|
||||
|
|
@ -47,8 +47,6 @@ start_link() ->
|
|||
supervisor:start_link({local, ?SERVER}, ?MODULE, []).
|
||||
|
||||
init([]) ->
|
||||
%% {_, Ps} = process_info(self(), links),
|
||||
%% [unlink(P) || P <- Ps],
|
||||
RestartStrategy = one_for_one,
|
||||
MaxRestarts = 1000,
|
||||
MaxSecondsBetweenRestarts = 3600,
|
||||
|
@ -62,9 +60,16 @@ init([]) ->
|
|||
ServerSup =
|
||||
{machi_flu_sup, {machi_flu_sup, start_link, []},
|
||||
Restart, Shutdown, Type, []},
|
||||
|
||||
{ok, {SupFlags, [ServerSup]}}.
|
||||
|
||||
%% AChild = {'AName', {'AModule', start_link, []},
|
||||
%% Restart, Shutdown, Type, ['AModule']},
|
||||
%% {ok, {SupFlags, [AChild]}}.
|
||||
RanchSup = {ranch_sup, {ranch_sup, start_link, []},
|
||||
Restart, Shutdown, supervisor, [ranch_sup]},
|
||||
LifecycleMgr =
|
||||
{machi_lifecycle_mgr, {machi_lifecycle_mgr, start_link, []},
|
||||
Restart, Shutdown, worker, []},
|
||||
RunningApps = [A || {A,_D,_V} <- application:which_applications()],
|
||||
Specs = case lists:member(ranch, RunningApps) of
|
||||
true ->
|
||||
[ServerSup, LifecycleMgr];
|
||||
false ->
|
||||
[ServerSup, RanchSup, LifecycleMgr]
|
||||
end,
|
||||
{ok, {SupFlags, Specs}}.
|
||||
|
|
|
@ -25,27 +25,35 @@
|
|||
-export([
|
||||
checksum_chunk/1,
|
||||
make_tagged_csum/1, make_tagged_csum/2,
|
||||
make_client_csum/1,
|
||||
unmake_tagged_csum/1,
|
||||
hexstr_to_bin/1, bin_to_hexstr/1,
|
||||
hexstr_to_int/1, int_to_hexstr/2, int_to_hexbin/2,
|
||||
make_binary/1, make_string/1,
|
||||
make_regname/1,
|
||||
make_config_filename/2,
|
||||
make_config_filename/4, make_config_filename/2,
|
||||
make_checksum_filename/4, make_checksum_filename/2,
|
||||
make_data_filename/4, make_data_filename/2,
|
||||
make_projection_filename/2,
|
||||
read_max_filenum/2, increment_max_filenum/2,
|
||||
make_data_filename/6, make_data_filename/2,
|
||||
make_projection_filename/2,
|
||||
is_valid_filename/1,
|
||||
parse_filename/1,
|
||||
read_max_filenum/4, increment_max_filenum/4,
|
||||
info_msg/2, verb/1, verb/2,
|
||||
mbytes/1,
|
||||
mbytes/1,
|
||||
pretty_time/0, pretty_time/2,
|
||||
%% TCP protocol helpers
|
||||
connect/2, connect/3,
|
||||
%% List twiddling
|
||||
permutations/1, perms/1,
|
||||
combinations/1, ordered_combinations/1,
|
||||
mk_order/2
|
||||
mk_order/2,
|
||||
%% Other
|
||||
wait_for_death/2, wait_for_life/2,
|
||||
bool2int/1,
|
||||
int2bool/1,
|
||||
read_opts_default/1,
|
||||
ns_info_default/1
|
||||
]).
|
||||
-compile(export_all).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
@ -63,17 +71,27 @@ make_regname(Prefix) when is_list(Prefix) ->
|
|||
|
||||
%% @doc Calculate a config file path, by common convention.
|
||||
|
||||
-spec make_config_filename(string(), machi_dt:namespace(), machi_dt:locator(), string()) ->
|
||||
string().
|
||||
make_config_filename(DataDir, NS, NSLocator, Prefix) ->
|
||||
NSLocator_str = int_to_hexstr(NSLocator, 32),
|
||||
lists:flatten(io_lib:format("~s/config/~s^~s^~s",
|
||||
[DataDir, Prefix, NS, NSLocator_str])).
|
||||
|
||||
%% @doc Calculate a config file path, by common convention.
|
||||
|
||||
-spec make_config_filename(string(), string()) ->
|
||||
string().
|
||||
make_config_filename(DataDir, Prefix) ->
|
||||
lists:flatten(io_lib:format("~s/config/~s", [DataDir, Prefix])).
|
||||
make_config_filename(DataDir, Filename) ->
|
||||
lists:flatten(io_lib:format("~s/config/~s",
|
||||
[DataDir, Filename])).
|
||||
|
||||
%% @doc Calculate a checksum file path, by common convention.
|
||||
|
||||
-spec make_checksum_filename(string(), string(), atom()|string()|binary(), integer()) ->
|
||||
string().
|
||||
make_checksum_filename(DataDir, Prefix, SequencerName, FileNum) ->
|
||||
lists:flatten(io_lib:format("~s/config/~s.~s.~w.csum",
|
||||
lists:flatten(io_lib:format("~s/config/~s^~s^~w.csum",
|
||||
[DataDir, Prefix, SequencerName, FileNum])).
|
||||
|
||||
%% @doc Calculate a checksum file path, by common convention.
|
||||
|
@ -87,11 +105,22 @@ make_checksum_filename(DataDir, FileName) ->
|
|||
|
||||
%% @doc Calculate a file data file path, by common convention.
|
||||
|
||||
-spec make_data_filename(string(), string(), atom()|string()|binary(), integer()) ->
|
||||
-spec make_data_filename(string(), machi_dt:namespace(), machi_dt:locator(), string(), atom()|string()|binary(), integer()|string()) ->
|
||||
{binary(), string()}.
|
||||
make_data_filename(DataDir, Prefix, SequencerName, FileNum) ->
|
||||
File = erlang:iolist_to_binary(io_lib:format("~s.~s.~w",
|
||||
[Prefix, SequencerName, FileNum])),
|
||||
make_data_filename(DataDir, NS, NSLocator, Prefix, SequencerName, FileNum)
|
||||
when is_integer(FileNum) ->
|
||||
NSLocator_str = int_to_hexstr(NSLocator, 32),
|
||||
File = erlang:iolist_to_binary(io_lib:format("~s^~s^~s^~s^~w",
|
||||
[Prefix, NS, NSLocator_str, SequencerName, FileNum])),
|
||||
make_data_filename2(DataDir, File);
|
||||
make_data_filename(DataDir, NS, NSLocator, Prefix, SequencerName, String)
|
||||
when is_list(String) ->
|
||||
NSLocator_str = int_to_hexstr(NSLocator, 32),
|
||||
File = erlang:iolist_to_binary(io_lib:format("~s^~s^~s^~s^~s",
|
||||
[Prefix, NS, NSLocator_str, SequencerName, string])),
|
||||
make_data_filename2(DataDir, File).
|
||||
|
||||
make_data_filename2(DataDir, File) ->
|
||||
FullPath = lists:flatten(io_lib:format("~s/data/~s", [DataDir, File])),
|
||||
{File, FullPath}.
|
||||
|
||||
|
@ -115,13 +144,49 @@ make_projection_filename(DataDir, "") ->
|
|||
make_projection_filename(DataDir, File) ->
|
||||
lists:flatten(io_lib:format("~s/projection/~s", [DataDir, File])).
|
||||
|
||||
%% @doc Given a filename, return true if it is a valid machi filename,
|
||||
%% false otherwise.
|
||||
-spec is_valid_filename( Filename :: string() ) -> true | false.
|
||||
is_valid_filename(Filename) ->
|
||||
case parse_filename(Filename) of
|
||||
{} -> false;
|
||||
{_,_,_,_,_} -> true
|
||||
end.
|
||||
|
||||
%% @doc Given a machi filename, return a set of components in a list.
|
||||
%% The components will be:
|
||||
%% <ul>
|
||||
%% <li>Prefix</li>
|
||||
%% <li>Cluster namespace</li>
|
||||
%% <li>Cluster locator</li>
|
||||
%% <li>UUID</li>
|
||||
%% <li>Sequence number</li>
|
||||
%% </ul>
|
||||
%%
|
||||
%% Invalid filenames will return an empty list.
|
||||
-spec parse_filename( Filename :: string() ) -> {} | {string(), machi_dt:namespace(), machi_dt:locator(), string(), string() }.
|
||||
parse_filename(Filename) ->
|
||||
case string:tokens(Filename, "^") of
|
||||
[Prefix, NS, NSLocator, UUID, SeqNo] ->
|
||||
{Prefix, NS, list_to_integer(NSLocator), UUID, SeqNo};
|
||||
[Prefix, NSLocator, UUID, SeqNo] ->
|
||||
%% string:tokens() doesn't consider "foo^^bar" as 3 tokens {sigh}
|
||||
case re:replace(Filename, "[^^]+", "x", [global,{return,binary}]) of
|
||||
<<"x^^x^x^x">> ->
|
||||
{Prefix, <<"">>, list_to_integer(NSLocator), UUID, SeqNo};
|
||||
_ ->
|
||||
{}
|
||||
end;
|
||||
_ -> {}
|
||||
end.
|
||||
|
||||
%% @doc Read the file size of a config file, which is used as the
|
||||
%% basis for a minimum sequence number.
|
||||
|
||||
-spec read_max_filenum(string(), string()) ->
|
||||
-spec read_max_filenum(string(), machi_dt:namespace(), machi_dt:locator(), string()) ->
|
||||
non_neg_integer().
|
||||
read_max_filenum(DataDir, Prefix) ->
|
||||
case file:read_file_info(make_config_filename(DataDir, Prefix)) of
|
||||
read_max_filenum(DataDir, NS, NSLocator, Prefix) ->
|
||||
case file:read_file_info(make_config_filename(DataDir, NS, NSLocator, Prefix)) of
|
||||
{error, enoent} ->
|
||||
0;
|
||||
{ok, FI} ->
|
||||
|
@ -131,11 +196,11 @@ read_max_filenum(DataDir, Prefix) ->
|
|||
%% @doc Increase the file size of a config file, which is used as the
|
||||
%% basis for a minimum sequence number.
|
||||
|
||||
-spec increment_max_filenum(string(), string()) ->
|
||||
-spec increment_max_filenum(string(), machi_dt:namespace(), machi_dt:locator(), string()) ->
|
||||
ok | {error, term()}.
|
||||
increment_max_filenum(DataDir, Prefix) ->
|
||||
increment_max_filenum(DataDir, NS, NSLocator, Prefix) ->
|
||||
try
|
||||
{ok, FH} = file:open(make_config_filename(DataDir, Prefix), [append]),
|
||||
{ok, FH} = file:open(make_config_filename(DataDir, NS, NSLocator, Prefix), [append]),
|
||||
ok = file:write(FH, "x"),
|
||||
ok = file:sync(FH),
|
||||
ok = file:close(FH)
|
||||
|
@ -224,22 +289,48 @@ int_to_hexbin(I, I_size) ->
|
|||
checksum_chunk(Chunk) when is_binary(Chunk); is_list(Chunk) ->
|
||||
crypto:hash(sha, Chunk).
|
||||
|
||||
convert_csum_tag(A) when is_atom(A)->
|
||||
A;
|
||||
convert_csum_tag(?CSUM_TAG_NONE) ->
|
||||
?CSUM_TAG_NONE_ATOM;
|
||||
convert_csum_tag(?CSUM_TAG_CLIENT_SHA) ->
|
||||
?CSUM_TAG_CLIENT_SHA_ATOM;
|
||||
convert_csum_tag(?CSUM_TAG_SERVER_SHA) ->
|
||||
?CSUM_TAG_SERVER_SHA_ATOM;
|
||||
convert_csum_tag(?CSUM_TAG_SERVER_REGEN_SHA) ->
|
||||
?CSUM_TAG_SERVER_REGEN_SHA_ATOM.
|
||||
|
||||
%% @doc Create a tagged checksum
|
||||
|
||||
make_tagged_csum(none) ->
|
||||
<<?CSUM_TAG_NONE:8>>;
|
||||
make_tagged_csum({Tag, CSum}) ->
|
||||
make_tagged_csum(Tag, CSum).
|
||||
|
||||
make_tagged_csum(none, _SHA) ->
|
||||
make_tagged_csum(<<>>) ->
|
||||
<<?CSUM_TAG_NONE:8>>;
|
||||
make_tagged_csum(client_sha, SHA) ->
|
||||
make_tagged_csum({Tag, CSum}) ->
|
||||
make_tagged_csum(convert_csum_tag(Tag), CSum).
|
||||
|
||||
%% @doc Makes tagged csum. Each meanings are:
|
||||
%% none / ?CSUM_TAG_NONE
|
||||
%% - a suspicious and nonsense checksum
|
||||
%% client_sha / ?CSUM_TAG_CLIENT_SHA
|
||||
%% - a valid checksum given by client and stored in server
|
||||
%% server_sha / ?CSUM_TAG_SERVER_SHA
|
||||
%% - a valid checksum generated by and stored in server
|
||||
%% server_regen_sha / ?CSUM_TAG_SERVER_REGEN_SHA
|
||||
%% - a valid checksum generated by server in an ad hoc manner, not stored in server
|
||||
-spec make_tagged_csum(machi_dt:csum_tag(), binary()) -> machi_dt:chunk_csum().
|
||||
make_tagged_csum(?CSUM_TAG_NONE_ATOM, _SHA) ->
|
||||
<<?CSUM_TAG_NONE:8>>;
|
||||
make_tagged_csum(?CSUM_TAG_CLIENT_SHA_ATOM, SHA) ->
|
||||
<<?CSUM_TAG_CLIENT_SHA:8, SHA/binary>>;
|
||||
make_tagged_csum(server_sha, SHA) ->
|
||||
make_tagged_csum(?CSUM_TAG_SERVER_SHA_ATOM, SHA) ->
|
||||
<<?CSUM_TAG_SERVER_SHA:8, SHA/binary>>;
|
||||
make_tagged_csum(server_regen_sha, SHA) ->
|
||||
make_tagged_csum(?CSUM_TAG_SERVER_REGEN_SHA_ATOM, SHA) ->
|
||||
<<?CSUM_TAG_SERVER_REGEN_SHA:8, SHA/binary>>.
|
||||
|
||||
make_client_csum(BinOrList) ->
|
||||
make_tagged_csum(?CSUM_TAG_CLIENT_SHA_ATOM, checksum_chunk(BinOrList)).
|
||||
|
||||
unmake_tagged_csum(<<Tag:8, Rest/binary>>) ->
|
||||
{Tag, Rest}.
|
||||
|
||||
|
@ -286,11 +377,22 @@ wait_for_death(Pid, Iters) when is_pid(Pid) ->
|
|||
case erlang:is_process_alive(Pid) of
|
||||
false ->
|
||||
ok;
|
||||
true ->
|
||||
timer:sleep(1),
|
||||
true ->
|
||||
timer:sleep(10),
|
||||
wait_for_death(Pid, Iters-1)
|
||||
end.
|
||||
|
||||
wait_for_life(Reg, 0) ->
|
||||
exit({not_alive_yet, Reg});
|
||||
wait_for_life(Reg, Iters) when is_atom(Reg) ->
|
||||
case erlang:whereis(Reg) of
|
||||
Pid when is_pid(Pid) ->
|
||||
{ok, Pid};
|
||||
_ ->
|
||||
timer:sleep(1),
|
||||
wait_for_life(Reg, Iters-1)
|
||||
end.
|
||||
|
||||
%%%%%%%%%%%%%%%%%
|
||||
|
||||
%% @doc Create a TCP connection to a remote Machi server.
|
||||
|
@ -341,3 +443,23 @@ mk_order(UPI2, Repair1) ->
|
|||
error -> error
|
||||
end || X <- UPI2],
|
||||
UPI2_order.
|
||||
|
||||
%% C-style conversion for PB usage.
|
||||
bool2int(true) -> 1;
|
||||
bool2int(false) -> 0.
|
||||
int2bool(0) -> false;
|
||||
int2bool(I) when is_integer(I) -> true.
|
||||
|
||||
read_opts_default(#read_opts{}=NSInfo) ->
|
||||
NSInfo;
|
||||
read_opts_default(A) when A == 'undefined'; A == 'noopt'; A == 'none' ->
|
||||
#read_opts{};
|
||||
read_opts_default(A) when is_atom(A) ->
|
||||
#read_opts{}.
|
||||
|
||||
ns_info_default(#ns_info{}=NSInfo) ->
|
||||
NSInfo;
|
||||
ns_info_default(A) when is_atom(A) ->
|
||||
#ns_info{}.
|
||||
|
||||
|
||||
|
|
|
@ -22,6 +22,8 @@
|
|||
|
||||
-module(machi_yessir_client).
|
||||
|
||||
-ifdef(TODO_refactoring_deferred).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
|
@ -30,7 +32,7 @@
|
|||
append_chunk/4, append_chunk/5,
|
||||
append_chunk_extra/5, append_chunk_extra/6,
|
||||
read_chunk/5, read_chunk/6,
|
||||
checksum_list/3, checksum_list/4,
|
||||
checksum_list/2, checksum_list/3,
|
||||
list_files/2, list_files/3,
|
||||
wedge_status/1, wedge_status/2,
|
||||
|
||||
|
@ -173,24 +175,24 @@ read_chunk(_Host, _TcpPort, EpochID, File, Offset, Size)
|
|||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
|
||||
checksum_list(#yessir{name=Name,chunk_size=ChunkSize}, _EpochID, File) ->
|
||||
checksum_list(#yessir{name=Name,chunk_size=ChunkSize}, File) ->
|
||||
case get({Name,offset,File}) of
|
||||
undefined ->
|
||||
{error, no_such_file};
|
||||
MaxOffset ->
|
||||
C = machi_util:make_tagged_csum(client_sha,
|
||||
make_csum(Name, ChunkSize)),
|
||||
Cs = [machi_flu1:encode_csum_file_entry_bin(Offset, ChunkSize, C) ||
|
||||
Offset <- lists:seq(?MINIMUM_OFFSET, MaxOffset, ChunkSize)],
|
||||
{ok, Cs}
|
||||
Cs = [{Offset, ChunkSize, C} ||
|
||||
Offset <- lists:seq(?MINIMUM_OFFSET, MaxOffset, ChunkSize)],
|
||||
{ok, term_to_binary(Cs)}
|
||||
end.
|
||||
|
||||
%% @doc Fetch the list of chunk checksums for `File'.
|
||||
|
||||
checksum_list(_Host, _TcpPort, EpochID, File) ->
|
||||
checksum_list(_Host, _TcpPort, File) ->
|
||||
Sock = connect(#p_srvr{proto_mod=?MODULE}),
|
||||
try
|
||||
checksum_list(Sock, EpochID, File)
|
||||
checksum_list(Sock, File)
|
||||
after
|
||||
disconnect(Sock)
|
||||
end.
|
||||
|
@ -450,18 +452,18 @@ connect(#p_srvr{name=Name, props=Props})->
|
|||
chunk_size=ChunkSize
|
||||
},
|
||||
%% Add fake dict entries for these files
|
||||
[begin
|
||||
Prefix = list_to_binary(io_lib:format("fake~w", [X])),
|
||||
{ok, _} = append_chunk_extra(Sock, {1,<<"unused">>}, Prefix, <<>>, FileSize)
|
||||
end || X <- lists:seq(1, NumFiles)],
|
||||
_ = [begin
|
||||
Prefix = list_to_binary(io_lib:format("fake~w", [X])),
|
||||
{ok, _} = append_chunk_extra(Sock, {1,<<"unused">>}, Prefix, <<>>, FileSize)
|
||||
end || X <- lists:seq(1, NumFiles)],
|
||||
|
||||
Sock.
|
||||
|
||||
disconnect(#yessir{name=Name}) ->
|
||||
[erase(K) || {{N,offset,_}=K, _V} <- get(), N == Name],
|
||||
[erase(K) || {{N,chunk,_}=K, _V} <- get(), N == Name],
|
||||
[erase(K) || {{N,csum,_}=K, _V} <- get(), N == Name],
|
||||
[erase(K) || {{N,proj,_,_}=K, _V} <- get(), N == Name],
|
||||
_ = [erase(K) || {{N,offset,_}=K, _V} <- get(), N == Name],
|
||||
_ = [erase(K) || {{N,chunk,_}=K, _V} <- get(), N == Name],
|
||||
_ = [erase(K) || {{N,csum,_}=K, _V} <- get(), N == Name],
|
||||
_ = [erase(K) || {{N,proj,_,_}=K, _V} <- get(), N == Name],
|
||||
ok.
|
||||
|
||||
%% Example use:
|
||||
|
@ -509,3 +511,5 @@ disconnect(#yessir{name=Name}) ->
|
|||
%% =INFO REPORT==== 17-May-2015::18:57:52 ===
|
||||
%% Repair success: tail a of [a] finished ap_mode repair ID {a,{1431,856671,140404}}: ok
|
||||
%% Stats [{t_in_files,0},{t_in_chunks,10413},{t_in_bytes,682426368},{t_out_files,0},{t_out_chunks,10413},{t_out_bytes,682426368},{t_bad_chunks,0},{t_elapsed_seconds,1.591}]
|
||||
|
||||
-endif. % TODO_refactoring_deferred
|
||||
|
|
|
@ -33,49 +33,59 @@
|
|||
-define(FLU_C, machi_flu1_client).
|
||||
|
||||
verify_file_checksums_test_() ->
|
||||
{timeout, 60, fun() -> verify_file_checksums_test2() end}.
|
||||
{setup,
|
||||
fun() -> os:cmd("rm -rf ./data") end,
|
||||
fun(_) -> os:cmd("rm -rf ./data") end,
|
||||
{timeout, 60, fun() -> verify_file_checksums_test2() end}
|
||||
}.
|
||||
|
||||
verify_file_checksums_test2() ->
|
||||
Host = "localhost",
|
||||
TcpPort = 32958,
|
||||
DataDir = "./data",
|
||||
W_props = [{initial_wedged, false}],
|
||||
FLU1 = machi_flu1_test:setup_test_flu(verify1_flu, TcpPort, DataDir,
|
||||
W_props),
|
||||
Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}),
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
try
|
||||
Prefix = <<"verify_prefix">>,
|
||||
NumChunks = 10,
|
||||
[{ok, _} = ?FLU_C:append_chunk(Sock1, ?DUMMY_PV1_EPOCH,
|
||||
Prefix, <<X:(X*8)/big>>) ||
|
||||
X <- lists:seq(1, NumChunks)],
|
||||
{ok, [{_FileSize,File}]} = ?FLU_C:list_files(Sock1, ?DUMMY_PV1_EPOCH),
|
||||
{ok, []} = machi_admin_util:verify_file_checksums_remote(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, File),
|
||||
machi_test_util:start_flu_package(verify1_flu, TcpPort, DataDir,
|
||||
W_props),
|
||||
Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}),
|
||||
try
|
||||
Prefix = <<"verify_prefix">>,
|
||||
NumChunks = 10,
|
||||
[{ok, _} = ?FLU_C:append_chunk(Sock1, NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
Prefix, <<X:(X*8)/big>>, NoCSum) ||
|
||||
X <- lists:seq(1, NumChunks)],
|
||||
{ok, [{_FileSize,File}]} = ?FLU_C:list_files(Sock1, ?DUMMY_PV1_EPOCH),
|
||||
?assertEqual({ok, []},
|
||||
machi_admin_util:verify_file_checksums_remote(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, File)),
|
||||
|
||||
%% Clobber the first 3 chunks, which are sizes 1/2/3.
|
||||
{_, Path} = machi_util:make_data_filename(DataDir,binary_to_list(File)),
|
||||
{ok, FH} = file:open(Path, [read,write]),
|
||||
{ok, _} = file:position(FH, ?MINIMUM_OFFSET),
|
||||
ok = file:write(FH, "y"),
|
||||
ok = file:write(FH, "yo"),
|
||||
ok = file:write(FH, "yo!"),
|
||||
ok = file:close(FH),
|
||||
%% Clobber the first 3 chunks, which are sizes 1/2/3.
|
||||
{_, Path} = machi_util:make_data_filename(DataDir,binary_to_list(File)),
|
||||
{ok, FH} = file:open(Path, [read,write]),
|
||||
{ok, _} = file:position(FH, ?MINIMUM_OFFSET),
|
||||
ok = file:write(FH, "y"),
|
||||
ok = file:write(FH, "yo"),
|
||||
ok = file:write(FH, "yo!"),
|
||||
ok = file:close(FH),
|
||||
|
||||
%% Check the local flavor of the API: should be 3 bad checksums
|
||||
{ok, Res1} = machi_admin_util:verify_file_checksums_local(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, Path),
|
||||
3 = length(Res1),
|
||||
%% Check the local flavor of the API: should be 3 bad checksums
|
||||
{ok, Res1} = machi_admin_util:verify_file_checksums_local(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, Path),
|
||||
3 = length(Res1),
|
||||
|
||||
%% Check the remote flavor of the API: should be 3 bad checksums
|
||||
{ok, Res2} = machi_admin_util:verify_file_checksums_remote(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, File),
|
||||
3 = length(Res2),
|
||||
%% Check the remote flavor of the API: should be 3 bad checksums
|
||||
{ok, Res2} = machi_admin_util:verify_file_checksums_remote(
|
||||
Host, TcpPort, ?DUMMY_PV1_EPOCH, File),
|
||||
3 = length(Res2),
|
||||
|
||||
ok
|
||||
ok
|
||||
after
|
||||
catch ?FLU_C:quit(Sock1)
|
||||
end
|
||||
after
|
||||
catch ?FLU_C:quit(Sock1),
|
||||
ok = ?FLU:stop(FLU1)
|
||||
catch machi_test_util:stop_flu_package()
|
||||
end.
|
||||
|
||||
-endif. % !PULSE
|
||||
|
|
616
test/machi_ap_repair_eqc.erl
Normal file
616
test/machi_ap_repair_eqc.erl
Normal file
|
@ -0,0 +1,616 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
%% EQC single-threaded and concurrent test for file operations and repair
|
||||
%% under simulated network partition.
|
||||
|
||||
%% The main purpose is to confirm no dataloss, i.e. every chunk that
|
||||
%% has been successfully written (ACK received) by append/write
|
||||
%% opration will be read after partition heals.
|
||||
%%
|
||||
%% All updating -- append, write and trim -- operations are executed
|
||||
%% through CR client, not directly by flu1 client, in order to be
|
||||
%% end-to-end test (in single chain point of veiw.) There may be churn
|
||||
%% for projections by simulated network partition.
|
||||
%%
|
||||
%% Test steps
|
||||
%% 1. Setup single chain.
|
||||
%% 2. Execute updating operations and simulated partition (by eqc_statem).
|
||||
%% Every updating results are recorded in ETS tables.
|
||||
%% 3. When {error, timeout|partition} happens, trigger management tick for
|
||||
%% every chain manager process.
|
||||
%% 4. After commands are executed, remove patition and wait for the chain
|
||||
%% without down nodes nor repairing nodes.
|
||||
%% 5. Asserting written results so that each record be read from the
|
||||
%% chain and data be the same with written one.
|
||||
|
||||
%% Improvements to-do's
|
||||
%% - Use higher concurrency, e.g. 10+
|
||||
%% - Random length for binary to write
|
||||
%% - Operations other than append, write, trim
|
||||
%% - Use checksum instead of binary to save memory
|
||||
%% - More variety for partitioning pattern: non-constant failure
|
||||
%% - Stop and restart
|
||||
%% - Suspend and resume of some erlang processes
|
||||
|
||||
-module(machi_ap_repair_eqc).
|
||||
|
||||
-ifdef(TEST).
|
||||
-ifdef(EQC).
|
||||
-compile(export_all).
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
-include("machi_verbose.hrl").
|
||||
-include_lib("eqc/include/eqc.hrl").
|
||||
-include_lib("eqc/include/eqc_statem.hrl").
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
-record(target, {verbose=false,
|
||||
flu_names,
|
||||
mgr_names}).
|
||||
|
||||
-record(state, {num,
|
||||
verbose=false,
|
||||
flu_names,
|
||||
mgr_names,
|
||||
cr_count}).
|
||||
|
||||
%% ETS table names
|
||||
-define(WRITTEN_TAB, written). % Successfully written data
|
||||
-define(ACCPT_TAB, accpt). % Errors with no harm, e.g. timeout
|
||||
-define(FAILED_TAB, failed). % Uncategorized errors, when happenes
|
||||
% it should be re-categorized to accept or critical
|
||||
-define(CRITICAL_TAB, critical). % Critical errors, e.g. double write to the same key
|
||||
|
||||
-define(QC_OUT(P),
|
||||
eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)).
|
||||
|
||||
%% EUNIT TEST DEFINITION
|
||||
prop_repair_test_() ->
|
||||
{PropTO, EUnitTO} = eqc_timeout(60),
|
||||
Verbose = eqc_verbose(),
|
||||
{spawn,
|
||||
[{timeout, EUnitTO,
|
||||
?_assertEqual(
|
||||
true,
|
||||
eqc:quickcheck(eqc:testing_time(
|
||||
PropTO, ?QC_OUT(noshrink(prop_repair(Verbose))))))}]}.
|
||||
|
||||
prop_repair_par_test_() ->
|
||||
{PropTO, EUnitTO} = eqc_timeout(60),
|
||||
Verbose = eqc_verbose(),
|
||||
{spawn,
|
||||
[{timeout, EUnitTO,
|
||||
?_assertEqual(
|
||||
true,
|
||||
eqc:quickcheck(eqc:testing_time(
|
||||
PropTO, ?QC_OUT(noshrink(prop_repair_par(Verbose))))))}]}.
|
||||
|
||||
%% Model
|
||||
|
||||
weight(_S, change_partition) -> 20;
|
||||
weight(_S, _) -> 100.
|
||||
|
||||
%% Append
|
||||
|
||||
append_args(#state{cr_count=CRCount}=S) ->
|
||||
[choose(1, CRCount), chunk(), S].
|
||||
|
||||
append(CRIndex, Bin, #state{verbose=V}=S) ->
|
||||
CRList = cr_list(),
|
||||
{_SimSelfName, C} = lists:nth(CRIndex, CRList),
|
||||
Prefix = <<"pre">>,
|
||||
Len = byte_size(Bin),
|
||||
NSInfo = #ns_info{},
|
||||
NoCSum = <<>>,
|
||||
Opts1 = #append_opts{},
|
||||
Res = (catch machi_cr_client:append_chunk(C, NSInfo, Prefix, Bin, NoCSum, Opts1, sec(1))),
|
||||
case Res of
|
||||
{ok, {_Off, Len, _FileName}=Key} ->
|
||||
case ets:insert_new(?WRITTEN_TAB, {Key, Bin}) of
|
||||
true ->
|
||||
[?V("<o>", []) || V],
|
||||
ok;
|
||||
false ->
|
||||
%% The Key is alread written, WHY!!!????
|
||||
case ets:lookup(?WRITTEN_TAB, Key) of
|
||||
[{Key, Bin}] ->
|
||||
%% TODO: The identical binary is alread inserted in
|
||||
%% written table. Is this acceptable??? Hmm, maybe NO...
|
||||
[?V("<dws:~w>", [Key]) || V],
|
||||
true = ets:insert_new(?ACCPT_TAB,
|
||||
{make_ref(), double_write_same, Key}),
|
||||
{acceptable_error, doublewrite_the_same};
|
||||
[{Key, OtherBin}] ->
|
||||
[?V("<dwd:~w:~w>", [Key, {OtherBin, Bin}]) || V],
|
||||
true = ets:insert_new(?CRITICAL_TAB,
|
||||
{make_ref(), double_write_diff, Key}),
|
||||
R = {critical_error,
|
||||
{doublewrite_diff, Key, {OtherBin, Bin}}},
|
||||
%% TODO: when double write happens, it seems that
|
||||
%% repair process got stack with endless loop. To
|
||||
%% avoit it, return error here.
|
||||
%% If this error/1 will be removed, one can possibly
|
||||
%% know double write frequency/rate.
|
||||
error(R)
|
||||
end
|
||||
end;
|
||||
{error, partition} ->
|
||||
[?V("<pt>", []) || V],
|
||||
true = ets:insert_new(?ACCPT_TAB, {make_ref(), timeout}),
|
||||
_ = tick(S),
|
||||
{acceptable_error, partition};
|
||||
{'EXIT', {timeout, _}} ->
|
||||
[?V("<to:~w:~w>", [_SimSelfName, C]) || V],
|
||||
true = ets:insert_new(?ACCPT_TAB, {make_ref(), timeout}),
|
||||
_ = tick(S),
|
||||
{acceptable_error, timeout};
|
||||
{ok, {_Off, UnexpectedLen, _FileName}=Key} ->
|
||||
[?V("<XX>", []) || V],
|
||||
true = ets:insert_new(?CRITICAL_TAB, {make_ref(), unexpected_len, Key}),
|
||||
{critical_error, {unexpected_len, Key, Len, UnexpectedLen}};
|
||||
{error, _Reason} = Error ->
|
||||
[?V("<er>", []) || V],
|
||||
true = ets:insert_new(?FAILED_TAB, {make_ref(), Error}),
|
||||
{other_error, Error};
|
||||
Other ->
|
||||
[?V("<er>", []) || V],
|
||||
true = ets:insert_new(?FAILED_TAB, {make_ref(), Other}),
|
||||
{other_error, Other}
|
||||
end.
|
||||
|
||||
%% Change partition
|
||||
|
||||
change_partition_args(#state{flu_names=FLUNames}=S) ->
|
||||
%% [partition(FLUNames), S].
|
||||
[partition_sym(FLUNames), S].
|
||||
|
||||
change_partition(Partition,
|
||||
#state{verbose=Verbose, flu_names=FLUNames}=S) ->
|
||||
[case Partition of
|
||||
[] -> ?V("## Turn OFF partition: ~w~n", [Partition]);
|
||||
_ -> ?V("## Turn ON partition: ~w~n", [Partition])
|
||||
end || Verbose],
|
||||
machi_partition_simulator:always_these_partitions(Partition),
|
||||
_ = machi_partition_simulator:get(FLUNames),
|
||||
%% Don't wait for stable chain, tick will be executed on demand
|
||||
%% in append oprations
|
||||
_ = tick(S),
|
||||
|
||||
ok.
|
||||
|
||||
%% Generators
|
||||
|
||||
num() ->
|
||||
choose(2, 5).
|
||||
|
||||
cr_count(Num) ->
|
||||
Num * 3.
|
||||
|
||||
%% Returns a list like
|
||||
%% `[{#p_srvr{name=a, port=7501, ..}, "./eqc/data.eqc.a/"}, ...]'
|
||||
all_list_extra(Num) ->
|
||||
{PortBase, DirBase} = get_port_dir_base(),
|
||||
[begin
|
||||
FLUNameStr = [$a + I - 1],
|
||||
FLUName = list_to_atom(FLUNameStr),
|
||||
MgrName = machi_flu_psup:make_mgr_supname(FLUName),
|
||||
{#p_srvr{name=FLUName, address="localhost", port=PortBase+I,
|
||||
props=[{chmgr, MgrName}]},
|
||||
DirBase ++ "/data.eqc." ++ FLUNameStr}
|
||||
end || I <- lists:seq(1, Num)].
|
||||
|
||||
sublist(L) ->
|
||||
?LET(K, nat(),
|
||||
?LET(L2, eqc_gen:vector(K, eqc_gen:oneof(L)),
|
||||
lists:usort(L2))).
|
||||
|
||||
%% Generator for possibly assymmetric partition information
|
||||
partition(FLUNames) ->
|
||||
frequency([{10, return([])},
|
||||
{20, non_empty(sublist(flu_ordered_pairs(FLUNames)))}]).
|
||||
|
||||
%% Generator for symmetric partition information
|
||||
partition_sym(FLUNames) ->
|
||||
?LET(Pairs, non_empty(sublist(flu_pairs(FLUNames))),
|
||||
lists:flatmap(fun({One, Another}) -> [{One, Another}, {Another, One}] end,
|
||||
Pairs)).
|
||||
|
||||
flu_ordered_pairs(FLUNames) ->
|
||||
[{From, To} || From <- FLUNames, To <- FLUNames, From =/= To].
|
||||
|
||||
flu_pairs(FLUNames) ->
|
||||
[{One, Another} || One <- FLUNames, Another <- FLUNames, One > Another].
|
||||
|
||||
chunk() ->
|
||||
non_empty(binary(10)).
|
||||
|
||||
%% Properties
|
||||
|
||||
prop_repair(Verbose) ->
|
||||
error_logger:tty(false),
|
||||
application:load(sasl),
|
||||
application:set_env(sasl, sasl_error_logger, false),
|
||||
|
||||
Seed = {1445,935441,287549},
|
||||
?FORALL(Num, num(),
|
||||
?FORALL(Cmds, commands(?MODULE, initial_state(Num, Verbose)),
|
||||
begin
|
||||
Target = setup_target(Num, Seed, Verbose),
|
||||
{H, S1, Res0} = run_commands(?MODULE, Cmds),
|
||||
%% ?V("S1=~w~n", [S1]),
|
||||
?V("==== Start post operations, stabilize and confirm results~n", []),
|
||||
_ = stabilize(commands_len(Cmds), Target),
|
||||
{Dataloss, Critical} = confirm_result(Target),
|
||||
_ = cleanup(Target),
|
||||
pretty_commands(
|
||||
?MODULE, Cmds, {H, S1, Res0},
|
||||
aggregate(with_title(cmds), command_names(Cmds),
|
||||
collect(with_title(length5), (length(Cmds) div 5) * 5,
|
||||
{Dataloss, Critical} =:= {0, 0})))
|
||||
end)).
|
||||
|
||||
prop_repair_par(Verbose) ->
|
||||
error_logger:tty(false),
|
||||
application:load(sasl),
|
||||
application:set_env(sasl, sasl_error_logger, false),
|
||||
|
||||
Seed = {1445,935441,287549},
|
||||
?FORALL(Num, num(),
|
||||
?FORALL(Cmds,
|
||||
%% Now try-and-err'ing, how to control command length and concurrency?
|
||||
?SUCHTHAT(Cmds0, ?SIZED(Size, resize(Size,
|
||||
parallel_commands(?MODULE, initial_state(Num, Verbose)))),
|
||||
commands_len(Cmds0) > 20
|
||||
andalso
|
||||
concurrency(Cmds0) > 2),
|
||||
begin
|
||||
CmdsLen= commands_len(Cmds),
|
||||
Target = setup_target(Num, Seed, Verbose),
|
||||
{Seq, Par, Res0} = run_parallel_commands(?MODULE, Cmds),
|
||||
%% ?V("Seq=~w~n", [Seq]),
|
||||
%% ?V("Par=~w~n", [Par]),
|
||||
?V("==== Start post operations, stabilize and confirm results~n", []),
|
||||
{FinalRes, {Dataloss, Critical}} =
|
||||
case Res0 of
|
||||
ok ->
|
||||
Res1 = stabilize(CmdsLen, Target),
|
||||
{Res1, confirm_result(Target)};
|
||||
_ ->
|
||||
?V("Res0=~w~n", [Res0]),
|
||||
{Res0, {undefined, undefined}}
|
||||
end,
|
||||
_ = cleanup(Target),
|
||||
%% Process is leaking? This log line can be removed after fix.
|
||||
[?V("process_count=~w~n", [erlang:system_info(process_count)]) || Verbose],
|
||||
pretty_commands(
|
||||
?MODULE, Cmds, {Seq, Par, Res0},
|
||||
aggregate(with_title(cmds), command_names(Cmds),
|
||||
collect(with_title(length5), (CmdsLen div 5) * 5,
|
||||
collect(with_title(conc), concurrency(Cmds),
|
||||
{FinalRes, {Dataloss, Critical}} =:= {ok, {0, 0}})))
|
||||
)
|
||||
end)).
|
||||
|
||||
%% Initilization / setup
|
||||
|
||||
%% Fake initialization function for debugging in shell like:
|
||||
%% > eqc_gen:sample(eqc_statem:commands(machi_ap_repair_eqc)).
|
||||
%% but not so helpful.
|
||||
initial_state() ->
|
||||
#state{cr_count=3}.
|
||||
|
||||
initial_state(Num, Verbose) ->
|
||||
AllListE = all_list_extra(Num),
|
||||
FLUNames = [P#p_srvr.name || {P, _Dir} <- AllListE],
|
||||
MgrNames = [{Name, machi_flu_psup:make_mgr_supname(Name)} || Name <- FLUNames],
|
||||
#state{num=Num, verbose=Verbose,
|
||||
flu_names=FLUNames, mgr_names=MgrNames,
|
||||
cr_count=cr_count(Num)}.
|
||||
|
||||
setup_target(Num, Seed, Verbose) ->
|
||||
%% ?V("setup_target(Num=~w, Seed=~w~nn", [Num, Seed]),
|
||||
AllListE = all_list_extra(Num),
|
||||
FLUNames = [P#p_srvr.name || {P, _Dir} <- AllListE],
|
||||
MgrNames = [{Name, machi_flu_psup:make_mgr_supname(Name)} || Name <- FLUNames],
|
||||
Dict = orddict:from_list([{P#p_srvr.name, P} || {P, _Dir} <- AllListE]),
|
||||
|
||||
setup_chain(Seed, AllListE, FLUNames, MgrNames, Dict),
|
||||
_ = setup_cpool(AllListE, FLUNames, Dict),
|
||||
|
||||
Target = #target{flu_names=FLUNames, mgr_names=MgrNames,
|
||||
verbose=Verbose},
|
||||
%% Don't wait for complete chain. Even partialy completed, the chain
|
||||
%% should work fine. Right?
|
||||
wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames,
|
||||
20, Verbose),
|
||||
Target.
|
||||
|
||||
setup_chain(Seed, AllListE, FLUNames, MgrNames, Dict) ->
|
||||
ok = shutdown_hard(),
|
||||
[begin
|
||||
machi_test_util:clean_up_dir(Dir),
|
||||
filelib:ensure_dir(Dir ++ "/not-used")
|
||||
end || {_P, Dir} <- AllListE],
|
||||
[catch ets:delete(T) || T <- tabs()],
|
||||
|
||||
[ets:new(T, [set, public, named_table,
|
||||
{write_concurrency, true}, {read_concurrency, true}]) ||
|
||||
T <- tabs()],
|
||||
{ok, _} = application:ensure_all_started(machi),
|
||||
|
||||
SimSpec = {part_sim,
|
||||
{machi_partition_simulator, start_link, [{0,0,0}, 0, 100]},
|
||||
permanent, 500, worker, []},
|
||||
{ok, _PSimPid} = supervisor:start_child(machi_sup, SimSpec),
|
||||
ok = machi_partition_simulator:set_seed(Seed),
|
||||
_Partitions = machi_partition_simulator:get(FLUNames),
|
||||
|
||||
%% Start FLUs and setup the chain
|
||||
FLUOpts = [{use_partition_simulator, true},
|
||||
%% {private_write_verbose, true},
|
||||
{active_mode, false},
|
||||
{simulate_repair, false}],
|
||||
[{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, FLUOpts) ||
|
||||
{#p_srvr{name=Name, port=Port}, Dir} <- AllListE],
|
||||
[machi_chain_manager1:set_chain_members(MgrName, Dict) || {_, MgrName} <- MgrNames],
|
||||
ok.
|
||||
|
||||
setup_cpool(AllListE, FLUNames, Dict) ->
|
||||
Num = length(AllListE),
|
||||
FCList = [begin
|
||||
{ok, PCPid} = machi_proxy_flu1_client:start_link(P),
|
||||
{Name, PCPid}
|
||||
end || {_, #p_srvr{name=Name}=P} <- Dict],
|
||||
%% CR clients are pooled, each has "name" which is interpreted "From"
|
||||
%% side of simulated partition.
|
||||
SimSelfNames = lists:append(lists:duplicate(cr_count(Num), FLUNames)),
|
||||
CRList = [begin
|
||||
{ok, C} = machi_cr_client:start_link(
|
||||
[P || {_, P} <- Dict],
|
||||
[{use_partition_simulator, true},
|
||||
{simulator_self_name, SimSelfName},
|
||||
{simulator_members, FLUNames}]),
|
||||
{SimSelfName, C}
|
||||
end || SimSelfName <- SimSelfNames],
|
||||
catch ets:delete(cpool),
|
||||
ets:new(cpool, [set, protected, named_table, {read_concurrency, true}]),
|
||||
ets:insert(cpool, {fc_list, FCList}),
|
||||
ets:insert(cpool, {cr_list, CRList}),
|
||||
{CRList, FCList}.
|
||||
|
||||
fc_list() ->
|
||||
[{fc_list, FCList}] = ets:lookup(cpool, fc_list),
|
||||
FCList.
|
||||
|
||||
cr_list() ->
|
||||
[{cr_list, CRList}] = ets:lookup(cpool, cr_list),
|
||||
CRList.
|
||||
|
||||
%% Post run_commands
|
||||
|
||||
stabilize(0, _T) ->
|
||||
ok;
|
||||
stabilize(_CmdsLen, #target{flu_names=FLUNames, mgr_names=MgrNames,
|
||||
verbose=Verbose}) ->
|
||||
machi_partition_simulator:no_partitions(),
|
||||
true = wait_until_stable(chain_state_all_ok(FLUNames), FLUNames, MgrNames,
|
||||
100, Verbose),
|
||||
ok.
|
||||
|
||||
chain_state_all_ok(FLUNames) ->
|
||||
[{FLUName, {FLUNames, [], []}} || FLUName <- FLUNames].
|
||||
|
||||
confirm_result(_T) ->
|
||||
[{_, C} | _] = cr_list(),
|
||||
[{written, _Written}, {accpt, Accpt},
|
||||
{failed, Failed}, {critical, Critical}] = tab_counts(),
|
||||
{OK, Dataloss} = confirm_written(C),
|
||||
?V(" Written=~w, DATALOSS=~w, Acceptable=~w~n", [OK, Dataloss, Accpt]),
|
||||
?V(" Failed=~w, Critical=~w~n~n", [Failed, Critical]),
|
||||
DirBase = get_dir_base(),
|
||||
Suffix = dump_file_suffix(),
|
||||
case Failed of
|
||||
0 -> ok;
|
||||
_ ->
|
||||
DumpFailed = filename:join(DirBase, "dump-failed-" ++ Suffix),
|
||||
?V("Dump failed ETS tab to: ~s~n", [DumpFailed]),
|
||||
ets:tab2file(?FAILED_TAB, DumpFailed)
|
||||
end,
|
||||
case Critical of
|
||||
0 -> ok;
|
||||
_ ->
|
||||
DumpCritical = filename:join(DirBase, "dump-critical-" ++ Suffix),
|
||||
?V("Dump critical ETS tab to: ~w~n", [DumpCritical]),
|
||||
ets:tab2file(?CRITICAL_TAB, DumpCritical)
|
||||
end,
|
||||
{Dataloss, Critical}.
|
||||
|
||||
confirm_written(C) ->
|
||||
ets:foldl(
|
||||
fun({Key, Bin}, {OK, NG}) ->
|
||||
case assert_chunk(C, Key, Bin) of
|
||||
ok -> {OK+1, NG};
|
||||
{error, _} -> {OK, NG+1}
|
||||
end
|
||||
end, {0, 0}, ?WRITTEN_TAB).
|
||||
|
||||
assert_chunk(C, {Off, Len, FileName}=Key, Bin) ->
|
||||
%% TODO: This probably a bug, read_chunk respnds with filename of `string()' type
|
||||
%% TODO : Use CSum instead of binary (after disuccsion about CSum is calmed down?)
|
||||
NSInfo = undefined,
|
||||
case (catch machi_cr_client:read_chunk(C, NSInfo, FileName, Off, Len, undefined, sec(3))) of
|
||||
{ok, {[{FileName, Off, Bin, _}], []}} ->
|
||||
ok;
|
||||
{ok, Got} ->
|
||||
?V("read_chunk got different binary for Key=~p~n", [Key]),
|
||||
?V(" Expected: ~p~n", [{[{FileName, Off, Bin, <<"CSum-NYI">>}], []}]),
|
||||
?V(" Got: ~p~n", [Got]),
|
||||
{error, different_binary};
|
||||
{error, Reason} ->
|
||||
?V("read_chunk error for Key=~p: ~p~n", [Key, Reason]),
|
||||
{error, Reason};
|
||||
Other ->
|
||||
?V("read_chunk other error for Key=~p: ~p~n", [Key, Other]),
|
||||
{error, Other}
|
||||
end.
|
||||
|
||||
cleanup(_Target) ->
|
||||
[begin unlink(FC), catch exit(FC, kill) end || {_, FC} <- fc_list()],
|
||||
[begin unlink(CR), catch exit(CR, kill) end || {_, CR} <- cr_list()],
|
||||
_ = shutdown_hard().
|
||||
|
||||
%% Internal misc utilities
|
||||
|
||||
eqc_verbose() ->
|
||||
os:getenv("EQC_VERBOSE") =:= "true".
|
||||
|
||||
eqc_timeout(Default) ->
|
||||
PropTimeout = case os:getenv("EQC_TIME") of
|
||||
false -> Default;
|
||||
V -> list_to_integer(V)
|
||||
end,
|
||||
{PropTimeout, PropTimeout * 300}.
|
||||
|
||||
get_port_dir_base() ->
|
||||
I = case os:getenv("EQC_BASE_PORT") of
|
||||
false -> 0;
|
||||
V -> list_to_integer(V)
|
||||
end,
|
||||
D = get_dir_base(),
|
||||
{7400 + (I * 100), D ++ "/" ++ integer_to_list(I)}.
|
||||
|
||||
get_dir_base() ->
|
||||
case os:getenv("EQC_BASE_DIR") of
|
||||
false -> "./eqc";
|
||||
DD -> DD
|
||||
end.
|
||||
|
||||
shutdown_hard() ->
|
||||
_STOP = application:stop(machi),
|
||||
timer:sleep(100).
|
||||
|
||||
tick(#state{flu_names=FLUNames, mgr_names=MgrNames,
|
||||
verbose=Verbose}) ->
|
||||
tick(FLUNames, MgrNames, Verbose).
|
||||
|
||||
tick(FLUNames, MgrNames, Verbose) ->
|
||||
tick(FLUNames, MgrNames, 2, 100, Verbose).
|
||||
|
||||
tick(FLUNames, MgrNames, Iter, SleepMax, Verbose) ->
|
||||
TickFun = tick_fun(FLUNames, MgrNames, self()),
|
||||
TickFun(Iter, 0, SleepMax),
|
||||
FCList = fc_list(),
|
||||
[?V("## Chain state after tick()=~w~n", [chain_state(FCList)]) || Verbose].
|
||||
|
||||
tick_fun(FLUNames, MgrNames, Parent) ->
|
||||
fun(Iters, SleepMin, SleepMax) ->
|
||||
%% ?V("^", []),
|
||||
Trigger =
|
||||
fun(FLUName, MgrName) ->
|
||||
random:seed(now()),
|
||||
[begin
|
||||
erlang:yield(),
|
||||
SleepMaxRand = random:uniform(SleepMax + 1),
|
||||
%% io:format(user, "{t}", []),
|
||||
Elapsed = machi_chain_manager1:sleep_ranked_order(
|
||||
SleepMin, SleepMaxRand,
|
||||
FLUName, FLUNames),
|
||||
MgrName ! tick_check_environment,
|
||||
%% Be more unfair by not sleeping here.
|
||||
timer:sleep(max(SleepMax - Elapsed, 1)),
|
||||
ok
|
||||
end || _ <- lists:seq(1, Iters)],
|
||||
Parent ! {done, self()}
|
||||
end,
|
||||
Pids = [{spawn(fun() -> Trigger(FLUName, MgrName) end), FLUName} ||
|
||||
{FLUName, MgrName} <- MgrNames ],
|
||||
[receive
|
||||
{done, ThePid} ->
|
||||
ok
|
||||
after 120*1000 ->
|
||||
exit({icky_timeout, M_name})
|
||||
end || {ThePid, M_name} <- Pids]
|
||||
end.
|
||||
|
||||
wait_until_stable(ExpectedChainState, FLUNames, MgrNames, Verbose) ->
|
||||
wait_until_stable(ExpectedChainState, FLUNames, MgrNames, 20, Verbose).
|
||||
|
||||
wait_until_stable(ExpectedChainState, FLUNames, MgrNames, Retries, Verbose) ->
|
||||
TickFun = tick_fun(FLUNames, MgrNames, self()),
|
||||
FCList = fc_list(),
|
||||
wait_until_stable1(ExpectedChainState, TickFun, FCList, Retries, Verbose).
|
||||
|
||||
wait_until_stable1(ExpectedChainState, _TickFun, FCList, 0, _Verbose) ->
|
||||
?V(" [ERROR] _ExpectedChainState ~p\n", [ExpectedChainState]),
|
||||
?V(" [ERROR] wait_until_stable failed.... : ~p~n", [chain_state(FCList)]),
|
||||
?V(" [ERROR] norm.... : ~p~n", [normalize_chain_state(chain_state(FCList))]),
|
||||
false;
|
||||
wait_until_stable1(ExpectedChainState, TickFun, FCList, Reties, Verbose) ->
|
||||
[TickFun(3, 0, 100) || _ <- lists:seq(1, 3)],
|
||||
Normalized = normalize_chain_state(chain_state(FCList)),
|
||||
case Normalized of
|
||||
ExpectedChainState ->
|
||||
[?V(" Got stable chain: ~w~n", [chain_state(FCList)]) || Verbose],
|
||||
true;
|
||||
_ ->
|
||||
[?V(" NOT YET stable chain: ~w~n", [chain_state(FCList)]) || Verbose],
|
||||
wait_until_stable1(ExpectedChainState, TickFun, FCList, Reties-1, Verbose)
|
||||
end.
|
||||
|
||||
normalize_chain_state(ChainState) ->
|
||||
lists:usort([{FLUName,
|
||||
{lists:usort(UPI), lists:usort(Repairing), lists:usort(Down)}} ||
|
||||
{FLUName, {_EpochNo, UPI, Repairing, Down}} <- ChainState]).
|
||||
|
||||
chain_state(FCList) ->
|
||||
lists:usort(
|
||||
[case (catch machi_proxy_flu1_client:read_latest_projection(C, private, sec(5))) of
|
||||
{ok, #projection_v1{epoch_number=EpochNo, upi=UPI,
|
||||
repairing=Repairing, down=Down}} ->
|
||||
{FLUName, {EpochNo, UPI, Repairing, Down}};
|
||||
Other ->
|
||||
{FLUName, Other}
|
||||
end || {FLUName, C} <- FCList]).
|
||||
|
||||
tabs() -> [?WRITTEN_TAB, ?ACCPT_TAB, ?FAILED_TAB, ?CRITICAL_TAB].
|
||||
|
||||
tab_counts() ->
|
||||
[{T, ets:info(T, size)} || T <- tabs()].
|
||||
|
||||
sec(Sec) ->
|
||||
timer:seconds(Sec).
|
||||
|
||||
commands_len({SeqCmds, ParCmdsList} = _Cmds) ->
|
||||
lists:sum([length(SeqCmds) | [length(P) || P <- ParCmdsList]]);
|
||||
commands_len(Cmds) ->
|
||||
length(Cmds).
|
||||
|
||||
concurrency({_SeqCmds, ParCmdsList} = _Cmds) -> length(ParCmdsList);
|
||||
concurrency(_) -> 1.
|
||||
|
||||
dump_file_suffix() ->
|
||||
{{Year, Month, Day}, {Hour, Min, Sec}} = calendar:local_time(),
|
||||
lists:flatten(
|
||||
io_lib:format("~4.10.0B-~2.10.0B-~2.10.0BT~2.10.0B:~2.10.0B:~2.10.0B.000Z",
|
||||
[Year, Month, Day, Hour, Min, Sec])).
|
||||
|
||||
-endif. % EQC
|
||||
-endif. % TEST
|
|
@ -45,14 +45,18 @@
|
|||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
help() ->
|
||||
io:format("~s\n", [short_doc()]).
|
||||
|
||||
short_doc() ->
|
||||
"
|
||||
A visualization of the convergence behavior of the chain self-management
|
||||
algorithm for Machi.
|
||||
1. Set up 4 FLUs and chain manager pairs.
|
||||
|
||||
1. Set up some server and chain manager pairs.
|
||||
2. Create a number of different network partition scenarios, where
|
||||
(simulated) partitions may be symmetric or asymmetric. Then halt changing
|
||||
the partitions and keep the simulated network stable and broken.
|
||||
(simulated) partitions may be symmetric or asymmetric. Then stop changing
|
||||
the partitions and keep the simulated network stable (and perhaps broken).
|
||||
3. Run a number of iterations of the algorithm in parallel by poking each
|
||||
of the manager processes on a random'ish basis.
|
||||
4. Afterward, fetch the chain transition changes made by each FLU and
|
||||
|
@ -61,73 +65,65 @@ algorithm for Machi.
|
|||
During the iteration periods, the following is a cheatsheet for the output.
|
||||
See the internal source for interpreting the rest of the output.
|
||||
|
||||
'Let loose the dogs of war!' Network instability
|
||||
'SET partitions = ' Network stability (but broken)
|
||||
'x uses:' The FLU x has made an internal state transition. The rest of
|
||||
the line is a dump of internal state.
|
||||
'{t}' This is a tick event which triggers one of the manager processes
|
||||
to evaluate its environment and perhaps make a state transition.
|
||||
'SET partitions = '
|
||||
|
||||
A long chain of '{t}{t}{t}{t}' means that the chain state has settled
|
||||
to a stable configuration, which is the goal of the algorithm.
|
||||
Press control-c to interrupt....".
|
||||
A pair-wise list of actors which cannot send messages. The
|
||||
list is uni-directional. If there are three servers (a,b,c),
|
||||
and if the partitions list is '[{a,b},{b,c}]' then all
|
||||
messages from a->b and b->c will be dropped, but any other
|
||||
sender->recipient messages will be delivered successfully.
|
||||
|
||||
long_doc() ->
|
||||
"
|
||||
'Let loose the dogs of war!'
|
||||
'x uses:'
|
||||
|
||||
The simulated network is very unstable for a few seconds.
|
||||
The FLU x has made an internal state transition and is using
|
||||
this epoch's projection as operating chain configuration. The
|
||||
rest of the line is a summary of the projection.
|
||||
|
||||
'x uses'
|
||||
'CONFIRM epoch {N}'
|
||||
|
||||
After a single iteration, server x has determined that the chain
|
||||
should be defined by the upi, repair, and down list in this record.
|
||||
If all participants reach the same conclusion at the same epoch
|
||||
number (and checksum, see next item below), then the chain is
|
||||
stable, fully configured, and can provide full service.
|
||||
This message confirms that all of the servers listed in the
|
||||
UPI and repairing lists of the projection at epoch {N} have
|
||||
agreed to use this projection because they all have written
|
||||
this projection to their respective private projection stores.
|
||||
The chain is now usable by/available to all clients.
|
||||
|
||||
'epoch,E'
|
||||
'Sweet, private projections are stable'
|
||||
|
||||
The epoch number for this decision is E. The checksum of the full
|
||||
record is not shown. For purposes of the protocol, a server will
|
||||
'wedge' itself and refuse service (until a new config is chosen)
|
||||
whenever: a). it sees a bigger epoch number mentioned somewhere, or
|
||||
b). it sees the same epoch number but a different checksum. In case
|
||||
of b), there was a network partition that has healed, and both sides
|
||||
had chosen to operate with an identical epoch number but different
|
||||
chain configs.
|
||||
This report announces that this iteration of the test cycle
|
||||
has passed successfully. The report that follows briefly
|
||||
summarizes the latest private projection used by each
|
||||
participating server. For example, when in strong consistency
|
||||
mode with 'a' as a witness and 'b' and 'c' as real servers:
|
||||
|
||||
'upi', 'repair', and 'down'
|
||||
%% Legend:
|
||||
%% server name, epoch ID, UPI list, repairing list, down list, ...
|
||||
%% ... witness list, 'false' (a constant value)
|
||||
|
||||
Members in the chain that are fully in sync and thus preserving the
|
||||
Update Propagation Invariant, up but under repair (simulated), and
|
||||
down, respectively.
|
||||
[{a,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}},
|
||||
{b,{{1116,<<23,143,246,55>>},[a,b],[],[c],[a],false}}]
|
||||
|
||||
'ps,[some list]'
|
||||
Both servers 'a' and 'b' agree on epoch 1116 with epoch ID
|
||||
{1116,<<23,143,246,55>>} where UPI=[a,b], repairing=[],
|
||||
down=[c], and witnesses=[a].
|
||||
|
||||
The list of asymmetric network partitions. {a,b} means that a
|
||||
cannot send to b, but b can send to a.
|
||||
Server 'c' is not shown because 'c' has wedged itself OOS (out
|
||||
of service) by configuring a chain length of zero.
|
||||
|
||||
This partition list is recorded for debugging purposes but is *not*
|
||||
used by the algorithm. The algorithm only 'feels' its effects via
|
||||
simulated timeout whenever there's a partition in one of the
|
||||
messaging directions.
|
||||
If no servers are listed in the report (i.e. only '[]' is
|
||||
displayed), then all servers have wedged themselves OOS, and
|
||||
the chain is unavailable.
|
||||
|
||||
'nodes_up,[list]'
|
||||
'DoIt,'
|
||||
|
||||
The best guess right now of which ndoes are up, relative to the
|
||||
author node, specified by '{author,X}'
|
||||
This marks a group of tick events which trigger the manager
|
||||
processes to evaluate their environment and perhaps make a
|
||||
state transition.
|
||||
|
||||
'SET partitions = [some list]'
|
||||
A long chain of 'DoIt,DoIt,DoIt,' means that the chain state has
|
||||
(probably) settled to a stable configuration, which is the goal of the
|
||||
algorithm.
|
||||
|
||||
All subsequent iterations should have a stable list of partitions,
|
||||
i.e. the 'ps' list described should be stable.
|
||||
|
||||
'{FLAP: x flaps n}!'
|
||||
|
||||
Server x has detected that it's flapping/oscillating after iteration
|
||||
n of a naive/1st draft detection algorithm.
|
||||
".
|
||||
Press control-c to interrupt the test....".
|
||||
|
||||
%% ' silly Emacs syntax highlighting....
|
||||
|
||||
|
@ -138,6 +134,7 @@ long_doc() ->
|
|||
%% convergence_demo_testfun(3).
|
||||
|
||||
-define(DEFAULT_MGR_OPTS, [{private_write_verbose, false},
|
||||
{private_write_verbose_confirm, true},
|
||||
{active_mode,false},
|
||||
{use_partition_simulator, true}]).
|
||||
|
||||
|
@ -154,7 +151,8 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
|
|||
%% Faster test startup, commented: io:format(user, short_doc(), []),
|
||||
%% Faster test startup, commented: timer:sleep(3000),
|
||||
|
||||
application:start(sasl),
|
||||
Apps = [sasl, ranch],
|
||||
[application:start(App) || App <- Apps],
|
||||
|
||||
MgrOpts = MgrOpts0 ++ ?DEFAULT_MGR_OPTS,
|
||||
TcpPort = proplists:get_value(port_base, MgrOpts, 62877),
|
||||
|
@ -191,15 +189,18 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
|
|||
end || #p_srvr{name=Name}=P <- Ps],
|
||||
MembersDict = machi_projection:make_members_dict(Ps),
|
||||
Witnesses = proplists:get_value(witnesses, MgrOpts, []),
|
||||
CMode = case {Witnesses, proplists:get_value(consistency_mode, MgrOpts,
|
||||
ap_mode)} of
|
||||
{[_|_], _} -> cp_mode;
|
||||
{_, cp_mode} -> cp_mode;
|
||||
{_, ap_mode} -> ap_mode
|
||||
end,
|
||||
MgrNamez = [begin
|
||||
MgrName = machi_flu_psup:make_mgr_supname(Name),
|
||||
ok = ?MGR:set_chain_members(MgrName,MembersDict,Witnesses),
|
||||
ok = ?MGR:set_chain_members(MgrName, ch_demo, 0, CMode,
|
||||
MembersDict,Witnesses),
|
||||
{Name, MgrName}
|
||||
end || #p_srvr{name=Name} <- Ps],
|
||||
CpApMode = case Witnesses /= [] of
|
||||
true -> cp_mode;
|
||||
false -> ap_mode
|
||||
end,
|
||||
|
||||
try
|
||||
[{_, Ma}|_] = MgrNamez,
|
||||
|
@ -295,7 +296,7 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
|
|||
private_projections_are_stable(Namez, DoIt)
|
||||
end, false, lists:seq(0, MaxIters)),
|
||||
io:format(user, "\n~s Sweet, private projections are stable\n", [machi_util:pretty_time()]),
|
||||
io:format(user, "\t~P\n", [get(stable), 14]),
|
||||
io:format(user, "\t~P\n", [get(stable), 24]),
|
||||
io:format(user, "Rolling sanity check ... ", []),
|
||||
PrivProjs = [{Name, begin
|
||||
{ok, Ps8} = ?FLU_PC:get_all_projections(
|
||||
|
@ -307,9 +308,9 @@ convergence_demo_testfun(NumFLUs, MgrOpts0) ->
|
|||
[{FLU, true} = {FLU, ?MGR:projection_transitions_are_sane_retrospective(Psx, FLU)} ||
|
||||
{FLU, Psx} <- PrivProjs]
|
||||
catch
|
||||
_Err:_What when CpApMode == cp_mode ->
|
||||
_Err:_What when CMode == cp_mode ->
|
||||
io:format(user, "none proj skip detected, TODO? ", []);
|
||||
_Err:_What when CpApMode == ap_mode ->
|
||||
_Err:_What when CMode == ap_mode ->
|
||||
io:format(user, "PrivProjs ~p\n", [PrivProjs]),
|
||||
exit({line, ?LINE, _Err, _What})
|
||||
end,
|
||||
|
@ -375,9 +376,9 @@ timer:sleep(1234),
|
|||
{FLU, Psx} <- PrivProjs],
|
||||
io:format(user, "\nAll sanity checks pass, hooray!\n", [])
|
||||
catch
|
||||
_Err:_What when CpApMode == cp_mode ->
|
||||
_Err:_What when CMode == cp_mode ->
|
||||
io:format(user, "none proj skip detected, TODO? ", []);
|
||||
_Err:_What when CpApMode == ap_mode ->
|
||||
_Err:_What when CMode == ap_mode ->
|
||||
io:format(user, "Report ~p\n", [Report]),
|
||||
io:format(user, "PrivProjs ~p\n", [PrivProjs]),
|
||||
exit({line, ?LINE, _Err, _What})
|
||||
|
@ -394,7 +395,8 @@ timer:sleep(1234),
|
|||
exit(SupPid, normal),
|
||||
ok = machi_partition_simulator:stop(),
|
||||
[ok = ?FLU_PC:quit(PPid) || {_, PPid} <- Namez],
|
||||
machi_util:wait_for_death(SupPid, 100)
|
||||
machi_util:wait_for_death(SupPid, 100),
|
||||
[application:start(App) || App <- lists:reverse(Apps)]
|
||||
end.
|
||||
|
||||
%% Many of the static partition lists below have been problematic at one
|
||||
|
@ -719,7 +721,7 @@ private_projections_are_stable(Namez, PollFunc) ->
|
|||
true
|
||||
end,
|
||||
|
||||
io:format(user, "\nPriv1 ~p\nPriv2 ~p\n1==2 ~w ap_disjoint ~w u_all_peers ~w cp_mode_agree ~w\n", [lists:sort(Private1), lists:sort(Private2), Private1 == Private2, AP_mode_disjoint_test_p, Unanimous_with_all_peers_p, CP_mode_agree_test_p]),
|
||||
%% io:format(user, "\nPriv1 ~p\nPriv2 ~p\n1==2 ~w ap_disjoint ~w u_all_peers ~w cp_mode_agree ~w\n", [lists:sort(Private1), lists:sort(Private2), Private1 == Private2, AP_mode_disjoint_test_p, Unanimous_with_all_peers_p, CP_mode_agree_test_p]),
|
||||
Private1 == Private2 andalso
|
||||
AP_mode_disjoint_test_p andalso
|
||||
(
|
||||
|
|
|
@ -273,78 +273,80 @@ make_prop_ets() ->
|
|||
|
||||
-endif. % EQC
|
||||
|
||||
make_advance_fun(FitList, FLUList, MgrList, Num) ->
|
||||
fun() ->
|
||||
[begin
|
||||
[catch machi_fitness:trigger_early_adjustment(Fit, Tgt) ||
|
||||
Fit <- FitList,
|
||||
Tgt <- FLUList ],
|
||||
[catch ?MGR:trigger_react_to_env(Mgr) || Mgr <- MgrList],
|
||||
ok
|
||||
end || _ <- lists:seq(1, Num)]
|
||||
end.
|
||||
|
||||
smoke0_test() ->
|
||||
{ok, _} = machi_partition_simulator:start_link({1,2,3}, 50, 50),
|
||||
Host = "localhost",
|
||||
TcpPort = 6623,
|
||||
{ok, FLUa} = machi_flu1:start_link([{a,TcpPort,"./data.a"}]),
|
||||
Pa = #p_srvr{name=a, address=Host, port=TcpPort},
|
||||
Members_Dict = machi_projection:make_members_dict([Pa]),
|
||||
%% Egadz, more racing on startup, yay. TODO fix.
|
||||
timer:sleep(1),
|
||||
{[Pa], [M0], _Dirs} = machi_test_util:start_flu_packages(
|
||||
1, TcpPort, "./data.", []),
|
||||
{ok, FLUaP} = ?FLU_PC:start_link(Pa),
|
||||
{ok, M0} = ?MGR:start_link(a, Members_Dict, [{active_mode, false}]),
|
||||
_SockA = machi_util:connect(Host, TcpPort),
|
||||
try
|
||||
pong = ?MGR:ping(M0)
|
||||
after
|
||||
ok = ?MGR:stop(M0),
|
||||
ok = machi_flu1:stop(FLUa),
|
||||
ok = ?FLU_PC:quit(FLUaP),
|
||||
ok = machi_partition_simulator:stop()
|
||||
machi_test_util:stop_flu_packages()
|
||||
end.
|
||||
|
||||
smoke1_test() ->
|
||||
machi_partition_simulator:start_link({1,2,3}, 100, 0),
|
||||
TcpPort = 62777,
|
||||
FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}, {c,TcpPort+2,"./data.c"}],
|
||||
P_s = [#p_srvr{name=Name, address="localhost", port=Port} ||
|
||||
{Name,Port,_Dir} <- FluInfo],
|
||||
smoke1_test_() ->
|
||||
{timeout, 1*60, fun() -> smoke1_test2() end}.
|
||||
|
||||
[machi_flu1_test:clean_up_data_dir(Dir) || {_,_,Dir} <- FluInfo],
|
||||
FLUs = [element(2, machi_flu1:start_link([{Name,Port,Dir}])) ||
|
||||
{Name,Port,Dir} <- FluInfo],
|
||||
MembersDict = machi_projection:make_members_dict(P_s),
|
||||
{ok, M0} = ?MGR:start_link(a, MembersDict, [{active_mode,false}]),
|
||||
smoke1_test2() ->
|
||||
TcpPort = 62777,
|
||||
MgrOpts = [{active_mode,false}],
|
||||
try
|
||||
{ok, P1} = ?MGR:test_calc_projection(M0, false),
|
||||
{Ps, MgrNames, _Dirs} = machi_test_util:start_flu_packages(
|
||||
3, TcpPort, "./data.", MgrOpts),
|
||||
MembersDict = machi_projection:make_members_dict(Ps),
|
||||
[machi_chain_manager1:set_chain_members(M, MembersDict) || M <- MgrNames],
|
||||
Ma = hd(MgrNames),
|
||||
|
||||
{ok, P1} = ?MGR:test_calc_projection(Ma, false),
|
||||
% DERP! Check for race with manager's proxy vs. proj listener
|
||||
case ?MGR:test_read_latest_public_projection(M0, false) of
|
||||
{error, partition} -> timer:sleep(500);
|
||||
_ -> ok
|
||||
end,
|
||||
{remote_write_results,{true,[{c,ok},{b,ok},{a,ok}]}} =
|
||||
?MGR:test_write_public_projection(M0, P1),
|
||||
{unanimous, P1, Extra1} = ?MGR:test_read_latest_public_projection(M0, false),
|
||||
ok = lists:foldl(
|
||||
fun(_, {_,{true,[{c,ok},{b,ok},{a,ok}]}}) ->
|
||||
ok; % Short-circuit remaining attempts
|
||||
(_, ok) ->
|
||||
ok; % Skip remaining!
|
||||
(_, _Else) ->
|
||||
timer:sleep(10),
|
||||
?MGR:test_write_public_projection(Ma, P1)
|
||||
end, not_ok, lists:seq(1, 1000)),
|
||||
%% Writing the exact same projection multiple times returns ok:
|
||||
%% no change!
|
||||
{_,{true,[{c,ok},{b,ok},{a,ok}]}} = ?MGR:test_write_public_projection(Ma, P1),
|
||||
{unanimous, P1, Extra1} = ?MGR:test_read_latest_public_projection(Ma, false),
|
||||
|
||||
ok
|
||||
after
|
||||
ok = ?MGR:stop(M0),
|
||||
[ok = machi_flu1:stop(X) || X <- FLUs],
|
||||
ok = machi_partition_simulator:stop()
|
||||
machi_test_util:stop_flu_packages()
|
||||
end.
|
||||
|
||||
nonunanimous_setup_and_fix_test() ->
|
||||
machi_partition_simulator:start_link({1,2,3}, 100, 0),
|
||||
nonunanimous_setup_and_fix_test_() ->
|
||||
os:cmd("rm -f /tmp/moomoo.*"),
|
||||
{timeout, 1*60, fun() -> nonunanimous_setup_and_fix_test2() end}.
|
||||
|
||||
nonunanimous_setup_and_fix_test2() ->
|
||||
TcpPort = 62877,
|
||||
FluInfo = [{a,TcpPort+0,"./data.a"}, {b,TcpPort+1,"./data.b"}],
|
||||
P_s = [#p_srvr{name=Name, address="localhost", port=Port} ||
|
||||
{Name,Port,_Dir} <- FluInfo],
|
||||
|
||||
[machi_flu1_test:clean_up_data_dir(Dir) || {_,_,Dir} <- FluInfo],
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
Opts = [{active_mode, false}],
|
||||
%% {ok, Mb} = ?MGR:start_link(b, MembersDict, [{active_mode, false}]++XX),
|
||||
[{ok,_}=machi_flu_psup:start_flu_package(Name, Port, Dir, Opts) ||
|
||||
{Name,Port,Dir} <- FluInfo],
|
||||
FLUs = [machi_flu_psup:make_flu_regname(Name) ||
|
||||
{Name,_Port,_Dir} <- FluInfo],
|
||||
[Proxy_a, Proxy_b] = Proxies =
|
||||
[element(2,?FLU_PC:start_link(P)) || P <- P_s],
|
||||
MembersDict = machi_projection:make_members_dict(P_s),
|
||||
[Ma,Mb] = [a_chmgr, b_chmgr],
|
||||
ok = machi_chain_manager1:set_chain_members(Ma, MembersDict, []),
|
||||
ok = machi_chain_manager1:set_chain_members(Mb, MembersDict, []),
|
||||
MgrOpts = [{active_mode,false}],
|
||||
{Ps, [Ma,Mb,Mc], Dirs} = machi_test_util:start_flu_packages(
|
||||
3, TcpPort, "./data.", MgrOpts),
|
||||
MembersDict = machi_projection:make_members_dict(Ps),
|
||||
ChainName = my_little_chain,
|
||||
[machi_chain_manager1:set_chain_members(M, ChainName, 0, ap_mode,
|
||||
MembersDict, []) || M <- [Ma, Mb]],
|
||||
|
||||
[Proxy_a, Proxy_b, Proxy_c] = Proxies =
|
||||
[element(2, ?FLU_PC:start_link(P)) || P <- Ps],
|
||||
|
||||
try
|
||||
{ok, P1} = ?MGR:test_calc_projection(Ma, false),
|
||||
|
||||
|
@ -381,16 +383,114 @@ nonunanimous_setup_and_fix_test() ->
|
|||
{ok, P2pb} = ?FLU_PC:read_latest_projection(Proxy_b, private),
|
||||
P2 = P2pb#projection_v1{dbg2=[]},
|
||||
|
||||
%% Pspam = machi_projection:update_checksum(
|
||||
%% P1b#projection_v1{epoch_number=?SPAM_PROJ_EPOCH,
|
||||
%% dbg=[hello_spam]}),
|
||||
%% ok = ?FLU_PC:write_projection(Proxy_b, public, Pspam),
|
||||
Mgrs = [a_chmgr, b_chmgr, c_chmgr],
|
||||
Advance = make_advance_fun([a_fitness,b_fitness,c_fitness],
|
||||
[a,b,c],
|
||||
Mgrs,
|
||||
3),
|
||||
Advance(),
|
||||
{_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Ma),
|
||||
{_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Mb),
|
||||
{_, _, TheEpoch_3} = ?MGR:trigger_react_to_env(Mc),
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Remove 'a' from the chain.\n", []),
|
||||
|
||||
MembersDict4 = machi_projection:make_members_dict(tl(Ps)),
|
||||
ok = machi_chain_manager1:set_chain_members(
|
||||
Mb, ChainName, TheEpoch_3, ap_mode, MembersDict4, []),
|
||||
|
||||
Advance(),
|
||||
{ok, {true, _,_,_}} = ?FLU_PC:wedge_status(Proxy_a),
|
||||
{_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Mb),
|
||||
{_, _, TheEpoch_4} = ?MGR:trigger_react_to_env(Mc),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Add a to the chain again (a is running).\n", []),
|
||||
|
||||
MembersDict5 = machi_projection:make_members_dict(Ps),
|
||||
ok = machi_chain_manager1:set_chain_members(
|
||||
Mb, ChainName, TheEpoch_4, ap_mode, MembersDict5, []),
|
||||
|
||||
Advance(),
|
||||
{_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Ma),
|
||||
{_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Mb),
|
||||
{_, _, TheEpoch_5} = ?MGR:trigger_react_to_env(Mc),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[a]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Stop a while a chain member, advance b&c.\n", []),
|
||||
|
||||
ok = machi_flu_psup:stop_flu_package(a),
|
||||
Advance(),
|
||||
{_, _, TheEpoch_6} = ?MGR:trigger_react_to_env(Mb),
|
||||
{_, _, TheEpoch_6} = ?MGR:trigger_react_to_env(Mc),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Remove 'a' from the chain.\n", []),
|
||||
|
||||
MembersDict7 = machi_projection:make_members_dict(tl(Ps)),
|
||||
ok = machi_chain_manager1:set_chain_members(
|
||||
Mb, ChainName, TheEpoch_6, ap_mode, MembersDict7, []),
|
||||
|
||||
Advance(),
|
||||
{_, _, TheEpoch_7} = ?MGR:trigger_react_to_env(Mb),
|
||||
{_, _, TheEpoch_7} = ?MGR:trigger_react_to_env(Mc),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Start a, advance.\n", []),
|
||||
|
||||
Opts = [{active_mode, false}, {initial_wedged, true}],
|
||||
#p_srvr{name=NameA} = hd(Ps),
|
||||
{ok,_}=machi_flu_psup:start_flu_package(NameA, TcpPort+1, hd(Dirs), Opts),
|
||||
Advance(),
|
||||
{ok, {true, _,_,_}} = ?FLU_PC:wedge_status(Proxy_a),
|
||||
{ok, {false, EpochID_8,_,_}} = ?FLU_PC:wedge_status(Proxy_b),
|
||||
{ok, {false, EpochID_8,_,_}} = ?FLU_PC:wedge_status(Proxy_c),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- tl(Proxies)],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Stop a, delete a's data, leave it stopped\n", []),
|
||||
|
||||
ok = machi_flu_psup:stop_flu_package(a),
|
||||
Advance(),
|
||||
machi_flu1_test:clean_up_data_dir(hd(Dirs)),
|
||||
{ok, {false, _,_,_}} = ?FLU_PC:wedge_status(Proxy_b),
|
||||
{ok, {false, _,_,_}} = ?FLU_PC:wedge_status(Proxy_c),
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Add a to the chain again (a is stopped).\n", []),
|
||||
|
||||
MembersDict9 = machi_projection:make_members_dict(Ps),
|
||||
{_, _, TheEpoch_9} = ?MGR:trigger_react_to_env(Mb),
|
||||
ok = machi_chain_manager1:set_chain_members(
|
||||
Mb, ChainName, TheEpoch_9, ap_mode, MembersDict9, []),
|
||||
Advance(),
|
||||
{_, _, TheEpoch_9b} = ?MGR:trigger_react_to_env(Mb),
|
||||
true = (TheEpoch_9b > TheEpoch_9),
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("STEP: Start a, and it joins like it ought to\n", []),
|
||||
|
||||
{ok,_}=machi_flu_psup:start_flu_package(NameA, TcpPort+1, hd(Dirs), Opts),
|
||||
Advance(),
|
||||
{ok, {false, {TheEpoch10,_},_,_}} = ?FLU_PC:wedge_status(Proxy_a),
|
||||
{ok, {false, {TheEpoch10,_},_,_}} = ?FLU_PC:wedge_status(Proxy_b),
|
||||
{ok, {false, {TheEpoch10,_},_,_}} = ?FLU_PC:wedge_status(Proxy_c),
|
||||
[{ok, #projection_v1{upi=[b,c], repairing=[a]}} =
|
||||
?FLU_PC:read_latest_projection(Pxy, private) || Pxy <- Proxies],
|
||||
ok
|
||||
after
|
||||
exit(SupPid, normal),
|
||||
[ok = ?FLU_PC:quit(X) || X <- Proxies],
|
||||
ok = machi_partition_simulator:stop()
|
||||
machi_test_util:stop_flu_packages()
|
||||
end.
|
||||
|
||||
unanimous_report_test() ->
|
||||
|
|
68
test/machi_cinfo_test.erl
Normal file
68
test/machi_cinfo_test.erl
Normal file
|
@ -0,0 +1,68 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_cinfo_test).
|
||||
|
||||
-ifdef(TEST).
|
||||
-ifndef(PULSE).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
%% smoke_test() will try just dump cluster_info and call each functions
|
||||
|
||||
smoke_test_() ->
|
||||
{setup,
|
||||
fun setup/0,
|
||||
fun cleanup/1,
|
||||
[
|
||||
fun() -> machi_cinfo:public_projection(a) end,
|
||||
fun() -> machi_cinfo:private_projection(a) end,
|
||||
fun() -> machi_cinfo:fitness(a) end,
|
||||
fun() -> machi_cinfo:chain_manager(a) end,
|
||||
fun() -> machi_cinfo:flu1(a) end,
|
||||
fun() -> machi_cinfo:dump() end
|
||||
]}.
|
||||
|
||||
setup() ->
|
||||
machi_cinfo:register(),
|
||||
Ps = [{a,#p_srvr{name=a, address="localhost", port=5555, props="./data.a"}},
|
||||
{b,#p_srvr{name=b, address="localhost", port=5556, props="./data.b"}},
|
||||
{c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}}
|
||||
],
|
||||
[os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps],
|
||||
{ok, SupPid} = machi_sup:start_link(),
|
||||
%% Only run a, don't run b & c so we have 100% failures talking to them
|
||||
[begin
|
||||
#p_srvr{name=Name, port=Port, props=Dir} = P,
|
||||
{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, [])
|
||||
end || {_,P} <- [hd(Ps)]],
|
||||
machi_chain_manager1:set_chain_members(a_chmgr, orddict:from_list(Ps)),
|
||||
{SupPid, Ps}.
|
||||
|
||||
cleanup({SupPid, Ps}) ->
|
||||
exit(SupPid, normal),
|
||||
[os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps],
|
||||
machi_util:wait_for_death(SupPid, 100),
|
||||
ok.
|
||||
|
||||
-endif. % !PULSE
|
||||
-endif. % TEST
|
|
@ -32,6 +32,7 @@ smoke_test_() -> {timeout, 1*60, fun() -> smoke_test2() end}.
|
|||
|
||||
setup_smoke_test(Host, PortBase, Os, Witness_list) ->
|
||||
os:cmd("rm -rf ./data.a ./data.b ./data.c"),
|
||||
{ok, _} = machi_util:wait_for_life(machi_flu_sup, 100),
|
||||
|
||||
F = fun(X) -> case lists:member(X, Witness_list) of
|
||||
true ->
|
||||
|
@ -57,9 +58,15 @@ setup_smoke_test(Host, PortBase, Os, Witness_list) ->
|
|||
%% 4. Wait until all others are using epoch id from #3.
|
||||
%%
|
||||
%% Damn, this is a pain to make 100% deterministic, bleh.
|
||||
ok = machi_chain_manager1:set_chain_members(a_chmgr, D, Witness_list),
|
||||
ok = machi_chain_manager1:set_chain_members(b_chmgr, D, Witness_list),
|
||||
ok = machi_chain_manager1:set_chain_members(c_chmgr, D, Witness_list),
|
||||
CMode = if Witness_list == [] -> ap_mode;
|
||||
Witness_list /= [] -> cp_mode
|
||||
end,
|
||||
ok = machi_chain_manager1:set_chain_members(a_chmgr, ch0, 0, CMode,
|
||||
D, Witness_list),
|
||||
ok = machi_chain_manager1:set_chain_members(b_chmgr, ch0, 0, CMode,
|
||||
D, Witness_list),
|
||||
ok = machi_chain_manager1:set_chain_members(c_chmgr, ch0, 0, CMode,
|
||||
D, Witness_list),
|
||||
run_ticks([a_chmgr,b_chmgr,c_chmgr]),
|
||||
%% Everyone is settled on the same damn epoch id.
|
||||
{ok, EpochID} = machi_flu1_client:get_latest_epochid(Host, PortBase+0,
|
||||
|
@ -95,11 +102,13 @@ run_ticks(MgrList) ->
|
|||
ok.
|
||||
|
||||
smoke_test2() ->
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
{ok, SupPid} = machi_sup:start_link(),
|
||||
error_logger:tty(false),
|
||||
try
|
||||
Prefix = <<"pre">>,
|
||||
Chunk1 = <<"yochunk">>,
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
Host = "localhost",
|
||||
PortBase = 64454,
|
||||
Os = [{ignore_stability_time, true}, {active_mode, false}],
|
||||
|
@ -107,95 +116,119 @@ smoke_test2() ->
|
|||
|
||||
%% Whew ... ok, now start some damn tests.
|
||||
{ok, C1} = machi_cr_client:start_link([P || {_,P}<-orddict:to_list(D)]),
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1),
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum),
|
||||
{ok, {Off1,Size1,File1}} =
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1),
|
||||
Chunk1_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, Chunk1},
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum),
|
||||
BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, "foo")},
|
||||
{error, bad_checksum} =
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1_badcs),
|
||||
{ok, Chunk1} = machi_cr_client:read_chunk(C1, File1, Off1, Size1),
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, BadCSum),
|
||||
{ok, {[{_, Off1, Chunk1, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, Size1, undefined),
|
||||
{ok, PPP} = machi_flu1_client:read_latest_projection(Host, PortBase+0,
|
||||
private),
|
||||
%% Verify that the client's CR wrote to all of them.
|
||||
[{ok, Chunk1} = machi_flu1_client:read_chunk(
|
||||
Host, PortBase+X, EpochID, File1, Off1, Size1) ||
|
||||
[{ok, {[{_, Off1, Chunk1, _}], []}} =
|
||||
machi_flu1_client:read_chunk(
|
||||
Host, PortBase+X, NSInfo, EpochID, File1, Off1, Size1, undefined) ||
|
||||
X <- [0,1,2] ],
|
||||
|
||||
%% Test read repair: Manually write to head, then verify that
|
||||
%% read-repair fixes all.
|
||||
FooOff1 = Off1 + (1024*1024),
|
||||
[{error, not_written} = machi_flu1_client:read_chunk(
|
||||
Host, PortBase+X, EpochID,
|
||||
File1, FooOff1, Size1) || X <- [0,1,2] ],
|
||||
ok = machi_flu1_client:write_chunk(Host, PortBase+0, EpochID,
|
||||
File1, FooOff1, Chunk1),
|
||||
{ok, Chunk1} = machi_cr_client:read_chunk(C1, File1, FooOff1, Size1),
|
||||
[{X,{ok, Chunk1}} = {X,machi_flu1_client:read_chunk(
|
||||
Host, PortBase+X, EpochID,
|
||||
File1, FooOff1, Size1)} || X <- [0,1,2] ],
|
||||
Host, PortBase+X, NSInfo, EpochID,
|
||||
File1, FooOff1, Size1, undefined) || X <- [0,1,2] ],
|
||||
ok = machi_flu1_client:write_chunk(Host, PortBase+0, NSInfo, EpochID,
|
||||
File1, FooOff1, Chunk1, NoCSum),
|
||||
{ok, {[{File1, FooOff1, Chunk1, _}=_YY], []}} =
|
||||
machi_flu1_client:read_chunk(Host, PortBase+0, NSInfo, EpochID,
|
||||
File1, FooOff1, Size1, undefined),
|
||||
{ok, {[{File1, FooOff1, Chunk1, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, FooOff1, Size1, undefined),
|
||||
[?assertMatch({X,{ok, {[{_, FooOff1, Chunk1, _}], []}}},
|
||||
{X,machi_flu1_client:read_chunk(
|
||||
Host, PortBase+X, NSInfo, EpochID,
|
||||
File1, FooOff1, Size1, undefined)})
|
||||
|| X <- [0,1,2] ],
|
||||
|
||||
%% Test read repair: Manually write to middle, then same checking.
|
||||
FooOff2 = Off1 + (2*1024*1024),
|
||||
Chunk2 = <<"Middle repair chunk">>,
|
||||
Size2 = size(Chunk2),
|
||||
ok = machi_flu1_client:write_chunk(Host, PortBase+1, EpochID,
|
||||
File1, FooOff2, Chunk2),
|
||||
{ok, Chunk2} = machi_cr_client:read_chunk(C1, File1, FooOff2, Size2),
|
||||
[{X,{ok, Chunk2}} = {X,machi_flu1_client:read_chunk(
|
||||
Host, PortBase+X, EpochID,
|
||||
File1, FooOff2, Size2)} || X <- [0,1,2] ],
|
||||
ok = machi_flu1_client:write_chunk(Host, PortBase+1, NSInfo, EpochID,
|
||||
File1, FooOff2, Chunk2, NoCSum),
|
||||
{ok, {[{File1, FooOff2, Chunk2, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, FooOff2, Size2, undefined),
|
||||
[{X,{ok, {[{File1, FooOff2, Chunk2, _}], []}}} =
|
||||
{X,machi_flu1_client:read_chunk(
|
||||
Host, PortBase+X, NSInfo, EpochID,
|
||||
File1, FooOff2, Size2, undefined)} || X <- [0,1,2] ],
|
||||
|
||||
%% Misc API smoke & minor regression checks
|
||||
{error, not_written} = machi_cr_client:read_chunk(C1, <<"no">>,
|
||||
999999999, 1),
|
||||
{error, partial_read} = machi_cr_client:read_chunk(C1, File1,
|
||||
Off1, 88888888),
|
||||
{error, bad_arg} = machi_cr_client:read_chunk(C1, NSInfo, <<"no">>,
|
||||
999999999, 1, undefined),
|
||||
{ok, {[{File1,Off1,Chunk1,_}, {File1,FooOff1,Chunk1,_}, {File1,FooOff2,Chunk2,_}],
|
||||
[]}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, 88888888, undefined),
|
||||
%% Checksum list return value is a primitive binary().
|
||||
{ok, KludgeBin} = machi_cr_client:checksum_list(C1, File1),
|
||||
true = is_binary(KludgeBin),
|
||||
|
||||
{error, no_such_file} = machi_cr_client:checksum_list(C1, <<"!!!!">>),
|
||||
%% Exactly one file right now
|
||||
{error, bad_arg} = machi_cr_client:checksum_list(C1, <<"!!!!">>),
|
||||
io:format(user, "\nFiles = ~p\n", [machi_cr_client:list_files(C1)]),
|
||||
%% Exactly one file right now, e.g.,
|
||||
%% {ok,[{2098202,<<"pre^b144ef13-db4d-4c9f-96e7-caff02dc754f^1">>}]}
|
||||
{ok, [_]} = machi_cr_client:list_files(C1),
|
||||
|
||||
%% Go back and test append_chunk_extra() and write_chunk()
|
||||
%% Go back and test append_chunk() + extra and write_chunk()
|
||||
Chunk10 = <<"It's a different chunk!">>,
|
||||
Size10 = byte_size(Chunk10),
|
||||
Extra10 = 5,
|
||||
Opts1 = #append_opts{chunk_extra=Extra10*Size10},
|
||||
{ok, {Off10,Size10,File10}} =
|
||||
machi_cr_client:append_chunk_extra(C1, Prefix, Chunk10,
|
||||
Extra10 * Size10),
|
||||
{ok, Chunk10} = machi_cr_client:read_chunk(C1, File10, Off10, Size10),
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk10,
|
||||
NoCSum, Opts1),
|
||||
{ok, {[{_, Off10, Chunk10, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File10, Off10, Size10, undefined),
|
||||
[begin
|
||||
Offx = Off10 + (Seq * Size10),
|
||||
%% TODO: uncomment written/not_written enforcement is available.
|
||||
%% {error,not_written} = machi_cr_client:read_chunk(C1, File10,
|
||||
%% {error,not_written} = machi_cr_client:read_chunk(C1, NSInfo, File10,
|
||||
%% Offx, Size10),
|
||||
{ok, {Offx,Size10,File10}} =
|
||||
machi_cr_client:write_chunk(C1, File10, Offx, Chunk10),
|
||||
{ok, Chunk10} = machi_cr_client:read_chunk(C1, File10, Offx,
|
||||
Size10)
|
||||
machi_cr_client:write_chunk(C1, NSInfo, File10, Offx, Chunk10, NoCSum),
|
||||
{ok, {[{_, Offx, Chunk10, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File10, Offx, Size10, undefined)
|
||||
end || Seq <- lists:seq(1, Extra10)],
|
||||
{ok, {Off11,Size11,File11}} =
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk10),
|
||||
%% Double-check that our reserved extra bytes were really honored!
|
||||
true = (Off11 > (Off10 + (Extra10 * Size10))),
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk10, NoCSum),
|
||||
%% %% Double-check that our reserved extra bytes were really honored!
|
||||
%% true = (Off11 > (Off10 + (Extra10 * Size10))),
|
||||
io:format(user, "\nFiles = ~p\n", [machi_cr_client:list_files(C1)]),
|
||||
|
||||
ok
|
||||
after
|
||||
exit(SupPid, normal),
|
||||
machi_util:wait_for_death(SupPid, 100),
|
||||
error_logger:tty(true),
|
||||
catch application:stop(machi),
|
||||
exit(SupPid, normal)
|
||||
catch application:stop(machi)
|
||||
end.
|
||||
|
||||
witness_smoke_test_() -> {timeout, 1*60, fun() -> witness_smoke_test2() end}.
|
||||
|
||||
witness_smoke_test2() ->
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
error_logger:tty(false),
|
||||
SupPid = case machi_sup:start_link() of
|
||||
{ok, P} -> P;
|
||||
{error, {already_started, P1}} -> P1;
|
||||
Other -> error(Other)
|
||||
end,
|
||||
%% TODO: I wonder why commenting this out makes this test pass
|
||||
%% error_logger:tty(true),
|
||||
try
|
||||
Prefix = <<"pre">>,
|
||||
Chunk1 = <<"yochunk">>,
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
Host = "localhost",
|
||||
PortBase = 64444,
|
||||
Os = [{ignore_stability_time, true}, {active_mode, false},
|
||||
|
@ -205,13 +238,15 @@ witness_smoke_test2() ->
|
|||
|
||||
%% Whew ... ok, now start some damn tests.
|
||||
{ok, C1} = machi_cr_client:start_link([P || {_,P}<-orddict:to_list(D)]),
|
||||
{ok, _} = machi_cr_client:append_chunk(C1, Prefix, Chunk1),
|
||||
{ok, _} = machi_cr_client:append_chunk(C1, NSInfo, Prefix,
|
||||
Chunk1, NoCSum),
|
||||
{ok, {Off1,Size1,File1}} =
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1),
|
||||
Chunk1_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, Chunk1},
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum),
|
||||
BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, "foo")},
|
||||
{error, bad_checksum} =
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1_badcs),
|
||||
{ok, Chunk1} = machi_cr_client:read_chunk(C1, File1, Off1, Size1),
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, BadCSum),
|
||||
{ok, {[{_, Off1, Chunk1, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, Size1, undefined),
|
||||
|
||||
%% Stop 'b' and let the chain reset.
|
||||
ok = machi_flu_psup:stop_flu_package(b),
|
||||
|
@ -224,15 +259,25 @@ witness_smoke_test2() ->
|
|||
%% Let's wedge OurWitness and see what happens: timeout/partition.
|
||||
#p_srvr{name=WitName, address=WitA, port=WitP} =
|
||||
orddict:fetch(OurWitness, D),
|
||||
{ok, {false, EpochID2}} = machi_flu1_client:wedge_status(WitA, WitP),
|
||||
{ok, {false, EpochID2,_,_}} = machi_flu1_client:wedge_status(WitA, WitP),
|
||||
machi_flu1:wedge_myself(WitName, EpochID2),
|
||||
{ok, {true, EpochID2}} = machi_flu1_client:wedge_status(WitA, WitP),
|
||||
case machi_flu1_client:wedge_status(WitA, WitP) of
|
||||
{ok, {true, EpochID2,_,_}} ->
|
||||
ok;
|
||||
{ok, {false, EpochID2,_,_}} ->
|
||||
%% This is racy. Work around it by sleeping a while.
|
||||
timer:sleep(6*1000),
|
||||
{ok, {true, EpochID2,_,_}} =
|
||||
machi_flu1_client:wedge_status(WitA, WitP)
|
||||
end,
|
||||
|
||||
%% Chunk1 is still readable: not affected by wedged witness head.
|
||||
{ok, Chunk1} = machi_cr_client:read_chunk(C1, File1, Off1, Size1),
|
||||
{ok, {[{_, Off1, Chunk1, _}], []}} =
|
||||
machi_cr_client:read_chunk(C1, NSInfo, File1, Off1, Size1, undefined),
|
||||
%% But because the head is wedged, an append will fail.
|
||||
{error, partition} =
|
||||
machi_cr_client:append_chunk(C1, Prefix, Chunk1, 1*1000),
|
||||
machi_cr_client:append_chunk(C1, NSInfo, Prefix, Chunk1, NoCSum,
|
||||
#append_opts{}, 1*1000),
|
||||
|
||||
%% The witness's wedge status should cause timeout/partition
|
||||
%% for write_chunk also.
|
||||
|
@ -241,7 +286,7 @@ witness_smoke_test2() ->
|
|||
File10 = File1,
|
||||
Offx = Off1 + (1 * Size10),
|
||||
{error, partition} =
|
||||
machi_cr_client:write_chunk(C1, File10, Offx, Chunk10, 1*1000),
|
||||
machi_cr_client:write_chunk(C1, NSInfo, File10, Offx, Chunk10, NoCSum, 1*1000),
|
||||
|
||||
ok
|
||||
after
|
||||
|
|
131
test/machi_csum_table_test.erl
Normal file
131
test/machi_csum_table_test.erl
Normal file
|
@ -0,0 +1,131 @@
|
|||
-module(machi_csum_table_test).
|
||||
-compile(export_all).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-define(HDR, {0, 1024, none}).
|
||||
|
||||
cleanup(Dir) ->
|
||||
os:cmd("rm -rf " ++ Dir).
|
||||
|
||||
smoke_test() ->
|
||||
Filename = "./temp-checksum-dumb-file",
|
||||
_ = cleanup(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
?assertEqual([{1024, infinity}],
|
||||
machi_csum_table:calc_unwritten_bytes(MC)),
|
||||
Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
|
||||
[] = machi_csum_table:find(MC, Offset, Size),
|
||||
ok = machi_csum_table:write(MC, Offset, Size, Checksum),
|
||||
[{1024, 40}, {1098, infinity}] = machi_csum_table:calc_unwritten_bytes(MC),
|
||||
?assertEqual([Entry], machi_csum_table:find(MC, Offset, Size)),
|
||||
ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined),
|
||||
?assertEqual([{Offset, Size, trimmed}],
|
||||
machi_csum_table:find(MC, Offset, Size)),
|
||||
ok = machi_csum_table:close(MC),
|
||||
ok = machi_csum_table:delete(MC).
|
||||
|
||||
close_test() ->
|
||||
Filename = "./temp-checksum-dumb-file-2",
|
||||
_ = cleanup(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
Entry = {Offset, Size, Checksum} = {1064, 34, <<"deadbeef">>},
|
||||
[] = machi_csum_table:find(MC, Offset, Size),
|
||||
ok = machi_csum_table:write(MC, Offset, Size, Checksum),
|
||||
[Entry] = machi_csum_table:find(MC, Offset, Size),
|
||||
ok = machi_csum_table:close(MC),
|
||||
|
||||
{ok, MC2} = machi_csum_table:open(Filename, []),
|
||||
[Entry] = machi_csum_table:find(MC2, Offset, Size),
|
||||
ok = machi_csum_table:trim(MC2, Offset, Size, undefined, undefined),
|
||||
[{Offset, Size, trimmed}] = machi_csum_table:find(MC2, Offset, Size),
|
||||
ok = machi_csum_table:delete(MC2).
|
||||
|
||||
smoke2_test() ->
|
||||
Filename = "./temp-checksum-dumb-file-3",
|
||||
_ = cleanup(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
Entry = {Offset, Size, Checksum} = {1025, 10, <<"deadbeef">>},
|
||||
ok = machi_csum_table:write(MC, Offset, Size, Checksum),
|
||||
?assertEqual([], machi_csum_table:find(MC, 0, 0)),
|
||||
?assertEqual([?HDR], machi_csum_table:find(MC, 0, 1)),
|
||||
[Entry] = machi_csum_table:find(MC, Offset, Size),
|
||||
[?HDR] = machi_csum_table:find(MC, 1, 1024),
|
||||
?assertEqual([?HDR, Entry],
|
||||
machi_csum_table:find(MC, 1023, 1024)),
|
||||
[Entry] = machi_csum_table:find(MC, 1024, 1024),
|
||||
[Entry] = machi_csum_table:find(MC, 1025, 1024),
|
||||
|
||||
ok = machi_csum_table:trim(MC, Offset, Size, undefined, undefined),
|
||||
[{Offset, Size, trimmed}] = machi_csum_table:find(MC, Offset, Size),
|
||||
ok = machi_csum_table:close(MC),
|
||||
ok = machi_csum_table:delete(MC).
|
||||
|
||||
smoke3_test() ->
|
||||
Filename = "./temp-checksum-dumb-file-4",
|
||||
_ = cleanup(Filename),
|
||||
{ok, MC} = machi_csum_table:open(Filename, []),
|
||||
Scenario =
|
||||
[%% Command, {Offset, Size, Csum}, LeftNeighbor, RightNeibor
|
||||
{?LINE, write, {2000, 10, <<"heh">>}, undefined, undefined},
|
||||
{?LINE, write, {3000, 10, <<"heh">>}, undefined, undefined},
|
||||
{?LINE, write, {4000, 10, <<"heh2">>}, undefined, undefined},
|
||||
{?LINE, write, {4000, 10, <<"heh2">>}, undefined, undefined},
|
||||
{?LINE, write, {4005, 10, <<"heh3">>}, {4000, 5, <<"heh2">>}, undefined},
|
||||
{?LINE, write, {4005, 10, <<"heh3">>}, undefined, undefined},
|
||||
{?LINE, trim, {3005, 10, <<>>}, {3000, 5, <<"heh">>}, undefined},
|
||||
{?LINE, trim, {2000, 10, <<>>}, undefined, undefined},
|
||||
{?LINE, trim, {2005, 5, <<>>}, {2000, 5, trimmed}, undefined},
|
||||
{?LINE, trim, {3000, 5, <<>>}, undefined, undefined},
|
||||
{?LINE, trim, {4000, 10, <<>>}, undefined, {4010, 5, <<"heh3">>}},
|
||||
{?LINE, trim, {4010, 5, <<>>}, undefined, undefined},
|
||||
{?LINE, trim, {0, 1024, <<>>}, undefined, undefined}
|
||||
],
|
||||
[ begin
|
||||
%% ?debugVal({_Line, Chunk}),
|
||||
{Offset, Size, Csum} = Chunk,
|
||||
?assertEqual(LeftN0,
|
||||
machi_csum_table:find_leftneighbor(MC, Offset)),
|
||||
?assertEqual(RightN0,
|
||||
machi_csum_table:find_rightneighbor(MC, Offset+Size)),
|
||||
LeftN = case LeftN0 of
|
||||
{OffsL, SizeL, trimmed} -> {OffsL, SizeL, trimmed};
|
||||
{OffsL, SizeL, _} -> {OffsL, SizeL, <<"boom">>};
|
||||
OtherL -> OtherL
|
||||
end,
|
||||
RightN = case RightN0 of
|
||||
{OffsR, SizeR, _} -> {OffsR, SizeR, <<"boot">>};
|
||||
OtherR -> OtherR
|
||||
end,
|
||||
case Cmd of
|
||||
write ->
|
||||
ok = machi_csum_table:write(MC, Offset, Size, Csum,
|
||||
LeftN, RightN);
|
||||
trim ->
|
||||
ok = machi_csum_table:trim(MC, Offset, Size,
|
||||
LeftN, RightN)
|
||||
end
|
||||
end || {_Line, Cmd, Chunk, LeftN0, RightN0} <- Scenario ],
|
||||
?assert(not machi_csum_table:all_trimmed(MC, 10000)),
|
||||
machi_csum_table:trim(MC, 0, 10000, undefined, undefined),
|
||||
?assert(machi_csum_table:all_trimmed(MC, 10000)),
|
||||
|
||||
ok = machi_csum_table:close(MC),
|
||||
ok = machi_csum_table:delete(MC).
|
||||
|
||||
%% TODO: add quickcheck test here
|
||||
|
||||
%% Previous implementation
|
||||
-spec all_trimmed2(machi_csum_table:table(),
|
||||
non_neg_integer(), non_neg_integer()) -> boolean().
|
||||
all_trimmed2(CsumT, Left, Right) ->
|
||||
Chunks = machi_csum_table:find(CsumT, Left, Right),
|
||||
runthru(Chunks, Left, Right).
|
||||
|
||||
%% @doc make sure all trimmed chunks are continously chained
|
||||
%% TODO: test with EQC
|
||||
runthru([], Pos, Pos) -> true;
|
||||
runthru([], Pos0, Pos) when Pos0 < Pos -> false;
|
||||
runthru([{Offset0, Size0, trimmed}|T], Offset, Pos) when Offset0 =< Offset ->
|
||||
runthru(T, Offset0+Size0, Pos);
|
||||
runthru(_L, _O, _P) ->
|
||||
false.
|
478
test/machi_file_proxy_eqc.erl
Normal file
478
test/machi_file_proxy_eqc.erl
Normal file
|
@ -0,0 +1,478 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_file_proxy_eqc).
|
||||
|
||||
-ifdef(TEST).
|
||||
-ifdef(EQC).
|
||||
-compile(export_all).
|
||||
-include("machi.hrl").
|
||||
-include_lib("eqc/include/eqc.hrl").
|
||||
-include_lib("eqc/include/eqc_statem.hrl").
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
-define(QC_OUT(P),
|
||||
eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)).
|
||||
|
||||
-define(TESTDIR, "./eqc").
|
||||
|
||||
%% EUNIT TEST DEFINITION
|
||||
eqc_test_() ->
|
||||
PropTimeout = case os:getenv("EQC_TIME") of
|
||||
false -> 30;
|
||||
V -> list_to_integer(V)
|
||||
end,
|
||||
{timeout, PropTimeout*2 + 30,
|
||||
{spawn,
|
||||
[
|
||||
?_assertEqual(true, eqc:quickcheck(eqc:testing_time(PropTimeout, ?QC_OUT(prop_ok()))))
|
||||
]
|
||||
}}.
|
||||
|
||||
%% SHELL HELPERS
|
||||
test() ->
|
||||
test(100).
|
||||
|
||||
test(N) ->
|
||||
quickcheck(numtests(N, prop_ok())).
|
||||
|
||||
check() ->
|
||||
check(prop_ok(), current_counterexample()).
|
||||
|
||||
%% GENERATORS
|
||||
|
||||
csum_type() ->
|
||||
elements([?CSUM_TAG_NONE, ?CSUM_TAG_CLIENT_SHA, ?CSUM_TAG_SERVER_SHA]).
|
||||
|
||||
csum(Type, Binary) ->
|
||||
case Type of
|
||||
?CSUM_TAG_NONE -> <<>>;
|
||||
_ -> machi_util:checksum_chunk(Binary)
|
||||
end.
|
||||
|
||||
position(P) ->
|
||||
?LET(O, offset(), P + O).
|
||||
|
||||
offset() ->
|
||||
?SUCHTHAT(X, int(), X >= 0).
|
||||
|
||||
offset_base() ->
|
||||
elements([4096, 6144, 7168, 8192, 20480, 100000, 1000000]).
|
||||
|
||||
big_offset() ->
|
||||
?LET(P, int(), ?LET(X, offset_base(), P+X)).
|
||||
|
||||
len() ->
|
||||
?SUCHTHAT(X, int(), X >= 1).
|
||||
|
||||
data_with_csum() ->
|
||||
?LET({B,T},{eqc_gen:largebinary(), csum_type()}, {B,T, csum(T, B)}).
|
||||
%?LET({B,T},{eqc_gen:binary(), csum_type()}, {B,T, csum(T, B)}).
|
||||
|
||||
data_with_csum(Limit) ->
|
||||
%?LET({B,T},{?LET(S, Limit, eqc_gen:largebinary(S)), csum_type()}, {B,T, csum(T, B)}).
|
||||
?LET({B,T},{?LET(S, Limit, eqc_gen:binary(S)), csum_type()}, {B,T, csum(T, B)}).
|
||||
|
||||
intervals([]) ->
|
||||
[];
|
||||
intervals([N]) ->
|
||||
[{N, choose(1,1)}];
|
||||
intervals([A,B|T]) ->
|
||||
[{A, oneof([choose(1, B-A), B-A])}|intervals([B|T])].
|
||||
|
||||
interval_list() ->
|
||||
?LET(L,
|
||||
oneof([list(choose(1025, 1033)), list(choose(1024, 4096))]),
|
||||
intervals(lists:usort(L))).
|
||||
|
||||
shuffle_interval() ->
|
||||
?LET(L, interval_list(), shuffle(L)).
|
||||
|
||||
get_written_interval(L) ->
|
||||
?LET({O, Ln}, elements(L), {O+1, Ln-1}).
|
||||
|
||||
%% INITIALIZATION
|
||||
|
||||
-record(state, {pid, prev_extra = 0,
|
||||
filename = undefined,
|
||||
planned_writes=[],
|
||||
planned_trims=[],
|
||||
written=[],
|
||||
trimmed=[]}).
|
||||
|
||||
initial_state() ->
|
||||
{_, _, MS} = os:timestamp(),
|
||||
Filename = test_server:temp_name("eqc_data") ++ "." ++ integer_to_list(MS),
|
||||
#state{filename=Filename, written=[{0,1024}]}.
|
||||
|
||||
initial_state(I, T) ->
|
||||
S=initial_state(),
|
||||
S#state{written=[{0,1024}],
|
||||
planned_writes=I,
|
||||
planned_trims=T}.
|
||||
|
||||
weight(_S, rewrite) -> 1;
|
||||
weight(_S, _) -> 2.
|
||||
|
||||
%% HELPERS
|
||||
|
||||
get_overlaps(_Offset, _Len, [], Acc) -> lists:reverse(Acc);
|
||||
get_overlaps(Offset, Len, [{Pos, Sz} = Ck|T], Acc0)
|
||||
%% Overlap judgement differnt from the one in machi_csum_table
|
||||
%% [a=Offset, b), [x=Pos, y) ...
|
||||
when
|
||||
%% a =< x && x < b && b =< y
|
||||
(Offset =< Pos andalso Pos < Offset + Len andalso Offset + Len =< Pos + Sz) orelse
|
||||
%% a =< x && y < b
|
||||
(Offset =< Pos andalso Pos + Sz < Offset + Len) orelse
|
||||
%% x < a && a < y && y =< b
|
||||
(Pos < Offset andalso Offset < Pos + Sz andalso Pos + Sz =< Offset + Len) orelse
|
||||
%% x < a && b < y
|
||||
(Pos < Offset + Len andalso Offset + Len < Pos + Sz) ->
|
||||
get_overlaps(Offset, Len, T, [Ck|Acc0]);
|
||||
get_overlaps(Offset, Len, [_Ck|T], Acc0) ->
|
||||
get_overlaps(Offset, Len, T, Acc0).
|
||||
|
||||
%% Inefficient but simple easy code to verify by eyes - returns all
|
||||
%% bytes that fits in (Offset, Len)
|
||||
chop(Offset, Len, List) ->
|
||||
ChopLeft = fun({Pos, Sz}) when Pos < Offset andalso Offset =< Pos + Sz ->
|
||||
{Offset, Sz + Pos - Offset};
|
||||
({Pos, Sz}) when Offset =< Pos andalso Pos + Sz < Offset + Len ->
|
||||
{Pos, Sz};
|
||||
({Pos, _Sz}) when Offset =< Pos ->
|
||||
{Pos, Offset + Len - Pos}
|
||||
end,
|
||||
ChopRight = fun({Pos, Sz}) when Offset + Len < Pos + Sz ->
|
||||
{Pos, Offset + Len - Pos};
|
||||
({Pos, Sz}) ->
|
||||
{Pos, Sz}
|
||||
end,
|
||||
Filter0 = fun({_, 0}) -> false;
|
||||
(Other) -> {true, Other} end,
|
||||
lists:filtermap(fun(E) -> Filter0(ChopRight(ChopLeft(E))) end,
|
||||
List).
|
||||
|
||||
%% Returns all bytes that are at left side of the Offset
|
||||
chopped_left(_Offset, []) -> undefined;
|
||||
chopped_left(Offset, [{Pos,_Sz}|_]) when Pos < Offset ->
|
||||
{Pos, Offset - Pos};
|
||||
chopped_left(_, _) ->
|
||||
undefined.
|
||||
|
||||
chopped_right(_Offset, []) -> undefined;
|
||||
chopped_right(Offset, List) ->
|
||||
{Pos, Sz} = lists:last(List),
|
||||
if Offset < Pos + Sz ->
|
||||
{Offset, Pos + Sz - Offset};
|
||||
true ->
|
||||
undefined
|
||||
end.
|
||||
|
||||
cleanup_chunk(Offset, Length, ChunkList) ->
|
||||
Overlaps = get_overlaps(Offset, Length, ChunkList, []),
|
||||
NewCL0 = lists:foldl(fun lists:delete/2,
|
||||
ChunkList, Overlaps),
|
||||
NewCL1 = case chopped_left(Offset, Overlaps) of
|
||||
undefined -> NewCL0;
|
||||
LeftRemain -> [LeftRemain|NewCL0]
|
||||
end,
|
||||
NewCL2 = case chopped_right(Offset+Length, Overlaps) of
|
||||
undefined -> NewCL1;
|
||||
RightRemain -> [RightRemain|NewCL1]
|
||||
end,
|
||||
lists:sort(NewCL2).
|
||||
|
||||
is_error({error, _}) -> true;
|
||||
is_error({error, _, _}) -> true;
|
||||
is_error(Other) -> {expected_ERROR, Other}.
|
||||
|
||||
is_ok({ok, _, _}) -> true;
|
||||
is_ok(ok) -> true;
|
||||
is_ok(Other) -> {expected_OK, Other}.
|
||||
|
||||
get_offset({ok, _Filename, Offset}) -> Offset;
|
||||
get_offset(_) -> error(badarg).
|
||||
|
||||
last_byte([]) -> 0;
|
||||
last_byte(L0) ->
|
||||
L1 = lists:map(fun({Pos, Sz}) -> Pos + Sz end, L0),
|
||||
lists:last(lists:sort(L1)).
|
||||
|
||||
cleanup() ->
|
||||
[begin
|
||||
Fs = filelib:wildcard(?TESTDIR ++ Glob),
|
||||
[file:delete(F) || F <- Fs],
|
||||
[file:del_dir(F) || F <- Fs]
|
||||
end || Glob <- ["*/*/*/*", "*/*/*", "*/*", "*"] ],
|
||||
_ = file:del_dir(?TESTDIR),
|
||||
ok.
|
||||
|
||||
%% start
|
||||
|
||||
start_pre(S) ->
|
||||
S#state.pid =:= undefined.
|
||||
|
||||
start_command(S) ->
|
||||
{call, ?MODULE, start, [S]}.
|
||||
|
||||
start(#state{filename=File}) ->
|
||||
{ok, Pid} = machi_file_proxy:start_link(some_flu, File, ?TESTDIR),
|
||||
unlink(Pid),
|
||||
Pid.
|
||||
|
||||
start_next(S, Pid, _) ->
|
||||
S#state{pid = Pid}.
|
||||
|
||||
%% read
|
||||
|
||||
read_pre(S) ->
|
||||
S#state.pid /= undefined.
|
||||
|
||||
read_args(S) ->
|
||||
[S#state.pid, oneof([offset(), big_offset()]), len()].
|
||||
|
||||
read_post(S, [_Pid, Off, L], Res) ->
|
||||
Written = get_overlaps(Off, L, S#state.written, []),
|
||||
Chopped = chop(Off, L, Written),
|
||||
Trimmed = get_overlaps(Off, L, S#state.trimmed, []),
|
||||
Eof = lists:max([Pos+Sz||{Pos,Sz}<-S#state.written]),
|
||||
case Res of
|
||||
{ok, {Written0, Trimmed0}} ->
|
||||
Written1 = lists:map(fun({_, Pos, Chunk, _}) ->
|
||||
{Pos, iolist_size(Chunk)}
|
||||
end, Written0),
|
||||
Trimmed1 = lists:map(fun({_, Pos, Sz}) -> {Pos, Sz} end, Trimmed0),
|
||||
Chopped =:= Written1
|
||||
andalso Trimmed =:= Trimmed1;
|
||||
%% TODO: such response are ugly, rethink the SPEC
|
||||
{error, not_written} when Eof < Off + L ->
|
||||
true;
|
||||
{error, not_written} when Chopped =:= [] andalso Trimmed =:= [] ->
|
||||
true;
|
||||
_Other ->
|
||||
is_error(Res)
|
||||
end.
|
||||
|
||||
read_next(S, _Res, _Args) -> S.
|
||||
|
||||
read(Pid, Offset, Length) ->
|
||||
machi_file_proxy:read(Pid, Offset, Length, [{needs_trimmed, true}]).
|
||||
|
||||
%% write
|
||||
|
||||
write_pre(S) ->
|
||||
S#state.pid /= undefined andalso S#state.planned_writes /= [].
|
||||
|
||||
%% do not allow writes with empty data
|
||||
write_pre(_S, [_Pid, _Extra, {<<>>, _Tag, _Csum}]) ->
|
||||
?assert(false),
|
||||
false;
|
||||
write_pre(_S, _Args) ->
|
||||
true.
|
||||
|
||||
write_args(S) ->
|
||||
{Off, Len} = hd(S#state.planned_writes),
|
||||
[S#state.pid, Off, data_with_csum(Len)].
|
||||
|
||||
write_post(S, [_Pid, Off, {Bin, _Tag, _Csum}] = _Args, Res) ->
|
||||
Size = iolist_size(Bin),
|
||||
case {get_overlaps(Off, Size, S#state.written, []),
|
||||
get_overlaps(Off, Size, S#state.trimmed, [])} of
|
||||
{[], []} ->
|
||||
%% No overlap neither with written ranges nor trimmed
|
||||
%% ranges; OK to write things.
|
||||
eq(Res, ok);
|
||||
{_, _} ->
|
||||
%% overlap found in either or both at written or at
|
||||
%% trimmed ranges; can't write.
|
||||
is_error(Res)
|
||||
end.
|
||||
|
||||
write_next(S, Res, [_Pid, Offset, {Bin, _Tag, _Csum}]) ->
|
||||
S0 = case is_ok(Res) of
|
||||
true ->
|
||||
S#state{written = lists:sort(S#state.written ++ [{Offset, iolist_size(Bin)}]) };
|
||||
_ ->
|
||||
S
|
||||
end,
|
||||
S0#state{prev_extra = 0, planned_writes=tl(S0#state.planned_writes)}.
|
||||
|
||||
|
||||
write(Pid, Offset, {Bin, Tag, Csum}) ->
|
||||
Meta = [{client_csum_tag, Tag},
|
||||
{client_csum, Csum}],
|
||||
machi_file_proxy:write(Pid, Offset, Meta, Bin).
|
||||
|
||||
%% append
|
||||
|
||||
append_pre(S) ->
|
||||
S#state.pid /= undefined.
|
||||
|
||||
%% do not allow appends with empty binary data
|
||||
append_pre(_S, [_Pid, _Extra, {<<>>, _Tag, _Csum}]) ->
|
||||
false;
|
||||
append_pre(_S, _Args) ->
|
||||
true.
|
||||
|
||||
append_args(S) ->
|
||||
[S#state.pid, default(0, len()), data_with_csum()].
|
||||
|
||||
append(Pid, Extra, {Bin, Tag, Csum}) ->
|
||||
Meta = [{client_csum_tag, Tag},
|
||||
{client_csum, Csum}],
|
||||
machi_file_proxy:append(Pid, Meta, Extra, Bin).
|
||||
|
||||
append_next(S, Res, [_Pid, Extra, {Bin, _Tag, _Csum}]) ->
|
||||
case is_ok(Res) of
|
||||
true ->
|
||||
Offset = get_offset(Res),
|
||||
S#state{prev_extra = Extra,
|
||||
written = lists:sort(S#state.written ++ [{Offset, iolist_size(Bin)}])};
|
||||
_Other ->
|
||||
S
|
||||
end.
|
||||
|
||||
%% appends should always succeed unless the disk is full
|
||||
%% or there's a hardware failure.
|
||||
append_post(S, _Args, Res) ->
|
||||
case is_ok(Res) of
|
||||
true ->
|
||||
Offset = get_offset(Res),
|
||||
case erlang:max(last_byte(S#state.written),
|
||||
last_byte(S#state.trimmed)) + S#state.prev_extra of
|
||||
Offset ->
|
||||
true;
|
||||
UnexpectedByte ->
|
||||
{wrong_offset_after_append,
|
||||
{Offset, UnexpectedByte},
|
||||
{S#state.written, S#state.prev_extra}}
|
||||
end;
|
||||
Error ->
|
||||
Error
|
||||
end.
|
||||
|
||||
%% rewrite
|
||||
|
||||
rewrite_pre(S) ->
|
||||
S#state.pid /= undefined andalso
|
||||
(S#state.written ++ S#state.trimmed) /= [] .
|
||||
|
||||
rewrite_args(S) ->
|
||||
?LET({Off, Len},
|
||||
get_written_interval(S#state.written ++ S#state.trimmed),
|
||||
[S#state.pid, Off, data_with_csum(Len)]).
|
||||
|
||||
rewrite(Pid, Offset, {Bin, Tag, Csum}) ->
|
||||
Meta = [{client_csum_tag, Tag},
|
||||
{client_csum, Csum}],
|
||||
machi_file_proxy:write(Pid, Offset, Meta, Bin).
|
||||
|
||||
rewrite_post(_S, _Args, Res) ->
|
||||
is_error(Res).
|
||||
|
||||
rewrite_next(S, _Res, _Args) ->
|
||||
S#state{prev_extra = 0}.
|
||||
|
||||
%% trim
|
||||
|
||||
trim_pre(S) ->
|
||||
S#state.pid /= undefined andalso S#state.planned_trims /= [].
|
||||
|
||||
trim_args(S) ->
|
||||
{Offset, Length} = hd(S#state.planned_trims),
|
||||
[S#state.pid, Offset, Length].
|
||||
|
||||
trim(Pid, Offset, Length) ->
|
||||
machi_file_proxy:trim(Pid, Offset, Length, false).
|
||||
|
||||
trim_post(_S, [_Pid, _Offset, _Length], ok) ->
|
||||
true;
|
||||
trim_post(_S, [_Pid, _Offset, _Length], _Res) ->
|
||||
false.
|
||||
|
||||
trim_next(S, Res, [_Pid, Offset, Length]) ->
|
||||
S1 = case is_ok(Res) of
|
||||
true ->
|
||||
NewWritten = cleanup_chunk(Offset, Length, S#state.written),
|
||||
Trimmed1 = cleanup_chunk(Offset, Length, S#state.trimmed),
|
||||
NewTrimmed = lists:sort([{Offset, Length}|Trimmed1]),
|
||||
S#state{trimmed=NewTrimmed,
|
||||
written=NewWritten};
|
||||
_Other ->
|
||||
S
|
||||
end,
|
||||
S1#state{prev_extra=0,
|
||||
planned_trims=tl(S#state.planned_trims)}.
|
||||
|
||||
stop_pre(S) ->
|
||||
S#state.pid /= undefined.
|
||||
|
||||
stop_args(S) ->
|
||||
[S#state.pid].
|
||||
|
||||
stop(Pid) ->
|
||||
catch machi_file_proxy:stop(Pid).
|
||||
|
||||
stop_post(_, _, _) -> true.
|
||||
|
||||
stop_next(S, _, _) ->
|
||||
S#state{pid=undefined, prev_extra=0}.
|
||||
|
||||
%% Property
|
||||
|
||||
prop_ok() ->
|
||||
cleanup(),
|
||||
?FORALL({I, T},
|
||||
{shuffle_interval(), shuffle_interval()},
|
||||
?FORALL(Cmds, parallel_commands(?MODULE, initial_state(I, T)),
|
||||
begin
|
||||
{H, S, Res} = run_parallel_commands(?MODULE, Cmds),
|
||||
cleanup(),
|
||||
pretty_commands(?MODULE, Cmds, {H, S, Res},
|
||||
aggregate(command_names(Cmds), Res == ok))
|
||||
end)).
|
||||
|
||||
%% Test for tester functions
|
||||
chopper_test_() ->
|
||||
[?_assertEqual([{0, 1024}],
|
||||
get_overlaps(1, 1, [{0, 1024}], [])),
|
||||
?_assertEqual([],
|
||||
get_overlaps(10, 5, [{9, 1}, {15, 1}], [])),
|
||||
?_assertEqual([{9,2},{14,1}],
|
||||
get_overlaps(10, 5, [{9, 2}, {14, 1}], [])),
|
||||
?_assertEqual([], chop(0, 0, [{0,2}])),
|
||||
?_assertEqual([{0, 1}], chop(0, 1, [{0,2}])),
|
||||
?_assertEqual([], chop(1, 0, [{0,2}])),
|
||||
?_assertEqual([{1, 1}], chop(1, 1, [{0,2}])),
|
||||
?_assertEqual([{1, 1}], chop(1, 2, [{0,2}])),
|
||||
?_assertEqual([], chop(2, 1, [{0,2}])),
|
||||
?_assertEqual([], chop(2, 2, [{0,2}])),
|
||||
?_assertEqual([{1, 1}], chop(1, 3, [{0,2}])),
|
||||
?_assertError(_, chop(3, 1, [{0,2}])),
|
||||
?_assertEqual([], chop(2, 3, [{0,2}])),
|
||||
?_assertEqual({0, 1}, chopped_left(1, [{0, 1024}])),
|
||||
?_assertEqual([{0, 1}, {2, 1022}], cleanup_chunk(1, 1, [{0, 1024}])),
|
||||
?_assertEqual([{2, 1022}], cleanup_chunk(0, 2, [{0, 1}, {2, 1022}])),
|
||||
?_assert(true)
|
||||
].
|
||||
|
||||
-endif. % EQC
|
||||
-endif. % TEST
|
142
test/machi_file_proxy_test.erl
Normal file
142
test/machi_file_proxy_test.erl
Normal file
|
@ -0,0 +1,142 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_file_proxy_test).
|
||||
|
||||
-ifdef(TEST).
|
||||
-compile(export_all).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-include("machi.hrl").
|
||||
|
||||
clean_up_data_dir(DataDir) ->
|
||||
[begin
|
||||
Fs = filelib:wildcard(DataDir ++ Glob),
|
||||
[file:delete(F) || F <- Fs],
|
||||
[file:del_dir(F) || F <- Fs]
|
||||
end || Glob <- ["*/*/*/*", "*/*/*", "*/*", "*"] ],
|
||||
_ = file:del_dir(DataDir),
|
||||
ok.
|
||||
|
||||
-ifndef(PULSE).
|
||||
|
||||
-define(TESTDIR, "./t").
|
||||
-define(HYOOGE, 75 * 1024 * 1024). % 75 MBytes
|
||||
|
||||
random_binary_single() ->
|
||||
%% OK, I guess it's not that random...
|
||||
<<"Four score and seven years ago our fathers brought forth on this
|
||||
continent a new nation, conceived in liberty, and dedicated to the
|
||||
proposition that all men are created equal.
|
||||
|
||||
Now we are engaged in a great civil war, testing whether that nation, or any
|
||||
nation so conceived and so dedicated, can long endure. We are met on a great
|
||||
battlefield of that war. We have come to dedicate a portion of that field, as a
|
||||
final resting place for those who here gave their lives that that nation
|
||||
might live. It is altogether fitting and proper that we should do this.
|
||||
|
||||
But, in a larger sense, we can not dedicate, we can not consecrate, we can not
|
||||
hallow this ground. The brave men, living and dead, who struggled here, have
|
||||
consecrated it, far above our poor power to add or detract. The world will
|
||||
little note, nor long remember what we say here, but it can never forget what
|
||||
they did here. It is for us the living, rather, to be dedicated here to the
|
||||
unfinished work which they who fought here have thus far so nobly advanced. It
|
||||
is rather for us to be here dedicated to the great task remaining before us—
|
||||
that from these honored dead we take increased devotion to that cause for which
|
||||
they gave the last full measure of devotion— that we here highly resolve that
|
||||
these dead shall not have died in vain— that this nation, under God, shall have
|
||||
a new birth of freedom— and that government of the people, by the people, for
|
||||
the people, shall not perish from the earth.">>.
|
||||
|
||||
random_binary(Start, End) ->
|
||||
Size = byte_size(random_binary_single()) - 1,
|
||||
case End > Size of
|
||||
true ->
|
||||
Copies = ( End div Size ) + 1,
|
||||
D0 = binary:copy(random_binary_single(), Copies),
|
||||
binary:part(<<D0/binary>>, Start, End);
|
||||
false ->
|
||||
binary:part(random_binary_single(), Start, End)
|
||||
end.
|
||||
|
||||
setup() ->
|
||||
{ok, Pid} = machi_file_proxy:start_link(fluname, "test", ?TESTDIR),
|
||||
Pid.
|
||||
|
||||
teardown(Pid) ->
|
||||
catch machi_file_proxy:stop(Pid).
|
||||
|
||||
machi_file_proxy_test_() ->
|
||||
clean_up_data_dir(?TESTDIR),
|
||||
{setup,
|
||||
fun setup/0,
|
||||
fun teardown/1,
|
||||
fun(Pid) ->
|
||||
[
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:read(Pid, -1, -1)),
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:write(Pid, -1, <<"yo">>)),
|
||||
?_assertEqual({error, bad_arg}, machi_file_proxy:append(Pid, [], -1, <<"krep">>)),
|
||||
?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, 1)),
|
||||
?_assertMatch({ok, {_, []}}, machi_file_proxy:read(Pid, 1, 1024)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, 1024, ?HYOOGE)),
|
||||
?_assertEqual({error, not_written}, machi_file_proxy:read(Pid, ?HYOOGE, 1)),
|
||||
{timeout, 10,
|
||||
?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, random_binary(0, ?HYOOGE)))},
|
||||
?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))),
|
||||
?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1024, <<"fail">>)),
|
||||
?_assertEqual({error, written}, machi_file_proxy:write(Pid, 1, <<"fail">>)),
|
||||
?_assertMatch({ok, {[{_, _, _, _}], []}}, machi_file_proxy:read(Pid, 1025, 1000)),
|
||||
?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, [], 1024, <<"mind the gap">>)),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 2060, [], random_binary(0, 1024)))
|
||||
]
|
||||
end}.
|
||||
|
||||
multiple_chunks_read_test_() ->
|
||||
clean_up_data_dir(?TESTDIR),
|
||||
{setup,
|
||||
fun setup/0,
|
||||
fun teardown/1,
|
||||
fun(Pid) ->
|
||||
[
|
||||
?_assertEqual(ok, machi_file_proxy:trim(Pid, 0, 1, false)),
|
||||
?_assertMatch({ok, {[], [{"test", 0, 1}]}},
|
||||
machi_file_proxy:read(Pid, 0, 1,
|
||||
#read_opts{needs_trimmed=true})),
|
||||
?_assertMatch({ok, "test", _}, machi_file_proxy:append(Pid, random_binary(0, 1024))),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 10000, <<"fail">>)),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 20000, <<"fail">>)),
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 30000, <<"fail">>)),
|
||||
%% Freeza
|
||||
?_assertEqual(ok, machi_file_proxy:write(Pid, 530000, <<"fail">>)),
|
||||
?_assertMatch({ok, {[{"test", 1024, _, _},
|
||||
{"test", 10000, <<"fail">>, _},
|
||||
{"test", 20000, <<"fail">>, _},
|
||||
{"test", 30000, <<"fail">>, _},
|
||||
{"test", 530000, <<"fail">>, _}], []}},
|
||||
machi_file_proxy:read(Pid, 1024, 530000)),
|
||||
?_assertMatch({ok, {[{"test", 1, _, _}], [{"test", 0, 1}]}},
|
||||
machi_file_proxy:read(Pid, 0, 1024,
|
||||
#read_opts{needs_trimmed=true}))
|
||||
]
|
||||
end}.
|
||||
|
||||
-endif. % !PULSE
|
||||
-endif. % TEST.
|
|
@ -30,6 +30,22 @@
|
|||
-define(FLU, machi_flu1).
|
||||
-define(FLU_C, machi_flu1_client).
|
||||
|
||||
get_env_vars(App, Ks) ->
|
||||
Raw = [application:get_env(App, K) || K <- Ks],
|
||||
Old = lists:zip(Ks, Raw),
|
||||
{App, Old}.
|
||||
|
||||
clean_up_env_vars({App, Old}) ->
|
||||
[case Res of
|
||||
undefined ->
|
||||
application:unset_env(App, K);
|
||||
{ok, V} ->
|
||||
application:set_env(App, K, V)
|
||||
end || {K, Res} <- Old].
|
||||
|
||||
filter_env_var({ok, V}) -> V;
|
||||
filter_env_var(Else) -> Else.
|
||||
|
||||
clean_up_data_dir(DataDir) ->
|
||||
[begin
|
||||
Fs = filelib:wildcard(DataDir ++ Glob),
|
||||
|
@ -39,10 +55,10 @@ clean_up_data_dir(DataDir) ->
|
|||
_ = file:del_dir(DataDir),
|
||||
ok.
|
||||
|
||||
setup_test_flu(RegName, TcpPort, DataDir) ->
|
||||
setup_test_flu(RegName, TcpPort, DataDir, []).
|
||||
start_flu_package(RegName, TcpPort, DataDir) ->
|
||||
start_flu_package(RegName, TcpPort, DataDir, []).
|
||||
|
||||
setup_test_flu(RegName, TcpPort, DataDir, Props) ->
|
||||
start_flu_package(RegName, TcpPort, DataDir, Props) ->
|
||||
case proplists:get_value(save_data_dir, Props) of
|
||||
true ->
|
||||
ok;
|
||||
|
@ -50,67 +66,89 @@ setup_test_flu(RegName, TcpPort, DataDir, Props) ->
|
|||
clean_up_data_dir(DataDir)
|
||||
end,
|
||||
|
||||
{ok, FLU1} = ?FLU:start_link([{RegName, TcpPort, DataDir}|Props]),
|
||||
%% TODO the process structuring/racy-ness of the various processes
|
||||
%% of the FLU needs to be deterministic to remove this sleep race
|
||||
%% "prevention".
|
||||
timer:sleep(10),
|
||||
FLU1.
|
||||
maybe_start_sup(),
|
||||
machi_flu_psup:start_flu_package(RegName, TcpPort, DataDir, Props).
|
||||
|
||||
stop_flu_package(FluName) ->
|
||||
machi_flu_psup:stop_flu_package(FluName),
|
||||
Pid = whereis(machi_sup),
|
||||
exit(Pid, normal),
|
||||
machi_util:wait_for_death(Pid, 100).
|
||||
|
||||
maybe_start_sup() ->
|
||||
case whereis(machi_sup) of
|
||||
undefined ->
|
||||
machi_sup:start_link(),
|
||||
%% evil but we have to let stuff start up
|
||||
timer:sleep(10),
|
||||
maybe_start_sup();
|
||||
Pid -> Pid
|
||||
end.
|
||||
|
||||
-ifndef(PULSE).
|
||||
|
||||
flu_smoke_test() ->
|
||||
Host = "localhost",
|
||||
TcpPort = 32957,
|
||||
TcpPort = 12957,
|
||||
DataDir = "./data",
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
Prefix = <<"prefix!">>,
|
||||
BadPrefix = BadFile = "no/good",
|
||||
|
||||
W_props = [{initial_wedged, false}],
|
||||
FLU1 = setup_test_flu(smoke_flu, TcpPort, DataDir, W_props),
|
||||
{_, _, _} = machi_test_util:start_flu_package(smoke_flu, TcpPort, DataDir, W_props),
|
||||
try
|
||||
Msg = "Hello, world!",
|
||||
Msg = ?FLU_C:echo(Host, TcpPort, Msg),
|
||||
{error, no_such_file} = ?FLU_C:checksum_list(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
"does-not-exist"),
|
||||
{error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH, BadFile),
|
||||
{error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort,"does-not-exist"),
|
||||
{error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort, BadFile),
|
||||
|
||||
{ok, []} = ?FLU_C:list_files(Host, TcpPort, ?DUMMY_PV1_EPOCH),
|
||||
{ok, {false, _}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
{ok, {false, _,_,_}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
|
||||
Chunk1 = <<"yo!">>,
|
||||
{ok, {Off1,Len1,File1}} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
{ok, {Off1,Len1,File1}} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix, Chunk1),
|
||||
{ok, Chunk1} = ?FLU_C:read_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH,
|
||||
File1, Off1, Len1),
|
||||
{ok, KludgeBin} = ?FLU_C:checksum_list(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH, File1),
|
||||
Prefix, Chunk1, NoCSum),
|
||||
{ok, {[{_, Off1, Chunk1, _}], _}} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
File1, Off1, Len1,
|
||||
noopt),
|
||||
{ok, KludgeBin} = ?FLU_C:checksum_list(Host, TcpPort, File1),
|
||||
true = is_binary(KludgeBin),
|
||||
{error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
{error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
BadPrefix, Chunk1),
|
||||
BadPrefix, Chunk1, NoCSum),
|
||||
{ok, [{_,File1}]} = ?FLU_C:list_files(Host, TcpPort, ?DUMMY_PV1_EPOCH),
|
||||
Len1 = size(Chunk1),
|
||||
{error, not_written} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
File1, Off1*983829323, Len1),
|
||||
{error, partial_read} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
File1, Off1, Len1*9999),
|
||||
NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
File1, Off1*983829323, Len1,
|
||||
noopt),
|
||||
%% XXX FIXME
|
||||
%%
|
||||
%% This is failing because the read extends past the end of the file.
|
||||
%% I guess the semantic here is that we should consider any read which
|
||||
%% *starts* at a valid offset to be a partial read, even if the length
|
||||
%% of the read will cause it to fail.
|
||||
%%
|
||||
%% {error, partial_read} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
%% NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
%% File1, Off1, Len1*9999),
|
||||
|
||||
{ok, {Off1b,Len1b,File1b}} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
{ok, {Off1b,Len1b,File1b}} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix, Chunk1),
|
||||
Prefix, Chunk1,NoCSum),
|
||||
Extra = 42,
|
||||
{ok, {Off1c,Len1c,File1c}} = ?FLU_C:append_chunk_extra(Host, TcpPort,
|
||||
Opts1 = #append_opts{chunk_extra=Extra},
|
||||
{ok, {Off1c,Len1c,File1c}} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix, Chunk1, Extra),
|
||||
Prefix, Chunk1, NoCSum,
|
||||
Opts1, infinity),
|
||||
{ok, {Off1d,Len1d,File1d}} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
NSInfo,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix, Chunk1),
|
||||
Prefix, Chunk1,NoCSum),
|
||||
if File1b == File1c, File1c == File1d ->
|
||||
true = (Off1c == Off1b + Len1b),
|
||||
true = (Off1d == Off1c + Len1c + Extra);
|
||||
|
@ -118,27 +156,44 @@ flu_smoke_test() ->
|
|||
exit(not_mandatory_but_test_expected_same_file_fixme)
|
||||
end,
|
||||
|
||||
Chunk1_cs = {<<?CSUM_TAG_NONE:8, 0:(8*20)>>, Chunk1},
|
||||
{ok, {Off1e,Len1e,File1e}} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix, Chunk1_cs),
|
||||
|
||||
Chunk2 = <<"yo yo">>,
|
||||
Len2 = byte_size(Chunk2),
|
||||
Off2 = ?MINIMUM_OFFSET + 77,
|
||||
File2 = "smoke-whole-file",
|
||||
ok = ?FLU_C:write_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH,
|
||||
File2, Off2, Chunk2),
|
||||
{error, bad_arg} = ?FLU_C:write_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH,
|
||||
BadFile, Off2, Chunk2),
|
||||
{ok, Chunk2} = ?FLU_C:read_chunk(Host, TcpPort, ?DUMMY_PV1_EPOCH,
|
||||
File2, Off2, Len2),
|
||||
{error, not_written} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
"no!!", Off2, Len2),
|
||||
File2 = "smoke-whole-file^^0^1^1",
|
||||
ok = ?FLU_C:write_chunk(Host, TcpPort, NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
File2, Off2, Chunk2, NoCSum),
|
||||
{error, bad_arg} = ?FLU_C:write_chunk(Host, TcpPort, NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
BadFile, Off2, Chunk2, NoCSum),
|
||||
{ok, {[{_, Off2, Chunk2, _}], _}} =
|
||||
?FLU_C:read_chunk(Host, TcpPort, NSInfo, ?DUMMY_PV1_EPOCH, File2, Off2, Len2, noopt),
|
||||
{error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
BadFile, Off2, Len2),
|
||||
NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
"no!!", Off2, Len2, noopt),
|
||||
{error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort,
|
||||
NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
BadFile, Off2, Len2, noopt),
|
||||
|
||||
%% Make a connected socket.
|
||||
Sock1 = ?FLU_C:connect(#p_srvr{address=Host, port=TcpPort}),
|
||||
|
||||
%% Let's test some cluster version enforcement.
|
||||
Good_EpochNum = 0,
|
||||
Good_NSVersion = 0,
|
||||
Good_NS = <<>>,
|
||||
{ok, {false, {Good_EpochNum,_}, Good_NSVersion, GoodNS}} =
|
||||
?FLU_C:wedge_status(Sock1),
|
||||
NS_good = #ns_info{version=Good_NSVersion, name=Good_NS},
|
||||
{ok, {[{_, Off2, Chunk2, _}], _}} =
|
||||
?FLU_C:read_chunk(Sock1, NS_good, ?DUMMY_PV1_EPOCH,
|
||||
File2, Off2, Len2, noopt),
|
||||
NS_bad_version = #ns_info{version=1, name=Good_NS},
|
||||
NS_bad_name = #ns_info{version=Good_NSVersion, name= <<"foons">>},
|
||||
{error, bad_epoch} =
|
||||
?FLU_C:read_chunk(Sock1, NS_bad_version, ?DUMMY_PV1_EPOCH,
|
||||
File2, Off2, Len2, noopt),
|
||||
{error, bad_arg} =
|
||||
?FLU_C:read_chunk(Sock1, NS_bad_name, ?DUMMY_PV1_EPOCH,
|
||||
File2, Off2, Len2, noopt),
|
||||
|
||||
%% We know that File1 still exists. Pretend that we've done a
|
||||
%% migration and exercise the delete_migration() API.
|
||||
|
@ -155,33 +210,32 @@ flu_smoke_test() ->
|
|||
{error, bad_arg} = ?FLU_C:trunc_hack(Host, TcpPort,
|
||||
?DUMMY_PV1_EPOCH, BadFile),
|
||||
|
||||
ok = ?FLU_C:quit(?FLU_C:connect(#p_srvr{address=Host,
|
||||
port=TcpPort}))
|
||||
ok = ?FLU_C:quit(Sock1)
|
||||
after
|
||||
ok = ?FLU:stop(FLU1)
|
||||
machi_test_util:stop_flu_package()
|
||||
end.
|
||||
|
||||
flu_projection_smoke_test() ->
|
||||
Host = "localhost",
|
||||
TcpPort = 32959,
|
||||
DataDir = "./data",
|
||||
|
||||
FLU1 = setup_test_flu(projection_test_flu, TcpPort, DataDir),
|
||||
TcpPort = 12959,
|
||||
DataDir = "./data.projst",
|
||||
{_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir),
|
||||
try
|
||||
[ok = flu_projection_common(Host, TcpPort, T) ||
|
||||
T <- [public, private] ]
|
||||
%% , {ok, {false, EpochID1}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
%% , {ok, {false, EpochID1,_,_}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
%% io:format(user, "EpochID1 ~p\n", [EpochID1])
|
||||
after
|
||||
ok = ?FLU:stop(FLU1)
|
||||
machi_test_util:stop_flu_package()
|
||||
end.
|
||||
|
||||
flu_projection_common(Host, TcpPort, T) ->
|
||||
{ok, {0,_}} = ?FLU_C:get_latest_epochid(Host, TcpPort, T),
|
||||
{error, not_written} =
|
||||
{ok, #projection_v1{epoch_number=0}} =
|
||||
?FLU_C:read_latest_projection(Host, TcpPort, T),
|
||||
{ok, []} = ?FLU_C:list_all_projections(Host, TcpPort, T),
|
||||
{ok, []} = ?FLU_C:get_all_projections(Host, TcpPort, T),
|
||||
{ok, [0]} = ?FLU_C:list_all_projections(Host, TcpPort, T),
|
||||
{ok, [#projection_v1{epoch_number=0}]} =
|
||||
?FLU_C:get_all_projections(Host, TcpPort, T),
|
||||
|
||||
P_a = #p_srvr{name=a, address="localhost", port=4321},
|
||||
P1 = machi_projection:new(1, a, [P_a], [], [a], [], []),
|
||||
|
@ -193,37 +247,39 @@ flu_projection_common(Host, TcpPort, T) ->
|
|||
{ok, P1} = ?FLU_C:read_projection(Host, TcpPort, T, 1),
|
||||
{ok, {1,_}} = ?FLU_C:get_latest_epochid(Host, TcpPort, T),
|
||||
{ok, P1} = ?FLU_C:read_latest_projection(Host, TcpPort, T),
|
||||
{ok, [1]} = ?FLU_C:list_all_projections(Host, TcpPort, T),
|
||||
{ok, [P1]} = ?FLU_C:get_all_projections(Host, TcpPort, T),
|
||||
{ok, [0,1]} = ?FLU_C:list_all_projections(Host, TcpPort, T),
|
||||
{ok, [_,P1]} = ?FLU_C:get_all_projections(Host, TcpPort, T),
|
||||
{error, not_written} = ?FLU_C:read_projection(Host, TcpPort, T, 2),
|
||||
ok.
|
||||
|
||||
bad_checksum_test() ->
|
||||
Host = "localhost",
|
||||
TcpPort = 32960,
|
||||
DataDir = "./data",
|
||||
|
||||
TcpPort = 12960,
|
||||
DataDir = "./data.bct",
|
||||
Opts = [{initial_wedged, false}],
|
||||
FLU1 = setup_test_flu(projection_test_flu, TcpPort, DataDir, Opts),
|
||||
{_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts),
|
||||
NSInfo = undefined,
|
||||
try
|
||||
Prefix = <<"some prefix">>,
|
||||
Chunk1 = <<"yo yo yo">>,
|
||||
Chunk1_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, Chunk1},
|
||||
{error, bad_checksum} = ?FLU_C:append_chunk(Host, TcpPort,
|
||||
BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, ".................")},
|
||||
{error, bad_checksum} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo,
|
||||
?DUMMY_PV1_EPOCH,
|
||||
Prefix, Chunk1_badcs),
|
||||
Prefix,
|
||||
Chunk1, BadCSum),
|
||||
ok
|
||||
after
|
||||
ok = ?FLU:stop(FLU1)
|
||||
machi_test_util:stop_flu_package()
|
||||
end.
|
||||
|
||||
witness_test() ->
|
||||
Host = "localhost",
|
||||
TcpPort = 32961,
|
||||
DataDir = "./data",
|
||||
|
||||
TcpPort = 12961,
|
||||
DataDir = "./data.witness",
|
||||
Opts = [{initial_wedged, false}, {witness_mode, true}],
|
||||
FLU1 = setup_test_flu(projection_test_flu, TcpPort, DataDir, Opts),
|
||||
{_,_,_} = machi_test_util:start_flu_package(projection_test_flu, TcpPort, DataDir, Opts),
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
try
|
||||
Prefix = <<"some prefix">>,
|
||||
Chunk1 = <<"yo yo yo">>,
|
||||
|
@ -236,15 +292,14 @@ witness_test() ->
|
|||
{ok, EpochID1} = ?FLU_C:get_latest_epochid(Host, TcpPort, private),
|
||||
|
||||
%% Witness-protected ops all fail
|
||||
{error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort, EpochID1,
|
||||
Prefix, Chunk1),
|
||||
{error, bad_arg} = ?FLU_C:append_chunk(Host, TcpPort, NSInfo, EpochID1,
|
||||
Prefix, Chunk1, NoCSum),
|
||||
File = <<"foofile">>,
|
||||
{error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort, EpochID1,
|
||||
File, 9999, 9999),
|
||||
{error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort, EpochID1,
|
||||
File),
|
||||
{error, bad_arg} = ?FLU_C:read_chunk(Host, TcpPort, NSInfo, EpochID1,
|
||||
File, 9999, 9999, noopt),
|
||||
{error, bad_arg} = ?FLU_C:checksum_list(Host, TcpPort, File),
|
||||
{error, bad_arg} = ?FLU_C:list_files(Host, TcpPort, EpochID1),
|
||||
{ok, {false, EpochID1}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
{ok, {false, EpochID1,_,_}} = ?FLU_C:wedge_status(Host, TcpPort),
|
||||
{ok, _} = ?FLU_C:get_latest_epochid(Host, TcpPort, public),
|
||||
{ok, _} = ?FLU_C:read_latest_projection(Host, TcpPort, public),
|
||||
{error, not_written} = ?FLU_C:read_projection(Host, TcpPort,
|
||||
|
@ -255,7 +310,7 @@ witness_test() ->
|
|||
|
||||
ok
|
||||
after
|
||||
ok = ?FLU:stop(FLU1)
|
||||
machi_test_util:stop_flu_package()
|
||||
end.
|
||||
|
||||
%% The purpose of timing_pb_encoding_test_ and timing_bif_encoding_test_ is
|
||||
|
@ -293,7 +348,7 @@ timing_pb_encoding_test2() ->
|
|||
RUN2 = timer:tc(fun() -> begin [_ = DoIt2() || _ <- XX], ok end end),
|
||||
erlang:garbage_collect(),
|
||||
Factor = (element(1, RUN1) / element(1, RUN2)),
|
||||
io:format(user, " speed factor=~.2f ", [Factor]),
|
||||
io:format(" speed factor=~.2f ", [Factor]),
|
||||
ok.
|
||||
|
||||
-endif. % !PULSE
|
||||
|
|
|
@ -38,12 +38,12 @@ smoke_test_() ->
|
|||
{timeout, 5*60, fun() -> smoke_test2() end}.
|
||||
|
||||
smoke_test2() ->
|
||||
Ps = [{a,#p_srvr{name=a, address="localhost", port=5555, props="./data.a"}},
|
||||
{b,#p_srvr{name=b, address="localhost", port=5556, props="./data.b"}},
|
||||
{c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}}
|
||||
Ps = [{a,#p_srvr{name=a, address="localhost", port=5550, props="./data.a"}},
|
||||
{b,#p_srvr{name=b, address="localhost", port=5551, props="./data.b"}},
|
||||
{c,#p_srvr{name=c, address="localhost", port=5552, props="./data.c"}}
|
||||
],
|
||||
[os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps],
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
{ok, SupPid} = machi_sup:start_link(),
|
||||
try
|
||||
%% Only run a, don't run b & c so we have 100% failures talking to them
|
||||
[begin
|
||||
|
@ -66,15 +66,15 @@ partial_stop_restart_test_() ->
|
|||
{timeout, 5*60, fun() -> partial_stop_restart2() end}.
|
||||
|
||||
partial_stop_restart2() ->
|
||||
Ps = [{a,#p_srvr{name=a, address="localhost", port=5555, props="./data.a"}},
|
||||
{b,#p_srvr{name=b, address="localhost", port=5556, props="./data.b"}},
|
||||
{c,#p_srvr{name=c, address="localhost", port=5557, props="./data.c"}}
|
||||
Ps = [{a,#p_srvr{name=a, address="localhost", port=5560, props="./data.a"}},
|
||||
{b,#p_srvr{name=b, address="localhost", port=5561, props="./data.b"}},
|
||||
{c,#p_srvr{name=c, address="localhost", port=5562, props="./data.c"}}
|
||||
],
|
||||
ChMgrs = [machi_flu_psup:make_mgr_supname(P#p_srvr.name) || {_,P} <-Ps],
|
||||
PStores = [machi_flu_psup:make_proj_supname(P#p_srvr.name) || {_,P} <-Ps],
|
||||
Dict = orddict:from_list(Ps),
|
||||
[os:cmd("rm -rf " ++ P#p_srvr.props) || {_,P} <- Ps],
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
{ok, SupPid} = machi_sup:start_link(),
|
||||
DbgProps = [{initial_wedged, true}],
|
||||
Start = fun({_,P}) ->
|
||||
#p_srvr{name=Name, port=Port, props=Dir} = P,
|
||||
|
@ -84,20 +84,23 @@ partial_stop_restart2() ->
|
|||
WedgeStatus = fun({_,#p_srvr{address=Addr, port=TcpPort}}) ->
|
||||
machi_flu1_client:wedge_status(Addr, TcpPort)
|
||||
end,
|
||||
NSInfo = undefined,
|
||||
Append = fun({_,#p_srvr{address=Addr, port=TcpPort}}, EpochID) ->
|
||||
NoCSum = <<>>,
|
||||
machi_flu1_client:append_chunk(Addr, TcpPort,
|
||||
EpochID,
|
||||
<<"prefix">>, <<"data">>)
|
||||
NSInfo, EpochID,
|
||||
<<"prefix">>,
|
||||
<<"data">>, NoCSum)
|
||||
end,
|
||||
try
|
||||
[Start(P) || P <- Ps],
|
||||
[{ok, {true, _}} = WedgeStatus(P) || P <- Ps], % all are wedged
|
||||
[{ok, {true, _,_,_}} = WedgeStatus(P) || P <- Ps], % all are wedged
|
||||
[{error,wedged} = Append(P, ?DUMMY_PV1_EPOCH) || P <- Ps], % all are wedged
|
||||
|
||||
[machi_chain_manager1:set_chain_members(ChMgr, Dict) ||
|
||||
ChMgr <- ChMgrs ],
|
||||
{ok, {false, EpochID1}} = WedgeStatus(hd(Ps)),
|
||||
[{ok, {false, EpochID1}} = WedgeStatus(P) || P <- Ps], % *not* wedged
|
||||
{ok, {false, EpochID1,_,_}} = WedgeStatus(hd(Ps)),
|
||||
[{ok, {false, EpochID1,_,_}} = WedgeStatus(P) || P <- Ps], % *not* wedged
|
||||
[{ok,_} = Append(P, EpochID1) || P <- Ps], % *not* wedged
|
||||
{ok, {_,_,File1}} = Append(hd(Ps), EpochID1),
|
||||
|
||||
|
@ -123,9 +126,9 @@ partial_stop_restart2() ->
|
|||
Epoch_m = Proj_m#projection_v1.epoch_number,
|
||||
%% Confirm that all FLUs are *not* wedged, with correct proj & epoch
|
||||
Proj_mCSum = Proj_m#projection_v1.epoch_csum,
|
||||
[{ok, {false, {Epoch_m, Proj_mCSum}}} = WedgeStatus(P) || % *not* wedged
|
||||
[{ok, {false, {Epoch_m, Proj_mCSum},_,_}} = WedgeStatus(P) || % *not* wedged
|
||||
P <- Ps],
|
||||
{ok, {false, EpochID1}} = WedgeStatus(hd(Ps)),
|
||||
{ok, {false, EpochID1,_,_}} = WedgeStatus(hd(Ps)),
|
||||
[{ok,_} = Append(P, EpochID1) || P <- Ps], % *not* wedged
|
||||
|
||||
%% Stop all but 'a'.
|
||||
|
@ -145,10 +148,10 @@ partial_stop_restart2() ->
|
|||
{error, wedged} = Append(hd(Ps), EpochID1),
|
||||
{_, #p_srvr{address=Addr_a, port=TcpPort_a}} = hd(Ps),
|
||||
{error, wedged} = machi_flu1_client:read_chunk(
|
||||
Addr_a, TcpPort_a, ?DUMMY_PV1_EPOCH,
|
||||
<<>>, 99999999, 1),
|
||||
{error, wedged} = machi_flu1_client:checksum_list(
|
||||
Addr_a, TcpPort_a, ?DUMMY_PV1_EPOCH, <<>>),
|
||||
Addr_a, TcpPort_a, NSInfo, ?DUMMY_PV1_EPOCH,
|
||||
<<>>, 99999999, 1, undefined),
|
||||
{error, bad_arg} = machi_flu1_client:checksum_list(
|
||||
Addr_a, TcpPort_a, <<>>),
|
||||
%% list_files() is permitted despite wedged status
|
||||
{ok, _} = machi_flu1_client:list_files(
|
||||
Addr_a, TcpPort_a, ?DUMMY_PV1_EPOCH),
|
||||
|
@ -157,7 +160,7 @@ partial_stop_restart2() ->
|
|||
{now_using,_,Epoch_n} = machi_chain_manager1:trigger_react_to_env(
|
||||
hd(ChMgrs)),
|
||||
true = (Epoch_n > Epoch_m),
|
||||
{ok, {false, EpochID3}} = WedgeStatus(hd(Ps)),
|
||||
{ok, {false, EpochID3,_,_}} = WedgeStatus(hd(Ps)),
|
||||
%% The file we're assigned should be different with the epoch change.
|
||||
{ok, {_,_,File3}} = Append(hd(Ps), EpochID3),
|
||||
true = (File1 /= File3),
|
||||
|
@ -173,6 +176,19 @@ partial_stop_restart2() ->
|
|||
ok
|
||||
end.
|
||||
|
||||
p_srvr_rec_test() ->
|
||||
P = #p_srvr{name=a, address="localhost", port=1024, props=[yo]},
|
||||
[P] = machi_flu_sup:sanitize_p_srvr_records([P]),
|
||||
[P] = machi_flu_sup:sanitize_p_srvr_records([P,P]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([nope]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{proto_mod=does_not_exist}]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{proto_mod="lists"}]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{address=7}]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{port=5}]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{port=foo}]),
|
||||
[] = machi_flu_sup:sanitize_p_srvr_records([#p_srvr{props=foo}]),
|
||||
ok.
|
||||
|
||||
-endif. % !PULSE
|
||||
-endif. % TEST
|
||||
|
||||
|
|
307
test/machi_lifecycle_mgr_test.erl
Normal file
307
test/machi_lifecycle_mgr_test.erl
Normal file
|
@ -0,0 +1,307 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2014 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_lifecycle_mgr_test).
|
||||
-compile(export_all).
|
||||
|
||||
-ifdef(TEST).
|
||||
-ifndef(PULSE).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
|
||||
-define(MGR, machi_chain_manager1).
|
||||
|
||||
setup() ->
|
||||
catch application:stop(machi),
|
||||
{ok, SupPid} = machi_sup:start_link(),
|
||||
error_logger:tty(false),
|
||||
Dir = "./" ++ atom_to_list(?MODULE) ++ ".datadir",
|
||||
machi_flu1_test:clean_up_data_dir(Dir ++ "/*/*"),
|
||||
machi_flu1_test:clean_up_data_dir(Dir),
|
||||
Envs = [{flu_data_dir, Dir ++ "/data/flu"},
|
||||
{flu_config_dir, Dir ++ "/etc/flu-config"},
|
||||
{chain_config_dir, Dir ++ "/etc/chain-config"},
|
||||
{platform_data_dir, Dir ++ "/data"},
|
||||
{platform_etc_dir, Dir ++ "/etc"},
|
||||
{not_used_pending, Dir ++ "/etc/pending"}
|
||||
],
|
||||
EnvKeys = [K || {K,_V} <- Envs],
|
||||
undefined = application:get_env(machi, yo),
|
||||
Cleanup = machi_flu1_test:get_env_vars(machi, EnvKeys ++ [yo]),
|
||||
[begin
|
||||
filelib:ensure_dir(V ++ "/unused"),
|
||||
application:set_env(machi, K, V)
|
||||
end || {K, V} <- Envs],
|
||||
{SupPid, Dir, Cleanup}.
|
||||
|
||||
cleanup({SupPid, Dir, Cleanup}) ->
|
||||
exit(SupPid, normal),
|
||||
machi_util:wait_for_death(SupPid, 100),
|
||||
error_logger:tty(true),
|
||||
catch application:stop(machi),
|
||||
machi_flu1_test:clean_up_data_dir(Dir ++ "/*/*"),
|
||||
machi_flu1_test:clean_up_data_dir(Dir),
|
||||
machi_flu1_test:clean_up_env_vars(Cleanup),
|
||||
undefined = application:get_env(machi, yo),
|
||||
ok.
|
||||
|
||||
smoke_test_() ->
|
||||
{timeout, 60, fun() -> smoke_test2() end}.
|
||||
|
||||
smoke_test2() ->
|
||||
YoCleanup = setup(),
|
||||
try
|
||||
Prefix = <<"pre">>,
|
||||
Chunk1 = <<"yochunk">>,
|
||||
Host = "localhost",
|
||||
PortBase = 60120,
|
||||
|
||||
Pa = #p_srvr{name=a,address="localhost",port=PortBase+0},
|
||||
Pb = #p_srvr{name=b,address="localhost",port=PortBase+1},
|
||||
Pc = #p_srvr{name=c,address="localhost",port=PortBase+2},
|
||||
%% Pstore_a = machi_flu1:make_projection_server_regname(a),
|
||||
%% Pstore_b = machi_flu1:make_projection_server_regname(b),
|
||||
%% Pstore_c = machi_flu1:make_projection_server_regname(c),
|
||||
Pstores = [Pstore_a, Pstore_b, Pstore_c] =
|
||||
[machi_flu1:make_projection_server_regname(a),
|
||||
machi_flu1:make_projection_server_regname(b),
|
||||
machi_flu1:make_projection_server_regname(c)],
|
||||
ChMgrs = [ChMgr_a, ChMgr_b, ChMgr_c] =
|
||||
[machi_chain_manager1:make_chmgr_regname(a),
|
||||
machi_chain_manager1:make_chmgr_regname(b),
|
||||
machi_chain_manager1:make_chmgr_regname(c)],
|
||||
Fits = [Fit_a, Fit_b, Fit_c] =
|
||||
[machi_flu_psup:make_fitness_regname(a),
|
||||
machi_flu_psup:make_fitness_regname(b),
|
||||
machi_flu_psup:make_fitness_regname(c)],
|
||||
Advance = machi_chain_manager1_test:make_advance_fun(
|
||||
Fits, [a,b,c], ChMgrs, 3),
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("\nSTEP: Start 3 FLUs, no chain.\n", []),
|
||||
|
||||
[machi_lifecycle_mgr:make_pending_config(P) || P <- [Pa,Pb,Pc] ],
|
||||
{[_,_,_],[]} = machi_lifecycle_mgr:process_pending(),
|
||||
[{ok, #projection_v1{epoch_number=0}} =
|
||||
machi_projection_store:read_latest_projection(PSTORE, private)
|
||||
|| PSTORE <- Pstores],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("\nSTEP: Start chain = [a,b,c]\n", []),
|
||||
|
||||
C1 = #chain_def_v1{name=cx, mode=ap_mode, full=[Pa,Pb,Pc],
|
||||
local_run=[a,b,c]},
|
||||
machi_lifecycle_mgr:make_pending_config(C1),
|
||||
{[],[_]} = machi_lifecycle_mgr:process_pending(),
|
||||
Advance(),
|
||||
[{ok, #projection_v1{all_members=[a,b,c]}} =
|
||||
machi_projection_store:read_latest_projection(PSTORE, private)
|
||||
|| PSTORE <- Pstores],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("\nSTEP: Reset chain = [b,c]\n", []),
|
||||
|
||||
C2 = #chain_def_v1{name=cx, mode=ap_mode, full=[Pb,Pc],
|
||||
old_full=[a,b,c], old_witnesses=[],
|
||||
local_stop=[a], local_run=[b,c]},
|
||||
machi_lifecycle_mgr:make_pending_config(C2),
|
||||
{[],[_]} = machi_lifecycle_mgr:process_pending(),
|
||||
Advance(),
|
||||
%% a should be down
|
||||
{'EXIT', _} = (catch machi_projection_store:read_latest_projection(
|
||||
hd(Pstores), private)),
|
||||
[{ok, #projection_v1{all_members=[b,c]}} =
|
||||
machi_projection_store:read_latest_projection(PSTORE, private)
|
||||
|| PSTORE <- tl(Pstores)],
|
||||
|
||||
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
|
||||
io:format("\nSTEP: Reset chain = []\n", []),
|
||||
|
||||
C3 = #chain_def_v1{name=cx, mode=ap_mode, full=[],
|
||||
old_full=[b,c], old_witnesses=[],
|
||||
local_stop=[b,c], local_run=[]},
|
||||
machi_lifecycle_mgr:make_pending_config(C3),
|
||||
{[],[_]} = machi_lifecycle_mgr:process_pending(),
|
||||
Advance(),
|
||||
%% a,b,c should be down
|
||||
[{'EXIT', _} = (catch machi_projection_store:read_latest_projection(
|
||||
PSTORE, private))
|
||||
|| PSTORE <- Pstores],
|
||||
|
||||
ok
|
||||
after
|
||||
cleanup(YoCleanup)
|
||||
end.
|
||||
|
||||
ast_tuple_syntax_test() ->
|
||||
T = fun(L) -> machi_lifecycle_mgr:check_ast_tuple_syntax(L) end,
|
||||
Canon1 = [ {host, "localhost", []},
|
||||
{host, "localhost", [{client_interface, "1.2.3.4"},
|
||||
{admin_interface, "5.6.7.8"}]},
|
||||
{flu, 'fx', "foohost", 4000, []},
|
||||
switch_old_and_new,
|
||||
{chain, 'cy', ['fx', 'fy'], [{foo,"yay"},{bar,baz}]} ],
|
||||
|
||||
{_Good,[]=_Bad} = T(Canon1),
|
||||
Canon1_norm = machi_lifecycle_mgr:normalize_ast_tuple_syntax(Canon1),
|
||||
true = (length(Canon1) == length(Canon1_norm)),
|
||||
{Canon1_norm_b, []} = T(Canon1_norm),
|
||||
true = (length(Canon1_norm) == length(Canon1_norm_b)),
|
||||
|
||||
{[],[_,_,_,_]} =
|
||||
T([ {host, 'localhost', []},
|
||||
{host, 'localhost', yo},
|
||||
{host, "localhost", [{client_interface, 77.88293829832}]},
|
||||
{host, "localhost", [{client_interface, "1.2.3.4"},
|
||||
{bummer, "5.6.7.8"}]} ]),
|
||||
{[],[_,_,_,_,_,_]} =
|
||||
T([ {flu, 'fx', 'foohost', 4000, []},
|
||||
{flu, 'fx', <<"foohost">>, 4000, []},
|
||||
{flu, 'fx', "foohost", -4000, []},
|
||||
{flu, 'fx', "foohost", 40009999, []},
|
||||
{flu, 'fx', "foohost", 4000, gack},
|
||||
{flu, 'fx', "foohost", 4000, [22]} ]),
|
||||
{[],[_,_,_]} =
|
||||
T([ {chain, 'cy', ["fx", "fy"], [foo,{bar,baz}]},
|
||||
yoloyolo,
|
||||
{chain, "cy", ["fx", 27], oops,arity,way,way,way,too,big,x}
|
||||
]).
|
||||
|
||||
ast_run_test() ->
|
||||
PortBase = 20300,
|
||||
R1 = [
|
||||
{host, "localhost", "localhost", "localhost", []},
|
||||
{flu, 'f0', "localhost", PortBase+0, []},
|
||||
{flu, 'f1', "localhost", PortBase+1, []},
|
||||
{chain, 'ca', ['f0'], []},
|
||||
{chain, 'cb', ['f1'], []},
|
||||
switch_old_and_new,
|
||||
{flu, 'f2', "localhost", PortBase+2, []},
|
||||
{flu, 'f3', "localhost", PortBase+3, []},
|
||||
{flu, 'f4', "localhost", PortBase+4, []},
|
||||
{chain, 'ca', ['f0', 'f2'], []},
|
||||
{chain, 'cc', ['f3', 'f4'], []}
|
||||
],
|
||||
|
||||
{ok, Env1} = machi_lifecycle_mgr:run_ast(R1),
|
||||
%% Uncomment to examine the Env trees.
|
||||
%% Y1 = {lists:sort(gb_trees:to_list(element(1, Env1))),
|
||||
%% lists:sort(gb_trees:to_list(element(2, Env1))),
|
||||
%% element(3, Env1)},
|
||||
%% io:format(user, "\nY1 ~p\n", [Y1]),
|
||||
|
||||
Negative_after_R1 =
|
||||
[
|
||||
{host, "localhost", "foo", "foo", []}, % dupe host
|
||||
{flu, 'f1', "other", PortBase+9999999, []}, % bogus port # (syntax)
|
||||
{flu, 'f1', "other", PortBase+888, []}, % dupe flu name
|
||||
{flu, 'f7', "localhost", PortBase+1, []}, % dupe host+port
|
||||
{chain, 'ca', ['f7'], []}, % unknown flu
|
||||
{chain, 'cc', ['f0'], []}, % flu previously assigned
|
||||
{chain, 'ca', cp_mode, ['f0', 'f1', 'f2'], [], []} % mode change
|
||||
],
|
||||
[begin
|
||||
%% io:format(user, "dbg: Neg ~p\n", [Neg]),
|
||||
{error, _} = machi_lifecycle_mgr:run_ast(R1 ++ [Neg])
|
||||
end || Neg <- Negative_after_R1],
|
||||
|
||||
%% The 'run' phase doesn't blow smoke. What about 'diff'?
|
||||
{X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, "localhost"),
|
||||
%% There's only one host, "localhost", so 'all' should be exactly equal.
|
||||
{X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, all),
|
||||
%% io:format(user, "X1b: ~p\n", [X1b]),
|
||||
|
||||
%% Append to the R1 scenario: for chain cc: add f5, remove f4
|
||||
%% Expect: see pattern matching below on X2b.
|
||||
R2 = (R1 -- [switch_old_and_new]) ++
|
||||
[switch_old_and_new,
|
||||
{flu, 'f5', "localhost", PortBase+5, []},
|
||||
{chain, 'cc', ['f3','f5'], []}],
|
||||
{ok, Env2} = machi_lifecycle_mgr:run_ast(R2),
|
||||
{_X2a, X2b} = machi_lifecycle_mgr:diff_env(Env2, "localhost"),
|
||||
%% io:format(user, "X2b: ~p\n", [X2b]),
|
||||
F5_port = PortBase+5,
|
||||
[#p_srvr{name='f5',address="localhost",port=F5_port},
|
||||
#chain_def_v1{name='cc',
|
||||
full=[#p_srvr{name='f3'},#p_srvr{name='f5'}], witnesses=[],
|
||||
old_full=[f3,f4], old_witnesses=[],
|
||||
local_run=[f5], local_stop=[f4]}] = X2b,
|
||||
|
||||
ok.
|
||||
|
||||
ast_then_apply_test_() ->
|
||||
{timeout, 60, fun() -> ast_then_apply_test2() end}.
|
||||
|
||||
ast_then_apply_test2() ->
|
||||
YoCleanup = setup(),
|
||||
try
|
||||
PortBase = 20400,
|
||||
NumChains = 4,
|
||||
ChainLen = 3,
|
||||
FLU_num = NumChains * ChainLen,
|
||||
FLU_defs = [{flu, list_to_atom("f"++integer_to_list(X)),
|
||||
"localhost", PortBase+X, []} || X <- lists:seq(1,FLU_num)],
|
||||
FLU_names = [FLU || {flu,FLU,_,_,_} <- FLU_defs],
|
||||
Ch_defs = [{chain, list_to_atom("c"++integer_to_list(X)),
|
||||
lists:sublist(FLU_names, X, 3),
|
||||
[]} || X <- lists:seq(1, FLU_num, 3)],
|
||||
|
||||
R1 = [switch_old_and_new,
|
||||
{host, "localhost", "localhost", "localhost", []}]
|
||||
++ FLU_defs ++ Ch_defs,
|
||||
{ok, Env1} = machi_lifecycle_mgr:run_ast(R1),
|
||||
{_X1a, X1b} = machi_lifecycle_mgr:diff_env(Env1, "localhost"),
|
||||
%% io:format(user, "X1b ~p\n", [X1b]),
|
||||
[machi_lifecycle_mgr:make_pending_config(X) || X <- X1b],
|
||||
{PassFLUs, PassChains} = machi_lifecycle_mgr:process_pending(),
|
||||
true = (length(PassFLUs) == length(FLU_defs)),
|
||||
true = (length(PassChains) == length(Ch_defs)),
|
||||
|
||||
%% Kick the chain managers into doing something useful right now.
|
||||
Pstores = [list_to_atom(atom_to_list(X) ++ "_pstore") || X <- FLU_names],
|
||||
Fits = [list_to_atom(atom_to_list(X) ++ "_fitness") || X <- FLU_names],
|
||||
ChMgrs = [list_to_atom(atom_to_list(X) ++ "_chmgr") || X <- FLU_names],
|
||||
Advance = machi_chain_manager1_test:make_advance_fun(
|
||||
Fits, FLU_names, ChMgrs, 3),
|
||||
Advance(),
|
||||
|
||||
%% Sanity check: everyone is configured properly.
|
||||
[begin
|
||||
{ok, #projection_v1{epoch_number=Epoch, all_members=All,
|
||||
chain_name=ChainName, upi=UPI}} =
|
||||
machi_projection_store:read_latest_projection(PStore, private),
|
||||
%% io:format(user, "~p: epoch ~p all ~p\n", [PStore, Epoch, All]),
|
||||
true = Epoch > 0,
|
||||
ChainLen = length(All),
|
||||
true = (length(UPI) > 0),
|
||||
{chain, _, Full, []} = lists:keyfind(ChainName, 2, Ch_defs),
|
||||
true = lists:sort(Full) == lists:sort(All)
|
||||
end || PStore <- Pstores],
|
||||
|
||||
ok
|
||||
after
|
||||
cleanup(YoCleanup)
|
||||
end.
|
||||
|
||||
-endif. % !PULSE
|
||||
-endif. % TEST
|
200
test/machi_merkle_tree_test.erl
Normal file
200
test/machi_merkle_tree_test.erl
Normal file
|
@ -0,0 +1,200 @@
|
|||
%% -------------------------------------------------------------------
|
||||
%%
|
||||
%% Copyright (c) 2007-2015 Basho Technologies, Inc. All Rights Reserved.
|
||||
%%
|
||||
%% This file is provided to you under the Apache License,
|
||||
%% Version 2.0 (the "License"); you may not use this file
|
||||
%% except in compliance with the License. You may obtain
|
||||
%% a copy of the License at
|
||||
%%
|
||||
%% http://www.apache.org/licenses/LICENSE-2.0
|
||||
%%
|
||||
%% Unless required by applicable law or agreed to in writing,
|
||||
%% software distributed under the License is distributed on an
|
||||
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
%% KIND, either express or implied. See the License for the
|
||||
%% specific language governing permissions and limitations
|
||||
%% under the License.
|
||||
%%
|
||||
%% -------------------------------------------------------------------
|
||||
|
||||
-module(machi_merkle_tree_test).
|
||||
-compile([export_all]).
|
||||
|
||||
-include("machi_merkle_tree.hrl").
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
-include_lib("kernel/include/file.hrl").
|
||||
|
||||
-define(GAP_CHANCE, 0.10).
|
||||
|
||||
%% unit tests
|
||||
basic_test() ->
|
||||
random:seed(os:timestamp()),
|
||||
Fsz = choose_size() * 1024,
|
||||
Filesize = max(Fsz, 10*1024*1024),
|
||||
ChunkSize = max(1048576, Filesize div 100),
|
||||
N = make_leaf_nodes(Filesize),
|
||||
D0 = #naive{ leaves = N, chunk_size = ChunkSize, recalc = true },
|
||||
T1 = machi_merkle_tree:build_tree(D0),
|
||||
|
||||
D1 = #naive{ leaves = tl(N), chunk_size = ChunkSize, recalc = true },
|
||||
T2 = machi_merkle_tree:build_tree(D1),
|
||||
|
||||
?assertNotEqual(T1#naive.root, T2#naive.root),
|
||||
?assertEqual(true, length(machi_merkle_tree:naive_diff(T1, T2)) == 1
|
||||
orelse
|
||||
Filesize > ChunkSize).
|
||||
|
||||
|
||||
make_leaf_nodes(Filesize) ->
|
||||
lists:reverse(
|
||||
lists:foldl(fun(T, Acc) -> machi_merkle_tree:update_acc(T, Acc) end,
|
||||
[],
|
||||
generate_offsets(Filesize, 1024, []))
|
||||
).
|
||||
|
||||
choose_int(Factor) ->
|
||||
random:uniform(1024*Factor).
|
||||
|
||||
small_int() ->
|
||||
choose_int(10).
|
||||
|
||||
medium_int() ->
|
||||
choose_int(1024).
|
||||
|
||||
large_int() ->
|
||||
choose_int(4096).
|
||||
|
||||
generate_offsets(Filesize, Current, Acc) when Current < Filesize ->
|
||||
Length0 = choose_size(),
|
||||
|
||||
Length = case Length0 + Current > Filesize of
|
||||
false -> Length0;
|
||||
true -> Filesize - Current
|
||||
end,
|
||||
Data = term_to_binary(os:timestamp()),
|
||||
Checksum = machi_util:make_tagged_csum(client_sha, machi_util:checksum_chunk(Data)),
|
||||
Gap = maybe_gap(random:uniform()),
|
||||
generate_offsets(Filesize, Current + Length + Gap, [ {Current, Length, Checksum} | Acc ]);
|
||||
generate_offsets(_Filesize, _Current, Acc) ->
|
||||
lists:reverse(Acc).
|
||||
|
||||
|
||||
random_from_list(L) ->
|
||||
N = random:uniform(length(L)),
|
||||
lists:nth(N, L).
|
||||
|
||||
choose_size() ->
|
||||
F = random_from_list([fun small_int/0, fun medium_int/0, fun large_int/0]),
|
||||
F().
|
||||
|
||||
maybe_gap(Chance) when Chance < ?GAP_CHANCE ->
|
||||
choose_size();
|
||||
maybe_gap(_) -> 0.
|
||||
|
||||
%% Define or remove these ifdefs if benchmarking is desired.
|
||||
-ifdef(BENCH).
|
||||
generate_offsets(FH, Filesize, Current, Acc) when Current < Filesize ->
|
||||
Length0 = choose_size(),
|
||||
|
||||
Length = case Length0 + Current > Filesize of
|
||||
false -> Length0;
|
||||
true -> Filesize - Current
|
||||
end,
|
||||
{ok, Data} = file:pread(FH, Current, Length),
|
||||
Checksum = machi_util:make_tagged_csum(client_sha, machi_util:checksum_chunk(Data)),
|
||||
Gap = maybe_gap(random:uniform()),
|
||||
generate_offsets(FH, Filesize, Current + Length + Gap, [ {Current, Length, Checksum} | Acc ]);
|
||||
generate_offsets(_FH, _Filesize, _Current, Acc) ->
|
||||
lists:reverse(Acc).
|
||||
|
||||
make_offsets_from_file(Filename) ->
|
||||
{ok, Info} = file:read_file_info(Filename),
|
||||
Filesize = Info#file_info.size,
|
||||
{ok, FH} = file:open(Filename, [read, raw, binary]),
|
||||
Offsets = generate_offsets(FH, Filesize, 1024, []),
|
||||
file:close(FH),
|
||||
Offsets.
|
||||
|
||||
choose_filename() ->
|
||||
random_from_list([
|
||||
"def^c5ea7511-d649-47d6-a8c3-2b619379c237^1",
|
||||
"jkl^b077eff7-b2be-4773-a73f-fea4acb8a732^1",
|
||||
"stu^553fa47a-157c-4fac-b10f-2252c7d8c37a^1",
|
||||
"vwx^ae015d68-7689-4c9f-9677-926c6664f513^1",
|
||||
"yza^4c784dc2-19bf-4ac6-91f6-58bbe5aa88e0^1"
|
||||
]).
|
||||
|
||||
|
||||
make_csum_file(DataDir, Filename, Offsets) ->
|
||||
Path = machi_util:make_checksum_filename(DataDir, Filename),
|
||||
filelib:ensure_dir(Path),
|
||||
{ok, MC} = machi_csum_table:open(Path, []),
|
||||
lists:foreach(fun({Offset, Size, Checksum}) ->
|
||||
machi_csum_table:write(MC, Offset, Size, Checksum) end,
|
||||
Offsets),
|
||||
machi_csum_table:close(MC).
|
||||
|
||||
|
||||
test() ->
|
||||
test(100).
|
||||
|
||||
test(N) ->
|
||||
{ok, F} = file:open("results.txt", [raw, write]),
|
||||
lists:foreach(fun(X) -> format_and_store(F, run_test(X)) end, lists:seq(1, N)).
|
||||
|
||||
format_and_store(F, {OffsetNum, {MTime, MSize}, {NTime, NSize}}) ->
|
||||
S = io_lib:format("~w\t~w\t~w\t~w\t~w\n", [OffsetNum, MTime, MSize, NTime, NSize]),
|
||||
ok = file:write(F, S).
|
||||
|
||||
run_test(C) ->
|
||||
random:seed(os:timestamp()),
|
||||
OffsetFn = "test/" ++ choose_filename(),
|
||||
O = make_offsets_from_file(OffsetFn),
|
||||
Fn = "csum_" ++ integer_to_list(C),
|
||||
make_csum_file(".", Fn, O),
|
||||
|
||||
Osize = length(O),
|
||||
|
||||
{MTime, {ok, M}} = timer:tc(fun() -> machi_merkle_tree:open(Fn, ".", merklet) end),
|
||||
{NTime, {ok, N}} = timer:tc(fun() -> machi_merkle_tree:open(Fn, ".", naive) end),
|
||||
|
||||
?assertEqual(Fn, machi_merkle_tree:filename(M)),
|
||||
?assertEqual(Fn, machi_merkle_tree:filename(N)),
|
||||
|
||||
MTree = machi_merkle_tree:tree(M),
|
||||
MSize = byte_size(term_to_binary(MTree)),
|
||||
|
||||
NTree = machi_merkle_tree:tree(N),
|
||||
NSize = byte_size(term_to_binary(NTree)),
|
||||
|
||||
?assertEqual(same, machi_merkle_tree:diff(N, N)),
|
||||
?assertEqual(same, machi_merkle_tree:diff(M, M)),
|
||||
{Osize, {MTime, MSize}, {NTime, NSize}}.
|
||||
|
||||
torture_test(C) ->
|
||||
Results = [ run_torture_test() || _ <- lists:seq(1, C) ],
|
||||
{ok, F} = file:open("torture_results.txt", [raw, write]),
|
||||
lists:foreach(fun({MSize, MTime, NSize, NTime}) ->
|
||||
file:write(F, io_lib:format("~p\t~p\t~p\t~p\n",
|
||||
[MSize, MTime, NSize, NTime]))
|
||||
end, Results),
|
||||
ok = file:close(F).
|
||||
|
||||
run_torture_test() ->
|
||||
{NTime, N} = timer:tc(fun() -> naive_torture() end),
|
||||
|
||||
MSize = byte_size(term_to_binary(M)),
|
||||
NSize = byte_size(term_to_binary(N)),
|
||||
|
||||
{MSize, MTime, NSize, NTime}.
|
||||
|
||||
naive_torture() ->
|
||||
N = lists:foldl(fun(T, Acc) -> machi_merkle_tree:update_acc(T, Acc) end, [], torture_generator()),
|
||||
T = #naive{ leaves = lists:reverse(N), chunk_size = 10010, recalc = true },
|
||||
machi_merkle_tree:build_tree(T).
|
||||
|
||||
torture_generator() ->
|
||||
[ {O, 1, crypto:hash(sha, term_to_binary(now()))} || O <- lists:seq(1024, 1000000) ].
|
||||
-endif. % BENCH
|
|
@ -24,6 +24,7 @@
|
|||
-ifdef(TEST).
|
||||
-ifndef(PULSE).
|
||||
|
||||
-include("machi.hrl").
|
||||
-include("machi_pb.hrl").
|
||||
-include("machi_projection.hrl").
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
@ -34,20 +35,15 @@ smoke_test_() ->
|
|||
{timeout, 5*60, fun() -> smoke_test2() end}.
|
||||
|
||||
smoke_test2() ->
|
||||
Port = 5720,
|
||||
Ps = [#p_srvr{name=a, address="localhost", port=Port, props="./data.a"}
|
||||
],
|
||||
D = orddict:from_list([{P#p_srvr.name, P} || P <- Ps]),
|
||||
|
||||
[os:cmd("rm -rf " ++ P#p_srvr.props) || P <- Ps],
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
PortBase = 5720,
|
||||
ok = application:set_env(machi, max_file_size, 1024*1024),
|
||||
try
|
||||
[begin
|
||||
#p_srvr{name=Name, port=Port, props=Dir} = P,
|
||||
{ok, _} = machi_flu_psup:start_flu_package(Name, Port, Dir, [])
|
||||
end || P <- Ps],
|
||||
ok = machi_chain_manager1:set_chain_members(a_chmgr, D),
|
||||
[machi_chain_manager1:trigger_react_to_env(a_chmgr) || _ <-lists:seq(1,5)],
|
||||
{Ps, MgrNames, Dirs} = machi_test_util:start_flu_packages(
|
||||
1, PortBase, "./data.", []),
|
||||
D = orddict:from_list([{P#p_srvr.name, P} || P <- Ps]),
|
||||
M0 = hd(MgrNames),
|
||||
ok = machi_chain_manager1:set_chain_members(M0, D),
|
||||
[machi_chain_manager1:trigger_react_to_env(M0) || _ <-lists:seq(1,5)],
|
||||
|
||||
{ok, Clnt} = ?C:start_link(Ps),
|
||||
try
|
||||
|
@ -60,16 +56,18 @@ smoke_test2() ->
|
|||
%% a separate test module? Or separate test func?
|
||||
{error, _} = ?C:auth(Clnt, "foo", "bar"),
|
||||
|
||||
PK = <<>>,
|
||||
Prefix = <<"prefix">>,
|
||||
Chunk1 = <<"Hello, chunk!">>,
|
||||
NS = "",
|
||||
NoCSum = <<>>,
|
||||
Opts1 = #append_opts{},
|
||||
{ok, {Off1, Size1, File1}} =
|
||||
?C:append_chunk(Clnt, PK, Prefix, Chunk1, none, 0),
|
||||
?C:append_chunk(Clnt, NS, Prefix, Chunk1, NoCSum, Opts1),
|
||||
true = is_binary(File1),
|
||||
Chunk2 = "It's another chunk",
|
||||
CSum2 = {client_sha, machi_util:checksum_chunk(Chunk2)},
|
||||
{ok, {Off2, Size2, File2}} =
|
||||
?C:append_chunk(Clnt, PK, Prefix, Chunk2, CSum2, 1024),
|
||||
?C:append_chunk(Clnt, NS, Prefix, Chunk2, CSum2, Opts1),
|
||||
Chunk3 = ["This is a ", <<"test,">>, 32, [["Hello, world!"]]],
|
||||
File3 = File2,
|
||||
Off3 = Off2 + iolist_size(Chunk2),
|
||||
|
@ -80,7 +78,9 @@ smoke_test2() ->
|
|||
{iolist_to_binary(Chunk2), File2, Off2, Size2},
|
||||
{iolist_to_binary(Chunk3), File3, Off3, Size3}],
|
||||
[begin
|
||||
{ok, Ch} = ?C:read_chunk(Clnt, Fl, Off, Sz)
|
||||
File = Fl,
|
||||
?assertMatch({ok, {[{File, Off, Ch, _}], []}},
|
||||
?C:read_chunk(Clnt, Fl, Off, Sz, undefined))
|
||||
end || {Ch, Fl, Off, Sz} <- Reads],
|
||||
|
||||
{ok, KludgeBin} = ?C:checksum_list(Clnt, File1),
|
||||
|
@ -88,15 +88,56 @@ smoke_test2() ->
|
|||
{ok, [{File1Size,File1}]} = ?C:list_files(Clnt),
|
||||
true = is_integer(File1Size),
|
||||
|
||||
File1Bin = binary_to_list(File1),
|
||||
[begin
|
||||
#p_srvr{name=Name, props=Props} = P,
|
||||
Dir = proplists:get_value(data_dir, Props),
|
||||
?assertEqual({ok, [File1Bin]},
|
||||
file:list_dir(filename:join([Dir, "data"]))),
|
||||
FileListFileName = filename:join([Dir, "known_files_" ++ atom_to_list(Name)]),
|
||||
{ok, Plist} = machi_plist:open(FileListFileName, []),
|
||||
?assertEqual([], machi_plist:all(Plist))
|
||||
end || P <- Ps],
|
||||
|
||||
[begin
|
||||
ok = ?C:trim_chunk(Clnt, Fl, Off, Sz)
|
||||
end || {_Ch, Fl, Off, Sz} <- Reads],
|
||||
[begin
|
||||
{ok, {[], Trimmed}} =
|
||||
?C:read_chunk(Clnt, Fl, Off, Sz, #read_opts{needs_trimmed=true}),
|
||||
Filename = Fl,
|
||||
?assertEqual([{Filename, Off, Sz}], Trimmed)
|
||||
end || {_Ch, Fl, Off, Sz} <- Reads],
|
||||
|
||||
LargeBytes = binary:copy(<<"x">>, 1024*1024),
|
||||
LBCsum = {client_sha, machi_util:checksum_chunk(LargeBytes)},
|
||||
{ok, {Offx, Sizex, Filex}} =
|
||||
?C:append_chunk(Clnt, NS,
|
||||
Prefix, LargeBytes, LBCsum, Opts1),
|
||||
ok = ?C:trim_chunk(Clnt, Filex, Offx, Sizex),
|
||||
|
||||
%% Make sure everything was trimmed
|
||||
File = binary_to_list(Filex),
|
||||
[begin
|
||||
#p_srvr{name=Name, props=Props} = P,
|
||||
Dir = proplists:get_value(data_dir, Props),
|
||||
?assertEqual({ok, []},
|
||||
file:list_dir(filename:join([Dir, "data"]))),
|
||||
FileListFileName = filename:join([Dir, "known_files_" ++ atom_to_list(Name)]),
|
||||
{ok, Plist} = machi_plist:open(FileListFileName, []),
|
||||
?assertEqual([File], machi_plist:all(Plist))
|
||||
end || P <- Ps],
|
||||
|
||||
[begin
|
||||
{error, trimmed} =
|
||||
?C:read_chunk(Clnt, Fl, Off, Sz, undefined)
|
||||
end || {_Ch, Fl, Off, Sz} <- Reads],
|
||||
ok
|
||||
after
|
||||
(catch ?C:quit(Clnt))
|
||||
end
|
||||
after
|
||||
exit(SupPid, normal),
|
||||
[os:cmd("rm -rf " ++ P#p_srvr.props) || P <- Ps],
|
||||
machi_util:wait_for_death(SupPid, 100),
|
||||
ok
|
||||
machi_test_util:stop_flu_packages()
|
||||
end.
|
||||
|
||||
-endif. % !PULSE
|
||||
|
|
17
test/machi_plist_test.erl
Normal file
17
test/machi_plist_test.erl
Normal file
|
@ -0,0 +1,17 @@
|
|||
-module(machi_plist_test).
|
||||
|
||||
-include_lib("eunit/include/eunit.hrl").
|
||||
|
||||
open_close_test() ->
|
||||
FileName = "bark-bark-one",
|
||||
file:delete(FileName),
|
||||
{ok, PList0} = machi_plist:open(FileName, []),
|
||||
{ok, PList1} = machi_plist:add(PList0, "boomar"),
|
||||
?assertEqual(["boomar"], machi_plist:all(PList1)),
|
||||
ok = machi_plist:close(PList1),
|
||||
|
||||
{ok, PList2} = machi_plist:open(FileName, []),
|
||||
?assertEqual(["boomar"], machi_plist:all(PList2)),
|
||||
ok = machi_plist:close(PList2),
|
||||
file:delete(FileName),
|
||||
ok.
|
|
@ -29,12 +29,11 @@
|
|||
-include("machi_projection.hrl").
|
||||
|
||||
smoke_test() ->
|
||||
{ok, SupPid} = machi_flu_sup:start_link(),
|
||||
PortBase = 64820,
|
||||
Dir = "./data.a",
|
||||
Os = [{ignore_stability_time, true}, {active_mode, false}],
|
||||
os:cmd("rm -rf " ++ Dir),
|
||||
{ok,Yo}=machi_flu_psup:start_flu_package(a, PortBase, "./data.a", Os),
|
||||
machi_test_util:start_flu_package(a, PortBase, "./data.a", Os),
|
||||
|
||||
try
|
||||
P1 = machi_projection:new(1, a, [], [], [], [], []),
|
||||
|
@ -59,9 +58,7 @@ smoke_test() ->
|
|||
|
||||
ok
|
||||
after
|
||||
machi_flu_psup:stop_flu_package(a),
|
||||
exit(SupPid, normal),
|
||||
timer:sleep(10)
|
||||
machi_test_util:stop_flu_package()
|
||||
end.
|
||||
|
||||
-endif. % !PULSE
|
||||
|
|
|
@ -32,48 +32,55 @@
|
|||
|
||||
api_smoke_test() ->
|
||||
RegName = api_smoke_flu,
|
||||
Host = "localhost",
|
||||
TcpPort = 57124,
|
||||
TcpPort = 17124,
|
||||
DataDir = "./data.api_smoke_flu",
|
||||
W_props = [{initial_wedged, false}],
|
||||
W_props = [{active_mode, false},{initial_wedged, false}],
|
||||
Prefix = <<"prefix">>,
|
||||
FLU1 = machi_flu1_test:setup_test_flu(RegName, TcpPort, DataDir,
|
||||
W_props),
|
||||
erase(flu_pid),
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
|
||||
try
|
||||
I = #p_srvr{name=RegName, address=Host, port=TcpPort},
|
||||
{[I], _, _} = machi_test_util:start_flu_package(
|
||||
RegName, TcpPort, DataDir, W_props),
|
||||
{ok, Prox1} = ?MUT:start_link(I),
|
||||
try
|
||||
FakeEpoch = ?DUMMY_PV1_EPOCH,
|
||||
[{ok, {_,_,_}} = ?MUT:append_chunk(Prox1,
|
||||
FakeEpoch, Prefix, <<"data">>,
|
||||
infinity) || _ <- lists:seq(1,5)],
|
||||
[{ok, {_,_,_}} = ?MUT:append_chunk(
|
||||
Prox1, NSInfo, FakeEpoch,
|
||||
Prefix, <<"data">>, NoCSum) ||
|
||||
_ <- lists:seq(1,5)],
|
||||
%% Stop the FLU, what happens?
|
||||
machi_flu1:stop(FLU1),
|
||||
[{error,partition} = ?MUT:append_chunk(Prox1,
|
||||
machi_test_util:stop_flu_package(),
|
||||
[{error,partition} = ?MUT:append_chunk(Prox1, NSInfo,
|
||||
FakeEpoch, Prefix, <<"data-stopped1">>,
|
||||
infinity) || _ <- lists:seq(1,3)],
|
||||
NoCSum) || _ <- lists:seq(1,3)],
|
||||
%% Start the FLU again, we should be able to do stuff immediately
|
||||
FLU1b = machi_flu1_test:setup_test_flu(RegName, TcpPort, DataDir,
|
||||
[save_data_dir|W_props]),
|
||||
put(flu_pid, FLU1b),
|
||||
machi_test_util:start_flu_package(RegName, TcpPort, DataDir,
|
||||
[no_cleanup|W_props]),
|
||||
MyChunk = <<"my chunk data">>,
|
||||
{ok, {MyOff,MySize,MyFile}} =
|
||||
?MUT:append_chunk(Prox1, FakeEpoch, Prefix, MyChunk,
|
||||
infinity),
|
||||
{ok, MyChunk} = ?MUT:read_chunk(Prox1, FakeEpoch, MyFile, MyOff, MySize),
|
||||
MyChunk2 = <<"my chunk data, yeah, again">>,
|
||||
?MUT:append_chunk(Prox1, NSInfo, FakeEpoch, Prefix, MyChunk,
|
||||
NoCSum),
|
||||
{ok, {[{_, MyOff, MyChunk, _MyChunkCSUM}], []}} =
|
||||
?MUT:read_chunk(Prox1, NSInfo, FakeEpoch, MyFile, MyOff, MySize, undefined),
|
||||
MyChunk2_parts = [<<"my chunk ">>, "data", <<", yeah, again">>],
|
||||
MyChunk2 = iolist_to_binary(MyChunk2_parts),
|
||||
Opts1 = #append_opts{chunk_extra=4242},
|
||||
{ok, {MyOff2,MySize2,MyFile2}} =
|
||||
?MUT:append_chunk_extra(Prox1, FakeEpoch, Prefix,
|
||||
MyChunk2, 4242, infinity),
|
||||
{ok, MyChunk2} = ?MUT:read_chunk(Prox1, FakeEpoch, MyFile2, MyOff2, MySize2),
|
||||
MyChunk_badcs = {<<?CSUM_TAG_CLIENT_SHA:8, 0:(8*20)>>, MyChunk},
|
||||
{error, bad_checksum} = ?MUT:append_chunk(Prox1, FakeEpoch,
|
||||
Prefix, MyChunk_badcs),
|
||||
{error, bad_checksum} = ?MUT:write_chunk(Prox1, FakeEpoch,
|
||||
<<"foo-file">>, 99832,
|
||||
MyChunk_badcs),
|
||||
?MUT:append_chunk(Prox1, NSInfo, FakeEpoch, Prefix,
|
||||
MyChunk2_parts, NoCSum, Opts1, infinity),
|
||||
[{ok, {[{_, MyOff2, MyChunk2, _}], []}} =
|
||||
?MUT:read_chunk(Prox1, NSInfo, FakeEpoch, MyFile2, MyOff2, MySize2, DefaultOptions) ||
|
||||
DefaultOptions <- [undefined, noopt, none, any_atom_at_all] ],
|
||||
|
||||
BadCSum = {?CSUM_TAG_CLIENT_SHA, crypto:hash(sha, "...................")},
|
||||
{error, bad_checksum} = ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
Prefix, MyChunk, BadCSum),
|
||||
{error, bad_checksum} = ?MUT:write_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
MyFile2,
|
||||
MyOff2 + size(MyChunk2),
|
||||
MyChunk, BadCSum,
|
||||
infinity),
|
||||
|
||||
%% Put kick_projection_reaction() in the middle of the test so
|
||||
%% that any problems with its async nature will (hopefully)
|
||||
|
@ -82,52 +89,52 @@ api_smoke_test() ->
|
|||
|
||||
%% Alright, now for the rest of the API, whee
|
||||
BadFile = <<"no-such-file">>,
|
||||
{error, no_such_file} = ?MUT:checksum_list(Prox1, FakeEpoch, BadFile),
|
||||
{error, bad_arg} = ?MUT:checksum_list(Prox1, BadFile),
|
||||
{ok, [_|_]} = ?MUT:list_files(Prox1, FakeEpoch),
|
||||
{ok, {false, _}} = ?MUT:wedge_status(Prox1),
|
||||
{ok, FakeEpoch} = ?MUT:get_latest_epochid(Prox1, public),
|
||||
{error, not_written} = ?MUT:read_latest_projection(Prox1, public),
|
||||
{ok, {false, _,_,_}} = ?MUT:wedge_status(Prox1),
|
||||
{ok, {0, _SomeCSum}} = ?MUT:get_latest_epochid(Prox1, public),
|
||||
{ok, #projection_v1{epoch_number=0}} =
|
||||
?MUT:read_latest_projection(Prox1, public),
|
||||
{error, not_written} = ?MUT:read_projection(Prox1, public, 44),
|
||||
P_a = #p_srvr{name=a, address="localhost", port=6622},
|
||||
P1 = machi_projection:new(1, a, [P_a], [], [a], [], []),
|
||||
ok = ?MUT:write_projection(Prox1, public, P1),
|
||||
{ok, P1} = ?MUT:read_projection(Prox1, public, 1),
|
||||
{ok, [P1]} = ?MUT:get_all_projections(Prox1, public),
|
||||
{ok, [1]} = ?MUT:list_all_projections(Prox1, public),
|
||||
{ok, [#projection_v1{epoch_number=0},P1]} =
|
||||
?MUT:get_all_projections(Prox1, public),
|
||||
{ok, [0,1]} = ?MUT:list_all_projections(Prox1, public),
|
||||
|
||||
ok
|
||||
after
|
||||
_ = (catch ?MUT:quit(Prox1))
|
||||
end
|
||||
after
|
||||
(catch machi_flu1:stop(FLU1)),
|
||||
(catch machi_flu1:stop(get(flu_pid)))
|
||||
(catch machi_test_util:stop_flu_package())
|
||||
end.
|
||||
|
||||
flu_restart_test() ->
|
||||
RegName = api_smoke_flu,
|
||||
Host = "localhost",
|
||||
TcpPort = 57125,
|
||||
flu_restart_test_() ->
|
||||
{timeout, 1*60, fun() -> flu_restart_test2() end}.
|
||||
|
||||
flu_restart_test2() ->
|
||||
RegName = a,
|
||||
TcpPort = 17125,
|
||||
DataDir = "./data.api_smoke_flu2",
|
||||
W_props = [{initial_wedged, false}],
|
||||
erase(flu_pid),
|
||||
put(flu_pid, []),
|
||||
FLU1 = machi_flu1_test:setup_test_flu(RegName, TcpPort, DataDir,
|
||||
W_props),
|
||||
put(flu_pid, [FLU1|get(flu_pid)]),
|
||||
W_props = [{initial_wedged, false}, {active_mode, false}],
|
||||
NSInfo = undefined,
|
||||
NoCSum = <<>>,
|
||||
|
||||
try
|
||||
I = #p_srvr{name=RegName, address=Host, port=TcpPort},
|
||||
{[I], _, _} = machi_test_util:start_flu_package(
|
||||
RegName, TcpPort, DataDir, W_props),
|
||||
{ok, Prox1} = ?MUT:start_link(I),
|
||||
try
|
||||
FakeEpoch = ?DUMMY_PV1_EPOCH,
|
||||
Data = <<"data!">>,
|
||||
Dataxx = <<"Fake!">>,
|
||||
{ok, {Off1,Size1,File1}} = ?MUT:append_chunk(Prox1,
|
||||
FakeEpoch, <<"prefix">>, Data,
|
||||
infinity),
|
||||
{ok, {Off1,Size1,File1}} = ?MUT:append_chunk(Prox1, NSInfo,
|
||||
FakeEpoch, <<"prefix">>, Data, NoCSum),
|
||||
P_a = #p_srvr{name=a, address="localhost", port=6622},
|
||||
P1 = machi_projection:new(1, a, [P_a], [], [a], [], []),
|
||||
P1 = machi_projection:new(1, RegName, [P_a], [], [RegName], [], []),
|
||||
P1xx = P1#projection_v1{dbg2=["dbg2 changes are ok"]},
|
||||
P1yy = P1#projection_v1{dbg=["not exactly the same as P1!!!"]},
|
||||
EpochID = {P1#projection_v1.epoch_number,
|
||||
|
@ -137,7 +144,7 @@ flu_restart_test() ->
|
|||
{ok, EpochID} = ?MUT:get_epoch_id(Prox1),
|
||||
{ok, EpochID} = ?MUT:get_latest_epochid(Prox1, public),
|
||||
{ok, EpochID} = ?MUT:get_latest_epochid(Prox1, private),
|
||||
ok = machi_flu1:stop(FLU1), timer:sleep(50),
|
||||
ok = machi_test_util:stop_flu_package(), timer:sleep(50),
|
||||
|
||||
%% Now that the last proxy op was successful and only
|
||||
%% after did we stop the FLU, let's check that both the
|
||||
|
@ -149,9 +156,10 @@ flu_restart_test() ->
|
|||
%% makes the code a bit convoluted. (No LFE or
|
||||
%% Elixir macros here, alas, they'd be useful.)
|
||||
|
||||
AppendOpts1 = #append_opts{chunk_extra=42},
|
||||
ExpectedOps =
|
||||
[
|
||||
fun(run) -> {ok, EpochID} = ?MUT:get_epoch_id(Prox1),
|
||||
fun(run) -> ?assertEqual({ok, EpochID}, ?MUT:get_epoch_id(Prox1)),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:get_epoch_id(Prox1) end,
|
||||
|
@ -217,48 +225,50 @@ flu_restart_test() ->
|
|||
(stop) -> ?MUT:write_projection(Prox1, private, P1yy)
|
||||
end,
|
||||
|
||||
fun(run) -> {ok, [_]} =
|
||||
fun(run) -> {ok, [#projection_v1{epoch_number=0}, #projection_v1{epoch_number=1}]} =
|
||||
?MUT:get_all_projections(Prox1, public),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:get_all_projections(Prox1, public)
|
||||
end,
|
||||
fun(run) -> {ok, [_]} =
|
||||
fun(run) -> {ok, [#projection_v1{epoch_number=0}, #projection_v1{epoch_number=1}]} =
|
||||
?MUT:get_all_projections(Prox1, private),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:get_all_projections(Prox1, private)
|
||||
end,
|
||||
fun(run) -> {ok, {_,_,_}} =
|
||||
?MUT:append_chunk(Prox1, FakeEpoch,
|
||||
<<"prefix">>, Data, infinity),
|
||||
?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
<<"prefix">>, Data, NoCSum),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:append_chunk(Prox1, FakeEpoch,
|
||||
<<"prefix">>, Data, infinity)
|
||||
(stop) -> ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
<<"prefix">>, Data, NoCSum)
|
||||
end,
|
||||
fun(run) -> {ok, {_,_,_}} =
|
||||
?MUT:append_chunk_extra(Prox1, FakeEpoch,
|
||||
<<"prefix">>, Data, 42, infinity),
|
||||
?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
<<"prefix">>, Data, NoCSum,
|
||||
AppendOpts1, infinity),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:append_chunk_extra(Prox1, FakeEpoch,
|
||||
<<"prefix">>, Data, 42, infinity)
|
||||
(stop) -> ?MUT:append_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
<<"prefix">>, Data, NoCSum,
|
||||
AppendOpts1, infinity)
|
||||
end,
|
||||
fun(run) -> {ok, Data} =
|
||||
?MUT:read_chunk(Prox1, FakeEpoch,
|
||||
File1, Off1, Size1),
|
||||
fun(run) -> {ok, {[{_, Off1, Data, _}], []}} =
|
||||
?MUT:read_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
File1, Off1, Size1, undefined),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:read_chunk(Prox1, FakeEpoch,
|
||||
File1, Off1, Size1)
|
||||
(stop) -> ?MUT:read_chunk(Prox1, NSInfo, FakeEpoch,
|
||||
File1, Off1, Size1, undefined)
|
||||
end,
|
||||
fun(run) -> {ok, KludgeBin} =
|
||||
?MUT:checksum_list(Prox1, FakeEpoch, File1),
|
||||
?MUT:checksum_list(Prox1, File1),
|
||||
true = is_binary(KludgeBin),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:checksum_list(Prox1, FakeEpoch, File1)
|
||||
(stop) -> ?MUT:checksum_list(Prox1, File1)
|
||||
end,
|
||||
fun(run) -> {ok, _} =
|
||||
?MUT:list_files(Prox1, FakeEpoch),
|
||||
|
@ -274,35 +284,32 @@ flu_restart_test() ->
|
|||
end,
|
||||
fun(run) ->
|
||||
ok =
|
||||
?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
|
||||
Data, infinity),
|
||||
?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
|
||||
Data, NoCSum, infinity),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
|
||||
Data, infinity)
|
||||
(stop) -> ?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
|
||||
Data, NoCSum, infinity)
|
||||
end,
|
||||
%% NOTE: When write-once enforcement is enabled, this test
|
||||
%% will fail: change ok -> {error, written}
|
||||
fun(run) -> %% {error, written} =
|
||||
ok =
|
||||
?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
|
||||
Dataxx, infinity),
|
||||
fun(run) ->
|
||||
{error, written} =
|
||||
?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
|
||||
Dataxx, NoCSum, infinity),
|
||||
ok;
|
||||
(line) -> io:format("line ~p, ", [?LINE]);
|
||||
(stop) -> ?MUT:write_chunk(Prox1, FakeEpoch, File1, Off1,
|
||||
Dataxx, infinity)
|
||||
(stop) -> ?MUT:write_chunk(Prox1, NSInfo, FakeEpoch, File1, Off1,
|
||||
Dataxx, NoCSum, infinity)
|
||||
end
|
||||
],
|
||||
|
||||
[begin
|
||||
FLU2 = machi_flu1_test:setup_test_flu(
|
||||
machi_test_util:start_flu_package(
|
||||
RegName, TcpPort, DataDir,
|
||||
[save_data_dir|W_props]),
|
||||
put(flu_pid, [FLU2|get(flu_pid)]),
|
||||
[no_cleanup|W_props]),
|
||||
_ = Fun(line),
|
||||
ok = Fun(run),
|
||||
ok = Fun(run),
|
||||
ok = machi_flu1:stop(FLU2),
|
||||
ok = machi_test_util:stop_flu_package(),
|
||||
{error, partition} = Fun(stop),
|
||||
{error, partition} = Fun(stop),
|
||||
ok
|
||||
|
@ -312,8 +319,8 @@ flu_restart_test() ->
|
|||
_ = (catch ?MUT:quit(Prox1))
|
||||
end
|
||||
after
|
||||
[catch machi_flu1:stop(Pid) || Pid <- get(flu_pid)]
|
||||
(catch machi_test_util:stop_flu_package())
|
||||
end.
|
||||
|
||||
|
||||
-endif. % !PULSE
|
||||
-endif. % TEST
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue