diff --git a/edoc/edoc-info b/edoc/edoc-info index 14345bb..362069c 100644 --- a/edoc/edoc-info +++ b/edoc/edoc-info @@ -4,6 +4,6 @@ {modules,[machi_admin_util,machi_app,machi_basho_bench_driver, machi_chain_manager1,machi_chain_repair,machi_chash,machi_cr_client, machi_dt,machi_flu1,machi_flu1_client,machi_flu_psup,machi_flu_sup, - machi_pb_high_client,machi_pb_server,machi_pb_wrap,machi_projection, + machi_pb_high_client,machi_pb_translate,machi_projection, machi_projection_store,machi_proxy_flu1_client,machi_sequencer, machi_sup,machi_util,machi_yessir_client]}. diff --git a/edoc/machi_admin_util.html b/edoc/machi_admin_util.html index 913907a..bdf67e2 100644 --- a/edoc/machi_admin_util.html +++ b/edoc/machi_admin_util.html @@ -45,6 +45,6 @@
Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_app.html b/edoc/machi_app.html index 1d7d77d..9f57f7d 100644 --- a/edoc/machi_app.html +++ b/edoc/machi_app.html @@ -37,6 +37,6 @@
Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_basho_bench_driver.html b/edoc/machi_basho_bench_driver.html index 99644be..3599cca 100644 --- a/edoc/machi_basho_bench_driver.html +++ b/edoc/machi_basho_bench_driver.html @@ -59,6 +59,6 @@ could add new entries to this ETS table.

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_chain_manager1.html b/edoc/machi_chain_manager1.html index 5a3de77..d5c6bbb 100644 --- a/edoc/machi_chain_manager1.html +++ b/edoc/machi_chain_manager1.html @@ -174,6 +174,6 @@ distributed state of a single Machi Chain Replication chain.

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_chain_repair.html b/edoc/machi_chain_repair.html index 0e0e5c8..256f8e7 100644 --- a/edoc/machi_chain_repair.html +++ b/edoc/machi_chain_repair.html @@ -86,6 +86,6 @@ in sync. In theory, that's lower overhead than the procedure used here.

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_chash.html b/edoc/machi_chash.html index bb61e53..2e1207c 100644 --- a/edoc/machi_chash.html +++ b/edoc/machi_chash.html @@ -166,6 +166,6 @@ list.

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_cr_client.html b/edoc/machi_cr_client.html index a1cb16f..488c456 100644 --- a/edoc/machi_cr_client.html +++ b/edoc/machi_cr_client.html @@ -240,6 +240,6 @@ function name.)

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_dt.html b/edoc/machi_dt.html index 12a129f..5569f39 100644 --- a/edoc/machi_dt.html +++ b/edoc/machi_dt.html @@ -101,6 +101,6 @@
Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_flu1.html b/edoc/machi_flu1.html index 5feaa3e..829d62e 100644 --- a/edoc/machi_flu1.html +++ b/edoc/machi_flu1.html @@ -65,15 +65,61 @@ replication/chain repair.

simple implementation would stop all existing processes that are running run_seq_append_server().

Function Index

- +
make_listener_regname/1
+ + + + + +
decode_csum_file_entry/1Decode a single binary() blob into an + {Offset,Size,TaggedCSum} tuple.
encode_csum_file_entry/3Encode Offset + Size + TaggedCSum into an iolist() type for + internal storage by the FLU.
encode_csum_file_entry_bin/3Encode Offset + Size + TaggedCSum into an binary() type for + internal storage by the FLU.
make_listener_regname/1
make_projection_server_regname/1
split_checksum_list_blob/1Split a binary() blob of checksum_list data into a list of + unparsed binary() blobs, one per entry.
split_checksum_list_blob_decode/1Split a binary() blob of checksum_list data into a list of + {Offset,Size,TaggedCSum} tuples.
start_link/1
stop/1
update_wedge_state/3
wedge_myself/2

Function Details

+

decode_csum_file_entry/1

+
+

decode_csum_file_entry(Else::binary()) -> error | {machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()}

+

Decode a single binary() blob into an + {Offset,Size,TaggedCSum} tuple.

+ +

The internal encoding (which is currently exposed to the outside world +via this function and related ones) is:

+ + + + See machi.hrl for the tagged checksum types, e.g., + ?CSUM_TAG_NONE.

+ +

encode_csum_file_entry/3

+
+

encode_csum_file_entry(Offset::machi_dt:file_offset(), Size::machi_dt:chunk_size(), TaggedCSum::machi_dt:chunk_s()) -> iolist()

+

Encode Offset + Size + TaggedCSum into an iolist() type for + internal storage by the FLU.

+ +

encode_csum_file_entry_bin/3

+
+

encode_csum_file_entry_bin(Offset::machi_dt:file_offset(), Size::machi_dt:chunk_size(), TaggedCSum::machi_dt:chunk_s()) -> binary()

+

Encode Offset + Size + TaggedCSum into an binary() type for + internal storage by the FLU.

+

make_listener_regname/1

make_listener_regname(BaseName) -> any()

@@ -84,6 +130,24 @@ replication/chain repair.

make_projection_server_regname(BaseName) -> any()

+

split_checksum_list_blob/1

+
+

split_checksum_list_blob(Bin::binary()) -> {[binary()], TrailingJunk::binary()}

+

Split a binary() blob of checksum_list data into a list of + unparsed binary() blobs, one per entry.

+ +

Decode the unparsed blobs with decode_csum_file_entry/1, if +desired.

+ + The return value TrailingJunk is unparseable bytes at the end of + the checksum list blob.

+ +

split_checksum_list_blob_decode/1

+
+

split_checksum_list_blob_decode(Bin::binary()) -> {[{machi_dt:file_offset(), machi_dt:chunk_size(), machi_dt:chunk_s()}], TrailingJunk::binary()}

+

Split a binary() blob of checksum_list data into a list of + {Offset,Size,TaggedCSum} tuples.

+

start_link/1

start_link(Rest) -> any()

@@ -98,9 +162,14 @@ replication/chain repair.

update_wedge_state(PidSpec, Boolean, EpochId) -> any()

+ +

wedge_myself/2

+
+

wedge_myself(PidSpec, EpochId) -> any()

+
Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_flu1_client.html b/edoc/machi_flu1_client.html index dbc94bb..8a78ea2 100644 --- a/edoc/machi_flu1_client.html +++ b/edoc/machi_flu1_client.html @@ -21,26 +21,17 @@ management can be found in machi_proxy_flu1_client and machi_cr_client.

-

TODO This EDoc was written first, and the EDoc and also -type and - -spec definitions for machi_proxy_flu1_client and machi_cr_client must be improved.

+

For the moment, this module implements a Protocol Buffers-based +protocol as the sole supported access method to the server, +sequencer, and projection store. Conceptually, those three +services are independent and ought to have their own protocols. As +a practical matter, there is no need for wire protocol +compatibility. Furthermore, from the perspective of failure +detection, it is very convenient that all three FLU-related +services are accessed using the same single TCP port.

-

Protocol origins

- -

The protocol implemented here is an artisanal, hand-crafted, silly -thing that was very quick to put together for a "demo day" proof of -concept. It will almost certainly be replaced with something else, -both in terms of wire format and better code separation of -serialization/deserialization vs. network transport management, -etc.

- - For the moment, this module implements a rudimentary TCP-based - protocol as the sole supported access method to the server, - sequencer, and projection store. Conceptually, those three - services are independent and ought to have their own protocols. As - a practical matter, there is no need for wire protocol - compatibility. Furthermore, from the perspective of failure - detection, it is very convenient that all three FLU-related - services are accessed using the same single TCP port. + TODO This EDoc was written first, and the EDoc and also -type and + -spec definitions for machi_proxy_flu1_client and machi_cr_client must be improved.

Data Types

port_wrap()

@@ -65,6 +56,8 @@ etc.

delete_migration/4Restricted API: Delete a file after it has been successfully migrated. disconnect/1 +echo/2Echo -- test protocol round-trip. +echo/3Get all epoch numbers from the FLU's projection store. get_all_projections/2Get all projections from the FLU's projection store. get_all_projections/3Get all projections from the FLU's projection store. get_latest_epochid/2Get the latest epoch number + checksum from the FLU's projection store. @@ -132,13 +125,33 @@ etc.

checksum_list/3

-

checksum_list(Sock::port_wrap(), EpochID::machi_dt:epoch_id(), File::machi_dt:file_name()) -> {ok, [machi_dt:chunk_summary()]} | {error, machi_dt:error_general() | no_such_file | partial_read} | {error, term()}

+

checksum_list(Sock::port_wrap(), EpochID::machi_dt:epoch_id(), File::machi_dt:file_name()) -> {ok, binary()} | {error, machi_dt:error_general() | no_such_file | partial_read} | {error, term()}

Fetch the list of chunk checksums for File.

checksum_list/4

-

checksum_list(Host::machi_dt:inet_host(), TcpPort::machi_dt:inet_port(), EpochID::machi_dt:epoch_id(), File::machi_dt:file_name()) -> {ok, [machi_dt:chunk_summary()]} | {error, machi_dt:error_general() | no_such_file} | {error, term()}

-

Fetch the list of chunk checksums for File.

+

checksum_list(Host::machi_dt:inet_host(), TcpPort::machi_dt:inet_port(), EpochID::machi_dt:epoch_id(), File::machi_dt:file_name()) -> {ok, binary()} | {error, machi_dt:error_general() | no_such_file} | {error, term()}

+

Fetch the list of chunk checksums for File.

+ + Why return a simple binary() type rather than + [machi_dt:chunk_summary()]? The two reasons are: +
    +
  1. Server overhead: the CPU required to chop up the implementation- + specific store into zillions of very small terms is very high. +
    2. +
  2. Protocol encoding and decoding overhead: the cost is non-zero, + and the sum of cost of encoding and decoding a zillion small terms + is substantial. +
    3. +
+ +

For both reasons, the server's protocol response is absurdly simple + and very fast: send back a binary() blob to the client. Then it +is the client's responsibility to spend the CPU time to parse the +blob.

+ + Details of the encoding used inside the binary() blog can be found + in the EDoc comments for machi_flu1:decode_csum_file_entry/1.

connect/1

@@ -167,6 +180,16 @@ etc.

disconnect(WS) -> any()

+

echo/2

+
+

echo(Sock::port_wrap(), String::string()) -> string() | {error, term()}

+

Echo -- test protocol round-trip.

+ +

echo/3

+
+

echo(Host::machi_dt:inet_host(), TcpPort::machi_dt:inet_port(), String::string()) -> string() | {error, term()}

+

Get all epoch numbers from the FLU's projection store.

+

get_all_projections/2

get_all_projections(Sock::port_wrap(), ProjType::machi_dt:projection_type()) -> {ok, [machi_dt:projection()]} | {error, term()}

@@ -288,6 +311,6 @@ etc.

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_flu_psup.html b/edoc/machi_flu_psup.html index 8858b53..2d116cc 100644 --- a/edoc/machi_flu_psup.html +++ b/edoc/machi_flu_psup.html @@ -107,6 +107,6 @@ at this time.

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_flu_sup.html b/edoc/machi_flu_sup.html index 82c9c90..97525cb 100644 --- a/edoc/machi_flu_sup.html +++ b/edoc/machi_flu_sup.html @@ -39,6 +39,6 @@ servers.

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_pb_high_client.html b/edoc/machi_pb_high_client.html index d073971..3cc220b 100644 --- a/edoc/machi_pb_high_client.html +++ b/edoc/machi_pb_high_client.html @@ -171,6 +171,6 @@
Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_projection.html b/edoc/machi_projection.html index e5e90f2..9e27a8e 100644 --- a/edoc/machi_projection.html +++ b/edoc/machi_projection.html @@ -89,6 +89,6 @@ correctly.

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_projection_store.html b/edoc/machi_projection_store.html index bc12fb5..d57ad4a 100644 --- a/edoc/machi_projection_store.html +++ b/edoc/machi_projection_store.html @@ -164,6 +164,6 @@ client access protocol (on the "server side" of the TCP connection).

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_proxy_flu1_client.html b/edoc/machi_proxy_flu1_client.html index 74236d6..845ab2a 100644 --- a/edoc/machi_proxy_flu1_client.html +++ b/edoc/machi_proxy_flu1_client.html @@ -288,6 +288,6 @@ TCP communication problems with the remote server.

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_sequencer.html b/edoc/machi_sequencer.html index ec86498..0129926 100644 --- a/edoc/machi_sequencer.html +++ b/edoc/machi_sequencer.html @@ -18,6 +18,6 @@ the future?
Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_sup.html b/edoc/machi_sup.html index c86d124..dc2ed41 100644 --- a/edoc/machi_sup.html +++ b/edoc/machi_sup.html @@ -37,6 +37,6 @@
Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_util.html b/edoc/machi_util.html index 20013b1..cfde6f5 100644 --- a/edoc/machi_util.html +++ b/edoc/machi_util.html @@ -43,6 +43,7 @@ mbytes/1 read_max_filenum/2Read the file size of a config file, which is used as the basis for a minimum sequence number. +unmake_tagged_csum/1 verb/1Log a verbose message. verb/2Log a verbose message. @@ -169,6 +170,11 @@

Read the file size of a config file, which is used as the basis for a minimum sequence number.

+

unmake_tagged_csum/1

+
+

unmake_tagged_csum(X1) -> any()

+
+

verb/1

verb(Fmt::string()) -> ok

@@ -181,6 +187,6 @@
Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/machi_yessir_client.html b/edoc/machi_yessir_client.html index f3ab4c8..d7ab5c2 100644 --- a/edoc/machi_yessir_client.html +++ b/edoc/machi_yessir_client.html @@ -255,6 +255,6 @@
Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.

diff --git a/edoc/modules-frame.html b/edoc/modules-frame.html index 2c5b968..45f3a7c 100644 --- a/edoc/modules-frame.html +++ b/edoc/modules-frame.html @@ -20,8 +20,7 @@ machi_flu_psup machi_flu_sup machi_pb_high_client -machi_pb_server -machi_pb_wrap +machi_pb_translate machi_projection machi_projection_store machi_proxy_flu1_client diff --git a/edoc/overview-summary.html b/edoc/overview-summary.html index e75fda4..677f84e 100644 --- a/edoc/overview-summary.html +++ b/edoc/overview-summary.html @@ -180,6 +180,6 @@ the code, is a reminder signal of unfinished work.

Overviewerlang logo
-

Generated by EDoc, Jun 25 2015, 16:39:10.

+

Generated by EDoc, Jul 1 2015, 18:32:19.