quoracle/quorums/quorum_system.py
2021-01-29 18:09:50 -08:00

516 lines
20 KiB
Python

# TODO(mwhittaker): We can define a set of read quorums that are not minimal.
# Does this mess things up?
from . import distribution
from . import geometry
from .distribution import Distribution
from .expr import Expr, Node
from .geometry import Point, Segment
from typing import *
import collections
import datetime
import itertools
import numpy as np
import pulp
T = TypeVar('T')
LOAD = 'load'
NETWORK = 'network'
LATENCY = 'latency'
# TODO(mwhittaker): Add some other non-optimal strategies.
# TODO(mwhittaker): Make it easy to make arbitrary strategies.
class QuorumSystem(Generic[T]):
def __init__(self, reads: Optional[Expr[T]] = None,
writes: Optional[Expr[T]] = None) -> None:
if reads is not None and writes is not None:
optimal_writes = reads.dual()
if not all(optimal_writes.is_quorum(write_quorum)
for write_quorum in writes.quorums()):
raise ValueError(
'Not all read quorums intersect all write quorums')
self.reads = reads
self.writes = writes
elif reads is not None and writes is None:
self.reads = reads
self.writes = reads.dual()
elif reads is None and writes is not None:
self.reads = writes.dual()
self.writes = writes
else:
raise ValueError('A QuorumSystem must be instantiated with a set '
'of read quorums or a set of write quorums')
self.x_to_node = {node.x: node for node in self.nodes()}
def __repr__(self) -> str:
return f'QuorumSystem(reads={self.reads}, writes={self.writes})'
def read_quorums(self) -> Iterator[Set[T]]:
return self.reads.quorums()
def write_quorums(self) -> Iterator[Set[T]]:
return self.writes.quorums()
def is_read_quorum(self, xs: Set[T]) -> bool:
return self.reads.is_quorum(xs)
def is_write_quorum(self, xs: Set[T]) -> bool:
return self.writes.is_quorum(xs)
def nodes(self) -> Set[Node[T]]:
return self.reads.nodes() | self.writes.nodes()
def resilience(self) -> int:
return min(self.read_resilience(), self.write_resilience())
def read_resilience(self) -> int:
return self.reads.resilience()
def write_resilience(self) -> int:
return self.writes.resilience()
def strategy(self,
optimize: str = LOAD,
load_limit: Optional[float] = None,
network_limit: Optional[float] = None,
latency_limit: Optional[datetime.timedelta] = None,
read_fraction: Optional[Distribution] = None,
write_fraction: Optional[Distribution] = None,
f: int = 0) \
-> 'Strategy[T]':
if f < 0:
raise ValueError('f must be >= 0')
if optimize == LOAD and load_limit is not None:
raise ValueError(
'a load limit cannot be set when optimizing for load')
if optimize == NETWORK and network_limit is not None:
raise ValueError(
'a network limit cannot be set when optimizing for network')
if optimize == LATENCY and latency_limit is not None:
raise ValueError(
'a latency limit cannot be set when optimizing for latency')
d = distribution.canonicalize_rw(read_fraction, write_fraction)
if f == 0:
return self._load_optimal_strategy(
list(self.read_quorums()),
list(self.write_quorums()),
d,
optimize=optimize,
load_limit=load_limit,
network_limit=network_limit,
latency_limit=latency_limit)
else:
xs = [node.x for node in self.nodes()]
read_quorums = list(self._f_resilient_quorums(f, xs, self.reads))
write_quorums = list(self._f_resilient_quorums(f, xs, self.reads))
if len(read_quorums) == 0:
raise ValueError(f'There are no {f}-resilient read quorums')
if len(write_quorums) == 0:
raise ValueError(f'There are no {f}-resilient write quorums')
return self._load_optimal_strategy(
read_quorums,
write_quorums,
d,
optimize=optimize,
load_limit=load_limit,
network_limit=network_limit,
latency_limit=latency_limit)
def dup_free(self) -> bool:
return self.reads.dup_free() and self.writes.dup_free()
def _f_resilient_quorums(self,
f: int,
xs: List[T],
e: Expr) -> Iterator[Set[T]]:
assert f >= 1
def helper(s: Set[T], i: int) -> Iterator[Set[T]]:
if all(e.is_quorum(s - set(failure))
for failure in itertools.combinations(s, min(f, len(s)))):
yield set(s)
return
for j in range(i, len(xs)):
s.add(xs[j])
yield from helper(s, j + 1)
s.remove(xs[j])
return helper(set(), 0)
def load(self,
read_fraction: Optional[Distribution] = None,
write_fraction: Optional[Distribution] = None,
f: int = 0) \
-> float:
return 0
# TODO(mwhittaker): Remove.
# sigma = self.strategy(read_fraction, write_fraction, f)
# return sigma.load(read_fraction, write_fraction)
def capacity(self,
read_fraction: Optional[Distribution] = None,
write_fraction: Optional[Distribution] = None,
f: int = 0) \
-> float:
return 0
# TODO(mwhittaker): Remove.
# return 1 / self.load(read_fraction, write_fraction, f)
def _read_quorum_latency(self, quorum: Set[Node[T]]) -> datetime.timedelta:
return self._quorum_latency(quorum, self.is_read_quorum)
def _write_quorum_latency(self, quorum: Set[Node[T]]) -> datetime.timedelta:
return self._quorum_latency(quorum, self.is_write_quorum)
def _quorum_latency(self,
quorum: Set[Node[T]],
is_quorum: Callable[[Set[T]], bool]) \
-> datetime.timedelta:
nodes = list(quorum)
nodes.sort(key=lambda node: node.latency)
for i in range(len(quorum)):
if is_quorum({node.x for node in nodes[:i+1]}):
return nodes[i].latency
raise ValueError('_quorum_latency called on a non-quorum')
def _load_optimal_strategy(
self,
read_quorums: List[Set[T]],
write_quorums: List[Set[T]],
read_fraction: Dict[float, float],
optimize: str = LOAD,
load_limit: Optional[float] = None,
network_limit: Optional[float] = None,
latency_limit: Optional[datetime.timedelta] = None) -> 'Strategy[T]':
"""
Consider the following 2x2 grid quorum system.
a b
c d
with
read_quorums = [{a, b}, {c, d}]
write_quorums = [{a, c}, {a, d}, {b, c}, {b, d}]
We can form a linear program to compute the optimal load of this quorum
system for some fixed read fraction fr as follows. First, we create a
variable ri for every read quorum i and a variable wi for every write
quorum i. ri represents the probabilty of selecting the ith read
quorum, and wi represents the probabilty of selecting the ith write
quorum. We introduce an additional variable l that represents the load
and solve the following linear program.
min L subject to
r0 + r1 + r2 = 1
w0 + w1 = 1
fr (r0) + (1 - fr) (w0 + w1) <= L # a's load
fr (r0) + (1 - fr) (w2 + w3) <= L # b's load
fr (r1) + (1 - fr) (w0 + w2) <= L # c's load
fr (r1) + (1 - fr) (w1 + w3) <= L # d's load
If we assume every element x has read capacity rcap_x and write
capacity wcap_x, then we adjust the linear program like this.
min L subject to
r0 + r1 + r2 = 1
w0 + w1 = 1
fr/rcap_a (r0) + (1 - fr)/wcap_a (w0 + w1) <= L # a's load
fr/rcap_b (r0) + (1 - fr)/wcap_b (w2 + w3) <= L # b's load
fr/rcap_c (r1) + (1 - fr)/wcap_c (w0 + w2) <= L # c's load
fr/rcap_d (r1) + (1 - fr)/wcap_d (w1 + w3) <= L # d's load
Assume we have fr = 0.9 with 80% probabilty and fr = 0.5 with 20%. Then
we adjust the linear program as follows to find the strategy that
minimzes the average load.
min 0.8 * L_0.9 + 0.2 * L_0.5 subject to
r0 + r1 + r2 = 1
w0 + w1 = 1
0.9/rcap_a (r0) + 0.1/wcap_a (w0 + w1) <= L_0.9 # a's load
0.9/rcap_b (r0) + 0.1/wcap_b (w2 + w3) <= L_0.9 # b's load
0.9/rcap_c (r1) + 0.1/wcap_c (w0 + w2) <= L_0.9 # c's load
0.9/rcap_d (r1) + 0.1/wcap_d (w1 + w3) <= L_0.9 # d's load
0.5/rcap_a (r0) + 0.5/wcap_a (w0 + w1) <= L_0.5 # a's load
0.5/rcap_b (r0) + 0.5/wcap_b (w2 + w3) <= L_0.5 # b's load
0.5/rcap_c (r1) + 0.5/wcap_c (w0 + w2) <= L_0.5 # c's load
0.5/rcap_d (r1) + 0.5/wcap_d (w1 + w3) <= L_0.5 # d's load
"""
nodes = self.nodes()
x_to_node = {node.x: node for node in nodes}
read_capacity = {node.x: node.read_capacity for node in nodes}
write_capacity = {node.x: node.write_capacity for node in nodes}
# Create a variable for every read quorum and every write quorum. While
# we do this, map each element x to the read and write quorums that
# it's in. For example, image we have the following read and write
# quorums:
#
# read_quorums = [{a}, {a, b}, {a, c}]
# write_quorums = [{a, b}, {a, b, c}]
#
# Then, we'd have
#
# read_quorum_vars = [r0, r1, 2]
# write_quorum_vars = [w0, w1]
# x_to_read_quorum_vars = {a: [r1, r2, r3], b: [r1], c: [r2]}
# x_to_write_quorum_vars = {a: [w1, w2], b: [w2, w2], c: [w2]}
read_quorum_vars: List[pulp.LpVariable] = []
x_to_read_quorum_vars: Dict[T, List[pulp.LpVariable]] = \
collections.defaultdict(list)
for (i, read_quorum) in enumerate(read_quorums):
v = pulp.LpVariable(f'r{i}', 0, 1)
read_quorum_vars.append(v)
for x in read_quorum:
x_to_read_quorum_vars[x].append(v)
write_quorum_vars: List[pulp.LpVariable] = []
x_to_write_quorum_vars: Dict[T, List[pulp.LpVariable]] = \
collections.defaultdict(list)
for (i, write_quorum) in enumerate(write_quorums):
v = pulp.LpVariable(f'w{i}', 0, 1)
write_quorum_vars.append(v)
for x in write_quorum:
x_to_write_quorum_vars[x].append(v)
fr = sum(weight * f for (f, weight) in read_fraction.items())
def network() -> pulp.LpAffineExpression:
read_network = fr * sum(
v * len(rq)
for (rq, v) in zip(read_quorums, read_quorum_vars)
)
write_network = (1 - fr) * sum(
v * len(wq)
for (wq, v) in zip(write_quorums, write_quorum_vars)
)
return read_network + write_network
def latency() -> pulp.LpAffineExpression:
read_latency = fr * sum(
v * self._read_quorum_latency(quorum).total_seconds()
for (rq, v) in zip(read_quorums, read_quorum_vars)
for quorum in [{x_to_node[x] for x in rq}]
)
write_latency = (1. - fr) * sum(
v * self._write_quorum_latency(quorum).total_seconds()
for (wq, v) in zip(write_quorums, write_quorum_vars)
for quorum in [{x_to_node[x] for x in wq}]
)
return read_latency + write_latency
def fr_load(problem: pulp.LpProblem, fr: float) -> pulp.LpAffineExpression:
l = pulp.LpVariable(f'l_{fr}', 0, 1)
for node in nodes:
x = node.x
x_load: pulp.LpAffineExpression = 0
if x in x_to_read_quorum_vars:
vs = x_to_read_quorum_vars[x]
x_load += fr * sum(vs) / read_capacity[x]
if x in x_to_write_quorum_vars:
vs = x_to_write_quorum_vars[x]
x_load += (1 - fr) * sum(vs) / write_capacity[x]
problem += (x_load <= l, f'{x}{fr}')
return l
def load(problem: pulp.LpProblem,
read_fraction: Dict[float, float]) -> pulp.LpAffineExpression:
return sum(weight * fr_load(problem, fr)
for (fr, weight) in read_fraction.items())
# Form the linear program to find the load.
problem = pulp.LpProblem("load", pulp.LpMinimize)
# We add these constraints to make sure that the probabilities we
# select form valid probabilty distributions.
problem += (sum(read_quorum_vars) == 1, 'valid read strategy')
problem += (sum(write_quorum_vars) == 1, 'valid write strategy')
# Add the objective.
if optimize == LOAD:
problem += load(problem, read_fraction)
elif optimize == NETWORK:
problem += network()
else:
assert optimize == LATENCY
problem += latency()
# Add any constraints.
if load_limit is not None:
problem += (load(problem, read_fraction) <= load_limit,
'load limit')
if network_limit is not None:
problem += (network() <= network_limit, 'network limit')
if latency_limit is not None:
problem += (latency() <= latency_limit.total_seconds(),
'latency limit')
# Solve the linear program.
print(problem)
problem.solve(pulp.apis.PULP_CBC_CMD(msg=False))
if problem.status != pulp.LpStatusOptimal:
raise ValueError('no strategy satisfies the given constraints')
# Prune out any quorums with 0 probability.
non_zero_read_quorums = [
(rq, v.varValue)
for (rq, v) in zip(read_quorums, read_quorum_vars)
if v.varValue != 0]
non_zero_write_quorums = [
(wq, v.varValue)
for (wq, v) in zip(write_quorums, write_quorum_vars)
if v.varValue != 0]
return Strategy(self,
[rq for (rq, _) in non_zero_read_quorums],
[weight for (_, weight) in non_zero_read_quorums],
[wq for (wq, _) in non_zero_write_quorums],
[weight for (_, weight) in non_zero_write_quorums])
class Strategy(Generic[T]):
def __init__(self,
qs: QuorumSystem[T],
reads: List[Set[T]],
read_weights: List[float],
writes: List[Set[T]],
write_weights: List[float]) -> None:
self.qs = qs
self.reads = reads
self.read_weights = read_weights
self.writes = writes
self.write_weights = write_weights
self.unweighted_read_load: Dict[T, float] = \
collections.defaultdict(float)
for (read_quorum, weight) in zip(self.reads, self.read_weights):
for x in read_quorum:
self.unweighted_read_load[x] += weight
self.unweighted_write_load: Dict[T, float] = \
collections.defaultdict(float)
for (write_quorum, weight) in zip(self.writes, self.write_weights):
for x in write_quorum:
self.unweighted_write_load[x] += weight
def __str__(self) -> str:
non_zero_reads = {tuple(r): p
for (r, p) in zip(self.reads, self.read_weights)
if p > 0}
non_zero_writes = {tuple(w): p
for (w, p) in zip(self.writes, self.write_weights)
if p > 0}
return f'Strategy(reads={non_zero_reads}, writes={non_zero_writes})'
def get_read_quorum(self) -> Set[T]:
return np.random.choice(self.reads, p=self.read_weights)
def get_write_quorum(self) -> Set[T]:
return np.random.choice(self.writes, p=self.write_weights)
def load(self,
read_fraction: Optional[Distribution] = None,
write_fraction: Optional[Distribution] = None) \
-> float:
d = distribution.canonicalize_rw(read_fraction, write_fraction)
return sum(weight * self._load(fr)
for (fr, weight) in d.items())
# TODO(mwhittaker): Rename throughput.
def capacity(self,
read_fraction: Optional[Distribution] = None,
write_fraction: Optional[Distribution] = None) \
-> float:
return 1 / self.load(read_fraction, write_fraction)
def network_load(self,
read_fraction: Optional[Distribution] = None,
write_fraction: Optional[Distribution] = None) -> float:
d = distribution.canonicalize_rw(read_fraction, write_fraction)
fr = sum(weight * f for (f, weight) in d.items())
read_network_load = fr * sum(
len(rq) * p
for (rq, p) in zip(self.reads, self.read_weights)
)
write_network_load = (1 - fr) * sum(
len(wq) * p
for (wq, p) in zip(self.writes, self.write_weights)
)
return read_network_load + write_network_load
def latency(self,
read_fraction: Optional[Distribution] = None,
write_fraction: Optional[Distribution] = None) \
-> datetime.timedelta:
d = distribution.canonicalize_rw(read_fraction, write_fraction)
fr = sum(weight * f for (f, weight) in d.items())
read_latency = fr * sum((
self.qs._read_quorum_latency(quorum) * p # type: ignore
for (rq, p) in zip(self.reads, self.read_weights)
for quorum in [{self.qs.x_to_node[x] for x in rq}]
), datetime.timedelta(seconds=0)) # type: ignore
write_latency = (1 - fr) * sum((
self.qs._write_quorum_latency(quorum) * p # type: ignore
for (wq, p) in zip(self.writes, self.write_weights)
for quorum in [{self.qs.x_to_node[x] for x in wq}]
), datetime.timedelta(seconds=0)) # type:ignore
return read_latency + write_latency # type: ignore
def node_load(self,
node: Node[T],
read_fraction: Optional[Distribution] = None,
write_fraction: Optional[Distribution] = None) \
-> float:
d = distribution.canonicalize_rw(read_fraction, write_fraction)
return sum(weight * self._node_load(node.x, fr)
for (fr, weight) in d.items())
def node_utilization(self,
node: Node[T],
read_fraction: Optional[Distribution] = None,
write_fraction: Optional[Distribution] = None) \
-> float:
# TODO(mwhittaker): Implement.
return 0.0
def node_throghput(self,
node: Node[T],
read_fraction: Optional[Distribution] = None,
write_fraction: Optional[Distribution] = None) \
-> float:
# TODO(mwhittaker): Implement.
return 0.0
def _node_load(self, x: T, fr: float) -> float:
"""
_node_load returns the load on x given a fixed read fraction fr.
"""
fw = 1 - fr
node = self.qs.x_to_node[x]
return (fr * self.unweighted_read_load[x] / node.read_capacity +
fw * self.unweighted_write_load[x] / node.write_capacity)
def _load(self, fr: float) -> float:
"""
_load returns the load given a fixed read fraction fr.
"""
return max(self._node_load(node.x, fr) for node in self.qs.nodes())