2017-02-04 00:51:13 +00:00
|
|
|
// Copyright 2016 Mozilla
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
|
|
|
|
// this file except in compliance with the License. You may obtain a copy of the
|
|
|
|
// License at http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
// Unless required by applicable law or agreed to in writing, software distributed
|
|
|
|
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
|
|
|
|
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
|
|
|
// specific language governing permissions and limitations under the License.
|
|
|
|
|
2018-02-01 17:17:07 +00:00
|
|
|
extern crate chrono;
|
2017-04-24 21:15:26 +00:00
|
|
|
extern crate enum_set;
|
2018-02-01 17:17:07 +00:00
|
|
|
extern crate ordered_float;
|
|
|
|
extern crate uuid;
|
2018-02-09 17:55:19 +00:00
|
|
|
extern crate serde;
|
2017-04-24 21:15:26 +00:00
|
|
|
|
2017-03-30 10:08:36 +00:00
|
|
|
#[macro_use]
|
|
|
|
extern crate lazy_static;
|
2017-02-14 03:20:20 +00:00
|
|
|
|
2018-02-09 17:55:19 +00:00
|
|
|
#[macro_use]
|
|
|
|
extern crate serde_derive;
|
|
|
|
|
2017-03-30 10:08:36 +00:00
|
|
|
extern crate edn;
|
|
|
|
|
|
|
|
pub mod values;
|
2018-02-14 00:51:21 +00:00
|
|
|
mod cache;
|
2017-03-30 10:08:36 +00:00
|
|
|
|
2017-11-30 23:02:07 +00:00
|
|
|
use std::collections::{
|
|
|
|
BTreeMap,
|
|
|
|
BTreeSet,
|
|
|
|
};
|
|
|
|
|
2017-04-18 04:09:45 +00:00
|
|
|
use std::fmt;
|
Use Rc for TypedValue, Variable, and query Ident keywords. (#395) r=nalexander
Part 1, core: use Rc for String and Keyword.
Part 2, query: use Rc for Variable.
Part 3, sql: use Rc for args in SQLiteQueryBuilder.
Part 4, query-algebrizer: use Rc.
Part 5, db: use Rc.
Part 6, query-parser: use Rc.
Part 7, query-projector: use Rc.
Part 8, query-translator: use Rc.
Part 9, top level: use Rc.
Part 10: intern Ident and IdentOrKeyword.
2017-03-29 20:18:17 +00:00
|
|
|
use std::rc::Rc;
|
2017-04-24 21:15:26 +00:00
|
|
|
|
|
|
|
use enum_set::EnumSet;
|
|
|
|
|
2017-02-08 21:59:56 +00:00
|
|
|
use self::ordered_float::OrderedFloat;
|
2017-04-29 03:11:55 +00:00
|
|
|
|
2017-12-06 20:42:59 +00:00
|
|
|
pub use uuid::Uuid;
|
2017-04-29 03:11:55 +00:00
|
|
|
|
2018-01-18 15:06:23 +00:00
|
|
|
pub use chrono::{
|
2017-04-29 03:11:55 +00:00
|
|
|
DateTime,
|
2018-01-18 15:06:23 +00:00
|
|
|
Timelike, // For truncation.
|
|
|
|
};
|
|
|
|
|
|
|
|
pub use edn::{
|
2017-04-29 03:11:55 +00:00
|
|
|
FromMicros,
|
2018-01-23 16:23:37 +00:00
|
|
|
NamespacedKeyword,
|
2017-04-29 03:11:55 +00:00
|
|
|
ToMicros,
|
2017-11-21 16:24:08 +00:00
|
|
|
Utc,
|
2017-04-29 03:11:55 +00:00
|
|
|
};
|
2017-02-08 21:59:56 +00:00
|
|
|
|
2018-03-06 17:01:20 +00:00
|
|
|
pub use cache::{
|
|
|
|
CachedAttributes,
|
|
|
|
UpdateableCache,
|
|
|
|
};
|
2018-02-14 00:51:21 +00:00
|
|
|
|
2017-02-04 00:51:13 +00:00
|
|
|
/// Core types defining a Mentat knowledge base.
|
|
|
|
|
|
|
|
/// Represents one entid in the entid space.
|
|
|
|
///
|
|
|
|
/// Per https://www.sqlite.org/datatype3.html (see also http://stackoverflow.com/a/8499544), SQLite
|
|
|
|
/// stores signed integers up to 64 bits in size. Since u32 is not appropriate for our use case, we
|
|
|
|
/// use i64 rather than manually truncating u64 to u63 and casting to i64 throughout the codebase.
|
|
|
|
pub type Entid = i64;
|
|
|
|
|
2018-01-23 16:31:27 +00:00
|
|
|
/// An entid that's either already in the store, or newly allocated to a tempid.
|
|
|
|
/// TODO: we'd like to link this in some way to the lifetime of a particular PartitionMap.
|
|
|
|
#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)]
|
|
|
|
pub struct KnownEntid(pub Entid);
|
|
|
|
|
|
|
|
impl From<KnownEntid> for Entid {
|
|
|
|
fn from(k: KnownEntid) -> Entid {
|
|
|
|
k.0
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<KnownEntid> for TypedValue {
|
|
|
|
fn from(k: KnownEntid) -> TypedValue {
|
|
|
|
TypedValue::Ref(k.0)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-04 00:51:13 +00:00
|
|
|
/// The attribute of each Mentat assertion has a :db/valueType constraining the value to a
|
|
|
|
/// particular set. Mentat recognizes the following :db/valueType values.
|
2017-04-24 21:15:26 +00:00
|
|
|
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialOrd, PartialEq)]
|
|
|
|
#[repr(u32)]
|
2017-02-04 00:51:13 +00:00
|
|
|
pub enum ValueType {
|
|
|
|
Ref,
|
|
|
|
Boolean,
|
|
|
|
Instant,
|
|
|
|
Long,
|
|
|
|
Double,
|
|
|
|
String,
|
|
|
|
Keyword,
|
2017-04-29 03:11:55 +00:00
|
|
|
Uuid,
|
2017-02-04 00:51:13 +00:00
|
|
|
}
|
|
|
|
|
2017-11-30 23:02:07 +00:00
|
|
|
pub type ValueTypeTag = i32;
|
|
|
|
|
2017-04-24 21:15:26 +00:00
|
|
|
impl ValueType {
|
|
|
|
pub fn all_enums() -> EnumSet<ValueType> {
|
|
|
|
// TODO: lazy_static.
|
|
|
|
let mut s = EnumSet::new();
|
|
|
|
s.insert(ValueType::Ref);
|
|
|
|
s.insert(ValueType::Boolean);
|
|
|
|
s.insert(ValueType::Instant);
|
|
|
|
s.insert(ValueType::Long);
|
|
|
|
s.insert(ValueType::Double);
|
|
|
|
s.insert(ValueType::String);
|
|
|
|
s.insert(ValueType::Keyword);
|
2017-04-29 03:11:55 +00:00
|
|
|
s.insert(ValueType::Uuid);
|
2017-04-24 21:15:26 +00:00
|
|
|
s
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl enum_set::CLike for ValueType {
|
|
|
|
fn to_u32(&self) -> u32 {
|
|
|
|
*self as u32
|
|
|
|
}
|
|
|
|
|
|
|
|
unsafe fn from_u32(v: u32) -> ValueType {
|
|
|
|
std::mem::transmute(v)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-30 10:08:36 +00:00
|
|
|
impl ValueType {
|
2018-01-23 16:23:37 +00:00
|
|
|
pub fn into_keyword(self) -> NamespacedKeyword {
|
|
|
|
NamespacedKeyword::new("db.type", match self {
|
|
|
|
ValueType::Ref => "ref",
|
|
|
|
ValueType::Boolean => "boolean",
|
|
|
|
ValueType::Instant => "instant",
|
|
|
|
ValueType::Long => "long",
|
|
|
|
ValueType::Double => "double",
|
|
|
|
ValueType::String => "string",
|
|
|
|
ValueType::Keyword => "keyword",
|
|
|
|
ValueType::Uuid => "uuid",
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn into_typed_value(self) -> TypedValue {
|
|
|
|
TypedValue::typed_ns_keyword("db.type", match self {
|
|
|
|
ValueType::Ref => "ref",
|
|
|
|
ValueType::Boolean => "boolean",
|
|
|
|
ValueType::Instant => "instant",
|
|
|
|
ValueType::Long => "long",
|
|
|
|
ValueType::Double => "double",
|
|
|
|
ValueType::String => "string",
|
|
|
|
ValueType::Keyword => "keyword",
|
|
|
|
ValueType::Uuid => "uuid",
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn into_edn_value(self) -> edn::Value {
|
2017-03-30 10:08:36 +00:00
|
|
|
match self {
|
2017-04-18 04:09:45 +00:00
|
|
|
ValueType::Ref => values::DB_TYPE_REF.clone(),
|
|
|
|
ValueType::Boolean => values::DB_TYPE_BOOLEAN.clone(),
|
|
|
|
ValueType::Instant => values::DB_TYPE_INSTANT.clone(),
|
|
|
|
ValueType::Long => values::DB_TYPE_LONG.clone(),
|
|
|
|
ValueType::Double => values::DB_TYPE_DOUBLE.clone(),
|
|
|
|
ValueType::String => values::DB_TYPE_STRING.clone(),
|
|
|
|
ValueType::Keyword => values::DB_TYPE_KEYWORD.clone(),
|
2017-04-29 03:11:55 +00:00
|
|
|
ValueType::Uuid => values::DB_TYPE_UUID.clone(),
|
2017-03-30 10:08:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-18 04:09:45 +00:00
|
|
|
impl fmt::Display for ValueType {
|
|
|
|
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
|
|
|
write!(f, "{}", match *self {
|
2017-04-18 20:50:58 +00:00
|
|
|
ValueType::Ref => ":db.type/ref",
|
|
|
|
ValueType::Boolean => ":db.type/boolean",
|
|
|
|
ValueType::Instant => ":db.type/instant",
|
|
|
|
ValueType::Long => ":db.type/long",
|
|
|
|
ValueType::Double => ":db.type/double",
|
|
|
|
ValueType::String => ":db.type/string",
|
|
|
|
ValueType::Keyword => ":db.type/keyword",
|
2017-04-29 03:11:55 +00:00
|
|
|
ValueType::Uuid => ":db.type/uuid",
|
2017-04-18 04:09:45 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-08 21:59:56 +00:00
|
|
|
/// Represents a Mentat value in a particular value set.
|
2018-04-24 22:04:00 +00:00
|
|
|
// TODO: expand to include :db.type/uri. https://github.com/mozilla/mentat/issues/201
|
|
|
|
// TODO: JSON data type? https://github.com/mozilla/mentat/issues/31
|
|
|
|
// TODO: BigInt? Bytes?
|
2018-02-09 17:55:19 +00:00
|
|
|
#[derive(Clone,Debug,Eq,Hash,Ord,PartialOrd,PartialEq,Serialize,Deserialize)]
|
2017-02-08 21:59:56 +00:00
|
|
|
pub enum TypedValue {
|
|
|
|
Ref(Entid),
|
|
|
|
Boolean(bool),
|
|
|
|
Long(i64),
|
|
|
|
Double(OrderedFloat<f64>),
|
2018-01-18 15:06:23 +00:00
|
|
|
Instant(DateTime<Utc>), // Use `into()` to ensure truncation.
|
2017-02-08 21:59:56 +00:00
|
|
|
// TODO: &str throughout?
|
Use Rc for TypedValue, Variable, and query Ident keywords. (#395) r=nalexander
Part 1, core: use Rc for String and Keyword.
Part 2, query: use Rc for Variable.
Part 3, sql: use Rc for args in SQLiteQueryBuilder.
Part 4, query-algebrizer: use Rc.
Part 5, db: use Rc.
Part 6, query-parser: use Rc.
Part 7, query-projector: use Rc.
Part 8, query-translator: use Rc.
Part 9, top level: use Rc.
Part 10: intern Ident and IdentOrKeyword.
2017-03-29 20:18:17 +00:00
|
|
|
String(Rc<String>),
|
|
|
|
Keyword(Rc<NamespacedKeyword>),
|
2017-04-29 03:11:55 +00:00
|
|
|
Uuid(Uuid), // It's only 128 bits, so this should be acceptable to clone.
|
2017-02-08 21:59:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl TypedValue {
|
2017-02-10 01:12:55 +00:00
|
|
|
/// Returns true if the provided type is `Some` and matches this value's type, or if the
|
|
|
|
/// provided type is `None`.
|
|
|
|
#[inline]
|
|
|
|
pub fn is_congruent_with<T: Into<Option<ValueType>>>(&self, t: T) -> bool {
|
|
|
|
t.into().map_or(true, |x| self.matches_type(x))
|
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
|
|
|
pub fn matches_type(&self, t: ValueType) -> bool {
|
|
|
|
self.value_type() == t
|
|
|
|
}
|
|
|
|
|
2017-02-08 21:59:56 +00:00
|
|
|
pub fn value_type(&self) -> ValueType {
|
|
|
|
match self {
|
|
|
|
&TypedValue::Ref(_) => ValueType::Ref,
|
|
|
|
&TypedValue::Boolean(_) => ValueType::Boolean,
|
|
|
|
&TypedValue::Long(_) => ValueType::Long,
|
2017-04-29 03:11:55 +00:00
|
|
|
&TypedValue::Instant(_) => ValueType::Instant,
|
2017-02-08 21:59:56 +00:00
|
|
|
&TypedValue::Double(_) => ValueType::Double,
|
|
|
|
&TypedValue::String(_) => ValueType::String,
|
|
|
|
&TypedValue::Keyword(_) => ValueType::Keyword,
|
2017-04-29 03:11:55 +00:00
|
|
|
&TypedValue::Uuid(_) => ValueType::Uuid,
|
2017-02-08 21:59:56 +00:00
|
|
|
}
|
|
|
|
}
|
2017-03-29 21:11:00 +00:00
|
|
|
|
|
|
|
/// Construct a new `TypedValue::Keyword` instance by cloning the provided
|
Use Rc for TypedValue, Variable, and query Ident keywords. (#395) r=nalexander
Part 1, core: use Rc for String and Keyword.
Part 2, query: use Rc for Variable.
Part 3, sql: use Rc for args in SQLiteQueryBuilder.
Part 4, query-algebrizer: use Rc.
Part 5, db: use Rc.
Part 6, query-parser: use Rc.
Part 7, query-projector: use Rc.
Part 8, query-translator: use Rc.
Part 9, top level: use Rc.
Part 10: intern Ident and IdentOrKeyword.
2017-03-29 20:18:17 +00:00
|
|
|
/// values and wrapping them in a new `Rc`. This is expensive, so this might
|
2017-03-29 21:11:00 +00:00
|
|
|
/// be best limited to tests.
|
|
|
|
pub fn typed_ns_keyword(ns: &str, name: &str) -> TypedValue {
|
Use Rc for TypedValue, Variable, and query Ident keywords. (#395) r=nalexander
Part 1, core: use Rc for String and Keyword.
Part 2, query: use Rc for Variable.
Part 3, sql: use Rc for args in SQLiteQueryBuilder.
Part 4, query-algebrizer: use Rc.
Part 5, db: use Rc.
Part 6, query-parser: use Rc.
Part 7, query-projector: use Rc.
Part 8, query-translator: use Rc.
Part 9, top level: use Rc.
Part 10: intern Ident and IdentOrKeyword.
2017-03-29 20:18:17 +00:00
|
|
|
TypedValue::Keyword(Rc::new(NamespacedKeyword::new(ns, name)))
|
2017-03-29 21:11:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Construct a new `TypedValue::String` instance by cloning the provided
|
Use Rc for TypedValue, Variable, and query Ident keywords. (#395) r=nalexander
Part 1, core: use Rc for String and Keyword.
Part 2, query: use Rc for Variable.
Part 3, sql: use Rc for args in SQLiteQueryBuilder.
Part 4, query-algebrizer: use Rc.
Part 5, db: use Rc.
Part 6, query-parser: use Rc.
Part 7, query-projector: use Rc.
Part 8, query-translator: use Rc.
Part 9, top level: use Rc.
Part 10: intern Ident and IdentOrKeyword.
2017-03-29 20:18:17 +00:00
|
|
|
/// value and wrapping it in a new `Rc`. This is expensive, so this might
|
2017-03-29 21:11:00 +00:00
|
|
|
/// be best limited to tests.
|
|
|
|
pub fn typed_string(s: &str) -> TypedValue {
|
Use Rc for TypedValue, Variable, and query Ident keywords. (#395) r=nalexander
Part 1, core: use Rc for String and Keyword.
Part 2, query: use Rc for Variable.
Part 3, sql: use Rc for args in SQLiteQueryBuilder.
Part 4, query-algebrizer: use Rc.
Part 5, db: use Rc.
Part 6, query-parser: use Rc.
Part 7, query-projector: use Rc.
Part 8, query-translator: use Rc.
Part 9, top level: use Rc.
Part 10: intern Ident and IdentOrKeyword.
2017-03-29 20:18:17 +00:00
|
|
|
TypedValue::String(Rc::new(s.to_string()))
|
2017-03-29 21:11:00 +00:00
|
|
|
}
|
2017-04-29 03:11:55 +00:00
|
|
|
|
|
|
|
pub fn current_instant() -> TypedValue {
|
2018-01-18 15:06:23 +00:00
|
|
|
Utc::now().into()
|
2017-04-29 03:11:55 +00:00
|
|
|
}
|
2018-02-01 17:17:07 +00:00
|
|
|
|
|
|
|
/// Construct a new `TypedValue::Instant` instance from the provided
|
|
|
|
/// microsecond timestamp.
|
|
|
|
pub fn instant(micros: i64) -> TypedValue {
|
|
|
|
DateTime::<Utc>::from_micros(micros).into()
|
|
|
|
}
|
2017-02-08 21:59:56 +00:00
|
|
|
}
|
|
|
|
|
2018-01-22 18:17:12 +00:00
|
|
|
trait MicrosecondPrecision {
|
|
|
|
/// Truncate the provided `DateTime` to microsecond precision.
|
|
|
|
fn microsecond_precision(self) -> Self;
|
|
|
|
}
|
|
|
|
|
|
|
|
impl MicrosecondPrecision for DateTime<Utc> {
|
|
|
|
fn microsecond_precision(self) -> DateTime<Utc> {
|
|
|
|
let nanoseconds = self.nanosecond();
|
|
|
|
if nanoseconds % 1000 == 0 {
|
|
|
|
return self;
|
|
|
|
}
|
|
|
|
let microseconds = nanoseconds / 1000;
|
|
|
|
let truncated = microseconds * 1000;
|
|
|
|
self.with_nanosecond(truncated).expect("valid timestamp")
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Return the current time as a UTC `DateTime` instance with microsecond precision.
|
|
|
|
pub fn now() -> DateTime<Utc> {
|
|
|
|
Utc::now().microsecond_precision()
|
|
|
|
}
|
|
|
|
|
2017-12-06 20:29:11 +00:00
|
|
|
// We don't do From<i64> or From<Entid> 'cos it's ambiguous.
|
|
|
|
|
|
|
|
impl From<bool> for TypedValue {
|
|
|
|
fn from(value: bool) -> TypedValue {
|
|
|
|
TypedValue::Boolean(value)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-18 15:06:23 +00:00
|
|
|
/// Truncate the provided `DateTime` to microsecond precision, and return the corresponding
|
|
|
|
/// `TypedValue::Instant`.
|
2017-12-06 20:29:11 +00:00
|
|
|
impl From<DateTime<Utc>> for TypedValue {
|
|
|
|
fn from(value: DateTime<Utc>) -> TypedValue {
|
2018-01-22 18:17:12 +00:00
|
|
|
TypedValue::Instant(value.microsecond_precision())
|
2017-12-06 20:29:11 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<Uuid> for TypedValue {
|
|
|
|
fn from(value: Uuid) -> TypedValue {
|
|
|
|
TypedValue::Uuid(value)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-12 22:18:50 +00:00
|
|
|
impl<'a> From<&'a str> for TypedValue {
|
|
|
|
fn from(value: &'a str) -> TypedValue {
|
|
|
|
TypedValue::String(Rc::new(value.to_string()))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-12-06 20:29:11 +00:00
|
|
|
impl From<String> for TypedValue {
|
|
|
|
fn from(value: String) -> TypedValue {
|
|
|
|
TypedValue::String(Rc::new(value))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<NamespacedKeyword> for TypedValue {
|
|
|
|
fn from(value: NamespacedKeyword) -> TypedValue {
|
|
|
|
TypedValue::Keyword(Rc::new(value))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<u32> for TypedValue {
|
|
|
|
fn from(value: u32) -> TypedValue {
|
|
|
|
TypedValue::Long(value as i64)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl From<i32> for TypedValue {
|
|
|
|
fn from(value: i32) -> TypedValue {
|
|
|
|
TypedValue::Long(value as i64)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-01 17:17:07 +00:00
|
|
|
impl From<f64> for TypedValue {
|
|
|
|
fn from(value: f64) -> TypedValue {
|
|
|
|
TypedValue::Double(OrderedFloat(value))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-05 10:08:20 +00:00
|
|
|
impl TypedValue {
|
2018-04-03 22:04:04 +00:00
|
|
|
pub fn into_known_entid(self) -> Option<KnownEntid> {
|
|
|
|
match self {
|
|
|
|
TypedValue::Ref(v) => Some(KnownEntid(v)),
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-05 10:08:20 +00:00
|
|
|
pub fn into_entid(self) -> Option<Entid> {
|
|
|
|
match self {
|
|
|
|
TypedValue::Ref(v) => Some(v),
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn into_kw(self) -> Option<Rc<NamespacedKeyword>> {
|
|
|
|
match self {
|
|
|
|
TypedValue::Keyword(v) => Some(v),
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn into_boolean(self) -> Option<bool> {
|
|
|
|
match self {
|
|
|
|
TypedValue::Boolean(v) => Some(v),
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn into_long(self) -> Option<i64> {
|
|
|
|
match self {
|
|
|
|
TypedValue::Long(v) => Some(v),
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn into_double(self) -> Option<f64> {
|
|
|
|
match self {
|
|
|
|
TypedValue::Double(v) => Some(v.into_inner()),
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn into_instant(self) -> Option<DateTime<Utc>> {
|
|
|
|
match self {
|
|
|
|
TypedValue::Instant(v) => Some(v),
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn into_timestamp(self) -> Option<i64> {
|
|
|
|
match self {
|
|
|
|
TypedValue::Instant(v) => Some(v.timestamp()),
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn into_string(self) -> Option<Rc<String>> {
|
|
|
|
match self {
|
|
|
|
TypedValue::String(v) => Some(v),
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn into_uuid(self) -> Option<Uuid> {
|
|
|
|
match self {
|
|
|
|
TypedValue::Uuid(v) => Some(v),
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn into_uuid_string(self) -> Option<String> {
|
|
|
|
match self {
|
|
|
|
TypedValue::Uuid(v) => Some(v.hyphenated().to_string()),
|
|
|
|
_ => None,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-01-29 22:29:16 +00:00
|
|
|
/// Type safe representation of the possible return values from SQLite's `typeof`
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialOrd, PartialEq)]
|
|
|
|
pub enum SQLTypeAffinity {
|
|
|
|
Null, // "null"
|
|
|
|
Integer, // "integer"
|
|
|
|
Real, // "real"
|
|
|
|
Text, // "text"
|
|
|
|
Blob, // "blob"
|
|
|
|
}
|
|
|
|
|
2017-03-07 04:14:50 +00:00
|
|
|
// Put this here rather than in `db` simply because it's widely needed.
|
|
|
|
pub trait SQLValueType {
|
2018-01-29 22:29:16 +00:00
|
|
|
fn value_type_tag(&self) -> ValueTypeTag;
|
2017-03-07 04:18:38 +00:00
|
|
|
fn accommodates_integer(&self, int: i64) -> bool;
|
2018-01-29 22:29:16 +00:00
|
|
|
|
|
|
|
/// Return a pair of the ValueTypeTag for this value type, and the SQLTypeAffinity required
|
|
|
|
/// to distinguish it from any other types that share the same tag.
|
|
|
|
///
|
|
|
|
/// Background: The tag alone is not enough to determine the type of a value, since multiple
|
|
|
|
/// ValueTypes may share the same tag (for example, ValueType::Long and ValueType::Double).
|
|
|
|
/// However, each ValueType can be determined by checking both the tag and the type's affinity.
|
|
|
|
fn sql_representation(&self) -> (ValueTypeTag, Option<SQLTypeAffinity>);
|
2017-03-07 04:14:50 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl SQLValueType for ValueType {
|
2018-01-29 22:29:16 +00:00
|
|
|
fn sql_representation(&self) -> (ValueTypeTag, Option<SQLTypeAffinity>) {
|
2017-03-07 04:14:50 +00:00
|
|
|
match *self {
|
2018-01-29 22:29:16 +00:00
|
|
|
ValueType::Ref => (0, None),
|
|
|
|
ValueType::Boolean => (1, None),
|
|
|
|
ValueType::Instant => (4, None),
|
|
|
|
|
2017-03-07 04:14:50 +00:00
|
|
|
// SQLite distinguishes integral from decimal types, allowing long and double to share a tag.
|
2018-01-29 22:29:16 +00:00
|
|
|
ValueType::Long => (5, Some(SQLTypeAffinity::Integer)),
|
|
|
|
ValueType::Double => (5, Some(SQLTypeAffinity::Real)),
|
|
|
|
ValueType::String => (10, None),
|
|
|
|
ValueType::Uuid => (11, None),
|
|
|
|
ValueType::Keyword => (13, None),
|
2017-03-07 04:14:50 +00:00
|
|
|
}
|
|
|
|
}
|
2017-03-07 04:18:38 +00:00
|
|
|
|
2018-01-29 22:29:16 +00:00
|
|
|
#[inline]
|
|
|
|
fn value_type_tag(&self) -> ValueTypeTag {
|
|
|
|
self.sql_representation().0
|
|
|
|
}
|
|
|
|
|
2017-03-07 04:18:38 +00:00
|
|
|
/// Returns true if the provided integer is in the SQLite value space of this type. For
|
|
|
|
/// example, `1` is how we encode `true`.
|
|
|
|
///
|
|
|
|
/// ```
|
|
|
|
/// use mentat_core::{ValueType, SQLValueType};
|
2017-04-29 03:11:55 +00:00
|
|
|
/// assert!(!ValueType::Instant.accommodates_integer(1493399581314));
|
|
|
|
/// assert!(!ValueType::Instant.accommodates_integer(1493399581314000));
|
2017-03-07 04:18:38 +00:00
|
|
|
/// assert!(ValueType::Boolean.accommodates_integer(1));
|
|
|
|
/// assert!(!ValueType::Boolean.accommodates_integer(-1));
|
|
|
|
/// assert!(!ValueType::Boolean.accommodates_integer(10));
|
|
|
|
/// assert!(!ValueType::String.accommodates_integer(10));
|
|
|
|
/// ```
|
|
|
|
fn accommodates_integer(&self, int: i64) -> bool {
|
|
|
|
use ValueType::*;
|
|
|
|
match *self {
|
2017-04-29 03:11:55 +00:00
|
|
|
Instant => false, // Always use #inst.
|
|
|
|
Long | Double => true,
|
2017-03-07 04:18:38 +00:00
|
|
|
Ref => int >= 0,
|
|
|
|
Boolean => (int == 0) || (int == 1),
|
|
|
|
ValueType::String => false,
|
|
|
|
Keyword => false,
|
2017-04-29 03:11:55 +00:00
|
|
|
Uuid => false,
|
2017-03-07 04:18:38 +00:00
|
|
|
}
|
|
|
|
}
|
2017-03-07 04:14:50 +00:00
|
|
|
}
|
|
|
|
|
2017-11-30 23:02:07 +00:00
|
|
|
trait EnumSetExtensions<T: enum_set::CLike + Clone> {
|
|
|
|
/// Return a set containing both `x` and `y`.
|
|
|
|
fn of_both(x: T, y: T) -> EnumSet<T>;
|
|
|
|
|
|
|
|
/// Return a clone of `self` with `y` added.
|
|
|
|
fn with(&self, y: T) -> EnumSet<T>;
|
|
|
|
}
|
|
|
|
|
|
|
|
impl<T: enum_set::CLike + Clone> EnumSetExtensions<T> for EnumSet<T> {
|
|
|
|
/// Return a set containing both `x` and `y`.
|
|
|
|
fn of_both(x: T, y: T) -> Self {
|
|
|
|
let mut o = EnumSet::new();
|
|
|
|
o.insert(x);
|
|
|
|
o.insert(y);
|
|
|
|
o
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Return a clone of `self` with `y` added.
|
|
|
|
fn with(&self, y: T) -> EnumSet<T> {
|
|
|
|
let mut o = self.clone();
|
|
|
|
o.insert(y);
|
|
|
|
o
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
|
|
|
pub struct ValueTypeSet(pub EnumSet<ValueType>);
|
|
|
|
|
|
|
|
impl Default for ValueTypeSet {
|
|
|
|
fn default() -> ValueTypeSet {
|
|
|
|
ValueTypeSet::any()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl ValueTypeSet {
|
|
|
|
pub fn any() -> ValueTypeSet {
|
|
|
|
ValueTypeSet(ValueType::all_enums())
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn none() -> ValueTypeSet {
|
|
|
|
ValueTypeSet(EnumSet::new())
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Return a set containing only `t`.
|
|
|
|
pub fn of_one(t: ValueType) -> ValueTypeSet {
|
|
|
|
let mut s = EnumSet::new();
|
|
|
|
s.insert(t);
|
|
|
|
ValueTypeSet(s)
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Return a set containing `Double` and `Long`.
|
|
|
|
pub fn of_numeric_types() -> ValueTypeSet {
|
|
|
|
ValueTypeSet(EnumSet::of_both(ValueType::Double, ValueType::Long))
|
|
|
|
}
|
|
|
|
|
2018-03-12 22:18:50 +00:00
|
|
|
/// Return a set containing `Double`, `Long`, and `Instant`.
|
|
|
|
pub fn of_numeric_and_instant_types() -> ValueTypeSet {
|
|
|
|
let mut s = EnumSet::new();
|
|
|
|
s.insert(ValueType::Double);
|
|
|
|
s.insert(ValueType::Long);
|
|
|
|
s.insert(ValueType::Instant);
|
|
|
|
ValueTypeSet(s)
|
|
|
|
}
|
|
|
|
|
2017-11-30 23:02:07 +00:00
|
|
|
/// Return a set containing `Ref` and `Keyword`.
|
|
|
|
pub fn of_keywords() -> ValueTypeSet {
|
|
|
|
ValueTypeSet(EnumSet::of_both(ValueType::Ref, ValueType::Keyword))
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Return a set containing `Ref` and `Long`.
|
|
|
|
pub fn of_longs() -> ValueTypeSet {
|
|
|
|
ValueTypeSet(EnumSet::of_both(ValueType::Ref, ValueType::Long))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl ValueTypeSet {
|
|
|
|
pub fn insert(&mut self, vt: ValueType) -> bool {
|
|
|
|
self.0.insert(vt)
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn len(&self) -> usize {
|
|
|
|
self.0.len()
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Returns a set containing all the types in this set and `other`.
|
|
|
|
pub fn union(&self, other: &ValueTypeSet) -> ValueTypeSet {
|
|
|
|
ValueTypeSet(self.0.union(other.0))
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn intersection(&self, other: &ValueTypeSet) -> ValueTypeSet {
|
|
|
|
ValueTypeSet(self.0.intersection(other.0))
|
|
|
|
}
|
|
|
|
|
2018-01-29 22:29:16 +00:00
|
|
|
/// Returns the set difference between `self` and `other`, which is the
|
|
|
|
/// set of items in `self` that are not in `other`.
|
|
|
|
pub fn difference(&self, other: &ValueTypeSet) -> ValueTypeSet {
|
|
|
|
ValueTypeSet(self.0 - other.0)
|
|
|
|
}
|
|
|
|
|
2017-11-30 23:02:07 +00:00
|
|
|
/// Return an arbitrary type that's part of this set.
|
|
|
|
/// For a set containing a single type, this will be that type.
|
|
|
|
pub fn exemplar(&self) -> Option<ValueType> {
|
|
|
|
self.0.iter().next()
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn is_subset(&self, other: &ValueTypeSet) -> bool {
|
|
|
|
self.0.is_subset(&other.0)
|
|
|
|
}
|
|
|
|
|
2018-01-29 22:29:16 +00:00
|
|
|
/// Returns true if `self` and `other` contain no items in common.
|
|
|
|
pub fn is_disjoint(&self, other: &ValueTypeSet) -> bool {
|
|
|
|
self.0.is_disjoint(&other.0)
|
|
|
|
}
|
|
|
|
|
2017-11-30 23:02:07 +00:00
|
|
|
pub fn contains(&self, vt: ValueType) -> bool {
|
|
|
|
self.0.contains(&vt)
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn is_empty(&self) -> bool {
|
|
|
|
self.0.is_empty()
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn is_unit(&self) -> bool {
|
|
|
|
self.0.len() == 1
|
|
|
|
}
|
2018-01-29 22:29:16 +00:00
|
|
|
|
|
|
|
pub fn iter(&self) -> ::enum_set::Iter<ValueType> {
|
|
|
|
self.0.iter()
|
|
|
|
}
|
2017-11-30 23:02:07 +00:00
|
|
|
}
|
|
|
|
|
2018-03-12 22:18:50 +00:00
|
|
|
impl From<ValueType> for ValueTypeSet {
|
|
|
|
fn from(t: ValueType) -> Self {
|
|
|
|
ValueTypeSet::of_one(t)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl ValueTypeSet {
|
|
|
|
pub fn is_only_numeric(&self) -> bool {
|
|
|
|
self.is_subset(&ValueTypeSet::of_numeric_types())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-11-30 23:02:07 +00:00
|
|
|
impl IntoIterator for ValueTypeSet {
|
|
|
|
type Item = ValueType;
|
|
|
|
type IntoIter = ::enum_set::Iter<ValueType>;
|
|
|
|
|
|
|
|
fn into_iter(self) -> Self::IntoIter {
|
|
|
|
self.0.into_iter()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl ::std::iter::FromIterator<ValueType> for ValueTypeSet {
|
|
|
|
fn from_iter<I: IntoIterator<Item = ValueType>>(iterator: I) -> Self {
|
|
|
|
let mut ret = Self::none();
|
|
|
|
ret.0.extend(iterator);
|
|
|
|
ret
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl ::std::iter::Extend<ValueType> for ValueTypeSet {
|
|
|
|
fn extend<I: IntoIterator<Item = ValueType>>(&mut self, iter: I) {
|
|
|
|
for element in iter {
|
|
|
|
self.0.insert(element);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-12 22:18:50 +00:00
|
|
|
/// We have an enum of types, `ValueType`. It can be collected into a set, `ValueTypeSet`. Each type
|
|
|
|
/// is associated with a type tag, which is how a type is represented in, e.g., SQL storage. Types
|
|
|
|
/// can share type tags, because backing SQL storage is able to differentiate between some types
|
|
|
|
/// (e.g., longs and doubles), and so distinct tags aren't necessary. That association is defined by
|
|
|
|
/// `SQLValueType`. That trait similarly extends to `ValueTypeSet`, which maps a collection of types
|
|
|
|
/// into a collection of tags.
|
2017-11-30 23:02:07 +00:00
|
|
|
pub trait SQLValueTypeSet {
|
|
|
|
fn value_type_tags(&self) -> BTreeSet<ValueTypeTag>;
|
2018-03-12 22:18:50 +00:00
|
|
|
fn has_unique_type_tag(&self) -> bool;
|
|
|
|
fn unique_type_tag(&self) -> Option<ValueTypeTag>;
|
2017-11-30 23:02:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl SQLValueTypeSet for ValueTypeSet {
|
|
|
|
// This is inefficient, but it'll do for now.
|
|
|
|
fn value_type_tags(&self) -> BTreeSet<ValueTypeTag> {
|
|
|
|
let mut out = BTreeSet::new();
|
|
|
|
for t in self.0.iter() {
|
|
|
|
out.insert(t.value_type_tag());
|
|
|
|
}
|
|
|
|
out
|
|
|
|
}
|
|
|
|
|
2018-03-12 22:18:50 +00:00
|
|
|
fn unique_type_tag(&self) -> Option<ValueTypeTag> {
|
|
|
|
if self.is_unit() || self.has_unique_type_tag() {
|
2017-11-30 23:02:07 +00:00
|
|
|
self.exemplar().map(|t| t.value_type_tag())
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-12 22:18:50 +00:00
|
|
|
fn has_unique_type_tag(&self) -> bool {
|
2017-11-30 23:02:07 +00:00
|
|
|
if self.is_unit() {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut acc = BTreeSet::new();
|
|
|
|
for t in self.0.iter() {
|
|
|
|
if acc.insert(t.value_type_tag()) && acc.len() > 1 {
|
|
|
|
// We inserted a second or subsequent value.
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
!acc.is_empty()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-10 01:12:55 +00:00
|
|
|
#[test]
|
|
|
|
fn test_typed_value() {
|
|
|
|
assert!(TypedValue::Boolean(false).is_congruent_with(None));
|
|
|
|
assert!(TypedValue::Boolean(false).is_congruent_with(ValueType::Boolean));
|
2017-03-29 21:11:00 +00:00
|
|
|
assert!(!TypedValue::typed_string("foo").is_congruent_with(ValueType::Boolean));
|
|
|
|
assert!(TypedValue::typed_string("foo").is_congruent_with(ValueType::String));
|
|
|
|
assert!(TypedValue::typed_string("foo").is_congruent_with(None));
|
2017-02-10 01:12:55 +00:00
|
|
|
}
|
|
|
|
|
2017-02-10 20:03:49 +00:00
|
|
|
/// Bit flags used in `flags0` column in temporary tables created during search,
|
|
|
|
/// such as the `search_results`, `inexact_searches` and `exact_searches` tables.
|
|
|
|
/// When moving to a more concrete table, such as `datoms`, they are expanded out
|
|
|
|
/// via these flags and put into their own column rather than a bit field.
|
|
|
|
pub enum AttributeBitFlags {
|
|
|
|
IndexAVET = 1 << 0,
|
|
|
|
IndexVAET = 1 << 1,
|
|
|
|
IndexFulltext = 1 << 2,
|
|
|
|
UniqueValue = 1 << 3,
|
|
|
|
}
|
|
|
|
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
pub mod attribute {
|
2018-01-23 16:23:37 +00:00
|
|
|
use TypedValue;
|
|
|
|
|
|
|
|
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialOrd, PartialEq)]
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
pub enum Unique {
|
|
|
|
Value,
|
|
|
|
Identity,
|
|
|
|
}
|
2018-01-23 16:23:37 +00:00
|
|
|
|
|
|
|
impl Unique {
|
|
|
|
// This is easier than rejigging DB_UNIQUE_VALUE to not be EDN.
|
|
|
|
pub fn into_typed_value(self) -> TypedValue {
|
|
|
|
match self {
|
|
|
|
Unique::Value => TypedValue::typed_ns_keyword("db.unique", "value"),
|
|
|
|
Unique::Identity => TypedValue::typed_ns_keyword("db.unique", "identity"),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
}
|
|
|
|
|
2017-02-04 00:51:13 +00:00
|
|
|
/// A Mentat schema attribute has a value type and several other flags determining how assertions
|
|
|
|
/// with the attribute are interpreted.
|
|
|
|
///
|
|
|
|
/// TODO: consider packing this into a bitfield or similar.
|
|
|
|
#[derive(Clone,Debug,Eq,Hash,Ord,PartialOrd,PartialEq)]
|
|
|
|
pub struct Attribute {
|
|
|
|
/// The associated value type, i.e., `:db/valueType`?
|
|
|
|
pub value_type: ValueType,
|
|
|
|
|
|
|
|
/// `true` if this attribute is multi-valued, i.e., it is `:db/cardinality
|
|
|
|
/// :db.cardinality/many`. `false` if this attribute is single-valued (the default), i.e., it
|
|
|
|
/// is `:db/cardinality :db.cardinality/one`.
|
|
|
|
pub multival: bool,
|
|
|
|
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
/// `None` if this attribute is neither unique-value nor unique-identity.
|
|
|
|
///
|
|
|
|
/// `Some(attribute::Unique::Value)` if this attribute is unique-value, i.e., it is `:db/unique
|
|
|
|
/// :db.unique/value`.
|
2017-02-04 00:51:13 +00:00
|
|
|
///
|
|
|
|
/// *Unique-value* means that there is at most one assertion with the attribute and a
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
/// particular value in the datom store. Unique-value attributes can be used in lookup-refs.
|
|
|
|
///
|
|
|
|
/// `Some(attribute::Unique::Identity)` if this attribute is unique-identity, i.e., it is `:db/unique
|
|
|
|
/// :db.unique/identity`.
|
2017-02-04 00:51:13 +00:00
|
|
|
///
|
|
|
|
/// Unique-identity attributes always have value type `Ref`.
|
|
|
|
///
|
|
|
|
/// *Unique-identity* means that the attribute is *unique-value* and that they can be used in
|
|
|
|
/// lookup-refs and will automatically upsert where appropriate.
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
pub unique: Option<attribute::Unique>,
|
2017-02-04 00:51:13 +00:00
|
|
|
|
|
|
|
/// `true` if this attribute is automatically indexed, i.e., it is `:db/indexing true`.
|
|
|
|
pub index: bool,
|
|
|
|
|
|
|
|
/// `true` if this attribute is automatically fulltext indexed, i.e., it is `:db/fulltext true`.
|
|
|
|
///
|
|
|
|
/// Fulltext attributes always have string values.
|
|
|
|
pub fulltext: bool,
|
|
|
|
|
|
|
|
/// `true` if this attribute is a component, i.e., it is `:db/isComponent true`.
|
|
|
|
///
|
|
|
|
/// Component attributes always have value type `Ref`.
|
|
|
|
///
|
|
|
|
/// They are used to compose entities from component sub-entities: they are fetched recursively
|
|
|
|
/// by pull expressions, and they are automatically recursively deleted where appropriate.
|
|
|
|
pub component: bool,
|
2018-04-03 21:23:46 +00:00
|
|
|
|
|
|
|
/// `true` if this attribute doesn't require history to be kept, i.e., it is `:db/noHistory true`.
|
|
|
|
pub no_history: bool,
|
2017-02-04 00:51:13 +00:00
|
|
|
}
|
|
|
|
|
2017-02-10 20:03:49 +00:00
|
|
|
impl Attribute {
|
|
|
|
/// Combine several attribute flags into a bitfield used in temporary search tables.
|
|
|
|
pub fn flags(&self) -> u8 {
|
|
|
|
let mut flags: u8 = 0;
|
|
|
|
|
|
|
|
if self.index {
|
|
|
|
flags |= AttributeBitFlags::IndexAVET as u8;
|
|
|
|
}
|
|
|
|
if self.value_type == ValueType::Ref {
|
|
|
|
flags |= AttributeBitFlags::IndexVAET as u8;
|
|
|
|
}
|
|
|
|
if self.fulltext {
|
|
|
|
flags |= AttributeBitFlags::IndexFulltext as u8;
|
|
|
|
}
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
if self.unique.is_some() {
|
2017-02-10 20:03:49 +00:00
|
|
|
flags |= AttributeBitFlags::UniqueValue as u8;
|
|
|
|
}
|
|
|
|
flags
|
|
|
|
}
|
2017-03-30 10:08:36 +00:00
|
|
|
|
|
|
|
pub fn to_edn_value(&self, ident: Option<NamespacedKeyword>) -> edn::Value {
|
|
|
|
let mut attribute_map: BTreeMap<edn::Value, edn::Value> = BTreeMap::default();
|
|
|
|
if let Some(ident) = ident {
|
|
|
|
attribute_map.insert(values::DB_IDENT.clone(), edn::Value::NamespacedKeyword(ident));
|
|
|
|
}
|
|
|
|
|
2018-01-23 16:23:37 +00:00
|
|
|
attribute_map.insert(values::DB_VALUE_TYPE.clone(), self.value_type.into_edn_value());
|
2017-03-30 10:08:36 +00:00
|
|
|
|
|
|
|
attribute_map.insert(values::DB_CARDINALITY.clone(), if self.multival { values::DB_CARDINALITY_MANY.clone() } else { values::DB_CARDINALITY_ONE.clone() });
|
|
|
|
|
|
|
|
match self.unique {
|
|
|
|
Some(attribute::Unique::Value) => { attribute_map.insert(values::DB_UNIQUE.clone(), values::DB_UNIQUE_VALUE.clone()); },
|
|
|
|
Some(attribute::Unique::Identity) => { attribute_map.insert(values::DB_UNIQUE.clone(), values::DB_UNIQUE_IDENTITY.clone()); },
|
2018-01-20 03:21:04 +00:00
|
|
|
None => (),
|
2017-03-30 10:08:36 +00:00
|
|
|
}
|
2018-01-20 03:21:04 +00:00
|
|
|
|
2017-03-30 10:08:36 +00:00
|
|
|
if self.index {
|
|
|
|
attribute_map.insert(values::DB_INDEX.clone(), edn::Value::Boolean(true));
|
|
|
|
}
|
|
|
|
|
|
|
|
if self.fulltext {
|
|
|
|
attribute_map.insert(values::DB_FULLTEXT.clone(), edn::Value::Boolean(true));
|
|
|
|
}
|
|
|
|
|
|
|
|
if self.component {
|
|
|
|
attribute_map.insert(values::DB_IS_COMPONENT.clone(), edn::Value::Boolean(true));
|
|
|
|
}
|
|
|
|
|
2018-04-03 21:23:46 +00:00
|
|
|
if self.no_history {
|
|
|
|
attribute_map.insert(values::DB_NO_HISTORY.clone(), edn::Value::Boolean(true));
|
|
|
|
}
|
|
|
|
|
2017-03-30 10:08:36 +00:00
|
|
|
edn::Value::Map(attribute_map)
|
|
|
|
}
|
2017-02-10 20:03:49 +00:00
|
|
|
}
|
|
|
|
|
2017-02-04 00:51:13 +00:00
|
|
|
impl Default for Attribute {
|
|
|
|
fn default() -> Attribute {
|
|
|
|
Attribute {
|
|
|
|
// There's no particular reason to favour one value type, so Ref it is.
|
|
|
|
value_type: ValueType::Ref,
|
|
|
|
fulltext: false,
|
|
|
|
index: false,
|
|
|
|
multival: false,
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
unique: None,
|
2017-02-04 00:51:13 +00:00
|
|
|
component: false,
|
2018-04-03 21:23:46 +00:00
|
|
|
no_history: false,
|
2017-02-04 00:51:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-02-17 21:55:36 +00:00
|
|
|
/// Map `NamespacedKeyword` idents (`:db/ident`) to positive integer entids (`1`).
|
|
|
|
pub type IdentMap = BTreeMap<NamespacedKeyword, Entid>;
|
2017-02-14 03:20:20 +00:00
|
|
|
|
2017-02-17 21:55:36 +00:00
|
|
|
/// Map positive integer entids (`1`) to `NamespacedKeyword` idents (`:db/ident`).
|
|
|
|
pub type EntidMap = BTreeMap<Entid, NamespacedKeyword>;
|
2017-02-14 03:20:20 +00:00
|
|
|
|
|
|
|
/// Map attribute entids to `Attribute` instances.
|
2018-01-23 16:23:37 +00:00
|
|
|
pub type AttributeMap = BTreeMap<Entid, Attribute>;
|
2017-02-14 03:20:20 +00:00
|
|
|
|
|
|
|
/// Represents a Mentat schema.
|
|
|
|
///
|
|
|
|
/// Maintains the mapping between string idents and positive integer entids; and exposes the schema
|
|
|
|
/// flags associated to a given entid (equivalently, ident).
|
|
|
|
///
|
|
|
|
/// TODO: consider a single bi-directional map instead of separate ident->entid and entid->ident
|
|
|
|
/// maps.
|
|
|
|
#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialOrd, PartialEq)]
|
|
|
|
pub struct Schema {
|
|
|
|
/// Map entid->ident.
|
|
|
|
///
|
|
|
|
/// Invariant: is the inverse map of `ident_map`.
|
|
|
|
pub entid_map: EntidMap,
|
|
|
|
|
|
|
|
/// Map ident->entid.
|
|
|
|
///
|
|
|
|
/// Invariant: is the inverse map of `entid_map`.
|
|
|
|
pub ident_map: IdentMap,
|
|
|
|
|
|
|
|
/// Map entid->attribute flags.
|
|
|
|
///
|
|
|
|
/// Invariant: key-set is the same as the key-set of `entid_map` (equivalently, the value-set of
|
|
|
|
/// `ident_map`).
|
2018-01-23 16:23:37 +00:00
|
|
|
pub attribute_map: AttributeMap,
|
2018-04-03 21:25:53 +00:00
|
|
|
|
|
|
|
/// Maintain a vec of unique attribute IDs for which the corresponding attribute in `attribute_map`
|
|
|
|
/// has `.component == true`.
|
|
|
|
pub component_attributes: Vec<Entid>,
|
2018-01-23 16:23:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pub trait HasSchema {
|
2018-01-23 16:31:27 +00:00
|
|
|
fn entid_for_type(&self, t: ValueType) -> Option<KnownEntid>;
|
|
|
|
|
|
|
|
fn get_ident<T>(&self, x: T) -> Option<&NamespacedKeyword> where T: Into<Entid>;
|
|
|
|
fn get_entid(&self, x: &NamespacedKeyword) -> Option<KnownEntid>;
|
|
|
|
fn attribute_for_entid<T>(&self, x: T) -> Option<&Attribute> where T: Into<Entid>;
|
|
|
|
|
|
|
|
// Returns the attribute and the entid named by the provided ident.
|
|
|
|
fn attribute_for_ident(&self, ident: &NamespacedKeyword) -> Option<(&Attribute, KnownEntid)>;
|
2018-01-23 16:23:37 +00:00
|
|
|
|
|
|
|
/// Return true if the provided entid identifies an attribute in this schema.
|
2018-01-23 16:31:27 +00:00
|
|
|
fn is_attribute<T>(&self, x: T) -> bool where T: Into<Entid>;
|
2018-01-23 16:23:37 +00:00
|
|
|
|
|
|
|
/// Return true if the provided ident identifies an attribute in this schema.
|
|
|
|
fn identifies_attribute(&self, x: &NamespacedKeyword) -> bool;
|
2018-04-03 21:25:53 +00:00
|
|
|
|
|
|
|
fn component_attributes(&self) -> &[Entid];
|
2017-02-14 03:20:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl Schema {
|
2018-04-03 21:25:53 +00:00
|
|
|
pub fn new(ident_map: IdentMap, entid_map: EntidMap, attribute_map: AttributeMap) -> Schema {
|
|
|
|
let mut s = Schema { ident_map, entid_map, attribute_map, component_attributes: Vec::new() };
|
|
|
|
s.update_component_attributes();
|
|
|
|
s
|
|
|
|
}
|
|
|
|
|
2018-01-23 16:23:37 +00:00
|
|
|
/// Returns an symbolic representation of the schema suitable for applying across Mentat stores.
|
|
|
|
pub fn to_edn_value(&self) -> edn::Value {
|
|
|
|
edn::Value::Vector((&self.attribute_map).iter()
|
|
|
|
.map(|(entid, attribute)|
|
|
|
|
attribute.to_edn_value(self.get_ident(*entid).cloned()))
|
|
|
|
.collect())
|
|
|
|
}
|
2018-01-23 16:31:27 +00:00
|
|
|
|
|
|
|
fn get_raw_entid(&self, x: &NamespacedKeyword) -> Option<Entid> {
|
|
|
|
self.ident_map.get(x).map(|x| *x)
|
|
|
|
}
|
2018-04-03 21:25:53 +00:00
|
|
|
|
|
|
|
pub fn update_component_attributes(&mut self) {
|
|
|
|
let mut components: Vec<Entid>;
|
|
|
|
components = self.attribute_map
|
|
|
|
.iter()
|
|
|
|
.filter_map(|(k, v)| if v.component { Some(*k) } else { None })
|
|
|
|
.collect();
|
|
|
|
components.sort_unstable();
|
|
|
|
self.component_attributes = components;
|
|
|
|
}
|
2018-01-23 16:23:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl HasSchema for Schema {
|
2018-01-23 16:31:27 +00:00
|
|
|
fn entid_for_type(&self, t: ValueType) -> Option<KnownEntid> {
|
|
|
|
// TODO: this can be made more efficient.
|
|
|
|
self.get_entid(&t.into_keyword())
|
2017-02-14 03:20:20 +00:00
|
|
|
}
|
|
|
|
|
2018-01-23 16:31:27 +00:00
|
|
|
fn get_ident<T>(&self, x: T) -> Option<&NamespacedKeyword> where T: Into<Entid> {
|
|
|
|
self.entid_map.get(&x.into())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn get_entid(&self, x: &NamespacedKeyword) -> Option<KnownEntid> {
|
|
|
|
self.get_raw_entid(x).map(KnownEntid)
|
2017-02-14 03:20:20 +00:00
|
|
|
}
|
|
|
|
|
2018-01-23 16:31:27 +00:00
|
|
|
fn attribute_for_entid<T>(&self, x: T) -> Option<&Attribute> where T: Into<Entid> {
|
|
|
|
self.attribute_map.get(&x.into())
|
2017-02-14 03:20:20 +00:00
|
|
|
}
|
|
|
|
|
2018-01-23 16:31:27 +00:00
|
|
|
fn attribute_for_ident(&self, ident: &NamespacedKeyword) -> Option<(&Attribute, KnownEntid)> {
|
|
|
|
self.get_raw_entid(&ident)
|
|
|
|
.and_then(|entid| {
|
|
|
|
self.attribute_for_entid(entid).map(|a| (a, KnownEntid(entid)))
|
|
|
|
})
|
2017-02-15 23:53:13 +00:00
|
|
|
}
|
|
|
|
|
2017-02-14 03:39:47 +00:00
|
|
|
/// Return true if the provided entid identifies an attribute in this schema.
|
2018-01-23 16:31:27 +00:00
|
|
|
fn is_attribute<T>(&self, x: T) -> bool where T: Into<Entid> {
|
|
|
|
self.attribute_map.contains_key(&x.into())
|
2017-02-14 03:20:20 +00:00
|
|
|
}
|
|
|
|
|
2017-02-14 03:39:47 +00:00
|
|
|
/// Return true if the provided ident identifies an attribute in this schema.
|
2018-01-23 16:23:37 +00:00
|
|
|
fn identifies_attribute(&self, x: &NamespacedKeyword) -> bool {
|
2018-01-23 16:31:27 +00:00
|
|
|
self.get_raw_entid(x).map(|e| self.is_attribute(e)).unwrap_or(false)
|
2017-02-14 03:20:20 +00:00
|
|
|
}
|
2018-04-03 21:25:53 +00:00
|
|
|
|
|
|
|
fn component_attributes(&self) -> &[Entid] {
|
|
|
|
&self.component_attributes
|
|
|
|
}
|
2017-02-14 03:20:20 +00:00
|
|
|
}
|
|
|
|
|
2018-04-24 22:04:00 +00:00
|
|
|
pub mod intern_set;
|
|
|
|
pub mod counter;
|
|
|
|
pub mod util;
|
|
|
|
|
2017-02-10 20:03:49 +00:00
|
|
|
#[cfg(test)]
|
|
|
|
mod test {
|
|
|
|
use super::*;
|
|
|
|
|
2018-01-18 15:06:23 +00:00
|
|
|
use std::str::FromStr;
|
|
|
|
|
2017-03-30 10:08:36 +00:00
|
|
|
fn associate_ident(schema: &mut Schema, i: NamespacedKeyword, e: Entid) {
|
|
|
|
schema.entid_map.insert(e, i.clone());
|
|
|
|
schema.ident_map.insert(i, e);
|
|
|
|
}
|
|
|
|
|
|
|
|
fn add_attribute(schema: &mut Schema, e: Entid, a: Attribute) {
|
2018-01-23 16:23:37 +00:00
|
|
|
schema.attribute_map.insert(e, a);
|
2017-03-30 10:08:36 +00:00
|
|
|
}
|
|
|
|
|
2017-02-10 20:03:49 +00:00
|
|
|
#[test]
|
|
|
|
fn test_attribute_flags() {
|
|
|
|
let attr1 = Attribute {
|
|
|
|
index: true,
|
|
|
|
value_type: ValueType::Ref,
|
|
|
|
fulltext: false,
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
unique: None,
|
2017-02-10 20:03:49 +00:00
|
|
|
multival: false,
|
|
|
|
component: false,
|
2018-04-03 21:23:46 +00:00
|
|
|
no_history: false,
|
2017-02-10 20:03:49 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
assert!(attr1.flags() & AttributeBitFlags::IndexAVET as u8 != 0);
|
|
|
|
assert!(attr1.flags() & AttributeBitFlags::IndexVAET as u8 != 0);
|
|
|
|
assert!(attr1.flags() & AttributeBitFlags::IndexFulltext as u8 == 0);
|
|
|
|
assert!(attr1.flags() & AttributeBitFlags::UniqueValue as u8 == 0);
|
|
|
|
|
|
|
|
let attr2 = Attribute {
|
|
|
|
index: false,
|
|
|
|
value_type: ValueType::Boolean,
|
|
|
|
fulltext: true,
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
unique: Some(attribute::Unique::Value),
|
2017-02-10 20:03:49 +00:00
|
|
|
multival: false,
|
|
|
|
component: false,
|
2018-04-03 21:23:46 +00:00
|
|
|
no_history: false,
|
2017-02-10 20:03:49 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
assert!(attr2.flags() & AttributeBitFlags::IndexAVET as u8 == 0);
|
|
|
|
assert!(attr2.flags() & AttributeBitFlags::IndexVAET as u8 == 0);
|
|
|
|
assert!(attr2.flags() & AttributeBitFlags::IndexFulltext as u8 != 0);
|
|
|
|
assert!(attr2.flags() & AttributeBitFlags::UniqueValue as u8 != 0);
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
|
|
|
|
let attr3 = Attribute {
|
|
|
|
index: false,
|
|
|
|
value_type: ValueType::Boolean,
|
|
|
|
fulltext: true,
|
|
|
|
unique: Some(attribute::Unique::Identity),
|
|
|
|
multival: false,
|
|
|
|
component: false,
|
2018-04-03 21:23:46 +00:00
|
|
|
no_history: false,
|
Schema alteration. Fixes #294 and #295. (#370) r=rnewman
* Pre: Don't retract :db/ident in test.
Datomic (and eventually Mentat) don't allow to retract :db/ident in
this way, so this runs afoul of future work to support mutating
metadata.
* Pre: s/VALUETYPE/VALUE_TYPE/.
This is consistent with the capitalization (which is "valueType") and
the other identifier.
* Pre: Remove some single quotes from error output.
* Part 1: Make materialized views be uniform [e a v value_type_tag].
This looks ahead to a time when we could support arbitrary
user-defined materialized views. For now, the "idents" materialized
view is those datoms of the form [e :db/ident :namespaced/keyword] and
the "schema" materialized view is those datoms of the form [e a v]
where a is in a particular set of attributes that will become clear in
the following commits.
This change is not backwards compatible, so I'm removing the open
current (really, v2) test. It'll be re-instated when we get to
https://github.com/mozilla/mentat/issues/194.
* Pre: Map TypedValue::Ref to TypedValue::Keyword in debug output.
* Part 3: Separate `schema_to_mutate` from the `schema` used to interpret.
This is just to keep track of the expected changes during
bootstrapping. I want bootstrap metadata mutations to flow through
the same code path as metadata mutations during regular transactions;
by differentiating the schema used for interpretation from the schema
that will be updated I expect to be able to apply bootstrap metadata
mutations to an empty schema and have things like materialized views
created (using the regular code paths).
This commit has been re-ordered for conceptual clarity, but it won't
compile because it references the metadata module. It's possible to
make it compile -- the functionality is there in the schema module --
but it's not worth the rebasing effort until after review (and
possibly not even then, since we'll squash down to a single commit to
land).
* Part 2: Maintain entids separately from idents.
In order to support historical idents, we need to distinguish the
"current" map from entid -> ident from the "complete historical" map
ident -> entid. This is what Datomic does; in Datomic, an ident is
never retracted (although it can be replaced). This approach is an
important part of allowing multiple consumers to share a schema
fragment as it migrates forward.
This fixes a limitation of the Clojure implementation, which did not
handle historical idents across knowledge base close and re-open.
The "entids" materialized view is naturally a slice of the "datoms"
table. The "idents" materialized view is a slice of the
"transactions" table. I hope that representing in this way, and
casting the problem in this light, might generalize to future
materialized views.
* Pre: Add DiffSet.
* Part 4: Collect mutations to a `Schema`.
I haven't taken your review comment about consuming AttributeBuilder
during each fluent function. If you read my response and still want
this, I'm happy to do it in review.
* Part 5: Handle :db/ident and :db.{install,alter}/attribute.
This "loops" the committed datoms out of the SQL store and back
through the metadata (schema, but in future also partition map)
processor. The metadata processor updates the schema and produces a
report of what changed; that report is then used to update the SQL
store. That update includes:
- the materialized views ("entids", "idents", and "schema");
- if needed, a subset of the datoms themselves (as flags change).
I've left a TODO for handling attribute retraction in the cases that
it makes sense. I expect that to be straight-forward.
* Review comment: Rename DiffSet to AddRetractAlterSet.
Also adds a little more commentary and a simple test.
* Review comment: Use ToIdent trait.
* Review comment: partially revert "Part 2: Maintain entids separately from idents."
This reverts commit 23a91df9c35e14398f2ddbd1ba25315821e67401.
Following our discussion, this removes the "entids" materialized
view. The next commit will remove historical idents from the "idents"
materialized view.
* Post: Use custom Either rather than std::result::Result.
This is not necessary, but it was suggested that we might be paying an
overhead creating Err instances while using error_chain. That seems
not to be the case, but this change shows that we don't actually use
any of the Result helper methods, so there's no reason to overload
Result. This change might avoid some future confusion, so I'm going
to land it anyway.
Signed-off-by: Nick Alexander <nalexander@mozilla.com>
* Review comment: Don't preserve historical idents.
* Review comment: More prepared statements when updating materialized views.
* Post: Test altering :db/cardinality and :db/unique.
These tests fail due to a Datomic limitation, namely that the marker
flag :db.alter/attribute can only be asserted once for an attribute!
That is, [:db.part/db :db.alter/attribute :attribute] will only be
transacted at most once. Since older versions of Datomic required the
:db.alter/attribute flag, I can only imagine they either never wrote
:db.alter/attribute to the store, or they handled it specially. I'll
need to remove the marker flag system from Mentat in order to address
this fundamental limitation.
* Post: Remove some more single quotes from error output.
* Post: Add assert_transact! macro to unwrap safely.
I was finding it very difficult to track unwrapping errors while
making changes, due to an underlying Mac OS X symbolication issue that
makes running tests with RUST_BACKTRACE=1 so slow that they all time
out.
* Post: Don't expect or recognize :db.{install,alter}/attribute.
I had this all working... except we will never see a repeated
`[:db.part/db :db.alter/attribute :attribute]` assertion in the store!
That means my approach would let you alter an attribute at most one
time. It's not worth hacking around this; it's better to just stop
expecting (and recognizing) the marker flags. (We have all the data
to distinguish the various cases that we need without the marker
flags.)
This brings Mentat in line with the thrust of newer Datomic versions,
but isn't compatible with Datomic, because (if I understand correctly)
Datomic automatically adds :db.{install,alter}/attribute assertions to
transactions.
I haven't purged the corresponding :db/ident and schema fragments just
yet:
- we might want them back
- we might want them in order to upgrade v1 and v2 databases to the
new on-disk layout we're fleshing out (v3?).
* Post: Don't make :db/unique :db.unique/* imply :db/index true.
This patch avoids a potential bug with the "schema" materialized view.
If :db/unique :db.unique/value implies :db/index true, then what
happens when you _retract_ :db.unique/value? I think Datomic defines
this in some way, but I really want the "schema" materialized view to
be a slice of "datoms" and not have these sort of ambiguities and
persistent effects. Therefore, to ensure that we don't retract a
schema characteristic and accidentally change more than we intended
to, this patch stops having any schema characteristic imply any other
schema characteristic(s). To achieve that, I added an
Option<Unique::{Value,Identity}> type to Attribute; this helps with
this patch, and also looks ahead to when we allow to retract
:db/unique attributes.
* Post: Allow to retract :db/ident.
* Post: Include more details about invalid schema changes.
The tests use strings, so they hide the chained errors which do in
fact provide more detail.
* Review comment: Fix outdated comment.
* Review comment: s/_SET/_SQL_LIST/.
* Review comment: Use a sub-select for checking cardinality.
This might be faster in practice.
* Review comment: Put `attribute::Unique` into its own namespace.
2017-03-20 20:18:59 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
assert!(attr3.flags() & AttributeBitFlags::IndexAVET as u8 == 0);
|
|
|
|
assert!(attr3.flags() & AttributeBitFlags::IndexVAET as u8 == 0);
|
|
|
|
assert!(attr3.flags() & AttributeBitFlags::IndexFulltext as u8 != 0);
|
|
|
|
assert!(attr3.flags() & AttributeBitFlags::UniqueValue as u8 != 0);
|
2017-02-10 20:03:49 +00:00
|
|
|
}
|
2017-03-30 10:08:36 +00:00
|
|
|
|
2018-01-18 15:06:23 +00:00
|
|
|
#[test]
|
|
|
|
fn test_datetime_truncation() {
|
|
|
|
let dt: DateTime<Utc> = DateTime::from_str("2018-01-11T00:34:09.273457004Z").expect("parsed");
|
|
|
|
let expected: DateTime<Utc> = DateTime::from_str("2018-01-11T00:34:09.273457Z").expect("parsed");
|
|
|
|
|
|
|
|
let tv: TypedValue = dt.into();
|
|
|
|
if let TypedValue::Instant(roundtripped) = tv {
|
|
|
|
assert_eq!(roundtripped, expected);
|
|
|
|
} else {
|
|
|
|
panic!();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-30 10:08:36 +00:00
|
|
|
#[test]
|
|
|
|
fn test_as_edn_value() {
|
|
|
|
let mut schema = Schema::default();
|
|
|
|
|
|
|
|
let attr1 = Attribute {
|
|
|
|
index: true,
|
|
|
|
value_type: ValueType::Ref,
|
|
|
|
fulltext: false,
|
|
|
|
unique: None,
|
|
|
|
multival: false,
|
|
|
|
component: false,
|
2018-04-03 21:23:46 +00:00
|
|
|
no_history: true,
|
2017-03-30 10:08:36 +00:00
|
|
|
};
|
|
|
|
associate_ident(&mut schema, NamespacedKeyword::new("foo", "bar"), 97);
|
|
|
|
add_attribute(&mut schema, 97, attr1);
|
|
|
|
|
|
|
|
let attr2 = Attribute {
|
|
|
|
index: false,
|
|
|
|
value_type: ValueType::String,
|
|
|
|
fulltext: true,
|
|
|
|
unique: Some(attribute::Unique::Value),
|
|
|
|
multival: true,
|
|
|
|
component: false,
|
2018-04-03 21:23:46 +00:00
|
|
|
no_history: false,
|
2017-03-30 10:08:36 +00:00
|
|
|
};
|
|
|
|
associate_ident(&mut schema, NamespacedKeyword::new("foo", "bas"), 98);
|
|
|
|
add_attribute(&mut schema, 98, attr2);
|
|
|
|
|
|
|
|
let attr3 = Attribute {
|
|
|
|
index: false,
|
|
|
|
value_type: ValueType::Boolean,
|
|
|
|
fulltext: false,
|
|
|
|
unique: Some(attribute::Unique::Identity),
|
|
|
|
multival: false,
|
|
|
|
component: true,
|
2018-04-03 21:23:46 +00:00
|
|
|
no_history: false,
|
2017-03-30 10:08:36 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
associate_ident(&mut schema, NamespacedKeyword::new("foo", "bat"), 99);
|
|
|
|
add_attribute(&mut schema, 99, attr3);
|
|
|
|
|
|
|
|
let value = schema.to_edn_value();
|
|
|
|
|
|
|
|
let expected_output = r#"[ { :db/ident :foo/bar
|
|
|
|
:db/valueType :db.type/ref
|
|
|
|
:db/cardinality :db.cardinality/one
|
2018-04-03 21:23:46 +00:00
|
|
|
:db/index true
|
|
|
|
:db/noHistory true },
|
2017-03-30 10:08:36 +00:00
|
|
|
{ :db/ident :foo/bas
|
|
|
|
:db/valueType :db.type/string
|
|
|
|
:db/cardinality :db.cardinality/many
|
|
|
|
:db/unique :db.unique/value
|
|
|
|
:db/fulltext true },
|
|
|
|
{ :db/ident :foo/bat
|
|
|
|
:db/valueType :db.type/boolean
|
|
|
|
:db/cardinality :db.cardinality/one
|
|
|
|
:db/unique :db.unique/identity
|
2018-04-03 21:25:28 +00:00
|
|
|
:db/isComponent true }, ]"#;
|
2017-03-30 10:08:36 +00:00
|
|
|
let expected_value = edn::parse::value(&expected_output).expect("to be able to parse").without_spans();
|
|
|
|
assert_eq!(expected_value, value);
|
|
|
|
|
|
|
|
// let's compare the whole thing again, just to make sure we are not changing anything when we convert to edn.
|
|
|
|
let value2 = schema.to_edn_value();
|
|
|
|
assert_eq!(expected_value, value2);
|
|
|
|
}
|
2017-02-10 20:03:49 +00:00
|
|
|
}
|