greg/mentat
1
0
Fork 0
Code Issues 294 Pull requests Projects Releases 9 Packages Wiki Activity

implement bytes (aka blobs) as native type

Browse source
This commit is contained in:
Mark Watts 2021-08-23 17:25:10 -04:00
parent d3821432bc
commit 73feb622cd
15 changed files with 64 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
core-traits/Cargo.toml
View file

@ -16,6 +16,7 @@ ordered-float = { version = "~2.7", features = ["serde"] }
uuid = { version = "~0.8", features = ["v4", "serde"] }
serde = { version = "~1.0", features = ["rc"] }
serde_derive = "~1.0"
bytes = { version = "1.0.1", features = ["serde"] }
[dependencies.edn]
path = "../edn"

24
core-traits/lib.rs
View file

@ -16,6 +16,7 @@ extern crate ordered_float;
extern crate serde_derive;
extern crate edn;
extern crate uuid;
extern crate bytes;
#[macro_use]
extern crate lazy_static;
@ -33,6 +34,7 @@ use std::sync::Arc;
use std::collections::BTreeMap;
use bytes::Bytes;
use indexmap::IndexMap;
use enum_set::EnumSet;
@ -280,6 +282,7 @@ pub enum ValueType {
String,
Keyword,
Uuid,
Bytes,
}
impl ValueType {
@ -294,6 +297,7 @@ impl ValueType {
s.insert(ValueType::String);
s.insert(ValueType::Keyword);
s.insert(ValueType::Uuid);
s.insert(ValueType::Bytes);
s
}
}
@ -321,6 +325,7 @@ impl ValueType {
ValueType::String => "string",
ValueType::Keyword => "keyword",
ValueType::Uuid => "uuid",
ValueType::Bytes => "bytes",
},
)
}
@ -338,6 +343,7 @@ impl ValueType {
"string" => Some(ValueType::String),
"keyword" => Some(ValueType::Keyword),
"uuid" => Some(ValueType::Uuid),
"bytes" => Some(ValueType::Bytes),
_ => None,
}
}
@ -355,6 +361,7 @@ impl ValueType {
ValueType::String => "string",
ValueType::Keyword => "keyword",
ValueType::Uuid => "uuid",
ValueType::Bytes => "bytes",
},
)
}
@ -369,6 +376,7 @@ impl ValueType {
ValueType::String => values::DB_TYPE_STRING.clone(),
ValueType::Keyword => values::DB_TYPE_KEYWORD.clone(),
ValueType::Uuid => values::DB_TYPE_UUID.clone(),
ValueType::Bytes => values::DB_TYPE_BYTES.clone(),
}
}
@ -391,6 +399,7 @@ impl fmt::Display for ValueType {
ValueType::String => ":db.type/string",
ValueType::Keyword => ":db.type/keyword",
ValueType::Uuid => ":db.type/uuid",
ValueType::Bytes => ":db.type/bytes",
}
)
}
@ -414,6 +423,7 @@ pub enum TypedValue {
String(ValueRc<String>),
Keyword(ValueRc<Keyword>),
Uuid(Uuid), // It's only 128 bits, so this should be acceptable to clone.
Bytes(Bytes),
}
impl From<KnownEntid> for TypedValue {
@ -445,6 +455,7 @@ impl TypedValue {
TypedValue::String(_) => ValueType::String,
TypedValue::Keyword(_) => ValueType::Keyword,
TypedValue::Uuid(_) => ValueType::Uuid,
TypedValue::Bytes(_) => ValueType::Bytes,
}
}
@ -596,6 +607,13 @@ impl TypedValue {
_ => None,
}
}
pub fn into_bytes(self) -> Option<Bytes> {
match self {
TypedValue::Bytes(b) => Some(b),
_ => None,
}
}
}
// We don't do From<i64> or From<Entid> 'cos it's ambiguous.
@ -686,6 +704,12 @@ impl From<f64> for TypedValue {
}
}
impl From<&[u8]> for TypedValue {
fn from(bslice: &[u8]) -> Self {
TypedValue::Bytes(Bytes::copy_from_slice(bslice))
}
}
trait MicrosecondPrecision {
/// Truncate the provided `DateTime` to microsecond precision.
fn microsecond_precision(self) -> Self;

1
core-traits/values.rs
View file

@ -58,6 +58,7 @@ lazy_static_namespaced_keyword_value!(DB_TYPE_REF, "db.type", "ref");
lazy_static_namespaced_keyword_value!(DB_TYPE_STRING, "db.type", "string");
lazy_static_namespaced_keyword_value!(DB_TYPE_URI, "db.type", "uri");
lazy_static_namespaced_keyword_value!(DB_TYPE_UUID, "db.type", "uuid");
lazy_static_namespaced_keyword_value!(DB_TYPE_BYTES, "db.type", "bytes");
lazy_static_namespaced_keyword_value!(DB_UNIQUE, "db", "unique");
lazy_static_namespaced_keyword_value!(DB_UNIQUE_IDENTITY, "db.unique", "identity");
lazy_static_namespaced_keyword_value!(DB_UNIQUE_VALUE, "db.unique", "value");

2
core/src/sql_types.rs
View file

@ -51,6 +51,7 @@ impl SQLValueType for ValueType {
ValueType::String => (10, None),
ValueType::Uuid => (11, None),
ValueType::Keyword => (13, None),
ValueType::Bytes => (15, Some(SQLTypeAffinity::Blob)),
}
}
@ -71,6 +72,7 @@ impl SQLValueType for ValueType {
ValueType::String => false,
Keyword => false,
Uuid => false,
Bytes => false,
}
}
}

6
db/src/db.rs
View file

@ -434,6 +434,9 @@ impl TypedSQLValue for TypedValue {
Ok(TypedValue::Uuid(u))
}
(13, rusqlite::types::Value::Text(x)) => to_namespaced_keyword(&x).map(|k| k.into()),
(15, rusqlite::types::Value::Blob(x)) => {
Ok(TypedValue::Bytes(x.into()))
}
(_, value) => bail!(DbErrorKind::BadSQLValuePair(value, value_type_tag)),
}
}
@ -454,6 +457,7 @@ impl TypedSQLValue for TypedValue {
Value::Float(ref x) => Some(TypedValue::Double(*x)),
Value::Text(ref x) => Some(x.clone().into()),
Value::Keyword(ref x) => Some(x.clone().into()),
Value::Bytes(b) => Some(TypedValue::Bytes(b.clone())),
_ => None,
}
}
@ -470,6 +474,7 @@ impl TypedSQLValue for TypedValue {
TypedValue::String(ref x) => (x.as_str().into(), 10),
TypedValue::Uuid(ref u) => (u.as_bytes().to_vec().into(), 11),
TypedValue::Keyword(ref x) => (x.to_string().into(), 13),
TypedValue::Bytes(b) => (b.to_vec().into(), 15),
}
}
@ -484,6 +489,7 @@ impl TypedSQLValue for TypedValue {
TypedValue::String(ref x) => (Value::Text(x.as_ref().clone()), ValueType::String),
TypedValue::Uuid(ref u) => (Value::Uuid(*u), ValueType::Uuid),
TypedValue::Keyword(ref x) => (Value::Keyword(x.as_ref().clone()), ValueType::Keyword),
TypedValue::Bytes(b) => (Value::Bytes(b.clone()), ValueType::Bytes),
}
}
}

3
db/src/internal_types.rs
View file

@ -105,7 +105,8 @@ impl TransactableValue for TypedValue {
| TypedValue::Long(_)
| TypedValue::Double(_)
| TypedValue::Instant(_)
| TypedValue::Uuid(_) => {
| TypedValue::Uuid(_)
| TypedValue::Bytes(_) => {
bail!(DbErrorKind::InputError(errors::InputError::BadEntityPlace))
}
}

1
db/src/metadata.rs
View file

@ -248,6 +248,7 @@ pub fn update_attribute_map_from_entid_triples(
TypedValue::Ref(entids::DB_TYPE_REF) => { builder.value_type(ValueType::Ref); },
TypedValue::Ref(entids::DB_TYPE_STRING) => { builder.value_type(ValueType::String); },
TypedValue::Ref(entids::DB_TYPE_UUID) => { builder.value_type(ValueType::Uuid); },
TypedValue::Ref(entids::DB_TYPE_BYTES) => { builder.value_type(ValueType::Bytes); },
_ => bail!(DbErrorKind::BadSchemaAssertion(format!("Expected [... :db/valueType :db.type/*] but got [... :db/valueType {:?}] for entid {} and attribute {}", value, entid, attr)))
}
},

2
db/src/schema.rs
View file

@ -362,6 +362,7 @@ impl SchemaTypeChecking for Schema {
(ValueType::Uuid, tv @ TypedValue::Uuid(_)) => Ok(tv),
(ValueType::Instant, tv @ TypedValue::Instant(_)) => Ok(tv),
(ValueType::Keyword, tv @ TypedValue::Keyword(_)) => Ok(tv),
(ValueType::Bytes, tv @ TypedValue::Bytes(_)) => Ok(tv),
// Ref coerces a little: we interpret some things depending on the schema as a Ref.
(ValueType::Ref, TypedValue::Long(x)) => Ok(TypedValue::Ref(x)),
(ValueType::Ref, TypedValue::Keyword(ref x)) => {
@ -379,6 +380,7 @@ impl SchemaTypeChecking for Schema {
| (vt @ ValueType::Uuid, _)
| (vt @ ValueType::Instant, _)
| (vt @ ValueType::Keyword, _)
| (vt @ ValueType::Bytes, _)
| (vt @ ValueType::Ref, _) => {
bail!(DbErrorKind::BadValuePair(format!("{}", value), vt))
}

5
db/tests/value_tests.rs
View file

@ -67,6 +67,11 @@ fn test_from_sql_value_pair() {
.unwrap(),
TypedValue::typed_ns_keyword("db", "keyword")
);
assert_eq!(
TypedValue::from_sql_value_pair(rusqlite::types::Value::Blob(vec![1,2,3,42]), 15)
.unwrap(),
TypedValue::Bytes((vec![1,2,3,42]).into())
);
}
#[test]

1
query-algebrizer/tests/type_reqs.rs
View file

@ -34,6 +34,7 @@ fn prepopulated_schema() -> Schema {
.define_simple_attr("test", "uuid", ValueType::Uuid, false)
.define_simple_attr("test", "instant", ValueType::Instant, false)
.define_simple_attr("test", "ref", ValueType::Ref, false)
.define_simple_attr("test", "bytes", ValueType::Bytes, false)
.schema
}

2
query-projector-traits/aggregates.rs
View file

@ -110,7 +110,7 @@ impl SimpleAggregationOp {
String => Ok(the_type),
// Unordered types.
Keyword | Ref | Uuid => {
Keyword | Ref | Uuid | Bytes => {
bail!(ProjectorError::CannotApplyAggregateOperationToTypes(
self,
possibilities

12
sql/src/lib.rs
View file

@ -181,6 +181,18 @@ impl QueryBuilder for SQLiteQueryBuilder {
let v = Rc::new(rusqlite::types::Value::Text(s.as_ref().to_string()));
self.push_static_arg(v);
}
Bytes(b) => {
let bytes = b.to_vec();
if let Some(arg) = self.byte_args.get(&bytes).cloned() {
// Why, borrow checker, why?!
self.push_named_arg(arg.as_str());
} else {
let arg = self.next_argument_name();
self.push_named_arg(arg.as_str());
self.byte_args.insert(bytes, arg);
}
},
}
Ok(())
}

4
tests/query.rs
View file

@ -835,6 +835,7 @@ fn test_type_reqs() {
{:db/ident :test/uuid :db/valueType :db.type/uuid :db/cardinality :db.cardinality/one}
{:db/ident :test/instant :db/valueType :db.type/instant :db/cardinality :db.cardinality/one}
{:db/ident :test/ref :db/valueType :db.type/ref :db/cardinality :db.cardinality/one}
{:db/ident :test/bytes :db/valueType :db.type/bytes :db/cardinality :db.cardinality/one}
]"#,
)
.unwrap();
@ -849,7 +850,8 @@ fn test_type_reqs() {
:test/keyword :foo/bar
:test/uuid #uuid "12341234-1234-1234-1234-123412341234"
:test/instant #inst "2018-01-01T11:00:00.000Z"
:test/ref 1}
:test/ref 1
:test/bytes #bytes 010203050403022a }
]"#,
)
.unwrap();

2
tools/cli/Cargo.toml
View file

@ -32,6 +32,8 @@ tabwriter = "~1.2"
tempfile = "~3.2"
termion = "~1.5"
time = "~0.3"
bytes = { version = "1.0.1", features = ["serde"] }
hex = "0.4.3"
[dependencies.rusqlite]
version = "~0.25"

1
tools/cli/src/mentat_cli/repl.rs
View file

@ -613,6 +613,7 @@ impl Repl {
Ref(r) => format!("{}", r),
String(ref s) => format!("{:?}", s.to_string()),
Uuid(ref u) => format!("{}", u),
Bytes(b) => format!("#bytes {:?}", b.to_vec()),
}
}
}