From 70b112801c7657dd8660a2412fb82b2081c4dee8 Mon Sep 17 00:00:00 2001 From: Richard Newman Date: Mon, 6 Mar 2017 14:40:10 -0800 Subject: [PATCH] Implement projection and querying. (#353) r=nalexander MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add a failing test for EDN parsing '…'. * Expose a SQLValueType trait to get value_type_tag values out of a ValueType. * Add accessors to FindSpec. * Implement querying. * Implement rudimentary projection. * Export mentat_db::new_connection. * Export symbols from mentat. * Add rudimentary end-to-end query tests. --- Cargo.toml | 10 +- db/src/db.rs | 19 + db/src/lib.rs | 6 + query-algebrizer/src/cc.rs | 4 +- query-algebrizer/src/lib.rs | 4 +- query-projector/Cargo.toml | 35 ++ query-projector/README.md | 6 + query-projector/src/lib.rs | 423 ++++++++++++++++++ query-sql/Cargo.toml | 21 + .../src/types.rs => query-sql/src/lib.rs | 13 +- query-translator/Cargo.toml | 6 + query-translator/src/lib.rs | 6 +- query-translator/src/translate.rs | 43 +- query-translator/tests/translate.rs | 31 +- query/src/lib.rs | 67 +-- src/errors.rs | 2 + src/lib.rs | 12 + src/query.rs | 45 +- tests/query.rs | 142 ++++++ 19 files changed, 821 insertions(+), 74 deletions(-) create mode 100644 query-projector/Cargo.toml create mode 100644 query-projector/README.md create mode 100644 query-projector/src/lib.rs create mode 100644 query-sql/Cargo.toml rename query-translator/src/types.rs => query-sql/src/lib.rs (97%) create mode 100644 tests/query.rs diff --git a/Cargo.toml b/Cargo.toml index 93da6d4e..2fad3dad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,11 +48,17 @@ path = "db" [dependencies.mentat_query] path = "query" +[dependencies.mentat_query_algebrizer] +path = "query-algebrizer" + [dependencies.mentat_query_parser] path = "query-parser" -[dependencies.mentat_query_algebrizer] -path = "query-algebrizer" +[dependencies.mentat_query_projector] +path = "query-projector" + +[dependencies.mentat_query_sql] +path = "query-sql" [dependencies.mentat_query_translator] path = "query-translator" diff --git a/db/src/db.rs b/db/src/db.rs index 37d457ce..ee59276f 100644 --- a/db/src/db.rs +++ b/db/src/db.rs @@ -331,6 +331,25 @@ pub fn ensure_current_version(conn: &mut rusqlite::Connection) -> Result { } } +pub trait SQLValueType { + fn value_type_tag(&self) -> i32; +} + +impl SQLValueType for ValueType { + fn value_type_tag(&self) -> i32 { + match *self { + ValueType::Ref => 0, + ValueType::Boolean => 1, + ValueType::Instant => 4, + // SQLite distinguishes integral from decimal types, allowing long and double to share a tag. + ValueType::Long => 5, + ValueType::Double => 5, + ValueType::String => 10, + ValueType::Keyword => 13, + } + } +} + pub trait TypedSQLValue { fn from_sql_value_pair(value: rusqlite::types::Value, value_type_tag: i32) -> Result; fn to_sql_value_pair<'a>(&'a self) -> (ToSqlOutput<'a>, i32); diff --git a/db/src/lib.rs b/db/src/lib.rs index 4fdf090f..287c1454 100644 --- a/db/src/lib.rs +++ b/db/src/lib.rs @@ -40,6 +40,12 @@ mod upsert_resolution; mod values; mod tx; +pub use db::{ + SQLValueType, + TypedSQLValue, + new_connection, +}; + pub use tx::transact; pub use types::{ DB, diff --git a/query-algebrizer/src/cc.rs b/query-algebrizer/src/cc.rs index 84d06496..d97ac7ed 100644 --- a/query-algebrizer/src/cc.rs +++ b/query-algebrizer/src/cc.rs @@ -193,12 +193,12 @@ pub struct ConjoiningClauses { pub wheres: Vec, /// A map from var to qualified columns. Used to project. - bindings: BTreeMap>, + pub bindings: BTreeMap>, /// A map from var to type. Whenever a var maps unambiguously to two different types, it cannot /// yield results, so we don't represent that case here. If a var isn't present in the map, it /// means that its type is not known in advance. - known_types: BTreeMap, + pub known_types: BTreeMap, /// A mapping, similar to `bindings`, but used to pull type tags out of the store at runtime. /// If a var isn't present in `known_types`, it should be present here. diff --git a/query-algebrizer/src/lib.rs b/query-algebrizer/src/lib.rs index e3ed9122..99741d04 100644 --- a/query-algebrizer/src/lib.rs +++ b/query-algebrizer/src/lib.rs @@ -27,9 +27,9 @@ use mentat_query::{ #[allow(dead_code)] pub struct AlgebraicQuery { default_source: SrcVar, - find_spec: FindSpec, + pub find_spec: FindSpec, has_aggregates: bool, - limit: Option, + pub limit: Option, pub cc: cc::ConjoiningClauses, } diff --git a/query-projector/Cargo.toml b/query-projector/Cargo.toml new file mode 100644 index 00000000..2f29722b --- /dev/null +++ b/query-projector/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "mentat_query_projector" +version = "0.0.1" +workspace = ".." + +[dependencies] +error-chain = "0.9.0" + +[dependencies.rusqlite] +version = "0.9.5" +# System sqlite might be very old. +features = ["bundled"] + +[dependencies.mentat_core] +path = "../core" + +[dependencies.mentat_db] +path = "../db" + +[dependencies.mentat_sql] +path = "../sql" + +[dependencies.mentat_query] +path = "../query" + +[dependencies.mentat_query_algebrizer] +path = "../query-algebrizer" + +# Only for tests. +[dev-dependencies.mentat_query_parser] +path = "../query-parser" + +[dependencies.mentat_query_sql] +path = "../query-sql" + diff --git a/query-projector/README.md b/query-projector/README.md new file mode 100644 index 00000000..efd43e5f --- /dev/null +++ b/query-projector/README.md @@ -0,0 +1,6 @@ +This module handles the derivation from an algebrized query of two things: + +- A SQL projection: a mapping from columns mentioned in the body of the query to columns in the output. +- A Datalog projection: a function that consumes rows of the appropriate shape (as defined by the SQL projection) to yield one of the four kinds of Datalog query result. + +These two must naturally coordinate, and so they are both produced here. diff --git a/query-projector/src/lib.rs b/query-projector/src/lib.rs new file mode 100644 index 00000000..364e5c2f --- /dev/null +++ b/query-projector/src/lib.rs @@ -0,0 +1,423 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#[macro_use] +extern crate error_chain; +extern crate rusqlite; + +extern crate mentat_core; +extern crate mentat_db; // For value conversion. +extern crate mentat_query; +extern crate mentat_query_algebrizer; +extern crate mentat_query_sql; +extern crate mentat_sql; + +use std::iter; + +use rusqlite::{ + Row, + Rows, +}; + +use mentat_core::{ + TypedValue, +}; + +use mentat_db::{ + SQLValueType, + TypedSQLValue, +}; + +use mentat_query::{ + Element, + FindSpec, + PlainSymbol, + Variable, +}; + +use mentat_query_algebrizer::{ + AlgebraicQuery, + DatomsColumn, + QualifiedAlias, + /* + ConjoiningClauses, + DatomsTable, + SourceAlias, + */ +}; + +use mentat_query_sql::{ + ColumnOrExpression, + /* + Constraint, + FromClause, + */ + Name, + Projection, + ProjectedColumn, + /* + SelectQuery, + TableList, + */ +}; + +error_chain! { + types { + Error, ErrorKind, ResultExt, Result; + } + + foreign_links { + Rusqlite(rusqlite::Error); + } + + links { + DbError(mentat_db::Error, mentat_db::ErrorKind); + } +} + +#[derive(Debug)] +pub enum QueryResults { + Scalar(Option), + Tuple(Option>), + Coll(Vec), + Rel(Vec>), +} + +impl QueryResults { + pub fn len(&self) -> usize { + use QueryResults::*; + match self { + &Scalar(ref o) => if o.is_some() { 1 } else { 0 }, + &Tuple(ref o) => if o.is_some() { 1 } else { 0 }, + &Coll(ref v) => v.len(), + &Rel(ref v) => v.len(), + } + } +} + +type Index = i32; // See rusqlite::RowIndex. +type ValueTypeTag = i32; +enum TypedIndex { + Known(Index, ValueTypeTag), + Unknown(Index, Index), +} + +impl TypedIndex { + /// Look up this index and type(index) pair in the provided row. + /// This function will panic if: + /// + /// - This is an `Unknown` and the retrieved type code isn't an i32. + /// - If the retrieved value can't be coerced to a rusqlite `Value`. + /// - Either index is out of bounds. + /// + /// Because we construct our SQL projection list, the code that stored the data, and this + /// consumer, a panic here implies that we have a bad bug — we put data of a very wrong type in + /// a row, and thus can't coerce to Value, we're retrieving from the wrong place, or our + /// generated SQL is junk. + /// + /// This function will return a runtime error if the type code is unknown, or the value is + /// otherwise not convertible by the DB layer. + fn lookup<'a, 'stmt>(&self, row: &Row<'a, 'stmt>) -> Result { + use TypedIndex::*; + + match self { + &Known(value_index, value_type) => { + let v: rusqlite::types::Value = row.get(value_index); + TypedValue::from_sql_value_pair(v, value_type).map_err(|e| e.into()) + }, + &Unknown(value_index, type_index) => { + let v: rusqlite::types::Value = row.get(value_index); + let value_type_tag: i32 = row.get(type_index); + TypedValue::from_sql_value_pair(v, value_type_tag).map_err(|e| e.into()) + }, + } + } +} + +fn column_name(var: &Variable) -> Name { + let &Variable(PlainSymbol(ref s)) = var; + s.clone() +} + +fn value_type_tag_name(var: &Variable) -> Name { + let &Variable(PlainSymbol(ref s)) = var; + format!("{}_value_type_tag", s) +} + +/// Walk an iterator of `Element`s, collecting projector templates and columns. +/// +/// Returns a pair: the SQL projection (which should always be a `Projection::Columns`) +/// and a `Vec` of `TypedIndex` 'keys' to use when looking up values. +/// +/// Callers must ensure that every `Element` is distinct -- a query like +/// +/// ```edn +/// [:find ?x ?x :where [?x _ _]] +/// ``` +/// +/// should fail to parse. See #358. +fn project_elements<'a, I: IntoIterator>( + count: usize, + elements: I, + query: &AlgebraicQuery) -> (Projection, Vec) { + + let mut cols = Vec::with_capacity(count); + let mut i: i32 = 0; + let mut templates = vec![]; + + for e in elements { + match e { + // Each time we come across a variable, we push a SQL column + // into the SQL projection, aliased to the name of the variable, + // and we push an annotated index into the projector. + &Element::Variable(ref var) => { + // Every variable should be bound by the top-level CC to at least + // one column in the query. If that constraint is violated it's a + // bug in our code, so it's appropriate to panic here. + let columns = query.cc + .bindings + .get(var) + .expect("Every variable has a binding"); + + let qa = columns[0].clone(); + let name = column_name(var); + + if let Some(t) = query.cc.known_types.get(var) { + cols.push(ProjectedColumn(ColumnOrExpression::Column(qa), name)); + let tag = t.value_type_tag(); + templates.push(TypedIndex::Known(i, tag)); + i += 1; // We used one SQL column. + } else { + let table = qa.0.clone(); + cols.push(ProjectedColumn(ColumnOrExpression::Column(qa), name)); + templates.push(TypedIndex::Unknown(i, i + 1)); + i += 2; // We used two SQL columns. + + // Also project the type from the SQL query. + let type_name = value_type_tag_name(var); + let type_qa = QualifiedAlias(table, DatomsColumn::ValueTypeTag); + cols.push(ProjectedColumn(ColumnOrExpression::Column(type_qa), type_name)); + } + } + } + } + + (Projection::Columns(cols), templates) +} + +pub trait Projector { + fn project<'stmt>(&self, rows: Rows<'stmt>) -> Result; +} + +struct ScalarProjector { + template: TypedIndex, +} + +impl ScalarProjector { + fn with_template(template: TypedIndex) -> ScalarProjector { + ScalarProjector { + template: template, + } + } + + fn combine(sql: Projection, mut templates: Vec) -> CombinedProjection { + let template = templates.pop().expect("Expected a single template"); + CombinedProjection { + sql_projection: sql, + datalog_projector: Box::new(ScalarProjector::with_template(template)), + } + } +} + +impl Projector for ScalarProjector { + fn project<'stmt>(&self, mut rows: Rows<'stmt>) -> Result { + if let Some(r) = rows.next() { + let row = r?; + let binding = self.template.lookup(&row)?; + Ok(QueryResults::Scalar(Some(binding))) + } else { + Ok(QueryResults::Scalar(None)) + } + } +} + +/// A tuple projector produces a single vector. It's the single-result version of rel. +struct TupleProjector { + len: usize, + templates: Vec, +} + +impl TupleProjector { + fn with_templates(len: usize, templates: Vec) -> TupleProjector { + TupleProjector { + len: len, + templates: templates, + } + } + + // This is exactly the same as for rel. + fn collect_bindings<'a, 'stmt>(&self, row: Row<'a, 'stmt>) -> Result> { + assert_eq!(row.column_count(), self.len as i32); + self.templates + .iter() + .map(|ti| ti.lookup(&row)) + .collect::>>() + } + + fn combine(column_count: usize, sql: Projection, templates: Vec) -> CombinedProjection { + let p = TupleProjector::with_templates(column_count, templates); + CombinedProjection { + sql_projection: sql, + datalog_projector: Box::new(p), + } + } +} + +impl Projector for TupleProjector { + fn project<'stmt>(&self, mut rows: Rows<'stmt>) -> Result { + if let Some(r) = rows.next() { + let row = r?; + let bindings = self.collect_bindings(row)?; + Ok(QueryResults::Tuple(Some(bindings))) + } else { + Ok(QueryResults::Tuple(None)) + } + } +} + +/// A rel projector produces a vector of vectors. +/// Each inner vector is the same size, and sourced from the same columns. +/// One inner vector is produced per `Row`. +/// Each column in the inner vector is the result of taking one or two columns from +/// the `Row`: one for the value and optionally one for the type tag. +struct RelProjector { + len: usize, + templates: Vec, +} + +impl RelProjector { + fn with_templates(len: usize, templates: Vec) -> RelProjector { + RelProjector { + len: len, + templates: templates, + } + } + + fn collect_bindings<'a, 'stmt>(&self, row: Row<'a, 'stmt>) -> Result> { + assert_eq!(row.column_count(), self.len as i32); + self.templates + .iter() + .map(|ti| ti.lookup(&row)) + .collect::>>() + } + + fn combine(column_count: usize, sql: Projection, templates: Vec) -> CombinedProjection { + let p = RelProjector::with_templates(column_count, templates); + CombinedProjection { + sql_projection: sql, + datalog_projector: Box::new(p), + } + } +} + +impl Projector for RelProjector { + fn project<'stmt>(&self, mut rows: Rows<'stmt>) -> Result { + let mut out: Vec> = vec![]; + while let Some(r) = rows.next() { + let row = r?; + let bindings = self.collect_bindings(row)?; + out.push(bindings); + } + Ok(QueryResults::Rel(out)) + } +} + +/// A coll projector produces a vector of values. +/// Each value is sourced from the same column. +struct CollProjector { + template: TypedIndex, +} + +impl CollProjector { + fn with_template(template: TypedIndex) -> CollProjector { + CollProjector { + template: template, + } + } + + fn combine(sql: Projection, mut templates: Vec) -> CombinedProjection { + let template = templates.pop().expect("Expected a single template"); + CombinedProjection { + sql_projection: sql, + datalog_projector: Box::new(CollProjector::with_template(template)), + } + } +} + +impl Projector for CollProjector { + fn project<'stmt>(&self, mut rows: Rows<'stmt>) -> Result { + let mut out: Vec = vec![]; + while let Some(r) = rows.next() { + let row = r?; + let binding = self.template.lookup(&row)?; + out.push(binding); + } + Ok(QueryResults::Coll(out)) + } +} + +/// Combines the two things you need to turn a query into SQL and turn its results into +/// `QueryResults`. +pub struct CombinedProjection { + /// A SQL projection, mapping columns mentioned in the body of the query to columns in the + /// output. + pub sql_projection: Projection, + + /// A Datalog projection. This consumes rows of the appropriate shape (as defined by + /// the SQL projection) to yield one of the four kinds of Datalog query result. + pub datalog_projector: Box, +} + +/// Compute a suitable SQL projection for an algebrized query. +/// This takes into account a number of things: +/// - The variable list in the find spec. +/// - The presence of any aggregate operations in the find spec. TODO: for now we only handle +/// simple variables +/// - The bindings established by the topmost CC. +/// - The types known at algebrizing time. +/// - The types extracted from the store for unknown attributes. +pub fn query_projection(query: &AlgebraicQuery) -> CombinedProjection { + use self::FindSpec::*; + + match query.find_spec { + FindColl(ref element) => { + let (cols, templates) = project_elements(1, iter::once(element), query); + CollProjector::combine(cols, templates) + }, + + FindScalar(ref element) => { + let (cols, templates) = project_elements(1, iter::once(element), query); + ScalarProjector::combine(cols, templates) + }, + + FindRel(ref elements) => { + let column_count = query.find_spec.expected_column_count(); + let (cols, templates) = project_elements(column_count, elements, query); + RelProjector::combine(column_count, cols, templates) + }, + + FindTuple(ref elements) => { + let column_count = query.find_spec.expected_column_count(); + let (cols, templates) = project_elements(column_count, elements, query); + TupleProjector::combine(column_count, cols, templates) + }, + } +} + diff --git a/query-sql/Cargo.toml b/query-sql/Cargo.toml new file mode 100644 index 00000000..cd5ed968 --- /dev/null +++ b/query-sql/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "mentat_query_sql" +version = "0.0.1" +workspace = ".." + +[dependencies] +[dependencies.mentat_core] +path = "../core" + +[dependencies.mentat_sql] +path = "../sql" + +[dependencies.mentat_query] +path = "../query" + +[dependencies.mentat_query_algebrizer] +path = "../query-algebrizer" + +# Only for tests. +[dev-dependencies.mentat_query_parser] +path = "../query-parser" diff --git a/query-translator/src/types.rs b/query-sql/src/lib.rs similarity index 97% rename from query-translator/src/types.rs rename to query-sql/src/lib.rs index 7484dac7..cbbb64c7 100644 --- a/query-translator/src/types.rs +++ b/query-sql/src/lib.rs @@ -8,7 +8,10 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. -#![allow(dead_code, unused_imports)] +extern crate mentat_core; +extern crate mentat_query; +extern crate mentat_query_algebrizer; +extern crate mentat_sql; use mentat_core::{ Entid, @@ -16,16 +19,11 @@ use mentat_core::{ }; use mentat_query_algebrizer::{ - AlgebraicQuery, - ConjoiningClauses, DatomsColumn, - DatomsTable, QualifiedAlias, SourceAlias, }; -use mentat_sql; - use mentat_sql::{ BuildQueryResult, QueryBuilder, @@ -80,6 +78,7 @@ impl Constraint { } } +#[allow(dead_code)] enum JoinOp { Inner, } @@ -94,6 +93,7 @@ pub struct Join { // TODO: constraints (ON, USING). } +#[allow(dead_code)] enum TableOrSubquery { Table(SourceAlias), // TODO: Subquery. @@ -278,6 +278,7 @@ impl SelectQuery { #[cfg(test)] mod tests { use super::*; + use mentat_query_algebrizer::DatomsTable; #[test] fn test_end_to_end() { diff --git a/query-translator/Cargo.toml b/query-translator/Cargo.toml index 51f1d2f2..e2a8f141 100644 --- a/query-translator/Cargo.toml +++ b/query-translator/Cargo.toml @@ -19,3 +19,9 @@ path = "../query-algebrizer" # Only for tests. [dev-dependencies.mentat_query_parser] path = "../query-parser" + +[dependencies.mentat_query_projector] +path = "../query-projector" + +[dependencies.mentat_query_sql] +path = "../query-sql" diff --git a/query-translator/src/lib.rs b/query-translator/src/lib.rs index 3a0e0050..e78302ac 100644 --- a/query-translator/src/lib.rs +++ b/query-translator/src/lib.rs @@ -11,16 +11,18 @@ extern crate mentat_core; extern crate mentat_query; extern crate mentat_query_algebrizer; +extern crate mentat_query_projector; +extern crate mentat_query_sql; extern crate mentat_sql; mod translate; -mod types; -pub use types::{ +pub use mentat_query_sql::{ Projection, }; pub use translate::{ cc_to_exists, cc_to_select, + query_to_select, }; diff --git a/query-translator/src/translate.rs b/query-translator/src/translate.rs index 20c61048..2942f228 100644 --- a/query-translator/src/translate.rs +++ b/query-translator/src/translate.rs @@ -10,6 +10,13 @@ #![allow(dead_code, unused_imports)] +use mentat_query::{ + Element, + FindSpec, + PlainSymbol, + Variable, +}; + use mentat_query_algebrizer::{ AlgebraicQuery, ColumnConstraint, @@ -20,11 +27,19 @@ use mentat_query_algebrizer::{ SourceAlias, }; -use types::{ +use mentat_query_projector::{ + CombinedProjection, + Projector, + query_projection, +}; + +use mentat_query_sql::{ ColumnOrExpression, Constraint, FromClause, + Name, Projection, + ProjectedColumn, SelectQuery, TableList, }; @@ -57,10 +72,12 @@ impl ToConstraint for ColumnConstraint { } } +pub struct CombinedSelectQuery { + pub query: SelectQuery, + pub projector: Box, +} -/// Consume a provided `ConjoiningClauses` to yield a new -/// `SelectQuery`. A projection list must also be provided. -pub fn cc_to_select(projection: Projection, cc: ConjoiningClauses) -> SelectQuery { +fn cc_to_select_query(projection: Projection, cc: ConjoiningClauses) -> SelectQuery { SelectQuery { projection: projection, from: FromClause::TableList(TableList(cc.from)), @@ -71,6 +88,20 @@ pub fn cc_to_select(projection: Projection, cc: ConjoiningClauses) -> SelectQuer } } -pub fn cc_to_exists(cc: ConjoiningClauses) -> SelectQuery { - cc_to_select(Projection::One, cc) +/// Consume a provided `ConjoiningClauses` to yield a new +/// `SelectQuery`. A projection list must also be provided. +pub fn cc_to_select(projection: CombinedProjection, cc: ConjoiningClauses) -> CombinedSelectQuery { + let CombinedProjection { sql_projection, datalog_projector } = projection; + CombinedSelectQuery { + query: cc_to_select_query(sql_projection, cc), + projector: datalog_projector, + } +} + +pub fn query_to_select(query: AlgebraicQuery) -> CombinedSelectQuery { + cc_to_select(query_projection(&query), query.cc) +} + +pub fn cc_to_exists(cc: ConjoiningClauses) -> SelectQuery { + cc_to_select_query(Projection::One, cc) } diff --git a/query-translator/tests/translate.rs b/query-translator/tests/translate.rs index 08720c73..9adac43f 100644 --- a/query-translator/tests/translate.rs +++ b/query-translator/tests/translate.rs @@ -27,7 +27,7 @@ use mentat_core::{ use mentat_query_parser::parse_find_string; use mentat_query_algebrizer::algebrize; use mentat_query_translator::{ - cc_to_exists, + query_to_select, }; use mentat_sql::SQLQuery; @@ -42,7 +42,26 @@ fn add_attribute(schema: &mut Schema, e: Entid, a: Attribute) { } #[test] -fn test_exists() { +#[should_panic(expected = "parse failed")] +fn test_coll() { + let mut schema = Schema::default(); + associate_ident(&mut schema, NamespacedKeyword::new("foo", "bar"), 99); + add_attribute(&mut schema, 99, Attribute { + value_type: ValueType::String, + ..Default::default() + }); + + let input = r#"[:find [?x ...] :where [?x :foo/bar "yyy"]]"#; + let parsed = parse_find_string(input).expect("parse failed"); + let algebrized = algebrize(&schema, parsed); + let select = query_to_select(algebrized); + let SQLQuery { sql, args } = select.query.to_sql_query().unwrap(); + assert_eq!(sql, "SELECT `datoms00`.e AS `?x` FROM `datoms` AS `datoms00` WHERE `datoms00`.a = 99 AND `datoms00`.v = $v0"); + assert_eq!(args, vec![("$v0".to_string(), "yyy".to_string())]); +} + +#[test] +fn test_rel() { let mut schema = Schema::default(); associate_ident(&mut schema, NamespacedKeyword::new("foo", "bar"), 99); add_attribute(&mut schema, 99, Attribute { @@ -51,10 +70,10 @@ fn test_exists() { }); let input = r#"[:find ?x :where [?x :foo/bar "yyy"]]"#; - let parsed = parse_find_string(input).unwrap(); + let parsed = parse_find_string(input).expect("parse failed"); let algebrized = algebrize(&schema, parsed); - let select = cc_to_exists(algebrized.cc); - let SQLQuery { sql, args } = select.to_sql_query().unwrap(); - assert_eq!(sql, "SELECT 1 FROM `datoms` AS `datoms00` WHERE `datoms00`.a = 99 AND `datoms00`.v = $v0"); + let select = query_to_select(algebrized); + let SQLQuery { sql, args } = select.query.to_sql_query().unwrap(); + assert_eq!(sql, "SELECT `datoms00`.e AS `?x` FROM `datoms` AS `datoms00` WHERE `datoms00`.a = 99 AND `datoms00`.v = $v0"); assert_eq!(args, vec![("$v0".to_string(), "yyy".to_string())]); } diff --git a/query/src/lib.rs b/query/src/lib.rs index fc98eceb..ecfa3529 100644 --- a/query/src/lib.rs +++ b/query/src/lib.rs @@ -340,35 +340,48 @@ pub enum FindSpec { } /// Returns true if the provided `FindSpec` returns at most one result. -pub fn is_unit_limited(spec: &FindSpec) -> bool { - match spec { - &FindSpec::FindScalar(..) => true, - &FindSpec::FindTuple(..) => true, - &FindSpec::FindRel(..) => false, - &FindSpec::FindColl(..) => false, +impl FindSpec { + pub fn is_unit_limited(&self) -> bool { + use FindSpec::*; + match self { + &FindScalar(..) => true, + &FindTuple(..) => true, + &FindRel(..) => false, + &FindColl(..) => false, + } } -} -/// Returns true if the provided `FindSpec` cares about distinct results. -/// -/// I use the words "cares about" because find is generally defined in terms of producing distinct -/// results at the Datalog level. -/// -/// Two of the find specs (scalar and tuple) produce only a single result. Those don't need to be -/// run with `SELECT DISTINCT`, because we're only consuming a single result. Those queries will be -/// run with `LIMIT 1`. -/// -/// Additionally, some projections cannot produce duplicate results: `[:find (max ?x) …]`, for -/// example. -/// -/// This function gives us the hook to add that logic when we're ready. -/// -/// Beyond this, `DISTINCT` is not always needed. For example, in some kinds of accumulation or -/// sampling projections we might not need to do it at the SQL level because we're consuming into -/// a dupe-eliminating data structure like a Set, or we know that a particular query cannot produce -/// duplicate results. -pub fn requires_distinct(spec: &FindSpec) -> bool { - return !is_unit_limited(spec); + pub fn expected_column_count(&self) -> usize { + use FindSpec::*; + match self { + &FindScalar(..) => 1, + &FindColl(..) => 1, + &FindTuple(ref elems) | &FindRel(ref elems) => elems.len(), + } + } + + + /// Returns true if the provided `FindSpec` cares about distinct results. + /// + /// I use the words "cares about" because find is generally defined in terms of producing distinct + /// results at the Datalog level. + /// + /// Two of the find specs (scalar and tuple) produce only a single result. Those don't need to be + /// run with `SELECT DISTINCT`, because we're only consuming a single result. Those queries will be + /// run with `LIMIT 1`. + /// + /// Additionally, some projections cannot produce duplicate results: `[:find (max ?x) …]`, for + /// example. + /// + /// This function gives us the hook to add that logic when we're ready. + /// + /// Beyond this, `DISTINCT` is not always needed. For example, in some kinds of accumulation or + /// sampling projections we might not need to do it at the SQL level because we're consuming into + /// a dupe-eliminating data structure like a Set, or we know that a particular query cannot produce + /// duplicate results. + pub fn requires_distinct(&self) -> bool { + !self.is_unit_limited() + } } // Note that the "implicit blank" rule applies. diff --git a/src/errors.rs b/src/errors.rs index 359072ce..f9cdb30c 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -15,6 +15,7 @@ use rusqlite; use edn; use mentat_db; use mentat_query_parser; +use mentat_query_projector; use mentat_sql; use mentat_tx_parser; @@ -31,6 +32,7 @@ error_chain! { links { DbError(mentat_db::Error, mentat_db::ErrorKind); QueryParseError(mentat_query_parser::Error, mentat_query_parser::ErrorKind); + ProjectorError(mentat_query_projector::Error, mentat_query_projector::ErrorKind); SqlError(mentat_sql::Error, mentat_sql::ErrorKind); TxParseError(mentat_tx_parser::Error, mentat_tx_parser::ErrorKind); } diff --git a/src/lib.rs b/src/lib.rs index f8d6d864..89b60890 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ extern crate mentat_db; extern crate mentat_query; extern crate mentat_query_algebrizer; extern crate mentat_query_parser; +extern crate mentat_query_projector; extern crate mentat_query_translator; extern crate mentat_sql; extern crate mentat_tx_parser; @@ -46,6 +47,17 @@ pub fn get_connection() -> Connection { return Connection::open_in_memory().unwrap(); } +pub use mentat_db::{ + new_connection, +}; + +pub use query::{ + NamespacedKeyword, + PlainSymbol, + QueryResults, + q_once, +}; + #[cfg(test)] mod tests { use edn::symbols::Keyword; diff --git a/src/query.rs b/src/query.rs index 25a7cb57..822748e8 100644 --- a/src/query.rs +++ b/src/query.rs @@ -10,6 +10,9 @@ use std::collections::HashMap; +use rusqlite; +use rusqlite::types::ToSql; + use mentat_core::{ Schema, TypedValue, @@ -17,6 +20,11 @@ use mentat_core::{ use mentat_query_algebrizer::algebrize; +pub use mentat_query::{ + NamespacedKeyword, + PlainSymbol, +}; + use mentat_query_parser::{ parse_find_string, }; @@ -26,21 +34,15 @@ use mentat_sql::{ }; use mentat_query_translator::{ - cc_to_select, - Projection, + query_to_select, +}; + +pub use mentat_query_projector::{ + QueryResults, }; use errors::Result; -use rusqlite; - -pub enum QueryResults { - Scalar(Option), - Tuple(Vec), - Coll(Vec), - Rel(Vec>), -} - pub type QueryExecutionResult = Result; /// Take an EDN query string, a reference to a open SQLite connection, a Mentat DB, and an optional @@ -59,21 +61,22 @@ pub fn q_once<'sqlite, 'schema, 'query> // TODO: validate inputs. let parsed = parse_find_string(query)?; let algebrized = algebrize(schema, parsed); - let projection = Projection::Star; - let select = cc_to_select(projection, algebrized.cc); - let SQLQuery { sql, args } = select.to_sql_query()?; + let select = query_to_select(algebrized); + let SQLQuery { sql, args } = select.query.to_sql_query()?; - /* let mut statement = sqlite.prepare(sql.as_str())?; - let mut rows = if args.is_empty() { + let rows = if args.is_empty() { statement.query(&[])? } else { - statement.query_named(args.map(|(k, v)| (k.as_str(), &v)))? + let refs: Vec<(&str, &ToSql)> = + args.iter() + .map(|&(ref k, ref v)| (k.as_str(), v as &ToSql)) + .collect(); + statement.query_named(refs.as_slice())? }; - */ - - - Ok(QueryResults::Scalar(Some(TypedValue::Boolean(true)))) + select.projector + .project(rows) + .map_err(|e| e.into()) } diff --git a/tests/query.rs b/tests/query.rs new file mode 100644 index 00000000..efa2c02d --- /dev/null +++ b/tests/query.rs @@ -0,0 +1,142 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +extern crate time; + +extern crate mentat; +extern crate mentat_core; +extern crate mentat_db; + +use mentat_core::{ + TypedValue, + ValueType, +}; + +use mentat::{ + NamespacedKeyword, + QueryResults, + new_connection, + q_once, +}; + +#[test] +fn test_rel() { + let mut c = new_connection("").expect("Couldn't open conn."); + let db = mentat_db::db::ensure_current_version(&mut c).expect("Couldn't open DB."); + + // Rel. + let start = time::PreciseTime::now(); + let results = q_once(&c, &db.schema, + "[:find ?x ?ident :where [?x :db/ident ?ident]]", None) + .expect("Query failed"); + let end = time::PreciseTime::now(); + + // This will need to change each time we add a default ident. + assert_eq!(37, results.len()); + + // Every row is a pair of a Ref and a Keyword. + if let QueryResults::Rel(ref rel) = results { + for r in rel { + assert_eq!(r.len(), 2); + assert!(r[0].matches_type(ValueType::Ref)); + assert!(r[1].matches_type(ValueType::Keyword)); + } + } else { + panic!("Expected rel."); + } + + println!("{:?}", results); + println!("Rel took {}µs", start.to(end).num_microseconds().unwrap()); +} + +#[test] +fn test_failing_scalar() { + let mut c = new_connection("").expect("Couldn't open conn."); + let db = mentat_db::db::ensure_current_version(&mut c).expect("Couldn't open DB."); + + // Scalar that fails. + let start = time::PreciseTime::now(); + let results = q_once(&c, &db.schema, + "[:find ?x . :where [?x :db/fulltext true]]", None) + .expect("Query failed"); + let end = time::PreciseTime::now(); + + assert_eq!(0, results.len()); + + if let QueryResults::Scalar(None) = results { + } else { + panic!("Expected failed scalar."); + } + + println!("Failing scalar took {}µs", start.to(end).num_microseconds().unwrap()); +} + +#[test] +fn test_scalar() { + let mut c = new_connection("").expect("Couldn't open conn."); + let db = mentat_db::db::ensure_current_version(&mut c).expect("Couldn't open DB."); + + // Scalar that succeeds. + let start = time::PreciseTime::now(); + let results = q_once(&c, &db.schema, + "[:find ?ident . :where [24 :db/ident ?ident]]", None) + .expect("Query failed"); + let end = time::PreciseTime::now(); + + assert_eq!(1, results.len()); + + if let QueryResults::Scalar(Some(TypedValue::Keyword(ref kw))) = results { + // Should be '24'. + assert_eq!(&NamespacedKeyword::new("db.type", "keyword"), kw); + assert_eq!(24, + db.schema.get_entid(kw).unwrap()); + } else { + panic!("Expected scalar."); + } + + println!("{:?}", results); + println!("Scalar took {}µs", start.to(end).num_microseconds().unwrap()); +} + +#[test] +fn test_tuple() { + let mut c = new_connection("").expect("Couldn't open conn."); + let db = mentat_db::db::ensure_current_version(&mut c).expect("Couldn't open DB."); + + // Tuple. + let start = time::PreciseTime::now(); + let results = q_once(&c, &db.schema, + "[:find [?index ?cardinality] + :where [:db/txInstant :db/index ?index] + [:db/txInstant :db/cardinality ?cardinality]]", + None) + .expect("Query failed"); + let end = time::PreciseTime::now(); + + assert_eq!(1, results.len()); + + if let QueryResults::Tuple(Some(ref tuple)) = results { + let cardinality_one = NamespacedKeyword::new("db.cardinality", "one"); + assert_eq!(tuple.len(), 2); + assert_eq!(tuple[0], TypedValue::Boolean(true)); + assert_eq!(tuple[1], TypedValue::Ref(db.schema.get_entid(&cardinality_one).unwrap())); + } else { + panic!("Expected tuple."); + } + + println!("{:?}", results); + println!("Tuple took {}µs", start.to(end).num_microseconds().unwrap()); +} + +#[test] +fn test_coll() { + // We can't test Coll yet, because the EDN parser is incomplete. +} +