greg/mentat
1
0
Fork 0
Code Issues 294 Pull requests Projects Releases 9 Packages Wiki Activity

Implement type annotations in queries. (#526) r=rnewman

Browse source
This commit is contained in:
Thom Chiovoloni 2018-01-29 14:29:16 -08:00 committed by Richard Newman
parent ef9f2d9c51
commit 98502eb68f
19 changed files with 775 additions and 141 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
core/src/lib.rs
View file

@ -281,27 +281,51 @@ impl From<i32> for TypedValue {
}
}
/// Type safe representation of the possible return values from SQLite's `typeof`
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialOrd, PartialEq)]
pub enum SQLTypeAffinity {
Null, // "null"
Integer, // "integer"
Real, // "real"
Text, // "text"
Blob, // "blob"
}
// Put this here rather than in `db` simply because it's widely needed.
pub trait SQLValueType {
fn value_type_tag(&self) -> i32;
fn value_type_tag(&self) -> ValueTypeTag;
fn accommodates_integer(&self, int: i64) -> bool;
/// Return a pair of the ValueTypeTag for this value type, and the SQLTypeAffinity required
/// to distinguish it from any other types that share the same tag.
///
/// Background: The tag alone is not enough to determine the type of a value, since multiple
/// ValueTypes may share the same tag (for example, ValueType::Long and ValueType::Double).
/// However, each ValueType can be determined by checking both the tag and the type's affinity.
fn sql_representation(&self) -> (ValueTypeTag, Option<SQLTypeAffinity>);
}
impl SQLValueType for ValueType {
fn value_type_tag(&self) -> i32 {
fn sql_representation(&self) -> (ValueTypeTag, Option<SQLTypeAffinity>) {
match *self {
ValueType::Ref => 0,
ValueType::Boolean => 1,
ValueType::Instant => 4,
ValueType::Ref => (0, None),
ValueType::Boolean => (1, None),
ValueType::Instant => (4, None),
// SQLite distinguishes integral from decimal types, allowing long and double to share a tag.
ValueType::Long => 5,
ValueType::Double => 5,
ValueType::String => 10,
ValueType::Uuid => 11,
ValueType::Keyword => 13,
ValueType::Long => (5, Some(SQLTypeAffinity::Integer)),
ValueType::Double => (5, Some(SQLTypeAffinity::Real)),
ValueType::String => (10, None),
ValueType::Uuid => (11, None),
ValueType::Keyword => (13, None),
}
}
#[inline]
fn value_type_tag(&self) -> ValueTypeTag {
self.sql_representation().0
}
/// Returns true if the provided integer is in the SQLite value space of this type. For
/// example, `1` is how we encode `true`.
///
@ -412,6 +436,12 @@ impl ValueTypeSet {
ValueTypeSet(self.0.intersection(other.0))
}
/// Returns the set difference between `self` and `other`, which is the
/// set of items in `self` that are not in `other`.
pub fn difference(&self, other: &ValueTypeSet) -> ValueTypeSet {
ValueTypeSet(self.0 - other.0)
}
/// Return an arbitrary type that's part of this set.
/// For a set containing a single type, this will be that type.
pub fn exemplar(&self) -> Option<ValueType> {
@ -422,6 +452,11 @@ impl ValueTypeSet {
self.0.is_subset(&other.0)
}
/// Returns true if `self` and `other` contain no items in common.
pub fn is_disjoint(&self, other: &ValueTypeSet) -> bool {
self.0.is_disjoint(&other.0)
}
pub fn contains(&self, vt: ValueType) -> bool {
self.0.contains(&vt)
}
@ -433,6 +468,10 @@ impl ValueTypeSet {
pub fn is_unit(&self) -> bool {
self.0.len() == 1
}
pub fn iter(&self) -> ::enum_set::Iter<ValueType> {
self.0.iter()
}
}
impl IntoIterator for ValueTypeSet {

144
query-algebrizer/src/clauses/mod.rs
View file

@ -46,6 +46,8 @@ use mentat_query::{
};
use errors::{
Error,
ErrorKind,
Result,
};
@ -214,6 +216,9 @@ pub struct ConjoiningClauses {
/// A mapping, similar to `column_bindings`, but used to pull type tags out of the store at runtime.
/// If a var isn't unit in `known_types`, it should be present here.
pub extracted_types: BTreeMap<Variable, QualifiedAlias>,
/// Map of variables to the set of type requirements we have for them.
required_types: BTreeMap<Variable, ValueTypeSet>,
}
impl PartialEq for ConjoiningClauses {
@ -226,7 +231,8 @@ impl PartialEq for ConjoiningClauses {
self.input_variables.eq(&other.input_variables) &&
self.value_bindings.eq(&other.value_bindings) &&
self.known_types.eq(&other.known_types) &&
self.extracted_types.eq(&other.extracted_types)
self.extracted_types.eq(&other.extracted_types) &&
self.required_types.eq(&other.required_types)
}
}
@ -244,6 +250,7 @@ impl Debug for ConjoiningClauses {
.field("value_bindings", &self.value_bindings)
.field("known_types", &self.known_types)
.field("extracted_types", &self.extracted_types)
.field("required_types", &self.required_types)
.finish()
}
}
@ -257,6 +264,7 @@ impl Default for ConjoiningClauses {
from: vec![],
computed_tables: vec![],
wheres: ColumnIntersection::default(),
required_types: BTreeMap::new(),
input_variables: BTreeSet::new(),
column_bindings: BTreeMap::new(),
value_bindings: BTreeMap::new(),
@ -320,6 +328,7 @@ impl ConjoiningClauses {
value_bindings: self.value_bindings.clone(),
known_types: self.known_types.clone(),
extracted_types: self.extracted_types.clone(),
required_types: self.required_types.clone(),
..Default::default()
}
}
@ -334,6 +343,7 @@ impl ConjoiningClauses {
value_bindings: self.value_bindings.with_intersected_keys(&vars),
known_types: self.known_types.with_intersected_keys(&vars),
extracted_types: self.extracted_types.with_intersected_keys(&vars),
required_types: self.required_types.with_intersected_keys(&vars),
..Default::default()
}
}
@ -356,7 +366,7 @@ impl ConjoiningClauses {
// Are we also trying to figure out the type of the value when the query runs?
// If so, constrain that!
if let Some(qa) = self.extracted_types.get(&var) {
self.wheres.add_intersection(ColumnConstraint::HasType(qa.0.clone(), vt));
self.wheres.add_intersection(ColumnConstraint::has_unit_type(qa.0.clone(), vt));
}
// Finally, store the binding for future use.
@ -541,6 +551,47 @@ impl ConjoiningClauses {
}
}
/// Require that `var` be one of the types in `types`. If any existing
/// type requirements exist for `var`, the requirement after this
/// function returns will be the intersection of the requested types and
/// the type requirements in place prior to calling `add_type_requirement`.
///
/// If the intersection will leave the variable so that it cannot be any
/// type, we'll call `mark_known_empty`.
pub fn add_type_requirement(&mut self, var: Variable, types: ValueTypeSet) {
if types.is_empty() {
// This shouldn't happen, but if it does…
self.mark_known_empty(EmptyBecause::NoValidTypes(var));
return;
}
// Optimize for the empty case.
let empty_because = match self.required_types.entry(var.clone()) {
Entry::Vacant(entry) => {
entry.insert(types);
return;
},
Entry::Occupied(mut entry) => {
// We have an existing requirement. The new requirement will be
// the intersection, but we'll `mark_known_empty` if that's empty.
let existing = *entry.get();
let intersection = types.intersection(&existing);
entry.insert(intersection);
if !intersection.is_empty() {
return;
}
EmptyBecause::TypeMismatch {
var: var,
existing: existing,
desired: types,
}
},
};
self.mark_known_empty(empty_because);
}
/// Like `constrain_var_to_type` but in reverse: this expands the set of types
/// with which a variable is associated.
///
@ -692,11 +743,13 @@ impl ConjoiningClauses {
// TODO: see if the variable is projected, aggregated, or compared elsewhere in
// the query. If it's not, we don't need to use all_datoms here.
&PatternValuePlace::Variable(ref v) => {
// Do we know that this variable can't be a string? If so, we don't need
// AllDatoms. None or String means it could be or definitely is.
match self.known_types.get(v).map(|types| types.contains(ValueType::String)) {
Some(false) => DatomsTable::Datoms,
_ => DatomsTable::AllDatoms,
// If `required_types` and `known_types` don't exclude strings,
// we need to query `all_datoms`.
if self.required_types.get(v).map_or(true, |s| s.contains(ValueType::String)) &&
self.known_types.get(v).map_or(true, |s| s.contains(ValueType::String)) {
DatomsTable::AllDatoms
} else {
DatomsTable::Datoms
}
}
&PatternValuePlace::Constant(NonIntegerConstant::Text(_)) =>
@ -848,7 +901,65 @@ impl ConjoiningClauses {
}
}
pub fn process_required_types(&mut self) -> Result<()> {
if self.empty_because.is_some() {
return Ok(())
}
// We can't call `mark_known_empty` inside the loop since it would be a
// mutable borrow on self while we're iterating over `self.required_types`.
// Doing it like this avoids needing to copy `self.required_types`.
let mut empty_because: Option<EmptyBecause> = None;
for (var, types) in self.required_types.iter() {
if let Some(already_known) = self.known_types.get(var) {
if already_known.is_disjoint(types) {
// If we know the constraint can't be one of the types
// the variable could take, then we know we're empty.
empty_because = Some(EmptyBecause::TypeMismatch {
var: var.clone(),
existing: *already_known,
desired: *types,
});
break;
}
if already_known.is_subset(types) {
// TODO: I'm not convinced that we can do nothing here.
//
// Consider `[:find ?x ?v :where [_ _ ?v] [(> ?v 10)] [?x :foo/long ?v]]`.
//
// That will produce SQL like:
//
// ```
// SELECT datoms01.e AS `?x`, datoms00.v AS `?v`
// FROM datoms datoms00, datoms01
// WHERE datoms00.v > 10
// AND datoms01.v = datoms00.v
// AND datoms01.value_type_tag = datoms00.value_type_tag
// AND datoms01.a = 65537
// ```
//
// Which is not optimal — the left side of the join will
// produce lots of spurious bindings for datoms00.v.
//
// See https://github.com/mozilla/mentat/issues/520, and
// https://github.com/mozilla/mentat/issues/293.
continue;
}
}
let qa = self.extracted_types
.get(&var)
.ok_or_else(|| Error::from_kind(ErrorKind::UnboundVariable(var.name())))?;
self.wheres.add_intersection(ColumnConstraint::HasTypes {
value: qa.0.clone(),
value_types: *types,
check_value: true,
});
}
if let Some(reason) = empty_because {
self.mark_known_empty(reason);
}
Ok(())
}
/// When a CC has accumulated all patterns, generate value_type_tag entries in `wheres`
/// to refine value types for which two things are true:
///
@ -873,6 +984,22 @@ impl ConjoiningClauses {
}
impl ConjoiningClauses {
pub fn apply_clauses(&mut self, schema: &Schema, where_clauses: Vec<WhereClause>) -> Result<()> {
// We apply (top level) type predicates first as an optimization.
for clause in where_clauses.iter() {
if let &WhereClause::TypeAnnotation(ref anno) = clause {
self.apply_type_anno(anno)?;
}
}
// Then we apply everything else.
for clause in where_clauses {
if let &WhereClause::TypeAnnotation(_) = &clause {
continue;
}
self.apply_clause(schema, clause)?;
}
Ok(())
}
// This is here, rather than in `lib.rs`, because it's recursive: `or` can contain `or`,
// and so on.
pub fn apply_clause(&mut self, schema: &Schema, where_clause: WhereClause) -> Result<()> {
@ -895,6 +1022,9 @@ impl ConjoiningClauses {
validate_not_join(&n)?;
self.apply_not_join(schema, n)
},
WhereClause::TypeAnnotation(anno) => {
self.apply_type_anno(&anno)
},
_ => unimplemented!(),
}
}

18
query-algebrizer/src/clauses/not.rs
View file

@ -49,16 +49,26 @@ impl ConjoiningClauses {
}
}
for clause in not_join.clauses.into_iter() {
template.apply_clause(&schema, clause)?;
}
template.apply_clauses(&schema, not_join.clauses)?;
if template.is_known_empty() {
return Ok(());
}
// We are only expanding column bindings here and not pruning extracted types as we are not projecting values.
template.expand_column_bindings();
if template.is_known_empty() {
return Ok(());
}
template.prune_extracted_types();
if template.is_known_empty() {
return Ok(());
}
template.process_required_types()?;
if template.is_known_empty() {
return Ok(());
}
let subquery = ComputedTable::Subquery(template);

9
query-algebrizer/src/clauses/or.rs
View file

@ -96,9 +96,7 @@ impl ConjoiningClauses {
// [:find ?x :where (or (and [?x _ 5] [?x :foo/bar 7]))]
// which is equivalent to dropping the `or` _and_ the `and`!
OrWhereClause::And(clauses) => {
for clause in clauses {
self.apply_clause(schema, clause)?;
}
self.apply_clauses(schema, clauses)?;
Ok(())
},
}
@ -564,9 +562,7 @@ impl ConjoiningClauses {
let mut receptacle = template.make_receptacle();
match clause {
OrWhereClause::And(clauses) => {
for clause in clauses {
receptacle.apply_clause(&schema, clause)?;
}
receptacle.apply_clauses(&schema, clauses)?;
},
OrWhereClause::Clause(clause) => {
receptacle.apply_clause(&schema, clause)?;
@ -577,6 +573,7 @@ impl ConjoiningClauses {
} else {
receptacle.expand_column_bindings();
receptacle.prune_extracted_types();
receptacle.process_required_types()?;
acc.push(receptacle);
}
}

9
query-algebrizer/src/clauses/pattern.rs
View file

@ -201,7 +201,7 @@ impl ConjoiningClauses {
} else {
// It must be a keyword.
self.constrain_column_to_constant(col.clone(), DatomsColumn::Value, TypedValue::Keyword(kw.clone()));
self.wheres.add_intersection(ColumnConstraint::HasType(col.clone(), ValueType::Keyword));
self.wheres.add_intersection(ColumnConstraint::has_unit_type(col.clone(), ValueType::Keyword));
};
},
PatternValuePlace::Constant(ref c) => {
@ -237,7 +237,8 @@ impl ConjoiningClauses {
// Because everything we handle here is unambiguous, we generate a single type
// restriction from the value type of the typed value.
if value_type.is_none() {
self.wheres.add_intersection(ColumnConstraint::HasType(col.clone(), typed_value_type));
self.wheres.add_intersection(
ColumnConstraint::has_unit_type(col.clone(), typed_value_type));
}
},
}
@ -445,7 +446,7 @@ mod testing {
// TODO: implement expand_type_tags.
assert_eq!(cc.wheres, vec![
ColumnConstraint::Equals(d0_v, QueryValue::TypedValue(TypedValue::Boolean(true))),
ColumnConstraint::HasType("datoms00".to_string(), ValueType::Boolean),
ColumnConstraint::has_unit_type("datoms00".to_string(), ValueType::Boolean),
].into());
}
@ -589,7 +590,7 @@ mod testing {
// TODO: implement expand_type_tags.
assert_eq!(cc.wheres, vec![
ColumnConstraint::Equals(d0_v, QueryValue::TypedValue(TypedValue::String(Rc::new("hello".to_string())))),
ColumnConstraint::HasType("all_datoms00".to_string(), ValueType::String),
ColumnConstraint::has_unit_type("all_datoms00".to_string(), ValueType::String),
].into());
}

8
query-algebrizer/src/clauses/predicate.rs
View file

@ -17,6 +17,7 @@ use mentat_core::{
use mentat_query::{
FnArg,
Predicate,
TypeAnnotation,
};
use clauses::ConjoiningClauses;
@ -59,6 +60,13 @@ impl ConjoiningClauses {
}
}
/// Apply a type annotation, which is a construct like a predicate that constrains the argument
/// to be a specific ValueType.
pub fn apply_type_anno(&mut self, anno: &TypeAnnotation) -> Result<()> {
self.add_type_requirement(anno.variable.clone(), ValueTypeSet::of_one(anno.value_type));
Ok(())
}
/// This function:
/// - Resolves variables and converts types to those more amenable to SQL.
/// - Ensures that the predicate functions name a known operator.

7
query-algebrizer/src/lib.rs
View file

@ -179,12 +179,11 @@ pub fn algebrize_with_inputs(schema: &Schema,
// TODO: integrate default source into pattern processing.
// TODO: flesh out the rest of find-into-context.
let where_clauses = parsed.where_clauses;
for where_clause in where_clauses {
cc.apply_clause(schema, where_clause)?;
}
cc.apply_clauses(schema, parsed.where_clauses)?;
cc.expand_column_bindings();
cc.prune_extracted_types();
cc.process_required_types()?;
let (order, extra_vars) = validate_and_simplify_order(&cc, parsed.order)?;
let with: BTreeSet<Variable> = parsed.with.into_iter().chain(extra_vars.into_iter()).collect();

32
query-algebrizer/src/types.rs
View file

@ -334,11 +334,25 @@ pub enum ColumnConstraint {
left: QueryValue,
right: QueryValue,
},
HasType(TableAlias, ValueType),
HasTypes {
value: TableAlias,
value_types: ValueTypeSet,
check_value: bool,
},
NotExists(ComputedTable),
Matches(QualifiedAlias, QueryValue),
}
impl ColumnConstraint {
pub fn has_unit_type(value: TableAlias, value_type: ValueType) -> ColumnConstraint {
ColumnConstraint::HasTypes {
value,
value_types: ValueTypeSet::of_one(value_type),
check_value: false,
}
}
}
#[derive(PartialEq, Eq, Debug)]
pub enum ColumnConstraintOrAlternation {
Constraint(ColumnConstraint),
@ -451,8 +465,20 @@ impl Debug for ColumnConstraint {
write!(f, "{:?} MATCHES {:?}", qa, thing)
},
&HasType(ref qa, value_type) => {
write!(f, "{:?}.value_type_tag = {:?}", qa, value_type)
&HasTypes { ref value, ref value_types, check_value } => {
// This is cludgey, but it's debug code.
write!(f, "(")?;
for value_type in value_types.iter() {
write!(f, "({:?}.value_type_tag = {:?}", value, value_type)?;
if check_value && value_type == ValueType::Double || value_type == ValueType::Long {
write!(f, " AND typeof({:?}) = '{:?}')", value,
if value_type == ValueType::Double { "real" } else { "integer" })?;
} else {
write!(f, ")")?;
}
write!(f, " OR ")?;
}
write!(f, "1)")
},
&NotExists(ref ct) => {
write!(f, "NOT EXISTS {:?}", ct)

30
query-algebrizer/tests/fulltext.rs
View file

@ -13,37 +13,24 @@ extern crate mentat_query;
extern crate mentat_query_algebrizer;
extern crate mentat_query_parser;
mod utils;
use mentat_core::{
Attribute,
Entid,
Schema,
ValueType,
};
use mentat_query_parser::{
parse_find_string,
};
use mentat_query::{
NamespacedKeyword,
};
use mentat_query_algebrizer::{
ConjoiningClauses,
algebrize,
use utils::{
add_attribute,
alg,
associate_ident,
};
// These are helpers that tests use to build Schema instances.
fn associate_ident(schema: &mut Schema, i: NamespacedKeyword, e: Entid) {
schema.entid_map.insert(e, i.clone());
schema.ident_map.insert(i.clone(), e);
}
fn add_attribute(schema: &mut Schema, e: Entid, a: Attribute) {
schema.attribute_map.insert(e, a);
}
fn prepopulated_schema() -> Schema {
let mut schema = Schema::default();
associate_ident(&mut schema, NamespacedKeyword::new("foo", "name"), 65);
@ -80,11 +67,6 @@ fn prepopulated_schema() -> Schema {
schema
}
fn alg(schema: &Schema, input: &str) -> ConjoiningClauses {
let parsed = parse_find_string(input).expect("query input to have parsed");
algebrize(schema.into(), parsed).expect("algebrizing to have succeeded").cc
}
#[test]
fn test_apply_fulltext() {
let schema = prepopulated_schema();

43
query-algebrizer/tests/ground.rs
View file

@ -13,20 +13,17 @@ extern crate mentat_query;
extern crate mentat_query_algebrizer;
extern crate mentat_query_parser;
mod utils;
use std::collections::BTreeMap;
use mentat_core::{
Attribute,
Entid,
Schema,
ValueType,
TypedValue,
};
use mentat_query_parser::{
parse_find_string,
};
use mentat_query::{
NamespacedKeyword,
PlainSymbol,
@ -35,26 +32,19 @@ use mentat_query::{
use mentat_query_algebrizer::{
BindingError,
ConjoiningClauses,
ComputedTable,
Error,
ErrorKind,
QueryInputs,
algebrize,
algebrize_with_inputs,
};
// These are helpers that tests use to build Schema instances.
#[cfg(test)]
fn associate_ident(schema: &mut Schema, i: NamespacedKeyword, e: Entid) {
schema.entid_map.insert(e, i.clone());
schema.ident_map.insert(i.clone(), e);
}
#[cfg(test)]
fn add_attribute(schema: &mut Schema, e: Entid, a: Attribute) {
schema.attribute_map.insert(e, a);
}
use utils::{
add_attribute,
alg,
associate_ident,
bails,
bails_with_inputs,
};
fn prepopulated_schema() -> Schema {
let mut schema = Schema::default();
@ -91,21 +81,6 @@ fn prepopulated_schema() -> Schema {
schema
}
fn bails(schema: &Schema, input: &str) -> Error {
let parsed = parse_find_string(input).expect("query input to have parsed");
algebrize(schema.into(), parsed).expect_err("algebrize to have failed")
}
fn bails_with_inputs(schema: &Schema, input: &str, inputs: QueryInputs) -> Error {
let parsed = parse_find_string(input).expect("query input to have parsed");
algebrize_with_inputs(schema, parsed, 0, inputs).expect_err("algebrize to have failed")
}
fn alg(schema: &Schema, input: &str) -> ConjoiningClauses {
let parsed = parse_find_string(input).expect("query input to have parsed");
algebrize(schema.into(), parsed).expect("algebrizing to have succeeded").cc
}
#[test]
fn test_ground_doesnt_bail_for_type_conflicts() {
// We know `?x` to be a ref, but we're attempting to ground it to a Double.

37
query-algebrizer/tests/predicate.rs
View file

@ -13,18 +13,15 @@ extern crate mentat_query;
extern crate mentat_query_algebrizer;
extern crate mentat_query_parser;
mod utils;
use mentat_core::{
Attribute,
Entid,
Schema,
ValueType,
ValueTypeSet,
};
use mentat_query_parser::{
parse_find_string,
};
use mentat_query::{
NamespacedKeyword,
PlainSymbol,
@ -32,24 +29,16 @@ use mentat_query::{
};
use mentat_query_algebrizer::{
ConjoiningClauses,
EmptyBecause,
Error,
ErrorKind,
algebrize,
};
// These are helpers that tests use to build Schema instances.
#[cfg(test)]
fn associate_ident(schema: &mut Schema, i: NamespacedKeyword, e: Entid) {
schema.entid_map.insert(e, i.clone());
schema.ident_map.insert(i.clone(), e);
}
#[cfg(test)]
fn add_attribute(schema: &mut Schema, e: Entid, a: Attribute) {
schema.attribute_map.insert(e, a);
}
use utils::{
add_attribute,
alg,
associate_ident,
bails,
};
fn prepopulated_schema() -> Schema {
let mut schema = Schema::default();
@ -68,16 +57,6 @@ fn prepopulated_schema() -> Schema {
schema
}
fn bails(schema: &Schema, input: &str) -> Error {
let parsed = parse_find_string(input).expect("query input to have parsed");
algebrize(schema.into(), parsed).expect_err("algebrize to have failed")
}
fn alg(schema: &Schema, input: &str) -> ConjoiningClauses {
let parsed = parse_find_string(input).expect("query input to have parsed");
algebrize(schema.into(), parsed).expect("algebrizing to have succeeded").cc
}
#[test]
fn test_instant_predicates_require_instants() {
let schema = prepopulated_schema();

80
query-algebrizer/tests/type_reqs.rs Normal file
View file

@ -0,0 +1,80 @@
// Copyright 2016 Mozilla
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.
extern crate mentat_core;
extern crate mentat_query;
extern crate mentat_query_algebrizer;
extern crate mentat_query_parser;
mod utils;
use utils::{
alg,
SchemaBuilder,
bails,
};
use mentat_core::{
Schema,
ValueType,
};
fn prepopulated_schema() -> Schema {
SchemaBuilder::new()
.define_simple_attr("test", "boolean", ValueType::Boolean, false)
.define_simple_attr("test", "long", ValueType::Long, false)
.define_simple_attr("test", "double", ValueType::Double, false)
.define_simple_attr("test", "string", ValueType::String, false)
.define_simple_attr("test", "keyword", ValueType::Keyword, false)
.define_simple_attr("test", "uuid", ValueType::Uuid, false)
.define_simple_attr("test", "instant", ValueType::Instant, false)
.define_simple_attr("test", "ref", ValueType::Ref, false)
.schema
}
#[test]
fn test_empty_known() {
let type_names = [
"boolean",
"long",
"double",
"string",
"keyword",
"uuid",
"instant",
"ref",
];
let schema = prepopulated_schema();
for known_type in type_names.iter() {
for required in type_names.iter() {
let q = format!("[:find ?e :where [?e :test/{} ?v] [({} ?v)]]",
known_type, required);
println!("Query: {}", q);
let cc = alg(&schema, &q);
// It should only be empty if the known type and our requirement differ.
assert_eq!(cc.empty_because.is_some(), known_type != required,
"known_type = {}; required = {}", known_type, required);
}
}
}
#[test]
fn test_multiple() {
let schema = prepopulated_schema();
let q = "[:find ?e :where [?e _ ?v] [(long ?v)] [(double ?v)]]";
let cc = alg(&schema, &q);
assert!(cc.empty_because.is_some());
}
#[test]
fn test_unbound() {
let schema = prepopulated_schema();
bails(&schema, "[:find ?e :where [(string ?e)]]");
}

99
query-algebrizer/tests/utils/mod.rs Normal file
View file

@ -0,0 +1,99 @@
// Copyright 2018 Mozilla
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.
// This is required to prevent warnings about unused functions in this file just
// because it's unused in a single file (tests that don't use every function in
// this module will get warnings otherwise).
#![allow(dead_code)]
use mentat_core::{
Attribute,
Entid,
Schema,
ValueType,
};
use mentat_query_parser::{
parse_find_string,
};
use mentat_query::{
NamespacedKeyword,
};
use mentat_query_algebrizer::{
algebrize,
algebrize_with_inputs,
ConjoiningClauses,
Error,
QueryInputs,
};
// Common utility functions used in multiple test files.
// These are helpers that tests use to build Schema instances.
pub fn associate_ident(schema: &mut Schema, i: NamespacedKeyword, e: Entid) {
schema.entid_map.insert(e, i.clone());
schema.ident_map.insert(i.clone(), e);
}
pub fn add_attribute(schema: &mut Schema, e: Entid, a: Attribute) {
schema.attribute_map.insert(e, a);
}
pub struct SchemaBuilder {
pub schema: Schema,
pub counter: Entid,
}
impl SchemaBuilder {
pub fn new() -> SchemaBuilder {
SchemaBuilder {
schema: Schema::default(),
counter: 65
}
}
pub fn define_attr(mut self, kw: NamespacedKeyword, attr: Attribute) -> Self {
associate_ident(&mut self.schema, kw, self.counter);
add_attribute(&mut self.schema, self.counter, attr);
self.counter += 1;
self
}
pub fn define_simple_attr<T>(self,
keyword_ns: T,
keyword_name: T,
value_type: ValueType,
multival: bool) -> Self
where T: Into<String>
{
self.define_attr(NamespacedKeyword::new(keyword_ns, keyword_name), Attribute {
value_type,
multival,
..Default::default()
})
}
}
pub fn bails(schema: &Schema, input: &str) -> Error {
let parsed = parse_find_string(input).expect("query input to have parsed");
algebrize(schema.into(), parsed).expect_err("algebrize to have failed")
}
pub fn bails_with_inputs(schema: &Schema, input: &str, inputs: QueryInputs) -> Error {
let parsed = parse_find_string(input).expect("query input to have parsed");
algebrize_with_inputs(schema, parsed, 0, inputs).expect_err("algebrize to have failed")
}
pub fn alg(schema: &Schema, input: &str) -> ConjoiningClauses {
let parsed = parse_find_string(input).expect("query input to have parsed");
algebrize(schema.into(), parsed).expect("algebrizing to have succeeded").cc
}

60
query-parser/src/parse.rs
View file

@ -12,6 +12,7 @@ extern crate combine;
extern crate edn;
extern crate mentat_parser_utils;
extern crate mentat_query;
extern crate mentat_core;
use std; // To refer to std::result::Result.
@ -20,6 +21,8 @@ use std::collections::BTreeSet;
use self::combine::{eof, many, many1, optional, parser, satisfy, satisfy_map, Parser, ParseResult, Stream};
use self::combine::combinator::{any, choice, or, try};
use self::mentat_core::ValueType;
use self::mentat_parser_utils::{
KeywordMapParser,
ResultParser,
@ -56,6 +59,7 @@ use self::mentat_query::{
Predicate,
QueryFunction,
SrcVar,
TypeAnnotation,
UnifyVars,
Variable,
VariableOrPlaceholder,
@ -286,6 +290,44 @@ def_parser!(Where, pred, WhereClause, {
})))
});
def_parser!(Query, type_anno_type, ValueType, {
satisfy_map(|v: &edn::ValueAndSpan| {
match v.inner {
edn::SpannedValue::PlainSymbol(ref s) => {
let name = s.0.as_str();
match name {
"ref" => Some(ValueType::Ref),
"boolean" => Some(ValueType::Boolean),
"instant" => Some(ValueType::Instant),
"long" => Some(ValueType::Long),
"double" => Some(ValueType::Double),
"string" => Some(ValueType::String),
"keyword" => Some(ValueType::Keyword),
"uuid" => Some(ValueType::Uuid),
_ => None
}
},
_ => None,
}
})
});
/// A type annotation.
def_parser!(Where, type_annotation, WhereClause, {
// Accept either a nested list or a nested vector here:
// `[(string ?x)]` or `[[string ?x]]`
vector()
.of_exactly(seq()
.of_exactly((Query::type_anno_type(), Query::variable())
.map(|(ty, var)| {
WhereClause::TypeAnnotation(
TypeAnnotation {
value_type: ty,
variable: var,
})
})))
});
/// A vector containing a parenthesized function expression and a binding.
def_parser!(Where, where_fn, WhereClause, {
// Accept either a nested list or a nested vector here:
@ -356,6 +398,7 @@ def_parser!(Where, clause, WhereClause, {
try(Where::not_join_clause()),
try(Where::not_clause()),
try(Where::type_annotation()),
try(Where::pred()),
try(Where::where_fn()),
])
@ -949,4 +992,21 @@ mod test {
VariableOrPlaceholder::Variable(Variable::from_valid_name("?y"))]),
}));
}
#[test]
fn test_type_anno() {
assert_edn_parses_to!(Where::type_annotation,
"[(string ?x)]",
WhereClause::TypeAnnotation(TypeAnnotation {
value_type: ValueType::String,
variable: Variable::from_valid_name("?x"),
}));
assert_edn_parses_to!(Where::clause,
"[[long ?foo]]",
WhereClause::TypeAnnotation(TypeAnnotation {
value_type: ValueType::Long,
variable: Variable::from_valid_name("?foo"),
}));
}
}

20
query-sql/src/lib.rs
View file

@ -19,6 +19,7 @@ use std::boxed::Box;
use mentat_core::{
Entid,
TypedValue,
SQLTypeAffinity,
};
use mentat_query::{
@ -105,6 +106,10 @@ pub enum Constraint {
},
NotExists {
subquery: TableOrSubquery,
},
TypeCheck {
value: ColumnOrExpression,
affinity: SQLTypeAffinity
}
}
@ -367,7 +372,20 @@ impl QueryFragment for Constraint {
subquery.push_sql(out)?;
out.push_sql(")");
Ok(())
}
},
&TypeCheck { ref value, ref affinity } => {
out.push_sql("typeof(");
value.push_sql(out)?;
out.push_sql(") = ");
out.push_sql(match *affinity {
SQLTypeAffinity::Null => "'null'",
SQLTypeAffinity::Integer => "'integer'",
SQLTypeAffinity::Real => "'real'",
SQLTypeAffinity::Text => "'text'",
SQLTypeAffinity::Blob => "'blob'",
});
Ok(())
},
}
}
}

72
query-translator/src/translate.rs
View file

@ -9,9 +9,12 @@
// specific language governing permissions and limitations under the License.
use mentat_core::{
SQLTypeAffinity,
SQLValueType,
TypedValue,
ValueType,
ValueTypeTag,
ValueTypeSet,
};
use mentat_query::Limit;
@ -55,6 +58,8 @@ use mentat_query_sql::{
Values,
};
use std::collections::HashMap;
use super::Result;
trait ToConstraint {
@ -97,6 +102,51 @@ impl ToConstraint for ColumnConstraintOrAlternation {
}
}
fn affinity_count(tag: i32) -> usize {
ValueTypeSet::any().into_iter()
.filter(|t| t.value_type_tag() == tag)
.count()
}
fn type_constraint(table: &TableAlias, tag: i32, to_check: Option<Vec<SQLTypeAffinity>>) -> Constraint {
let type_column = QualifiedAlias::new(table.clone(),
DatomsColumn::ValueTypeTag).to_column();
let check_type_tag = Constraint::equal(type_column, ColumnOrExpression::Integer(tag));
if let Some(affinities) = to_check {
let check_affinities = Constraint::Or {
constraints: affinities.into_iter().map(|affinity| {
Constraint::TypeCheck {
value: QualifiedAlias::new(table.clone(),
DatomsColumn::Value).to_column(),
affinity,
}
}).collect()
};
Constraint::And {
constraints: vec![
check_type_tag,
check_affinities
]
}
} else {
check_type_tag
}
}
// Returns a map of tags to a vector of all the possible affinities that those tags can represent
// given the types in `value_types`.
fn possible_affinities(value_types: ValueTypeSet) -> HashMap<ValueTypeTag, Vec<SQLTypeAffinity>> {
let mut result = HashMap::with_capacity(value_types.len());
for ty in value_types {
let (tag, affinity_to_check) = ty.sql_representation();
let mut affinities = result.entry(tag).or_insert_with(Vec::new);
if let Some(affinity) = affinity_to_check {
affinities.push(affinity);
}
}
result
}
impl ToConstraint for ColumnConstraint {
fn to_constraint(self) -> Constraint {
use self::ColumnConstraint::*;
@ -157,10 +207,24 @@ impl ToConstraint for ColumnConstraint {
right: right.into(),
}
},
HasType(table, value_type) => {
let column = QualifiedAlias::new(table, DatomsColumn::ValueTypeTag).to_column();
Constraint::equal(column, ColumnOrExpression::Integer(value_type.value_type_tag()))
HasTypes { value: table, value_types, check_value } => {
let constraints = if check_value {
possible_affinities(value_types)
.into_iter()
.map(|(tag, affinities)| {
let to_check = if affinities.is_empty() || affinities.len() == affinity_count(tag) {
None
} else {
Some(affinities)
};
type_constraint(&table, tag, to_check)
}).collect()
} else {
value_types.into_iter()
.map(|vt| type_constraint(&table, vt.value_type_tag(), None))
.collect()
};
Constraint::Or { constraints }
},
NotExists(computed_table) => {

66
query-translator/tests/translate.rs
View file

@ -209,7 +209,7 @@ fn test_unknown_attribute_keyword_value() {
let SQLQuery { sql, args } = translate(&schema, query);
// Only match keywords, not strings: tag = 13.
assert_eq!(sql, "SELECT DISTINCT `datoms00`.e AS `?x` FROM `datoms` AS `datoms00` WHERE `datoms00`.v = $v0 AND `datoms00`.value_type_tag = 13");
assert_eq!(sql, "SELECT DISTINCT `datoms00`.e AS `?x` FROM `datoms` AS `datoms00` WHERE `datoms00`.v = $v0 AND (`datoms00`.value_type_tag = 13)");
assert_eq!(args, vec![make_arg("$v0", ":ab/yyy")]);
}
@ -222,7 +222,7 @@ fn test_unknown_attribute_string_value() {
// We expect all_datoms because we're querying for a string. Magic, that.
// We don't want keywords etc., so tag = 10.
assert_eq!(sql, "SELECT DISTINCT `all_datoms00`.e AS `?x` FROM `all_datoms` AS `all_datoms00` WHERE `all_datoms00`.v = $v0 AND `all_datoms00`.value_type_tag = 10");
assert_eq!(sql, "SELECT DISTINCT `all_datoms00`.e AS `?x` FROM `all_datoms` AS `all_datoms00` WHERE `all_datoms00`.v = $v0 AND (`all_datoms00`.value_type_tag = 10)");
assert_eq!(args, vec![make_arg("$v0", "horses")]);
}
@ -235,7 +235,7 @@ fn test_unknown_attribute_double_value() {
// In general, doubles _could_ be 1.0, which might match a boolean or a ref. Set tag = 5 to
// make sure we only match numbers.
assert_eq!(sql, "SELECT DISTINCT `datoms00`.e AS `?x` FROM `datoms` AS `datoms00` WHERE `datoms00`.v = 9.95e0 AND `datoms00`.value_type_tag = 5");
assert_eq!(sql, "SELECT DISTINCT `datoms00`.e AS `?x` FROM `datoms` AS `datoms00` WHERE `datoms00`.v = 9.95e0 AND (`datoms00`.value_type_tag = 5)");
assert_eq!(args, vec![]);
}
@ -286,6 +286,64 @@ fn test_unknown_ident() {
assert_eq!("SELECT 1 LIMIT 0", sql);
}
#[test]
fn test_type_required_long() {
let schema = Schema::default();
let query = r#"[:find ?x :where [?x _ ?e] [(long ?e)]]"#;
let SQLQuery { sql, args } = translate(&schema, query);
assert_eq!(sql, "SELECT DISTINCT `datoms00`.e AS `?x` \
FROM `datoms` AS `datoms00` \
WHERE ((`datoms00`.value_type_tag = 5 AND \
(typeof(`datoms00`.v) = 'integer')))");
assert_eq!(args, vec![]);
}
#[test]
fn test_type_required_double() {
let schema = Schema::default();
let query = r#"[:find ?x :where [?x _ ?e] [(double ?e)]]"#;
let SQLQuery { sql, args } = translate(&schema, query);
assert_eq!(sql, "SELECT DISTINCT `datoms00`.e AS `?x` \
FROM `datoms` AS `datoms00` \
WHERE ((`datoms00`.value_type_tag = 5 AND \
(typeof(`datoms00`.v) = 'real')))");
assert_eq!(args, vec![]);
}
#[test]
fn test_type_required_boolean() {
let schema = Schema::default();
let query = r#"[:find ?x :where [?x _ ?e] [(boolean ?e)]]"#;
let SQLQuery { sql, args } = translate(&schema, query);
assert_eq!(sql, "SELECT DISTINCT `datoms00`.e AS `?x` \
FROM `datoms` AS `datoms00` \
WHERE (`datoms00`.value_type_tag = 1)");
assert_eq!(args, vec![]);
}
#[test]
fn test_type_required_string() {
let schema = Schema::default();
let query = r#"[:find ?x :where [?x _ ?e] [(string ?e)]]"#;
let SQLQuery { sql, args } = translate(&schema, query);
// Note: strings should use `all_datoms` and not `datoms`.
assert_eq!(sql, "SELECT DISTINCT `all_datoms00`.e AS `?x` \
FROM `all_datoms` AS `all_datoms00` \
WHERE (`all_datoms00`.value_type_tag = 10)");
assert_eq!(args, vec![]);
}
#[test]
fn test_numeric_less_than_unknown_attribute() {
let schema = Schema::default();
@ -751,7 +809,7 @@ fn test_unbound_attribute_with_ground() {
`all_datoms00`.value_type_tag AS `?v_value_type_tag` \
FROM `all_datoms` AS `all_datoms00` \
WHERE NOT EXISTS (SELECT 1 WHERE `all_datoms00`.v = 17 AND \
`all_datoms00`.value_type_tag = 5)");
(`all_datoms00`.value_type_tag = 5))");
}

28
query/src/lib.rs
View file

@ -56,6 +56,7 @@ pub use edn::{
use mentat_core::{
TypedValue,
ValueType,
};
pub type SrcVarName = String; // Do not include the required syntactic '$'.
@ -769,6 +770,12 @@ pub struct NotJoin {
pub clauses: Vec<WhereClause>,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct TypeAnnotation {
pub value_type: ValueType,
pub variable: Variable,
}
#[allow(dead_code)]
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum WhereClause {
@ -778,6 +785,7 @@ pub enum WhereClause {
WhereFn(WhereFn),
RuleExpr,
Pattern(Pattern),
TypeAnnotation(TypeAnnotation),
}
#[allow(dead_code)]
@ -852,12 +860,13 @@ impl ContainsVariables for WhereClause {
fn accumulate_mentioned_variables(&self, acc: &mut BTreeSet<Variable>) {
use WhereClause::*;
match self {
&OrJoin(ref o) => o.accumulate_mentioned_variables(acc),
&Pred(ref p) => p.accumulate_mentioned_variables(acc),
&Pattern(ref p) => p.accumulate_mentioned_variables(acc),
&NotJoin(ref n) => n.accumulate_mentioned_variables(acc),
&WhereFn(ref f) => f.accumulate_mentioned_variables(acc),
&RuleExpr => (),
&OrJoin(ref o) => o.accumulate_mentioned_variables(acc),
&Pred(ref p) => p.accumulate_mentioned_variables(acc),
&Pattern(ref p) => p.accumulate_mentioned_variables(acc),
&NotJoin(ref n) => n.accumulate_mentioned_variables(acc),
&WhereFn(ref f) => f.accumulate_mentioned_variables(acc),
&TypeAnnotation(ref a) => a.accumulate_mentioned_variables(acc),
&RuleExpr => (),
}
}
}
@ -920,6 +929,13 @@ impl ContainsVariables for Predicate {
}
}
impl ContainsVariables for TypeAnnotation {
fn accumulate_mentioned_variables(&self, acc: &mut BTreeSet<Variable>) {
acc_ref(acc, &self.variable);
}
}
impl ContainsVariables for Binding {
fn accumulate_mentioned_variables(&self, acc: &mut BTreeSet<Variable>) {
match self {

95
tests/query.rs
View file

@ -25,9 +25,9 @@ use mentat_core::{
HasSchema,
KnownEntid,
TypedValue,
ValueType,
Utc,
Uuid,
ValueType,
};
use mentat::{
@ -501,3 +501,96 @@ fn test_lookup() {
let fetched_many = conn.lookup_value_for_attribute(&c, *entid, &foo_many).unwrap().unwrap();
assert!(two_longs.contains(&fetched_many));
}
#[test]
fn test_type_reqs() {
let mut c = new_connection("").expect("Couldn't open conn.");
let mut conn = Conn::connect(&mut c).expect("Couldn't open DB.");
conn.transact(&mut c, r#"[
{:db/ident :test/boolean :db/valueType :db.type/boolean :db/cardinality :db.cardinality/one}
{:db/ident :test/long :db/valueType :db.type/long :db/cardinality :db.cardinality/one}
{:db/ident :test/double :db/valueType :db.type/double :db/cardinality :db.cardinality/one}
{:db/ident :test/string :db/valueType :db.type/string :db/cardinality :db.cardinality/one}
{:db/ident :test/keyword :db/valueType :db.type/keyword :db/cardinality :db.cardinality/one}
{:db/ident :test/uuid :db/valueType :db.type/uuid :db/cardinality :db.cardinality/one}
{:db/ident :test/instant :db/valueType :db.type/instant :db/cardinality :db.cardinality/one}
{:db/ident :test/ref :db/valueType :db.type/ref :db/cardinality :db.cardinality/one}
]"#).unwrap();
conn.transact(&mut c, r#"[
{:test/boolean true
:test/long 33
:test/double 1.4
:test/string "foo"
:test/keyword :foo/bar
:test/uuid #uuid "12341234-1234-1234-1234-123412341234"
:test/instant #inst "2018-01-01T11:00:00.000Z"
:test/ref 1}
]"#).unwrap();
let eid_query = r#"[:find ?eid :where [?eid :test/string "foo"]]"#;
let res = conn.q_once(&mut c, eid_query, None).unwrap();
let entid = match res {
QueryResults::Rel(ref vs) if vs.len() == 1 && vs[0].len() == 1 && vs[0][0].matches_type(ValueType::Ref) =>
if let TypedValue::Ref(eid) = vs[0][0] {
eid
} else {
// Already checked this.
unreachable!();
}
unexpected => {
panic!("Query to get the entity id returned unexpected result {:?}", unexpected);
}
};
let type_names = &[
"boolean",
"long",
"double",
"string",
"keyword",
"uuid",
"instant",
"ref",
];
for name in type_names {
let q = format!("[:find [?v ...] :in ?e :where [?e _ ?v] [({} ?v)]]", name);
let results = conn.q_once(&mut c, &q, QueryInputs::with_value_sequence(vec![
(Variable::from_valid_name("?e"), TypedValue::Ref(entid)),
])).unwrap();
match results {
QueryResults::Coll(vals) => {
assert_eq!(vals.len(), 1, "Query should find exactly 1 item");
},
v => {
panic!("Query returned unexpected type: {:?}", v);
}
}
}
conn.transact(&mut c, r#"[
{:db/ident :test/long2 :db/valueType :db.type/long :db/cardinality :db.cardinality/one}
]"#).unwrap();
conn.transact(&mut c, &format!("[[:db/add {} :test/long2 5]]", entid)).unwrap();
let longs_query = r#"[:find [?v ...]
:order (asc ?v)
:in ?e
:where [?e _ ?v] [(long ?v)]]"#;
let res = conn.q_once(&mut c, longs_query, QueryInputs::with_value_sequence(vec![
(Variable::from_valid_name("?e"), TypedValue::Ref(entid)),
])).unwrap();
match res {
QueryResults::Coll(vals) => {
assert_eq!(vals, vec![TypedValue::Long(5), TypedValue::Long(33)])
},
v => {
panic!("Query returned unexpected type: {:?}", v);
}
};
}