(query) Implement tx-log API: (tx-ids ...) and (tx-data ...) functions. r=rnewman

Cargo.toml

@@ -30,7 +30,7 @@ chrono = "0.4"
 error-chain = { git = "https://github.com/rnewman/error-chain", branch = "rnewman/sync" }
 lazy_static = "0.2"
 time = "0.1"
-uuid = "0.5"
+uuid = { version = "0.5", features = ["v4", "serde"] }
 
 [dependencies.rusqlite]
 version = "0.13"

core/Cargo.toml

@@ -9,7 +9,7 @@ enum-set = { git = "https://github.com/rnewman/enum-set" }
 lazy_static = "0.2"
 num = "0.1"
 ordered-float = { version = "0.5", features = ["serde"] }
-uuid = "0.5"
+uuid = { version = "0.5", features = ["v4", "serde"] }
 serde = { version = "1.0", features = ["rc"] }
 serde_derive = "1.0"
 

edn/Cargo.toml

@@ -16,7 +16,7 @@ itertools = "0.7"
 num = "0.1"
 ordered-float = "0.5"
 pretty = "0.2"
-uuid = "0.5"
+uuid = { version = "0.5", features = ["v4", "serde"] }
 serde = { version = "1.0", optional = true }
 serde_derive = { version = "1.0", optional = true }
 

query-algebrizer/src/clauses/mod.rs

@@ -84,6 +84,7 @@ mod resolve;
 
 mod ground;
 mod fulltext;
+mod tx_log_api;
 mod where_fn;
 
 use validate::{
@@ -449,6 +450,10 @@ impl ConjoiningClauses {
                     self.constrain_column_to_constant(table, column, bound_val);
                 },
 
+                Column::Transactions(_) => {
+                    self.constrain_column_to_constant(table, column, bound_val);
+                },
+
                 Column::Fulltext(FulltextColumn::Rowid) |
                 Column::Fulltext(FulltextColumn::Text) => {
                     // We never expose `rowid` via queries.  We do expose `text`, but only
@@ -512,9 +517,6 @@ impl ConjoiningClauses {
         // to get its type, record that we can get it from this table.
         let needs_type_extraction =
             !late_binding &&                                // Never need to extract for bound vars.
-            // Never need to extract types for refs, and var columns are handled elsewhere:
-            // a subquery will be projecting a type tag.
-            column == Column::Fixed(DatomsColumn::Value) &&
             self.known_type(&var).is_none() &&              // Don't need to extract if we know a single type.
             !self.extracted_types.contains_key(&var);       // We're already extracting the type.
 
@@ -523,8 +525,11 @@ impl ConjoiningClauses {
         // If we subsequently find out its type, we'll remove this later -- see
         // the removal in `constrain_var_to_type`.
         if needs_type_extraction {
-            self.extracted_types.insert(var.clone(), alias.for_type_tag());
+            if let Some(tag_alias) = alias.for_associated_type_tag() {
+                self.extracted_types.insert(var.clone(), tag_alias);
+            }
         }
+
         self.column_bindings.entry(var).or_insert(vec![]).push(alias);
     }
 

query-algebrizer/src/clauses/resolve.rs

@@ -137,6 +137,14 @@ impl ConjoiningClauses {
         }
     }
 
+    /// Take a transaction ID function argument and turn it into a `QueryValue` suitable for use in
+    /// a concrete constraint.
+    pub(crate) fn resolve_tx_argument(&mut self, schema: &Schema, function: &PlainSymbol, position: usize, arg: FnArg) -> Result<QueryValue> {
+        // Under the hood there's nothing special about a transaction ID -- it's just another ref.
+        // In the future, we might handle instants specially.
+        self.resolve_ref_argument(schema, function, position, arg)
+    }
+
     /// Take a function argument and turn it into a `QueryValue` suitable for use in a concrete
     /// constraint.
     #[allow(dead_code)]

query-algebrizer/src/clauses/tx_log_api.rs

@@ -0,0 +1,394 @@
+// Copyright 2018 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use mentat_core::{
+    ValueType,
+};
+
+use mentat_query::{
+    Binding,
+    FnArg,
+    SrcVar,
+    VariableOrPlaceholder,
+    WhereFn,
+};
+
+use clauses::{
+    ConjoiningClauses,
+};
+
+use errors::{
+    BindingError,
+    ErrorKind,
+    Result,
+};
+
+use types::{
+    Column,
+    ColumnConstraint,
+    DatomsTable,
+    Inequality,
+    QualifiedAlias,
+    QueryValue,
+    SourceAlias,
+    TransactionsColumn,
+};
+
+use Known;
+
+impl ConjoiningClauses {
+    // Log in Query: tx-ids and tx-data
+    //
+    // The log API includes two convenience functions that are available within query. The tx-ids
+    // function takes arguments similar to txRange above, but returns a collection of transaction
+    // entity ids. You will typically use the collection binding form [?tx …] to capture the
+    // results.
+    //
+    // [(tx-ids ?log ?tx1 ?tx2) [?tx ...]]
+    //
+    // TODO: handle tx1 == 0 (more generally, tx1 < bootstrap::TX0) specially (no after constraint).
+    // TODO: allow tx2 to be missing (no before constraint).
+    // TODO: allow txK arguments to be instants.
+    // TODO: allow arbitrary additional attribute arguments that restrict the tx-ids to those
+    // transactions that impact one of the given attributes.
+    pub(crate) fn apply_tx_ids(&mut self, known: Known, where_fn: WhereFn) -> Result<()> {
+        if where_fn.args.len() != 3 {
+            bail!(ErrorKind::InvalidNumberOfArguments(where_fn.operator.clone(), where_fn.args.len(), 3));
+        }
+
+        if where_fn.binding.is_empty() {
+            // The binding must introduce at least one bound variable.
+            bail!(ErrorKind::InvalidBinding(where_fn.operator.clone(), BindingError::NoBoundVariable));
+        }
+
+        if !where_fn.binding.is_valid() {
+            // The binding must not duplicate bound variables.
+            bail!(ErrorKind::InvalidBinding(where_fn.operator.clone(), BindingError::RepeatedBoundVariable));
+        }
+
+        // We should have exactly one binding. Destructure it now.
+        let tx_var = match where_fn.binding {
+            Binding::BindRel(bindings) => {
+                let bindings_count = bindings.len();
+                if bindings_count != 1 {
+                    bail!(ErrorKind::InvalidBinding(where_fn.operator.clone(),
+                                                    BindingError::InvalidNumberOfBindings {
+                                                        number: bindings_count,
+                                                        expected: 1,
+                                                    }));
+                }
+                match bindings.into_iter().next().unwrap() {
+                    VariableOrPlaceholder::Placeholder => unreachable!("binding.is_empty()!"),
+                    VariableOrPlaceholder::Variable(v) => v,
+                }
+            },
+            Binding::BindColl(v) => v,
+            Binding::BindScalar(_) |
+            Binding::BindTuple(_) => {
+                bail!(ErrorKind::InvalidBinding(where_fn.operator.clone(), BindingError::ExpectedBindRelOrBindColl))
+            },
+        };
+
+        let mut args = where_fn.args.into_iter();
+
+        // TODO: process source variables.
+        match args.next().unwrap() {
+            FnArg::SrcVar(SrcVar::DefaultSrc) => {},
+            _ => bail!(ErrorKind::InvalidArgument(where_fn.operator.clone(), "source variable", 0)),
+        }
+
+        let tx1 = self.resolve_tx_argument(&known.schema, &where_fn.operator, 1, args.next().unwrap())?;
+        let tx2 = self.resolve_tx_argument(&known.schema, &where_fn.operator, 2, args.next().unwrap())?;
+
+        let transactions = self.next_alias_for_table(DatomsTable::Transactions);
+
+        self.from.push(SourceAlias(DatomsTable::Transactions, transactions.clone()));
+
+        // Bound variable must be a ref.
+        self.constrain_var_to_type(tx_var.clone(), ValueType::Ref);
+        if self.is_known_empty() {
+            return Ok(());
+        }
+
+        self.bind_column_to_var(known.schema, transactions.clone(), TransactionsColumn::Tx, tx_var.clone());
+
+        let after_constraint = ColumnConstraint::Inequality {
+            operator: Inequality::LessThanOrEquals,
+            left: tx1,
+            right: QueryValue::Column(QualifiedAlias(transactions.clone(), Column::Transactions(TransactionsColumn::Tx))),
+        };
+        self.wheres.add_intersection(after_constraint);
+
+        let before_constraint = ColumnConstraint::Inequality {
+            operator: Inequality::LessThan,
+            left: QueryValue::Column(QualifiedAlias(transactions.clone(), Column::Transactions(TransactionsColumn::Tx))),
+            right: tx2,
+        };
+        self.wheres.add_intersection(before_constraint);
+
+        Ok(())
+    }
+
+    pub(crate) fn apply_tx_data(&mut self, known: Known, where_fn: WhereFn) -> Result<()> {
+        if where_fn.args.len() != 2 {
+            bail!(ErrorKind::InvalidNumberOfArguments(where_fn.operator.clone(), where_fn.args.len(), 2));
+        }
+
+        if where_fn.binding.is_empty() {
+            // The binding must introduce at least one bound variable.
+            bail!(ErrorKind::InvalidBinding(where_fn.operator.clone(), BindingError::NoBoundVariable));
+        }
+
+        if !where_fn.binding.is_valid() {
+            // The binding must not duplicate bound variables.
+            bail!(ErrorKind::InvalidBinding(where_fn.operator.clone(), BindingError::RepeatedBoundVariable));
+        }
+
+        // We should have at most five bindings. Destructure them now.
+        let bindings = match where_fn.binding {
+            Binding::BindRel(bindings) => {
+                let bindings_count = bindings.len();
+                if bindings_count < 1 || bindings_count > 5 {
+                    bail!(ErrorKind::InvalidBinding(where_fn.operator.clone(),
+                                                    BindingError::InvalidNumberOfBindings {
+                                                        number: bindings.len(),
+                                                        expected: 5,
+                                                    }));
+                }
+                bindings
+            },
+            Binding::BindScalar(_) |
+            Binding::BindTuple(_) |
+            Binding::BindColl(_) => bail!(ErrorKind::InvalidBinding(where_fn.operator.clone(), BindingError::ExpectedBindRel)),
+        };
+        let mut bindings = bindings.into_iter();
+        let b_e = bindings.next().unwrap_or(VariableOrPlaceholder::Placeholder);
+        let b_a = bindings.next().unwrap_or(VariableOrPlaceholder::Placeholder);
+        let b_v = bindings.next().unwrap_or(VariableOrPlaceholder::Placeholder);
+        let b_tx = bindings.next().unwrap_or(VariableOrPlaceholder::Placeholder);
+        let b_op = bindings.next().unwrap_or(VariableOrPlaceholder::Placeholder);
+
+        let mut args = where_fn.args.into_iter();
+
+        // TODO: process source variables.
+        match args.next().unwrap() {
+            FnArg::SrcVar(SrcVar::DefaultSrc) => {},
+            _ => bail!(ErrorKind::InvalidArgument(where_fn.operator.clone(), "source variable", 0)),
+        }
+
+        let tx = self.resolve_tx_argument(&known.schema, &where_fn.operator, 1, args.next().unwrap())?;
+
+        let transactions = self.next_alias_for_table(DatomsTable::Transactions);
+
+        self.from.push(SourceAlias(DatomsTable::Transactions, transactions.clone()));
+
+        let tx_constraint = ColumnConstraint::Equals(
+            QualifiedAlias(transactions.clone(), Column::Transactions(TransactionsColumn::Tx)),
+            tx);
+        self.wheres.add_intersection(tx_constraint);
+
+        if let VariableOrPlaceholder::Variable(ref var) = b_e {
+            // It must be a ref.
+            self.constrain_var_to_type(var.clone(), ValueType::Ref);
+            if self.is_known_empty() {
+                return Ok(());
+            }
+
+            self.bind_column_to_var(known.schema, transactions.clone(), TransactionsColumn::Entity, var.clone());
+        }
+
+        if let VariableOrPlaceholder::Variable(ref var) = b_a {
+            // It must be a ref.
+            self.constrain_var_to_type(var.clone(), ValueType::Ref);
+            if self.is_known_empty() {
+                return Ok(());
+            }
+
+            self.bind_column_to_var(known.schema, transactions.clone(), TransactionsColumn::Attribute, var.clone());
+        }
+
+        if let VariableOrPlaceholder::Variable(ref var) = b_v {
+            self.bind_column_to_var(known.schema, transactions.clone(), TransactionsColumn::Value, var.clone());
+        }
+
+        if let VariableOrPlaceholder::Variable(ref var) = b_tx {
+            // It must be a ref.
+            self.constrain_var_to_type(var.clone(), ValueType::Ref);
+            if self.is_known_empty() {
+                return Ok(());
+            }
+
+            // TODO: this might be a programming error if var is our tx argument.  Perhaps we can be
+            // helpful in that case.
+            self.bind_column_to_var(known.schema, transactions.clone(), TransactionsColumn::Tx, var.clone());
+        }
+
+        if let VariableOrPlaceholder::Variable(ref var) = b_op {
+            // It must be a boolean.
+            self.constrain_var_to_type(var.clone(), ValueType::Boolean);
+            if self.is_known_empty() {
+                return Ok(());
+            }
+
+            self.bind_column_to_var(known.schema, transactions.clone(), TransactionsColumn::Added, var.clone());
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod testing {
+    use super::*;
+
+    use mentat_core::{
+        Schema,
+        TypedValue,
+        ValueType,
+    };
+
+    use mentat_query::{
+        Binding,
+        FnArg,
+        PlainSymbol,
+        Variable,
+    };
+
+    #[test]
+    fn test_apply_tx_ids() {
+        let mut cc = ConjoiningClauses::default();
+        let schema = Schema::default();
+
+        let known = Known::for_schema(&schema);
+
+        let op = PlainSymbol::new("tx-ids");
+        cc.apply_tx_ids(known, WhereFn {
+            operator: op,
+            args: vec![
+                FnArg::SrcVar(SrcVar::DefaultSrc),
+                FnArg::EntidOrInteger(1000),
+                FnArg::EntidOrInteger(2000),
+            ],
+            binding: Binding::BindRel(vec![VariableOrPlaceholder::Variable(Variable::from_valid_name("?tx")),
+            ]),
+        }).expect("to be able to apply_tx_ids");
+
+        assert!(!cc.is_known_empty());
+
+        // Finally, expand column bindings.
+        cc.expand_column_bindings();
+        assert!(!cc.is_known_empty());
+
+        let clauses = cc.wheres;
+        assert_eq!(clauses.len(), 2);
+
+        assert_eq!(clauses.0[0],
+                   ColumnConstraint::Inequality {
+                       operator: Inequality::LessThanOrEquals,
+                       left: QueryValue::TypedValue(TypedValue::Ref(1000)),
+                       right: QueryValue::Column(QualifiedAlias("transactions00".to_string(), Column::Transactions(TransactionsColumn::Tx))),
+                   }.into());
+
+        assert_eq!(clauses.0[1],
+                   ColumnConstraint::Inequality {
+                       operator: Inequality::LessThan,
+                       left: QueryValue::Column(QualifiedAlias("transactions00".to_string(), Column::Transactions(TransactionsColumn::Tx))),
+                       right: QueryValue::TypedValue(TypedValue::Ref(2000)),
+                   }.into());
+
+        let bindings = cc.column_bindings;
+        assert_eq!(bindings.len(), 1);
+
+        assert_eq!(bindings.get(&Variable::from_valid_name("?tx")).expect("column binding for ?tx").clone(),
+                   vec![QualifiedAlias("transactions00".to_string(), Column::Transactions(TransactionsColumn::Tx))]);
+
+        let known_types = cc.known_types;
+        assert_eq!(known_types.len(), 1);
+
+        assert_eq!(known_types.get(&Variable::from_valid_name("?tx")).expect("known types for ?tx").clone(),
+                   vec![ValueType::Ref].into_iter().collect());
+    }
+
+    #[test]
+    fn test_apply_tx_data() {
+        let mut cc = ConjoiningClauses::default();
+        let schema = Schema::default();
+
+        let known = Known::for_schema(&schema);
+
+        let op = PlainSymbol::new("tx-data");
+        cc.apply_tx_data(known, WhereFn {
+            operator: op,
+            args: vec![
+                FnArg::SrcVar(SrcVar::DefaultSrc),
+                FnArg::EntidOrInteger(1000),
+            ],
+            binding: Binding::BindRel(vec![
+                VariableOrPlaceholder::Variable(Variable::from_valid_name("?e")),
+                VariableOrPlaceholder::Variable(Variable::from_valid_name("?a")),
+                VariableOrPlaceholder::Variable(Variable::from_valid_name("?v")),
+                VariableOrPlaceholder::Variable(Variable::from_valid_name("?tx")),
+                VariableOrPlaceholder::Variable(Variable::from_valid_name("?added")),
+            ]),
+        }).expect("to be able to apply_tx_data");
+
+        assert!(!cc.is_known_empty());
+
+        // Finally, expand column bindings.
+        cc.expand_column_bindings();
+        assert!(!cc.is_known_empty());
+
+        let clauses = cc.wheres;
+        assert_eq!(clauses.len(), 1);
+
+        assert_eq!(clauses.0[0],
+                   ColumnConstraint::Equals(QualifiedAlias("transactions00".to_string(), Column::Transactions(TransactionsColumn::Tx)),
+                                            QueryValue::TypedValue(TypedValue::Ref(1000))).into());
+
+        let bindings = cc.column_bindings;
+        assert_eq!(bindings.len(), 5);
+
+        assert_eq!(bindings.get(&Variable::from_valid_name("?e")).expect("column binding for ?e").clone(),
+                   vec![QualifiedAlias("transactions00".to_string(), Column::Transactions(TransactionsColumn::Entity))]);
+
+        assert_eq!(bindings.get(&Variable::from_valid_name("?a")).expect("column binding for ?a").clone(),
+                   vec![QualifiedAlias("transactions00".to_string(), Column::Transactions(TransactionsColumn::Attribute))]);
+
+        assert_eq!(bindings.get(&Variable::from_valid_name("?v")).expect("column binding for ?v").clone(),
+                   vec![QualifiedAlias("transactions00".to_string(), Column::Transactions(TransactionsColumn::Value))]);
+
+        assert_eq!(bindings.get(&Variable::from_valid_name("?tx")).expect("column binding for ?tx").clone(),
+                   vec![QualifiedAlias("transactions00".to_string(), Column::Transactions(TransactionsColumn::Tx))]);
+
+        assert_eq!(bindings.get(&Variable::from_valid_name("?added")).expect("column binding for ?added").clone(),
+                   vec![QualifiedAlias("transactions00".to_string(), Column::Transactions(TransactionsColumn::Added))]);
+
+        let known_types = cc.known_types;
+        assert_eq!(known_types.len(), 4);
+
+        assert_eq!(known_types.get(&Variable::from_valid_name("?e")).expect("known types for ?e").clone(),
+                   vec![ValueType::Ref].into_iter().collect());
+
+        assert_eq!(known_types.get(&Variable::from_valid_name("?a")).expect("known types for ?a").clone(),
+                   vec![ValueType::Ref].into_iter().collect());
+
+        assert_eq!(known_types.get(&Variable::from_valid_name("?tx")).expect("known types for ?tx").clone(),
+                   vec![ValueType::Ref].into_iter().collect());
+
+        assert_eq!(known_types.get(&Variable::from_valid_name("?added")).expect("known types for ?added").clone(),
+                   vec![ValueType::Boolean].into_iter().collect());
+
+        let extracted_types = cc.extracted_types;
+        assert_eq!(extracted_types.len(), 1);
+
+        assert_eq!(extracted_types.get(&Variable::from_valid_name("?v")).expect("extracted types for ?v").clone(),
+                   QualifiedAlias("transactions00".to_string(), Column::Transactions(TransactionsColumn::ValueTypeTag)));
+    }
+}

query-algebrizer/src/clauses/where_fn.rs

@@ -37,6 +37,8 @@ impl ConjoiningClauses {
         match where_fn.operator.0.as_str() {
             "fulltext" => self.apply_fulltext(known, where_fn),
             "ground" => self.apply_ground(known, where_fn),
+            "tx-data" => self.apply_tx_data(known, where_fn),
+            "tx-ids" => self.apply_tx_ids(known, where_fn),
             _ => bail!(ErrorKind::UnknownFunction(where_fn.operator.clone())),
         }
     }

query-algebrizer/src/errors.rs

@@ -29,6 +29,11 @@ pub enum BindingError {
     /// than Datomic: we won't try to make sense of non-obvious (and potentially erroneous) bindings.
     ExpectedBindRel,
 
+    /// Expected `[[?x ?y]]` or `[?x ...]` but got some other type of binding.  Mentat is
+    /// deliberately more strict than Datomic: we won't try to make sense of non-obvious (and
+    /// potentially erroneous) bindings.
+    ExpectedBindRelOrBindColl,
+
     /// Expected `[?x1 … ?xN]` or `[[?x1 … ?xN]]` but got some other number of bindings.  Mentat is
     /// deliberately more strict than Datomic: we prefer placeholders to omission.
     InvalidNumberOfBindings { number: usize, expected: usize },

query-algebrizer/src/types.rs

@@ -41,6 +41,7 @@ pub enum DatomsTable {
     FulltextDatoms,     // The fulltext-datoms view.
     AllDatoms,          // Fulltext and non-fulltext datoms.
     Computed(usize),    // A computed table, tracked elsewhere in the query.
+    Transactions,       // The transactions table, which makes the tx-data log API efficient.
 }
 
 /// A source of rows that isn't a named table -- typically a subquery or union.
@@ -66,6 +67,7 @@ impl DatomsTable {
             DatomsTable::FulltextDatoms => "fulltext_datoms",
             DatomsTable::AllDatoms => "all_datoms",
             DatomsTable::Computed(_) => "c",
+            DatomsTable::Transactions => "transactions",
         }
     }
 }
@@ -91,6 +93,17 @@ pub enum FulltextColumn {
     Text,
 }
 
+/// One of the named columns of our transactions table.
+#[derive(PartialEq, Eq, Clone)]
+pub enum TransactionsColumn {
+    Entity,
+    Attribute,
+    Value,
+    Tx,
+    Added,
+    ValueTypeTag,
+}
+
 #[derive(PartialEq, Eq, Clone)]
 pub enum VariableColumn {
     Variable(Variable),
@@ -102,6 +115,7 @@ pub enum Column {
     Fixed(DatomsColumn),
     Fulltext(FulltextColumn),
     Variable(VariableColumn),
+    Transactions(TransactionsColumn),
 }
 
 impl From<DatomsColumn> for Column {
@@ -116,6 +130,12 @@ impl From<VariableColumn> for Column {
     }
 }
 
+impl From<TransactionsColumn> for Column {
+    fn from(from: TransactionsColumn) -> Column {
+        Column::Transactions(from)
+    }
+}
+
 impl DatomsColumn {
     pub fn as_str(&self) -> &'static str {
         use self::DatomsColumn::*;
@@ -127,6 +147,16 @@ impl DatomsColumn {
             ValueTypeTag => "value_type_tag",
         }
     }
+
+    /// The type of the `v` column is determined by the `value_type_tag` column.  Return the
+    /// associated column determining the type of this column, if there is one.
+    pub fn associated_type_tag_column(&self) -> Option<DatomsColumn> {
+        use self::DatomsColumn::*;
+        match *self {
+            Value => Some(ValueTypeTag),
+            _ => None,
+        }
+    }
 }
 
 impl ColumnName for DatomsColumn {
@@ -166,6 +196,7 @@ impl Debug for Column {
             &Column::Fixed(ref c) => c.fmt(f),
             &Column::Fulltext(ref c) => c.fmt(f),
             &Column::Variable(ref v) => v.fmt(f),
+            &Column::Transactions(ref t) => t.fmt(f),
         }
     }
 }
@@ -192,6 +223,40 @@ impl Debug for FulltextColumn {
     }
 }
 
+impl TransactionsColumn {
+    pub fn as_str(&self) -> &'static str {
+        use self::TransactionsColumn::*;
+        match *self {
+            Entity => "e",
+            Attribute => "a",
+            Value => "v",
+            Tx => "tx",
+            Added => "added",
+            ValueTypeTag => "value_type_tag",
+        }
+    }
+
+    pub fn associated_type_tag_column(&self) -> Option<TransactionsColumn> {
+        use self::TransactionsColumn::*;
+        match *self {
+            Value => Some(ValueTypeTag),
+            _ => None,
+        }
+    }
+}
+
+impl ColumnName for TransactionsColumn {
+    fn column_name(&self) -> String {
+        self.as_str().to_string()
+    }
+}
+
+impl Debug for TransactionsColumn {
+    fn fmt(&self, f: &mut Formatter) -> Result {
+        write!(f, "{}", self.as_str())
+    }
+}
+
 /// A specific instance of a table within a query. E.g., "datoms123".
 pub type TableAlias = String;
 
@@ -220,9 +285,13 @@ impl QualifiedAlias {
         QualifiedAlias(table, column.into())
     }
 
-    pub fn for_type_tag(&self) -> QualifiedAlias {
+    pub fn for_associated_type_tag(&self) -> Option<QualifiedAlias> {
-        // TODO: this only makes sense for `DatomsColumn` tables.
+        match self.1 {
-        QualifiedAlias(self.0.clone(), Column::Fixed(DatomsColumn::ValueTypeTag))
+            Column::Fixed(ref c) => c.associated_type_tag_column().map(Column::Fixed),
+            Column::Fulltext(_) => None,
+            Column::Variable(_) => None,
+            Column::Transactions(ref c) => c.associated_type_tag_column().map(Column::Transactions),
+        }.map(|d| QualifiedAlias(self.0.clone(), d))
     }
 }
 

query-sql/src/lib.rs

@@ -243,6 +243,10 @@ fn push_column(qb: &mut QueryBuilder, col: &Column) -> BuildQueryResult {
             Ok(())
         },
         &Column::Variable(ref vc) => push_variable_column(qb, vc),
+        &Column::Transactions(ref d) => {
+            qb.push_sql(d.as_str());
+            Ok(())
+        },
     }
 }
 

query-translator/src/translate.rs

@@ -170,7 +170,7 @@ impl ToConstraint for ColumnConstraint {
                 Constraint::equal(left.to_column(), right.to_column()),
 
             Equals(qa, QueryValue::PrimitiveLong(value)) => {
-                let tag_column = qa.for_type_tag().to_column();
+                let tag_column = qa.for_associated_type_tag().expect("an associated type tag alias").to_column();
                 let value_column = qa.to_column();
 
                 // A bare long in a query might match a ref, an instant, a long (obviously), or a

query-translator/tests/translate.rs

@@ -1134,3 +1134,103 @@ fn test_tx_before_and_after() {
                      WHERE `datoms00`.tx < 12345");
     assert_eq!(args, vec![]);
 }
+
+
+#[test]
+fn test_tx_ids() {
+    let mut schema = prepopulated_typed_schema(ValueType::Double);
+    associate_ident(&mut schema, NamespacedKeyword::new("db", "txInstant"), 101);
+    add_attribute(&mut schema, 101, Attribute {
+        value_type: ValueType::Instant,
+        multival: false,
+        index: true,
+        ..Default::default()
+    });
+
+    let query = r#"[:find ?tx :where [(tx-ids $ 1000 2000) [[?tx]]]]"#;
+    let SQLQuery { sql, args } = translate(&schema, query);
+    assert_eq!(sql, "SELECT DISTINCT `transactions00`.tx AS `?tx` \
+                     FROM `transactions` AS `transactions00` \
+                     WHERE 1000 <= `transactions00`.tx \
+                     AND `transactions00`.tx < 2000");
+    assert_eq!(args, vec![]);
+
+    // This is rather artificial but verifies that binding the arguments to (tx-ids) works.
+    let query = r#"[:find ?tx :where [?first :db/txInstant #inst "2016-01-01T11:00:00.000Z"] [?last :db/txInstant #inst "2017-01-01T11:00:00.000Z"] [(tx-ids $ ?first ?last) [?tx ...]]]"#;
+    let SQLQuery { sql, args } = translate(&schema, query);
+    assert_eq!(sql, "SELECT DISTINCT `transactions02`.tx AS `?tx` \
+                     FROM `datoms` AS `datoms00`, \
+                     `datoms` AS `datoms01`, \
+                     `transactions` AS `transactions02` \
+                     WHERE `datoms00`.a = 101 \
+                     AND `datoms00`.v = 1451646000000000 \
+                     AND `datoms01`.a = 101 \
+                     AND `datoms01`.v = 1483268400000000 \
+                     AND `datoms00`.e <= `transactions02`.tx \
+                     AND `transactions02`.tx < `datoms01`.e");
+    assert_eq!(args, vec![]);
+
+    // In practice the following query would be inefficient because of the filter on all_datoms.tx,
+    // but that is what (tx-data) is for.
+    let query = r#"[:find ?e ?a ?v ?tx :where [(tx-ids $ 1000 2000) [[?tx]]] [?e ?a ?v ?tx]]"#;
+    let SQLQuery { sql, args } = translate(&schema, query);
+    assert_eq!(sql, "SELECT DISTINCT `all_datoms01`.e AS `?e`, \
+	                   `all_datoms01`.a AS `?a`, \
+                     `all_datoms01`.v AS `?v`, \
+                     `all_datoms01`.value_type_tag AS `?v_value_type_tag`, \
+                     `transactions00`.tx AS `?tx` \
+                     FROM `transactions` AS `transactions00`, \
+                     `all_datoms` AS `all_datoms01` \
+                     WHERE 1000 <= `transactions00`.tx \
+                     AND `transactions00`.tx < 2000 \
+                     AND `transactions00`.tx = `all_datoms01`.tx");
+    assert_eq!(args, vec![]);
+}
+
+#[test]
+fn test_tx_data() {
+    let schema = prepopulated_typed_schema(ValueType::Double);
+
+    let query = r#"[:find ?e ?a ?v ?tx ?added :where [(tx-data $ 1000) [[?e ?a ?v ?tx ?added]]]]"#;
+    let SQLQuery { sql, args } = translate(&schema, query);
+    assert_eq!(sql, "SELECT DISTINCT `transactions00`.e AS `?e`, \
+                     `transactions00`.a AS `?a`, \
+                     `transactions00`.v AS `?v`, \
+                     `transactions00`.value_type_tag AS `?v_value_type_tag`, \
+                     `transactions00`.tx AS `?tx`, \
+                     `transactions00`.added AS `?added` \
+                     FROM `transactions` AS `transactions00` \
+                     WHERE `transactions00`.tx = 1000");
+    assert_eq!(args, vec![]);
+
+    // Ensure that we don't project columns that we don't need, even if they are bound to named
+    // variables or to placeholders.
+    let query = r#"[:find [?a ?v ?added] :where [(tx-data $ 1000) [[?e ?a ?v _ ?added]]]]"#;
+    let SQLQuery { sql, args } = translate(&schema, query);
+    assert_eq!(sql, "SELECT `transactions00`.a AS `?a`, \
+                     `transactions00`.v AS `?v`, \
+                     `transactions00`.value_type_tag AS `?v_value_type_tag`, \
+                     `transactions00`.added AS `?added` \
+                     FROM `transactions` AS `transactions00` \
+                     WHERE `transactions00`.tx = 1000 \
+                     LIMIT 1");
+    assert_eq!(args, vec![]);
+
+    // This is awkward since the transactions table is queried twice, once to list transaction IDs
+    // and a second time to extract data.  https://github.com/mozilla/mentat/issues/644 tracks
+    // improving this, perhaps by optimizing certain combinations of functions and bindings.
+    let query = r#"[:find ?e ?a ?v ?tx ?added :where [(tx-ids $ 1000 2000) [[?tx]]] [(tx-data $ ?tx) [[?e ?a ?v _ ?added]]]]"#;
+    let SQLQuery { sql, args } = translate(&schema, query);
+    assert_eq!(sql, "SELECT DISTINCT `transactions01`.e AS `?e`, \
+                     `transactions01`.a AS `?a`, \
+                     `transactions01`.v AS `?v`, \
+                     `transactions01`.value_type_tag AS `?v_value_type_tag`, \
+                     `transactions00`.tx AS `?tx`, \
+                     `transactions01`.added AS `?added` \
+                     FROM `transactions` AS `transactions00`, \
+                     `transactions` AS `transactions01` \
+                     WHERE 1000 <= `transactions00`.tx \
+                     AND `transactions00`.tx < 2000 \
+                     AND `transactions01`.tx = `transactions00`.tx");
+    assert_eq!(args, vec![]);
+}

tests/query.rs

@@ -27,6 +27,7 @@ use chrono::FixedOffset;
 
 use mentat_core::{
     DateTime,
+    Entid,
     HasSchema,
     KnownEntid,
     TypedValue,
@@ -48,6 +49,7 @@ use mentat::{
     Queryable,
     QueryResults,
     Store,
+    TxReport,
     Variable,
     new_connection,
 };
@@ -1339,3 +1341,115 @@ fn test_aggregation_implicit_grouping() {
         x => panic!("Got unexpected results {:?}", x),
     }
 }
+
+#[test]
+fn test_tx_ids() {
+    let mut store = Store::open("").expect("opened");
+
+    store.transact(r#"[
+        [:db/add "a" :db/ident :foo/term]
+        [:db/add "a" :db/valueType :db.type/string]
+        [:db/add "a" :db/fulltext false]
+        [:db/add "a" :db/cardinality :db.cardinality/many]
+    ]"#).unwrap();
+
+    let tx1 = store.transact(r#"[
+        [:db/add "v" :foo/term "1"]
+    ]"#).expect("tx1 to apply").tx_id;
+
+    let tx2 = store.transact(r#"[
+        [:db/add "v" :foo/term "2"]
+    ]"#).expect("tx2 to apply").tx_id;
+
+    let tx3 = store.transact(r#"[
+        [:db/add "v" :foo/term "3"]
+    ]"#).expect("tx3 to apply").tx_id;
+
+    fn assert_tx_id_range(store: &Store, after: Entid, before: Entid, expected: Vec<TypedValue>) {
+        // TODO: after https://github.com/mozilla/mentat/issues/641, use q_prepare with inputs bound
+        // at execution time.
+        let r = store.q_once(r#"[:find [?tx ...]
+                                 :in ?after ?before
+                                 :where
+                                 [(tx-ids $ ?after ?before) [?tx ...]]
+                                ]"#,
+                             QueryInputs::with_value_sequence(vec![
+                                 (Variable::from_valid_name("?after"),  TypedValue::Ref(after)),
+                                 (Variable::from_valid_name("?before"), TypedValue::Ref(before)),
+                             ]))
+            .expect("results")
+            .into();
+        match r {
+            QueryResults::Coll(txs) => {
+                assert_eq!(txs, expected);
+            },
+            x => panic!("Got unexpected results {:?}", x),
+        }
+    }
+
+    assert_tx_id_range(&store, tx1, tx2, vec![TypedValue::Ref(tx1)]);
+    assert_tx_id_range(&store, tx1, tx3, vec![TypedValue::Ref(tx1), TypedValue::Ref(tx2)]);
+    assert_tx_id_range(&store, tx2, tx3, vec![TypedValue::Ref(tx2)]);
+    assert_tx_id_range(&store, tx2, tx3 + 1, vec![TypedValue::Ref(tx2), TypedValue::Ref(tx3)]);
+}
+
+#[test]
+fn test_tx_data() {
+    let mut store = Store::open("").expect("opened");
+
+    store.transact(r#"[
+        [:db/add "a" :db/ident :foo/term]
+        [:db/add "a" :db/valueType :db.type/string]
+        [:db/add "a" :db/fulltext false]
+        [:db/add "a" :db/cardinality :db.cardinality/many]
+    ]"#).unwrap();
+
+    let tx1 = store.transact(r#"[
+        [:db/add "e" :foo/term "1"]
+    ]"#).expect("tx1 to apply");
+
+    let tx2 = store.transact(r#"[
+        [:db/add "e" :foo/term "2"]
+    ]"#).expect("tx2 to apply");
+
+    fn assert_tx_data(store: &Store, tx: &TxReport, value: TypedValue) {
+        // TODO: after https://github.com/mozilla/mentat/issues/641, use q_prepare with inputs bound
+        // at execution time.
+        let r = store.q_once(r#"[:find ?e ?a-name ?v ?tx ?added
+                                 :in ?tx-in
+                                 :where
+                                 [(tx-data $ ?tx-in) [[?e ?a ?v ?tx ?added]]]
+                                 [?a :db/ident ?a-name]
+                                 :order ?e
+                                ]"#,
+                             QueryInputs::with_value_sequence(vec![
+                                 (Variable::from_valid_name("?tx-in"),  TypedValue::Ref(tx.tx_id)),
+                             ]))
+            .expect("results")
+            .into();
+
+        let e = tx.tempids.get("e").cloned().expect("tempid");
+
+        match r {
+            QueryResults::Rel(vals) => {
+                assert_eq!(vals,
+                           vec![
+                               vec![TypedValue::Ref(e),
+                                    TypedValue::typed_ns_keyword("foo", "term"),
+                                    value,
+                                    TypedValue::Ref(tx.tx_id),
+                                    TypedValue::Boolean(true)],
+                               vec![TypedValue::Ref(tx.tx_id),
+                                    TypedValue::typed_ns_keyword("db", "txInstant"),
+                                    TypedValue::Instant(tx.tx_instant),
+                                    TypedValue::Ref(tx.tx_id),
+                                    TypedValue::Boolean(true)],
+                           ]);
+            },
+            x => panic!("Got unexpected results {:?}", x),
+        }
+    };
+
+    assert_tx_data(&store, &tx1, TypedValue::String("1".to_string().into()));
+    assert_tx_data(&store, &tx2, TypedValue::String("2".to_string().into()));
+}