From 2b53abbd851070687cb69d71e3fc2237070a900e Mon Sep 17 00:00:00 2001 From: Richard Newman Date: Fri, 28 Apr 2017 10:31:31 -0700 Subject: [PATCH] Part 1: parse #inst in EDN and throughout query engine. --- core/src/lib.rs | 13 +++++ edn/Cargo.toml | 1 + edn/src/edn.rustpeg | 40 ++++++++++++++- edn/src/lib.rs | 11 +++- edn/src/types.rs | 68 +++++++++++++++++++++---- edn/tests/tests.rs | 18 ++++++- query-algebrizer/src/clauses/resolve.rs | 2 + query/src/lib.rs | 6 +++ sql/src/lib.rs | 8 ++- 9 files changed, 153 insertions(+), 14 deletions(-) diff --git a/core/src/lib.rs b/core/src/lib.rs index dffa4717..65b6b2ba 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -33,6 +33,13 @@ pub use edn::{ Uuid, }; +pub use edn::{ + DateTime, + FromMicros, + ToMicros, + UTC, +}; + /// Core types defining a Mentat knowledge base. /// Represents one entid in the entid space. @@ -123,6 +130,7 @@ pub enum TypedValue { Boolean(bool), Long(i64), Double(OrderedFloat), + Instant(DateTime), // TODO: &str throughout? String(Rc), Keyword(Rc), @@ -147,6 +155,7 @@ impl TypedValue { &TypedValue::Ref(_) => ValueType::Ref, &TypedValue::Boolean(_) => ValueType::Boolean, &TypedValue::Long(_) => ValueType::Long, + &TypedValue::Instant(_) => ValueType::Instant, &TypedValue::Double(_) => ValueType::Double, &TypedValue::String(_) => ValueType::String, &TypedValue::Keyword(_) => ValueType::Keyword, @@ -167,6 +176,10 @@ impl TypedValue { pub fn typed_string(s: &str) -> TypedValue { TypedValue::String(Rc::new(s.to_string())) } + + pub fn current_instant() -> TypedValue { + TypedValue::Instant(UTC::now()) + } } // Put this here rather than in `db` simply because it's widely needed. diff --git a/edn/Cargo.toml b/edn/Cargo.toml index 8540fc96..6f0c7f62 100644 --- a/edn/Cargo.toml +++ b/edn/Cargo.toml @@ -11,6 +11,7 @@ build = "build.rs" readme = "./README.md" [dependencies] +chrono = "0.3" itertools = "0.5.9" num = "0.1.35" ordered-float = "0.4.0" diff --git a/edn/src/edn.rustpeg b/edn/src/edn.rustpeg index ddc10bea..21b6fa9d 100644 --- a/edn/src/edn.rustpeg +++ b/edn/src/edn.rustpeg @@ -14,6 +14,11 @@ use std::collections::{BTreeSet, BTreeMap, LinkedList}; use std::iter::FromIterator; use std::f64::{NAN, INFINITY, NEG_INFINITY}; +use chrono::{ + DateTime, + TimeZone, + UTC +}; use num::BigInt; use ordered_float::OrderedFloat; use uuid::Uuid; @@ -143,6 +148,39 @@ pub text -> ValueAndSpan = } } + +// RFC 3339 timestamps. #inst "1985-04-12T23:20:50.52Z" +// We accept an arbitrary depth of decimals. +// Note that we discard the timezone information -- all times are translated to UTC. +pub inst_string -> DateTime = + "\"" d:$( [0-9]*<4> "-" [0-2][0-9] "-" [0-3][0-9] + "T" + [0-2][0-9] ":" [0-5][0-9] ":" [0-6][0-9] + ("." [0-9]+)? + "Z" / (("+" / "-") [0-2][0-9] ":" [0-5][0-9]) + ) + "\"" {? + DateTime::parse_from_rfc3339(d) + .map(|t| t.with_timezone(&UTC)) + .map_err(|_| "invalid datetime") // Oh, rustpeg. + } + +pub inst_millis -> DateTime = + d:$( digit+ ) { + let millis = d.parse::().unwrap(); + let seconds: i64 = millis / 1000; + let nanos: u32 = ((millis % 1000).abs() as u32) * 1000000; + UTC.timestamp(seconds, nanos) + } + +pub inst -> ValueAndSpan = + start:#position "#inst" whitespace+ t:(inst_millis / inst_string) end:#position { + ValueAndSpan { + inner: SpannedValue::Instant(t), + span: Span::new(start, end) + } + } + pub uuid_string -> Uuid = "\"" u:$( [a-f0-9]*<8> "-" [a-f0-9]*<4> "-" [a-f0-9]*<4> "-" [a-f0-9]*<4> "-" [a-f0-9]*<12> ) "\"" { Uuid::parse_str(u).expect("this is a valid UUID string") @@ -234,7 +272,7 @@ pub map -> ValueAndSpan = // It's important that float comes before integer or the parser assumes that // floats are integers and fails to parse pub value -> ValueAndSpan = - __ v:(nil / nan / infinity / boolean / float / octalinteger / hexinteger / basedinteger / uuid / bigint / integer / text / keyword / symbol / list / vector / map / set) __ { + __ v:(nil / nan / infinity / boolean / float / octalinteger / hexinteger / basedinteger / inst / uuid / bigint / integer / text / keyword / symbol / list / vector / map / set) __ { v } diff --git a/edn/src/lib.rs b/edn/src/lib.rs index 32aa4ad7..dce9a9c6 100644 --- a/edn/src/lib.rs +++ b/edn/src/lib.rs @@ -8,6 +8,7 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. +extern crate chrono; extern crate itertools; extern crate num; extern crate ordered_float; @@ -25,11 +26,19 @@ pub mod parse { } // Re-export the types we use. +pub use chrono::{DateTime, UTC}; pub use num::BigInt; pub use ordered_float::OrderedFloat; pub use uuid::Uuid; // Export from our modules. pub use parse::ParseError; -pub use types::{Span, SpannedValue, Value, ValueAndSpan}; +pub use types::{ + FromMicros, + Span, + SpannedValue, + ToMicros, + Value, + ValueAndSpan, +}; pub use symbols::{Keyword, NamespacedKeyword, PlainSymbol, NamespacedSymbol}; diff --git a/edn/src/types.rs b/edn/src/types.rs index 81eb797c..241664ec 100644 --- a/edn/src/types.rs +++ b/edn/src/types.rs @@ -15,6 +15,11 @@ use std::cmp::{Ordering, Ord, PartialOrd}; use std::fmt::{Display, Formatter}; use std::f64; +use chrono::{ + DateTime, + TimeZone, // For UTC::timestamp. The compiler incorrectly complains that this is unused. + UTC, +}; use num::BigInt; use ordered_float::OrderedFloat; use uuid::Uuid; @@ -27,6 +32,7 @@ pub enum Value { Nil, Boolean(bool), Integer(i64), + Instant(DateTime), BigInteger(BigInt), Float(OrderedFloat), Text(String), @@ -55,6 +61,7 @@ pub enum SpannedValue { Nil, Boolean(bool), Integer(i64), + Instant(DateTime), BigInteger(BigInt), Float(OrderedFloat), Text(String), @@ -127,6 +134,7 @@ impl From for Value { SpannedValue::Nil => Value::Nil, SpannedValue::Boolean(v) => Value::Boolean(v), SpannedValue::Integer(v) => Value::Integer(v), + SpannedValue::Instant(v) => Value::Instant(v), SpannedValue::BigInteger(v) => Value::BigInteger(v), SpannedValue::Float(v) => Value::Float(v), SpannedValue::Text(v) => Value::Text(v), @@ -273,6 +281,7 @@ macro_rules! def_common_value_methods { def_is!(is_nil, $t::Nil); def_is!(is_boolean, $t::Boolean(_)); def_is!(is_integer, $t::Integer(_)); + def_is!(is_instant, $t::Instant(_)); def_is!(is_big_integer, $t::BigInteger(_)); def_is!(is_float, $t::Float(_)); def_is!(is_text, $t::Text(_)); @@ -294,6 +303,7 @@ macro_rules! def_common_value_methods { def_as!(as_boolean, $t::Boolean, bool,); def_as!(as_integer, $t::Integer, i64,); + def_as!(as_instant, $t::Instant, DateTime,); def_as!(as_float, $t::Float, f64, |v: OrderedFloat| v.into_inner()); def_as_ref!(as_big_integer, $t::BigInteger, BigInt); @@ -311,6 +321,7 @@ macro_rules! def_common_value_methods { def_into!(into_boolean, $t::Boolean, bool,); def_into!(into_integer, $t::Integer, i64,); + def_into!(into_instant, $t::Instant, DateTime,); def_into!(into_big_integer, $t::BigInteger, BigInt,); def_into!(into_ordered_float, $t::Float, OrderedFloat,); def_into!(into_float, $t::Float, f64, |v: OrderedFloat| v.into_inner()); @@ -344,16 +355,17 @@ macro_rules! def_common_value_methods { $t::Integer(_) => 2, $t::BigInteger(_) => 3, $t::Float(_) => 4, - $t::Text(_) => 5, - $t::Uuid(_) => 6, - $t::PlainSymbol(_) => 7, - $t::NamespacedSymbol(_) => 8, - $t::Keyword(_) => 9, - $t::NamespacedKeyword(_) => 10, - $t::Vector(_) => 11, - $t::List(_) => 12, - $t::Set(_) => 13, - $t::Map(_) => 14, + $t::Instant(_) => 5, + $t::Text(_) => 6, + $t::Uuid(_) => 7, + $t::PlainSymbol(_) => 8, + $t::NamespacedSymbol(_) => 9, + $t::Keyword(_) => 10, + $t::NamespacedKeyword(_) => 11, + $t::Vector(_) => 12, + $t::List(_) => 13, + $t::Set(_) => 14, + $t::Map(_) => 15, } } @@ -362,6 +374,7 @@ macro_rules! def_common_value_methods { $t::Nil => false, $t::Boolean(_) => false, $t::Integer(_) => false, + $t::Instant(_) => false, $t::BigInteger(_) => false, $t::Float(_) => false, $t::Text(_) => false, @@ -399,6 +412,7 @@ macro_rules! def_common_value_ord { (&$t::Nil, &$t::Nil) => Ordering::Equal, (&$t::Boolean(a), &$t::Boolean(b)) => b.cmp(&a), (&$t::Integer(a), &$t::Integer(b)) => b.cmp(&a), + (&$t::Instant(a), &$t::Instant(b)) => b.cmp(&a), (&$t::BigInteger(ref a), &$t::BigInteger(ref b)) => b.cmp(a), (&$t::Float(ref a), &$t::Float(ref b)) => b.cmp(a), (&$t::Text(ref a), &$t::Text(ref b)) => b.cmp(a), @@ -425,6 +439,7 @@ macro_rules! def_common_value_display { $t::Nil => write!($f, "nil"), $t::Boolean(v) => write!($f, "{}", v), $t::Integer(v) => write!($f, "{}", v), + $t::Instant(v) => write!($f, "{}", v), $t::BigInteger(ref v) => write!($f, "{}N", v), // TODO: make sure float syntax is correct. $t::Float(ref v) => { @@ -530,8 +545,29 @@ impl Display for ValueAndSpan { } } +pub trait FromMicros { + fn from_micros(ts: i64) -> Self; +} + +impl FromMicros for DateTime { + fn from_micros(ts: i64) -> Self { + UTC.timestamp(ts / 100_000, ((ts % 100_000).abs() as u32) * 1_000) + } +} + +pub trait ToMicros { + fn to_micros(&self) -> i64; +} + +impl ToMicros for DateTime { + fn to_micros(&self) -> i64 { + (self.timestamp() * 100_000) + (self.timestamp_subsec_micros() as i64) + } +} + #[cfg(test)] mod test { + extern crate chrono; extern crate ordered_float; extern crate num; @@ -544,9 +580,21 @@ mod test { use parse; + use chrono::{ + DateTime, + TimeZone, + UTC, + }; use num::BigInt; use ordered_float::OrderedFloat; + #[test] + fn test_micros_roundtrip() { + let ts_micros: i64 = 1493399581314000; + let dt = DateTime::::from_micros(ts_micros); + assert_eq!(dt.to_micros(), ts_micros); + } + #[test] fn test_value_from() { assert_eq!(Value::from_float(42f64), Value::Float(OrderedFloat::from(42f64))); diff --git a/edn/tests/tests.rs b/edn/tests/tests.rs index c47e57f3..5dcc9f51 100644 --- a/edn/tests/tests.rs +++ b/edn/tests/tests.rs @@ -8,6 +8,7 @@ // CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. +extern crate chrono; extern crate edn; extern crate num; extern crate ordered_float; @@ -22,7 +23,17 @@ use num::traits::{Zero, One}; use ordered_float::OrderedFloat; use edn::parse::{self, ParseError}; -use edn::types::{Value, SpannedValue, Span, ValueAndSpan}; +use edn::types::{ + Value, + ValueAndSpan, + Span, + SpannedValue, +}; +use uuid::Uuid; +use chrono::{ + TimeZone, + UTC, +}; use edn::symbols; use edn::utils; @@ -412,6 +423,11 @@ fn test_value() { assert_eq!(value("(1)").unwrap(), List(LinkedList::from_iter(vec![Integer(1)]))); assert_eq!(value("#{1}").unwrap(), Set(BTreeSet::from_iter(vec![Integer(1)]))); assert_eq!(value("{1 2}").unwrap(), Map(BTreeMap::from_iter(vec![(Integer(1), Integer(2))]))); + assert_eq!(value("#uuid \"e43c6f3e-3123-49b7-8098-9b47a7bc0fa4\"").unwrap(), + Uuid(uuid::Uuid::parse_str("e43c6f3e-3123-49b7-8098-9b47a7bc0fa4").unwrap())); + assert_eq!(value("#inst 1493410985187").unwrap(), Instant(UTC.timestamp(1493410985, 187000000))); + assert_eq!(value("#inst \"2017-04-28T20:23:05.187Z\"").unwrap(), + Instant(UTC.timestamp(1493410985, 187000000))); } #[test] diff --git a/query-algebrizer/src/clauses/resolve.rs b/query-algebrizer/src/clauses/resolve.rs index 6231bd23..9cf27807 100644 --- a/query-algebrizer/src/clauses/resolve.rs +++ b/query-algebrizer/src/clauses/resolve.rs @@ -55,6 +55,7 @@ impl ConjoiningClauses { Constant(NonIntegerConstant::Boolean(_)) | Constant(NonIntegerConstant::Text(_)) | Constant(NonIntegerConstant::Uuid(_)) | + Constant(NonIntegerConstant::Instant(_)) | // Instants are covered elsewhere. Constant(NonIntegerConstant::BigInteger(_)) => { self.mark_known_empty(EmptyBecause::NonNumericArgument); bail!(ErrorKind::NonNumericArgument(function.clone(), position)); @@ -82,6 +83,7 @@ impl ConjoiningClauses { Constant(NonIntegerConstant::Float(f)) => Ok(QueryValue::TypedValue(TypedValue::Double(f))), Constant(NonIntegerConstant::Text(s)) => Ok(QueryValue::TypedValue(TypedValue::typed_string(s.as_str()))), Constant(NonIntegerConstant::Uuid(u)) => Ok(QueryValue::TypedValue(TypedValue::Uuid(u))), + Constant(NonIntegerConstant::Instant(u)) => Ok(QueryValue::TypedValue(TypedValue::Instant(u))), Constant(NonIntegerConstant::BigInteger(_)) => unimplemented!(), SrcVar(_) => unimplemented!(), } diff --git a/query/src/lib.rs b/query/src/lib.rs index d5d2bdbe..1f2a6d3e 100644 --- a/query/src/lib.rs +++ b/query/src/lib.rs @@ -42,8 +42,10 @@ use std::rc::Rc; use edn::{ BigInt, + DateTime, OrderedFloat, Uuid, + UTC, }; pub use edn::{ @@ -186,6 +188,7 @@ pub enum NonIntegerConstant { BigInteger(BigInt), Float(OrderedFloat), Text(Rc), + Instant(DateTime), Uuid(Uuid), } @@ -196,6 +199,7 @@ impl NonIntegerConstant { NonIntegerConstant::Boolean(v) => TypedValue::Boolean(v), NonIntegerConstant::Float(v) => TypedValue::Double(v), NonIntegerConstant::Text(v) => TypedValue::String(v), + NonIntegerConstant::Instant(v) => TypedValue::Instant(v), NonIntegerConstant::Uuid(v) => TypedValue::Uuid(v), } } @@ -320,6 +324,8 @@ impl FromValue for PatternValuePlace { Some(PatternValuePlace::Constant(NonIntegerConstant::Float(x))), edn::SpannedValue::BigInteger(ref x) => Some(PatternValuePlace::Constant(NonIntegerConstant::BigInteger(x.clone()))), + edn::SpannedValue::Instant(x) => + Some(PatternValuePlace::Constant(NonIntegerConstant::Instant(x))), edn::SpannedValue::Text(ref x) => // TODO: intern strings. #398. Some(PatternValuePlace::Constant(NonIntegerConstant::Text(Rc::new(x.clone())))), diff --git a/sql/src/lib.rs b/sql/src/lib.rs index 312225dd..7b926de4 100644 --- a/sql/src/lib.rs +++ b/sql/src/lib.rs @@ -19,7 +19,10 @@ use std::rc::Rc; use ordered_float::OrderedFloat; -use mentat_core::TypedValue; +use mentat_core::{ + ToMicros, + TypedValue, +}; pub use rusqlite::types::Value; @@ -140,6 +143,9 @@ impl QueryBuilder for SQLiteQueryBuilder { &Boolean(v) => self.push_sql(if v { "1" } else { "0" }), &Long(v) => self.push_sql(v.to_string().as_str()), &Double(OrderedFloat(v)) => self.push_sql(v.to_string().as_str()), + &Instant(dt) => { + self.push_sql(format!("{}", dt.to_micros()).as_str()); // TODO: argument instead? + }, &Uuid(ref u) => { // Get a byte array. let bytes = u.as_bytes().clone();