From b11b9b909c5923fd82246636d63454a8b774c582 Mon Sep 17 00:00:00 2001 From: Nick Alexander Date: Thu, 12 Jan 2017 16:08:29 -0800 Subject: [PATCH] Add tx{-parser} crates; start parsing transactions. (#164) r=rnewman This depends on edn and uses the combine parser combinator library. --- .travis.yml | 1 + Cargo.toml | 3 + edn/src/types.rs | 2 +- tx-parser/Cargo.toml | 12 ++ tx-parser/src/lib.rs | 337 ++++++++++++++++++++++++++++++++++++++ tx-parser/tests/parser.rs | 49 ++++++ tx/Cargo.toml | 7 + tx/README.md | 1 + tx/src/entities.rs | 58 +++++++ tx/src/lib.rs | 11 ++ 10 files changed, 480 insertions(+), 1 deletion(-) create mode 100644 tx-parser/Cargo.toml create mode 100644 tx-parser/src/lib.rs create mode 100644 tx-parser/tests/parser.rs create mode 100644 tx/Cargo.toml create mode 100644 tx/README.md create mode 100644 tx/src/entities.rs create mode 100644 tx/src/lib.rs diff --git a/.travis.yml b/.travis.yml index c7242366..e2b872d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,3 +4,4 @@ script: - cargo test --verbose - cargo test --verbose -p edn - cargo test --verbose -p mentat_query_parser + - cargo test --verbose -p mentat_tx_parser diff --git a/Cargo.toml b/Cargo.toml index ecf0f84f..ce71e598 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,3 +15,6 @@ clap = "2.19.3" [dependencies.mentat_query_parser] path = "query-parser" + +[dependencies.mentat_tx_parser] + path = "tx-parser" diff --git a/edn/src/types.rs b/edn/src/types.rs index 1a9748f3..7f969d07 100644 --- a/edn/src/types.rs +++ b/edn/src/types.rs @@ -16,7 +16,7 @@ use num::BigInt; use ordered_float::OrderedFloat; /// Value represents one of the allowed values in an EDN string. -#[derive(PartialEq, Eq, Hash, Debug)] +#[derive(PartialEq, Eq, Hash, Clone, Debug)] pub enum Value { Nil, Boolean(bool), diff --git a/tx-parser/Cargo.toml b/tx-parser/Cargo.toml new file mode 100644 index 00000000..7601720b --- /dev/null +++ b/tx-parser/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "mentat_tx_parser" +version = "0.0.1" + +[dependencies] +combine = "2.1.1" + +[dependencies.edn] + path = "../edn" + +[dependencies.mentat_tx] + path = "../tx" diff --git a/tx-parser/src/lib.rs b/tx-parser/src/lib.rs new file mode 100644 index 00000000..8b136ba4 --- /dev/null +++ b/tx-parser/src/lib.rs @@ -0,0 +1,337 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#![allow(dead_code)] + +extern crate edn; +extern crate combine; +extern crate mentat_tx; + +use combine::{any, eof, many, optional, parser, satisfy_map, token, Parser, ParseResult, Stream}; +use combine::combinator::{Expected, FnParser}; +use edn::symbols::NamespacedKeyword; +use edn::types::Value; +use mentat_tx::entities::*; + +pub struct Tx(::std::marker::PhantomData I>); + +type TxParser = Expected ParseResult>>; + +fn fn_parser(f: fn(I) -> ParseResult, err: &'static str) -> TxParser + where I: Stream +{ + parser(f).expected(err) +} + +impl Tx + where I: Stream +{ + fn integer() -> TxParser { + fn_parser(Tx::::integer_, "integer") + } + + fn integer_(input: I) -> ParseResult { + return satisfy_map(|x: Value| if let Value::Integer(y) = x { + Some(y) + } else { + None + }) + .parse_stream(input); + } + + fn keyword() -> TxParser { + fn_parser(Tx::::keyword_, "keyword") + } + + fn keyword_(input: I) -> ParseResult { + return satisfy_map(|x: Value| if let Value::NamespacedKeyword(y) = x { + Some(y) + } else { + None + }) + .parse_stream(input); + } + + fn entid() -> TxParser { + fn_parser(Tx::::entid_, "entid") + } + + fn entid_(input: I) -> ParseResult { + let p = Tx::::integer() + .map(|x| Entid::Entid(x)) + .or(Tx::::keyword().map(|x| Entid::Ident(x))) + .parse_lazy(input) + .into(); + return p; + } + + fn lookup_ref() -> TxParser { + fn_parser(Tx::::lookup_ref_, "lookup-ref") + } + + fn lookup_ref_(input: I) -> ParseResult { + return satisfy_map(|x: Value| if let Value::Vector(y) = x { + let mut p = (Tx::<&[Value]>::entid(), any(), eof()) + .map(|(a, v, _)| LookupRef { a: a, v: v }); + let r = p.parse_lazy(&y[..]).into(); + match r { + Ok((r, _)) => Some(r), + _ => None, + } + } else { + None + }) + .parse_stream(input); + } + + fn entid_or_lookup_ref() -> TxParser { + fn_parser(Tx::::entid_or_lookup_ref_, "entid|lookup-ref") + } + + fn entid_or_lookup_ref_(input: I) -> ParseResult { + let p = Tx::::entid() + .map(|x| EntidOrLookupRef::Entid(x)) + .or(Tx::::lookup_ref().map(|x| EntidOrLookupRef::LookupRef(x))) + .parse_lazy(input) + .into(); + return p; + } + + // TODO: abstract the "match Vector, parse internal stream" pattern to remove this boilerplate. + fn add_(input: I) -> ParseResult { + return satisfy_map(|x: Value| -> Option { + if let Value::Vector(y) = x { + let mut p = (token(Value::NamespacedKeyword(NamespacedKeyword::new("db", + "add"))), + Tx::<&[Value]>::entid_or_lookup_ref(), + Tx::<&[Value]>::entid(), + // TODO: handle lookup-ref. + any(), + // TODO: entid or special keyword :db/tx? + optional(Tx::<&[Value]>::entid()), + eof()) + .map(|(_, e, a, v, tx, _)| { + Entity::Add { + e: e, + a: a, + v: ValueOrLookupRef::Value(v), + tx: tx, + } + }); + // TODO: use ok() with a type annotation rather than explicit match. + match p.parse_lazy(&y[..]).into() { + Ok((r, _)) => Some(r), + _ => None, + } + } else { + None + } + }) + .parse_stream(input); + } + + fn add() -> TxParser { + fn_parser(Tx::::add_, "[:db/add e a v tx?]") + } + + fn retract_(input: I) -> ParseResult { + return satisfy_map(|x: Value| -> Option { + if let Value::Vector(y) = x { + let mut p = (token(Value::NamespacedKeyword(NamespacedKeyword::new("db", + "retract"))), + Tx::<&[Value]>::entid_or_lookup_ref(), + Tx::<&[Value]>::entid(), + // TODO: handle lookup-ref. + any(), + eof()) + .map(|(_, e, a, v, _)| { + Entity::Retract { + e: e, + a: a, + v: ValueOrLookupRef::Value(v), + } + }); + // TODO: use ok() with a type annotation rather than explicit match. + match p.parse_lazy(&y[..]).into() { + Ok((r, _)) => Some(r), + _ => None, + } + } else { + None + } + }) + .parse_stream(input); + } + + fn retract() -> TxParser { + fn_parser(Tx::::retract_, "[:db/retract e a v]") + } + + fn retract_attribute_(input: I) -> ParseResult { + return satisfy_map(|x: Value| -> Option { + if let Value::Vector(y) = x { + let mut p = (token(Value::NamespacedKeyword(NamespacedKeyword::new("db", "retractAttribute"))), + Tx::<&[Value]>::entid_or_lookup_ref(), + Tx::<&[Value]>::entid(), + eof()) + .map(|(_, e, a, _)| Entity::RetractAttribute { e: e, a: a }); + // TODO: use ok() with a type annotation rather than explicit match. + match p.parse_lazy(&y[..]).into() { + Ok((r, _)) => Some(r), + _ => None, + } + } else { + None + } + }) + .parse_stream(input); + } + + fn retract_attribute() -> TxParser { + fn_parser(Tx::::retract_attribute_, "[:db/retractAttribute e a]") + } + + fn retract_entity_(input: I) -> ParseResult { + return satisfy_map(|x: Value| -> Option { + if let Value::Vector(y) = x { + let mut p = + (token(Value::NamespacedKeyword(NamespacedKeyword::new("db", + "retractEntity"))), + Tx::<&[Value]>::entid_or_lookup_ref(), + eof()) + .map(|(_, e, _)| Entity::RetractEntity { e: e }); + // TODO: use ok() with a type annotation rather than explicit match. + match p.parse_lazy(&y[..]).into() { + Ok((r, _)) => Some(r), + _ => None, + } + } else { + None + } + }) + .parse_stream(input); + } + + fn retract_entity() -> TxParser { + fn_parser(Tx::::retract_entity_, "[:db/retractEntity e]") + } + + fn entity_(input: I) -> ParseResult { + let mut p = Tx::::add() + .or(Tx::::retract()) + .or(Tx::::retract_attribute()) + .or(Tx::::retract_entity()); + p.parse_stream(input) + } + + fn entity() -> TxParser { + fn_parser(Tx::::entity_, + "[:db/add|:db/retract|:db/retractAttribute|:db/retractEntity ...]") + } + + fn entities_(input: I) -> ParseResult, I> { + return satisfy_map(|x: Value| -> Option> { + if let Value::Vector(y) = x { + let mut p = (many(Tx::<&[Value]>::entity()), eof()).map(|(es, _)| es); + // TODO: use ok() with a type annotation rather than explicit match. + match p.parse_lazy(&y[..]).into() { + Ok((r, _)) => Some(r), + _ => None, + } + } else { + None + } + }) + .parse_stream(input); + } + + fn entities() -> TxParser, I> { + fn_parser(Tx::::entities_, + "[[:db/add|:db/retract|:db/retractAttribute|:db/retractEntity ...]*]") + } + + pub fn parse(input: I) -> Result, combine::ParseError> { + (Tx::::entities(), eof()) + .map(|(es, _)| es) + .parse(input) + .map(|x| x.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use combine::Parser; + use edn::symbols::NamespacedKeyword; + use edn::types::Value; + use mentat_tx::entities::*; + + fn kw(namespace: &str, name: &str) -> Value { + Value::NamespacedKeyword(NamespacedKeyword::new(namespace, name)) + } + + #[test] + fn test_add() { + let input = [Value::Vector(vec![kw("db", "add"), + kw("test", "entid"), + kw("test", "a"), + Value::Text("v".into())])]; + let mut parser = Tx::entity(); + let result = parser.parse(&input[..]); + assert_eq!(result, + Ok((Entity::Add { + e: EntidOrLookupRef::Entid(Entid::Ident(NamespacedKeyword::new("test", + "entid"))), + a: Entid::Ident(NamespacedKeyword::new("test", "a")), + v: ValueOrLookupRef::Value(Value::Text("v".into())), + tx: None, + }, + &[][..]))); + } + + #[test] + fn test_retract() { + let input = [Value::Vector(vec![kw("db", "retract"), + Value::Integer(101), + kw("test", "a"), + Value::Text("v".into())])]; + let mut parser = Tx::entity(); + let result = parser.parse(&input[..]); + assert_eq!(result, + Ok((Entity::Retract { + e: EntidOrLookupRef::Entid(Entid::Entid(101)), + a: Entid::Ident(NamespacedKeyword::new("test", "a")), + v: ValueOrLookupRef::Value(Value::Text("v".into())), + }, + &[][..]))); + } + + #[test] + fn test_lookup_ref() { + let input = [Value::Vector(vec![kw("db", "add"), + Value::Vector(vec![kw("test", "a1"), + Value::Text("v1".into())]), + kw("test", "a"), + Value::Text("v".into())])]; + let mut parser = Tx::entity(); + let result = parser.parse(&input[..]); + assert_eq!(result, + Ok((Entity::Add { + e: EntidOrLookupRef::LookupRef(LookupRef { + a: Entid::Ident(NamespacedKeyword::new("test", "a1")), + v: Value::Text("v1".into()), + }), + a: Entid::Ident(NamespacedKeyword::new("test", "a")), + v: ValueOrLookupRef::Value(Value::Text("v".into())), + tx: None, + }, + &[][..]))); + } +} diff --git a/tx-parser/tests/parser.rs b/tx-parser/tests/parser.rs new file mode 100644 index 00000000..77424560 --- /dev/null +++ b/tx-parser/tests/parser.rs @@ -0,0 +1,49 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +extern crate edn; +extern crate combine; +extern crate mentat_tx; +extern crate mentat_tx_parser; + +use edn::parse; +use edn::symbols::NamespacedKeyword; +use edn::types::Value; +use mentat_tx::entities::*; +use mentat_tx_parser::Tx; + +#[test] +fn test_entities() { + + // TODO: align with whitespace after the EDN parser ignores more whitespace. + let input = r#"[[:db/add 101 :test/a "v"] +[:db/retract 102 :test/b "w"]]"#; + + let edn = parse::value(input).unwrap(); + let input = [edn]; + + let result = Tx::parse(&input[..]); + assert_eq!(result, + Ok(vec![ + Entity::Add { + e: EntidOrLookupRef::Entid(Entid::Entid(101)), + a: Entid::Ident(NamespacedKeyword::new("test", "a")), + v: ValueOrLookupRef::Value(Value::Text("v".into())), + tx: None, + }, + Entity::Retract { + e: EntidOrLookupRef::Entid(Entid::Entid(102)), + a: Entid::Ident(NamespacedKeyword::new("test", "b")), + v: ValueOrLookupRef::Value(Value::Text("w".into())), + }, + ])); +} + +// TODO: test error handling in select cases. diff --git a/tx/Cargo.toml b/tx/Cargo.toml new file mode 100644 index 00000000..72010b56 --- /dev/null +++ b/tx/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "mentat_tx" +version = "0.0.1" + +[dependencies] +[dependencies.edn] + path = "../edn" diff --git a/tx/README.md b/tx/README.md new file mode 100644 index 00000000..22949767 --- /dev/null +++ b/tx/README.md @@ -0,0 +1 @@ +This sub-crate implements the core types used by the transaction processor. \ No newline at end of file diff --git a/tx/src/entities.rs b/tx/src/entities.rs new file mode 100644 index 00000000..3c99b622 --- /dev/null +++ b/tx/src/entities.rs @@ -0,0 +1,58 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +///! This module defines core types that support the transaction processor. + +extern crate edn; + +use self::edn::types::Value; +use self::edn::symbols::NamespacedKeyword; + +#[derive(Clone, Debug, PartialEq)] +pub enum Entid { + Entid(i64), + Ident(NamespacedKeyword), +} + +#[derive(Clone, Debug, PartialEq)] +pub struct LookupRef { + pub a: Entid, + // TODO: consider boxing to allow recursive lookup refs. + pub v: Value, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum EntidOrLookupRef { + Entid(Entid), + LookupRef(LookupRef), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ValueOrLookupRef { + Value(Value), + LookupRef(LookupRef), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Entity { + Add { + e: EntidOrLookupRef, + a: Entid, + v: ValueOrLookupRef, + tx: Option, + }, + Retract { + e: EntidOrLookupRef, + a: Entid, + v: ValueOrLookupRef, + }, + RetractAttribute { e: EntidOrLookupRef, a: Entid }, + RetractEntity { e: EntidOrLookupRef }, +} diff --git a/tx/src/lib.rs b/tx/src/lib.rs new file mode 100644 index 00000000..78eac748 --- /dev/null +++ b/tx/src/lib.rs @@ -0,0 +1,11 @@ +// Copyright 2016 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +pub mod entities;